15433956dSTimothy McDaniel /* SPDX-License-Identifier: BSD-3-Clause 25433956dSTimothy McDaniel * Copyright(c) 2016-2020 Intel Corporation 35433956dSTimothy McDaniel */ 45433956dSTimothy McDaniel 55433956dSTimothy McDaniel #include <stdint.h> 65433956dSTimothy McDaniel #include <stdbool.h> 75433956dSTimothy McDaniel #include <stdio.h> 85433956dSTimothy McDaniel #include <errno.h> 95433956dSTimothy McDaniel #include <assert.h> 105433956dSTimothy McDaniel #include <unistd.h> 115433956dSTimothy McDaniel #include <string.h> 125433956dSTimothy McDaniel 135433956dSTimothy McDaniel #include <rte_malloc.h> 145433956dSTimothy McDaniel #include <rte_errno.h> 155433956dSTimothy McDaniel 16d4a06a39STimothy McDaniel #include "base/dlb2_regs.h" 17037a6167STimothy McDaniel #include "base/dlb2_hw_types.h" 18dfdd11a8STimothy McDaniel #include "base/dlb2_resource.h" 195433956dSTimothy McDaniel #include "base/dlb2_osdep.h" 205433956dSTimothy McDaniel #include "dlb2_main.h" 215433956dSTimothy McDaniel #include "../dlb2_user.h" 225433956dSTimothy McDaniel #include "../dlb2_priv.h" 23e7c9971aSTimothy McDaniel #include "../dlb2_iface.h" 245433956dSTimothy McDaniel #include "../dlb2_inline_fns.h" 255433956dSTimothy McDaniel 265433956dSTimothy McDaniel #define PF_ID_ZERO 0 /* PF ONLY! */ 275433956dSTimothy McDaniel #define NO_OWNER_VF 0 /* PF ONLY! */ 285433956dSTimothy McDaniel #define NOT_VF_REQ false /* PF ONLY! */ 295a687833SAbdullah Sevincer #define DLB2_PCI_PASID_CAP_OFFSET 0x148 /* PASID capability offset */ 305433956dSTimothy McDaniel 315433956dSTimothy McDaniel static int 325433956dSTimothy McDaniel dlb2_pf_init_driver_state(struct dlb2_dev *dlb2_dev) 335433956dSTimothy McDaniel { 345433956dSTimothy McDaniel rte_spinlock_init(&dlb2_dev->resource_mutex); 355433956dSTimothy McDaniel 365433956dSTimothy McDaniel return 0; 375433956dSTimothy McDaniel } 385433956dSTimothy McDaniel 395433956dSTimothy McDaniel static void dlb2_pf_enable_pm(struct dlb2_dev *dlb2_dev) 405433956dSTimothy McDaniel { 410d65f8f9STimothy McDaniel int version; 420d65f8f9STimothy McDaniel version = DLB2_HW_DEVICE_FROM_PCI_ID(dlb2_dev->pdev); 430d65f8f9STimothy McDaniel 440d65f8f9STimothy McDaniel dlb2_clr_pmcsr_disable(&dlb2_dev->hw, version); 455433956dSTimothy McDaniel } 465433956dSTimothy McDaniel 475433956dSTimothy McDaniel #define DLB2_READY_RETRY_LIMIT 1000 480d65f8f9STimothy McDaniel static int dlb2_pf_wait_for_device_ready(struct dlb2_dev *dlb2_dev, 490d65f8f9STimothy McDaniel int dlb_version) 505433956dSTimothy McDaniel { 515433956dSTimothy McDaniel u32 retries = 0; 525433956dSTimothy McDaniel 535433956dSTimothy McDaniel /* Allow at least 1s for the device to become active after power-on */ 545433956dSTimothy McDaniel for (retries = 0; retries < DLB2_READY_RETRY_LIMIT; retries++) { 550d65f8f9STimothy McDaniel u32 idle_val; 560d65f8f9STimothy McDaniel u32 idle_dlb_func_idle; 570d65f8f9STimothy McDaniel u32 pm_st_val; 580d65f8f9STimothy McDaniel u32 pm_st_pmsm; 595433956dSTimothy McDaniel u32 addr; 605433956dSTimothy McDaniel 610d65f8f9STimothy McDaniel addr = DLB2_CM_CFG_PM_STATUS(dlb_version); 620d65f8f9STimothy McDaniel pm_st_val = DLB2_CSR_RD(&dlb2_dev->hw, addr); 630d65f8f9STimothy McDaniel addr = DLB2_CM_CFG_DIAGNOSTIC_IDLE_STATUS(dlb_version); 640d65f8f9STimothy McDaniel idle_val = DLB2_CSR_RD(&dlb2_dev->hw, addr); 650d65f8f9STimothy McDaniel idle_dlb_func_idle = idle_val & 660d65f8f9STimothy McDaniel DLB2_CM_CFG_DIAGNOSTIC_IDLE_STATUS_DLB_FUNC_IDLE; 670d65f8f9STimothy McDaniel pm_st_pmsm = pm_st_val & DLB2_CM_CFG_PM_STATUS_PMSM; 680d65f8f9STimothy McDaniel if (pm_st_pmsm && idle_dlb_func_idle) 695433956dSTimothy McDaniel break; 705433956dSTimothy McDaniel 715433956dSTimothy McDaniel rte_delay_ms(1); 725433956dSTimothy McDaniel }; 735433956dSTimothy McDaniel 745433956dSTimothy McDaniel if (retries == DLB2_READY_RETRY_LIMIT) { 75*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] wait for device ready timed out", 765433956dSTimothy McDaniel __func__); 775433956dSTimothy McDaniel return -1; 785433956dSTimothy McDaniel } 795433956dSTimothy McDaniel 805433956dSTimothy McDaniel return 0; 815433956dSTimothy McDaniel } 825433956dSTimothy McDaniel 835433956dSTimothy McDaniel struct dlb2_dev * 848d1d9070SAbdullah Sevincer dlb2_probe(struct rte_pci_device *pdev, const void *probe_args) 855433956dSTimothy McDaniel { 865433956dSTimothy McDaniel struct dlb2_dev *dlb2_dev; 875433956dSTimothy McDaniel int ret = 0; 880d65f8f9STimothy McDaniel int dlb_version = 0; 895433956dSTimothy McDaniel 905433956dSTimothy McDaniel DLB2_INFO(dlb2_dev, "probe\n"); 915433956dSTimothy McDaniel 925433956dSTimothy McDaniel dlb2_dev = rte_malloc("DLB2_PF", sizeof(struct dlb2_dev), 935433956dSTimothy McDaniel RTE_CACHE_LINE_SIZE); 945433956dSTimothy McDaniel 955433956dSTimothy McDaniel if (dlb2_dev == NULL) { 965433956dSTimothy McDaniel ret = -ENOMEM; 975433956dSTimothy McDaniel goto dlb2_dev_malloc_fail; 985433956dSTimothy McDaniel } 995433956dSTimothy McDaniel 1000d65f8f9STimothy McDaniel dlb_version = DLB2_HW_DEVICE_FROM_PCI_ID(pdev); 1010d65f8f9STimothy McDaniel 1025433956dSTimothy McDaniel /* PCI Bus driver has already mapped bar space into process. 1035433956dSTimothy McDaniel * Save off our IO register and FUNC addresses. 1045433956dSTimothy McDaniel */ 1055433956dSTimothy McDaniel 1065433956dSTimothy McDaniel /* BAR 0 */ 1075433956dSTimothy McDaniel if (pdev->mem_resource[0].addr == NULL) { 1085433956dSTimothy McDaniel DLB2_ERR(dlb2_dev, "probe: BAR 0 addr (csr_kva) is NULL\n"); 1095433956dSTimothy McDaniel ret = -EINVAL; 1105433956dSTimothy McDaniel goto pci_mmap_bad_addr; 1115433956dSTimothy McDaniel } 1125433956dSTimothy McDaniel dlb2_dev->hw.func_kva = (void *)(uintptr_t)pdev->mem_resource[0].addr; 1135433956dSTimothy McDaniel dlb2_dev->hw.func_phys_addr = pdev->mem_resource[0].phys_addr; 1145433956dSTimothy McDaniel 1155433956dSTimothy McDaniel DLB2_INFO(dlb2_dev, "DLB2 FUNC VA=%p, PA=%p, len=%p\n", 1165433956dSTimothy McDaniel (void *)dlb2_dev->hw.func_kva, 1175433956dSTimothy McDaniel (void *)dlb2_dev->hw.func_phys_addr, 1185433956dSTimothy McDaniel (void *)(pdev->mem_resource[0].len)); 1195433956dSTimothy McDaniel 1205433956dSTimothy McDaniel /* BAR 2 */ 1215433956dSTimothy McDaniel if (pdev->mem_resource[2].addr == NULL) { 1225433956dSTimothy McDaniel DLB2_ERR(dlb2_dev, "probe: BAR 2 addr (func_kva) is NULL\n"); 1235433956dSTimothy McDaniel ret = -EINVAL; 1245433956dSTimothy McDaniel goto pci_mmap_bad_addr; 1255433956dSTimothy McDaniel } 1265433956dSTimothy McDaniel dlb2_dev->hw.csr_kva = (void *)(uintptr_t)pdev->mem_resource[2].addr; 1275433956dSTimothy McDaniel dlb2_dev->hw.csr_phys_addr = pdev->mem_resource[2].phys_addr; 1285433956dSTimothy McDaniel 1295433956dSTimothy McDaniel DLB2_INFO(dlb2_dev, "DLB2 CSR VA=%p, PA=%p, len=%p\n", 1305433956dSTimothy McDaniel (void *)dlb2_dev->hw.csr_kva, 1315433956dSTimothy McDaniel (void *)dlb2_dev->hw.csr_phys_addr, 1325433956dSTimothy McDaniel (void *)(pdev->mem_resource[2].len)); 1335433956dSTimothy McDaniel 1345433956dSTimothy McDaniel dlb2_dev->pdev = pdev; 1355433956dSTimothy McDaniel 1365433956dSTimothy McDaniel /* PM enable must be done before any other MMIO accesses, and this 1375433956dSTimothy McDaniel * setting is persistent across device reset. 1385433956dSTimothy McDaniel */ 1395433956dSTimothy McDaniel dlb2_pf_enable_pm(dlb2_dev); 1405433956dSTimothy McDaniel 1410d65f8f9STimothy McDaniel ret = dlb2_pf_wait_for_device_ready(dlb2_dev, dlb_version); 1425433956dSTimothy McDaniel if (ret) 1435433956dSTimothy McDaniel goto wait_for_device_ready_fail; 1445433956dSTimothy McDaniel 1458d1d9070SAbdullah Sevincer ret = dlb2_resource_probe(&dlb2_dev->hw, probe_args); 1468d1d9070SAbdullah Sevincer if (ret) 1478d1d9070SAbdullah Sevincer goto resource_probe_fail; 1488d1d9070SAbdullah Sevincer 1495433956dSTimothy McDaniel ret = dlb2_pf_reset(dlb2_dev); 1505433956dSTimothy McDaniel if (ret) 1515433956dSTimothy McDaniel goto dlb2_reset_fail; 1525433956dSTimothy McDaniel 1535433956dSTimothy McDaniel ret = dlb2_pf_init_driver_state(dlb2_dev); 1545433956dSTimothy McDaniel if (ret) 1555433956dSTimothy McDaniel goto init_driver_state_fail; 1565433956dSTimothy McDaniel 1578d1d9070SAbdullah Sevincer ret = dlb2_resource_init(&dlb2_dev->hw, dlb_version, probe_args); 1585433956dSTimothy McDaniel if (ret) 1595433956dSTimothy McDaniel goto resource_init_fail; 1605433956dSTimothy McDaniel 1615433956dSTimothy McDaniel return dlb2_dev; 1625433956dSTimothy McDaniel 1635433956dSTimothy McDaniel resource_init_fail: 1645433956dSTimothy McDaniel dlb2_resource_free(&dlb2_dev->hw); 1655433956dSTimothy McDaniel init_driver_state_fail: 1665433956dSTimothy McDaniel dlb2_reset_fail: 1675433956dSTimothy McDaniel pci_mmap_bad_addr: 1688d1d9070SAbdullah Sevincer resource_probe_fail: 1695433956dSTimothy McDaniel wait_for_device_ready_fail: 1705433956dSTimothy McDaniel rte_free(dlb2_dev); 1715433956dSTimothy McDaniel dlb2_dev_malloc_fail: 1725433956dSTimothy McDaniel rte_errno = ret; 1735433956dSTimothy McDaniel return NULL; 1745433956dSTimothy McDaniel } 1755433956dSTimothy McDaniel 1765433956dSTimothy McDaniel int 1775433956dSTimothy McDaniel dlb2_pf_reset(struct dlb2_dev *dlb2_dev) 1785433956dSTimothy McDaniel { 1795433956dSTimothy McDaniel int ret = 0; 1805433956dSTimothy McDaniel int i = 0; 1815433956dSTimothy McDaniel uint32_t dword[16]; 1825433956dSTimothy McDaniel uint16_t cmd; 1835433956dSTimothy McDaniel off_t off; 1845433956dSTimothy McDaniel 1855433956dSTimothy McDaniel uint16_t dev_ctl_word; 1865433956dSTimothy McDaniel uint16_t dev_ctl2_word; 1875433956dSTimothy McDaniel uint16_t lnk_word; 1885433956dSTimothy McDaniel uint16_t lnk_word2; 1895433956dSTimothy McDaniel uint16_t slt_word; 1905433956dSTimothy McDaniel uint16_t slt_word2; 1915433956dSTimothy McDaniel uint16_t rt_ctl_word; 1925433956dSTimothy McDaniel uint32_t pri_reqs_dword; 1935433956dSTimothy McDaniel uint16_t pri_ctrl_word; 1945433956dSTimothy McDaniel 195a10b6e53SDavid Marchand off_t pcie_cap_offset; 1965433956dSTimothy McDaniel int pri_cap_offset; 197a10b6e53SDavid Marchand off_t msix_cap_offset; 1985433956dSTimothy McDaniel int err_cap_offset; 1995433956dSTimothy McDaniel int acs_cap_offset; 2005433956dSTimothy McDaniel int wait_count; 2015433956dSTimothy McDaniel 2025433956dSTimothy McDaniel uint16_t devsta_busy_word; 2035433956dSTimothy McDaniel uint16_t devctl_word; 2045433956dSTimothy McDaniel 2055433956dSTimothy McDaniel struct rte_pci_device *pdev = dlb2_dev->pdev; 2065433956dSTimothy McDaniel 2075433956dSTimothy McDaniel /* Save PCI config state */ 2085433956dSTimothy McDaniel 2095433956dSTimothy McDaniel for (i = 0; i < 16; i++) { 2105433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &dword[i], 4, i * 4) != 4) 2115433956dSTimothy McDaniel return ret; 2125433956dSTimothy McDaniel } 2135433956dSTimothy McDaniel 214baa9c550SDavid Marchand pcie_cap_offset = rte_pci_find_capability(pdev, RTE_PCI_CAP_ID_EXP); 2155433956dSTimothy McDaniel 2165433956dSTimothy McDaniel if (pcie_cap_offset < 0) { 217*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to find the pcie capability", 2185433956dSTimothy McDaniel __func__); 2195433956dSTimothy McDaniel return pcie_cap_offset; 2205433956dSTimothy McDaniel } 2215433956dSTimothy McDaniel 222a00c150fSTimothy McDaniel off = pcie_cap_offset + RTE_PCI_EXP_DEVCTL; 2235433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &dev_ctl_word, 2, off) != 2) 2245433956dSTimothy McDaniel dev_ctl_word = 0; 2255433956dSTimothy McDaniel 2263a40df8dSDavid Marchand off = pcie_cap_offset + RTE_PCI_EXP_LNKCTL; 2275433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &lnk_word, 2, off) != 2) 2285433956dSTimothy McDaniel lnk_word = 0; 2295433956dSTimothy McDaniel 2303a40df8dSDavid Marchand off = pcie_cap_offset + RTE_PCI_EXP_SLTCTL; 2315433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &slt_word, 2, off) != 2) 2325433956dSTimothy McDaniel slt_word = 0; 2335433956dSTimothy McDaniel 2343a40df8dSDavid Marchand off = pcie_cap_offset + RTE_PCI_EXP_RTCTL; 2355433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &rt_ctl_word, 2, off) != 2) 2365433956dSTimothy McDaniel rt_ctl_word = 0; 2375433956dSTimothy McDaniel 2383a40df8dSDavid Marchand off = pcie_cap_offset + RTE_PCI_EXP_DEVCTL2; 2395433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &dev_ctl2_word, 2, off) != 2) 2405433956dSTimothy McDaniel dev_ctl2_word = 0; 2415433956dSTimothy McDaniel 2423a40df8dSDavid Marchand off = pcie_cap_offset + RTE_PCI_EXP_LNKCTL2; 2435433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &lnk_word2, 2, off) != 2) 2445433956dSTimothy McDaniel lnk_word2 = 0; 2455433956dSTimothy McDaniel 2463a40df8dSDavid Marchand off = pcie_cap_offset + RTE_PCI_EXP_SLTCTL2; 2475433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &slt_word2, 2, off) != 2) 2485433956dSTimothy McDaniel slt_word2 = 0; 2495433956dSTimothy McDaniel 2500138cf92SDavid Marchand off = RTE_PCI_EXT_CAP_ID_PRI; 251a00c150fSTimothy McDaniel pri_cap_offset = rte_pci_find_ext_capability(pdev, off); 2525433956dSTimothy McDaniel 2535433956dSTimothy McDaniel if (pri_cap_offset >= 0) { 2544ac34947SDavid Marchand off = pri_cap_offset + RTE_PCI_PRI_ALLOC_REQ; 2555433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &pri_reqs_dword, 4, off) != 4) 2565433956dSTimothy McDaniel pri_reqs_dword = 0; 2575433956dSTimothy McDaniel } 2585433956dSTimothy McDaniel 2595433956dSTimothy McDaniel /* clear the PCI command register before issuing the FLR */ 2605433956dSTimothy McDaniel 261c89450cbSDavid Marchand off = RTE_PCI_COMMAND; 2625433956dSTimothy McDaniel cmd = 0; 2635433956dSTimothy McDaniel if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) { 264*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pci command", 2655433956dSTimothy McDaniel __func__); 2665433956dSTimothy McDaniel return ret; 2675433956dSTimothy McDaniel } 2685433956dSTimothy McDaniel 2695433956dSTimothy McDaniel /* issue the FLR */ 2705433956dSTimothy McDaniel for (wait_count = 0; wait_count < 4; wait_count++) { 2715433956dSTimothy McDaniel int sleep_time; 2725433956dSTimothy McDaniel 2733a40df8dSDavid Marchand off = pcie_cap_offset + RTE_PCI_EXP_DEVSTA; 2745433956dSTimothy McDaniel ret = rte_pci_read_config(pdev, &devsta_busy_word, 2, off); 2755433956dSTimothy McDaniel if (ret != 2) { 276*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to read the pci device status", 2775433956dSTimothy McDaniel __func__); 2785433956dSTimothy McDaniel return ret; 2795433956dSTimothy McDaniel } 2805433956dSTimothy McDaniel 2813a40df8dSDavid Marchand if (!(devsta_busy_word & RTE_PCI_EXP_DEVSTA_TRPND)) 2825433956dSTimothy McDaniel break; 2835433956dSTimothy McDaniel 2845433956dSTimothy McDaniel sleep_time = (1 << (wait_count)) * 100; 2855433956dSTimothy McDaniel rte_delay_ms(sleep_time); 2865433956dSTimothy McDaniel } 2875433956dSTimothy McDaniel 2885433956dSTimothy McDaniel if (wait_count == 4) { 289*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] wait for pci pending transactions timed out", 2905433956dSTimothy McDaniel __func__); 2915433956dSTimothy McDaniel return -1; 2925433956dSTimothy McDaniel } 2935433956dSTimothy McDaniel 294a00c150fSTimothy McDaniel off = pcie_cap_offset + RTE_PCI_EXP_DEVCTL; 2955433956dSTimothy McDaniel ret = rte_pci_read_config(pdev, &devctl_word, 2, off); 2965433956dSTimothy McDaniel if (ret != 2) { 297*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to read the pcie device control", 2985433956dSTimothy McDaniel __func__); 2995433956dSTimothy McDaniel return ret; 3005433956dSTimothy McDaniel } 3015433956dSTimothy McDaniel 3023a40df8dSDavid Marchand devctl_word |= RTE_PCI_EXP_DEVCTL_BCR_FLR; 3035433956dSTimothy McDaniel 3045433956dSTimothy McDaniel ret = rte_pci_write_config(pdev, &devctl_word, 2, off); 3055433956dSTimothy McDaniel if (ret != 2) { 306*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie device control", 3075433956dSTimothy McDaniel __func__); 3085433956dSTimothy McDaniel return ret; 3095433956dSTimothy McDaniel } 3105433956dSTimothy McDaniel 3115433956dSTimothy McDaniel rte_delay_ms(100); 3125433956dSTimothy McDaniel 3135433956dSTimothy McDaniel /* Restore PCI config state */ 3145433956dSTimothy McDaniel 3155433956dSTimothy McDaniel if (pcie_cap_offset >= 0) { 316a00c150fSTimothy McDaniel off = pcie_cap_offset + RTE_PCI_EXP_DEVCTL; 3175433956dSTimothy McDaniel ret = rte_pci_write_config(pdev, &dev_ctl_word, 2, off); 3185433956dSTimothy McDaniel if (ret != 2) { 319*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie device control at offset %d", 3205433956dSTimothy McDaniel __func__, (int)off); 3215433956dSTimothy McDaniel return ret; 3225433956dSTimothy McDaniel } 3235433956dSTimothy McDaniel 3243a40df8dSDavid Marchand off = pcie_cap_offset + RTE_PCI_EXP_LNKCTL; 3255433956dSTimothy McDaniel ret = rte_pci_write_config(pdev, &lnk_word, 2, off); 3265433956dSTimothy McDaniel if (ret != 2) { 327*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d", 3285433956dSTimothy McDaniel __func__, (int)off); 3295433956dSTimothy McDaniel return ret; 3305433956dSTimothy McDaniel } 3315433956dSTimothy McDaniel 3323a40df8dSDavid Marchand off = pcie_cap_offset + RTE_PCI_EXP_SLTCTL; 3335433956dSTimothy McDaniel ret = rte_pci_write_config(pdev, &slt_word, 2, off); 3345433956dSTimothy McDaniel if (ret != 2) { 335*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d", 3365433956dSTimothy McDaniel __func__, (int)off); 3375433956dSTimothy McDaniel return ret; 3385433956dSTimothy McDaniel } 3395433956dSTimothy McDaniel 3403a40df8dSDavid Marchand off = pcie_cap_offset + RTE_PCI_EXP_RTCTL; 3415433956dSTimothy McDaniel ret = rte_pci_write_config(pdev, &rt_ctl_word, 2, off); 3425433956dSTimothy McDaniel if (ret != 2) { 343*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d", 3445433956dSTimothy McDaniel __func__, (int)off); 3455433956dSTimothy McDaniel return ret; 3465433956dSTimothy McDaniel } 3475433956dSTimothy McDaniel 3483a40df8dSDavid Marchand off = pcie_cap_offset + RTE_PCI_EXP_DEVCTL2; 3495433956dSTimothy McDaniel ret = rte_pci_write_config(pdev, &dev_ctl2_word, 2, off); 3505433956dSTimothy McDaniel if (ret != 2) { 351*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d", 3525433956dSTimothy McDaniel __func__, (int)off); 3535433956dSTimothy McDaniel return ret; 3545433956dSTimothy McDaniel } 3555433956dSTimothy McDaniel 3563a40df8dSDavid Marchand off = pcie_cap_offset + RTE_PCI_EXP_LNKCTL2; 3575433956dSTimothy McDaniel ret = rte_pci_write_config(pdev, &lnk_word2, 2, off); 3585433956dSTimothy McDaniel if (ret != 2) { 359*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d", 3605433956dSTimothy McDaniel __func__, (int)off); 3615433956dSTimothy McDaniel return ret; 3625433956dSTimothy McDaniel } 3635433956dSTimothy McDaniel 3643a40df8dSDavid Marchand off = pcie_cap_offset + RTE_PCI_EXP_SLTCTL2; 3655433956dSTimothy McDaniel ret = rte_pci_write_config(pdev, &slt_word2, 2, off); 3665433956dSTimothy McDaniel if (ret != 2) { 367*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d", 3685433956dSTimothy McDaniel __func__, (int)off); 3695433956dSTimothy McDaniel return ret; 3705433956dSTimothy McDaniel } 3715433956dSTimothy McDaniel } 3725433956dSTimothy McDaniel 3735433956dSTimothy McDaniel if (pri_cap_offset >= 0) { 3744ac34947SDavid Marchand pri_ctrl_word = RTE_PCI_PRI_CTRL_ENABLE; 3755433956dSTimothy McDaniel 3764ac34947SDavid Marchand off = pri_cap_offset + RTE_PCI_PRI_ALLOC_REQ; 3775433956dSTimothy McDaniel ret = rte_pci_write_config(pdev, &pri_reqs_dword, 4, off); 3785433956dSTimothy McDaniel if (ret != 4) { 379*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d", 3805433956dSTimothy McDaniel __func__, (int)off); 3815433956dSTimothy McDaniel return ret; 3825433956dSTimothy McDaniel } 3835433956dSTimothy McDaniel 3844ac34947SDavid Marchand off = pri_cap_offset + RTE_PCI_PRI_CTRL; 3855433956dSTimothy McDaniel ret = rte_pci_write_config(pdev, &pri_ctrl_word, 2, off); 3865433956dSTimothy McDaniel if (ret != 2) { 387*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d", 3885433956dSTimothy McDaniel __func__, (int)off); 3895433956dSTimothy McDaniel return ret; 3905433956dSTimothy McDaniel } 3915433956dSTimothy McDaniel } 3925433956dSTimothy McDaniel 393a00c150fSTimothy McDaniel off = RTE_PCI_EXT_CAP_ID_ERR; 394a00c150fSTimothy McDaniel err_cap_offset = rte_pci_find_ext_capability(pdev, off); 3955433956dSTimothy McDaniel 3965433956dSTimothy McDaniel if (err_cap_offset >= 0) { 3975433956dSTimothy McDaniel uint32_t tmp; 3985433956dSTimothy McDaniel 399fce1a19aSDavid Marchand off = err_cap_offset + RTE_PCI_ERR_ROOT_STATUS; 4005433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &tmp, 4, off) != 4) 4015433956dSTimothy McDaniel tmp = 0; 4025433956dSTimothy McDaniel 4035433956dSTimothy McDaniel ret = rte_pci_write_config(pdev, &tmp, 4, off); 4045433956dSTimothy McDaniel if (ret != 4) { 405*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d", 4065433956dSTimothy McDaniel __func__, (int)off); 4075433956dSTimothy McDaniel return ret; 4085433956dSTimothy McDaniel } 4095433956dSTimothy McDaniel 410fce1a19aSDavid Marchand off = err_cap_offset + RTE_PCI_ERR_COR_STATUS; 4115433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &tmp, 4, off) != 4) 4125433956dSTimothy McDaniel tmp = 0; 4135433956dSTimothy McDaniel 4145433956dSTimothy McDaniel ret = rte_pci_write_config(pdev, &tmp, 4, off); 4155433956dSTimothy McDaniel if (ret != 4) { 416*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d", 4175433956dSTimothy McDaniel __func__, (int)off); 4185433956dSTimothy McDaniel return ret; 4195433956dSTimothy McDaniel } 4205433956dSTimothy McDaniel 421fce1a19aSDavid Marchand off = err_cap_offset + RTE_PCI_ERR_UNCOR_STATUS; 4225433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &tmp, 4, off) != 4) 4235433956dSTimothy McDaniel tmp = 0; 4245433956dSTimothy McDaniel 4255433956dSTimothy McDaniel ret = rte_pci_write_config(pdev, &tmp, 4, off); 4265433956dSTimothy McDaniel if (ret != 4) { 427*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d", 4285433956dSTimothy McDaniel __func__, (int)off); 4295433956dSTimothy McDaniel return ret; 4305433956dSTimothy McDaniel } 4315433956dSTimothy McDaniel } 4325433956dSTimothy McDaniel 4335433956dSTimothy McDaniel for (i = 16; i > 0; i--) { 4345433956dSTimothy McDaniel off = (i - 1) * 4; 4355433956dSTimothy McDaniel ret = rte_pci_write_config(pdev, &dword[i - 1], 4, off); 4365433956dSTimothy McDaniel if (ret != 4) { 437*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d", 4385433956dSTimothy McDaniel __func__, (int)off); 4395433956dSTimothy McDaniel return ret; 4405433956dSTimothy McDaniel } 4415433956dSTimothy McDaniel } 4425433956dSTimothy McDaniel 443c89450cbSDavid Marchand off = RTE_PCI_COMMAND; 4445433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) { 445c89450cbSDavid Marchand cmd &= ~RTE_PCI_COMMAND_INTX_DISABLE; 4465433956dSTimothy McDaniel if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) { 447*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pci command", 4485433956dSTimothy McDaniel __func__); 4495433956dSTimothy McDaniel return ret; 4505433956dSTimothy McDaniel } 4515433956dSTimothy McDaniel } 4525433956dSTimothy McDaniel 453baa9c550SDavid Marchand msix_cap_offset = rte_pci_find_capability(pdev, RTE_PCI_CAP_ID_MSIX); 4545433956dSTimothy McDaniel if (msix_cap_offset >= 0) { 4557bb1168dSDavid Marchand off = msix_cap_offset + RTE_PCI_MSIX_FLAGS; 4565433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) { 4577bb1168dSDavid Marchand cmd |= RTE_PCI_MSIX_FLAGS_ENABLE; 4587bb1168dSDavid Marchand cmd |= RTE_PCI_MSIX_FLAGS_MASKALL; 4595433956dSTimothy McDaniel if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) { 460*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write msix flags", 4615433956dSTimothy McDaniel __func__); 4625433956dSTimothy McDaniel return ret; 4635433956dSTimothy McDaniel } 4645433956dSTimothy McDaniel } 4655433956dSTimothy McDaniel 4667bb1168dSDavid Marchand off = msix_cap_offset + RTE_PCI_MSIX_FLAGS; 4675433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) { 4687bb1168dSDavid Marchand cmd &= ~RTE_PCI_MSIX_FLAGS_MASKALL; 4695433956dSTimothy McDaniel if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) { 470*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write msix flags", 4715433956dSTimothy McDaniel __func__); 4725433956dSTimothy McDaniel return ret; 4735433956dSTimothy McDaniel } 4745433956dSTimothy McDaniel } 4755433956dSTimothy McDaniel } 4765433956dSTimothy McDaniel 4770138cf92SDavid Marchand off = RTE_PCI_EXT_CAP_ID_ACS; 478a00c150fSTimothy McDaniel acs_cap_offset = rte_pci_find_ext_capability(pdev, off); 4795433956dSTimothy McDaniel 4805433956dSTimothy McDaniel if (acs_cap_offset >= 0) { 4815433956dSTimothy McDaniel uint16_t acs_cap, acs_ctrl, acs_mask; 48205026b99SDavid Marchand off = acs_cap_offset + RTE_PCI_ACS_CAP; 4835433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &acs_cap, 2, off) != 2) 4845433956dSTimothy McDaniel acs_cap = 0; 4855433956dSTimothy McDaniel 48605026b99SDavid Marchand off = acs_cap_offset + RTE_PCI_ACS_CTRL; 4875433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2) 4885433956dSTimothy McDaniel acs_ctrl = 0; 4895433956dSTimothy McDaniel 49005026b99SDavid Marchand acs_mask = RTE_PCI_ACS_SV | RTE_PCI_ACS_RR; 49105026b99SDavid Marchand acs_mask |= (RTE_PCI_ACS_CR | RTE_PCI_ACS_UF); 4925433956dSTimothy McDaniel acs_ctrl |= (acs_cap & acs_mask); 4935433956dSTimothy McDaniel 4945433956dSTimothy McDaniel ret = rte_pci_write_config(pdev, &acs_ctrl, 2, off); 4955433956dSTimothy McDaniel if (ret != 2) { 496*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d", 4975433956dSTimothy McDaniel __func__, (int)off); 4985433956dSTimothy McDaniel return ret; 4995433956dSTimothy McDaniel } 5005433956dSTimothy McDaniel 50105026b99SDavid Marchand off = acs_cap_offset + RTE_PCI_ACS_CTRL; 5025433956dSTimothy McDaniel if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2) 5035433956dSTimothy McDaniel acs_ctrl = 0; 5045433956dSTimothy McDaniel 50505026b99SDavid Marchand acs_mask = RTE_PCI_ACS_RR | RTE_PCI_ACS_CR; 50605026b99SDavid Marchand acs_mask |= RTE_PCI_ACS_EC; 5075433956dSTimothy McDaniel acs_ctrl &= ~acs_mask; 5085433956dSTimothy McDaniel 50905026b99SDavid Marchand off = acs_cap_offset + RTE_PCI_ACS_CTRL; 5105433956dSTimothy McDaniel ret = rte_pci_write_config(pdev, &acs_ctrl, 2, off); 5115433956dSTimothy McDaniel if (ret != 2) { 512*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d", 5135433956dSTimothy McDaniel __func__, (int)off); 5145433956dSTimothy McDaniel return ret; 5155433956dSTimothy McDaniel } 5165433956dSTimothy McDaniel } 5175433956dSTimothy McDaniel 5185a687833SAbdullah Sevincer /* Disable PASID if it is enabled by default, which 5195a687833SAbdullah Sevincer * breaks the DLB if enabled. 5205a687833SAbdullah Sevincer */ 521baaa446dSAbdullah Sevincer off = DLB2_PCI_PASID_CAP_OFFSET; 522baaa446dSAbdullah Sevincer if (rte_pci_pasid_set_state(pdev, off, false) < 0) { 523*f665790aSDavid Marchand DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d", 5245a687833SAbdullah Sevincer __func__, (int)off); 5255a687833SAbdullah Sevincer return -1; 5265a687833SAbdullah Sevincer } 5275a687833SAbdullah Sevincer 5285433956dSTimothy McDaniel return 0; 5295433956dSTimothy McDaniel } 530f3cad285STimothy McDaniel 531f3cad285STimothy McDaniel int 532f3cad285STimothy McDaniel dlb2_pf_create_sched_domain(struct dlb2_hw *hw, 533f3cad285STimothy McDaniel struct dlb2_create_sched_domain_args *args, 534f3cad285STimothy McDaniel struct dlb2_cmd_response *resp) 535f3cad285STimothy McDaniel { 536f3cad285STimothy McDaniel return dlb2_hw_create_sched_domain(hw, args, resp, NOT_VF_REQ, 537f3cad285STimothy McDaniel PF_ID_ZERO); 538f3cad285STimothy McDaniel } 539f3cad285STimothy McDaniel 540f3cad285STimothy McDaniel int 541f3cad285STimothy McDaniel dlb2_pf_reset_domain(struct dlb2_hw *hw, u32 id) 542f3cad285STimothy McDaniel { 543f3cad285STimothy McDaniel return dlb2_reset_domain(hw, id, NOT_VF_REQ, PF_ID_ZERO); 544f3cad285STimothy McDaniel } 5457e668e57STimothy McDaniel 5467e668e57STimothy McDaniel int 5477e668e57STimothy McDaniel dlb2_pf_create_ldb_queue(struct dlb2_hw *hw, 5487e668e57STimothy McDaniel u32 id, 5497e668e57STimothy McDaniel struct dlb2_create_ldb_queue_args *args, 5507e668e57STimothy McDaniel struct dlb2_cmd_response *resp) 5517e668e57STimothy McDaniel { 5527e668e57STimothy McDaniel return dlb2_hw_create_ldb_queue(hw, id, args, resp, NOT_VF_REQ, 5537e668e57STimothy McDaniel PF_ID_ZERO); 5547e668e57STimothy McDaniel } 5553a6d0c04STimothy McDaniel 5563a6d0c04STimothy McDaniel int 5573a6d0c04STimothy McDaniel dlb2_pf_create_ldb_port(struct dlb2_hw *hw, 5583a6d0c04STimothy McDaniel u32 id, 5593a6d0c04STimothy McDaniel struct dlb2_create_ldb_port_args *args, 5603a6d0c04STimothy McDaniel uintptr_t cq_dma_base, 5613a6d0c04STimothy McDaniel struct dlb2_cmd_response *resp) 5623a6d0c04STimothy McDaniel { 5633a6d0c04STimothy McDaniel return dlb2_hw_create_ldb_port(hw, id, args, 5643a6d0c04STimothy McDaniel cq_dma_base, 5653a6d0c04STimothy McDaniel resp, 5663a6d0c04STimothy McDaniel NOT_VF_REQ, 5673a6d0c04STimothy McDaniel PF_ID_ZERO); 5683a6d0c04STimothy McDaniel } 5693a6d0c04STimothy McDaniel 5703a6d0c04STimothy McDaniel int 5713a6d0c04STimothy McDaniel dlb2_pf_create_dir_port(struct dlb2_hw *hw, 5723a6d0c04STimothy McDaniel u32 id, 5733a6d0c04STimothy McDaniel struct dlb2_create_dir_port_args *args, 5743a6d0c04STimothy McDaniel uintptr_t cq_dma_base, 5753a6d0c04STimothy McDaniel struct dlb2_cmd_response *resp) 5763a6d0c04STimothy McDaniel { 5773a6d0c04STimothy McDaniel return dlb2_hw_create_dir_port(hw, id, args, 5783a6d0c04STimothy McDaniel cq_dma_base, 5793a6d0c04STimothy McDaniel resp, 5803a6d0c04STimothy McDaniel NOT_VF_REQ, 5813a6d0c04STimothy McDaniel PF_ID_ZERO); 5823a6d0c04STimothy McDaniel } 5831acd82c0STimothy McDaniel 5841acd82c0STimothy McDaniel int 5851acd82c0STimothy McDaniel dlb2_pf_create_dir_queue(struct dlb2_hw *hw, 5861acd82c0STimothy McDaniel u32 id, 5871acd82c0STimothy McDaniel struct dlb2_create_dir_queue_args *args, 5881acd82c0STimothy McDaniel struct dlb2_cmd_response *resp) 5891acd82c0STimothy McDaniel { 5901acd82c0STimothy McDaniel return dlb2_hw_create_dir_queue(hw, id, args, resp, NOT_VF_REQ, 5911acd82c0STimothy McDaniel PF_ID_ZERO); 5921acd82c0STimothy McDaniel } 59359e1a966STimothy McDaniel 59459e1a966STimothy McDaniel int 59559e1a966STimothy McDaniel dlb2_pf_start_domain(struct dlb2_hw *hw, 59659e1a966STimothy McDaniel u32 id, 59759e1a966STimothy McDaniel struct dlb2_start_domain_args *args, 59859e1a966STimothy McDaniel struct dlb2_cmd_response *resp) 59959e1a966STimothy McDaniel { 60059e1a966STimothy McDaniel return dlb2_hw_start_domain(hw, id, args, resp, NOT_VF_REQ, 60159e1a966STimothy McDaniel PF_ID_ZERO); 60259e1a966STimothy McDaniel } 603