xref: /dpdk/drivers/event/dlb2/pf/dlb2_main.c (revision f665790a5dbad7b645ff46f31d65e977324e7bfc)
15433956dSTimothy McDaniel /* SPDX-License-Identifier: BSD-3-Clause
25433956dSTimothy McDaniel  * Copyright(c) 2016-2020 Intel Corporation
35433956dSTimothy McDaniel  */
45433956dSTimothy McDaniel 
55433956dSTimothy McDaniel #include <stdint.h>
65433956dSTimothy McDaniel #include <stdbool.h>
75433956dSTimothy McDaniel #include <stdio.h>
85433956dSTimothy McDaniel #include <errno.h>
95433956dSTimothy McDaniel #include <assert.h>
105433956dSTimothy McDaniel #include <unistd.h>
115433956dSTimothy McDaniel #include <string.h>
125433956dSTimothy McDaniel 
135433956dSTimothy McDaniel #include <rte_malloc.h>
145433956dSTimothy McDaniel #include <rte_errno.h>
155433956dSTimothy McDaniel 
16d4a06a39STimothy McDaniel #include "base/dlb2_regs.h"
17037a6167STimothy McDaniel #include "base/dlb2_hw_types.h"
18dfdd11a8STimothy McDaniel #include "base/dlb2_resource.h"
195433956dSTimothy McDaniel #include "base/dlb2_osdep.h"
205433956dSTimothy McDaniel #include "dlb2_main.h"
215433956dSTimothy McDaniel #include "../dlb2_user.h"
225433956dSTimothy McDaniel #include "../dlb2_priv.h"
23e7c9971aSTimothy McDaniel #include "../dlb2_iface.h"
245433956dSTimothy McDaniel #include "../dlb2_inline_fns.h"
255433956dSTimothy McDaniel 
265433956dSTimothy McDaniel #define PF_ID_ZERO 0	/* PF ONLY! */
275433956dSTimothy McDaniel #define NO_OWNER_VF 0	/* PF ONLY! */
285433956dSTimothy McDaniel #define NOT_VF_REQ false /* PF ONLY! */
295a687833SAbdullah Sevincer #define DLB2_PCI_PASID_CAP_OFFSET        0x148   /* PASID capability offset */
305433956dSTimothy McDaniel 
315433956dSTimothy McDaniel static int
325433956dSTimothy McDaniel dlb2_pf_init_driver_state(struct dlb2_dev *dlb2_dev)
335433956dSTimothy McDaniel {
345433956dSTimothy McDaniel 	rte_spinlock_init(&dlb2_dev->resource_mutex);
355433956dSTimothy McDaniel 
365433956dSTimothy McDaniel 	return 0;
375433956dSTimothy McDaniel }
385433956dSTimothy McDaniel 
395433956dSTimothy McDaniel static void dlb2_pf_enable_pm(struct dlb2_dev *dlb2_dev)
405433956dSTimothy McDaniel {
410d65f8f9STimothy McDaniel 	int version;
420d65f8f9STimothy McDaniel 	version = DLB2_HW_DEVICE_FROM_PCI_ID(dlb2_dev->pdev);
430d65f8f9STimothy McDaniel 
440d65f8f9STimothy McDaniel 	dlb2_clr_pmcsr_disable(&dlb2_dev->hw, version);
455433956dSTimothy McDaniel }
465433956dSTimothy McDaniel 
475433956dSTimothy McDaniel #define DLB2_READY_RETRY_LIMIT 1000
480d65f8f9STimothy McDaniel static int dlb2_pf_wait_for_device_ready(struct dlb2_dev *dlb2_dev,
490d65f8f9STimothy McDaniel 					 int dlb_version)
505433956dSTimothy McDaniel {
515433956dSTimothy McDaniel 	u32 retries = 0;
525433956dSTimothy McDaniel 
535433956dSTimothy McDaniel 	/* Allow at least 1s for the device to become active after power-on */
545433956dSTimothy McDaniel 	for (retries = 0; retries < DLB2_READY_RETRY_LIMIT; retries++) {
550d65f8f9STimothy McDaniel 		u32 idle_val;
560d65f8f9STimothy McDaniel 		u32 idle_dlb_func_idle;
570d65f8f9STimothy McDaniel 		u32 pm_st_val;
580d65f8f9STimothy McDaniel 		u32 pm_st_pmsm;
595433956dSTimothy McDaniel 		u32 addr;
605433956dSTimothy McDaniel 
610d65f8f9STimothy McDaniel 		addr = DLB2_CM_CFG_PM_STATUS(dlb_version);
620d65f8f9STimothy McDaniel 		pm_st_val = DLB2_CSR_RD(&dlb2_dev->hw, addr);
630d65f8f9STimothy McDaniel 		addr = DLB2_CM_CFG_DIAGNOSTIC_IDLE_STATUS(dlb_version);
640d65f8f9STimothy McDaniel 		idle_val = DLB2_CSR_RD(&dlb2_dev->hw, addr);
650d65f8f9STimothy McDaniel 		idle_dlb_func_idle = idle_val &
660d65f8f9STimothy McDaniel 			DLB2_CM_CFG_DIAGNOSTIC_IDLE_STATUS_DLB_FUNC_IDLE;
670d65f8f9STimothy McDaniel 		pm_st_pmsm = pm_st_val & DLB2_CM_CFG_PM_STATUS_PMSM;
680d65f8f9STimothy McDaniel 		if (pm_st_pmsm && idle_dlb_func_idle)
695433956dSTimothy McDaniel 			break;
705433956dSTimothy McDaniel 
715433956dSTimothy McDaniel 		rte_delay_ms(1);
725433956dSTimothy McDaniel 	};
735433956dSTimothy McDaniel 
745433956dSTimothy McDaniel 	if (retries == DLB2_READY_RETRY_LIMIT) {
75*f665790aSDavid Marchand 		DLB2_LOG_ERR("[%s()] wait for device ready timed out",
765433956dSTimothy McDaniel 		       __func__);
775433956dSTimothy McDaniel 		return -1;
785433956dSTimothy McDaniel 	}
795433956dSTimothy McDaniel 
805433956dSTimothy McDaniel 	return 0;
815433956dSTimothy McDaniel }
825433956dSTimothy McDaniel 
835433956dSTimothy McDaniel struct dlb2_dev *
848d1d9070SAbdullah Sevincer dlb2_probe(struct rte_pci_device *pdev, const void *probe_args)
855433956dSTimothy McDaniel {
865433956dSTimothy McDaniel 	struct dlb2_dev *dlb2_dev;
875433956dSTimothy McDaniel 	int ret = 0;
880d65f8f9STimothy McDaniel 	int dlb_version = 0;
895433956dSTimothy McDaniel 
905433956dSTimothy McDaniel 	DLB2_INFO(dlb2_dev, "probe\n");
915433956dSTimothy McDaniel 
925433956dSTimothy McDaniel 	dlb2_dev = rte_malloc("DLB2_PF", sizeof(struct dlb2_dev),
935433956dSTimothy McDaniel 			      RTE_CACHE_LINE_SIZE);
945433956dSTimothy McDaniel 
955433956dSTimothy McDaniel 	if (dlb2_dev == NULL) {
965433956dSTimothy McDaniel 		ret = -ENOMEM;
975433956dSTimothy McDaniel 		goto dlb2_dev_malloc_fail;
985433956dSTimothy McDaniel 	}
995433956dSTimothy McDaniel 
1000d65f8f9STimothy McDaniel 	dlb_version = DLB2_HW_DEVICE_FROM_PCI_ID(pdev);
1010d65f8f9STimothy McDaniel 
1025433956dSTimothy McDaniel 	/* PCI Bus driver has already mapped bar space into process.
1035433956dSTimothy McDaniel 	 * Save off our IO register and FUNC addresses.
1045433956dSTimothy McDaniel 	 */
1055433956dSTimothy McDaniel 
1065433956dSTimothy McDaniel 	/* BAR 0 */
1075433956dSTimothy McDaniel 	if (pdev->mem_resource[0].addr == NULL) {
1085433956dSTimothy McDaniel 		DLB2_ERR(dlb2_dev, "probe: BAR 0 addr (csr_kva) is NULL\n");
1095433956dSTimothy McDaniel 		ret = -EINVAL;
1105433956dSTimothy McDaniel 		goto pci_mmap_bad_addr;
1115433956dSTimothy McDaniel 	}
1125433956dSTimothy McDaniel 	dlb2_dev->hw.func_kva = (void *)(uintptr_t)pdev->mem_resource[0].addr;
1135433956dSTimothy McDaniel 	dlb2_dev->hw.func_phys_addr = pdev->mem_resource[0].phys_addr;
1145433956dSTimothy McDaniel 
1155433956dSTimothy McDaniel 	DLB2_INFO(dlb2_dev, "DLB2 FUNC VA=%p, PA=%p, len=%p\n",
1165433956dSTimothy McDaniel 		  (void *)dlb2_dev->hw.func_kva,
1175433956dSTimothy McDaniel 		  (void *)dlb2_dev->hw.func_phys_addr,
1185433956dSTimothy McDaniel 		  (void *)(pdev->mem_resource[0].len));
1195433956dSTimothy McDaniel 
1205433956dSTimothy McDaniel 	/* BAR 2 */
1215433956dSTimothy McDaniel 	if (pdev->mem_resource[2].addr == NULL) {
1225433956dSTimothy McDaniel 		DLB2_ERR(dlb2_dev, "probe: BAR 2 addr (func_kva) is NULL\n");
1235433956dSTimothy McDaniel 		ret = -EINVAL;
1245433956dSTimothy McDaniel 		goto pci_mmap_bad_addr;
1255433956dSTimothy McDaniel 	}
1265433956dSTimothy McDaniel 	dlb2_dev->hw.csr_kva = (void *)(uintptr_t)pdev->mem_resource[2].addr;
1275433956dSTimothy McDaniel 	dlb2_dev->hw.csr_phys_addr = pdev->mem_resource[2].phys_addr;
1285433956dSTimothy McDaniel 
1295433956dSTimothy McDaniel 	DLB2_INFO(dlb2_dev, "DLB2 CSR VA=%p, PA=%p, len=%p\n",
1305433956dSTimothy McDaniel 		  (void *)dlb2_dev->hw.csr_kva,
1315433956dSTimothy McDaniel 		  (void *)dlb2_dev->hw.csr_phys_addr,
1325433956dSTimothy McDaniel 		  (void *)(pdev->mem_resource[2].len));
1335433956dSTimothy McDaniel 
1345433956dSTimothy McDaniel 	dlb2_dev->pdev = pdev;
1355433956dSTimothy McDaniel 
1365433956dSTimothy McDaniel 	/* PM enable must be done before any other MMIO accesses, and this
1375433956dSTimothy McDaniel 	 * setting is persistent across device reset.
1385433956dSTimothy McDaniel 	 */
1395433956dSTimothy McDaniel 	dlb2_pf_enable_pm(dlb2_dev);
1405433956dSTimothy McDaniel 
1410d65f8f9STimothy McDaniel 	ret = dlb2_pf_wait_for_device_ready(dlb2_dev, dlb_version);
1425433956dSTimothy McDaniel 	if (ret)
1435433956dSTimothy McDaniel 		goto wait_for_device_ready_fail;
1445433956dSTimothy McDaniel 
1458d1d9070SAbdullah Sevincer 	ret = dlb2_resource_probe(&dlb2_dev->hw, probe_args);
1468d1d9070SAbdullah Sevincer 	if (ret)
1478d1d9070SAbdullah Sevincer 		goto resource_probe_fail;
1488d1d9070SAbdullah Sevincer 
1495433956dSTimothy McDaniel 	ret = dlb2_pf_reset(dlb2_dev);
1505433956dSTimothy McDaniel 	if (ret)
1515433956dSTimothy McDaniel 		goto dlb2_reset_fail;
1525433956dSTimothy McDaniel 
1535433956dSTimothy McDaniel 	ret = dlb2_pf_init_driver_state(dlb2_dev);
1545433956dSTimothy McDaniel 	if (ret)
1555433956dSTimothy McDaniel 		goto init_driver_state_fail;
1565433956dSTimothy McDaniel 
1578d1d9070SAbdullah Sevincer 	ret = dlb2_resource_init(&dlb2_dev->hw, dlb_version, probe_args);
1585433956dSTimothy McDaniel 	if (ret)
1595433956dSTimothy McDaniel 		goto resource_init_fail;
1605433956dSTimothy McDaniel 
1615433956dSTimothy McDaniel 	return dlb2_dev;
1625433956dSTimothy McDaniel 
1635433956dSTimothy McDaniel resource_init_fail:
1645433956dSTimothy McDaniel 	dlb2_resource_free(&dlb2_dev->hw);
1655433956dSTimothy McDaniel init_driver_state_fail:
1665433956dSTimothy McDaniel dlb2_reset_fail:
1675433956dSTimothy McDaniel pci_mmap_bad_addr:
1688d1d9070SAbdullah Sevincer resource_probe_fail:
1695433956dSTimothy McDaniel wait_for_device_ready_fail:
1705433956dSTimothy McDaniel 	rte_free(dlb2_dev);
1715433956dSTimothy McDaniel dlb2_dev_malloc_fail:
1725433956dSTimothy McDaniel 	rte_errno = ret;
1735433956dSTimothy McDaniel 	return NULL;
1745433956dSTimothy McDaniel }
1755433956dSTimothy McDaniel 
1765433956dSTimothy McDaniel int
1775433956dSTimothy McDaniel dlb2_pf_reset(struct dlb2_dev *dlb2_dev)
1785433956dSTimothy McDaniel {
1795433956dSTimothy McDaniel 	int ret = 0;
1805433956dSTimothy McDaniel 	int i = 0;
1815433956dSTimothy McDaniel 	uint32_t dword[16];
1825433956dSTimothy McDaniel 	uint16_t cmd;
1835433956dSTimothy McDaniel 	off_t off;
1845433956dSTimothy McDaniel 
1855433956dSTimothy McDaniel 	uint16_t dev_ctl_word;
1865433956dSTimothy McDaniel 	uint16_t dev_ctl2_word;
1875433956dSTimothy McDaniel 	uint16_t lnk_word;
1885433956dSTimothy McDaniel 	uint16_t lnk_word2;
1895433956dSTimothy McDaniel 	uint16_t slt_word;
1905433956dSTimothy McDaniel 	uint16_t slt_word2;
1915433956dSTimothy McDaniel 	uint16_t rt_ctl_word;
1925433956dSTimothy McDaniel 	uint32_t pri_reqs_dword;
1935433956dSTimothy McDaniel 	uint16_t pri_ctrl_word;
1945433956dSTimothy McDaniel 
195a10b6e53SDavid Marchand 	off_t pcie_cap_offset;
1965433956dSTimothy McDaniel 	int pri_cap_offset;
197a10b6e53SDavid Marchand 	off_t msix_cap_offset;
1985433956dSTimothy McDaniel 	int err_cap_offset;
1995433956dSTimothy McDaniel 	int acs_cap_offset;
2005433956dSTimothy McDaniel 	int wait_count;
2015433956dSTimothy McDaniel 
2025433956dSTimothy McDaniel 	uint16_t devsta_busy_word;
2035433956dSTimothy McDaniel 	uint16_t devctl_word;
2045433956dSTimothy McDaniel 
2055433956dSTimothy McDaniel 	struct rte_pci_device *pdev = dlb2_dev->pdev;
2065433956dSTimothy McDaniel 
2075433956dSTimothy McDaniel 	/* Save PCI config state */
2085433956dSTimothy McDaniel 
2095433956dSTimothy McDaniel 	for (i = 0; i < 16; i++) {
2105433956dSTimothy McDaniel 		if (rte_pci_read_config(pdev, &dword[i], 4, i * 4) != 4)
2115433956dSTimothy McDaniel 			return ret;
2125433956dSTimothy McDaniel 	}
2135433956dSTimothy McDaniel 
214baa9c550SDavid Marchand 	pcie_cap_offset = rte_pci_find_capability(pdev, RTE_PCI_CAP_ID_EXP);
2155433956dSTimothy McDaniel 
2165433956dSTimothy McDaniel 	if (pcie_cap_offset < 0) {
217*f665790aSDavid Marchand 		DLB2_LOG_ERR("[%s()] failed to find the pcie capability",
2185433956dSTimothy McDaniel 		       __func__);
2195433956dSTimothy McDaniel 		return pcie_cap_offset;
2205433956dSTimothy McDaniel 	}
2215433956dSTimothy McDaniel 
222a00c150fSTimothy McDaniel 	off = pcie_cap_offset + RTE_PCI_EXP_DEVCTL;
2235433956dSTimothy McDaniel 	if (rte_pci_read_config(pdev, &dev_ctl_word, 2, off) != 2)
2245433956dSTimothy McDaniel 		dev_ctl_word = 0;
2255433956dSTimothy McDaniel 
2263a40df8dSDavid Marchand 	off = pcie_cap_offset + RTE_PCI_EXP_LNKCTL;
2275433956dSTimothy McDaniel 	if (rte_pci_read_config(pdev, &lnk_word, 2, off) != 2)
2285433956dSTimothy McDaniel 		lnk_word = 0;
2295433956dSTimothy McDaniel 
2303a40df8dSDavid Marchand 	off = pcie_cap_offset + RTE_PCI_EXP_SLTCTL;
2315433956dSTimothy McDaniel 	if (rte_pci_read_config(pdev, &slt_word, 2, off) != 2)
2325433956dSTimothy McDaniel 		slt_word = 0;
2335433956dSTimothy McDaniel 
2343a40df8dSDavid Marchand 	off = pcie_cap_offset + RTE_PCI_EXP_RTCTL;
2355433956dSTimothy McDaniel 	if (rte_pci_read_config(pdev, &rt_ctl_word, 2, off) != 2)
2365433956dSTimothy McDaniel 		rt_ctl_word = 0;
2375433956dSTimothy McDaniel 
2383a40df8dSDavid Marchand 	off = pcie_cap_offset + RTE_PCI_EXP_DEVCTL2;
2395433956dSTimothy McDaniel 	if (rte_pci_read_config(pdev, &dev_ctl2_word, 2, off) != 2)
2405433956dSTimothy McDaniel 		dev_ctl2_word = 0;
2415433956dSTimothy McDaniel 
2423a40df8dSDavid Marchand 	off = pcie_cap_offset + RTE_PCI_EXP_LNKCTL2;
2435433956dSTimothy McDaniel 	if (rte_pci_read_config(pdev, &lnk_word2, 2, off) != 2)
2445433956dSTimothy McDaniel 		lnk_word2 = 0;
2455433956dSTimothy McDaniel 
2463a40df8dSDavid Marchand 	off = pcie_cap_offset + RTE_PCI_EXP_SLTCTL2;
2475433956dSTimothy McDaniel 	if (rte_pci_read_config(pdev, &slt_word2, 2, off) != 2)
2485433956dSTimothy McDaniel 		slt_word2 = 0;
2495433956dSTimothy McDaniel 
2500138cf92SDavid Marchand 	off = RTE_PCI_EXT_CAP_ID_PRI;
251a00c150fSTimothy McDaniel 	pri_cap_offset = rte_pci_find_ext_capability(pdev, off);
2525433956dSTimothy McDaniel 
2535433956dSTimothy McDaniel 	if (pri_cap_offset >= 0) {
2544ac34947SDavid Marchand 		off = pri_cap_offset + RTE_PCI_PRI_ALLOC_REQ;
2555433956dSTimothy McDaniel 		if (rte_pci_read_config(pdev, &pri_reqs_dword, 4, off) != 4)
2565433956dSTimothy McDaniel 			pri_reqs_dword = 0;
2575433956dSTimothy McDaniel 	}
2585433956dSTimothy McDaniel 
2595433956dSTimothy McDaniel 	/* clear the PCI command register before issuing the FLR */
2605433956dSTimothy McDaniel 
261c89450cbSDavid Marchand 	off = RTE_PCI_COMMAND;
2625433956dSTimothy McDaniel 	cmd = 0;
2635433956dSTimothy McDaniel 	if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
264*f665790aSDavid Marchand 		DLB2_LOG_ERR("[%s()] failed to write the pci command",
2655433956dSTimothy McDaniel 		       __func__);
2665433956dSTimothy McDaniel 		return ret;
2675433956dSTimothy McDaniel 	}
2685433956dSTimothy McDaniel 
2695433956dSTimothy McDaniel 	/* issue the FLR */
2705433956dSTimothy McDaniel 	for (wait_count = 0; wait_count < 4; wait_count++) {
2715433956dSTimothy McDaniel 		int sleep_time;
2725433956dSTimothy McDaniel 
2733a40df8dSDavid Marchand 		off = pcie_cap_offset + RTE_PCI_EXP_DEVSTA;
2745433956dSTimothy McDaniel 		ret = rte_pci_read_config(pdev, &devsta_busy_word, 2, off);
2755433956dSTimothy McDaniel 		if (ret != 2) {
276*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to read the pci device status",
2775433956dSTimothy McDaniel 			       __func__);
2785433956dSTimothy McDaniel 			return ret;
2795433956dSTimothy McDaniel 		}
2805433956dSTimothy McDaniel 
2813a40df8dSDavid Marchand 		if (!(devsta_busy_word & RTE_PCI_EXP_DEVSTA_TRPND))
2825433956dSTimothy McDaniel 			break;
2835433956dSTimothy McDaniel 
2845433956dSTimothy McDaniel 		sleep_time = (1 << (wait_count)) * 100;
2855433956dSTimothy McDaniel 		rte_delay_ms(sleep_time);
2865433956dSTimothy McDaniel 	}
2875433956dSTimothy McDaniel 
2885433956dSTimothy McDaniel 	if (wait_count == 4) {
289*f665790aSDavid Marchand 		DLB2_LOG_ERR("[%s()] wait for pci pending transactions timed out",
2905433956dSTimothy McDaniel 		       __func__);
2915433956dSTimothy McDaniel 		return -1;
2925433956dSTimothy McDaniel 	}
2935433956dSTimothy McDaniel 
294a00c150fSTimothy McDaniel 	off = pcie_cap_offset + RTE_PCI_EXP_DEVCTL;
2955433956dSTimothy McDaniel 	ret = rte_pci_read_config(pdev, &devctl_word, 2, off);
2965433956dSTimothy McDaniel 	if (ret != 2) {
297*f665790aSDavid Marchand 		DLB2_LOG_ERR("[%s()] failed to read the pcie device control",
2985433956dSTimothy McDaniel 		       __func__);
2995433956dSTimothy McDaniel 		return ret;
3005433956dSTimothy McDaniel 	}
3015433956dSTimothy McDaniel 
3023a40df8dSDavid Marchand 	devctl_word |= RTE_PCI_EXP_DEVCTL_BCR_FLR;
3035433956dSTimothy McDaniel 
3045433956dSTimothy McDaniel 	ret = rte_pci_write_config(pdev, &devctl_word, 2, off);
3055433956dSTimothy McDaniel 	if (ret != 2) {
306*f665790aSDavid Marchand 		DLB2_LOG_ERR("[%s()] failed to write the pcie device control",
3075433956dSTimothy McDaniel 		       __func__);
3085433956dSTimothy McDaniel 		return ret;
3095433956dSTimothy McDaniel 	}
3105433956dSTimothy McDaniel 
3115433956dSTimothy McDaniel 	rte_delay_ms(100);
3125433956dSTimothy McDaniel 
3135433956dSTimothy McDaniel 	/* Restore PCI config state */
3145433956dSTimothy McDaniel 
3155433956dSTimothy McDaniel 	if (pcie_cap_offset >= 0) {
316a00c150fSTimothy McDaniel 		off = pcie_cap_offset + RTE_PCI_EXP_DEVCTL;
3175433956dSTimothy McDaniel 		ret = rte_pci_write_config(pdev, &dev_ctl_word, 2, off);
3185433956dSTimothy McDaniel 		if (ret != 2) {
319*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to write the pcie device control at offset %d",
3205433956dSTimothy McDaniel 				__func__, (int)off);
3215433956dSTimothy McDaniel 			return ret;
3225433956dSTimothy McDaniel 		}
3235433956dSTimothy McDaniel 
3243a40df8dSDavid Marchand 		off = pcie_cap_offset + RTE_PCI_EXP_LNKCTL;
3255433956dSTimothy McDaniel 		ret = rte_pci_write_config(pdev, &lnk_word, 2, off);
3265433956dSTimothy McDaniel 		if (ret != 2) {
327*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d",
3285433956dSTimothy McDaniel 				__func__, (int)off);
3295433956dSTimothy McDaniel 			return ret;
3305433956dSTimothy McDaniel 		}
3315433956dSTimothy McDaniel 
3323a40df8dSDavid Marchand 		off = pcie_cap_offset + RTE_PCI_EXP_SLTCTL;
3335433956dSTimothy McDaniel 		ret = rte_pci_write_config(pdev, &slt_word, 2, off);
3345433956dSTimothy McDaniel 		if (ret != 2) {
335*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d",
3365433956dSTimothy McDaniel 				__func__, (int)off);
3375433956dSTimothy McDaniel 			return ret;
3385433956dSTimothy McDaniel 		}
3395433956dSTimothy McDaniel 
3403a40df8dSDavid Marchand 		off = pcie_cap_offset + RTE_PCI_EXP_RTCTL;
3415433956dSTimothy McDaniel 		ret = rte_pci_write_config(pdev, &rt_ctl_word, 2, off);
3425433956dSTimothy McDaniel 		if (ret != 2) {
343*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d",
3445433956dSTimothy McDaniel 				__func__, (int)off);
3455433956dSTimothy McDaniel 			return ret;
3465433956dSTimothy McDaniel 		}
3475433956dSTimothy McDaniel 
3483a40df8dSDavid Marchand 		off = pcie_cap_offset + RTE_PCI_EXP_DEVCTL2;
3495433956dSTimothy McDaniel 		ret = rte_pci_write_config(pdev, &dev_ctl2_word, 2, off);
3505433956dSTimothy McDaniel 		if (ret != 2) {
351*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d",
3525433956dSTimothy McDaniel 				__func__, (int)off);
3535433956dSTimothy McDaniel 			return ret;
3545433956dSTimothy McDaniel 		}
3555433956dSTimothy McDaniel 
3563a40df8dSDavid Marchand 		off = pcie_cap_offset + RTE_PCI_EXP_LNKCTL2;
3575433956dSTimothy McDaniel 		ret = rte_pci_write_config(pdev, &lnk_word2, 2, off);
3585433956dSTimothy McDaniel 		if (ret != 2) {
359*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d",
3605433956dSTimothy McDaniel 				__func__, (int)off);
3615433956dSTimothy McDaniel 			return ret;
3625433956dSTimothy McDaniel 		}
3635433956dSTimothy McDaniel 
3643a40df8dSDavid Marchand 		off = pcie_cap_offset + RTE_PCI_EXP_SLTCTL2;
3655433956dSTimothy McDaniel 		ret = rte_pci_write_config(pdev, &slt_word2, 2, off);
3665433956dSTimothy McDaniel 		if (ret != 2) {
367*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d",
3685433956dSTimothy McDaniel 				__func__, (int)off);
3695433956dSTimothy McDaniel 			return ret;
3705433956dSTimothy McDaniel 		}
3715433956dSTimothy McDaniel 	}
3725433956dSTimothy McDaniel 
3735433956dSTimothy McDaniel 	if (pri_cap_offset >= 0) {
3744ac34947SDavid Marchand 		pri_ctrl_word = RTE_PCI_PRI_CTRL_ENABLE;
3755433956dSTimothy McDaniel 
3764ac34947SDavid Marchand 		off = pri_cap_offset + RTE_PCI_PRI_ALLOC_REQ;
3775433956dSTimothy McDaniel 		ret = rte_pci_write_config(pdev, &pri_reqs_dword, 4, off);
3785433956dSTimothy McDaniel 		if (ret != 4) {
379*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d",
3805433956dSTimothy McDaniel 				__func__, (int)off);
3815433956dSTimothy McDaniel 			return ret;
3825433956dSTimothy McDaniel 		}
3835433956dSTimothy McDaniel 
3844ac34947SDavid Marchand 		off = pri_cap_offset + RTE_PCI_PRI_CTRL;
3855433956dSTimothy McDaniel 		ret = rte_pci_write_config(pdev, &pri_ctrl_word, 2, off);
3865433956dSTimothy McDaniel 		if (ret != 2) {
387*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d",
3885433956dSTimothy McDaniel 				__func__, (int)off);
3895433956dSTimothy McDaniel 			return ret;
3905433956dSTimothy McDaniel 		}
3915433956dSTimothy McDaniel 	}
3925433956dSTimothy McDaniel 
393a00c150fSTimothy McDaniel 	off = RTE_PCI_EXT_CAP_ID_ERR;
394a00c150fSTimothy McDaniel 	err_cap_offset = rte_pci_find_ext_capability(pdev, off);
3955433956dSTimothy McDaniel 
3965433956dSTimothy McDaniel 	if (err_cap_offset >= 0) {
3975433956dSTimothy McDaniel 		uint32_t tmp;
3985433956dSTimothy McDaniel 
399fce1a19aSDavid Marchand 		off = err_cap_offset + RTE_PCI_ERR_ROOT_STATUS;
4005433956dSTimothy McDaniel 		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
4015433956dSTimothy McDaniel 			tmp = 0;
4025433956dSTimothy McDaniel 
4035433956dSTimothy McDaniel 		ret = rte_pci_write_config(pdev, &tmp, 4, off);
4045433956dSTimothy McDaniel 		if (ret != 4) {
405*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d",
4065433956dSTimothy McDaniel 				__func__, (int)off);
4075433956dSTimothy McDaniel 			return ret;
4085433956dSTimothy McDaniel 		}
4095433956dSTimothy McDaniel 
410fce1a19aSDavid Marchand 		off = err_cap_offset + RTE_PCI_ERR_COR_STATUS;
4115433956dSTimothy McDaniel 		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
4125433956dSTimothy McDaniel 			tmp = 0;
4135433956dSTimothy McDaniel 
4145433956dSTimothy McDaniel 		ret = rte_pci_write_config(pdev, &tmp, 4, off);
4155433956dSTimothy McDaniel 		if (ret != 4) {
416*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d",
4175433956dSTimothy McDaniel 				__func__, (int)off);
4185433956dSTimothy McDaniel 			return ret;
4195433956dSTimothy McDaniel 		}
4205433956dSTimothy McDaniel 
421fce1a19aSDavid Marchand 		off = err_cap_offset + RTE_PCI_ERR_UNCOR_STATUS;
4225433956dSTimothy McDaniel 		if (rte_pci_read_config(pdev, &tmp, 4, off) != 4)
4235433956dSTimothy McDaniel 			tmp = 0;
4245433956dSTimothy McDaniel 
4255433956dSTimothy McDaniel 		ret = rte_pci_write_config(pdev, &tmp, 4, off);
4265433956dSTimothy McDaniel 		if (ret != 4) {
427*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d",
4285433956dSTimothy McDaniel 				__func__, (int)off);
4295433956dSTimothy McDaniel 			return ret;
4305433956dSTimothy McDaniel 		}
4315433956dSTimothy McDaniel 	}
4325433956dSTimothy McDaniel 
4335433956dSTimothy McDaniel 	for (i = 16; i > 0; i--) {
4345433956dSTimothy McDaniel 		off = (i - 1) * 4;
4355433956dSTimothy McDaniel 		ret = rte_pci_write_config(pdev, &dword[i - 1], 4, off);
4365433956dSTimothy McDaniel 		if (ret != 4) {
437*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d",
4385433956dSTimothy McDaniel 				__func__, (int)off);
4395433956dSTimothy McDaniel 			return ret;
4405433956dSTimothy McDaniel 		}
4415433956dSTimothy McDaniel 	}
4425433956dSTimothy McDaniel 
443c89450cbSDavid Marchand 	off = RTE_PCI_COMMAND;
4445433956dSTimothy McDaniel 	if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
445c89450cbSDavid Marchand 		cmd &= ~RTE_PCI_COMMAND_INTX_DISABLE;
4465433956dSTimothy McDaniel 		if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
447*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to write the pci command",
4485433956dSTimothy McDaniel 			       __func__);
4495433956dSTimothy McDaniel 			return ret;
4505433956dSTimothy McDaniel 		}
4515433956dSTimothy McDaniel 	}
4525433956dSTimothy McDaniel 
453baa9c550SDavid Marchand 	msix_cap_offset = rte_pci_find_capability(pdev, RTE_PCI_CAP_ID_MSIX);
4545433956dSTimothy McDaniel 	if (msix_cap_offset >= 0) {
4557bb1168dSDavid Marchand 		off = msix_cap_offset + RTE_PCI_MSIX_FLAGS;
4565433956dSTimothy McDaniel 		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
4577bb1168dSDavid Marchand 			cmd |= RTE_PCI_MSIX_FLAGS_ENABLE;
4587bb1168dSDavid Marchand 			cmd |= RTE_PCI_MSIX_FLAGS_MASKALL;
4595433956dSTimothy McDaniel 			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
460*f665790aSDavid Marchand 				DLB2_LOG_ERR("[%s()] failed to write msix flags",
4615433956dSTimothy McDaniel 				       __func__);
4625433956dSTimothy McDaniel 				return ret;
4635433956dSTimothy McDaniel 			}
4645433956dSTimothy McDaniel 		}
4655433956dSTimothy McDaniel 
4667bb1168dSDavid Marchand 		off = msix_cap_offset + RTE_PCI_MSIX_FLAGS;
4675433956dSTimothy McDaniel 		if (rte_pci_read_config(pdev, &cmd, 2, off) == 2) {
4687bb1168dSDavid Marchand 			cmd &= ~RTE_PCI_MSIX_FLAGS_MASKALL;
4695433956dSTimothy McDaniel 			if (rte_pci_write_config(pdev, &cmd, 2, off) != 2) {
470*f665790aSDavid Marchand 				DLB2_LOG_ERR("[%s()] failed to write msix flags",
4715433956dSTimothy McDaniel 				       __func__);
4725433956dSTimothy McDaniel 				return ret;
4735433956dSTimothy McDaniel 			}
4745433956dSTimothy McDaniel 		}
4755433956dSTimothy McDaniel 	}
4765433956dSTimothy McDaniel 
4770138cf92SDavid Marchand 	off = RTE_PCI_EXT_CAP_ID_ACS;
478a00c150fSTimothy McDaniel 	acs_cap_offset = rte_pci_find_ext_capability(pdev, off);
4795433956dSTimothy McDaniel 
4805433956dSTimothy McDaniel 	if (acs_cap_offset >= 0) {
4815433956dSTimothy McDaniel 		uint16_t acs_cap, acs_ctrl, acs_mask;
48205026b99SDavid Marchand 		off = acs_cap_offset + RTE_PCI_ACS_CAP;
4835433956dSTimothy McDaniel 		if (rte_pci_read_config(pdev, &acs_cap, 2, off) != 2)
4845433956dSTimothy McDaniel 			acs_cap = 0;
4855433956dSTimothy McDaniel 
48605026b99SDavid Marchand 		off = acs_cap_offset + RTE_PCI_ACS_CTRL;
4875433956dSTimothy McDaniel 		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
4885433956dSTimothy McDaniel 			acs_ctrl = 0;
4895433956dSTimothy McDaniel 
49005026b99SDavid Marchand 		acs_mask = RTE_PCI_ACS_SV | RTE_PCI_ACS_RR;
49105026b99SDavid Marchand 		acs_mask |= (RTE_PCI_ACS_CR | RTE_PCI_ACS_UF);
4925433956dSTimothy McDaniel 		acs_ctrl |= (acs_cap & acs_mask);
4935433956dSTimothy McDaniel 
4945433956dSTimothy McDaniel 		ret = rte_pci_write_config(pdev, &acs_ctrl, 2, off);
4955433956dSTimothy McDaniel 		if (ret != 2) {
496*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d",
4975433956dSTimothy McDaniel 				__func__, (int)off);
4985433956dSTimothy McDaniel 			return ret;
4995433956dSTimothy McDaniel 		}
5005433956dSTimothy McDaniel 
50105026b99SDavid Marchand 		off = acs_cap_offset + RTE_PCI_ACS_CTRL;
5025433956dSTimothy McDaniel 		if (rte_pci_read_config(pdev, &acs_ctrl, 2, off) != 2)
5035433956dSTimothy McDaniel 			acs_ctrl = 0;
5045433956dSTimothy McDaniel 
50505026b99SDavid Marchand 		acs_mask = RTE_PCI_ACS_RR | RTE_PCI_ACS_CR;
50605026b99SDavid Marchand 		acs_mask |= RTE_PCI_ACS_EC;
5075433956dSTimothy McDaniel 		acs_ctrl &= ~acs_mask;
5085433956dSTimothy McDaniel 
50905026b99SDavid Marchand 		off = acs_cap_offset + RTE_PCI_ACS_CTRL;
5105433956dSTimothy McDaniel 		ret = rte_pci_write_config(pdev, &acs_ctrl, 2, off);
5115433956dSTimothy McDaniel 		if (ret != 2) {
512*f665790aSDavid Marchand 			DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d",
5135433956dSTimothy McDaniel 				__func__, (int)off);
5145433956dSTimothy McDaniel 			return ret;
5155433956dSTimothy McDaniel 		}
5165433956dSTimothy McDaniel 	}
5175433956dSTimothy McDaniel 
5185a687833SAbdullah Sevincer 	/* Disable PASID if it is enabled by default, which
5195a687833SAbdullah Sevincer 	 * breaks the DLB if enabled.
5205a687833SAbdullah Sevincer 	 */
521baaa446dSAbdullah Sevincer 	off = DLB2_PCI_PASID_CAP_OFFSET;
522baaa446dSAbdullah Sevincer 	if (rte_pci_pasid_set_state(pdev, off, false) < 0) {
523*f665790aSDavid Marchand 		DLB2_LOG_ERR("[%s()] failed to write the pcie config space at offset %d",
5245a687833SAbdullah Sevincer 				__func__, (int)off);
5255a687833SAbdullah Sevincer 		return -1;
5265a687833SAbdullah Sevincer 	}
5275a687833SAbdullah Sevincer 
5285433956dSTimothy McDaniel 	return 0;
5295433956dSTimothy McDaniel }
530f3cad285STimothy McDaniel 
531f3cad285STimothy McDaniel int
532f3cad285STimothy McDaniel dlb2_pf_create_sched_domain(struct dlb2_hw *hw,
533f3cad285STimothy McDaniel 			    struct dlb2_create_sched_domain_args *args,
534f3cad285STimothy McDaniel 			    struct dlb2_cmd_response *resp)
535f3cad285STimothy McDaniel {
536f3cad285STimothy McDaniel 	return dlb2_hw_create_sched_domain(hw, args, resp, NOT_VF_REQ,
537f3cad285STimothy McDaniel 					   PF_ID_ZERO);
538f3cad285STimothy McDaniel }
539f3cad285STimothy McDaniel 
540f3cad285STimothy McDaniel int
541f3cad285STimothy McDaniel dlb2_pf_reset_domain(struct dlb2_hw *hw, u32 id)
542f3cad285STimothy McDaniel {
543f3cad285STimothy McDaniel 	return dlb2_reset_domain(hw, id, NOT_VF_REQ, PF_ID_ZERO);
544f3cad285STimothy McDaniel }
5457e668e57STimothy McDaniel 
5467e668e57STimothy McDaniel int
5477e668e57STimothy McDaniel dlb2_pf_create_ldb_queue(struct dlb2_hw *hw,
5487e668e57STimothy McDaniel 			 u32 id,
5497e668e57STimothy McDaniel 			 struct dlb2_create_ldb_queue_args *args,
5507e668e57STimothy McDaniel 			 struct dlb2_cmd_response *resp)
5517e668e57STimothy McDaniel {
5527e668e57STimothy McDaniel 	return dlb2_hw_create_ldb_queue(hw, id, args, resp, NOT_VF_REQ,
5537e668e57STimothy McDaniel 					PF_ID_ZERO);
5547e668e57STimothy McDaniel }
5553a6d0c04STimothy McDaniel 
5563a6d0c04STimothy McDaniel int
5573a6d0c04STimothy McDaniel dlb2_pf_create_ldb_port(struct dlb2_hw *hw,
5583a6d0c04STimothy McDaniel 			u32 id,
5593a6d0c04STimothy McDaniel 			struct dlb2_create_ldb_port_args *args,
5603a6d0c04STimothy McDaniel 			uintptr_t cq_dma_base,
5613a6d0c04STimothy McDaniel 			struct dlb2_cmd_response *resp)
5623a6d0c04STimothy McDaniel {
5633a6d0c04STimothy McDaniel 	return dlb2_hw_create_ldb_port(hw, id, args,
5643a6d0c04STimothy McDaniel 				       cq_dma_base,
5653a6d0c04STimothy McDaniel 				       resp,
5663a6d0c04STimothy McDaniel 				       NOT_VF_REQ,
5673a6d0c04STimothy McDaniel 				       PF_ID_ZERO);
5683a6d0c04STimothy McDaniel }
5693a6d0c04STimothy McDaniel 
5703a6d0c04STimothy McDaniel int
5713a6d0c04STimothy McDaniel dlb2_pf_create_dir_port(struct dlb2_hw *hw,
5723a6d0c04STimothy McDaniel 			u32 id,
5733a6d0c04STimothy McDaniel 			struct dlb2_create_dir_port_args *args,
5743a6d0c04STimothy McDaniel 			uintptr_t cq_dma_base,
5753a6d0c04STimothy McDaniel 			struct dlb2_cmd_response *resp)
5763a6d0c04STimothy McDaniel {
5773a6d0c04STimothy McDaniel 	return dlb2_hw_create_dir_port(hw, id, args,
5783a6d0c04STimothy McDaniel 				       cq_dma_base,
5793a6d0c04STimothy McDaniel 				       resp,
5803a6d0c04STimothy McDaniel 				       NOT_VF_REQ,
5813a6d0c04STimothy McDaniel 				       PF_ID_ZERO);
5823a6d0c04STimothy McDaniel }
5831acd82c0STimothy McDaniel 
5841acd82c0STimothy McDaniel int
5851acd82c0STimothy McDaniel dlb2_pf_create_dir_queue(struct dlb2_hw *hw,
5861acd82c0STimothy McDaniel 			 u32 id,
5871acd82c0STimothy McDaniel 			 struct dlb2_create_dir_queue_args *args,
5881acd82c0STimothy McDaniel 			 struct dlb2_cmd_response *resp)
5891acd82c0STimothy McDaniel {
5901acd82c0STimothy McDaniel 	return dlb2_hw_create_dir_queue(hw, id, args, resp, NOT_VF_REQ,
5911acd82c0STimothy McDaniel 					PF_ID_ZERO);
5921acd82c0STimothy McDaniel }
59359e1a966STimothy McDaniel 
59459e1a966STimothy McDaniel int
59559e1a966STimothy McDaniel dlb2_pf_start_domain(struct dlb2_hw *hw,
59659e1a966STimothy McDaniel 		     u32 id,
59759e1a966STimothy McDaniel 		     struct dlb2_start_domain_args *args,
59859e1a966STimothy McDaniel 		     struct dlb2_cmd_response *resp)
59959e1a966STimothy McDaniel {
60059e1a966STimothy McDaniel 	return dlb2_hw_start_domain(hw, id, args, resp, NOT_VF_REQ,
60159e1a966STimothy McDaniel 				    PF_ID_ZERO);
60259e1a966STimothy McDaniel }
603