xref: /dpdk/drivers/net/virtio/virtio_pci.c (revision a10b6e53fef4ef322d95a15db02d567b9d5daafd)
15566a3e3SBruce Richardson /* SPDX-License-Identifier: BSD-3-Clause
25566a3e3SBruce Richardson  * Copyright(c) 2010-2014 Intel Corporation
36c3169a3SBruce Richardson  */
46c3169a3SBruce Richardson #include <stdint.h>
56c3169a3SBruce Richardson 
6742bde12SBruce Richardson #ifdef RTE_EXEC_ENV_LINUX
7c52afa68SYuanhan Liu  #include <dirent.h>
8c52afa68SYuanhan Liu  #include <fcntl.h>
9c52afa68SYuanhan Liu #endif
10c52afa68SYuanhan Liu 
11631d4ee4SSantosh Shukla #include <rte_io.h>
12a04322f6SDavid Marchand #include <bus_driver.h>
13631d4ee4SSantosh Shukla 
146c3169a3SBruce Richardson #include "virtio_pci.h"
156c3169a3SBruce Richardson #include "virtio_logs.h"
16d5bbeefcSYuanhan Liu #include "virtqueue.h"
176c3169a3SBruce Richardson 
186ba1f63bSYuanhan Liu /*
196ba1f63bSYuanhan Liu  * Following macros are derived from linux/pci_regs.h, however,
206ba1f63bSYuanhan Liu  * we can't simply include that header here, as there is no such
216ba1f63bSYuanhan Liu  * file for non-Linux platform.
226ba1f63bSYuanhan Liu  */
236ba1f63bSYuanhan Liu #define PCI_CAP_ID_VNDR		0x09
24554b6d3eSJianfeng Tan #define PCI_CAP_ID_MSIX		0x11
256ba1f63bSYuanhan Liu 
26b8f04520SDavid Marchand /*
27b8f04520SDavid Marchand  * The remaining space is defined by each driver as the per-driver
28b8f04520SDavid Marchand  * configuration space.
29b8f04520SDavid Marchand  */
306a504290SMaxime Coquelin #define VIRTIO_PCI_CONFIG(dev) \
316a504290SMaxime Coquelin 		(((dev)->msix_status == VIRTIO_MSIX_ENABLED) ? 24 : 20)
32b86af7b1SYuanhan Liu 
33c8d4b02fSMaxime Coquelin struct virtio_pci_internal virtio_pci_internal[RTE_MAX_ETHPORTS];
34c8d4b02fSMaxime Coquelin 
357793d293SMaxime Coquelin #define PCI_MSIX_ENABLE 0x8000
367793d293SMaxime Coquelin 
377793d293SMaxime Coquelin static enum virtio_msix_status
387793d293SMaxime Coquelin vtpci_msix_detect(struct rte_pci_device *dev)
397793d293SMaxime Coquelin {
407793d293SMaxime Coquelin 	uint16_t flags;
41*a10b6e53SDavid Marchand 	off_t pos;
427793d293SMaxime Coquelin 
43*a10b6e53SDavid Marchand 	pos = rte_pci_find_capability(dev, PCI_CAP_ID_MSIX);
44*a10b6e53SDavid Marchand 	if (pos > 0 && rte_pci_read_config(dev, &flags, sizeof(flags),
45*a10b6e53SDavid Marchand 			pos + 2) == sizeof(flags)) {
467793d293SMaxime Coquelin 		if (flags & PCI_MSIX_ENABLE)
477793d293SMaxime Coquelin 			return VIRTIO_MSIX_ENABLED;
487793d293SMaxime Coquelin 		else
497793d293SMaxime Coquelin 			return VIRTIO_MSIX_DISABLED;
507793d293SMaxime Coquelin 	}
517793d293SMaxime Coquelin 
527793d293SMaxime Coquelin 	return VIRTIO_MSIX_NONE;
537793d293SMaxime Coquelin }
547793d293SMaxime Coquelin 
55281ccccbSDavid Marchand /*
56281ccccbSDavid Marchand  * Since we are in legacy mode:
57281ccccbSDavid Marchand  * http://ozlabs.org/~rusty/virtio-spec/virtio-0.9.5.pdf
58281ccccbSDavid Marchand  *
59281ccccbSDavid Marchand  * "Note that this is possible because while the virtio header is PCI (i.e.
60281ccccbSDavid Marchand  * little) endian, the device-specific region is encoded in the native endian of
61281ccccbSDavid Marchand  * the guest (where such distinction is applicable)."
62281ccccbSDavid Marchand  *
63281ccccbSDavid Marchand  * For powerpc which supports both, qemu supposes that cpu is big endian and
64281ccccbSDavid Marchand  * enforces this for the virtio-net stuff.
65281ccccbSDavid Marchand  */
66d5bbeefcSYuanhan Liu static void
67d5bbeefcSYuanhan Liu legacy_read_dev_config(struct virtio_hw *hw, size_t offset,
686c3169a3SBruce Richardson 		       void *dst, int length)
696c3169a3SBruce Richardson {
706a504290SMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
71281ccccbSDavid Marchand #ifdef RTE_ARCH_PPC_64
72281ccccbSDavid Marchand 	int size;
73281ccccbSDavid Marchand 
74281ccccbSDavid Marchand 	while (length > 0) {
75281ccccbSDavid Marchand 		if (length >= 4) {
76281ccccbSDavid Marchand 			size = 4;
773dcfe039SThomas Monjalon 			rte_pci_ioport_read(VTPCI_IO(hw), dst, size,
786a504290SMaxime Coquelin 				VIRTIO_PCI_CONFIG(dev) + offset);
79281ccccbSDavid Marchand 			*(uint32_t *)dst = rte_be_to_cpu_32(*(uint32_t *)dst);
80281ccccbSDavid Marchand 		} else if (length >= 2) {
81281ccccbSDavid Marchand 			size = 2;
823dcfe039SThomas Monjalon 			rte_pci_ioport_read(VTPCI_IO(hw), dst, size,
836a504290SMaxime Coquelin 				VIRTIO_PCI_CONFIG(dev) + offset);
84281ccccbSDavid Marchand 			*(uint16_t *)dst = rte_be_to_cpu_16(*(uint16_t *)dst);
85281ccccbSDavid Marchand 		} else {
86281ccccbSDavid Marchand 			size = 1;
873dcfe039SThomas Monjalon 			rte_pci_ioport_read(VTPCI_IO(hw), dst, size,
886a504290SMaxime Coquelin 				VIRTIO_PCI_CONFIG(dev) + offset);
89281ccccbSDavid Marchand 		}
90281ccccbSDavid Marchand 
91281ccccbSDavid Marchand 		dst = (char *)dst + size;
92281ccccbSDavid Marchand 		offset += size;
93281ccccbSDavid Marchand 		length -= size;
94281ccccbSDavid Marchand 	}
95281ccccbSDavid Marchand #else
963dcfe039SThomas Monjalon 	rte_pci_ioport_read(VTPCI_IO(hw), dst, length,
976a504290SMaxime Coquelin 		VIRTIO_PCI_CONFIG(dev) + offset);
98281ccccbSDavid Marchand #endif
996c3169a3SBruce Richardson }
1006c3169a3SBruce Richardson 
101d5bbeefcSYuanhan Liu static void
102d5bbeefcSYuanhan Liu legacy_write_dev_config(struct virtio_hw *hw, size_t offset,
103d5bbeefcSYuanhan Liu 			const void *src, int length)
1046c3169a3SBruce Richardson {
1056a504290SMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
106281ccccbSDavid Marchand #ifdef RTE_ARCH_PPC_64
107281ccccbSDavid Marchand 	union {
108281ccccbSDavid Marchand 		uint32_t u32;
109281ccccbSDavid Marchand 		uint16_t u16;
110281ccccbSDavid Marchand 	} tmp;
111281ccccbSDavid Marchand 	int size;
112281ccccbSDavid Marchand 
113281ccccbSDavid Marchand 	while (length > 0) {
114281ccccbSDavid Marchand 		if (length >= 4) {
115281ccccbSDavid Marchand 			size = 4;
116281ccccbSDavid Marchand 			tmp.u32 = rte_cpu_to_be_32(*(const uint32_t *)src);
1173dcfe039SThomas Monjalon 			rte_pci_ioport_write(VTPCI_IO(hw), &tmp.u32, size,
1186a504290SMaxime Coquelin 				VIRTIO_PCI_CONFIG(dev) + offset);
119281ccccbSDavid Marchand 		} else if (length >= 2) {
120281ccccbSDavid Marchand 			size = 2;
121281ccccbSDavid Marchand 			tmp.u16 = rte_cpu_to_be_16(*(const uint16_t *)src);
1223dcfe039SThomas Monjalon 			rte_pci_ioport_write(VTPCI_IO(hw), &tmp.u16, size,
1236a504290SMaxime Coquelin 				VIRTIO_PCI_CONFIG(dev) + offset);
124281ccccbSDavid Marchand 		} else {
125281ccccbSDavid Marchand 			size = 1;
1263dcfe039SThomas Monjalon 			rte_pci_ioport_write(VTPCI_IO(hw), src, size,
1276a504290SMaxime Coquelin 				VIRTIO_PCI_CONFIG(dev) + offset);
128281ccccbSDavid Marchand 		}
129281ccccbSDavid Marchand 
130281ccccbSDavid Marchand 		src = (const char *)src + size;
131281ccccbSDavid Marchand 		offset += size;
132281ccccbSDavid Marchand 		length -= size;
133281ccccbSDavid Marchand 	}
134281ccccbSDavid Marchand #else
1353dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), src, length,
1366a504290SMaxime Coquelin 		VIRTIO_PCI_CONFIG(dev) + offset);
137281ccccbSDavid Marchand #endif
1386c3169a3SBruce Richardson }
1396c3169a3SBruce Richardson 
1403891f233SYuanhan Liu static uint64_t
141d5bbeefcSYuanhan Liu legacy_get_features(struct virtio_hw *hw)
142d5bbeefcSYuanhan Liu {
14336ea36efSYuanhan Liu 	uint32_t dst;
144b8f04520SDavid Marchand 
1453dcfe039SThomas Monjalon 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 4, VIRTIO_PCI_HOST_FEATURES);
146b8f04520SDavid Marchand 	return dst;
147d5bbeefcSYuanhan Liu }
148d5bbeefcSYuanhan Liu 
149d5bbeefcSYuanhan Liu static void
1503891f233SYuanhan Liu legacy_set_features(struct virtio_hw *hw, uint64_t features)
151d5bbeefcSYuanhan Liu {
1523891f233SYuanhan Liu 	if ((features >> 32) != 0) {
1533891f233SYuanhan Liu 		PMD_DRV_LOG(ERR,
1543891f233SYuanhan Liu 			"only 32 bit features are allowed for legacy virtio!");
1553891f233SYuanhan Liu 		return;
1563891f233SYuanhan Liu 	}
1573dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &features, 4,
158b8f04520SDavid Marchand 		VIRTIO_PCI_GUEST_FEATURES);
159d5bbeefcSYuanhan Liu }
160d5bbeefcSYuanhan Liu 
161cbb135b3SMaxime Coquelin static int
162cbb135b3SMaxime Coquelin legacy_features_ok(struct virtio_hw *hw __rte_unused)
163cbb135b3SMaxime Coquelin {
164cbb135b3SMaxime Coquelin 	return 0;
165cbb135b3SMaxime Coquelin }
166cbb135b3SMaxime Coquelin 
167d5bbeefcSYuanhan Liu static uint8_t
168d5bbeefcSYuanhan Liu legacy_get_status(struct virtio_hw *hw)
169d5bbeefcSYuanhan Liu {
170b8f04520SDavid Marchand 	uint8_t dst;
171b8f04520SDavid Marchand 
1723dcfe039SThomas Monjalon 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 1, VIRTIO_PCI_STATUS);
173b8f04520SDavid Marchand 	return dst;
174d5bbeefcSYuanhan Liu }
175d5bbeefcSYuanhan Liu 
176d5bbeefcSYuanhan Liu static void
177d5bbeefcSYuanhan Liu legacy_set_status(struct virtio_hw *hw, uint8_t status)
178d5bbeefcSYuanhan Liu {
1793dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &status, 1, VIRTIO_PCI_STATUS);
180d5bbeefcSYuanhan Liu }
181d5bbeefcSYuanhan Liu 
182d5bbeefcSYuanhan Liu static uint8_t
183d5bbeefcSYuanhan Liu legacy_get_isr(struct virtio_hw *hw)
184d5bbeefcSYuanhan Liu {
185b8f04520SDavid Marchand 	uint8_t dst;
186b8f04520SDavid Marchand 
1873dcfe039SThomas Monjalon 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 1, VIRTIO_PCI_ISR);
188b8f04520SDavid Marchand 	return dst;
189d5bbeefcSYuanhan Liu }
190d5bbeefcSYuanhan Liu 
1917be78d02SJosh Soref /* Enable one vector (0) for Link State Interrupt */
192d5bbeefcSYuanhan Liu static uint16_t
193d5bbeefcSYuanhan Liu legacy_set_config_irq(struct virtio_hw *hw, uint16_t vec)
194d5bbeefcSYuanhan Liu {
195b8f04520SDavid Marchand 	uint16_t dst;
196b8f04520SDavid Marchand 
1973dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &vec, 2, VIRTIO_MSI_CONFIG_VECTOR);
1983dcfe039SThomas Monjalon 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 2, VIRTIO_MSI_CONFIG_VECTOR);
199b8f04520SDavid Marchand 	return dst;
200d5bbeefcSYuanhan Liu }
201d5bbeefcSYuanhan Liu 
202d5bbeefcSYuanhan Liu static uint16_t
203c49526acSJianfeng Tan legacy_set_queue_irq(struct virtio_hw *hw, struct virtqueue *vq, uint16_t vec)
204c49526acSJianfeng Tan {
205c49526acSJianfeng Tan 	uint16_t dst;
206c49526acSJianfeng Tan 
2073dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2,
208c49526acSJianfeng Tan 		VIRTIO_PCI_QUEUE_SEL);
2093dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &vec, 2, VIRTIO_MSI_QUEUE_VECTOR);
2103dcfe039SThomas Monjalon 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 2, VIRTIO_MSI_QUEUE_VECTOR);
211c49526acSJianfeng Tan 	return dst;
212c49526acSJianfeng Tan }
213c49526acSJianfeng Tan 
214c49526acSJianfeng Tan static uint16_t
215d5bbeefcSYuanhan Liu legacy_get_queue_num(struct virtio_hw *hw, uint16_t queue_id)
216d5bbeefcSYuanhan Liu {
217b8f04520SDavid Marchand 	uint16_t dst;
218b8f04520SDavid Marchand 
2193dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &queue_id, 2, VIRTIO_PCI_QUEUE_SEL);
2203dcfe039SThomas Monjalon 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 2, VIRTIO_PCI_QUEUE_NUM);
221b8f04520SDavid Marchand 	return dst;
222d5bbeefcSYuanhan Liu }
223d5bbeefcSYuanhan Liu 
224595454c5SJianfeng Tan static int
225d5bbeefcSYuanhan Liu legacy_setup_queue(struct virtio_hw *hw, struct virtqueue *vq)
226d5bbeefcSYuanhan Liu {
227b8f04520SDavid Marchand 	uint32_t src;
228d5bbeefcSYuanhan Liu 
2298b76b3a0SDavid Marchand 	/* Virtio PCI device VIRTIO_PCI_QUEUE_PFN register is 32bit,
2308b76b3a0SDavid Marchand 	 * and only accepts 32 bit page frame number.
2318b76b3a0SDavid Marchand 	 * Check if the allocated physical memory exceeds 16TB.
2328b76b3a0SDavid Marchand 	 */
2338b76b3a0SDavid Marchand 	if ((vq->vq_ring_mem + vq->vq_ring_size - 1) >>
2348b76b3a0SDavid Marchand 			(VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) {
2358b76b3a0SDavid Marchand 		PMD_INIT_LOG(ERR, "vring address shouldn't be above 16TB!");
236595454c5SJianfeng Tan 		return -1;
2378b76b3a0SDavid Marchand 	}
238595454c5SJianfeng Tan 
2393dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2,
240b8f04520SDavid Marchand 		VIRTIO_PCI_QUEUE_SEL);
24101ad44fdSHuawei Xie 	src = vq->vq_ring_mem >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
2423dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &src, 4, VIRTIO_PCI_QUEUE_PFN);
243595454c5SJianfeng Tan 
244595454c5SJianfeng Tan 	return 0;
245d5bbeefcSYuanhan Liu }
246d5bbeefcSYuanhan Liu 
247d5bbeefcSYuanhan Liu static void
248d5bbeefcSYuanhan Liu legacy_del_queue(struct virtio_hw *hw, struct virtqueue *vq)
249d5bbeefcSYuanhan Liu {
250b8f04520SDavid Marchand 	uint32_t src = 0;
251d5bbeefcSYuanhan Liu 
2523dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2,
253b8f04520SDavid Marchand 		VIRTIO_PCI_QUEUE_SEL);
2543dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &src, 4, VIRTIO_PCI_QUEUE_PFN);
255d5bbeefcSYuanhan Liu }
256d5bbeefcSYuanhan Liu 
257d5bbeefcSYuanhan Liu static void
258d5bbeefcSYuanhan Liu legacy_notify_queue(struct virtio_hw *hw, struct virtqueue *vq)
259d5bbeefcSYuanhan Liu {
2603dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2,
261b8f04520SDavid Marchand 		VIRTIO_PCI_QUEUE_NOTIFY);
262d5bbeefcSYuanhan Liu }
263d5bbeefcSYuanhan Liu 
2647793d293SMaxime Coquelin static void
2657793d293SMaxime Coquelin legacy_intr_detect(struct virtio_hw *hw)
2667793d293SMaxime Coquelin {
267c8d4b02fSMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
268c8d4b02fSMaxime Coquelin 
269aa0d4b8aSMaxime Coquelin 	dev->msix_status = vtpci_msix_detect(VTPCI_DEV(hw));
2706a504290SMaxime Coquelin 	hw->intr_lsc = !!dev->msix_status;
2717793d293SMaxime Coquelin }
2727793d293SMaxime Coquelin 
273f12908c8SMaxime Coquelin static int
274f12908c8SMaxime Coquelin legacy_dev_close(struct virtio_hw *hw)
275f12908c8SMaxime Coquelin {
276aa0d4b8aSMaxime Coquelin 	rte_pci_unmap_device(VTPCI_DEV(hw));
277f12908c8SMaxime Coquelin 	rte_pci_ioport_unmap(VTPCI_IO(hw));
278f12908c8SMaxime Coquelin 
279f12908c8SMaxime Coquelin 	return 0;
280f12908c8SMaxime Coquelin }
281f12908c8SMaxime Coquelin 
282f8b60756SMaxime Coquelin const struct virtio_ops legacy_ops = {
283d5bbeefcSYuanhan Liu 	.read_dev_cfg	= legacy_read_dev_config,
284d5bbeefcSYuanhan Liu 	.write_dev_cfg	= legacy_write_dev_config,
285d5bbeefcSYuanhan Liu 	.get_status	= legacy_get_status,
286d5bbeefcSYuanhan Liu 	.set_status	= legacy_set_status,
287d5bbeefcSYuanhan Liu 	.get_features	= legacy_get_features,
288d5bbeefcSYuanhan Liu 	.set_features	= legacy_set_features,
289cbb135b3SMaxime Coquelin 	.features_ok	= legacy_features_ok,
290d5bbeefcSYuanhan Liu 	.get_isr	= legacy_get_isr,
291d5bbeefcSYuanhan Liu 	.set_config_irq	= legacy_set_config_irq,
292c49526acSJianfeng Tan 	.set_queue_irq  = legacy_set_queue_irq,
293d5bbeefcSYuanhan Liu 	.get_queue_num	= legacy_get_queue_num,
294d5bbeefcSYuanhan Liu 	.setup_queue	= legacy_setup_queue,
295d5bbeefcSYuanhan Liu 	.del_queue	= legacy_del_queue,
296d5bbeefcSYuanhan Liu 	.notify_queue	= legacy_notify_queue,
2977793d293SMaxime Coquelin 	.intr_detect	= legacy_intr_detect,
298f12908c8SMaxime Coquelin 	.dev_close	= legacy_dev_close,
299d5bbeefcSYuanhan Liu };
300d5bbeefcSYuanhan Liu 
3016ba1f63bSYuanhan Liu static inline void
3026ba1f63bSYuanhan Liu io_write64_twopart(uint64_t val, uint32_t *lo, uint32_t *hi)
3036ba1f63bSYuanhan Liu {
304631d4ee4SSantosh Shukla 	rte_write32(val & ((1ULL << 32) - 1), lo);
305631d4ee4SSantosh Shukla 	rte_write32(val >> 32,		     hi);
3066ba1f63bSYuanhan Liu }
3076ba1f63bSYuanhan Liu 
3086ba1f63bSYuanhan Liu static void
3096ba1f63bSYuanhan Liu modern_read_dev_config(struct virtio_hw *hw, size_t offset,
3106ba1f63bSYuanhan Liu 		       void *dst, int length)
3116ba1f63bSYuanhan Liu {
312266ece29SMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
3136ba1f63bSYuanhan Liu 	int i;
3146ba1f63bSYuanhan Liu 	uint8_t *p;
3156ba1f63bSYuanhan Liu 	uint8_t old_gen, new_gen;
3166ba1f63bSYuanhan Liu 
3176ba1f63bSYuanhan Liu 	do {
318266ece29SMaxime Coquelin 		old_gen = rte_read8(&dev->common_cfg->config_generation);
3196ba1f63bSYuanhan Liu 
3206ba1f63bSYuanhan Liu 		p = dst;
3216ba1f63bSYuanhan Liu 		for (i = 0;  i < length; i++)
322266ece29SMaxime Coquelin 			*p++ = rte_read8((uint8_t *)dev->dev_cfg + offset + i);
3236ba1f63bSYuanhan Liu 
324266ece29SMaxime Coquelin 		new_gen = rte_read8(&dev->common_cfg->config_generation);
3256ba1f63bSYuanhan Liu 	} while (old_gen != new_gen);
3266ba1f63bSYuanhan Liu }
3276ba1f63bSYuanhan Liu 
3286ba1f63bSYuanhan Liu static void
3296ba1f63bSYuanhan Liu modern_write_dev_config(struct virtio_hw *hw, size_t offset,
3306ba1f63bSYuanhan Liu 			const void *src, int length)
3316ba1f63bSYuanhan Liu {
332266ece29SMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
3336ba1f63bSYuanhan Liu 	int i;
3346ba1f63bSYuanhan Liu 	const uint8_t *p = src;
3356ba1f63bSYuanhan Liu 
3366ba1f63bSYuanhan Liu 	for (i = 0;  i < length; i++)
337266ece29SMaxime Coquelin 		rte_write8((*p++), (((uint8_t *)dev->dev_cfg) + offset + i));
3386ba1f63bSYuanhan Liu }
3396ba1f63bSYuanhan Liu 
3406ba1f63bSYuanhan Liu static uint64_t
3416ba1f63bSYuanhan Liu modern_get_features(struct virtio_hw *hw)
3426ba1f63bSYuanhan Liu {
343266ece29SMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
3446ba1f63bSYuanhan Liu 	uint32_t features_lo, features_hi;
3456ba1f63bSYuanhan Liu 
346266ece29SMaxime Coquelin 	rte_write32(0, &dev->common_cfg->device_feature_select);
347266ece29SMaxime Coquelin 	features_lo = rte_read32(&dev->common_cfg->device_feature);
3486ba1f63bSYuanhan Liu 
349266ece29SMaxime Coquelin 	rte_write32(1, &dev->common_cfg->device_feature_select);
350266ece29SMaxime Coquelin 	features_hi = rte_read32(&dev->common_cfg->device_feature);
3516ba1f63bSYuanhan Liu 
3526ba1f63bSYuanhan Liu 	return ((uint64_t)features_hi << 32) | features_lo;
3536ba1f63bSYuanhan Liu }
3546ba1f63bSYuanhan Liu 
3556ba1f63bSYuanhan Liu static void
3566ba1f63bSYuanhan Liu modern_set_features(struct virtio_hw *hw, uint64_t features)
3576ba1f63bSYuanhan Liu {
358266ece29SMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
3596ba1f63bSYuanhan Liu 
360266ece29SMaxime Coquelin 	rte_write32(0, &dev->common_cfg->guest_feature_select);
361266ece29SMaxime Coquelin 	rte_write32(features & ((1ULL << 32) - 1),
362266ece29SMaxime Coquelin 		    &dev->common_cfg->guest_feature);
363266ece29SMaxime Coquelin 
364266ece29SMaxime Coquelin 	rte_write32(1, &dev->common_cfg->guest_feature_select);
365631d4ee4SSantosh Shukla 	rte_write32(features >> 32,
366266ece29SMaxime Coquelin 		    &dev->common_cfg->guest_feature);
3676ba1f63bSYuanhan Liu }
3686ba1f63bSYuanhan Liu 
369cbb135b3SMaxime Coquelin static int
370cbb135b3SMaxime Coquelin modern_features_ok(struct virtio_hw *hw)
371cbb135b3SMaxime Coquelin {
372b4f9a45aSMaxime Coquelin 	if (!virtio_with_feature(hw, VIRTIO_F_VERSION_1)) {
373f3854ebaSThomas Monjalon 		PMD_INIT_LOG(ERR, "Version 1+ required with modern devices");
374cbb135b3SMaxime Coquelin 		return -1;
375cbb135b3SMaxime Coquelin 	}
376cbb135b3SMaxime Coquelin 
377cbb135b3SMaxime Coquelin 	return 0;
378cbb135b3SMaxime Coquelin }
379cbb135b3SMaxime Coquelin 
3806ba1f63bSYuanhan Liu static uint8_t
3816ba1f63bSYuanhan Liu modern_get_status(struct virtio_hw *hw)
3826ba1f63bSYuanhan Liu {
383266ece29SMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
384266ece29SMaxime Coquelin 
385266ece29SMaxime Coquelin 	return rte_read8(&dev->common_cfg->device_status);
3866ba1f63bSYuanhan Liu }
3876ba1f63bSYuanhan Liu 
3886ba1f63bSYuanhan Liu static void
3896ba1f63bSYuanhan Liu modern_set_status(struct virtio_hw *hw, uint8_t status)
3906ba1f63bSYuanhan Liu {
391266ece29SMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
392266ece29SMaxime Coquelin 
393266ece29SMaxime Coquelin 	rte_write8(status, &dev->common_cfg->device_status);
3946ba1f63bSYuanhan Liu }
3956ba1f63bSYuanhan Liu 
3966ba1f63bSYuanhan Liu static uint8_t
3976ba1f63bSYuanhan Liu modern_get_isr(struct virtio_hw *hw)
3986ba1f63bSYuanhan Liu {
399266ece29SMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
400266ece29SMaxime Coquelin 
401266ece29SMaxime Coquelin 	return rte_read8(dev->isr);
4026ba1f63bSYuanhan Liu }
4036ba1f63bSYuanhan Liu 
4046ba1f63bSYuanhan Liu static uint16_t
4056ba1f63bSYuanhan Liu modern_set_config_irq(struct virtio_hw *hw, uint16_t vec)
4066ba1f63bSYuanhan Liu {
407266ece29SMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
408266ece29SMaxime Coquelin 
409266ece29SMaxime Coquelin 	rte_write16(vec, &dev->common_cfg->msix_config);
410266ece29SMaxime Coquelin 	return rte_read16(&dev->common_cfg->msix_config);
4116ba1f63bSYuanhan Liu }
4126ba1f63bSYuanhan Liu 
4136ba1f63bSYuanhan Liu static uint16_t
414c49526acSJianfeng Tan modern_set_queue_irq(struct virtio_hw *hw, struct virtqueue *vq, uint16_t vec)
415c49526acSJianfeng Tan {
416266ece29SMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
417266ece29SMaxime Coquelin 
418266ece29SMaxime Coquelin 	rte_write16(vq->vq_queue_index, &dev->common_cfg->queue_select);
419266ece29SMaxime Coquelin 	rte_write16(vec, &dev->common_cfg->queue_msix_vector);
420266ece29SMaxime Coquelin 	return rte_read16(&dev->common_cfg->queue_msix_vector);
421c49526acSJianfeng Tan }
422c49526acSJianfeng Tan 
423c49526acSJianfeng Tan static uint16_t
4246ba1f63bSYuanhan Liu modern_get_queue_num(struct virtio_hw *hw, uint16_t queue_id)
4256ba1f63bSYuanhan Liu {
426266ece29SMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
427266ece29SMaxime Coquelin 
428266ece29SMaxime Coquelin 	rte_write16(queue_id, &dev->common_cfg->queue_select);
429266ece29SMaxime Coquelin 	return rte_read16(&dev->common_cfg->queue_size);
4306ba1f63bSYuanhan Liu }
4316ba1f63bSYuanhan Liu 
432595454c5SJianfeng Tan static int
4336ba1f63bSYuanhan Liu modern_setup_queue(struct virtio_hw *hw, struct virtqueue *vq)
4346ba1f63bSYuanhan Liu {
435266ece29SMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
4366ba1f63bSYuanhan Liu 	uint64_t desc_addr, avail_addr, used_addr;
4376ba1f63bSYuanhan Liu 	uint16_t notify_off;
4386ba1f63bSYuanhan Liu 
43901ad44fdSHuawei Xie 	desc_addr = vq->vq_ring_mem;
4406ba1f63bSYuanhan Liu 	avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc);
4416ba1f63bSYuanhan Liu 	used_addr = RTE_ALIGN_CEIL(avail_addr + offsetof(struct vring_avail,
4426ba1f63bSYuanhan Liu 							 ring[vq->vq_nentries]),
443df968842SMaxime Coquelin 				   VIRTIO_VRING_ALIGN);
4446ba1f63bSYuanhan Liu 
445266ece29SMaxime Coquelin 	rte_write16(vq->vq_queue_index, &dev->common_cfg->queue_select);
4466ba1f63bSYuanhan Liu 
447266ece29SMaxime Coquelin 	io_write64_twopart(desc_addr, &dev->common_cfg->queue_desc_lo,
448266ece29SMaxime Coquelin 				      &dev->common_cfg->queue_desc_hi);
449266ece29SMaxime Coquelin 	io_write64_twopart(avail_addr, &dev->common_cfg->queue_avail_lo,
450266ece29SMaxime Coquelin 				       &dev->common_cfg->queue_avail_hi);
451266ece29SMaxime Coquelin 	io_write64_twopart(used_addr, &dev->common_cfg->queue_used_lo,
452266ece29SMaxime Coquelin 				      &dev->common_cfg->queue_used_hi);
4536ba1f63bSYuanhan Liu 
454266ece29SMaxime Coquelin 	notify_off = rte_read16(&dev->common_cfg->queue_notify_off);
455266ece29SMaxime Coquelin 	vq->notify_addr = (void *)((uint8_t *)dev->notify_base +
456266ece29SMaxime Coquelin 				notify_off * dev->notify_off_multiplier);
4576ba1f63bSYuanhan Liu 
458266ece29SMaxime Coquelin 	rte_write16(1, &dev->common_cfg->queue_enable);
4596ba1f63bSYuanhan Liu 
4606ba1f63bSYuanhan Liu 	PMD_INIT_LOG(DEBUG, "queue %u addresses:", vq->vq_queue_index);
4616ba1f63bSYuanhan Liu 	PMD_INIT_LOG(DEBUG, "\t desc_addr: %" PRIx64, desc_addr);
4626ba1f63bSYuanhan Liu 	PMD_INIT_LOG(DEBUG, "\t aval_addr: %" PRIx64, avail_addr);
4636ba1f63bSYuanhan Liu 	PMD_INIT_LOG(DEBUG, "\t used_addr: %" PRIx64, used_addr);
4646ba1f63bSYuanhan Liu 	PMD_INIT_LOG(DEBUG, "\t notify addr: %p (notify offset: %u)",
4656ba1f63bSYuanhan Liu 		vq->notify_addr, notify_off);
466595454c5SJianfeng Tan 
467595454c5SJianfeng Tan 	return 0;
4686ba1f63bSYuanhan Liu }
4696ba1f63bSYuanhan Liu 
4706ba1f63bSYuanhan Liu static void
4716ba1f63bSYuanhan Liu modern_del_queue(struct virtio_hw *hw, struct virtqueue *vq)
4726ba1f63bSYuanhan Liu {
473266ece29SMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
4746ba1f63bSYuanhan Liu 
475266ece29SMaxime Coquelin 	rte_write16(vq->vq_queue_index, &dev->common_cfg->queue_select);
4766ba1f63bSYuanhan Liu 
477266ece29SMaxime Coquelin 	io_write64_twopart(0, &dev->common_cfg->queue_desc_lo,
478266ece29SMaxime Coquelin 				  &dev->common_cfg->queue_desc_hi);
479266ece29SMaxime Coquelin 	io_write64_twopart(0, &dev->common_cfg->queue_avail_lo,
480266ece29SMaxime Coquelin 				  &dev->common_cfg->queue_avail_hi);
481266ece29SMaxime Coquelin 	io_write64_twopart(0, &dev->common_cfg->queue_used_lo,
482266ece29SMaxime Coquelin 				  &dev->common_cfg->queue_used_hi);
483266ece29SMaxime Coquelin 
484266ece29SMaxime Coquelin 	rte_write16(0, &dev->common_cfg->queue_enable);
4856ba1f63bSYuanhan Liu }
4866ba1f63bSYuanhan Liu 
4876ba1f63bSYuanhan Liu static void
4887e72f3ecSCheng Jiang modern_notify_queue(struct virtio_hw *hw, struct virtqueue *vq)
4896ba1f63bSYuanhan Liu {
4907e72f3ecSCheng Jiang 	uint32_t notify_data;
4917e72f3ecSCheng Jiang 
492b4f9a45aSMaxime Coquelin 	if (!virtio_with_feature(hw, VIRTIO_F_NOTIFICATION_DATA)) {
493518208f3SXiao Wang 		rte_write16(vq->vq_queue_index, vq->notify_addr);
4947e72f3ecSCheng Jiang 		return;
4957e72f3ecSCheng Jiang 	}
4967e72f3ecSCheng Jiang 
497b4f9a45aSMaxime Coquelin 	if (virtio_with_packed_queue(hw)) {
4987e72f3ecSCheng Jiang 		/*
4997e72f3ecSCheng Jiang 		 * Bit[0:15]: vq queue index
5007e72f3ecSCheng Jiang 		 * Bit[16:30]: avail index
5017e72f3ecSCheng Jiang 		 * Bit[31]: avail wrap counter
5027e72f3ecSCheng Jiang 		 */
5037e72f3ecSCheng Jiang 		notify_data = ((uint32_t)(!!(vq->vq_packed.cached_flags &
5047e72f3ecSCheng Jiang 				VRING_PACKED_DESC_F_AVAIL)) << 31) |
5057e72f3ecSCheng Jiang 				((uint32_t)vq->vq_avail_idx << 16) |
5067e72f3ecSCheng Jiang 				vq->vq_queue_index;
5077e72f3ecSCheng Jiang 	} else {
5087e72f3ecSCheng Jiang 		/*
5097e72f3ecSCheng Jiang 		 * Bit[0:15]: vq queue index
5107e72f3ecSCheng Jiang 		 * Bit[16:31]: avail index
5117e72f3ecSCheng Jiang 		 */
5127e72f3ecSCheng Jiang 		notify_data = ((uint32_t)vq->vq_avail_idx << 16) |
5137e72f3ecSCheng Jiang 				vq->vq_queue_index;
5147e72f3ecSCheng Jiang 	}
5157e72f3ecSCheng Jiang 	rte_write32(notify_data, vq->notify_addr);
5166ba1f63bSYuanhan Liu }
5176ba1f63bSYuanhan Liu 
5187793d293SMaxime Coquelin 
5197793d293SMaxime Coquelin 
5207793d293SMaxime Coquelin static void
5217793d293SMaxime Coquelin modern_intr_detect(struct virtio_hw *hw)
5227793d293SMaxime Coquelin {
523c8d4b02fSMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
524c8d4b02fSMaxime Coquelin 
525aa0d4b8aSMaxime Coquelin 	dev->msix_status = vtpci_msix_detect(VTPCI_DEV(hw));
5266a504290SMaxime Coquelin 	hw->intr_lsc = !!dev->msix_status;
5277793d293SMaxime Coquelin }
5287793d293SMaxime Coquelin 
529f12908c8SMaxime Coquelin static int
530f12908c8SMaxime Coquelin modern_dev_close(struct virtio_hw *hw)
531f12908c8SMaxime Coquelin {
532aa0d4b8aSMaxime Coquelin 	rte_pci_unmap_device(VTPCI_DEV(hw));
533f12908c8SMaxime Coquelin 
534f12908c8SMaxime Coquelin 	return 0;
535f12908c8SMaxime Coquelin }
536f12908c8SMaxime Coquelin 
537f8b60756SMaxime Coquelin const struct virtio_ops modern_ops = {
5386ba1f63bSYuanhan Liu 	.read_dev_cfg	= modern_read_dev_config,
5396ba1f63bSYuanhan Liu 	.write_dev_cfg	= modern_write_dev_config,
5406ba1f63bSYuanhan Liu 	.get_status	= modern_get_status,
5416ba1f63bSYuanhan Liu 	.set_status	= modern_set_status,
5426ba1f63bSYuanhan Liu 	.get_features	= modern_get_features,
5436ba1f63bSYuanhan Liu 	.set_features	= modern_set_features,
544cbb135b3SMaxime Coquelin 	.features_ok	= modern_features_ok,
5456ba1f63bSYuanhan Liu 	.get_isr	= modern_get_isr,
5466ba1f63bSYuanhan Liu 	.set_config_irq	= modern_set_config_irq,
547c49526acSJianfeng Tan 	.set_queue_irq  = modern_set_queue_irq,
5486ba1f63bSYuanhan Liu 	.get_queue_num	= modern_get_queue_num,
5496ba1f63bSYuanhan Liu 	.setup_queue	= modern_setup_queue,
5506ba1f63bSYuanhan Liu 	.del_queue	= modern_del_queue,
5516ba1f63bSYuanhan Liu 	.notify_queue	= modern_notify_queue,
5527793d293SMaxime Coquelin 	.intr_detect	= modern_intr_detect,
553f12908c8SMaxime Coquelin 	.dev_close	= modern_dev_close,
5546ba1f63bSYuanhan Liu };
5556ba1f63bSYuanhan Liu 
5566ba1f63bSYuanhan Liu static void *
5576ba1f63bSYuanhan Liu get_cfg_addr(struct rte_pci_device *dev, struct virtio_pci_cap *cap)
5586ba1f63bSYuanhan Liu {
5596ba1f63bSYuanhan Liu 	uint8_t  bar    = cap->bar;
5606ba1f63bSYuanhan Liu 	uint32_t length = cap->length;
5616ba1f63bSYuanhan Liu 	uint32_t offset = cap->offset;
5626ba1f63bSYuanhan Liu 	uint8_t *base;
5636ba1f63bSYuanhan Liu 
5640373ab9bSZhiyong Yang 	if (bar >= PCI_MAX_RESOURCE) {
5656ba1f63bSYuanhan Liu 		PMD_INIT_LOG(ERR, "invalid bar: %u", bar);
5666ba1f63bSYuanhan Liu 		return NULL;
5676ba1f63bSYuanhan Liu 	}
5686ba1f63bSYuanhan Liu 
5696ba1f63bSYuanhan Liu 	if (offset + length < offset) {
5706ba1f63bSYuanhan Liu 		PMD_INIT_LOG(ERR, "offset(%u) + length(%u) overflows",
5716ba1f63bSYuanhan Liu 			offset, length);
5726ba1f63bSYuanhan Liu 		return NULL;
5736ba1f63bSYuanhan Liu 	}
5746ba1f63bSYuanhan Liu 
5756ba1f63bSYuanhan Liu 	if (offset + length > dev->mem_resource[bar].len) {
5766ba1f63bSYuanhan Liu 		PMD_INIT_LOG(ERR,
5776ba1f63bSYuanhan Liu 			"invalid cap: overflows bar space: %u > %" PRIu64,
5786ba1f63bSYuanhan Liu 			offset + length, dev->mem_resource[bar].len);
5796ba1f63bSYuanhan Liu 		return NULL;
5806ba1f63bSYuanhan Liu 	}
5816ba1f63bSYuanhan Liu 
5826ba1f63bSYuanhan Liu 	base = dev->mem_resource[bar].addr;
5836ba1f63bSYuanhan Liu 	if (base == NULL) {
5846ba1f63bSYuanhan Liu 		PMD_INIT_LOG(ERR, "bar %u base addr is NULL", bar);
5856ba1f63bSYuanhan Liu 		return NULL;
5866ba1f63bSYuanhan Liu 	}
5876ba1f63bSYuanhan Liu 
5886ba1f63bSYuanhan Liu 	return base + offset;
5896ba1f63bSYuanhan Liu }
5906ba1f63bSYuanhan Liu 
5916ba1f63bSYuanhan Liu static int
592266ece29SMaxime Coquelin virtio_read_caps(struct rte_pci_device *pci_dev, struct virtio_hw *hw)
5936ba1f63bSYuanhan Liu {
594266ece29SMaxime Coquelin 	struct virtio_pci_dev *dev = virtio_pci_get_dev(hw);
5956ba1f63bSYuanhan Liu 	struct virtio_pci_cap cap;
596*a10b6e53SDavid Marchand 	off_t pos;
5976ba1f63bSYuanhan Liu 	int ret;
5986ba1f63bSYuanhan Liu 
599266ece29SMaxime Coquelin 	if (rte_pci_map_device(pci_dev)) {
6006ba1f63bSYuanhan Liu 		PMD_INIT_LOG(DEBUG, "failed to map pci device!");
6016ba1f63bSYuanhan Liu 		return -1;
6026ba1f63bSYuanhan Liu 	}
6036ba1f63bSYuanhan Liu 
604*a10b6e53SDavid Marchand 	/*
605*a10b6e53SDavid Marchand 	 * Transitional devices would also have this capability,
606cb482cb3SJianfeng Tan 	 * that's why we also check if msix is enabled.
607cb482cb3SJianfeng Tan 	 */
608*a10b6e53SDavid Marchand 	dev->msix_status = vtpci_msix_detect(pci_dev);
60949bb1f7aSBrian Russell 
610*a10b6e53SDavid Marchand 	pos = rte_pci_find_capability(pci_dev, PCI_CAP_ID_VNDR);
611*a10b6e53SDavid Marchand 	while (pos > 0) {
612*a10b6e53SDavid Marchand 		if (rte_pci_read_config(pci_dev, &cap, sizeof(cap), pos) != sizeof(cap))
61349bb1f7aSBrian Russell 			break;
6146ba1f63bSYuanhan Liu 		PMD_INIT_LOG(DEBUG,
6156ba1f63bSYuanhan Liu 			"[%2x] cfg type: %u, bar: %u, offset: %04x, len: %u",
616*a10b6e53SDavid Marchand 			(unsigned int)pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
6176ba1f63bSYuanhan Liu 
6186ba1f63bSYuanhan Liu 		switch (cap.cfg_type) {
6196ba1f63bSYuanhan Liu 		case VIRTIO_PCI_CAP_COMMON_CFG:
620266ece29SMaxime Coquelin 			dev->common_cfg = get_cfg_addr(pci_dev, &cap);
6216ba1f63bSYuanhan Liu 			break;
6226ba1f63bSYuanhan Liu 		case VIRTIO_PCI_CAP_NOTIFY_CFG:
623*a10b6e53SDavid Marchand 			ret = rte_pci_read_config(pci_dev, &dev->notify_off_multiplier,
6246ba1f63bSYuanhan Liu 				4, pos + sizeof(cap));
625ecfae151STiwei Bie 			if (ret != 4)
626ecfae151STiwei Bie 				PMD_INIT_LOG(DEBUG,
627ecfae151STiwei Bie 					"failed to read notify_off_multiplier, ret %d",
628ecfae151STiwei Bie 					ret);
629ecfae151STiwei Bie 			else
630266ece29SMaxime Coquelin 				dev->notify_base = get_cfg_addr(pci_dev, &cap);
6316ba1f63bSYuanhan Liu 			break;
6326ba1f63bSYuanhan Liu 		case VIRTIO_PCI_CAP_DEVICE_CFG:
633266ece29SMaxime Coquelin 			dev->dev_cfg = get_cfg_addr(pci_dev, &cap);
6346ba1f63bSYuanhan Liu 			break;
6356ba1f63bSYuanhan Liu 		case VIRTIO_PCI_CAP_ISR_CFG:
636266ece29SMaxime Coquelin 			dev->isr = get_cfg_addr(pci_dev, &cap);
6376ba1f63bSYuanhan Liu 			break;
6386ba1f63bSYuanhan Liu 		}
6396ba1f63bSYuanhan Liu 
640*a10b6e53SDavid Marchand 		pos = rte_pci_find_next_capability(pci_dev, PCI_CAP_ID_VNDR, pos);
6416ba1f63bSYuanhan Liu 	}
6426ba1f63bSYuanhan Liu 
643266ece29SMaxime Coquelin 	if (dev->common_cfg == NULL || dev->notify_base == NULL ||
644266ece29SMaxime Coquelin 	    dev->dev_cfg == NULL    || dev->isr == NULL) {
6456ba1f63bSYuanhan Liu 		PMD_INIT_LOG(INFO, "no modern virtio pci device found.");
6466ba1f63bSYuanhan Liu 		return -1;
6476ba1f63bSYuanhan Liu 	}
6486ba1f63bSYuanhan Liu 
6496ba1f63bSYuanhan Liu 	PMD_INIT_LOG(INFO, "found modern virtio pci device.");
6506ba1f63bSYuanhan Liu 
651266ece29SMaxime Coquelin 	PMD_INIT_LOG(DEBUG, "common cfg mapped at: %p", dev->common_cfg);
652266ece29SMaxime Coquelin 	PMD_INIT_LOG(DEBUG, "device cfg mapped at: %p", dev->dev_cfg);
653266ece29SMaxime Coquelin 	PMD_INIT_LOG(DEBUG, "isr cfg mapped at: %p", dev->isr);
6546ba1f63bSYuanhan Liu 	PMD_INIT_LOG(DEBUG, "notify base: %p, notify off multiplier: %u",
655266ece29SMaxime Coquelin 		dev->notify_base, dev->notify_off_multiplier);
6566ba1f63bSYuanhan Liu 
6576ba1f63bSYuanhan Liu 	return 0;
6586ba1f63bSYuanhan Liu }
6596ba1f63bSYuanhan Liu 
660ac5e1d83SHuawei Xie /*
661ac5e1d83SHuawei Xie  * Return -1:
662ac5e1d83SHuawei Xie  *   if there is error mapping with VFIO/UIO.
663ac5e1d83SHuawei Xie  *   if port map error when driver type is KDRV_NONE.
664a65a34a8SStephen Hemminger  *   if marked as allowed but driver type is KDRV_UNKNOWN.
665ac5e1d83SHuawei Xie  * Return 1 if kernel driver is managing the device.
666ac5e1d83SHuawei Xie  * Return 0 on success.
667ac5e1d83SHuawei Xie  */
668d5bbeefcSYuanhan Liu int
6691ac79346SMaxime Coquelin vtpci_init(struct rte_pci_device *pci_dev, struct virtio_pci_dev *dev)
670d5bbeefcSYuanhan Liu {
6711ac79346SMaxime Coquelin 	struct virtio_hw *hw = &dev->hw;
6721ac79346SMaxime Coquelin 
673f305ecbbSMaxime Coquelin 	RTE_BUILD_BUG_ON(offsetof(struct virtio_pci_dev, hw) != 0);
674f305ecbbSMaxime Coquelin 
6756ba1f63bSYuanhan Liu 	/*
6766ba1f63bSYuanhan Liu 	 * Try if we can succeed reading virtio pci caps, which exists
6776ba1f63bSYuanhan Liu 	 * only on modern pci device. If failed, we fallback to legacy
6786ba1f63bSYuanhan Liu 	 * virtio handling.
6796ba1f63bSYuanhan Liu 	 */
6801ac79346SMaxime Coquelin 	if (virtio_read_caps(pci_dev, hw) == 0) {
6816ba1f63bSYuanhan Liu 		PMD_INIT_LOG(INFO, "modern virtio pci detected.");
682f8b60756SMaxime Coquelin 		VIRTIO_OPS(hw) = &modern_ops;
6831ac79346SMaxime Coquelin 		dev->modern = true;
6847793d293SMaxime Coquelin 		goto msix_detect;
6856ba1f63bSYuanhan Liu 	}
6866ba1f63bSYuanhan Liu 
6876ba1f63bSYuanhan Liu 	PMD_INIT_LOG(INFO, "trying with legacy virtio pci.");
6881ac79346SMaxime Coquelin 	if (rte_pci_ioport_map(pci_dev, 0, VTPCI_IO(hw)) < 0) {
6891ac79346SMaxime Coquelin 		rte_pci_unmap_device(pci_dev);
6901ac79346SMaxime Coquelin 		if (pci_dev->kdrv == RTE_PCI_KDRV_UNKNOWN &&
6911ac79346SMaxime Coquelin 		    (!pci_dev->device.devargs ||
6921ac79346SMaxime Coquelin 		     pci_dev->device.devargs->bus !=
6932b0e39c1SGaetan Rivet 		     rte_bus_find_by_name("pci"))) {
694ac5e1d83SHuawei Xie 			PMD_INIT_LOG(INFO,
695ac5e1d83SHuawei Xie 				"skip kernel managed virtio device.");
696ac5e1d83SHuawei Xie 			return 1;
697ac5e1d83SHuawei Xie 		}
698c52afa68SYuanhan Liu 		return -1;
699ac5e1d83SHuawei Xie 	}
7006ba1f63bSYuanhan Liu 
701f8b60756SMaxime Coquelin 	VIRTIO_OPS(hw) = &legacy_ops;
7021ac79346SMaxime Coquelin 	dev->modern = false;
703c52afa68SYuanhan Liu 
7047793d293SMaxime Coquelin msix_detect:
705f8b60756SMaxime Coquelin 	VIRTIO_OPS(hw)->intr_detect(hw);
7067793d293SMaxime Coquelin 
707d5bbeefcSYuanhan Liu 	return 0;
7086c3169a3SBruce Richardson }
709fe19d49cSZhiyong Yang 
710c8d4b02fSMaxime Coquelin void vtpci_legacy_ioport_unmap(struct virtio_hw *hw)
711c8d4b02fSMaxime Coquelin {
712c8d4b02fSMaxime Coquelin 	rte_pci_ioport_unmap(VTPCI_IO(hw));
713c8d4b02fSMaxime Coquelin }
714c8d4b02fSMaxime Coquelin 
715c8d4b02fSMaxime Coquelin int vtpci_legacy_ioport_map(struct virtio_hw *hw)
716c8d4b02fSMaxime Coquelin {
717aa0d4b8aSMaxime Coquelin 	return rte_pci_ioport_map(VTPCI_DEV(hw), 0, VTPCI_IO(hw));
718c8d4b02fSMaxime Coquelin }
719