xref: /dpdk/drivers/net/virtio/virtio_pci.c (revision a65a34a85ebf5ee6dda53d680393b0342ff7fdb9)
15566a3e3SBruce Richardson /* SPDX-License-Identifier: BSD-3-Clause
25566a3e3SBruce Richardson  * Copyright(c) 2010-2014 Intel Corporation
36c3169a3SBruce Richardson  */
46c3169a3SBruce Richardson #include <stdint.h>
56c3169a3SBruce Richardson 
6742bde12SBruce Richardson #ifdef RTE_EXEC_ENV_LINUX
7c52afa68SYuanhan Liu  #include <dirent.h>
8c52afa68SYuanhan Liu  #include <fcntl.h>
9c52afa68SYuanhan Liu #endif
10c52afa68SYuanhan Liu 
11631d4ee4SSantosh Shukla #include <rte_io.h>
122b0e39c1SGaetan Rivet #include <rte_bus.h>
13631d4ee4SSantosh Shukla 
146c3169a3SBruce Richardson #include "virtio_pci.h"
156c3169a3SBruce Richardson #include "virtio_logs.h"
16d5bbeefcSYuanhan Liu #include "virtqueue.h"
176c3169a3SBruce Richardson 
186ba1f63bSYuanhan Liu /*
196ba1f63bSYuanhan Liu  * Following macros are derived from linux/pci_regs.h, however,
206ba1f63bSYuanhan Liu  * we can't simply include that header here, as there is no such
216ba1f63bSYuanhan Liu  * file for non-Linux platform.
226ba1f63bSYuanhan Liu  */
236ba1f63bSYuanhan Liu #define PCI_CAPABILITY_LIST	0x34
246ba1f63bSYuanhan Liu #define PCI_CAP_ID_VNDR		0x09
25554b6d3eSJianfeng Tan #define PCI_CAP_ID_MSIX		0x11
266ba1f63bSYuanhan Liu 
27b8f04520SDavid Marchand /*
28b8f04520SDavid Marchand  * The remaining space is defined by each driver as the per-driver
29b8f04520SDavid Marchand  * configuration space.
30b8f04520SDavid Marchand  */
31fe19d49cSZhiyong Yang #define VIRTIO_PCI_CONFIG(hw) \
32fe19d49cSZhiyong Yang 		(((hw)->use_msix == VIRTIO_MSIX_ENABLED) ? 24 : 20)
33b86af7b1SYuanhan Liu 
34595454c5SJianfeng Tan static inline int
35595454c5SJianfeng Tan check_vq_phys_addr_ok(struct virtqueue *vq)
36595454c5SJianfeng Tan {
37595454c5SJianfeng Tan 	/* Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit,
38595454c5SJianfeng Tan 	 * and only accepts 32 bit page frame number.
39595454c5SJianfeng Tan 	 * Check if the allocated physical memory exceeds 16TB.
40595454c5SJianfeng Tan 	 */
41595454c5SJianfeng Tan 	if ((vq->vq_ring_mem + vq->vq_ring_size - 1) >>
42595454c5SJianfeng Tan 			(VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) {
43595454c5SJianfeng Tan 		PMD_INIT_LOG(ERR, "vring address shouldn't be above 16TB!");
44595454c5SJianfeng Tan 		return 0;
45595454c5SJianfeng Tan 	}
46595454c5SJianfeng Tan 
47595454c5SJianfeng Tan 	return 1;
48595454c5SJianfeng Tan }
49595454c5SJianfeng Tan 
50281ccccbSDavid Marchand /*
51281ccccbSDavid Marchand  * Since we are in legacy mode:
52281ccccbSDavid Marchand  * http://ozlabs.org/~rusty/virtio-spec/virtio-0.9.5.pdf
53281ccccbSDavid Marchand  *
54281ccccbSDavid Marchand  * "Note that this is possible because while the virtio header is PCI (i.e.
55281ccccbSDavid Marchand  * little) endian, the device-specific region is encoded in the native endian of
56281ccccbSDavid Marchand  * the guest (where such distinction is applicable)."
57281ccccbSDavid Marchand  *
58281ccccbSDavid Marchand  * For powerpc which supports both, qemu supposes that cpu is big endian and
59281ccccbSDavid Marchand  * enforces this for the virtio-net stuff.
60281ccccbSDavid Marchand  */
61d5bbeefcSYuanhan Liu static void
62d5bbeefcSYuanhan Liu legacy_read_dev_config(struct virtio_hw *hw, size_t offset,
636c3169a3SBruce Richardson 		       void *dst, int length)
646c3169a3SBruce Richardson {
65281ccccbSDavid Marchand #ifdef RTE_ARCH_PPC_64
66281ccccbSDavid Marchand 	int size;
67281ccccbSDavid Marchand 
68281ccccbSDavid Marchand 	while (length > 0) {
69281ccccbSDavid Marchand 		if (length >= 4) {
70281ccccbSDavid Marchand 			size = 4;
713dcfe039SThomas Monjalon 			rte_pci_ioport_read(VTPCI_IO(hw), dst, size,
72281ccccbSDavid Marchand 				VIRTIO_PCI_CONFIG(hw) + offset);
73281ccccbSDavid Marchand 			*(uint32_t *)dst = rte_be_to_cpu_32(*(uint32_t *)dst);
74281ccccbSDavid Marchand 		} else if (length >= 2) {
75281ccccbSDavid Marchand 			size = 2;
763dcfe039SThomas Monjalon 			rte_pci_ioport_read(VTPCI_IO(hw), dst, size,
77281ccccbSDavid Marchand 				VIRTIO_PCI_CONFIG(hw) + offset);
78281ccccbSDavid Marchand 			*(uint16_t *)dst = rte_be_to_cpu_16(*(uint16_t *)dst);
79281ccccbSDavid Marchand 		} else {
80281ccccbSDavid Marchand 			size = 1;
813dcfe039SThomas Monjalon 			rte_pci_ioport_read(VTPCI_IO(hw), dst, size,
82281ccccbSDavid Marchand 				VIRTIO_PCI_CONFIG(hw) + offset);
83281ccccbSDavid Marchand 		}
84281ccccbSDavid Marchand 
85281ccccbSDavid Marchand 		dst = (char *)dst + size;
86281ccccbSDavid Marchand 		offset += size;
87281ccccbSDavid Marchand 		length -= size;
88281ccccbSDavid Marchand 	}
89281ccccbSDavid Marchand #else
903dcfe039SThomas Monjalon 	rte_pci_ioport_read(VTPCI_IO(hw), dst, length,
91b8f04520SDavid Marchand 		VIRTIO_PCI_CONFIG(hw) + offset);
92281ccccbSDavid Marchand #endif
936c3169a3SBruce Richardson }
946c3169a3SBruce Richardson 
95d5bbeefcSYuanhan Liu static void
96d5bbeefcSYuanhan Liu legacy_write_dev_config(struct virtio_hw *hw, size_t offset,
97d5bbeefcSYuanhan Liu 			const void *src, int length)
986c3169a3SBruce Richardson {
99281ccccbSDavid Marchand #ifdef RTE_ARCH_PPC_64
100281ccccbSDavid Marchand 	union {
101281ccccbSDavid Marchand 		uint32_t u32;
102281ccccbSDavid Marchand 		uint16_t u16;
103281ccccbSDavid Marchand 	} tmp;
104281ccccbSDavid Marchand 	int size;
105281ccccbSDavid Marchand 
106281ccccbSDavid Marchand 	while (length > 0) {
107281ccccbSDavid Marchand 		if (length >= 4) {
108281ccccbSDavid Marchand 			size = 4;
109281ccccbSDavid Marchand 			tmp.u32 = rte_cpu_to_be_32(*(const uint32_t *)src);
1103dcfe039SThomas Monjalon 			rte_pci_ioport_write(VTPCI_IO(hw), &tmp.u32, size,
111281ccccbSDavid Marchand 				VIRTIO_PCI_CONFIG(hw) + offset);
112281ccccbSDavid Marchand 		} else if (length >= 2) {
113281ccccbSDavid Marchand 			size = 2;
114281ccccbSDavid Marchand 			tmp.u16 = rte_cpu_to_be_16(*(const uint16_t *)src);
1153dcfe039SThomas Monjalon 			rte_pci_ioport_write(VTPCI_IO(hw), &tmp.u16, size,
116281ccccbSDavid Marchand 				VIRTIO_PCI_CONFIG(hw) + offset);
117281ccccbSDavid Marchand 		} else {
118281ccccbSDavid Marchand 			size = 1;
1193dcfe039SThomas Monjalon 			rte_pci_ioport_write(VTPCI_IO(hw), src, size,
120281ccccbSDavid Marchand 				VIRTIO_PCI_CONFIG(hw) + offset);
121281ccccbSDavid Marchand 		}
122281ccccbSDavid Marchand 
123281ccccbSDavid Marchand 		src = (const char *)src + size;
124281ccccbSDavid Marchand 		offset += size;
125281ccccbSDavid Marchand 		length -= size;
126281ccccbSDavid Marchand 	}
127281ccccbSDavid Marchand #else
1283dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), src, length,
129b8f04520SDavid Marchand 		VIRTIO_PCI_CONFIG(hw) + offset);
130281ccccbSDavid Marchand #endif
1316c3169a3SBruce Richardson }
1326c3169a3SBruce Richardson 
1333891f233SYuanhan Liu static uint64_t
134d5bbeefcSYuanhan Liu legacy_get_features(struct virtio_hw *hw)
135d5bbeefcSYuanhan Liu {
13636ea36efSYuanhan Liu 	uint32_t dst;
137b8f04520SDavid Marchand 
1383dcfe039SThomas Monjalon 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 4, VIRTIO_PCI_HOST_FEATURES);
139b8f04520SDavid Marchand 	return dst;
140d5bbeefcSYuanhan Liu }
141d5bbeefcSYuanhan Liu 
142d5bbeefcSYuanhan Liu static void
1433891f233SYuanhan Liu legacy_set_features(struct virtio_hw *hw, uint64_t features)
144d5bbeefcSYuanhan Liu {
1453891f233SYuanhan Liu 	if ((features >> 32) != 0) {
1463891f233SYuanhan Liu 		PMD_DRV_LOG(ERR,
1473891f233SYuanhan Liu 			"only 32 bit features are allowed for legacy virtio!");
1483891f233SYuanhan Liu 		return;
1493891f233SYuanhan Liu 	}
1503dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &features, 4,
151b8f04520SDavid Marchand 		VIRTIO_PCI_GUEST_FEATURES);
152d5bbeefcSYuanhan Liu }
153d5bbeefcSYuanhan Liu 
154d5bbeefcSYuanhan Liu static uint8_t
155d5bbeefcSYuanhan Liu legacy_get_status(struct virtio_hw *hw)
156d5bbeefcSYuanhan Liu {
157b8f04520SDavid Marchand 	uint8_t dst;
158b8f04520SDavid Marchand 
1593dcfe039SThomas Monjalon 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 1, VIRTIO_PCI_STATUS);
160b8f04520SDavid Marchand 	return dst;
161d5bbeefcSYuanhan Liu }
162d5bbeefcSYuanhan Liu 
163d5bbeefcSYuanhan Liu static void
164d5bbeefcSYuanhan Liu legacy_set_status(struct virtio_hw *hw, uint8_t status)
165d5bbeefcSYuanhan Liu {
1663dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &status, 1, VIRTIO_PCI_STATUS);
167d5bbeefcSYuanhan Liu }
168d5bbeefcSYuanhan Liu 
169d5bbeefcSYuanhan Liu static uint8_t
170d5bbeefcSYuanhan Liu legacy_get_isr(struct virtio_hw *hw)
171d5bbeefcSYuanhan Liu {
172b8f04520SDavid Marchand 	uint8_t dst;
173b8f04520SDavid Marchand 
1743dcfe039SThomas Monjalon 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 1, VIRTIO_PCI_ISR);
175b8f04520SDavid Marchand 	return dst;
176d5bbeefcSYuanhan Liu }
177d5bbeefcSYuanhan Liu 
178d5bbeefcSYuanhan Liu /* Enable one vector (0) for Link State Intrerrupt */
179d5bbeefcSYuanhan Liu static uint16_t
180d5bbeefcSYuanhan Liu legacy_set_config_irq(struct virtio_hw *hw, uint16_t vec)
181d5bbeefcSYuanhan Liu {
182b8f04520SDavid Marchand 	uint16_t dst;
183b8f04520SDavid Marchand 
1843dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &vec, 2, VIRTIO_MSI_CONFIG_VECTOR);
1853dcfe039SThomas Monjalon 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 2, VIRTIO_MSI_CONFIG_VECTOR);
186b8f04520SDavid Marchand 	return dst;
187d5bbeefcSYuanhan Liu }
188d5bbeefcSYuanhan Liu 
189d5bbeefcSYuanhan Liu static uint16_t
190c49526acSJianfeng Tan legacy_set_queue_irq(struct virtio_hw *hw, struct virtqueue *vq, uint16_t vec)
191c49526acSJianfeng Tan {
192c49526acSJianfeng Tan 	uint16_t dst;
193c49526acSJianfeng Tan 
1943dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2,
195c49526acSJianfeng Tan 		VIRTIO_PCI_QUEUE_SEL);
1963dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &vec, 2, VIRTIO_MSI_QUEUE_VECTOR);
1973dcfe039SThomas Monjalon 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 2, VIRTIO_MSI_QUEUE_VECTOR);
198c49526acSJianfeng Tan 	return dst;
199c49526acSJianfeng Tan }
200c49526acSJianfeng Tan 
201c49526acSJianfeng Tan static uint16_t
202d5bbeefcSYuanhan Liu legacy_get_queue_num(struct virtio_hw *hw, uint16_t queue_id)
203d5bbeefcSYuanhan Liu {
204b8f04520SDavid Marchand 	uint16_t dst;
205b8f04520SDavid Marchand 
2063dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &queue_id, 2, VIRTIO_PCI_QUEUE_SEL);
2073dcfe039SThomas Monjalon 	rte_pci_ioport_read(VTPCI_IO(hw), &dst, 2, VIRTIO_PCI_QUEUE_NUM);
208b8f04520SDavid Marchand 	return dst;
209d5bbeefcSYuanhan Liu }
210d5bbeefcSYuanhan Liu 
211595454c5SJianfeng Tan static int
212d5bbeefcSYuanhan Liu legacy_setup_queue(struct virtio_hw *hw, struct virtqueue *vq)
213d5bbeefcSYuanhan Liu {
214b8f04520SDavid Marchand 	uint32_t src;
215d5bbeefcSYuanhan Liu 
216595454c5SJianfeng Tan 	if (!check_vq_phys_addr_ok(vq))
217595454c5SJianfeng Tan 		return -1;
218595454c5SJianfeng Tan 
2193dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2,
220b8f04520SDavid Marchand 		VIRTIO_PCI_QUEUE_SEL);
22101ad44fdSHuawei Xie 	src = vq->vq_ring_mem >> VIRTIO_PCI_QUEUE_ADDR_SHIFT;
2223dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &src, 4, VIRTIO_PCI_QUEUE_PFN);
223595454c5SJianfeng Tan 
224595454c5SJianfeng Tan 	return 0;
225d5bbeefcSYuanhan Liu }
226d5bbeefcSYuanhan Liu 
227d5bbeefcSYuanhan Liu static void
228d5bbeefcSYuanhan Liu legacy_del_queue(struct virtio_hw *hw, struct virtqueue *vq)
229d5bbeefcSYuanhan Liu {
230b8f04520SDavid Marchand 	uint32_t src = 0;
231d5bbeefcSYuanhan Liu 
2323dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2,
233b8f04520SDavid Marchand 		VIRTIO_PCI_QUEUE_SEL);
2343dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &src, 4, VIRTIO_PCI_QUEUE_PFN);
235d5bbeefcSYuanhan Liu }
236d5bbeefcSYuanhan Liu 
237d5bbeefcSYuanhan Liu static void
238d5bbeefcSYuanhan Liu legacy_notify_queue(struct virtio_hw *hw, struct virtqueue *vq)
239d5bbeefcSYuanhan Liu {
2403dcfe039SThomas Monjalon 	rte_pci_ioport_write(VTPCI_IO(hw), &vq->vq_queue_index, 2,
241b8f04520SDavid Marchand 		VIRTIO_PCI_QUEUE_NOTIFY);
242d5bbeefcSYuanhan Liu }
243d5bbeefcSYuanhan Liu 
2446d890f8aSYuanhan Liu const struct virtio_pci_ops legacy_ops = {
245d5bbeefcSYuanhan Liu 	.read_dev_cfg	= legacy_read_dev_config,
246d5bbeefcSYuanhan Liu 	.write_dev_cfg	= legacy_write_dev_config,
247d5bbeefcSYuanhan Liu 	.get_status	= legacy_get_status,
248d5bbeefcSYuanhan Liu 	.set_status	= legacy_set_status,
249d5bbeefcSYuanhan Liu 	.get_features	= legacy_get_features,
250d5bbeefcSYuanhan Liu 	.set_features	= legacy_set_features,
251d5bbeefcSYuanhan Liu 	.get_isr	= legacy_get_isr,
252d5bbeefcSYuanhan Liu 	.set_config_irq	= legacy_set_config_irq,
253c49526acSJianfeng Tan 	.set_queue_irq  = legacy_set_queue_irq,
254d5bbeefcSYuanhan Liu 	.get_queue_num	= legacy_get_queue_num,
255d5bbeefcSYuanhan Liu 	.setup_queue	= legacy_setup_queue,
256d5bbeefcSYuanhan Liu 	.del_queue	= legacy_del_queue,
257d5bbeefcSYuanhan Liu 	.notify_queue	= legacy_notify_queue,
258d5bbeefcSYuanhan Liu };
259d5bbeefcSYuanhan Liu 
2606ba1f63bSYuanhan Liu static inline void
2616ba1f63bSYuanhan Liu io_write64_twopart(uint64_t val, uint32_t *lo, uint32_t *hi)
2626ba1f63bSYuanhan Liu {
263631d4ee4SSantosh Shukla 	rte_write32(val & ((1ULL << 32) - 1), lo);
264631d4ee4SSantosh Shukla 	rte_write32(val >> 32,		     hi);
2656ba1f63bSYuanhan Liu }
2666ba1f63bSYuanhan Liu 
2676ba1f63bSYuanhan Liu static void
2686ba1f63bSYuanhan Liu modern_read_dev_config(struct virtio_hw *hw, size_t offset,
2696ba1f63bSYuanhan Liu 		       void *dst, int length)
2706ba1f63bSYuanhan Liu {
2716ba1f63bSYuanhan Liu 	int i;
2726ba1f63bSYuanhan Liu 	uint8_t *p;
2736ba1f63bSYuanhan Liu 	uint8_t old_gen, new_gen;
2746ba1f63bSYuanhan Liu 
2756ba1f63bSYuanhan Liu 	do {
276631d4ee4SSantosh Shukla 		old_gen = rte_read8(&hw->common_cfg->config_generation);
2776ba1f63bSYuanhan Liu 
2786ba1f63bSYuanhan Liu 		p = dst;
2796ba1f63bSYuanhan Liu 		for (i = 0;  i < length; i++)
280631d4ee4SSantosh Shukla 			*p++ = rte_read8((uint8_t *)hw->dev_cfg + offset + i);
2816ba1f63bSYuanhan Liu 
282631d4ee4SSantosh Shukla 		new_gen = rte_read8(&hw->common_cfg->config_generation);
2836ba1f63bSYuanhan Liu 	} while (old_gen != new_gen);
2846ba1f63bSYuanhan Liu }
2856ba1f63bSYuanhan Liu 
2866ba1f63bSYuanhan Liu static void
2876ba1f63bSYuanhan Liu modern_write_dev_config(struct virtio_hw *hw, size_t offset,
2886ba1f63bSYuanhan Liu 			const void *src, int length)
2896ba1f63bSYuanhan Liu {
2906ba1f63bSYuanhan Liu 	int i;
2916ba1f63bSYuanhan Liu 	const uint8_t *p = src;
2926ba1f63bSYuanhan Liu 
2936ba1f63bSYuanhan Liu 	for (i = 0;  i < length; i++)
294631d4ee4SSantosh Shukla 		rte_write8((*p++), (((uint8_t *)hw->dev_cfg) + offset + i));
2956ba1f63bSYuanhan Liu }
2966ba1f63bSYuanhan Liu 
2976ba1f63bSYuanhan Liu static uint64_t
2986ba1f63bSYuanhan Liu modern_get_features(struct virtio_hw *hw)
2996ba1f63bSYuanhan Liu {
3006ba1f63bSYuanhan Liu 	uint32_t features_lo, features_hi;
3016ba1f63bSYuanhan Liu 
302631d4ee4SSantosh Shukla 	rte_write32(0, &hw->common_cfg->device_feature_select);
303631d4ee4SSantosh Shukla 	features_lo = rte_read32(&hw->common_cfg->device_feature);
3046ba1f63bSYuanhan Liu 
305631d4ee4SSantosh Shukla 	rte_write32(1, &hw->common_cfg->device_feature_select);
306631d4ee4SSantosh Shukla 	features_hi = rte_read32(&hw->common_cfg->device_feature);
3076ba1f63bSYuanhan Liu 
3086ba1f63bSYuanhan Liu 	return ((uint64_t)features_hi << 32) | features_lo;
3096ba1f63bSYuanhan Liu }
3106ba1f63bSYuanhan Liu 
3116ba1f63bSYuanhan Liu static void
3126ba1f63bSYuanhan Liu modern_set_features(struct virtio_hw *hw, uint64_t features)
3136ba1f63bSYuanhan Liu {
314631d4ee4SSantosh Shukla 	rte_write32(0, &hw->common_cfg->guest_feature_select);
315631d4ee4SSantosh Shukla 	rte_write32(features & ((1ULL << 32) - 1),
3166ba1f63bSYuanhan Liu 		    &hw->common_cfg->guest_feature);
3176ba1f63bSYuanhan Liu 
318631d4ee4SSantosh Shukla 	rte_write32(1, &hw->common_cfg->guest_feature_select);
319631d4ee4SSantosh Shukla 	rte_write32(features >> 32,
3206ba1f63bSYuanhan Liu 		    &hw->common_cfg->guest_feature);
3216ba1f63bSYuanhan Liu }
3226ba1f63bSYuanhan Liu 
3236ba1f63bSYuanhan Liu static uint8_t
3246ba1f63bSYuanhan Liu modern_get_status(struct virtio_hw *hw)
3256ba1f63bSYuanhan Liu {
326631d4ee4SSantosh Shukla 	return rte_read8(&hw->common_cfg->device_status);
3276ba1f63bSYuanhan Liu }
3286ba1f63bSYuanhan Liu 
3296ba1f63bSYuanhan Liu static void
3306ba1f63bSYuanhan Liu modern_set_status(struct virtio_hw *hw, uint8_t status)
3316ba1f63bSYuanhan Liu {
332631d4ee4SSantosh Shukla 	rte_write8(status, &hw->common_cfg->device_status);
3336ba1f63bSYuanhan Liu }
3346ba1f63bSYuanhan Liu 
3356ba1f63bSYuanhan Liu static uint8_t
3366ba1f63bSYuanhan Liu modern_get_isr(struct virtio_hw *hw)
3376ba1f63bSYuanhan Liu {
338631d4ee4SSantosh Shukla 	return rte_read8(hw->isr);
3396ba1f63bSYuanhan Liu }
3406ba1f63bSYuanhan Liu 
3416ba1f63bSYuanhan Liu static uint16_t
3426ba1f63bSYuanhan Liu modern_set_config_irq(struct virtio_hw *hw, uint16_t vec)
3436ba1f63bSYuanhan Liu {
344631d4ee4SSantosh Shukla 	rte_write16(vec, &hw->common_cfg->msix_config);
345631d4ee4SSantosh Shukla 	return rte_read16(&hw->common_cfg->msix_config);
3466ba1f63bSYuanhan Liu }
3476ba1f63bSYuanhan Liu 
3486ba1f63bSYuanhan Liu static uint16_t
349c49526acSJianfeng Tan modern_set_queue_irq(struct virtio_hw *hw, struct virtqueue *vq, uint16_t vec)
350c49526acSJianfeng Tan {
351631d4ee4SSantosh Shukla 	rte_write16(vq->vq_queue_index, &hw->common_cfg->queue_select);
352631d4ee4SSantosh Shukla 	rte_write16(vec, &hw->common_cfg->queue_msix_vector);
353631d4ee4SSantosh Shukla 	return rte_read16(&hw->common_cfg->queue_msix_vector);
354c49526acSJianfeng Tan }
355c49526acSJianfeng Tan 
356c49526acSJianfeng Tan static uint16_t
3576ba1f63bSYuanhan Liu modern_get_queue_num(struct virtio_hw *hw, uint16_t queue_id)
3586ba1f63bSYuanhan Liu {
359631d4ee4SSantosh Shukla 	rte_write16(queue_id, &hw->common_cfg->queue_select);
360631d4ee4SSantosh Shukla 	return rte_read16(&hw->common_cfg->queue_size);
3616ba1f63bSYuanhan Liu }
3626ba1f63bSYuanhan Liu 
363595454c5SJianfeng Tan static int
3646ba1f63bSYuanhan Liu modern_setup_queue(struct virtio_hw *hw, struct virtqueue *vq)
3656ba1f63bSYuanhan Liu {
3666ba1f63bSYuanhan Liu 	uint64_t desc_addr, avail_addr, used_addr;
3676ba1f63bSYuanhan Liu 	uint16_t notify_off;
3686ba1f63bSYuanhan Liu 
369595454c5SJianfeng Tan 	if (!check_vq_phys_addr_ok(vq))
370595454c5SJianfeng Tan 		return -1;
371595454c5SJianfeng Tan 
37201ad44fdSHuawei Xie 	desc_addr = vq->vq_ring_mem;
3736ba1f63bSYuanhan Liu 	avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc);
3746ba1f63bSYuanhan Liu 	used_addr = RTE_ALIGN_CEIL(avail_addr + offsetof(struct vring_avail,
3756ba1f63bSYuanhan Liu 							 ring[vq->vq_nentries]),
3766ba1f63bSYuanhan Liu 				   VIRTIO_PCI_VRING_ALIGN);
3776ba1f63bSYuanhan Liu 
378631d4ee4SSantosh Shukla 	rte_write16(vq->vq_queue_index, &hw->common_cfg->queue_select);
3796ba1f63bSYuanhan Liu 
3806ba1f63bSYuanhan Liu 	io_write64_twopart(desc_addr, &hw->common_cfg->queue_desc_lo,
3816ba1f63bSYuanhan Liu 				      &hw->common_cfg->queue_desc_hi);
3826ba1f63bSYuanhan Liu 	io_write64_twopart(avail_addr, &hw->common_cfg->queue_avail_lo,
3836ba1f63bSYuanhan Liu 				       &hw->common_cfg->queue_avail_hi);
3846ba1f63bSYuanhan Liu 	io_write64_twopart(used_addr, &hw->common_cfg->queue_used_lo,
3856ba1f63bSYuanhan Liu 				      &hw->common_cfg->queue_used_hi);
3866ba1f63bSYuanhan Liu 
387631d4ee4SSantosh Shukla 	notify_off = rte_read16(&hw->common_cfg->queue_notify_off);
3886ba1f63bSYuanhan Liu 	vq->notify_addr = (void *)((uint8_t *)hw->notify_base +
3896ba1f63bSYuanhan Liu 				notify_off * hw->notify_off_multiplier);
3906ba1f63bSYuanhan Liu 
391631d4ee4SSantosh Shukla 	rte_write16(1, &hw->common_cfg->queue_enable);
3926ba1f63bSYuanhan Liu 
3936ba1f63bSYuanhan Liu 	PMD_INIT_LOG(DEBUG, "queue %u addresses:", vq->vq_queue_index);
3946ba1f63bSYuanhan Liu 	PMD_INIT_LOG(DEBUG, "\t desc_addr: %" PRIx64, desc_addr);
3956ba1f63bSYuanhan Liu 	PMD_INIT_LOG(DEBUG, "\t aval_addr: %" PRIx64, avail_addr);
3966ba1f63bSYuanhan Liu 	PMD_INIT_LOG(DEBUG, "\t used_addr: %" PRIx64, used_addr);
3976ba1f63bSYuanhan Liu 	PMD_INIT_LOG(DEBUG, "\t notify addr: %p (notify offset: %u)",
3986ba1f63bSYuanhan Liu 		vq->notify_addr, notify_off);
399595454c5SJianfeng Tan 
400595454c5SJianfeng Tan 	return 0;
4016ba1f63bSYuanhan Liu }
4026ba1f63bSYuanhan Liu 
4036ba1f63bSYuanhan Liu static void
4046ba1f63bSYuanhan Liu modern_del_queue(struct virtio_hw *hw, struct virtqueue *vq)
4056ba1f63bSYuanhan Liu {
406631d4ee4SSantosh Shukla 	rte_write16(vq->vq_queue_index, &hw->common_cfg->queue_select);
4076ba1f63bSYuanhan Liu 
4086ba1f63bSYuanhan Liu 	io_write64_twopart(0, &hw->common_cfg->queue_desc_lo,
4096ba1f63bSYuanhan Liu 				  &hw->common_cfg->queue_desc_hi);
4106ba1f63bSYuanhan Liu 	io_write64_twopart(0, &hw->common_cfg->queue_avail_lo,
4116ba1f63bSYuanhan Liu 				  &hw->common_cfg->queue_avail_hi);
4126ba1f63bSYuanhan Liu 	io_write64_twopart(0, &hw->common_cfg->queue_used_lo,
4136ba1f63bSYuanhan Liu 				  &hw->common_cfg->queue_used_hi);
4146ba1f63bSYuanhan Liu 
415631d4ee4SSantosh Shukla 	rte_write16(0, &hw->common_cfg->queue_enable);
4166ba1f63bSYuanhan Liu }
4176ba1f63bSYuanhan Liu 
4186ba1f63bSYuanhan Liu static void
4197e72f3ecSCheng Jiang modern_notify_queue(struct virtio_hw *hw, struct virtqueue *vq)
4206ba1f63bSYuanhan Liu {
4217e72f3ecSCheng Jiang 	uint32_t notify_data;
4227e72f3ecSCheng Jiang 
4237e72f3ecSCheng Jiang 	if (!vtpci_with_feature(hw, VIRTIO_F_NOTIFICATION_DATA)) {
424518208f3SXiao Wang 		rte_write16(vq->vq_queue_index, vq->notify_addr);
4257e72f3ecSCheng Jiang 		return;
4267e72f3ecSCheng Jiang 	}
4277e72f3ecSCheng Jiang 
4287e72f3ecSCheng Jiang 	if (vtpci_with_feature(hw, VIRTIO_F_RING_PACKED)) {
4297e72f3ecSCheng Jiang 		/*
4307e72f3ecSCheng Jiang 		 * Bit[0:15]: vq queue index
4317e72f3ecSCheng Jiang 		 * Bit[16:30]: avail index
4327e72f3ecSCheng Jiang 		 * Bit[31]: avail wrap counter
4337e72f3ecSCheng Jiang 		 */
4347e72f3ecSCheng Jiang 		notify_data = ((uint32_t)(!!(vq->vq_packed.cached_flags &
4357e72f3ecSCheng Jiang 				VRING_PACKED_DESC_F_AVAIL)) << 31) |
4367e72f3ecSCheng Jiang 				((uint32_t)vq->vq_avail_idx << 16) |
4377e72f3ecSCheng Jiang 				vq->vq_queue_index;
4387e72f3ecSCheng Jiang 	} else {
4397e72f3ecSCheng Jiang 		/*
4407e72f3ecSCheng Jiang 		 * Bit[0:15]: vq queue index
4417e72f3ecSCheng Jiang 		 * Bit[16:31]: avail index
4427e72f3ecSCheng Jiang 		 */
4437e72f3ecSCheng Jiang 		notify_data = ((uint32_t)vq->vq_avail_idx << 16) |
4447e72f3ecSCheng Jiang 				vq->vq_queue_index;
4457e72f3ecSCheng Jiang 	}
4467e72f3ecSCheng Jiang 	rte_write32(notify_data, vq->notify_addr);
4476ba1f63bSYuanhan Liu }
4486ba1f63bSYuanhan Liu 
4496d890f8aSYuanhan Liu const struct virtio_pci_ops modern_ops = {
4506ba1f63bSYuanhan Liu 	.read_dev_cfg	= modern_read_dev_config,
4516ba1f63bSYuanhan Liu 	.write_dev_cfg	= modern_write_dev_config,
4526ba1f63bSYuanhan Liu 	.get_status	= modern_get_status,
4536ba1f63bSYuanhan Liu 	.set_status	= modern_set_status,
4546ba1f63bSYuanhan Liu 	.get_features	= modern_get_features,
4556ba1f63bSYuanhan Liu 	.set_features	= modern_set_features,
4566ba1f63bSYuanhan Liu 	.get_isr	= modern_get_isr,
4576ba1f63bSYuanhan Liu 	.set_config_irq	= modern_set_config_irq,
458c49526acSJianfeng Tan 	.set_queue_irq  = modern_set_queue_irq,
4596ba1f63bSYuanhan Liu 	.get_queue_num	= modern_get_queue_num,
4606ba1f63bSYuanhan Liu 	.setup_queue	= modern_setup_queue,
4616ba1f63bSYuanhan Liu 	.del_queue	= modern_del_queue,
4626ba1f63bSYuanhan Liu 	.notify_queue	= modern_notify_queue,
4636ba1f63bSYuanhan Liu };
4646ba1f63bSYuanhan Liu 
4656ba1f63bSYuanhan Liu 
466d5bbeefcSYuanhan Liu void
467d5bbeefcSYuanhan Liu vtpci_read_dev_config(struct virtio_hw *hw, size_t offset,
468d5bbeefcSYuanhan Liu 		      void *dst, int length)
469d5bbeefcSYuanhan Liu {
470553f4593SYuanhan Liu 	VTPCI_OPS(hw)->read_dev_cfg(hw, offset, dst, length);
471d5bbeefcSYuanhan Liu }
472d5bbeefcSYuanhan Liu 
473d5bbeefcSYuanhan Liu void
474d5bbeefcSYuanhan Liu vtpci_write_dev_config(struct virtio_hw *hw, size_t offset,
475d5bbeefcSYuanhan Liu 		       const void *src, int length)
476d5bbeefcSYuanhan Liu {
477553f4593SYuanhan Liu 	VTPCI_OPS(hw)->write_dev_cfg(hw, offset, src, length);
478d5bbeefcSYuanhan Liu }
479d5bbeefcSYuanhan Liu 
4803891f233SYuanhan Liu uint64_t
4813891f233SYuanhan Liu vtpci_negotiate_features(struct virtio_hw *hw, uint64_t host_features)
4826c3169a3SBruce Richardson {
4833891f233SYuanhan Liu 	uint64_t features;
484d5bbeefcSYuanhan Liu 
4856c3169a3SBruce Richardson 	/*
4866c3169a3SBruce Richardson 	 * Limit negotiated features to what the driver, virtqueue, and
4876c3169a3SBruce Richardson 	 * host all support.
4886c3169a3SBruce Richardson 	 */
4896c3169a3SBruce Richardson 	features = host_features & hw->guest_features;
490553f4593SYuanhan Liu 	VTPCI_OPS(hw)->set_features(hw, features);
4916c3169a3SBruce Richardson 
4926c3169a3SBruce Richardson 	return features;
4936c3169a3SBruce Richardson }
4946c3169a3SBruce Richardson 
4956c3169a3SBruce Richardson void
4966c3169a3SBruce Richardson vtpci_reset(struct virtio_hw *hw)
4976c3169a3SBruce Richardson {
498553f4593SYuanhan Liu 	VTPCI_OPS(hw)->set_status(hw, VIRTIO_CONFIG_STATUS_RESET);
499d5bbeefcSYuanhan Liu 	/* flush status write */
500553f4593SYuanhan Liu 	VTPCI_OPS(hw)->get_status(hw);
5016c3169a3SBruce Richardson }
5026c3169a3SBruce Richardson 
5036c3169a3SBruce Richardson void
5046c3169a3SBruce Richardson vtpci_reinit_complete(struct virtio_hw *hw)
5056c3169a3SBruce Richardson {
5066c3169a3SBruce Richardson 	vtpci_set_status(hw, VIRTIO_CONFIG_STATUS_DRIVER_OK);
5076c3169a3SBruce Richardson }
5086c3169a3SBruce Richardson 
5096c3169a3SBruce Richardson void
5106c3169a3SBruce Richardson vtpci_set_status(struct virtio_hw *hw, uint8_t status)
5116c3169a3SBruce Richardson {
5126c3169a3SBruce Richardson 	if (status != VIRTIO_CONFIG_STATUS_RESET)
513553f4593SYuanhan Liu 		status |= VTPCI_OPS(hw)->get_status(hw);
5146c3169a3SBruce Richardson 
515553f4593SYuanhan Liu 	VTPCI_OPS(hw)->set_status(hw, status);
5166c3169a3SBruce Richardson }
5176c3169a3SBruce Richardson 
5186c3169a3SBruce Richardson uint8_t
5196ba1f63bSYuanhan Liu vtpci_get_status(struct virtio_hw *hw)
5206ba1f63bSYuanhan Liu {
521553f4593SYuanhan Liu 	return VTPCI_OPS(hw)->get_status(hw);
5226ba1f63bSYuanhan Liu }
5236ba1f63bSYuanhan Liu 
5246ba1f63bSYuanhan Liu uint8_t
5256c3169a3SBruce Richardson vtpci_isr(struct virtio_hw *hw)
5266c3169a3SBruce Richardson {
527553f4593SYuanhan Liu 	return VTPCI_OPS(hw)->get_isr(hw);
5286c3169a3SBruce Richardson }
5296c3169a3SBruce Richardson 
5306ba1f63bSYuanhan Liu static void *
5316ba1f63bSYuanhan Liu get_cfg_addr(struct rte_pci_device *dev, struct virtio_pci_cap *cap)
5326ba1f63bSYuanhan Liu {
5336ba1f63bSYuanhan Liu 	uint8_t  bar    = cap->bar;
5346ba1f63bSYuanhan Liu 	uint32_t length = cap->length;
5356ba1f63bSYuanhan Liu 	uint32_t offset = cap->offset;
5366ba1f63bSYuanhan Liu 	uint8_t *base;
5376ba1f63bSYuanhan Liu 
5380373ab9bSZhiyong Yang 	if (bar >= PCI_MAX_RESOURCE) {
5396ba1f63bSYuanhan Liu 		PMD_INIT_LOG(ERR, "invalid bar: %u", bar);
5406ba1f63bSYuanhan Liu 		return NULL;
5416ba1f63bSYuanhan Liu 	}
5426ba1f63bSYuanhan Liu 
5436ba1f63bSYuanhan Liu 	if (offset + length < offset) {
5446ba1f63bSYuanhan Liu 		PMD_INIT_LOG(ERR, "offset(%u) + length(%u) overflows",
5456ba1f63bSYuanhan Liu 			offset, length);
5466ba1f63bSYuanhan Liu 		return NULL;
5476ba1f63bSYuanhan Liu 	}
5486ba1f63bSYuanhan Liu 
5496ba1f63bSYuanhan Liu 	if (offset + length > dev->mem_resource[bar].len) {
5506ba1f63bSYuanhan Liu 		PMD_INIT_LOG(ERR,
5516ba1f63bSYuanhan Liu 			"invalid cap: overflows bar space: %u > %" PRIu64,
5526ba1f63bSYuanhan Liu 			offset + length, dev->mem_resource[bar].len);
5536ba1f63bSYuanhan Liu 		return NULL;
5546ba1f63bSYuanhan Liu 	}
5556ba1f63bSYuanhan Liu 
5566ba1f63bSYuanhan Liu 	base = dev->mem_resource[bar].addr;
5576ba1f63bSYuanhan Liu 	if (base == NULL) {
5586ba1f63bSYuanhan Liu 		PMD_INIT_LOG(ERR, "bar %u base addr is NULL", bar);
5596ba1f63bSYuanhan Liu 		return NULL;
5606ba1f63bSYuanhan Liu 	}
5616ba1f63bSYuanhan Liu 
5626ba1f63bSYuanhan Liu 	return base + offset;
5636ba1f63bSYuanhan Liu }
5646ba1f63bSYuanhan Liu 
565cb482cb3SJianfeng Tan #define PCI_MSIX_ENABLE 0x8000
566cb482cb3SJianfeng Tan 
5676ba1f63bSYuanhan Liu static int
5686ba1f63bSYuanhan Liu virtio_read_caps(struct rte_pci_device *dev, struct virtio_hw *hw)
5696ba1f63bSYuanhan Liu {
5706ba1f63bSYuanhan Liu 	uint8_t pos;
5716ba1f63bSYuanhan Liu 	struct virtio_pci_cap cap;
5726ba1f63bSYuanhan Liu 	int ret;
5736ba1f63bSYuanhan Liu 
5743dcfe039SThomas Monjalon 	if (rte_pci_map_device(dev)) {
5756ba1f63bSYuanhan Liu 		PMD_INIT_LOG(DEBUG, "failed to map pci device!");
5766ba1f63bSYuanhan Liu 		return -1;
5776ba1f63bSYuanhan Liu 	}
5786ba1f63bSYuanhan Liu 
5793dcfe039SThomas Monjalon 	ret = rte_pci_read_config(dev, &pos, 1, PCI_CAPABILITY_LIST);
58049bb1f7aSBrian Russell 	if (ret != 1) {
58149bb1f7aSBrian Russell 		PMD_INIT_LOG(DEBUG,
58249bb1f7aSBrian Russell 			     "failed to read pci capability list, ret %d", ret);
5836ba1f63bSYuanhan Liu 		return -1;
5846ba1f63bSYuanhan Liu 	}
5856ba1f63bSYuanhan Liu 
5866ba1f63bSYuanhan Liu 	while (pos) {
58749bb1f7aSBrian Russell 		ret = rte_pci_read_config(dev, &cap, 2, pos);
58849bb1f7aSBrian Russell 		if (ret != 2) {
58949bb1f7aSBrian Russell 			PMD_INIT_LOG(DEBUG,
59049bb1f7aSBrian Russell 				     "failed to read pci cap at pos: %x ret %d",
59149bb1f7aSBrian Russell 				     pos, ret);
5926ba1f63bSYuanhan Liu 			break;
5936ba1f63bSYuanhan Liu 		}
5946ba1f63bSYuanhan Liu 
595cb482cb3SJianfeng Tan 		if (cap.cap_vndr == PCI_CAP_ID_MSIX) {
596cb482cb3SJianfeng Tan 			/* Transitional devices would also have this capability,
597cb482cb3SJianfeng Tan 			 * that's why we also check if msix is enabled.
598cb482cb3SJianfeng Tan 			 * 1st byte is cap ID; 2nd byte is the position of next
599cb482cb3SJianfeng Tan 			 * cap; next two bytes are the flags.
600cb482cb3SJianfeng Tan 			 */
60149bb1f7aSBrian Russell 			uint16_t flags;
60249bb1f7aSBrian Russell 
60349bb1f7aSBrian Russell 			ret = rte_pci_read_config(dev, &flags, sizeof(flags),
60449bb1f7aSBrian Russell 					pos + 2);
60549bb1f7aSBrian Russell 			if (ret != sizeof(flags)) {
60649bb1f7aSBrian Russell 				PMD_INIT_LOG(DEBUG,
60749bb1f7aSBrian Russell 					     "failed to read pci cap at pos:"
60849bb1f7aSBrian Russell 					     " %x ret %d", pos + 2, ret);
60949bb1f7aSBrian Russell 				break;
61049bb1f7aSBrian Russell 			}
611cb482cb3SJianfeng Tan 
612cb482cb3SJianfeng Tan 			if (flags & PCI_MSIX_ENABLE)
613fe19d49cSZhiyong Yang 				hw->use_msix = VIRTIO_MSIX_ENABLED;
614fe19d49cSZhiyong Yang 			else
615fe19d49cSZhiyong Yang 				hw->use_msix = VIRTIO_MSIX_DISABLED;
616cb482cb3SJianfeng Tan 		}
617554b6d3eSJianfeng Tan 
6186ba1f63bSYuanhan Liu 		if (cap.cap_vndr != PCI_CAP_ID_VNDR) {
6196ba1f63bSYuanhan Liu 			PMD_INIT_LOG(DEBUG,
6206ba1f63bSYuanhan Liu 				"[%2x] skipping non VNDR cap id: %02x",
6216ba1f63bSYuanhan Liu 				pos, cap.cap_vndr);
6226ba1f63bSYuanhan Liu 			goto next;
6236ba1f63bSYuanhan Liu 		}
6246ba1f63bSYuanhan Liu 
62549bb1f7aSBrian Russell 		ret = rte_pci_read_config(dev, &cap, sizeof(cap), pos);
62649bb1f7aSBrian Russell 		if (ret != sizeof(cap)) {
62749bb1f7aSBrian Russell 			PMD_INIT_LOG(DEBUG,
62849bb1f7aSBrian Russell 				     "failed to read pci cap at pos: %x ret %d",
62949bb1f7aSBrian Russell 				     pos, ret);
63049bb1f7aSBrian Russell 			break;
63149bb1f7aSBrian Russell 		}
63249bb1f7aSBrian Russell 
6336ba1f63bSYuanhan Liu 		PMD_INIT_LOG(DEBUG,
6346ba1f63bSYuanhan Liu 			"[%2x] cfg type: %u, bar: %u, offset: %04x, len: %u",
6356ba1f63bSYuanhan Liu 			pos, cap.cfg_type, cap.bar, cap.offset, cap.length);
6366ba1f63bSYuanhan Liu 
6376ba1f63bSYuanhan Liu 		switch (cap.cfg_type) {
6386ba1f63bSYuanhan Liu 		case VIRTIO_PCI_CAP_COMMON_CFG:
6396ba1f63bSYuanhan Liu 			hw->common_cfg = get_cfg_addr(dev, &cap);
6406ba1f63bSYuanhan Liu 			break;
6416ba1f63bSYuanhan Liu 		case VIRTIO_PCI_CAP_NOTIFY_CFG:
642ecfae151STiwei Bie 			ret = rte_pci_read_config(dev,
643ecfae151STiwei Bie 					&hw->notify_off_multiplier,
6446ba1f63bSYuanhan Liu 					4, pos + sizeof(cap));
645ecfae151STiwei Bie 			if (ret != 4)
646ecfae151STiwei Bie 				PMD_INIT_LOG(DEBUG,
647ecfae151STiwei Bie 					"failed to read notify_off_multiplier, ret %d",
648ecfae151STiwei Bie 					ret);
649ecfae151STiwei Bie 			else
6506ba1f63bSYuanhan Liu 				hw->notify_base = get_cfg_addr(dev, &cap);
6516ba1f63bSYuanhan Liu 			break;
6526ba1f63bSYuanhan Liu 		case VIRTIO_PCI_CAP_DEVICE_CFG:
6536ba1f63bSYuanhan Liu 			hw->dev_cfg = get_cfg_addr(dev, &cap);
6546ba1f63bSYuanhan Liu 			break;
6556ba1f63bSYuanhan Liu 		case VIRTIO_PCI_CAP_ISR_CFG:
6566ba1f63bSYuanhan Liu 			hw->isr = get_cfg_addr(dev, &cap);
6576ba1f63bSYuanhan Liu 			break;
6586ba1f63bSYuanhan Liu 		}
6596ba1f63bSYuanhan Liu 
6606ba1f63bSYuanhan Liu next:
6616ba1f63bSYuanhan Liu 		pos = cap.cap_next;
6626ba1f63bSYuanhan Liu 	}
6636ba1f63bSYuanhan Liu 
6646ba1f63bSYuanhan Liu 	if (hw->common_cfg == NULL || hw->notify_base == NULL ||
6656ba1f63bSYuanhan Liu 	    hw->dev_cfg == NULL    || hw->isr == NULL) {
6666ba1f63bSYuanhan Liu 		PMD_INIT_LOG(INFO, "no modern virtio pci device found.");
6676ba1f63bSYuanhan Liu 		return -1;
6686ba1f63bSYuanhan Liu 	}
6696ba1f63bSYuanhan Liu 
6706ba1f63bSYuanhan Liu 	PMD_INIT_LOG(INFO, "found modern virtio pci device.");
6716ba1f63bSYuanhan Liu 
6726ba1f63bSYuanhan Liu 	PMD_INIT_LOG(DEBUG, "common cfg mapped at: %p", hw->common_cfg);
6736ba1f63bSYuanhan Liu 	PMD_INIT_LOG(DEBUG, "device cfg mapped at: %p", hw->dev_cfg);
6746ba1f63bSYuanhan Liu 	PMD_INIT_LOG(DEBUG, "isr cfg mapped at: %p", hw->isr);
6756ba1f63bSYuanhan Liu 	PMD_INIT_LOG(DEBUG, "notify base: %p, notify off multiplier: %u",
6766ba1f63bSYuanhan Liu 		hw->notify_base, hw->notify_off_multiplier);
6776ba1f63bSYuanhan Liu 
6786ba1f63bSYuanhan Liu 	return 0;
6796ba1f63bSYuanhan Liu }
6806ba1f63bSYuanhan Liu 
681ac5e1d83SHuawei Xie /*
682ac5e1d83SHuawei Xie  * Return -1:
683ac5e1d83SHuawei Xie  *   if there is error mapping with VFIO/UIO.
684ac5e1d83SHuawei Xie  *   if port map error when driver type is KDRV_NONE.
685*a65a34a8SStephen Hemminger  *   if marked as allowed but driver type is KDRV_UNKNOWN.
686ac5e1d83SHuawei Xie  * Return 1 if kernel driver is managing the device.
687ac5e1d83SHuawei Xie  * Return 0 on success.
688ac5e1d83SHuawei Xie  */
689d5bbeefcSYuanhan Liu int
690a60a0c15SJianfeng Tan vtpci_init(struct rte_pci_device *dev, struct virtio_hw *hw)
691d5bbeefcSYuanhan Liu {
6926ba1f63bSYuanhan Liu 	/*
6936ba1f63bSYuanhan Liu 	 * Try if we can succeed reading virtio pci caps, which exists
6946ba1f63bSYuanhan Liu 	 * only on modern pci device. If failed, we fallback to legacy
6956ba1f63bSYuanhan Liu 	 * virtio handling.
6966ba1f63bSYuanhan Liu 	 */
6976ba1f63bSYuanhan Liu 	if (virtio_read_caps(dev, hw) == 0) {
6986ba1f63bSYuanhan Liu 		PMD_INIT_LOG(INFO, "modern virtio pci detected.");
699553f4593SYuanhan Liu 		virtio_hw_internal[hw->port_id].vtpci_ops = &modern_ops;
7006ba1f63bSYuanhan Liu 		hw->modern = 1;
7016ba1f63bSYuanhan Liu 		return 0;
7026ba1f63bSYuanhan Liu 	}
7036ba1f63bSYuanhan Liu 
7046ba1f63bSYuanhan Liu 	PMD_INIT_LOG(INFO, "trying with legacy virtio pci.");
7053dcfe039SThomas Monjalon 	if (rte_pci_ioport_map(dev, 0, VTPCI_IO(hw)) < 0) {
7063484c8d8STiwei Bie 		rte_pci_unmap_device(dev);
7077c0d798aSDavid Marchand 		if (dev->kdrv == RTE_PCI_KDRV_UNKNOWN &&
70813a1317dSJan Viktorin 		    (!dev->device.devargs ||
7092b0e39c1SGaetan Rivet 		     dev->device.devargs->bus !=
7102b0e39c1SGaetan Rivet 		     rte_bus_find_by_name("pci"))) {
711ac5e1d83SHuawei Xie 			PMD_INIT_LOG(INFO,
712ac5e1d83SHuawei Xie 				"skip kernel managed virtio device.");
713ac5e1d83SHuawei Xie 			return 1;
714ac5e1d83SHuawei Xie 		}
715c52afa68SYuanhan Liu 		return -1;
716ac5e1d83SHuawei Xie 	}
7176ba1f63bSYuanhan Liu 
718553f4593SYuanhan Liu 	virtio_hw_internal[hw->port_id].vtpci_ops = &legacy_ops;
7196ba1f63bSYuanhan Liu 	hw->modern   = 0;
720c52afa68SYuanhan Liu 
721d5bbeefcSYuanhan Liu 	return 0;
7226c3169a3SBruce Richardson }
723fe19d49cSZhiyong Yang 
724fe19d49cSZhiyong Yang enum virtio_msix_status
725fe19d49cSZhiyong Yang vtpci_msix_detect(struct rte_pci_device *dev)
726fe19d49cSZhiyong Yang {
727fe19d49cSZhiyong Yang 	uint8_t pos;
728fe19d49cSZhiyong Yang 	int ret;
729fe19d49cSZhiyong Yang 
730fe19d49cSZhiyong Yang 	ret = rte_pci_read_config(dev, &pos, 1, PCI_CAPABILITY_LIST);
73149bb1f7aSBrian Russell 	if (ret != 1) {
73249bb1f7aSBrian Russell 		PMD_INIT_LOG(DEBUG,
73349bb1f7aSBrian Russell 			     "failed to read pci capability list, ret %d", ret);
734fe19d49cSZhiyong Yang 		return VIRTIO_MSIX_NONE;
735fe19d49cSZhiyong Yang 	}
736fe19d49cSZhiyong Yang 
737fe19d49cSZhiyong Yang 	while (pos) {
73849bb1f7aSBrian Russell 		uint8_t cap[2];
73949bb1f7aSBrian Russell 
74049bb1f7aSBrian Russell 		ret = rte_pci_read_config(dev, cap, sizeof(cap), pos);
74149bb1f7aSBrian Russell 		if (ret != sizeof(cap)) {
74249bb1f7aSBrian Russell 			PMD_INIT_LOG(DEBUG,
74349bb1f7aSBrian Russell 				     "failed to read pci cap at pos: %x ret %d",
74449bb1f7aSBrian Russell 				     pos, ret);
745fe19d49cSZhiyong Yang 			break;
746fe19d49cSZhiyong Yang 		}
747fe19d49cSZhiyong Yang 
74849bb1f7aSBrian Russell 		if (cap[0] == PCI_CAP_ID_MSIX) {
74949bb1f7aSBrian Russell 			uint16_t flags;
75049bb1f7aSBrian Russell 
75149bb1f7aSBrian Russell 			ret = rte_pci_read_config(dev, &flags, sizeof(flags),
75249bb1f7aSBrian Russell 					pos + sizeof(cap));
75349bb1f7aSBrian Russell 			if (ret != sizeof(flags)) {
75449bb1f7aSBrian Russell 				PMD_INIT_LOG(DEBUG,
75549bb1f7aSBrian Russell 					     "failed to read pci cap at pos:"
75649bb1f7aSBrian Russell 					     " %x ret %d", pos + 2, ret);
75749bb1f7aSBrian Russell 				break;
75849bb1f7aSBrian Russell 			}
759fe19d49cSZhiyong Yang 
760fe19d49cSZhiyong Yang 			if (flags & PCI_MSIX_ENABLE)
761fe19d49cSZhiyong Yang 				return VIRTIO_MSIX_ENABLED;
762fe19d49cSZhiyong Yang 			else
763fe19d49cSZhiyong Yang 				return VIRTIO_MSIX_DISABLED;
764fe19d49cSZhiyong Yang 		}
765fe19d49cSZhiyong Yang 
76649bb1f7aSBrian Russell 		pos = cap[1];
767fe19d49cSZhiyong Yang 	}
768fe19d49cSZhiyong Yang 
769fe19d49cSZhiyong Yang 	return VIRTIO_MSIX_NONE;
770fe19d49cSZhiyong Yang }
771