xref: /dpdk/drivers/net/virtio/virtio_user_ethdev.c (revision ff11fc60c5d8d9ae5a0f0114db4c3bc834090548)
15566a3e3SBruce Richardson /* SPDX-License-Identifier: BSD-3-Clause
25566a3e3SBruce Richardson  * Copyright(c) 2010-2016 Intel Corporation
3e9efa4d9SJianfeng Tan  */
4e9efa4d9SJianfeng Tan 
5e9efa4d9SJianfeng Tan #include <stdint.h>
672b452c5SDmitry Kozlyuk #include <stdlib.h>
7e9efa4d9SJianfeng Tan #include <sys/types.h>
8e9efa4d9SJianfeng Tan #include <unistd.h>
9ef53b603SJianfeng Tan #include <fcntl.h>
107d62bf6fSMaxime Coquelin #include <linux/major.h>
11f908b22eSAdrian Moreno #include <sys/stat.h>
127d62bf6fSMaxime Coquelin #include <sys/sysmacros.h>
13ef53b603SJianfeng Tan #include <sys/socket.h>
14e9efa4d9SJianfeng Tan 
15ce2eabddSJianfeng Tan #include <rte_malloc.h>
16ce2eabddSJianfeng Tan #include <rte_kvargs.h>
17df96fd0dSBruce Richardson #include <ethdev_vdev.h>
184851ef2bSDavid Marchand #include <bus_vdev_driver.h>
19ef53b603SJianfeng Tan #include <rte_alarm.h>
206ebbf410SXuan Ding #include <rte_cycles.h>
210fd27826SSrujana Challa #include <rte_io.h>
22ce2eabddSJianfeng Tan 
23ce2eabddSJianfeng Tan #include "virtio_ethdev.h"
24e9efa4d9SJianfeng Tan #include "virtio_logs.h"
25f8b60756SMaxime Coquelin #include "virtio.h"
26e9efa4d9SJianfeng Tan #include "virtqueue.h"
271b69528eSJianfeng Tan #include "virtio_rxtx.h"
28e9efa4d9SJianfeng Tan #include "virtio_user/virtio_user_dev.h"
297f468b2eSTiwei Bie #include "virtio_user/vhost.h"
30e9efa4d9SJianfeng Tan 
3165832c82SMaxime Coquelin #define virtio_user_get_dev(hwp) container_of(hwp, struct virtio_user_dev, hw)
32e9efa4d9SJianfeng Tan 
336ebbf410SXuan Ding static void
34e9efa4d9SJianfeng Tan virtio_user_read_dev_config(struct virtio_hw *hw, size_t offset,
35e9efa4d9SJianfeng Tan 		     void *dst, int length)
36e9efa4d9SJianfeng Tan {
37e9efa4d9SJianfeng Tan 	int i;
38e9efa4d9SJianfeng Tan 	struct virtio_user_dev *dev = virtio_user_get_dev(hw);
39e9efa4d9SJianfeng Tan 
40e9efa4d9SJianfeng Tan 	if (offset == offsetof(struct virtio_net_config, mac) &&
4135b2d13fSOlivier Matz 	    length == RTE_ETHER_ADDR_LEN) {
4235b2d13fSOlivier Matz 		for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i)
43e9efa4d9SJianfeng Tan 			((uint8_t *)dst)[i] = dev->mac_addr[i];
44e9efa4d9SJianfeng Tan 		return;
45e9efa4d9SJianfeng Tan 	}
46e9efa4d9SJianfeng Tan 
47ef53b603SJianfeng Tan 	if (offset == offsetof(struct virtio_net_config, status)) {
4894973531SMaxime Coquelin 		virtio_user_dev_update_link_state(dev);
49bd8f50a4SZhiyong Yang 
50d0131e49SXiao Wang 		*(uint16_t *)dst = dev->net_status;
51ef53b603SJianfeng Tan 	}
52e9efa4d9SJianfeng Tan 
53e9efa4d9SJianfeng Tan 	if (offset == offsetof(struct virtio_net_config, max_virtqueue_pairs))
54e9efa4d9SJianfeng Tan 		*(uint16_t *)dst = dev->max_queue_pairs;
55dbce501eSSrujana Challa 
56dbce501eSSrujana Challa 	if (offset >= offsetof(struct virtio_net_config, rss_max_key_size))
57dbce501eSSrujana Challa 		virtio_user_dev_get_rss_config(dev, dst, offset, length);
58e9efa4d9SJianfeng Tan }
59e9efa4d9SJianfeng Tan 
60e9efa4d9SJianfeng Tan static void
61e9efa4d9SJianfeng Tan virtio_user_write_dev_config(struct virtio_hw *hw, size_t offset,
62e9efa4d9SJianfeng Tan 		      const void *src, int length)
63e9efa4d9SJianfeng Tan {
64e9efa4d9SJianfeng Tan 	int i;
65e9efa4d9SJianfeng Tan 	struct virtio_user_dev *dev = virtio_user_get_dev(hw);
66e9efa4d9SJianfeng Tan 
67e9efa4d9SJianfeng Tan 	if ((offset == offsetof(struct virtio_net_config, mac)) &&
68c995b005SMaxime Coquelin 	    (length == RTE_ETHER_ADDR_LEN)) {
6935b2d13fSOlivier Matz 		for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i)
70e9efa4d9SJianfeng Tan 			dev->mac_addr[i] = ((const uint8_t *)src)[i];
71c995b005SMaxime Coquelin 		virtio_user_dev_set_mac(dev);
72c995b005SMaxime Coquelin 		virtio_user_dev_get_mac(dev);
73c995b005SMaxime Coquelin 	} else {
74f2462150SFerruh Yigit 		PMD_DRV_LOG(ERR, "not supported offset=%zu, len=%d",
75e9efa4d9SJianfeng Tan 			    offset, length);
76e9efa4d9SJianfeng Tan 	}
77c995b005SMaxime Coquelin }
78e9efa4d9SJianfeng Tan 
79e9efa4d9SJianfeng Tan static void
80c12a26eeSJianfeng Tan virtio_user_reset(struct virtio_hw *hw)
81c12a26eeSJianfeng Tan {
82c12a26eeSJianfeng Tan 	struct virtio_user_dev *dev = virtio_user_get_dev(hw);
83c12a26eeSJianfeng Tan 
84c12a26eeSJianfeng Tan 	if (dev->status & VIRTIO_CONFIG_STATUS_DRIVER_OK)
85c12a26eeSJianfeng Tan 		virtio_user_stop_device(dev);
86c12a26eeSJianfeng Tan }
87c12a26eeSJianfeng Tan 
88c12a26eeSJianfeng Tan static void
89e9efa4d9SJianfeng Tan virtio_user_set_status(struct virtio_hw *hw, uint8_t status)
90e9efa4d9SJianfeng Tan {
91e9efa4d9SJianfeng Tan 	struct virtio_user_dev *dev = virtio_user_get_dev(hw);
92844e4683SMaxime Coquelin 	uint8_t old_status = dev->status;
93e9efa4d9SJianfeng Tan 
94844e4683SMaxime Coquelin 	if (status & VIRTIO_CONFIG_STATUS_FEATURES_OK &&
95844e4683SMaxime Coquelin 			~old_status & VIRTIO_CONFIG_STATUS_FEATURES_OK)
96844e4683SMaxime Coquelin 		virtio_user_dev_set_features(dev);
97c1a6f787SMaxime Coquelin 
98c1a6f787SMaxime Coquelin 	if (status & VIRTIO_CONFIG_STATUS_DRIVER_OK) {
99c1a6f787SMaxime Coquelin 		if (virtio_user_start_device(dev)) {
100c1a6f787SMaxime Coquelin 			virtio_user_dev_update_status(dev);
101c1a6f787SMaxime Coquelin 			return;
102c1a6f787SMaxime Coquelin 		}
103c1a6f787SMaxime Coquelin 	} else if (status == VIRTIO_CONFIG_STATUS_RESET) {
104c12a26eeSJianfeng Tan 		virtio_user_reset(hw);
105c1a6f787SMaxime Coquelin 	}
106d7e10ea9SAdrian Moreno 
107d7e10ea9SAdrian Moreno 	virtio_user_dev_set_status(dev, status);
108e9efa4d9SJianfeng Tan }
109e9efa4d9SJianfeng Tan 
110e9efa4d9SJianfeng Tan static uint8_t
111e9efa4d9SJianfeng Tan virtio_user_get_status(struct virtio_hw *hw)
112e9efa4d9SJianfeng Tan {
113e9efa4d9SJianfeng Tan 	struct virtio_user_dev *dev = virtio_user_get_dev(hw);
114e9efa4d9SJianfeng Tan 
115d7e10ea9SAdrian Moreno 	virtio_user_dev_update_status(dev);
1160b0dc66cSAdrian Moreno 
117e9efa4d9SJianfeng Tan 	return dev->status;
118e9efa4d9SJianfeng Tan }
119e9efa4d9SJianfeng Tan 
120e9efa4d9SJianfeng Tan static uint64_t
121e9efa4d9SJianfeng Tan virtio_user_get_features(struct virtio_hw *hw)
122e9efa4d9SJianfeng Tan {
123e9efa4d9SJianfeng Tan 	struct virtio_user_dev *dev = virtio_user_get_dev(hw);
124e9efa4d9SJianfeng Tan 
125142678d4SJianfeng Tan 	/* unmask feature bits defined in vhost user protocol */
126f078c2f0SMaxime Coquelin 	return (dev->device_features | dev->frontend_features) &
127f078c2f0SMaxime Coquelin 		VIRTIO_PMD_SUPPORTED_GUEST_FEATURES;
128e9efa4d9SJianfeng Tan }
129e9efa4d9SJianfeng Tan 
130e9efa4d9SJianfeng Tan static void
131e9efa4d9SJianfeng Tan virtio_user_set_features(struct virtio_hw *hw, uint64_t features)
132e9efa4d9SJianfeng Tan {
133e9efa4d9SJianfeng Tan 	struct virtio_user_dev *dev = virtio_user_get_dev(hw);
134e9efa4d9SJianfeng Tan 
135f078c2f0SMaxime Coquelin 	dev->features = features & (dev->device_features | dev->frontend_features);
136e9efa4d9SJianfeng Tan }
137e9efa4d9SJianfeng Tan 
138cbb135b3SMaxime Coquelin static int
139cbb135b3SMaxime Coquelin virtio_user_features_ok(struct virtio_hw *hw __rte_unused)
140cbb135b3SMaxime Coquelin {
141cbb135b3SMaxime Coquelin 	return 0;
142cbb135b3SMaxime Coquelin }
143cbb135b3SMaxime Coquelin 
144e9efa4d9SJianfeng Tan static uint8_t
145e9efa4d9SJianfeng Tan virtio_user_get_isr(struct virtio_hw *hw __rte_unused)
146e9efa4d9SJianfeng Tan {
14735c4f855SJianfeng Tan 	/* rxq interrupts and config interrupt are separated in virtio-user,
14835c4f855SJianfeng Tan 	 * here we only report config change.
149e9efa4d9SJianfeng Tan 	 */
1506a504290SMaxime Coquelin 	return VIRTIO_ISR_CONFIG;
151e9efa4d9SJianfeng Tan }
152e9efa4d9SJianfeng Tan 
153e9efa4d9SJianfeng Tan static uint16_t
154e9efa4d9SJianfeng Tan virtio_user_set_config_irq(struct virtio_hw *hw __rte_unused,
155e9efa4d9SJianfeng Tan 		    uint16_t vec __rte_unused)
156e9efa4d9SJianfeng Tan {
15735c4f855SJianfeng Tan 	return 0;
158e9efa4d9SJianfeng Tan }
159e9efa4d9SJianfeng Tan 
1603d4fb6fdSJianfeng Tan static uint16_t
1613d4fb6fdSJianfeng Tan virtio_user_set_queue_irq(struct virtio_hw *hw __rte_unused,
1623d4fb6fdSJianfeng Tan 			  struct virtqueue *vq __rte_unused,
1633d4fb6fdSJianfeng Tan 			  uint16_t vec)
1643d4fb6fdSJianfeng Tan {
1653d4fb6fdSJianfeng Tan 	/* pretend we have done that */
1663d4fb6fdSJianfeng Tan 	return vec;
1673d4fb6fdSJianfeng Tan }
1683d4fb6fdSJianfeng Tan 
169e9efa4d9SJianfeng Tan /* This function is to get the queue size, aka, number of descs, of a specified
170e9efa4d9SJianfeng Tan  * queue. Different with the VHOST_USER_GET_QUEUE_NUM, which is used to get the
171e9efa4d9SJianfeng Tan  * max supported queues.
172e9efa4d9SJianfeng Tan  */
173e9efa4d9SJianfeng Tan static uint16_t
174e9efa4d9SJianfeng Tan virtio_user_get_queue_num(struct virtio_hw *hw, uint16_t queue_id __rte_unused)
175e9efa4d9SJianfeng Tan {
176e9efa4d9SJianfeng Tan 	struct virtio_user_dev *dev = virtio_user_get_dev(hw);
177e9efa4d9SJianfeng Tan 
178e9efa4d9SJianfeng Tan 	/* Currently, each queue has same queue size */
179e9efa4d9SJianfeng Tan 	return dev->queue_size;
180e9efa4d9SJianfeng Tan }
181e9efa4d9SJianfeng Tan 
18248a44640SJens Freimann static void
18348a44640SJens Freimann virtio_user_setup_queue_packed(struct virtqueue *vq,
18448a44640SJens Freimann 			       struct virtio_user_dev *dev)
185e9efa4d9SJianfeng Tan {
18648a44640SJens Freimann 	uint16_t queue_idx = vq->vq_queue_index;
18748a44640SJens Freimann 	struct vring_packed *vring;
18848a44640SJens Freimann 	uint64_t desc_addr;
18948a44640SJens Freimann 	uint64_t avail_addr;
19048a44640SJens Freimann 	uint64_t used_addr;
19148a44640SJens Freimann 	uint16_t i;
19248a44640SJens Freimann 
1936fdf32d1SMaxime Coquelin 	vring  = &dev->vrings.packed[queue_idx];
19448a44640SJens Freimann 	desc_addr = (uintptr_t)vq->vq_ring_virt_mem;
19548a44640SJens Freimann 	avail_addr = desc_addr + vq->vq_nentries *
19648a44640SJens Freimann 		sizeof(struct vring_packed_desc);
19748a44640SJens Freimann 	used_addr = RTE_ALIGN_CEIL(avail_addr +
19848a44640SJens Freimann 			   sizeof(struct vring_packed_desc_event),
199df968842SMaxime Coquelin 			   VIRTIO_VRING_ALIGN);
20048a44640SJens Freimann 	vring->num = vq->vq_nentries;
201666ef294SSrujana Challa 	vring->desc_iova = vq->vq_ring_mem;
2024cdc4d98STiwei Bie 	vring->desc = (void *)(uintptr_t)desc_addr;
2034cdc4d98STiwei Bie 	vring->driver = (void *)(uintptr_t)avail_addr;
2044cdc4d98STiwei Bie 	vring->device = (void *)(uintptr_t)used_addr;
20548a44640SJens Freimann 	dev->packed_queues[queue_idx].avail_wrap_counter = true;
20648a44640SJens Freimann 	dev->packed_queues[queue_idx].used_wrap_counter = true;
207*ff11fc60SKommula Shiva Shankar 	dev->packed_queues[queue_idx].used_idx = 0;
20848a44640SJens Freimann 
20945c224e7STiwei Bie 	for (i = 0; i < vring->num; i++)
2104cdc4d98STiwei Bie 		vring->desc[i].flags = 0;
21148a44640SJens Freimann }
21248a44640SJens Freimann 
21348a44640SJens Freimann static void
21448a44640SJens Freimann virtio_user_setup_queue_split(struct virtqueue *vq, struct virtio_user_dev *dev)
21548a44640SJens Freimann {
216e9efa4d9SJianfeng Tan 	uint16_t queue_idx = vq->vq_queue_index;
217e9efa4d9SJianfeng Tan 	uint64_t desc_addr, avail_addr, used_addr;
218e9efa4d9SJianfeng Tan 
219e9efa4d9SJianfeng Tan 	desc_addr = (uintptr_t)vq->vq_ring_virt_mem;
220e9efa4d9SJianfeng Tan 	avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc);
221e9efa4d9SJianfeng Tan 	used_addr = RTE_ALIGN_CEIL(avail_addr + offsetof(struct vring_avail,
222e9efa4d9SJianfeng Tan 							 ring[vq->vq_nentries]),
223df968842SMaxime Coquelin 				   VIRTIO_VRING_ALIGN);
224e9efa4d9SJianfeng Tan 
2256fdf32d1SMaxime Coquelin 	dev->vrings.split[queue_idx].num = vq->vq_nentries;
226666ef294SSrujana Challa 	dev->vrings.split[queue_idx].desc_iova = vq->vq_ring_mem;
2276fdf32d1SMaxime Coquelin 	dev->vrings.split[queue_idx].desc = (void *)(uintptr_t)desc_addr;
2286fdf32d1SMaxime Coquelin 	dev->vrings.split[queue_idx].avail = (void *)(uintptr_t)avail_addr;
2296fdf32d1SMaxime Coquelin 	dev->vrings.split[queue_idx].used = (void *)(uintptr_t)used_addr;
23048a44640SJens Freimann }
23148a44640SJens Freimann 
23248a44640SJens Freimann static int
23348a44640SJens Freimann virtio_user_setup_queue(struct virtio_hw *hw, struct virtqueue *vq)
23448a44640SJens Freimann {
23548a44640SJens Freimann 	struct virtio_user_dev *dev = virtio_user_get_dev(hw);
23648a44640SJens Freimann 
237b4f9a45aSMaxime Coquelin 	if (virtio_with_packed_queue(hw))
23848a44640SJens Freimann 		virtio_user_setup_queue_packed(vq, dev);
23948a44640SJens Freimann 	else
24048a44640SJens Freimann 		virtio_user_setup_queue_split(vq, dev);
241e9efa4d9SJianfeng Tan 
2420fd27826SSrujana Challa 	if (dev->notify_area)
2430fd27826SSrujana Challa 		vq->notify_addr = dev->notify_area[vq->vq_queue_index];
2440fd27826SSrujana Challa 
24566b45ceaSMaxime Coquelin 	if (dev->hw_cvq && hw->cvq && (virtnet_cq_to_vq(hw->cvq) == vq))
24666b45ceaSMaxime Coquelin 		return virtio_user_dev_create_shadow_cvq(dev, vq);
24766b45ceaSMaxime Coquelin 
248e9efa4d9SJianfeng Tan 	return 0;
249e9efa4d9SJianfeng Tan }
250e9efa4d9SJianfeng Tan 
251e9efa4d9SJianfeng Tan static void
252e9efa4d9SJianfeng Tan virtio_user_del_queue(struct virtio_hw *hw, struct virtqueue *vq)
253e9efa4d9SJianfeng Tan {
254e9efa4d9SJianfeng Tan 	/* For legacy devices, write 0 to VIRTIO_PCI_QUEUE_PFN port, QEMU
255e9efa4d9SJianfeng Tan 	 * correspondingly stops the ioeventfds, and reset the status of
256e9efa4d9SJianfeng Tan 	 * the device.
257e9efa4d9SJianfeng Tan 	 * For modern devices, set queue desc, avail, used in PCI bar to 0,
258e9efa4d9SJianfeng Tan 	 * not see any more behavior in QEMU.
259e9efa4d9SJianfeng Tan 	 *
260e9efa4d9SJianfeng Tan 	 * Here we just care about what information to deliver to vhost-user
261e9efa4d9SJianfeng Tan 	 * or vhost-kernel. So we just close ioeventfd for now.
262e9efa4d9SJianfeng Tan 	 */
263e9efa4d9SJianfeng Tan 	struct virtio_user_dev *dev = virtio_user_get_dev(hw);
264e9efa4d9SJianfeng Tan 
265e9efa4d9SJianfeng Tan 	close(dev->callfds[vq->vq_queue_index]);
266e9efa4d9SJianfeng Tan 	close(dev->kickfds[vq->vq_queue_index]);
26766b45ceaSMaxime Coquelin 
26866b45ceaSMaxime Coquelin 	if (hw->cvq && (virtnet_cq_to_vq(hw->cvq) == vq))
26966b45ceaSMaxime Coquelin 		virtio_user_dev_destroy_shadow_cvq(dev);
270e9efa4d9SJianfeng Tan }
271e9efa4d9SJianfeng Tan 
272e9efa4d9SJianfeng Tan static void
273e9efa4d9SJianfeng Tan virtio_user_notify_queue(struct virtio_hw *hw, struct virtqueue *vq)
274e9efa4d9SJianfeng Tan {
275e9efa4d9SJianfeng Tan 	struct virtio_user_dev *dev = virtio_user_get_dev(hw);
2760fd27826SSrujana Challa 	uint64_t notify_data = 1;
277e9efa4d9SJianfeng Tan 
2783169550fSMaxime Coquelin 	if (hw->cvq && (virtnet_cq_to_vq(hw->cvq) == vq)) {
2791b69528eSJianfeng Tan 		virtio_user_handle_cq(dev, vq->vq_queue_index);
280fcdb603aSMaxime Coquelin 
2811b69528eSJianfeng Tan 		return;
2821b69528eSJianfeng Tan 	}
2831b69528eSJianfeng Tan 
2840fd27826SSrujana Challa 	if (!dev->notify_area) {
2850fd27826SSrujana Challa 		if (write(dev->kickfds[vq->vq_queue_index], &notify_data,
2860fd27826SSrujana Challa 			  sizeof(notify_data)) < 0)
287f2462150SFerruh Yigit 			PMD_DRV_LOG(ERR, "failed to kick backend: %s",
288e9efa4d9SJianfeng Tan 				    strerror(errno));
2890fd27826SSrujana Challa 		return;
2900fd27826SSrujana Challa 	} else if (!virtio_with_feature(hw, VIRTIO_F_NOTIFICATION_DATA)) {
2910fd27826SSrujana Challa 		rte_write16(vq->vq_queue_index, vq->notify_addr);
2920fd27826SSrujana Challa 		return;
2930fd27826SSrujana Challa 	}
2940fd27826SSrujana Challa 
2950fd27826SSrujana Challa 	if (virtio_with_packed_queue(hw)) {
2960fd27826SSrujana Challa 		/* Bit[0:15]: vq queue index
2970fd27826SSrujana Challa 		 * Bit[16:30]: avail index
2980fd27826SSrujana Challa 		 * Bit[31]: avail wrap counter
2990fd27826SSrujana Challa 		 */
3000fd27826SSrujana Challa 		notify_data = ((uint32_t)(!!(vq->vq_packed.cached_flags &
3010fd27826SSrujana Challa 				VRING_PACKED_DESC_F_AVAIL)) << 31) |
3020fd27826SSrujana Challa 				((uint32_t)vq->vq_avail_idx << 16) |
3030fd27826SSrujana Challa 				vq->vq_queue_index;
3040fd27826SSrujana Challa 	} else {
3050fd27826SSrujana Challa 		/* Bit[0:15]: vq queue index
3060fd27826SSrujana Challa 		 * Bit[16:31]: avail index
3070fd27826SSrujana Challa 		 */
3080fd27826SSrujana Challa 		notify_data = ((uint32_t)vq->vq_avail_idx << 16) |
3090fd27826SSrujana Challa 				vq->vq_queue_index;
3100fd27826SSrujana Challa 	}
3110fd27826SSrujana Challa 	rte_write32(notify_data, vq->notify_addr);
312e9efa4d9SJianfeng Tan }
313e9efa4d9SJianfeng Tan 
314f12908c8SMaxime Coquelin static int
315f12908c8SMaxime Coquelin virtio_user_dev_close(struct virtio_hw *hw)
316f12908c8SMaxime Coquelin {
317f12908c8SMaxime Coquelin 	struct virtio_user_dev *dev = virtio_user_get_dev(hw);
318f12908c8SMaxime Coquelin 
319f12908c8SMaxime Coquelin 	virtio_user_dev_uninit(dev);
320f12908c8SMaxime Coquelin 
321f12908c8SMaxime Coquelin 	return 0;
322f12908c8SMaxime Coquelin }
323f12908c8SMaxime Coquelin 
324f8b60756SMaxime Coquelin const struct virtio_ops virtio_user_ops = {
325e9efa4d9SJianfeng Tan 	.read_dev_cfg	= virtio_user_read_dev_config,
326e9efa4d9SJianfeng Tan 	.write_dev_cfg	= virtio_user_write_dev_config,
327e9efa4d9SJianfeng Tan 	.get_status	= virtio_user_get_status,
328e9efa4d9SJianfeng Tan 	.set_status	= virtio_user_set_status,
329e9efa4d9SJianfeng Tan 	.get_features	= virtio_user_get_features,
330e9efa4d9SJianfeng Tan 	.set_features	= virtio_user_set_features,
331cbb135b3SMaxime Coquelin 	.features_ok	= virtio_user_features_ok,
332e9efa4d9SJianfeng Tan 	.get_isr	= virtio_user_get_isr,
333e9efa4d9SJianfeng Tan 	.set_config_irq	= virtio_user_set_config_irq,
3343d4fb6fdSJianfeng Tan 	.set_queue_irq	= virtio_user_set_queue_irq,
335e9efa4d9SJianfeng Tan 	.get_queue_num	= virtio_user_get_queue_num,
336e9efa4d9SJianfeng Tan 	.setup_queue	= virtio_user_setup_queue,
337e9efa4d9SJianfeng Tan 	.del_queue	= virtio_user_del_queue,
338e9efa4d9SJianfeng Tan 	.notify_queue	= virtio_user_notify_queue,
339f12908c8SMaxime Coquelin 	.dev_close	= virtio_user_dev_close,
340e9efa4d9SJianfeng Tan };
341ce2eabddSJianfeng Tan 
342ce2eabddSJianfeng Tan static const char *valid_args[] = {
343ce2eabddSJianfeng Tan #define VIRTIO_USER_ARG_QUEUES_NUM     "queues"
344ce2eabddSJianfeng Tan 	VIRTIO_USER_ARG_QUEUES_NUM,
345ce2eabddSJianfeng Tan #define VIRTIO_USER_ARG_CQ_NUM         "cq"
346ce2eabddSJianfeng Tan 	VIRTIO_USER_ARG_CQ_NUM,
347ce2eabddSJianfeng Tan #define VIRTIO_USER_ARG_MAC            "mac"
348ce2eabddSJianfeng Tan 	VIRTIO_USER_ARG_MAC,
349ce2eabddSJianfeng Tan #define VIRTIO_USER_ARG_PATH           "path"
350ce2eabddSJianfeng Tan 	VIRTIO_USER_ARG_PATH,
351ce2eabddSJianfeng Tan #define VIRTIO_USER_ARG_QUEUE_SIZE     "queue_size"
352ce2eabddSJianfeng Tan 	VIRTIO_USER_ARG_QUEUE_SIZE,
3534214a1b4SWenfeng Liu #define VIRTIO_USER_ARG_INTERFACE_NAME "iface"
3544214a1b4SWenfeng Liu 	VIRTIO_USER_ARG_INTERFACE_NAME,
355bd8f50a4SZhiyong Yang #define VIRTIO_USER_ARG_SERVER_MODE    "server"
356bd8f50a4SZhiyong Yang 	VIRTIO_USER_ARG_SERVER_MODE,
357488ed97aSMarvin Liu #define VIRTIO_USER_ARG_MRG_RXBUF      "mrg_rxbuf"
358488ed97aSMarvin Liu 	VIRTIO_USER_ARG_MRG_RXBUF,
359488ed97aSMarvin Liu #define VIRTIO_USER_ARG_IN_ORDER       "in_order"
360488ed97aSMarvin Liu 	VIRTIO_USER_ARG_IN_ORDER,
36134f3966cSYuanhan Liu #define VIRTIO_USER_ARG_PACKED_VQ      "packed_vq"
36234f3966cSYuanhan Liu 	VIRTIO_USER_ARG_PACKED_VQ,
363b0db4beaSIvan Dyukov #define VIRTIO_USER_ARG_SPEED          "speed"
364b0db4beaSIvan Dyukov 	VIRTIO_USER_ARG_SPEED,
3656b7eefbcSMarvin Liu #define VIRTIO_USER_ARG_VECTORIZED     "vectorized"
3666b7eefbcSMarvin Liu 	VIRTIO_USER_ARG_VECTORIZED,
367ce2eabddSJianfeng Tan 	NULL
368ce2eabddSJianfeng Tan };
369ce2eabddSJianfeng Tan 
370ce2eabddSJianfeng Tan #define VIRTIO_USER_DEF_CQ_EN	0
371ce2eabddSJianfeng Tan #define VIRTIO_USER_DEF_Q_NUM	1
372ce2eabddSJianfeng Tan #define VIRTIO_USER_DEF_Q_SZ	256
373bd8f50a4SZhiyong Yang #define VIRTIO_USER_DEF_SERVER_MODE	0
374ce2eabddSJianfeng Tan 
375ce2eabddSJianfeng Tan static int
376ce2eabddSJianfeng Tan get_string_arg(const char *key __rte_unused,
377ce2eabddSJianfeng Tan 	       const char *value, void *extra_args)
378ce2eabddSJianfeng Tan {
379ce2eabddSJianfeng Tan 	if (!value || !extra_args)
380ce2eabddSJianfeng Tan 		return -EINVAL;
381ce2eabddSJianfeng Tan 
382ce2eabddSJianfeng Tan 	*(char **)extra_args = strdup(value);
383ce2eabddSJianfeng Tan 
3844214a1b4SWenfeng Liu 	if (!*(char **)extra_args)
3854214a1b4SWenfeng Liu 		return -ENOMEM;
3864214a1b4SWenfeng Liu 
387ce2eabddSJianfeng Tan 	return 0;
388ce2eabddSJianfeng Tan }
389ce2eabddSJianfeng Tan 
390ce2eabddSJianfeng Tan static int
391ce2eabddSJianfeng Tan get_integer_arg(const char *key __rte_unused,
392ce2eabddSJianfeng Tan 		const char *value, void *extra_args)
393ce2eabddSJianfeng Tan {
394bc5b6c11SIvan Dyukov 	uint64_t integer = 0;
395ce2eabddSJianfeng Tan 	if (!value || !extra_args)
396ce2eabddSJianfeng Tan 		return -EINVAL;
397bc5b6c11SIvan Dyukov 	errno = 0;
398bc5b6c11SIvan Dyukov 	integer = strtoull(value, NULL, 0);
399bc5b6c11SIvan Dyukov 	/* extra_args keeps default value, it should be replaced
400bc5b6c11SIvan Dyukov 	 * only in case of successful parsing of the 'value' arg
401bc5b6c11SIvan Dyukov 	 */
402bc5b6c11SIvan Dyukov 	if (errno == 0)
403bc5b6c11SIvan Dyukov 		*(uint64_t *)extra_args = integer;
404bc5b6c11SIvan Dyukov 	return -errno;
405ce2eabddSJianfeng Tan }
406ce2eabddSJianfeng Tan 
4077d62bf6fSMaxime Coquelin static uint32_t
4087d62bf6fSMaxime Coquelin vdpa_dynamic_major_num(void)
4097d62bf6fSMaxime Coquelin {
4107d62bf6fSMaxime Coquelin 	FILE *fp;
4117d62bf6fSMaxime Coquelin 	char *line = NULL;
4129f528374SChengwen Feng 	size_t size = 0;
4137d62bf6fSMaxime Coquelin 	char name[11];
4147d62bf6fSMaxime Coquelin 	bool found = false;
4157d62bf6fSMaxime Coquelin 	uint32_t num;
4167d62bf6fSMaxime Coquelin 
4177d62bf6fSMaxime Coquelin 	fp = fopen("/proc/devices", "r");
4187d62bf6fSMaxime Coquelin 	if (fp == NULL) {
4197d62bf6fSMaxime Coquelin 		PMD_INIT_LOG(ERR, "Cannot open /proc/devices: %s",
4207d62bf6fSMaxime Coquelin 			     strerror(errno));
4217d62bf6fSMaxime Coquelin 		return UNNAMED_MAJOR;
4227d62bf6fSMaxime Coquelin 	}
4237d62bf6fSMaxime Coquelin 
4247d62bf6fSMaxime Coquelin 	while (getline(&line, &size, fp) > 0) {
4257d62bf6fSMaxime Coquelin 		char *stripped = line + strspn(line, " ");
4267d62bf6fSMaxime Coquelin 		if ((sscanf(stripped, "%u %10s", &num, name) == 2) &&
4277d62bf6fSMaxime Coquelin 		    (strncmp(name, "vhost-vdpa", 10) == 0)) {
4287d62bf6fSMaxime Coquelin 			found = true;
4297d62bf6fSMaxime Coquelin 			break;
4307d62bf6fSMaxime Coquelin 		}
4317d62bf6fSMaxime Coquelin 	}
4329f528374SChengwen Feng 	free(line);
4337d62bf6fSMaxime Coquelin 	fclose(fp);
4347d62bf6fSMaxime Coquelin 	return found ? num : UNNAMED_MAJOR;
4357d62bf6fSMaxime Coquelin }
4367d62bf6fSMaxime Coquelin 
437f908b22eSAdrian Moreno static enum virtio_user_backend_type
438f908b22eSAdrian Moreno virtio_user_backend_type(const char *path)
439f908b22eSAdrian Moreno {
440f908b22eSAdrian Moreno 	struct stat sb;
441f908b22eSAdrian Moreno 
4427d62bf6fSMaxime Coquelin 	if (stat(path, &sb) == -1) {
443b0395dc8SAdrian Moreno 		if (errno == ENOENT)
444b0395dc8SAdrian Moreno 			return VIRTIO_USER_BACKEND_VHOST_USER;
445b0395dc8SAdrian Moreno 
446f3854ebaSThomas Monjalon 		PMD_INIT_LOG(ERR, "Stat fails: %s (%s)", path,
4477d62bf6fSMaxime Coquelin 			     strerror(errno));
448f908b22eSAdrian Moreno 		return VIRTIO_USER_BACKEND_UNKNOWN;
4497d62bf6fSMaxime Coquelin 	}
450f908b22eSAdrian Moreno 
4517d62bf6fSMaxime Coquelin 	if (S_ISSOCK(sb.st_mode)) {
4527d62bf6fSMaxime Coquelin 		return VIRTIO_USER_BACKEND_VHOST_USER;
4537d62bf6fSMaxime Coquelin 	} else if (S_ISCHR(sb.st_mode)) {
4547d62bf6fSMaxime Coquelin 		if (major(sb.st_rdev) == MISC_MAJOR)
4557d62bf6fSMaxime Coquelin 			return VIRTIO_USER_BACKEND_VHOST_KERNEL;
4567d62bf6fSMaxime Coquelin 		if (major(sb.st_rdev) == vdpa_dynamic_major_num())
4577d62bf6fSMaxime Coquelin 			return VIRTIO_USER_BACKEND_VHOST_VDPA;
4587d62bf6fSMaxime Coquelin 	}
4597d62bf6fSMaxime Coquelin 	return VIRTIO_USER_BACKEND_UNKNOWN;
460f908b22eSAdrian Moreno }
461f908b22eSAdrian Moreno 
462ce2eabddSJianfeng Tan static struct rte_eth_dev *
463050fe6e9SJan Blunck virtio_user_eth_dev_alloc(struct rte_vdev_device *vdev)
464ce2eabddSJianfeng Tan {
465ce2eabddSJianfeng Tan 	struct rte_eth_dev *eth_dev;
466ce2eabddSJianfeng Tan 	struct rte_eth_dev_data *data;
467ce2eabddSJianfeng Tan 	struct virtio_hw *hw;
468ce2eabddSJianfeng Tan 	struct virtio_user_dev *dev;
469ce2eabddSJianfeng Tan 
47065832c82SMaxime Coquelin 	eth_dev = rte_eth_vdev_allocate(vdev, sizeof(*dev));
471ce2eabddSJianfeng Tan 	if (!eth_dev) {
472ce2eabddSJianfeng Tan 		PMD_INIT_LOG(ERR, "cannot alloc rte_eth_dev");
473ce2eabddSJianfeng Tan 		return NULL;
474ce2eabddSJianfeng Tan 	}
475ce2eabddSJianfeng Tan 
476ce2eabddSJianfeng Tan 	data = eth_dev->data;
47765832c82SMaxime Coquelin 	dev = eth_dev->data->dev_private;
47865832c82SMaxime Coquelin 	hw = &dev->hw;
479ce2eabddSJianfeng Tan 
480553f4593SYuanhan Liu 	hw->port_id = data->port_id;
481f8b60756SMaxime Coquelin 	VIRTIO_OPS(hw) = &virtio_user_ops;
4826a504290SMaxime Coquelin 
4836a504290SMaxime Coquelin 	hw->intr_lsc = 1;
4844710e16aSMarvin Liu 	hw->use_vec_rx = 0;
48577d66da8SMarvin Liu 	hw->use_vec_tx = 0;
4869f233f54SMarvin Liu 	hw->use_inorder_rx = 0;
4879f233f54SMarvin Liu 	hw->use_inorder_tx = 0;
48865832c82SMaxime Coquelin 
489ce2eabddSJianfeng Tan 	return eth_dev;
490ce2eabddSJianfeng Tan }
491ce2eabddSJianfeng Tan 
492ca8326a9SJianfeng Tan static void
493ca8326a9SJianfeng Tan virtio_user_eth_dev_free(struct rte_eth_dev *eth_dev)
494ca8326a9SJianfeng Tan {
495ca8326a9SJianfeng Tan 	rte_eth_dev_release_port(eth_dev);
496ca8326a9SJianfeng Tan }
497ca8326a9SJianfeng Tan 
498ce2eabddSJianfeng Tan /* Dev initialization routine. Invoked once for each virtio vdev at
499c3b2fdfeSYong Wang  * EAL init time, see rte_bus_probe().
500ce2eabddSJianfeng Tan  * Returns 0 on success.
501ce2eabddSJianfeng Tan  */
502ce2eabddSJianfeng Tan static int
50365832c82SMaxime Coquelin virtio_user_pmd_probe(struct rte_vdev_device *vdev)
504ce2eabddSJianfeng Tan {
50514f06474SJianfeng Tan 	struct rte_kvargs *kvlist = NULL;
506ce2eabddSJianfeng Tan 	struct rte_eth_dev *eth_dev;
507ce2eabddSJianfeng Tan 	struct virtio_hw *hw;
50865832c82SMaxime Coquelin 	struct virtio_user_dev *dev;
509f908b22eSAdrian Moreno 	enum virtio_user_backend_type backend_type = VIRTIO_USER_BACKEND_UNKNOWN;
510ce2eabddSJianfeng Tan 	uint64_t queues = VIRTIO_USER_DEF_Q_NUM;
511ce2eabddSJianfeng Tan 	uint64_t cq = VIRTIO_USER_DEF_CQ_EN;
512ce2eabddSJianfeng Tan 	uint64_t queue_size = VIRTIO_USER_DEF_Q_SZ;
513bd8f50a4SZhiyong Yang 	uint64_t server_mode = VIRTIO_USER_DEF_SERVER_MODE;
514488ed97aSMarvin Liu 	uint64_t mrg_rxbuf = 1;
515488ed97aSMarvin Liu 	uint64_t in_order = 1;
5169070f88bSTiwei Bie 	uint64_t packed_vq = 0;
5176b7eefbcSMarvin Liu 	uint64_t vectorized = 0;
518ce2eabddSJianfeng Tan 	char *path = NULL;
5194214a1b4SWenfeng Liu 	char *ifname = NULL;
520ce2eabddSJianfeng Tan 	char *mac_addr = NULL;
521ce2eabddSJianfeng Tan 	int ret = -1;
522ce2eabddSJianfeng Tan 
52365832c82SMaxime Coquelin 	RTE_BUILD_BUG_ON(offsetof(struct virtio_user_dev, hw) != 0);
52465832c82SMaxime Coquelin 
5251c8489daSTiwei Bie 	if (rte_eal_process_type() == RTE_PROC_SECONDARY) {
52665832c82SMaxime Coquelin 		const char *name = rte_vdev_device_name(vdev);
5271c8489daSTiwei Bie 		eth_dev = rte_eth_dev_attach_secondary(name);
5281c8489daSTiwei Bie 		if (!eth_dev) {
52988fa5bb6SStephen Hemminger 			PMD_INIT_LOG(ERR, "Failed to probe %s", name);
5301c8489daSTiwei Bie 			return -1;
5311c8489daSTiwei Bie 		}
5321c8489daSTiwei Bie 
533c4fa9cc7SMaxime Coquelin 		dev = eth_dev->data->dev_private;
534c4fa9cc7SMaxime Coquelin 		hw = &dev->hw;
535f8b60756SMaxime Coquelin 		VIRTIO_OPS(hw) = &virtio_user_ops;
536c4fa9cc7SMaxime Coquelin 
5371c8489daSTiwei Bie 		if (eth_virtio_dev_init(eth_dev) < 0) {
5381c8489daSTiwei Bie 			PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails");
5391c8489daSTiwei Bie 			rte_eth_dev_release_port(eth_dev);
5401c8489daSTiwei Bie 			return -1;
5411c8489daSTiwei Bie 		}
5421c8489daSTiwei Bie 
5431c8489daSTiwei Bie 		eth_dev->dev_ops = &virtio_user_secondary_eth_dev_ops;
54465832c82SMaxime Coquelin 		eth_dev->device = &vdev->device;
5451c8489daSTiwei Bie 		rte_eth_dev_probing_finish(eth_dev);
5461c8489daSTiwei Bie 		return 0;
5471c8489daSTiwei Bie 	}
5481c8489daSTiwei Bie 
54965832c82SMaxime Coquelin 	kvlist = rte_kvargs_parse(rte_vdev_device_args(vdev), valid_args);
550ce2eabddSJianfeng Tan 	if (!kvlist) {
551ce2eabddSJianfeng Tan 		PMD_INIT_LOG(ERR, "error when parsing param");
552ce2eabddSJianfeng Tan 		goto end;
553ce2eabddSJianfeng Tan 	}
554ce2eabddSJianfeng Tan 
5559cca159eSMaxime Coquelin 	if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_PATH) == 1) {
556ca8326a9SJianfeng Tan 		if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_PATH,
557ca8326a9SJianfeng Tan 				       &get_string_arg, &path) < 0) {
558404bd6bfSJianfeng Tan 			PMD_INIT_LOG(ERR, "error to parse %s",
559404bd6bfSJianfeng Tan 				     VIRTIO_USER_ARG_PATH);
560404bd6bfSJianfeng Tan 			goto end;
561404bd6bfSJianfeng Tan 		}
5629cca159eSMaxime Coquelin 	} else {
563f2462150SFerruh Yigit 		PMD_INIT_LOG(ERR, "arg %s is mandatory for virtio_user",
5642dac0df4STiwei Bie 			     VIRTIO_USER_ARG_PATH);
565ce2eabddSJianfeng Tan 		goto end;
566ce2eabddSJianfeng Tan 	}
567ce2eabddSJianfeng Tan 
568f908b22eSAdrian Moreno 	backend_type = virtio_user_backend_type(path);
569f908b22eSAdrian Moreno 	if (backend_type == VIRTIO_USER_BACKEND_UNKNOWN) {
570f908b22eSAdrian Moreno 		PMD_INIT_LOG(ERR,
571f908b22eSAdrian Moreno 			     "unable to determine backend type for path %s",
572f908b22eSAdrian Moreno 			path);
573f908b22eSAdrian Moreno 		goto end;
574f908b22eSAdrian Moreno 	}
575b0395dc8SAdrian Moreno 	PMD_INIT_LOG(INFO, "Backend type detected: %s",
576b0395dc8SAdrian Moreno 		     virtio_user_backend_strings[backend_type]);
577f908b22eSAdrian Moreno 
5784214a1b4SWenfeng Liu 	if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_INTERFACE_NAME) == 1) {
579f908b22eSAdrian Moreno 		if (backend_type != VIRTIO_USER_BACKEND_VHOST_KERNEL) {
5804214a1b4SWenfeng Liu 			PMD_INIT_LOG(ERR,
5814214a1b4SWenfeng Liu 				"arg %s applies only to vhost-kernel backend",
5824214a1b4SWenfeng Liu 				VIRTIO_USER_ARG_INTERFACE_NAME);
5834214a1b4SWenfeng Liu 			goto end;
5844214a1b4SWenfeng Liu 		}
5854214a1b4SWenfeng Liu 
5864214a1b4SWenfeng Liu 		if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_INTERFACE_NAME,
5874214a1b4SWenfeng Liu 				       &get_string_arg, &ifname) < 0) {
5884214a1b4SWenfeng Liu 			PMD_INIT_LOG(ERR, "error to parse %s",
5894214a1b4SWenfeng Liu 				     VIRTIO_USER_ARG_INTERFACE_NAME);
5904214a1b4SWenfeng Liu 			goto end;
5914214a1b4SWenfeng Liu 		}
5924214a1b4SWenfeng Liu 	}
5934214a1b4SWenfeng Liu 
594404bd6bfSJianfeng Tan 	if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_MAC) == 1) {
595ca8326a9SJianfeng Tan 		if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MAC,
596ca8326a9SJianfeng Tan 				       &get_string_arg, &mac_addr) < 0) {
597404bd6bfSJianfeng Tan 			PMD_INIT_LOG(ERR, "error to parse %s",
598404bd6bfSJianfeng Tan 				     VIRTIO_USER_ARG_MAC);
599404bd6bfSJianfeng Tan 			goto end;
600404bd6bfSJianfeng Tan 		}
601404bd6bfSJianfeng Tan 	}
602ce2eabddSJianfeng Tan 
603404bd6bfSJianfeng Tan 	if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_QUEUE_SIZE) == 1) {
604ca8326a9SJianfeng Tan 		if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUE_SIZE,
605ca8326a9SJianfeng Tan 				       &get_integer_arg, &queue_size) < 0) {
606404bd6bfSJianfeng Tan 			PMD_INIT_LOG(ERR, "error to parse %s",
607404bd6bfSJianfeng Tan 				     VIRTIO_USER_ARG_QUEUE_SIZE);
608404bd6bfSJianfeng Tan 			goto end;
609404bd6bfSJianfeng Tan 		}
610404bd6bfSJianfeng Tan 	}
611ce2eabddSJianfeng Tan 
612404bd6bfSJianfeng Tan 	if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_QUEUES_NUM) == 1) {
613ca8326a9SJianfeng Tan 		if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_QUEUES_NUM,
614ca8326a9SJianfeng Tan 				       &get_integer_arg, &queues) < 0) {
615404bd6bfSJianfeng Tan 			PMD_INIT_LOG(ERR, "error to parse %s",
616404bd6bfSJianfeng Tan 				     VIRTIO_USER_ARG_QUEUES_NUM);
617404bd6bfSJianfeng Tan 			goto end;
618404bd6bfSJianfeng Tan 		}
619404bd6bfSJianfeng Tan 	}
620ce2eabddSJianfeng Tan 
621bd8f50a4SZhiyong Yang 	if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_SERVER_MODE) == 1) {
622bd8f50a4SZhiyong Yang 		if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_SERVER_MODE,
623bd8f50a4SZhiyong Yang 				       &get_integer_arg, &server_mode) < 0) {
624bd8f50a4SZhiyong Yang 			PMD_INIT_LOG(ERR, "error to parse %s",
625bd8f50a4SZhiyong Yang 				     VIRTIO_USER_ARG_SERVER_MODE);
626bd8f50a4SZhiyong Yang 			goto end;
627bd8f50a4SZhiyong Yang 		}
628bd8f50a4SZhiyong Yang 	}
629bd8f50a4SZhiyong Yang 
630404bd6bfSJianfeng Tan 	if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_CQ_NUM) == 1) {
631ca8326a9SJianfeng Tan 		if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_CQ_NUM,
632ca8326a9SJianfeng Tan 				       &get_integer_arg, &cq) < 0) {
633404bd6bfSJianfeng Tan 			PMD_INIT_LOG(ERR, "error to parse %s",
634404bd6bfSJianfeng Tan 				     VIRTIO_USER_ARG_CQ_NUM);
635404bd6bfSJianfeng Tan 			goto end;
636404bd6bfSJianfeng Tan 		}
637404bd6bfSJianfeng Tan 	}
6381b69528eSJianfeng Tan 
63934f3966cSYuanhan Liu 	if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_PACKED_VQ) == 1) {
64034f3966cSYuanhan Liu 		if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_PACKED_VQ,
64134f3966cSYuanhan Liu 				       &get_integer_arg, &packed_vq) < 0) {
64234f3966cSYuanhan Liu 			PMD_INIT_LOG(ERR, "error to parse %s",
64334f3966cSYuanhan Liu 				     VIRTIO_USER_ARG_PACKED_VQ);
64434f3966cSYuanhan Liu 			goto end;
64534f3966cSYuanhan Liu 		}
64634f3966cSYuanhan Liu 	}
64734f3966cSYuanhan Liu 
6486b7eefbcSMarvin Liu 	if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_VECTORIZED) == 1) {
6496b7eefbcSMarvin Liu 		if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_VECTORIZED,
6506b7eefbcSMarvin Liu 				       &get_integer_arg, &vectorized) < 0) {
6516b7eefbcSMarvin Liu 			PMD_INIT_LOG(ERR, "error to parse %s",
6526b7eefbcSMarvin Liu 				     VIRTIO_USER_ARG_VECTORIZED);
6536b7eefbcSMarvin Liu 			goto end;
6546b7eefbcSMarvin Liu 		}
6556b7eefbcSMarvin Liu 	}
6566b7eefbcSMarvin Liu 
657488ed97aSMarvin Liu 	if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_MRG_RXBUF) == 1) {
658488ed97aSMarvin Liu 		if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_MRG_RXBUF,
659488ed97aSMarvin Liu 				       &get_integer_arg, &mrg_rxbuf) < 0) {
660488ed97aSMarvin Liu 			PMD_INIT_LOG(ERR, "error to parse %s",
661488ed97aSMarvin Liu 				     VIRTIO_USER_ARG_MRG_RXBUF);
662488ed97aSMarvin Liu 			goto end;
663488ed97aSMarvin Liu 		}
664488ed97aSMarvin Liu 	}
665488ed97aSMarvin Liu 
666488ed97aSMarvin Liu 	if (rte_kvargs_count(kvlist, VIRTIO_USER_ARG_IN_ORDER) == 1) {
667488ed97aSMarvin Liu 		if (rte_kvargs_process(kvlist, VIRTIO_USER_ARG_IN_ORDER,
668488ed97aSMarvin Liu 				       &get_integer_arg, &in_order) < 0) {
669488ed97aSMarvin Liu 			PMD_INIT_LOG(ERR, "error to parse %s",
670488ed97aSMarvin Liu 				     VIRTIO_USER_ARG_IN_ORDER);
671488ed97aSMarvin Liu 			goto end;
672488ed97aSMarvin Liu 		}
673488ed97aSMarvin Liu 	}
674488ed97aSMarvin Liu 
67565832c82SMaxime Coquelin 	eth_dev = virtio_user_eth_dev_alloc(vdev);
676ce2eabddSJianfeng Tan 	if (!eth_dev) {
677e8df94b8SJianfeng Tan 		PMD_INIT_LOG(ERR, "virtio_user fails to alloc device");
678ce2eabddSJianfeng Tan 		goto end;
679ce2eabddSJianfeng Tan 	}
680ce2eabddSJianfeng Tan 
68165832c82SMaxime Coquelin 	dev = eth_dev->data->dev_private;
68265832c82SMaxime Coquelin 	hw = &dev->hw;
68352901852SMaxime Coquelin 	if (virtio_user_dev_init(dev, path, (uint16_t)queues, cq,
6841c8489daSTiwei Bie 			 queue_size, mac_addr, &ifname, server_mode,
685f908b22eSAdrian Moreno 			 mrg_rxbuf, in_order, packed_vq, backend_type) < 0) {
686ca8326a9SJianfeng Tan 		PMD_INIT_LOG(ERR, "virtio_user_dev_init fails");
687ca8326a9SJianfeng Tan 		virtio_user_eth_dev_free(eth_dev);
688ce2eabddSJianfeng Tan 		goto end;
689ca8326a9SJianfeng Tan 	}
690fbe90cddSThomas Monjalon 
691ba55c94aSMaxime Coquelin 	/*
692ba55c94aSMaxime Coquelin 	 * Virtio-user requires using virtual addresses for the descriptors
693ba55c94aSMaxime Coquelin 	 * buffers, whatever other devices require
694ba55c94aSMaxime Coquelin 	 */
695666ef294SSrujana Challa 	if (backend_type == VIRTIO_USER_BACKEND_VHOST_VDPA)
696666ef294SSrujana Challa 		/* vDPA backend requires using IOVA for the buffers
697666ef294SSrujana Challa 		 * to make it work in IOVA as PA mode also.
698666ef294SSrujana Challa 		 */
699666ef294SSrujana Challa 		hw->use_va = false;
700666ef294SSrujana Challa 	else
701ba55c94aSMaxime Coquelin 		hw->use_va = true;
702ba55c94aSMaxime Coquelin 
70387db93e0SDavid Marchand 	/* previously called by pci probing for physical dev */
704ce2eabddSJianfeng Tan 	if (eth_virtio_dev_init(eth_dev) < 0) {
705ce2eabddSJianfeng Tan 		PMD_INIT_LOG(ERR, "eth_virtio_dev_init fails");
706edca47a1SHarold Huang 		virtio_user_dev_uninit(dev);
707ca8326a9SJianfeng Tan 		virtio_user_eth_dev_free(eth_dev);
708ce2eabddSJianfeng Tan 		goto end;
709ce2eabddSJianfeng Tan 	}
710fbe90cddSThomas Monjalon 
71177d66da8SMarvin Liu 	if (vectorized) {
71277d66da8SMarvin Liu 		if (packed_vq) {
7139ef38ddbSJoyce Kong #if defined(CC_AVX512_SUPPORT) || defined(RTE_ARCH_ARM)
7146b7eefbcSMarvin Liu 			hw->use_vec_rx = 1;
71577d66da8SMarvin Liu 			hw->use_vec_tx = 1;
71677d66da8SMarvin Liu #else
71777d66da8SMarvin Liu 			PMD_INIT_LOG(INFO,
71877d66da8SMarvin Liu 				"building environment do not support packed ring vectorized");
71977d66da8SMarvin Liu #endif
72077d66da8SMarvin Liu 		} else {
72177d66da8SMarvin Liu 			hw->use_vec_rx = 1;
72277d66da8SMarvin Liu 		}
72377d66da8SMarvin Liu 	}
7246b7eefbcSMarvin Liu 
725fbe90cddSThomas Monjalon 	rte_eth_dev_probing_finish(eth_dev);
726ce2eabddSJianfeng Tan 	ret = 0;
727ce2eabddSJianfeng Tan 
728ce2eabddSJianfeng Tan end:
72914f06474SJianfeng Tan 	rte_kvargs_free(kvlist);
730ce2eabddSJianfeng Tan 	free(path);
731ce2eabddSJianfeng Tan 	free(mac_addr);
7324214a1b4SWenfeng Liu 	free(ifname);
733ce2eabddSJianfeng Tan 	return ret;
734ce2eabddSJianfeng Tan }
735ce2eabddSJianfeng Tan 
736ce2eabddSJianfeng Tan static int
7375d2aa461SJan Blunck virtio_user_pmd_remove(struct rte_vdev_device *vdev)
738ce2eabddSJianfeng Tan {
7395d2aa461SJan Blunck 	const char *name;
740ce2eabddSJianfeng Tan 	struct rte_eth_dev *eth_dev;
741ce2eabddSJianfeng Tan 
7425d2aa461SJan Blunck 	if (!vdev)
743ce2eabddSJianfeng Tan 		return -EINVAL;
744ce2eabddSJianfeng Tan 
7455d2aa461SJan Blunck 	name = rte_vdev_device_name(vdev);
746f2462150SFerruh Yigit 	PMD_DRV_LOG(INFO, "Un-Initializing %s", name);
747ce2eabddSJianfeng Tan 	eth_dev = rte_eth_dev_allocated(name);
7487f468b2eSTiwei Bie 	/* Port has already been released by close. */
749ce2eabddSJianfeng Tan 	if (!eth_dev)
7507f468b2eSTiwei Bie 		return 0;
751ce2eabddSJianfeng Tan 
7521c8489daSTiwei Bie 	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
7531c8489daSTiwei Bie 		return rte_eth_dev_release_port(eth_dev);
7541c8489daSTiwei Bie 
755ce2eabddSJianfeng Tan 	/* make sure the device is stopped, queues freed */
7568a5a0aadSThomas Monjalon 	return rte_eth_dev_close(eth_dev->data->port_id);
757ce2eabddSJianfeng Tan }
758ce2eabddSJianfeng Tan 
75986e71eb2SMaxime Coquelin static int virtio_user_pmd_dma_map(struct rte_vdev_device *vdev, void *addr,
76086e71eb2SMaxime Coquelin 		uint64_t iova, size_t len)
76186e71eb2SMaxime Coquelin {
76286e71eb2SMaxime Coquelin 	const char *name;
76386e71eb2SMaxime Coquelin 	struct rte_eth_dev *eth_dev;
76486e71eb2SMaxime Coquelin 	struct virtio_user_dev *dev;
76586e71eb2SMaxime Coquelin 
76686e71eb2SMaxime Coquelin 	if (!vdev)
76786e71eb2SMaxime Coquelin 		return -EINVAL;
76886e71eb2SMaxime Coquelin 
76986e71eb2SMaxime Coquelin 	name = rte_vdev_device_name(vdev);
77086e71eb2SMaxime Coquelin 	eth_dev = rte_eth_dev_allocated(name);
77186e71eb2SMaxime Coquelin 	/* Port has already been released by close. */
77286e71eb2SMaxime Coquelin 	if (!eth_dev)
77386e71eb2SMaxime Coquelin 		return 0;
77486e71eb2SMaxime Coquelin 
77565832c82SMaxime Coquelin 	dev = eth_dev->data->dev_private;
77686e71eb2SMaxime Coquelin 
77786e71eb2SMaxime Coquelin 	if (dev->ops->dma_map)
77886e71eb2SMaxime Coquelin 		return dev->ops->dma_map(dev, addr, iova, len);
77986e71eb2SMaxime Coquelin 
78086e71eb2SMaxime Coquelin 	return 0;
78186e71eb2SMaxime Coquelin }
78286e71eb2SMaxime Coquelin 
78386e71eb2SMaxime Coquelin static int virtio_user_pmd_dma_unmap(struct rte_vdev_device *vdev, void *addr,
78486e71eb2SMaxime Coquelin 		uint64_t iova, size_t len)
78586e71eb2SMaxime Coquelin {
78686e71eb2SMaxime Coquelin 	const char *name;
78786e71eb2SMaxime Coquelin 	struct rte_eth_dev *eth_dev;
78886e71eb2SMaxime Coquelin 	struct virtio_user_dev *dev;
78986e71eb2SMaxime Coquelin 
79086e71eb2SMaxime Coquelin 	if (!vdev)
79186e71eb2SMaxime Coquelin 		return -EINVAL;
79286e71eb2SMaxime Coquelin 
79386e71eb2SMaxime Coquelin 	name = rte_vdev_device_name(vdev);
79486e71eb2SMaxime Coquelin 	eth_dev = rte_eth_dev_allocated(name);
79586e71eb2SMaxime Coquelin 	/* Port has already been released by close. */
79686e71eb2SMaxime Coquelin 	if (!eth_dev)
79786e71eb2SMaxime Coquelin 		return 0;
79886e71eb2SMaxime Coquelin 
79965832c82SMaxime Coquelin 	dev = eth_dev->data->dev_private;
80086e71eb2SMaxime Coquelin 
80186e71eb2SMaxime Coquelin 	if (dev->ops->dma_unmap)
80286e71eb2SMaxime Coquelin 		return dev->ops->dma_unmap(dev, addr, iova, len);
80386e71eb2SMaxime Coquelin 
80486e71eb2SMaxime Coquelin 	return 0;
80586e71eb2SMaxime Coquelin }
80686e71eb2SMaxime Coquelin 
807fe363dd4SJan Viktorin static struct rte_vdev_driver virtio_user_driver = {
80850a3345fSShreyansh Jain 	.probe = virtio_user_pmd_probe,
80950a3345fSShreyansh Jain 	.remove = virtio_user_pmd_remove,
81086e71eb2SMaxime Coquelin 	.dma_map = virtio_user_pmd_dma_map,
81186e71eb2SMaxime Coquelin 	.dma_unmap = virtio_user_pmd_dma_unmap,
812ce2eabddSJianfeng Tan };
813ce2eabddSJianfeng Tan 
81401f19227SShreyansh Jain RTE_PMD_REGISTER_VDEV(net_virtio_user, virtio_user_driver);
8159fa80cb2SJan Blunck RTE_PMD_REGISTER_ALIAS(net_virtio_user, virtio_user);
81601f19227SShreyansh Jain RTE_PMD_REGISTER_PARAM_STRING(net_virtio_user,
81744e32a67SPablo de Lara 	"path=<path> "
81844e32a67SPablo de Lara 	"mac=<mac addr> "
81944e32a67SPablo de Lara 	"cq=<int> "
82044e32a67SPablo de Lara 	"queue_size=<int> "
8214214a1b4SWenfeng Liu 	"queues=<int> "
822488ed97aSMarvin Liu 	"iface=<string> "
82362758c76STiwei Bie 	"server=<0|1> "
824488ed97aSMarvin Liu 	"mrg_rxbuf=<0|1> "
8259070f88bSTiwei Bie 	"in_order=<0|1> "
826b0db4beaSIvan Dyukov 	"packed_vq=<0|1> "
8276b7eefbcSMarvin Liu 	"speed=<int> "
8286b7eefbcSMarvin Liu 	"vectorized=<0|1>");
829