xref: /dpdk/drivers/net/virtio/virtio_user/virtio_user_dev.c (revision 76f80881ef1e14cce2c11020ca913ea5d7186cba)
15566a3e3SBruce Richardson /* SPDX-License-Identifier: BSD-3-Clause
25566a3e3SBruce Richardson  * Copyright(c) 2010-2016 Intel Corporation
337a7eb2aSJianfeng Tan  */
437a7eb2aSJianfeng Tan 
537a7eb2aSJianfeng Tan #include <stdint.h>
637a7eb2aSJianfeng Tan #include <stdio.h>
737a7eb2aSJianfeng Tan #include <fcntl.h>
837a7eb2aSJianfeng Tan #include <string.h>
937a7eb2aSJianfeng Tan #include <errno.h>
1037a7eb2aSJianfeng Tan #include <sys/mman.h>
1137a7eb2aSJianfeng Tan #include <unistd.h>
1237a7eb2aSJianfeng Tan #include <sys/eventfd.h>
1333d24d65SJianfeng Tan #include <sys/types.h>
1433d24d65SJianfeng Tan #include <sys/stat.h>
1537a7eb2aSJianfeng Tan 
166723c0fcSBruce Richardson #include <rte_string_fns.h>
177ff26957STiwei Bie #include <rte_eal_memconfig.h>
187ff26957STiwei Bie 
1937a7eb2aSJianfeng Tan #include "vhost.h"
2037a7eb2aSJianfeng Tan #include "virtio_user_dev.h"
2137a7eb2aSJianfeng Tan #include "../virtio_ethdev.h"
2237a7eb2aSJianfeng Tan 
2312ecb2f6SMaxime Coquelin #define VIRTIO_USER_MEM_EVENT_CLB_NAME "virtio_user_mem_event_clb"
2412ecb2f6SMaxime Coquelin 
2537a7eb2aSJianfeng Tan static int
2657ae79a7SJianfeng Tan virtio_user_create_queue(struct virtio_user_dev *dev, uint32_t queue_sel)
2757ae79a7SJianfeng Tan {
2857ae79a7SJianfeng Tan 	/* Of all per virtqueue MSGs, make sure VHOST_SET_VRING_CALL come
2957ae79a7SJianfeng Tan 	 * firstly because vhost depends on this msg to allocate virtqueue
3057ae79a7SJianfeng Tan 	 * pair.
3157ae79a7SJianfeng Tan 	 */
3257ae79a7SJianfeng Tan 	struct vhost_vring_file file;
3357ae79a7SJianfeng Tan 
3457ae79a7SJianfeng Tan 	file.index = queue_sel;
35e6e7ad8bSJianfeng Tan 	file.fd = dev->callfds[queue_sel];
3633d24d65SJianfeng Tan 	dev->ops->send_request(dev, VHOST_USER_SET_VRING_CALL, &file);
3757ae79a7SJianfeng Tan 
3857ae79a7SJianfeng Tan 	return 0;
3957ae79a7SJianfeng Tan }
4057ae79a7SJianfeng Tan 
4157ae79a7SJianfeng Tan static int
4237a7eb2aSJianfeng Tan virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel)
4337a7eb2aSJianfeng Tan {
4437a7eb2aSJianfeng Tan 	struct vhost_vring_file file;
4537a7eb2aSJianfeng Tan 	struct vhost_vring_state state;
4637a7eb2aSJianfeng Tan 	struct vring *vring = &dev->vrings[queue_sel];
4748a44640SJens Freimann 	struct vring_packed *pq_vring = &dev->packed_vrings[queue_sel];
4837a7eb2aSJianfeng Tan 	struct vhost_vring_addr addr = {
4937a7eb2aSJianfeng Tan 		.index = queue_sel,
5037a7eb2aSJianfeng Tan 		.log_guest_addr = 0,
5137a7eb2aSJianfeng Tan 		.flags = 0, /* disable log */
5237a7eb2aSJianfeng Tan 	};
5337a7eb2aSJianfeng Tan 
5448a44640SJens Freimann 	if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) {
5548a44640SJens Freimann 		addr.desc_user_addr =
564cdc4d98STiwei Bie 			(uint64_t)(uintptr_t)pq_vring->desc;
5748a44640SJens Freimann 		addr.avail_user_addr =
584cdc4d98STiwei Bie 			(uint64_t)(uintptr_t)pq_vring->driver;
5948a44640SJens Freimann 		addr.used_user_addr =
604cdc4d98STiwei Bie 			(uint64_t)(uintptr_t)pq_vring->device;
6148a44640SJens Freimann 	} else {
6248a44640SJens Freimann 		addr.desc_user_addr = (uint64_t)(uintptr_t)vring->desc;
6348a44640SJens Freimann 		addr.avail_user_addr = (uint64_t)(uintptr_t)vring->avail;
6448a44640SJens Freimann 		addr.used_user_addr = (uint64_t)(uintptr_t)vring->used;
6548a44640SJens Freimann 	}
6648a44640SJens Freimann 
6737a7eb2aSJianfeng Tan 	state.index = queue_sel;
6837a7eb2aSJianfeng Tan 	state.num = vring->num;
6933d24d65SJianfeng Tan 	dev->ops->send_request(dev, VHOST_USER_SET_VRING_NUM, &state);
7037a7eb2aSJianfeng Tan 
71be7a4707SJianfeng Tan 	state.index = queue_sel;
7237a7eb2aSJianfeng Tan 	state.num = 0; /* no reservation */
7334f3966cSYuanhan Liu 	if (dev->features & (1ULL << VIRTIO_F_RING_PACKED))
7434f3966cSYuanhan Liu 		state.num |= (1 << 15);
7533d24d65SJianfeng Tan 	dev->ops->send_request(dev, VHOST_USER_SET_VRING_BASE, &state);
7637a7eb2aSJianfeng Tan 
7733d24d65SJianfeng Tan 	dev->ops->send_request(dev, VHOST_USER_SET_VRING_ADDR, &addr);
7837a7eb2aSJianfeng Tan 
7937a7eb2aSJianfeng Tan 	/* Of all per virtqueue MSGs, make sure VHOST_USER_SET_VRING_KICK comes
8037a7eb2aSJianfeng Tan 	 * lastly because vhost depends on this msg to judge if
8137a7eb2aSJianfeng Tan 	 * virtio is ready.
8237a7eb2aSJianfeng Tan 	 */
8357ae79a7SJianfeng Tan 	file.index = queue_sel;
84e6e7ad8bSJianfeng Tan 	file.fd = dev->kickfds[queue_sel];
8533d24d65SJianfeng Tan 	dev->ops->send_request(dev, VHOST_USER_SET_VRING_KICK, &file);
8637a7eb2aSJianfeng Tan 
8737a7eb2aSJianfeng Tan 	return 0;
8837a7eb2aSJianfeng Tan }
8937a7eb2aSJianfeng Tan 
9057ae79a7SJianfeng Tan static int
9157ae79a7SJianfeng Tan virtio_user_queue_setup(struct virtio_user_dev *dev,
9257ae79a7SJianfeng Tan 			int (*fn)(struct virtio_user_dev *, uint32_t))
9357ae79a7SJianfeng Tan {
9457ae79a7SJianfeng Tan 	uint32_t i, queue_sel;
9557ae79a7SJianfeng Tan 
9657ae79a7SJianfeng Tan 	for (i = 0; i < dev->max_queue_pairs; ++i) {
9757ae79a7SJianfeng Tan 		queue_sel = 2 * i + VTNET_SQ_RQ_QUEUE_IDX;
9857ae79a7SJianfeng Tan 		if (fn(dev, queue_sel) < 0) {
9957ae79a7SJianfeng Tan 			PMD_DRV_LOG(INFO, "setup rx vq fails: %u", i);
10057ae79a7SJianfeng Tan 			return -1;
10157ae79a7SJianfeng Tan 		}
10257ae79a7SJianfeng Tan 	}
10357ae79a7SJianfeng Tan 	for (i = 0; i < dev->max_queue_pairs; ++i) {
10457ae79a7SJianfeng Tan 		queue_sel = 2 * i + VTNET_SQ_TQ_QUEUE_IDX;
10557ae79a7SJianfeng Tan 		if (fn(dev, queue_sel) < 0) {
10657ae79a7SJianfeng Tan 			PMD_DRV_LOG(INFO, "setup tx vq fails: %u", i);
10757ae79a7SJianfeng Tan 			return -1;
10857ae79a7SJianfeng Tan 		}
10957ae79a7SJianfeng Tan 	}
11057ae79a7SJianfeng Tan 
11157ae79a7SJianfeng Tan 	return 0;
11257ae79a7SJianfeng Tan }
11357ae79a7SJianfeng Tan 
11437a7eb2aSJianfeng Tan int
115bd8f50a4SZhiyong Yang is_vhost_user_by_type(const char *path)
116bd8f50a4SZhiyong Yang {
117bd8f50a4SZhiyong Yang 	struct stat sb;
118bd8f50a4SZhiyong Yang 
119bd8f50a4SZhiyong Yang 	if (stat(path, &sb) == -1)
120bd8f50a4SZhiyong Yang 		return 0;
121bd8f50a4SZhiyong Yang 
122bd8f50a4SZhiyong Yang 	return S_ISSOCK(sb.st_mode);
123bd8f50a4SZhiyong Yang }
124bd8f50a4SZhiyong Yang 
125bd8f50a4SZhiyong Yang int
12637a7eb2aSJianfeng Tan virtio_user_start_device(struct virtio_user_dev *dev)
12737a7eb2aSJianfeng Tan {
12837a7eb2aSJianfeng Tan 	uint64_t features;
12937a7eb2aSJianfeng Tan 	int ret;
13037a7eb2aSJianfeng Tan 
1317ff26957STiwei Bie 	/*
1327ff26957STiwei Bie 	 * XXX workaround!
1337ff26957STiwei Bie 	 *
1347ff26957STiwei Bie 	 * We need to make sure that the locks will be
1357ff26957STiwei Bie 	 * taken in the correct order to avoid deadlocks.
1367ff26957STiwei Bie 	 *
1377ff26957STiwei Bie 	 * Before releasing this lock, this thread should
1387ff26957STiwei Bie 	 * not trigger any memory hotplug events.
1397ff26957STiwei Bie 	 *
1407ff26957STiwei Bie 	 * This is a temporary workaround, and should be
1417ff26957STiwei Bie 	 * replaced when we get proper supports from the
1427ff26957STiwei Bie 	 * memory subsystem in the future.
1437ff26957STiwei Bie 	 */
144*76f80881SAnatoly Burakov 	rte_mcfg_mem_read_lock();
14512ecb2f6SMaxime Coquelin 	pthread_mutex_lock(&dev->mutex);
14612ecb2f6SMaxime Coquelin 
147bd8f50a4SZhiyong Yang 	if (is_vhost_user_by_type(dev->path) && dev->vhostfd < 0)
14812ecb2f6SMaxime Coquelin 		goto error;
149bd8f50a4SZhiyong Yang 
15057ae79a7SJianfeng Tan 	/* Step 0: tell vhost to create queues */
15157ae79a7SJianfeng Tan 	if (virtio_user_queue_setup(dev, virtio_user_create_queue) < 0)
15237a7eb2aSJianfeng Tan 		goto error;
15337a7eb2aSJianfeng Tan 
15433d24d65SJianfeng Tan 	/* Step 1: set features */
15537a7eb2aSJianfeng Tan 	features = dev->features;
15633d24d65SJianfeng Tan 	/* Strip VIRTIO_NET_F_MAC, as MAC address is handled in vdev init */
15737a7eb2aSJianfeng Tan 	features &= ~(1ull << VIRTIO_NET_F_MAC);
15833d24d65SJianfeng Tan 	/* Strip VIRTIO_NET_F_CTRL_VQ, as devices do not really need to know */
159142678d4SJianfeng Tan 	features &= ~(1ull << VIRTIO_NET_F_CTRL_VQ);
16035c4f855SJianfeng Tan 	features &= ~(1ull << VIRTIO_NET_F_STATUS);
16133d24d65SJianfeng Tan 	ret = dev->ops->send_request(dev, VHOST_USER_SET_FEATURES, &features);
16237a7eb2aSJianfeng Tan 	if (ret < 0)
16337a7eb2aSJianfeng Tan 		goto error;
16437a7eb2aSJianfeng Tan 	PMD_DRV_LOG(INFO, "set features: %" PRIx64, features);
16537a7eb2aSJianfeng Tan 
16657ae79a7SJianfeng Tan 	/* Step 2: share memory regions */
16733d24d65SJianfeng Tan 	ret = dev->ops->send_request(dev, VHOST_USER_SET_MEM_TABLE, NULL);
16857ae79a7SJianfeng Tan 	if (ret < 0)
16957ae79a7SJianfeng Tan 		goto error;
17057ae79a7SJianfeng Tan 
17157ae79a7SJianfeng Tan 	/* Step 3: kick queues */
17257ae79a7SJianfeng Tan 	if (virtio_user_queue_setup(dev, virtio_user_kick_queue) < 0)
17357ae79a7SJianfeng Tan 		goto error;
17457ae79a7SJianfeng Tan 
17557ae79a7SJianfeng Tan 	/* Step 4: enable queues
17657ae79a7SJianfeng Tan 	 * we enable the 1st queue pair by default.
17757ae79a7SJianfeng Tan 	 */
17833d24d65SJianfeng Tan 	dev->ops->enable_qp(dev, 0, 1);
17957ae79a7SJianfeng Tan 
18012ecb2f6SMaxime Coquelin 	dev->started = true;
18112ecb2f6SMaxime Coquelin 	pthread_mutex_unlock(&dev->mutex);
182*76f80881SAnatoly Burakov 	rte_mcfg_mem_read_unlock();
18312ecb2f6SMaxime Coquelin 
18437a7eb2aSJianfeng Tan 	return 0;
18537a7eb2aSJianfeng Tan error:
18612ecb2f6SMaxime Coquelin 	pthread_mutex_unlock(&dev->mutex);
187*76f80881SAnatoly Burakov 	rte_mcfg_mem_read_unlock();
18837a7eb2aSJianfeng Tan 	/* TODO: free resource here or caller to check */
18937a7eb2aSJianfeng Tan 	return -1;
19037a7eb2aSJianfeng Tan }
19137a7eb2aSJianfeng Tan 
19237a7eb2aSJianfeng Tan int virtio_user_stop_device(struct virtio_user_dev *dev)
19337a7eb2aSJianfeng Tan {
19474dc6746STiwei Bie 	struct vhost_vring_state state;
195c12a26eeSJianfeng Tan 	uint32_t i;
19674dc6746STiwei Bie 	int error = 0;
197c12a26eeSJianfeng Tan 
19812ecb2f6SMaxime Coquelin 	pthread_mutex_lock(&dev->mutex);
199f457e900STiwei Bie 	if (!dev->started)
200f457e900STiwei Bie 		goto out;
201f457e900STiwei Bie 
202c12a26eeSJianfeng Tan 	for (i = 0; i < dev->max_queue_pairs; ++i)
20333d24d65SJianfeng Tan 		dev->ops->enable_qp(dev, i, 0);
204c12a26eeSJianfeng Tan 
20574dc6746STiwei Bie 	/* Stop the backend. */
20674dc6746STiwei Bie 	for (i = 0; i < dev->max_queue_pairs * 2; ++i) {
20774dc6746STiwei Bie 		state.index = i;
20874dc6746STiwei Bie 		if (dev->ops->send_request(dev, VHOST_USER_GET_VRING_BASE,
20974dc6746STiwei Bie 					   &state) < 0) {
21074dc6746STiwei Bie 			PMD_DRV_LOG(ERR, "get_vring_base failed, index=%u\n",
21174dc6746STiwei Bie 				    i);
21274dc6746STiwei Bie 			error = -1;
21374dc6746STiwei Bie 			goto out;
2140d6a8752SJianfeng Tan 		}
21574dc6746STiwei Bie 	}
21674dc6746STiwei Bie 
21712ecb2f6SMaxime Coquelin 	dev->started = false;
218f457e900STiwei Bie out:
21912ecb2f6SMaxime Coquelin 	pthread_mutex_unlock(&dev->mutex);
2200d6a8752SJianfeng Tan 
22174dc6746STiwei Bie 	return error;
22237a7eb2aSJianfeng Tan }
22337a7eb2aSJianfeng Tan 
22437a7eb2aSJianfeng Tan static inline void
22537a7eb2aSJianfeng Tan parse_mac(struct virtio_user_dev *dev, const char *mac)
22637a7eb2aSJianfeng Tan {
22737a7eb2aSJianfeng Tan 	int i, r;
22835b2d13fSOlivier Matz 	uint32_t tmp[RTE_ETHER_ADDR_LEN];
22937a7eb2aSJianfeng Tan 
23037a7eb2aSJianfeng Tan 	if (!mac)
23137a7eb2aSJianfeng Tan 		return;
23237a7eb2aSJianfeng Tan 
23337a7eb2aSJianfeng Tan 	r = sscanf(mac, "%x:%x:%x:%x:%x:%x", &tmp[0],
23437a7eb2aSJianfeng Tan 			&tmp[1], &tmp[2], &tmp[3], &tmp[4], &tmp[5]);
23535b2d13fSOlivier Matz 	if (r == RTE_ETHER_ADDR_LEN) {
23635b2d13fSOlivier Matz 		for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i)
23737a7eb2aSJianfeng Tan 			dev->mac_addr[i] = (uint8_t)tmp[i];
23837a7eb2aSJianfeng Tan 		dev->mac_specified = 1;
23937a7eb2aSJianfeng Tan 	} else {
24037a7eb2aSJianfeng Tan 		/* ignore the wrong mac, use random mac */
24137a7eb2aSJianfeng Tan 		PMD_DRV_LOG(ERR, "wrong format of mac: %s", mac);
24237a7eb2aSJianfeng Tan 	}
24337a7eb2aSJianfeng Tan }
24437a7eb2aSJianfeng Tan 
24533d24d65SJianfeng Tan static int
246e6e7ad8bSJianfeng Tan virtio_user_dev_init_notify(struct virtio_user_dev *dev)
24733d24d65SJianfeng Tan {
248e6e7ad8bSJianfeng Tan 	uint32_t i, j;
249e6e7ad8bSJianfeng Tan 	int callfd;
250e6e7ad8bSJianfeng Tan 	int kickfd;
25133d24d65SJianfeng Tan 
2522269b9aeSWenfeng Liu 	for (i = 0; i < VIRTIO_MAX_VIRTQUEUES; ++i) {
253e6e7ad8bSJianfeng Tan 		if (i >= dev->max_queue_pairs * 2) {
25433d24d65SJianfeng Tan 			dev->kickfds[i] = -1;
25533d24d65SJianfeng Tan 			dev->callfds[i] = -1;
256e6e7ad8bSJianfeng Tan 			continue;
25733d24d65SJianfeng Tan 		}
25833d24d65SJianfeng Tan 
259e6e7ad8bSJianfeng Tan 		/* May use invalid flag, but some backend uses kickfd and
260e6e7ad8bSJianfeng Tan 		 * callfd as criteria to judge if dev is alive. so finally we
261e6e7ad8bSJianfeng Tan 		 * use real event_fd.
262e6e7ad8bSJianfeng Tan 		 */
263e6e7ad8bSJianfeng Tan 		callfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
264e6e7ad8bSJianfeng Tan 		if (callfd < 0) {
265e6e7ad8bSJianfeng Tan 			PMD_DRV_LOG(ERR, "callfd error, %s", strerror(errno));
266e6e7ad8bSJianfeng Tan 			break;
267e6e7ad8bSJianfeng Tan 		}
268e6e7ad8bSJianfeng Tan 		kickfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK);
269e6e7ad8bSJianfeng Tan 		if (kickfd < 0) {
270e6e7ad8bSJianfeng Tan 			PMD_DRV_LOG(ERR, "kickfd error, %s", strerror(errno));
271e6e7ad8bSJianfeng Tan 			break;
272e6e7ad8bSJianfeng Tan 		}
273e6e7ad8bSJianfeng Tan 		dev->callfds[i] = callfd;
274e6e7ad8bSJianfeng Tan 		dev->kickfds[i] = kickfd;
275e6e7ad8bSJianfeng Tan 	}
276e6e7ad8bSJianfeng Tan 
277e6e7ad8bSJianfeng Tan 	if (i < VIRTIO_MAX_VIRTQUEUES) {
278e6e7ad8bSJianfeng Tan 		for (j = 0; j <= i; ++j) {
279e6e7ad8bSJianfeng Tan 			close(dev->callfds[j]);
280e6e7ad8bSJianfeng Tan 			close(dev->kickfds[j]);
281e6e7ad8bSJianfeng Tan 		}
282e6e7ad8bSJianfeng Tan 
283e6e7ad8bSJianfeng Tan 		return -1;
284e6e7ad8bSJianfeng Tan 	}
285e6e7ad8bSJianfeng Tan 
286e6e7ad8bSJianfeng Tan 	return 0;
287e6e7ad8bSJianfeng Tan }
288e6e7ad8bSJianfeng Tan 
289e6e7ad8bSJianfeng Tan static int
2903d4fb6fdSJianfeng Tan virtio_user_fill_intr_handle(struct virtio_user_dev *dev)
2913d4fb6fdSJianfeng Tan {
2923d4fb6fdSJianfeng Tan 	uint32_t i;
2933d4fb6fdSJianfeng Tan 	struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id];
2943d4fb6fdSJianfeng Tan 
2953d4fb6fdSJianfeng Tan 	if (!eth_dev->intr_handle) {
2963d4fb6fdSJianfeng Tan 		eth_dev->intr_handle = malloc(sizeof(*eth_dev->intr_handle));
2973d4fb6fdSJianfeng Tan 		if (!eth_dev->intr_handle) {
2983d4fb6fdSJianfeng Tan 			PMD_DRV_LOG(ERR, "fail to allocate intr_handle");
2993d4fb6fdSJianfeng Tan 			return -1;
3003d4fb6fdSJianfeng Tan 		}
3013d4fb6fdSJianfeng Tan 		memset(eth_dev->intr_handle, 0, sizeof(*eth_dev->intr_handle));
3023d4fb6fdSJianfeng Tan 	}
3033d4fb6fdSJianfeng Tan 
3043d4fb6fdSJianfeng Tan 	for (i = 0; i < dev->max_queue_pairs; ++i)
3053d4fb6fdSJianfeng Tan 		eth_dev->intr_handle->efds[i] = dev->callfds[i];
3063d4fb6fdSJianfeng Tan 	eth_dev->intr_handle->nb_efd = dev->max_queue_pairs;
3073d4fb6fdSJianfeng Tan 	eth_dev->intr_handle->max_intr = dev->max_queue_pairs + 1;
3083d4fb6fdSJianfeng Tan 	eth_dev->intr_handle->type = RTE_INTR_HANDLE_VDEV;
30929906b97SJingjing Wu 	/* For virtio vdev, no need to read counter for clean */
31029906b97SJingjing Wu 	eth_dev->intr_handle->efd_counter_size = 0;
311eae6e70cSJianfeng Tan 	eth_dev->intr_handle->fd = -1;
312cc4690e9SJianfeng Tan 	if (dev->vhostfd >= 0)
313cc4690e9SJianfeng Tan 		eth_dev->intr_handle->fd = dev->vhostfd;
314bd8f50a4SZhiyong Yang 	else if (dev->is_server)
315bd8f50a4SZhiyong Yang 		eth_dev->intr_handle->fd = dev->listenfd;
3163d4fb6fdSJianfeng Tan 
3173d4fb6fdSJianfeng Tan 	return 0;
3183d4fb6fdSJianfeng Tan }
3193d4fb6fdSJianfeng Tan 
32012ecb2f6SMaxime Coquelin static void
32112ecb2f6SMaxime Coquelin virtio_user_mem_event_cb(enum rte_mem_event type __rte_unused,
32212ecb2f6SMaxime Coquelin 						 const void *addr __rte_unused,
32312ecb2f6SMaxime Coquelin 						 size_t len __rte_unused,
32412ecb2f6SMaxime Coquelin 						 void *arg)
32512ecb2f6SMaxime Coquelin {
32612ecb2f6SMaxime Coquelin 	struct virtio_user_dev *dev = arg;
327f32c7c9dSAnatoly Burakov 	struct rte_memseg_list *msl;
32812ecb2f6SMaxime Coquelin 	uint16_t i;
32912ecb2f6SMaxime Coquelin 
330f32c7c9dSAnatoly Burakov 	/* ignore externally allocated memory */
331f32c7c9dSAnatoly Burakov 	msl = rte_mem_virt2memseg_list(addr);
332f32c7c9dSAnatoly Burakov 	if (msl->external)
333f32c7c9dSAnatoly Burakov 		return;
334f32c7c9dSAnatoly Burakov 
33512ecb2f6SMaxime Coquelin 	pthread_mutex_lock(&dev->mutex);
33612ecb2f6SMaxime Coquelin 
33712ecb2f6SMaxime Coquelin 	if (dev->started == false)
33812ecb2f6SMaxime Coquelin 		goto exit;
33912ecb2f6SMaxime Coquelin 
34012ecb2f6SMaxime Coquelin 	/* Step 1: pause the active queues */
34112ecb2f6SMaxime Coquelin 	for (i = 0; i < dev->queue_pairs; i++)
34212ecb2f6SMaxime Coquelin 		dev->ops->enable_qp(dev, i, 0);
34312ecb2f6SMaxime Coquelin 
34412ecb2f6SMaxime Coquelin 	/* Step 2: update memory regions */
34512ecb2f6SMaxime Coquelin 	dev->ops->send_request(dev, VHOST_USER_SET_MEM_TABLE, NULL);
34612ecb2f6SMaxime Coquelin 
34712ecb2f6SMaxime Coquelin 	/* Step 3: resume the active queues */
34812ecb2f6SMaxime Coquelin 	for (i = 0; i < dev->queue_pairs; i++)
34912ecb2f6SMaxime Coquelin 		dev->ops->enable_qp(dev, i, 1);
35012ecb2f6SMaxime Coquelin 
35112ecb2f6SMaxime Coquelin exit:
35212ecb2f6SMaxime Coquelin 	pthread_mutex_unlock(&dev->mutex);
35312ecb2f6SMaxime Coquelin }
35412ecb2f6SMaxime Coquelin 
3553d4fb6fdSJianfeng Tan static int
356e6e7ad8bSJianfeng Tan virtio_user_dev_setup(struct virtio_user_dev *dev)
357e6e7ad8bSJianfeng Tan {
358e6e7ad8bSJianfeng Tan 	uint32_t q;
359e6e7ad8bSJianfeng Tan 
360e6e7ad8bSJianfeng Tan 	dev->vhostfd = -1;
361e3b43481SJianfeng Tan 	dev->vhostfds = NULL;
362e3b43481SJianfeng Tan 	dev->tapfds = NULL;
363e3b43481SJianfeng Tan 
364bd8f50a4SZhiyong Yang 	if (dev->is_server) {
365bd8f50a4SZhiyong Yang 		if (access(dev->path, F_OK) == 0 &&
366bd8f50a4SZhiyong Yang 		    !is_vhost_user_by_type(dev->path)) {
367bd8f50a4SZhiyong Yang 			PMD_DRV_LOG(ERR, "Server mode doesn't support vhost-kernel!");
368bd8f50a4SZhiyong Yang 			return -1;
369bd8f50a4SZhiyong Yang 		}
370520dd992SFerruh Yigit 		dev->ops = &virtio_ops_user;
371bd8f50a4SZhiyong Yang 	} else {
37233d24d65SJianfeng Tan 		if (is_vhost_user_by_type(dev->path)) {
373520dd992SFerruh Yigit 			dev->ops = &virtio_ops_user;
374e3b43481SJianfeng Tan 		} else {
375520dd992SFerruh Yigit 			dev->ops = &virtio_ops_kernel;
376e3b43481SJianfeng Tan 
377bd8f50a4SZhiyong Yang 			dev->vhostfds = malloc(dev->max_queue_pairs *
378bd8f50a4SZhiyong Yang 					       sizeof(int));
379bd8f50a4SZhiyong Yang 			dev->tapfds = malloc(dev->max_queue_pairs *
380bd8f50a4SZhiyong Yang 					     sizeof(int));
381e3b43481SJianfeng Tan 			if (!dev->vhostfds || !dev->tapfds) {
382e3b43481SJianfeng Tan 				PMD_INIT_LOG(ERR, "Failed to malloc");
383e3b43481SJianfeng Tan 				return -1;
38433d24d65SJianfeng Tan 			}
38533d24d65SJianfeng Tan 
386e3b43481SJianfeng Tan 			for (q = 0; q < dev->max_queue_pairs; ++q) {
387e3b43481SJianfeng Tan 				dev->vhostfds[q] = -1;
388e3b43481SJianfeng Tan 				dev->tapfds[q] = -1;
389e3b43481SJianfeng Tan 			}
390e3b43481SJianfeng Tan 		}
391bd8f50a4SZhiyong Yang 	}
392e3b43481SJianfeng Tan 
393cc4690e9SJianfeng Tan 	if (dev->ops->setup(dev) < 0)
394cc4690e9SJianfeng Tan 		return -1;
395cc4690e9SJianfeng Tan 
396cc4690e9SJianfeng Tan 	if (virtio_user_dev_init_notify(dev) < 0)
397cc4690e9SJianfeng Tan 		return -1;
398cc4690e9SJianfeng Tan 
399cc4690e9SJianfeng Tan 	if (virtio_user_fill_intr_handle(dev) < 0)
400cc4690e9SJianfeng Tan 		return -1;
401cc4690e9SJianfeng Tan 
402cc4690e9SJianfeng Tan 	return 0;
40333d24d65SJianfeng Tan }
40433d24d65SJianfeng Tan 
405bed3b24cSJianfeng Tan /* Use below macro to filter features from vhost backend */
406bed3b24cSJianfeng Tan #define VIRTIO_USER_SUPPORTED_FEATURES			\
407bed3b24cSJianfeng Tan 	(1ULL << VIRTIO_NET_F_MAC		|	\
408bed3b24cSJianfeng Tan 	 1ULL << VIRTIO_NET_F_STATUS		|	\
409bed3b24cSJianfeng Tan 	 1ULL << VIRTIO_NET_F_MQ		|	\
410bed3b24cSJianfeng Tan 	 1ULL << VIRTIO_NET_F_CTRL_MAC_ADDR	|	\
411bed3b24cSJianfeng Tan 	 1ULL << VIRTIO_NET_F_CTRL_VQ		|	\
412bed3b24cSJianfeng Tan 	 1ULL << VIRTIO_NET_F_CTRL_RX		|	\
413bed3b24cSJianfeng Tan 	 1ULL << VIRTIO_NET_F_CTRL_VLAN		|	\
414bed3b24cSJianfeng Tan 	 1ULL << VIRTIO_NET_F_CSUM		|	\
415bed3b24cSJianfeng Tan 	 1ULL << VIRTIO_NET_F_HOST_TSO4		|	\
416bed3b24cSJianfeng Tan 	 1ULL << VIRTIO_NET_F_HOST_TSO6		|	\
417bed3b24cSJianfeng Tan 	 1ULL << VIRTIO_NET_F_MRG_RXBUF		|	\
418bed3b24cSJianfeng Tan 	 1ULL << VIRTIO_RING_F_INDIRECT_DESC	|	\
419bed3b24cSJianfeng Tan 	 1ULL << VIRTIO_NET_F_GUEST_CSUM	|	\
420bed3b24cSJianfeng Tan 	 1ULL << VIRTIO_NET_F_GUEST_TSO4	|	\
421bed3b24cSJianfeng Tan 	 1ULL << VIRTIO_NET_F_GUEST_TSO6	|	\
42241e45c90SMarvin Liu 	 1ULL << VIRTIO_F_IN_ORDER		|	\
42334f3966cSYuanhan Liu 	 1ULL << VIRTIO_F_VERSION_1		|	\
4248532a0fcSTiwei Bie 	 1ULL << VIRTIO_F_RING_PACKED)
425bed3b24cSJianfeng Tan 
42637a7eb2aSJianfeng Tan int
42737a7eb2aSJianfeng Tan virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues,
428488ed97aSMarvin Liu 		     int cq, int queue_size, const char *mac, char **ifname,
4291c8489daSTiwei Bie 		     int server, int mrg_rxbuf, int in_order, int packed_vq)
43037a7eb2aSJianfeng Tan {
43112ecb2f6SMaxime Coquelin 	pthread_mutex_init(&dev->mutex, NULL);
4326723c0fcSBruce Richardson 	strlcpy(dev->path, path, PATH_MAX);
43312ecb2f6SMaxime Coquelin 	dev->started = 0;
43437a7eb2aSJianfeng Tan 	dev->max_queue_pairs = queues;
43537a7eb2aSJianfeng Tan 	dev->queue_pairs = 1; /* mq disabled by default */
43637a7eb2aSJianfeng Tan 	dev->queue_size = queue_size;
4371c8489daSTiwei Bie 	dev->is_server = server;
43837a7eb2aSJianfeng Tan 	dev->mac_specified = 0;
439bb97d2ddSTiwei Bie 	dev->frontend_features = 0;
440bd9568f3STiwei Bie 	dev->unsupported_features = ~VIRTIO_USER_SUPPORTED_FEATURES;
44137a7eb2aSJianfeng Tan 	parse_mac(dev, mac);
44237a7eb2aSJianfeng Tan 
4434214a1b4SWenfeng Liu 	if (*ifname) {
4444214a1b4SWenfeng Liu 		dev->ifname = *ifname;
4454214a1b4SWenfeng Liu 		*ifname = NULL;
4464214a1b4SWenfeng Liu 	}
4474214a1b4SWenfeng Liu 
44833d24d65SJianfeng Tan 	if (virtio_user_dev_setup(dev) < 0) {
44937a7eb2aSJianfeng Tan 		PMD_INIT_LOG(ERR, "backend set up fails");
45037a7eb2aSJianfeng Tan 		return -1;
45137a7eb2aSJianfeng Tan 	}
452bce7e905SJianfeng Tan 
4536e1e5904SJiayu Hu 	if (!dev->is_server) {
454bd8f50a4SZhiyong Yang 		if (dev->ops->send_request(dev, VHOST_USER_SET_OWNER,
455bd8f50a4SZhiyong Yang 					   NULL) < 0) {
456bd8f50a4SZhiyong Yang 			PMD_INIT_LOG(ERR, "set_owner fails: %s",
457bd8f50a4SZhiyong Yang 				     strerror(errno));
45837a7eb2aSJianfeng Tan 			return -1;
45937a7eb2aSJianfeng Tan 		}
46037a7eb2aSJianfeng Tan 
46133d24d65SJianfeng Tan 		if (dev->ops->send_request(dev, VHOST_USER_GET_FEATURES,
462142678d4SJianfeng Tan 					   &dev->device_features) < 0) {
463bd8f50a4SZhiyong Yang 			PMD_INIT_LOG(ERR, "get_features failed: %s",
464bd8f50a4SZhiyong Yang 				     strerror(errno));
46537a7eb2aSJianfeng Tan 			return -1;
46637a7eb2aSJianfeng Tan 		}
467bd8f50a4SZhiyong Yang 	} else {
468bd8f50a4SZhiyong Yang 		/* We just pretend vhost-user can support all these features.
469bd8f50a4SZhiyong Yang 		 * Note that this could be problematic that if some feature is
470bd8f50a4SZhiyong Yang 		 * negotiated but not supported by the vhost-user which comes
471bd8f50a4SZhiyong Yang 		 * later.
472bd8f50a4SZhiyong Yang 		 */
473bd8f50a4SZhiyong Yang 		dev->device_features = VIRTIO_USER_SUPPORTED_FEATURES;
474bd8f50a4SZhiyong Yang 	}
475bd8f50a4SZhiyong Yang 
476bd9568f3STiwei Bie 	if (!mrg_rxbuf)
477488ed97aSMarvin Liu 		dev->unsupported_features |= (1ull << VIRTIO_NET_F_MRG_RXBUF);
478488ed97aSMarvin Liu 
479bd9568f3STiwei Bie 	if (!in_order)
480488ed97aSMarvin Liu 		dev->unsupported_features |= (1ull << VIRTIO_F_IN_ORDER);
481488ed97aSMarvin Liu 
48248a44640SJens Freimann 	if (!packed_vq)
4839070f88bSTiwei Bie 		dev->unsupported_features |= (1ull << VIRTIO_F_RING_PACKED);
48434f3966cSYuanhan Liu 
4859070f88bSTiwei Bie 	if (dev->mac_specified)
4869070f88bSTiwei Bie 		dev->frontend_features |= (1ull << VIRTIO_NET_F_MAC);
4879070f88bSTiwei Bie 	else
4887c66ff61SMarvin Liu 		dev->unsupported_features |= (1ull << VIRTIO_NET_F_MAC);
489f9b9d1a5SJianfeng Tan 
490142678d4SJianfeng Tan 	if (cq) {
491142678d4SJianfeng Tan 		/* device does not really need to know anything about CQ,
492142678d4SJianfeng Tan 		 * so if necessary, we just claim to support CQ
493f9b9d1a5SJianfeng Tan 		 */
494bb97d2ddSTiwei Bie 		dev->frontend_features |= (1ull << VIRTIO_NET_F_CTRL_VQ);
495142678d4SJianfeng Tan 	} else {
4967c66ff61SMarvin Liu 		dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VQ);
497bd9568f3STiwei Bie 		/* Also disable features that depend on VIRTIO_NET_F_CTRL_VQ */
4987c66ff61SMarvin Liu 		dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_RX);
4997c66ff61SMarvin Liu 		dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VLAN);
5007c66ff61SMarvin Liu 		dev->unsupported_features |=
5017c66ff61SMarvin Liu 			(1ull << VIRTIO_NET_F_GUEST_ANNOUNCE);
5027c66ff61SMarvin Liu 		dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ);
5037c66ff61SMarvin Liu 		dev->unsupported_features |=
5047c66ff61SMarvin Liu 			(1ull << VIRTIO_NET_F_CTRL_MAC_ADDR);
505f9b9d1a5SJianfeng Tan 	}
506f9b9d1a5SJianfeng Tan 
50735c4f855SJianfeng Tan 	/* The backend will not report this feature, we add it explicitly */
5084bf9e26dSJianfeng Tan 	if (is_vhost_user_by_type(dev->path))
509bb97d2ddSTiwei Bie 		dev->frontend_features |= (1ull << VIRTIO_NET_F_STATUS);
51035c4f855SJianfeng Tan 
511bd9568f3STiwei Bie 	/*
512bd9568f3STiwei Bie 	 * Device features =
513bd9568f3STiwei Bie 	 *     (frontend_features | backend_features) & ~unsupported_features;
514bd9568f3STiwei Bie 	 */
515bb97d2ddSTiwei Bie 	dev->device_features |= dev->frontend_features;
516bd9568f3STiwei Bie 	dev->device_features &= ~dev->unsupported_features;
517bed3b24cSJianfeng Tan 
51812ecb2f6SMaxime Coquelin 	if (rte_mem_event_callback_register(VIRTIO_USER_MEM_EVENT_CLB_NAME,
51912ecb2f6SMaxime Coquelin 				virtio_user_mem_event_cb, dev)) {
52088e5469fSXiao Wang 		if (rte_errno != ENOTSUP) {
52188e5469fSXiao Wang 			PMD_INIT_LOG(ERR, "Failed to register mem event"
52288e5469fSXiao Wang 					" callback\n");
52312ecb2f6SMaxime Coquelin 			return -1;
52412ecb2f6SMaxime Coquelin 		}
52588e5469fSXiao Wang 	}
52612ecb2f6SMaxime Coquelin 
52737a7eb2aSJianfeng Tan 	return 0;
52837a7eb2aSJianfeng Tan }
52937a7eb2aSJianfeng Tan 
53037a7eb2aSJianfeng Tan void
53137a7eb2aSJianfeng Tan virtio_user_dev_uninit(struct virtio_user_dev *dev)
53237a7eb2aSJianfeng Tan {
533e3b43481SJianfeng Tan 	uint32_t i;
534e3b43481SJianfeng Tan 
535e3b43481SJianfeng Tan 	virtio_user_stop_device(dev);
536e3b43481SJianfeng Tan 
53712ecb2f6SMaxime Coquelin 	rte_mem_event_callback_unregister(VIRTIO_USER_MEM_EVENT_CLB_NAME, dev);
53812ecb2f6SMaxime Coquelin 
539e6e7ad8bSJianfeng Tan 	for (i = 0; i < dev->max_queue_pairs * 2; ++i) {
540e6e7ad8bSJianfeng Tan 		close(dev->callfds[i]);
541e6e7ad8bSJianfeng Tan 		close(dev->kickfds[i]);
542e6e7ad8bSJianfeng Tan 	}
543e6e7ad8bSJianfeng Tan 
54437a7eb2aSJianfeng Tan 	close(dev->vhostfd);
545e3b43481SJianfeng Tan 
546bd8f50a4SZhiyong Yang 	if (dev->is_server && dev->listenfd >= 0) {
547bd8f50a4SZhiyong Yang 		close(dev->listenfd);
548bd8f50a4SZhiyong Yang 		dev->listenfd = -1;
549bd8f50a4SZhiyong Yang 	}
550bd8f50a4SZhiyong Yang 
551e3b43481SJianfeng Tan 	if (dev->vhostfds) {
552e3b43481SJianfeng Tan 		for (i = 0; i < dev->max_queue_pairs; ++i)
553e3b43481SJianfeng Tan 			close(dev->vhostfds[i]);
554e3b43481SJianfeng Tan 		free(dev->vhostfds);
555e3b43481SJianfeng Tan 		free(dev->tapfds);
556e3b43481SJianfeng Tan 	}
5574214a1b4SWenfeng Liu 
5584214a1b4SWenfeng Liu 	free(dev->ifname);
559bd8f50a4SZhiyong Yang 
560bd8f50a4SZhiyong Yang 	if (dev->is_server)
561bd8f50a4SZhiyong Yang 		unlink(dev->path);
56237a7eb2aSJianfeng Tan }
563f9b9d1a5SJianfeng Tan 
564201a4165SZhiyong Yang uint8_t
565f9b9d1a5SJianfeng Tan virtio_user_handle_mq(struct virtio_user_dev *dev, uint16_t q_pairs)
566f9b9d1a5SJianfeng Tan {
567f9b9d1a5SJianfeng Tan 	uint16_t i;
568f9b9d1a5SJianfeng Tan 	uint8_t ret = 0;
569f9b9d1a5SJianfeng Tan 
570f9b9d1a5SJianfeng Tan 	if (q_pairs > dev->max_queue_pairs) {
571f9b9d1a5SJianfeng Tan 		PMD_INIT_LOG(ERR, "multi-q config %u, but only %u supported",
572f9b9d1a5SJianfeng Tan 			     q_pairs, dev->max_queue_pairs);
573f9b9d1a5SJianfeng Tan 		return -1;
574f9b9d1a5SJianfeng Tan 	}
575f9b9d1a5SJianfeng Tan 
576201a4165SZhiyong Yang 	/* Server mode can't enable queue pairs if vhostfd is invalid,
577201a4165SZhiyong Yang 	 * always return 0 in this case.
578201a4165SZhiyong Yang 	 */
57921b90f79STiwei Bie 	if (!dev->is_server || dev->vhostfd >= 0) {
580f9b9d1a5SJianfeng Tan 		for (i = 0; i < q_pairs; ++i)
58133d24d65SJianfeng Tan 			ret |= dev->ops->enable_qp(dev, i, 1);
582f9b9d1a5SJianfeng Tan 		for (i = q_pairs; i < dev->max_queue_pairs; ++i)
58333d24d65SJianfeng Tan 			ret |= dev->ops->enable_qp(dev, i, 0);
584201a4165SZhiyong Yang 	}
585f9b9d1a5SJianfeng Tan 	dev->queue_pairs = q_pairs;
586f9b9d1a5SJianfeng Tan 
587f9b9d1a5SJianfeng Tan 	return ret;
588f9b9d1a5SJianfeng Tan }
589f9b9d1a5SJianfeng Tan 
590f9b9d1a5SJianfeng Tan static uint32_t
591f9b9d1a5SJianfeng Tan virtio_user_handle_ctrl_msg(struct virtio_user_dev *dev, struct vring *vring,
592f9b9d1a5SJianfeng Tan 			    uint16_t idx_hdr)
593f9b9d1a5SJianfeng Tan {
594f9b9d1a5SJianfeng Tan 	struct virtio_net_ctrl_hdr *hdr;
595f9b9d1a5SJianfeng Tan 	virtio_net_ctrl_ack status = ~0;
596f9b9d1a5SJianfeng Tan 	uint16_t i, idx_data, idx_status;
597f9b9d1a5SJianfeng Tan 	uint32_t n_descs = 0;
598f9b9d1a5SJianfeng Tan 
599f9b9d1a5SJianfeng Tan 	/* locate desc for header, data, and status */
600f9b9d1a5SJianfeng Tan 	idx_data = vring->desc[idx_hdr].next;
601f9b9d1a5SJianfeng Tan 	n_descs++;
602f9b9d1a5SJianfeng Tan 
603f9b9d1a5SJianfeng Tan 	i = idx_data;
604f9b9d1a5SJianfeng Tan 	while (vring->desc[i].flags == VRING_DESC_F_NEXT) {
605f9b9d1a5SJianfeng Tan 		i = vring->desc[i].next;
606f9b9d1a5SJianfeng Tan 		n_descs++;
607f9b9d1a5SJianfeng Tan 	}
608f9b9d1a5SJianfeng Tan 
609f9b9d1a5SJianfeng Tan 	/* locate desc for status */
610f9b9d1a5SJianfeng Tan 	idx_status = i;
611f9b9d1a5SJianfeng Tan 	n_descs++;
612f9b9d1a5SJianfeng Tan 
613f9b9d1a5SJianfeng Tan 	hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr;
614f9b9d1a5SJianfeng Tan 	if (hdr->class == VIRTIO_NET_CTRL_MQ &&
615f9b9d1a5SJianfeng Tan 	    hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) {
616f9b9d1a5SJianfeng Tan 		uint16_t queues;
617f9b9d1a5SJianfeng Tan 
618f9b9d1a5SJianfeng Tan 		queues = *(uint16_t *)(uintptr_t)vring->desc[idx_data].addr;
619f9b9d1a5SJianfeng Tan 		status = virtio_user_handle_mq(dev, queues);
620f9b9d1a5SJianfeng Tan 	}
621f9b9d1a5SJianfeng Tan 
622f9b9d1a5SJianfeng Tan 	/* Update status */
623f9b9d1a5SJianfeng Tan 	*(virtio_net_ctrl_ack *)(uintptr_t)vring->desc[idx_status].addr = status;
624f9b9d1a5SJianfeng Tan 
625f9b9d1a5SJianfeng Tan 	return n_descs;
626f9b9d1a5SJianfeng Tan }
627f9b9d1a5SJianfeng Tan 
62848a44640SJens Freimann static inline int
62948a44640SJens Freimann desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter)
63048a44640SJens Freimann {
63112e9e70cSTiwei Bie 	uint16_t flags = desc->flags;
63212e9e70cSTiwei Bie 
63312e9e70cSTiwei Bie 	return wrap_counter == !!(flags & VRING_PACKED_DESC_F_AVAIL) &&
63412e9e70cSTiwei Bie 		wrap_counter != !!(flags & VRING_PACKED_DESC_F_USED);
63548a44640SJens Freimann }
63648a44640SJens Freimann 
63748a44640SJens Freimann static uint32_t
63845c224e7STiwei Bie virtio_user_handle_ctrl_msg_packed(struct virtio_user_dev *dev,
63948a44640SJens Freimann 				   struct vring_packed *vring,
64048a44640SJens Freimann 				   uint16_t idx_hdr)
64148a44640SJens Freimann {
64248a44640SJens Freimann 	struct virtio_net_ctrl_hdr *hdr;
64348a44640SJens Freimann 	virtio_net_ctrl_ack status = ~0;
64448a44640SJens Freimann 	uint16_t idx_data, idx_status;
64548a44640SJens Freimann 	/* initialize to one, header is first */
64648a44640SJens Freimann 	uint32_t n_descs = 1;
64748a44640SJens Freimann 
64848a44640SJens Freimann 	/* locate desc for header, data, and status */
64948a44640SJens Freimann 	idx_data = idx_hdr + 1;
65048a44640SJens Freimann 	if (idx_data >= dev->queue_size)
65148a44640SJens Freimann 		idx_data -= dev->queue_size;
65248a44640SJens Freimann 
65348a44640SJens Freimann 	n_descs++;
65448a44640SJens Freimann 
65548a44640SJens Freimann 	idx_status = idx_data;
6564cdc4d98STiwei Bie 	while (vring->desc[idx_status].flags & VRING_DESC_F_NEXT) {
65748a44640SJens Freimann 		idx_status++;
65848a44640SJens Freimann 		if (idx_status >= dev->queue_size)
65948a44640SJens Freimann 			idx_status -= dev->queue_size;
66048a44640SJens Freimann 		n_descs++;
66148a44640SJens Freimann 	}
66248a44640SJens Freimann 
6634cdc4d98STiwei Bie 	hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr;
66448a44640SJens Freimann 	if (hdr->class == VIRTIO_NET_CTRL_MQ &&
66548a44640SJens Freimann 	    hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) {
66648a44640SJens Freimann 		uint16_t queues;
66748a44640SJens Freimann 
66848a44640SJens Freimann 		queues = *(uint16_t *)(uintptr_t)
6694cdc4d98STiwei Bie 				vring->desc[idx_data].addr;
67048a44640SJens Freimann 		status = virtio_user_handle_mq(dev, queues);
67148a44640SJens Freimann 	}
67248a44640SJens Freimann 
67348a44640SJens Freimann 	/* Update status */
67448a44640SJens Freimann 	*(virtio_net_ctrl_ack *)(uintptr_t)
6754cdc4d98STiwei Bie 		vring->desc[idx_status].addr = status;
67648a44640SJens Freimann 
67745c224e7STiwei Bie 	/* Update used descriptor */
6784cdc4d98STiwei Bie 	vring->desc[idx_hdr].id = vring->desc[idx_status].id;
6794cdc4d98STiwei Bie 	vring->desc[idx_hdr].len = sizeof(status);
68045c224e7STiwei Bie 
68148a44640SJens Freimann 	return n_descs;
68248a44640SJens Freimann }
68348a44640SJens Freimann 
68448a44640SJens Freimann void
68548a44640SJens Freimann virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx)
68648a44640SJens Freimann {
68748a44640SJens Freimann 	struct virtio_user_queue *vq = &dev->packed_queues[queue_idx];
68848a44640SJens Freimann 	struct vring_packed *vring = &dev->packed_vrings[queue_idx];
68912e9e70cSTiwei Bie 	uint16_t n_descs, flags;
69048a44640SJens Freimann 
6914cdc4d98STiwei Bie 	while (desc_is_avail(&vring->desc[vq->used_idx],
69248a44640SJens Freimann 			     vq->used_wrap_counter)) {
69348a44640SJens Freimann 
69445c224e7STiwei Bie 		n_descs = virtio_user_handle_ctrl_msg_packed(dev, vring,
69545c224e7STiwei Bie 				vq->used_idx);
69648a44640SJens Freimann 
69712e9e70cSTiwei Bie 		flags = VRING_DESC_F_WRITE;
69812e9e70cSTiwei Bie 		if (vq->used_wrap_counter)
69912e9e70cSTiwei Bie 			flags |= VRING_PACKED_DESC_F_AVAIL_USED;
70012e9e70cSTiwei Bie 
70145c224e7STiwei Bie 		rte_smp_wmb();
70212e9e70cSTiwei Bie 		vring->desc[vq->used_idx].flags = flags;
70345c224e7STiwei Bie 
70445c224e7STiwei Bie 		vq->used_idx += n_descs;
70545c224e7STiwei Bie 		if (vq->used_idx >= dev->queue_size) {
70648a44640SJens Freimann 			vq->used_idx -= dev->queue_size;
70748a44640SJens Freimann 			vq->used_wrap_counter ^= 1;
70848a44640SJens Freimann 		}
70948a44640SJens Freimann 	}
71048a44640SJens Freimann }
71148a44640SJens Freimann 
712f9b9d1a5SJianfeng Tan void
713f9b9d1a5SJianfeng Tan virtio_user_handle_cq(struct virtio_user_dev *dev, uint16_t queue_idx)
714f9b9d1a5SJianfeng Tan {
715f9b9d1a5SJianfeng Tan 	uint16_t avail_idx, desc_idx;
716f9b9d1a5SJianfeng Tan 	struct vring_used_elem *uep;
717f9b9d1a5SJianfeng Tan 	uint32_t n_descs;
718f9b9d1a5SJianfeng Tan 	struct vring *vring = &dev->vrings[queue_idx];
719f9b9d1a5SJianfeng Tan 
720f9b9d1a5SJianfeng Tan 	/* Consume avail ring, using used ring idx as first one */
721f9b9d1a5SJianfeng Tan 	while (vring->used->idx != vring->avail->idx) {
722f9b9d1a5SJianfeng Tan 		avail_idx = (vring->used->idx) & (vring->num - 1);
723f9b9d1a5SJianfeng Tan 		desc_idx = vring->avail->ring[avail_idx];
724f9b9d1a5SJianfeng Tan 
725f9b9d1a5SJianfeng Tan 		n_descs = virtio_user_handle_ctrl_msg(dev, vring, desc_idx);
726f9b9d1a5SJianfeng Tan 
727f9b9d1a5SJianfeng Tan 		/* Update used ring */
728f9b9d1a5SJianfeng Tan 		uep = &vring->used->ring[avail_idx];
7290403e37aSTiwei Bie 		uep->id = desc_idx;
730f9b9d1a5SJianfeng Tan 		uep->len = n_descs;
731f9b9d1a5SJianfeng Tan 
732f9b9d1a5SJianfeng Tan 		vring->used->idx++;
733f9b9d1a5SJianfeng Tan 	}
734f9b9d1a5SJianfeng Tan }
735