15566a3e3SBruce Richardson /* SPDX-License-Identifier: BSD-3-Clause 25566a3e3SBruce Richardson * Copyright(c) 2010-2016 Intel Corporation 337a7eb2aSJianfeng Tan */ 437a7eb2aSJianfeng Tan 537a7eb2aSJianfeng Tan #include <stdint.h> 637a7eb2aSJianfeng Tan #include <stdio.h> 737a7eb2aSJianfeng Tan #include <fcntl.h> 837a7eb2aSJianfeng Tan #include <string.h> 937a7eb2aSJianfeng Tan #include <errno.h> 1037a7eb2aSJianfeng Tan #include <sys/mman.h> 1137a7eb2aSJianfeng Tan #include <unistd.h> 1237a7eb2aSJianfeng Tan #include <sys/eventfd.h> 1333d24d65SJianfeng Tan #include <sys/types.h> 1433d24d65SJianfeng Tan #include <sys/stat.h> 1537a7eb2aSJianfeng Tan 166723c0fcSBruce Richardson #include <rte_string_fns.h> 177ff26957STiwei Bie #include <rte_eal_memconfig.h> 187ff26957STiwei Bie 1937a7eb2aSJianfeng Tan #include "vhost.h" 2037a7eb2aSJianfeng Tan #include "virtio_user_dev.h" 2137a7eb2aSJianfeng Tan #include "../virtio_ethdev.h" 2237a7eb2aSJianfeng Tan 2312ecb2f6SMaxime Coquelin #define VIRTIO_USER_MEM_EVENT_CLB_NAME "virtio_user_mem_event_clb" 2412ecb2f6SMaxime Coquelin 2537a7eb2aSJianfeng Tan static int 2657ae79a7SJianfeng Tan virtio_user_create_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 2757ae79a7SJianfeng Tan { 2857ae79a7SJianfeng Tan /* Of all per virtqueue MSGs, make sure VHOST_SET_VRING_CALL come 2957ae79a7SJianfeng Tan * firstly because vhost depends on this msg to allocate virtqueue 3057ae79a7SJianfeng Tan * pair. 3157ae79a7SJianfeng Tan */ 3257ae79a7SJianfeng Tan struct vhost_vring_file file; 3357ae79a7SJianfeng Tan 3457ae79a7SJianfeng Tan file.index = queue_sel; 35e6e7ad8bSJianfeng Tan file.fd = dev->callfds[queue_sel]; 3633d24d65SJianfeng Tan dev->ops->send_request(dev, VHOST_USER_SET_VRING_CALL, &file); 3757ae79a7SJianfeng Tan 3857ae79a7SJianfeng Tan return 0; 3957ae79a7SJianfeng Tan } 4057ae79a7SJianfeng Tan 4157ae79a7SJianfeng Tan static int 4237a7eb2aSJianfeng Tan virtio_user_kick_queue(struct virtio_user_dev *dev, uint32_t queue_sel) 4337a7eb2aSJianfeng Tan { 4437a7eb2aSJianfeng Tan struct vhost_vring_file file; 4537a7eb2aSJianfeng Tan struct vhost_vring_state state; 4637a7eb2aSJianfeng Tan struct vring *vring = &dev->vrings[queue_sel]; 4748a44640SJens Freimann struct vring_packed *pq_vring = &dev->packed_vrings[queue_sel]; 4837a7eb2aSJianfeng Tan struct vhost_vring_addr addr = { 4937a7eb2aSJianfeng Tan .index = queue_sel, 5037a7eb2aSJianfeng Tan .log_guest_addr = 0, 5137a7eb2aSJianfeng Tan .flags = 0, /* disable log */ 5237a7eb2aSJianfeng Tan }; 5337a7eb2aSJianfeng Tan 5448a44640SJens Freimann if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) { 5548a44640SJens Freimann addr.desc_user_addr = 564cdc4d98STiwei Bie (uint64_t)(uintptr_t)pq_vring->desc; 5748a44640SJens Freimann addr.avail_user_addr = 584cdc4d98STiwei Bie (uint64_t)(uintptr_t)pq_vring->driver; 5948a44640SJens Freimann addr.used_user_addr = 604cdc4d98STiwei Bie (uint64_t)(uintptr_t)pq_vring->device; 6148a44640SJens Freimann } else { 6248a44640SJens Freimann addr.desc_user_addr = (uint64_t)(uintptr_t)vring->desc; 6348a44640SJens Freimann addr.avail_user_addr = (uint64_t)(uintptr_t)vring->avail; 6448a44640SJens Freimann addr.used_user_addr = (uint64_t)(uintptr_t)vring->used; 6548a44640SJens Freimann } 6648a44640SJens Freimann 6737a7eb2aSJianfeng Tan state.index = queue_sel; 6837a7eb2aSJianfeng Tan state.num = vring->num; 6933d24d65SJianfeng Tan dev->ops->send_request(dev, VHOST_USER_SET_VRING_NUM, &state); 7037a7eb2aSJianfeng Tan 71be7a4707SJianfeng Tan state.index = queue_sel; 7237a7eb2aSJianfeng Tan state.num = 0; /* no reservation */ 7334f3966cSYuanhan Liu if (dev->features & (1ULL << VIRTIO_F_RING_PACKED)) 7434f3966cSYuanhan Liu state.num |= (1 << 15); 7533d24d65SJianfeng Tan dev->ops->send_request(dev, VHOST_USER_SET_VRING_BASE, &state); 7637a7eb2aSJianfeng Tan 7733d24d65SJianfeng Tan dev->ops->send_request(dev, VHOST_USER_SET_VRING_ADDR, &addr); 7837a7eb2aSJianfeng Tan 7937a7eb2aSJianfeng Tan /* Of all per virtqueue MSGs, make sure VHOST_USER_SET_VRING_KICK comes 8037a7eb2aSJianfeng Tan * lastly because vhost depends on this msg to judge if 8137a7eb2aSJianfeng Tan * virtio is ready. 8237a7eb2aSJianfeng Tan */ 8357ae79a7SJianfeng Tan file.index = queue_sel; 84e6e7ad8bSJianfeng Tan file.fd = dev->kickfds[queue_sel]; 8533d24d65SJianfeng Tan dev->ops->send_request(dev, VHOST_USER_SET_VRING_KICK, &file); 8637a7eb2aSJianfeng Tan 8737a7eb2aSJianfeng Tan return 0; 8837a7eb2aSJianfeng Tan } 8937a7eb2aSJianfeng Tan 9057ae79a7SJianfeng Tan static int 9157ae79a7SJianfeng Tan virtio_user_queue_setup(struct virtio_user_dev *dev, 9257ae79a7SJianfeng Tan int (*fn)(struct virtio_user_dev *, uint32_t)) 9357ae79a7SJianfeng Tan { 9457ae79a7SJianfeng Tan uint32_t i, queue_sel; 9557ae79a7SJianfeng Tan 9657ae79a7SJianfeng Tan for (i = 0; i < dev->max_queue_pairs; ++i) { 9757ae79a7SJianfeng Tan queue_sel = 2 * i + VTNET_SQ_RQ_QUEUE_IDX; 9857ae79a7SJianfeng Tan if (fn(dev, queue_sel) < 0) { 9957ae79a7SJianfeng Tan PMD_DRV_LOG(INFO, "setup rx vq fails: %u", i); 10057ae79a7SJianfeng Tan return -1; 10157ae79a7SJianfeng Tan } 10257ae79a7SJianfeng Tan } 10357ae79a7SJianfeng Tan for (i = 0; i < dev->max_queue_pairs; ++i) { 10457ae79a7SJianfeng Tan queue_sel = 2 * i + VTNET_SQ_TQ_QUEUE_IDX; 10557ae79a7SJianfeng Tan if (fn(dev, queue_sel) < 0) { 10657ae79a7SJianfeng Tan PMD_DRV_LOG(INFO, "setup tx vq fails: %u", i); 10757ae79a7SJianfeng Tan return -1; 10857ae79a7SJianfeng Tan } 10957ae79a7SJianfeng Tan } 11057ae79a7SJianfeng Tan 11157ae79a7SJianfeng Tan return 0; 11257ae79a7SJianfeng Tan } 11357ae79a7SJianfeng Tan 11437a7eb2aSJianfeng Tan int 115bd8f50a4SZhiyong Yang is_vhost_user_by_type(const char *path) 116bd8f50a4SZhiyong Yang { 117bd8f50a4SZhiyong Yang struct stat sb; 118bd8f50a4SZhiyong Yang 119bd8f50a4SZhiyong Yang if (stat(path, &sb) == -1) 120bd8f50a4SZhiyong Yang return 0; 121bd8f50a4SZhiyong Yang 122bd8f50a4SZhiyong Yang return S_ISSOCK(sb.st_mode); 123bd8f50a4SZhiyong Yang } 124bd8f50a4SZhiyong Yang 125bd8f50a4SZhiyong Yang int 12637a7eb2aSJianfeng Tan virtio_user_start_device(struct virtio_user_dev *dev) 12737a7eb2aSJianfeng Tan { 12837a7eb2aSJianfeng Tan uint64_t features; 12937a7eb2aSJianfeng Tan int ret; 13037a7eb2aSJianfeng Tan 1317ff26957STiwei Bie /* 1327ff26957STiwei Bie * XXX workaround! 1337ff26957STiwei Bie * 1347ff26957STiwei Bie * We need to make sure that the locks will be 1357ff26957STiwei Bie * taken in the correct order to avoid deadlocks. 1367ff26957STiwei Bie * 1377ff26957STiwei Bie * Before releasing this lock, this thread should 1387ff26957STiwei Bie * not trigger any memory hotplug events. 1397ff26957STiwei Bie * 1407ff26957STiwei Bie * This is a temporary workaround, and should be 1417ff26957STiwei Bie * replaced when we get proper supports from the 1427ff26957STiwei Bie * memory subsystem in the future. 1437ff26957STiwei Bie */ 144*76f80881SAnatoly Burakov rte_mcfg_mem_read_lock(); 14512ecb2f6SMaxime Coquelin pthread_mutex_lock(&dev->mutex); 14612ecb2f6SMaxime Coquelin 147bd8f50a4SZhiyong Yang if (is_vhost_user_by_type(dev->path) && dev->vhostfd < 0) 14812ecb2f6SMaxime Coquelin goto error; 149bd8f50a4SZhiyong Yang 15057ae79a7SJianfeng Tan /* Step 0: tell vhost to create queues */ 15157ae79a7SJianfeng Tan if (virtio_user_queue_setup(dev, virtio_user_create_queue) < 0) 15237a7eb2aSJianfeng Tan goto error; 15337a7eb2aSJianfeng Tan 15433d24d65SJianfeng Tan /* Step 1: set features */ 15537a7eb2aSJianfeng Tan features = dev->features; 15633d24d65SJianfeng Tan /* Strip VIRTIO_NET_F_MAC, as MAC address is handled in vdev init */ 15737a7eb2aSJianfeng Tan features &= ~(1ull << VIRTIO_NET_F_MAC); 15833d24d65SJianfeng Tan /* Strip VIRTIO_NET_F_CTRL_VQ, as devices do not really need to know */ 159142678d4SJianfeng Tan features &= ~(1ull << VIRTIO_NET_F_CTRL_VQ); 16035c4f855SJianfeng Tan features &= ~(1ull << VIRTIO_NET_F_STATUS); 16133d24d65SJianfeng Tan ret = dev->ops->send_request(dev, VHOST_USER_SET_FEATURES, &features); 16237a7eb2aSJianfeng Tan if (ret < 0) 16337a7eb2aSJianfeng Tan goto error; 16437a7eb2aSJianfeng Tan PMD_DRV_LOG(INFO, "set features: %" PRIx64, features); 16537a7eb2aSJianfeng Tan 16657ae79a7SJianfeng Tan /* Step 2: share memory regions */ 16733d24d65SJianfeng Tan ret = dev->ops->send_request(dev, VHOST_USER_SET_MEM_TABLE, NULL); 16857ae79a7SJianfeng Tan if (ret < 0) 16957ae79a7SJianfeng Tan goto error; 17057ae79a7SJianfeng Tan 17157ae79a7SJianfeng Tan /* Step 3: kick queues */ 17257ae79a7SJianfeng Tan if (virtio_user_queue_setup(dev, virtio_user_kick_queue) < 0) 17357ae79a7SJianfeng Tan goto error; 17457ae79a7SJianfeng Tan 17557ae79a7SJianfeng Tan /* Step 4: enable queues 17657ae79a7SJianfeng Tan * we enable the 1st queue pair by default. 17757ae79a7SJianfeng Tan */ 17833d24d65SJianfeng Tan dev->ops->enable_qp(dev, 0, 1); 17957ae79a7SJianfeng Tan 18012ecb2f6SMaxime Coquelin dev->started = true; 18112ecb2f6SMaxime Coquelin pthread_mutex_unlock(&dev->mutex); 182*76f80881SAnatoly Burakov rte_mcfg_mem_read_unlock(); 18312ecb2f6SMaxime Coquelin 18437a7eb2aSJianfeng Tan return 0; 18537a7eb2aSJianfeng Tan error: 18612ecb2f6SMaxime Coquelin pthread_mutex_unlock(&dev->mutex); 187*76f80881SAnatoly Burakov rte_mcfg_mem_read_unlock(); 18837a7eb2aSJianfeng Tan /* TODO: free resource here or caller to check */ 18937a7eb2aSJianfeng Tan return -1; 19037a7eb2aSJianfeng Tan } 19137a7eb2aSJianfeng Tan 19237a7eb2aSJianfeng Tan int virtio_user_stop_device(struct virtio_user_dev *dev) 19337a7eb2aSJianfeng Tan { 19474dc6746STiwei Bie struct vhost_vring_state state; 195c12a26eeSJianfeng Tan uint32_t i; 19674dc6746STiwei Bie int error = 0; 197c12a26eeSJianfeng Tan 19812ecb2f6SMaxime Coquelin pthread_mutex_lock(&dev->mutex); 199f457e900STiwei Bie if (!dev->started) 200f457e900STiwei Bie goto out; 201f457e900STiwei Bie 202c12a26eeSJianfeng Tan for (i = 0; i < dev->max_queue_pairs; ++i) 20333d24d65SJianfeng Tan dev->ops->enable_qp(dev, i, 0); 204c12a26eeSJianfeng Tan 20574dc6746STiwei Bie /* Stop the backend. */ 20674dc6746STiwei Bie for (i = 0; i < dev->max_queue_pairs * 2; ++i) { 20774dc6746STiwei Bie state.index = i; 20874dc6746STiwei Bie if (dev->ops->send_request(dev, VHOST_USER_GET_VRING_BASE, 20974dc6746STiwei Bie &state) < 0) { 21074dc6746STiwei Bie PMD_DRV_LOG(ERR, "get_vring_base failed, index=%u\n", 21174dc6746STiwei Bie i); 21274dc6746STiwei Bie error = -1; 21374dc6746STiwei Bie goto out; 2140d6a8752SJianfeng Tan } 21574dc6746STiwei Bie } 21674dc6746STiwei Bie 21712ecb2f6SMaxime Coquelin dev->started = false; 218f457e900STiwei Bie out: 21912ecb2f6SMaxime Coquelin pthread_mutex_unlock(&dev->mutex); 2200d6a8752SJianfeng Tan 22174dc6746STiwei Bie return error; 22237a7eb2aSJianfeng Tan } 22337a7eb2aSJianfeng Tan 22437a7eb2aSJianfeng Tan static inline void 22537a7eb2aSJianfeng Tan parse_mac(struct virtio_user_dev *dev, const char *mac) 22637a7eb2aSJianfeng Tan { 22737a7eb2aSJianfeng Tan int i, r; 22835b2d13fSOlivier Matz uint32_t tmp[RTE_ETHER_ADDR_LEN]; 22937a7eb2aSJianfeng Tan 23037a7eb2aSJianfeng Tan if (!mac) 23137a7eb2aSJianfeng Tan return; 23237a7eb2aSJianfeng Tan 23337a7eb2aSJianfeng Tan r = sscanf(mac, "%x:%x:%x:%x:%x:%x", &tmp[0], 23437a7eb2aSJianfeng Tan &tmp[1], &tmp[2], &tmp[3], &tmp[4], &tmp[5]); 23535b2d13fSOlivier Matz if (r == RTE_ETHER_ADDR_LEN) { 23635b2d13fSOlivier Matz for (i = 0; i < RTE_ETHER_ADDR_LEN; ++i) 23737a7eb2aSJianfeng Tan dev->mac_addr[i] = (uint8_t)tmp[i]; 23837a7eb2aSJianfeng Tan dev->mac_specified = 1; 23937a7eb2aSJianfeng Tan } else { 24037a7eb2aSJianfeng Tan /* ignore the wrong mac, use random mac */ 24137a7eb2aSJianfeng Tan PMD_DRV_LOG(ERR, "wrong format of mac: %s", mac); 24237a7eb2aSJianfeng Tan } 24337a7eb2aSJianfeng Tan } 24437a7eb2aSJianfeng Tan 24533d24d65SJianfeng Tan static int 246e6e7ad8bSJianfeng Tan virtio_user_dev_init_notify(struct virtio_user_dev *dev) 24733d24d65SJianfeng Tan { 248e6e7ad8bSJianfeng Tan uint32_t i, j; 249e6e7ad8bSJianfeng Tan int callfd; 250e6e7ad8bSJianfeng Tan int kickfd; 25133d24d65SJianfeng Tan 2522269b9aeSWenfeng Liu for (i = 0; i < VIRTIO_MAX_VIRTQUEUES; ++i) { 253e6e7ad8bSJianfeng Tan if (i >= dev->max_queue_pairs * 2) { 25433d24d65SJianfeng Tan dev->kickfds[i] = -1; 25533d24d65SJianfeng Tan dev->callfds[i] = -1; 256e6e7ad8bSJianfeng Tan continue; 25733d24d65SJianfeng Tan } 25833d24d65SJianfeng Tan 259e6e7ad8bSJianfeng Tan /* May use invalid flag, but some backend uses kickfd and 260e6e7ad8bSJianfeng Tan * callfd as criteria to judge if dev is alive. so finally we 261e6e7ad8bSJianfeng Tan * use real event_fd. 262e6e7ad8bSJianfeng Tan */ 263e6e7ad8bSJianfeng Tan callfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); 264e6e7ad8bSJianfeng Tan if (callfd < 0) { 265e6e7ad8bSJianfeng Tan PMD_DRV_LOG(ERR, "callfd error, %s", strerror(errno)); 266e6e7ad8bSJianfeng Tan break; 267e6e7ad8bSJianfeng Tan } 268e6e7ad8bSJianfeng Tan kickfd = eventfd(0, EFD_CLOEXEC | EFD_NONBLOCK); 269e6e7ad8bSJianfeng Tan if (kickfd < 0) { 270e6e7ad8bSJianfeng Tan PMD_DRV_LOG(ERR, "kickfd error, %s", strerror(errno)); 271e6e7ad8bSJianfeng Tan break; 272e6e7ad8bSJianfeng Tan } 273e6e7ad8bSJianfeng Tan dev->callfds[i] = callfd; 274e6e7ad8bSJianfeng Tan dev->kickfds[i] = kickfd; 275e6e7ad8bSJianfeng Tan } 276e6e7ad8bSJianfeng Tan 277e6e7ad8bSJianfeng Tan if (i < VIRTIO_MAX_VIRTQUEUES) { 278e6e7ad8bSJianfeng Tan for (j = 0; j <= i; ++j) { 279e6e7ad8bSJianfeng Tan close(dev->callfds[j]); 280e6e7ad8bSJianfeng Tan close(dev->kickfds[j]); 281e6e7ad8bSJianfeng Tan } 282e6e7ad8bSJianfeng Tan 283e6e7ad8bSJianfeng Tan return -1; 284e6e7ad8bSJianfeng Tan } 285e6e7ad8bSJianfeng Tan 286e6e7ad8bSJianfeng Tan return 0; 287e6e7ad8bSJianfeng Tan } 288e6e7ad8bSJianfeng Tan 289e6e7ad8bSJianfeng Tan static int 2903d4fb6fdSJianfeng Tan virtio_user_fill_intr_handle(struct virtio_user_dev *dev) 2913d4fb6fdSJianfeng Tan { 2923d4fb6fdSJianfeng Tan uint32_t i; 2933d4fb6fdSJianfeng Tan struct rte_eth_dev *eth_dev = &rte_eth_devices[dev->port_id]; 2943d4fb6fdSJianfeng Tan 2953d4fb6fdSJianfeng Tan if (!eth_dev->intr_handle) { 2963d4fb6fdSJianfeng Tan eth_dev->intr_handle = malloc(sizeof(*eth_dev->intr_handle)); 2973d4fb6fdSJianfeng Tan if (!eth_dev->intr_handle) { 2983d4fb6fdSJianfeng Tan PMD_DRV_LOG(ERR, "fail to allocate intr_handle"); 2993d4fb6fdSJianfeng Tan return -1; 3003d4fb6fdSJianfeng Tan } 3013d4fb6fdSJianfeng Tan memset(eth_dev->intr_handle, 0, sizeof(*eth_dev->intr_handle)); 3023d4fb6fdSJianfeng Tan } 3033d4fb6fdSJianfeng Tan 3043d4fb6fdSJianfeng Tan for (i = 0; i < dev->max_queue_pairs; ++i) 3053d4fb6fdSJianfeng Tan eth_dev->intr_handle->efds[i] = dev->callfds[i]; 3063d4fb6fdSJianfeng Tan eth_dev->intr_handle->nb_efd = dev->max_queue_pairs; 3073d4fb6fdSJianfeng Tan eth_dev->intr_handle->max_intr = dev->max_queue_pairs + 1; 3083d4fb6fdSJianfeng Tan eth_dev->intr_handle->type = RTE_INTR_HANDLE_VDEV; 30929906b97SJingjing Wu /* For virtio vdev, no need to read counter for clean */ 31029906b97SJingjing Wu eth_dev->intr_handle->efd_counter_size = 0; 311eae6e70cSJianfeng Tan eth_dev->intr_handle->fd = -1; 312cc4690e9SJianfeng Tan if (dev->vhostfd >= 0) 313cc4690e9SJianfeng Tan eth_dev->intr_handle->fd = dev->vhostfd; 314bd8f50a4SZhiyong Yang else if (dev->is_server) 315bd8f50a4SZhiyong Yang eth_dev->intr_handle->fd = dev->listenfd; 3163d4fb6fdSJianfeng Tan 3173d4fb6fdSJianfeng Tan return 0; 3183d4fb6fdSJianfeng Tan } 3193d4fb6fdSJianfeng Tan 32012ecb2f6SMaxime Coquelin static void 32112ecb2f6SMaxime Coquelin virtio_user_mem_event_cb(enum rte_mem_event type __rte_unused, 32212ecb2f6SMaxime Coquelin const void *addr __rte_unused, 32312ecb2f6SMaxime Coquelin size_t len __rte_unused, 32412ecb2f6SMaxime Coquelin void *arg) 32512ecb2f6SMaxime Coquelin { 32612ecb2f6SMaxime Coquelin struct virtio_user_dev *dev = arg; 327f32c7c9dSAnatoly Burakov struct rte_memseg_list *msl; 32812ecb2f6SMaxime Coquelin uint16_t i; 32912ecb2f6SMaxime Coquelin 330f32c7c9dSAnatoly Burakov /* ignore externally allocated memory */ 331f32c7c9dSAnatoly Burakov msl = rte_mem_virt2memseg_list(addr); 332f32c7c9dSAnatoly Burakov if (msl->external) 333f32c7c9dSAnatoly Burakov return; 334f32c7c9dSAnatoly Burakov 33512ecb2f6SMaxime Coquelin pthread_mutex_lock(&dev->mutex); 33612ecb2f6SMaxime Coquelin 33712ecb2f6SMaxime Coquelin if (dev->started == false) 33812ecb2f6SMaxime Coquelin goto exit; 33912ecb2f6SMaxime Coquelin 34012ecb2f6SMaxime Coquelin /* Step 1: pause the active queues */ 34112ecb2f6SMaxime Coquelin for (i = 0; i < dev->queue_pairs; i++) 34212ecb2f6SMaxime Coquelin dev->ops->enable_qp(dev, i, 0); 34312ecb2f6SMaxime Coquelin 34412ecb2f6SMaxime Coquelin /* Step 2: update memory regions */ 34512ecb2f6SMaxime Coquelin dev->ops->send_request(dev, VHOST_USER_SET_MEM_TABLE, NULL); 34612ecb2f6SMaxime Coquelin 34712ecb2f6SMaxime Coquelin /* Step 3: resume the active queues */ 34812ecb2f6SMaxime Coquelin for (i = 0; i < dev->queue_pairs; i++) 34912ecb2f6SMaxime Coquelin dev->ops->enable_qp(dev, i, 1); 35012ecb2f6SMaxime Coquelin 35112ecb2f6SMaxime Coquelin exit: 35212ecb2f6SMaxime Coquelin pthread_mutex_unlock(&dev->mutex); 35312ecb2f6SMaxime Coquelin } 35412ecb2f6SMaxime Coquelin 3553d4fb6fdSJianfeng Tan static int 356e6e7ad8bSJianfeng Tan virtio_user_dev_setup(struct virtio_user_dev *dev) 357e6e7ad8bSJianfeng Tan { 358e6e7ad8bSJianfeng Tan uint32_t q; 359e6e7ad8bSJianfeng Tan 360e6e7ad8bSJianfeng Tan dev->vhostfd = -1; 361e3b43481SJianfeng Tan dev->vhostfds = NULL; 362e3b43481SJianfeng Tan dev->tapfds = NULL; 363e3b43481SJianfeng Tan 364bd8f50a4SZhiyong Yang if (dev->is_server) { 365bd8f50a4SZhiyong Yang if (access(dev->path, F_OK) == 0 && 366bd8f50a4SZhiyong Yang !is_vhost_user_by_type(dev->path)) { 367bd8f50a4SZhiyong Yang PMD_DRV_LOG(ERR, "Server mode doesn't support vhost-kernel!"); 368bd8f50a4SZhiyong Yang return -1; 369bd8f50a4SZhiyong Yang } 370520dd992SFerruh Yigit dev->ops = &virtio_ops_user; 371bd8f50a4SZhiyong Yang } else { 37233d24d65SJianfeng Tan if (is_vhost_user_by_type(dev->path)) { 373520dd992SFerruh Yigit dev->ops = &virtio_ops_user; 374e3b43481SJianfeng Tan } else { 375520dd992SFerruh Yigit dev->ops = &virtio_ops_kernel; 376e3b43481SJianfeng Tan 377bd8f50a4SZhiyong Yang dev->vhostfds = malloc(dev->max_queue_pairs * 378bd8f50a4SZhiyong Yang sizeof(int)); 379bd8f50a4SZhiyong Yang dev->tapfds = malloc(dev->max_queue_pairs * 380bd8f50a4SZhiyong Yang sizeof(int)); 381e3b43481SJianfeng Tan if (!dev->vhostfds || !dev->tapfds) { 382e3b43481SJianfeng Tan PMD_INIT_LOG(ERR, "Failed to malloc"); 383e3b43481SJianfeng Tan return -1; 38433d24d65SJianfeng Tan } 38533d24d65SJianfeng Tan 386e3b43481SJianfeng Tan for (q = 0; q < dev->max_queue_pairs; ++q) { 387e3b43481SJianfeng Tan dev->vhostfds[q] = -1; 388e3b43481SJianfeng Tan dev->tapfds[q] = -1; 389e3b43481SJianfeng Tan } 390e3b43481SJianfeng Tan } 391bd8f50a4SZhiyong Yang } 392e3b43481SJianfeng Tan 393cc4690e9SJianfeng Tan if (dev->ops->setup(dev) < 0) 394cc4690e9SJianfeng Tan return -1; 395cc4690e9SJianfeng Tan 396cc4690e9SJianfeng Tan if (virtio_user_dev_init_notify(dev) < 0) 397cc4690e9SJianfeng Tan return -1; 398cc4690e9SJianfeng Tan 399cc4690e9SJianfeng Tan if (virtio_user_fill_intr_handle(dev) < 0) 400cc4690e9SJianfeng Tan return -1; 401cc4690e9SJianfeng Tan 402cc4690e9SJianfeng Tan return 0; 40333d24d65SJianfeng Tan } 40433d24d65SJianfeng Tan 405bed3b24cSJianfeng Tan /* Use below macro to filter features from vhost backend */ 406bed3b24cSJianfeng Tan #define VIRTIO_USER_SUPPORTED_FEATURES \ 407bed3b24cSJianfeng Tan (1ULL << VIRTIO_NET_F_MAC | \ 408bed3b24cSJianfeng Tan 1ULL << VIRTIO_NET_F_STATUS | \ 409bed3b24cSJianfeng Tan 1ULL << VIRTIO_NET_F_MQ | \ 410bed3b24cSJianfeng Tan 1ULL << VIRTIO_NET_F_CTRL_MAC_ADDR | \ 411bed3b24cSJianfeng Tan 1ULL << VIRTIO_NET_F_CTRL_VQ | \ 412bed3b24cSJianfeng Tan 1ULL << VIRTIO_NET_F_CTRL_RX | \ 413bed3b24cSJianfeng Tan 1ULL << VIRTIO_NET_F_CTRL_VLAN | \ 414bed3b24cSJianfeng Tan 1ULL << VIRTIO_NET_F_CSUM | \ 415bed3b24cSJianfeng Tan 1ULL << VIRTIO_NET_F_HOST_TSO4 | \ 416bed3b24cSJianfeng Tan 1ULL << VIRTIO_NET_F_HOST_TSO6 | \ 417bed3b24cSJianfeng Tan 1ULL << VIRTIO_NET_F_MRG_RXBUF | \ 418bed3b24cSJianfeng Tan 1ULL << VIRTIO_RING_F_INDIRECT_DESC | \ 419bed3b24cSJianfeng Tan 1ULL << VIRTIO_NET_F_GUEST_CSUM | \ 420bed3b24cSJianfeng Tan 1ULL << VIRTIO_NET_F_GUEST_TSO4 | \ 421bed3b24cSJianfeng Tan 1ULL << VIRTIO_NET_F_GUEST_TSO6 | \ 42241e45c90SMarvin Liu 1ULL << VIRTIO_F_IN_ORDER | \ 42334f3966cSYuanhan Liu 1ULL << VIRTIO_F_VERSION_1 | \ 4248532a0fcSTiwei Bie 1ULL << VIRTIO_F_RING_PACKED) 425bed3b24cSJianfeng Tan 42637a7eb2aSJianfeng Tan int 42737a7eb2aSJianfeng Tan virtio_user_dev_init(struct virtio_user_dev *dev, char *path, int queues, 428488ed97aSMarvin Liu int cq, int queue_size, const char *mac, char **ifname, 4291c8489daSTiwei Bie int server, int mrg_rxbuf, int in_order, int packed_vq) 43037a7eb2aSJianfeng Tan { 43112ecb2f6SMaxime Coquelin pthread_mutex_init(&dev->mutex, NULL); 4326723c0fcSBruce Richardson strlcpy(dev->path, path, PATH_MAX); 43312ecb2f6SMaxime Coquelin dev->started = 0; 43437a7eb2aSJianfeng Tan dev->max_queue_pairs = queues; 43537a7eb2aSJianfeng Tan dev->queue_pairs = 1; /* mq disabled by default */ 43637a7eb2aSJianfeng Tan dev->queue_size = queue_size; 4371c8489daSTiwei Bie dev->is_server = server; 43837a7eb2aSJianfeng Tan dev->mac_specified = 0; 439bb97d2ddSTiwei Bie dev->frontend_features = 0; 440bd9568f3STiwei Bie dev->unsupported_features = ~VIRTIO_USER_SUPPORTED_FEATURES; 44137a7eb2aSJianfeng Tan parse_mac(dev, mac); 44237a7eb2aSJianfeng Tan 4434214a1b4SWenfeng Liu if (*ifname) { 4444214a1b4SWenfeng Liu dev->ifname = *ifname; 4454214a1b4SWenfeng Liu *ifname = NULL; 4464214a1b4SWenfeng Liu } 4474214a1b4SWenfeng Liu 44833d24d65SJianfeng Tan if (virtio_user_dev_setup(dev) < 0) { 44937a7eb2aSJianfeng Tan PMD_INIT_LOG(ERR, "backend set up fails"); 45037a7eb2aSJianfeng Tan return -1; 45137a7eb2aSJianfeng Tan } 452bce7e905SJianfeng Tan 4536e1e5904SJiayu Hu if (!dev->is_server) { 454bd8f50a4SZhiyong Yang if (dev->ops->send_request(dev, VHOST_USER_SET_OWNER, 455bd8f50a4SZhiyong Yang NULL) < 0) { 456bd8f50a4SZhiyong Yang PMD_INIT_LOG(ERR, "set_owner fails: %s", 457bd8f50a4SZhiyong Yang strerror(errno)); 45837a7eb2aSJianfeng Tan return -1; 45937a7eb2aSJianfeng Tan } 46037a7eb2aSJianfeng Tan 46133d24d65SJianfeng Tan if (dev->ops->send_request(dev, VHOST_USER_GET_FEATURES, 462142678d4SJianfeng Tan &dev->device_features) < 0) { 463bd8f50a4SZhiyong Yang PMD_INIT_LOG(ERR, "get_features failed: %s", 464bd8f50a4SZhiyong Yang strerror(errno)); 46537a7eb2aSJianfeng Tan return -1; 46637a7eb2aSJianfeng Tan } 467bd8f50a4SZhiyong Yang } else { 468bd8f50a4SZhiyong Yang /* We just pretend vhost-user can support all these features. 469bd8f50a4SZhiyong Yang * Note that this could be problematic that if some feature is 470bd8f50a4SZhiyong Yang * negotiated but not supported by the vhost-user which comes 471bd8f50a4SZhiyong Yang * later. 472bd8f50a4SZhiyong Yang */ 473bd8f50a4SZhiyong Yang dev->device_features = VIRTIO_USER_SUPPORTED_FEATURES; 474bd8f50a4SZhiyong Yang } 475bd8f50a4SZhiyong Yang 476bd9568f3STiwei Bie if (!mrg_rxbuf) 477488ed97aSMarvin Liu dev->unsupported_features |= (1ull << VIRTIO_NET_F_MRG_RXBUF); 478488ed97aSMarvin Liu 479bd9568f3STiwei Bie if (!in_order) 480488ed97aSMarvin Liu dev->unsupported_features |= (1ull << VIRTIO_F_IN_ORDER); 481488ed97aSMarvin Liu 48248a44640SJens Freimann if (!packed_vq) 4839070f88bSTiwei Bie dev->unsupported_features |= (1ull << VIRTIO_F_RING_PACKED); 48434f3966cSYuanhan Liu 4859070f88bSTiwei Bie if (dev->mac_specified) 4869070f88bSTiwei Bie dev->frontend_features |= (1ull << VIRTIO_NET_F_MAC); 4879070f88bSTiwei Bie else 4887c66ff61SMarvin Liu dev->unsupported_features |= (1ull << VIRTIO_NET_F_MAC); 489f9b9d1a5SJianfeng Tan 490142678d4SJianfeng Tan if (cq) { 491142678d4SJianfeng Tan /* device does not really need to know anything about CQ, 492142678d4SJianfeng Tan * so if necessary, we just claim to support CQ 493f9b9d1a5SJianfeng Tan */ 494bb97d2ddSTiwei Bie dev->frontend_features |= (1ull << VIRTIO_NET_F_CTRL_VQ); 495142678d4SJianfeng Tan } else { 4967c66ff61SMarvin Liu dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VQ); 497bd9568f3STiwei Bie /* Also disable features that depend on VIRTIO_NET_F_CTRL_VQ */ 4987c66ff61SMarvin Liu dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_RX); 4997c66ff61SMarvin Liu dev->unsupported_features |= (1ull << VIRTIO_NET_F_CTRL_VLAN); 5007c66ff61SMarvin Liu dev->unsupported_features |= 5017c66ff61SMarvin Liu (1ull << VIRTIO_NET_F_GUEST_ANNOUNCE); 5027c66ff61SMarvin Liu dev->unsupported_features |= (1ull << VIRTIO_NET_F_MQ); 5037c66ff61SMarvin Liu dev->unsupported_features |= 5047c66ff61SMarvin Liu (1ull << VIRTIO_NET_F_CTRL_MAC_ADDR); 505f9b9d1a5SJianfeng Tan } 506f9b9d1a5SJianfeng Tan 50735c4f855SJianfeng Tan /* The backend will not report this feature, we add it explicitly */ 5084bf9e26dSJianfeng Tan if (is_vhost_user_by_type(dev->path)) 509bb97d2ddSTiwei Bie dev->frontend_features |= (1ull << VIRTIO_NET_F_STATUS); 51035c4f855SJianfeng Tan 511bd9568f3STiwei Bie /* 512bd9568f3STiwei Bie * Device features = 513bd9568f3STiwei Bie * (frontend_features | backend_features) & ~unsupported_features; 514bd9568f3STiwei Bie */ 515bb97d2ddSTiwei Bie dev->device_features |= dev->frontend_features; 516bd9568f3STiwei Bie dev->device_features &= ~dev->unsupported_features; 517bed3b24cSJianfeng Tan 51812ecb2f6SMaxime Coquelin if (rte_mem_event_callback_register(VIRTIO_USER_MEM_EVENT_CLB_NAME, 51912ecb2f6SMaxime Coquelin virtio_user_mem_event_cb, dev)) { 52088e5469fSXiao Wang if (rte_errno != ENOTSUP) { 52188e5469fSXiao Wang PMD_INIT_LOG(ERR, "Failed to register mem event" 52288e5469fSXiao Wang " callback\n"); 52312ecb2f6SMaxime Coquelin return -1; 52412ecb2f6SMaxime Coquelin } 52588e5469fSXiao Wang } 52612ecb2f6SMaxime Coquelin 52737a7eb2aSJianfeng Tan return 0; 52837a7eb2aSJianfeng Tan } 52937a7eb2aSJianfeng Tan 53037a7eb2aSJianfeng Tan void 53137a7eb2aSJianfeng Tan virtio_user_dev_uninit(struct virtio_user_dev *dev) 53237a7eb2aSJianfeng Tan { 533e3b43481SJianfeng Tan uint32_t i; 534e3b43481SJianfeng Tan 535e3b43481SJianfeng Tan virtio_user_stop_device(dev); 536e3b43481SJianfeng Tan 53712ecb2f6SMaxime Coquelin rte_mem_event_callback_unregister(VIRTIO_USER_MEM_EVENT_CLB_NAME, dev); 53812ecb2f6SMaxime Coquelin 539e6e7ad8bSJianfeng Tan for (i = 0; i < dev->max_queue_pairs * 2; ++i) { 540e6e7ad8bSJianfeng Tan close(dev->callfds[i]); 541e6e7ad8bSJianfeng Tan close(dev->kickfds[i]); 542e6e7ad8bSJianfeng Tan } 543e6e7ad8bSJianfeng Tan 54437a7eb2aSJianfeng Tan close(dev->vhostfd); 545e3b43481SJianfeng Tan 546bd8f50a4SZhiyong Yang if (dev->is_server && dev->listenfd >= 0) { 547bd8f50a4SZhiyong Yang close(dev->listenfd); 548bd8f50a4SZhiyong Yang dev->listenfd = -1; 549bd8f50a4SZhiyong Yang } 550bd8f50a4SZhiyong Yang 551e3b43481SJianfeng Tan if (dev->vhostfds) { 552e3b43481SJianfeng Tan for (i = 0; i < dev->max_queue_pairs; ++i) 553e3b43481SJianfeng Tan close(dev->vhostfds[i]); 554e3b43481SJianfeng Tan free(dev->vhostfds); 555e3b43481SJianfeng Tan free(dev->tapfds); 556e3b43481SJianfeng Tan } 5574214a1b4SWenfeng Liu 5584214a1b4SWenfeng Liu free(dev->ifname); 559bd8f50a4SZhiyong Yang 560bd8f50a4SZhiyong Yang if (dev->is_server) 561bd8f50a4SZhiyong Yang unlink(dev->path); 56237a7eb2aSJianfeng Tan } 563f9b9d1a5SJianfeng Tan 564201a4165SZhiyong Yang uint8_t 565f9b9d1a5SJianfeng Tan virtio_user_handle_mq(struct virtio_user_dev *dev, uint16_t q_pairs) 566f9b9d1a5SJianfeng Tan { 567f9b9d1a5SJianfeng Tan uint16_t i; 568f9b9d1a5SJianfeng Tan uint8_t ret = 0; 569f9b9d1a5SJianfeng Tan 570f9b9d1a5SJianfeng Tan if (q_pairs > dev->max_queue_pairs) { 571f9b9d1a5SJianfeng Tan PMD_INIT_LOG(ERR, "multi-q config %u, but only %u supported", 572f9b9d1a5SJianfeng Tan q_pairs, dev->max_queue_pairs); 573f9b9d1a5SJianfeng Tan return -1; 574f9b9d1a5SJianfeng Tan } 575f9b9d1a5SJianfeng Tan 576201a4165SZhiyong Yang /* Server mode can't enable queue pairs if vhostfd is invalid, 577201a4165SZhiyong Yang * always return 0 in this case. 578201a4165SZhiyong Yang */ 57921b90f79STiwei Bie if (!dev->is_server || dev->vhostfd >= 0) { 580f9b9d1a5SJianfeng Tan for (i = 0; i < q_pairs; ++i) 58133d24d65SJianfeng Tan ret |= dev->ops->enable_qp(dev, i, 1); 582f9b9d1a5SJianfeng Tan for (i = q_pairs; i < dev->max_queue_pairs; ++i) 58333d24d65SJianfeng Tan ret |= dev->ops->enable_qp(dev, i, 0); 584201a4165SZhiyong Yang } 585f9b9d1a5SJianfeng Tan dev->queue_pairs = q_pairs; 586f9b9d1a5SJianfeng Tan 587f9b9d1a5SJianfeng Tan return ret; 588f9b9d1a5SJianfeng Tan } 589f9b9d1a5SJianfeng Tan 590f9b9d1a5SJianfeng Tan static uint32_t 591f9b9d1a5SJianfeng Tan virtio_user_handle_ctrl_msg(struct virtio_user_dev *dev, struct vring *vring, 592f9b9d1a5SJianfeng Tan uint16_t idx_hdr) 593f9b9d1a5SJianfeng Tan { 594f9b9d1a5SJianfeng Tan struct virtio_net_ctrl_hdr *hdr; 595f9b9d1a5SJianfeng Tan virtio_net_ctrl_ack status = ~0; 596f9b9d1a5SJianfeng Tan uint16_t i, idx_data, idx_status; 597f9b9d1a5SJianfeng Tan uint32_t n_descs = 0; 598f9b9d1a5SJianfeng Tan 599f9b9d1a5SJianfeng Tan /* locate desc for header, data, and status */ 600f9b9d1a5SJianfeng Tan idx_data = vring->desc[idx_hdr].next; 601f9b9d1a5SJianfeng Tan n_descs++; 602f9b9d1a5SJianfeng Tan 603f9b9d1a5SJianfeng Tan i = idx_data; 604f9b9d1a5SJianfeng Tan while (vring->desc[i].flags == VRING_DESC_F_NEXT) { 605f9b9d1a5SJianfeng Tan i = vring->desc[i].next; 606f9b9d1a5SJianfeng Tan n_descs++; 607f9b9d1a5SJianfeng Tan } 608f9b9d1a5SJianfeng Tan 609f9b9d1a5SJianfeng Tan /* locate desc for status */ 610f9b9d1a5SJianfeng Tan idx_status = i; 611f9b9d1a5SJianfeng Tan n_descs++; 612f9b9d1a5SJianfeng Tan 613f9b9d1a5SJianfeng Tan hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr; 614f9b9d1a5SJianfeng Tan if (hdr->class == VIRTIO_NET_CTRL_MQ && 615f9b9d1a5SJianfeng Tan hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 616f9b9d1a5SJianfeng Tan uint16_t queues; 617f9b9d1a5SJianfeng Tan 618f9b9d1a5SJianfeng Tan queues = *(uint16_t *)(uintptr_t)vring->desc[idx_data].addr; 619f9b9d1a5SJianfeng Tan status = virtio_user_handle_mq(dev, queues); 620f9b9d1a5SJianfeng Tan } 621f9b9d1a5SJianfeng Tan 622f9b9d1a5SJianfeng Tan /* Update status */ 623f9b9d1a5SJianfeng Tan *(virtio_net_ctrl_ack *)(uintptr_t)vring->desc[idx_status].addr = status; 624f9b9d1a5SJianfeng Tan 625f9b9d1a5SJianfeng Tan return n_descs; 626f9b9d1a5SJianfeng Tan } 627f9b9d1a5SJianfeng Tan 62848a44640SJens Freimann static inline int 62948a44640SJens Freimann desc_is_avail(struct vring_packed_desc *desc, bool wrap_counter) 63048a44640SJens Freimann { 63112e9e70cSTiwei Bie uint16_t flags = desc->flags; 63212e9e70cSTiwei Bie 63312e9e70cSTiwei Bie return wrap_counter == !!(flags & VRING_PACKED_DESC_F_AVAIL) && 63412e9e70cSTiwei Bie wrap_counter != !!(flags & VRING_PACKED_DESC_F_USED); 63548a44640SJens Freimann } 63648a44640SJens Freimann 63748a44640SJens Freimann static uint32_t 63845c224e7STiwei Bie virtio_user_handle_ctrl_msg_packed(struct virtio_user_dev *dev, 63948a44640SJens Freimann struct vring_packed *vring, 64048a44640SJens Freimann uint16_t idx_hdr) 64148a44640SJens Freimann { 64248a44640SJens Freimann struct virtio_net_ctrl_hdr *hdr; 64348a44640SJens Freimann virtio_net_ctrl_ack status = ~0; 64448a44640SJens Freimann uint16_t idx_data, idx_status; 64548a44640SJens Freimann /* initialize to one, header is first */ 64648a44640SJens Freimann uint32_t n_descs = 1; 64748a44640SJens Freimann 64848a44640SJens Freimann /* locate desc for header, data, and status */ 64948a44640SJens Freimann idx_data = idx_hdr + 1; 65048a44640SJens Freimann if (idx_data >= dev->queue_size) 65148a44640SJens Freimann idx_data -= dev->queue_size; 65248a44640SJens Freimann 65348a44640SJens Freimann n_descs++; 65448a44640SJens Freimann 65548a44640SJens Freimann idx_status = idx_data; 6564cdc4d98STiwei Bie while (vring->desc[idx_status].flags & VRING_DESC_F_NEXT) { 65748a44640SJens Freimann idx_status++; 65848a44640SJens Freimann if (idx_status >= dev->queue_size) 65948a44640SJens Freimann idx_status -= dev->queue_size; 66048a44640SJens Freimann n_descs++; 66148a44640SJens Freimann } 66248a44640SJens Freimann 6634cdc4d98STiwei Bie hdr = (void *)(uintptr_t)vring->desc[idx_hdr].addr; 66448a44640SJens Freimann if (hdr->class == VIRTIO_NET_CTRL_MQ && 66548a44640SJens Freimann hdr->cmd == VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET) { 66648a44640SJens Freimann uint16_t queues; 66748a44640SJens Freimann 66848a44640SJens Freimann queues = *(uint16_t *)(uintptr_t) 6694cdc4d98STiwei Bie vring->desc[idx_data].addr; 67048a44640SJens Freimann status = virtio_user_handle_mq(dev, queues); 67148a44640SJens Freimann } 67248a44640SJens Freimann 67348a44640SJens Freimann /* Update status */ 67448a44640SJens Freimann *(virtio_net_ctrl_ack *)(uintptr_t) 6754cdc4d98STiwei Bie vring->desc[idx_status].addr = status; 67648a44640SJens Freimann 67745c224e7STiwei Bie /* Update used descriptor */ 6784cdc4d98STiwei Bie vring->desc[idx_hdr].id = vring->desc[idx_status].id; 6794cdc4d98STiwei Bie vring->desc[idx_hdr].len = sizeof(status); 68045c224e7STiwei Bie 68148a44640SJens Freimann return n_descs; 68248a44640SJens Freimann } 68348a44640SJens Freimann 68448a44640SJens Freimann void 68548a44640SJens Freimann virtio_user_handle_cq_packed(struct virtio_user_dev *dev, uint16_t queue_idx) 68648a44640SJens Freimann { 68748a44640SJens Freimann struct virtio_user_queue *vq = &dev->packed_queues[queue_idx]; 68848a44640SJens Freimann struct vring_packed *vring = &dev->packed_vrings[queue_idx]; 68912e9e70cSTiwei Bie uint16_t n_descs, flags; 69048a44640SJens Freimann 6914cdc4d98STiwei Bie while (desc_is_avail(&vring->desc[vq->used_idx], 69248a44640SJens Freimann vq->used_wrap_counter)) { 69348a44640SJens Freimann 69445c224e7STiwei Bie n_descs = virtio_user_handle_ctrl_msg_packed(dev, vring, 69545c224e7STiwei Bie vq->used_idx); 69648a44640SJens Freimann 69712e9e70cSTiwei Bie flags = VRING_DESC_F_WRITE; 69812e9e70cSTiwei Bie if (vq->used_wrap_counter) 69912e9e70cSTiwei Bie flags |= VRING_PACKED_DESC_F_AVAIL_USED; 70012e9e70cSTiwei Bie 70145c224e7STiwei Bie rte_smp_wmb(); 70212e9e70cSTiwei Bie vring->desc[vq->used_idx].flags = flags; 70345c224e7STiwei Bie 70445c224e7STiwei Bie vq->used_idx += n_descs; 70545c224e7STiwei Bie if (vq->used_idx >= dev->queue_size) { 70648a44640SJens Freimann vq->used_idx -= dev->queue_size; 70748a44640SJens Freimann vq->used_wrap_counter ^= 1; 70848a44640SJens Freimann } 70948a44640SJens Freimann } 71048a44640SJens Freimann } 71148a44640SJens Freimann 712f9b9d1a5SJianfeng Tan void 713f9b9d1a5SJianfeng Tan virtio_user_handle_cq(struct virtio_user_dev *dev, uint16_t queue_idx) 714f9b9d1a5SJianfeng Tan { 715f9b9d1a5SJianfeng Tan uint16_t avail_idx, desc_idx; 716f9b9d1a5SJianfeng Tan struct vring_used_elem *uep; 717f9b9d1a5SJianfeng Tan uint32_t n_descs; 718f9b9d1a5SJianfeng Tan struct vring *vring = &dev->vrings[queue_idx]; 719f9b9d1a5SJianfeng Tan 720f9b9d1a5SJianfeng Tan /* Consume avail ring, using used ring idx as first one */ 721f9b9d1a5SJianfeng Tan while (vring->used->idx != vring->avail->idx) { 722f9b9d1a5SJianfeng Tan avail_idx = (vring->used->idx) & (vring->num - 1); 723f9b9d1a5SJianfeng Tan desc_idx = vring->avail->ring[avail_idx]; 724f9b9d1a5SJianfeng Tan 725f9b9d1a5SJianfeng Tan n_descs = virtio_user_handle_ctrl_msg(dev, vring, desc_idx); 726f9b9d1a5SJianfeng Tan 727f9b9d1a5SJianfeng Tan /* Update used ring */ 728f9b9d1a5SJianfeng Tan uep = &vring->used->ring[avail_idx]; 7290403e37aSTiwei Bie uep->id = desc_idx; 730f9b9d1a5SJianfeng Tan uep->len = n_descs; 731f9b9d1a5SJianfeng Tan 732f9b9d1a5SJianfeng Tan vring->used->idx++; 733f9b9d1a5SJianfeng Tan } 734f9b9d1a5SJianfeng Tan } 735