xref: /dpdk/drivers/vdpa/nfp/nfp_vdpa.c (revision b6de43530dfa30cbf6b70857e3835099701063d4)

/* SPDX-License-Identifier: BSD-3-Clause
 * Copyright (c) 2023 Corigine, Inc.
 * All rights reserved.
 */

#include <pthread.h>
#include <sys/epoll.h>
#include <sys/ioctl.h>
#include <unistd.h>

#include <nfp_common_pci.h>
#include <nfp_dev.h>
#include <rte_vfio.h>
#include <rte_eal_paging.h>
#include <rte_malloc.h>
#include <vdpa_driver.h>

#include "nfp_vdpa_core.h"
#include "nfp_vdpa_log.h"

#define NFP_VDPA_DRIVER_NAME nfp_vdpa

#define MSIX_IRQ_SET_BUF_LEN (sizeof(struct vfio_irq_set) + \
		sizeof(int) * (NFP_VDPA_MAX_QUEUES * 2 + 1))

#define NFP_VDPA_USED_RING_LEN(size) \
		((size) * sizeof(struct vring_used_elem) + sizeof(struct vring_used))

#define EPOLL_DATA_INTR        1

struct nfp_vdpa_dev {
	struct rte_pci_device *pci_dev;
	struct rte_vdpa_device *vdev;
	struct nfp_vdpa_hw hw;

	int vfio_container_fd;
	int vfio_group_fd;
	int vfio_dev_fd;
	int iommu_group;

	rte_thread_t tid;    /**< Thread for notify relay */
	int epoll_fd;

	int vid;
	uint16_t max_queues;
	RTE_ATOMIC(uint32_t) started;
	RTE_ATOMIC(uint32_t) dev_attached;
	RTE_ATOMIC(uint32_t) running;
	rte_spinlock_t lock;

	/** Eventfd for used ring interrupt */
	int intr_fd[NFP_VDPA_MAX_QUEUES * 2];
};

struct nfp_vdpa_dev_node {
	TAILQ_ENTRY(nfp_vdpa_dev_node) next;
	struct nfp_vdpa_dev *device;
};

TAILQ_HEAD(vdpa_dev_list_head, nfp_vdpa_dev_node);

static struct vdpa_dev_list_head vdpa_dev_list =
	TAILQ_HEAD_INITIALIZER(vdpa_dev_list);

static pthread_mutex_t vdpa_list_lock = PTHREAD_MUTEX_INITIALIZER;

static struct nfp_vdpa_dev_node *
nfp_vdpa_find_node_by_vdev(struct rte_vdpa_device *vdev)
{
	bool found = false;
	struct nfp_vdpa_dev_node *node;

	pthread_mutex_lock(&vdpa_list_lock);

	TAILQ_FOREACH(node, &vdpa_dev_list, next) {
		if (vdev == node->device->vdev) {
			found = true;
			break;
		}
	}

	pthread_mutex_unlock(&vdpa_list_lock);

	if (found)
		return node;

	return NULL;
}

static struct nfp_vdpa_dev_node *
nfp_vdpa_find_node_by_pdev(struct rte_pci_device *pdev)
{
	bool found = false;
	struct nfp_vdpa_dev_node *node;

	pthread_mutex_lock(&vdpa_list_lock);

	TAILQ_FOREACH(node, &vdpa_dev_list, next) {
		if (pdev == node->device->pci_dev) {
			found = true;
			break;
		}
	}

	pthread_mutex_unlock(&vdpa_list_lock);

	if (found)
		return node;

	return NULL;
}

static int
nfp_vdpa_vfio_setup(struct nfp_vdpa_dev *device)
{
	int ret;
	char dev_name[RTE_DEV_NAME_MAX_LEN] = {0};
	struct rte_pci_device *pci_dev = device->pci_dev;

	rte_pci_unmap_device(pci_dev);

	rte_pci_device_name(&pci_dev->addr, dev_name, RTE_DEV_NAME_MAX_LEN);
	ret = rte_vfio_get_group_num(rte_pci_get_sysfs_path(), dev_name,
			&device->iommu_group);
	if (ret <= 0)
		return -1;

	device->vfio_container_fd = rte_vfio_container_create();
	if (device->vfio_container_fd < 0)
		return -1;

	device->vfio_group_fd = rte_vfio_container_group_bind(
			device->vfio_container_fd, device->iommu_group);
	if (device->vfio_group_fd < 0)
		goto container_destroy;

	DRV_VDPA_LOG(DEBUG, "The container_fd=%d, group_fd=%d.",
			device->vfio_container_fd, device->vfio_group_fd);

	ret = rte_pci_map_device(pci_dev);
	if (ret != 0)
		goto group_unbind;

	device->vfio_dev_fd = rte_intr_dev_fd_get(pci_dev->intr_handle);

	return 0;

group_unbind:
	rte_vfio_container_group_unbind(device->vfio_container_fd, device->iommu_group);
container_destroy:
	rte_vfio_container_destroy(device->vfio_container_fd);

	return -1;
}

static void
nfp_vdpa_vfio_teardown(struct nfp_vdpa_dev *device)
{
	rte_pci_unmap_device(device->pci_dev);
	rte_vfio_container_group_unbind(device->vfio_container_fd, device->iommu_group);
	rte_vfio_container_destroy(device->vfio_container_fd);
}

static int
nfp_vdpa_dma_do_unmap(struct rte_vhost_memory *mem,
		uint32_t times,
		int vfio_container_fd)
{
	uint32_t i;
	int ret = 0;
	struct rte_vhost_mem_region *region;

	for (i = 0; i < times; i++) {
		region = &mem->regions[i];

		ret = rte_vfio_container_dma_unmap(vfio_container_fd,
				region->host_user_addr, region->guest_phys_addr,
				region->size);
		if (ret < 0) {
			/* Here should not return, even error happened. */
			DRV_VDPA_LOG(ERR, "DMA unmap failed. Times: %u.", i);
		}
	}

	return ret;
}

static int
nfp_vdpa_dma_do_map(struct rte_vhost_memory *mem,
		uint32_t times,
		int vfio_container_fd)
{
	int ret;
	uint32_t i;
	struct rte_vhost_mem_region *region;

	for (i = 0; i < times; i++) {
		region = &mem->regions[i];

		ret = rte_vfio_container_dma_map(vfio_container_fd,
				region->host_user_addr, region->guest_phys_addr,
				region->size);
		if (ret < 0) {
			DRV_VDPA_LOG(ERR, "DMA map failed.");
			nfp_vdpa_dma_do_unmap(mem, i, vfio_container_fd);
			return ret;
		}
	}

	return 0;
}

static int
nfp_vdpa_dma_map(struct nfp_vdpa_dev *device,
		bool do_map)
{
	int ret;
	int vfio_container_fd;
	struct rte_vhost_memory *mem = NULL;

	ret = rte_vhost_get_mem_table(device->vid, &mem);
	if (ret < 0) {
		DRV_VDPA_LOG(ERR, "Failed to get memory layout.");
		return ret;
	}

	vfio_container_fd = device->vfio_container_fd;
	DRV_VDPA_LOG(DEBUG, "The vfio_container_fd %d.", vfio_container_fd);

	if (do_map)
		ret = nfp_vdpa_dma_do_map(mem, mem->nregions, vfio_container_fd);
	else
		ret = nfp_vdpa_dma_do_unmap(mem, mem->nregions, vfio_container_fd);

	free(mem);

	return ret;
}

static uint64_t
nfp_vdpa_qva_to_gpa(int vid,
		uint64_t qva)
{
	int ret;
	uint32_t i;
	uint64_t gpa = 0;
	struct rte_vhost_memory *mem = NULL;
	struct rte_vhost_mem_region *region;

	ret = rte_vhost_get_mem_table(vid, &mem);
	if (ret < 0) {
		DRV_VDPA_LOG(ERR, "Failed to get memory layout.");
		return gpa;
	}

	for (i = 0; i < mem->nregions; i++) {
		region = &mem->regions[i];

		if (qva >= region->host_user_addr &&
				qva < region->host_user_addr + region->size) {
			gpa = qva - region->host_user_addr + region->guest_phys_addr;
			break;
		}
	}

	free(mem);

	return gpa;
}

static void
nfp_vdpa_relay_vring_free(struct nfp_vdpa_dev *device,
		uint16_t vring_index)
{
	uint16_t i;
	uint64_t size;
	struct rte_vhost_vring vring;
	uint64_t m_vring_iova = NFP_VDPA_RELAY_VRING;

	for (i = 0; i < vring_index; i++) {
		rte_vhost_get_vhost_vring(device->vid, i, &vring);

		size = RTE_ALIGN_CEIL(vring_size(vring.size, rte_mem_page_size()),
				rte_mem_page_size());
		rte_vfio_container_dma_unmap(device->vfio_container_fd,
				(uint64_t)(uintptr_t)device->hw.m_vring[i].desc,
				m_vring_iova, size);

		rte_free(device->hw.m_vring[i].desc);
		m_vring_iova += size;
	}
}

static int
nfp_vdpa_relay_vring_alloc(struct nfp_vdpa_dev *device)
{
	int ret;
	uint16_t i;
	uint64_t size;
	void *vring_buf;
	uint64_t page_size;
	struct rte_vhost_vring vring;
	struct nfp_vdpa_hw *vdpa_hw = &device->hw;
	uint64_t m_vring_iova = NFP_VDPA_RELAY_VRING;

	page_size = rte_mem_page_size();

	for (i = 0; i < vdpa_hw->nr_vring; i++) {
		rte_vhost_get_vhost_vring(device->vid, i, &vring);

		size = RTE_ALIGN_CEIL(vring_size(vring.size, page_size), page_size);
		vring_buf = rte_zmalloc("nfp_vdpa_relay", size, page_size);
		if (vring_buf == NULL)
			goto vring_free_all;

		vring_init(&vdpa_hw->m_vring[i], vring.size, vring_buf, page_size);

		ret = rte_vfio_container_dma_map(device->vfio_container_fd,
				(uint64_t)(uintptr_t)vring_buf, m_vring_iova, size);
		if (ret != 0) {
			DRV_VDPA_LOG(ERR, "vDPA vring relay dma map failed.");
			goto vring_free_one;
		}

		m_vring_iova += size;
	}

	return 0;

vring_free_one:
	rte_free(device->hw.m_vring[i].desc);
vring_free_all:
	nfp_vdpa_relay_vring_free(device, i);

	return -ENOSPC;
}

static int
nfp_vdpa_start(struct nfp_vdpa_dev *device,
		bool relay)
{
	int ret;
	int vid;
	uint16_t i;
	uint64_t gpa;
	uint16_t size;
	struct rte_vhost_vring vring;
	struct nfp_vdpa_hw *vdpa_hw = &device->hw;
	uint64_t m_vring_iova = NFP_VDPA_RELAY_VRING;

	vid = device->vid;
	vdpa_hw->nr_vring = rte_vhost_get_vring_num(vid);

	ret = rte_vhost_get_negotiated_features(vid, &vdpa_hw->req_features);
	if (ret != 0)
		return ret;

	if (relay) {
		ret = nfp_vdpa_relay_vring_alloc(device);
		if (ret != 0)
			return ret;
	}

	for (i = 0; i < vdpa_hw->nr_vring; i++) {
		ret = rte_vhost_get_vhost_vring(vid, i, &vring);
		if (ret != 0)
			goto relay_vring_free;

		gpa = nfp_vdpa_qva_to_gpa(vid, (uint64_t)(uintptr_t)vring.desc);
		if (gpa == 0) {
			DRV_VDPA_LOG(ERR, "Fail to get GPA for descriptor ring.");
			goto relay_vring_free;
		}

		vdpa_hw->vring[i].desc = gpa;

		gpa = nfp_vdpa_qva_to_gpa(vid, (uint64_t)(uintptr_t)vring.avail);
		if (gpa == 0) {
			DRV_VDPA_LOG(ERR, "Fail to get GPA for available ring.");
			goto relay_vring_free;
		}

		vdpa_hw->vring[i].avail = gpa;

		/* Direct I/O for Tx queue, relay for Rx queue */
		if (relay && ((i & 1) == 0)) {
			vdpa_hw->vring[i].used = m_vring_iova +
					(char *)vdpa_hw->m_vring[i].used -
					(char *)vdpa_hw->m_vring[i].desc;

			ret = rte_vhost_get_vring_base(vid, i,
					&vdpa_hw->m_vring[i].avail->idx,
					&vdpa_hw->m_vring[i].used->idx);
			if (ret != 0)
				goto relay_vring_free;
		} else {
			gpa = nfp_vdpa_qva_to_gpa(vid, (uint64_t)(uintptr_t)vring.used);
			if (gpa == 0) {
				DRV_VDPA_LOG(ERR, "Fail to get GPA for used ring.");
				goto relay_vring_free;
			}

			vdpa_hw->vring[i].used = gpa;
		}

		vdpa_hw->vring[i].size = vring.size;

		if (relay) {
			size = RTE_ALIGN_CEIL(vring_size(vring.size,
					rte_mem_page_size()), rte_mem_page_size());
			m_vring_iova += size;
		}

		ret = rte_vhost_get_vring_base(vid, i,
				&vdpa_hw->vring[i].last_avail_idx,
				&vdpa_hw->vring[i].last_used_idx);
		if (ret != 0)
			goto relay_vring_free;
	}

	if (relay)
		return nfp_vdpa_relay_hw_start(&device->hw, vid);
	else
		return nfp_vdpa_hw_start(&device->hw, vid);

relay_vring_free:
	if (relay)
		nfp_vdpa_relay_vring_free(device, vdpa_hw->nr_vring);

	return -EFAULT;
}

static void
nfp_vdpa_update_used_ring(struct nfp_vdpa_dev *dev,
		uint16_t qid)
{
	rte_vdpa_relay_vring_used(dev->vid, qid, &dev->hw.m_vring[qid]);
	rte_vhost_vring_call(dev->vid, qid);
}

static void
nfp_vdpa_relay_stop(struct nfp_vdpa_dev *device)
{
	int vid;
	uint32_t i;
	uint64_t len;
	struct rte_vhost_vring vring;
	struct nfp_vdpa_hw *vdpa_hw = &device->hw;

	nfp_vdpa_hw_stop(vdpa_hw);

	vid = device->vid;
	for (i = 0; i < vdpa_hw->nr_vring; i++) {
		/* Synchronize remaining new used entries if any */
		if ((i & 1) == 0)
			nfp_vdpa_update_used_ring(device, i);

		rte_vhost_get_vhost_vring(vid, i, &vring);
		len = NFP_VDPA_USED_RING_LEN(vring.size);
		vdpa_hw->vring[i].last_avail_idx = vring.avail->idx;
		vdpa_hw->vring[i].last_used_idx = vring.used->idx;

		rte_vhost_set_vring_base(vid, i,
				vdpa_hw->vring[i].last_avail_idx,
				vdpa_hw->vring[i].last_used_idx);

		rte_vhost_log_used_vring(vid, i, 0, len);

		if (vring.used->idx != vring.avail->idx)
			rte_atomic_store_explicit(
					(unsigned short __rte_atomic *)&vring.used->idx,
					vring.avail->idx, rte_memory_order_release);
	}

	nfp_vdpa_relay_vring_free(device, vdpa_hw->nr_vring);
}

static void
nfp_vdpa_stop(struct nfp_vdpa_dev *device,
		bool relay)
{
	int vid;
	uint32_t i;
	struct nfp_vdpa_hw *vdpa_hw = &device->hw;

	nfp_vdpa_hw_stop(vdpa_hw);

	vid = device->vid;
	if (relay)
		nfp_vdpa_relay_stop(device);
	else
		for (i = 0; i < vdpa_hw->nr_vring; i++)
			rte_vhost_set_vring_base(vid, i,
					vdpa_hw->vring[i].last_avail_idx,
					vdpa_hw->vring[i].last_used_idx);

}

static int
nfp_vdpa_enable_vfio_intr(struct nfp_vdpa_dev *device,
		bool relay)
{
	int fd;
	int ret;
	uint16_t i;
	int *fd_ptr;
	uint16_t nr_vring;
	struct vfio_irq_set *irq_set;
	struct rte_vhost_vring vring;
	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];

	nr_vring = rte_vhost_get_vring_num(device->vid);

	irq_set = (struct vfio_irq_set *)irq_set_buf;
	irq_set->argsz = sizeof(irq_set_buf);
	irq_set->count = nr_vring + 1;
	irq_set->flags = VFIO_IRQ_SET_DATA_EVENTFD | VFIO_IRQ_SET_ACTION_TRIGGER;
	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
	irq_set->start = 0;

	fd_ptr = (int *)&irq_set->data;
	fd_ptr[RTE_INTR_VEC_ZERO_OFFSET] = rte_intr_fd_get(device->pci_dev->intr_handle);

	for (i = 0; i < nr_vring; i++)
		device->intr_fd[i] = -1;

	for (i = 0; i < nr_vring; i++) {
		rte_vhost_get_vhost_vring(device->vid, i, &vring);
		fd_ptr[RTE_INTR_VEC_RXTX_OFFSET + i] = vring.callfd;
	}

	if (relay) {
		for (i = 0; i < nr_vring; i += 2) {
			fd = eventfd(0, EFD_NONBLOCK | EFD_CLOEXEC);
			if (fd < 0) {
				DRV_VDPA_LOG(ERR, "Can't setup eventfd.");
				return -EINVAL;
			}

			device->intr_fd[i] = fd;
			fd_ptr[RTE_INTR_VEC_RXTX_OFFSET + i] = fd;
		}
	}

	ret = ioctl(device->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
	if (ret != 0) {
		DRV_VDPA_LOG(ERR, "Error enabling MSI-X interrupts.");
		return -EIO;
	}

	return 0;
}

static int
nfp_vdpa_disable_vfio_intr(struct nfp_vdpa_dev *device)
{
	int ret;
	struct vfio_irq_set *irq_set;
	char irq_set_buf[MSIX_IRQ_SET_BUF_LEN];

	irq_set = (struct vfio_irq_set *)irq_set_buf;
	irq_set->argsz = sizeof(irq_set_buf);
	irq_set->count = 0;
	irq_set->flags = VFIO_IRQ_SET_DATA_NONE | VFIO_IRQ_SET_ACTION_TRIGGER;
	irq_set->index = VFIO_PCI_MSIX_IRQ_INDEX;
	irq_set->start = 0;

	ret = ioctl(device->vfio_dev_fd, VFIO_DEVICE_SET_IRQS, irq_set);
	if (ret != 0) {
		DRV_VDPA_LOG(ERR, "Error disabling MSI-X interrupts.");
		return -EIO;
	}

	return 0;
}

static void
nfp_vdpa_read_kickfd(int kickfd)
{
	int bytes;
	uint64_t buf;

	for (;;) {
		bytes = read(kickfd, &buf, 8);
		if (bytes >= 0)
			break;

		if (errno != EINTR && errno != EWOULDBLOCK &&
				errno != EAGAIN) {
			DRV_VDPA_LOG(ERR, "Error reading kickfd.");
			break;
		}
	}
}

static int
nfp_vdpa_notify_epoll_ctl(uint32_t queue_num,
		struct nfp_vdpa_dev *device)
{
	int ret;
	uint32_t qid;

	for (qid = 0; qid < queue_num; qid++) {
		struct epoll_event ev;
		struct rte_vhost_vring vring;

		ev.events = EPOLLIN | EPOLLPRI;
		rte_vhost_get_vhost_vring(device->vid, qid, &vring);
		ev.data.u64 = qid | (uint64_t)vring.kickfd << 32;
		ret = epoll_ctl(device->epoll_fd, EPOLL_CTL_ADD, vring.kickfd, &ev);
		if (ret < 0) {
			DRV_VDPA_LOG(ERR, "Epoll add error for queue %d.", qid);
			return ret;
		}
	}

	return 0;
}

static int
nfp_vdpa_notify_epoll_wait(uint32_t queue_num,
		struct nfp_vdpa_dev *device)
{
	int i;
	int fds;
	int kickfd;
	uint32_t qid;
	struct epoll_event events[NFP_VDPA_MAX_QUEUES * 2];

	for (;;) {
		fds = epoll_wait(device->epoll_fd, events, queue_num, -1);
		if (fds < 0) {
			if (errno == EINTR)
				continue;

			DRV_VDPA_LOG(ERR, "Epoll wait fail.");
			return -EACCES;
		}

		for (i = 0; i < fds; i++) {
			qid = events[i].data.u32;
			kickfd = (uint32_t)(events[i].data.u64 >> 32);

			nfp_vdpa_read_kickfd(kickfd);
			nfp_vdpa_notify_queue(&device->hw, qid);
		}
	}

	return 0;
}

static uint32_t
nfp_vdpa_notify_relay(void *arg)
{
	int ret;
	int epoll_fd;
	uint32_t queue_num;
	struct nfp_vdpa_dev *device = arg;

	epoll_fd = epoll_create(NFP_VDPA_MAX_QUEUES * 2);
	if (epoll_fd < 0) {
		DRV_VDPA_LOG(ERR, "Failed to create epoll instance.");
		return 1;
	}

	device->epoll_fd = epoll_fd;

	queue_num = rte_vhost_get_vring_num(device->vid);

	ret = nfp_vdpa_notify_epoll_ctl(queue_num, device);
	if (ret != 0)
		goto notify_exit;

	ret = nfp_vdpa_notify_epoll_wait(queue_num, device);
	if (ret != 0)
		goto notify_exit;

	return 0;

notify_exit:
	close(device->epoll_fd);
	device->epoll_fd = -1;

	return 1;
}

static int
nfp_vdpa_setup_notify_relay(struct nfp_vdpa_dev *device)
{
	int ret;
	char name[RTE_THREAD_INTERNAL_NAME_SIZE];

	snprintf(name, sizeof(name), "nfp-noti%d", device->vid);
	ret = rte_thread_create_internal_control(&device->tid, name,
			nfp_vdpa_notify_relay, (void *)device);
	if (ret != 0) {
		DRV_VDPA_LOG(ERR, "Failed to create notify relay pthread.");
		return -1;
	}

	return 0;
}

static void
nfp_vdpa_unset_notify_relay(struct nfp_vdpa_dev *device)
{
	if (device->tid.opaque_id != 0) {
		pthread_cancel((pthread_t)device->tid.opaque_id);
		rte_thread_join(device->tid, NULL);
		device->tid.opaque_id = 0;
	}

	if (device->epoll_fd >= 0) {
		close(device->epoll_fd);
		device->epoll_fd = -1;
	}
}

static int
update_datapath(struct nfp_vdpa_dev *device)
{
	int ret;

	rte_spinlock_lock(&device->lock);

	if ((rte_atomic_load_explicit(&device->running, rte_memory_order_relaxed) == 0) &&
			(rte_atomic_load_explicit(&device->started,
					rte_memory_order_relaxed) != 0) &&
			(rte_atomic_load_explicit(&device->dev_attached,
					rte_memory_order_relaxed) != 0)) {
		ret = nfp_vdpa_dma_map(device, true);
		if (ret != 0)
			goto unlock_exit;

		ret = nfp_vdpa_enable_vfio_intr(device, false);
		if (ret != 0)
			goto dma_map_rollback;

		ret = nfp_vdpa_start(device, false);
		if (ret != 0)
			goto disable_vfio_intr;

		ret = nfp_vdpa_setup_notify_relay(device);
		if (ret != 0)
			goto vdpa_stop;

		rte_atomic_store_explicit(&device->running, 1, rte_memory_order_relaxed);
	} else if ((rte_atomic_load_explicit(&device->running, rte_memory_order_relaxed) != 0) &&
			((rte_atomic_load_explicit(&device->started,
					rte_memory_order_relaxed) != 0) ||
			(rte_atomic_load_explicit(&device->dev_attached,
					rte_memory_order_relaxed) != 0))) {
		nfp_vdpa_unset_notify_relay(device);

		nfp_vdpa_stop(device, false);

		ret = nfp_vdpa_disable_vfio_intr(device);
		if (ret != 0)
			goto unlock_exit;

		ret = nfp_vdpa_dma_map(device, false);
		if (ret != 0)
			goto unlock_exit;

		rte_atomic_store_explicit(&device->running, 0, rte_memory_order_relaxed);
	}

	rte_spinlock_unlock(&device->lock);
	return 0;

vdpa_stop:
	nfp_vdpa_stop(device, false);
disable_vfio_intr:
	nfp_vdpa_disable_vfio_intr(device);
dma_map_rollback:
	nfp_vdpa_dma_map(device, false);
unlock_exit:
	rte_spinlock_unlock(&device->lock);
	return ret;
}

static int
nfp_vdpa_vring_epoll_ctl(uint32_t queue_num,
		struct nfp_vdpa_dev *device)
{
	int ret;
	uint32_t qid;
	struct epoll_event ev;
	struct rte_vhost_vring vring;

	for (qid = 0; qid < queue_num; qid++) {
		ev.events = EPOLLIN | EPOLLPRI;
		rte_vhost_get_vhost_vring(device->vid, qid, &vring);
		ev.data.u64 = qid << 1 | (uint64_t)vring.kickfd << 32;
		ret = epoll_ctl(device->epoll_fd, EPOLL_CTL_ADD, vring.kickfd, &ev);
		if (ret < 0) {
			DRV_VDPA_LOG(ERR, "Epoll add error for queue %u.", qid);
			return ret;
		}
	}

	/* vDPA driver interrupt */
	for (qid = 0; qid < queue_num; qid += 2) {
		ev.events = EPOLLIN | EPOLLPRI;
		/* Leave a flag to mark it's for interrupt */
		ev.data.u64 = EPOLL_DATA_INTR | qid << 1 |
				(uint64_t)device->intr_fd[qid] << 32;
		ret = epoll_ctl(device->epoll_fd, EPOLL_CTL_ADD,
				device->intr_fd[qid], &ev);
		if (ret < 0) {
			DRV_VDPA_LOG(ERR, "Epoll add error for queue %u.", qid);
			return ret;
		}

		nfp_vdpa_update_used_ring(device, qid);
	}

	return 0;
}

static int
nfp_vdpa_vring_epoll_wait(uint32_t queue_num,
		struct nfp_vdpa_dev *device)
{
	int i;
	int fds;
	int kickfd;
	uint32_t qid;
	struct epoll_event events[NFP_VDPA_MAX_QUEUES * 2];

	for (;;) {
		fds = epoll_wait(device->epoll_fd, events, queue_num * 2, -1);
		if (fds < 0) {
			if (errno == EINTR)
				continue;

			DRV_VDPA_LOG(ERR, "Epoll wait fail.");
			return -EACCES;
		}

		for (i = 0; i < fds; i++) {
			qid = events[i].data.u32 >> 1;
			kickfd = (uint32_t)(events[i].data.u64 >> 32);

			nfp_vdpa_read_kickfd(kickfd);
			if ((events[i].data.u32 & EPOLL_DATA_INTR) != 0) {
				nfp_vdpa_update_used_ring(device, qid);
				nfp_vdpa_irq_unmask(&device->hw);
			} else {
				nfp_vdpa_notify_queue(&device->hw, qid);
			}
		}
	}

	return 0;
}

static uint32_t
nfp_vdpa_vring_relay(void *arg)
{
	int ret;
	int epoll_fd;
	uint16_t queue_id;
	uint32_t queue_num;
	struct nfp_vdpa_dev *device = arg;

	epoll_fd = epoll_create(NFP_VDPA_MAX_QUEUES * 2);
	if (epoll_fd < 0) {
		DRV_VDPA_LOG(ERR, "failed to create epoll instance.");
		return 1;
	}

	device->epoll_fd = epoll_fd;

	queue_num = rte_vhost_get_vring_num(device->vid);

	ret = nfp_vdpa_vring_epoll_ctl(queue_num, device);
	if (ret != 0)
		goto notify_exit;

	/* Start relay with a first kick */
	for (queue_id = 0; queue_id < queue_num; queue_id++)
		nfp_vdpa_notify_queue(&device->hw, queue_id);

	ret = nfp_vdpa_vring_epoll_wait(queue_num, device);
	if (ret != 0)
		goto notify_exit;

	return 0;

notify_exit:
	close(device->epoll_fd);
	device->epoll_fd = -1;

	return 1;
}

static int
nfp_vdpa_setup_vring_relay(struct nfp_vdpa_dev *device)
{
	int ret;
	char name[RTE_THREAD_INTERNAL_NAME_SIZE];

	snprintf(name, sizeof(name), "nfp_vring%d", device->vid);
	ret = rte_thread_create_internal_control(&device->tid, name,
			nfp_vdpa_vring_relay, (void *)device);
	if (ret != 0) {
		DRV_VDPA_LOG(ERR, "Failed to create vring relay pthread.");
		return -EPERM;
	}

	return 0;
}

static int
nfp_vdpa_sw_fallback(struct nfp_vdpa_dev *device)
{
	int ret;
	int vid = device->vid;

	/* Stop the direct IO data path */
	nfp_vdpa_unset_notify_relay(device);
	nfp_vdpa_disable_vfio_intr(device);

	ret = rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, false);
	if ((ret != 0) && (ret != -ENOTSUP)) {
		DRV_VDPA_LOG(ERR, "Unset the host notifier failed.");
		goto error;
	}

	/* Setup interrupt for vring relay */
	ret = nfp_vdpa_enable_vfio_intr(device, true);
	if (ret != 0)
		goto error;

	/* Config the VF */
	ret = nfp_vdpa_start(device, true);
	if (ret != 0)
		goto unset_intr;

	/* Setup vring relay thread */
	ret = nfp_vdpa_setup_vring_relay(device);
	if (ret != 0)
		goto stop_vf;

	device->hw.sw_fallback_running = true;

	return 0;

stop_vf:
	nfp_vdpa_stop(device, true);
unset_intr:
	nfp_vdpa_disable_vfio_intr(device);
error:
	return ret;
}

static int
nfp_vdpa_dev_config(int vid)
{
	int ret;
	struct nfp_vdpa_dev *device;
	struct rte_vdpa_device *vdev;
	struct nfp_vdpa_dev_node *node;

	vdev = rte_vhost_get_vdpa_device(vid);
	node = nfp_vdpa_find_node_by_vdev(vdev);
	if (node == NULL) {
		DRV_VDPA_LOG(ERR, "Invalid vDPA device: %p.", vdev);
		return -ENODEV;
	}

	device = node->device;
	device->vid = vid;
	rte_atomic_store_explicit(&device->dev_attached, 1, rte_memory_order_relaxed);
	update_datapath(device);

	ret = rte_vhost_host_notifier_ctrl(vid, RTE_VHOST_QUEUE_ALL, true);
	if (ret != 0)
		DRV_VDPA_LOG(INFO, "vDPA (%s): software relay is used.",
				vdev->device->name);

	return 0;
}

static int
nfp_vdpa_dev_close(int vid)
{
	struct nfp_vdpa_dev *device;
	struct rte_vdpa_device *vdev;
	struct nfp_vdpa_dev_node *node;

	vdev = rte_vhost_get_vdpa_device(vid);
	node = nfp_vdpa_find_node_by_vdev(vdev);
	if (node == NULL) {
		DRV_VDPA_LOG(ERR, "Invalid vDPA device: %p.", vdev);
		return -ENODEV;
	}

	device = node->device;
	if (device->hw.sw_fallback_running) {
		/* Reset VF */
		nfp_vdpa_stop(device, true);

		/* Remove interrupt setting */
		nfp_vdpa_disable_vfio_intr(device);

		/* Unset DMA map for guest memory */
		nfp_vdpa_dma_map(device, false);

		device->hw.sw_fallback_running = false;

		rte_atomic_store_explicit(&device->dev_attached, 0,
				rte_memory_order_relaxed);
		rte_atomic_store_explicit(&device->running, 0,
				rte_memory_order_relaxed);
	} else {
		rte_atomic_store_explicit(&device->dev_attached, 0,
				rte_memory_order_relaxed);
		update_datapath(device);
	}

	return 0;
}

static int
nfp_vdpa_get_vfio_group_fd(int vid)
{
	struct rte_vdpa_device *vdev;
	struct nfp_vdpa_dev_node *node;

	vdev = rte_vhost_get_vdpa_device(vid);
	node = nfp_vdpa_find_node_by_vdev(vdev);
	if (node == NULL) {
		DRV_VDPA_LOG(ERR, "Invalid vDPA device: %p.", vdev);
		return -ENODEV;
	}

	return node->device->vfio_group_fd;
}

static int
nfp_vdpa_get_vfio_device_fd(int vid)
{
	struct rte_vdpa_device *vdev;
	struct nfp_vdpa_dev_node *node;

	vdev = rte_vhost_get_vdpa_device(vid);
	node = nfp_vdpa_find_node_by_vdev(vdev);
	if (node == NULL) {
		DRV_VDPA_LOG(ERR, "Invalid vDPA device: %p.", vdev);
		return -ENODEV;
	}

	return node->device->vfio_dev_fd;
}

static int
nfp_vdpa_get_notify_area(int vid,
		int qid,
		uint64_t *offset,
		uint64_t *size)
{
	int ret;
	struct nfp_vdpa_dev *device;
	struct rte_vdpa_device *vdev;
	struct nfp_vdpa_dev_node *node;
	struct vfio_region_info region = {
		.argsz = sizeof(region)
	};

	vdev = rte_vhost_get_vdpa_device(vid);
	node = nfp_vdpa_find_node_by_vdev(vdev);
	if (node == NULL) {
		DRV_VDPA_LOG(ERR,  "Invalid vDPA device: %p", vdev);
		return -ENODEV;
	}

	device = node->device;
	region.index = device->hw.notify_region;

	ret = ioctl(device->vfio_dev_fd, VFIO_DEVICE_GET_REGION_INFO, &region);
	if (ret != 0) {
		DRV_VDPA_LOG(ERR, "Get not get device region info.");
		return -EIO;
	}

	*offset = nfp_vdpa_get_queue_notify_offset(&device->hw, qid) + region.offset;
	*size = NFP_VDPA_NOTIFY_ADDR_INTERVAL;

	return 0;
}

static int
nfp_vdpa_get_queue_num(struct rte_vdpa_device *vdev,
		uint32_t *queue_num)
{
	struct nfp_vdpa_dev_node *node;

	node = nfp_vdpa_find_node_by_vdev(vdev);
	if (node == NULL) {
		DRV_VDPA_LOG(ERR, "Invalid vDPA device: %p.", vdev);
		return -ENODEV;
	}

	*queue_num = node->device->max_queues;

	return 0;
}

static int
nfp_vdpa_get_vdpa_features(struct rte_vdpa_device *vdev,
		uint64_t *features)
{
	struct nfp_vdpa_dev_node *node;

	node = nfp_vdpa_find_node_by_vdev(vdev);
	if (node == NULL) {
		DRV_VDPA_LOG(ERR,  "Invalid vDPA device: %p", vdev);
		return -ENODEV;
	}

	*features = node->device->hw.features;

	return 0;
}

static int
nfp_vdpa_get_protocol_features(struct rte_vdpa_device *vdev __rte_unused,
		uint64_t *features)
{
	*features = 1ULL << VHOST_USER_PROTOCOL_F_LOG_SHMFD |
			1ULL << VHOST_USER_PROTOCOL_F_REPLY_ACK |
			1ULL << VHOST_USER_PROTOCOL_F_BACKEND_REQ |
			1ULL << VHOST_USER_PROTOCOL_F_BACKEND_SEND_FD |
			1ULL << VHOST_USER_PROTOCOL_F_HOST_NOTIFIER;

	return 0;
}

static int
nfp_vdpa_set_features(int32_t vid)
{
	int ret;
	uint64_t features = 0;
	struct nfp_vdpa_dev *device;
	struct rte_vdpa_device *vdev;
	struct nfp_vdpa_dev_node *node;

	DRV_VDPA_LOG(DEBUG, "Start vid=%d.", vid);

	vdev = rte_vhost_get_vdpa_device(vid);
	node = nfp_vdpa_find_node_by_vdev(vdev);
	if (node == NULL) {
		DRV_VDPA_LOG(ERR, "Invalid vDPA device: %p.", vdev);
		return -ENODEV;
	}

	rte_vhost_get_negotiated_features(vid, &features);

	if (RTE_VHOST_NEED_LOG(features) == 0)
		return 0;

	device = node->device;
	if (device->hw.sw_lm) {
		ret = nfp_vdpa_sw_fallback(device);
		if (ret != 0) {
			DRV_VDPA_LOG(ERR, "Software fallback start failed.");
			return -1;
		}
	}

	return 0;
}

static int
nfp_vdpa_set_vring_state(int vid,
		int vring,
		int state)
{
	DRV_VDPA_LOG(DEBUG, "Start vid=%d, vring=%d, state=%d.", vid, vring, state);
	return 0;
}

struct rte_vdpa_dev_ops nfp_vdpa_ops = {
	.get_queue_num = nfp_vdpa_get_queue_num,
	.get_features = nfp_vdpa_get_vdpa_features,
	.get_protocol_features = nfp_vdpa_get_protocol_features,
	.dev_conf = nfp_vdpa_dev_config,
	.dev_close = nfp_vdpa_dev_close,
	.set_vring_state = nfp_vdpa_set_vring_state,
	.set_features = nfp_vdpa_set_features,
	.get_vfio_group_fd = nfp_vdpa_get_vfio_group_fd,
	.get_vfio_device_fd = nfp_vdpa_get_vfio_device_fd,
	.get_notify_area = nfp_vdpa_get_notify_area,
};

static int
nfp_vdpa_pci_probe(struct rte_pci_device *pci_dev)
{
	int ret;
	struct nfp_vdpa_dev *device;
	struct nfp_vdpa_dev_node *node;

	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
		return 0;

	node = calloc(1, sizeof(*node));
	if (node == NULL)
		return -ENOMEM;

	device = calloc(1, sizeof(*device));
	if (device == NULL)
		goto free_node;

	device->pci_dev = pci_dev;

	ret = nfp_vdpa_vfio_setup(device);
	if (ret != 0)
		goto free_device;

	ret = nfp_vdpa_hw_init(&device->hw, pci_dev);
	if (ret != 0)
		goto vfio_teardown;

	device->max_queues = NFP_VDPA_MAX_QUEUES;

	device->vdev = rte_vdpa_register_device(&pci_dev->device, &nfp_vdpa_ops);
	if (device->vdev == NULL) {
		DRV_VDPA_LOG(ERR, "Failed to register device %s.", pci_dev->name);
		goto vfio_teardown;
	}

	node->device = device;
	pthread_mutex_lock(&vdpa_list_lock);
	TAILQ_INSERT_TAIL(&vdpa_dev_list, node, next);
	pthread_mutex_unlock(&vdpa_list_lock);

	rte_spinlock_init(&device->lock);
	rte_atomic_store_explicit(&device->started, 1, rte_memory_order_relaxed);
	update_datapath(device);

	return 0;

vfio_teardown:
	nfp_vdpa_vfio_teardown(device);
free_device:
	free(device);
free_node:
	free(node);

	return -1;
}

static int
nfp_vdpa_pci_remove(struct rte_pci_device *pci_dev)
{
	struct nfp_vdpa_dev *device;
	struct nfp_vdpa_dev_node *node;

	if (rte_eal_process_type() != RTE_PROC_PRIMARY)
		return 0;

	node = nfp_vdpa_find_node_by_pdev(pci_dev);
	if (node == NULL) {
		DRV_VDPA_LOG(ERR, "Invalid device: %s.", pci_dev->name);
		return -ENODEV;
	}

	device = node->device;

	rte_atomic_store_explicit(&device->started, 0, rte_memory_order_relaxed);
	update_datapath(device);

	pthread_mutex_lock(&vdpa_list_lock);
	TAILQ_REMOVE(&vdpa_dev_list, node, next);
	pthread_mutex_unlock(&vdpa_list_lock);

	rte_vdpa_unregister_device(device->vdev);
	nfp_vdpa_vfio_teardown(device);

	free(device);
	free(node);

	return 0;
}

static const struct rte_pci_id pci_id_nfp_vdpa_map[] = {
	{
		RTE_PCI_DEVICE(PCI_VENDOR_ID_NETRONOME,
				PCI_DEVICE_ID_NFP6000_VF_NIC)
	},
	{
		.vendor_id = 0,
	},
};

static struct nfp_class_driver nfp_vdpa = {
	.drv_class = NFP_CLASS_VDPA,
	.name = RTE_STR(NFP_VDPA_DRIVER_NAME),
	.id_table = pci_id_nfp_vdpa_map,
	.drv_flags =  RTE_PCI_DRV_NEED_MAPPING | RTE_PCI_DRV_INTR_LSC,
	.probe = nfp_vdpa_pci_probe,
	.remove = nfp_vdpa_pci_remove,
};

RTE_INIT(nfp_vdpa_init)
{
	nfp_class_driver_register(&nfp_vdpa);
}

RTE_PMD_REGISTER_PCI_TABLE(NFP_VDPA_DRIVER_NAME, pci_id_nfp_vdpa_map);
RTE_PMD_REGISTER_KMOD_DEP(NFP_VDPA_DRIVER_NAME, "* vfio-pci");