xref: /freebsd-src/sys/dev/virtio/virtqueue.c (revision d1bdc2821fcd416ab9b238580386eb605a6128d0)
110b59a9bSPeter Grehan /*-
24d846d26SWarner Losh  * SPDX-License-Identifier: BSD-2-Clause
3718cf2ccSPedro F. Giffuni  *
4abd6790cSBryan Venteicher  * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
510b59a9bSPeter Grehan  * All rights reserved.
610b59a9bSPeter Grehan  *
710b59a9bSPeter Grehan  * Redistribution and use in source and binary forms, with or without
810b59a9bSPeter Grehan  * modification, are permitted provided that the following conditions
910b59a9bSPeter Grehan  * are met:
1010b59a9bSPeter Grehan  * 1. Redistributions of source code must retain the above copyright
1110b59a9bSPeter Grehan  *    notice unmodified, this list of conditions, and the following
1210b59a9bSPeter Grehan  *    disclaimer.
1310b59a9bSPeter Grehan  * 2. Redistributions in binary form must reproduce the above copyright
1410b59a9bSPeter Grehan  *    notice, this list of conditions and the following disclaimer in the
1510b59a9bSPeter Grehan  *    documentation and/or other materials provided with the distribution.
1610b59a9bSPeter Grehan  *
1710b59a9bSPeter Grehan  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
1810b59a9bSPeter Grehan  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
1910b59a9bSPeter Grehan  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
2010b59a9bSPeter Grehan  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
2110b59a9bSPeter Grehan  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
2210b59a9bSPeter Grehan  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
2310b59a9bSPeter Grehan  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
2410b59a9bSPeter Grehan  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
2510b59a9bSPeter Grehan  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
2610b59a9bSPeter Grehan  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2710b59a9bSPeter Grehan  */
2810b59a9bSPeter Grehan 
2910b59a9bSPeter Grehan /*
3010b59a9bSPeter Grehan  * Implements the virtqueue interface as basically described
3110b59a9bSPeter Grehan  * in the original VirtIO paper.
3210b59a9bSPeter Grehan  */
3310b59a9bSPeter Grehan 
3410b59a9bSPeter Grehan #include <sys/param.h>
3510b59a9bSPeter Grehan #include <sys/systm.h>
3610b59a9bSPeter Grehan #include <sys/kernel.h>
3710b59a9bSPeter Grehan #include <sys/malloc.h>
38fd80208aSStephen J. Kiernan #include <sys/sdt.h>
3910b59a9bSPeter Grehan #include <sys/sglist.h>
4010b59a9bSPeter Grehan #include <vm/vm.h>
4110b59a9bSPeter Grehan #include <vm/pmap.h>
4210b59a9bSPeter Grehan 
4310b59a9bSPeter Grehan #include <machine/cpu.h>
4410b59a9bSPeter Grehan #include <machine/bus.h>
4510b59a9bSPeter Grehan #include <machine/atomic.h>
4610b59a9bSPeter Grehan #include <machine/resource.h>
4710b59a9bSPeter Grehan #include <sys/bus.h>
4810b59a9bSPeter Grehan #include <sys/rman.h>
4910b59a9bSPeter Grehan 
5010b59a9bSPeter Grehan #include <dev/virtio/virtio.h>
5110b59a9bSPeter Grehan #include <dev/virtio/virtqueue.h>
5210b59a9bSPeter Grehan #include <dev/virtio/virtio_ring.h>
5310b59a9bSPeter Grehan 
5410b59a9bSPeter Grehan #include "virtio_bus_if.h"
5510b59a9bSPeter Grehan 
5610b59a9bSPeter Grehan struct virtqueue {
5710b59a9bSPeter Grehan 	device_t		 vq_dev;
5810b59a9bSPeter Grehan 	uint16_t		 vq_queue_index;
5910b59a9bSPeter Grehan 	uint16_t		 vq_nentries;
6010b59a9bSPeter Grehan 	uint32_t		 vq_flags;
61fbe0c4f4SBryan Venteicher #define	VIRTQUEUE_FLAG_MODERN	 0x0001
62fbe0c4f4SBryan Venteicher #define	VIRTQUEUE_FLAG_INDIRECT	 0x0002
63fbe0c4f4SBryan Venteicher #define	VIRTQUEUE_FLAG_EVENT_IDX 0x0004
6410b59a9bSPeter Grehan 
6510b59a9bSPeter Grehan 	int			 vq_max_indirect_size;
66fbe0c4f4SBryan Venteicher 	bus_size_t		 vq_notify_offset;
6710b59a9bSPeter Grehan 	virtqueue_intr_t	*vq_intrhand;
6810b59a9bSPeter Grehan 	void			*vq_intrhand_arg;
6910b59a9bSPeter Grehan 
7010b59a9bSPeter Grehan 	struct vring		 vq_ring;
7110b59a9bSPeter Grehan 	uint16_t		 vq_free_cnt;
7210b59a9bSPeter Grehan 	uint16_t		 vq_queued_cnt;
7310b59a9bSPeter Grehan 	/*
7410b59a9bSPeter Grehan 	 * Head of the free chain in the descriptor table. If
7510b59a9bSPeter Grehan 	 * there are no free descriptors, this will be set to
7610b59a9bSPeter Grehan 	 * VQ_RING_DESC_CHAIN_END.
7710b59a9bSPeter Grehan 	 */
7810b59a9bSPeter Grehan 	uint16_t		 vq_desc_head_idx;
7910b59a9bSPeter Grehan 	/*
8010b59a9bSPeter Grehan 	 * Last consumed descriptor in the used table,
8110b59a9bSPeter Grehan 	 * trails vq_ring.used->idx.
8210b59a9bSPeter Grehan 	 */
8310b59a9bSPeter Grehan 	uint16_t		 vq_used_cons_idx;
8410b59a9bSPeter Grehan 
85fbe0c4f4SBryan Venteicher 	void			*vq_ring_mem;
86fbe0c4f4SBryan Venteicher 	int			 vq_indirect_mem_size;
87fbe0c4f4SBryan Venteicher 	int			 vq_alignment;
88fbe0c4f4SBryan Venteicher 	int			 vq_ring_size;
89fbe0c4f4SBryan Venteicher 	char			 vq_name[VIRTQUEUE_MAX_NAME_SZ];
90fbe0c4f4SBryan Venteicher 
9110b59a9bSPeter Grehan 	struct vq_desc_extra {
9210b59a9bSPeter Grehan 		void		  *cookie;
9310b59a9bSPeter Grehan 		struct vring_desc *indirect;
9410b59a9bSPeter Grehan 		vm_paddr_t	   indirect_paddr;
9510b59a9bSPeter Grehan 		uint16_t	   ndescs;
9610b59a9bSPeter Grehan 	} vq_descx[0];
9710b59a9bSPeter Grehan };
9810b59a9bSPeter Grehan 
9910b59a9bSPeter Grehan /*
10010b59a9bSPeter Grehan  * The maximum virtqueue size is 2^15. Use that value as the end of
10110b59a9bSPeter Grehan  * descriptor chain terminator since it will never be a valid index
10210b59a9bSPeter Grehan  * in the descriptor table. This is used to verify we are correctly
10310b59a9bSPeter Grehan  * handling vq_free_cnt.
10410b59a9bSPeter Grehan  */
10510b59a9bSPeter Grehan #define VQ_RING_DESC_CHAIN_END 32768
10610b59a9bSPeter Grehan 
10710b59a9bSPeter Grehan #define VQASSERT(_vq, _exp, _msg, ...)				\
10810b59a9bSPeter Grehan     KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name,	\
10910b59a9bSPeter Grehan 	##__VA_ARGS__))
11010b59a9bSPeter Grehan 
11110b59a9bSPeter Grehan #define VQ_RING_ASSERT_VALID_IDX(_vq, _idx)			\
11210b59a9bSPeter Grehan     VQASSERT((_vq), (_idx) < (_vq)->vq_nentries,		\
11310b59a9bSPeter Grehan 	"invalid ring index: %d, max: %d", (_idx),		\
11410b59a9bSPeter Grehan 	(_vq)->vq_nentries)
11510b59a9bSPeter Grehan 
11610b59a9bSPeter Grehan #define VQ_RING_ASSERT_CHAIN_TERM(_vq)				\
11710b59a9bSPeter Grehan     VQASSERT((_vq), (_vq)->vq_desc_head_idx ==			\
11810b59a9bSPeter Grehan 	VQ_RING_DESC_CHAIN_END,	"full ring terminated "		\
11910b59a9bSPeter Grehan 	"incorrectly: head idx: %d", (_vq)->vq_desc_head_idx)
12010b59a9bSPeter Grehan 
12110b59a9bSPeter Grehan static int	virtqueue_init_indirect(struct virtqueue *vq, int);
12210b59a9bSPeter Grehan static void	virtqueue_free_indirect(struct virtqueue *vq);
12310b59a9bSPeter Grehan static void	virtqueue_init_indirect_list(struct virtqueue *,
12410b59a9bSPeter Grehan 		    struct vring_desc *);
12510b59a9bSPeter Grehan 
12610b59a9bSPeter Grehan static void	vq_ring_init(struct virtqueue *);
12710b59a9bSPeter Grehan static void	vq_ring_update_avail(struct virtqueue *, uint16_t);
12810b59a9bSPeter Grehan static uint16_t	vq_ring_enqueue_segments(struct virtqueue *,
12910b59a9bSPeter Grehan 		    struct vring_desc *, uint16_t, struct sglist *, int, int);
130ccb576a8SMina Galić static bool	vq_ring_use_indirect(struct virtqueue *, int);
13110b59a9bSPeter Grehan static void	vq_ring_enqueue_indirect(struct virtqueue *, void *,
13210b59a9bSPeter Grehan 		    struct sglist *, int, int);
133310dacd0SPeter Grehan static int	vq_ring_enable_interrupt(struct virtqueue *, uint16_t);
134b8a58707SPeter Grehan static int	vq_ring_must_notify_host(struct virtqueue *);
135b8a58707SPeter Grehan static void	vq_ring_notify_host(struct virtqueue *);
13610b59a9bSPeter Grehan static void	vq_ring_free_chain(struct virtqueue *, uint16_t);
13710b59a9bSPeter Grehan 
138fd80208aSStephen J. Kiernan SDT_PROVIDER_DEFINE(virtqueue);
139fd80208aSStephen J. Kiernan SDT_PROBE_DEFINE6(virtqueue, , enqueue_segments, entry, "struct virtqueue *",
140fd80208aSStephen J. Kiernan     "struct vring_desc *", "uint16_t", "struct sglist *", "int", "int");
141fd80208aSStephen J. Kiernan SDT_PROBE_DEFINE1(virtqueue, , enqueue_segments, return, "uint16_t");
142fd80208aSStephen J. Kiernan 
143fbe0c4f4SBryan Venteicher #define vq_modern(_vq) 		(((_vq)->vq_flags & VIRTQUEUE_FLAG_MODERN) != 0)
144fbe0c4f4SBryan Venteicher #define vq_htog16(_vq, _val) 	virtio_htog16(vq_modern(_vq), _val)
145fbe0c4f4SBryan Venteicher #define vq_htog32(_vq, _val) 	virtio_htog32(vq_modern(_vq), _val)
146fbe0c4f4SBryan Venteicher #define vq_htog64(_vq, _val) 	virtio_htog64(vq_modern(_vq), _val)
147fbe0c4f4SBryan Venteicher #define vq_gtoh16(_vq, _val) 	virtio_gtoh16(vq_modern(_vq), _val)
148fbe0c4f4SBryan Venteicher #define vq_gtoh32(_vq, _val) 	virtio_gtoh32(vq_modern(_vq), _val)
149fbe0c4f4SBryan Venteicher #define vq_gtoh64(_vq, _val) 	virtio_gtoh64(vq_modern(_vq), _val)
15010b59a9bSPeter Grehan 
15110b59a9bSPeter Grehan int
1529da9560cSBryan Venteicher virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size,
1539da9560cSBryan Venteicher     bus_size_t notify_offset, int align, vm_paddr_t highaddr,
1549da9560cSBryan Venteicher     struct vq_alloc_info *info, struct virtqueue **vqp)
15510b59a9bSPeter Grehan {
15610b59a9bSPeter Grehan 	struct virtqueue *vq;
15710b59a9bSPeter Grehan 	int error;
15810b59a9bSPeter Grehan 
15910b59a9bSPeter Grehan 	*vqp = NULL;
16010b59a9bSPeter Grehan 	error = 0;
16110b59a9bSPeter Grehan 
16210b59a9bSPeter Grehan 	if (size == 0) {
16310b59a9bSPeter Grehan 		device_printf(dev,
16410b59a9bSPeter Grehan 		    "virtqueue %d (%s) does not exist (size is zero)\n",
16510b59a9bSPeter Grehan 		    queue, info->vqai_name);
16610b59a9bSPeter Grehan 		return (ENODEV);
16710b59a9bSPeter Grehan 	} else if (!powerof2(size)) {
16810b59a9bSPeter Grehan 		device_printf(dev,
16910b59a9bSPeter Grehan 		    "virtqueue %d (%s) size is not a power of 2: %d\n",
17010b59a9bSPeter Grehan 		    queue, info->vqai_name, size);
17110b59a9bSPeter Grehan 		return (ENXIO);
17210b59a9bSPeter Grehan 	} else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) {
17310b59a9bSPeter Grehan 		device_printf(dev, "virtqueue %d (%s) requested too many "
17410b59a9bSPeter Grehan 		    "indirect descriptors: %d, max %d\n",
17510b59a9bSPeter Grehan 		    queue, info->vqai_name, info->vqai_maxindirsz,
17610b59a9bSPeter Grehan 		    VIRTIO_MAX_INDIRECT);
17710b59a9bSPeter Grehan 		return (EINVAL);
17810b59a9bSPeter Grehan 	}
17910b59a9bSPeter Grehan 
18010b59a9bSPeter Grehan 	vq = malloc(sizeof(struct virtqueue) +
18110b59a9bSPeter Grehan 	    size * sizeof(struct vq_desc_extra), M_DEVBUF, M_NOWAIT | M_ZERO);
18210b59a9bSPeter Grehan 	if (vq == NULL) {
18310b59a9bSPeter Grehan 		device_printf(dev, "cannot allocate virtqueue\n");
18410b59a9bSPeter Grehan 		return (ENOMEM);
18510b59a9bSPeter Grehan 	}
18610b59a9bSPeter Grehan 
18710b59a9bSPeter Grehan 	vq->vq_dev = dev;
18810b59a9bSPeter Grehan 	strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name));
18910b59a9bSPeter Grehan 	vq->vq_queue_index = queue;
1909da9560cSBryan Venteicher 	vq->vq_notify_offset = notify_offset;
19110b59a9bSPeter Grehan 	vq->vq_alignment = align;
19210b59a9bSPeter Grehan 	vq->vq_nentries = size;
19310b59a9bSPeter Grehan 	vq->vq_free_cnt = size;
19410b59a9bSPeter Grehan 	vq->vq_intrhand = info->vqai_intr;
19510b59a9bSPeter Grehan 	vq->vq_intrhand_arg = info->vqai_intr_arg;
19610b59a9bSPeter Grehan 
197fbe0c4f4SBryan Venteicher 	if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_F_VERSION_1) != 0)
198fbe0c4f4SBryan Venteicher 		vq->vq_flags |= VIRTQUEUE_FLAG_MODERN;
199b8a58707SPeter Grehan 	if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0)
200b8a58707SPeter Grehan 		vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX;
201b8a58707SPeter Grehan 
20210b59a9bSPeter Grehan 	if (info->vqai_maxindirsz > 1) {
20310b59a9bSPeter Grehan 		error = virtqueue_init_indirect(vq, info->vqai_maxindirsz);
20410b59a9bSPeter Grehan 		if (error)
20510b59a9bSPeter Grehan 			goto fail;
20610b59a9bSPeter Grehan 	}
20710b59a9bSPeter Grehan 
20810b59a9bSPeter Grehan 	vq->vq_ring_size = round_page(vring_size(size, align));
20910b59a9bSPeter Grehan 	vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF,
21010b59a9bSPeter Grehan 	    M_NOWAIT | M_ZERO, 0, highaddr, PAGE_SIZE, 0);
21110b59a9bSPeter Grehan 	if (vq->vq_ring_mem == NULL) {
21210b59a9bSPeter Grehan 		device_printf(dev,
21310b59a9bSPeter Grehan 		    "cannot allocate memory for virtqueue ring\n");
21410b59a9bSPeter Grehan 		error = ENOMEM;
21510b59a9bSPeter Grehan 		goto fail;
21610b59a9bSPeter Grehan 	}
21710b59a9bSPeter Grehan 
21810b59a9bSPeter Grehan 	vq_ring_init(vq);
21910b59a9bSPeter Grehan 	virtqueue_disable_intr(vq);
22010b59a9bSPeter Grehan 
22110b59a9bSPeter Grehan 	*vqp = vq;
22210b59a9bSPeter Grehan 
22310b59a9bSPeter Grehan fail:
22410b59a9bSPeter Grehan 	if (error)
22510b59a9bSPeter Grehan 		virtqueue_free(vq);
22610b59a9bSPeter Grehan 
22710b59a9bSPeter Grehan 	return (error);
22810b59a9bSPeter Grehan }
22910b59a9bSPeter Grehan 
23010b59a9bSPeter Grehan static int
23110b59a9bSPeter Grehan virtqueue_init_indirect(struct virtqueue *vq, int indirect_size)
23210b59a9bSPeter Grehan {
23310b59a9bSPeter Grehan 	device_t dev;
23410b59a9bSPeter Grehan 	struct vq_desc_extra *dxp;
23510b59a9bSPeter Grehan 	int i, size;
23610b59a9bSPeter Grehan 
23710b59a9bSPeter Grehan 	dev = vq->vq_dev;
23810b59a9bSPeter Grehan 
23910b59a9bSPeter Grehan 	if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) {
24010b59a9bSPeter Grehan 		/*
24110b59a9bSPeter Grehan 		 * Indirect descriptors requested by the driver but not
24210b59a9bSPeter Grehan 		 * negotiated. Return zero to keep the initialization
24310b59a9bSPeter Grehan 		 * going: we'll run fine without.
24410b59a9bSPeter Grehan 		 */
24510b59a9bSPeter Grehan 		if (bootverbose)
24610b59a9bSPeter Grehan 			device_printf(dev, "virtqueue %d (%s) requested "
24710b59a9bSPeter Grehan 			    "indirect descriptors but not negotiated\n",
24810b59a9bSPeter Grehan 			    vq->vq_queue_index, vq->vq_name);
24910b59a9bSPeter Grehan 		return (0);
25010b59a9bSPeter Grehan 	}
25110b59a9bSPeter Grehan 
25210b59a9bSPeter Grehan 	size = indirect_size * sizeof(struct vring_desc);
25310b59a9bSPeter Grehan 	vq->vq_max_indirect_size = indirect_size;
25410b59a9bSPeter Grehan 	vq->vq_indirect_mem_size = size;
25510b59a9bSPeter Grehan 	vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT;
25610b59a9bSPeter Grehan 
25710b59a9bSPeter Grehan 	for (i = 0; i < vq->vq_nentries; i++) {
25810b59a9bSPeter Grehan 		dxp = &vq->vq_descx[i];
25910b59a9bSPeter Grehan 
26010b59a9bSPeter Grehan 		dxp->indirect = malloc(size, M_DEVBUF, M_NOWAIT);
26110b59a9bSPeter Grehan 		if (dxp->indirect == NULL) {
26210b59a9bSPeter Grehan 			device_printf(dev, "cannot allocate indirect list\n");
26310b59a9bSPeter Grehan 			return (ENOMEM);
26410b59a9bSPeter Grehan 		}
26510b59a9bSPeter Grehan 
26610b59a9bSPeter Grehan 		dxp->indirect_paddr = vtophys(dxp->indirect);
26710b59a9bSPeter Grehan 		virtqueue_init_indirect_list(vq, dxp->indirect);
26810b59a9bSPeter Grehan 	}
26910b59a9bSPeter Grehan 
27010b59a9bSPeter Grehan 	return (0);
27110b59a9bSPeter Grehan }
27210b59a9bSPeter Grehan 
27310b59a9bSPeter Grehan static void
27410b59a9bSPeter Grehan virtqueue_free_indirect(struct virtqueue *vq)
27510b59a9bSPeter Grehan {
27610b59a9bSPeter Grehan 	struct vq_desc_extra *dxp;
27710b59a9bSPeter Grehan 	int i;
27810b59a9bSPeter Grehan 
27910b59a9bSPeter Grehan 	for (i = 0; i < vq->vq_nentries; i++) {
28010b59a9bSPeter Grehan 		dxp = &vq->vq_descx[i];
28110b59a9bSPeter Grehan 
28210b59a9bSPeter Grehan 		if (dxp->indirect == NULL)
28310b59a9bSPeter Grehan 			break;
28410b59a9bSPeter Grehan 
28510b59a9bSPeter Grehan 		free(dxp->indirect, M_DEVBUF);
28610b59a9bSPeter Grehan 		dxp->indirect = NULL;
28710b59a9bSPeter Grehan 		dxp->indirect_paddr = 0;
28810b59a9bSPeter Grehan 	}
28910b59a9bSPeter Grehan 
29010b59a9bSPeter Grehan 	vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT;
29110b59a9bSPeter Grehan 	vq->vq_indirect_mem_size = 0;
29210b59a9bSPeter Grehan }
29310b59a9bSPeter Grehan 
29410b59a9bSPeter Grehan static void
29510b59a9bSPeter Grehan virtqueue_init_indirect_list(struct virtqueue *vq,
29610b59a9bSPeter Grehan     struct vring_desc *indirect)
29710b59a9bSPeter Grehan {
29810b59a9bSPeter Grehan 	int i;
29910b59a9bSPeter Grehan 
30010b59a9bSPeter Grehan 	bzero(indirect, vq->vq_indirect_mem_size);
30110b59a9bSPeter Grehan 
30210b59a9bSPeter Grehan 	for (i = 0; i < vq->vq_max_indirect_size - 1; i++)
303fbe0c4f4SBryan Venteicher 		indirect[i].next = vq_gtoh16(vq, i + 1);
304fbe0c4f4SBryan Venteicher 	indirect[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END);
30510b59a9bSPeter Grehan }
30610b59a9bSPeter Grehan 
30710b59a9bSPeter Grehan int
30810b59a9bSPeter Grehan virtqueue_reinit(struct virtqueue *vq, uint16_t size)
30910b59a9bSPeter Grehan {
31010b59a9bSPeter Grehan 	struct vq_desc_extra *dxp;
31110b59a9bSPeter Grehan 	int i;
31210b59a9bSPeter Grehan 
31310b59a9bSPeter Grehan 	if (vq->vq_nentries != size) {
31410b59a9bSPeter Grehan 		device_printf(vq->vq_dev,
31510b59a9bSPeter Grehan 		    "%s: '%s' changed size; old=%hu, new=%hu\n",
31610b59a9bSPeter Grehan 		    __func__, vq->vq_name, vq->vq_nentries, size);
31710b59a9bSPeter Grehan 		return (EINVAL);
31810b59a9bSPeter Grehan 	}
31910b59a9bSPeter Grehan 
32010b59a9bSPeter Grehan 	/* Warn if the virtqueue was not properly cleaned up. */
32110b59a9bSPeter Grehan 	if (vq->vq_free_cnt != vq->vq_nentries) {
32210b59a9bSPeter Grehan 		device_printf(vq->vq_dev,
323310dacd0SPeter Grehan 		    "%s: warning '%s' virtqueue not empty, "
32410b59a9bSPeter Grehan 		    "leaking %d entries\n", __func__, vq->vq_name,
32510b59a9bSPeter Grehan 		    vq->vq_nentries - vq->vq_free_cnt);
32610b59a9bSPeter Grehan 	}
32710b59a9bSPeter Grehan 
32810b59a9bSPeter Grehan 	vq->vq_desc_head_idx = 0;
32910b59a9bSPeter Grehan 	vq->vq_used_cons_idx = 0;
33010b59a9bSPeter Grehan 	vq->vq_queued_cnt = 0;
33110b59a9bSPeter Grehan 	vq->vq_free_cnt = vq->vq_nentries;
33210b59a9bSPeter Grehan 
33310b59a9bSPeter Grehan 	/* To be safe, reset all our allocated memory. */
33410b59a9bSPeter Grehan 	bzero(vq->vq_ring_mem, vq->vq_ring_size);
33510b59a9bSPeter Grehan 	for (i = 0; i < vq->vq_nentries; i++) {
33610b59a9bSPeter Grehan 		dxp = &vq->vq_descx[i];
33710b59a9bSPeter Grehan 		dxp->cookie = NULL;
33810b59a9bSPeter Grehan 		dxp->ndescs = 0;
33910b59a9bSPeter Grehan 		if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
34010b59a9bSPeter Grehan 			virtqueue_init_indirect_list(vq, dxp->indirect);
34110b59a9bSPeter Grehan 	}
34210b59a9bSPeter Grehan 
34310b59a9bSPeter Grehan 	vq_ring_init(vq);
34410b59a9bSPeter Grehan 	virtqueue_disable_intr(vq);
34510b59a9bSPeter Grehan 
34610b59a9bSPeter Grehan 	return (0);
34710b59a9bSPeter Grehan }
34810b59a9bSPeter Grehan 
34910b59a9bSPeter Grehan void
35010b59a9bSPeter Grehan virtqueue_free(struct virtqueue *vq)
35110b59a9bSPeter Grehan {
35210b59a9bSPeter Grehan 
35310b59a9bSPeter Grehan 	if (vq->vq_free_cnt != vq->vq_nentries) {
35410b59a9bSPeter Grehan 		device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, "
35510b59a9bSPeter Grehan 		    "leaking %d entries\n", vq->vq_name,
35610b59a9bSPeter Grehan 		    vq->vq_nentries - vq->vq_free_cnt);
35710b59a9bSPeter Grehan 	}
35810b59a9bSPeter Grehan 
35910b59a9bSPeter Grehan 	if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT)
36010b59a9bSPeter Grehan 		virtqueue_free_indirect(vq);
36110b59a9bSPeter Grehan 
36210b59a9bSPeter Grehan 	if (vq->vq_ring_mem != NULL) {
363*d1bdc282SBjoern A. Zeeb 		free(vq->vq_ring_mem, M_DEVBUF);
36410b59a9bSPeter Grehan 		vq->vq_ring_size = 0;
36510b59a9bSPeter Grehan 		vq->vq_ring_mem = NULL;
36610b59a9bSPeter Grehan 	}
36710b59a9bSPeter Grehan 
36810b59a9bSPeter Grehan 	free(vq, M_DEVBUF);
36910b59a9bSPeter Grehan }
37010b59a9bSPeter Grehan 
37110b59a9bSPeter Grehan vm_paddr_t
37210b59a9bSPeter Grehan virtqueue_paddr(struct virtqueue *vq)
37310b59a9bSPeter Grehan {
37410b59a9bSPeter Grehan 
37510b59a9bSPeter Grehan 	return (vtophys(vq->vq_ring_mem));
37610b59a9bSPeter Grehan }
37710b59a9bSPeter Grehan 
37888126356SBryan Venteicher vm_paddr_t
37988126356SBryan Venteicher virtqueue_desc_paddr(struct virtqueue *vq)
38088126356SBryan Venteicher {
38188126356SBryan Venteicher 
38288126356SBryan Venteicher 	return (vtophys(vq->vq_ring.desc));
38388126356SBryan Venteicher }
38488126356SBryan Venteicher 
38588126356SBryan Venteicher vm_paddr_t
38688126356SBryan Venteicher virtqueue_avail_paddr(struct virtqueue *vq)
38788126356SBryan Venteicher {
38888126356SBryan Venteicher 
38988126356SBryan Venteicher 	return (vtophys(vq->vq_ring.avail));
39088126356SBryan Venteicher }
39188126356SBryan Venteicher 
39288126356SBryan Venteicher vm_paddr_t
39388126356SBryan Venteicher virtqueue_used_paddr(struct virtqueue *vq)
39488126356SBryan Venteicher {
39588126356SBryan Venteicher 
39688126356SBryan Venteicher 	return (vtophys(vq->vq_ring.used));
39788126356SBryan Venteicher }
39888126356SBryan Venteicher 
39988126356SBryan Venteicher uint16_t
40088126356SBryan Venteicher virtqueue_index(struct virtqueue *vq)
40188126356SBryan Venteicher {
402fbe0c4f4SBryan Venteicher 
40388126356SBryan Venteicher 	return (vq->vq_queue_index);
40488126356SBryan Venteicher }
40588126356SBryan Venteicher 
40610b59a9bSPeter Grehan int
40710b59a9bSPeter Grehan virtqueue_size(struct virtqueue *vq)
40810b59a9bSPeter Grehan {
40910b59a9bSPeter Grehan 
41010b59a9bSPeter Grehan 	return (vq->vq_nentries);
41110b59a9bSPeter Grehan }
41210b59a9bSPeter Grehan 
41310b59a9bSPeter Grehan int
4144b59668fSBryan Venteicher virtqueue_nfree(struct virtqueue *vq)
4154b59668fSBryan Venteicher {
4164b59668fSBryan Venteicher 
4174b59668fSBryan Venteicher 	return (vq->vq_free_cnt);
4184b59668fSBryan Venteicher }
4194b59668fSBryan Venteicher 
420ccb576a8SMina Galić bool
42110b59a9bSPeter Grehan virtqueue_empty(struct virtqueue *vq)
42210b59a9bSPeter Grehan {
42310b59a9bSPeter Grehan 
42410b59a9bSPeter Grehan 	return (vq->vq_nentries == vq->vq_free_cnt);
42510b59a9bSPeter Grehan }
42610b59a9bSPeter Grehan 
427ccb576a8SMina Galić bool
42810b59a9bSPeter Grehan virtqueue_full(struct virtqueue *vq)
42910b59a9bSPeter Grehan {
43010b59a9bSPeter Grehan 
43110b59a9bSPeter Grehan 	return (vq->vq_free_cnt == 0);
43210b59a9bSPeter Grehan }
43310b59a9bSPeter Grehan 
43410b59a9bSPeter Grehan void
43510b59a9bSPeter Grehan virtqueue_notify(struct virtqueue *vq)
43610b59a9bSPeter Grehan {
437310dacd0SPeter Grehan 
438b8a58707SPeter Grehan 	/* Ensure updated avail->idx is visible to host. */
439b8a58707SPeter Grehan 	mb();
44010b59a9bSPeter Grehan 
441b8a58707SPeter Grehan 	if (vq_ring_must_notify_host(vq))
442b8a58707SPeter Grehan 		vq_ring_notify_host(vq);
44310b59a9bSPeter Grehan 	vq->vq_queued_cnt = 0;
44410b59a9bSPeter Grehan }
44510b59a9bSPeter Grehan 
44610b59a9bSPeter Grehan int
44710b59a9bSPeter Grehan virtqueue_nused(struct virtqueue *vq)
44810b59a9bSPeter Grehan {
44910b59a9bSPeter Grehan 	uint16_t used_idx, nused;
45010b59a9bSPeter Grehan 
451fbe0c4f4SBryan Venteicher 	used_idx = vq_htog16(vq, vq->vq_ring.used->idx);
452b8a58707SPeter Grehan 
453b8a58707SPeter Grehan 	nused = (uint16_t)(used_idx - vq->vq_used_cons_idx);
45410b59a9bSPeter Grehan 	VQASSERT(vq, nused <= vq->vq_nentries, "used more than available");
45510b59a9bSPeter Grehan 
45610b59a9bSPeter Grehan 	return (nused);
45710b59a9bSPeter Grehan }
45810b59a9bSPeter Grehan 
45910b59a9bSPeter Grehan int
4606632efe4SBryan Venteicher virtqueue_intr_filter(struct virtqueue *vq)
4616632efe4SBryan Venteicher {
4626632efe4SBryan Venteicher 
463fbe0c4f4SBryan Venteicher 	if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx))
4646632efe4SBryan Venteicher 		return (0);
4656632efe4SBryan Venteicher 
4666632efe4SBryan Venteicher 	virtqueue_disable_intr(vq);
4676632efe4SBryan Venteicher 
4686632efe4SBryan Venteicher 	return (1);
4696632efe4SBryan Venteicher }
4706632efe4SBryan Venteicher 
4716632efe4SBryan Venteicher void
47210b59a9bSPeter Grehan virtqueue_intr(struct virtqueue *vq)
47310b59a9bSPeter Grehan {
47410b59a9bSPeter Grehan 
47510b59a9bSPeter Grehan 	vq->vq_intrhand(vq->vq_intrhand_arg);
47610b59a9bSPeter Grehan }
47710b59a9bSPeter Grehan 
47810b59a9bSPeter Grehan int
47910b59a9bSPeter Grehan virtqueue_enable_intr(struct virtqueue *vq)
48010b59a9bSPeter Grehan {
48110b59a9bSPeter Grehan 
482310dacd0SPeter Grehan 	return (vq_ring_enable_interrupt(vq, 0));
48310b59a9bSPeter Grehan }
48410b59a9bSPeter Grehan 
485b8a58707SPeter Grehan int
486b619f40aSBryan Venteicher virtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint)
487b8a58707SPeter Grehan {
488310dacd0SPeter Grehan 	uint16_t ndesc, avail_idx;
489b8a58707SPeter Grehan 
490fbe0c4f4SBryan Venteicher 	avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
491b619f40aSBryan Venteicher 	ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx);
492b619f40aSBryan Venteicher 
493b619f40aSBryan Venteicher 	switch (hint) {
494b619f40aSBryan Venteicher 	case VQ_POSTPONE_SHORT:
4954142b1cbSBryan Venteicher 		ndesc = ndesc / 4;
496b619f40aSBryan Venteicher 		break;
497b619f40aSBryan Venteicher 	case VQ_POSTPONE_LONG:
4984142b1cbSBryan Venteicher 		ndesc = (ndesc * 3) / 4;
499b619f40aSBryan Venteicher 		break;
500b619f40aSBryan Venteicher 	case VQ_POSTPONE_EMPTIED:
501b619f40aSBryan Venteicher 		break;
502b619f40aSBryan Venteicher 	}
503b8a58707SPeter Grehan 
504310dacd0SPeter Grehan 	return (vq_ring_enable_interrupt(vq, ndesc));
505b8a58707SPeter Grehan }
506b8a58707SPeter Grehan 
507b619f40aSBryan Venteicher /*
508b619f40aSBryan Venteicher  * Note this is only considered a hint to the host.
509b619f40aSBryan Venteicher  */
51010b59a9bSPeter Grehan void
51110b59a9bSPeter Grehan virtqueue_disable_intr(struct virtqueue *vq)
51210b59a9bSPeter Grehan {
51310b59a9bSPeter Grehan 
514b619f40aSBryan Venteicher 	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
515fbe0c4f4SBryan Venteicher 		vring_used_event(&vq->vq_ring) = vq_gtoh16(vq,
516fbe0c4f4SBryan Venteicher 		    vq->vq_used_cons_idx - vq->vq_nentries - 1);
517fbe0c4f4SBryan Venteicher 		return;
518fbe0c4f4SBryan Venteicher 	}
519fbe0c4f4SBryan Venteicher 
520fbe0c4f4SBryan Venteicher 	vq->vq_ring.avail->flags |= vq_gtoh16(vq, VRING_AVAIL_F_NO_INTERRUPT);
52110b59a9bSPeter Grehan }
52210b59a9bSPeter Grehan 
52310b59a9bSPeter Grehan int
52410b59a9bSPeter Grehan virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg,
52510b59a9bSPeter Grehan     int readable, int writable)
52610b59a9bSPeter Grehan {
52710b59a9bSPeter Grehan 	struct vq_desc_extra *dxp;
52810b59a9bSPeter Grehan 	int needed;
52910b59a9bSPeter Grehan 	uint16_t head_idx, idx;
53010b59a9bSPeter Grehan 
53110b59a9bSPeter Grehan 	needed = readable + writable;
53210b59a9bSPeter Grehan 
53310b59a9bSPeter Grehan 	VQASSERT(vq, cookie != NULL, "enqueuing with no cookie");
53410b59a9bSPeter Grehan 	VQASSERT(vq, needed == sg->sg_nseg,
53510b59a9bSPeter Grehan 	    "segment count mismatch, %d, %d", needed, sg->sg_nseg);
53610b59a9bSPeter Grehan 	VQASSERT(vq,
53710b59a9bSPeter Grehan 	    needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size,
53810b59a9bSPeter Grehan 	    "too many segments to enqueue: %d, %d/%d", needed,
53910b59a9bSPeter Grehan 	    vq->vq_nentries, vq->vq_max_indirect_size);
54010b59a9bSPeter Grehan 
54110b59a9bSPeter Grehan 	if (needed < 1)
54210b59a9bSPeter Grehan 		return (EINVAL);
54310b59a9bSPeter Grehan 	if (vq->vq_free_cnt == 0)
54410b59a9bSPeter Grehan 		return (ENOSPC);
54510b59a9bSPeter Grehan 
54610b59a9bSPeter Grehan 	if (vq_ring_use_indirect(vq, needed)) {
54710b59a9bSPeter Grehan 		vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable);
54810b59a9bSPeter Grehan 		return (0);
54910b59a9bSPeter Grehan 	} else if (vq->vq_free_cnt < needed)
55010b59a9bSPeter Grehan 		return (EMSGSIZE);
55110b59a9bSPeter Grehan 
55210b59a9bSPeter Grehan 	head_idx = vq->vq_desc_head_idx;
55310b59a9bSPeter Grehan 	VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
55410b59a9bSPeter Grehan 	dxp = &vq->vq_descx[head_idx];
55510b59a9bSPeter Grehan 
55610b59a9bSPeter Grehan 	VQASSERT(vq, dxp->cookie == NULL,
55710b59a9bSPeter Grehan 	    "cookie already exists for index %d", head_idx);
55810b59a9bSPeter Grehan 	dxp->cookie = cookie;
55910b59a9bSPeter Grehan 	dxp->ndescs = needed;
56010b59a9bSPeter Grehan 
56110b59a9bSPeter Grehan 	idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx,
56210b59a9bSPeter Grehan 	    sg, readable, writable);
56310b59a9bSPeter Grehan 
56410b59a9bSPeter Grehan 	vq->vq_desc_head_idx = idx;
56510b59a9bSPeter Grehan 	vq->vq_free_cnt -= needed;
56610b59a9bSPeter Grehan 	if (vq->vq_free_cnt == 0)
56710b59a9bSPeter Grehan 		VQ_RING_ASSERT_CHAIN_TERM(vq);
56810b59a9bSPeter Grehan 	else
56910b59a9bSPeter Grehan 		VQ_RING_ASSERT_VALID_IDX(vq, idx);
57010b59a9bSPeter Grehan 
57110b59a9bSPeter Grehan 	vq_ring_update_avail(vq, head_idx);
57210b59a9bSPeter Grehan 
57310b59a9bSPeter Grehan 	return (0);
57410b59a9bSPeter Grehan }
57510b59a9bSPeter Grehan 
57610b59a9bSPeter Grehan void *
57710b59a9bSPeter Grehan virtqueue_dequeue(struct virtqueue *vq, uint32_t *len)
57810b59a9bSPeter Grehan {
57910b59a9bSPeter Grehan 	struct vring_used_elem *uep;
58010b59a9bSPeter Grehan 	void *cookie;
58110b59a9bSPeter Grehan 	uint16_t used_idx, desc_idx;
58210b59a9bSPeter Grehan 
583fbe0c4f4SBryan Venteicher 	if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx))
58410b59a9bSPeter Grehan 		return (NULL);
58510b59a9bSPeter Grehan 
58610b59a9bSPeter Grehan 	used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1);
58710b59a9bSPeter Grehan 	uep = &vq->vq_ring.used->ring[used_idx];
58810b59a9bSPeter Grehan 
589c44ef550SPeter Grehan 	rmb();
590fbe0c4f4SBryan Venteicher 	desc_idx = (uint16_t) vq_htog32(vq, uep->id);
59110b59a9bSPeter Grehan 	if (len != NULL)
592fbe0c4f4SBryan Venteicher 		*len = vq_htog32(vq, uep->len);
59310b59a9bSPeter Grehan 
59410b59a9bSPeter Grehan 	vq_ring_free_chain(vq, desc_idx);
59510b59a9bSPeter Grehan 
59610b59a9bSPeter Grehan 	cookie = vq->vq_descx[desc_idx].cookie;
59710b59a9bSPeter Grehan 	VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx);
59810b59a9bSPeter Grehan 	vq->vq_descx[desc_idx].cookie = NULL;
59910b59a9bSPeter Grehan 
60010b59a9bSPeter Grehan 	return (cookie);
60110b59a9bSPeter Grehan }
60210b59a9bSPeter Grehan 
60310b59a9bSPeter Grehan void *
60410b59a9bSPeter Grehan virtqueue_poll(struct virtqueue *vq, uint32_t *len)
60510b59a9bSPeter Grehan {
60610b59a9bSPeter Grehan 	void *cookie;
60710b59a9bSPeter Grehan 
608156b97faSRuslan Bukin 	VIRTIO_BUS_POLL(vq->vq_dev);
609156b97faSRuslan Bukin 	while ((cookie = virtqueue_dequeue(vq, len)) == NULL) {
61010b59a9bSPeter Grehan 		cpu_spinwait();
611156b97faSRuslan Bukin 		VIRTIO_BUS_POLL(vq->vq_dev);
612156b97faSRuslan Bukin 	}
61310b59a9bSPeter Grehan 
61410b59a9bSPeter Grehan 	return (cookie);
61510b59a9bSPeter Grehan }
61610b59a9bSPeter Grehan 
61710b59a9bSPeter Grehan void *
61810b59a9bSPeter Grehan virtqueue_drain(struct virtqueue *vq, int *last)
61910b59a9bSPeter Grehan {
62010b59a9bSPeter Grehan 	void *cookie;
62110b59a9bSPeter Grehan 	int idx;
62210b59a9bSPeter Grehan 
62310b59a9bSPeter Grehan 	cookie = NULL;
62410b59a9bSPeter Grehan 	idx = *last;
62510b59a9bSPeter Grehan 
62610b59a9bSPeter Grehan 	while (idx < vq->vq_nentries && cookie == NULL) {
62710b59a9bSPeter Grehan 		if ((cookie = vq->vq_descx[idx].cookie) != NULL) {
62810b59a9bSPeter Grehan 			vq->vq_descx[idx].cookie = NULL;
62910b59a9bSPeter Grehan 			/* Free chain to keep free count consistent. */
63010b59a9bSPeter Grehan 			vq_ring_free_chain(vq, idx);
63110b59a9bSPeter Grehan 		}
63210b59a9bSPeter Grehan 		idx++;
63310b59a9bSPeter Grehan 	}
63410b59a9bSPeter Grehan 
63510b59a9bSPeter Grehan 	*last = idx;
63610b59a9bSPeter Grehan 
63710b59a9bSPeter Grehan 	return (cookie);
63810b59a9bSPeter Grehan }
63910b59a9bSPeter Grehan 
64010b59a9bSPeter Grehan void
64110b59a9bSPeter Grehan virtqueue_dump(struct virtqueue *vq)
64210b59a9bSPeter Grehan {
64310b59a9bSPeter Grehan 
64410b59a9bSPeter Grehan 	if (vq == NULL)
64510b59a9bSPeter Grehan 		return;
64610b59a9bSPeter Grehan 
64710b59a9bSPeter Grehan 	printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; "
64810b59a9bSPeter Grehan 	    "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; "
649a5b6123eSLuigi Rizzo 	    "used.idx=%d; used_event_idx=%d; avail.flags=0x%x; used.flags=0x%x\n",
650fbe0c4f4SBryan Venteicher 	    vq->vq_name, vq->vq_nentries, vq->vq_free_cnt, virtqueue_nused(vq),
651fbe0c4f4SBryan Venteicher 	    vq->vq_queued_cnt, vq->vq_desc_head_idx,
652fbe0c4f4SBryan Venteicher 	    vq_htog16(vq, vq->vq_ring.avail->idx), vq->vq_used_cons_idx,
653fbe0c4f4SBryan Venteicher 	    vq_htog16(vq, vq->vq_ring.used->idx),
654fbe0c4f4SBryan Venteicher 	    vq_htog16(vq, vring_used_event(&vq->vq_ring)),
655fbe0c4f4SBryan Venteicher 	    vq_htog16(vq, vq->vq_ring.avail->flags),
656fbe0c4f4SBryan Venteicher 	    vq_htog16(vq, vq->vq_ring.used->flags));
65710b59a9bSPeter Grehan }
65810b59a9bSPeter Grehan 
65910b59a9bSPeter Grehan static void
66010b59a9bSPeter Grehan vq_ring_init(struct virtqueue *vq)
66110b59a9bSPeter Grehan {
66210b59a9bSPeter Grehan 	struct vring *vr;
66310b59a9bSPeter Grehan 	char *ring_mem;
66410b59a9bSPeter Grehan 	int i, size;
66510b59a9bSPeter Grehan 
66610b59a9bSPeter Grehan 	ring_mem = vq->vq_ring_mem;
66710b59a9bSPeter Grehan 	size = vq->vq_nentries;
66810b59a9bSPeter Grehan 	vr = &vq->vq_ring;
66910b59a9bSPeter Grehan 
67010b59a9bSPeter Grehan 	vring_init(vr, size, ring_mem, vq->vq_alignment);
67110b59a9bSPeter Grehan 
67210b59a9bSPeter Grehan 	for (i = 0; i < size - 1; i++)
673fbe0c4f4SBryan Venteicher 		vr->desc[i].next = vq_gtoh16(vq, i + 1);
674fbe0c4f4SBryan Venteicher 	vr->desc[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END);
67510b59a9bSPeter Grehan }
67610b59a9bSPeter Grehan 
67710b59a9bSPeter Grehan static void
67810b59a9bSPeter Grehan vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx)
67910b59a9bSPeter Grehan {
680fbe0c4f4SBryan Venteicher 	uint16_t avail_idx, avail_ring_idx;
68110b59a9bSPeter Grehan 
68210b59a9bSPeter Grehan 	/*
68310b59a9bSPeter Grehan 	 * Place the head of the descriptor chain into the next slot and make
68410b59a9bSPeter Grehan 	 * it usable to the host. The chain is made available now rather than
68510b59a9bSPeter Grehan 	 * deferring to virtqueue_notify() in the hopes that if the host is
68610b59a9bSPeter Grehan 	 * currently running on another CPU, we can keep it processing the new
68710b59a9bSPeter Grehan 	 * descriptor.
68810b59a9bSPeter Grehan 	 */
689fbe0c4f4SBryan Venteicher 	avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
690fbe0c4f4SBryan Venteicher 	avail_ring_idx = avail_idx & (vq->vq_nentries - 1);
691fbe0c4f4SBryan Venteicher 	vq->vq_ring.avail->ring[avail_ring_idx] = vq_gtoh16(vq, desc_idx);
69210b59a9bSPeter Grehan 
693c44ef550SPeter Grehan 	wmb();
694fbe0c4f4SBryan Venteicher 	vq->vq_ring.avail->idx = vq_gtoh16(vq, avail_idx + 1);
69510b59a9bSPeter Grehan 
696b8a58707SPeter Grehan 	/* Keep pending count until virtqueue_notify(). */
69710b59a9bSPeter Grehan 	vq->vq_queued_cnt++;
69810b59a9bSPeter Grehan }
69910b59a9bSPeter Grehan 
70010b59a9bSPeter Grehan static uint16_t
70110b59a9bSPeter Grehan vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc,
70210b59a9bSPeter Grehan     uint16_t head_idx, struct sglist *sg, int readable, int writable)
70310b59a9bSPeter Grehan {
70410b59a9bSPeter Grehan 	struct sglist_seg *seg;
70510b59a9bSPeter Grehan 	struct vring_desc *dp;
70610b59a9bSPeter Grehan 	int i, needed;
70710b59a9bSPeter Grehan 	uint16_t idx;
70810b59a9bSPeter Grehan 
709fd80208aSStephen J. Kiernan 	SDT_PROBE6(virtqueue, , enqueue_segments, entry, vq, desc, head_idx,
710fd80208aSStephen J. Kiernan 	    sg, readable, writable);
711fd80208aSStephen J. Kiernan 
71210b59a9bSPeter Grehan 	needed = readable + writable;
71310b59a9bSPeter Grehan 
71410b59a9bSPeter Grehan 	for (i = 0, idx = head_idx, seg = sg->sg_segs;
71510b59a9bSPeter Grehan 	     i < needed;
716fbe0c4f4SBryan Venteicher 	     i++, idx = vq_htog16(vq, dp->next), seg++) {
71710b59a9bSPeter Grehan 		VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END,
71810b59a9bSPeter Grehan 		    "premature end of free desc chain");
71910b59a9bSPeter Grehan 
72010b59a9bSPeter Grehan 		dp = &desc[idx];
721fbe0c4f4SBryan Venteicher 		dp->addr = vq_gtoh64(vq, seg->ss_paddr);
722fbe0c4f4SBryan Venteicher 		dp->len = vq_gtoh32(vq, seg->ss_len);
72310b59a9bSPeter Grehan 		dp->flags = 0;
72410b59a9bSPeter Grehan 
72510b59a9bSPeter Grehan 		if (i < needed - 1)
726fbe0c4f4SBryan Venteicher 			dp->flags |= vq_gtoh16(vq, VRING_DESC_F_NEXT);
72710b59a9bSPeter Grehan 		if (i >= readable)
728fbe0c4f4SBryan Venteicher 			dp->flags |= vq_gtoh16(vq, VRING_DESC_F_WRITE);
72910b59a9bSPeter Grehan 	}
73010b59a9bSPeter Grehan 
731fd80208aSStephen J. Kiernan 	SDT_PROBE1(virtqueue, , enqueue_segments, return, idx);
73210b59a9bSPeter Grehan 	return (idx);
73310b59a9bSPeter Grehan }
73410b59a9bSPeter Grehan 
735ccb576a8SMina Galić static bool
73610b59a9bSPeter Grehan vq_ring_use_indirect(struct virtqueue *vq, int needed)
73710b59a9bSPeter Grehan {
73810b59a9bSPeter Grehan 
73910b59a9bSPeter Grehan 	if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0)
740ccb576a8SMina Galić 		return (false);
74110b59a9bSPeter Grehan 
74210b59a9bSPeter Grehan 	if (vq->vq_max_indirect_size < needed)
743ccb576a8SMina Galić 		return (false);
74410b59a9bSPeter Grehan 
74510b59a9bSPeter Grehan 	if (needed < 2)
746ccb576a8SMina Galić 		return (false);
74710b59a9bSPeter Grehan 
748ccb576a8SMina Galić 	return (true);
74910b59a9bSPeter Grehan }
75010b59a9bSPeter Grehan 
75110b59a9bSPeter Grehan static void
75210b59a9bSPeter Grehan vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie,
75310b59a9bSPeter Grehan     struct sglist *sg, int readable, int writable)
75410b59a9bSPeter Grehan {
75510b59a9bSPeter Grehan 	struct vring_desc *dp;
75610b59a9bSPeter Grehan 	struct vq_desc_extra *dxp;
75710b59a9bSPeter Grehan 	int needed;
75810b59a9bSPeter Grehan 	uint16_t head_idx;
75910b59a9bSPeter Grehan 
76010b59a9bSPeter Grehan 	needed = readable + writable;
76110b59a9bSPeter Grehan 	VQASSERT(vq, needed <= vq->vq_max_indirect_size,
76210b59a9bSPeter Grehan 	    "enqueuing too many indirect descriptors");
76310b59a9bSPeter Grehan 
76410b59a9bSPeter Grehan 	head_idx = vq->vq_desc_head_idx;
76510b59a9bSPeter Grehan 	VQ_RING_ASSERT_VALID_IDX(vq, head_idx);
76610b59a9bSPeter Grehan 	dp = &vq->vq_ring.desc[head_idx];
76710b59a9bSPeter Grehan 	dxp = &vq->vq_descx[head_idx];
76810b59a9bSPeter Grehan 
76910b59a9bSPeter Grehan 	VQASSERT(vq, dxp->cookie == NULL,
77010b59a9bSPeter Grehan 	    "cookie already exists for index %d", head_idx);
77110b59a9bSPeter Grehan 	dxp->cookie = cookie;
77210b59a9bSPeter Grehan 	dxp->ndescs = 1;
77310b59a9bSPeter Grehan 
774fbe0c4f4SBryan Venteicher 	dp->addr = vq_gtoh64(vq, dxp->indirect_paddr);
775fbe0c4f4SBryan Venteicher 	dp->len = vq_gtoh32(vq, needed * sizeof(struct vring_desc));
776fbe0c4f4SBryan Venteicher 	dp->flags = vq_gtoh16(vq, VRING_DESC_F_INDIRECT);
77710b59a9bSPeter Grehan 
77810b59a9bSPeter Grehan 	vq_ring_enqueue_segments(vq, dxp->indirect, 0,
77910b59a9bSPeter Grehan 	    sg, readable, writable);
78010b59a9bSPeter Grehan 
781fbe0c4f4SBryan Venteicher 	vq->vq_desc_head_idx = vq_htog16(vq, dp->next);
78210b59a9bSPeter Grehan 	vq->vq_free_cnt--;
78310b59a9bSPeter Grehan 	if (vq->vq_free_cnt == 0)
78410b59a9bSPeter Grehan 		VQ_RING_ASSERT_CHAIN_TERM(vq);
78510b59a9bSPeter Grehan 	else
78610b59a9bSPeter Grehan 		VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx);
78710b59a9bSPeter Grehan 
78810b59a9bSPeter Grehan 	vq_ring_update_avail(vq, head_idx);
78910b59a9bSPeter Grehan }
79010b59a9bSPeter Grehan 
791b8a58707SPeter Grehan static int
792310dacd0SPeter Grehan vq_ring_enable_interrupt(struct virtqueue *vq, uint16_t ndesc)
793310dacd0SPeter Grehan {
794310dacd0SPeter Grehan 
795310dacd0SPeter Grehan 	/*
796310dacd0SPeter Grehan 	 * Enable interrupts, making sure we get the latest index of
797310dacd0SPeter Grehan 	 * what's already been consumed.
798310dacd0SPeter Grehan 	 */
799fbe0c4f4SBryan Venteicher 	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
800fbe0c4f4SBryan Venteicher 		vring_used_event(&vq->vq_ring) =
801fbe0c4f4SBryan Venteicher 		    vq_gtoh16(vq, vq->vq_used_cons_idx + ndesc);
802fbe0c4f4SBryan Venteicher 	} else {
803fbe0c4f4SBryan Venteicher 		vq->vq_ring.avail->flags &=
804fbe0c4f4SBryan Venteicher 		    vq_gtoh16(vq, ~VRING_AVAIL_F_NO_INTERRUPT);
805fbe0c4f4SBryan Venteicher 	}
806310dacd0SPeter Grehan 
807310dacd0SPeter Grehan 	mb();
808310dacd0SPeter Grehan 
809310dacd0SPeter Grehan 	/*
810310dacd0SPeter Grehan 	 * Enough items may have already been consumed to meet our threshold
811310dacd0SPeter Grehan 	 * since we last checked. Let our caller know so it processes the new
812310dacd0SPeter Grehan 	 * entries.
813310dacd0SPeter Grehan 	 */
814310dacd0SPeter Grehan 	if (virtqueue_nused(vq) > ndesc)
815310dacd0SPeter Grehan 		return (1);
816310dacd0SPeter Grehan 
817310dacd0SPeter Grehan 	return (0);
818310dacd0SPeter Grehan }
819310dacd0SPeter Grehan 
820310dacd0SPeter Grehan static int
821b8a58707SPeter Grehan vq_ring_must_notify_host(struct virtqueue *vq)
822b8a58707SPeter Grehan {
823fbe0c4f4SBryan Venteicher 	uint16_t new_idx, prev_idx, event_idx, flags;
824b8a58707SPeter Grehan 
825b8a58707SPeter Grehan 	if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) {
826fbe0c4f4SBryan Venteicher 		new_idx = vq_htog16(vq, vq->vq_ring.avail->idx);
827b8a58707SPeter Grehan 		prev_idx = new_idx - vq->vq_queued_cnt;
828fbe0c4f4SBryan Venteicher 		event_idx = vq_htog16(vq, vring_avail_event(&vq->vq_ring));
829b8a58707SPeter Grehan 
830b8a58707SPeter Grehan 		return (vring_need_event(event_idx, new_idx, prev_idx) != 0);
831b8a58707SPeter Grehan 	}
832b8a58707SPeter Grehan 
833fbe0c4f4SBryan Venteicher 	flags = vq->vq_ring.used->flags;
834fbe0c4f4SBryan Venteicher 	return ((flags & vq_gtoh16(vq, VRING_USED_F_NO_NOTIFY)) == 0);
835b8a58707SPeter Grehan }
836b8a58707SPeter Grehan 
83710b59a9bSPeter Grehan static void
838b8a58707SPeter Grehan vq_ring_notify_host(struct virtqueue *vq)
83910b59a9bSPeter Grehan {
84010b59a9bSPeter Grehan 
8419da9560cSBryan Venteicher 	VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index,
8429da9560cSBryan Venteicher 	    vq->vq_notify_offset);
84310b59a9bSPeter Grehan }
84410b59a9bSPeter Grehan 
84510b59a9bSPeter Grehan static void
84610b59a9bSPeter Grehan vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx)
84710b59a9bSPeter Grehan {
84810b59a9bSPeter Grehan 	struct vring_desc *dp;
84910b59a9bSPeter Grehan 	struct vq_desc_extra *dxp;
85010b59a9bSPeter Grehan 
85110b59a9bSPeter Grehan 	VQ_RING_ASSERT_VALID_IDX(vq, desc_idx);
85210b59a9bSPeter Grehan 	dp = &vq->vq_ring.desc[desc_idx];
85310b59a9bSPeter Grehan 	dxp = &vq->vq_descx[desc_idx];
85410b59a9bSPeter Grehan 
85510b59a9bSPeter Grehan 	if (vq->vq_free_cnt == 0)
85610b59a9bSPeter Grehan 		VQ_RING_ASSERT_CHAIN_TERM(vq);
85710b59a9bSPeter Grehan 
85810b59a9bSPeter Grehan 	vq->vq_free_cnt += dxp->ndescs;
8591dbd8bc9SPeter Grehan 	dxp->ndescs--;
86010b59a9bSPeter Grehan 
861fbe0c4f4SBryan Venteicher 	if ((dp->flags & vq_gtoh16(vq, VRING_DESC_F_INDIRECT)) == 0) {
862fbe0c4f4SBryan Venteicher 		while (dp->flags & vq_gtoh16(vq, VRING_DESC_F_NEXT)) {
863fbe0c4f4SBryan Venteicher 			uint16_t next_idx = vq_htog16(vq, dp->next);
864fbe0c4f4SBryan Venteicher 			VQ_RING_ASSERT_VALID_IDX(vq, next_idx);
865fbe0c4f4SBryan Venteicher 			dp = &vq->vq_ring.desc[next_idx];
86610b59a9bSPeter Grehan 			dxp->ndescs--;
86710b59a9bSPeter Grehan 		}
86810b59a9bSPeter Grehan 	}
8691dbd8bc9SPeter Grehan 
8701dbd8bc9SPeter Grehan 	VQASSERT(vq, dxp->ndescs == 0,
871310dacd0SPeter Grehan 	    "failed to free entire desc chain, remaining: %d", dxp->ndescs);
87210b59a9bSPeter Grehan 
87310b59a9bSPeter Grehan 	/*
87410b59a9bSPeter Grehan 	 * We must append the existing free chain, if any, to the end of
87510b59a9bSPeter Grehan 	 * newly freed chain. If the virtqueue was completely used, then
87610b59a9bSPeter Grehan 	 * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above).
87710b59a9bSPeter Grehan 	 */
878fbe0c4f4SBryan Venteicher 	dp->next = vq_gtoh16(vq, vq->vq_desc_head_idx);
87910b59a9bSPeter Grehan 	vq->vq_desc_head_idx = desc_idx;
88010b59a9bSPeter Grehan }
881