110b59a9bSPeter Grehan /*- 24d846d26SWarner Losh * SPDX-License-Identifier: BSD-2-Clause 3718cf2ccSPedro F. Giffuni * 4abd6790cSBryan Venteicher * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org> 510b59a9bSPeter Grehan * All rights reserved. 610b59a9bSPeter Grehan * 710b59a9bSPeter Grehan * Redistribution and use in source and binary forms, with or without 810b59a9bSPeter Grehan * modification, are permitted provided that the following conditions 910b59a9bSPeter Grehan * are met: 1010b59a9bSPeter Grehan * 1. Redistributions of source code must retain the above copyright 1110b59a9bSPeter Grehan * notice unmodified, this list of conditions, and the following 1210b59a9bSPeter Grehan * disclaimer. 1310b59a9bSPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright 1410b59a9bSPeter Grehan * notice, this list of conditions and the following disclaimer in the 1510b59a9bSPeter Grehan * documentation and/or other materials provided with the distribution. 1610b59a9bSPeter Grehan * 1710b59a9bSPeter Grehan * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 1810b59a9bSPeter Grehan * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 1910b59a9bSPeter Grehan * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 2010b59a9bSPeter Grehan * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 2110b59a9bSPeter Grehan * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 2210b59a9bSPeter Grehan * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2310b59a9bSPeter Grehan * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2410b59a9bSPeter Grehan * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2510b59a9bSPeter Grehan * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 2610b59a9bSPeter Grehan * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2710b59a9bSPeter Grehan */ 2810b59a9bSPeter Grehan 2910b59a9bSPeter Grehan /* 3010b59a9bSPeter Grehan * Implements the virtqueue interface as basically described 3110b59a9bSPeter Grehan * in the original VirtIO paper. 3210b59a9bSPeter Grehan */ 3310b59a9bSPeter Grehan 3410b59a9bSPeter Grehan #include <sys/param.h> 3510b59a9bSPeter Grehan #include <sys/systm.h> 3610b59a9bSPeter Grehan #include <sys/kernel.h> 3710b59a9bSPeter Grehan #include <sys/malloc.h> 38fd80208aSStephen J. Kiernan #include <sys/sdt.h> 3910b59a9bSPeter Grehan #include <sys/sglist.h> 4010b59a9bSPeter Grehan #include <vm/vm.h> 4110b59a9bSPeter Grehan #include <vm/pmap.h> 4210b59a9bSPeter Grehan 4310b59a9bSPeter Grehan #include <machine/cpu.h> 4410b59a9bSPeter Grehan #include <machine/bus.h> 4510b59a9bSPeter Grehan #include <machine/atomic.h> 4610b59a9bSPeter Grehan #include <machine/resource.h> 4710b59a9bSPeter Grehan #include <sys/bus.h> 4810b59a9bSPeter Grehan #include <sys/rman.h> 4910b59a9bSPeter Grehan 5010b59a9bSPeter Grehan #include <dev/virtio/virtio.h> 5110b59a9bSPeter Grehan #include <dev/virtio/virtqueue.h> 5210b59a9bSPeter Grehan #include <dev/virtio/virtio_ring.h> 5310b59a9bSPeter Grehan 5410b59a9bSPeter Grehan #include "virtio_bus_if.h" 5510b59a9bSPeter Grehan 5610b59a9bSPeter Grehan struct virtqueue { 5710b59a9bSPeter Grehan device_t vq_dev; 5810b59a9bSPeter Grehan uint16_t vq_queue_index; 5910b59a9bSPeter Grehan uint16_t vq_nentries; 6010b59a9bSPeter Grehan uint32_t vq_flags; 61fbe0c4f4SBryan Venteicher #define VIRTQUEUE_FLAG_MODERN 0x0001 62fbe0c4f4SBryan Venteicher #define VIRTQUEUE_FLAG_INDIRECT 0x0002 63fbe0c4f4SBryan Venteicher #define VIRTQUEUE_FLAG_EVENT_IDX 0x0004 6410b59a9bSPeter Grehan 6510b59a9bSPeter Grehan int vq_max_indirect_size; 66fbe0c4f4SBryan Venteicher bus_size_t vq_notify_offset; 6710b59a9bSPeter Grehan virtqueue_intr_t *vq_intrhand; 6810b59a9bSPeter Grehan void *vq_intrhand_arg; 6910b59a9bSPeter Grehan 7010b59a9bSPeter Grehan struct vring vq_ring; 7110b59a9bSPeter Grehan uint16_t vq_free_cnt; 7210b59a9bSPeter Grehan uint16_t vq_queued_cnt; 7310b59a9bSPeter Grehan /* 7410b59a9bSPeter Grehan * Head of the free chain in the descriptor table. If 7510b59a9bSPeter Grehan * there are no free descriptors, this will be set to 7610b59a9bSPeter Grehan * VQ_RING_DESC_CHAIN_END. 7710b59a9bSPeter Grehan */ 7810b59a9bSPeter Grehan uint16_t vq_desc_head_idx; 7910b59a9bSPeter Grehan /* 8010b59a9bSPeter Grehan * Last consumed descriptor in the used table, 8110b59a9bSPeter Grehan * trails vq_ring.used->idx. 8210b59a9bSPeter Grehan */ 8310b59a9bSPeter Grehan uint16_t vq_used_cons_idx; 8410b59a9bSPeter Grehan 85fbe0c4f4SBryan Venteicher void *vq_ring_mem; 86fbe0c4f4SBryan Venteicher int vq_indirect_mem_size; 87fbe0c4f4SBryan Venteicher int vq_alignment; 88fbe0c4f4SBryan Venteicher int vq_ring_size; 89fbe0c4f4SBryan Venteicher char vq_name[VIRTQUEUE_MAX_NAME_SZ]; 90fbe0c4f4SBryan Venteicher 9110b59a9bSPeter Grehan struct vq_desc_extra { 9210b59a9bSPeter Grehan void *cookie; 9310b59a9bSPeter Grehan struct vring_desc *indirect; 9410b59a9bSPeter Grehan vm_paddr_t indirect_paddr; 9510b59a9bSPeter Grehan uint16_t ndescs; 9610b59a9bSPeter Grehan } vq_descx[0]; 9710b59a9bSPeter Grehan }; 9810b59a9bSPeter Grehan 9910b59a9bSPeter Grehan /* 10010b59a9bSPeter Grehan * The maximum virtqueue size is 2^15. Use that value as the end of 10110b59a9bSPeter Grehan * descriptor chain terminator since it will never be a valid index 10210b59a9bSPeter Grehan * in the descriptor table. This is used to verify we are correctly 10310b59a9bSPeter Grehan * handling vq_free_cnt. 10410b59a9bSPeter Grehan */ 10510b59a9bSPeter Grehan #define VQ_RING_DESC_CHAIN_END 32768 10610b59a9bSPeter Grehan 10710b59a9bSPeter Grehan #define VQASSERT(_vq, _exp, _msg, ...) \ 10810b59a9bSPeter Grehan KASSERT((_exp),("%s: %s - "_msg, __func__, (_vq)->vq_name, \ 10910b59a9bSPeter Grehan ##__VA_ARGS__)) 11010b59a9bSPeter Grehan 11110b59a9bSPeter Grehan #define VQ_RING_ASSERT_VALID_IDX(_vq, _idx) \ 11210b59a9bSPeter Grehan VQASSERT((_vq), (_idx) < (_vq)->vq_nentries, \ 11310b59a9bSPeter Grehan "invalid ring index: %d, max: %d", (_idx), \ 11410b59a9bSPeter Grehan (_vq)->vq_nentries) 11510b59a9bSPeter Grehan 11610b59a9bSPeter Grehan #define VQ_RING_ASSERT_CHAIN_TERM(_vq) \ 11710b59a9bSPeter Grehan VQASSERT((_vq), (_vq)->vq_desc_head_idx == \ 11810b59a9bSPeter Grehan VQ_RING_DESC_CHAIN_END, "full ring terminated " \ 11910b59a9bSPeter Grehan "incorrectly: head idx: %d", (_vq)->vq_desc_head_idx) 12010b59a9bSPeter Grehan 12110b59a9bSPeter Grehan static int virtqueue_init_indirect(struct virtqueue *vq, int); 12210b59a9bSPeter Grehan static void virtqueue_free_indirect(struct virtqueue *vq); 12310b59a9bSPeter Grehan static void virtqueue_init_indirect_list(struct virtqueue *, 12410b59a9bSPeter Grehan struct vring_desc *); 12510b59a9bSPeter Grehan 12610b59a9bSPeter Grehan static void vq_ring_init(struct virtqueue *); 12710b59a9bSPeter Grehan static void vq_ring_update_avail(struct virtqueue *, uint16_t); 12810b59a9bSPeter Grehan static uint16_t vq_ring_enqueue_segments(struct virtqueue *, 12910b59a9bSPeter Grehan struct vring_desc *, uint16_t, struct sglist *, int, int); 130ccb576a8SMina Galić static bool vq_ring_use_indirect(struct virtqueue *, int); 13110b59a9bSPeter Grehan static void vq_ring_enqueue_indirect(struct virtqueue *, void *, 13210b59a9bSPeter Grehan struct sglist *, int, int); 133310dacd0SPeter Grehan static int vq_ring_enable_interrupt(struct virtqueue *, uint16_t); 134b8a58707SPeter Grehan static int vq_ring_must_notify_host(struct virtqueue *); 135b8a58707SPeter Grehan static void vq_ring_notify_host(struct virtqueue *); 13610b59a9bSPeter Grehan static void vq_ring_free_chain(struct virtqueue *, uint16_t); 13710b59a9bSPeter Grehan 138fd80208aSStephen J. Kiernan SDT_PROVIDER_DEFINE(virtqueue); 139fd80208aSStephen J. Kiernan SDT_PROBE_DEFINE6(virtqueue, , enqueue_segments, entry, "struct virtqueue *", 140fd80208aSStephen J. Kiernan "struct vring_desc *", "uint16_t", "struct sglist *", "int", "int"); 141fd80208aSStephen J. Kiernan SDT_PROBE_DEFINE1(virtqueue, , enqueue_segments, return, "uint16_t"); 142fd80208aSStephen J. Kiernan 143fbe0c4f4SBryan Venteicher #define vq_modern(_vq) (((_vq)->vq_flags & VIRTQUEUE_FLAG_MODERN) != 0) 144fbe0c4f4SBryan Venteicher #define vq_htog16(_vq, _val) virtio_htog16(vq_modern(_vq), _val) 145fbe0c4f4SBryan Venteicher #define vq_htog32(_vq, _val) virtio_htog32(vq_modern(_vq), _val) 146fbe0c4f4SBryan Venteicher #define vq_htog64(_vq, _val) virtio_htog64(vq_modern(_vq), _val) 147fbe0c4f4SBryan Venteicher #define vq_gtoh16(_vq, _val) virtio_gtoh16(vq_modern(_vq), _val) 148fbe0c4f4SBryan Venteicher #define vq_gtoh32(_vq, _val) virtio_gtoh32(vq_modern(_vq), _val) 149fbe0c4f4SBryan Venteicher #define vq_gtoh64(_vq, _val) virtio_gtoh64(vq_modern(_vq), _val) 15010b59a9bSPeter Grehan 15110b59a9bSPeter Grehan int 1529da9560cSBryan Venteicher virtqueue_alloc(device_t dev, uint16_t queue, uint16_t size, 1539da9560cSBryan Venteicher bus_size_t notify_offset, int align, vm_paddr_t highaddr, 1549da9560cSBryan Venteicher struct vq_alloc_info *info, struct virtqueue **vqp) 15510b59a9bSPeter Grehan { 15610b59a9bSPeter Grehan struct virtqueue *vq; 15710b59a9bSPeter Grehan int error; 15810b59a9bSPeter Grehan 15910b59a9bSPeter Grehan *vqp = NULL; 16010b59a9bSPeter Grehan error = 0; 16110b59a9bSPeter Grehan 16210b59a9bSPeter Grehan if (size == 0) { 16310b59a9bSPeter Grehan device_printf(dev, 16410b59a9bSPeter Grehan "virtqueue %d (%s) does not exist (size is zero)\n", 16510b59a9bSPeter Grehan queue, info->vqai_name); 16610b59a9bSPeter Grehan return (ENODEV); 16710b59a9bSPeter Grehan } else if (!powerof2(size)) { 16810b59a9bSPeter Grehan device_printf(dev, 16910b59a9bSPeter Grehan "virtqueue %d (%s) size is not a power of 2: %d\n", 17010b59a9bSPeter Grehan queue, info->vqai_name, size); 17110b59a9bSPeter Grehan return (ENXIO); 17210b59a9bSPeter Grehan } else if (info->vqai_maxindirsz > VIRTIO_MAX_INDIRECT) { 17310b59a9bSPeter Grehan device_printf(dev, "virtqueue %d (%s) requested too many " 17410b59a9bSPeter Grehan "indirect descriptors: %d, max %d\n", 17510b59a9bSPeter Grehan queue, info->vqai_name, info->vqai_maxindirsz, 17610b59a9bSPeter Grehan VIRTIO_MAX_INDIRECT); 17710b59a9bSPeter Grehan return (EINVAL); 17810b59a9bSPeter Grehan } 17910b59a9bSPeter Grehan 18010b59a9bSPeter Grehan vq = malloc(sizeof(struct virtqueue) + 18110b59a9bSPeter Grehan size * sizeof(struct vq_desc_extra), M_DEVBUF, M_NOWAIT | M_ZERO); 18210b59a9bSPeter Grehan if (vq == NULL) { 18310b59a9bSPeter Grehan device_printf(dev, "cannot allocate virtqueue\n"); 18410b59a9bSPeter Grehan return (ENOMEM); 18510b59a9bSPeter Grehan } 18610b59a9bSPeter Grehan 18710b59a9bSPeter Grehan vq->vq_dev = dev; 18810b59a9bSPeter Grehan strlcpy(vq->vq_name, info->vqai_name, sizeof(vq->vq_name)); 18910b59a9bSPeter Grehan vq->vq_queue_index = queue; 1909da9560cSBryan Venteicher vq->vq_notify_offset = notify_offset; 19110b59a9bSPeter Grehan vq->vq_alignment = align; 19210b59a9bSPeter Grehan vq->vq_nentries = size; 19310b59a9bSPeter Grehan vq->vq_free_cnt = size; 19410b59a9bSPeter Grehan vq->vq_intrhand = info->vqai_intr; 19510b59a9bSPeter Grehan vq->vq_intrhand_arg = info->vqai_intr_arg; 19610b59a9bSPeter Grehan 197fbe0c4f4SBryan Venteicher if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_F_VERSION_1) != 0) 198fbe0c4f4SBryan Venteicher vq->vq_flags |= VIRTQUEUE_FLAG_MODERN; 199b8a58707SPeter Grehan if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_EVENT_IDX) != 0) 200b8a58707SPeter Grehan vq->vq_flags |= VIRTQUEUE_FLAG_EVENT_IDX; 201b8a58707SPeter Grehan 20210b59a9bSPeter Grehan if (info->vqai_maxindirsz > 1) { 20310b59a9bSPeter Grehan error = virtqueue_init_indirect(vq, info->vqai_maxindirsz); 20410b59a9bSPeter Grehan if (error) 20510b59a9bSPeter Grehan goto fail; 20610b59a9bSPeter Grehan } 20710b59a9bSPeter Grehan 20810b59a9bSPeter Grehan vq->vq_ring_size = round_page(vring_size(size, align)); 20910b59a9bSPeter Grehan vq->vq_ring_mem = contigmalloc(vq->vq_ring_size, M_DEVBUF, 21010b59a9bSPeter Grehan M_NOWAIT | M_ZERO, 0, highaddr, PAGE_SIZE, 0); 21110b59a9bSPeter Grehan if (vq->vq_ring_mem == NULL) { 21210b59a9bSPeter Grehan device_printf(dev, 21310b59a9bSPeter Grehan "cannot allocate memory for virtqueue ring\n"); 21410b59a9bSPeter Grehan error = ENOMEM; 21510b59a9bSPeter Grehan goto fail; 21610b59a9bSPeter Grehan } 21710b59a9bSPeter Grehan 21810b59a9bSPeter Grehan vq_ring_init(vq); 21910b59a9bSPeter Grehan virtqueue_disable_intr(vq); 22010b59a9bSPeter Grehan 22110b59a9bSPeter Grehan *vqp = vq; 22210b59a9bSPeter Grehan 22310b59a9bSPeter Grehan fail: 22410b59a9bSPeter Grehan if (error) 22510b59a9bSPeter Grehan virtqueue_free(vq); 22610b59a9bSPeter Grehan 22710b59a9bSPeter Grehan return (error); 22810b59a9bSPeter Grehan } 22910b59a9bSPeter Grehan 23010b59a9bSPeter Grehan static int 23110b59a9bSPeter Grehan virtqueue_init_indirect(struct virtqueue *vq, int indirect_size) 23210b59a9bSPeter Grehan { 23310b59a9bSPeter Grehan device_t dev; 23410b59a9bSPeter Grehan struct vq_desc_extra *dxp; 23510b59a9bSPeter Grehan int i, size; 23610b59a9bSPeter Grehan 23710b59a9bSPeter Grehan dev = vq->vq_dev; 23810b59a9bSPeter Grehan 23910b59a9bSPeter Grehan if (VIRTIO_BUS_WITH_FEATURE(dev, VIRTIO_RING_F_INDIRECT_DESC) == 0) { 24010b59a9bSPeter Grehan /* 24110b59a9bSPeter Grehan * Indirect descriptors requested by the driver but not 24210b59a9bSPeter Grehan * negotiated. Return zero to keep the initialization 24310b59a9bSPeter Grehan * going: we'll run fine without. 24410b59a9bSPeter Grehan */ 24510b59a9bSPeter Grehan if (bootverbose) 24610b59a9bSPeter Grehan device_printf(dev, "virtqueue %d (%s) requested " 24710b59a9bSPeter Grehan "indirect descriptors but not negotiated\n", 24810b59a9bSPeter Grehan vq->vq_queue_index, vq->vq_name); 24910b59a9bSPeter Grehan return (0); 25010b59a9bSPeter Grehan } 25110b59a9bSPeter Grehan 25210b59a9bSPeter Grehan size = indirect_size * sizeof(struct vring_desc); 25310b59a9bSPeter Grehan vq->vq_max_indirect_size = indirect_size; 25410b59a9bSPeter Grehan vq->vq_indirect_mem_size = size; 25510b59a9bSPeter Grehan vq->vq_flags |= VIRTQUEUE_FLAG_INDIRECT; 25610b59a9bSPeter Grehan 25710b59a9bSPeter Grehan for (i = 0; i < vq->vq_nentries; i++) { 25810b59a9bSPeter Grehan dxp = &vq->vq_descx[i]; 25910b59a9bSPeter Grehan 26010b59a9bSPeter Grehan dxp->indirect = malloc(size, M_DEVBUF, M_NOWAIT); 26110b59a9bSPeter Grehan if (dxp->indirect == NULL) { 26210b59a9bSPeter Grehan device_printf(dev, "cannot allocate indirect list\n"); 26310b59a9bSPeter Grehan return (ENOMEM); 26410b59a9bSPeter Grehan } 26510b59a9bSPeter Grehan 26610b59a9bSPeter Grehan dxp->indirect_paddr = vtophys(dxp->indirect); 26710b59a9bSPeter Grehan virtqueue_init_indirect_list(vq, dxp->indirect); 26810b59a9bSPeter Grehan } 26910b59a9bSPeter Grehan 27010b59a9bSPeter Grehan return (0); 27110b59a9bSPeter Grehan } 27210b59a9bSPeter Grehan 27310b59a9bSPeter Grehan static void 27410b59a9bSPeter Grehan virtqueue_free_indirect(struct virtqueue *vq) 27510b59a9bSPeter Grehan { 27610b59a9bSPeter Grehan struct vq_desc_extra *dxp; 27710b59a9bSPeter Grehan int i; 27810b59a9bSPeter Grehan 27910b59a9bSPeter Grehan for (i = 0; i < vq->vq_nentries; i++) { 28010b59a9bSPeter Grehan dxp = &vq->vq_descx[i]; 28110b59a9bSPeter Grehan 28210b59a9bSPeter Grehan if (dxp->indirect == NULL) 28310b59a9bSPeter Grehan break; 28410b59a9bSPeter Grehan 28510b59a9bSPeter Grehan free(dxp->indirect, M_DEVBUF); 28610b59a9bSPeter Grehan dxp->indirect = NULL; 28710b59a9bSPeter Grehan dxp->indirect_paddr = 0; 28810b59a9bSPeter Grehan } 28910b59a9bSPeter Grehan 29010b59a9bSPeter Grehan vq->vq_flags &= ~VIRTQUEUE_FLAG_INDIRECT; 29110b59a9bSPeter Grehan vq->vq_indirect_mem_size = 0; 29210b59a9bSPeter Grehan } 29310b59a9bSPeter Grehan 29410b59a9bSPeter Grehan static void 29510b59a9bSPeter Grehan virtqueue_init_indirect_list(struct virtqueue *vq, 29610b59a9bSPeter Grehan struct vring_desc *indirect) 29710b59a9bSPeter Grehan { 29810b59a9bSPeter Grehan int i; 29910b59a9bSPeter Grehan 30010b59a9bSPeter Grehan bzero(indirect, vq->vq_indirect_mem_size); 30110b59a9bSPeter Grehan 30210b59a9bSPeter Grehan for (i = 0; i < vq->vq_max_indirect_size - 1; i++) 303fbe0c4f4SBryan Venteicher indirect[i].next = vq_gtoh16(vq, i + 1); 304fbe0c4f4SBryan Venteicher indirect[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END); 30510b59a9bSPeter Grehan } 30610b59a9bSPeter Grehan 30710b59a9bSPeter Grehan int 30810b59a9bSPeter Grehan virtqueue_reinit(struct virtqueue *vq, uint16_t size) 30910b59a9bSPeter Grehan { 31010b59a9bSPeter Grehan struct vq_desc_extra *dxp; 31110b59a9bSPeter Grehan int i; 31210b59a9bSPeter Grehan 31310b59a9bSPeter Grehan if (vq->vq_nentries != size) { 31410b59a9bSPeter Grehan device_printf(vq->vq_dev, 31510b59a9bSPeter Grehan "%s: '%s' changed size; old=%hu, new=%hu\n", 31610b59a9bSPeter Grehan __func__, vq->vq_name, vq->vq_nentries, size); 31710b59a9bSPeter Grehan return (EINVAL); 31810b59a9bSPeter Grehan } 31910b59a9bSPeter Grehan 32010b59a9bSPeter Grehan /* Warn if the virtqueue was not properly cleaned up. */ 32110b59a9bSPeter Grehan if (vq->vq_free_cnt != vq->vq_nentries) { 32210b59a9bSPeter Grehan device_printf(vq->vq_dev, 323310dacd0SPeter Grehan "%s: warning '%s' virtqueue not empty, " 32410b59a9bSPeter Grehan "leaking %d entries\n", __func__, vq->vq_name, 32510b59a9bSPeter Grehan vq->vq_nentries - vq->vq_free_cnt); 32610b59a9bSPeter Grehan } 32710b59a9bSPeter Grehan 32810b59a9bSPeter Grehan vq->vq_desc_head_idx = 0; 32910b59a9bSPeter Grehan vq->vq_used_cons_idx = 0; 33010b59a9bSPeter Grehan vq->vq_queued_cnt = 0; 33110b59a9bSPeter Grehan vq->vq_free_cnt = vq->vq_nentries; 33210b59a9bSPeter Grehan 33310b59a9bSPeter Grehan /* To be safe, reset all our allocated memory. */ 33410b59a9bSPeter Grehan bzero(vq->vq_ring_mem, vq->vq_ring_size); 33510b59a9bSPeter Grehan for (i = 0; i < vq->vq_nentries; i++) { 33610b59a9bSPeter Grehan dxp = &vq->vq_descx[i]; 33710b59a9bSPeter Grehan dxp->cookie = NULL; 33810b59a9bSPeter Grehan dxp->ndescs = 0; 33910b59a9bSPeter Grehan if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) 34010b59a9bSPeter Grehan virtqueue_init_indirect_list(vq, dxp->indirect); 34110b59a9bSPeter Grehan } 34210b59a9bSPeter Grehan 34310b59a9bSPeter Grehan vq_ring_init(vq); 34410b59a9bSPeter Grehan virtqueue_disable_intr(vq); 34510b59a9bSPeter Grehan 34610b59a9bSPeter Grehan return (0); 34710b59a9bSPeter Grehan } 34810b59a9bSPeter Grehan 34910b59a9bSPeter Grehan void 35010b59a9bSPeter Grehan virtqueue_free(struct virtqueue *vq) 35110b59a9bSPeter Grehan { 35210b59a9bSPeter Grehan 35310b59a9bSPeter Grehan if (vq->vq_free_cnt != vq->vq_nentries) { 35410b59a9bSPeter Grehan device_printf(vq->vq_dev, "%s: freeing non-empty virtqueue, " 35510b59a9bSPeter Grehan "leaking %d entries\n", vq->vq_name, 35610b59a9bSPeter Grehan vq->vq_nentries - vq->vq_free_cnt); 35710b59a9bSPeter Grehan } 35810b59a9bSPeter Grehan 35910b59a9bSPeter Grehan if (vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) 36010b59a9bSPeter Grehan virtqueue_free_indirect(vq); 36110b59a9bSPeter Grehan 36210b59a9bSPeter Grehan if (vq->vq_ring_mem != NULL) { 363*d1bdc282SBjoern A. Zeeb free(vq->vq_ring_mem, M_DEVBUF); 36410b59a9bSPeter Grehan vq->vq_ring_size = 0; 36510b59a9bSPeter Grehan vq->vq_ring_mem = NULL; 36610b59a9bSPeter Grehan } 36710b59a9bSPeter Grehan 36810b59a9bSPeter Grehan free(vq, M_DEVBUF); 36910b59a9bSPeter Grehan } 37010b59a9bSPeter Grehan 37110b59a9bSPeter Grehan vm_paddr_t 37210b59a9bSPeter Grehan virtqueue_paddr(struct virtqueue *vq) 37310b59a9bSPeter Grehan { 37410b59a9bSPeter Grehan 37510b59a9bSPeter Grehan return (vtophys(vq->vq_ring_mem)); 37610b59a9bSPeter Grehan } 37710b59a9bSPeter Grehan 37888126356SBryan Venteicher vm_paddr_t 37988126356SBryan Venteicher virtqueue_desc_paddr(struct virtqueue *vq) 38088126356SBryan Venteicher { 38188126356SBryan Venteicher 38288126356SBryan Venteicher return (vtophys(vq->vq_ring.desc)); 38388126356SBryan Venteicher } 38488126356SBryan Venteicher 38588126356SBryan Venteicher vm_paddr_t 38688126356SBryan Venteicher virtqueue_avail_paddr(struct virtqueue *vq) 38788126356SBryan Venteicher { 38888126356SBryan Venteicher 38988126356SBryan Venteicher return (vtophys(vq->vq_ring.avail)); 39088126356SBryan Venteicher } 39188126356SBryan Venteicher 39288126356SBryan Venteicher vm_paddr_t 39388126356SBryan Venteicher virtqueue_used_paddr(struct virtqueue *vq) 39488126356SBryan Venteicher { 39588126356SBryan Venteicher 39688126356SBryan Venteicher return (vtophys(vq->vq_ring.used)); 39788126356SBryan Venteicher } 39888126356SBryan Venteicher 39988126356SBryan Venteicher uint16_t 40088126356SBryan Venteicher virtqueue_index(struct virtqueue *vq) 40188126356SBryan Venteicher { 402fbe0c4f4SBryan Venteicher 40388126356SBryan Venteicher return (vq->vq_queue_index); 40488126356SBryan Venteicher } 40588126356SBryan Venteicher 40610b59a9bSPeter Grehan int 40710b59a9bSPeter Grehan virtqueue_size(struct virtqueue *vq) 40810b59a9bSPeter Grehan { 40910b59a9bSPeter Grehan 41010b59a9bSPeter Grehan return (vq->vq_nentries); 41110b59a9bSPeter Grehan } 41210b59a9bSPeter Grehan 41310b59a9bSPeter Grehan int 4144b59668fSBryan Venteicher virtqueue_nfree(struct virtqueue *vq) 4154b59668fSBryan Venteicher { 4164b59668fSBryan Venteicher 4174b59668fSBryan Venteicher return (vq->vq_free_cnt); 4184b59668fSBryan Venteicher } 4194b59668fSBryan Venteicher 420ccb576a8SMina Galić bool 42110b59a9bSPeter Grehan virtqueue_empty(struct virtqueue *vq) 42210b59a9bSPeter Grehan { 42310b59a9bSPeter Grehan 42410b59a9bSPeter Grehan return (vq->vq_nentries == vq->vq_free_cnt); 42510b59a9bSPeter Grehan } 42610b59a9bSPeter Grehan 427ccb576a8SMina Galić bool 42810b59a9bSPeter Grehan virtqueue_full(struct virtqueue *vq) 42910b59a9bSPeter Grehan { 43010b59a9bSPeter Grehan 43110b59a9bSPeter Grehan return (vq->vq_free_cnt == 0); 43210b59a9bSPeter Grehan } 43310b59a9bSPeter Grehan 43410b59a9bSPeter Grehan void 43510b59a9bSPeter Grehan virtqueue_notify(struct virtqueue *vq) 43610b59a9bSPeter Grehan { 437310dacd0SPeter Grehan 438b8a58707SPeter Grehan /* Ensure updated avail->idx is visible to host. */ 439b8a58707SPeter Grehan mb(); 44010b59a9bSPeter Grehan 441b8a58707SPeter Grehan if (vq_ring_must_notify_host(vq)) 442b8a58707SPeter Grehan vq_ring_notify_host(vq); 44310b59a9bSPeter Grehan vq->vq_queued_cnt = 0; 44410b59a9bSPeter Grehan } 44510b59a9bSPeter Grehan 44610b59a9bSPeter Grehan int 44710b59a9bSPeter Grehan virtqueue_nused(struct virtqueue *vq) 44810b59a9bSPeter Grehan { 44910b59a9bSPeter Grehan uint16_t used_idx, nused; 45010b59a9bSPeter Grehan 451fbe0c4f4SBryan Venteicher used_idx = vq_htog16(vq, vq->vq_ring.used->idx); 452b8a58707SPeter Grehan 453b8a58707SPeter Grehan nused = (uint16_t)(used_idx - vq->vq_used_cons_idx); 45410b59a9bSPeter Grehan VQASSERT(vq, nused <= vq->vq_nentries, "used more than available"); 45510b59a9bSPeter Grehan 45610b59a9bSPeter Grehan return (nused); 45710b59a9bSPeter Grehan } 45810b59a9bSPeter Grehan 45910b59a9bSPeter Grehan int 4606632efe4SBryan Venteicher virtqueue_intr_filter(struct virtqueue *vq) 4616632efe4SBryan Venteicher { 4626632efe4SBryan Venteicher 463fbe0c4f4SBryan Venteicher if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx)) 4646632efe4SBryan Venteicher return (0); 4656632efe4SBryan Venteicher 4666632efe4SBryan Venteicher virtqueue_disable_intr(vq); 4676632efe4SBryan Venteicher 4686632efe4SBryan Venteicher return (1); 4696632efe4SBryan Venteicher } 4706632efe4SBryan Venteicher 4716632efe4SBryan Venteicher void 47210b59a9bSPeter Grehan virtqueue_intr(struct virtqueue *vq) 47310b59a9bSPeter Grehan { 47410b59a9bSPeter Grehan 47510b59a9bSPeter Grehan vq->vq_intrhand(vq->vq_intrhand_arg); 47610b59a9bSPeter Grehan } 47710b59a9bSPeter Grehan 47810b59a9bSPeter Grehan int 47910b59a9bSPeter Grehan virtqueue_enable_intr(struct virtqueue *vq) 48010b59a9bSPeter Grehan { 48110b59a9bSPeter Grehan 482310dacd0SPeter Grehan return (vq_ring_enable_interrupt(vq, 0)); 48310b59a9bSPeter Grehan } 48410b59a9bSPeter Grehan 485b8a58707SPeter Grehan int 486b619f40aSBryan Venteicher virtqueue_postpone_intr(struct virtqueue *vq, vq_postpone_t hint) 487b8a58707SPeter Grehan { 488310dacd0SPeter Grehan uint16_t ndesc, avail_idx; 489b8a58707SPeter Grehan 490fbe0c4f4SBryan Venteicher avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx); 491b619f40aSBryan Venteicher ndesc = (uint16_t)(avail_idx - vq->vq_used_cons_idx); 492b619f40aSBryan Venteicher 493b619f40aSBryan Venteicher switch (hint) { 494b619f40aSBryan Venteicher case VQ_POSTPONE_SHORT: 4954142b1cbSBryan Venteicher ndesc = ndesc / 4; 496b619f40aSBryan Venteicher break; 497b619f40aSBryan Venteicher case VQ_POSTPONE_LONG: 4984142b1cbSBryan Venteicher ndesc = (ndesc * 3) / 4; 499b619f40aSBryan Venteicher break; 500b619f40aSBryan Venteicher case VQ_POSTPONE_EMPTIED: 501b619f40aSBryan Venteicher break; 502b619f40aSBryan Venteicher } 503b8a58707SPeter Grehan 504310dacd0SPeter Grehan return (vq_ring_enable_interrupt(vq, ndesc)); 505b8a58707SPeter Grehan } 506b8a58707SPeter Grehan 507b619f40aSBryan Venteicher /* 508b619f40aSBryan Venteicher * Note this is only considered a hint to the host. 509b619f40aSBryan Venteicher */ 51010b59a9bSPeter Grehan void 51110b59a9bSPeter Grehan virtqueue_disable_intr(struct virtqueue *vq) 51210b59a9bSPeter Grehan { 51310b59a9bSPeter Grehan 514b619f40aSBryan Venteicher if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) { 515fbe0c4f4SBryan Venteicher vring_used_event(&vq->vq_ring) = vq_gtoh16(vq, 516fbe0c4f4SBryan Venteicher vq->vq_used_cons_idx - vq->vq_nentries - 1); 517fbe0c4f4SBryan Venteicher return; 518fbe0c4f4SBryan Venteicher } 519fbe0c4f4SBryan Venteicher 520fbe0c4f4SBryan Venteicher vq->vq_ring.avail->flags |= vq_gtoh16(vq, VRING_AVAIL_F_NO_INTERRUPT); 52110b59a9bSPeter Grehan } 52210b59a9bSPeter Grehan 52310b59a9bSPeter Grehan int 52410b59a9bSPeter Grehan virtqueue_enqueue(struct virtqueue *vq, void *cookie, struct sglist *sg, 52510b59a9bSPeter Grehan int readable, int writable) 52610b59a9bSPeter Grehan { 52710b59a9bSPeter Grehan struct vq_desc_extra *dxp; 52810b59a9bSPeter Grehan int needed; 52910b59a9bSPeter Grehan uint16_t head_idx, idx; 53010b59a9bSPeter Grehan 53110b59a9bSPeter Grehan needed = readable + writable; 53210b59a9bSPeter Grehan 53310b59a9bSPeter Grehan VQASSERT(vq, cookie != NULL, "enqueuing with no cookie"); 53410b59a9bSPeter Grehan VQASSERT(vq, needed == sg->sg_nseg, 53510b59a9bSPeter Grehan "segment count mismatch, %d, %d", needed, sg->sg_nseg); 53610b59a9bSPeter Grehan VQASSERT(vq, 53710b59a9bSPeter Grehan needed <= vq->vq_nentries || needed <= vq->vq_max_indirect_size, 53810b59a9bSPeter Grehan "too many segments to enqueue: %d, %d/%d", needed, 53910b59a9bSPeter Grehan vq->vq_nentries, vq->vq_max_indirect_size); 54010b59a9bSPeter Grehan 54110b59a9bSPeter Grehan if (needed < 1) 54210b59a9bSPeter Grehan return (EINVAL); 54310b59a9bSPeter Grehan if (vq->vq_free_cnt == 0) 54410b59a9bSPeter Grehan return (ENOSPC); 54510b59a9bSPeter Grehan 54610b59a9bSPeter Grehan if (vq_ring_use_indirect(vq, needed)) { 54710b59a9bSPeter Grehan vq_ring_enqueue_indirect(vq, cookie, sg, readable, writable); 54810b59a9bSPeter Grehan return (0); 54910b59a9bSPeter Grehan } else if (vq->vq_free_cnt < needed) 55010b59a9bSPeter Grehan return (EMSGSIZE); 55110b59a9bSPeter Grehan 55210b59a9bSPeter Grehan head_idx = vq->vq_desc_head_idx; 55310b59a9bSPeter Grehan VQ_RING_ASSERT_VALID_IDX(vq, head_idx); 55410b59a9bSPeter Grehan dxp = &vq->vq_descx[head_idx]; 55510b59a9bSPeter Grehan 55610b59a9bSPeter Grehan VQASSERT(vq, dxp->cookie == NULL, 55710b59a9bSPeter Grehan "cookie already exists for index %d", head_idx); 55810b59a9bSPeter Grehan dxp->cookie = cookie; 55910b59a9bSPeter Grehan dxp->ndescs = needed; 56010b59a9bSPeter Grehan 56110b59a9bSPeter Grehan idx = vq_ring_enqueue_segments(vq, vq->vq_ring.desc, head_idx, 56210b59a9bSPeter Grehan sg, readable, writable); 56310b59a9bSPeter Grehan 56410b59a9bSPeter Grehan vq->vq_desc_head_idx = idx; 56510b59a9bSPeter Grehan vq->vq_free_cnt -= needed; 56610b59a9bSPeter Grehan if (vq->vq_free_cnt == 0) 56710b59a9bSPeter Grehan VQ_RING_ASSERT_CHAIN_TERM(vq); 56810b59a9bSPeter Grehan else 56910b59a9bSPeter Grehan VQ_RING_ASSERT_VALID_IDX(vq, idx); 57010b59a9bSPeter Grehan 57110b59a9bSPeter Grehan vq_ring_update_avail(vq, head_idx); 57210b59a9bSPeter Grehan 57310b59a9bSPeter Grehan return (0); 57410b59a9bSPeter Grehan } 57510b59a9bSPeter Grehan 57610b59a9bSPeter Grehan void * 57710b59a9bSPeter Grehan virtqueue_dequeue(struct virtqueue *vq, uint32_t *len) 57810b59a9bSPeter Grehan { 57910b59a9bSPeter Grehan struct vring_used_elem *uep; 58010b59a9bSPeter Grehan void *cookie; 58110b59a9bSPeter Grehan uint16_t used_idx, desc_idx; 58210b59a9bSPeter Grehan 583fbe0c4f4SBryan Venteicher if (vq->vq_used_cons_idx == vq_htog16(vq, vq->vq_ring.used->idx)) 58410b59a9bSPeter Grehan return (NULL); 58510b59a9bSPeter Grehan 58610b59a9bSPeter Grehan used_idx = vq->vq_used_cons_idx++ & (vq->vq_nentries - 1); 58710b59a9bSPeter Grehan uep = &vq->vq_ring.used->ring[used_idx]; 58810b59a9bSPeter Grehan 589c44ef550SPeter Grehan rmb(); 590fbe0c4f4SBryan Venteicher desc_idx = (uint16_t) vq_htog32(vq, uep->id); 59110b59a9bSPeter Grehan if (len != NULL) 592fbe0c4f4SBryan Venteicher *len = vq_htog32(vq, uep->len); 59310b59a9bSPeter Grehan 59410b59a9bSPeter Grehan vq_ring_free_chain(vq, desc_idx); 59510b59a9bSPeter Grehan 59610b59a9bSPeter Grehan cookie = vq->vq_descx[desc_idx].cookie; 59710b59a9bSPeter Grehan VQASSERT(vq, cookie != NULL, "no cookie for index %d", desc_idx); 59810b59a9bSPeter Grehan vq->vq_descx[desc_idx].cookie = NULL; 59910b59a9bSPeter Grehan 60010b59a9bSPeter Grehan return (cookie); 60110b59a9bSPeter Grehan } 60210b59a9bSPeter Grehan 60310b59a9bSPeter Grehan void * 60410b59a9bSPeter Grehan virtqueue_poll(struct virtqueue *vq, uint32_t *len) 60510b59a9bSPeter Grehan { 60610b59a9bSPeter Grehan void *cookie; 60710b59a9bSPeter Grehan 608156b97faSRuslan Bukin VIRTIO_BUS_POLL(vq->vq_dev); 609156b97faSRuslan Bukin while ((cookie = virtqueue_dequeue(vq, len)) == NULL) { 61010b59a9bSPeter Grehan cpu_spinwait(); 611156b97faSRuslan Bukin VIRTIO_BUS_POLL(vq->vq_dev); 612156b97faSRuslan Bukin } 61310b59a9bSPeter Grehan 61410b59a9bSPeter Grehan return (cookie); 61510b59a9bSPeter Grehan } 61610b59a9bSPeter Grehan 61710b59a9bSPeter Grehan void * 61810b59a9bSPeter Grehan virtqueue_drain(struct virtqueue *vq, int *last) 61910b59a9bSPeter Grehan { 62010b59a9bSPeter Grehan void *cookie; 62110b59a9bSPeter Grehan int idx; 62210b59a9bSPeter Grehan 62310b59a9bSPeter Grehan cookie = NULL; 62410b59a9bSPeter Grehan idx = *last; 62510b59a9bSPeter Grehan 62610b59a9bSPeter Grehan while (idx < vq->vq_nentries && cookie == NULL) { 62710b59a9bSPeter Grehan if ((cookie = vq->vq_descx[idx].cookie) != NULL) { 62810b59a9bSPeter Grehan vq->vq_descx[idx].cookie = NULL; 62910b59a9bSPeter Grehan /* Free chain to keep free count consistent. */ 63010b59a9bSPeter Grehan vq_ring_free_chain(vq, idx); 63110b59a9bSPeter Grehan } 63210b59a9bSPeter Grehan idx++; 63310b59a9bSPeter Grehan } 63410b59a9bSPeter Grehan 63510b59a9bSPeter Grehan *last = idx; 63610b59a9bSPeter Grehan 63710b59a9bSPeter Grehan return (cookie); 63810b59a9bSPeter Grehan } 63910b59a9bSPeter Grehan 64010b59a9bSPeter Grehan void 64110b59a9bSPeter Grehan virtqueue_dump(struct virtqueue *vq) 64210b59a9bSPeter Grehan { 64310b59a9bSPeter Grehan 64410b59a9bSPeter Grehan if (vq == NULL) 64510b59a9bSPeter Grehan return; 64610b59a9bSPeter Grehan 64710b59a9bSPeter Grehan printf("VQ: %s - size=%d; free=%d; used=%d; queued=%d; " 64810b59a9bSPeter Grehan "desc_head_idx=%d; avail.idx=%d; used_cons_idx=%d; " 649a5b6123eSLuigi Rizzo "used.idx=%d; used_event_idx=%d; avail.flags=0x%x; used.flags=0x%x\n", 650fbe0c4f4SBryan Venteicher vq->vq_name, vq->vq_nentries, vq->vq_free_cnt, virtqueue_nused(vq), 651fbe0c4f4SBryan Venteicher vq->vq_queued_cnt, vq->vq_desc_head_idx, 652fbe0c4f4SBryan Venteicher vq_htog16(vq, vq->vq_ring.avail->idx), vq->vq_used_cons_idx, 653fbe0c4f4SBryan Venteicher vq_htog16(vq, vq->vq_ring.used->idx), 654fbe0c4f4SBryan Venteicher vq_htog16(vq, vring_used_event(&vq->vq_ring)), 655fbe0c4f4SBryan Venteicher vq_htog16(vq, vq->vq_ring.avail->flags), 656fbe0c4f4SBryan Venteicher vq_htog16(vq, vq->vq_ring.used->flags)); 65710b59a9bSPeter Grehan } 65810b59a9bSPeter Grehan 65910b59a9bSPeter Grehan static void 66010b59a9bSPeter Grehan vq_ring_init(struct virtqueue *vq) 66110b59a9bSPeter Grehan { 66210b59a9bSPeter Grehan struct vring *vr; 66310b59a9bSPeter Grehan char *ring_mem; 66410b59a9bSPeter Grehan int i, size; 66510b59a9bSPeter Grehan 66610b59a9bSPeter Grehan ring_mem = vq->vq_ring_mem; 66710b59a9bSPeter Grehan size = vq->vq_nentries; 66810b59a9bSPeter Grehan vr = &vq->vq_ring; 66910b59a9bSPeter Grehan 67010b59a9bSPeter Grehan vring_init(vr, size, ring_mem, vq->vq_alignment); 67110b59a9bSPeter Grehan 67210b59a9bSPeter Grehan for (i = 0; i < size - 1; i++) 673fbe0c4f4SBryan Venteicher vr->desc[i].next = vq_gtoh16(vq, i + 1); 674fbe0c4f4SBryan Venteicher vr->desc[i].next = vq_gtoh16(vq, VQ_RING_DESC_CHAIN_END); 67510b59a9bSPeter Grehan } 67610b59a9bSPeter Grehan 67710b59a9bSPeter Grehan static void 67810b59a9bSPeter Grehan vq_ring_update_avail(struct virtqueue *vq, uint16_t desc_idx) 67910b59a9bSPeter Grehan { 680fbe0c4f4SBryan Venteicher uint16_t avail_idx, avail_ring_idx; 68110b59a9bSPeter Grehan 68210b59a9bSPeter Grehan /* 68310b59a9bSPeter Grehan * Place the head of the descriptor chain into the next slot and make 68410b59a9bSPeter Grehan * it usable to the host. The chain is made available now rather than 68510b59a9bSPeter Grehan * deferring to virtqueue_notify() in the hopes that if the host is 68610b59a9bSPeter Grehan * currently running on another CPU, we can keep it processing the new 68710b59a9bSPeter Grehan * descriptor. 68810b59a9bSPeter Grehan */ 689fbe0c4f4SBryan Venteicher avail_idx = vq_htog16(vq, vq->vq_ring.avail->idx); 690fbe0c4f4SBryan Venteicher avail_ring_idx = avail_idx & (vq->vq_nentries - 1); 691fbe0c4f4SBryan Venteicher vq->vq_ring.avail->ring[avail_ring_idx] = vq_gtoh16(vq, desc_idx); 69210b59a9bSPeter Grehan 693c44ef550SPeter Grehan wmb(); 694fbe0c4f4SBryan Venteicher vq->vq_ring.avail->idx = vq_gtoh16(vq, avail_idx + 1); 69510b59a9bSPeter Grehan 696b8a58707SPeter Grehan /* Keep pending count until virtqueue_notify(). */ 69710b59a9bSPeter Grehan vq->vq_queued_cnt++; 69810b59a9bSPeter Grehan } 69910b59a9bSPeter Grehan 70010b59a9bSPeter Grehan static uint16_t 70110b59a9bSPeter Grehan vq_ring_enqueue_segments(struct virtqueue *vq, struct vring_desc *desc, 70210b59a9bSPeter Grehan uint16_t head_idx, struct sglist *sg, int readable, int writable) 70310b59a9bSPeter Grehan { 70410b59a9bSPeter Grehan struct sglist_seg *seg; 70510b59a9bSPeter Grehan struct vring_desc *dp; 70610b59a9bSPeter Grehan int i, needed; 70710b59a9bSPeter Grehan uint16_t idx; 70810b59a9bSPeter Grehan 709fd80208aSStephen J. Kiernan SDT_PROBE6(virtqueue, , enqueue_segments, entry, vq, desc, head_idx, 710fd80208aSStephen J. Kiernan sg, readable, writable); 711fd80208aSStephen J. Kiernan 71210b59a9bSPeter Grehan needed = readable + writable; 71310b59a9bSPeter Grehan 71410b59a9bSPeter Grehan for (i = 0, idx = head_idx, seg = sg->sg_segs; 71510b59a9bSPeter Grehan i < needed; 716fbe0c4f4SBryan Venteicher i++, idx = vq_htog16(vq, dp->next), seg++) { 71710b59a9bSPeter Grehan VQASSERT(vq, idx != VQ_RING_DESC_CHAIN_END, 71810b59a9bSPeter Grehan "premature end of free desc chain"); 71910b59a9bSPeter Grehan 72010b59a9bSPeter Grehan dp = &desc[idx]; 721fbe0c4f4SBryan Venteicher dp->addr = vq_gtoh64(vq, seg->ss_paddr); 722fbe0c4f4SBryan Venteicher dp->len = vq_gtoh32(vq, seg->ss_len); 72310b59a9bSPeter Grehan dp->flags = 0; 72410b59a9bSPeter Grehan 72510b59a9bSPeter Grehan if (i < needed - 1) 726fbe0c4f4SBryan Venteicher dp->flags |= vq_gtoh16(vq, VRING_DESC_F_NEXT); 72710b59a9bSPeter Grehan if (i >= readable) 728fbe0c4f4SBryan Venteicher dp->flags |= vq_gtoh16(vq, VRING_DESC_F_WRITE); 72910b59a9bSPeter Grehan } 73010b59a9bSPeter Grehan 731fd80208aSStephen J. Kiernan SDT_PROBE1(virtqueue, , enqueue_segments, return, idx); 73210b59a9bSPeter Grehan return (idx); 73310b59a9bSPeter Grehan } 73410b59a9bSPeter Grehan 735ccb576a8SMina Galić static bool 73610b59a9bSPeter Grehan vq_ring_use_indirect(struct virtqueue *vq, int needed) 73710b59a9bSPeter Grehan { 73810b59a9bSPeter Grehan 73910b59a9bSPeter Grehan if ((vq->vq_flags & VIRTQUEUE_FLAG_INDIRECT) == 0) 740ccb576a8SMina Galić return (false); 74110b59a9bSPeter Grehan 74210b59a9bSPeter Grehan if (vq->vq_max_indirect_size < needed) 743ccb576a8SMina Galić return (false); 74410b59a9bSPeter Grehan 74510b59a9bSPeter Grehan if (needed < 2) 746ccb576a8SMina Galić return (false); 74710b59a9bSPeter Grehan 748ccb576a8SMina Galić return (true); 74910b59a9bSPeter Grehan } 75010b59a9bSPeter Grehan 75110b59a9bSPeter Grehan static void 75210b59a9bSPeter Grehan vq_ring_enqueue_indirect(struct virtqueue *vq, void *cookie, 75310b59a9bSPeter Grehan struct sglist *sg, int readable, int writable) 75410b59a9bSPeter Grehan { 75510b59a9bSPeter Grehan struct vring_desc *dp; 75610b59a9bSPeter Grehan struct vq_desc_extra *dxp; 75710b59a9bSPeter Grehan int needed; 75810b59a9bSPeter Grehan uint16_t head_idx; 75910b59a9bSPeter Grehan 76010b59a9bSPeter Grehan needed = readable + writable; 76110b59a9bSPeter Grehan VQASSERT(vq, needed <= vq->vq_max_indirect_size, 76210b59a9bSPeter Grehan "enqueuing too many indirect descriptors"); 76310b59a9bSPeter Grehan 76410b59a9bSPeter Grehan head_idx = vq->vq_desc_head_idx; 76510b59a9bSPeter Grehan VQ_RING_ASSERT_VALID_IDX(vq, head_idx); 76610b59a9bSPeter Grehan dp = &vq->vq_ring.desc[head_idx]; 76710b59a9bSPeter Grehan dxp = &vq->vq_descx[head_idx]; 76810b59a9bSPeter Grehan 76910b59a9bSPeter Grehan VQASSERT(vq, dxp->cookie == NULL, 77010b59a9bSPeter Grehan "cookie already exists for index %d", head_idx); 77110b59a9bSPeter Grehan dxp->cookie = cookie; 77210b59a9bSPeter Grehan dxp->ndescs = 1; 77310b59a9bSPeter Grehan 774fbe0c4f4SBryan Venteicher dp->addr = vq_gtoh64(vq, dxp->indirect_paddr); 775fbe0c4f4SBryan Venteicher dp->len = vq_gtoh32(vq, needed * sizeof(struct vring_desc)); 776fbe0c4f4SBryan Venteicher dp->flags = vq_gtoh16(vq, VRING_DESC_F_INDIRECT); 77710b59a9bSPeter Grehan 77810b59a9bSPeter Grehan vq_ring_enqueue_segments(vq, dxp->indirect, 0, 77910b59a9bSPeter Grehan sg, readable, writable); 78010b59a9bSPeter Grehan 781fbe0c4f4SBryan Venteicher vq->vq_desc_head_idx = vq_htog16(vq, dp->next); 78210b59a9bSPeter Grehan vq->vq_free_cnt--; 78310b59a9bSPeter Grehan if (vq->vq_free_cnt == 0) 78410b59a9bSPeter Grehan VQ_RING_ASSERT_CHAIN_TERM(vq); 78510b59a9bSPeter Grehan else 78610b59a9bSPeter Grehan VQ_RING_ASSERT_VALID_IDX(vq, vq->vq_desc_head_idx); 78710b59a9bSPeter Grehan 78810b59a9bSPeter Grehan vq_ring_update_avail(vq, head_idx); 78910b59a9bSPeter Grehan } 79010b59a9bSPeter Grehan 791b8a58707SPeter Grehan static int 792310dacd0SPeter Grehan vq_ring_enable_interrupt(struct virtqueue *vq, uint16_t ndesc) 793310dacd0SPeter Grehan { 794310dacd0SPeter Grehan 795310dacd0SPeter Grehan /* 796310dacd0SPeter Grehan * Enable interrupts, making sure we get the latest index of 797310dacd0SPeter Grehan * what's already been consumed. 798310dacd0SPeter Grehan */ 799fbe0c4f4SBryan Venteicher if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) { 800fbe0c4f4SBryan Venteicher vring_used_event(&vq->vq_ring) = 801fbe0c4f4SBryan Venteicher vq_gtoh16(vq, vq->vq_used_cons_idx + ndesc); 802fbe0c4f4SBryan Venteicher } else { 803fbe0c4f4SBryan Venteicher vq->vq_ring.avail->flags &= 804fbe0c4f4SBryan Venteicher vq_gtoh16(vq, ~VRING_AVAIL_F_NO_INTERRUPT); 805fbe0c4f4SBryan Venteicher } 806310dacd0SPeter Grehan 807310dacd0SPeter Grehan mb(); 808310dacd0SPeter Grehan 809310dacd0SPeter Grehan /* 810310dacd0SPeter Grehan * Enough items may have already been consumed to meet our threshold 811310dacd0SPeter Grehan * since we last checked. Let our caller know so it processes the new 812310dacd0SPeter Grehan * entries. 813310dacd0SPeter Grehan */ 814310dacd0SPeter Grehan if (virtqueue_nused(vq) > ndesc) 815310dacd0SPeter Grehan return (1); 816310dacd0SPeter Grehan 817310dacd0SPeter Grehan return (0); 818310dacd0SPeter Grehan } 819310dacd0SPeter Grehan 820310dacd0SPeter Grehan static int 821b8a58707SPeter Grehan vq_ring_must_notify_host(struct virtqueue *vq) 822b8a58707SPeter Grehan { 823fbe0c4f4SBryan Venteicher uint16_t new_idx, prev_idx, event_idx, flags; 824b8a58707SPeter Grehan 825b8a58707SPeter Grehan if (vq->vq_flags & VIRTQUEUE_FLAG_EVENT_IDX) { 826fbe0c4f4SBryan Venteicher new_idx = vq_htog16(vq, vq->vq_ring.avail->idx); 827b8a58707SPeter Grehan prev_idx = new_idx - vq->vq_queued_cnt; 828fbe0c4f4SBryan Venteicher event_idx = vq_htog16(vq, vring_avail_event(&vq->vq_ring)); 829b8a58707SPeter Grehan 830b8a58707SPeter Grehan return (vring_need_event(event_idx, new_idx, prev_idx) != 0); 831b8a58707SPeter Grehan } 832b8a58707SPeter Grehan 833fbe0c4f4SBryan Venteicher flags = vq->vq_ring.used->flags; 834fbe0c4f4SBryan Venteicher return ((flags & vq_gtoh16(vq, VRING_USED_F_NO_NOTIFY)) == 0); 835b8a58707SPeter Grehan } 836b8a58707SPeter Grehan 83710b59a9bSPeter Grehan static void 838b8a58707SPeter Grehan vq_ring_notify_host(struct virtqueue *vq) 83910b59a9bSPeter Grehan { 84010b59a9bSPeter Grehan 8419da9560cSBryan Venteicher VIRTIO_BUS_NOTIFY_VQ(vq->vq_dev, vq->vq_queue_index, 8429da9560cSBryan Venteicher vq->vq_notify_offset); 84310b59a9bSPeter Grehan } 84410b59a9bSPeter Grehan 84510b59a9bSPeter Grehan static void 84610b59a9bSPeter Grehan vq_ring_free_chain(struct virtqueue *vq, uint16_t desc_idx) 84710b59a9bSPeter Grehan { 84810b59a9bSPeter Grehan struct vring_desc *dp; 84910b59a9bSPeter Grehan struct vq_desc_extra *dxp; 85010b59a9bSPeter Grehan 85110b59a9bSPeter Grehan VQ_RING_ASSERT_VALID_IDX(vq, desc_idx); 85210b59a9bSPeter Grehan dp = &vq->vq_ring.desc[desc_idx]; 85310b59a9bSPeter Grehan dxp = &vq->vq_descx[desc_idx]; 85410b59a9bSPeter Grehan 85510b59a9bSPeter Grehan if (vq->vq_free_cnt == 0) 85610b59a9bSPeter Grehan VQ_RING_ASSERT_CHAIN_TERM(vq); 85710b59a9bSPeter Grehan 85810b59a9bSPeter Grehan vq->vq_free_cnt += dxp->ndescs; 8591dbd8bc9SPeter Grehan dxp->ndescs--; 86010b59a9bSPeter Grehan 861fbe0c4f4SBryan Venteicher if ((dp->flags & vq_gtoh16(vq, VRING_DESC_F_INDIRECT)) == 0) { 862fbe0c4f4SBryan Venteicher while (dp->flags & vq_gtoh16(vq, VRING_DESC_F_NEXT)) { 863fbe0c4f4SBryan Venteicher uint16_t next_idx = vq_htog16(vq, dp->next); 864fbe0c4f4SBryan Venteicher VQ_RING_ASSERT_VALID_IDX(vq, next_idx); 865fbe0c4f4SBryan Venteicher dp = &vq->vq_ring.desc[next_idx]; 86610b59a9bSPeter Grehan dxp->ndescs--; 86710b59a9bSPeter Grehan } 86810b59a9bSPeter Grehan } 8691dbd8bc9SPeter Grehan 8701dbd8bc9SPeter Grehan VQASSERT(vq, dxp->ndescs == 0, 871310dacd0SPeter Grehan "failed to free entire desc chain, remaining: %d", dxp->ndescs); 87210b59a9bSPeter Grehan 87310b59a9bSPeter Grehan /* 87410b59a9bSPeter Grehan * We must append the existing free chain, if any, to the end of 87510b59a9bSPeter Grehan * newly freed chain. If the virtqueue was completely used, then 87610b59a9bSPeter Grehan * head would be VQ_RING_DESC_CHAIN_END (ASSERTed above). 87710b59a9bSPeter Grehan */ 878fbe0c4f4SBryan Venteicher dp->next = vq_gtoh16(vq, vq->vq_desc_head_idx); 87910b59a9bSPeter Grehan vq->vq_desc_head_idx = desc_idx; 88010b59a9bSPeter Grehan } 881