178eb3293SRoger Pau Monné /*- 278eb3293SRoger Pau Monné * Copyright (c) 2016 Akshay Jaggi <jaggi@FreeBSD.org> 378eb3293SRoger Pau Monné * All rights reserved. 478eb3293SRoger Pau Monné * 578eb3293SRoger Pau Monné * Redistribution and use in source and binary forms, with or without 678eb3293SRoger Pau Monné * modification, are permitted provided that the following conditions 778eb3293SRoger Pau Monné * are met: 878eb3293SRoger Pau Monné * 1. Redistributions of source code must retain the above copyright 978eb3293SRoger Pau Monné * notice, this list of conditions and the following disclaimer. 1078eb3293SRoger Pau Monné * 2. Redistributions in binary form must reproduce the above copyright 1178eb3293SRoger Pau Monné * notice, this list of conditions and the following disclaimer in the 1278eb3293SRoger Pau Monné * documentation and/or other materials provided with the distribution. 1378eb3293SRoger Pau Monné * 1478eb3293SRoger Pau Monné * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 1578eb3293SRoger Pau Monné * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 1678eb3293SRoger Pau Monné * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 1778eb3293SRoger Pau Monné * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 1878eb3293SRoger Pau Monné * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 1978eb3293SRoger Pau Monné * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2078eb3293SRoger Pau Monné * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2178eb3293SRoger Pau Monné * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2278eb3293SRoger Pau Monné * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 2378eb3293SRoger Pau Monné * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 2478eb3293SRoger Pau Monné * SUCH DAMAGE. 2578eb3293SRoger Pau Monné * 2678eb3293SRoger Pau Monné * gntdev.c 2778eb3293SRoger Pau Monné * 2878eb3293SRoger Pau Monné * Interface to /dev/xen/gntdev. 2978eb3293SRoger Pau Monné * 3078eb3293SRoger Pau Monné */ 3178eb3293SRoger Pau Monné 3278eb3293SRoger Pau Monné #include <sys/param.h> 3378eb3293SRoger Pau Monné #include <sys/systm.h> 3478eb3293SRoger Pau Monné #include <sys/uio.h> 3578eb3293SRoger Pau Monné #include <sys/bus.h> 3678eb3293SRoger Pau Monné #include <sys/malloc.h> 3778eb3293SRoger Pau Monné #include <sys/kernel.h> 3878eb3293SRoger Pau Monné #include <sys/lock.h> 3978eb3293SRoger Pau Monné #include <sys/mutex.h> 4078eb3293SRoger Pau Monné #include <sys/rwlock.h> 4178eb3293SRoger Pau Monné #include <sys/selinfo.h> 4278eb3293SRoger Pau Monné #include <sys/poll.h> 4378eb3293SRoger Pau Monné #include <sys/conf.h> 4478eb3293SRoger Pau Monné #include <sys/fcntl.h> 4578eb3293SRoger Pau Monné #include <sys/ioccom.h> 4678eb3293SRoger Pau Monné #include <sys/rman.h> 4778eb3293SRoger Pau Monné #include <sys/tree.h> 4878eb3293SRoger Pau Monné #include <sys/module.h> 4978eb3293SRoger Pau Monné #include <sys/proc.h> 5078eb3293SRoger Pau Monné #include <sys/bitset.h> 5178eb3293SRoger Pau Monné #include <sys/queue.h> 5278eb3293SRoger Pau Monné #include <sys/mman.h> 5378eb3293SRoger Pau Monné #include <sys/syslog.h> 5478eb3293SRoger Pau Monné #include <sys/taskqueue.h> 5578eb3293SRoger Pau Monné 5678eb3293SRoger Pau Monné #include <vm/vm.h> 5778eb3293SRoger Pau Monné #include <vm/vm_param.h> 5878eb3293SRoger Pau Monné #include <vm/vm_extern.h> 5978eb3293SRoger Pau Monné #include <vm/vm_kern.h> 6078eb3293SRoger Pau Monné #include <vm/vm_page.h> 6178eb3293SRoger Pau Monné #include <vm/vm_map.h> 6278eb3293SRoger Pau Monné #include <vm/vm_object.h> 6378eb3293SRoger Pau Monné #include <vm/vm_pager.h> 6478eb3293SRoger Pau Monné 6578eb3293SRoger Pau Monné #include <machine/md_var.h> 6678eb3293SRoger Pau Monné 6778eb3293SRoger Pau Monné #include <xen/xen-os.h> 6878eb3293SRoger Pau Monné #include <xen/hypervisor.h> 6978eb3293SRoger Pau Monné #include <xen/error.h> 7078eb3293SRoger Pau Monné #include <xen/xen_intr.h> 7178eb3293SRoger Pau Monné #include <xen/gnttab.h> 7278eb3293SRoger Pau Monné #include <xen/gntdev.h> 7378eb3293SRoger Pau Monné 7478eb3293SRoger Pau Monné MALLOC_DEFINE(M_GNTDEV, "gntdev", "Xen grant-table user-space device"); 7578eb3293SRoger Pau Monné 7678eb3293SRoger Pau Monné #define MAX_OFFSET_COUNT ((0xffffffffffffffffull >> PAGE_SHIFT) + 1) 7778eb3293SRoger Pau Monné 7878eb3293SRoger Pau Monné static d_open_t gntdev_open; 7978eb3293SRoger Pau Monné static d_ioctl_t gntdev_ioctl; 8078eb3293SRoger Pau Monné static d_mmap_single_t gntdev_mmap_single; 8178eb3293SRoger Pau Monné 8278eb3293SRoger Pau Monné static struct cdevsw gntdev_devsw = { 8378eb3293SRoger Pau Monné .d_version = D_VERSION, 8478eb3293SRoger Pau Monné .d_open = gntdev_open, 8578eb3293SRoger Pau Monné .d_ioctl = gntdev_ioctl, 8678eb3293SRoger Pau Monné .d_mmap_single = gntdev_mmap_single, 8778eb3293SRoger Pau Monné .d_name = "gntdev", 8878eb3293SRoger Pau Monné }; 8978eb3293SRoger Pau Monné 9078eb3293SRoger Pau Monné static device_t gntdev_dev = NULL; 9178eb3293SRoger Pau Monné 9278eb3293SRoger Pau Monné struct gntdev_gref; 9378eb3293SRoger Pau Monné struct gntdev_gmap; 9478eb3293SRoger Pau Monné STAILQ_HEAD(gref_list_head, gntdev_gref); 9578eb3293SRoger Pau Monné STAILQ_HEAD(gmap_list_head, gntdev_gmap); 9678eb3293SRoger Pau Monné RB_HEAD(gref_tree_head, gntdev_gref); 9778eb3293SRoger Pau Monné RB_HEAD(gmap_tree_head, gntdev_gmap); 9878eb3293SRoger Pau Monné 9978eb3293SRoger Pau Monné struct file_offset_struct { 10078eb3293SRoger Pau Monné RB_ENTRY(file_offset_struct) next; 10178eb3293SRoger Pau Monné uint64_t file_offset; 10278eb3293SRoger Pau Monné uint64_t count; 10378eb3293SRoger Pau Monné }; 10478eb3293SRoger Pau Monné 10578eb3293SRoger Pau Monné static int 10678eb3293SRoger Pau Monné offset_cmp(struct file_offset_struct *f1, struct file_offset_struct *f2) 10778eb3293SRoger Pau Monné { 10878eb3293SRoger Pau Monné return (f1->file_offset - f2->file_offset); 10978eb3293SRoger Pau Monné } 11078eb3293SRoger Pau Monné 11178eb3293SRoger Pau Monné RB_HEAD(file_offset_head, file_offset_struct); 11278eb3293SRoger Pau Monné RB_GENERATE_STATIC(file_offset_head, file_offset_struct, next, offset_cmp); 11378eb3293SRoger Pau Monné 11478eb3293SRoger Pau Monné struct per_user_data { 11578eb3293SRoger Pau Monné struct mtx user_data_lock; 11678eb3293SRoger Pau Monné struct gref_tree_head gref_tree; 11778eb3293SRoger Pau Monné struct gmap_tree_head gmap_tree; 11878eb3293SRoger Pau Monné struct file_offset_head file_offset; 11978eb3293SRoger Pau Monné }; 12078eb3293SRoger Pau Monné 12178eb3293SRoger Pau Monné /* 12278eb3293SRoger Pau Monné * Get offset into the file which will be used while mmapping the 12378eb3293SRoger Pau Monné * appropriate pages by the userspace program. 12478eb3293SRoger Pau Monné */ 12578eb3293SRoger Pau Monné static int 12678eb3293SRoger Pau Monné get_file_offset(struct per_user_data *priv_user, uint32_t count, 12778eb3293SRoger Pau Monné uint64_t *file_offset) 12878eb3293SRoger Pau Monné { 12978eb3293SRoger Pau Monné struct file_offset_struct *offset, *offset_tmp; 13078eb3293SRoger Pau Monné 13178eb3293SRoger Pau Monné if (count == 0) 13278eb3293SRoger Pau Monné return (EINVAL); 13378eb3293SRoger Pau Monné mtx_lock(&priv_user->user_data_lock); 13478eb3293SRoger Pau Monné RB_FOREACH_SAFE(offset, file_offset_head, &priv_user->file_offset, 13578eb3293SRoger Pau Monné offset_tmp) { 13678eb3293SRoger Pau Monné if (offset->count >= count) { 13778eb3293SRoger Pau Monné offset->count -= count; 13878eb3293SRoger Pau Monné *file_offset = offset->file_offset + offset->count * 13978eb3293SRoger Pau Monné PAGE_SIZE; 14078eb3293SRoger Pau Monné if (offset->count == 0) { 14178eb3293SRoger Pau Monné RB_REMOVE(file_offset_head, 14278eb3293SRoger Pau Monné &priv_user->file_offset, offset); 14378eb3293SRoger Pau Monné free(offset, M_GNTDEV); 14478eb3293SRoger Pau Monné } 14578eb3293SRoger Pau Monné mtx_unlock(&priv_user->user_data_lock); 14678eb3293SRoger Pau Monné return (0); 14778eb3293SRoger Pau Monné } 14878eb3293SRoger Pau Monné } 14978eb3293SRoger Pau Monné mtx_unlock(&priv_user->user_data_lock); 15078eb3293SRoger Pau Monné 15178eb3293SRoger Pau Monné return (ENOSPC); 15278eb3293SRoger Pau Monné } 15378eb3293SRoger Pau Monné 15478eb3293SRoger Pau Monné static void 15578eb3293SRoger Pau Monné put_file_offset(struct per_user_data *priv_user, uint32_t count, 15678eb3293SRoger Pau Monné uint64_t file_offset) 15778eb3293SRoger Pau Monné { 15878eb3293SRoger Pau Monné struct file_offset_struct *offset, *offset_nxt, *offset_prv; 15978eb3293SRoger Pau Monné 16078eb3293SRoger Pau Monné offset = malloc(sizeof(*offset), M_GNTDEV, M_WAITOK | M_ZERO); 16178eb3293SRoger Pau Monné offset->file_offset = file_offset; 16278eb3293SRoger Pau Monné offset->count = count; 16378eb3293SRoger Pau Monné 16478eb3293SRoger Pau Monné mtx_lock(&priv_user->user_data_lock); 16578eb3293SRoger Pau Monné RB_INSERT(file_offset_head, &priv_user->file_offset, offset); 16678eb3293SRoger Pau Monné offset_nxt = RB_NEXT(file_offset_head, &priv_user->file_offset, offset); 16778eb3293SRoger Pau Monné offset_prv = RB_PREV(file_offset_head, &priv_user->file_offset, offset); 16878eb3293SRoger Pau Monné if (offset_nxt != NULL && 16978eb3293SRoger Pau Monné offset_nxt->file_offset == offset->file_offset + offset->count * 17078eb3293SRoger Pau Monné PAGE_SIZE) { 17178eb3293SRoger Pau Monné offset->count += offset_nxt->count; 17278eb3293SRoger Pau Monné RB_REMOVE(file_offset_head, &priv_user->file_offset, 17378eb3293SRoger Pau Monné offset_nxt); 17478eb3293SRoger Pau Monné free(offset_nxt, M_GNTDEV); 17578eb3293SRoger Pau Monné } 17678eb3293SRoger Pau Monné if (offset_prv != NULL && 17778eb3293SRoger Pau Monné offset->file_offset == offset_prv->file_offset + offset_prv->count * 17878eb3293SRoger Pau Monné PAGE_SIZE) { 17978eb3293SRoger Pau Monné offset_prv->count += offset->count; 18078eb3293SRoger Pau Monné RB_REMOVE(file_offset_head, &priv_user->file_offset, offset); 18178eb3293SRoger Pau Monné free(offset, M_GNTDEV); 18278eb3293SRoger Pau Monné } 18378eb3293SRoger Pau Monné mtx_unlock(&priv_user->user_data_lock); 18478eb3293SRoger Pau Monné } 18578eb3293SRoger Pau Monné 18678eb3293SRoger Pau Monné static int gntdev_gmap_pg_ctor(void *handle, vm_ooffset_t size, 18778eb3293SRoger Pau Monné vm_prot_t prot, vm_ooffset_t foff, struct ucred *cred, u_short *color); 18878eb3293SRoger Pau Monné static void gntdev_gmap_pg_dtor(void *handle); 18978eb3293SRoger Pau Monné static int gntdev_gmap_pg_fault(vm_object_t object, vm_ooffset_t offset, 19078eb3293SRoger Pau Monné int prot, vm_page_t *mres); 19178eb3293SRoger Pau Monné 19278eb3293SRoger Pau Monné static struct cdev_pager_ops gntdev_gmap_pg_ops = { 19378eb3293SRoger Pau Monné .cdev_pg_fault = gntdev_gmap_pg_fault, 19478eb3293SRoger Pau Monné .cdev_pg_ctor = gntdev_gmap_pg_ctor, 19578eb3293SRoger Pau Monné .cdev_pg_dtor = gntdev_gmap_pg_dtor, 19678eb3293SRoger Pau Monné }; 19778eb3293SRoger Pau Monné 19878eb3293SRoger Pau Monné struct cleanup_data_struct { 19978eb3293SRoger Pau Monné struct mtx to_kill_grefs_mtx; 20078eb3293SRoger Pau Monné struct mtx to_kill_gmaps_mtx; 20178eb3293SRoger Pau Monné struct gref_list_head to_kill_grefs; 20278eb3293SRoger Pau Monné struct gmap_list_head to_kill_gmaps; 20378eb3293SRoger Pau Monné }; 20478eb3293SRoger Pau Monné 20578eb3293SRoger Pau Monné static struct cleanup_data_struct cleanup_data = { 20678eb3293SRoger Pau Monné .to_kill_grefs = STAILQ_HEAD_INITIALIZER(cleanup_data.to_kill_grefs), 20778eb3293SRoger Pau Monné .to_kill_gmaps = STAILQ_HEAD_INITIALIZER(cleanup_data.to_kill_gmaps), 20878eb3293SRoger Pau Monné }; 20978eb3293SRoger Pau Monné MTX_SYSINIT(to_kill_grefs_mtx, &cleanup_data.to_kill_grefs_mtx, 21078eb3293SRoger Pau Monné "gntdev to_kill_grefs mutex", MTX_DEF); 21178eb3293SRoger Pau Monné MTX_SYSINIT(to_kill_gmaps_mtx, &cleanup_data.to_kill_gmaps_mtx, 21278eb3293SRoger Pau Monné "gntdev to_kill_gmaps mutex", MTX_DEF); 21378eb3293SRoger Pau Monné 21478eb3293SRoger Pau Monné static void cleanup_function(void *arg, __unused int pending); 21578eb3293SRoger Pau Monné static struct task cleanup_task = TASK_INITIALIZER(0, cleanup_function, 21678eb3293SRoger Pau Monné &cleanup_data); 21778eb3293SRoger Pau Monné 21878eb3293SRoger Pau Monné struct notify_data { 21978eb3293SRoger Pau Monné uint64_t index; 22078eb3293SRoger Pau Monné uint32_t action; 22178eb3293SRoger Pau Monné uint32_t event_channel_port; 22278eb3293SRoger Pau Monné xen_intr_handle_t notify_evtchn_handle; 22378eb3293SRoger Pau Monné }; 22478eb3293SRoger Pau Monné 22578eb3293SRoger Pau Monné static void notify(struct notify_data *notify, vm_page_t page); 22678eb3293SRoger Pau Monné 22778eb3293SRoger Pau Monné /*-------------------- Grant Allocation Methods -----------------------------*/ 22878eb3293SRoger Pau Monné 22978eb3293SRoger Pau Monné struct gntdev_gref { 23078eb3293SRoger Pau Monné union gref_next_union { 23178eb3293SRoger Pau Monné STAILQ_ENTRY(gntdev_gref) list; 23278eb3293SRoger Pau Monné RB_ENTRY(gntdev_gref) tree; 23378eb3293SRoger Pau Monné } gref_next; 23478eb3293SRoger Pau Monné uint64_t file_index; 23578eb3293SRoger Pau Monné grant_ref_t gref_id; 23678eb3293SRoger Pau Monné vm_page_t page; 23778eb3293SRoger Pau Monné struct notify_data *notify; 23878eb3293SRoger Pau Monné }; 23978eb3293SRoger Pau Monné 24078eb3293SRoger Pau Monné static int 24178eb3293SRoger Pau Monné gref_cmp(struct gntdev_gref *g1, struct gntdev_gref *g2) 24278eb3293SRoger Pau Monné { 24378eb3293SRoger Pau Monné return (g1->file_index - g2->file_index); 24478eb3293SRoger Pau Monné } 24578eb3293SRoger Pau Monné 24678eb3293SRoger Pau Monné RB_GENERATE_STATIC(gref_tree_head, gntdev_gref, gref_next.tree, gref_cmp); 24778eb3293SRoger Pau Monné 24878eb3293SRoger Pau Monné /* 24978eb3293SRoger Pau Monné * Traverse over the device-list of to-be-deleted grants allocated, and 25078eb3293SRoger Pau Monné * if all accesses, both local mmaps and foreign maps, to them have ended, 25178eb3293SRoger Pau Monné * destroy them. 25278eb3293SRoger Pau Monné */ 25378eb3293SRoger Pau Monné static void 25478eb3293SRoger Pau Monné gref_list_dtor(struct cleanup_data_struct *cleanup_data) 25578eb3293SRoger Pau Monné { 25678eb3293SRoger Pau Monné struct gref_list_head tmp_grefs; 25778eb3293SRoger Pau Monné struct gntdev_gref *gref, *gref_tmp, *gref_previous; 25878eb3293SRoger Pau Monné 25978eb3293SRoger Pau Monné STAILQ_INIT(&tmp_grefs); 26078eb3293SRoger Pau Monné mtx_lock(&cleanup_data->to_kill_grefs_mtx); 26178eb3293SRoger Pau Monné STAILQ_SWAP(&cleanup_data->to_kill_grefs, &tmp_grefs, gntdev_gref); 26278eb3293SRoger Pau Monné mtx_unlock(&cleanup_data->to_kill_grefs_mtx); 26378eb3293SRoger Pau Monné 26478eb3293SRoger Pau Monné gref_previous = NULL; 26578eb3293SRoger Pau Monné STAILQ_FOREACH_SAFE(gref, &tmp_grefs, gref_next.list, gref_tmp) { 26678eb3293SRoger Pau Monné if (gref->page && gref->page->object == NULL) { 26778eb3293SRoger Pau Monné if (gref->notify) { 26878eb3293SRoger Pau Monné notify(gref->notify, gref->page); 26978eb3293SRoger Pau Monné } 27078eb3293SRoger Pau Monné if (gref->gref_id != GRANT_REF_INVALID) { 27178eb3293SRoger Pau Monné if (gnttab_query_foreign_access(gref->gref_id)) 27278eb3293SRoger Pau Monné continue; 27378eb3293SRoger Pau Monné if (gnttab_end_foreign_access_ref(gref->gref_id) 27478eb3293SRoger Pau Monné == 0) 27578eb3293SRoger Pau Monné continue; 27678eb3293SRoger Pau Monné gnttab_free_grant_reference(gref->gref_id); 27778eb3293SRoger Pau Monné } 27888ea538aSMark Johnston vm_page_unwire_noq(gref->page); 27978eb3293SRoger Pau Monné vm_page_free(gref->page); 28078eb3293SRoger Pau Monné gref->page = NULL; 28178eb3293SRoger Pau Monné } 28278eb3293SRoger Pau Monné if (gref->page == NULL) { 28378eb3293SRoger Pau Monné if (gref_previous == NULL) 28478eb3293SRoger Pau Monné STAILQ_REMOVE_HEAD(&tmp_grefs, gref_next.list); 28578eb3293SRoger Pau Monné else 28678eb3293SRoger Pau Monné STAILQ_REMOVE_AFTER(&tmp_grefs, gref_previous, 28778eb3293SRoger Pau Monné gref_next.list); 28878eb3293SRoger Pau Monné if (gref->notify) 28978eb3293SRoger Pau Monné free(gref->notify, M_GNTDEV); 29078eb3293SRoger Pau Monné free(gref, M_GNTDEV); 29178eb3293SRoger Pau Monné } 29278eb3293SRoger Pau Monné else 29378eb3293SRoger Pau Monné gref_previous = gref; 29478eb3293SRoger Pau Monné } 29578eb3293SRoger Pau Monné 29678eb3293SRoger Pau Monné if (!STAILQ_EMPTY(&tmp_grefs)) { 29778eb3293SRoger Pau Monné mtx_lock(&cleanup_data->to_kill_grefs_mtx); 29878eb3293SRoger Pau Monné STAILQ_CONCAT(&cleanup_data->to_kill_grefs, &tmp_grefs); 29978eb3293SRoger Pau Monné mtx_unlock(&cleanup_data->to_kill_grefs_mtx); 30078eb3293SRoger Pau Monné } 30178eb3293SRoger Pau Monné } 30278eb3293SRoger Pau Monné 30378eb3293SRoger Pau Monné /* 30478eb3293SRoger Pau Monné * Find count number of contiguous allocated grants for a given userspace 30578eb3293SRoger Pau Monné * program by file-offset (index). 30678eb3293SRoger Pau Monné */ 30778eb3293SRoger Pau Monné static struct gntdev_gref* 30878eb3293SRoger Pau Monné gntdev_find_grefs(struct per_user_data *priv_user, 30978eb3293SRoger Pau Monné uint64_t index, uint32_t count) 31078eb3293SRoger Pau Monné { 31178eb3293SRoger Pau Monné struct gntdev_gref find_gref, *gref, *gref_start = NULL; 31278eb3293SRoger Pau Monné 31378eb3293SRoger Pau Monné find_gref.file_index = index; 31478eb3293SRoger Pau Monné 31578eb3293SRoger Pau Monné mtx_lock(&priv_user->user_data_lock); 31678eb3293SRoger Pau Monné gref_start = RB_FIND(gref_tree_head, &priv_user->gref_tree, &find_gref); 31778eb3293SRoger Pau Monné for (gref = gref_start; gref != NULL && count > 0; gref = 31878eb3293SRoger Pau Monné RB_NEXT(gref_tree_head, &priv_user->gref_tree, gref)) { 31978eb3293SRoger Pau Monné if (index != gref->file_index) 32078eb3293SRoger Pau Monné break; 32178eb3293SRoger Pau Monné index += PAGE_SIZE; 32278eb3293SRoger Pau Monné count--; 32378eb3293SRoger Pau Monné } 32478eb3293SRoger Pau Monné mtx_unlock(&priv_user->user_data_lock); 32578eb3293SRoger Pau Monné 32678eb3293SRoger Pau Monné if (count) 32778eb3293SRoger Pau Monné return (NULL); 32878eb3293SRoger Pau Monné return (gref_start); 32978eb3293SRoger Pau Monné } 33078eb3293SRoger Pau Monné 33178eb3293SRoger Pau Monné /* 33278eb3293SRoger Pau Monné * IOCTL_GNTDEV_ALLOC_GREF 33378eb3293SRoger Pau Monné * Allocate required number of wired pages for the request, grant foreign 33478eb3293SRoger Pau Monné * access to the physical frames for these pages, and add details about 33578eb3293SRoger Pau Monné * this allocation to the per user private data, so that these pages can 33678eb3293SRoger Pau Monné * be mmapped by the userspace program. 33778eb3293SRoger Pau Monné */ 33878eb3293SRoger Pau Monné static int 33978eb3293SRoger Pau Monné gntdev_alloc_gref(struct ioctl_gntdev_alloc_gref *arg) 34078eb3293SRoger Pau Monné { 34178eb3293SRoger Pau Monné uint32_t i; 34278eb3293SRoger Pau Monné int error, readonly; 34378eb3293SRoger Pau Monné uint64_t file_offset; 34478eb3293SRoger Pau Monné struct gntdev_gref *grefs; 34578eb3293SRoger Pau Monné struct per_user_data *priv_user; 34678eb3293SRoger Pau Monné 34778eb3293SRoger Pau Monné readonly = !(arg->flags & GNTDEV_ALLOC_FLAG_WRITABLE); 34878eb3293SRoger Pau Monné 34978eb3293SRoger Pau Monné error = devfs_get_cdevpriv((void**) &priv_user); 35078eb3293SRoger Pau Monné if (error != 0) 35178eb3293SRoger Pau Monné return (EINVAL); 35278eb3293SRoger Pau Monné 35378eb3293SRoger Pau Monné /* Cleanup grefs and free pages. */ 35478eb3293SRoger Pau Monné taskqueue_enqueue(taskqueue_thread, &cleanup_task); 35578eb3293SRoger Pau Monné 35678eb3293SRoger Pau Monné /* Get file offset for this request. */ 35778eb3293SRoger Pau Monné error = get_file_offset(priv_user, arg->count, &file_offset); 35878eb3293SRoger Pau Monné if (error != 0) 35978eb3293SRoger Pau Monné return (error); 36078eb3293SRoger Pau Monné 36178eb3293SRoger Pau Monné /* Allocate grefs. */ 36278eb3293SRoger Pau Monné grefs = malloc(sizeof(*grefs) * arg->count, M_GNTDEV, M_WAITOK); 36378eb3293SRoger Pau Monné 36478eb3293SRoger Pau Monné for (i = 0; i < arg->count; i++) { 36578eb3293SRoger Pau Monné grefs[i].file_index = file_offset + i * PAGE_SIZE; 36678eb3293SRoger Pau Monné grefs[i].gref_id = GRANT_REF_INVALID; 36778eb3293SRoger Pau Monné grefs[i].notify = NULL; 368a4667e09SMark Johnston grefs[i].page = vm_page_alloc_noobj(VM_ALLOC_WIRED | 369a4667e09SMark Johnston VM_ALLOC_ZERO); 37078eb3293SRoger Pau Monné if (grefs[i].page == NULL) { 37178eb3293SRoger Pau Monné log(LOG_ERR, "Page allocation failed."); 37278eb3293SRoger Pau Monné error = ENOMEM; 37378eb3293SRoger Pau Monné break; 37478eb3293SRoger Pau Monné } 37578eb3293SRoger Pau Monné grefs[i].page->valid = VM_PAGE_BITS_ALL; 37678eb3293SRoger Pau Monné 37778eb3293SRoger Pau Monné error = gnttab_grant_foreign_access(arg->domid, 37878eb3293SRoger Pau Monné (VM_PAGE_TO_PHYS(grefs[i].page) >> PAGE_SHIFT), 37978eb3293SRoger Pau Monné readonly, &grefs[i].gref_id); 38078eb3293SRoger Pau Monné if (error != 0) { 38178eb3293SRoger Pau Monné log(LOG_ERR, "Grant Table Hypercall failed."); 38278eb3293SRoger Pau Monné break; 38378eb3293SRoger Pau Monné } 38478eb3293SRoger Pau Monné } 38578eb3293SRoger Pau Monné 3866cdff09cSMark Johnston /* Copy the output values. */ 3876cdff09cSMark Johnston arg->index = file_offset; 3886cdff09cSMark Johnston for (i = 0; error == 0 && i < arg->count; i++) { 3896cdff09cSMark Johnston if (suword32(&arg->gref_ids[i], grefs[i].gref_id) != 0) 3906cdff09cSMark Johnston error = EFAULT; 3916cdff09cSMark Johnston } 3926cdff09cSMark Johnston 39378eb3293SRoger Pau Monné if (error != 0) { 39478eb3293SRoger Pau Monné /* 39578eb3293SRoger Pau Monné * If target domain maps the gref (by guessing the gref-id), 39678eb3293SRoger Pau Monné * then we can't clean it up yet and we have to leave the 39778eb3293SRoger Pau Monné * page in place so as to not leak our memory to that domain. 39878eb3293SRoger Pau Monné * Add it to a global list to be cleaned up later. 39978eb3293SRoger Pau Monné */ 40078eb3293SRoger Pau Monné mtx_lock(&cleanup_data.to_kill_grefs_mtx); 40178eb3293SRoger Pau Monné for (i = 0; i < arg->count; i++) 40278eb3293SRoger Pau Monné STAILQ_INSERT_TAIL(&cleanup_data.to_kill_grefs, 40378eb3293SRoger Pau Monné &grefs[i], gref_next.list); 40478eb3293SRoger Pau Monné mtx_unlock(&cleanup_data.to_kill_grefs_mtx); 40578eb3293SRoger Pau Monné 40678eb3293SRoger Pau Monné taskqueue_enqueue(taskqueue_thread, &cleanup_task); 40778eb3293SRoger Pau Monné 40878eb3293SRoger Pau Monné return (error); 40978eb3293SRoger Pau Monné } 41078eb3293SRoger Pau Monné 41178eb3293SRoger Pau Monné /* Modify the per user private data. */ 41278eb3293SRoger Pau Monné mtx_lock(&priv_user->user_data_lock); 41378eb3293SRoger Pau Monné for (i = 0; i < arg->count; i++) 41478eb3293SRoger Pau Monné RB_INSERT(gref_tree_head, &priv_user->gref_tree, &grefs[i]); 41578eb3293SRoger Pau Monné mtx_unlock(&priv_user->user_data_lock); 41678eb3293SRoger Pau Monné 41778eb3293SRoger Pau Monné return (error); 41878eb3293SRoger Pau Monné } 41978eb3293SRoger Pau Monné 42078eb3293SRoger Pau Monné /* 42178eb3293SRoger Pau Monné * IOCTL_GNTDEV_DEALLOC_GREF 42278eb3293SRoger Pau Monné * Remove grant allocation information from the per user private data, so 42378eb3293SRoger Pau Monné * that it can't be mmapped anymore by the userspace program, and add it 42478eb3293SRoger Pau Monné * to the to-be-deleted grants global device-list. 42578eb3293SRoger Pau Monné */ 42678eb3293SRoger Pau Monné static int 42778eb3293SRoger Pau Monné gntdev_dealloc_gref(struct ioctl_gntdev_dealloc_gref *arg) 42878eb3293SRoger Pau Monné { 42978eb3293SRoger Pau Monné int error; 43078eb3293SRoger Pau Monné uint32_t count; 43178eb3293SRoger Pau Monné struct gntdev_gref *gref, *gref_tmp; 43278eb3293SRoger Pau Monné struct per_user_data *priv_user; 43378eb3293SRoger Pau Monné 43478eb3293SRoger Pau Monné error = devfs_get_cdevpriv((void**) &priv_user); 43578eb3293SRoger Pau Monné if (error != 0) 43678eb3293SRoger Pau Monné return (EINVAL); 43778eb3293SRoger Pau Monné 43878eb3293SRoger Pau Monné gref = gntdev_find_grefs(priv_user, arg->index, arg->count); 43978eb3293SRoger Pau Monné if (gref == NULL) { 44078eb3293SRoger Pau Monné log(LOG_ERR, "Can't find requested grant-refs."); 44178eb3293SRoger Pau Monné return (EINVAL); 44278eb3293SRoger Pau Monné } 44378eb3293SRoger Pau Monné 44478eb3293SRoger Pau Monné /* Remove the grefs from user private data. */ 44578eb3293SRoger Pau Monné count = arg->count; 44678eb3293SRoger Pau Monné mtx_lock(&priv_user->user_data_lock); 44778eb3293SRoger Pau Monné mtx_lock(&cleanup_data.to_kill_grefs_mtx); 44878eb3293SRoger Pau Monné for (; gref != NULL && count > 0; gref = gref_tmp) { 44978eb3293SRoger Pau Monné gref_tmp = RB_NEXT(gref_tree_head, &priv_user->gref_tree, gref); 45078eb3293SRoger Pau Monné RB_REMOVE(gref_tree_head, &priv_user->gref_tree, gref); 45178eb3293SRoger Pau Monné STAILQ_INSERT_TAIL(&cleanup_data.to_kill_grefs, gref, 45278eb3293SRoger Pau Monné gref_next.list); 45378eb3293SRoger Pau Monné count--; 45478eb3293SRoger Pau Monné } 45578eb3293SRoger Pau Monné mtx_unlock(&cleanup_data.to_kill_grefs_mtx); 45678eb3293SRoger Pau Monné mtx_unlock(&priv_user->user_data_lock); 45778eb3293SRoger Pau Monné 45878eb3293SRoger Pau Monné taskqueue_enqueue(taskqueue_thread, &cleanup_task); 45978eb3293SRoger Pau Monné put_file_offset(priv_user, arg->count, arg->index); 46078eb3293SRoger Pau Monné 46178eb3293SRoger Pau Monné return (0); 46278eb3293SRoger Pau Monné } 46378eb3293SRoger Pau Monné 46478eb3293SRoger Pau Monné /*-------------------- Grant Mapping Methods --------------------------------*/ 46578eb3293SRoger Pau Monné 46678eb3293SRoger Pau Monné struct gntdev_gmap_map { 46778eb3293SRoger Pau Monné vm_object_t mem; 46878eb3293SRoger Pau Monné struct resource *pseudo_phys_res; 46978eb3293SRoger Pau Monné int pseudo_phys_res_id; 47078eb3293SRoger Pau Monné vm_paddr_t phys_base_addr; 47178eb3293SRoger Pau Monné }; 47278eb3293SRoger Pau Monné 47378eb3293SRoger Pau Monné struct gntdev_gmap { 47478eb3293SRoger Pau Monné union gmap_next_union { 47578eb3293SRoger Pau Monné STAILQ_ENTRY(gntdev_gmap) list; 47678eb3293SRoger Pau Monné RB_ENTRY(gntdev_gmap) tree; 47778eb3293SRoger Pau Monné } gmap_next; 47878eb3293SRoger Pau Monné uint64_t file_index; 47978eb3293SRoger Pau Monné uint32_t count; 48078eb3293SRoger Pau Monné struct gnttab_map_grant_ref *grant_map_ops; 48178eb3293SRoger Pau Monné struct gntdev_gmap_map *map; 48278eb3293SRoger Pau Monné struct notify_data *notify; 48378eb3293SRoger Pau Monné }; 48478eb3293SRoger Pau Monné 48578eb3293SRoger Pau Monné static int 48678eb3293SRoger Pau Monné gmap_cmp(struct gntdev_gmap *g1, struct gntdev_gmap *g2) 48778eb3293SRoger Pau Monné { 48878eb3293SRoger Pau Monné return (g1->file_index - g2->file_index); 48978eb3293SRoger Pau Monné } 49078eb3293SRoger Pau Monné 49178eb3293SRoger Pau Monné RB_GENERATE_STATIC(gmap_tree_head, gntdev_gmap, gmap_next.tree, gmap_cmp); 49278eb3293SRoger Pau Monné 49378eb3293SRoger Pau Monné /* 49478eb3293SRoger Pau Monné * Traverse over the device-list of to-be-deleted grant mappings, and if 49578eb3293SRoger Pau Monné * the region is no longer mmapped by anyone, free the memory used to 49678eb3293SRoger Pau Monné * store information about the mapping. 49778eb3293SRoger Pau Monné */ 49878eb3293SRoger Pau Monné static void 49978eb3293SRoger Pau Monné gmap_list_dtor(struct cleanup_data_struct *cleanup_data) 50078eb3293SRoger Pau Monné { 50178eb3293SRoger Pau Monné struct gmap_list_head tmp_gmaps; 50278eb3293SRoger Pau Monné struct gntdev_gmap *gmap, *gmap_tmp, *gmap_previous; 50378eb3293SRoger Pau Monné 50478eb3293SRoger Pau Monné STAILQ_INIT(&tmp_gmaps); 50578eb3293SRoger Pau Monné mtx_lock(&cleanup_data->to_kill_gmaps_mtx); 50678eb3293SRoger Pau Monné STAILQ_SWAP(&cleanup_data->to_kill_gmaps, &tmp_gmaps, gntdev_gmap); 50778eb3293SRoger Pau Monné mtx_unlock(&cleanup_data->to_kill_gmaps_mtx); 50878eb3293SRoger Pau Monné 50978eb3293SRoger Pau Monné gmap_previous = NULL; 51078eb3293SRoger Pau Monné STAILQ_FOREACH_SAFE(gmap, &tmp_gmaps, gmap_next.list, gmap_tmp) { 51178eb3293SRoger Pau Monné if (gmap->map == NULL) { 51278eb3293SRoger Pau Monné if (gmap_previous == NULL) 51378eb3293SRoger Pau Monné STAILQ_REMOVE_HEAD(&tmp_gmaps, gmap_next.list); 51478eb3293SRoger Pau Monné else 51578eb3293SRoger Pau Monné STAILQ_REMOVE_AFTER(&tmp_gmaps, gmap_previous, 51678eb3293SRoger Pau Monné gmap_next.list); 51778eb3293SRoger Pau Monné 51878eb3293SRoger Pau Monné if (gmap->notify) 51978eb3293SRoger Pau Monné free(gmap->notify, M_GNTDEV); 52078eb3293SRoger Pau Monné free(gmap->grant_map_ops, M_GNTDEV); 52178eb3293SRoger Pau Monné free(gmap, M_GNTDEV); 52278eb3293SRoger Pau Monné } 52378eb3293SRoger Pau Monné else 52478eb3293SRoger Pau Monné gmap_previous = gmap; 52578eb3293SRoger Pau Monné } 52678eb3293SRoger Pau Monné 52778eb3293SRoger Pau Monné if (!STAILQ_EMPTY(&tmp_gmaps)) { 52878eb3293SRoger Pau Monné mtx_lock(&cleanup_data->to_kill_gmaps_mtx); 52978eb3293SRoger Pau Monné STAILQ_CONCAT(&cleanup_data->to_kill_gmaps, &tmp_gmaps); 53078eb3293SRoger Pau Monné mtx_unlock(&cleanup_data->to_kill_gmaps_mtx); 53178eb3293SRoger Pau Monné } 53278eb3293SRoger Pau Monné } 53378eb3293SRoger Pau Monné 53478eb3293SRoger Pau Monné /* 53578eb3293SRoger Pau Monné * Find mapped grants for a given userspace program, by file-offset (index) 53678eb3293SRoger Pau Monné * and count, as supplied during the map-ioctl. 53778eb3293SRoger Pau Monné */ 53878eb3293SRoger Pau Monné static struct gntdev_gmap* 53978eb3293SRoger Pau Monné gntdev_find_gmap(struct per_user_data *priv_user, 54078eb3293SRoger Pau Monné uint64_t index, uint32_t count) 54178eb3293SRoger Pau Monné { 54278eb3293SRoger Pau Monné struct gntdev_gmap find_gmap, *gmap; 54378eb3293SRoger Pau Monné 54478eb3293SRoger Pau Monné find_gmap.file_index = index; 54578eb3293SRoger Pau Monné 54678eb3293SRoger Pau Monné mtx_lock(&priv_user->user_data_lock); 54778eb3293SRoger Pau Monné gmap = RB_FIND(gmap_tree_head, &priv_user->gmap_tree, &find_gmap); 54878eb3293SRoger Pau Monné mtx_unlock(&priv_user->user_data_lock); 54978eb3293SRoger Pau Monné 55078eb3293SRoger Pau Monné if (gmap != NULL && gmap->count == count) 55178eb3293SRoger Pau Monné return (gmap); 55278eb3293SRoger Pau Monné return (NULL); 55378eb3293SRoger Pau Monné } 55478eb3293SRoger Pau Monné 55578eb3293SRoger Pau Monné /* 55678eb3293SRoger Pau Monné * Remove the pages from the mgtdevice pager, call the unmap hypercall, 55778eb3293SRoger Pau Monné * free the xenmem resource. This function is called during the 55878eb3293SRoger Pau Monné * destruction of the mgtdevice pager, which happens when all mmaps to 55978eb3293SRoger Pau Monné * it have been removed, and the unmap-ioctl has been performed. 56078eb3293SRoger Pau Monné */ 56178eb3293SRoger Pau Monné static int 56278eb3293SRoger Pau Monné notify_unmap_cleanup(struct gntdev_gmap *gmap) 56378eb3293SRoger Pau Monné { 56478eb3293SRoger Pau Monné uint32_t i; 56578eb3293SRoger Pau Monné int error, count; 56678eb3293SRoger Pau Monné struct gnttab_unmap_grant_ref *unmap_ops; 56778eb3293SRoger Pau Monné 56878eb3293SRoger Pau Monné unmap_ops = malloc(sizeof(struct gnttab_unmap_grant_ref) * gmap->count, 56978eb3293SRoger Pau Monné M_GNTDEV, M_WAITOK); 57078eb3293SRoger Pau Monné 57178eb3293SRoger Pau Monné /* Enumerate freeable maps. */ 57278eb3293SRoger Pau Monné count = 0; 57378eb3293SRoger Pau Monné for (i = 0; i < gmap->count; i++) { 57478eb3293SRoger Pau Monné if (gmap->grant_map_ops[i].handle != -1) { 57578eb3293SRoger Pau Monné unmap_ops[count].handle = gmap->grant_map_ops[i].handle; 57678eb3293SRoger Pau Monné unmap_ops[count].host_addr = 57778eb3293SRoger Pau Monné gmap->grant_map_ops[i].host_addr; 57878eb3293SRoger Pau Monné unmap_ops[count].dev_bus_addr = 0; 57978eb3293SRoger Pau Monné count++; 58078eb3293SRoger Pau Monné } 58178eb3293SRoger Pau Monné } 58278eb3293SRoger Pau Monné 58378eb3293SRoger Pau Monné /* Perform notification. */ 58478eb3293SRoger Pau Monné if (count > 0 && gmap->notify) { 58578eb3293SRoger Pau Monné vm_page_t page; 58678eb3293SRoger Pau Monné uint64_t page_offset; 58778eb3293SRoger Pau Monné 58878eb3293SRoger Pau Monné page_offset = gmap->notify->index - gmap->file_index; 58978eb3293SRoger Pau Monné page = PHYS_TO_VM_PAGE(gmap->map->phys_base_addr + page_offset); 59078eb3293SRoger Pau Monné notify(gmap->notify, page); 59178eb3293SRoger Pau Monné } 59278eb3293SRoger Pau Monné 59378eb3293SRoger Pau Monné /* Free the pages. */ 594*38e3125dSDoug Moore cdev_mgtdev_pager_free_pages(gmap->map->mem); 59578eb3293SRoger Pau Monné 59678eb3293SRoger Pau Monné /* Perform unmap hypercall. */ 59778eb3293SRoger Pau Monné error = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, 59878eb3293SRoger Pau Monné unmap_ops, count); 59978eb3293SRoger Pau Monné 60078eb3293SRoger Pau Monné for (i = 0; i < gmap->count; i++) { 60178eb3293SRoger Pau Monné gmap->grant_map_ops[i].handle = -1; 60278eb3293SRoger Pau Monné gmap->grant_map_ops[i].host_addr = 0; 60378eb3293SRoger Pau Monné } 60478eb3293SRoger Pau Monné 60578eb3293SRoger Pau Monné if (gmap->map) { 60678eb3293SRoger Pau Monné error = xenmem_free(gntdev_dev, gmap->map->pseudo_phys_res_id, 60778eb3293SRoger Pau Monné gmap->map->pseudo_phys_res); 60878eb3293SRoger Pau Monné KASSERT(error == 0, 60978eb3293SRoger Pau Monné ("Unable to release memory resource: %d", error)); 61078eb3293SRoger Pau Monné 61178eb3293SRoger Pau Monné free(gmap->map, M_GNTDEV); 61278eb3293SRoger Pau Monné gmap->map = NULL; 61378eb3293SRoger Pau Monné } 61478eb3293SRoger Pau Monné 61578eb3293SRoger Pau Monné free(unmap_ops, M_GNTDEV); 61678eb3293SRoger Pau Monné 61778eb3293SRoger Pau Monné return (error); 61878eb3293SRoger Pau Monné } 61978eb3293SRoger Pau Monné 62078eb3293SRoger Pau Monné /* 62178eb3293SRoger Pau Monné * IOCTL_GNTDEV_MAP_GRANT_REF 62278eb3293SRoger Pau Monné * Populate structures for mapping the grant reference in the per user 62378eb3293SRoger Pau Monné * private data. Actual resource allocation and map hypercall is performed 62478eb3293SRoger Pau Monné * during the mmap. 62578eb3293SRoger Pau Monné */ 62678eb3293SRoger Pau Monné static int 62778eb3293SRoger Pau Monné gntdev_map_grant_ref(struct ioctl_gntdev_map_grant_ref *arg) 62878eb3293SRoger Pau Monné { 62978eb3293SRoger Pau Monné uint32_t i; 63078eb3293SRoger Pau Monné int error; 63178eb3293SRoger Pau Monné struct gntdev_gmap *gmap; 63278eb3293SRoger Pau Monné struct per_user_data *priv_user; 63378eb3293SRoger Pau Monné 63478eb3293SRoger Pau Monné error = devfs_get_cdevpriv((void**) &priv_user); 63578eb3293SRoger Pau Monné if (error != 0) 63678eb3293SRoger Pau Monné return (EINVAL); 63778eb3293SRoger Pau Monné 63878eb3293SRoger Pau Monné gmap = malloc(sizeof(*gmap), M_GNTDEV, M_WAITOK | M_ZERO); 63978eb3293SRoger Pau Monné gmap->count = arg->count; 64078eb3293SRoger Pau Monné gmap->grant_map_ops = 64178eb3293SRoger Pau Monné malloc(sizeof(struct gnttab_map_grant_ref) * arg->count, 64278eb3293SRoger Pau Monné M_GNTDEV, M_WAITOK | M_ZERO); 64378eb3293SRoger Pau Monné 64478eb3293SRoger Pau Monné for (i = 0; i < arg->count; i++) { 6452602ef7cSRoger Pau Monné struct ioctl_gntdev_grant_ref ref; 6462602ef7cSRoger Pau Monné 6472602ef7cSRoger Pau Monné error = copyin(&arg->refs[i], &ref, sizeof(ref)); 6482602ef7cSRoger Pau Monné if (error != 0) { 6492602ef7cSRoger Pau Monné free(gmap->grant_map_ops, M_GNTDEV); 6502602ef7cSRoger Pau Monné free(gmap, M_GNTDEV); 6512602ef7cSRoger Pau Monné return (error); 6522602ef7cSRoger Pau Monné } 6532602ef7cSRoger Pau Monné gmap->grant_map_ops[i].dom = ref.domid; 6542602ef7cSRoger Pau Monné gmap->grant_map_ops[i].ref = ref.ref; 65578eb3293SRoger Pau Monné gmap->grant_map_ops[i].handle = -1; 65678eb3293SRoger Pau Monné gmap->grant_map_ops[i].flags = GNTMAP_host_map; 65778eb3293SRoger Pau Monné } 65878eb3293SRoger Pau Monné 6592602ef7cSRoger Pau Monné error = get_file_offset(priv_user, arg->count, &gmap->file_index); 6602602ef7cSRoger Pau Monné if (error != 0) { 6612602ef7cSRoger Pau Monné free(gmap->grant_map_ops, M_GNTDEV); 6622602ef7cSRoger Pau Monné free(gmap, M_GNTDEV); 6632602ef7cSRoger Pau Monné return (error); 6642602ef7cSRoger Pau Monné } 6652602ef7cSRoger Pau Monné 66678eb3293SRoger Pau Monné mtx_lock(&priv_user->user_data_lock); 66778eb3293SRoger Pau Monné RB_INSERT(gmap_tree_head, &priv_user->gmap_tree, gmap); 66878eb3293SRoger Pau Monné mtx_unlock(&priv_user->user_data_lock); 66978eb3293SRoger Pau Monné 67078eb3293SRoger Pau Monné arg->index = gmap->file_index; 67178eb3293SRoger Pau Monné 67278eb3293SRoger Pau Monné return (error); 67378eb3293SRoger Pau Monné } 67478eb3293SRoger Pau Monné 67578eb3293SRoger Pau Monné /* 67678eb3293SRoger Pau Monné * IOCTL_GNTDEV_UNMAP_GRANT_REF 67778eb3293SRoger Pau Monné * Remove the map information from the per user private data and add it 67878eb3293SRoger Pau Monné * to the global device-list of mappings to be deleted. A reference to 67978eb3293SRoger Pau Monné * the mgtdevice pager is also decreased, the reason for which is 68078eb3293SRoger Pau Monné * explained in mmap_gmap(). 68178eb3293SRoger Pau Monné */ 68278eb3293SRoger Pau Monné static int 68378eb3293SRoger Pau Monné gntdev_unmap_grant_ref(struct ioctl_gntdev_unmap_grant_ref *arg) 68478eb3293SRoger Pau Monné { 68578eb3293SRoger Pau Monné int error; 68678eb3293SRoger Pau Monné struct gntdev_gmap *gmap; 68778eb3293SRoger Pau Monné struct per_user_data *priv_user; 68878eb3293SRoger Pau Monné 68978eb3293SRoger Pau Monné error = devfs_get_cdevpriv((void**) &priv_user); 69078eb3293SRoger Pau Monné if (error != 0) 69178eb3293SRoger Pau Monné return (EINVAL); 69278eb3293SRoger Pau Monné 69378eb3293SRoger Pau Monné gmap = gntdev_find_gmap(priv_user, arg->index, arg->count); 69478eb3293SRoger Pau Monné if (gmap == NULL) { 69578eb3293SRoger Pau Monné log(LOG_ERR, "Can't find requested grant-map."); 69678eb3293SRoger Pau Monné return (EINVAL); 69778eb3293SRoger Pau Monné } 69878eb3293SRoger Pau Monné 69978eb3293SRoger Pau Monné mtx_lock(&priv_user->user_data_lock); 70078eb3293SRoger Pau Monné mtx_lock(&cleanup_data.to_kill_gmaps_mtx); 70178eb3293SRoger Pau Monné RB_REMOVE(gmap_tree_head, &priv_user->gmap_tree, gmap); 70278eb3293SRoger Pau Monné STAILQ_INSERT_TAIL(&cleanup_data.to_kill_gmaps, gmap, gmap_next.list); 70378eb3293SRoger Pau Monné mtx_unlock(&cleanup_data.to_kill_gmaps_mtx); 70478eb3293SRoger Pau Monné mtx_unlock(&priv_user->user_data_lock); 70578eb3293SRoger Pau Monné 70678eb3293SRoger Pau Monné if (gmap->map) 70778eb3293SRoger Pau Monné vm_object_deallocate(gmap->map->mem); 70878eb3293SRoger Pau Monné 70978eb3293SRoger Pau Monné taskqueue_enqueue(taskqueue_thread, &cleanup_task); 71078eb3293SRoger Pau Monné put_file_offset(priv_user, arg->count, arg->index); 71178eb3293SRoger Pau Monné 71278eb3293SRoger Pau Monné return (0); 71378eb3293SRoger Pau Monné } 71478eb3293SRoger Pau Monné 71578eb3293SRoger Pau Monné /* 71678eb3293SRoger Pau Monné * IOCTL_GNTDEV_GET_OFFSET_FOR_VADDR 71778eb3293SRoger Pau Monné * Get file-offset and count for a given mapping, from the virtual address 71878eb3293SRoger Pau Monné * where the mapping is mmapped. 71978eb3293SRoger Pau Monné * Please note, this only works for grants mapped by this domain, and not 72078eb3293SRoger Pau Monné * grants allocated. Count doesn't make much sense in reference to grants 72178eb3293SRoger Pau Monné * allocated. Also, because this function is present in the linux gntdev 72278eb3293SRoger Pau Monné * device, but not in the linux gntalloc one, most userspace code only use 72378eb3293SRoger Pau Monné * it for mapped grants. 72478eb3293SRoger Pau Monné */ 72578eb3293SRoger Pau Monné static int 72678eb3293SRoger Pau Monné gntdev_get_offset_for_vaddr(struct ioctl_gntdev_get_offset_for_vaddr *arg, 72778eb3293SRoger Pau Monné struct thread *td) 72878eb3293SRoger Pau Monné { 72978eb3293SRoger Pau Monné int error; 73078eb3293SRoger Pau Monné vm_map_t map; 73178eb3293SRoger Pau Monné vm_map_entry_t entry; 73278eb3293SRoger Pau Monné vm_object_t mem; 73378eb3293SRoger Pau Monné vm_pindex_t pindex; 73478eb3293SRoger Pau Monné vm_prot_t prot; 73578eb3293SRoger Pau Monné boolean_t wired; 73678eb3293SRoger Pau Monné struct gntdev_gmap *gmap; 737b8aa60dbSRoger Pau Monné int rc; 73878eb3293SRoger Pau Monné 73978eb3293SRoger Pau Monné map = &td->td_proc->p_vmspace->vm_map; 74078eb3293SRoger Pau Monné error = vm_map_lookup(&map, arg->vaddr, VM_PROT_NONE, &entry, 74178eb3293SRoger Pau Monné &mem, &pindex, &prot, &wired); 74278eb3293SRoger Pau Monné if (error != KERN_SUCCESS) 74378eb3293SRoger Pau Monné return (EINVAL); 74478eb3293SRoger Pau Monné 74578eb3293SRoger Pau Monné if ((mem->type != OBJT_MGTDEVICE) || 746b8aa60dbSRoger Pau Monné (mem->un_pager.devp.ops != &gntdev_gmap_pg_ops)) { 747b8aa60dbSRoger Pau Monné rc = EINVAL; 748b8aa60dbSRoger Pau Monné goto out; 749b8aa60dbSRoger Pau Monné } 75078eb3293SRoger Pau Monné 75178eb3293SRoger Pau Monné gmap = mem->handle; 75278eb3293SRoger Pau Monné if (gmap == NULL || 753b8aa60dbSRoger Pau Monné (entry->end - entry->start) != (gmap->count * PAGE_SIZE)) { 754b8aa60dbSRoger Pau Monné rc = EINVAL; 755b8aa60dbSRoger Pau Monné goto out; 756b8aa60dbSRoger Pau Monné } 75778eb3293SRoger Pau Monné 75878eb3293SRoger Pau Monné arg->count = gmap->count; 75978eb3293SRoger Pau Monné arg->offset = gmap->file_index; 760b8aa60dbSRoger Pau Monné rc = 0; 761b8aa60dbSRoger Pau Monné 762b8aa60dbSRoger Pau Monné out: 763b8aa60dbSRoger Pau Monné vm_map_lookup_done(map, entry); 764b8aa60dbSRoger Pau Monné return (rc); 76578eb3293SRoger Pau Monné } 76678eb3293SRoger Pau Monné 76778eb3293SRoger Pau Monné /*-------------------- Grant Mapping Pager ----------------------------------*/ 76878eb3293SRoger Pau Monné 76978eb3293SRoger Pau Monné static int 77078eb3293SRoger Pau Monné gntdev_gmap_pg_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 77178eb3293SRoger Pau Monné vm_ooffset_t foff, struct ucred *cred, u_short *color) 77278eb3293SRoger Pau Monné { 77378eb3293SRoger Pau Monné 77478eb3293SRoger Pau Monné return (0); 77578eb3293SRoger Pau Monné } 77678eb3293SRoger Pau Monné 77778eb3293SRoger Pau Monné static void 77878eb3293SRoger Pau Monné gntdev_gmap_pg_dtor(void *handle) 77978eb3293SRoger Pau Monné { 78078eb3293SRoger Pau Monné 78178eb3293SRoger Pau Monné notify_unmap_cleanup((struct gntdev_gmap *)handle); 78278eb3293SRoger Pau Monné } 78378eb3293SRoger Pau Monné 78478eb3293SRoger Pau Monné static int 78578eb3293SRoger Pau Monné gntdev_gmap_pg_fault(vm_object_t object, vm_ooffset_t offset, int prot, 78678eb3293SRoger Pau Monné vm_page_t *mres) 78778eb3293SRoger Pau Monné { 78878eb3293SRoger Pau Monné struct gntdev_gmap *gmap = object->handle; 78978eb3293SRoger Pau Monné vm_pindex_t pidx, ridx; 7903cf3b4e6SJeff Roberson vm_page_t page; 79178eb3293SRoger Pau Monné vm_ooffset_t relative_offset; 79278eb3293SRoger Pau Monné 79378eb3293SRoger Pau Monné if (gmap->map == NULL) 79478eb3293SRoger Pau Monné return (VM_PAGER_FAIL); 79578eb3293SRoger Pau Monné 79678eb3293SRoger Pau Monné relative_offset = offset - gmap->file_index; 79778eb3293SRoger Pau Monné 79810d9120cSKonstantin Belousov pidx = OFF_TO_IDX(offset); 79910d9120cSKonstantin Belousov ridx = OFF_TO_IDX(relative_offset); 80078eb3293SRoger Pau Monné if (ridx >= gmap->count || 80178eb3293SRoger Pau Monné gmap->grant_map_ops[ridx].status != GNTST_okay) 80278eb3293SRoger Pau Monné return (VM_PAGER_FAIL); 80378eb3293SRoger Pau Monné 80478eb3293SRoger Pau Monné page = PHYS_TO_VM_PAGE(gmap->map->phys_base_addr + relative_offset); 80578eb3293SRoger Pau Monné if (page == NULL) 80678eb3293SRoger Pau Monné return (VM_PAGER_FAIL); 80778eb3293SRoger Pau Monné 80878eb3293SRoger Pau Monné KASSERT((page->flags & PG_FICTITIOUS) != 0, 80978eb3293SRoger Pau Monné ("not fictitious %p", page)); 810fee2a2faSMark Johnston KASSERT(vm_page_wired(page), ("page %p is not wired", page)); 811fee2a2faSMark Johnston KASSERT(!vm_page_busied(page), ("page %p is busy", page)); 81278eb3293SRoger Pau Monné 81363e97555SJeff Roberson vm_page_busy_acquire(page, 0); 8140012f373SJeff Roberson vm_page_valid(page); 8153cf3b4e6SJeff Roberson if (*mres != NULL) 8163cf3b4e6SJeff Roberson vm_page_replace(page, object, pidx, *mres); 8173cf3b4e6SJeff Roberson else 81878eb3293SRoger Pau Monné vm_page_insert(page, object, pidx); 81978eb3293SRoger Pau Monné *mres = page; 82078eb3293SRoger Pau Monné return (VM_PAGER_OK); 82178eb3293SRoger Pau Monné } 82278eb3293SRoger Pau Monné 82378eb3293SRoger Pau Monné /*------------------ Grant Table Methods ------------------------------------*/ 82478eb3293SRoger Pau Monné 82578eb3293SRoger Pau Monné static void 82678eb3293SRoger Pau Monné notify(struct notify_data *notify, vm_page_t page) 82778eb3293SRoger Pau Monné { 82878eb3293SRoger Pau Monné if (notify->action & UNMAP_NOTIFY_CLEAR_BYTE) { 82978eb3293SRoger Pau Monné uint8_t *mem; 83078eb3293SRoger Pau Monné uint64_t offset; 83178eb3293SRoger Pau Monné 83278eb3293SRoger Pau Monné offset = notify->index & PAGE_MASK; 83378eb3293SRoger Pau Monné mem = (uint8_t *)pmap_quick_enter_page(page); 83478eb3293SRoger Pau Monné mem[offset] = 0; 83578eb3293SRoger Pau Monné pmap_quick_remove_page((vm_offset_t)mem); 83678eb3293SRoger Pau Monné } 83778eb3293SRoger Pau Monné if (notify->action & UNMAP_NOTIFY_SEND_EVENT) { 83878eb3293SRoger Pau Monné xen_intr_signal(notify->notify_evtchn_handle); 83978eb3293SRoger Pau Monné xen_intr_unbind(¬ify->notify_evtchn_handle); 84078eb3293SRoger Pau Monné } 84178eb3293SRoger Pau Monné notify->action = 0; 84278eb3293SRoger Pau Monné } 84378eb3293SRoger Pau Monné 84478eb3293SRoger Pau Monné /* 84578eb3293SRoger Pau Monné * Helper to copy new arguments from the notify ioctl into 84678eb3293SRoger Pau Monné * the existing notify data. 84778eb3293SRoger Pau Monné */ 84878eb3293SRoger Pau Monné static int 84978eb3293SRoger Pau Monné copy_notify_helper(struct notify_data *destination, 85078eb3293SRoger Pau Monné struct ioctl_gntdev_unmap_notify *source) 85178eb3293SRoger Pau Monné { 85278eb3293SRoger Pau Monné xen_intr_handle_t handlep = NULL; 85378eb3293SRoger Pau Monné 85478eb3293SRoger Pau Monné /* 85578eb3293SRoger Pau Monné * "Get" before "Put"ting previous reference, as we might be 85678eb3293SRoger Pau Monné * holding the last reference to the event channel port. 85778eb3293SRoger Pau Monné */ 85878eb3293SRoger Pau Monné if (source->action & UNMAP_NOTIFY_SEND_EVENT) 85978eb3293SRoger Pau Monné if (xen_intr_get_evtchn_from_port(source->event_channel_port, 86078eb3293SRoger Pau Monné &handlep) != 0) 86178eb3293SRoger Pau Monné return (EINVAL); 86278eb3293SRoger Pau Monné 86378eb3293SRoger Pau Monné if (destination->action & UNMAP_NOTIFY_SEND_EVENT) 86478eb3293SRoger Pau Monné xen_intr_unbind(&destination->notify_evtchn_handle); 86578eb3293SRoger Pau Monné 86678eb3293SRoger Pau Monné destination->action = source->action; 86778eb3293SRoger Pau Monné destination->event_channel_port = source->event_channel_port; 86878eb3293SRoger Pau Monné destination->index = source->index; 86978eb3293SRoger Pau Monné destination->notify_evtchn_handle = handlep; 87078eb3293SRoger Pau Monné 87178eb3293SRoger Pau Monné return (0); 87278eb3293SRoger Pau Monné } 87378eb3293SRoger Pau Monné 87478eb3293SRoger Pau Monné /* 87578eb3293SRoger Pau Monné * IOCTL_GNTDEV_SET_UNMAP_NOTIFY 87678eb3293SRoger Pau Monné * Set unmap notification inside the appropriate grant. It sends a 87778eb3293SRoger Pau Monné * notification when the grant is completely munmapped by this domain 87878eb3293SRoger Pau Monné * and ready for destruction. 87978eb3293SRoger Pau Monné */ 88078eb3293SRoger Pau Monné static int 88178eb3293SRoger Pau Monné gntdev_set_unmap_notify(struct ioctl_gntdev_unmap_notify *arg) 88278eb3293SRoger Pau Monné { 88378eb3293SRoger Pau Monné int error; 88478eb3293SRoger Pau Monné uint64_t index; 88578eb3293SRoger Pau Monné struct per_user_data *priv_user; 88678eb3293SRoger Pau Monné struct gntdev_gref *gref = NULL; 88778eb3293SRoger Pau Monné struct gntdev_gmap *gmap; 88878eb3293SRoger Pau Monné 88978eb3293SRoger Pau Monné error = devfs_get_cdevpriv((void**) &priv_user); 89078eb3293SRoger Pau Monné if (error != 0) 89178eb3293SRoger Pau Monné return (EINVAL); 89278eb3293SRoger Pau Monné 89378eb3293SRoger Pau Monné if (arg->action & ~(UNMAP_NOTIFY_CLEAR_BYTE|UNMAP_NOTIFY_SEND_EVENT)) 89478eb3293SRoger Pau Monné return (EINVAL); 89578eb3293SRoger Pau Monné 89678eb3293SRoger Pau Monné index = arg->index & ~PAGE_MASK; 89778eb3293SRoger Pau Monné gref = gntdev_find_grefs(priv_user, index, 1); 89878eb3293SRoger Pau Monné if (gref) { 89978eb3293SRoger Pau Monné if (gref->notify == NULL) 90078eb3293SRoger Pau Monné gref->notify = malloc(sizeof(*arg), M_GNTDEV, 90178eb3293SRoger Pau Monné M_WAITOK | M_ZERO); 90278eb3293SRoger Pau Monné return (copy_notify_helper(gref->notify, arg)); 90378eb3293SRoger Pau Monné } 90478eb3293SRoger Pau Monné 90578eb3293SRoger Pau Monné error = EINVAL; 90678eb3293SRoger Pau Monné mtx_lock(&priv_user->user_data_lock); 90778eb3293SRoger Pau Monné RB_FOREACH(gmap, gmap_tree_head, &priv_user->gmap_tree) { 90878eb3293SRoger Pau Monné if (arg->index >= gmap->file_index && 90978eb3293SRoger Pau Monné arg->index < gmap->file_index + gmap->count * PAGE_SIZE) { 91078eb3293SRoger Pau Monné if (gmap->notify == NULL) 91178eb3293SRoger Pau Monné gmap->notify = malloc(sizeof(*arg), M_GNTDEV, 91278eb3293SRoger Pau Monné M_WAITOK | M_ZERO); 91378eb3293SRoger Pau Monné error = copy_notify_helper(gmap->notify, arg); 91478eb3293SRoger Pau Monné break; 91578eb3293SRoger Pau Monné } 91678eb3293SRoger Pau Monné } 91778eb3293SRoger Pau Monné mtx_unlock(&priv_user->user_data_lock); 91878eb3293SRoger Pau Monné 91978eb3293SRoger Pau Monné return (error); 92078eb3293SRoger Pau Monné } 92178eb3293SRoger Pau Monné 92278eb3293SRoger Pau Monné /*------------------ Gntdev Char Device Methods -----------------------------*/ 92378eb3293SRoger Pau Monné 92478eb3293SRoger Pau Monné static void 92578eb3293SRoger Pau Monné cleanup_function(void *arg, __unused int pending) 92678eb3293SRoger Pau Monné { 92778eb3293SRoger Pau Monné 92878eb3293SRoger Pau Monné gref_list_dtor((struct cleanup_data_struct *) arg); 92978eb3293SRoger Pau Monné gmap_list_dtor((struct cleanup_data_struct *) arg); 93078eb3293SRoger Pau Monné } 93178eb3293SRoger Pau Monné 93278eb3293SRoger Pau Monné static void 93378eb3293SRoger Pau Monné per_user_data_dtor(void *arg) 93478eb3293SRoger Pau Monné { 93578eb3293SRoger Pau Monné struct gntdev_gref *gref, *gref_tmp; 93678eb3293SRoger Pau Monné struct gntdev_gmap *gmap, *gmap_tmp; 93778eb3293SRoger Pau Monné struct file_offset_struct *offset, *offset_tmp; 93878eb3293SRoger Pau Monné struct per_user_data *priv_user; 93978eb3293SRoger Pau Monné 94078eb3293SRoger Pau Monné priv_user = (struct per_user_data *) arg; 94178eb3293SRoger Pau Monné 94278eb3293SRoger Pau Monné mtx_lock(&priv_user->user_data_lock); 94378eb3293SRoger Pau Monné 94478eb3293SRoger Pau Monné mtx_lock(&cleanup_data.to_kill_grefs_mtx); 94578eb3293SRoger Pau Monné RB_FOREACH_SAFE(gref, gref_tree_head, &priv_user->gref_tree, gref_tmp) { 94678eb3293SRoger Pau Monné RB_REMOVE(gref_tree_head, &priv_user->gref_tree, gref); 94778eb3293SRoger Pau Monné STAILQ_INSERT_TAIL(&cleanup_data.to_kill_grefs, gref, 94878eb3293SRoger Pau Monné gref_next.list); 94978eb3293SRoger Pau Monné } 95078eb3293SRoger Pau Monné mtx_unlock(&cleanup_data.to_kill_grefs_mtx); 95178eb3293SRoger Pau Monné 95278eb3293SRoger Pau Monné mtx_lock(&cleanup_data.to_kill_gmaps_mtx); 95378eb3293SRoger Pau Monné RB_FOREACH_SAFE(gmap, gmap_tree_head, &priv_user->gmap_tree, gmap_tmp) { 95478eb3293SRoger Pau Monné RB_REMOVE(gmap_tree_head, &priv_user->gmap_tree, gmap); 95578eb3293SRoger Pau Monné STAILQ_INSERT_TAIL(&cleanup_data.to_kill_gmaps, gmap, 95678eb3293SRoger Pau Monné gmap_next.list); 95778eb3293SRoger Pau Monné if (gmap->map) 95878eb3293SRoger Pau Monné vm_object_deallocate(gmap->map->mem); 95978eb3293SRoger Pau Monné } 96078eb3293SRoger Pau Monné mtx_unlock(&cleanup_data.to_kill_gmaps_mtx); 96178eb3293SRoger Pau Monné 96278eb3293SRoger Pau Monné RB_FOREACH_SAFE(offset, file_offset_head, &priv_user->file_offset, 96378eb3293SRoger Pau Monné offset_tmp) { 96478eb3293SRoger Pau Monné RB_REMOVE(file_offset_head, &priv_user->file_offset, offset); 96578eb3293SRoger Pau Monné free(offset, M_GNTDEV); 96678eb3293SRoger Pau Monné } 96778eb3293SRoger Pau Monné 96878eb3293SRoger Pau Monné mtx_unlock(&priv_user->user_data_lock); 96978eb3293SRoger Pau Monné 97078eb3293SRoger Pau Monné taskqueue_enqueue(taskqueue_thread, &cleanup_task); 97178eb3293SRoger Pau Monné 97278eb3293SRoger Pau Monné mtx_destroy(&priv_user->user_data_lock); 97378eb3293SRoger Pau Monné free(priv_user, M_GNTDEV); 97478eb3293SRoger Pau Monné } 97578eb3293SRoger Pau Monné 97678eb3293SRoger Pau Monné static int 97778eb3293SRoger Pau Monné gntdev_open(struct cdev *dev, int flag, int otyp, struct thread *td) 97878eb3293SRoger Pau Monné { 97978eb3293SRoger Pau Monné int error; 98078eb3293SRoger Pau Monné struct per_user_data *priv_user; 98178eb3293SRoger Pau Monné struct file_offset_struct *offset; 98278eb3293SRoger Pau Monné 98378eb3293SRoger Pau Monné priv_user = malloc(sizeof(*priv_user), M_GNTDEV, M_WAITOK | M_ZERO); 98478eb3293SRoger Pau Monné RB_INIT(&priv_user->gref_tree); 98578eb3293SRoger Pau Monné RB_INIT(&priv_user->gmap_tree); 98678eb3293SRoger Pau Monné RB_INIT(&priv_user->file_offset); 98778eb3293SRoger Pau Monné offset = malloc(sizeof(*offset), M_GNTDEV, M_WAITOK | M_ZERO); 98878eb3293SRoger Pau Monné offset->file_offset = 0; 98978eb3293SRoger Pau Monné offset->count = MAX_OFFSET_COUNT; 99078eb3293SRoger Pau Monné RB_INSERT(file_offset_head, &priv_user->file_offset, offset); 99178eb3293SRoger Pau Monné mtx_init(&priv_user->user_data_lock, 99278eb3293SRoger Pau Monné "per user data mutex", NULL, MTX_DEF); 99378eb3293SRoger Pau Monné 99478eb3293SRoger Pau Monné error = devfs_set_cdevpriv(priv_user, per_user_data_dtor); 99578eb3293SRoger Pau Monné if (error != 0) 99678eb3293SRoger Pau Monné per_user_data_dtor(priv_user); 99778eb3293SRoger Pau Monné 99878eb3293SRoger Pau Monné return (error); 99978eb3293SRoger Pau Monné } 100078eb3293SRoger Pau Monné 100178eb3293SRoger Pau Monné static int 100278eb3293SRoger Pau Monné gntdev_ioctl(struct cdev *dev, u_long cmd, caddr_t data, 100378eb3293SRoger Pau Monné int fflag, struct thread *td) 100478eb3293SRoger Pau Monné { 100578eb3293SRoger Pau Monné int error; 100678eb3293SRoger Pau Monné 100778eb3293SRoger Pau Monné switch (cmd) { 100878eb3293SRoger Pau Monné case IOCTL_GNTDEV_SET_UNMAP_NOTIFY: 100978eb3293SRoger Pau Monné error = gntdev_set_unmap_notify( 101078eb3293SRoger Pau Monné (struct ioctl_gntdev_unmap_notify*) data); 101178eb3293SRoger Pau Monné break; 101278eb3293SRoger Pau Monné case IOCTL_GNTDEV_ALLOC_GREF: 101378eb3293SRoger Pau Monné error = gntdev_alloc_gref( 101478eb3293SRoger Pau Monné (struct ioctl_gntdev_alloc_gref*) data); 101578eb3293SRoger Pau Monné break; 101678eb3293SRoger Pau Monné case IOCTL_GNTDEV_DEALLOC_GREF: 101778eb3293SRoger Pau Monné error = gntdev_dealloc_gref( 101878eb3293SRoger Pau Monné (struct ioctl_gntdev_dealloc_gref*) data); 101978eb3293SRoger Pau Monné break; 102078eb3293SRoger Pau Monné case IOCTL_GNTDEV_MAP_GRANT_REF: 102178eb3293SRoger Pau Monné error = gntdev_map_grant_ref( 102278eb3293SRoger Pau Monné (struct ioctl_gntdev_map_grant_ref*) data); 102378eb3293SRoger Pau Monné break; 102478eb3293SRoger Pau Monné case IOCTL_GNTDEV_UNMAP_GRANT_REF: 102578eb3293SRoger Pau Monné error = gntdev_unmap_grant_ref( 102678eb3293SRoger Pau Monné (struct ioctl_gntdev_unmap_grant_ref*) data); 102778eb3293SRoger Pau Monné break; 102878eb3293SRoger Pau Monné case IOCTL_GNTDEV_GET_OFFSET_FOR_VADDR: 102978eb3293SRoger Pau Monné error = gntdev_get_offset_for_vaddr( 103078eb3293SRoger Pau Monné (struct ioctl_gntdev_get_offset_for_vaddr*) data, td); 103178eb3293SRoger Pau Monné break; 103278eb3293SRoger Pau Monné default: 103378eb3293SRoger Pau Monné error = ENOSYS; 103478eb3293SRoger Pau Monné break; 103578eb3293SRoger Pau Monné } 103678eb3293SRoger Pau Monné 103778eb3293SRoger Pau Monné return (error); 103878eb3293SRoger Pau Monné } 103978eb3293SRoger Pau Monné 104078eb3293SRoger Pau Monné /* 104178eb3293SRoger Pau Monné * MMAP an allocated grant into user memory. 104278eb3293SRoger Pau Monné * Please note, that the grants must not already be mmapped, otherwise 104378eb3293SRoger Pau Monné * this function will fail. 104478eb3293SRoger Pau Monné */ 104578eb3293SRoger Pau Monné static int 104678eb3293SRoger Pau Monné mmap_gref(struct per_user_data *priv_user, struct gntdev_gref *gref_start, 104778eb3293SRoger Pau Monné uint32_t count, vm_size_t size, struct vm_object **object) 104878eb3293SRoger Pau Monné { 104978eb3293SRoger Pau Monné vm_object_t mem_obj; 105078eb3293SRoger Pau Monné struct gntdev_gref *gref; 105178eb3293SRoger Pau Monné 1052fbf2a778SKonstantin Belousov mem_obj = vm_pager_allocate(OBJT_PHYS, NULL, size, VM_PROT_ALL, 0, 1053fbf2a778SKonstantin Belousov curthread->td_ucred); 105478eb3293SRoger Pau Monné if (mem_obj == NULL) 105578eb3293SRoger Pau Monné return (ENOMEM); 105678eb3293SRoger Pau Monné 105778eb3293SRoger Pau Monné mtx_lock(&priv_user->user_data_lock); 105878eb3293SRoger Pau Monné VM_OBJECT_WLOCK(mem_obj); 105978eb3293SRoger Pau Monné for (gref = gref_start; gref != NULL && count > 0; gref = 106078eb3293SRoger Pau Monné RB_NEXT(gref_tree_head, &priv_user->gref_tree, gref)) { 106178eb3293SRoger Pau Monné if (gref->page->object) 106278eb3293SRoger Pau Monné break; 106378eb3293SRoger Pau Monné 106478eb3293SRoger Pau Monné vm_page_insert(gref->page, mem_obj, 106510d9120cSKonstantin Belousov OFF_TO_IDX(gref->file_index)); 106678eb3293SRoger Pau Monné 106778eb3293SRoger Pau Monné count--; 106878eb3293SRoger Pau Monné } 106978eb3293SRoger Pau Monné VM_OBJECT_WUNLOCK(mem_obj); 107078eb3293SRoger Pau Monné mtx_unlock(&priv_user->user_data_lock); 107178eb3293SRoger Pau Monné 107278eb3293SRoger Pau Monné if (count) { 107378eb3293SRoger Pau Monné vm_object_deallocate(mem_obj); 107478eb3293SRoger Pau Monné return (EINVAL); 107578eb3293SRoger Pau Monné } 107678eb3293SRoger Pau Monné 107778eb3293SRoger Pau Monné *object = mem_obj; 107878eb3293SRoger Pau Monné 107978eb3293SRoger Pau Monné return (0); 108078eb3293SRoger Pau Monné 108178eb3293SRoger Pau Monné } 108278eb3293SRoger Pau Monné 108378eb3293SRoger Pau Monné /* 108478eb3293SRoger Pau Monné * MMAP a mapped grant into user memory. 108578eb3293SRoger Pau Monné */ 108678eb3293SRoger Pau Monné static int 108778eb3293SRoger Pau Monné mmap_gmap(struct per_user_data *priv_user, struct gntdev_gmap *gmap_start, 108878eb3293SRoger Pau Monné vm_ooffset_t *offset, vm_size_t size, struct vm_object **object, int nprot) 108978eb3293SRoger Pau Monné { 109078eb3293SRoger Pau Monné uint32_t i; 109178eb3293SRoger Pau Monné int error; 109278eb3293SRoger Pau Monné 109378eb3293SRoger Pau Monné /* 109478eb3293SRoger Pau Monné * The grant map hypercall might already be done. 109578eb3293SRoger Pau Monné * If that is the case, increase a reference to the 109678eb3293SRoger Pau Monné * vm object and return the already allocated object. 109778eb3293SRoger Pau Monné */ 109878eb3293SRoger Pau Monné if (gmap_start->map) { 109978eb3293SRoger Pau Monné vm_object_reference(gmap_start->map->mem); 110078eb3293SRoger Pau Monné *object = gmap_start->map->mem; 110178eb3293SRoger Pau Monné return (0); 110278eb3293SRoger Pau Monné } 110378eb3293SRoger Pau Monné 110478eb3293SRoger Pau Monné gmap_start->map = malloc(sizeof(*(gmap_start->map)), M_GNTDEV, 110578eb3293SRoger Pau Monné M_WAITOK | M_ZERO); 110678eb3293SRoger Pau Monné 110778eb3293SRoger Pau Monné /* Allocate the xen pseudo physical memory resource. */ 110878eb3293SRoger Pau Monné gmap_start->map->pseudo_phys_res_id = 0; 110978eb3293SRoger Pau Monné gmap_start->map->pseudo_phys_res = xenmem_alloc(gntdev_dev, 111078eb3293SRoger Pau Monné &gmap_start->map->pseudo_phys_res_id, size); 111178eb3293SRoger Pau Monné if (gmap_start->map->pseudo_phys_res == NULL) { 111278eb3293SRoger Pau Monné free(gmap_start->map, M_GNTDEV); 111378eb3293SRoger Pau Monné gmap_start->map = NULL; 111478eb3293SRoger Pau Monné return (ENOMEM); 111578eb3293SRoger Pau Monné } 111678eb3293SRoger Pau Monné gmap_start->map->phys_base_addr = 111778eb3293SRoger Pau Monné rman_get_start(gmap_start->map->pseudo_phys_res); 111878eb3293SRoger Pau Monné 111978eb3293SRoger Pau Monné /* Allocate the mgtdevice pager. */ 112078eb3293SRoger Pau Monné gmap_start->map->mem = cdev_pager_allocate(gmap_start, OBJT_MGTDEVICE, 112178eb3293SRoger Pau Monné &gntdev_gmap_pg_ops, size, nprot, *offset, NULL); 112278eb3293SRoger Pau Monné if (gmap_start->map->mem == NULL) { 112378eb3293SRoger Pau Monné xenmem_free(gntdev_dev, gmap_start->map->pseudo_phys_res_id, 112478eb3293SRoger Pau Monné gmap_start->map->pseudo_phys_res); 112578eb3293SRoger Pau Monné free(gmap_start->map, M_GNTDEV); 112678eb3293SRoger Pau Monné gmap_start->map = NULL; 112778eb3293SRoger Pau Monné return (ENOMEM); 112878eb3293SRoger Pau Monné } 112978eb3293SRoger Pau Monné 113078eb3293SRoger Pau Monné for (i = 0; i < gmap_start->count; i++) { 113178eb3293SRoger Pau Monné gmap_start->grant_map_ops[i].host_addr = 113278eb3293SRoger Pau Monné gmap_start->map->phys_base_addr + i * PAGE_SIZE; 113378eb3293SRoger Pau Monné 113478eb3293SRoger Pau Monné if ((nprot & PROT_WRITE) == 0) 113578eb3293SRoger Pau Monné gmap_start->grant_map_ops[i].flags |= GNTMAP_readonly; 113678eb3293SRoger Pau Monné } 113778eb3293SRoger Pau Monné /* Make the MAP hypercall. */ 113878eb3293SRoger Pau Monné error = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, 113978eb3293SRoger Pau Monné gmap_start->grant_map_ops, gmap_start->count); 114078eb3293SRoger Pau Monné if (error != 0) { 114178eb3293SRoger Pau Monné /* 114278eb3293SRoger Pau Monné * Deallocate pager. 114378eb3293SRoger Pau Monné * Pager deallocation will automatically take care of 114478eb3293SRoger Pau Monné * xenmem deallocation, etc. 114578eb3293SRoger Pau Monné */ 114678eb3293SRoger Pau Monné vm_object_deallocate(gmap_start->map->mem); 114778eb3293SRoger Pau Monné 114878eb3293SRoger Pau Monné return (EINVAL); 114978eb3293SRoger Pau Monné } 115078eb3293SRoger Pau Monné 115178eb3293SRoger Pau Monné /* Retry EAGAIN maps. */ 115278eb3293SRoger Pau Monné for (i = 0; i < gmap_start->count; i++) { 115378eb3293SRoger Pau Monné int delay = 1; 115478eb3293SRoger Pau Monné while (delay < 256 && 115578eb3293SRoger Pau Monné gmap_start->grant_map_ops[i].status == GNTST_eagain) { 115678eb3293SRoger Pau Monné HYPERVISOR_grant_table_op( GNTTABOP_map_grant_ref, 115778eb3293SRoger Pau Monné &gmap_start->grant_map_ops[i], 1); 115878eb3293SRoger Pau Monné pause(("gntmap"), delay * SBT_1MS); 115978eb3293SRoger Pau Monné delay++; 116078eb3293SRoger Pau Monné } 116178eb3293SRoger Pau Monné if (gmap_start->grant_map_ops[i].status == GNTST_eagain) 116278eb3293SRoger Pau Monné gmap_start->grant_map_ops[i].status = GNTST_bad_page; 116378eb3293SRoger Pau Monné 116478eb3293SRoger Pau Monné if (gmap_start->grant_map_ops[i].status != GNTST_okay) { 116578eb3293SRoger Pau Monné /* 116678eb3293SRoger Pau Monné * Deallocate pager. 116778eb3293SRoger Pau Monné * Pager deallocation will automatically take care of 116878eb3293SRoger Pau Monné * xenmem deallocation, notification, unmap hypercall, 116978eb3293SRoger Pau Monné * etc. 117078eb3293SRoger Pau Monné */ 117178eb3293SRoger Pau Monné vm_object_deallocate(gmap_start->map->mem); 117278eb3293SRoger Pau Monné 117378eb3293SRoger Pau Monné return (EINVAL); 117478eb3293SRoger Pau Monné } 117578eb3293SRoger Pau Monné } 117678eb3293SRoger Pau Monné 117778eb3293SRoger Pau Monné /* 117878eb3293SRoger Pau Monné * Add a reference to the vm object. We do not want 117978eb3293SRoger Pau Monné * the vm object to be deleted when all the mmaps are 118078eb3293SRoger Pau Monné * unmapped, because it may be re-mmapped. Instead, 118178eb3293SRoger Pau Monné * we want the object to be deleted, when along with 118278eb3293SRoger Pau Monné * munmaps, we have also processed the unmap-ioctl. 118378eb3293SRoger Pau Monné */ 118478eb3293SRoger Pau Monné vm_object_reference(gmap_start->map->mem); 118578eb3293SRoger Pau Monné 118678eb3293SRoger Pau Monné *object = gmap_start->map->mem; 118778eb3293SRoger Pau Monné 118878eb3293SRoger Pau Monné return (0); 118978eb3293SRoger Pau Monné } 119078eb3293SRoger Pau Monné 119178eb3293SRoger Pau Monné static int 119278eb3293SRoger Pau Monné gntdev_mmap_single(struct cdev *cdev, vm_ooffset_t *offset, vm_size_t size, 119378eb3293SRoger Pau Monné struct vm_object **object, int nprot) 119478eb3293SRoger Pau Monné { 119578eb3293SRoger Pau Monné int error; 119678eb3293SRoger Pau Monné uint32_t count; 119778eb3293SRoger Pau Monné struct gntdev_gref *gref_start; 119878eb3293SRoger Pau Monné struct gntdev_gmap *gmap_start; 119978eb3293SRoger Pau Monné struct per_user_data *priv_user; 120078eb3293SRoger Pau Monné 120178eb3293SRoger Pau Monné error = devfs_get_cdevpriv((void**) &priv_user); 120278eb3293SRoger Pau Monné if (error != 0) 120378eb3293SRoger Pau Monné return (EINVAL); 120478eb3293SRoger Pau Monné 120510d9120cSKonstantin Belousov count = OFF_TO_IDX(size); 120678eb3293SRoger Pau Monné 120778eb3293SRoger Pau Monné gref_start = gntdev_find_grefs(priv_user, *offset, count); 120878eb3293SRoger Pau Monné if (gref_start) { 120978eb3293SRoger Pau Monné error = mmap_gref(priv_user, gref_start, count, size, object); 121078eb3293SRoger Pau Monné return (error); 121178eb3293SRoger Pau Monné } 121278eb3293SRoger Pau Monné 121378eb3293SRoger Pau Monné gmap_start = gntdev_find_gmap(priv_user, *offset, count); 121478eb3293SRoger Pau Monné if (gmap_start) { 121578eb3293SRoger Pau Monné error = mmap_gmap(priv_user, gmap_start, offset, size, object, 121678eb3293SRoger Pau Monné nprot); 121778eb3293SRoger Pau Monné return (error); 121878eb3293SRoger Pau Monné } 121978eb3293SRoger Pau Monné 122078eb3293SRoger Pau Monné return (EINVAL); 122178eb3293SRoger Pau Monné } 122278eb3293SRoger Pau Monné 122378eb3293SRoger Pau Monné /*------------------ Private Device Attachment Functions --------------------*/ 122478eb3293SRoger Pau Monné static void 122578eb3293SRoger Pau Monné gntdev_identify(driver_t *driver, device_t parent) 122678eb3293SRoger Pau Monné { 122778eb3293SRoger Pau Monné 122878eb3293SRoger Pau Monné KASSERT((xen_domain()), 122978eb3293SRoger Pau Monné ("Trying to attach gntdev device on non Xen domain")); 123078eb3293SRoger Pau Monné 123178eb3293SRoger Pau Monné if (BUS_ADD_CHILD(parent, 0, "gntdev", 0) == NULL) 123278eb3293SRoger Pau Monné panic("unable to attach gntdev user-space device"); 123378eb3293SRoger Pau Monné } 123478eb3293SRoger Pau Monné 123578eb3293SRoger Pau Monné static int 123678eb3293SRoger Pau Monné gntdev_probe(device_t dev) 123778eb3293SRoger Pau Monné { 123878eb3293SRoger Pau Monné 123978eb3293SRoger Pau Monné gntdev_dev = dev; 124078eb3293SRoger Pau Monné device_set_desc(dev, "Xen grant-table user-space device"); 124178eb3293SRoger Pau Monné return (BUS_PROBE_NOWILDCARD); 124278eb3293SRoger Pau Monné } 124378eb3293SRoger Pau Monné 124478eb3293SRoger Pau Monné static int 124578eb3293SRoger Pau Monné gntdev_attach(device_t dev) 124678eb3293SRoger Pau Monné { 124778eb3293SRoger Pau Monné 124878eb3293SRoger Pau Monné make_dev_credf(MAKEDEV_ETERNAL, &gntdev_devsw, 0, NULL, UID_ROOT, 124978eb3293SRoger Pau Monné GID_WHEEL, 0600, "xen/gntdev"); 125078eb3293SRoger Pau Monné return (0); 125178eb3293SRoger Pau Monné } 125278eb3293SRoger Pau Monné 125378eb3293SRoger Pau Monné /*-------------------- Private Device Attachment Data -----------------------*/ 125478eb3293SRoger Pau Monné static device_method_t gntdev_methods[] = { 125578eb3293SRoger Pau Monné DEVMETHOD(device_identify, gntdev_identify), 125678eb3293SRoger Pau Monné DEVMETHOD(device_probe, gntdev_probe), 125778eb3293SRoger Pau Monné DEVMETHOD(device_attach, gntdev_attach), 125878eb3293SRoger Pau Monné DEVMETHOD_END 125978eb3293SRoger Pau Monné }; 126078eb3293SRoger Pau Monné 126178eb3293SRoger Pau Monné static driver_t gntdev_driver = { 126278eb3293SRoger Pau Monné "gntdev", 126378eb3293SRoger Pau Monné gntdev_methods, 126478eb3293SRoger Pau Monné 0, 126578eb3293SRoger Pau Monné }; 126678eb3293SRoger Pau Monné 1267f929eb1eSJohn Baldwin DRIVER_MODULE(gntdev, xenpv, gntdev_driver, 0, 0); 126878eb3293SRoger Pau Monné MODULE_DEPEND(gntdev, xenpv, 1, 1, 1); 1269