1 /* $NetBSD: ttm_bo_vm.c,v 1.10 2015/07/28 01:25:00 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2014 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Taylor R. Campbell. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: ttm_bo_vm.c,v 1.10 2015/07/28 01:25:00 riastradh Exp $"); 34 35 #include <sys/types.h> 36 37 #include <uvm/uvm.h> 38 #include <uvm/uvm_extern.h> 39 #include <uvm/uvm_fault.h> 40 41 #include <linux/bitops.h> 42 43 #include <drm/drm_vma_manager.h> 44 45 #include <ttm/ttm_bo_driver.h> 46 47 static int ttm_bo_uvm_fault_idle(struct ttm_buffer_object *, 48 struct uvm_faultinfo *); 49 static int ttm_bo_uvm_lookup(struct ttm_bo_device *, unsigned long, 50 unsigned long, struct ttm_buffer_object **); 51 52 void 53 ttm_bo_uvm_reference(struct uvm_object *uobj) 54 { 55 struct ttm_buffer_object *const bo = container_of(uobj, 56 struct ttm_buffer_object, uvmobj); 57 58 (void)ttm_bo_reference(bo); 59 } 60 61 void 62 ttm_bo_uvm_detach(struct uvm_object *uobj) 63 { 64 struct ttm_buffer_object *bo = container_of(uobj, 65 struct ttm_buffer_object, uvmobj); 66 67 ttm_bo_unref(&bo); 68 KASSERT(bo == NULL); 69 } 70 71 int 72 ttm_bo_uvm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, 73 struct vm_page **pps, int npages, int centeridx, vm_prot_t access_type, 74 int flags) 75 { 76 struct uvm_object *const uobj = ufi->entry->object.uvm_obj; 77 struct ttm_buffer_object *const bo = container_of(uobj, 78 struct ttm_buffer_object, uvmobj); 79 struct ttm_bo_device *const bdev = bo->bdev; 80 struct ttm_mem_type_manager *man = 81 &bdev->man[bo->mem.mem_type]; 82 union { 83 bus_addr_t base; 84 struct ttm_tt *ttm; 85 } u; 86 size_t size __diagused; 87 voff_t uoffset; /* offset in bytes into bo */ 88 unsigned startpage; /* offset in pages into bo */ 89 unsigned i; 90 vm_prot_t vm_prot; /* VM_PROT_* */ 91 pgprot_t pgprot; /* VM_PROT_* | PMAP_* cacheability flags */ 92 unsigned mmapflags; 93 int ret; 94 95 /* Thanks, uvm, but we don't need this lock. */ 96 mutex_exit(uobj->vmobjlock); 97 98 /* Copy-on-write mappings make no sense for the graphics aperture. */ 99 if (UVM_ET_ISCOPYONWRITE(ufi->entry)) { 100 ret = -EIO; 101 goto out0; 102 } 103 104 /* Try to lock the buffer. */ 105 ret = ttm_bo_reserve(bo, true, true, false, NULL); 106 if (ret) { 107 if (ret != -EBUSY) 108 goto out0; 109 /* 110 * It's currently locked. Unlock the fault, wait for 111 * it, and start over. 112 */ 113 uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, NULL); 114 (void)ttm_bo_wait_unreserved(bo); 115 return -ERESTART; 116 } 117 118 /* drm prime buffers are not mappable. XXX Catch this earlier? */ 119 if (bo->ttm && ISSET(bo->ttm->page_flags, TTM_PAGE_FLAG_SG)) { 120 ret = -EINVAL; 121 goto out1; 122 } 123 124 /* Notify the driver of a fault if it wants. */ 125 if (bdev->driver->fault_reserve_notify) { 126 ret = (*bdev->driver->fault_reserve_notify)(bo); 127 if (ret) { 128 if (ret == -ERESTART) 129 ret = -EIO; 130 goto out1; 131 } 132 } 133 134 ret = ttm_bo_uvm_fault_idle(bo, ufi); 135 if (ret) { 136 KASSERT(ret == -ERESTART); 137 /* ttm_bo_uvm_fault_idle calls uvmfault_unlockall for us. */ 138 ttm_bo_unreserve(bo); 139 /* XXX errno Linux->NetBSD */ 140 return -ret; 141 } 142 143 ret = ttm_mem_io_lock(man, true); 144 if (ret) { 145 ret = -EIO; 146 goto out1; 147 } 148 ret = ttm_mem_io_reserve_vm(bo); 149 if (ret) { 150 ret = -EIO; 151 goto out2; 152 } 153 154 vm_prot = ufi->entry->protection; 155 if (bo->mem.bus.is_iomem) { 156 u.base = (bo->mem.bus.base + bo->mem.bus.offset); 157 size = bo->mem.bus.size; 158 pgprot = ttm_io_prot(bo->mem.placement, vm_prot); 159 } else { 160 u.ttm = bo->ttm; 161 size = (bo->ttm->num_pages << PAGE_SHIFT); 162 if (ISSET(bo->mem.placement, TTM_PL_FLAG_CACHED)) 163 pgprot = vm_prot; 164 else 165 pgprot = ttm_io_prot(bo->mem.placement, vm_prot); 166 if ((*u.ttm->bdev->driver->ttm_tt_populate)(u.ttm)) { 167 ret = -ENOMEM; 168 goto out2; 169 } 170 } 171 172 KASSERT(ufi->entry->start <= vaddr); 173 KASSERT((ufi->entry->offset & (PAGE_SIZE - 1)) == 0); 174 KASSERT(ufi->entry->offset <= size); 175 KASSERT((vaddr - ufi->entry->start) <= (size - ufi->entry->offset)); 176 KASSERT(npages <= ((size - ufi->entry->offset) - 177 (vaddr - ufi->entry->start))); 178 uoffset = (ufi->entry->offset + (vaddr - ufi->entry->start)); 179 startpage = (uoffset >> PAGE_SHIFT); 180 for (i = 0; i < npages; i++) { 181 paddr_t paddr; 182 183 /* XXX PGO_ALLPAGES? */ 184 if (pps[i] == PGO_DONTCARE) 185 continue; 186 if (bo->mem.bus.is_iomem) { 187 const paddr_t cookie = bus_space_mmap(bdev->memt, 188 u.base, ((startpage + i) << PAGE_SHIFT), vm_prot, 189 0); 190 191 paddr = pmap_phys_address(cookie); 192 mmapflags = pmap_mmap_flags(cookie); 193 } else { 194 paddr = page_to_phys(u.ttm->pages[startpage + i]); 195 mmapflags = 0; 196 } 197 ret = -pmap_enter(ufi->orig_map->pmap, vaddr + i*PAGE_SIZE, 198 paddr, vm_prot, (PMAP_CANFAIL | pgprot | mmapflags)); 199 if (ret) 200 goto out3; 201 } 202 203 out3: pmap_update(ufi->orig_map->pmap); 204 out2: ttm_mem_io_unlock(man); 205 out1: ttm_bo_unreserve(bo); 206 out0: uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, NULL); 207 /* XXX errno Linux->NetBSD */ 208 return -ret; 209 } 210 211 static int 212 ttm_bo_uvm_fault_idle(struct ttm_buffer_object *bo, struct uvm_faultinfo *ufi) 213 { 214 struct ttm_bo_device *const bdev = bo->bdev; 215 int ret = 0; 216 217 spin_lock(&bdev->fence_lock); 218 if (__predict_true(!test_bit(TTM_BO_PRIV_FLAG_MOVING, 219 &bo->priv_flags))) 220 goto out; 221 if (ttm_bo_wait(bo, false, false, true) == 0) 222 goto out; 223 224 uvmfault_unlockall(ufi, ufi->entry->aref.ar_amap, NULL); 225 (void)ttm_bo_wait(bo, false, true, false); 226 ret = -ERESTART; 227 228 out: spin_unlock(&bdev->fence_lock); 229 return ret; 230 } 231 232 int 233 ttm_bo_mmap_object(struct ttm_bo_device *bdev, off_t offset, size_t size, 234 vm_prot_t prot, struct uvm_object **uobjp, voff_t *uoffsetp, 235 struct file *file) 236 { 237 const unsigned long startpage = (offset >> PAGE_SHIFT); 238 const unsigned long npages = (size >> PAGE_SHIFT); 239 struct ttm_buffer_object *bo; 240 int ret; 241 242 KASSERT(0 == (offset & (PAGE_SIZE - 1))); 243 KASSERT(0 == (size & (PAGE_SIZE - 1))); 244 245 ret = ttm_bo_uvm_lookup(bdev, startpage, npages, &bo); 246 if (ret) 247 goto fail0; 248 KASSERT(drm_vma_node_start(&bo->vma_node) <= offset); 249 /* XXX Just assert this? */ 250 if (__predict_false(bdev->driver->verify_access == NULL)) { 251 ret = -EPERM; 252 goto fail1; 253 } 254 ret = (*bdev->driver->verify_access)(bo, file); 255 if (ret) 256 goto fail1; 257 258 /* Success! */ 259 *uobjp = &bo->uvmobj; 260 *uoffsetp = (offset - 261 (drm_vma_node_start(&bo->vma_node) << PAGE_SHIFT)); 262 return 0; 263 264 fail1: ttm_bo_unref(&bo); 265 fail0: KASSERT(ret); 266 return ret; 267 } 268 269 static int 270 ttm_bo_uvm_lookup(struct ttm_bo_device *bdev, unsigned long startpage, 271 unsigned long npages, struct ttm_buffer_object **bop) 272 { 273 struct ttm_buffer_object *bo = NULL; 274 struct drm_vma_offset_node *node; 275 276 drm_vma_offset_lock_lookup(&bdev->vma_manager); 277 node = drm_vma_offset_lookup_locked(&bdev->vma_manager, startpage, 278 npages); 279 if (node != NULL) { 280 bo = container_of(node, struct ttm_buffer_object, vma_node); 281 if (!kref_get_unless_zero(&bo->kref)) 282 bo = NULL; 283 } 284 drm_vma_offset_unlock_lookup(&bdev->vma_manager); 285 286 if (bo == NULL) 287 return -ENOENT; 288 289 *bop = bo; 290 return 0; 291 } 292