1 /* $NetBSD: amdgpu_gem.c,v 1.7 2021/12/18 23:44:58 riastradh Exp $ */ 2 3 /* 4 * Copyright 2008 Advanced Micro Devices, Inc. 5 * Copyright 2008 Red Hat Inc. 6 * Copyright 2009 Jerome Glisse. 7 * 8 * Permission is hereby granted, free of charge, to any person obtaining a 9 * copy of this software and associated documentation files (the "Software"), 10 * to deal in the Software without restriction, including without limitation 11 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 12 * and/or sell copies of the Software, and to permit persons to whom the 13 * Software is furnished to do so, subject to the following conditions: 14 * 15 * The above copyright notice and this permission notice shall be included in 16 * all copies or substantial portions of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 22 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 23 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 * OTHER DEALINGS IN THE SOFTWARE. 25 * 26 * Authors: Dave Airlie 27 * Alex Deucher 28 * Jerome Glisse 29 */ 30 #include <sys/cdefs.h> 31 __KERNEL_RCSID(0, "$NetBSD: amdgpu_gem.c,v 1.7 2021/12/18 23:44:58 riastradh Exp $"); 32 33 #include <linux/ktime.h> 34 #include <linux/module.h> 35 #include <linux/pagemap.h> 36 #include <linux/pci.h> 37 38 #include <drm/amdgpu_drm.h> 39 #include <drm/drm_debugfs.h> 40 41 #include "amdgpu.h" 42 #include "amdgpu_display.h" 43 #include "amdgpu_xgmi.h" 44 45 #include <linux/nbsd-namespace.h> 46 47 void amdgpu_gem_object_free(struct drm_gem_object *gobj) 48 { 49 struct amdgpu_bo *robj = gem_to_amdgpu_bo(gobj); 50 51 if (robj) { 52 amdgpu_mn_unregister(robj); 53 amdgpu_bo_unref(&robj); 54 } 55 } 56 57 int amdgpu_gem_object_create(struct amdgpu_device *adev, unsigned long size, 58 int alignment, u32 initial_domain, 59 u64 flags, enum ttm_bo_type type, 60 struct dma_resv *resv, 61 struct drm_gem_object **obj) 62 { 63 struct amdgpu_bo *bo; 64 struct amdgpu_bo_param bp; 65 int r; 66 67 memset(&bp, 0, sizeof(bp)); 68 *obj = NULL; 69 70 bp.size = size; 71 bp.byte_align = alignment; 72 bp.type = type; 73 bp.resv = resv; 74 bp.preferred_domain = initial_domain; 75 retry: 76 bp.flags = flags; 77 bp.domain = initial_domain; 78 r = amdgpu_bo_create(adev, &bp, &bo); 79 if (r) { 80 if (r != -ERESTARTSYS) { 81 if (flags & AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED) { 82 flags &= ~AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED; 83 goto retry; 84 } 85 86 if (initial_domain == AMDGPU_GEM_DOMAIN_VRAM) { 87 initial_domain |= AMDGPU_GEM_DOMAIN_GTT; 88 goto retry; 89 } 90 DRM_DEBUG("Failed to allocate GEM object (%ld, %d, %u, %d)\n", 91 size, initial_domain, alignment, r); 92 } 93 return r; 94 } 95 *obj = &bo->tbo.base; 96 97 return 0; 98 } 99 100 void amdgpu_gem_force_release(struct amdgpu_device *adev) 101 { 102 struct drm_device *ddev = adev->ddev; 103 struct drm_file *file; 104 105 mutex_lock(&ddev->filelist_mutex); 106 107 list_for_each_entry(file, &ddev->filelist, lhead) { 108 struct drm_gem_object *gobj; 109 int handle; 110 111 WARN_ONCE(1, "Still active user space clients!\n"); 112 spin_lock(&file->table_lock); 113 idr_for_each_entry(&file->object_idr, gobj, handle) { 114 WARN_ONCE(1, "And also active allocations!\n"); 115 drm_gem_object_put_unlocked(gobj); 116 } 117 idr_destroy(&file->object_idr); 118 spin_unlock(&file->table_lock); 119 } 120 121 mutex_unlock(&ddev->filelist_mutex); 122 } 123 124 /* 125 * Call from drm_gem_handle_create which appear in both new and open ioctl 126 * case. 127 */ 128 int amdgpu_gem_object_open(struct drm_gem_object *obj, 129 struct drm_file *file_priv) 130 { 131 struct amdgpu_bo *abo = gem_to_amdgpu_bo(obj); 132 struct amdgpu_device *adev = amdgpu_ttm_adev(abo->tbo.bdev); 133 struct amdgpu_fpriv *fpriv = file_priv->driver_priv; 134 struct amdgpu_vm *vm = &fpriv->vm; 135 struct amdgpu_bo_va *bo_va; 136 struct mm_struct *mm; 137 int r; 138 139 mm = amdgpu_ttm_tt_get_usermm(abo->tbo.ttm); 140 if (mm && mm != current->mm) 141 return -EPERM; 142 143 if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID && 144 abo->tbo.base.resv != vm->root.base.bo->tbo.base.resv) 145 return -EPERM; 146 147 r = amdgpu_bo_reserve(abo, false); 148 if (r) 149 return r; 150 151 bo_va = amdgpu_vm_bo_find(vm, abo); 152 if (!bo_va) { 153 bo_va = amdgpu_vm_bo_add(adev, vm, abo); 154 } else { 155 ++bo_va->ref_count; 156 } 157 amdgpu_bo_unreserve(abo); 158 return 0; 159 } 160 161 void amdgpu_gem_object_close(struct drm_gem_object *obj, 162 struct drm_file *file_priv) 163 { 164 struct amdgpu_bo *bo = gem_to_amdgpu_bo(obj); 165 struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev); 166 struct amdgpu_fpriv *fpriv = file_priv->driver_priv; 167 struct amdgpu_vm *vm = &fpriv->vm; 168 169 struct amdgpu_bo_list_entry vm_pd; 170 struct list_head list, duplicates; 171 struct ttm_validate_buffer tv; 172 struct ww_acquire_ctx ticket; 173 struct amdgpu_bo_va *bo_va; 174 int r; 175 176 INIT_LIST_HEAD(&list); 177 INIT_LIST_HEAD(&duplicates); 178 179 tv.bo = &bo->tbo; 180 tv.num_shared = 1; 181 list_add(&tv.head, &list); 182 183 amdgpu_vm_get_pd_bo(vm, &list, &vm_pd); 184 185 r = ttm_eu_reserve_buffers(&ticket, &list, false, &duplicates); 186 if (r) { 187 dev_err(adev->dev, "leaking bo va because " 188 "we fail to reserve bo (%d)\n", r); 189 return; 190 } 191 bo_va = amdgpu_vm_bo_find(vm, bo); 192 if (bo_va && --bo_va->ref_count == 0) { 193 amdgpu_vm_bo_rmv(adev, bo_va); 194 195 if (amdgpu_vm_ready(vm)) { 196 struct dma_fence *fence = NULL; 197 198 r = amdgpu_vm_clear_freed(adev, vm, &fence); 199 if (unlikely(r)) { 200 dev_err(adev->dev, "failed to clear page " 201 "tables on GEM object close (%d)\n", r); 202 } 203 204 if (fence) { 205 amdgpu_bo_fence(bo, fence, true); 206 dma_fence_put(fence); 207 } 208 } 209 } 210 ttm_eu_backoff_reservation(&ticket, &list); 211 } 212 213 /* 214 * GEM ioctls. 215 */ 216 int amdgpu_gem_create_ioctl(struct drm_device *dev, void *data, 217 struct drm_file *filp) 218 { 219 struct amdgpu_device *adev = dev->dev_private; 220 struct amdgpu_fpriv *fpriv = filp->driver_priv; 221 struct amdgpu_vm *vm = &fpriv->vm; 222 union drm_amdgpu_gem_create *args = data; 223 uint64_t flags = args->in.domain_flags; 224 uint64_t size = args->in.bo_size; 225 struct dma_resv *resv = NULL; 226 struct drm_gem_object *gobj; 227 uint32_t handle; 228 int r; 229 230 /* reject invalid gem flags */ 231 if (flags & ~(AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | 232 AMDGPU_GEM_CREATE_NO_CPU_ACCESS | 233 AMDGPU_GEM_CREATE_CPU_GTT_USWC | 234 AMDGPU_GEM_CREATE_VRAM_CLEARED | 235 AMDGPU_GEM_CREATE_VM_ALWAYS_VALID | 236 AMDGPU_GEM_CREATE_EXPLICIT_SYNC)) 237 238 return -EINVAL; 239 240 /* reject invalid gem domains */ 241 if (args->in.domains & ~AMDGPU_GEM_DOMAIN_MASK) 242 return -EINVAL; 243 244 /* create a gem object to contain this object in */ 245 if (args->in.domains & (AMDGPU_GEM_DOMAIN_GDS | 246 AMDGPU_GEM_DOMAIN_GWS | AMDGPU_GEM_DOMAIN_OA)) { 247 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) { 248 /* if gds bo is created from user space, it must be 249 * passed to bo list 250 */ 251 DRM_ERROR("GDS bo cannot be per-vm-bo\n"); 252 return -EINVAL; 253 } 254 flags |= AMDGPU_GEM_CREATE_NO_CPU_ACCESS; 255 } 256 257 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) { 258 r = amdgpu_bo_reserve(vm->root.base.bo, false); 259 if (r) 260 return r; 261 262 resv = vm->root.base.bo->tbo.base.resv; 263 } 264 265 r = amdgpu_gem_object_create(adev, size, args->in.alignment, 266 (u32)(0xffffffff & args->in.domains), 267 flags, ttm_bo_type_device, resv, &gobj); 268 if (flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) { 269 if (!r) { 270 struct amdgpu_bo *abo = gem_to_amdgpu_bo(gobj); 271 272 abo->parent = amdgpu_bo_ref(vm->root.base.bo); 273 } 274 amdgpu_bo_unreserve(vm->root.base.bo); 275 } 276 if (r) 277 return r; 278 279 r = drm_gem_handle_create(filp, gobj, &handle); 280 /* drop reference from allocate - handle holds it now */ 281 drm_gem_object_put_unlocked(gobj); 282 if (r) 283 return r; 284 285 memset(args, 0, sizeof(*args)); 286 args->out.handle = handle; 287 return 0; 288 } 289 290 int amdgpu_gem_userptr_ioctl(struct drm_device *dev, void *data, 291 struct drm_file *filp) 292 { 293 struct ttm_operation_ctx ctx = { true, false }; 294 struct amdgpu_device *adev = dev->dev_private; 295 struct drm_amdgpu_gem_userptr *args = data; 296 struct drm_gem_object *gobj; 297 struct amdgpu_bo *bo; 298 uint32_t handle; 299 int r; 300 301 args->addr = untagged_addr(args->addr); 302 303 if (offset_in_page(args->addr | args->size)) 304 return -EINVAL; 305 306 /* reject unknown flag values */ 307 if (args->flags & ~(AMDGPU_GEM_USERPTR_READONLY | 308 AMDGPU_GEM_USERPTR_ANONONLY | AMDGPU_GEM_USERPTR_VALIDATE | 309 AMDGPU_GEM_USERPTR_REGISTER)) 310 return -EINVAL; 311 312 if (!(args->flags & AMDGPU_GEM_USERPTR_READONLY) && 313 !(args->flags & AMDGPU_GEM_USERPTR_REGISTER)) { 314 315 /* if we want to write to it we must install a MMU notifier */ 316 return -EACCES; 317 } 318 319 /* create a gem object to contain this object in */ 320 r = amdgpu_gem_object_create(adev, args->size, 0, AMDGPU_GEM_DOMAIN_CPU, 321 0, ttm_bo_type_device, NULL, &gobj); 322 if (r) 323 return r; 324 325 bo = gem_to_amdgpu_bo(gobj); 326 bo->preferred_domains = AMDGPU_GEM_DOMAIN_GTT; 327 bo->allowed_domains = AMDGPU_GEM_DOMAIN_GTT; 328 r = amdgpu_ttm_tt_set_userptr(bo->tbo.ttm, args->addr, args->flags); 329 if (r) 330 goto release_object; 331 332 if (args->flags & AMDGPU_GEM_USERPTR_REGISTER) { 333 r = amdgpu_mn_register(bo, args->addr); 334 if (r) 335 goto release_object; 336 } 337 338 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) { 339 r = amdgpu_ttm_tt_get_user_pages(bo, bo->tbo.ttm->pages); 340 if (r) 341 goto release_object; 342 343 r = amdgpu_bo_reserve(bo, true); 344 if (r) 345 goto user_pages_done; 346 347 amdgpu_bo_placement_from_domain(bo, AMDGPU_GEM_DOMAIN_GTT); 348 r = ttm_bo_validate(&bo->tbo, &bo->placement, &ctx); 349 amdgpu_bo_unreserve(bo); 350 if (r) 351 goto user_pages_done; 352 } 353 354 r = drm_gem_handle_create(filp, gobj, &handle); 355 if (r) 356 goto user_pages_done; 357 358 args->handle = handle; 359 360 user_pages_done: 361 if (args->flags & AMDGPU_GEM_USERPTR_VALIDATE) 362 amdgpu_ttm_tt_get_user_pages_done(bo->tbo.ttm); 363 364 release_object: 365 drm_gem_object_put_unlocked(gobj); 366 367 return r; 368 } 369 370 int amdgpu_mode_dumb_mmap(struct drm_file *filp, 371 struct drm_device *dev, 372 uint32_t handle, uint64_t *offset_p) 373 { 374 struct drm_gem_object *gobj; 375 struct amdgpu_bo *robj; 376 377 gobj = drm_gem_object_lookup(filp, handle); 378 if (gobj == NULL) { 379 return -ENOENT; 380 } 381 robj = gem_to_amdgpu_bo(gobj); 382 if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm) || 383 (robj->flags & AMDGPU_GEM_CREATE_NO_CPU_ACCESS)) { 384 drm_gem_object_put_unlocked(gobj); 385 return -EPERM; 386 } 387 *offset_p = amdgpu_bo_mmap_offset(robj); 388 drm_gem_object_put_unlocked(gobj); 389 return 0; 390 } 391 392 int amdgpu_gem_mmap_ioctl(struct drm_device *dev, void *data, 393 struct drm_file *filp) 394 { 395 union drm_amdgpu_gem_mmap *args = data; 396 uint32_t handle = args->in.handle; 397 memset(args, 0, sizeof(*args)); 398 return amdgpu_mode_dumb_mmap(filp, dev, handle, &args->out.addr_ptr); 399 } 400 401 /** 402 * amdgpu_gem_timeout - calculate jiffies timeout from absolute value 403 * 404 * @timeout_ns: timeout in ns 405 * 406 * Calculate the timeout in jiffies from an absolute timeout in ns. 407 */ 408 unsigned long amdgpu_gem_timeout(uint64_t timeout_ns) 409 { 410 unsigned long timeout_jiffies; 411 ktime_t timeout; 412 413 /* clamp timeout if it's to large */ 414 if (((int64_t)timeout_ns) < 0) 415 return MAX_SCHEDULE_TIMEOUT; 416 417 timeout = ktime_sub(ns_to_ktime(timeout_ns), ktime_get()); 418 if (ktime_to_ns(timeout) < 0) 419 return 0; 420 421 timeout_jiffies = nsecs_to_jiffies(ktime_to_ns(timeout)); 422 /* clamp timeout to avoid unsigned-> signed overflow */ 423 if (timeout_jiffies > MAX_SCHEDULE_TIMEOUT ) 424 return MAX_SCHEDULE_TIMEOUT - 1; 425 426 return timeout_jiffies; 427 } 428 429 int amdgpu_gem_wait_idle_ioctl(struct drm_device *dev, void *data, 430 struct drm_file *filp) 431 { 432 union drm_amdgpu_gem_wait_idle *args = data; 433 struct drm_gem_object *gobj; 434 struct amdgpu_bo *robj; 435 uint32_t handle = args->in.handle; 436 unsigned long timeout = amdgpu_gem_timeout(args->in.timeout); 437 int r = 0; 438 long ret; 439 440 gobj = drm_gem_object_lookup(filp, handle); 441 if (gobj == NULL) { 442 return -ENOENT; 443 } 444 robj = gem_to_amdgpu_bo(gobj); 445 ret = dma_resv_wait_timeout_rcu(robj->tbo.base.resv, true, true, 446 timeout); 447 448 /* ret == 0 means not signaled, 449 * ret > 0 means signaled 450 * ret < 0 means interrupted before timeout 451 */ 452 if (ret >= 0) { 453 memset(args, 0, sizeof(*args)); 454 args->out.status = (ret == 0); 455 } else 456 r = ret; 457 458 drm_gem_object_put_unlocked(gobj); 459 return r; 460 } 461 462 int amdgpu_gem_metadata_ioctl(struct drm_device *dev, void *data, 463 struct drm_file *filp) 464 { 465 struct drm_amdgpu_gem_metadata *args = data; 466 struct drm_gem_object *gobj; 467 struct amdgpu_bo *robj; 468 int r = -1; 469 470 DRM_DEBUG("%d \n", args->handle); 471 gobj = drm_gem_object_lookup(filp, args->handle); 472 if (gobj == NULL) 473 return -ENOENT; 474 robj = gem_to_amdgpu_bo(gobj); 475 476 r = amdgpu_bo_reserve(robj, false); 477 if (unlikely(r != 0)) 478 goto out; 479 480 if (args->op == AMDGPU_GEM_METADATA_OP_GET_METADATA) { 481 amdgpu_bo_get_tiling_flags(robj, &args->data.tiling_info); 482 r = amdgpu_bo_get_metadata(robj, args->data.data, 483 sizeof(args->data.data), 484 &args->data.data_size_bytes, 485 &args->data.flags); 486 } else if (args->op == AMDGPU_GEM_METADATA_OP_SET_METADATA) { 487 if (args->data.data_size_bytes > sizeof(args->data.data)) { 488 r = -EINVAL; 489 goto unreserve; 490 } 491 r = amdgpu_bo_set_tiling_flags(robj, args->data.tiling_info); 492 if (!r) 493 r = amdgpu_bo_set_metadata(robj, args->data.data, 494 args->data.data_size_bytes, 495 args->data.flags); 496 } 497 498 unreserve: 499 amdgpu_bo_unreserve(robj); 500 out: 501 drm_gem_object_put_unlocked(gobj); 502 return r; 503 } 504 505 /** 506 * amdgpu_gem_va_update_vm -update the bo_va in its VM 507 * 508 * @adev: amdgpu_device pointer 509 * @vm: vm to update 510 * @bo_va: bo_va to update 511 * @operation: map, unmap or clear 512 * 513 * Update the bo_va directly after setting its address. Errors are not 514 * vital here, so they are not reported back to userspace. 515 */ 516 static void amdgpu_gem_va_update_vm(struct amdgpu_device *adev, 517 struct amdgpu_vm *vm, 518 struct amdgpu_bo_va *bo_va, 519 uint32_t operation) 520 { 521 int r; 522 523 if (!amdgpu_vm_ready(vm)) 524 return; 525 526 r = amdgpu_vm_clear_freed(adev, vm, NULL); 527 if (r) 528 goto error; 529 530 if (operation == AMDGPU_VA_OP_MAP || 531 operation == AMDGPU_VA_OP_REPLACE) { 532 r = amdgpu_vm_bo_update(adev, bo_va, false); 533 if (r) 534 goto error; 535 } 536 537 r = amdgpu_vm_update_pdes(adev, vm, false); 538 539 error: 540 if (r && r != -ERESTARTSYS) 541 DRM_ERROR("Couldn't update BO_VA (%d)\n", r); 542 } 543 544 /** 545 * amdgpu_gem_va_map_flags - map GEM UAPI flags into hardware flags 546 * 547 * @adev: amdgpu_device pointer 548 * @flags: GEM UAPI flags 549 * 550 * Returns the GEM UAPI flags mapped into hardware for the ASIC. 551 */ 552 uint64_t amdgpu_gem_va_map_flags(struct amdgpu_device *adev, uint32_t flags) 553 { 554 uint64_t pte_flag = 0; 555 556 if (flags & AMDGPU_VM_PAGE_EXECUTABLE) 557 pte_flag |= AMDGPU_PTE_EXECUTABLE; 558 if (flags & AMDGPU_VM_PAGE_READABLE) 559 pte_flag |= AMDGPU_PTE_READABLE; 560 if (flags & AMDGPU_VM_PAGE_WRITEABLE) 561 pte_flag |= AMDGPU_PTE_WRITEABLE; 562 if (flags & AMDGPU_VM_PAGE_PRT) 563 pte_flag |= AMDGPU_PTE_PRT; 564 565 if (adev->gmc.gmc_funcs->map_mtype) 566 pte_flag |= amdgpu_gmc_map_mtype(adev, 567 flags & AMDGPU_VM_MTYPE_MASK); 568 569 return pte_flag; 570 } 571 572 int amdgpu_gem_va_ioctl(struct drm_device *dev, void *data, 573 struct drm_file *filp) 574 { 575 const uint32_t valid_flags = AMDGPU_VM_DELAY_UPDATE | 576 AMDGPU_VM_PAGE_READABLE | AMDGPU_VM_PAGE_WRITEABLE | 577 AMDGPU_VM_PAGE_EXECUTABLE | AMDGPU_VM_MTYPE_MASK; 578 const uint32_t prt_flags = AMDGPU_VM_DELAY_UPDATE | 579 AMDGPU_VM_PAGE_PRT; 580 581 struct drm_amdgpu_gem_va *args = data; 582 struct drm_gem_object *gobj; 583 struct amdgpu_device *adev = dev->dev_private; 584 struct amdgpu_fpriv *fpriv = filp->driver_priv; 585 struct amdgpu_bo *abo; 586 struct amdgpu_bo_va *bo_va; 587 struct amdgpu_bo_list_entry vm_pd; 588 struct ttm_validate_buffer tv; 589 struct ww_acquire_ctx ticket; 590 struct list_head list, duplicates; 591 uint64_t va_flags; 592 int r = 0; 593 594 if (args->va_address < AMDGPU_VA_RESERVED_SIZE) { 595 dev_dbg(&dev->pdev->dev, 596 "va_address 0x%LX is in reserved area 0x%LX\n", 597 args->va_address, AMDGPU_VA_RESERVED_SIZE); 598 return -EINVAL; 599 } 600 601 if (args->va_address >= AMDGPU_GMC_HOLE_START && 602 args->va_address < AMDGPU_GMC_HOLE_END) { 603 dev_dbg(&dev->pdev->dev, 604 "va_address 0x%LX is in VA hole 0x%LX-0x%LX\n", 605 args->va_address, AMDGPU_GMC_HOLE_START, 606 AMDGPU_GMC_HOLE_END); 607 return -EINVAL; 608 } 609 610 args->va_address &= AMDGPU_GMC_HOLE_MASK; 611 612 if ((args->flags & ~valid_flags) && (args->flags & ~prt_flags)) { 613 dev_dbg(&dev->pdev->dev, "invalid flags combination 0x%08X\n", 614 args->flags); 615 return -EINVAL; 616 } 617 618 switch (args->operation) { 619 case AMDGPU_VA_OP_MAP: 620 case AMDGPU_VA_OP_UNMAP: 621 case AMDGPU_VA_OP_CLEAR: 622 case AMDGPU_VA_OP_REPLACE: 623 break; 624 default: 625 dev_err(pci_dev_dev(dev->pdev), "unsupported operation %d\n", 626 args->operation); 627 return -EINVAL; 628 } 629 630 INIT_LIST_HEAD(&list); 631 INIT_LIST_HEAD(&duplicates); 632 if ((args->operation != AMDGPU_VA_OP_CLEAR) && 633 !(args->flags & AMDGPU_VM_PAGE_PRT)) { 634 gobj = drm_gem_object_lookup(filp, args->handle); 635 if (gobj == NULL) 636 return -ENOENT; 637 abo = gem_to_amdgpu_bo(gobj); 638 tv.bo = &abo->tbo; 639 if (abo->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) 640 tv.num_shared = 1; 641 else 642 tv.num_shared = 0; 643 list_add(&tv.head, &list); 644 } else { 645 gobj = NULL; 646 abo = NULL; 647 } 648 649 amdgpu_vm_get_pd_bo(&fpriv->vm, &list, &vm_pd); 650 651 r = ttm_eu_reserve_buffers(&ticket, &list, true, &duplicates); 652 if (r) 653 goto error_unref; 654 655 if (abo) { 656 bo_va = amdgpu_vm_bo_find(&fpriv->vm, abo); 657 if (!bo_va) { 658 r = -ENOENT; 659 goto error_backoff; 660 } 661 } else if (args->operation != AMDGPU_VA_OP_CLEAR) { 662 bo_va = fpriv->prt_va; 663 } else { 664 bo_va = NULL; 665 } 666 667 switch (args->operation) { 668 case AMDGPU_VA_OP_MAP: 669 va_flags = amdgpu_gem_va_map_flags(adev, args->flags); 670 r = amdgpu_vm_bo_map(adev, bo_va, args->va_address, 671 args->offset_in_bo, args->map_size, 672 va_flags); 673 break; 674 case AMDGPU_VA_OP_UNMAP: 675 r = amdgpu_vm_bo_unmap(adev, bo_va, args->va_address); 676 break; 677 678 case AMDGPU_VA_OP_CLEAR: 679 r = amdgpu_vm_bo_clear_mappings(adev, &fpriv->vm, 680 args->va_address, 681 args->map_size); 682 break; 683 case AMDGPU_VA_OP_REPLACE: 684 va_flags = amdgpu_gem_va_map_flags(adev, args->flags); 685 r = amdgpu_vm_bo_replace_map(adev, bo_va, args->va_address, 686 args->offset_in_bo, args->map_size, 687 va_flags); 688 break; 689 default: 690 break; 691 } 692 if (!r && !(args->flags & AMDGPU_VM_DELAY_UPDATE) && !amdgpu_vm_debug) 693 amdgpu_gem_va_update_vm(adev, &fpriv->vm, bo_va, 694 args->operation); 695 696 error_backoff: 697 ttm_eu_backoff_reservation(&ticket, &list); 698 699 error_unref: 700 drm_gem_object_put_unlocked(gobj); 701 return r; 702 } 703 704 int amdgpu_gem_op_ioctl(struct drm_device *dev, void *data, 705 struct drm_file *filp) 706 { 707 struct amdgpu_device *adev = dev->dev_private; 708 struct drm_amdgpu_gem_op *args = data; 709 struct drm_gem_object *gobj; 710 struct amdgpu_vm_bo_base *base; 711 struct amdgpu_bo *robj; 712 int r; 713 714 gobj = drm_gem_object_lookup(filp, args->handle); 715 if (gobj == NULL) { 716 return -ENOENT; 717 } 718 robj = gem_to_amdgpu_bo(gobj); 719 720 r = amdgpu_bo_reserve(robj, false); 721 if (unlikely(r)) 722 goto out; 723 724 switch (args->op) { 725 case AMDGPU_GEM_OP_GET_GEM_CREATE_INFO: { 726 struct drm_amdgpu_gem_create_in info; 727 void __user *out = u64_to_user_ptr(args->value); 728 729 info.bo_size = robj->tbo.base.size; 730 info.alignment = robj->tbo.mem.page_alignment << PAGE_SHIFT; 731 info.domains = robj->preferred_domains; 732 info.domain_flags = robj->flags; 733 amdgpu_bo_unreserve(robj); 734 if (copy_to_user(out, &info, sizeof(info))) 735 r = -EFAULT; 736 break; 737 } 738 case AMDGPU_GEM_OP_SET_PLACEMENT: 739 if (robj->prime_shared_count && (args->value & AMDGPU_GEM_DOMAIN_VRAM)) { 740 r = -EINVAL; 741 amdgpu_bo_unreserve(robj); 742 break; 743 } 744 if (amdgpu_ttm_tt_get_usermm(robj->tbo.ttm)) { 745 r = -EPERM; 746 amdgpu_bo_unreserve(robj); 747 break; 748 } 749 for (base = robj->vm_bo; base; base = base->next) 750 if (amdgpu_xgmi_same_hive(amdgpu_ttm_adev(robj->tbo.bdev), 751 amdgpu_ttm_adev(base->vm->root.base.bo->tbo.bdev))) { 752 r = -EINVAL; 753 amdgpu_bo_unreserve(robj); 754 goto out; 755 } 756 757 758 robj->preferred_domains = args->value & (AMDGPU_GEM_DOMAIN_VRAM | 759 AMDGPU_GEM_DOMAIN_GTT | 760 AMDGPU_GEM_DOMAIN_CPU); 761 robj->allowed_domains = robj->preferred_domains; 762 if (robj->allowed_domains == AMDGPU_GEM_DOMAIN_VRAM) 763 robj->allowed_domains |= AMDGPU_GEM_DOMAIN_GTT; 764 765 if (robj->flags & AMDGPU_GEM_CREATE_VM_ALWAYS_VALID) 766 amdgpu_vm_bo_invalidate(adev, robj, true); 767 768 amdgpu_bo_unreserve(robj); 769 break; 770 default: 771 amdgpu_bo_unreserve(robj); 772 r = -EINVAL; 773 } 774 775 out: 776 drm_gem_object_put_unlocked(gobj); 777 return r; 778 } 779 780 int amdgpu_mode_dumb_create(struct drm_file *file_priv, 781 struct drm_device *dev, 782 struct drm_mode_create_dumb *args) 783 { 784 struct amdgpu_device *adev = dev->dev_private; 785 struct drm_gem_object *gobj; 786 uint32_t handle; 787 u64 flags = AMDGPU_GEM_CREATE_CPU_ACCESS_REQUIRED | 788 AMDGPU_GEM_CREATE_CPU_GTT_USWC; 789 u32 domain; 790 int r; 791 792 /* 793 * The buffer returned from this function should be cleared, but 794 * it can only be done if the ring is enabled or we'll fail to 795 * create the buffer. 796 */ 797 if (adev->mman.buffer_funcs_enabled) 798 flags |= AMDGPU_GEM_CREATE_VRAM_CLEARED; 799 800 args->pitch = amdgpu_align_pitch(adev, args->width, 801 DIV_ROUND_UP(args->bpp, 8), 0); 802 args->size = (u64)args->pitch * args->height; 803 args->size = ALIGN(args->size, PAGE_SIZE); 804 domain = amdgpu_bo_get_preferred_pin_domain(adev, 805 amdgpu_display_supported_domains(adev, flags)); 806 r = amdgpu_gem_object_create(adev, args->size, 0, domain, flags, 807 ttm_bo_type_device, NULL, &gobj); 808 if (r) 809 return -ENOMEM; 810 811 r = drm_gem_handle_create(file_priv, gobj, &handle); 812 /* drop reference from allocate - handle holds it now */ 813 drm_gem_object_put_unlocked(gobj); 814 if (r) { 815 return r; 816 } 817 args->handle = handle; 818 return 0; 819 } 820 821 #if defined(CONFIG_DEBUG_FS) 822 823 #define amdgpu_debugfs_gem_bo_print_flag(m, bo, flag) \ 824 if (bo->flags & (AMDGPU_GEM_CREATE_ ## flag)) { \ 825 seq_printf((m), " " #flag); \ 826 } 827 828 static int amdgpu_debugfs_gem_bo_info(int id, void *ptr, void *data) 829 { 830 struct drm_gem_object *gobj = ptr; 831 struct amdgpu_bo *bo = gem_to_amdgpu_bo(gobj); 832 struct seq_file *m = data; 833 834 struct dma_buf_attachment *attachment; 835 struct dma_buf *dma_buf; 836 unsigned domain; 837 const char *placement; 838 unsigned pin_count; 839 840 domain = amdgpu_mem_type_to_domain(bo->tbo.mem.mem_type); 841 switch (domain) { 842 case AMDGPU_GEM_DOMAIN_VRAM: 843 placement = "VRAM"; 844 break; 845 case AMDGPU_GEM_DOMAIN_GTT: 846 placement = " GTT"; 847 break; 848 case AMDGPU_GEM_DOMAIN_CPU: 849 default: 850 placement = " CPU"; 851 break; 852 } 853 seq_printf(m, "\t0x%08x: %12ld byte %s", 854 id, amdgpu_bo_size(bo), placement); 855 856 pin_count = READ_ONCE(bo->pin_count); 857 if (pin_count) 858 seq_printf(m, " pin count %d", pin_count); 859 860 dma_buf = READ_ONCE(bo->tbo.base.dma_buf); 861 attachment = READ_ONCE(bo->tbo.base.import_attach); 862 863 if (attachment) 864 seq_printf(m, " imported from %p", dma_buf); 865 else if (dma_buf) 866 seq_printf(m, " exported as %p", dma_buf); 867 868 amdgpu_debugfs_gem_bo_print_flag(m, bo, CPU_ACCESS_REQUIRED); 869 amdgpu_debugfs_gem_bo_print_flag(m, bo, NO_CPU_ACCESS); 870 amdgpu_debugfs_gem_bo_print_flag(m, bo, CPU_GTT_USWC); 871 amdgpu_debugfs_gem_bo_print_flag(m, bo, VRAM_CLEARED); 872 amdgpu_debugfs_gem_bo_print_flag(m, bo, SHADOW); 873 amdgpu_debugfs_gem_bo_print_flag(m, bo, VRAM_CONTIGUOUS); 874 amdgpu_debugfs_gem_bo_print_flag(m, bo, VM_ALWAYS_VALID); 875 amdgpu_debugfs_gem_bo_print_flag(m, bo, EXPLICIT_SYNC); 876 877 seq_printf(m, "\n"); 878 879 return 0; 880 } 881 882 static int amdgpu_debugfs_gem_info(struct seq_file *m, void *data) 883 { 884 struct drm_info_node *node = (struct drm_info_node *)m->private; 885 struct drm_device *dev = node->minor->dev; 886 struct drm_file *file; 887 int r; 888 889 r = mutex_lock_interruptible(&dev->filelist_mutex); 890 if (r) 891 return r; 892 893 list_for_each_entry(file, &dev->filelist, lhead) { 894 struct task_struct *task; 895 896 /* 897 * Although we have a valid reference on file->pid, that does 898 * not guarantee that the task_struct who called get_pid() is 899 * still alive (e.g. get_pid(current) => fork() => exit()). 900 * Therefore, we need to protect this ->comm access using RCU. 901 */ 902 rcu_read_lock(); 903 task = pid_task(file->pid, PIDTYPE_PID); 904 seq_printf(m, "pid %8d command %s:\n", pid_nr(file->pid), 905 task ? task->comm : "<unknown>"); 906 rcu_read_unlock(); 907 908 spin_lock(&file->table_lock); 909 idr_for_each(&file->object_idr, amdgpu_debugfs_gem_bo_info, m); 910 spin_unlock(&file->table_lock); 911 } 912 913 mutex_unlock(&dev->filelist_mutex); 914 return 0; 915 } 916 917 static const struct drm_info_list amdgpu_debugfs_gem_list[] = { 918 {"amdgpu_gem_info", &amdgpu_debugfs_gem_info, 0, NULL}, 919 }; 920 #endif 921 922 int amdgpu_debugfs_gem_init(struct amdgpu_device *adev) 923 { 924 #if defined(CONFIG_DEBUG_FS) 925 return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_gem_list, 1); 926 #endif 927 return 0; 928 } 929