1 /************************************************************************** 2 * 3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 4 * All Rights Reserved. 5 * 6 * Permission is hereby granted, free of charge, to any person obtaining a 7 * copy of this software and associated documentation files (the 8 * "Software"), to deal in the Software without restriction, including 9 * without limitation the rights to use, copy, modify, merge, publish, 10 * distribute, sub license, and/or sell copies of the Software, and to 11 * permit persons to whom the Software is furnished to do so, subject to 12 * the following conditions: 13 * 14 * The above copyright notice and this permission notice (including the 15 * next paragraph) shall be included in all copies or substantial portions 16 * of the Software. 17 * 18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 24 * USE OR OTHER DEALINGS IN THE SOFTWARE. 25 * 26 **************************************************************************/ 27 /* 28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 29 * 30 * $FreeBSD: head/sys/dev/drm2/ttm/ttm_bo.c 248060 2013-03-08 18:11:02Z dumbbell $ 31 */ 32 33 #include <drm/drmP.h> 34 #include <dev/drm/ttm/ttm_module.h> 35 #include <dev/drm/ttm/ttm_bo_driver.h> 36 #include <dev/drm/ttm/ttm_placement.h> 37 38 #define TTM_ASSERT_LOCKED(param) 39 #define TTM_DEBUG(fmt, arg...) 40 #define TTM_BO_HASH_ORDER 13 41 42 static int ttm_bo_setup_vm(struct ttm_buffer_object *bo); 43 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink); 44 static void ttm_bo_global_kobj_release(struct ttm_bo_global *glob); 45 46 MALLOC_DEFINE(M_TTM_BO, "ttm_bo", "TTM Buffer Objects"); 47 48 static inline int ttm_mem_type_from_flags(uint32_t flags, uint32_t *mem_type) 49 { 50 int i; 51 52 for (i = 0; i <= TTM_PL_PRIV5; i++) 53 if (flags & (1 << i)) { 54 *mem_type = i; 55 return 0; 56 } 57 return -EINVAL; 58 } 59 60 static void ttm_mem_type_debug(struct ttm_bo_device *bdev, int mem_type) 61 { 62 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 63 64 kprintf(" has_type: %d\n", man->has_type); 65 kprintf(" use_type: %d\n", man->use_type); 66 kprintf(" flags: 0x%08X\n", man->flags); 67 kprintf(" gpu_offset: 0x%08lX\n", man->gpu_offset); 68 kprintf(" size: %ju\n", (uintmax_t)man->size); 69 kprintf(" available_caching: 0x%08X\n", man->available_caching); 70 kprintf(" default_caching: 0x%08X\n", man->default_caching); 71 if (mem_type != TTM_PL_SYSTEM) 72 (*man->func->debug)(man, TTM_PFX); 73 } 74 75 static void ttm_bo_mem_space_debug(struct ttm_buffer_object *bo, 76 struct ttm_placement *placement) 77 { 78 int i, ret, mem_type; 79 80 kprintf("No space for %p (%lu pages, %luK, %luM)\n", 81 bo, bo->mem.num_pages, bo->mem.size >> 10, 82 bo->mem.size >> 20); 83 for (i = 0; i < placement->num_placement; i++) { 84 ret = ttm_mem_type_from_flags(placement->placement[i], 85 &mem_type); 86 if (ret) 87 return; 88 kprintf(" placement[%d]=0x%08X (%d)\n", 89 i, placement->placement[i], mem_type); 90 ttm_mem_type_debug(bo->bdev, mem_type); 91 } 92 } 93 94 #if 0 95 static ssize_t ttm_bo_global_show(struct ttm_bo_global *glob, 96 char *buffer) 97 { 98 99 return snprintf(buffer, PAGE_SIZE, "%lu\n", 100 (unsigned long) atomic_read(&glob->bo_count)); 101 } 102 #endif 103 104 static inline uint32_t ttm_bo_type_flags(unsigned type) 105 { 106 return 1 << (type); 107 } 108 109 static void ttm_bo_release_list(struct ttm_buffer_object *bo) 110 { 111 struct ttm_bo_device *bdev = bo->bdev; 112 size_t acc_size = bo->acc_size; 113 114 KKASSERT(atomic_read(&bo->list_kref) == 0); 115 KKASSERT(atomic_read(&bo->kref) == 0); 116 KKASSERT(atomic_read(&bo->cpu_writers) == 0); 117 KKASSERT(bo->sync_obj == NULL); 118 KKASSERT(bo->mem.mm_node == NULL); 119 KKASSERT(list_empty(&bo->lru)); 120 KKASSERT(list_empty(&bo->ddestroy)); 121 122 if (bo->ttm) 123 ttm_tt_destroy(bo->ttm); 124 atomic_dec(&bo->glob->bo_count); 125 if (bo->destroy) 126 bo->destroy(bo); 127 else { 128 drm_free(bo, M_TTM_BO); 129 } 130 ttm_mem_global_free(bdev->glob->mem_glob, acc_size); 131 } 132 133 static int 134 ttm_bo_wait_unreserved_locked(struct ttm_buffer_object *bo, bool interruptible) 135 { 136 const char *wmsg; 137 int flags, ret; 138 139 ret = 0; 140 if (interruptible) { 141 flags = PCATCH; 142 wmsg = "ttbowi"; 143 } else { 144 flags = 0; 145 wmsg = "ttbowu"; 146 } 147 while (ttm_bo_is_reserved(bo)) { 148 ret = -lksleep(bo, &bo->glob->lru_lock, 0, wmsg, 0); 149 if (ret != 0) 150 break; 151 } 152 return (ret); 153 } 154 155 void ttm_bo_add_to_lru(struct ttm_buffer_object *bo) 156 { 157 struct ttm_bo_device *bdev = bo->bdev; 158 struct ttm_mem_type_manager *man; 159 160 KKASSERT(ttm_bo_is_reserved(bo)); 161 162 if (!(bo->mem.placement & TTM_PL_FLAG_NO_EVICT)) { 163 164 KKASSERT(list_empty(&bo->lru)); 165 166 man = &bdev->man[bo->mem.mem_type]; 167 list_add_tail(&bo->lru, &man->lru); 168 refcount_acquire(&bo->list_kref); 169 170 if (bo->ttm != NULL) { 171 list_add_tail(&bo->swap, &bo->glob->swap_lru); 172 refcount_acquire(&bo->list_kref); 173 } 174 } 175 } 176 177 int ttm_bo_del_from_lru(struct ttm_buffer_object *bo) 178 { 179 int put_count = 0; 180 181 if (!list_empty(&bo->swap)) { 182 list_del_init(&bo->swap); 183 ++put_count; 184 } 185 if (!list_empty(&bo->lru)) { 186 list_del_init(&bo->lru); 187 ++put_count; 188 } 189 190 /* 191 * TODO: Add a driver hook to delete from 192 * driver-specific LRU's here. 193 */ 194 195 return put_count; 196 } 197 198 int ttm_bo_reserve_nolru(struct ttm_buffer_object *bo, 199 bool interruptible, 200 bool no_wait, bool use_sequence, uint32_t sequence) 201 { 202 int ret; 203 204 while (unlikely(atomic_xchg(&bo->reserved, 1) != 0)) { 205 /** 206 * Deadlock avoidance for multi-bo reserving. 207 */ 208 if (use_sequence && bo->seq_valid) { 209 /** 210 * We've already reserved this one. 211 */ 212 if (unlikely(sequence == bo->val_seq)) 213 return -EDEADLK; 214 /** 215 * Already reserved by a thread that will not back 216 * off for us. We need to back off. 217 */ 218 if (unlikely(sequence - bo->val_seq < (1 << 31))) 219 return -EAGAIN; 220 } 221 222 if (no_wait) 223 return -EBUSY; 224 225 ret = ttm_bo_wait_unreserved_locked(bo, interruptible); 226 227 if (unlikely(ret)) 228 return ret; 229 } 230 231 if (use_sequence) { 232 bool wake_up = false; 233 /** 234 * Wake up waiters that may need to recheck for deadlock, 235 * if we decreased the sequence number. 236 */ 237 if (unlikely((bo->val_seq - sequence < (1 << 31)) 238 || !bo->seq_valid)) 239 wake_up = true; 240 241 /* 242 * In the worst case with memory ordering these values can be 243 * seen in the wrong order. However since we call wake_up_all 244 * in that case, this will hopefully not pose a problem, 245 * and the worst case would only cause someone to accidentally 246 * hit -EAGAIN in ttm_bo_reserve when they see old value of 247 * val_seq. However this would only happen if seq_valid was 248 * written before val_seq was, and just means some slightly 249 * increased cpu usage 250 */ 251 bo->val_seq = sequence; 252 bo->seq_valid = true; 253 if (wake_up) 254 wakeup(bo); 255 } else { 256 bo->seq_valid = false; 257 } 258 259 return 0; 260 } 261 262 void ttm_bo_list_ref_sub(struct ttm_buffer_object *bo, int count, 263 bool never_free) 264 { 265 u_int old; 266 267 old = atomic_fetchadd_int(&bo->list_kref, -count); 268 if (old <= count) { 269 if (never_free) 270 panic("ttm_bo_ref_buf"); 271 ttm_bo_release_list(bo); 272 } 273 } 274 275 int ttm_bo_reserve(struct ttm_buffer_object *bo, 276 bool interruptible, 277 bool no_wait, bool use_sequence, uint32_t sequence) 278 { 279 struct ttm_bo_global *glob = bo->glob; 280 int put_count = 0; 281 int ret; 282 283 lockmgr(&bo->glob->lru_lock, LK_EXCLUSIVE); 284 ret = ttm_bo_reserve_nolru(bo, interruptible, no_wait, use_sequence, 285 sequence); 286 if (likely(ret == 0)) { 287 put_count = ttm_bo_del_from_lru(bo); 288 lockmgr(&glob->lru_lock, LK_RELEASE); 289 ttm_bo_list_ref_sub(bo, put_count, true); 290 } else 291 lockmgr(&bo->glob->lru_lock, LK_RELEASE); 292 293 return ret; 294 } 295 296 int ttm_bo_reserve_slowpath_nolru(struct ttm_buffer_object *bo, 297 bool interruptible, uint32_t sequence) 298 { 299 bool wake_up = false; 300 int ret; 301 302 while (unlikely(atomic_xchg(&bo->reserved, 1) != 0)) { 303 if (bo->seq_valid && sequence == bo->val_seq) { 304 DRM_ERROR( 305 "%s: bo->seq_valid && sequence == bo->val_seq", 306 __func__); 307 } 308 309 ret = ttm_bo_wait_unreserved_locked(bo, interruptible); 310 311 if (unlikely(ret)) 312 return ret; 313 } 314 315 if ((bo->val_seq - sequence < (1 << 31)) || !bo->seq_valid) 316 wake_up = true; 317 318 /** 319 * Wake up waiters that may need to recheck for deadlock, 320 * if we decreased the sequence number. 321 */ 322 bo->val_seq = sequence; 323 bo->seq_valid = true; 324 if (wake_up) 325 wakeup(bo); 326 327 return 0; 328 } 329 330 int ttm_bo_reserve_slowpath(struct ttm_buffer_object *bo, 331 bool interruptible, uint32_t sequence) 332 { 333 struct ttm_bo_global *glob = bo->glob; 334 int put_count, ret; 335 336 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 337 ret = ttm_bo_reserve_slowpath_nolru(bo, interruptible, sequence); 338 if (likely(!ret)) { 339 put_count = ttm_bo_del_from_lru(bo); 340 lockmgr(&glob->lru_lock, LK_RELEASE); 341 ttm_bo_list_ref_sub(bo, put_count, true); 342 } else 343 lockmgr(&glob->lru_lock, LK_RELEASE); 344 return ret; 345 } 346 347 void ttm_bo_unreserve_locked(struct ttm_buffer_object *bo) 348 { 349 ttm_bo_add_to_lru(bo); 350 atomic_set(&bo->reserved, 0); 351 wakeup(bo); 352 } 353 354 void ttm_bo_unreserve(struct ttm_buffer_object *bo) 355 { 356 struct ttm_bo_global *glob = bo->glob; 357 358 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 359 ttm_bo_unreserve_locked(bo); 360 lockmgr(&glob->lru_lock, LK_RELEASE); 361 } 362 363 /* 364 * Call bo->mutex locked. 365 */ 366 static int ttm_bo_add_ttm(struct ttm_buffer_object *bo, bool zero_alloc) 367 { 368 struct ttm_bo_device *bdev = bo->bdev; 369 struct ttm_bo_global *glob = bo->glob; 370 int ret = 0; 371 uint32_t page_flags = 0; 372 373 TTM_ASSERT_LOCKED(&bo->mutex); 374 bo->ttm = NULL; 375 376 if (bdev->need_dma32) 377 page_flags |= TTM_PAGE_FLAG_DMA32; 378 379 switch (bo->type) { 380 case ttm_bo_type_device: 381 if (zero_alloc) 382 page_flags |= TTM_PAGE_FLAG_ZERO_ALLOC; 383 case ttm_bo_type_kernel: 384 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 385 page_flags, glob->dummy_read_page); 386 if (unlikely(bo->ttm == NULL)) 387 ret = -ENOMEM; 388 break; 389 case ttm_bo_type_sg: 390 bo->ttm = bdev->driver->ttm_tt_create(bdev, bo->num_pages << PAGE_SHIFT, 391 page_flags | TTM_PAGE_FLAG_SG, 392 glob->dummy_read_page); 393 if (unlikely(bo->ttm == NULL)) { 394 ret = -ENOMEM; 395 break; 396 } 397 bo->ttm->sg = bo->sg; 398 break; 399 default: 400 kprintf("[TTM] Illegal buffer object type\n"); 401 ret = -EINVAL; 402 break; 403 } 404 405 return ret; 406 } 407 408 static int ttm_bo_handle_move_mem(struct ttm_buffer_object *bo, 409 struct ttm_mem_reg *mem, 410 bool evict, bool interruptible, 411 bool no_wait_gpu) 412 { 413 struct ttm_bo_device *bdev = bo->bdev; 414 bool old_is_pci = ttm_mem_reg_is_pci(bdev, &bo->mem); 415 bool new_is_pci = ttm_mem_reg_is_pci(bdev, mem); 416 struct ttm_mem_type_manager *old_man = &bdev->man[bo->mem.mem_type]; 417 struct ttm_mem_type_manager *new_man = &bdev->man[mem->mem_type]; 418 int ret = 0; 419 420 if (old_is_pci || new_is_pci || 421 ((mem->placement & bo->mem.placement & TTM_PL_MASK_CACHING) == 0)) { 422 ret = ttm_mem_io_lock(old_man, true); 423 if (unlikely(ret != 0)) 424 goto out_err; 425 ttm_bo_unmap_virtual_locked(bo); 426 ttm_mem_io_unlock(old_man); 427 } 428 429 /* 430 * Create and bind a ttm if required. 431 */ 432 433 if (!(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 434 if (bo->ttm == NULL) { 435 bool zero = !(old_man->flags & TTM_MEMTYPE_FLAG_FIXED); 436 ret = ttm_bo_add_ttm(bo, zero); 437 if (ret) 438 goto out_err; 439 } 440 441 ret = ttm_tt_set_placement_caching(bo->ttm, mem->placement); 442 if (ret) 443 goto out_err; 444 445 if (mem->mem_type != TTM_PL_SYSTEM) { 446 ret = ttm_tt_bind(bo->ttm, mem); 447 if (ret) 448 goto out_err; 449 } 450 451 if (bo->mem.mem_type == TTM_PL_SYSTEM) { 452 if (bdev->driver->move_notify) 453 bdev->driver->move_notify(bo, mem); 454 bo->mem = *mem; 455 mem->mm_node = NULL; 456 goto moved; 457 } 458 } 459 460 if (bdev->driver->move_notify) 461 bdev->driver->move_notify(bo, mem); 462 463 if (!(old_man->flags & TTM_MEMTYPE_FLAG_FIXED) && 464 !(new_man->flags & TTM_MEMTYPE_FLAG_FIXED)) 465 ret = ttm_bo_move_ttm(bo, evict, no_wait_gpu, mem); 466 else if (bdev->driver->move) 467 ret = bdev->driver->move(bo, evict, interruptible, 468 no_wait_gpu, mem); 469 else 470 ret = ttm_bo_move_memcpy(bo, evict, no_wait_gpu, mem); 471 472 if (ret) { 473 if (bdev->driver->move_notify) { 474 struct ttm_mem_reg tmp_mem = *mem; 475 *mem = bo->mem; 476 bo->mem = tmp_mem; 477 bdev->driver->move_notify(bo, mem); 478 bo->mem = *mem; 479 *mem = tmp_mem; 480 } 481 482 goto out_err; 483 } 484 485 moved: 486 if (bo->evicted) { 487 ret = bdev->driver->invalidate_caches(bdev, bo->mem.placement); 488 if (ret) 489 kprintf("[TTM] Can not flush read caches\n"); 490 bo->evicted = false; 491 } 492 493 if (bo->mem.mm_node) { 494 bo->offset = (bo->mem.start << PAGE_SHIFT) + 495 bdev->man[bo->mem.mem_type].gpu_offset; 496 bo->cur_placement = bo->mem.placement; 497 } else 498 bo->offset = 0; 499 500 return 0; 501 502 out_err: 503 new_man = &bdev->man[bo->mem.mem_type]; 504 if ((new_man->flags & TTM_MEMTYPE_FLAG_FIXED) && bo->ttm) { 505 ttm_tt_unbind(bo->ttm); 506 ttm_tt_destroy(bo->ttm); 507 bo->ttm = NULL; 508 } 509 510 return ret; 511 } 512 513 /** 514 * Call bo::reserved. 515 * Will release GPU memory type usage on destruction. 516 * This is the place to put in driver specific hooks to release 517 * driver private resources. 518 * Will release the bo::reserved lock. 519 */ 520 521 static void ttm_bo_cleanup_memtype_use(struct ttm_buffer_object *bo) 522 { 523 if (bo->bdev->driver->move_notify) 524 bo->bdev->driver->move_notify(bo, NULL); 525 526 if (bo->ttm) { 527 ttm_tt_unbind(bo->ttm); 528 ttm_tt_destroy(bo->ttm); 529 bo->ttm = NULL; 530 } 531 ttm_bo_mem_put(bo, &bo->mem); 532 533 atomic_set(&bo->reserved, 0); 534 wakeup(&bo); 535 536 /* 537 * Since the final reference to this bo may not be dropped by 538 * the current task we have to put a memory barrier here to make 539 * sure the changes done in this function are always visible. 540 * 541 * This function only needs protection against the final kref_put. 542 */ 543 cpu_mfence(); 544 } 545 546 static void ttm_bo_cleanup_refs_or_queue(struct ttm_buffer_object *bo) 547 { 548 struct ttm_bo_device *bdev = bo->bdev; 549 struct ttm_bo_global *glob = bo->glob; 550 struct ttm_bo_driver *driver = bdev->driver; 551 void *sync_obj = NULL; 552 int put_count; 553 int ret; 554 555 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 556 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 557 558 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 559 (void) ttm_bo_wait(bo, false, false, true); 560 if (!ret && !bo->sync_obj) { 561 lockmgr(&bdev->fence_lock, LK_RELEASE); 562 put_count = ttm_bo_del_from_lru(bo); 563 564 lockmgr(&glob->lru_lock, LK_RELEASE); 565 ttm_bo_cleanup_memtype_use(bo); 566 567 ttm_bo_list_ref_sub(bo, put_count, true); 568 569 return; 570 } 571 if (bo->sync_obj) 572 sync_obj = driver->sync_obj_ref(bo->sync_obj); 573 lockmgr(&bdev->fence_lock, LK_RELEASE); 574 575 if (!ret) { 576 atomic_set(&bo->reserved, 0); 577 wakeup(bo); 578 } 579 580 refcount_acquire(&bo->list_kref); 581 list_add_tail(&bo->ddestroy, &bdev->ddestroy); 582 lockmgr(&glob->lru_lock, LK_RELEASE); 583 584 if (sync_obj) { 585 driver->sync_obj_flush(sync_obj); 586 driver->sync_obj_unref(&sync_obj); 587 } 588 taskqueue_enqueue_timeout(taskqueue_thread[mycpuid], &bdev->wq, 589 ((hz / 100) < 1) ? 1 : hz / 100); 590 } 591 592 /** 593 * function ttm_bo_cleanup_refs_and_unlock 594 * If bo idle, remove from delayed- and lru lists, and unref. 595 * If not idle, do nothing. 596 * 597 * Must be called with lru_lock and reservation held, this function 598 * will drop both before returning. 599 * 600 * @interruptible Any sleeps should occur interruptibly. 601 * @no_wait_gpu Never wait for gpu. Return -EBUSY instead. 602 */ 603 604 static int ttm_bo_cleanup_refs_and_unlock(struct ttm_buffer_object *bo, 605 bool interruptible, 606 bool no_wait_gpu) 607 { 608 struct ttm_bo_device *bdev = bo->bdev; 609 struct ttm_bo_driver *driver = bdev->driver; 610 struct ttm_bo_global *glob = bo->glob; 611 int put_count; 612 int ret; 613 614 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 615 ret = ttm_bo_wait(bo, false, false, true); 616 617 if (ret && !no_wait_gpu) { 618 void *sync_obj; 619 620 /* 621 * Take a reference to the fence and unreserve, 622 * at this point the buffer should be dead, so 623 * no new sync objects can be attached. 624 */ 625 sync_obj = driver->sync_obj_ref(bo->sync_obj); 626 lockmgr(&bdev->fence_lock, LK_RELEASE); 627 628 atomic_set(&bo->reserved, 0); 629 wakeup(bo); 630 lockmgr(&glob->lru_lock, LK_RELEASE); 631 632 ret = driver->sync_obj_wait(sync_obj, false, interruptible); 633 driver->sync_obj_unref(&sync_obj); 634 if (ret) 635 return ret; 636 637 /* 638 * remove sync_obj with ttm_bo_wait, the wait should be 639 * finished, and no new wait object should have been added. 640 */ 641 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 642 ret = ttm_bo_wait(bo, false, false, true); 643 lockmgr(&bdev->fence_lock, LK_RELEASE); 644 if (ret) 645 return ret; 646 647 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 648 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 649 650 /* 651 * We raced, and lost, someone else holds the reservation now, 652 * and is probably busy in ttm_bo_cleanup_memtype_use. 653 * 654 * Even if it's not the case, because we finished waiting any 655 * delayed destruction would succeed, so just return success 656 * here. 657 */ 658 if (ret) { 659 lockmgr(&glob->lru_lock, LK_RELEASE); 660 return 0; 661 } 662 } else 663 lockmgr(&bdev->fence_lock, LK_RELEASE); 664 665 if (ret || unlikely(list_empty(&bo->ddestroy))) { 666 atomic_set(&bo->reserved, 0); 667 wakeup(bo); 668 lockmgr(&glob->lru_lock, LK_RELEASE); 669 return ret; 670 } 671 672 put_count = ttm_bo_del_from_lru(bo); 673 list_del_init(&bo->ddestroy); 674 ++put_count; 675 676 lockmgr(&glob->lru_lock, LK_RELEASE); 677 ttm_bo_cleanup_memtype_use(bo); 678 679 ttm_bo_list_ref_sub(bo, put_count, true); 680 681 return 0; 682 } 683 684 /** 685 * Traverse the delayed list, and call ttm_bo_cleanup_refs on all 686 * encountered buffers. 687 */ 688 689 static int ttm_bo_delayed_delete(struct ttm_bo_device *bdev, bool remove_all) 690 { 691 struct ttm_bo_global *glob = bdev->glob; 692 struct ttm_buffer_object *entry = NULL; 693 int ret = 0; 694 695 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 696 if (list_empty(&bdev->ddestroy)) 697 goto out_unlock; 698 699 entry = list_first_entry(&bdev->ddestroy, 700 struct ttm_buffer_object, ddestroy); 701 refcount_acquire(&entry->list_kref); 702 703 for (;;) { 704 struct ttm_buffer_object *nentry = NULL; 705 706 if (entry->ddestroy.next != &bdev->ddestroy) { 707 nentry = list_first_entry(&entry->ddestroy, 708 struct ttm_buffer_object, ddestroy); 709 refcount_acquire(&nentry->list_kref); 710 } 711 712 ret = ttm_bo_reserve_nolru(entry, false, true, false, 0); 713 if (remove_all && ret) { 714 ret = ttm_bo_reserve_nolru(entry, false, false, 715 false, 0); 716 } 717 718 if (!ret) 719 ret = ttm_bo_cleanup_refs_and_unlock(entry, false, 720 !remove_all); 721 else 722 lockmgr(&glob->lru_lock, LK_RELEASE); 723 724 if (refcount_release(&entry->list_kref)) 725 ttm_bo_release_list(entry); 726 entry = nentry; 727 728 if (ret || !entry) 729 goto out; 730 731 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 732 if (list_empty(&entry->ddestroy)) 733 break; 734 } 735 736 out_unlock: 737 lockmgr(&glob->lru_lock, LK_RELEASE); 738 out: 739 if (entry && refcount_release(&entry->list_kref)) 740 ttm_bo_release_list(entry); 741 return ret; 742 } 743 744 static void ttm_bo_delayed_workqueue(void *arg, int pending __unused) 745 { 746 struct ttm_bo_device *bdev = arg; 747 748 if (ttm_bo_delayed_delete(bdev, false)) { 749 taskqueue_enqueue_timeout(taskqueue_thread[mycpuid], &bdev->wq, 750 ((hz / 100) < 1) ? 1 : hz / 100); 751 } 752 } 753 754 static void ttm_bo_release(struct ttm_buffer_object *bo) 755 { 756 struct ttm_bo_device *bdev = bo->bdev; 757 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 758 759 lockmgr(&bdev->vm_lock, LK_EXCLUSIVE); 760 if (likely(bo->vm_node != NULL)) { 761 RB_REMOVE(ttm_bo_device_buffer_objects, 762 &bdev->addr_space_rb, bo); 763 drm_mm_put_block(bo->vm_node); 764 bo->vm_node = NULL; 765 } 766 lockmgr(&bdev->vm_lock, LK_RELEASE); 767 ttm_mem_io_lock(man, false); 768 ttm_mem_io_free_vm(bo); 769 ttm_mem_io_unlock(man); 770 ttm_bo_cleanup_refs_or_queue(bo); 771 if (refcount_release(&bo->list_kref)) 772 ttm_bo_release_list(bo); 773 } 774 775 void ttm_bo_unref(struct ttm_buffer_object **p_bo) 776 { 777 struct ttm_buffer_object *bo = *p_bo; 778 779 *p_bo = NULL; 780 if (refcount_release(&bo->kref)) 781 ttm_bo_release(bo); 782 } 783 784 int ttm_bo_lock_delayed_workqueue(struct ttm_bo_device *bdev) 785 { 786 int pending; 787 788 taskqueue_cancel_timeout(taskqueue_thread[mycpuid], &bdev->wq, &pending); 789 if (pending) 790 taskqueue_drain_timeout(taskqueue_thread[mycpuid], &bdev->wq); 791 return (pending); 792 } 793 794 void ttm_bo_unlock_delayed_workqueue(struct ttm_bo_device *bdev, int resched) 795 { 796 if (resched) { 797 taskqueue_enqueue_timeout(taskqueue_thread[mycpuid], &bdev->wq, 798 ((hz / 100) < 1) ? 1 : hz / 100); 799 } 800 } 801 802 static int ttm_bo_evict(struct ttm_buffer_object *bo, bool interruptible, 803 bool no_wait_gpu) 804 { 805 struct ttm_bo_device *bdev = bo->bdev; 806 struct ttm_mem_reg evict_mem; 807 struct ttm_placement placement; 808 int ret = 0; 809 810 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 811 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 812 lockmgr(&bdev->fence_lock, LK_RELEASE); 813 814 if (unlikely(ret != 0)) { 815 if (ret != -ERESTART) { 816 kprintf("[TTM] Failed to expire sync object before buffer eviction\n"); 817 } 818 goto out; 819 } 820 821 KKASSERT(ttm_bo_is_reserved(bo)); 822 823 evict_mem = bo->mem; 824 evict_mem.mm_node = NULL; 825 evict_mem.bus.io_reserved_vm = false; 826 evict_mem.bus.io_reserved_count = 0; 827 828 placement.fpfn = 0; 829 placement.lpfn = 0; 830 placement.num_placement = 0; 831 placement.num_busy_placement = 0; 832 bdev->driver->evict_flags(bo, &placement); 833 ret = ttm_bo_mem_space(bo, &placement, &evict_mem, interruptible, 834 no_wait_gpu); 835 if (ret) { 836 if (ret != -ERESTART) { 837 kprintf("[TTM] Failed to find memory space for buffer 0x%p eviction\n", 838 bo); 839 ttm_bo_mem_space_debug(bo, &placement); 840 } 841 goto out; 842 } 843 844 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, interruptible, 845 no_wait_gpu); 846 if (ret) { 847 if (ret != -ERESTART) 848 kprintf("[TTM] Buffer eviction failed\n"); 849 ttm_bo_mem_put(bo, &evict_mem); 850 goto out; 851 } 852 bo->evicted = true; 853 out: 854 return ret; 855 } 856 857 static int ttm_mem_evict_first(struct ttm_bo_device *bdev, 858 uint32_t mem_type, 859 bool interruptible, 860 bool no_wait_gpu) 861 { 862 struct ttm_bo_global *glob = bdev->glob; 863 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 864 struct ttm_buffer_object *bo; 865 int ret = -EBUSY, put_count; 866 867 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 868 list_for_each_entry(bo, &man->lru, lru) { 869 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 870 if (!ret) 871 break; 872 } 873 874 if (ret) { 875 lockmgr(&glob->lru_lock, LK_RELEASE); 876 return ret; 877 } 878 879 refcount_acquire(&bo->list_kref); 880 881 if (!list_empty(&bo->ddestroy)) { 882 ret = ttm_bo_cleanup_refs_and_unlock(bo, interruptible, 883 no_wait_gpu); 884 if (refcount_release(&bo->list_kref)) 885 ttm_bo_release_list(bo); 886 return ret; 887 } 888 889 put_count = ttm_bo_del_from_lru(bo); 890 lockmgr(&glob->lru_lock, LK_RELEASE); 891 892 KKASSERT(ret == 0); 893 894 ttm_bo_list_ref_sub(bo, put_count, true); 895 896 ret = ttm_bo_evict(bo, interruptible, no_wait_gpu); 897 ttm_bo_unreserve(bo); 898 899 if (refcount_release(&bo->list_kref)) 900 ttm_bo_release_list(bo); 901 return ret; 902 } 903 904 void ttm_bo_mem_put(struct ttm_buffer_object *bo, struct ttm_mem_reg *mem) 905 { 906 struct ttm_mem_type_manager *man = &bo->bdev->man[mem->mem_type]; 907 908 if (mem->mm_node) 909 (*man->func->put_node)(man, mem); 910 } 911 912 /** 913 * Repeatedly evict memory from the LRU for @mem_type until we create enough 914 * space, or we've evicted everything and there isn't enough space. 915 */ 916 static int ttm_bo_mem_force_space(struct ttm_buffer_object *bo, 917 uint32_t mem_type, 918 struct ttm_placement *placement, 919 struct ttm_mem_reg *mem, 920 bool interruptible, 921 bool no_wait_gpu) 922 { 923 struct ttm_bo_device *bdev = bo->bdev; 924 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 925 int ret; 926 927 do { 928 ret = (*man->func->get_node)(man, bo, placement, mem); 929 if (unlikely(ret != 0)) 930 return ret; 931 if (mem->mm_node) 932 break; 933 ret = ttm_mem_evict_first(bdev, mem_type, 934 interruptible, no_wait_gpu); 935 if (unlikely(ret != 0)) 936 return ret; 937 } while (1); 938 if (mem->mm_node == NULL) 939 return -ENOMEM; 940 mem->mem_type = mem_type; 941 return 0; 942 } 943 944 static uint32_t ttm_bo_select_caching(struct ttm_mem_type_manager *man, 945 uint32_t cur_placement, 946 uint32_t proposed_placement) 947 { 948 uint32_t caching = proposed_placement & TTM_PL_MASK_CACHING; 949 uint32_t result = proposed_placement & ~TTM_PL_MASK_CACHING; 950 951 /** 952 * Keep current caching if possible. 953 */ 954 955 if ((cur_placement & caching) != 0) 956 result |= (cur_placement & caching); 957 else if ((man->default_caching & caching) != 0) 958 result |= man->default_caching; 959 else if ((TTM_PL_FLAG_CACHED & caching) != 0) 960 result |= TTM_PL_FLAG_CACHED; 961 else if ((TTM_PL_FLAG_WC & caching) != 0) 962 result |= TTM_PL_FLAG_WC; 963 else if ((TTM_PL_FLAG_UNCACHED & caching) != 0) 964 result |= TTM_PL_FLAG_UNCACHED; 965 966 return result; 967 } 968 969 static bool ttm_bo_mt_compatible(struct ttm_mem_type_manager *man, 970 uint32_t mem_type, 971 uint32_t proposed_placement, 972 uint32_t *masked_placement) 973 { 974 uint32_t cur_flags = ttm_bo_type_flags(mem_type); 975 976 if ((cur_flags & proposed_placement & TTM_PL_MASK_MEM) == 0) 977 return false; 978 979 if ((proposed_placement & man->available_caching) == 0) 980 return false; 981 982 cur_flags |= (proposed_placement & man->available_caching); 983 984 *masked_placement = cur_flags; 985 return true; 986 } 987 988 /** 989 * Creates space for memory region @mem according to its type. 990 * 991 * This function first searches for free space in compatible memory types in 992 * the priority order defined by the driver. If free space isn't found, then 993 * ttm_bo_mem_force_space is attempted in priority order to evict and find 994 * space. 995 */ 996 int ttm_bo_mem_space(struct ttm_buffer_object *bo, 997 struct ttm_placement *placement, 998 struct ttm_mem_reg *mem, 999 bool interruptible, 1000 bool no_wait_gpu) 1001 { 1002 struct ttm_bo_device *bdev = bo->bdev; 1003 struct ttm_mem_type_manager *man; 1004 uint32_t mem_type = TTM_PL_SYSTEM; 1005 uint32_t cur_flags = 0; 1006 bool type_found = false; 1007 bool type_ok = false; 1008 bool has_erestartsys = false; 1009 int i, ret; 1010 1011 mem->mm_node = NULL; 1012 for (i = 0; i < placement->num_placement; ++i) { 1013 ret = ttm_mem_type_from_flags(placement->placement[i], 1014 &mem_type); 1015 if (ret) 1016 return ret; 1017 man = &bdev->man[mem_type]; 1018 1019 type_ok = ttm_bo_mt_compatible(man, 1020 mem_type, 1021 placement->placement[i], 1022 &cur_flags); 1023 1024 if (!type_ok) 1025 continue; 1026 1027 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 1028 cur_flags); 1029 /* 1030 * Use the access and other non-mapping-related flag bits from 1031 * the memory placement flags to the current flags 1032 */ 1033 ttm_flag_masked(&cur_flags, placement->placement[i], 1034 ~TTM_PL_MASK_MEMTYPE); 1035 1036 if (mem_type == TTM_PL_SYSTEM) 1037 break; 1038 1039 if (man->has_type && man->use_type) { 1040 type_found = true; 1041 ret = (*man->func->get_node)(man, bo, placement, mem); 1042 if (unlikely(ret)) 1043 return ret; 1044 } 1045 if (mem->mm_node) 1046 break; 1047 } 1048 1049 if ((type_ok && (mem_type == TTM_PL_SYSTEM)) || mem->mm_node) { 1050 mem->mem_type = mem_type; 1051 mem->placement = cur_flags; 1052 return 0; 1053 } 1054 1055 if (!type_found) 1056 return -EINVAL; 1057 1058 for (i = 0; i < placement->num_busy_placement; ++i) { 1059 ret = ttm_mem_type_from_flags(placement->busy_placement[i], 1060 &mem_type); 1061 if (ret) 1062 return ret; 1063 man = &bdev->man[mem_type]; 1064 if (!man->has_type) 1065 continue; 1066 if (!ttm_bo_mt_compatible(man, 1067 mem_type, 1068 placement->busy_placement[i], 1069 &cur_flags)) 1070 continue; 1071 1072 cur_flags = ttm_bo_select_caching(man, bo->mem.placement, 1073 cur_flags); 1074 /* 1075 * Use the access and other non-mapping-related flag bits from 1076 * the memory placement flags to the current flags 1077 */ 1078 ttm_flag_masked(&cur_flags, placement->busy_placement[i], 1079 ~TTM_PL_MASK_MEMTYPE); 1080 1081 1082 if (mem_type == TTM_PL_SYSTEM) { 1083 mem->mem_type = mem_type; 1084 mem->placement = cur_flags; 1085 mem->mm_node = NULL; 1086 return 0; 1087 } 1088 1089 ret = ttm_bo_mem_force_space(bo, mem_type, placement, mem, 1090 interruptible, no_wait_gpu); 1091 if (ret == 0 && mem->mm_node) { 1092 mem->placement = cur_flags; 1093 return 0; 1094 } 1095 if (ret == -ERESTART) 1096 has_erestartsys = true; 1097 } 1098 ret = (has_erestartsys) ? -ERESTART : -ENOMEM; 1099 return ret; 1100 } 1101 1102 static 1103 int ttm_bo_move_buffer(struct ttm_buffer_object *bo, 1104 struct ttm_placement *placement, 1105 bool interruptible, 1106 bool no_wait_gpu) 1107 { 1108 int ret = 0; 1109 struct ttm_mem_reg mem; 1110 struct ttm_bo_device *bdev = bo->bdev; 1111 1112 KKASSERT(ttm_bo_is_reserved(bo)); 1113 1114 /* 1115 * FIXME: It's possible to pipeline buffer moves. 1116 * Have the driver move function wait for idle when necessary, 1117 * instead of doing it here. 1118 */ 1119 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1120 ret = ttm_bo_wait(bo, false, interruptible, no_wait_gpu); 1121 lockmgr(&bdev->fence_lock, LK_RELEASE); 1122 if (ret) 1123 return ret; 1124 mem.num_pages = bo->num_pages; 1125 mem.size = mem.num_pages << PAGE_SHIFT; 1126 mem.page_alignment = bo->mem.page_alignment; 1127 mem.bus.io_reserved_vm = false; 1128 mem.bus.io_reserved_count = 0; 1129 /* 1130 * Determine where to move the buffer. 1131 */ 1132 ret = ttm_bo_mem_space(bo, placement, &mem, 1133 interruptible, no_wait_gpu); 1134 if (ret) 1135 goto out_unlock; 1136 ret = ttm_bo_handle_move_mem(bo, &mem, false, 1137 interruptible, no_wait_gpu); 1138 out_unlock: 1139 if (ret && mem.mm_node) 1140 ttm_bo_mem_put(bo, &mem); 1141 return ret; 1142 } 1143 1144 static int ttm_bo_mem_compat(struct ttm_placement *placement, 1145 struct ttm_mem_reg *mem) 1146 { 1147 int i; 1148 1149 if (mem->mm_node && placement->lpfn != 0 && 1150 (mem->start < placement->fpfn || 1151 mem->start + mem->num_pages > placement->lpfn)) 1152 return -1; 1153 1154 for (i = 0; i < placement->num_placement; i++) { 1155 if ((placement->placement[i] & mem->placement & 1156 TTM_PL_MASK_CACHING) && 1157 (placement->placement[i] & mem->placement & 1158 TTM_PL_MASK_MEM)) 1159 return i; 1160 } 1161 return -1; 1162 } 1163 1164 int ttm_bo_validate(struct ttm_buffer_object *bo, 1165 struct ttm_placement *placement, 1166 bool interruptible, 1167 bool no_wait_gpu) 1168 { 1169 int ret; 1170 1171 KKASSERT(ttm_bo_is_reserved(bo)); 1172 /* Check that range is valid */ 1173 if (placement->lpfn || placement->fpfn) 1174 if (placement->fpfn > placement->lpfn || 1175 (placement->lpfn - placement->fpfn) < bo->num_pages) 1176 return -EINVAL; 1177 /* 1178 * Check whether we need to move buffer. 1179 */ 1180 ret = ttm_bo_mem_compat(placement, &bo->mem); 1181 if (ret < 0) { 1182 ret = ttm_bo_move_buffer(bo, placement, interruptible, 1183 no_wait_gpu); 1184 if (ret) 1185 return ret; 1186 } else { 1187 /* 1188 * Use the access and other non-mapping-related flag bits from 1189 * the compatible memory placement flags to the active flags 1190 */ 1191 ttm_flag_masked(&bo->mem.placement, placement->placement[ret], 1192 ~TTM_PL_MASK_MEMTYPE); 1193 } 1194 /* 1195 * We might need to add a TTM. 1196 */ 1197 if (bo->mem.mem_type == TTM_PL_SYSTEM && bo->ttm == NULL) { 1198 ret = ttm_bo_add_ttm(bo, true); 1199 if (ret) 1200 return ret; 1201 } 1202 return 0; 1203 } 1204 1205 int ttm_bo_check_placement(struct ttm_buffer_object *bo, 1206 struct ttm_placement *placement) 1207 { 1208 KKASSERT(!((placement->fpfn || placement->lpfn) && 1209 (bo->mem.num_pages > (placement->lpfn - placement->fpfn)))); 1210 1211 return 0; 1212 } 1213 1214 int ttm_bo_init(struct ttm_bo_device *bdev, 1215 struct ttm_buffer_object *bo, 1216 unsigned long size, 1217 enum ttm_bo_type type, 1218 struct ttm_placement *placement, 1219 uint32_t page_alignment, 1220 bool interruptible, 1221 struct vm_object *persistent_swap_storage, 1222 size_t acc_size, 1223 struct sg_table *sg, 1224 void (*destroy) (struct ttm_buffer_object *)) 1225 { 1226 int ret = 0; 1227 unsigned long num_pages; 1228 struct ttm_mem_global *mem_glob = bdev->glob->mem_glob; 1229 1230 ret = ttm_mem_global_alloc(mem_glob, acc_size, false, false); 1231 if (ret) { 1232 kprintf("[TTM] Out of kernel memory\n"); 1233 if (destroy) 1234 (*destroy)(bo); 1235 else 1236 drm_free(bo, M_TTM_BO); 1237 return -ENOMEM; 1238 } 1239 1240 num_pages = (size + PAGE_SIZE - 1) >> PAGE_SHIFT; 1241 if (num_pages == 0) { 1242 kprintf("[TTM] Illegal buffer object size\n"); 1243 if (destroy) 1244 (*destroy)(bo); 1245 else 1246 drm_free(bo, M_TTM_BO); 1247 ttm_mem_global_free(mem_glob, acc_size); 1248 return -EINVAL; 1249 } 1250 bo->destroy = destroy; 1251 1252 refcount_init(&bo->kref, 1); 1253 refcount_init(&bo->list_kref, 1); 1254 atomic_set(&bo->cpu_writers, 0); 1255 atomic_set(&bo->reserved, 1); 1256 INIT_LIST_HEAD(&bo->lru); 1257 INIT_LIST_HEAD(&bo->ddestroy); 1258 INIT_LIST_HEAD(&bo->swap); 1259 INIT_LIST_HEAD(&bo->io_reserve_lru); 1260 bo->bdev = bdev; 1261 bo->glob = bdev->glob; 1262 bo->type = type; 1263 bo->num_pages = num_pages; 1264 bo->mem.size = num_pages << PAGE_SHIFT; 1265 bo->mem.mem_type = TTM_PL_SYSTEM; 1266 bo->mem.num_pages = bo->num_pages; 1267 bo->mem.mm_node = NULL; 1268 bo->mem.page_alignment = page_alignment; 1269 bo->mem.bus.io_reserved_vm = false; 1270 bo->mem.bus.io_reserved_count = 0; 1271 bo->priv_flags = 0; 1272 bo->mem.placement = (TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED); 1273 bo->seq_valid = false; 1274 bo->persistent_swap_storage = persistent_swap_storage; 1275 bo->acc_size = acc_size; 1276 bo->sg = sg; 1277 atomic_inc(&bo->glob->bo_count); 1278 1279 ret = ttm_bo_check_placement(bo, placement); 1280 if (unlikely(ret != 0)) 1281 goto out_err; 1282 1283 /* 1284 * For ttm_bo_type_device buffers, allocate 1285 * address space from the device. 1286 */ 1287 if (bo->type == ttm_bo_type_device || 1288 bo->type == ttm_bo_type_sg) { 1289 ret = ttm_bo_setup_vm(bo); 1290 if (ret) 1291 goto out_err; 1292 } 1293 1294 ret = ttm_bo_validate(bo, placement, interruptible, false); 1295 if (ret) 1296 goto out_err; 1297 1298 ttm_bo_unreserve(bo); 1299 return 0; 1300 1301 out_err: 1302 ttm_bo_unreserve(bo); 1303 ttm_bo_unref(&bo); 1304 1305 return ret; 1306 } 1307 1308 size_t ttm_bo_acc_size(struct ttm_bo_device *bdev, 1309 unsigned long bo_size, 1310 unsigned struct_size) 1311 { 1312 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1313 size_t size = 0; 1314 1315 size += ttm_round_pot(struct_size); 1316 size += PAGE_ALIGN(npages * sizeof(void *)); 1317 size += ttm_round_pot(sizeof(struct ttm_tt)); 1318 return size; 1319 } 1320 1321 size_t ttm_bo_dma_acc_size(struct ttm_bo_device *bdev, 1322 unsigned long bo_size, 1323 unsigned struct_size) 1324 { 1325 unsigned npages = (PAGE_ALIGN(bo_size)) >> PAGE_SHIFT; 1326 size_t size = 0; 1327 1328 size += ttm_round_pot(struct_size); 1329 size += PAGE_ALIGN(npages * sizeof(void *)); 1330 size += PAGE_ALIGN(npages * sizeof(dma_addr_t)); 1331 size += ttm_round_pot(sizeof(struct ttm_dma_tt)); 1332 return size; 1333 } 1334 1335 int ttm_bo_create(struct ttm_bo_device *bdev, 1336 unsigned long size, 1337 enum ttm_bo_type type, 1338 struct ttm_placement *placement, 1339 uint32_t page_alignment, 1340 bool interruptible, 1341 struct vm_object *persistent_swap_storage, 1342 struct ttm_buffer_object **p_bo) 1343 { 1344 struct ttm_buffer_object *bo; 1345 size_t acc_size; 1346 int ret; 1347 1348 bo = kmalloc(sizeof(*bo), M_TTM_BO, M_WAITOK | M_ZERO); 1349 acc_size = ttm_bo_acc_size(bdev, size, sizeof(struct ttm_buffer_object)); 1350 ret = ttm_bo_init(bdev, bo, size, type, placement, page_alignment, 1351 interruptible, persistent_swap_storage, acc_size, 1352 NULL, NULL); 1353 if (likely(ret == 0)) 1354 *p_bo = bo; 1355 1356 return ret; 1357 } 1358 1359 static int ttm_bo_force_list_clean(struct ttm_bo_device *bdev, 1360 unsigned mem_type, bool allow_errors) 1361 { 1362 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1363 struct ttm_bo_global *glob = bdev->glob; 1364 int ret; 1365 1366 /* 1367 * Can't use standard list traversal since we're unlocking. 1368 */ 1369 1370 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1371 while (!list_empty(&man->lru)) { 1372 lockmgr(&glob->lru_lock, LK_RELEASE); 1373 ret = ttm_mem_evict_first(bdev, mem_type, false, false); 1374 if (ret) { 1375 if (allow_errors) { 1376 return ret; 1377 } else { 1378 kprintf("[TTM] Cleanup eviction failed\n"); 1379 } 1380 } 1381 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1382 } 1383 lockmgr(&glob->lru_lock, LK_RELEASE); 1384 return 0; 1385 } 1386 1387 int ttm_bo_clean_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1388 { 1389 struct ttm_mem_type_manager *man; 1390 int ret = -EINVAL; 1391 1392 if (mem_type >= TTM_NUM_MEM_TYPES) { 1393 kprintf("[TTM] Illegal memory type %d\n", mem_type); 1394 return ret; 1395 } 1396 man = &bdev->man[mem_type]; 1397 1398 if (!man->has_type) { 1399 kprintf("[TTM] Trying to take down uninitialized memory manager type %u\n", 1400 mem_type); 1401 return ret; 1402 } 1403 1404 man->use_type = false; 1405 man->has_type = false; 1406 1407 ret = 0; 1408 if (mem_type > 0) { 1409 ttm_bo_force_list_clean(bdev, mem_type, false); 1410 1411 ret = (*man->func->takedown)(man); 1412 } 1413 1414 return ret; 1415 } 1416 1417 int ttm_bo_evict_mm(struct ttm_bo_device *bdev, unsigned mem_type) 1418 { 1419 struct ttm_mem_type_manager *man = &bdev->man[mem_type]; 1420 1421 if (mem_type == 0 || mem_type >= TTM_NUM_MEM_TYPES) { 1422 kprintf("[TTM] Illegal memory manager memory type %u\n", mem_type); 1423 return -EINVAL; 1424 } 1425 1426 if (!man->has_type) { 1427 kprintf("[TTM] Memory type %u has not been initialized\n", mem_type); 1428 return 0; 1429 } 1430 1431 return ttm_bo_force_list_clean(bdev, mem_type, true); 1432 } 1433 1434 int ttm_bo_init_mm(struct ttm_bo_device *bdev, unsigned type, 1435 unsigned long p_size) 1436 { 1437 int ret = -EINVAL; 1438 struct ttm_mem_type_manager *man; 1439 1440 KKASSERT(type < TTM_NUM_MEM_TYPES); 1441 man = &bdev->man[type]; 1442 KKASSERT(!man->has_type); 1443 man->io_reserve_fastpath = true; 1444 man->use_io_reserve_lru = false; 1445 lockinit(&man->io_reserve_mutex, "ttmman", 0, LK_CANRECURSE); 1446 INIT_LIST_HEAD(&man->io_reserve_lru); 1447 1448 ret = bdev->driver->init_mem_type(bdev, type, man); 1449 if (ret) 1450 return ret; 1451 man->bdev = bdev; 1452 1453 ret = 0; 1454 if (type != TTM_PL_SYSTEM) { 1455 ret = (*man->func->init)(man, p_size); 1456 if (ret) 1457 return ret; 1458 } 1459 man->has_type = true; 1460 man->use_type = true; 1461 man->size = p_size; 1462 1463 INIT_LIST_HEAD(&man->lru); 1464 1465 return 0; 1466 } 1467 1468 static void ttm_bo_global_kobj_release(struct ttm_bo_global *glob) 1469 { 1470 1471 ttm_mem_unregister_shrink(glob->mem_glob, &glob->shrink); 1472 vm_page_free(glob->dummy_read_page); 1473 } 1474 1475 void ttm_bo_global_release(struct drm_global_reference *ref) 1476 { 1477 struct ttm_bo_global *glob = ref->object; 1478 1479 if (refcount_release(&glob->kobj_ref)) 1480 ttm_bo_global_kobj_release(glob); 1481 } 1482 1483 int ttm_bo_global_init(struct drm_global_reference *ref) 1484 { 1485 struct ttm_bo_global_ref *bo_ref = 1486 container_of(ref, struct ttm_bo_global_ref, ref); 1487 struct ttm_bo_global *glob = ref->object; 1488 int ret; 1489 1490 lockinit(&glob->device_list_mutex, "ttmdlm", 0, LK_CANRECURSE); 1491 lockinit(&glob->lru_lock, "ttmlru", 0, LK_CANRECURSE); 1492 glob->mem_glob = bo_ref->mem_glob; 1493 glob->dummy_read_page = vm_page_alloc_contig( 1494 0, VM_MAX_ADDRESS, PAGE_SIZE, 0, 1*PAGE_SIZE, VM_MEMATTR_UNCACHEABLE); 1495 1496 if (unlikely(glob->dummy_read_page == NULL)) { 1497 ret = -ENOMEM; 1498 goto out_no_drp; 1499 } 1500 1501 INIT_LIST_HEAD(&glob->swap_lru); 1502 INIT_LIST_HEAD(&glob->device_list); 1503 1504 ttm_mem_init_shrink(&glob->shrink, ttm_bo_swapout); 1505 ret = ttm_mem_register_shrink(glob->mem_glob, &glob->shrink); 1506 if (unlikely(ret != 0)) { 1507 kprintf("[TTM] Could not register buffer object swapout\n"); 1508 goto out_no_shrink; 1509 } 1510 1511 atomic_set(&glob->bo_count, 0); 1512 1513 refcount_init(&glob->kobj_ref, 1); 1514 return (0); 1515 1516 out_no_shrink: 1517 vm_page_free(glob->dummy_read_page); 1518 out_no_drp: 1519 drm_free(glob, M_DRM_GLOBAL); 1520 return ret; 1521 } 1522 1523 int ttm_bo_device_release(struct ttm_bo_device *bdev) 1524 { 1525 int ret = 0; 1526 unsigned i = TTM_NUM_MEM_TYPES; 1527 struct ttm_mem_type_manager *man; 1528 struct ttm_bo_global *glob = bdev->glob; 1529 1530 while (i--) { 1531 man = &bdev->man[i]; 1532 if (man->has_type) { 1533 man->use_type = false; 1534 if ((i != TTM_PL_SYSTEM) && ttm_bo_clean_mm(bdev, i)) { 1535 ret = -EBUSY; 1536 kprintf("[TTM] DRM memory manager type %d is not clean\n", 1537 i); 1538 } 1539 man->has_type = false; 1540 } 1541 } 1542 1543 lockmgr(&glob->device_list_mutex, LK_EXCLUSIVE); 1544 list_del(&bdev->device_list); 1545 lockmgr(&glob->device_list_mutex, LK_RELEASE); 1546 1547 if (taskqueue_cancel_timeout(taskqueue_thread[mycpuid], &bdev->wq, NULL)) 1548 taskqueue_drain_timeout(taskqueue_thread[mycpuid], &bdev->wq); 1549 1550 while (ttm_bo_delayed_delete(bdev, true)) 1551 ; 1552 1553 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1554 if (list_empty(&bdev->ddestroy)) 1555 TTM_DEBUG("Delayed destroy list was clean\n"); 1556 1557 if (list_empty(&bdev->man[0].lru)) 1558 TTM_DEBUG("Swap list was clean\n"); 1559 lockmgr(&glob->lru_lock, LK_RELEASE); 1560 1561 KKASSERT(drm_mm_clean(&bdev->addr_space_mm)); 1562 lockmgr(&bdev->vm_lock, LK_EXCLUSIVE); 1563 drm_mm_takedown(&bdev->addr_space_mm); 1564 lockmgr(&bdev->vm_lock, LK_RELEASE); 1565 1566 return ret; 1567 } 1568 1569 int ttm_bo_device_init(struct ttm_bo_device *bdev, 1570 struct ttm_bo_global *glob, 1571 struct ttm_bo_driver *driver, 1572 uint64_t file_page_offset, 1573 bool need_dma32) 1574 { 1575 int ret = -EINVAL; 1576 1577 lockinit(&bdev->vm_lock, "ttmvml", 0, LK_CANRECURSE); 1578 bdev->driver = driver; 1579 1580 memset(bdev->man, 0, sizeof(bdev->man)); 1581 1582 /* 1583 * Initialize the system memory buffer type. 1584 * Other types need to be driver / IOCTL initialized. 1585 */ 1586 ret = ttm_bo_init_mm(bdev, TTM_PL_SYSTEM, 0); 1587 if (unlikely(ret != 0)) 1588 goto out_no_sys; 1589 1590 RB_INIT(&bdev->addr_space_rb); 1591 ret = drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000); 1592 if (unlikely(ret != 0)) 1593 goto out_no_addr_mm; 1594 1595 TIMEOUT_TASK_INIT(taskqueue_thread[mycpuid], &bdev->wq, 0, 1596 ttm_bo_delayed_workqueue, bdev); 1597 INIT_LIST_HEAD(&bdev->ddestroy); 1598 bdev->dev_mapping = NULL; 1599 bdev->glob = glob; 1600 bdev->need_dma32 = need_dma32; 1601 bdev->val_seq = 0; 1602 lockinit(&bdev->fence_lock, "ttmfence", 0, LK_CANRECURSE); 1603 lockmgr(&glob->device_list_mutex, LK_EXCLUSIVE); 1604 list_add_tail(&bdev->device_list, &glob->device_list); 1605 lockmgr(&glob->device_list_mutex, LK_RELEASE); 1606 1607 return 0; 1608 out_no_addr_mm: 1609 ttm_bo_clean_mm(bdev, 0); 1610 out_no_sys: 1611 return ret; 1612 } 1613 1614 /* 1615 * buffer object vm functions. 1616 */ 1617 1618 bool ttm_mem_reg_is_pci(struct ttm_bo_device *bdev, struct ttm_mem_reg *mem) 1619 { 1620 struct ttm_mem_type_manager *man = &bdev->man[mem->mem_type]; 1621 1622 if (!(man->flags & TTM_MEMTYPE_FLAG_FIXED)) { 1623 if (mem->mem_type == TTM_PL_SYSTEM) 1624 return false; 1625 1626 if (man->flags & TTM_MEMTYPE_FLAG_CMA) 1627 return false; 1628 1629 if (mem->placement & TTM_PL_FLAG_CACHED) 1630 return false; 1631 } 1632 return true; 1633 } 1634 1635 void ttm_bo_unmap_virtual_locked(struct ttm_buffer_object *bo) 1636 { 1637 1638 ttm_bo_release_mmap(bo); 1639 ttm_mem_io_free_vm(bo); 1640 } 1641 1642 void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) 1643 { 1644 struct ttm_bo_device *bdev = bo->bdev; 1645 struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; 1646 1647 ttm_mem_io_lock(man, false); 1648 ttm_bo_unmap_virtual_locked(bo); 1649 ttm_mem_io_unlock(man); 1650 } 1651 1652 static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo) 1653 { 1654 struct ttm_bo_device *bdev = bo->bdev; 1655 1656 /* The caller acquired bdev->vm_lock. */ 1657 RB_INSERT(ttm_bo_device_buffer_objects, &bdev->addr_space_rb, bo); 1658 } 1659 1660 /** 1661 * ttm_bo_setup_vm: 1662 * 1663 * @bo: the buffer to allocate address space for 1664 * 1665 * Allocate address space in the drm device so that applications 1666 * can mmap the buffer and access the contents. This only 1667 * applies to ttm_bo_type_device objects as others are not 1668 * placed in the drm device address space. 1669 */ 1670 1671 static int ttm_bo_setup_vm(struct ttm_buffer_object *bo) 1672 { 1673 struct ttm_bo_device *bdev = bo->bdev; 1674 int ret; 1675 1676 retry_pre_get: 1677 ret = drm_mm_pre_get(&bdev->addr_space_mm); 1678 if (unlikely(ret != 0)) 1679 return ret; 1680 1681 lockmgr(&bdev->vm_lock, LK_EXCLUSIVE); 1682 bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm, 1683 bo->mem.num_pages, 0, 0); 1684 1685 if (unlikely(bo->vm_node == NULL)) { 1686 ret = -ENOMEM; 1687 goto out_unlock; 1688 } 1689 1690 bo->vm_node = drm_mm_get_block_atomic(bo->vm_node, 1691 bo->mem.num_pages, 0); 1692 1693 if (unlikely(bo->vm_node == NULL)) { 1694 lockmgr(&bdev->vm_lock, LK_RELEASE); 1695 goto retry_pre_get; 1696 } 1697 1698 ttm_bo_vm_insert_rb(bo); 1699 lockmgr(&bdev->vm_lock, LK_RELEASE); 1700 bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT; 1701 1702 return 0; 1703 out_unlock: 1704 lockmgr(&bdev->vm_lock, LK_RELEASE); 1705 return ret; 1706 } 1707 1708 int ttm_bo_wait(struct ttm_buffer_object *bo, 1709 bool lazy, bool interruptible, bool no_wait) 1710 { 1711 struct ttm_bo_driver *driver = bo->bdev->driver; 1712 struct ttm_bo_device *bdev = bo->bdev; 1713 void *sync_obj; 1714 int ret = 0; 1715 1716 if (likely(bo->sync_obj == NULL)) 1717 return 0; 1718 1719 while (bo->sync_obj) { 1720 1721 if (driver->sync_obj_signaled(bo->sync_obj)) { 1722 void *tmp_obj = bo->sync_obj; 1723 bo->sync_obj = NULL; 1724 clear_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags); 1725 lockmgr(&bdev->fence_lock, LK_RELEASE); 1726 driver->sync_obj_unref(&tmp_obj); 1727 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1728 continue; 1729 } 1730 1731 if (no_wait) 1732 return -EBUSY; 1733 1734 sync_obj = driver->sync_obj_ref(bo->sync_obj); 1735 lockmgr(&bdev->fence_lock, LK_RELEASE); 1736 ret = driver->sync_obj_wait(sync_obj, 1737 lazy, interruptible); 1738 if (unlikely(ret != 0)) { 1739 driver->sync_obj_unref(&sync_obj); 1740 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1741 return ret; 1742 } 1743 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1744 if (likely(bo->sync_obj == sync_obj)) { 1745 void *tmp_obj = bo->sync_obj; 1746 bo->sync_obj = NULL; 1747 clear_bit(TTM_BO_PRIV_FLAG_MOVING, 1748 &bo->priv_flags); 1749 lockmgr(&bdev->fence_lock, LK_RELEASE); 1750 driver->sync_obj_unref(&sync_obj); 1751 driver->sync_obj_unref(&tmp_obj); 1752 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1753 } else { 1754 lockmgr(&bdev->fence_lock, LK_RELEASE); 1755 driver->sync_obj_unref(&sync_obj); 1756 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1757 } 1758 } 1759 return 0; 1760 } 1761 1762 int ttm_bo_synccpu_write_grab(struct ttm_buffer_object *bo, bool no_wait) 1763 { 1764 struct ttm_bo_device *bdev = bo->bdev; 1765 int ret = 0; 1766 1767 /* 1768 * Using ttm_bo_reserve makes sure the lru lists are updated. 1769 */ 1770 1771 ret = ttm_bo_reserve(bo, true, no_wait, false, 0); 1772 if (unlikely(ret != 0)) 1773 return ret; 1774 lockmgr(&bdev->fence_lock, LK_EXCLUSIVE); 1775 ret = ttm_bo_wait(bo, false, true, no_wait); 1776 lockmgr(&bdev->fence_lock, LK_RELEASE); 1777 if (likely(ret == 0)) 1778 atomic_inc(&bo->cpu_writers); 1779 ttm_bo_unreserve(bo); 1780 return ret; 1781 } 1782 1783 void ttm_bo_synccpu_write_release(struct ttm_buffer_object *bo) 1784 { 1785 atomic_dec(&bo->cpu_writers); 1786 } 1787 1788 /** 1789 * A buffer object shrink method that tries to swap out the first 1790 * buffer object on the bo_global::swap_lru list. 1791 */ 1792 1793 static int ttm_bo_swapout(struct ttm_mem_shrink *shrink) 1794 { 1795 struct ttm_bo_global *glob = 1796 container_of(shrink, struct ttm_bo_global, shrink); 1797 struct ttm_buffer_object *bo; 1798 int ret = -EBUSY; 1799 int put_count; 1800 uint32_t swap_placement = (TTM_PL_FLAG_CACHED | TTM_PL_FLAG_SYSTEM); 1801 1802 lockmgr(&glob->lru_lock, LK_EXCLUSIVE); 1803 list_for_each_entry(bo, &glob->swap_lru, swap) { 1804 ret = ttm_bo_reserve_nolru(bo, false, true, false, 0); 1805 if (!ret) 1806 break; 1807 } 1808 1809 if (ret) { 1810 lockmgr(&glob->lru_lock, LK_RELEASE); 1811 return ret; 1812 } 1813 1814 refcount_acquire(&bo->list_kref); 1815 1816 if (!list_empty(&bo->ddestroy)) { 1817 ret = ttm_bo_cleanup_refs_and_unlock(bo, false, false); 1818 if (refcount_release(&bo->list_kref)) 1819 ttm_bo_release_list(bo); 1820 return ret; 1821 } 1822 1823 put_count = ttm_bo_del_from_lru(bo); 1824 lockmgr(&glob->lru_lock, LK_RELEASE); 1825 1826 ttm_bo_list_ref_sub(bo, put_count, true); 1827 1828 /** 1829 * Wait for GPU, then move to system cached. 1830 */ 1831 1832 lockmgr(&bo->bdev->fence_lock, LK_EXCLUSIVE); 1833 ret = ttm_bo_wait(bo, false, false, false); 1834 lockmgr(&bo->bdev->fence_lock, LK_RELEASE); 1835 1836 if (unlikely(ret != 0)) 1837 goto out; 1838 1839 if ((bo->mem.placement & swap_placement) != swap_placement) { 1840 struct ttm_mem_reg evict_mem; 1841 1842 evict_mem = bo->mem; 1843 evict_mem.mm_node = NULL; 1844 evict_mem.placement = TTM_PL_FLAG_SYSTEM | TTM_PL_FLAG_CACHED; 1845 evict_mem.mem_type = TTM_PL_SYSTEM; 1846 1847 ret = ttm_bo_handle_move_mem(bo, &evict_mem, true, 1848 false, false); 1849 if (unlikely(ret != 0)) 1850 goto out; 1851 } 1852 1853 ttm_bo_unmap_virtual(bo); 1854 1855 /** 1856 * Swap out. Buffer will be swapped in again as soon as 1857 * anyone tries to access a ttm page. 1858 */ 1859 1860 if (bo->bdev->driver->swap_notify) 1861 bo->bdev->driver->swap_notify(bo); 1862 1863 ret = ttm_tt_swapout(bo->ttm, bo->persistent_swap_storage); 1864 out: 1865 1866 /** 1867 * 1868 * Unreserve without putting on LRU to avoid swapping out an 1869 * already swapped buffer. 1870 */ 1871 1872 atomic_set(&bo->reserved, 0); 1873 wakeup(bo); 1874 if (refcount_release(&bo->list_kref)) 1875 ttm_bo_release_list(bo); 1876 return ret; 1877 } 1878 1879 void ttm_bo_swapout_all(struct ttm_bo_device *bdev) 1880 { 1881 while (ttm_bo_swapout(&bdev->glob->shrink) == 0) 1882 ; 1883 } 1884