1 /*- 2 * Copyright © 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 * Copyright (c) 2011 The FreeBSD Foundation 27 * All rights reserved. 28 * 29 * This software was developed by Konstantin Belousov under sponsorship from 30 * the FreeBSD Foundation. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 51 * SUCH DAMAGE. 52 * 53 * $FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 253497 2013-07-20 13:52:40Z kib $ 54 */ 55 56 #include <sys/resourcevar.h> 57 #include <sys/sfbuf.h> 58 59 #include <dev/drm/drmP.h> 60 #include <dev/drm/drm.h> 61 #include "i915_drm.h" 62 #include "i915_drv.h" 63 #include "intel_drv.h" 64 #include "intel_ringbuffer.h" 65 66 static void i915_gem_object_flush_cpu_write_domain( 67 struct drm_i915_gem_object *obj); 68 static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, 69 int tiling_mode); 70 static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev, 71 uint32_t size, int tiling_mode); 72 static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 73 unsigned alignment, bool map_and_fenceable); 74 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 75 int flags); 76 static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj); 77 static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, 78 bool write); 79 static void i915_gem_object_set_to_full_cpu_read_domain( 80 struct drm_i915_gem_object *obj); 81 static int i915_gem_object_set_cpu_read_domain_range( 82 struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size); 83 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj); 84 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 85 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj); 86 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj); 87 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj); 88 static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex); 89 static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 90 uint32_t flush_domains); 91 static void i915_gem_clear_fence_reg(struct drm_device *dev, 92 struct drm_i915_fence_reg *reg); 93 static void i915_gem_reset_fences(struct drm_device *dev); 94 static void i915_gem_retire_task_handler(void *arg, int pending); 95 static int i915_gem_phys_pwrite(struct drm_device *dev, 96 struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset, 97 uint64_t size, struct drm_file *file_priv); 98 static void i915_gem_lowmem(void *arg); 99 100 MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); 101 long i915_gem_wired_pages_cnt; 102 103 static void 104 i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size) 105 { 106 107 dev_priv->mm.object_count++; 108 dev_priv->mm.object_memory += size; 109 } 110 111 static void 112 i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, size_t size) 113 { 114 115 dev_priv->mm.object_count--; 116 dev_priv->mm.object_memory -= size; 117 } 118 119 static int 120 i915_gem_wait_for_error(struct drm_device *dev) 121 { 122 struct drm_i915_private *dev_priv; 123 int ret; 124 125 dev_priv = dev->dev_private; 126 if (!atomic_load_acq_int(&dev_priv->mm.wedged)) 127 return (0); 128 129 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE); 130 while (dev_priv->error_completion == 0) { 131 ret = -lksleep(&dev_priv->error_completion, 132 &dev_priv->error_completion_lock, PCATCH, "915wco", 0); 133 if (ret != 0) { 134 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 135 return (ret); 136 } 137 } 138 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 139 140 if (atomic_load_acq_int(&dev_priv->mm.wedged)) { 141 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE); 142 dev_priv->error_completion++; 143 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 144 } 145 return (0); 146 } 147 148 int 149 i915_mutex_lock_interruptible(struct drm_device *dev) 150 { 151 struct drm_i915_private *dev_priv; 152 int ret; 153 154 dev_priv = dev->dev_private; 155 ret = i915_gem_wait_for_error(dev); 156 if (ret != 0) 157 return (ret); 158 159 /* 160 * interruptible shall it be. might indeed be if dev_lock is 161 * changed to sx 162 */ 163 ret = lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_SLEEPFAIL); 164 if (ret != 0) 165 return (-ret); 166 167 return (0); 168 } 169 170 171 static void 172 i915_gem_free_object_tail(struct drm_i915_gem_object *obj) 173 { 174 struct drm_device *dev; 175 drm_i915_private_t *dev_priv; 176 int ret; 177 178 dev = obj->base.dev; 179 dev_priv = dev->dev_private; 180 181 ret = i915_gem_object_unbind(obj); 182 if (ret == -ERESTART) { 183 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list); 184 return; 185 } 186 187 drm_gem_free_mmap_offset(&obj->base); 188 drm_gem_object_release(&obj->base); 189 i915_gem_info_remove_obj(dev_priv, obj->base.size); 190 191 drm_free(obj->page_cpu_valid, DRM_I915_GEM); 192 drm_free(obj->bit_17, DRM_I915_GEM); 193 drm_free(obj, DRM_I915_GEM); 194 } 195 196 void 197 i915_gem_free_object(struct drm_gem_object *gem_obj) 198 { 199 struct drm_i915_gem_object *obj; 200 struct drm_device *dev; 201 202 obj = to_intel_bo(gem_obj); 203 dev = obj->base.dev; 204 205 while (obj->pin_count > 0) 206 i915_gem_object_unpin(obj); 207 208 if (obj->phys_obj != NULL) 209 i915_gem_detach_phys_object(dev, obj); 210 211 i915_gem_free_object_tail(obj); 212 } 213 214 static void 215 init_ring_lists(struct intel_ring_buffer *ring) 216 { 217 218 INIT_LIST_HEAD(&ring->active_list); 219 INIT_LIST_HEAD(&ring->request_list); 220 INIT_LIST_HEAD(&ring->gpu_write_list); 221 } 222 223 void 224 i915_gem_load(struct drm_device *dev) 225 { 226 drm_i915_private_t *dev_priv; 227 int i; 228 229 dev_priv = dev->dev_private; 230 231 INIT_LIST_HEAD(&dev_priv->mm.active_list); 232 INIT_LIST_HEAD(&dev_priv->mm.flushing_list); 233 INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 234 INIT_LIST_HEAD(&dev_priv->mm.pinned_list); 235 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 236 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); 237 INIT_LIST_HEAD(&dev_priv->mm.gtt_list); 238 for (i = 0; i < I915_NUM_RINGS; i++) 239 init_ring_lists(&dev_priv->rings[i]); 240 for (i = 0; i < I915_MAX_NUM_FENCES; i++) 241 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 242 TIMEOUT_TASK_INIT(dev_priv->tq, &dev_priv->mm.retire_task, 0, 243 i915_gem_retire_task_handler, dev_priv); 244 dev_priv->error_completion = 0; 245 246 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 247 if (IS_GEN3(dev)) { 248 u32 tmp = I915_READ(MI_ARB_STATE); 249 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) { 250 /* 251 * arb state is a masked write, so set bit + 252 * bit in mask. 253 */ 254 tmp = MI_ARB_C3_LP_WRITE_ENABLE | 255 (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT); 256 I915_WRITE(MI_ARB_STATE, tmp); 257 } 258 } 259 260 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 261 262 /* Old X drivers will take 0-2 for front, back, depth buffers */ 263 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 264 dev_priv->fence_reg_start = 3; 265 266 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || 267 IS_G33(dev)) 268 dev_priv->num_fence_regs = 16; 269 else 270 dev_priv->num_fence_regs = 8; 271 272 /* Initialize fence registers to zero */ 273 for (i = 0; i < dev_priv->num_fence_regs; i++) { 274 i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]); 275 } 276 i915_gem_detect_bit_6_swizzle(dev); 277 dev_priv->mm.interruptible = true; 278 279 dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, 280 i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY); 281 } 282 283 int 284 i915_gem_do_init(struct drm_device *dev, unsigned long start, 285 unsigned long mappable_end, unsigned long end) 286 { 287 drm_i915_private_t *dev_priv; 288 unsigned long mappable; 289 int error; 290 291 dev_priv = dev->dev_private; 292 mappable = min(end, mappable_end) - start; 293 294 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start); 295 296 dev_priv->mm.gtt_start = start; 297 dev_priv->mm.gtt_mappable_end = mappable_end; 298 dev_priv->mm.gtt_end = end; 299 dev_priv->mm.gtt_total = end - start; 300 dev_priv->mm.mappable_gtt_total = mappable; 301 302 /* Take over this portion of the GTT */ 303 intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); 304 device_printf(dev->device, 305 "taking over the fictitious range 0x%lx-0x%lx\n", 306 dev->agp->base + start, dev->agp->base + start + mappable); 307 error = -vm_phys_fictitious_reg_range(dev->agp->base + start, 308 dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING); 309 return (error); 310 } 311 312 int 313 i915_gem_init_ioctl(struct drm_device *dev, void *data, 314 struct drm_file *file) 315 { 316 struct drm_i915_gem_init *args; 317 drm_i915_private_t *dev_priv; 318 319 dev_priv = dev->dev_private; 320 args = data; 321 322 if (args->gtt_start >= args->gtt_end || 323 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) 324 return (-EINVAL); 325 326 /* 327 * XXXKIB. The second-time initialization should be guarded 328 * against. 329 */ 330 lockmgr(&dev->dev_lock, LK_EXCLUSIVE|LK_RETRY|LK_CANRECURSE); 331 i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end); 332 lockmgr(&dev->dev_lock, LK_RELEASE); 333 334 return 0; 335 } 336 337 int 338 i915_gem_idle(struct drm_device *dev) 339 { 340 drm_i915_private_t *dev_priv; 341 int ret; 342 343 dev_priv = dev->dev_private; 344 if (dev_priv->mm.suspended) 345 return (0); 346 347 ret = i915_gpu_idle(dev, true); 348 if (ret != 0) 349 return (ret); 350 351 /* Under UMS, be paranoid and evict. */ 352 if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 353 ret = i915_gem_evict_inactive(dev, false); 354 if (ret != 0) 355 return ret; 356 } 357 358 i915_gem_reset_fences(dev); 359 360 /* Hack! Don't let anybody do execbuf while we don't control the chip. 361 * We need to replace this with a semaphore, or something. 362 * And not confound mm.suspended! 363 */ 364 dev_priv->mm.suspended = 1; 365 callout_stop(&dev_priv->hangcheck_timer); 366 367 i915_kernel_lost_context(dev); 368 i915_gem_cleanup_ringbuffer(dev); 369 370 /* Cancel the retire work handler, which should be idle now. */ 371 taskqueue_cancel_timeout(dev_priv->tq, &dev_priv->mm.retire_task, NULL); 372 return (ret); 373 } 374 375 void 376 i915_gem_init_swizzling(struct drm_device *dev) 377 { 378 drm_i915_private_t *dev_priv; 379 380 dev_priv = dev->dev_private; 381 382 if (INTEL_INFO(dev)->gen < 5 || 383 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 384 return; 385 386 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 387 DISP_TILE_SURFACE_SWIZZLING); 388 389 if (IS_GEN5(dev)) 390 return; 391 392 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 393 if (IS_GEN6(dev)) 394 I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_SNB)); 395 else 396 I915_WRITE(ARB_MODE, ARB_MODE_ENABLE(ARB_MODE_SWIZZLE_IVB)); 397 } 398 399 void 400 i915_gem_init_ppgtt(struct drm_device *dev) 401 { 402 drm_i915_private_t *dev_priv; 403 struct i915_hw_ppgtt *ppgtt; 404 uint32_t pd_offset, pd_entry; 405 vm_paddr_t pt_addr; 406 struct intel_ring_buffer *ring; 407 u_int first_pd_entry_in_global_pt, i; 408 409 dev_priv = dev->dev_private; 410 ppgtt = dev_priv->mm.aliasing_ppgtt; 411 if (ppgtt == NULL) 412 return; 413 414 first_pd_entry_in_global_pt = 512 * 1024 - I915_PPGTT_PD_ENTRIES; 415 for (i = 0; i < ppgtt->num_pd_entries; i++) { 416 pt_addr = VM_PAGE_TO_PHYS(ppgtt->pt_pages[i]); 417 pd_entry = GEN6_PDE_ADDR_ENCODE(pt_addr); 418 pd_entry |= GEN6_PDE_VALID; 419 intel_gtt_write(first_pd_entry_in_global_pt + i, pd_entry); 420 } 421 intel_gtt_read_pte(first_pd_entry_in_global_pt); 422 423 pd_offset = ppgtt->pd_offset; 424 pd_offset /= 64; /* in cachelines, */ 425 pd_offset <<= 16; 426 427 if (INTEL_INFO(dev)->gen == 6) { 428 uint32_t ecochk = I915_READ(GAM_ECOCHK); 429 I915_WRITE(GAM_ECOCHK, ecochk | ECOCHK_SNB_BIT | 430 ECOCHK_PPGTT_CACHE64B); 431 I915_WRITE(GFX_MODE, GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); 432 } else if (INTEL_INFO(dev)->gen >= 7) { 433 I915_WRITE(GAM_ECOCHK, ECOCHK_PPGTT_CACHE64B); 434 /* GFX_MODE is per-ring on gen7+ */ 435 } 436 437 for (i = 0; i < I915_NUM_RINGS; i++) { 438 ring = &dev_priv->rings[i]; 439 440 if (INTEL_INFO(dev)->gen >= 7) 441 I915_WRITE(RING_MODE_GEN7(ring), 442 GFX_MODE_ENABLE(GFX_PPGTT_ENABLE)); 443 444 I915_WRITE(RING_PP_DIR_DCLV(ring), PP_DIR_DCLV_2G); 445 I915_WRITE(RING_PP_DIR_BASE(ring), pd_offset); 446 } 447 } 448 449 int 450 i915_gem_init_hw(struct drm_device *dev) 451 { 452 drm_i915_private_t *dev_priv; 453 int ret; 454 455 dev_priv = dev->dev_private; 456 457 i915_gem_init_swizzling(dev); 458 459 ret = intel_init_render_ring_buffer(dev); 460 if (ret != 0) 461 return (ret); 462 463 if (HAS_BSD(dev)) { 464 ret = intel_init_bsd_ring_buffer(dev); 465 if (ret != 0) 466 goto cleanup_render_ring; 467 } 468 469 if (HAS_BLT(dev)) { 470 ret = intel_init_blt_ring_buffer(dev); 471 if (ret != 0) 472 goto cleanup_bsd_ring; 473 } 474 475 dev_priv->next_seqno = 1; 476 i915_gem_init_ppgtt(dev); 477 return (0); 478 479 cleanup_bsd_ring: 480 intel_cleanup_ring_buffer(&dev_priv->rings[VCS]); 481 cleanup_render_ring: 482 intel_cleanup_ring_buffer(&dev_priv->rings[RCS]); 483 return (ret); 484 } 485 486 int 487 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 488 struct drm_file *file) 489 { 490 struct drm_i915_private *dev_priv; 491 struct drm_i915_gem_get_aperture *args; 492 struct drm_i915_gem_object *obj; 493 size_t pinned; 494 495 dev_priv = dev->dev_private; 496 args = data; 497 498 if (!(dev->driver->driver_features & DRIVER_GEM)) 499 return (-ENODEV); 500 501 pinned = 0; 502 DRM_LOCK(dev); 503 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list) 504 pinned += obj->gtt_space->size; 505 DRM_UNLOCK(dev); 506 507 args->aper_size = dev_priv->mm.gtt_total; 508 args->aper_available_size = args->aper_size - pinned; 509 510 return (0); 511 } 512 513 int 514 i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, 515 bool map_and_fenceable) 516 { 517 struct drm_device *dev; 518 struct drm_i915_private *dev_priv; 519 int ret; 520 521 dev = obj->base.dev; 522 dev_priv = dev->dev_private; 523 524 KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT, 525 ("Max pin count")); 526 527 if (obj->gtt_space != NULL) { 528 if ((alignment && obj->gtt_offset & (alignment - 1)) || 529 (map_and_fenceable && !obj->map_and_fenceable)) { 530 DRM_DEBUG("bo is already pinned with incorrect alignment:" 531 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," 532 " obj->map_and_fenceable=%d\n", 533 obj->gtt_offset, alignment, 534 map_and_fenceable, 535 obj->map_and_fenceable); 536 ret = i915_gem_object_unbind(obj); 537 if (ret != 0) 538 return (ret); 539 } 540 } 541 542 if (obj->gtt_space == NULL) { 543 ret = i915_gem_object_bind_to_gtt(obj, alignment, 544 map_and_fenceable); 545 if (ret) 546 return (ret); 547 } 548 549 if (obj->pin_count++ == 0 && !obj->active) 550 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 551 obj->pin_mappable |= map_and_fenceable; 552 553 #if 1 554 KIB_NOTYET(); 555 #else 556 WARN_ON(i915_verify_lists(dev)); 557 #endif 558 return (0); 559 } 560 561 void 562 i915_gem_object_unpin(struct drm_i915_gem_object *obj) 563 { 564 struct drm_device *dev; 565 drm_i915_private_t *dev_priv; 566 567 dev = obj->base.dev; 568 dev_priv = dev->dev_private; 569 570 #if 1 571 KIB_NOTYET(); 572 #else 573 WARN_ON(i915_verify_lists(dev)); 574 #endif 575 576 KASSERT(obj->pin_count != 0, ("zero pin count")); 577 KASSERT(obj->gtt_space != NULL, ("No gtt mapping")); 578 579 if (--obj->pin_count == 0) { 580 if (!obj->active) 581 list_move_tail(&obj->mm_list, 582 &dev_priv->mm.inactive_list); 583 obj->pin_mappable = false; 584 } 585 #if 1 586 KIB_NOTYET(); 587 #else 588 WARN_ON(i915_verify_lists(dev)); 589 #endif 590 } 591 592 int 593 i915_gem_pin_ioctl(struct drm_device *dev, void *data, 594 struct drm_file *file) 595 { 596 struct drm_i915_gem_pin *args; 597 struct drm_i915_gem_object *obj; 598 struct drm_gem_object *gobj; 599 int ret; 600 601 args = data; 602 603 ret = i915_mutex_lock_interruptible(dev); 604 if (ret != 0) 605 return ret; 606 607 gobj = drm_gem_object_lookup(dev, file, args->handle); 608 if (gobj == NULL) { 609 ret = -ENOENT; 610 goto unlock; 611 } 612 obj = to_intel_bo(gobj); 613 614 if (obj->madv != I915_MADV_WILLNEED) { 615 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 616 ret = -EINVAL; 617 goto out; 618 } 619 620 if (obj->pin_filp != NULL && obj->pin_filp != file) { 621 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 622 args->handle); 623 ret = -EINVAL; 624 goto out; 625 } 626 627 obj->user_pin_count++; 628 obj->pin_filp = file; 629 if (obj->user_pin_count == 1) { 630 ret = i915_gem_object_pin(obj, args->alignment, true); 631 if (ret != 0) 632 goto out; 633 } 634 635 /* XXX - flush the CPU caches for pinned objects 636 * as the X server doesn't manage domains yet 637 */ 638 i915_gem_object_flush_cpu_write_domain(obj); 639 args->offset = obj->gtt_offset; 640 out: 641 drm_gem_object_unreference(&obj->base); 642 unlock: 643 DRM_UNLOCK(dev); 644 return (ret); 645 } 646 647 int 648 i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 649 struct drm_file *file) 650 { 651 struct drm_i915_gem_pin *args; 652 struct drm_i915_gem_object *obj; 653 int ret; 654 655 args = data; 656 ret = i915_mutex_lock_interruptible(dev); 657 if (ret != 0) 658 return (ret); 659 660 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 661 if (&obj->base == NULL) { 662 ret = -ENOENT; 663 goto unlock; 664 } 665 666 if (obj->pin_filp != file) { 667 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 668 args->handle); 669 ret = -EINVAL; 670 goto out; 671 } 672 obj->user_pin_count--; 673 if (obj->user_pin_count == 0) { 674 obj->pin_filp = NULL; 675 i915_gem_object_unpin(obj); 676 } 677 678 out: 679 drm_gem_object_unreference(&obj->base); 680 unlock: 681 DRM_UNLOCK(dev); 682 return (ret); 683 } 684 685 int 686 i915_gem_busy_ioctl(struct drm_device *dev, void *data, 687 struct drm_file *file) 688 { 689 struct drm_i915_gem_busy *args; 690 struct drm_i915_gem_object *obj; 691 struct drm_i915_gem_request *request; 692 int ret; 693 694 args = data; 695 696 ret = i915_mutex_lock_interruptible(dev); 697 if (ret != 0) 698 return ret; 699 700 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 701 if (&obj->base == NULL) { 702 ret = -ENOENT; 703 goto unlock; 704 } 705 706 args->busy = obj->active; 707 if (args->busy) { 708 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 709 ret = i915_gem_flush_ring(obj->ring, 710 0, obj->base.write_domain); 711 } else if (obj->ring->outstanding_lazy_request == 712 obj->last_rendering_seqno) { 713 request = kmalloc(sizeof(*request), DRM_I915_GEM, 714 M_WAITOK | M_ZERO); 715 ret = i915_add_request(obj->ring, NULL, request); 716 if (ret != 0) 717 drm_free(request, DRM_I915_GEM); 718 } 719 720 i915_gem_retire_requests_ring(obj->ring); 721 args->busy = obj->active; 722 } 723 724 drm_gem_object_unreference(&obj->base); 725 unlock: 726 DRM_UNLOCK(dev); 727 return (ret); 728 } 729 730 /* Throttle our rendering by waiting until the ring has completed our requests 731 * emitted over 20 msec ago. 732 * 733 * Note that if we were to use the current jiffies each time around the loop, 734 * we wouldn't escape the function with any frames outstanding if the time to 735 * render a frame was over 20ms. 736 * 737 * This should get us reasonable parallelism between CPU and GPU but also 738 * relatively low latency when blocking on a particular request to finish. 739 */ 740 static int 741 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 742 { 743 struct drm_i915_private *dev_priv = dev->dev_private; 744 struct drm_i915_file_private *file_priv = file->driver_priv; 745 unsigned long recent_enough = ticks - (20 * hz / 1000); 746 struct drm_i915_gem_request *request; 747 struct intel_ring_buffer *ring = NULL; 748 u32 seqno = 0; 749 int ret; 750 751 dev_priv = dev->dev_private; 752 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 753 return (-EIO); 754 755 recent_enough = ticks - (20 * hz / 1000); 756 ring = NULL; 757 seqno = 0; 758 759 spin_lock(&file_priv->mm.lock); 760 list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 761 if (time_after_eq(request->emitted_jiffies, recent_enough)) 762 break; 763 764 ring = request->ring; 765 seqno = request->seqno; 766 } 767 spin_unlock(&file_priv->mm.lock); 768 769 if (seqno == 0) 770 return 0; 771 772 ret = 0; 773 lockmgr(&ring->irq_lock, LK_EXCLUSIVE); 774 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { 775 if (ring->irq_get(ring)) { 776 while (ret == 0 && 777 !(i915_seqno_passed(ring->get_seqno(ring), seqno) || 778 atomic_load_acq_int(&dev_priv->mm.wedged))) 779 ret = -lksleep(ring, &ring->irq_lock, PCATCH, 780 "915thr", 0); 781 ring->irq_put(ring); 782 if (ret == 0 && atomic_load_acq_int(&dev_priv->mm.wedged)) 783 ret = -EIO; 784 } else if (_intel_wait_for(dev, 785 i915_seqno_passed(ring->get_seqno(ring), seqno) || 786 atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 0, "915rtr")) { 787 ret = -EBUSY; 788 } 789 } 790 lockmgr(&ring->irq_lock, LK_RELEASE); 791 792 if (ret == 0) 793 taskqueue_enqueue_timeout(dev_priv->tq, 794 &dev_priv->mm.retire_task, 0); 795 796 return ret; 797 } 798 799 int 800 i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 801 struct drm_file *file_priv) 802 { 803 804 return (i915_gem_ring_throttle(dev, file_priv)); 805 } 806 807 int 808 i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 809 struct drm_file *file_priv) 810 { 811 struct drm_i915_gem_madvise *args; 812 struct drm_i915_gem_object *obj; 813 int ret; 814 815 args = data; 816 switch (args->madv) { 817 case I915_MADV_DONTNEED: 818 case I915_MADV_WILLNEED: 819 break; 820 default: 821 return (-EINVAL); 822 } 823 824 ret = i915_mutex_lock_interruptible(dev); 825 if (ret != 0) 826 return (ret); 827 828 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); 829 if (&obj->base == NULL) { 830 ret = -ENOENT; 831 goto unlock; 832 } 833 834 if (obj->pin_count != 0) { 835 ret = -EINVAL; 836 goto out; 837 } 838 839 if (obj->madv != I915_MADV_PURGED_INTERNAL) 840 obj->madv = args->madv; 841 if (i915_gem_object_is_purgeable(obj) && obj->gtt_space == NULL) 842 i915_gem_object_truncate(obj); 843 args->retained = obj->madv != I915_MADV_PURGED_INTERNAL; 844 845 out: 846 drm_gem_object_unreference(&obj->base); 847 unlock: 848 DRM_UNLOCK(dev); 849 return (ret); 850 } 851 852 void 853 i915_gem_cleanup_ringbuffer(struct drm_device *dev) 854 { 855 drm_i915_private_t *dev_priv; 856 int i; 857 858 dev_priv = dev->dev_private; 859 for (i = 0; i < I915_NUM_RINGS; i++) 860 intel_cleanup_ring_buffer(&dev_priv->rings[i]); 861 } 862 863 int 864 i915_gem_entervt_ioctl(struct drm_device *dev, void *data, 865 struct drm_file *file_priv) 866 { 867 drm_i915_private_t *dev_priv; 868 int ret, i; 869 870 if (drm_core_check_feature(dev, DRIVER_MODESET)) 871 return (0); 872 dev_priv = dev->dev_private; 873 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 874 DRM_ERROR("Reenabling wedged hardware, good luck\n"); 875 atomic_store_rel_int(&dev_priv->mm.wedged, 0); 876 } 877 878 dev_priv->mm.suspended = 0; 879 880 ret = i915_gem_init_hw(dev); 881 if (ret != 0) { 882 return (ret); 883 } 884 885 KASSERT(list_empty(&dev_priv->mm.active_list), ("active list")); 886 KASSERT(list_empty(&dev_priv->mm.flushing_list), ("flushing list")); 887 KASSERT(list_empty(&dev_priv->mm.inactive_list), ("inactive list")); 888 for (i = 0; i < I915_NUM_RINGS; i++) { 889 KASSERT(list_empty(&dev_priv->rings[i].active_list), 890 ("ring %d active list", i)); 891 KASSERT(list_empty(&dev_priv->rings[i].request_list), 892 ("ring %d request list", i)); 893 } 894 895 DRM_UNLOCK(dev); 896 ret = drm_irq_install(dev); 897 DRM_LOCK(dev); 898 if (ret) 899 goto cleanup_ringbuffer; 900 901 return (0); 902 903 cleanup_ringbuffer: 904 i915_gem_cleanup_ringbuffer(dev); 905 dev_priv->mm.suspended = 1; 906 907 return (ret); 908 } 909 910 int 911 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 912 struct drm_file *file_priv) 913 { 914 915 if (drm_core_check_feature(dev, DRIVER_MODESET)) 916 return 0; 917 918 drm_irq_uninstall(dev); 919 return (i915_gem_idle(dev)); 920 } 921 922 int 923 i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size, 924 uint32_t *handle_p) 925 { 926 struct drm_i915_gem_object *obj; 927 uint32_t handle; 928 int ret; 929 930 size = roundup(size, PAGE_SIZE); 931 if (size == 0) 932 return (-EINVAL); 933 934 obj = i915_gem_alloc_object(dev, size); 935 if (obj == NULL) 936 return (-ENOMEM); 937 938 handle = 0; 939 ret = drm_gem_handle_create(file, &obj->base, &handle); 940 if (ret != 0) { 941 drm_gem_object_release(&obj->base); 942 i915_gem_info_remove_obj(dev->dev_private, obj->base.size); 943 drm_free(obj, DRM_I915_GEM); 944 return (-ret); 945 } 946 947 /* drop reference from allocate - handle holds it now */ 948 drm_gem_object_unreference(&obj->base); 949 *handle_p = handle; 950 return (0); 951 } 952 953 int 954 i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev, 955 struct drm_mode_create_dumb *args) 956 { 957 958 /* have to work out size/pitch and return them */ 959 args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); 960 args->size = args->pitch * args->height; 961 return (i915_gem_create(file, dev, args->size, &args->handle)); 962 } 963 964 int 965 i915_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, 966 uint32_t handle) 967 { 968 969 return (drm_gem_handle_delete(file, handle)); 970 } 971 972 int 973 i915_gem_create_ioctl(struct drm_device *dev, void *data, 974 struct drm_file *file) 975 { 976 struct drm_i915_gem_create *args = data; 977 978 return (i915_gem_create(file, dev, args->size, &args->handle)); 979 } 980 981 static inline void vm_page_reference(vm_page_t m) 982 { 983 vm_page_flag_set(m, PG_REFERENCED); 984 } 985 986 static int 987 i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj, 988 uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw, 989 struct drm_file *file) 990 { 991 vm_object_t vm_obj; 992 vm_page_t m; 993 struct sf_buf *sf; 994 vm_offset_t mkva; 995 vm_pindex_t obj_pi; 996 int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po; 997 998 if (obj->gtt_offset != 0 && rw == UIO_READ) 999 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 1000 else 1001 do_bit17_swizzling = 0; 1002 1003 obj->dirty = 1; 1004 vm_obj = obj->base.vm_obj; 1005 ret = 0; 1006 1007 VM_OBJECT_LOCK(vm_obj); 1008 vm_object_pip_add(vm_obj, 1); 1009 while (size > 0) { 1010 obj_pi = OFF_TO_IDX(offset); 1011 obj_po = offset & PAGE_MASK; 1012 1013 m = i915_gem_wire_page(vm_obj, obj_pi); 1014 VM_OBJECT_UNLOCK(vm_obj); 1015 1016 sf = sf_buf_alloc(m); 1017 mkva = sf_buf_kva(sf); 1018 length = min(size, PAGE_SIZE - obj_po); 1019 while (length > 0) { 1020 if (do_bit17_swizzling && 1021 (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) { 1022 cnt = roundup2(obj_po + 1, 64); 1023 cnt = min(cnt - obj_po, length); 1024 swizzled_po = obj_po ^ 64; 1025 } else { 1026 cnt = length; 1027 swizzled_po = obj_po; 1028 } 1029 if (rw == UIO_READ) 1030 ret = -copyout_nofault( 1031 (char *)mkva + swizzled_po, 1032 (void *)(uintptr_t)data_ptr, cnt); 1033 else 1034 ret = -copyin_nofault( 1035 (void *)(uintptr_t)data_ptr, 1036 (char *)mkva + swizzled_po, cnt); 1037 if (ret != 0) 1038 break; 1039 data_ptr += cnt; 1040 size -= cnt; 1041 length -= cnt; 1042 offset += cnt; 1043 obj_po += cnt; 1044 } 1045 sf_buf_free(sf); 1046 VM_OBJECT_LOCK(vm_obj); 1047 if (rw == UIO_WRITE) 1048 vm_page_dirty(m); 1049 vm_page_reference(m); 1050 vm_page_busy_wait(m, FALSE, "i915gem"); 1051 vm_page_unwire(m, 1); 1052 vm_page_wakeup(m); 1053 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 1054 1055 if (ret != 0) 1056 break; 1057 } 1058 vm_object_pip_wakeup(vm_obj); 1059 VM_OBJECT_UNLOCK(vm_obj); 1060 1061 return (ret); 1062 } 1063 1064 static int 1065 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj, 1066 uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file) 1067 { 1068 vm_offset_t mkva; 1069 int ret; 1070 1071 /* 1072 * Pass the unaligned physical address and size to pmap_mapdev_attr() 1073 * so it can properly calculate whether an extra page needs to be 1074 * mapped or not to cover the requested range. The function will 1075 * add the page offset into the returned mkva for us. 1076 */ 1077 mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset + 1078 offset, size, PAT_WRITE_COMBINING); 1079 ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva, size); 1080 pmap_unmapdev(mkva, size); 1081 return (ret); 1082 } 1083 1084 static int 1085 i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr, 1086 uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file) 1087 { 1088 struct drm_i915_gem_object *obj; 1089 vm_page_t *ma; 1090 vm_offset_t start, end; 1091 int npages, ret; 1092 1093 if (size == 0) 1094 return (0); 1095 start = trunc_page(data_ptr); 1096 end = round_page(data_ptr + size); 1097 npages = howmany(end - start, PAGE_SIZE); 1098 ma = kmalloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK | 1099 M_ZERO); 1100 npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, 1101 (vm_offset_t)data_ptr, size, 1102 (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages); 1103 if (npages == -1) { 1104 ret = -EFAULT; 1105 goto free_ma; 1106 } 1107 1108 ret = i915_mutex_lock_interruptible(dev); 1109 if (ret != 0) 1110 goto unlocked; 1111 1112 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1113 if (&obj->base == NULL) { 1114 ret = -ENOENT; 1115 goto unlock; 1116 } 1117 if (offset > obj->base.size || size > obj->base.size - offset) { 1118 ret = -EINVAL; 1119 goto out; 1120 } 1121 1122 if (rw == UIO_READ) { 1123 ret = i915_gem_object_set_cpu_read_domain_range(obj, 1124 offset, size); 1125 if (ret != 0) 1126 goto out; 1127 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 1128 UIO_READ, file); 1129 } else { 1130 if (obj->phys_obj) { 1131 ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset, 1132 size, file); 1133 } else if (obj->gtt_space && 1134 obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 1135 ret = i915_gem_object_pin(obj, 0, true); 1136 if (ret != 0) 1137 goto out; 1138 ret = i915_gem_object_set_to_gtt_domain(obj, true); 1139 if (ret != 0) 1140 goto out_unpin; 1141 ret = i915_gem_object_put_fence(obj); 1142 if (ret != 0) 1143 goto out_unpin; 1144 ret = i915_gem_gtt_write(dev, obj, data_ptr, size, 1145 offset, file); 1146 out_unpin: 1147 i915_gem_object_unpin(obj); 1148 } else { 1149 ret = i915_gem_object_set_to_cpu_domain(obj, true); 1150 if (ret != 0) 1151 goto out; 1152 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 1153 UIO_WRITE, file); 1154 } 1155 } 1156 out: 1157 drm_gem_object_unreference(&obj->base); 1158 unlock: 1159 DRM_UNLOCK(dev); 1160 unlocked: 1161 vm_page_unhold_pages(ma, npages); 1162 free_ma: 1163 drm_free(ma, DRM_I915_GEM); 1164 return (ret); 1165 } 1166 1167 int 1168 i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 1169 { 1170 struct drm_i915_gem_pread *args; 1171 1172 args = data; 1173 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 1174 args->offset, UIO_READ, file)); 1175 } 1176 1177 int 1178 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 1179 { 1180 struct drm_i915_gem_pwrite *args; 1181 1182 args = data; 1183 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 1184 args->offset, UIO_WRITE, file)); 1185 } 1186 1187 int 1188 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1189 struct drm_file *file) 1190 { 1191 struct drm_i915_gem_set_domain *args; 1192 struct drm_i915_gem_object *obj; 1193 uint32_t read_domains; 1194 uint32_t write_domain; 1195 int ret; 1196 1197 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1198 return (-ENODEV); 1199 1200 args = data; 1201 read_domains = args->read_domains; 1202 write_domain = args->write_domain; 1203 1204 if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 || 1205 (read_domains & I915_GEM_GPU_DOMAINS) != 0 || 1206 (write_domain != 0 && read_domains != write_domain)) 1207 return (-EINVAL); 1208 1209 ret = i915_mutex_lock_interruptible(dev); 1210 if (ret != 0) 1211 return (ret); 1212 1213 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1214 if (&obj->base == NULL) { 1215 ret = -ENOENT; 1216 goto unlock; 1217 } 1218 1219 if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) { 1220 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1221 if (ret == -EINVAL) 1222 ret = 0; 1223 } else 1224 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1225 1226 drm_gem_object_unreference(&obj->base); 1227 unlock: 1228 DRM_UNLOCK(dev); 1229 return (ret); 1230 } 1231 1232 int 1233 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1234 struct drm_file *file) 1235 { 1236 struct drm_i915_gem_sw_finish *args; 1237 struct drm_i915_gem_object *obj; 1238 int ret; 1239 1240 args = data; 1241 ret = 0; 1242 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1243 return (ENODEV); 1244 ret = i915_mutex_lock_interruptible(dev); 1245 if (ret != 0) 1246 return (ret); 1247 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1248 if (&obj->base == NULL) { 1249 ret = -ENOENT; 1250 goto unlock; 1251 } 1252 if (obj->pin_count != 0) 1253 i915_gem_object_flush_cpu_write_domain(obj); 1254 drm_gem_object_unreference(&obj->base); 1255 unlock: 1256 DRM_UNLOCK(dev); 1257 return (ret); 1258 } 1259 1260 int 1261 i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1262 struct drm_file *file) 1263 { 1264 struct drm_i915_gem_mmap *args; 1265 struct drm_gem_object *obj; 1266 struct proc *p; 1267 vm_map_t map; 1268 vm_offset_t addr; 1269 vm_size_t size; 1270 int error, rv; 1271 1272 args = data; 1273 1274 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1275 return (-ENODEV); 1276 1277 obj = drm_gem_object_lookup(dev, file, args->handle); 1278 if (obj == NULL) 1279 return (-ENOENT); 1280 error = 0; 1281 if (args->size == 0) 1282 goto out; 1283 p = curproc; 1284 map = &p->p_vmspace->vm_map; 1285 size = round_page(args->size); 1286 PROC_LOCK(p); 1287 if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) { 1288 PROC_UNLOCK(p); 1289 error = ENOMEM; 1290 goto out; 1291 } 1292 PROC_UNLOCK(p); 1293 1294 addr = 0; 1295 vm_object_hold(obj->vm_obj); 1296 vm_object_reference_locked(obj->vm_obj); 1297 vm_object_drop(obj->vm_obj); 1298 DRM_UNLOCK(dev); 1299 rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, 1300 PAGE_SIZE, /* align */ 1301 TRUE, /* fitit */ 1302 VM_MAPTYPE_NORMAL, /* maptype */ 1303 VM_PROT_READ | VM_PROT_WRITE, /* prot */ 1304 VM_PROT_READ | VM_PROT_WRITE, /* max */ 1305 MAP_SHARED /* cow */); 1306 if (rv != KERN_SUCCESS) { 1307 vm_object_deallocate(obj->vm_obj); 1308 error = -vm_mmap_to_errno(rv); 1309 } else { 1310 args->addr_ptr = (uint64_t)addr; 1311 } 1312 DRM_LOCK(dev); 1313 out: 1314 drm_gem_object_unreference(obj); 1315 return (error); 1316 } 1317 1318 static int 1319 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 1320 vm_ooffset_t foff, struct ucred *cred, u_short *color) 1321 { 1322 1323 *color = 0; /* XXXKIB */ 1324 return (0); 1325 } 1326 1327 int i915_intr_pf; 1328 1329 static int 1330 i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, 1331 vm_page_t *mres) 1332 { 1333 struct drm_gem_object *gem_obj; 1334 struct drm_i915_gem_object *obj; 1335 struct drm_device *dev; 1336 drm_i915_private_t *dev_priv; 1337 vm_page_t m, oldm; 1338 int cause, ret; 1339 bool write; 1340 1341 gem_obj = vm_obj->handle; 1342 obj = to_intel_bo(gem_obj); 1343 dev = obj->base.dev; 1344 dev_priv = dev->dev_private; 1345 #if 0 1346 write = (prot & VM_PROT_WRITE) != 0; 1347 #else 1348 write = true; 1349 #endif 1350 vm_object_pip_add(vm_obj, 1); 1351 1352 /* 1353 * Remove the placeholder page inserted by vm_fault() from the 1354 * object before dropping the object lock. If 1355 * i915_gem_release_mmap() is active in parallel on this gem 1356 * object, then it owns the drm device sx and might find the 1357 * placeholder already. Then, since the page is busy, 1358 * i915_gem_release_mmap() sleeps waiting for the busy state 1359 * of the page cleared. We will be not able to acquire drm 1360 * device lock until i915_gem_release_mmap() is able to make a 1361 * progress. 1362 */ 1363 if (*mres != NULL) { 1364 oldm = *mres; 1365 vm_page_remove(oldm); 1366 *mres = NULL; 1367 } else 1368 oldm = NULL; 1369 retry: 1370 VM_OBJECT_UNLOCK(vm_obj); 1371 unlocked_vmobj: 1372 cause = ret = 0; 1373 m = NULL; 1374 1375 if (i915_intr_pf) { 1376 ret = i915_mutex_lock_interruptible(dev); 1377 if (ret != 0) { 1378 cause = 10; 1379 goto out; 1380 } 1381 } else 1382 DRM_LOCK(dev); 1383 1384 /* 1385 * Since the object lock was dropped, other thread might have 1386 * faulted on the same GTT address and instantiated the 1387 * mapping for the page. Recheck. 1388 */ 1389 VM_OBJECT_LOCK(vm_obj); 1390 m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); 1391 if (m != NULL) { 1392 if ((m->flags & PG_BUSY) != 0) { 1393 DRM_UNLOCK(dev); 1394 #if 0 /* XXX */ 1395 vm_page_sleep(m, "915pee"); 1396 #endif 1397 goto retry; 1398 } 1399 goto have_page; 1400 } else 1401 VM_OBJECT_UNLOCK(vm_obj); 1402 1403 /* Now bind it into the GTT if needed */ 1404 if (!obj->map_and_fenceable) { 1405 ret = i915_gem_object_unbind(obj); 1406 if (ret != 0) { 1407 cause = 20; 1408 goto unlock; 1409 } 1410 } 1411 if (!obj->gtt_space) { 1412 ret = i915_gem_object_bind_to_gtt(obj, 0, true); 1413 if (ret != 0) { 1414 cause = 30; 1415 goto unlock; 1416 } 1417 1418 ret = i915_gem_object_set_to_gtt_domain(obj, write); 1419 if (ret != 0) { 1420 cause = 40; 1421 goto unlock; 1422 } 1423 } 1424 1425 if (obj->tiling_mode == I915_TILING_NONE) 1426 ret = i915_gem_object_put_fence(obj); 1427 else 1428 ret = i915_gem_object_get_fence(obj, NULL); 1429 if (ret != 0) { 1430 cause = 50; 1431 goto unlock; 1432 } 1433 1434 if (i915_gem_object_is_inactive(obj)) 1435 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 1436 1437 obj->fault_mappable = true; 1438 VM_OBJECT_LOCK(vm_obj); 1439 m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset + 1440 offset); 1441 if (m == NULL) { 1442 cause = 60; 1443 ret = -EFAULT; 1444 goto unlock; 1445 } 1446 KASSERT((m->flags & PG_FICTITIOUS) != 0, 1447 ("not fictitious %p", m)); 1448 KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m)); 1449 1450 if ((m->flags & PG_BUSY) != 0) { 1451 DRM_UNLOCK(dev); 1452 #if 0 /* XXX */ 1453 vm_page_sleep(m, "915pbs"); 1454 #endif 1455 goto retry; 1456 } 1457 m->valid = VM_PAGE_BITS_ALL; 1458 vm_page_insert(m, vm_obj, OFF_TO_IDX(offset)); 1459 have_page: 1460 *mres = m; 1461 vm_page_busy_try(m, false); 1462 1463 DRM_UNLOCK(dev); 1464 if (oldm != NULL) { 1465 vm_page_free(oldm); 1466 } 1467 vm_object_pip_wakeup(vm_obj); 1468 return (VM_PAGER_OK); 1469 1470 unlock: 1471 DRM_UNLOCK(dev); 1472 out: 1473 KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); 1474 if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) { 1475 goto unlocked_vmobj; 1476 } 1477 VM_OBJECT_LOCK(vm_obj); 1478 vm_object_pip_wakeup(vm_obj); 1479 return (VM_PAGER_ERROR); 1480 } 1481 1482 static void 1483 i915_gem_pager_dtor(void *handle) 1484 { 1485 struct drm_gem_object *obj; 1486 struct drm_device *dev; 1487 1488 obj = handle; 1489 dev = obj->dev; 1490 1491 DRM_LOCK(dev); 1492 drm_gem_free_mmap_offset(obj); 1493 i915_gem_release_mmap(to_intel_bo(obj)); 1494 drm_gem_object_unreference(obj); 1495 DRM_UNLOCK(dev); 1496 } 1497 1498 struct cdev_pager_ops i915_gem_pager_ops = { 1499 .cdev_pg_fault = i915_gem_pager_fault, 1500 .cdev_pg_ctor = i915_gem_pager_ctor, 1501 .cdev_pg_dtor = i915_gem_pager_dtor 1502 }; 1503 1504 int 1505 i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev, 1506 uint32_t handle, uint64_t *offset) 1507 { 1508 struct drm_i915_private *dev_priv; 1509 struct drm_i915_gem_object *obj; 1510 int ret; 1511 1512 if (!(dev->driver->driver_features & DRIVER_GEM)) 1513 return (-ENODEV); 1514 1515 dev_priv = dev->dev_private; 1516 1517 ret = i915_mutex_lock_interruptible(dev); 1518 if (ret != 0) 1519 return (ret); 1520 1521 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1522 if (&obj->base == NULL) { 1523 ret = -ENOENT; 1524 goto unlock; 1525 } 1526 1527 if (obj->base.size > dev_priv->mm.gtt_mappable_end) { 1528 ret = -E2BIG; 1529 goto out; 1530 } 1531 1532 if (obj->madv != I915_MADV_WILLNEED) { 1533 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1534 ret = -EINVAL; 1535 goto out; 1536 } 1537 1538 ret = drm_gem_create_mmap_offset(&obj->base); 1539 if (ret != 0) 1540 goto out; 1541 1542 *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 1543 DRM_GEM_MAPPING_KEY; 1544 out: 1545 drm_gem_object_unreference(&obj->base); 1546 unlock: 1547 DRM_UNLOCK(dev); 1548 return (ret); 1549 } 1550 1551 int 1552 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1553 struct drm_file *file) 1554 { 1555 struct drm_i915_private *dev_priv; 1556 struct drm_i915_gem_mmap_gtt *args; 1557 1558 dev_priv = dev->dev_private; 1559 args = data; 1560 1561 return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset)); 1562 } 1563 1564 struct drm_i915_gem_object * 1565 i915_gem_alloc_object(struct drm_device *dev, size_t size) 1566 { 1567 struct drm_i915_private *dev_priv; 1568 struct drm_i915_gem_object *obj; 1569 1570 dev_priv = dev->dev_private; 1571 1572 obj = kmalloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO); 1573 1574 if (drm_gem_object_init(dev, &obj->base, size) != 0) { 1575 drm_free(obj, DRM_I915_GEM); 1576 return (NULL); 1577 } 1578 1579 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1580 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1581 1582 if (HAS_LLC(dev)) 1583 obj->cache_level = I915_CACHE_LLC; 1584 else 1585 obj->cache_level = I915_CACHE_NONE; 1586 obj->base.driver_private = NULL; 1587 obj->fence_reg = I915_FENCE_REG_NONE; 1588 INIT_LIST_HEAD(&obj->mm_list); 1589 INIT_LIST_HEAD(&obj->gtt_list); 1590 INIT_LIST_HEAD(&obj->ring_list); 1591 INIT_LIST_HEAD(&obj->exec_list); 1592 INIT_LIST_HEAD(&obj->gpu_write_list); 1593 obj->madv = I915_MADV_WILLNEED; 1594 /* Avoid an unnecessary call to unbind on the first bind. */ 1595 obj->map_and_fenceable = true; 1596 1597 i915_gem_info_add_obj(dev_priv, size); 1598 1599 return (obj); 1600 } 1601 1602 void 1603 i915_gem_clflush_object(struct drm_i915_gem_object *obj) 1604 { 1605 1606 /* If we don't have a page list set up, then we're not pinned 1607 * to GPU, and we can ignore the cache flush because it'll happen 1608 * again at bind time. 1609 */ 1610 if (obj->pages == NULL) 1611 return; 1612 1613 /* If the GPU is snooping the contents of the CPU cache, 1614 * we do not need to manually clear the CPU cache lines. However, 1615 * the caches are only snooped when the render cache is 1616 * flushed/invalidated. As we always have to emit invalidations 1617 * and flushes when moving into and out of the RENDER domain, correct 1618 * snooping behaviour occurs naturally as the result of our domain 1619 * tracking. 1620 */ 1621 if (obj->cache_level != I915_CACHE_NONE) 1622 return; 1623 1624 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); 1625 } 1626 1627 static void 1628 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) 1629 { 1630 uint32_t old_write_domain; 1631 1632 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 1633 return; 1634 1635 i915_gem_clflush_object(obj); 1636 intel_gtt_chipset_flush(); 1637 old_write_domain = obj->base.write_domain; 1638 obj->base.write_domain = 0; 1639 } 1640 1641 static int 1642 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj) 1643 { 1644 1645 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) 1646 return (0); 1647 return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain)); 1648 } 1649 1650 static void 1651 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 1652 { 1653 uint32_t old_write_domain; 1654 1655 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 1656 return; 1657 1658 cpu_sfence(); 1659 1660 old_write_domain = obj->base.write_domain; 1661 obj->base.write_domain = 0; 1662 } 1663 1664 int 1665 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 1666 { 1667 uint32_t old_write_domain, old_read_domains; 1668 int ret; 1669 1670 if (obj->gtt_space == NULL) 1671 return (-EINVAL); 1672 1673 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 1674 return 0; 1675 1676 ret = i915_gem_object_flush_gpu_write_domain(obj); 1677 if (ret != 0) 1678 return (ret); 1679 1680 if (obj->pending_gpu_write || write) { 1681 ret = i915_gem_object_wait_rendering(obj); 1682 if (ret != 0) 1683 return (ret); 1684 } 1685 1686 i915_gem_object_flush_cpu_write_domain(obj); 1687 1688 old_write_domain = obj->base.write_domain; 1689 old_read_domains = obj->base.read_domains; 1690 1691 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 1692 ("In GTT write domain")); 1693 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 1694 if (write) { 1695 obj->base.read_domains = I915_GEM_DOMAIN_GTT; 1696 obj->base.write_domain = I915_GEM_DOMAIN_GTT; 1697 obj->dirty = 1; 1698 } 1699 1700 return (0); 1701 } 1702 1703 int 1704 i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 1705 enum i915_cache_level cache_level) 1706 { 1707 struct drm_device *dev; 1708 drm_i915_private_t *dev_priv; 1709 int ret; 1710 1711 if (obj->cache_level == cache_level) 1712 return 0; 1713 1714 if (obj->pin_count) { 1715 DRM_DEBUG("can not change the cache level of pinned objects\n"); 1716 return (-EBUSY); 1717 } 1718 1719 dev = obj->base.dev; 1720 dev_priv = dev->dev_private; 1721 if (obj->gtt_space) { 1722 ret = i915_gem_object_finish_gpu(obj); 1723 if (ret != 0) 1724 return (ret); 1725 1726 i915_gem_object_finish_gtt(obj); 1727 1728 /* Before SandyBridge, you could not use tiling or fence 1729 * registers with snooped memory, so relinquish any fences 1730 * currently pointing to our region in the aperture. 1731 */ 1732 if (INTEL_INFO(obj->base.dev)->gen < 6) { 1733 ret = i915_gem_object_put_fence(obj); 1734 if (ret != 0) 1735 return (ret); 1736 } 1737 1738 i915_gem_gtt_rebind_object(obj, cache_level); 1739 if (obj->has_aliasing_ppgtt_mapping) 1740 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, 1741 obj, cache_level); 1742 } 1743 1744 if (cache_level == I915_CACHE_NONE) { 1745 u32 old_read_domains, old_write_domain; 1746 1747 /* If we're coming from LLC cached, then we haven't 1748 * actually been tracking whether the data is in the 1749 * CPU cache or not, since we only allow one bit set 1750 * in obj->write_domain and have been skipping the clflushes. 1751 * Just set it to the CPU cache for now. 1752 */ 1753 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1754 ("obj %p in CPU write domain", obj)); 1755 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0, 1756 ("obj %p in CPU read domain", obj)); 1757 1758 old_read_domains = obj->base.read_domains; 1759 old_write_domain = obj->base.write_domain; 1760 1761 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1762 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1763 1764 } 1765 1766 obj->cache_level = cache_level; 1767 return (0); 1768 } 1769 1770 int 1771 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 1772 u32 alignment, struct intel_ring_buffer *pipelined) 1773 { 1774 u32 old_read_domains, old_write_domain; 1775 int ret; 1776 1777 ret = i915_gem_object_flush_gpu_write_domain(obj); 1778 if (ret != 0) 1779 return (ret); 1780 1781 if (pipelined != obj->ring) { 1782 ret = i915_gem_object_wait_rendering(obj); 1783 if (ret == -ERESTART || ret == -EINTR) 1784 return (ret); 1785 } 1786 1787 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); 1788 if (ret != 0) 1789 return (ret); 1790 1791 ret = i915_gem_object_pin(obj, alignment, true); 1792 if (ret != 0) 1793 return (ret); 1794 1795 i915_gem_object_flush_cpu_write_domain(obj); 1796 1797 old_write_domain = obj->base.write_domain; 1798 old_read_domains = obj->base.read_domains; 1799 1800 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 1801 ("obj %p in GTT write domain", obj)); 1802 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 1803 1804 return (0); 1805 } 1806 1807 int 1808 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) 1809 { 1810 int ret; 1811 1812 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) 1813 return (0); 1814 1815 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 1816 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); 1817 if (ret != 0) 1818 return (ret); 1819 } 1820 1821 ret = i915_gem_object_wait_rendering(obj); 1822 if (ret != 0) 1823 return (ret); 1824 1825 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 1826 1827 return (0); 1828 } 1829 1830 static int 1831 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 1832 { 1833 uint32_t old_write_domain, old_read_domains; 1834 int ret; 1835 1836 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 1837 return 0; 1838 1839 ret = i915_gem_object_flush_gpu_write_domain(obj); 1840 if (ret != 0) 1841 return (ret); 1842 1843 ret = i915_gem_object_wait_rendering(obj); 1844 if (ret != 0) 1845 return (ret); 1846 1847 i915_gem_object_flush_gtt_write_domain(obj); 1848 i915_gem_object_set_to_full_cpu_read_domain(obj); 1849 1850 old_write_domain = obj->base.write_domain; 1851 old_read_domains = obj->base.read_domains; 1852 1853 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 1854 i915_gem_clflush_object(obj); 1855 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 1856 } 1857 1858 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1859 ("In cpu write domain")); 1860 1861 if (write) { 1862 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1863 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1864 } 1865 1866 return (0); 1867 } 1868 1869 static void 1870 i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj) 1871 { 1872 int i; 1873 1874 if (obj->page_cpu_valid == NULL) 1875 return; 1876 1877 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) { 1878 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) { 1879 if (obj->page_cpu_valid[i] != 0) 1880 continue; 1881 drm_clflush_pages(obj->pages + i, 1); 1882 } 1883 } 1884 1885 drm_free(obj->page_cpu_valid, DRM_I915_GEM); 1886 obj->page_cpu_valid = NULL; 1887 } 1888 1889 static int 1890 i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj, 1891 uint64_t offset, uint64_t size) 1892 { 1893 uint32_t old_read_domains; 1894 int i, ret; 1895 1896 if (offset == 0 && size == obj->base.size) 1897 return (i915_gem_object_set_to_cpu_domain(obj, 0)); 1898 1899 ret = i915_gem_object_flush_gpu_write_domain(obj); 1900 if (ret != 0) 1901 return (ret); 1902 ret = i915_gem_object_wait_rendering(obj); 1903 if (ret != 0) 1904 return (ret); 1905 1906 i915_gem_object_flush_gtt_write_domain(obj); 1907 1908 if (obj->page_cpu_valid == NULL && 1909 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) 1910 return (0); 1911 1912 if (obj->page_cpu_valid == NULL) { 1913 obj->page_cpu_valid = kmalloc(obj->base.size / PAGE_SIZE, 1914 DRM_I915_GEM, M_WAITOK | M_ZERO); 1915 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) 1916 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE); 1917 1918 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; 1919 i++) { 1920 if (obj->page_cpu_valid[i]) 1921 continue; 1922 drm_clflush_pages(obj->pages + i, 1); 1923 obj->page_cpu_valid[i] = 1; 1924 } 1925 1926 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1927 ("In gpu write domain")); 1928 1929 old_read_domains = obj->base.read_domains; 1930 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 1931 1932 return (0); 1933 } 1934 1935 static uint32_t 1936 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 1937 { 1938 uint32_t gtt_size; 1939 1940 if (INTEL_INFO(dev)->gen >= 4 || 1941 tiling_mode == I915_TILING_NONE) 1942 return (size); 1943 1944 /* Previous chips need a power-of-two fence region when tiling */ 1945 if (INTEL_INFO(dev)->gen == 3) 1946 gtt_size = 1024*1024; 1947 else 1948 gtt_size = 512*1024; 1949 1950 while (gtt_size < size) 1951 gtt_size <<= 1; 1952 1953 return (gtt_size); 1954 } 1955 1956 /** 1957 * i915_gem_get_gtt_alignment - return required GTT alignment for an object 1958 * @obj: object to check 1959 * 1960 * Return the required GTT alignment for an object, taking into account 1961 * potential fence register mapping. 1962 */ 1963 static uint32_t 1964 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, 1965 int tiling_mode) 1966 { 1967 1968 /* 1969 * Minimum alignment is 4k (GTT page size), but might be greater 1970 * if a fence register is needed for the object. 1971 */ 1972 if (INTEL_INFO(dev)->gen >= 4 || 1973 tiling_mode == I915_TILING_NONE) 1974 return (4096); 1975 1976 /* 1977 * Previous chips need to be aligned to the size of the smallest 1978 * fence register that can contain the object. 1979 */ 1980 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 1981 } 1982 1983 uint32_t 1984 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size, 1985 int tiling_mode) 1986 { 1987 1988 if (tiling_mode == I915_TILING_NONE) 1989 return (4096); 1990 1991 /* 1992 * Minimum alignment is 4k (GTT page size) for sane hw. 1993 */ 1994 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev)) 1995 return (4096); 1996 1997 /* 1998 * Previous hardware however needs to be aligned to a power-of-two 1999 * tile height. The simplest method for determining this is to reuse 2000 * the power-of-tile object size. 2001 */ 2002 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 2003 } 2004 2005 static int 2006 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 2007 unsigned alignment, bool map_and_fenceable) 2008 { 2009 struct drm_device *dev; 2010 struct drm_i915_private *dev_priv; 2011 struct drm_mm_node *free_space; 2012 uint32_t size, fence_size, fence_alignment, unfenced_alignment; 2013 bool mappable, fenceable; 2014 int ret; 2015 2016 dev = obj->base.dev; 2017 dev_priv = dev->dev_private; 2018 2019 if (obj->madv != I915_MADV_WILLNEED) { 2020 DRM_ERROR("Attempting to bind a purgeable object\n"); 2021 return (-EINVAL); 2022 } 2023 2024 fence_size = i915_gem_get_gtt_size(dev, obj->base.size, 2025 obj->tiling_mode); 2026 fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size, 2027 obj->tiling_mode); 2028 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev, 2029 obj->base.size, obj->tiling_mode); 2030 if (alignment == 0) 2031 alignment = map_and_fenceable ? fence_alignment : 2032 unfenced_alignment; 2033 if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) { 2034 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 2035 return (-EINVAL); 2036 } 2037 2038 size = map_and_fenceable ? fence_size : obj->base.size; 2039 2040 /* If the object is bigger than the entire aperture, reject it early 2041 * before evicting everything in a vain attempt to find space. 2042 */ 2043 if (obj->base.size > (map_and_fenceable ? 2044 dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { 2045 DRM_ERROR( 2046 "Attempting to bind an object larger than the aperture\n"); 2047 return (-E2BIG); 2048 } 2049 2050 search_free: 2051 if (map_and_fenceable) 2052 free_space = drm_mm_search_free_in_range( 2053 &dev_priv->mm.gtt_space, size, alignment, 0, 2054 dev_priv->mm.gtt_mappable_end, 0); 2055 else 2056 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 2057 size, alignment, 0); 2058 if (free_space != NULL) { 2059 if (map_and_fenceable) 2060 obj->gtt_space = drm_mm_get_block_range_generic( 2061 free_space, size, alignment, 0, 2062 dev_priv->mm.gtt_mappable_end, 1); 2063 else 2064 obj->gtt_space = drm_mm_get_block_generic(free_space, 2065 size, alignment, 1); 2066 } 2067 if (obj->gtt_space == NULL) { 2068 ret = i915_gem_evict_something(dev, size, alignment, 2069 map_and_fenceable); 2070 if (ret != 0) 2071 return (ret); 2072 goto search_free; 2073 } 2074 ret = i915_gem_object_get_pages_gtt(obj, 0); 2075 if (ret != 0) { 2076 drm_mm_put_block(obj->gtt_space); 2077 obj->gtt_space = NULL; 2078 /* 2079 * i915_gem_object_get_pages_gtt() cannot return 2080 * ENOMEM, since we use vm_page_grab(VM_ALLOC_RETRY) 2081 * (which does not support operation without a flag 2082 * anyway). 2083 */ 2084 return (ret); 2085 } 2086 2087 ret = i915_gem_gtt_bind_object(obj); 2088 if (ret != 0) { 2089 i915_gem_object_put_pages_gtt(obj); 2090 drm_mm_put_block(obj->gtt_space); 2091 obj->gtt_space = NULL; 2092 if (i915_gem_evict_everything(dev, false)) 2093 return (ret); 2094 goto search_free; 2095 } 2096 2097 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); 2098 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2099 2100 KASSERT((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0, 2101 ("Object in gpu read domain")); 2102 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2103 ("Object in gpu write domain")); 2104 2105 obj->gtt_offset = obj->gtt_space->start; 2106 2107 fenceable = 2108 obj->gtt_space->size == fence_size && 2109 (obj->gtt_space->start & (fence_alignment - 1)) == 0; 2110 2111 mappable = 2112 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; 2113 obj->map_and_fenceable = mappable && fenceable; 2114 2115 return (0); 2116 } 2117 2118 static void 2119 i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 2120 { 2121 u32 old_write_domain, old_read_domains; 2122 2123 /* Act a barrier for all accesses through the GTT */ 2124 cpu_mfence(); 2125 2126 /* Force a pagefault for domain tracking on next user access */ 2127 i915_gem_release_mmap(obj); 2128 2129 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 2130 return; 2131 2132 old_read_domains = obj->base.read_domains; 2133 old_write_domain = obj->base.write_domain; 2134 2135 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 2136 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 2137 2138 } 2139 2140 int 2141 i915_gem_object_unbind(struct drm_i915_gem_object *obj) 2142 { 2143 drm_i915_private_t *dev_priv; 2144 int ret; 2145 2146 dev_priv = obj->base.dev->dev_private; 2147 ret = 0; 2148 if (obj->gtt_space == NULL) 2149 return (0); 2150 if (obj->pin_count != 0) { 2151 DRM_ERROR("Attempting to unbind pinned buffer\n"); 2152 return (-EINVAL); 2153 } 2154 2155 ret = i915_gem_object_finish_gpu(obj); 2156 if (ret == -ERESTART || ret == -EINTR) 2157 return (ret); 2158 2159 i915_gem_object_finish_gtt(obj); 2160 2161 if (ret == 0) 2162 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 2163 if (ret == -ERESTART || ret == -EINTR) 2164 return (ret); 2165 if (ret != 0) { 2166 i915_gem_clflush_object(obj); 2167 obj->base.read_domains = obj->base.write_domain = 2168 I915_GEM_DOMAIN_CPU; 2169 } 2170 2171 ret = i915_gem_object_put_fence(obj); 2172 if (ret == -ERESTART) 2173 return (ret); 2174 2175 i915_gem_gtt_unbind_object(obj); 2176 if (obj->has_aliasing_ppgtt_mapping) { 2177 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); 2178 obj->has_aliasing_ppgtt_mapping = 0; 2179 } 2180 i915_gem_object_put_pages_gtt(obj); 2181 2182 list_del_init(&obj->gtt_list); 2183 list_del_init(&obj->mm_list); 2184 obj->map_and_fenceable = true; 2185 2186 drm_mm_put_block(obj->gtt_space); 2187 obj->gtt_space = NULL; 2188 obj->gtt_offset = 0; 2189 2190 if (i915_gem_object_is_purgeable(obj)) 2191 i915_gem_object_truncate(obj); 2192 2193 return (ret); 2194 } 2195 2196 static int 2197 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 2198 int flags) 2199 { 2200 struct drm_device *dev; 2201 vm_object_t vm_obj; 2202 vm_page_t m; 2203 int page_count, i, j; 2204 2205 dev = obj->base.dev; 2206 KASSERT(obj->pages == NULL, ("Obj already has pages")); 2207 page_count = obj->base.size / PAGE_SIZE; 2208 obj->pages = kmalloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, 2209 M_WAITOK); 2210 vm_obj = obj->base.vm_obj; 2211 VM_OBJECT_LOCK(vm_obj); 2212 for (i = 0; i < page_count; i++) { 2213 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL) 2214 goto failed; 2215 } 2216 VM_OBJECT_UNLOCK(vm_obj); 2217 if (i915_gem_object_needs_bit17_swizzle(obj)) 2218 i915_gem_object_do_bit_17_swizzle(obj); 2219 return (0); 2220 2221 failed: 2222 for (j = 0; j < i; j++) { 2223 m = obj->pages[j]; 2224 vm_page_busy_wait(m, FALSE, "i915gem"); 2225 vm_page_unwire(m, 0); 2226 vm_page_wakeup(m); 2227 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2228 } 2229 VM_OBJECT_UNLOCK(vm_obj); 2230 drm_free(obj->pages, DRM_I915_GEM); 2231 obj->pages = NULL; 2232 return (-EIO); 2233 } 2234 2235 #define GEM_PARANOID_CHECK_GTT 0 2236 #if GEM_PARANOID_CHECK_GTT 2237 static void 2238 i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma, 2239 int page_count) 2240 { 2241 struct drm_i915_private *dev_priv; 2242 vm_paddr_t pa; 2243 unsigned long start, end; 2244 u_int i; 2245 int j; 2246 2247 dev_priv = dev->dev_private; 2248 start = OFF_TO_IDX(dev_priv->mm.gtt_start); 2249 end = OFF_TO_IDX(dev_priv->mm.gtt_end); 2250 for (i = start; i < end; i++) { 2251 pa = intel_gtt_read_pte_paddr(i); 2252 for (j = 0; j < page_count; j++) { 2253 if (pa == VM_PAGE_TO_PHYS(ma[j])) { 2254 panic("Page %p in GTT pte index %d pte %x", 2255 ma[i], i, intel_gtt_read_pte(i)); 2256 } 2257 } 2258 } 2259 } 2260 #endif 2261 2262 static void 2263 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 2264 { 2265 vm_page_t m; 2266 int page_count, i; 2267 2268 KASSERT(obj->madv != I915_MADV_PURGED_INTERNAL, ("Purged object")); 2269 2270 if (obj->tiling_mode != I915_TILING_NONE) 2271 i915_gem_object_save_bit_17_swizzle(obj); 2272 if (obj->madv == I915_MADV_DONTNEED) 2273 obj->dirty = 0; 2274 page_count = obj->base.size / PAGE_SIZE; 2275 VM_OBJECT_LOCK(obj->base.vm_obj); 2276 #if GEM_PARANOID_CHECK_GTT 2277 i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count); 2278 #endif 2279 for (i = 0; i < page_count; i++) { 2280 m = obj->pages[i]; 2281 if (obj->dirty) 2282 vm_page_dirty(m); 2283 if (obj->madv == I915_MADV_WILLNEED) 2284 vm_page_reference(m); 2285 vm_page_busy_wait(obj->pages[i], FALSE, "i915gem"); 2286 vm_page_unwire(obj->pages[i], 1); 2287 vm_page_wakeup(obj->pages[i]); 2288 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2289 } 2290 VM_OBJECT_UNLOCK(obj->base.vm_obj); 2291 obj->dirty = 0; 2292 drm_free(obj->pages, DRM_I915_GEM); 2293 obj->pages = NULL; 2294 } 2295 2296 void 2297 i915_gem_release_mmap(struct drm_i915_gem_object *obj) 2298 { 2299 vm_object_t devobj; 2300 vm_page_t m; 2301 int i, page_count; 2302 2303 if (!obj->fault_mappable) 2304 return; 2305 2306 devobj = cdev_pager_lookup(obj); 2307 if (devobj != NULL) { 2308 page_count = OFF_TO_IDX(obj->base.size); 2309 2310 VM_OBJECT_LOCK(devobj); 2311 for (i = 0; i < page_count; i++) { 2312 m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm"); 2313 if (m == NULL) 2314 continue; 2315 cdev_pager_free_page(devobj, m); 2316 } 2317 VM_OBJECT_UNLOCK(devobj); 2318 vm_object_deallocate(devobj); 2319 } 2320 2321 obj->fault_mappable = false; 2322 } 2323 2324 int 2325 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj) 2326 { 2327 int ret; 2328 2329 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2330 ("In GPU write domain")); 2331 2332 if (obj->active) { 2333 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno, 2334 true); 2335 if (ret != 0) 2336 return (ret); 2337 } 2338 return (0); 2339 } 2340 2341 void 2342 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, 2343 struct intel_ring_buffer *ring, uint32_t seqno) 2344 { 2345 struct drm_device *dev = obj->base.dev; 2346 struct drm_i915_private *dev_priv = dev->dev_private; 2347 struct drm_i915_fence_reg *reg; 2348 2349 obj->ring = ring; 2350 KASSERT(ring != NULL, ("NULL ring")); 2351 2352 /* Add a reference if we're newly entering the active list. */ 2353 if (!obj->active) { 2354 drm_gem_object_reference(&obj->base); 2355 obj->active = 1; 2356 } 2357 2358 /* Move from whatever list we were on to the tail of execution. */ 2359 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); 2360 list_move_tail(&obj->ring_list, &ring->active_list); 2361 2362 obj->last_rendering_seqno = seqno; 2363 if (obj->fenced_gpu_access) { 2364 obj->last_fenced_seqno = seqno; 2365 obj->last_fenced_ring = ring; 2366 2367 /* Bump MRU to take account of the delayed flush */ 2368 if (obj->fence_reg != I915_FENCE_REG_NONE) { 2369 reg = &dev_priv->fence_regs[obj->fence_reg]; 2370 list_move_tail(®->lru_list, 2371 &dev_priv->mm.fence_list); 2372 } 2373 } 2374 } 2375 2376 static void 2377 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj) 2378 { 2379 list_del_init(&obj->ring_list); 2380 obj->last_rendering_seqno = 0; 2381 obj->last_fenced_seqno = 0; 2382 } 2383 2384 static void 2385 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj) 2386 { 2387 struct drm_device *dev = obj->base.dev; 2388 drm_i915_private_t *dev_priv = dev->dev_private; 2389 2390 KASSERT(obj->active, ("Object not active")); 2391 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list); 2392 2393 i915_gem_object_move_off_active(obj); 2394 } 2395 2396 static void 2397 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) 2398 { 2399 struct drm_device *dev = obj->base.dev; 2400 struct drm_i915_private *dev_priv = dev->dev_private; 2401 2402 if (obj->pin_count != 0) 2403 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 2404 else 2405 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2406 2407 KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list")); 2408 KASSERT(obj->active, ("Object not active")); 2409 obj->ring = NULL; 2410 obj->last_fenced_ring = NULL; 2411 2412 i915_gem_object_move_off_active(obj); 2413 obj->fenced_gpu_access = false; 2414 2415 obj->active = 0; 2416 obj->pending_gpu_write = false; 2417 drm_gem_object_unreference(&obj->base); 2418 2419 #if 1 2420 KIB_NOTYET(); 2421 #else 2422 WARN_ON(i915_verify_lists(dev)); 2423 #endif 2424 } 2425 2426 static void 2427 i915_gem_object_truncate(struct drm_i915_gem_object *obj) 2428 { 2429 vm_object_t vm_obj; 2430 2431 vm_obj = obj->base.vm_obj; 2432 VM_OBJECT_LOCK(vm_obj); 2433 vm_object_page_remove(vm_obj, 0, 0, false); 2434 VM_OBJECT_UNLOCK(vm_obj); 2435 obj->madv = I915_MADV_PURGED_INTERNAL; 2436 } 2437 2438 static inline int 2439 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) 2440 { 2441 2442 return (obj->madv == I915_MADV_DONTNEED); 2443 } 2444 2445 static void 2446 i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 2447 uint32_t flush_domains) 2448 { 2449 struct drm_i915_gem_object *obj, *next; 2450 uint32_t old_write_domain; 2451 2452 list_for_each_entry_safe(obj, next, &ring->gpu_write_list, 2453 gpu_write_list) { 2454 if (obj->base.write_domain & flush_domains) { 2455 old_write_domain = obj->base.write_domain; 2456 obj->base.write_domain = 0; 2457 list_del_init(&obj->gpu_write_list); 2458 i915_gem_object_move_to_active(obj, ring, 2459 i915_gem_next_request_seqno(ring)); 2460 } 2461 } 2462 } 2463 2464 static int 2465 i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) 2466 { 2467 drm_i915_private_t *dev_priv; 2468 2469 dev_priv = obj->base.dev->dev_private; 2470 return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 2471 obj->tiling_mode != I915_TILING_NONE); 2472 } 2473 2474 #define VM_OBJECT_LOCK_ASSERT_OWNED(object) 2475 2476 static vm_page_t 2477 i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex) 2478 { 2479 vm_page_t m; 2480 int rv; 2481 2482 VM_OBJECT_LOCK_ASSERT_OWNED(object); 2483 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); 2484 if (m->valid != VM_PAGE_BITS_ALL) { 2485 if (vm_pager_has_page(object, pindex)) { 2486 rv = vm_pager_get_page(object, &m, 1); 2487 m = vm_page_lookup(object, pindex); 2488 if (m == NULL) 2489 return (NULL); 2490 if (rv != VM_PAGER_OK) { 2491 vm_page_free(m); 2492 return (NULL); 2493 } 2494 } else { 2495 pmap_zero_page(VM_PAGE_TO_PHYS(m)); 2496 m->valid = VM_PAGE_BITS_ALL; 2497 m->dirty = 0; 2498 } 2499 } 2500 vm_page_wire(m); 2501 vm_page_wakeup(m); 2502 atomic_add_long(&i915_gem_wired_pages_cnt, 1); 2503 return (m); 2504 } 2505 2506 int 2507 i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains, 2508 uint32_t flush_domains) 2509 { 2510 int ret; 2511 2512 if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0) 2513 return 0; 2514 2515 ret = ring->flush(ring, invalidate_domains, flush_domains); 2516 if (ret) 2517 return ret; 2518 2519 if (flush_domains & I915_GEM_GPU_DOMAINS) 2520 i915_gem_process_flushing_list(ring, flush_domains); 2521 return 0; 2522 } 2523 2524 static int 2525 i915_ring_idle(struct intel_ring_buffer *ring, bool do_retire) 2526 { 2527 int ret; 2528 2529 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) 2530 return 0; 2531 2532 if (!list_empty(&ring->gpu_write_list)) { 2533 ret = i915_gem_flush_ring(ring, I915_GEM_GPU_DOMAINS, 2534 I915_GEM_GPU_DOMAINS); 2535 if (ret != 0) 2536 return ret; 2537 } 2538 2539 return (i915_wait_request(ring, i915_gem_next_request_seqno(ring), 2540 do_retire)); 2541 } 2542 2543 int 2544 i915_gpu_idle(struct drm_device *dev, bool do_retire) 2545 { 2546 drm_i915_private_t *dev_priv = dev->dev_private; 2547 int ret, i; 2548 2549 /* Flush everything onto the inactive list. */ 2550 for (i = 0; i < I915_NUM_RINGS; i++) { 2551 ret = i915_ring_idle(&dev_priv->rings[i], do_retire); 2552 if (ret) 2553 return ret; 2554 } 2555 2556 return 0; 2557 } 2558 2559 int 2560 i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno, bool do_retire) 2561 { 2562 drm_i915_private_t *dev_priv; 2563 struct drm_i915_gem_request *request; 2564 uint32_t ier; 2565 int flags, ret; 2566 bool recovery_complete; 2567 2568 KASSERT(seqno != 0, ("Zero seqno")); 2569 2570 dev_priv = ring->dev->dev_private; 2571 ret = 0; 2572 2573 if (atomic_load_acq_int(&dev_priv->mm.wedged) != 0) { 2574 /* Give the error handler a chance to run. */ 2575 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE); 2576 recovery_complete = (&dev_priv->error_completion) > 0; 2577 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 2578 return (recovery_complete ? -EIO : -EAGAIN); 2579 } 2580 2581 if (seqno == ring->outstanding_lazy_request) { 2582 request = kmalloc(sizeof(*request), DRM_I915_GEM, 2583 M_WAITOK | M_ZERO); 2584 if (request == NULL) 2585 return (-ENOMEM); 2586 2587 ret = i915_add_request(ring, NULL, request); 2588 if (ret != 0) { 2589 drm_free(request, DRM_I915_GEM); 2590 return (ret); 2591 } 2592 2593 seqno = request->seqno; 2594 } 2595 2596 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { 2597 if (HAS_PCH_SPLIT(ring->dev)) 2598 ier = I915_READ(DEIER) | I915_READ(GTIER); 2599 else 2600 ier = I915_READ(IER); 2601 if (!ier) { 2602 DRM_ERROR("something (likely vbetool) disabled " 2603 "interrupts, re-enabling\n"); 2604 ring->dev->driver->irq_preinstall(ring->dev); 2605 ring->dev->driver->irq_postinstall(ring->dev); 2606 } 2607 2608 ring->waiting_seqno = seqno; 2609 lockmgr(&ring->irq_lock, LK_EXCLUSIVE); 2610 if (ring->irq_get(ring)) { 2611 flags = dev_priv->mm.interruptible ? PCATCH : 0; 2612 while (!i915_seqno_passed(ring->get_seqno(ring), seqno) 2613 && !atomic_load_acq_int(&dev_priv->mm.wedged) && 2614 ret == 0) { 2615 ret = -lksleep(ring, &ring->irq_lock, flags, 2616 "915gwr", 0); 2617 } 2618 ring->irq_put(ring); 2619 lockmgr(&ring->irq_lock, LK_RELEASE); 2620 } else { 2621 lockmgr(&ring->irq_lock, LK_RELEASE); 2622 if (_intel_wait_for(ring->dev, 2623 i915_seqno_passed(ring->get_seqno(ring), seqno) || 2624 atomic_load_acq_int(&dev_priv->mm.wedged), 3000, 2625 0, "i915wrq") != 0) 2626 ret = -EBUSY; 2627 } 2628 ring->waiting_seqno = 0; 2629 2630 } 2631 if (atomic_load_acq_int(&dev_priv->mm.wedged)) 2632 ret = -EAGAIN; 2633 2634 /* Directly dispatch request retiring. While we have the work queue 2635 * to handle this, the waiter on a request often wants an associated 2636 * buffer to have made it to the inactive list, and we would need 2637 * a separate wait queue to handle that. 2638 */ 2639 if (ret == 0 && do_retire) 2640 i915_gem_retire_requests_ring(ring); 2641 2642 return (ret); 2643 } 2644 2645 static u32 2646 i915_gem_get_seqno(struct drm_device *dev) 2647 { 2648 drm_i915_private_t *dev_priv = dev->dev_private; 2649 u32 seqno = dev_priv->next_seqno; 2650 2651 /* reserve 0 for non-seqno */ 2652 if (++dev_priv->next_seqno == 0) 2653 dev_priv->next_seqno = 1; 2654 2655 return seqno; 2656 } 2657 2658 u32 2659 i915_gem_next_request_seqno(struct intel_ring_buffer *ring) 2660 { 2661 if (ring->outstanding_lazy_request == 0) 2662 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev); 2663 2664 return ring->outstanding_lazy_request; 2665 } 2666 2667 int 2668 i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file, 2669 struct drm_i915_gem_request *request) 2670 { 2671 drm_i915_private_t *dev_priv; 2672 struct drm_i915_file_private *file_priv; 2673 uint32_t seqno; 2674 u32 request_ring_position; 2675 int was_empty; 2676 int ret; 2677 2678 KASSERT(request != NULL, ("NULL request in add")); 2679 DRM_LOCK_ASSERT(ring->dev); 2680 dev_priv = ring->dev->dev_private; 2681 2682 seqno = i915_gem_next_request_seqno(ring); 2683 request_ring_position = intel_ring_get_tail(ring); 2684 2685 ret = ring->add_request(ring, &seqno); 2686 if (ret != 0) 2687 return ret; 2688 2689 request->seqno = seqno; 2690 request->ring = ring; 2691 request->tail = request_ring_position; 2692 request->emitted_jiffies = ticks; 2693 was_empty = list_empty(&ring->request_list); 2694 list_add_tail(&request->list, &ring->request_list); 2695 2696 if (file != NULL) { 2697 file_priv = file->driver_priv; 2698 2699 spin_lock(&file_priv->mm.lock); 2700 request->file_priv = file_priv; 2701 list_add_tail(&request->client_list, 2702 &file_priv->mm.request_list); 2703 spin_unlock(&file_priv->mm.lock); 2704 } 2705 2706 ring->outstanding_lazy_request = 0; 2707 2708 if (!dev_priv->mm.suspended) { 2709 if (i915_enable_hangcheck) { 2710 callout_reset(&dev_priv->hangcheck_timer, 2711 DRM_I915_HANGCHECK_PERIOD, i915_hangcheck_elapsed, ring->dev); 2712 } 2713 if (was_empty) 2714 taskqueue_enqueue_timeout(dev_priv->tq, 2715 &dev_priv->mm.retire_task, hz); 2716 } 2717 return (0); 2718 } 2719 2720 static inline void 2721 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 2722 { 2723 struct drm_i915_file_private *file_priv = request->file_priv; 2724 2725 if (!file_priv) 2726 return; 2727 2728 DRM_LOCK_ASSERT(request->ring->dev); 2729 2730 spin_lock(&file_priv->mm.lock); 2731 if (request->file_priv != NULL) { 2732 list_del(&request->client_list); 2733 request->file_priv = NULL; 2734 } 2735 spin_unlock(&file_priv->mm.lock); 2736 } 2737 2738 void 2739 i915_gem_release(struct drm_device *dev, struct drm_file *file) 2740 { 2741 struct drm_i915_file_private *file_priv; 2742 struct drm_i915_gem_request *request; 2743 2744 file_priv = file->driver_priv; 2745 2746 /* Clean up our request list when the client is going away, so that 2747 * later retire_requests won't dereference our soon-to-be-gone 2748 * file_priv. 2749 */ 2750 spin_lock(&file_priv->mm.lock); 2751 while (!list_empty(&file_priv->mm.request_list)) { 2752 request = list_first_entry(&file_priv->mm.request_list, 2753 struct drm_i915_gem_request, 2754 client_list); 2755 list_del(&request->client_list); 2756 request->file_priv = NULL; 2757 } 2758 spin_unlock(&file_priv->mm.lock); 2759 } 2760 2761 static void 2762 i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, 2763 struct intel_ring_buffer *ring) 2764 { 2765 2766 if (ring->dev != NULL) 2767 DRM_LOCK_ASSERT(ring->dev); 2768 2769 while (!list_empty(&ring->request_list)) { 2770 struct drm_i915_gem_request *request; 2771 2772 request = list_first_entry(&ring->request_list, 2773 struct drm_i915_gem_request, list); 2774 2775 list_del(&request->list); 2776 i915_gem_request_remove_from_client(request); 2777 drm_free(request, DRM_I915_GEM); 2778 } 2779 2780 while (!list_empty(&ring->active_list)) { 2781 struct drm_i915_gem_object *obj; 2782 2783 obj = list_first_entry(&ring->active_list, 2784 struct drm_i915_gem_object, ring_list); 2785 2786 obj->base.write_domain = 0; 2787 list_del_init(&obj->gpu_write_list); 2788 i915_gem_object_move_to_inactive(obj); 2789 } 2790 } 2791 2792 static void 2793 i915_gem_reset_fences(struct drm_device *dev) 2794 { 2795 struct drm_i915_private *dev_priv = dev->dev_private; 2796 int i; 2797 2798 for (i = 0; i < dev_priv->num_fence_regs; i++) { 2799 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 2800 struct drm_i915_gem_object *obj = reg->obj; 2801 2802 if (!obj) 2803 continue; 2804 2805 if (obj->tiling_mode) 2806 i915_gem_release_mmap(obj); 2807 2808 reg->obj->fence_reg = I915_FENCE_REG_NONE; 2809 reg->obj->fenced_gpu_access = false; 2810 reg->obj->last_fenced_seqno = 0; 2811 reg->obj->last_fenced_ring = NULL; 2812 i915_gem_clear_fence_reg(dev, reg); 2813 } 2814 } 2815 2816 void 2817 i915_gem_reset(struct drm_device *dev) 2818 { 2819 struct drm_i915_private *dev_priv = dev->dev_private; 2820 struct drm_i915_gem_object *obj; 2821 int i; 2822 2823 for (i = 0; i < I915_NUM_RINGS; i++) 2824 i915_gem_reset_ring_lists(dev_priv, &dev_priv->rings[i]); 2825 2826 /* Remove anything from the flushing lists. The GPU cache is likely 2827 * to be lost on reset along with the data, so simply move the 2828 * lost bo to the inactive list. 2829 */ 2830 while (!list_empty(&dev_priv->mm.flushing_list)) { 2831 obj = list_first_entry(&dev_priv->mm.flushing_list, 2832 struct drm_i915_gem_object, 2833 mm_list); 2834 2835 obj->base.write_domain = 0; 2836 list_del_init(&obj->gpu_write_list); 2837 i915_gem_object_move_to_inactive(obj); 2838 } 2839 2840 /* Move everything out of the GPU domains to ensure we do any 2841 * necessary invalidation upon reuse. 2842 */ 2843 list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { 2844 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 2845 } 2846 2847 /* The fence registers are invalidated so clear them out */ 2848 i915_gem_reset_fences(dev); 2849 } 2850 2851 /** 2852 * This function clears the request list as sequence numbers are passed. 2853 */ 2854 void 2855 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) 2856 { 2857 uint32_t seqno; 2858 int i; 2859 2860 if (list_empty(&ring->request_list)) 2861 return; 2862 2863 seqno = ring->get_seqno(ring); 2864 for (i = 0; i < DRM_ARRAY_SIZE(ring->sync_seqno); i++) 2865 if (seqno >= ring->sync_seqno[i]) 2866 ring->sync_seqno[i] = 0; 2867 2868 while (!list_empty(&ring->request_list)) { 2869 struct drm_i915_gem_request *request; 2870 2871 request = list_first_entry(&ring->request_list, 2872 struct drm_i915_gem_request, 2873 list); 2874 2875 if (!i915_seqno_passed(seqno, request->seqno)) 2876 break; 2877 2878 ring->last_retired_head = request->tail; 2879 2880 list_del(&request->list); 2881 i915_gem_request_remove_from_client(request); 2882 drm_free(request, DRM_I915_GEM); 2883 } 2884 2885 /* Move any buffers on the active list that are no longer referenced 2886 * by the ringbuffer to the flushing/inactive lists as appropriate. 2887 */ 2888 while (!list_empty(&ring->active_list)) { 2889 struct drm_i915_gem_object *obj; 2890 2891 obj = list_first_entry(&ring->active_list, 2892 struct drm_i915_gem_object, 2893 ring_list); 2894 2895 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno)) 2896 break; 2897 2898 if (obj->base.write_domain != 0) 2899 i915_gem_object_move_to_flushing(obj); 2900 else 2901 i915_gem_object_move_to_inactive(obj); 2902 } 2903 2904 if (ring->trace_irq_seqno && 2905 i915_seqno_passed(seqno, ring->trace_irq_seqno)) { 2906 lockmgr(&ring->irq_lock, LK_EXCLUSIVE); 2907 ring->irq_put(ring); 2908 lockmgr(&ring->irq_lock, LK_RELEASE); 2909 ring->trace_irq_seqno = 0; 2910 } 2911 } 2912 2913 void 2914 i915_gem_retire_requests(struct drm_device *dev) 2915 { 2916 drm_i915_private_t *dev_priv = dev->dev_private; 2917 struct drm_i915_gem_object *obj, *next; 2918 int i; 2919 2920 if (!list_empty(&dev_priv->mm.deferred_free_list)) { 2921 list_for_each_entry_safe(obj, next, 2922 &dev_priv->mm.deferred_free_list, mm_list) 2923 i915_gem_free_object_tail(obj); 2924 } 2925 2926 for (i = 0; i < I915_NUM_RINGS; i++) 2927 i915_gem_retire_requests_ring(&dev_priv->rings[i]); 2928 } 2929 2930 static int 2931 sandybridge_write_fence_reg(struct drm_i915_gem_object *obj, 2932 struct intel_ring_buffer *pipelined) 2933 { 2934 struct drm_device *dev = obj->base.dev; 2935 drm_i915_private_t *dev_priv = dev->dev_private; 2936 u32 size = obj->gtt_space->size; 2937 int regnum = obj->fence_reg; 2938 uint64_t val; 2939 2940 val = (uint64_t)((obj->gtt_offset + size - 4096) & 2941 0xfffff000) << 32; 2942 val |= obj->gtt_offset & 0xfffff000; 2943 val |= (uint64_t)((obj->stride / 128) - 1) << 2944 SANDYBRIDGE_FENCE_PITCH_SHIFT; 2945 2946 if (obj->tiling_mode == I915_TILING_Y) 2947 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2948 val |= I965_FENCE_REG_VALID; 2949 2950 if (pipelined) { 2951 int ret = intel_ring_begin(pipelined, 6); 2952 if (ret) 2953 return ret; 2954 2955 intel_ring_emit(pipelined, MI_NOOP); 2956 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 2957 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8); 2958 intel_ring_emit(pipelined, (u32)val); 2959 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4); 2960 intel_ring_emit(pipelined, (u32)(val >> 32)); 2961 intel_ring_advance(pipelined); 2962 } else 2963 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val); 2964 2965 return 0; 2966 } 2967 2968 static int 2969 i965_write_fence_reg(struct drm_i915_gem_object *obj, 2970 struct intel_ring_buffer *pipelined) 2971 { 2972 struct drm_device *dev = obj->base.dev; 2973 drm_i915_private_t *dev_priv = dev->dev_private; 2974 u32 size = obj->gtt_space->size; 2975 int regnum = obj->fence_reg; 2976 uint64_t val; 2977 2978 val = (uint64_t)((obj->gtt_offset + size - 4096) & 2979 0xfffff000) << 32; 2980 val |= obj->gtt_offset & 0xfffff000; 2981 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 2982 if (obj->tiling_mode == I915_TILING_Y) 2983 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2984 val |= I965_FENCE_REG_VALID; 2985 2986 if (pipelined) { 2987 int ret = intel_ring_begin(pipelined, 6); 2988 if (ret) 2989 return ret; 2990 2991 intel_ring_emit(pipelined, MI_NOOP); 2992 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 2993 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8); 2994 intel_ring_emit(pipelined, (u32)val); 2995 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4); 2996 intel_ring_emit(pipelined, (u32)(val >> 32)); 2997 intel_ring_advance(pipelined); 2998 } else 2999 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val); 3000 3001 return 0; 3002 } 3003 3004 static int 3005 i915_write_fence_reg(struct drm_i915_gem_object *obj, 3006 struct intel_ring_buffer *pipelined) 3007 { 3008 struct drm_device *dev = obj->base.dev; 3009 drm_i915_private_t *dev_priv = dev->dev_private; 3010 u32 size = obj->gtt_space->size; 3011 u32 fence_reg, val, pitch_val; 3012 int tile_width; 3013 3014 if ((obj->gtt_offset & ~I915_FENCE_START_MASK) || 3015 (size & -size) != size || (obj->gtt_offset & (size - 1))) { 3016 kprintf( 3017 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", 3018 obj->gtt_offset, obj->map_and_fenceable, size); 3019 return -EINVAL; 3020 } 3021 3022 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) 3023 tile_width = 128; 3024 else 3025 tile_width = 512; 3026 3027 /* Note: pitch better be a power of two tile widths */ 3028 pitch_val = obj->stride / tile_width; 3029 pitch_val = ffs(pitch_val) - 1; 3030 3031 val = obj->gtt_offset; 3032 if (obj->tiling_mode == I915_TILING_Y) 3033 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3034 val |= I915_FENCE_SIZE_BITS(size); 3035 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3036 val |= I830_FENCE_REG_VALID; 3037 3038 fence_reg = obj->fence_reg; 3039 if (fence_reg < 8) 3040 fence_reg = FENCE_REG_830_0 + fence_reg * 4; 3041 else 3042 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 3043 3044 if (pipelined) { 3045 int ret = intel_ring_begin(pipelined, 4); 3046 if (ret) 3047 return ret; 3048 3049 intel_ring_emit(pipelined, MI_NOOP); 3050 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 3051 intel_ring_emit(pipelined, fence_reg); 3052 intel_ring_emit(pipelined, val); 3053 intel_ring_advance(pipelined); 3054 } else 3055 I915_WRITE(fence_reg, val); 3056 3057 return 0; 3058 } 3059 3060 static int 3061 i830_write_fence_reg(struct drm_i915_gem_object *obj, 3062 struct intel_ring_buffer *pipelined) 3063 { 3064 struct drm_device *dev = obj->base.dev; 3065 drm_i915_private_t *dev_priv = dev->dev_private; 3066 u32 size = obj->gtt_space->size; 3067 int regnum = obj->fence_reg; 3068 uint32_t val; 3069 uint32_t pitch_val; 3070 3071 if ((obj->gtt_offset & ~I830_FENCE_START_MASK) || 3072 (size & -size) != size || (obj->gtt_offset & (size - 1))) { 3073 kprintf( 3074 "object 0x%08x not 512K or pot-size 0x%08x aligned\n", 3075 obj->gtt_offset, size); 3076 return -EINVAL; 3077 } 3078 3079 pitch_val = obj->stride / 128; 3080 pitch_val = ffs(pitch_val) - 1; 3081 3082 val = obj->gtt_offset; 3083 if (obj->tiling_mode == I915_TILING_Y) 3084 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 3085 val |= I830_FENCE_SIZE_BITS(size); 3086 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 3087 val |= I830_FENCE_REG_VALID; 3088 3089 if (pipelined) { 3090 int ret = intel_ring_begin(pipelined, 4); 3091 if (ret) 3092 return ret; 3093 3094 intel_ring_emit(pipelined, MI_NOOP); 3095 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 3096 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4); 3097 intel_ring_emit(pipelined, val); 3098 intel_ring_advance(pipelined); 3099 } else 3100 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val); 3101 3102 return 0; 3103 } 3104 3105 static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno) 3106 { 3107 return i915_seqno_passed(ring->get_seqno(ring), seqno); 3108 } 3109 3110 static int 3111 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj, 3112 struct intel_ring_buffer *pipelined) 3113 { 3114 int ret; 3115 3116 if (obj->fenced_gpu_access) { 3117 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 3118 ret = i915_gem_flush_ring(obj->last_fenced_ring, 0, 3119 obj->base.write_domain); 3120 if (ret) 3121 return ret; 3122 } 3123 3124 obj->fenced_gpu_access = false; 3125 } 3126 3127 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) { 3128 if (!ring_passed_seqno(obj->last_fenced_ring, 3129 obj->last_fenced_seqno)) { 3130 ret = i915_wait_request(obj->last_fenced_ring, 3131 obj->last_fenced_seqno, 3132 true); 3133 if (ret) 3134 return ret; 3135 } 3136 3137 obj->last_fenced_seqno = 0; 3138 obj->last_fenced_ring = NULL; 3139 } 3140 3141 /* Ensure that all CPU reads are completed before installing a fence 3142 * and all writes before removing the fence. 3143 */ 3144 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) 3145 cpu_mfence(); 3146 3147 return 0; 3148 } 3149 3150 int 3151 i915_gem_object_put_fence(struct drm_i915_gem_object *obj) 3152 { 3153 int ret; 3154 3155 if (obj->tiling_mode) 3156 i915_gem_release_mmap(obj); 3157 3158 ret = i915_gem_object_flush_fence(obj, NULL); 3159 if (ret) 3160 return ret; 3161 3162 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3163 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3164 3165 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0) 3166 kprintf("%s: pin_count %d\n", __func__, 3167 dev_priv->fence_regs[obj->fence_reg].pin_count); 3168 i915_gem_clear_fence_reg(obj->base.dev, 3169 &dev_priv->fence_regs[obj->fence_reg]); 3170 3171 obj->fence_reg = I915_FENCE_REG_NONE; 3172 } 3173 3174 return 0; 3175 } 3176 3177 static struct drm_i915_fence_reg * 3178 i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined) 3179 { 3180 struct drm_i915_private *dev_priv = dev->dev_private; 3181 struct drm_i915_fence_reg *reg, *first, *avail; 3182 int i; 3183 3184 /* First try to find a free reg */ 3185 avail = NULL; 3186 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 3187 reg = &dev_priv->fence_regs[i]; 3188 if (!reg->obj) 3189 return reg; 3190 3191 if (!reg->pin_count) 3192 avail = reg; 3193 } 3194 3195 if (avail == NULL) 3196 return NULL; 3197 3198 /* None available, try to steal one or wait for a user to finish */ 3199 avail = first = NULL; 3200 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { 3201 if (reg->pin_count) 3202 continue; 3203 3204 if (first == NULL) 3205 first = reg; 3206 3207 if (!pipelined || 3208 !reg->obj->last_fenced_ring || 3209 reg->obj->last_fenced_ring == pipelined) { 3210 avail = reg; 3211 break; 3212 } 3213 } 3214 3215 if (avail == NULL) 3216 avail = first; 3217 3218 return avail; 3219 } 3220 3221 int 3222 i915_gem_object_get_fence(struct drm_i915_gem_object *obj, 3223 struct intel_ring_buffer *pipelined) 3224 { 3225 struct drm_device *dev = obj->base.dev; 3226 struct drm_i915_private *dev_priv = dev->dev_private; 3227 struct drm_i915_fence_reg *reg; 3228 int ret; 3229 3230 pipelined = NULL; 3231 ret = 0; 3232 3233 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3234 reg = &dev_priv->fence_regs[obj->fence_reg]; 3235 list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 3236 3237 if (obj->tiling_changed) { 3238 ret = i915_gem_object_flush_fence(obj, pipelined); 3239 if (ret) 3240 return ret; 3241 3242 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 3243 pipelined = NULL; 3244 3245 if (pipelined) { 3246 reg->setup_seqno = 3247 i915_gem_next_request_seqno(pipelined); 3248 obj->last_fenced_seqno = reg->setup_seqno; 3249 obj->last_fenced_ring = pipelined; 3250 } 3251 3252 goto update; 3253 } 3254 3255 if (!pipelined) { 3256 if (reg->setup_seqno) { 3257 if (!ring_passed_seqno(obj->last_fenced_ring, 3258 reg->setup_seqno)) { 3259 ret = i915_wait_request( 3260 obj->last_fenced_ring, 3261 reg->setup_seqno, 3262 true); 3263 if (ret) 3264 return ret; 3265 } 3266 3267 reg->setup_seqno = 0; 3268 } 3269 } else if (obj->last_fenced_ring && 3270 obj->last_fenced_ring != pipelined) { 3271 ret = i915_gem_object_flush_fence(obj, pipelined); 3272 if (ret) 3273 return ret; 3274 } 3275 3276 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 3277 pipelined = NULL; 3278 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined")); 3279 3280 if (obj->tiling_changed) { 3281 if (pipelined) { 3282 reg->setup_seqno = 3283 i915_gem_next_request_seqno(pipelined); 3284 obj->last_fenced_seqno = reg->setup_seqno; 3285 obj->last_fenced_ring = pipelined; 3286 } 3287 goto update; 3288 } 3289 3290 return 0; 3291 } 3292 3293 reg = i915_find_fence_reg(dev, pipelined); 3294 if (reg == NULL) 3295 return -EDEADLK; 3296 3297 ret = i915_gem_object_flush_fence(obj, pipelined); 3298 if (ret) 3299 return ret; 3300 3301 if (reg->obj) { 3302 struct drm_i915_gem_object *old = reg->obj; 3303 3304 drm_gem_object_reference(&old->base); 3305 3306 if (old->tiling_mode) 3307 i915_gem_release_mmap(old); 3308 3309 ret = i915_gem_object_flush_fence(old, pipelined); 3310 if (ret) { 3311 drm_gem_object_unreference(&old->base); 3312 return ret; 3313 } 3314 3315 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0) 3316 pipelined = NULL; 3317 3318 old->fence_reg = I915_FENCE_REG_NONE; 3319 old->last_fenced_ring = pipelined; 3320 old->last_fenced_seqno = 3321 pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 3322 3323 drm_gem_object_unreference(&old->base); 3324 } else if (obj->last_fenced_seqno == 0) 3325 pipelined = NULL; 3326 3327 reg->obj = obj; 3328 list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 3329 obj->fence_reg = reg - dev_priv->fence_regs; 3330 obj->last_fenced_ring = pipelined; 3331 3332 reg->setup_seqno = 3333 pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 3334 obj->last_fenced_seqno = reg->setup_seqno; 3335 3336 update: 3337 obj->tiling_changed = false; 3338 switch (INTEL_INFO(dev)->gen) { 3339 case 7: 3340 case 6: 3341 ret = sandybridge_write_fence_reg(obj, pipelined); 3342 break; 3343 case 5: 3344 case 4: 3345 ret = i965_write_fence_reg(obj, pipelined); 3346 break; 3347 case 3: 3348 ret = i915_write_fence_reg(obj, pipelined); 3349 break; 3350 case 2: 3351 ret = i830_write_fence_reg(obj, pipelined); 3352 break; 3353 } 3354 3355 return ret; 3356 } 3357 3358 static void 3359 i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg) 3360 { 3361 drm_i915_private_t *dev_priv = dev->dev_private; 3362 uint32_t fence_reg = reg - dev_priv->fence_regs; 3363 3364 switch (INTEL_INFO(dev)->gen) { 3365 case 7: 3366 case 6: 3367 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0); 3368 break; 3369 case 5: 3370 case 4: 3371 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0); 3372 break; 3373 case 3: 3374 if (fence_reg >= 8) 3375 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 3376 else 3377 case 2: 3378 fence_reg = FENCE_REG_830_0 + fence_reg * 4; 3379 3380 I915_WRITE(fence_reg, 0); 3381 break; 3382 } 3383 3384 list_del_init(®->lru_list); 3385 reg->obj = NULL; 3386 reg->setup_seqno = 0; 3387 reg->pin_count = 0; 3388 } 3389 3390 int 3391 i915_gem_init_object(struct drm_gem_object *obj) 3392 { 3393 3394 kprintf("i915_gem_init_object called\n"); 3395 return (0); 3396 } 3397 3398 static bool 3399 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) 3400 { 3401 3402 return (obj->gtt_space && !obj->active && obj->pin_count == 0); 3403 } 3404 3405 static void 3406 i915_gem_retire_task_handler(void *arg, int pending) 3407 { 3408 drm_i915_private_t *dev_priv; 3409 struct drm_device *dev; 3410 bool idle; 3411 int i; 3412 3413 dev_priv = arg; 3414 dev = dev_priv->dev; 3415 3416 /* Come back later if the device is busy... */ 3417 if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) { 3418 taskqueue_enqueue_timeout(dev_priv->tq, 3419 &dev_priv->mm.retire_task, hz); 3420 return; 3421 } 3422 3423 i915_gem_retire_requests(dev); 3424 3425 /* Send a periodic flush down the ring so we don't hold onto GEM 3426 * objects indefinitely. 3427 */ 3428 idle = true; 3429 for (i = 0; i < I915_NUM_RINGS; i++) { 3430 struct intel_ring_buffer *ring = &dev_priv->rings[i]; 3431 3432 if (!list_empty(&ring->gpu_write_list)) { 3433 struct drm_i915_gem_request *request; 3434 int ret; 3435 3436 ret = i915_gem_flush_ring(ring, 3437 0, I915_GEM_GPU_DOMAINS); 3438 request = kmalloc(sizeof(*request), DRM_I915_GEM, 3439 M_WAITOK | M_ZERO); 3440 if (ret || request == NULL || 3441 i915_add_request(ring, NULL, request)) 3442 drm_free(request, DRM_I915_GEM); 3443 } 3444 3445 idle &= list_empty(&ring->request_list); 3446 } 3447 3448 if (!dev_priv->mm.suspended && !idle) 3449 taskqueue_enqueue_timeout(dev_priv->tq, 3450 &dev_priv->mm.retire_task, hz); 3451 3452 DRM_UNLOCK(dev); 3453 } 3454 3455 void 3456 i915_gem_lastclose(struct drm_device *dev) 3457 { 3458 int ret; 3459 3460 if (drm_core_check_feature(dev, DRIVER_MODESET)) 3461 return; 3462 3463 ret = i915_gem_idle(dev); 3464 if (ret != 0) 3465 DRM_ERROR("failed to idle hardware: %d\n", ret); 3466 } 3467 3468 static int 3469 i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align) 3470 { 3471 drm_i915_private_t *dev_priv; 3472 struct drm_i915_gem_phys_object *phys_obj; 3473 int ret; 3474 3475 dev_priv = dev->dev_private; 3476 if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0) 3477 return (0); 3478 3479 phys_obj = kmalloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM, 3480 M_WAITOK | M_ZERO); 3481 3482 phys_obj->id = id; 3483 3484 phys_obj->handle = drm_pci_alloc(dev, size, align, ~0); 3485 if (phys_obj->handle == NULL) { 3486 ret = -ENOMEM; 3487 goto free_obj; 3488 } 3489 pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, 3490 size / PAGE_SIZE, PAT_WRITE_COMBINING); 3491 3492 dev_priv->mm.phys_objs[id - 1] = phys_obj; 3493 3494 return (0); 3495 3496 free_obj: 3497 drm_free(phys_obj, DRM_I915_GEM); 3498 return (ret); 3499 } 3500 3501 static void 3502 i915_gem_free_phys_object(struct drm_device *dev, int id) 3503 { 3504 drm_i915_private_t *dev_priv; 3505 struct drm_i915_gem_phys_object *phys_obj; 3506 3507 dev_priv = dev->dev_private; 3508 if (dev_priv->mm.phys_objs[id - 1] == NULL) 3509 return; 3510 3511 phys_obj = dev_priv->mm.phys_objs[id - 1]; 3512 if (phys_obj->cur_obj != NULL) 3513 i915_gem_detach_phys_object(dev, phys_obj->cur_obj); 3514 3515 drm_pci_free(dev, phys_obj->handle); 3516 drm_free(phys_obj, DRM_I915_GEM); 3517 dev_priv->mm.phys_objs[id - 1] = NULL; 3518 } 3519 3520 void 3521 i915_gem_free_all_phys_object(struct drm_device *dev) 3522 { 3523 int i; 3524 3525 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) 3526 i915_gem_free_phys_object(dev, i); 3527 } 3528 3529 void 3530 i915_gem_detach_phys_object(struct drm_device *dev, 3531 struct drm_i915_gem_object *obj) 3532 { 3533 vm_page_t m; 3534 struct sf_buf *sf; 3535 char *vaddr, *dst; 3536 int i, page_count; 3537 3538 if (obj->phys_obj == NULL) 3539 return; 3540 vaddr = obj->phys_obj->handle->vaddr; 3541 3542 page_count = obj->base.size / PAGE_SIZE; 3543 VM_OBJECT_LOCK(obj->base.vm_obj); 3544 for (i = 0; i < page_count; i++) { 3545 m = i915_gem_wire_page(obj->base.vm_obj, i); 3546 if (m == NULL) 3547 continue; /* XXX */ 3548 3549 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3550 sf = sf_buf_alloc(m); 3551 if (sf != NULL) { 3552 dst = (char *)sf_buf_kva(sf); 3553 memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE); 3554 sf_buf_free(sf); 3555 } 3556 drm_clflush_pages(&m, 1); 3557 3558 VM_OBJECT_LOCK(obj->base.vm_obj); 3559 vm_page_reference(m); 3560 vm_page_dirty(m); 3561 vm_page_busy_wait(m, FALSE, "i915gem"); 3562 vm_page_unwire(m, 0); 3563 vm_page_wakeup(m); 3564 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3565 } 3566 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3567 intel_gtt_chipset_flush(); 3568 3569 obj->phys_obj->cur_obj = NULL; 3570 obj->phys_obj = NULL; 3571 } 3572 3573 int 3574 i915_gem_attach_phys_object(struct drm_device *dev, 3575 struct drm_i915_gem_object *obj, int id, int align) 3576 { 3577 drm_i915_private_t *dev_priv; 3578 vm_page_t m; 3579 struct sf_buf *sf; 3580 char *dst, *src; 3581 int i, page_count, ret; 3582 3583 if (id > I915_MAX_PHYS_OBJECT) 3584 return (-EINVAL); 3585 3586 if (obj->phys_obj != NULL) { 3587 if (obj->phys_obj->id == id) 3588 return (0); 3589 i915_gem_detach_phys_object(dev, obj); 3590 } 3591 3592 dev_priv = dev->dev_private; 3593 if (dev_priv->mm.phys_objs[id - 1] == NULL) { 3594 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align); 3595 if (ret != 0) { 3596 DRM_ERROR("failed to init phys object %d size: %zu\n", 3597 id, obj->base.size); 3598 return (ret); 3599 } 3600 } 3601 3602 /* bind to the object */ 3603 obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; 3604 obj->phys_obj->cur_obj = obj; 3605 3606 page_count = obj->base.size / PAGE_SIZE; 3607 3608 VM_OBJECT_LOCK(obj->base.vm_obj); 3609 ret = 0; 3610 for (i = 0; i < page_count; i++) { 3611 m = i915_gem_wire_page(obj->base.vm_obj, i); 3612 if (m == NULL) { 3613 ret = -EIO; 3614 break; 3615 } 3616 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3617 sf = sf_buf_alloc(m); 3618 src = (char *)sf_buf_kva(sf); 3619 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i); 3620 memcpy(dst, src, PAGE_SIZE); 3621 sf_buf_free(sf); 3622 3623 VM_OBJECT_LOCK(obj->base.vm_obj); 3624 3625 vm_page_reference(m); 3626 vm_page_busy_wait(m, FALSE, "i915gem"); 3627 vm_page_unwire(m, 0); 3628 vm_page_wakeup(m); 3629 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3630 } 3631 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3632 3633 return (0); 3634 } 3635 3636 static int 3637 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj, 3638 uint64_t data_ptr, uint64_t offset, uint64_t size, 3639 struct drm_file *file_priv) 3640 { 3641 char *user_data, *vaddr; 3642 int ret; 3643 3644 vaddr = (char *)obj->phys_obj->handle->vaddr + offset; 3645 user_data = (char *)(uintptr_t)data_ptr; 3646 3647 if (copyin_nofault(user_data, vaddr, size) != 0) { 3648 /* The physical object once assigned is fixed for the lifetime 3649 * of the obj, so we can safely drop the lock and continue 3650 * to access vaddr. 3651 */ 3652 DRM_UNLOCK(dev); 3653 ret = -copyin(user_data, vaddr, size); 3654 DRM_LOCK(dev); 3655 if (ret != 0) 3656 return (ret); 3657 } 3658 3659 intel_gtt_chipset_flush(); 3660 return (0); 3661 } 3662 3663 static int 3664 i915_gpu_is_active(struct drm_device *dev) 3665 { 3666 drm_i915_private_t *dev_priv; 3667 3668 dev_priv = dev->dev_private; 3669 return (!list_empty(&dev_priv->mm.flushing_list) || 3670 !list_empty(&dev_priv->mm.active_list)); 3671 } 3672 3673 static void 3674 i915_gem_lowmem(void *arg) 3675 { 3676 struct drm_device *dev; 3677 struct drm_i915_private *dev_priv; 3678 struct drm_i915_gem_object *obj, *next; 3679 int cnt, cnt_fail, cnt_total; 3680 3681 dev = arg; 3682 dev_priv = dev->dev_private; 3683 3684 if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) 3685 return; 3686 3687 rescan: 3688 /* first scan for clean buffers */ 3689 i915_gem_retire_requests(dev); 3690 3691 cnt_total = cnt_fail = cnt = 0; 3692 3693 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3694 mm_list) { 3695 if (i915_gem_object_is_purgeable(obj)) { 3696 if (i915_gem_object_unbind(obj) != 0) 3697 cnt_total++; 3698 } else 3699 cnt_total++; 3700 } 3701 3702 /* second pass, evict/count anything still on the inactive list */ 3703 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3704 mm_list) { 3705 if (i915_gem_object_unbind(obj) == 0) 3706 cnt++; 3707 else 3708 cnt_fail++; 3709 } 3710 3711 if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) { 3712 /* 3713 * We are desperate for pages, so as a last resort, wait 3714 * for the GPU to finish and discard whatever we can. 3715 * This has a dramatic impact to reduce the number of 3716 * OOM-killer events whilst running the GPU aggressively. 3717 */ 3718 if (i915_gpu_idle(dev, true) == 0) 3719 goto rescan; 3720 } 3721 DRM_UNLOCK(dev); 3722 } 3723 3724 void 3725 i915_gem_unload(struct drm_device *dev) 3726 { 3727 struct drm_i915_private *dev_priv; 3728 3729 dev_priv = dev->dev_private; 3730 EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem); 3731 } 3732