1 /*- 2 * Copyright © 2008 Intel Corporation 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice (including the next 12 * paragraph) shall be included in all copies or substantial portions of the 13 * Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 21 * IN THE SOFTWARE. 22 * 23 * Authors: 24 * Eric Anholt <eric@anholt.net> 25 * 26 * Copyright (c) 2011 The FreeBSD Foundation 27 * All rights reserved. 28 * 29 * This software was developed by Konstantin Belousov under sponsorship from 30 * the FreeBSD Foundation. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 41 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 42 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 43 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 44 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 45 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 46 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 47 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 48 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 49 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 50 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 51 * SUCH DAMAGE. 52 * 53 * $FreeBSD: head/sys/dev/drm2/i915/i915_gem.c 253497 2013-07-20 13:52:40Z kib $ 54 */ 55 56 #include <sys/resourcevar.h> 57 #include <sys/sfbuf.h> 58 59 #include <drm/drmP.h> 60 #include <drm/i915_drm.h> 61 #include "i915_drv.h" 62 #include "intel_drv.h" 63 #include "intel_ringbuffer.h" 64 65 static void i915_gem_object_flush_cpu_write_domain( 66 struct drm_i915_gem_object *obj); 67 static uint32_t i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, 68 int tiling_mode); 69 static uint32_t i915_gem_get_gtt_alignment(struct drm_device *dev, 70 uint32_t size, int tiling_mode); 71 static int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 72 unsigned alignment, bool map_and_fenceable); 73 static int i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 74 int flags); 75 static void i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj); 76 static int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, 77 bool write); 78 static void i915_gem_object_set_to_full_cpu_read_domain( 79 struct drm_i915_gem_object *obj); 80 static int i915_gem_object_set_cpu_read_domain_range( 81 struct drm_i915_gem_object *obj, uint64_t offset, uint64_t size); 82 static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj); 83 static void i915_gem_object_truncate(struct drm_i915_gem_object *obj); 84 static int i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj); 85 static bool i915_gem_object_is_inactive(struct drm_i915_gem_object *obj); 86 static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj); 87 static vm_page_t i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex); 88 static void i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 89 uint32_t flush_domains); 90 static void i915_gem_clear_fence_reg(struct drm_device *dev, 91 struct drm_i915_fence_reg *reg); 92 static void i915_gem_reset_fences(struct drm_device *dev); 93 static int i915_gem_phys_pwrite(struct drm_device *dev, 94 struct drm_i915_gem_object *obj, uint64_t data_ptr, uint64_t offset, 95 uint64_t size, struct drm_file *file_priv); 96 static void i915_gem_lowmem(void *arg); 97 98 MALLOC_DEFINE(DRM_I915_GEM, "i915gem", "Allocations from i915 gem"); 99 long i915_gem_wired_pages_cnt; 100 101 static void 102 i915_gem_info_add_obj(struct drm_i915_private *dev_priv, size_t size) 103 { 104 105 dev_priv->mm.object_count++; 106 dev_priv->mm.object_memory += size; 107 } 108 109 static void 110 i915_gem_info_remove_obj(struct drm_i915_private *dev_priv, size_t size) 111 { 112 113 dev_priv->mm.object_count--; 114 dev_priv->mm.object_memory -= size; 115 } 116 117 static int 118 i915_gem_wait_for_error(struct drm_device *dev) 119 { 120 struct drm_i915_private *dev_priv = dev->dev_private; 121 int ret; 122 123 if (!atomic_read(&dev_priv->mm.wedged)) 124 return 0; 125 126 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE); 127 while (dev_priv->error_completion == 0) { 128 ret = -lksleep(&dev_priv->error_completion, 129 &dev_priv->error_completion_lock, PCATCH, "915wco", 0); 130 if (ret != 0) { 131 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 132 return (ret); 133 } 134 } 135 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 136 137 if (atomic_read(&dev_priv->mm.wedged)) { 138 /* GPU is hung, bump the completion count to account for 139 * the token we just consumed so that we never hit zero and 140 * end up waiting upon a subsequent completion event that 141 * will never happen. 142 */ 143 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE); 144 dev_priv->error_completion++; 145 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 146 } 147 return 0; 148 } 149 150 int 151 i915_mutex_lock_interruptible(struct drm_device *dev) 152 { 153 struct drm_i915_private *dev_priv; 154 int ret; 155 156 dev_priv = dev->dev_private; 157 ret = i915_gem_wait_for_error(dev); 158 if (ret != 0) 159 return (ret); 160 161 /* 162 * interruptible shall it be. might indeed be if dev_lock is 163 * changed to sx 164 */ 165 ret = lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_SLEEPFAIL); 166 if (ret != 0) 167 return (-ret); 168 169 return (0); 170 } 171 172 173 static void 174 i915_gem_free_object_tail(struct drm_i915_gem_object *obj) 175 { 176 struct drm_device *dev; 177 drm_i915_private_t *dev_priv; 178 int ret; 179 180 dev = obj->base.dev; 181 dev_priv = dev->dev_private; 182 183 ret = i915_gem_object_unbind(obj); 184 if (ret == -ERESTART) { 185 list_move(&obj->mm_list, &dev_priv->mm.deferred_free_list); 186 return; 187 } 188 189 drm_gem_free_mmap_offset(&obj->base); 190 drm_gem_object_release(&obj->base); 191 i915_gem_info_remove_obj(dev_priv, obj->base.size); 192 193 drm_free(obj->page_cpu_valid, DRM_I915_GEM); 194 drm_free(obj->bit_17, DRM_I915_GEM); 195 drm_free(obj, DRM_I915_GEM); 196 } 197 198 void 199 i915_gem_free_object(struct drm_gem_object *gem_obj) 200 { 201 struct drm_i915_gem_object *obj; 202 struct drm_device *dev; 203 204 obj = to_intel_bo(gem_obj); 205 dev = obj->base.dev; 206 207 while (obj->pin_count > 0) 208 i915_gem_object_unpin(obj); 209 210 if (obj->phys_obj != NULL) 211 i915_gem_detach_phys_object(dev, obj); 212 213 i915_gem_free_object_tail(obj); 214 } 215 216 int 217 i915_gem_do_init(struct drm_device *dev, unsigned long start, 218 unsigned long mappable_end, unsigned long end) 219 { 220 drm_i915_private_t *dev_priv; 221 unsigned long mappable; 222 int error; 223 224 dev_priv = dev->dev_private; 225 mappable = min(end, mappable_end) - start; 226 227 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start); 228 229 dev_priv->mm.gtt_start = start; 230 dev_priv->mm.gtt_mappable_end = mappable_end; 231 dev_priv->mm.gtt_end = end; 232 dev_priv->mm.gtt_total = end - start; 233 dev_priv->mm.mappable_gtt_total = mappable; 234 235 /* Take over this portion of the GTT */ 236 intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE); 237 device_printf(dev->dev, 238 "taking over the fictitious range 0x%lx-0x%lx\n", 239 dev->agp->base + start, dev->agp->base + start + mappable); 240 error = -vm_phys_fictitious_reg_range(dev->agp->base + start, 241 dev->agp->base + start + mappable, VM_MEMATTR_WRITE_COMBINING); 242 return (error); 243 } 244 245 int 246 i915_gem_init_ioctl(struct drm_device *dev, void *data, 247 struct drm_file *file) 248 { 249 struct drm_i915_gem_init *args; 250 drm_i915_private_t *dev_priv; 251 252 dev_priv = dev->dev_private; 253 args = data; 254 255 if (args->gtt_start >= args->gtt_end || 256 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1)) 257 return (-EINVAL); 258 259 /* 260 * XXXKIB. The second-time initialization should be guarded 261 * against. 262 */ 263 lockmgr(&dev->dev_lock, LK_EXCLUSIVE|LK_RETRY|LK_CANRECURSE); 264 i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end); 265 lockmgr(&dev->dev_lock, LK_RELEASE); 266 267 return 0; 268 } 269 270 int 271 i915_gem_idle(struct drm_device *dev) 272 { 273 drm_i915_private_t *dev_priv; 274 int ret; 275 276 dev_priv = dev->dev_private; 277 if (dev_priv->mm.suspended) 278 return (0); 279 280 ret = i915_gpu_idle(dev, true); 281 if (ret != 0) 282 return (ret); 283 284 /* Under UMS, be paranoid and evict. */ 285 if (!drm_core_check_feature(dev, DRIVER_MODESET)) { 286 ret = i915_gem_evict_inactive(dev, false); 287 if (ret != 0) 288 return ret; 289 } 290 291 i915_gem_reset_fences(dev); 292 293 /* Hack! Don't let anybody do execbuf while we don't control the chip. 294 * We need to replace this with a semaphore, or something. 295 * And not confound mm.suspended! 296 */ 297 dev_priv->mm.suspended = 1; 298 del_timer_sync(&dev_priv->hangcheck_timer); 299 300 i915_kernel_lost_context(dev); 301 i915_gem_cleanup_ringbuffer(dev); 302 303 /* Cancel the retire work handler, which should be idle now. */ 304 cancel_delayed_work_sync(&dev_priv->mm.retire_work); 305 306 return (ret); 307 } 308 309 void 310 i915_gem_init_swizzling(struct drm_device *dev) 311 { 312 drm_i915_private_t *dev_priv; 313 314 dev_priv = dev->dev_private; 315 316 if (INTEL_INFO(dev)->gen < 5 || 317 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE) 318 return; 319 320 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) | 321 DISP_TILE_SURFACE_SWIZZLING); 322 323 if (IS_GEN5(dev)) 324 return; 325 326 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL); 327 if (IS_GEN6(dev)) 328 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB)); 329 else 330 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB)); 331 } 332 333 int 334 i915_gem_init_hw(struct drm_device *dev) 335 { 336 drm_i915_private_t *dev_priv; 337 int ret; 338 339 dev_priv = dev->dev_private; 340 341 i915_gem_init_swizzling(dev); 342 343 ret = intel_init_render_ring_buffer(dev); 344 if (ret != 0) 345 return (ret); 346 347 if (HAS_BSD(dev)) { 348 ret = intel_init_bsd_ring_buffer(dev); 349 if (ret != 0) 350 goto cleanup_render_ring; 351 } 352 353 if (HAS_BLT(dev)) { 354 ret = intel_init_blt_ring_buffer(dev); 355 if (ret != 0) 356 goto cleanup_bsd_ring; 357 } 358 359 dev_priv->next_seqno = 1; 360 i915_gem_init_ppgtt(dev); 361 return (0); 362 363 cleanup_bsd_ring: 364 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]); 365 cleanup_render_ring: 366 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]); 367 return (ret); 368 } 369 370 int 371 i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data, 372 struct drm_file *file) 373 { 374 struct drm_i915_private *dev_priv; 375 struct drm_i915_gem_get_aperture *args; 376 struct drm_i915_gem_object *obj; 377 size_t pinned; 378 379 dev_priv = dev->dev_private; 380 args = data; 381 382 if (!(dev->driver->driver_features & DRIVER_GEM)) 383 return (-ENODEV); 384 385 pinned = 0; 386 DRM_LOCK(dev); 387 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list) 388 pinned += obj->gtt_space->size; 389 DRM_UNLOCK(dev); 390 391 args->aper_size = dev_priv->mm.gtt_total; 392 args->aper_available_size = args->aper_size - pinned; 393 394 return (0); 395 } 396 397 int 398 i915_gem_object_pin(struct drm_i915_gem_object *obj, uint32_t alignment, 399 bool map_and_fenceable) 400 { 401 struct drm_device *dev; 402 struct drm_i915_private *dev_priv; 403 int ret; 404 405 dev = obj->base.dev; 406 dev_priv = dev->dev_private; 407 408 KASSERT(obj->pin_count != DRM_I915_GEM_OBJECT_MAX_PIN_COUNT, 409 ("Max pin count")); 410 411 if (obj->gtt_space != NULL) { 412 if ((alignment && obj->gtt_offset & (alignment - 1)) || 413 (map_and_fenceable && !obj->map_and_fenceable)) { 414 DRM_DEBUG("bo is already pinned with incorrect alignment:" 415 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d," 416 " obj->map_and_fenceable=%d\n", 417 obj->gtt_offset, alignment, 418 map_and_fenceable, 419 obj->map_and_fenceable); 420 ret = i915_gem_object_unbind(obj); 421 if (ret != 0) 422 return (ret); 423 } 424 } 425 426 if (obj->gtt_space == NULL) { 427 ret = i915_gem_object_bind_to_gtt(obj, alignment, 428 map_and_fenceable); 429 if (ret) 430 return (ret); 431 } 432 433 if (obj->pin_count++ == 0 && !obj->active) 434 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 435 obj->pin_mappable |= map_and_fenceable; 436 437 #if 1 438 KIB_NOTYET(); 439 #else 440 WARN_ON(i915_verify_lists(dev)); 441 #endif 442 return (0); 443 } 444 445 void 446 i915_gem_object_unpin(struct drm_i915_gem_object *obj) 447 { 448 struct drm_device *dev; 449 drm_i915_private_t *dev_priv; 450 451 dev = obj->base.dev; 452 dev_priv = dev->dev_private; 453 454 #if 1 455 KIB_NOTYET(); 456 #else 457 WARN_ON(i915_verify_lists(dev)); 458 #endif 459 460 KASSERT(obj->pin_count != 0, ("zero pin count")); 461 KASSERT(obj->gtt_space != NULL, ("No gtt mapping")); 462 463 if (--obj->pin_count == 0) { 464 if (!obj->active) 465 list_move_tail(&obj->mm_list, 466 &dev_priv->mm.inactive_list); 467 obj->pin_mappable = false; 468 } 469 #if 1 470 KIB_NOTYET(); 471 #else 472 WARN_ON(i915_verify_lists(dev)); 473 #endif 474 } 475 476 int 477 i915_gem_pin_ioctl(struct drm_device *dev, void *data, 478 struct drm_file *file) 479 { 480 struct drm_i915_gem_pin *args; 481 struct drm_i915_gem_object *obj; 482 struct drm_gem_object *gobj; 483 int ret; 484 485 args = data; 486 487 ret = i915_mutex_lock_interruptible(dev); 488 if (ret != 0) 489 return ret; 490 491 gobj = drm_gem_object_lookup(dev, file, args->handle); 492 if (gobj == NULL) { 493 ret = -ENOENT; 494 goto unlock; 495 } 496 obj = to_intel_bo(gobj); 497 498 if (obj->madv != I915_MADV_WILLNEED) { 499 DRM_ERROR("Attempting to pin a purgeable buffer\n"); 500 ret = -EINVAL; 501 goto out; 502 } 503 504 if (obj->pin_filp != NULL && obj->pin_filp != file) { 505 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n", 506 args->handle); 507 ret = -EINVAL; 508 goto out; 509 } 510 511 obj->user_pin_count++; 512 obj->pin_filp = file; 513 if (obj->user_pin_count == 1) { 514 ret = i915_gem_object_pin(obj, args->alignment, true); 515 if (ret != 0) 516 goto out; 517 } 518 519 /* XXX - flush the CPU caches for pinned objects 520 * as the X server doesn't manage domains yet 521 */ 522 i915_gem_object_flush_cpu_write_domain(obj); 523 args->offset = obj->gtt_offset; 524 out: 525 drm_gem_object_unreference(&obj->base); 526 unlock: 527 DRM_UNLOCK(dev); 528 return (ret); 529 } 530 531 int 532 i915_gem_unpin_ioctl(struct drm_device *dev, void *data, 533 struct drm_file *file) 534 { 535 struct drm_i915_gem_pin *args; 536 struct drm_i915_gem_object *obj; 537 int ret; 538 539 args = data; 540 ret = i915_mutex_lock_interruptible(dev); 541 if (ret != 0) 542 return (ret); 543 544 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 545 if (&obj->base == NULL) { 546 ret = -ENOENT; 547 goto unlock; 548 } 549 550 if (obj->pin_filp != file) { 551 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n", 552 args->handle); 553 ret = -EINVAL; 554 goto out; 555 } 556 obj->user_pin_count--; 557 if (obj->user_pin_count == 0) { 558 obj->pin_filp = NULL; 559 i915_gem_object_unpin(obj); 560 } 561 562 out: 563 drm_gem_object_unreference(&obj->base); 564 unlock: 565 DRM_UNLOCK(dev); 566 return (ret); 567 } 568 569 int 570 i915_gem_busy_ioctl(struct drm_device *dev, void *data, 571 struct drm_file *file) 572 { 573 struct drm_i915_gem_busy *args; 574 struct drm_i915_gem_object *obj; 575 struct drm_i915_gem_request *request; 576 int ret; 577 578 args = data; 579 580 ret = i915_mutex_lock_interruptible(dev); 581 if (ret != 0) 582 return ret; 583 584 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 585 if (&obj->base == NULL) { 586 ret = -ENOENT; 587 goto unlock; 588 } 589 590 args->busy = obj->active; 591 if (args->busy) { 592 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 593 ret = i915_gem_flush_ring(obj->ring, 594 0, obj->base.write_domain); 595 } else if (obj->ring->outstanding_lazy_request == 596 obj->last_rendering_seqno) { 597 request = kmalloc(sizeof(*request), DRM_I915_GEM, 598 M_WAITOK | M_ZERO); 599 ret = i915_add_request(obj->ring, NULL, request); 600 if (ret != 0) 601 drm_free(request, DRM_I915_GEM); 602 } 603 604 i915_gem_retire_requests_ring(obj->ring); 605 args->busy = obj->active; 606 } 607 608 drm_gem_object_unreference(&obj->base); 609 unlock: 610 DRM_UNLOCK(dev); 611 return (ret); 612 } 613 614 /* Throttle our rendering by waiting until the ring has completed our requests 615 * emitted over 20 msec ago. 616 * 617 * Note that if we were to use the current jiffies each time around the loop, 618 * we wouldn't escape the function with any frames outstanding if the time to 619 * render a frame was over 20ms. 620 * 621 * This should get us reasonable parallelism between CPU and GPU but also 622 * relatively low latency when blocking on a particular request to finish. 623 */ 624 static int 625 i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file) 626 { 627 struct drm_i915_private *dev_priv = dev->dev_private; 628 struct drm_i915_file_private *file_priv = file->driver_priv; 629 unsigned long recent_enough = ticks - (20 * hz / 1000); 630 struct drm_i915_gem_request *request; 631 struct intel_ring_buffer *ring = NULL; 632 u32 seqno = 0; 633 int ret; 634 635 dev_priv = dev->dev_private; 636 if (atomic_read(&dev_priv->mm.wedged)) 637 return -EIO; 638 639 recent_enough = ticks - (20 * hz / 1000); 640 ring = NULL; 641 seqno = 0; 642 643 spin_lock(&file_priv->mm.lock); 644 list_for_each_entry(request, &file_priv->mm.request_list, client_list) { 645 if (time_after_eq(request->emitted_jiffies, recent_enough)) 646 break; 647 648 ring = request->ring; 649 seqno = request->seqno; 650 } 651 spin_unlock(&file_priv->mm.lock); 652 653 if (seqno == 0) 654 return 0; 655 656 ret = 0; 657 lockmgr(&ring->irq_lock, LK_EXCLUSIVE); 658 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { 659 if (ring->irq_get(ring)) { 660 while (ret == 0 && 661 !(i915_seqno_passed(ring->get_seqno(ring), seqno) || 662 atomic_read(&dev_priv->mm.wedged))) 663 ret = -lksleep(ring, &ring->irq_lock, PCATCH, 664 "915thr", 0); 665 ring->irq_put(ring); 666 if (ret == 0 && atomic_read(&dev_priv->mm.wedged)) 667 ret = -EIO; 668 } else if (_intel_wait_for(dev, 669 i915_seqno_passed(ring->get_seqno(ring), seqno) || 670 atomic_read(&dev_priv->mm.wedged), 3000, 0, "915rtr")) { 671 ret = -EBUSY; 672 } 673 } 674 lockmgr(&ring->irq_lock, LK_RELEASE); 675 676 if (ret == 0) 677 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0); 678 679 return ret; 680 } 681 682 int 683 i915_gem_throttle_ioctl(struct drm_device *dev, void *data, 684 struct drm_file *file_priv) 685 { 686 687 return (i915_gem_ring_throttle(dev, file_priv)); 688 } 689 690 int 691 i915_gem_madvise_ioctl(struct drm_device *dev, void *data, 692 struct drm_file *file_priv) 693 { 694 struct drm_i915_gem_madvise *args = data; 695 struct drm_i915_gem_object *obj; 696 int ret; 697 698 switch (args->madv) { 699 case I915_MADV_DONTNEED: 700 case I915_MADV_WILLNEED: 701 break; 702 default: 703 return -EINVAL; 704 } 705 706 ret = i915_mutex_lock_interruptible(dev); 707 if (ret) 708 return ret; 709 710 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle)); 711 if (&obj->base == NULL) { 712 ret = -ENOENT; 713 goto unlock; 714 } 715 716 if (obj->pin_count) { 717 ret = -EINVAL; 718 goto out; 719 } 720 721 if (obj->madv != __I915_MADV_PURGED) 722 obj->madv = args->madv; 723 724 /* if the object is no longer attached, discard its backing storage */ 725 if (i915_gem_object_is_purgeable(obj) && obj->pages == NULL) 726 i915_gem_object_truncate(obj); 727 728 args->retained = obj->madv != __I915_MADV_PURGED; 729 730 out: 731 drm_gem_object_unreference(&obj->base); 732 unlock: 733 DRM_UNLOCK(dev); 734 return ret; 735 } 736 737 void 738 i915_gem_cleanup_ringbuffer(struct drm_device *dev) 739 { 740 drm_i915_private_t *dev_priv; 741 int i; 742 743 dev_priv = dev->dev_private; 744 for (i = 0; i < I915_NUM_RINGS; i++) 745 intel_cleanup_ring_buffer(&dev_priv->ring[i]); 746 } 747 748 int 749 i915_gem_entervt_ioctl(struct drm_device *dev, void *data, 750 struct drm_file *file_priv) 751 { 752 drm_i915_private_t *dev_priv = dev->dev_private; 753 int ret; 754 755 if (drm_core_check_feature(dev, DRIVER_MODESET)) 756 return 0; 757 758 if (atomic_read(&dev_priv->mm.wedged)) { 759 DRM_ERROR("Reenabling wedged hardware, good luck\n"); 760 atomic_set(&dev_priv->mm.wedged, 0); 761 } 762 763 DRM_LOCK(dev); 764 dev_priv->mm.suspended = 0; 765 766 ret = i915_gem_init_hw(dev); 767 if (ret != 0) { 768 DRM_UNLOCK(dev); 769 return ret; 770 } 771 772 KASSERT(list_empty(&dev_priv->mm.active_list), ("active list")); 773 DRM_UNLOCK(dev); 774 775 ret = drm_irq_install(dev); 776 if (ret) 777 goto cleanup_ringbuffer; 778 779 return 0; 780 781 cleanup_ringbuffer: 782 DRM_LOCK(dev); 783 i915_gem_cleanup_ringbuffer(dev); 784 dev_priv->mm.suspended = 1; 785 DRM_UNLOCK(dev); 786 787 return ret; 788 } 789 790 int 791 i915_gem_leavevt_ioctl(struct drm_device *dev, void *data, 792 struct drm_file *file_priv) 793 { 794 795 if (drm_core_check_feature(dev, DRIVER_MODESET)) 796 return 0; 797 798 drm_irq_uninstall(dev); 799 return (i915_gem_idle(dev)); 800 } 801 802 int 803 i915_gem_create(struct drm_file *file, struct drm_device *dev, uint64_t size, 804 uint32_t *handle_p) 805 { 806 struct drm_i915_gem_object *obj; 807 uint32_t handle; 808 int ret; 809 810 size = roundup(size, PAGE_SIZE); 811 if (size == 0) 812 return (-EINVAL); 813 814 obj = i915_gem_alloc_object(dev, size); 815 if (obj == NULL) 816 return (-ENOMEM); 817 818 handle = 0; 819 ret = drm_gem_handle_create(file, &obj->base, &handle); 820 if (ret != 0) { 821 drm_gem_object_release(&obj->base); 822 i915_gem_info_remove_obj(dev->dev_private, obj->base.size); 823 drm_free(obj, DRM_I915_GEM); 824 return (-ret); 825 } 826 827 /* drop reference from allocate - handle holds it now */ 828 drm_gem_object_unreference(&obj->base); 829 *handle_p = handle; 830 return (0); 831 } 832 833 int 834 i915_gem_dumb_create(struct drm_file *file, struct drm_device *dev, 835 struct drm_mode_create_dumb *args) 836 { 837 838 /* have to work out size/pitch and return them */ 839 args->pitch = roundup2(args->width * ((args->bpp + 7) / 8), 64); 840 args->size = args->pitch * args->height; 841 return (i915_gem_create(file, dev, args->size, &args->handle)); 842 } 843 844 int 845 i915_gem_dumb_destroy(struct drm_file *file, struct drm_device *dev, 846 uint32_t handle) 847 { 848 849 return (drm_gem_handle_delete(file, handle)); 850 } 851 852 int 853 i915_gem_create_ioctl(struct drm_device *dev, void *data, 854 struct drm_file *file) 855 { 856 struct drm_i915_gem_create *args = data; 857 858 return (i915_gem_create(file, dev, args->size, &args->handle)); 859 } 860 861 static inline void vm_page_reference(vm_page_t m) 862 { 863 vm_page_flag_set(m, PG_REFERENCED); 864 } 865 866 static int 867 i915_gem_swap_io(struct drm_device *dev, struct drm_i915_gem_object *obj, 868 uint64_t data_ptr, uint64_t size, uint64_t offset, enum uio_rw rw, 869 struct drm_file *file) 870 { 871 vm_object_t vm_obj; 872 vm_page_t m; 873 struct sf_buf *sf; 874 vm_offset_t mkva; 875 vm_pindex_t obj_pi; 876 int cnt, do_bit17_swizzling, length, obj_po, ret, swizzled_po; 877 878 if (obj->gtt_offset != 0 && rw == UIO_READ) 879 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj); 880 else 881 do_bit17_swizzling = 0; 882 883 obj->dirty = 1; 884 vm_obj = obj->base.vm_obj; 885 ret = 0; 886 887 VM_OBJECT_LOCK(vm_obj); 888 vm_object_pip_add(vm_obj, 1); 889 while (size > 0) { 890 obj_pi = OFF_TO_IDX(offset); 891 obj_po = offset & PAGE_MASK; 892 893 m = i915_gem_wire_page(vm_obj, obj_pi); 894 VM_OBJECT_UNLOCK(vm_obj); 895 896 sf = sf_buf_alloc(m); 897 mkva = sf_buf_kva(sf); 898 length = min(size, PAGE_SIZE - obj_po); 899 while (length > 0) { 900 if (do_bit17_swizzling && 901 (VM_PAGE_TO_PHYS(m) & (1 << 17)) != 0) { 902 cnt = roundup2(obj_po + 1, 64); 903 cnt = min(cnt - obj_po, length); 904 swizzled_po = obj_po ^ 64; 905 } else { 906 cnt = length; 907 swizzled_po = obj_po; 908 } 909 if (rw == UIO_READ) 910 ret = -copyout_nofault( 911 (char *)mkva + swizzled_po, 912 (void *)(uintptr_t)data_ptr, cnt); 913 else 914 ret = -copyin_nofault( 915 (void *)(uintptr_t)data_ptr, 916 (char *)mkva + swizzled_po, cnt); 917 if (ret != 0) 918 break; 919 data_ptr += cnt; 920 size -= cnt; 921 length -= cnt; 922 offset += cnt; 923 obj_po += cnt; 924 } 925 sf_buf_free(sf); 926 VM_OBJECT_LOCK(vm_obj); 927 if (rw == UIO_WRITE) 928 vm_page_dirty(m); 929 vm_page_reference(m); 930 vm_page_busy_wait(m, FALSE, "i915gem"); 931 vm_page_unwire(m, 1); 932 vm_page_wakeup(m); 933 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 934 935 if (ret != 0) 936 break; 937 } 938 vm_object_pip_wakeup(vm_obj); 939 VM_OBJECT_UNLOCK(vm_obj); 940 941 return (ret); 942 } 943 944 static int 945 i915_gem_gtt_write(struct drm_device *dev, struct drm_i915_gem_object *obj, 946 uint64_t data_ptr, uint64_t size, uint64_t offset, struct drm_file *file) 947 { 948 vm_offset_t mkva; 949 int ret; 950 951 /* 952 * Pass the unaligned physical address and size to pmap_mapdev_attr() 953 * so it can properly calculate whether an extra page needs to be 954 * mapped or not to cover the requested range. The function will 955 * add the page offset into the returned mkva for us. 956 */ 957 mkva = (vm_offset_t)pmap_mapdev_attr(dev->agp->base + obj->gtt_offset + 958 offset, size, PAT_WRITE_COMBINING); 959 ret = -copyin_nofault((void *)(uintptr_t)data_ptr, (char *)mkva, size); 960 pmap_unmapdev(mkva, size); 961 return (ret); 962 } 963 964 static int 965 i915_gem_obj_io(struct drm_device *dev, uint32_t handle, uint64_t data_ptr, 966 uint64_t size, uint64_t offset, enum uio_rw rw, struct drm_file *file) 967 { 968 struct drm_i915_gem_object *obj; 969 vm_page_t *ma; 970 vm_offset_t start, end; 971 int npages, ret; 972 973 if (size == 0) 974 return (0); 975 start = trunc_page(data_ptr); 976 end = round_page(data_ptr + size); 977 npages = howmany(end - start, PAGE_SIZE); 978 ma = kmalloc(npages * sizeof(vm_page_t), DRM_I915_GEM, M_WAITOK | 979 M_ZERO); 980 npages = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map, 981 (vm_offset_t)data_ptr, size, 982 (rw == UIO_READ ? VM_PROT_WRITE : 0 ) | VM_PROT_READ, ma, npages); 983 if (npages == -1) { 984 ret = -EFAULT; 985 goto free_ma; 986 } 987 988 ret = i915_mutex_lock_interruptible(dev); 989 if (ret != 0) 990 goto unlocked; 991 992 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 993 if (&obj->base == NULL) { 994 ret = -ENOENT; 995 goto unlock; 996 } 997 if (offset > obj->base.size || size > obj->base.size - offset) { 998 ret = -EINVAL; 999 goto out; 1000 } 1001 1002 if (rw == UIO_READ) { 1003 ret = i915_gem_object_set_cpu_read_domain_range(obj, 1004 offset, size); 1005 if (ret != 0) 1006 goto out; 1007 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 1008 UIO_READ, file); 1009 } else { 1010 if (obj->phys_obj) { 1011 ret = i915_gem_phys_pwrite(dev, obj, data_ptr, offset, 1012 size, file); 1013 } else if (obj->gtt_space && 1014 obj->base.write_domain != I915_GEM_DOMAIN_CPU) { 1015 ret = i915_gem_object_pin(obj, 0, true); 1016 if (ret != 0) 1017 goto out; 1018 ret = i915_gem_object_set_to_gtt_domain(obj, true); 1019 if (ret != 0) 1020 goto out_unpin; 1021 ret = i915_gem_object_put_fence(obj); 1022 if (ret != 0) 1023 goto out_unpin; 1024 ret = i915_gem_gtt_write(dev, obj, data_ptr, size, 1025 offset, file); 1026 out_unpin: 1027 i915_gem_object_unpin(obj); 1028 } else { 1029 ret = i915_gem_object_set_to_cpu_domain(obj, true); 1030 if (ret != 0) 1031 goto out; 1032 ret = i915_gem_swap_io(dev, obj, data_ptr, size, offset, 1033 UIO_WRITE, file); 1034 } 1035 } 1036 out: 1037 drm_gem_object_unreference(&obj->base); 1038 unlock: 1039 DRM_UNLOCK(dev); 1040 unlocked: 1041 vm_page_unhold_pages(ma, npages); 1042 free_ma: 1043 drm_free(ma, DRM_I915_GEM); 1044 return (ret); 1045 } 1046 1047 int 1048 i915_gem_pread_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 1049 { 1050 struct drm_i915_gem_pread *args; 1051 1052 args = data; 1053 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 1054 args->offset, UIO_READ, file)); 1055 } 1056 1057 int 1058 i915_gem_pwrite_ioctl(struct drm_device *dev, void *data, struct drm_file *file) 1059 { 1060 struct drm_i915_gem_pwrite *args; 1061 1062 args = data; 1063 return (i915_gem_obj_io(dev, args->handle, args->data_ptr, args->size, 1064 args->offset, UIO_WRITE, file)); 1065 } 1066 1067 int 1068 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data, 1069 struct drm_file *file) 1070 { 1071 struct drm_i915_gem_set_domain *args; 1072 struct drm_i915_gem_object *obj; 1073 uint32_t read_domains; 1074 uint32_t write_domain; 1075 int ret; 1076 1077 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1078 return (-ENODEV); 1079 1080 args = data; 1081 read_domains = args->read_domains; 1082 write_domain = args->write_domain; 1083 1084 if ((write_domain & I915_GEM_GPU_DOMAINS) != 0 || 1085 (read_domains & I915_GEM_GPU_DOMAINS) != 0 || 1086 (write_domain != 0 && read_domains != write_domain)) 1087 return (-EINVAL); 1088 1089 ret = i915_mutex_lock_interruptible(dev); 1090 if (ret != 0) 1091 return (ret); 1092 1093 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1094 if (&obj->base == NULL) { 1095 ret = -ENOENT; 1096 goto unlock; 1097 } 1098 1099 if ((read_domains & I915_GEM_DOMAIN_GTT) != 0) { 1100 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0); 1101 if (ret == -EINVAL) 1102 ret = 0; 1103 } else 1104 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0); 1105 1106 drm_gem_object_unreference(&obj->base); 1107 unlock: 1108 DRM_UNLOCK(dev); 1109 return (ret); 1110 } 1111 1112 int 1113 i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data, 1114 struct drm_file *file) 1115 { 1116 struct drm_i915_gem_sw_finish *args; 1117 struct drm_i915_gem_object *obj; 1118 int ret; 1119 1120 args = data; 1121 ret = 0; 1122 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1123 return (ENODEV); 1124 ret = i915_mutex_lock_interruptible(dev); 1125 if (ret != 0) 1126 return (ret); 1127 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle)); 1128 if (&obj->base == NULL) { 1129 ret = -ENOENT; 1130 goto unlock; 1131 } 1132 if (obj->pin_count != 0) 1133 i915_gem_object_flush_cpu_write_domain(obj); 1134 drm_gem_object_unreference(&obj->base); 1135 unlock: 1136 DRM_UNLOCK(dev); 1137 return (ret); 1138 } 1139 1140 int 1141 i915_gem_mmap_ioctl(struct drm_device *dev, void *data, 1142 struct drm_file *file) 1143 { 1144 struct drm_i915_gem_mmap *args; 1145 struct drm_gem_object *obj; 1146 struct proc *p; 1147 vm_map_t map; 1148 vm_offset_t addr; 1149 vm_size_t size; 1150 int error, rv; 1151 1152 args = data; 1153 1154 if ((dev->driver->driver_features & DRIVER_GEM) == 0) 1155 return (-ENODEV); 1156 1157 obj = drm_gem_object_lookup(dev, file, args->handle); 1158 if (obj == NULL) 1159 return (-ENOENT); 1160 error = 0; 1161 if (args->size == 0) 1162 goto out; 1163 p = curproc; 1164 map = &p->p_vmspace->vm_map; 1165 size = round_page(args->size); 1166 PROC_LOCK(p); 1167 if (map->size + size > p->p_rlimit[RLIMIT_VMEM].rlim_cur) { 1168 PROC_UNLOCK(p); 1169 error = ENOMEM; 1170 goto out; 1171 } 1172 PROC_UNLOCK(p); 1173 1174 addr = 0; 1175 vm_object_hold(obj->vm_obj); 1176 vm_object_reference_locked(obj->vm_obj); 1177 vm_object_drop(obj->vm_obj); 1178 DRM_UNLOCK(dev); 1179 rv = vm_map_find(map, obj->vm_obj, args->offset, &addr, args->size, 1180 PAGE_SIZE, /* align */ 1181 TRUE, /* fitit */ 1182 VM_MAPTYPE_NORMAL, /* maptype */ 1183 VM_PROT_READ | VM_PROT_WRITE, /* prot */ 1184 VM_PROT_READ | VM_PROT_WRITE, /* max */ 1185 MAP_SHARED /* cow */); 1186 if (rv != KERN_SUCCESS) { 1187 vm_object_deallocate(obj->vm_obj); 1188 error = -vm_mmap_to_errno(rv); 1189 } else { 1190 args->addr_ptr = (uint64_t)addr; 1191 } 1192 DRM_LOCK(dev); 1193 out: 1194 drm_gem_object_unreference(obj); 1195 return (error); 1196 } 1197 1198 static int 1199 i915_gem_pager_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot, 1200 vm_ooffset_t foff, struct ucred *cred, u_short *color) 1201 { 1202 1203 *color = 0; /* XXXKIB */ 1204 return (0); 1205 } 1206 1207 int i915_intr_pf; 1208 1209 static int 1210 i915_gem_pager_fault(vm_object_t vm_obj, vm_ooffset_t offset, int prot, 1211 vm_page_t *mres) 1212 { 1213 struct drm_gem_object *gem_obj; 1214 struct drm_i915_gem_object *obj; 1215 struct drm_device *dev; 1216 drm_i915_private_t *dev_priv; 1217 vm_page_t m, oldm; 1218 int cause, ret; 1219 bool write; 1220 1221 gem_obj = vm_obj->handle; 1222 obj = to_intel_bo(gem_obj); 1223 dev = obj->base.dev; 1224 dev_priv = dev->dev_private; 1225 #if 0 1226 write = (prot & VM_PROT_WRITE) != 0; 1227 #else 1228 write = true; 1229 #endif 1230 vm_object_pip_add(vm_obj, 1); 1231 1232 /* 1233 * Remove the placeholder page inserted by vm_fault() from the 1234 * object before dropping the object lock. If 1235 * i915_gem_release_mmap() is active in parallel on this gem 1236 * object, then it owns the drm device sx and might find the 1237 * placeholder already. Then, since the page is busy, 1238 * i915_gem_release_mmap() sleeps waiting for the busy state 1239 * of the page cleared. We will be not able to acquire drm 1240 * device lock until i915_gem_release_mmap() is able to make a 1241 * progress. 1242 */ 1243 if (*mres != NULL) { 1244 oldm = *mres; 1245 vm_page_remove(oldm); 1246 *mres = NULL; 1247 } else 1248 oldm = NULL; 1249 retry: 1250 VM_OBJECT_UNLOCK(vm_obj); 1251 unlocked_vmobj: 1252 cause = ret = 0; 1253 m = NULL; 1254 1255 if (i915_intr_pf) { 1256 ret = i915_mutex_lock_interruptible(dev); 1257 if (ret != 0) { 1258 cause = 10; 1259 goto out; 1260 } 1261 } else 1262 DRM_LOCK(dev); 1263 1264 /* 1265 * Since the object lock was dropped, other thread might have 1266 * faulted on the same GTT address and instantiated the 1267 * mapping for the page. Recheck. 1268 */ 1269 VM_OBJECT_LOCK(vm_obj); 1270 m = vm_page_lookup(vm_obj, OFF_TO_IDX(offset)); 1271 if (m != NULL) { 1272 if ((m->flags & PG_BUSY) != 0) { 1273 DRM_UNLOCK(dev); 1274 #if 0 /* XXX */ 1275 vm_page_sleep(m, "915pee"); 1276 #endif 1277 goto retry; 1278 } 1279 goto have_page; 1280 } else 1281 VM_OBJECT_UNLOCK(vm_obj); 1282 1283 /* Now bind it into the GTT if needed */ 1284 if (!obj->map_and_fenceable) { 1285 ret = i915_gem_object_unbind(obj); 1286 if (ret != 0) { 1287 cause = 20; 1288 goto unlock; 1289 } 1290 } 1291 if (!obj->gtt_space) { 1292 ret = i915_gem_object_bind_to_gtt(obj, 0, true); 1293 if (ret != 0) { 1294 cause = 30; 1295 goto unlock; 1296 } 1297 1298 ret = i915_gem_object_set_to_gtt_domain(obj, write); 1299 if (ret != 0) { 1300 cause = 40; 1301 goto unlock; 1302 } 1303 } 1304 1305 if (obj->tiling_mode == I915_TILING_NONE) 1306 ret = i915_gem_object_put_fence(obj); 1307 else 1308 ret = i915_gem_object_get_fence(obj, NULL); 1309 if (ret != 0) { 1310 cause = 50; 1311 goto unlock; 1312 } 1313 1314 if (i915_gem_object_is_inactive(obj)) 1315 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 1316 1317 obj->fault_mappable = true; 1318 VM_OBJECT_LOCK(vm_obj); 1319 m = vm_phys_fictitious_to_vm_page(dev->agp->base + obj->gtt_offset + 1320 offset); 1321 if (m == NULL) { 1322 cause = 60; 1323 ret = -EFAULT; 1324 goto unlock; 1325 } 1326 KASSERT((m->flags & PG_FICTITIOUS) != 0, 1327 ("not fictitious %p", m)); 1328 KASSERT(m->wire_count == 1, ("wire_count not 1 %p", m)); 1329 1330 if ((m->flags & PG_BUSY) != 0) { 1331 DRM_UNLOCK(dev); 1332 #if 0 /* XXX */ 1333 vm_page_sleep(m, "915pbs"); 1334 #endif 1335 goto retry; 1336 } 1337 m->valid = VM_PAGE_BITS_ALL; 1338 vm_page_insert(m, vm_obj, OFF_TO_IDX(offset)); 1339 have_page: 1340 *mres = m; 1341 vm_page_busy_try(m, false); 1342 1343 DRM_UNLOCK(dev); 1344 if (oldm != NULL) { 1345 vm_page_free(oldm); 1346 } 1347 vm_object_pip_wakeup(vm_obj); 1348 return (VM_PAGER_OK); 1349 1350 unlock: 1351 DRM_UNLOCK(dev); 1352 out: 1353 KASSERT(ret != 0, ("i915_gem_pager_fault: wrong return")); 1354 if (ret == -EAGAIN || ret == -EIO || ret == -EINTR) { 1355 goto unlocked_vmobj; 1356 } 1357 VM_OBJECT_LOCK(vm_obj); 1358 vm_object_pip_wakeup(vm_obj); 1359 return (VM_PAGER_ERROR); 1360 } 1361 1362 static void 1363 i915_gem_pager_dtor(void *handle) 1364 { 1365 struct drm_gem_object *obj; 1366 struct drm_device *dev; 1367 1368 obj = handle; 1369 dev = obj->dev; 1370 1371 DRM_LOCK(dev); 1372 drm_gem_free_mmap_offset(obj); 1373 i915_gem_release_mmap(to_intel_bo(obj)); 1374 drm_gem_object_unreference(obj); 1375 DRM_UNLOCK(dev); 1376 } 1377 1378 struct cdev_pager_ops i915_gem_pager_ops = { 1379 .cdev_pg_fault = i915_gem_pager_fault, 1380 .cdev_pg_ctor = i915_gem_pager_ctor, 1381 .cdev_pg_dtor = i915_gem_pager_dtor 1382 }; 1383 1384 int 1385 i915_gem_mmap_gtt(struct drm_file *file, struct drm_device *dev, 1386 uint32_t handle, uint64_t *offset) 1387 { 1388 struct drm_i915_private *dev_priv; 1389 struct drm_i915_gem_object *obj; 1390 int ret; 1391 1392 if (!(dev->driver->driver_features & DRIVER_GEM)) 1393 return (-ENODEV); 1394 1395 dev_priv = dev->dev_private; 1396 1397 ret = i915_mutex_lock_interruptible(dev); 1398 if (ret != 0) 1399 return (ret); 1400 1401 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle)); 1402 if (&obj->base == NULL) { 1403 ret = -ENOENT; 1404 goto unlock; 1405 } 1406 1407 if (obj->base.size > dev_priv->mm.gtt_mappable_end) { 1408 ret = -E2BIG; 1409 goto out; 1410 } 1411 1412 if (obj->madv != I915_MADV_WILLNEED) { 1413 DRM_ERROR("Attempting to mmap a purgeable buffer\n"); 1414 ret = -EINVAL; 1415 goto out; 1416 } 1417 1418 ret = drm_gem_create_mmap_offset(&obj->base); 1419 if (ret != 0) 1420 goto out; 1421 1422 *offset = DRM_GEM_MAPPING_OFF(obj->base.map_list.key) | 1423 DRM_GEM_MAPPING_KEY; 1424 out: 1425 drm_gem_object_unreference(&obj->base); 1426 unlock: 1427 DRM_UNLOCK(dev); 1428 return (ret); 1429 } 1430 1431 int 1432 i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data, 1433 struct drm_file *file) 1434 { 1435 struct drm_i915_private *dev_priv; 1436 struct drm_i915_gem_mmap_gtt *args; 1437 1438 dev_priv = dev->dev_private; 1439 args = data; 1440 1441 return (i915_gem_mmap_gtt(file, dev, args->handle, &args->offset)); 1442 } 1443 1444 struct drm_i915_gem_object * 1445 i915_gem_alloc_object(struct drm_device *dev, size_t size) 1446 { 1447 struct drm_i915_private *dev_priv; 1448 struct drm_i915_gem_object *obj; 1449 1450 dev_priv = dev->dev_private; 1451 1452 obj = kmalloc(sizeof(*obj), DRM_I915_GEM, M_WAITOK | M_ZERO); 1453 1454 if (drm_gem_object_init(dev, &obj->base, size) != 0) { 1455 drm_free(obj, DRM_I915_GEM); 1456 return (NULL); 1457 } 1458 1459 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1460 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1461 1462 if (HAS_LLC(dev)) 1463 obj->cache_level = I915_CACHE_LLC; 1464 else 1465 obj->cache_level = I915_CACHE_NONE; 1466 obj->base.driver_private = NULL; 1467 obj->fence_reg = I915_FENCE_REG_NONE; 1468 INIT_LIST_HEAD(&obj->mm_list); 1469 INIT_LIST_HEAD(&obj->gtt_list); 1470 INIT_LIST_HEAD(&obj->ring_list); 1471 INIT_LIST_HEAD(&obj->exec_list); 1472 INIT_LIST_HEAD(&obj->gpu_write_list); 1473 obj->madv = I915_MADV_WILLNEED; 1474 /* Avoid an unnecessary call to unbind on the first bind. */ 1475 obj->map_and_fenceable = true; 1476 1477 i915_gem_info_add_obj(dev_priv, size); 1478 1479 return (obj); 1480 } 1481 1482 void 1483 i915_gem_clflush_object(struct drm_i915_gem_object *obj) 1484 { 1485 1486 /* If we don't have a page list set up, then we're not pinned 1487 * to GPU, and we can ignore the cache flush because it'll happen 1488 * again at bind time. 1489 */ 1490 if (obj->pages == NULL) 1491 return; 1492 1493 /* If the GPU is snooping the contents of the CPU cache, 1494 * we do not need to manually clear the CPU cache lines. However, 1495 * the caches are only snooped when the render cache is 1496 * flushed/invalidated. As we always have to emit invalidations 1497 * and flushes when moving into and out of the RENDER domain, correct 1498 * snooping behaviour occurs naturally as the result of our domain 1499 * tracking. 1500 */ 1501 if (obj->cache_level != I915_CACHE_NONE) 1502 return; 1503 1504 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE); 1505 } 1506 1507 static void 1508 i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj) 1509 { 1510 uint32_t old_write_domain; 1511 1512 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) 1513 return; 1514 1515 i915_gem_clflush_object(obj); 1516 intel_gtt_chipset_flush(); 1517 old_write_domain = obj->base.write_domain; 1518 obj->base.write_domain = 0; 1519 } 1520 1521 static int 1522 i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj) 1523 { 1524 1525 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0) 1526 return (0); 1527 return (i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain)); 1528 } 1529 1530 static void 1531 i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj) 1532 { 1533 uint32_t old_write_domain; 1534 1535 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT) 1536 return; 1537 1538 cpu_sfence(); 1539 1540 old_write_domain = obj->base.write_domain; 1541 obj->base.write_domain = 0; 1542 } 1543 1544 int 1545 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write) 1546 { 1547 uint32_t old_write_domain, old_read_domains; 1548 int ret; 1549 1550 if (obj->gtt_space == NULL) 1551 return (-EINVAL); 1552 1553 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT) 1554 return 0; 1555 1556 ret = i915_gem_object_flush_gpu_write_domain(obj); 1557 if (ret != 0) 1558 return (ret); 1559 1560 if (obj->pending_gpu_write || write) { 1561 ret = i915_gem_object_wait_rendering(obj); 1562 if (ret != 0) 1563 return (ret); 1564 } 1565 1566 i915_gem_object_flush_cpu_write_domain(obj); 1567 1568 old_write_domain = obj->base.write_domain; 1569 old_read_domains = obj->base.read_domains; 1570 1571 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 1572 ("In GTT write domain")); 1573 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 1574 if (write) { 1575 obj->base.read_domains = I915_GEM_DOMAIN_GTT; 1576 obj->base.write_domain = I915_GEM_DOMAIN_GTT; 1577 obj->dirty = 1; 1578 } 1579 1580 return (0); 1581 } 1582 1583 int 1584 i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj, 1585 enum i915_cache_level cache_level) 1586 { 1587 struct drm_device *dev; 1588 drm_i915_private_t *dev_priv; 1589 int ret; 1590 1591 if (obj->cache_level == cache_level) 1592 return 0; 1593 1594 if (obj->pin_count) { 1595 DRM_DEBUG("can not change the cache level of pinned objects\n"); 1596 return (-EBUSY); 1597 } 1598 1599 dev = obj->base.dev; 1600 dev_priv = dev->dev_private; 1601 if (obj->gtt_space) { 1602 ret = i915_gem_object_finish_gpu(obj); 1603 if (ret != 0) 1604 return (ret); 1605 1606 i915_gem_object_finish_gtt(obj); 1607 1608 /* Before SandyBridge, you could not use tiling or fence 1609 * registers with snooped memory, so relinquish any fences 1610 * currently pointing to our region in the aperture. 1611 */ 1612 if (INTEL_INFO(obj->base.dev)->gen < 6) { 1613 ret = i915_gem_object_put_fence(obj); 1614 if (ret != 0) 1615 return (ret); 1616 } 1617 1618 i915_gem_gtt_rebind_object(obj, cache_level); 1619 if (obj->has_aliasing_ppgtt_mapping) 1620 i915_ppgtt_bind_object(dev_priv->mm.aliasing_ppgtt, 1621 obj, cache_level); 1622 } 1623 1624 if (cache_level == I915_CACHE_NONE) { 1625 u32 old_read_domains, old_write_domain; 1626 1627 /* If we're coming from LLC cached, then we haven't 1628 * actually been tracking whether the data is in the 1629 * CPU cache or not, since we only allow one bit set 1630 * in obj->write_domain and have been skipping the clflushes. 1631 * Just set it to the CPU cache for now. 1632 */ 1633 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1634 ("obj %p in CPU write domain", obj)); 1635 KASSERT((obj->base.read_domains & ~I915_GEM_DOMAIN_CPU) == 0, 1636 ("obj %p in CPU read domain", obj)); 1637 1638 old_read_domains = obj->base.read_domains; 1639 old_write_domain = obj->base.write_domain; 1640 1641 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1642 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1643 1644 } 1645 1646 obj->cache_level = cache_level; 1647 return (0); 1648 } 1649 1650 int 1651 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj, 1652 u32 alignment, struct intel_ring_buffer *pipelined) 1653 { 1654 u32 old_read_domains, old_write_domain; 1655 int ret; 1656 1657 ret = i915_gem_object_flush_gpu_write_domain(obj); 1658 if (ret != 0) 1659 return (ret); 1660 1661 if (pipelined != obj->ring) { 1662 ret = i915_gem_object_wait_rendering(obj); 1663 if (ret == -ERESTART || ret == -EINTR) 1664 return (ret); 1665 } 1666 1667 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE); 1668 if (ret != 0) 1669 return (ret); 1670 1671 ret = i915_gem_object_pin(obj, alignment, true); 1672 if (ret != 0) 1673 return (ret); 1674 1675 i915_gem_object_flush_cpu_write_domain(obj); 1676 1677 old_write_domain = obj->base.write_domain; 1678 old_read_domains = obj->base.read_domains; 1679 1680 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) == 0, 1681 ("obj %p in GTT write domain", obj)); 1682 obj->base.read_domains |= I915_GEM_DOMAIN_GTT; 1683 1684 return (0); 1685 } 1686 1687 int 1688 i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj) 1689 { 1690 int ret; 1691 1692 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0) 1693 return (0); 1694 1695 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 1696 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain); 1697 if (ret != 0) 1698 return (ret); 1699 } 1700 1701 ret = i915_gem_object_wait_rendering(obj); 1702 if (ret != 0) 1703 return (ret); 1704 1705 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 1706 1707 return (0); 1708 } 1709 1710 static int 1711 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write) 1712 { 1713 uint32_t old_write_domain, old_read_domains; 1714 int ret; 1715 1716 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU) 1717 return 0; 1718 1719 ret = i915_gem_object_flush_gpu_write_domain(obj); 1720 if (ret != 0) 1721 return (ret); 1722 1723 ret = i915_gem_object_wait_rendering(obj); 1724 if (ret != 0) 1725 return (ret); 1726 1727 i915_gem_object_flush_gtt_write_domain(obj); 1728 i915_gem_object_set_to_full_cpu_read_domain(obj); 1729 1730 old_write_domain = obj->base.write_domain; 1731 old_read_domains = obj->base.read_domains; 1732 1733 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) { 1734 i915_gem_clflush_object(obj); 1735 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 1736 } 1737 1738 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1739 ("In cpu write domain")); 1740 1741 if (write) { 1742 obj->base.read_domains = I915_GEM_DOMAIN_CPU; 1743 obj->base.write_domain = I915_GEM_DOMAIN_CPU; 1744 } 1745 1746 return (0); 1747 } 1748 1749 static void 1750 i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj) 1751 { 1752 int i; 1753 1754 if (obj->page_cpu_valid == NULL) 1755 return; 1756 1757 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) { 1758 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) { 1759 if (obj->page_cpu_valid[i] != 0) 1760 continue; 1761 drm_clflush_pages(obj->pages + i, 1); 1762 } 1763 } 1764 1765 drm_free(obj->page_cpu_valid, DRM_I915_GEM); 1766 obj->page_cpu_valid = NULL; 1767 } 1768 1769 static int 1770 i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj, 1771 uint64_t offset, uint64_t size) 1772 { 1773 uint32_t old_read_domains; 1774 int i, ret; 1775 1776 if (offset == 0 && size == obj->base.size) 1777 return (i915_gem_object_set_to_cpu_domain(obj, 0)); 1778 1779 ret = i915_gem_object_flush_gpu_write_domain(obj); 1780 if (ret != 0) 1781 return (ret); 1782 ret = i915_gem_object_wait_rendering(obj); 1783 if (ret != 0) 1784 return (ret); 1785 1786 i915_gem_object_flush_gtt_write_domain(obj); 1787 1788 if (obj->page_cpu_valid == NULL && 1789 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0) 1790 return (0); 1791 1792 if (obj->page_cpu_valid == NULL) { 1793 obj->page_cpu_valid = kmalloc(obj->base.size / PAGE_SIZE, 1794 DRM_I915_GEM, M_WAITOK | M_ZERO); 1795 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) 1796 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE); 1797 1798 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE; 1799 i++) { 1800 if (obj->page_cpu_valid[i]) 1801 continue; 1802 drm_clflush_pages(obj->pages + i, 1); 1803 obj->page_cpu_valid[i] = 1; 1804 } 1805 1806 KASSERT((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) == 0, 1807 ("In gpu write domain")); 1808 1809 old_read_domains = obj->base.read_domains; 1810 obj->base.read_domains |= I915_GEM_DOMAIN_CPU; 1811 1812 return (0); 1813 } 1814 1815 static uint32_t 1816 i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode) 1817 { 1818 uint32_t gtt_size; 1819 1820 if (INTEL_INFO(dev)->gen >= 4 || 1821 tiling_mode == I915_TILING_NONE) 1822 return (size); 1823 1824 /* Previous chips need a power-of-two fence region when tiling */ 1825 if (INTEL_INFO(dev)->gen == 3) 1826 gtt_size = 1024*1024; 1827 else 1828 gtt_size = 512*1024; 1829 1830 while (gtt_size < size) 1831 gtt_size <<= 1; 1832 1833 return (gtt_size); 1834 } 1835 1836 /** 1837 * i915_gem_get_gtt_alignment - return required GTT alignment for an object 1838 * @obj: object to check 1839 * 1840 * Return the required GTT alignment for an object, taking into account 1841 * potential fence register mapping. 1842 */ 1843 static uint32_t 1844 i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size, 1845 int tiling_mode) 1846 { 1847 1848 /* 1849 * Minimum alignment is 4k (GTT page size), but might be greater 1850 * if a fence register is needed for the object. 1851 */ 1852 if (INTEL_INFO(dev)->gen >= 4 || 1853 tiling_mode == I915_TILING_NONE) 1854 return (4096); 1855 1856 /* 1857 * Previous chips need to be aligned to the size of the smallest 1858 * fence register that can contain the object. 1859 */ 1860 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 1861 } 1862 1863 uint32_t 1864 i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev, uint32_t size, 1865 int tiling_mode) 1866 { 1867 1868 if (tiling_mode == I915_TILING_NONE) 1869 return (4096); 1870 1871 /* 1872 * Minimum alignment is 4k (GTT page size) for sane hw. 1873 */ 1874 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev)) 1875 return (4096); 1876 1877 /* 1878 * Previous hardware however needs to be aligned to a power-of-two 1879 * tile height. The simplest method for determining this is to reuse 1880 * the power-of-tile object size. 1881 */ 1882 return (i915_gem_get_gtt_size(dev, size, tiling_mode)); 1883 } 1884 1885 static int 1886 i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj, 1887 unsigned alignment, bool map_and_fenceable) 1888 { 1889 struct drm_device *dev; 1890 struct drm_i915_private *dev_priv; 1891 struct drm_mm_node *free_space; 1892 uint32_t size, fence_size, fence_alignment, unfenced_alignment; 1893 bool mappable, fenceable; 1894 int ret; 1895 1896 dev = obj->base.dev; 1897 dev_priv = dev->dev_private; 1898 1899 if (obj->madv != I915_MADV_WILLNEED) { 1900 DRM_ERROR("Attempting to bind a purgeable object\n"); 1901 return (-EINVAL); 1902 } 1903 1904 fence_size = i915_gem_get_gtt_size(dev, obj->base.size, 1905 obj->tiling_mode); 1906 fence_alignment = i915_gem_get_gtt_alignment(dev, obj->base.size, 1907 obj->tiling_mode); 1908 unfenced_alignment = i915_gem_get_unfenced_gtt_alignment(dev, 1909 obj->base.size, obj->tiling_mode); 1910 if (alignment == 0) 1911 alignment = map_and_fenceable ? fence_alignment : 1912 unfenced_alignment; 1913 if (map_and_fenceable && (alignment & (fence_alignment - 1)) != 0) { 1914 DRM_ERROR("Invalid object alignment requested %u\n", alignment); 1915 return (-EINVAL); 1916 } 1917 1918 size = map_and_fenceable ? fence_size : obj->base.size; 1919 1920 /* If the object is bigger than the entire aperture, reject it early 1921 * before evicting everything in a vain attempt to find space. 1922 */ 1923 if (obj->base.size > (map_and_fenceable ? 1924 dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) { 1925 DRM_ERROR( 1926 "Attempting to bind an object larger than the aperture\n"); 1927 return (-E2BIG); 1928 } 1929 1930 search_free: 1931 if (map_and_fenceable) 1932 free_space = drm_mm_search_free_in_range( 1933 &dev_priv->mm.gtt_space, size, alignment, 0, 1934 dev_priv->mm.gtt_mappable_end, 0); 1935 else 1936 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space, 1937 size, alignment, 0); 1938 if (free_space != NULL) { 1939 int color = 0; 1940 if (map_and_fenceable) 1941 obj->gtt_space = drm_mm_get_block_range_generic( 1942 free_space, size, alignment, color, 0, 1943 dev_priv->mm.gtt_mappable_end, 1); 1944 else 1945 obj->gtt_space = drm_mm_get_block_generic(free_space, 1946 size, alignment, color, 1); 1947 } 1948 if (obj->gtt_space == NULL) { 1949 ret = i915_gem_evict_something(dev, size, alignment, 1950 map_and_fenceable); 1951 if (ret != 0) 1952 return (ret); 1953 goto search_free; 1954 } 1955 ret = i915_gem_object_get_pages_gtt(obj, 0); 1956 if (ret != 0) { 1957 drm_mm_put_block(obj->gtt_space); 1958 obj->gtt_space = NULL; 1959 /* 1960 * i915_gem_object_get_pages_gtt() cannot return 1961 * ENOMEM, since we use vm_page_grab(VM_ALLOC_RETRY) 1962 * (which does not support operation without a flag 1963 * anyway). 1964 */ 1965 return (ret); 1966 } 1967 1968 ret = i915_gem_gtt_bind_object(obj); 1969 if (ret != 0) { 1970 i915_gem_object_put_pages_gtt(obj); 1971 drm_mm_put_block(obj->gtt_space); 1972 obj->gtt_space = NULL; 1973 if (i915_gem_evict_everything(dev, false)) 1974 return (ret); 1975 goto search_free; 1976 } 1977 1978 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list); 1979 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 1980 1981 KASSERT((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0, 1982 ("Object in gpu read domain")); 1983 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 1984 ("Object in gpu write domain")); 1985 1986 obj->gtt_offset = obj->gtt_space->start; 1987 1988 fenceable = 1989 obj->gtt_space->size == fence_size && 1990 (obj->gtt_space->start & (fence_alignment - 1)) == 0; 1991 1992 mappable = 1993 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end; 1994 obj->map_and_fenceable = mappable && fenceable; 1995 1996 return (0); 1997 } 1998 1999 static void 2000 i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj) 2001 { 2002 u32 old_write_domain, old_read_domains; 2003 2004 /* Act a barrier for all accesses through the GTT */ 2005 cpu_mfence(); 2006 2007 /* Force a pagefault for domain tracking on next user access */ 2008 i915_gem_release_mmap(obj); 2009 2010 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0) 2011 return; 2012 2013 old_read_domains = obj->base.read_domains; 2014 old_write_domain = obj->base.write_domain; 2015 2016 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT; 2017 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT; 2018 2019 } 2020 2021 int 2022 i915_gem_object_unbind(struct drm_i915_gem_object *obj) 2023 { 2024 drm_i915_private_t *dev_priv; 2025 int ret; 2026 2027 dev_priv = obj->base.dev->dev_private; 2028 ret = 0; 2029 if (obj->gtt_space == NULL) 2030 return (0); 2031 if (obj->pin_count != 0) { 2032 DRM_ERROR("Attempting to unbind pinned buffer\n"); 2033 return (-EINVAL); 2034 } 2035 2036 ret = i915_gem_object_finish_gpu(obj); 2037 if (ret == -ERESTART || ret == -EINTR) 2038 return (ret); 2039 2040 i915_gem_object_finish_gtt(obj); 2041 2042 if (ret == 0) 2043 ret = i915_gem_object_set_to_cpu_domain(obj, 1); 2044 if (ret == -ERESTART || ret == -EINTR) 2045 return (ret); 2046 if (ret != 0) { 2047 i915_gem_clflush_object(obj); 2048 obj->base.read_domains = obj->base.write_domain = 2049 I915_GEM_DOMAIN_CPU; 2050 } 2051 2052 ret = i915_gem_object_put_fence(obj); 2053 if (ret == -ERESTART) 2054 return (ret); 2055 2056 i915_gem_gtt_unbind_object(obj); 2057 if (obj->has_aliasing_ppgtt_mapping) { 2058 i915_ppgtt_unbind_object(dev_priv->mm.aliasing_ppgtt, obj); 2059 obj->has_aliasing_ppgtt_mapping = 0; 2060 } 2061 i915_gem_object_put_pages_gtt(obj); 2062 2063 list_del_init(&obj->gtt_list); 2064 list_del_init(&obj->mm_list); 2065 obj->map_and_fenceable = true; 2066 2067 drm_mm_put_block(obj->gtt_space); 2068 obj->gtt_space = NULL; 2069 obj->gtt_offset = 0; 2070 2071 if (i915_gem_object_is_purgeable(obj)) 2072 i915_gem_object_truncate(obj); 2073 2074 return (ret); 2075 } 2076 2077 static int 2078 i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj, 2079 int flags) 2080 { 2081 struct drm_device *dev; 2082 vm_object_t vm_obj; 2083 vm_page_t m; 2084 int page_count, i, j; 2085 2086 dev = obj->base.dev; 2087 KASSERT(obj->pages == NULL, ("Obj already has pages")); 2088 page_count = obj->base.size / PAGE_SIZE; 2089 obj->pages = kmalloc(page_count * sizeof(vm_page_t), DRM_I915_GEM, 2090 M_WAITOK); 2091 vm_obj = obj->base.vm_obj; 2092 VM_OBJECT_LOCK(vm_obj); 2093 for (i = 0; i < page_count; i++) { 2094 if ((obj->pages[i] = i915_gem_wire_page(vm_obj, i)) == NULL) 2095 goto failed; 2096 } 2097 VM_OBJECT_UNLOCK(vm_obj); 2098 if (i915_gem_object_needs_bit17_swizzle(obj)) 2099 i915_gem_object_do_bit_17_swizzle(obj); 2100 return (0); 2101 2102 failed: 2103 for (j = 0; j < i; j++) { 2104 m = obj->pages[j]; 2105 vm_page_busy_wait(m, FALSE, "i915gem"); 2106 vm_page_unwire(m, 0); 2107 vm_page_wakeup(m); 2108 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2109 } 2110 VM_OBJECT_UNLOCK(vm_obj); 2111 drm_free(obj->pages, DRM_I915_GEM); 2112 obj->pages = NULL; 2113 return (-EIO); 2114 } 2115 2116 #define GEM_PARANOID_CHECK_GTT 0 2117 #if GEM_PARANOID_CHECK_GTT 2118 static void 2119 i915_gem_assert_pages_not_mapped(struct drm_device *dev, vm_page_t *ma, 2120 int page_count) 2121 { 2122 struct drm_i915_private *dev_priv; 2123 vm_paddr_t pa; 2124 unsigned long start, end; 2125 u_int i; 2126 int j; 2127 2128 dev_priv = dev->dev_private; 2129 start = OFF_TO_IDX(dev_priv->mm.gtt_start); 2130 end = OFF_TO_IDX(dev_priv->mm.gtt_end); 2131 for (i = start; i < end; i++) { 2132 pa = intel_gtt_read_pte_paddr(i); 2133 for (j = 0; j < page_count; j++) { 2134 if (pa == VM_PAGE_TO_PHYS(ma[j])) { 2135 panic("Page %p in GTT pte index %d pte %x", 2136 ma[i], i, intel_gtt_read_pte(i)); 2137 } 2138 } 2139 } 2140 } 2141 #endif 2142 2143 static void 2144 i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj) 2145 { 2146 vm_page_t m; 2147 int page_count, i; 2148 2149 BUG_ON(obj->madv == __I915_MADV_PURGED); 2150 2151 if (obj->tiling_mode != I915_TILING_NONE) 2152 i915_gem_object_save_bit_17_swizzle(obj); 2153 if (obj->madv == I915_MADV_DONTNEED) 2154 obj->dirty = 0; 2155 page_count = obj->base.size / PAGE_SIZE; 2156 VM_OBJECT_LOCK(obj->base.vm_obj); 2157 #if GEM_PARANOID_CHECK_GTT 2158 i915_gem_assert_pages_not_mapped(obj->base.dev, obj->pages, page_count); 2159 #endif 2160 for (i = 0; i < page_count; i++) { 2161 m = obj->pages[i]; 2162 if (obj->dirty) 2163 vm_page_dirty(m); 2164 if (obj->madv == I915_MADV_WILLNEED) 2165 vm_page_reference(m); 2166 vm_page_busy_wait(obj->pages[i], FALSE, "i915gem"); 2167 vm_page_unwire(obj->pages[i], 1); 2168 vm_page_wakeup(obj->pages[i]); 2169 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 2170 } 2171 VM_OBJECT_UNLOCK(obj->base.vm_obj); 2172 obj->dirty = 0; 2173 drm_free(obj->pages, DRM_I915_GEM); 2174 obj->pages = NULL; 2175 } 2176 2177 void 2178 i915_gem_release_mmap(struct drm_i915_gem_object *obj) 2179 { 2180 vm_object_t devobj; 2181 vm_page_t m; 2182 int i, page_count; 2183 2184 if (!obj->fault_mappable) 2185 return; 2186 2187 devobj = cdev_pager_lookup(obj); 2188 if (devobj != NULL) { 2189 page_count = OFF_TO_IDX(obj->base.size); 2190 2191 VM_OBJECT_LOCK(devobj); 2192 for (i = 0; i < page_count; i++) { 2193 m = vm_page_lookup_busy_wait(devobj, i, TRUE, "915unm"); 2194 if (m == NULL) 2195 continue; 2196 cdev_pager_free_page(devobj, m); 2197 } 2198 VM_OBJECT_UNLOCK(devobj); 2199 vm_object_deallocate(devobj); 2200 } 2201 2202 obj->fault_mappable = false; 2203 } 2204 2205 int 2206 i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj) 2207 { 2208 int ret; 2209 2210 KASSERT((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0, 2211 ("In GPU write domain")); 2212 2213 if (obj->active) { 2214 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno, 2215 true); 2216 if (ret != 0) 2217 return (ret); 2218 } 2219 return (0); 2220 } 2221 2222 void 2223 i915_gem_object_move_to_active(struct drm_i915_gem_object *obj, 2224 struct intel_ring_buffer *ring, uint32_t seqno) 2225 { 2226 struct drm_device *dev = obj->base.dev; 2227 struct drm_i915_private *dev_priv = dev->dev_private; 2228 struct drm_i915_fence_reg *reg; 2229 2230 obj->ring = ring; 2231 KASSERT(ring != NULL, ("NULL ring")); 2232 2233 /* Add a reference if we're newly entering the active list. */ 2234 if (!obj->active) { 2235 drm_gem_object_reference(&obj->base); 2236 obj->active = 1; 2237 } 2238 2239 /* Move from whatever list we were on to the tail of execution. */ 2240 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list); 2241 list_move_tail(&obj->ring_list, &ring->active_list); 2242 2243 obj->last_rendering_seqno = seqno; 2244 if (obj->fenced_gpu_access) { 2245 obj->last_fenced_seqno = seqno; 2246 obj->last_fenced_ring = ring; 2247 2248 /* Bump MRU to take account of the delayed flush */ 2249 if (obj->fence_reg != I915_FENCE_REG_NONE) { 2250 reg = &dev_priv->fence_regs[obj->fence_reg]; 2251 list_move_tail(®->lru_list, 2252 &dev_priv->mm.fence_list); 2253 } 2254 } 2255 } 2256 2257 static void 2258 i915_gem_object_move_off_active(struct drm_i915_gem_object *obj) 2259 { 2260 list_del_init(&obj->ring_list); 2261 obj->last_rendering_seqno = 0; 2262 obj->last_fenced_seqno = 0; 2263 } 2264 2265 static void 2266 i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj) 2267 { 2268 struct drm_device *dev = obj->base.dev; 2269 drm_i915_private_t *dev_priv = dev->dev_private; 2270 2271 KASSERT(obj->active, ("Object not active")); 2272 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list); 2273 2274 i915_gem_object_move_off_active(obj); 2275 } 2276 2277 static void 2278 i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj) 2279 { 2280 struct drm_device *dev = obj->base.dev; 2281 struct drm_i915_private *dev_priv = dev->dev_private; 2282 2283 if (obj->pin_count != 0) 2284 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list); 2285 else 2286 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list); 2287 2288 KASSERT(list_empty(&obj->gpu_write_list), ("On gpu_write_list")); 2289 KASSERT(obj->active, ("Object not active")); 2290 obj->ring = NULL; 2291 obj->last_fenced_ring = NULL; 2292 2293 i915_gem_object_move_off_active(obj); 2294 obj->fenced_gpu_access = false; 2295 2296 obj->active = 0; 2297 obj->pending_gpu_write = false; 2298 drm_gem_object_unreference(&obj->base); 2299 2300 #if 1 2301 KIB_NOTYET(); 2302 #else 2303 WARN_ON(i915_verify_lists(dev)); 2304 #endif 2305 } 2306 2307 /* Immediately discard the backing storage */ 2308 static void 2309 i915_gem_object_truncate(struct drm_i915_gem_object *obj) 2310 { 2311 vm_object_t vm_obj; 2312 2313 vm_obj = obj->base.vm_obj; 2314 VM_OBJECT_LOCK(vm_obj); 2315 vm_object_page_remove(vm_obj, 0, 0, false); 2316 VM_OBJECT_UNLOCK(vm_obj); 2317 obj->madv = __I915_MADV_PURGED; 2318 } 2319 2320 static inline int 2321 i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj) 2322 { 2323 2324 return (obj->madv == I915_MADV_DONTNEED); 2325 } 2326 2327 static void 2328 i915_gem_process_flushing_list(struct intel_ring_buffer *ring, 2329 uint32_t flush_domains) 2330 { 2331 struct drm_i915_gem_object *obj, *next; 2332 uint32_t old_write_domain; 2333 2334 list_for_each_entry_safe(obj, next, &ring->gpu_write_list, 2335 gpu_write_list) { 2336 if (obj->base.write_domain & flush_domains) { 2337 old_write_domain = obj->base.write_domain; 2338 obj->base.write_domain = 0; 2339 list_del_init(&obj->gpu_write_list); 2340 i915_gem_object_move_to_active(obj, ring, 2341 i915_gem_next_request_seqno(ring)); 2342 } 2343 } 2344 } 2345 2346 static int 2347 i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj) 2348 { 2349 drm_i915_private_t *dev_priv; 2350 2351 dev_priv = obj->base.dev->dev_private; 2352 return (dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 && 2353 obj->tiling_mode != I915_TILING_NONE); 2354 } 2355 2356 #define VM_OBJECT_LOCK_ASSERT_OWNED(object) 2357 2358 static vm_page_t 2359 i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex) 2360 { 2361 vm_page_t m; 2362 int rv; 2363 2364 VM_OBJECT_LOCK_ASSERT_OWNED(object); 2365 m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); 2366 if (m->valid != VM_PAGE_BITS_ALL) { 2367 if (vm_pager_has_page(object, pindex)) { 2368 rv = vm_pager_get_page(object, &m, 1); 2369 m = vm_page_lookup(object, pindex); 2370 if (m == NULL) 2371 return (NULL); 2372 if (rv != VM_PAGER_OK) { 2373 vm_page_free(m); 2374 return (NULL); 2375 } 2376 } else { 2377 pmap_zero_page(VM_PAGE_TO_PHYS(m)); 2378 m->valid = VM_PAGE_BITS_ALL; 2379 m->dirty = 0; 2380 } 2381 } 2382 vm_page_wire(m); 2383 vm_page_wakeup(m); 2384 atomic_add_long(&i915_gem_wired_pages_cnt, 1); 2385 return (m); 2386 } 2387 2388 int 2389 i915_gem_flush_ring(struct intel_ring_buffer *ring, uint32_t invalidate_domains, 2390 uint32_t flush_domains) 2391 { 2392 int ret; 2393 2394 if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0) 2395 return 0; 2396 2397 ret = ring->flush(ring, invalidate_domains, flush_domains); 2398 if (ret) 2399 return ret; 2400 2401 if (flush_domains & I915_GEM_GPU_DOMAINS) 2402 i915_gem_process_flushing_list(ring, flush_domains); 2403 return 0; 2404 } 2405 2406 static int 2407 i915_ring_idle(struct intel_ring_buffer *ring, bool do_retire) 2408 { 2409 int ret; 2410 2411 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list)) 2412 return 0; 2413 2414 if (!list_empty(&ring->gpu_write_list)) { 2415 ret = i915_gem_flush_ring(ring, I915_GEM_GPU_DOMAINS, 2416 I915_GEM_GPU_DOMAINS); 2417 if (ret != 0) 2418 return ret; 2419 } 2420 2421 return (i915_wait_request(ring, i915_gem_next_request_seqno(ring), 2422 do_retire)); 2423 } 2424 2425 int 2426 i915_gpu_idle(struct drm_device *dev, bool do_retire) 2427 { 2428 drm_i915_private_t *dev_priv = dev->dev_private; 2429 int ret, i; 2430 2431 /* Flush everything onto the inactive list. */ 2432 for (i = 0; i < I915_NUM_RINGS; i++) { 2433 ret = i915_ring_idle(&dev_priv->ring[i], do_retire); 2434 if (ret) 2435 return ret; 2436 } 2437 2438 return 0; 2439 } 2440 2441 int 2442 i915_wait_request(struct intel_ring_buffer *ring, uint32_t seqno, bool do_retire) 2443 { 2444 drm_i915_private_t *dev_priv; 2445 struct drm_i915_gem_request *request; 2446 uint32_t ier; 2447 int flags, ret; 2448 bool recovery_complete; 2449 2450 KASSERT(seqno != 0, ("Zero seqno")); 2451 2452 dev_priv = ring->dev->dev_private; 2453 ret = 0; 2454 2455 if (atomic_read(&dev_priv->mm.wedged) != 0) { 2456 /* Give the error handler a chance to run. */ 2457 lockmgr(&dev_priv->error_completion_lock, LK_EXCLUSIVE); 2458 recovery_complete = (&dev_priv->error_completion) > 0; 2459 lockmgr(&dev_priv->error_completion_lock, LK_RELEASE); 2460 return (recovery_complete ? -EIO : -EAGAIN); 2461 } 2462 2463 if (seqno == ring->outstanding_lazy_request) { 2464 request = kmalloc(sizeof(*request), DRM_I915_GEM, 2465 M_WAITOK | M_ZERO); 2466 if (request == NULL) 2467 return (-ENOMEM); 2468 2469 ret = i915_add_request(ring, NULL, request); 2470 if (ret != 0) { 2471 drm_free(request, DRM_I915_GEM); 2472 return (ret); 2473 } 2474 2475 seqno = request->seqno; 2476 } 2477 2478 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) { 2479 if (HAS_PCH_SPLIT(ring->dev)) 2480 ier = I915_READ(DEIER) | I915_READ(GTIER); 2481 else 2482 ier = I915_READ(IER); 2483 if (!ier) { 2484 DRM_ERROR("something (likely vbetool) disabled " 2485 "interrupts, re-enabling\n"); 2486 ring->dev->driver->irq_preinstall(ring->dev); 2487 ring->dev->driver->irq_postinstall(ring->dev); 2488 } 2489 2490 ring->waiting_seqno = seqno; 2491 lockmgr(&ring->irq_lock, LK_EXCLUSIVE); 2492 if (ring->irq_get(ring)) { 2493 flags = dev_priv->mm.interruptible ? PCATCH : 0; 2494 while (!i915_seqno_passed(ring->get_seqno(ring), seqno) 2495 && !atomic_read(&dev_priv->mm.wedged) && 2496 ret == 0) { 2497 ret = -lksleep(ring, &ring->irq_lock, flags, 2498 "915gwr", 0); 2499 } 2500 ring->irq_put(ring); 2501 lockmgr(&ring->irq_lock, LK_RELEASE); 2502 } else { 2503 lockmgr(&ring->irq_lock, LK_RELEASE); 2504 if (_intel_wait_for(ring->dev, 2505 i915_seqno_passed(ring->get_seqno(ring), seqno) || 2506 atomic_read(&dev_priv->mm.wedged), 3000, 2507 0, "i915wrq") != 0) 2508 ret = -EBUSY; 2509 } 2510 ring->waiting_seqno = 0; 2511 2512 } 2513 if (atomic_read(&dev_priv->mm.wedged)) 2514 ret = -EAGAIN; 2515 2516 /* Directly dispatch request retiring. While we have the work queue 2517 * to handle this, the waiter on a request often wants an associated 2518 * buffer to have made it to the inactive list, and we would need 2519 * a separate wait queue to handle that. 2520 */ 2521 if (ret == 0 && do_retire) 2522 i915_gem_retire_requests_ring(ring); 2523 2524 return (ret); 2525 } 2526 2527 static u32 2528 i915_gem_get_seqno(struct drm_device *dev) 2529 { 2530 drm_i915_private_t *dev_priv = dev->dev_private; 2531 u32 seqno = dev_priv->next_seqno; 2532 2533 /* reserve 0 for non-seqno */ 2534 if (++dev_priv->next_seqno == 0) 2535 dev_priv->next_seqno = 1; 2536 2537 return seqno; 2538 } 2539 2540 u32 2541 i915_gem_next_request_seqno(struct intel_ring_buffer *ring) 2542 { 2543 if (ring->outstanding_lazy_request == 0) 2544 ring->outstanding_lazy_request = i915_gem_get_seqno(ring->dev); 2545 2546 return ring->outstanding_lazy_request; 2547 } 2548 2549 int 2550 i915_add_request(struct intel_ring_buffer *ring, struct drm_file *file, 2551 struct drm_i915_gem_request *request) 2552 { 2553 drm_i915_private_t *dev_priv; 2554 struct drm_i915_file_private *file_priv; 2555 uint32_t seqno; 2556 u32 request_ring_position; 2557 int was_empty; 2558 int ret; 2559 2560 KASSERT(request != NULL, ("NULL request in add")); 2561 DRM_LOCK_ASSERT(ring->dev); 2562 dev_priv = ring->dev->dev_private; 2563 2564 seqno = i915_gem_next_request_seqno(ring); 2565 request_ring_position = intel_ring_get_tail(ring); 2566 2567 ret = ring->add_request(ring, &seqno); 2568 if (ret != 0) 2569 return ret; 2570 2571 request->seqno = seqno; 2572 request->ring = ring; 2573 request->tail = request_ring_position; 2574 request->emitted_jiffies = ticks; 2575 was_empty = list_empty(&ring->request_list); 2576 list_add_tail(&request->list, &ring->request_list); 2577 2578 if (file != NULL) { 2579 file_priv = file->driver_priv; 2580 2581 spin_lock(&file_priv->mm.lock); 2582 request->file_priv = file_priv; 2583 list_add_tail(&request->client_list, 2584 &file_priv->mm.request_list); 2585 spin_unlock(&file_priv->mm.lock); 2586 } 2587 2588 ring->outstanding_lazy_request = 0; 2589 2590 if (!dev_priv->mm.suspended) { 2591 if (i915_enable_hangcheck) { 2592 mod_timer(&dev_priv->hangcheck_timer, 2593 round_jiffies_up(jiffies + DRM_I915_HANGCHECK_JIFFIES)); 2594 } 2595 if (was_empty) { 2596 queue_delayed_work(dev_priv->wq, 2597 &dev_priv->mm.retire_work, 2598 round_jiffies_up_relative(hz)); 2599 intel_mark_busy(dev_priv->dev); 2600 } 2601 } 2602 return (0); 2603 } 2604 2605 static inline void 2606 i915_gem_request_remove_from_client(struct drm_i915_gem_request *request) 2607 { 2608 struct drm_i915_file_private *file_priv = request->file_priv; 2609 2610 if (!file_priv) 2611 return; 2612 2613 DRM_LOCK_ASSERT(request->ring->dev); 2614 2615 spin_lock(&file_priv->mm.lock); 2616 if (request->file_priv != NULL) { 2617 list_del(&request->client_list); 2618 request->file_priv = NULL; 2619 } 2620 spin_unlock(&file_priv->mm.lock); 2621 } 2622 2623 void 2624 i915_gem_release(struct drm_device *dev, struct drm_file *file) 2625 { 2626 struct drm_i915_file_private *file_priv; 2627 struct drm_i915_gem_request *request; 2628 2629 file_priv = file->driver_priv; 2630 2631 /* Clean up our request list when the client is going away, so that 2632 * later retire_requests won't dereference our soon-to-be-gone 2633 * file_priv. 2634 */ 2635 spin_lock(&file_priv->mm.lock); 2636 while (!list_empty(&file_priv->mm.request_list)) { 2637 request = list_first_entry(&file_priv->mm.request_list, 2638 struct drm_i915_gem_request, 2639 client_list); 2640 list_del(&request->client_list); 2641 request->file_priv = NULL; 2642 } 2643 spin_unlock(&file_priv->mm.lock); 2644 } 2645 2646 static void 2647 i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv, 2648 struct intel_ring_buffer *ring) 2649 { 2650 2651 if (ring->dev != NULL) 2652 DRM_LOCK_ASSERT(ring->dev); 2653 2654 while (!list_empty(&ring->request_list)) { 2655 struct drm_i915_gem_request *request; 2656 2657 request = list_first_entry(&ring->request_list, 2658 struct drm_i915_gem_request, list); 2659 2660 list_del(&request->list); 2661 i915_gem_request_remove_from_client(request); 2662 drm_free(request, DRM_I915_GEM); 2663 } 2664 2665 while (!list_empty(&ring->active_list)) { 2666 struct drm_i915_gem_object *obj; 2667 2668 obj = list_first_entry(&ring->active_list, 2669 struct drm_i915_gem_object, ring_list); 2670 2671 obj->base.write_domain = 0; 2672 list_del_init(&obj->gpu_write_list); 2673 i915_gem_object_move_to_inactive(obj); 2674 } 2675 } 2676 2677 static void 2678 i915_gem_reset_fences(struct drm_device *dev) 2679 { 2680 struct drm_i915_private *dev_priv = dev->dev_private; 2681 int i; 2682 2683 for (i = 0; i < dev_priv->num_fence_regs; i++) { 2684 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i]; 2685 struct drm_i915_gem_object *obj = reg->obj; 2686 2687 if (!obj) 2688 continue; 2689 2690 if (obj->tiling_mode) 2691 i915_gem_release_mmap(obj); 2692 2693 reg->obj->fence_reg = I915_FENCE_REG_NONE; 2694 reg->obj->fenced_gpu_access = false; 2695 reg->obj->last_fenced_seqno = 0; 2696 reg->obj->last_fenced_ring = NULL; 2697 i915_gem_clear_fence_reg(dev, reg); 2698 } 2699 } 2700 2701 void 2702 i915_gem_reset(struct drm_device *dev) 2703 { 2704 struct drm_i915_private *dev_priv = dev->dev_private; 2705 struct drm_i915_gem_object *obj; 2706 int i; 2707 2708 for (i = 0; i < I915_NUM_RINGS; i++) 2709 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]); 2710 2711 /* Remove anything from the flushing lists. The GPU cache is likely 2712 * to be lost on reset along with the data, so simply move the 2713 * lost bo to the inactive list. 2714 */ 2715 while (!list_empty(&dev_priv->mm.flushing_list)) { 2716 obj = list_first_entry(&dev_priv->mm.flushing_list, 2717 struct drm_i915_gem_object, 2718 mm_list); 2719 2720 obj->base.write_domain = 0; 2721 list_del_init(&obj->gpu_write_list); 2722 i915_gem_object_move_to_inactive(obj); 2723 } 2724 2725 /* Move everything out of the GPU domains to ensure we do any 2726 * necessary invalidation upon reuse. 2727 */ 2728 list_for_each_entry(obj, &dev_priv->mm.inactive_list, mm_list) { 2729 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS; 2730 } 2731 2732 /* The fence registers are invalidated so clear them out */ 2733 i915_gem_reset_fences(dev); 2734 } 2735 2736 /** 2737 * This function clears the request list as sequence numbers are passed. 2738 */ 2739 void 2740 i915_gem_retire_requests_ring(struct intel_ring_buffer *ring) 2741 { 2742 uint32_t seqno; 2743 int i; 2744 2745 if (list_empty(&ring->request_list)) 2746 return; 2747 2748 seqno = ring->get_seqno(ring); 2749 for (i = 0; i < DRM_ARRAY_SIZE(ring->sync_seqno); i++) 2750 if (seqno >= ring->sync_seqno[i]) 2751 ring->sync_seqno[i] = 0; 2752 2753 while (!list_empty(&ring->request_list)) { 2754 struct drm_i915_gem_request *request; 2755 2756 request = list_first_entry(&ring->request_list, 2757 struct drm_i915_gem_request, 2758 list); 2759 2760 if (!i915_seqno_passed(seqno, request->seqno)) 2761 break; 2762 2763 ring->last_retired_head = request->tail; 2764 2765 list_del(&request->list); 2766 i915_gem_request_remove_from_client(request); 2767 drm_free(request, DRM_I915_GEM); 2768 } 2769 2770 /* Move any buffers on the active list that are no longer referenced 2771 * by the ringbuffer to the flushing/inactive lists as appropriate. 2772 */ 2773 while (!list_empty(&ring->active_list)) { 2774 struct drm_i915_gem_object *obj; 2775 2776 obj = list_first_entry(&ring->active_list, 2777 struct drm_i915_gem_object, 2778 ring_list); 2779 2780 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno)) 2781 break; 2782 2783 if (obj->base.write_domain != 0) 2784 i915_gem_object_move_to_flushing(obj); 2785 else 2786 i915_gem_object_move_to_inactive(obj); 2787 } 2788 2789 if (ring->trace_irq_seqno && 2790 i915_seqno_passed(seqno, ring->trace_irq_seqno)) { 2791 lockmgr(&ring->irq_lock, LK_EXCLUSIVE); 2792 ring->irq_put(ring); 2793 lockmgr(&ring->irq_lock, LK_RELEASE); 2794 ring->trace_irq_seqno = 0; 2795 } 2796 } 2797 2798 void 2799 i915_gem_retire_requests(struct drm_device *dev) 2800 { 2801 drm_i915_private_t *dev_priv = dev->dev_private; 2802 struct drm_i915_gem_object *obj, *next; 2803 int i; 2804 2805 if (!list_empty(&dev_priv->mm.deferred_free_list)) { 2806 list_for_each_entry_safe(obj, next, 2807 &dev_priv->mm.deferred_free_list, mm_list) 2808 i915_gem_free_object_tail(obj); 2809 } 2810 2811 for (i = 0; i < I915_NUM_RINGS; i++) 2812 i915_gem_retire_requests_ring(&dev_priv->ring[i]); 2813 } 2814 2815 static int 2816 sandybridge_write_fence_reg(struct drm_i915_gem_object *obj, 2817 struct intel_ring_buffer *pipelined) 2818 { 2819 struct drm_device *dev = obj->base.dev; 2820 drm_i915_private_t *dev_priv = dev->dev_private; 2821 u32 size = obj->gtt_space->size; 2822 int regnum = obj->fence_reg; 2823 uint64_t val; 2824 2825 val = (uint64_t)((obj->gtt_offset + size - 4096) & 2826 0xfffff000) << 32; 2827 val |= obj->gtt_offset & 0xfffff000; 2828 val |= (uint64_t)((obj->stride / 128) - 1) << 2829 SANDYBRIDGE_FENCE_PITCH_SHIFT; 2830 2831 if (obj->tiling_mode == I915_TILING_Y) 2832 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2833 val |= I965_FENCE_REG_VALID; 2834 2835 if (pipelined) { 2836 int ret = intel_ring_begin(pipelined, 6); 2837 if (ret) 2838 return ret; 2839 2840 intel_ring_emit(pipelined, MI_NOOP); 2841 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 2842 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8); 2843 intel_ring_emit(pipelined, (u32)val); 2844 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4); 2845 intel_ring_emit(pipelined, (u32)(val >> 32)); 2846 intel_ring_advance(pipelined); 2847 } else 2848 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val); 2849 2850 return 0; 2851 } 2852 2853 static int 2854 i965_write_fence_reg(struct drm_i915_gem_object *obj, 2855 struct intel_ring_buffer *pipelined) 2856 { 2857 struct drm_device *dev = obj->base.dev; 2858 drm_i915_private_t *dev_priv = dev->dev_private; 2859 u32 size = obj->gtt_space->size; 2860 int regnum = obj->fence_reg; 2861 uint64_t val; 2862 2863 val = (uint64_t)((obj->gtt_offset + size - 4096) & 2864 0xfffff000) << 32; 2865 val |= obj->gtt_offset & 0xfffff000; 2866 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT; 2867 if (obj->tiling_mode == I915_TILING_Y) 2868 val |= 1 << I965_FENCE_TILING_Y_SHIFT; 2869 val |= I965_FENCE_REG_VALID; 2870 2871 if (pipelined) { 2872 int ret = intel_ring_begin(pipelined, 6); 2873 if (ret) 2874 return ret; 2875 2876 intel_ring_emit(pipelined, MI_NOOP); 2877 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2)); 2878 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8); 2879 intel_ring_emit(pipelined, (u32)val); 2880 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4); 2881 intel_ring_emit(pipelined, (u32)(val >> 32)); 2882 intel_ring_advance(pipelined); 2883 } else 2884 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val); 2885 2886 return 0; 2887 } 2888 2889 static int 2890 i915_write_fence_reg(struct drm_i915_gem_object *obj, 2891 struct intel_ring_buffer *pipelined) 2892 { 2893 struct drm_device *dev = obj->base.dev; 2894 drm_i915_private_t *dev_priv = dev->dev_private; 2895 u32 size = obj->gtt_space->size; 2896 u32 fence_reg, val, pitch_val; 2897 int tile_width; 2898 2899 if ((obj->gtt_offset & ~I915_FENCE_START_MASK) || 2900 (size & -size) != size || (obj->gtt_offset & (size - 1))) { 2901 kprintf( 2902 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n", 2903 obj->gtt_offset, obj->map_and_fenceable, size); 2904 return -EINVAL; 2905 } 2906 2907 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev)) 2908 tile_width = 128; 2909 else 2910 tile_width = 512; 2911 2912 /* Note: pitch better be a power of two tile widths */ 2913 pitch_val = obj->stride / tile_width; 2914 pitch_val = ffs(pitch_val) - 1; 2915 2916 val = obj->gtt_offset; 2917 if (obj->tiling_mode == I915_TILING_Y) 2918 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2919 val |= I915_FENCE_SIZE_BITS(size); 2920 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2921 val |= I830_FENCE_REG_VALID; 2922 2923 fence_reg = obj->fence_reg; 2924 if (fence_reg < 8) 2925 fence_reg = FENCE_REG_830_0 + fence_reg * 4; 2926 else 2927 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 2928 2929 if (pipelined) { 2930 int ret = intel_ring_begin(pipelined, 4); 2931 if (ret) 2932 return ret; 2933 2934 intel_ring_emit(pipelined, MI_NOOP); 2935 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 2936 intel_ring_emit(pipelined, fence_reg); 2937 intel_ring_emit(pipelined, val); 2938 intel_ring_advance(pipelined); 2939 } else 2940 I915_WRITE(fence_reg, val); 2941 2942 return 0; 2943 } 2944 2945 static int 2946 i830_write_fence_reg(struct drm_i915_gem_object *obj, 2947 struct intel_ring_buffer *pipelined) 2948 { 2949 struct drm_device *dev = obj->base.dev; 2950 drm_i915_private_t *dev_priv = dev->dev_private; 2951 u32 size = obj->gtt_space->size; 2952 int regnum = obj->fence_reg; 2953 uint32_t val; 2954 uint32_t pitch_val; 2955 2956 if ((obj->gtt_offset & ~I830_FENCE_START_MASK) || 2957 (size & -size) != size || (obj->gtt_offset & (size - 1))) { 2958 kprintf( 2959 "object 0x%08x not 512K or pot-size 0x%08x aligned\n", 2960 obj->gtt_offset, size); 2961 return -EINVAL; 2962 } 2963 2964 pitch_val = obj->stride / 128; 2965 pitch_val = ffs(pitch_val) - 1; 2966 2967 val = obj->gtt_offset; 2968 if (obj->tiling_mode == I915_TILING_Y) 2969 val |= 1 << I830_FENCE_TILING_Y_SHIFT; 2970 val |= I830_FENCE_SIZE_BITS(size); 2971 val |= pitch_val << I830_FENCE_PITCH_SHIFT; 2972 val |= I830_FENCE_REG_VALID; 2973 2974 if (pipelined) { 2975 int ret = intel_ring_begin(pipelined, 4); 2976 if (ret) 2977 return ret; 2978 2979 intel_ring_emit(pipelined, MI_NOOP); 2980 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1)); 2981 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4); 2982 intel_ring_emit(pipelined, val); 2983 intel_ring_advance(pipelined); 2984 } else 2985 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val); 2986 2987 return 0; 2988 } 2989 2990 static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno) 2991 { 2992 return i915_seqno_passed(ring->get_seqno(ring), seqno); 2993 } 2994 2995 static int 2996 i915_gem_object_flush_fence(struct drm_i915_gem_object *obj, 2997 struct intel_ring_buffer *pipelined) 2998 { 2999 int ret; 3000 3001 if (obj->fenced_gpu_access) { 3002 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) { 3003 ret = i915_gem_flush_ring(obj->last_fenced_ring, 0, 3004 obj->base.write_domain); 3005 if (ret) 3006 return ret; 3007 } 3008 3009 obj->fenced_gpu_access = false; 3010 } 3011 3012 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) { 3013 if (!ring_passed_seqno(obj->last_fenced_ring, 3014 obj->last_fenced_seqno)) { 3015 ret = i915_wait_request(obj->last_fenced_ring, 3016 obj->last_fenced_seqno, 3017 true); 3018 if (ret) 3019 return ret; 3020 } 3021 3022 obj->last_fenced_seqno = 0; 3023 obj->last_fenced_ring = NULL; 3024 } 3025 3026 /* Ensure that all CPU reads are completed before installing a fence 3027 * and all writes before removing the fence. 3028 */ 3029 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT) 3030 cpu_mfence(); 3031 3032 return 0; 3033 } 3034 3035 int 3036 i915_gem_object_put_fence(struct drm_i915_gem_object *obj) 3037 { 3038 int ret; 3039 3040 if (obj->tiling_mode) 3041 i915_gem_release_mmap(obj); 3042 3043 ret = i915_gem_object_flush_fence(obj, NULL); 3044 if (ret) 3045 return ret; 3046 3047 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3048 struct drm_i915_private *dev_priv = obj->base.dev->dev_private; 3049 3050 if (dev_priv->fence_regs[obj->fence_reg].pin_count != 0) 3051 kprintf("%s: pin_count %d\n", __func__, 3052 dev_priv->fence_regs[obj->fence_reg].pin_count); 3053 i915_gem_clear_fence_reg(obj->base.dev, 3054 &dev_priv->fence_regs[obj->fence_reg]); 3055 3056 obj->fence_reg = I915_FENCE_REG_NONE; 3057 } 3058 3059 return 0; 3060 } 3061 3062 static struct drm_i915_fence_reg * 3063 i915_find_fence_reg(struct drm_device *dev, struct intel_ring_buffer *pipelined) 3064 { 3065 struct drm_i915_private *dev_priv = dev->dev_private; 3066 struct drm_i915_fence_reg *reg, *first, *avail; 3067 int i; 3068 3069 /* First try to find a free reg */ 3070 avail = NULL; 3071 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) { 3072 reg = &dev_priv->fence_regs[i]; 3073 if (!reg->obj) 3074 return reg; 3075 3076 if (!reg->pin_count) 3077 avail = reg; 3078 } 3079 3080 if (avail == NULL) 3081 return NULL; 3082 3083 /* None available, try to steal one or wait for a user to finish */ 3084 avail = first = NULL; 3085 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) { 3086 if (reg->pin_count) 3087 continue; 3088 3089 if (first == NULL) 3090 first = reg; 3091 3092 if (!pipelined || 3093 !reg->obj->last_fenced_ring || 3094 reg->obj->last_fenced_ring == pipelined) { 3095 avail = reg; 3096 break; 3097 } 3098 } 3099 3100 if (avail == NULL) 3101 avail = first; 3102 3103 return avail; 3104 } 3105 3106 int 3107 i915_gem_object_get_fence(struct drm_i915_gem_object *obj, 3108 struct intel_ring_buffer *pipelined) 3109 { 3110 struct drm_device *dev = obj->base.dev; 3111 struct drm_i915_private *dev_priv = dev->dev_private; 3112 struct drm_i915_fence_reg *reg; 3113 int ret; 3114 3115 pipelined = NULL; 3116 ret = 0; 3117 3118 if (obj->fence_reg != I915_FENCE_REG_NONE) { 3119 reg = &dev_priv->fence_regs[obj->fence_reg]; 3120 list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 3121 3122 if (obj->tiling_changed) { 3123 ret = i915_gem_object_flush_fence(obj, pipelined); 3124 if (ret) 3125 return ret; 3126 3127 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 3128 pipelined = NULL; 3129 3130 if (pipelined) { 3131 reg->setup_seqno = 3132 i915_gem_next_request_seqno(pipelined); 3133 obj->last_fenced_seqno = reg->setup_seqno; 3134 obj->last_fenced_ring = pipelined; 3135 } 3136 3137 goto update; 3138 } 3139 3140 if (!pipelined) { 3141 if (reg->setup_seqno) { 3142 if (!ring_passed_seqno(obj->last_fenced_ring, 3143 reg->setup_seqno)) { 3144 ret = i915_wait_request( 3145 obj->last_fenced_ring, 3146 reg->setup_seqno, 3147 true); 3148 if (ret) 3149 return ret; 3150 } 3151 3152 reg->setup_seqno = 0; 3153 } 3154 } else if (obj->last_fenced_ring && 3155 obj->last_fenced_ring != pipelined) { 3156 ret = i915_gem_object_flush_fence(obj, pipelined); 3157 if (ret) 3158 return ret; 3159 } 3160 3161 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno) 3162 pipelined = NULL; 3163 KASSERT(pipelined || reg->setup_seqno == 0, ("!pipelined")); 3164 3165 if (obj->tiling_changed) { 3166 if (pipelined) { 3167 reg->setup_seqno = 3168 i915_gem_next_request_seqno(pipelined); 3169 obj->last_fenced_seqno = reg->setup_seqno; 3170 obj->last_fenced_ring = pipelined; 3171 } 3172 goto update; 3173 } 3174 3175 return 0; 3176 } 3177 3178 reg = i915_find_fence_reg(dev, pipelined); 3179 if (reg == NULL) 3180 return -EDEADLK; 3181 3182 ret = i915_gem_object_flush_fence(obj, pipelined); 3183 if (ret) 3184 return ret; 3185 3186 if (reg->obj) { 3187 struct drm_i915_gem_object *old = reg->obj; 3188 3189 drm_gem_object_reference(&old->base); 3190 3191 if (old->tiling_mode) 3192 i915_gem_release_mmap(old); 3193 3194 ret = i915_gem_object_flush_fence(old, pipelined); 3195 if (ret) { 3196 drm_gem_object_unreference(&old->base); 3197 return ret; 3198 } 3199 3200 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0) 3201 pipelined = NULL; 3202 3203 old->fence_reg = I915_FENCE_REG_NONE; 3204 old->last_fenced_ring = pipelined; 3205 old->last_fenced_seqno = 3206 pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 3207 3208 drm_gem_object_unreference(&old->base); 3209 } else if (obj->last_fenced_seqno == 0) 3210 pipelined = NULL; 3211 3212 reg->obj = obj; 3213 list_move_tail(®->lru_list, &dev_priv->mm.fence_list); 3214 obj->fence_reg = reg - dev_priv->fence_regs; 3215 obj->last_fenced_ring = pipelined; 3216 3217 reg->setup_seqno = 3218 pipelined ? i915_gem_next_request_seqno(pipelined) : 0; 3219 obj->last_fenced_seqno = reg->setup_seqno; 3220 3221 update: 3222 obj->tiling_changed = false; 3223 switch (INTEL_INFO(dev)->gen) { 3224 case 7: 3225 case 6: 3226 ret = sandybridge_write_fence_reg(obj, pipelined); 3227 break; 3228 case 5: 3229 case 4: 3230 ret = i965_write_fence_reg(obj, pipelined); 3231 break; 3232 case 3: 3233 ret = i915_write_fence_reg(obj, pipelined); 3234 break; 3235 case 2: 3236 ret = i830_write_fence_reg(obj, pipelined); 3237 break; 3238 } 3239 3240 return ret; 3241 } 3242 3243 static void 3244 i915_gem_clear_fence_reg(struct drm_device *dev, struct drm_i915_fence_reg *reg) 3245 { 3246 drm_i915_private_t *dev_priv = dev->dev_private; 3247 uint32_t fence_reg = reg - dev_priv->fence_regs; 3248 3249 switch (INTEL_INFO(dev)->gen) { 3250 case 7: 3251 case 6: 3252 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0); 3253 break; 3254 case 5: 3255 case 4: 3256 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0); 3257 break; 3258 case 3: 3259 if (fence_reg >= 8) 3260 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4; 3261 else 3262 case 2: 3263 fence_reg = FENCE_REG_830_0 + fence_reg * 4; 3264 3265 I915_WRITE(fence_reg, 0); 3266 break; 3267 } 3268 3269 list_del_init(®->lru_list); 3270 reg->obj = NULL; 3271 reg->setup_seqno = 0; 3272 reg->pin_count = 0; 3273 } 3274 3275 int 3276 i915_gem_init_object(struct drm_gem_object *obj) 3277 { 3278 3279 kprintf("i915_gem_init_object called\n"); 3280 return (0); 3281 } 3282 3283 static bool 3284 i915_gem_object_is_inactive(struct drm_i915_gem_object *obj) 3285 { 3286 3287 return (obj->gtt_space && !obj->active && obj->pin_count == 0); 3288 } 3289 3290 static void 3291 i915_gem_retire_work_handler(struct work_struct *work) 3292 { 3293 drm_i915_private_t *dev_priv; 3294 struct drm_device *dev; 3295 struct intel_ring_buffer *ring; 3296 bool idle; 3297 int i; 3298 3299 dev_priv = container_of(work, drm_i915_private_t, 3300 mm.retire_work.work); 3301 dev = dev_priv->dev; 3302 3303 /* Come back later if the device is busy... */ 3304 if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) { 3305 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 3306 round_jiffies_up_relative(hz)); 3307 return; 3308 } 3309 3310 i915_gem_retire_requests(dev); 3311 3312 /* Send a periodic flush down the ring so we don't hold onto GEM 3313 * objects indefinitely. 3314 */ 3315 idle = true; 3316 for (i = 0; i < I915_NUM_RINGS; i++) { 3317 ring = &dev_priv->ring[i]; 3318 3319 if (!list_empty(&ring->gpu_write_list)) { 3320 struct drm_i915_gem_request *request; 3321 int ret; 3322 3323 ret = i915_gem_flush_ring(ring, 3324 0, I915_GEM_GPU_DOMAINS); 3325 request = kmalloc(sizeof(*request), DRM_I915_GEM, 3326 M_WAITOK | M_ZERO); 3327 if (ret || request == NULL || 3328 i915_add_request(ring, NULL, request)) 3329 drm_free(request, DRM_I915_GEM); 3330 } 3331 3332 idle &= list_empty(&ring->request_list); 3333 } 3334 3335 if (!dev_priv->mm.suspended && !idle) 3336 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 3337 round_jiffies_up_relative(hz)); 3338 3339 DRM_UNLOCK(dev); 3340 } 3341 3342 void 3343 i915_gem_lastclose(struct drm_device *dev) 3344 { 3345 int ret; 3346 3347 if (drm_core_check_feature(dev, DRIVER_MODESET)) 3348 return; 3349 3350 ret = i915_gem_idle(dev); 3351 if (ret != 0) 3352 DRM_ERROR("failed to idle hardware: %d\n", ret); 3353 } 3354 3355 static int 3356 i915_gem_init_phys_object(struct drm_device *dev, int id, int size, int align) 3357 { 3358 drm_i915_private_t *dev_priv; 3359 struct drm_i915_gem_phys_object *phys_obj; 3360 int ret; 3361 3362 dev_priv = dev->dev_private; 3363 if (dev_priv->mm.phys_objs[id - 1] != NULL || size == 0) 3364 return (0); 3365 3366 phys_obj = kmalloc(sizeof(struct drm_i915_gem_phys_object), DRM_I915_GEM, 3367 M_WAITOK | M_ZERO); 3368 3369 phys_obj->id = id; 3370 3371 phys_obj->handle = drm_pci_alloc(dev, size, align, ~0); 3372 if (phys_obj->handle == NULL) { 3373 ret = -ENOMEM; 3374 goto free_obj; 3375 } 3376 pmap_change_attr((vm_offset_t)phys_obj->handle->vaddr, 3377 size / PAGE_SIZE, PAT_WRITE_COMBINING); 3378 3379 dev_priv->mm.phys_objs[id - 1] = phys_obj; 3380 3381 return (0); 3382 3383 free_obj: 3384 drm_free(phys_obj, DRM_I915_GEM); 3385 return (ret); 3386 } 3387 3388 static void 3389 i915_gem_free_phys_object(struct drm_device *dev, int id) 3390 { 3391 drm_i915_private_t *dev_priv; 3392 struct drm_i915_gem_phys_object *phys_obj; 3393 3394 dev_priv = dev->dev_private; 3395 if (dev_priv->mm.phys_objs[id - 1] == NULL) 3396 return; 3397 3398 phys_obj = dev_priv->mm.phys_objs[id - 1]; 3399 if (phys_obj->cur_obj != NULL) 3400 i915_gem_detach_phys_object(dev, phys_obj->cur_obj); 3401 3402 drm_pci_free(dev, phys_obj->handle); 3403 drm_free(phys_obj, DRM_I915_GEM); 3404 dev_priv->mm.phys_objs[id - 1] = NULL; 3405 } 3406 3407 void 3408 i915_gem_free_all_phys_object(struct drm_device *dev) 3409 { 3410 int i; 3411 3412 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++) 3413 i915_gem_free_phys_object(dev, i); 3414 } 3415 3416 void 3417 i915_gem_detach_phys_object(struct drm_device *dev, 3418 struct drm_i915_gem_object *obj) 3419 { 3420 vm_page_t m; 3421 struct sf_buf *sf; 3422 char *vaddr, *dst; 3423 int i, page_count; 3424 3425 if (obj->phys_obj == NULL) 3426 return; 3427 vaddr = obj->phys_obj->handle->vaddr; 3428 3429 page_count = obj->base.size / PAGE_SIZE; 3430 VM_OBJECT_LOCK(obj->base.vm_obj); 3431 for (i = 0; i < page_count; i++) { 3432 m = i915_gem_wire_page(obj->base.vm_obj, i); 3433 if (m == NULL) 3434 continue; /* XXX */ 3435 3436 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3437 sf = sf_buf_alloc(m); 3438 if (sf != NULL) { 3439 dst = (char *)sf_buf_kva(sf); 3440 memcpy(dst, vaddr + IDX_TO_OFF(i), PAGE_SIZE); 3441 sf_buf_free(sf); 3442 } 3443 drm_clflush_pages(&m, 1); 3444 3445 VM_OBJECT_LOCK(obj->base.vm_obj); 3446 vm_page_reference(m); 3447 vm_page_dirty(m); 3448 vm_page_busy_wait(m, FALSE, "i915gem"); 3449 vm_page_unwire(m, 0); 3450 vm_page_wakeup(m); 3451 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3452 } 3453 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3454 intel_gtt_chipset_flush(); 3455 3456 obj->phys_obj->cur_obj = NULL; 3457 obj->phys_obj = NULL; 3458 } 3459 3460 static void 3461 init_ring_lists(struct intel_ring_buffer *ring) 3462 { 3463 3464 INIT_LIST_HEAD(&ring->active_list); 3465 INIT_LIST_HEAD(&ring->request_list); 3466 INIT_LIST_HEAD(&ring->gpu_write_list); 3467 } 3468 3469 void 3470 i915_gem_load(struct drm_device *dev) 3471 { 3472 int i; 3473 drm_i915_private_t *dev_priv = dev->dev_private; 3474 3475 INIT_LIST_HEAD(&dev_priv->mm.active_list); 3476 INIT_LIST_HEAD(&dev_priv->mm.flushing_list); 3477 INIT_LIST_HEAD(&dev_priv->mm.inactive_list); 3478 INIT_LIST_HEAD(&dev_priv->mm.pinned_list); 3479 INIT_LIST_HEAD(&dev_priv->mm.fence_list); 3480 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list); 3481 INIT_LIST_HEAD(&dev_priv->mm.gtt_list); 3482 for (i = 0; i < I915_NUM_RINGS; i++) 3483 init_ring_lists(&dev_priv->ring[i]); 3484 for (i = 0; i < I915_MAX_NUM_FENCES; i++) 3485 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list); 3486 INIT_DELAYED_WORK(&dev_priv->mm.retire_work, 3487 i915_gem_retire_work_handler); 3488 dev_priv->error_completion = 0; 3489 3490 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */ 3491 if (IS_GEN3(dev)) { 3492 I915_WRITE(MI_ARB_STATE, 3493 _MASKED_BIT_ENABLE(MI_ARB_C3_LP_WRITE_ENABLE)); 3494 } 3495 3496 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL; 3497 3498 /* Old X drivers will take 0-2 for front, back, depth buffers */ 3499 if (!drm_core_check_feature(dev, DRIVER_MODESET)) 3500 dev_priv->fence_reg_start = 3; 3501 3502 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev)) 3503 dev_priv->num_fence_regs = 16; 3504 else 3505 dev_priv->num_fence_regs = 8; 3506 3507 /* Initialize fence registers to zero */ 3508 i915_gem_reset_fences(dev); 3509 3510 i915_gem_detect_bit_6_swizzle(dev); 3511 3512 dev_priv->mm.interruptible = true; 3513 3514 dev_priv->mm.i915_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, 3515 i915_gem_lowmem, dev, EVENTHANDLER_PRI_ANY); 3516 } 3517 3518 int 3519 i915_gem_attach_phys_object(struct drm_device *dev, 3520 struct drm_i915_gem_object *obj, int id, int align) 3521 { 3522 drm_i915_private_t *dev_priv; 3523 vm_page_t m; 3524 struct sf_buf *sf; 3525 char *dst, *src; 3526 int i, page_count, ret; 3527 3528 if (id > I915_MAX_PHYS_OBJECT) 3529 return (-EINVAL); 3530 3531 if (obj->phys_obj != NULL) { 3532 if (obj->phys_obj->id == id) 3533 return (0); 3534 i915_gem_detach_phys_object(dev, obj); 3535 } 3536 3537 dev_priv = dev->dev_private; 3538 if (dev_priv->mm.phys_objs[id - 1] == NULL) { 3539 ret = i915_gem_init_phys_object(dev, id, obj->base.size, align); 3540 if (ret != 0) { 3541 DRM_ERROR("failed to init phys object %d size: %zu\n", 3542 id, obj->base.size); 3543 return (ret); 3544 } 3545 } 3546 3547 /* bind to the object */ 3548 obj->phys_obj = dev_priv->mm.phys_objs[id - 1]; 3549 obj->phys_obj->cur_obj = obj; 3550 3551 page_count = obj->base.size / PAGE_SIZE; 3552 3553 VM_OBJECT_LOCK(obj->base.vm_obj); 3554 ret = 0; 3555 for (i = 0; i < page_count; i++) { 3556 m = i915_gem_wire_page(obj->base.vm_obj, i); 3557 if (m == NULL) { 3558 ret = -EIO; 3559 break; 3560 } 3561 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3562 sf = sf_buf_alloc(m); 3563 src = (char *)sf_buf_kva(sf); 3564 dst = (char *)obj->phys_obj->handle->vaddr + IDX_TO_OFF(i); 3565 memcpy(dst, src, PAGE_SIZE); 3566 sf_buf_free(sf); 3567 3568 VM_OBJECT_LOCK(obj->base.vm_obj); 3569 3570 vm_page_reference(m); 3571 vm_page_busy_wait(m, FALSE, "i915gem"); 3572 vm_page_unwire(m, 0); 3573 vm_page_wakeup(m); 3574 atomic_add_long(&i915_gem_wired_pages_cnt, -1); 3575 } 3576 VM_OBJECT_UNLOCK(obj->base.vm_obj); 3577 3578 return (0); 3579 } 3580 3581 static int 3582 i915_gem_phys_pwrite(struct drm_device *dev, struct drm_i915_gem_object *obj, 3583 uint64_t data_ptr, uint64_t offset, uint64_t size, 3584 struct drm_file *file_priv) 3585 { 3586 char *user_data, *vaddr; 3587 int ret; 3588 3589 vaddr = (char *)obj->phys_obj->handle->vaddr + offset; 3590 user_data = (char *)(uintptr_t)data_ptr; 3591 3592 if (copyin_nofault(user_data, vaddr, size) != 0) { 3593 /* The physical object once assigned is fixed for the lifetime 3594 * of the obj, so we can safely drop the lock and continue 3595 * to access vaddr. 3596 */ 3597 DRM_UNLOCK(dev); 3598 ret = -copyin(user_data, vaddr, size); 3599 DRM_LOCK(dev); 3600 if (ret != 0) 3601 return (ret); 3602 } 3603 3604 intel_gtt_chipset_flush(); 3605 return (0); 3606 } 3607 3608 static int 3609 i915_gpu_is_active(struct drm_device *dev) 3610 { 3611 drm_i915_private_t *dev_priv; 3612 3613 dev_priv = dev->dev_private; 3614 return (!list_empty(&dev_priv->mm.flushing_list) || 3615 !list_empty(&dev_priv->mm.active_list)); 3616 } 3617 3618 static void 3619 i915_gem_lowmem(void *arg) 3620 { 3621 struct drm_device *dev; 3622 struct drm_i915_private *dev_priv; 3623 struct drm_i915_gem_object *obj, *next; 3624 int cnt, cnt_fail, cnt_total; 3625 3626 dev = arg; 3627 dev_priv = dev->dev_private; 3628 3629 if (lockmgr(&dev->dev_struct_lock, LK_EXCLUSIVE|LK_NOWAIT)) 3630 return; 3631 3632 rescan: 3633 /* first scan for clean buffers */ 3634 i915_gem_retire_requests(dev); 3635 3636 cnt_total = cnt_fail = cnt = 0; 3637 3638 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3639 mm_list) { 3640 if (i915_gem_object_is_purgeable(obj)) { 3641 if (i915_gem_object_unbind(obj) != 0) 3642 cnt_total++; 3643 } else 3644 cnt_total++; 3645 } 3646 3647 /* second pass, evict/count anything still on the inactive list */ 3648 list_for_each_entry_safe(obj, next, &dev_priv->mm.inactive_list, 3649 mm_list) { 3650 if (i915_gem_object_unbind(obj) == 0) 3651 cnt++; 3652 else 3653 cnt_fail++; 3654 } 3655 3656 if (cnt_fail > cnt_total / 100 && i915_gpu_is_active(dev)) { 3657 /* 3658 * We are desperate for pages, so as a last resort, wait 3659 * for the GPU to finish and discard whatever we can. 3660 * This has a dramatic impact to reduce the number of 3661 * OOM-killer events whilst running the GPU aggressively. 3662 */ 3663 if (i915_gpu_idle(dev, true) == 0) 3664 goto rescan; 3665 } 3666 DRM_UNLOCK(dev); 3667 } 3668 3669 void 3670 i915_gem_unload(struct drm_device *dev) 3671 { 3672 struct drm_i915_private *dev_priv; 3673 3674 dev_priv = dev->dev_private; 3675 EVENTHANDLER_DEREGISTER(vm_lowmem, dev_priv->mm.i915_lowmem); 3676 } 3677