1 /* $OpenBSD: intel_uncore.c,v 1.3 2015/11/01 03:54:20 jsg Exp $ */ 2 /* 3 * Copyright © 2013 Intel Corporation 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice (including the next 13 * paragraph) shall be included in all copies or substantial portions of the 14 * Software. 15 * 16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS 22 * IN THE SOFTWARE. 23 */ 24 25 #include "i915_drv.h" 26 #include "intel_drv.h" 27 28 #define FORCEWAKE_ACK_TIMEOUT_MS 2 29 30 #define __raw_i915_read8(dev_priv__, reg__) bus_space_read_1((dev_priv__)->regs->bst, (dev_priv__)->regs->bsh, (reg__)) 31 #define __raw_i915_write8(dev_priv__, reg__, val__) bus_space_write_1((dev_priv__)->regs->bst, (dev_priv__)->regs->bsh, (reg__), (val__)) 32 33 #define __raw_i915_read16(dev_priv__, reg__) bus_space_read_2((dev_priv__)->regs->bst, (dev_priv__)->regs->bsh, (reg__)) 34 #define __raw_i915_write16(dev_priv__, reg__, val__) bus_space_write_2((dev_priv__)->regs->bst, (dev_priv__)->regs->bsh, (reg__), (val__)) 35 36 #define __raw_i915_read32(dev_priv__, reg__) bus_space_read_4((dev_priv__)->regs->bst, (dev_priv__)->regs->bsh, (reg__)) 37 #define __raw_i915_write32(dev_priv__, reg__, val__) bus_space_write_4((dev_priv__)->regs->bst, (dev_priv__)->regs->bsh, (reg__), (val__)) 38 39 #define __raw_i915_read64(dev_priv__, reg__) bus_space_read_8((dev_priv__)->regs->bst, (dev_priv__)->regs->bsh, (reg__)) 40 #define __raw_i915_write64(dev_priv__, reg__, val__) bus_space_write_8((dev_priv__)->regs->bst, (dev_priv__)->regs->bsh, (reg__), (val__)) 41 42 #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32(dev_priv__, reg__) 43 44 45 static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv) 46 { 47 u32 gt_thread_status_mask; 48 49 if (IS_HASWELL(dev_priv->dev)) 50 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK_HSW; 51 else 52 gt_thread_status_mask = GEN6_GT_THREAD_STATUS_CORE_MASK; 53 54 /* w/a for a sporadic read returning 0 by waiting for the GT 55 * thread to wake up. 56 */ 57 if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) & gt_thread_status_mask) == 0, 500)) 58 DRM_ERROR("GT thread status wait timed out\n"); 59 } 60 61 static void __gen6_gt_force_wake_reset(struct drm_i915_private *dev_priv) 62 { 63 __raw_i915_write32(dev_priv, FORCEWAKE, 0); 64 /* something from same cacheline, but !FORCEWAKE */ 65 __raw_posting_read(dev_priv, ECOBUS); 66 } 67 68 static void __gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, 69 int fw_engine) 70 { 71 if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1) == 0, 72 FORCEWAKE_ACK_TIMEOUT_MS)) 73 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n"); 74 75 __raw_i915_write32(dev_priv, FORCEWAKE, 1); 76 /* something from same cacheline, but !FORCEWAKE */ 77 __raw_posting_read(dev_priv, ECOBUS); 78 79 if (wait_for_atomic((__raw_i915_read32(dev_priv, FORCEWAKE_ACK) & 1), 80 FORCEWAKE_ACK_TIMEOUT_MS)) 81 DRM_ERROR("Timed out waiting for forcewake to ack request.\n"); 82 83 /* WaRsForcewakeWaitTC0:snb */ 84 __gen6_gt_wait_for_thread_c0(dev_priv); 85 } 86 87 static void __gen6_gt_force_wake_mt_reset(struct drm_i915_private *dev_priv) 88 { 89 __raw_i915_write32(dev_priv, FORCEWAKE_MT, _MASKED_BIT_DISABLE(0xffff)); 90 /* something from same cacheline, but !FORCEWAKE_MT */ 91 __raw_posting_read(dev_priv, ECOBUS); 92 } 93 94 static void __gen6_gt_force_wake_mt_get(struct drm_i915_private *dev_priv, 95 int fw_engine) 96 { 97 u32 forcewake_ack; 98 99 if (IS_HASWELL(dev_priv->dev) || IS_GEN8(dev_priv->dev)) 100 forcewake_ack = FORCEWAKE_ACK_HSW; 101 else 102 forcewake_ack = FORCEWAKE_MT_ACK; 103 104 if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL) == 0, 105 FORCEWAKE_ACK_TIMEOUT_MS)) 106 DRM_ERROR("Timed out waiting for forcewake old ack to clear.\n"); 107 108 __raw_i915_write32(dev_priv, FORCEWAKE_MT, 109 _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); 110 /* something from same cacheline, but !FORCEWAKE_MT */ 111 __raw_posting_read(dev_priv, ECOBUS); 112 113 if (wait_for_atomic((__raw_i915_read32(dev_priv, forcewake_ack) & FORCEWAKE_KERNEL), 114 FORCEWAKE_ACK_TIMEOUT_MS)) 115 DRM_ERROR("Timed out waiting for forcewake to ack request.\n"); 116 117 /* WaRsForcewakeWaitTC0:ivb,hsw */ 118 if (INTEL_INFO(dev_priv->dev)->gen < 8) 119 __gen6_gt_wait_for_thread_c0(dev_priv); 120 } 121 122 static void gen6_gt_check_fifodbg(struct drm_i915_private *dev_priv) 123 { 124 u32 gtfifodbg; 125 126 gtfifodbg = __raw_i915_read32(dev_priv, GTFIFODBG); 127 if (WARN(gtfifodbg, "GT wake FIFO error 0x%x\n", gtfifodbg)) 128 __raw_i915_write32(dev_priv, GTFIFODBG, gtfifodbg); 129 } 130 131 static void __gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, 132 int fw_engine) 133 { 134 __raw_i915_write32(dev_priv, FORCEWAKE, 0); 135 /* something from same cacheline, but !FORCEWAKE */ 136 __raw_posting_read(dev_priv, ECOBUS); 137 gen6_gt_check_fifodbg(dev_priv); 138 } 139 140 static void __gen6_gt_force_wake_mt_put(struct drm_i915_private *dev_priv, 141 int fw_engine) 142 { 143 __raw_i915_write32(dev_priv, FORCEWAKE_MT, 144 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); 145 /* something from same cacheline, but !FORCEWAKE_MT */ 146 __raw_posting_read(dev_priv, ECOBUS); 147 gen6_gt_check_fifodbg(dev_priv); 148 } 149 150 static int __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv) 151 { 152 int ret = 0; 153 154 /* On VLV, FIFO will be shared by both SW and HW. 155 * So, we need to read the FREE_ENTRIES everytime */ 156 if (IS_VALLEYVIEW(dev_priv->dev)) 157 dev_priv->uncore.fifo_count = 158 __raw_i915_read32(dev_priv, GTFIFOCTL) & 159 GT_FIFO_FREE_ENTRIES_MASK; 160 161 if (dev_priv->uncore.fifo_count < GT_FIFO_NUM_RESERVED_ENTRIES) { 162 int loop = 500; 163 u32 fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK; 164 while (fifo <= GT_FIFO_NUM_RESERVED_ENTRIES && loop--) { 165 udelay(10); 166 fifo = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK; 167 } 168 if (WARN_ON(loop < 0 && fifo <= GT_FIFO_NUM_RESERVED_ENTRIES)) 169 ++ret; 170 dev_priv->uncore.fifo_count = fifo; 171 } 172 dev_priv->uncore.fifo_count--; 173 174 return ret; 175 } 176 177 static void vlv_force_wake_reset(struct drm_i915_private *dev_priv) 178 { 179 __raw_i915_write32(dev_priv, FORCEWAKE_VLV, 180 _MASKED_BIT_DISABLE(0xffff)); 181 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV, 182 _MASKED_BIT_DISABLE(0xffff)); 183 /* something from same cacheline, but !FORCEWAKE_VLV */ 184 __raw_posting_read(dev_priv, FORCEWAKE_ACK_VLV); 185 } 186 187 static void __vlv_force_wake_get(struct drm_i915_private *dev_priv, 188 int fw_engine) 189 { 190 /* Check for Render Engine */ 191 if (FORCEWAKE_RENDER & fw_engine) { 192 if (wait_for_atomic((__raw_i915_read32(dev_priv, 193 FORCEWAKE_ACK_VLV) & 194 FORCEWAKE_KERNEL) == 0, 195 FORCEWAKE_ACK_TIMEOUT_MS)) 196 DRM_ERROR("Timed out: Render forcewake old ack to clear.\n"); 197 198 __raw_i915_write32(dev_priv, FORCEWAKE_VLV, 199 _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); 200 201 if (wait_for_atomic((__raw_i915_read32(dev_priv, 202 FORCEWAKE_ACK_VLV) & 203 FORCEWAKE_KERNEL), 204 FORCEWAKE_ACK_TIMEOUT_MS)) 205 DRM_ERROR("Timed out: waiting for Render to ack.\n"); 206 } 207 208 /* Check for Media Engine */ 209 if (FORCEWAKE_MEDIA & fw_engine) { 210 if (wait_for_atomic((__raw_i915_read32(dev_priv, 211 FORCEWAKE_ACK_MEDIA_VLV) & 212 FORCEWAKE_KERNEL) == 0, 213 FORCEWAKE_ACK_TIMEOUT_MS)) 214 DRM_ERROR("Timed out: Media forcewake old ack to clear.\n"); 215 216 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV, 217 _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL)); 218 219 if (wait_for_atomic((__raw_i915_read32(dev_priv, 220 FORCEWAKE_ACK_MEDIA_VLV) & 221 FORCEWAKE_KERNEL), 222 FORCEWAKE_ACK_TIMEOUT_MS)) 223 DRM_ERROR("Timed out: waiting for media to ack.\n"); 224 } 225 226 /* WaRsForcewakeWaitTC0:vlv */ 227 __gen6_gt_wait_for_thread_c0(dev_priv); 228 229 } 230 231 static void __vlv_force_wake_put(struct drm_i915_private *dev_priv, 232 int fw_engine) 233 { 234 235 /* Check for Render Engine */ 236 if (FORCEWAKE_RENDER & fw_engine) 237 __raw_i915_write32(dev_priv, FORCEWAKE_VLV, 238 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); 239 240 241 /* Check for Media Engine */ 242 if (FORCEWAKE_MEDIA & fw_engine) 243 __raw_i915_write32(dev_priv, FORCEWAKE_MEDIA_VLV, 244 _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL)); 245 246 /* The below doubles as a POSTING_READ */ 247 gen6_gt_check_fifodbg(dev_priv); 248 249 } 250 251 void vlv_force_wake_get(struct drm_i915_private *dev_priv, 252 int fw_engine) 253 { 254 unsigned long irqflags; 255 256 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 257 if (FORCEWAKE_RENDER & fw_engine) { 258 if (dev_priv->uncore.fw_rendercount++ == 0) 259 dev_priv->uncore.funcs.force_wake_get(dev_priv, 260 FORCEWAKE_RENDER); 261 } 262 if (FORCEWAKE_MEDIA & fw_engine) { 263 if (dev_priv->uncore.fw_mediacount++ == 0) 264 dev_priv->uncore.funcs.force_wake_get(dev_priv, 265 FORCEWAKE_MEDIA); 266 } 267 268 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 269 } 270 271 void vlv_force_wake_put(struct drm_i915_private *dev_priv, 272 int fw_engine) 273 { 274 unsigned long irqflags; 275 276 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 277 278 if (FORCEWAKE_RENDER & fw_engine) { 279 WARN_ON(dev_priv->uncore.fw_rendercount == 0); 280 if (--dev_priv->uncore.fw_rendercount == 0) 281 dev_priv->uncore.funcs.force_wake_put(dev_priv, 282 FORCEWAKE_RENDER); 283 } 284 285 if (FORCEWAKE_MEDIA & fw_engine) { 286 WARN_ON(dev_priv->uncore.fw_mediacount == 0); 287 if (--dev_priv->uncore.fw_mediacount == 0) 288 dev_priv->uncore.funcs.force_wake_put(dev_priv, 289 FORCEWAKE_MEDIA); 290 } 291 292 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 293 } 294 295 static void gen6_force_wake_work(struct work_struct *work) 296 { 297 struct drm_i915_private *dev_priv = 298 container_of(work, typeof(*dev_priv), uncore.force_wake_work.work); 299 unsigned long irqflags; 300 301 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 302 if (--dev_priv->uncore.forcewake_count == 0) 303 dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL); 304 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 305 } 306 307 static void intel_uncore_forcewake_reset(struct drm_device *dev) 308 { 309 struct drm_i915_private *dev_priv = dev->dev_private; 310 311 if (IS_VALLEYVIEW(dev)) 312 vlv_force_wake_reset(dev_priv); 313 else if (IS_GEN6(dev) || IS_GEN7(dev)) 314 __gen6_gt_force_wake_reset(dev_priv); 315 316 if (IS_IVYBRIDGE(dev) || IS_HASWELL(dev) || IS_GEN8(dev)) 317 __gen6_gt_force_wake_mt_reset(dev_priv); 318 } 319 320 void intel_uncore_early_sanitize(struct drm_device *dev) 321 { 322 struct drm_i915_private *dev_priv = dev->dev_private; 323 324 if (HAS_FPGA_DBG_UNCLAIMED(dev)) 325 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); 326 327 if ((IS_HASWELL(dev) || IS_BROADWELL(dev)) && 328 (__raw_i915_read32(dev_priv, HSW_EDRAM_PRESENT) == 1)) { 329 /* The docs do not explain exactly how the calculation can be 330 * made. It is somewhat guessable, but for now, it's always 331 * 128MB. 332 * NB: We can't write IDICR yet because we do not have gt funcs 333 * set up */ 334 dev_priv->ellc_size = 128; 335 DRM_INFO("Found %zuMB of eLLC\n", dev_priv->ellc_size); 336 } 337 338 /* clear out old GT FIFO errors */ 339 if (IS_GEN6(dev) || IS_GEN7(dev)) 340 __raw_i915_write32(dev_priv, GTFIFODBG, 341 __raw_i915_read32(dev_priv, GTFIFODBG)); 342 343 intel_uncore_forcewake_reset(dev); 344 } 345 346 void intel_uncore_sanitize(struct drm_device *dev) 347 { 348 struct drm_i915_private *dev_priv = dev->dev_private; 349 u32 reg_val; 350 351 /* BIOS often leaves RC6 enabled, but disable it for hw init */ 352 intel_disable_gt_powersave(dev); 353 354 /* Turn off power gate, require especially for the BIOS less system */ 355 if (IS_VALLEYVIEW(dev)) { 356 357 mutex_lock(&dev_priv->rps.hw_lock); 358 reg_val = vlv_punit_read(dev_priv, PUNIT_REG_PWRGT_STATUS); 359 360 if (reg_val & (RENDER_PWRGT | MEDIA_PWRGT | DISP2D_PWRGT)) 361 vlv_punit_write(dev_priv, PUNIT_REG_PWRGT_CTRL, 0x0); 362 363 mutex_unlock(&dev_priv->rps.hw_lock); 364 365 } 366 } 367 368 /* 369 * Generally this is called implicitly by the register read function. However, 370 * if some sequence requires the GT to not power down then this function should 371 * be called at the beginning of the sequence followed by a call to 372 * gen6_gt_force_wake_put() at the end of the sequence. 373 */ 374 void gen6_gt_force_wake_get(struct drm_i915_private *dev_priv, int fw_engine) 375 { 376 unsigned long irqflags; 377 378 if (!dev_priv->uncore.funcs.force_wake_get) 379 return; 380 381 intel_runtime_pm_get(dev_priv); 382 383 /* Redirect to VLV specific routine */ 384 if (IS_VALLEYVIEW(dev_priv->dev)) 385 return vlv_force_wake_get(dev_priv, fw_engine); 386 387 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 388 if (dev_priv->uncore.forcewake_count++ == 0) 389 dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL); 390 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 391 } 392 393 /* 394 * see gen6_gt_force_wake_get() 395 */ 396 void gen6_gt_force_wake_put(struct drm_i915_private *dev_priv, int fw_engine) 397 { 398 unsigned long irqflags; 399 400 if (!dev_priv->uncore.funcs.force_wake_put) 401 return; 402 403 /* Redirect to VLV specific routine */ 404 if (IS_VALLEYVIEW(dev_priv->dev)) 405 return vlv_force_wake_put(dev_priv, fw_engine); 406 407 408 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 409 if (--dev_priv->uncore.forcewake_count == 0) { 410 dev_priv->uncore.forcewake_count++; 411 mod_delayed_work(dev_priv->wq, 412 &dev_priv->uncore.force_wake_work, 413 1); 414 } 415 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 416 417 intel_runtime_pm_put(dev_priv); 418 } 419 420 /* We give fast paths for the really cool registers */ 421 #define NEEDS_FORCE_WAKE(dev_priv, reg) \ 422 ((reg) < 0x40000 && (reg) != FORCEWAKE) 423 424 static void 425 ilk_dummy_write(struct drm_i915_private *dev_priv) 426 { 427 /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up 428 * the chip from rc6 before touching it for real. MI_MODE is masked, 429 * hence harmless to write 0 into. */ 430 __raw_i915_write32(dev_priv, MI_MODE, 0); 431 } 432 433 static void 434 hsw_unclaimed_reg_clear(struct drm_i915_private *dev_priv, u32 reg) 435 { 436 if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) { 437 DRM_ERROR("Unknown unclaimed register before writing to %x\n", 438 reg); 439 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); 440 } 441 } 442 443 static void 444 hsw_unclaimed_reg_check(struct drm_i915_private *dev_priv, u32 reg) 445 { 446 if (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM) { 447 DRM_ERROR("Unclaimed write to %x\n", reg); 448 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); 449 } 450 } 451 452 static void 453 assert_device_not_suspended(struct drm_i915_private *dev_priv) 454 { 455 WARN_ONCE(HAS_RUNTIME_PM(dev_priv->dev) && dev_priv->pm.suspended, 456 "Device suspended\n"); 457 } 458 459 #define REG_READ_HEADER(x) \ 460 unsigned long irqflags; \ 461 u##x val = 0; \ 462 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags) 463 464 #define REG_READ_FOOTER \ 465 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \ 466 trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \ 467 return val 468 469 #define __gen4_read(x) \ 470 static u##x \ 471 gen4_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ 472 REG_READ_HEADER(x); \ 473 val = __raw_i915_read##x(dev_priv, reg); \ 474 REG_READ_FOOTER; \ 475 } 476 477 #define __gen5_read(x) \ 478 static u##x \ 479 gen5_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ 480 REG_READ_HEADER(x); \ 481 ilk_dummy_write(dev_priv); \ 482 val = __raw_i915_read##x(dev_priv, reg); \ 483 REG_READ_FOOTER; \ 484 } 485 486 #define __gen6_read(x) \ 487 static u##x \ 488 gen6_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ 489 REG_READ_HEADER(x); \ 490 if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ 491 if (dev_priv->uncore.forcewake_count == 0) \ 492 dev_priv->uncore.funcs.force_wake_get(dev_priv, \ 493 FORCEWAKE_ALL); \ 494 val = __raw_i915_read##x(dev_priv, reg); \ 495 if (dev_priv->uncore.forcewake_count == 0) \ 496 dev_priv->uncore.funcs.force_wake_put(dev_priv, \ 497 FORCEWAKE_ALL); \ 498 } else { \ 499 val = __raw_i915_read##x(dev_priv, reg); \ 500 } \ 501 REG_READ_FOOTER; \ 502 } 503 504 #define __vlv_read(x) \ 505 static u##x \ 506 vlv_read##x(struct drm_i915_private *dev_priv, off_t reg, bool trace) { \ 507 unsigned fwengine = 0; \ 508 unsigned *fwcount; \ 509 REG_READ_HEADER(x); \ 510 if (FORCEWAKE_VLV_RENDER_RANGE_OFFSET(reg)) { \ 511 fwengine = FORCEWAKE_RENDER; \ 512 fwcount = &dev_priv->uncore.fw_rendercount; \ 513 } \ 514 else if (FORCEWAKE_VLV_MEDIA_RANGE_OFFSET(reg)) { \ 515 fwengine = FORCEWAKE_MEDIA; \ 516 fwcount = &dev_priv->uncore.fw_mediacount; \ 517 } \ 518 if (fwengine != 0) { \ 519 if ((*fwcount)++ == 0) \ 520 (dev_priv)->uncore.funcs.force_wake_get(dev_priv, \ 521 fwengine); \ 522 val = __raw_i915_read##x(dev_priv, reg); \ 523 if (--(*fwcount) == 0) \ 524 (dev_priv)->uncore.funcs.force_wake_put(dev_priv, \ 525 fwengine); \ 526 } else { \ 527 val = __raw_i915_read##x(dev_priv, reg); \ 528 } \ 529 REG_READ_FOOTER; \ 530 } 531 532 533 __vlv_read(8) 534 __vlv_read(16) 535 __vlv_read(32) 536 __vlv_read(64) 537 __gen6_read(8) 538 __gen6_read(16) 539 __gen6_read(32) 540 __gen6_read(64) 541 __gen5_read(8) 542 __gen5_read(16) 543 __gen5_read(32) 544 __gen5_read(64) 545 __gen4_read(8) 546 __gen4_read(16) 547 __gen4_read(32) 548 __gen4_read(64) 549 550 #undef __vlv_read 551 #undef __gen6_read 552 #undef __gen5_read 553 #undef __gen4_read 554 #undef REG_READ_FOOTER 555 #undef REG_READ_HEADER 556 557 #define REG_WRITE_HEADER \ 558 unsigned long irqflags; \ 559 trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \ 560 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags) 561 562 #define REG_WRITE_FOOTER \ 563 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags) 564 565 #define __gen4_write(x) \ 566 static void \ 567 gen4_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ 568 REG_WRITE_HEADER; \ 569 __raw_i915_write##x(dev_priv, reg, val); \ 570 REG_WRITE_FOOTER; \ 571 } 572 573 #define __gen5_write(x) \ 574 static void \ 575 gen5_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ 576 REG_WRITE_HEADER; \ 577 ilk_dummy_write(dev_priv); \ 578 __raw_i915_write##x(dev_priv, reg, val); \ 579 REG_WRITE_FOOTER; \ 580 } 581 582 #define __gen6_write(x) \ 583 static void \ 584 gen6_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ 585 u32 __fifo_ret = 0; \ 586 REG_WRITE_HEADER; \ 587 if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ 588 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \ 589 } \ 590 assert_device_not_suspended(dev_priv); \ 591 __raw_i915_write##x(dev_priv, reg, val); \ 592 if (unlikely(__fifo_ret)) { \ 593 gen6_gt_check_fifodbg(dev_priv); \ 594 } \ 595 REG_WRITE_FOOTER; \ 596 } 597 598 #define __hsw_write(x) \ 599 static void \ 600 hsw_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ 601 u32 __fifo_ret = 0; \ 602 REG_WRITE_HEADER; \ 603 if (NEEDS_FORCE_WAKE((dev_priv), (reg))) { \ 604 __fifo_ret = __gen6_gt_wait_for_fifo(dev_priv); \ 605 } \ 606 assert_device_not_suspended(dev_priv); \ 607 hsw_unclaimed_reg_clear(dev_priv, reg); \ 608 __raw_i915_write##x(dev_priv, reg, val); \ 609 if (unlikely(__fifo_ret)) { \ 610 gen6_gt_check_fifodbg(dev_priv); \ 611 } \ 612 hsw_unclaimed_reg_check(dev_priv, reg); \ 613 REG_WRITE_FOOTER; \ 614 } 615 616 static const u32 gen8_shadowed_regs[] = { 617 FORCEWAKE_MT, 618 GEN6_RPNSWREQ, 619 GEN6_RC_VIDEO_FREQ, 620 RING_TAIL(RENDER_RING_BASE), 621 RING_TAIL(GEN6_BSD_RING_BASE), 622 RING_TAIL(VEBOX_RING_BASE), 623 RING_TAIL(BLT_RING_BASE), 624 /* TODO: Other registers are not yet used */ 625 }; 626 627 static bool is_gen8_shadowed(struct drm_i915_private *dev_priv, u32 reg) 628 { 629 int i; 630 for (i = 0; i < ARRAY_SIZE(gen8_shadowed_regs); i++) 631 if (reg == gen8_shadowed_regs[i]) 632 return true; 633 634 return false; 635 } 636 637 #define __gen8_write(x) \ 638 static void \ 639 gen8_write##x(struct drm_i915_private *dev_priv, off_t reg, u##x val, bool trace) { \ 640 REG_WRITE_HEADER; \ 641 if (reg < 0x40000 && !is_gen8_shadowed(dev_priv, reg)) { \ 642 if (dev_priv->uncore.forcewake_count == 0) \ 643 dev_priv->uncore.funcs.force_wake_get(dev_priv, \ 644 FORCEWAKE_ALL); \ 645 __raw_i915_write##x(dev_priv, reg, val); \ 646 if (dev_priv->uncore.forcewake_count == 0) \ 647 dev_priv->uncore.funcs.force_wake_put(dev_priv, \ 648 FORCEWAKE_ALL); \ 649 } else { \ 650 __raw_i915_write##x(dev_priv, reg, val); \ 651 } \ 652 REG_WRITE_FOOTER; \ 653 } 654 655 __gen8_write(8) 656 __gen8_write(16) 657 __gen8_write(32) 658 __gen8_write(64) 659 __hsw_write(8) 660 __hsw_write(16) 661 __hsw_write(32) 662 __hsw_write(64) 663 __gen6_write(8) 664 __gen6_write(16) 665 __gen6_write(32) 666 __gen6_write(64) 667 __gen5_write(8) 668 __gen5_write(16) 669 __gen5_write(32) 670 __gen5_write(64) 671 __gen4_write(8) 672 __gen4_write(16) 673 __gen4_write(32) 674 __gen4_write(64) 675 676 #undef __gen8_write 677 #undef __hsw_write 678 #undef __gen6_write 679 #undef __gen5_write 680 #undef __gen4_write 681 #undef REG_WRITE_FOOTER 682 #undef REG_WRITE_HEADER 683 684 void intel_uncore_init(struct drm_device *dev) 685 { 686 struct drm_i915_private *dev_priv = dev->dev_private; 687 688 INIT_DELAYED_WORK(&dev_priv->uncore.force_wake_work, 689 gen6_force_wake_work); 690 691 if (IS_VALLEYVIEW(dev)) { 692 dev_priv->uncore.funcs.force_wake_get = __vlv_force_wake_get; 693 dev_priv->uncore.funcs.force_wake_put = __vlv_force_wake_put; 694 } else if (IS_HASWELL(dev) || IS_GEN8(dev)) { 695 dev_priv->uncore.funcs.force_wake_get = __gen6_gt_force_wake_mt_get; 696 dev_priv->uncore.funcs.force_wake_put = __gen6_gt_force_wake_mt_put; 697 } else if (IS_IVYBRIDGE(dev)) { 698 u32 ecobus; 699 700 /* IVB configs may use multi-threaded forcewake */ 701 702 /* A small trick here - if the bios hasn't configured 703 * MT forcewake, and if the device is in RC6, then 704 * force_wake_mt_get will not wake the device and the 705 * ECOBUS read will return zero. Which will be 706 * (correctly) interpreted by the test below as MT 707 * forcewake being disabled. 708 */ 709 mutex_lock(&dev->struct_mutex); 710 __gen6_gt_force_wake_mt_get(dev_priv, FORCEWAKE_ALL); 711 ecobus = __raw_i915_read32(dev_priv, ECOBUS); 712 __gen6_gt_force_wake_mt_put(dev_priv, FORCEWAKE_ALL); 713 mutex_unlock(&dev->struct_mutex); 714 715 if (ecobus & FORCEWAKE_MT_ENABLE) { 716 dev_priv->uncore.funcs.force_wake_get = 717 __gen6_gt_force_wake_mt_get; 718 dev_priv->uncore.funcs.force_wake_put = 719 __gen6_gt_force_wake_mt_put; 720 } else { 721 DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n"); 722 DRM_INFO("when using vblank-synced partial screen updates.\n"); 723 dev_priv->uncore.funcs.force_wake_get = 724 __gen6_gt_force_wake_get; 725 dev_priv->uncore.funcs.force_wake_put = 726 __gen6_gt_force_wake_put; 727 } 728 } else if (IS_GEN6(dev)) { 729 dev_priv->uncore.funcs.force_wake_get = 730 __gen6_gt_force_wake_get; 731 dev_priv->uncore.funcs.force_wake_put = 732 __gen6_gt_force_wake_put; 733 } 734 735 switch (INTEL_INFO(dev)->gen) { 736 default: 737 dev_priv->uncore.funcs.mmio_writeb = gen8_write8; 738 dev_priv->uncore.funcs.mmio_writew = gen8_write16; 739 dev_priv->uncore.funcs.mmio_writel = gen8_write32; 740 dev_priv->uncore.funcs.mmio_writeq = gen8_write64; 741 dev_priv->uncore.funcs.mmio_readb = gen6_read8; 742 dev_priv->uncore.funcs.mmio_readw = gen6_read16; 743 dev_priv->uncore.funcs.mmio_readl = gen6_read32; 744 dev_priv->uncore.funcs.mmio_readq = gen6_read64; 745 break; 746 case 7: 747 case 6: 748 if (IS_HASWELL(dev)) { 749 dev_priv->uncore.funcs.mmio_writeb = hsw_write8; 750 dev_priv->uncore.funcs.mmio_writew = hsw_write16; 751 dev_priv->uncore.funcs.mmio_writel = hsw_write32; 752 dev_priv->uncore.funcs.mmio_writeq = hsw_write64; 753 } else { 754 dev_priv->uncore.funcs.mmio_writeb = gen6_write8; 755 dev_priv->uncore.funcs.mmio_writew = gen6_write16; 756 dev_priv->uncore.funcs.mmio_writel = gen6_write32; 757 dev_priv->uncore.funcs.mmio_writeq = gen6_write64; 758 } 759 760 if (IS_VALLEYVIEW(dev)) { 761 dev_priv->uncore.funcs.mmio_readb = vlv_read8; 762 dev_priv->uncore.funcs.mmio_readw = vlv_read16; 763 dev_priv->uncore.funcs.mmio_readl = vlv_read32; 764 dev_priv->uncore.funcs.mmio_readq = vlv_read64; 765 } else { 766 dev_priv->uncore.funcs.mmio_readb = gen6_read8; 767 dev_priv->uncore.funcs.mmio_readw = gen6_read16; 768 dev_priv->uncore.funcs.mmio_readl = gen6_read32; 769 dev_priv->uncore.funcs.mmio_readq = gen6_read64; 770 } 771 break; 772 case 5: 773 dev_priv->uncore.funcs.mmio_writeb = gen5_write8; 774 dev_priv->uncore.funcs.mmio_writew = gen5_write16; 775 dev_priv->uncore.funcs.mmio_writel = gen5_write32; 776 dev_priv->uncore.funcs.mmio_writeq = gen5_write64; 777 dev_priv->uncore.funcs.mmio_readb = gen5_read8; 778 dev_priv->uncore.funcs.mmio_readw = gen5_read16; 779 dev_priv->uncore.funcs.mmio_readl = gen5_read32; 780 dev_priv->uncore.funcs.mmio_readq = gen5_read64; 781 break; 782 case 4: 783 case 3: 784 case 2: 785 dev_priv->uncore.funcs.mmio_writeb = gen4_write8; 786 dev_priv->uncore.funcs.mmio_writew = gen4_write16; 787 dev_priv->uncore.funcs.mmio_writel = gen4_write32; 788 dev_priv->uncore.funcs.mmio_writeq = gen4_write64; 789 dev_priv->uncore.funcs.mmio_readb = gen4_read8; 790 dev_priv->uncore.funcs.mmio_readw = gen4_read16; 791 dev_priv->uncore.funcs.mmio_readl = gen4_read32; 792 dev_priv->uncore.funcs.mmio_readq = gen4_read64; 793 break; 794 } 795 } 796 797 void intel_uncore_fini(struct drm_device *dev) 798 { 799 struct drm_i915_private *dev_priv = dev->dev_private; 800 801 flush_delayed_work(&dev_priv->uncore.force_wake_work); 802 803 /* Paranoia: make sure we have disabled everything before we exit. */ 804 intel_uncore_sanitize(dev); 805 } 806 807 static const struct register_whitelist { 808 uint64_t offset; 809 uint32_t size; 810 uint32_t gen_bitmask; /* support gens, 0x10 for 4, 0x30 for 4 and 5, etc. */ 811 } whitelist[] = { 812 { RING_TIMESTAMP(RENDER_RING_BASE), 8, 0x1F0 }, 813 }; 814 815 int i915_reg_read_ioctl(struct drm_device *dev, 816 void *data, struct drm_file *file) 817 { 818 struct drm_i915_private *dev_priv = dev->dev_private; 819 struct drm_i915_reg_read *reg = data; 820 struct register_whitelist const *entry = whitelist; 821 int i; 822 823 for (i = 0; i < ARRAY_SIZE(whitelist); i++, entry++) { 824 if (entry->offset == reg->offset && 825 (1 << INTEL_INFO(dev)->gen & entry->gen_bitmask)) 826 break; 827 } 828 829 if (i == ARRAY_SIZE(whitelist)) 830 return -EINVAL; 831 832 switch (entry->size) { 833 case 8: 834 reg->val = I915_READ64(reg->offset); 835 break; 836 case 4: 837 reg->val = I915_READ(reg->offset); 838 break; 839 case 2: 840 reg->val = I915_READ16(reg->offset); 841 break; 842 case 1: 843 reg->val = I915_READ8(reg->offset); 844 break; 845 default: 846 WARN_ON(1); 847 return -EINVAL; 848 } 849 850 return 0; 851 } 852 853 int i915_get_reset_stats_ioctl(struct drm_device *dev, 854 void *data, struct drm_file *file) 855 { 856 struct drm_i915_private *dev_priv = dev->dev_private; 857 struct drm_i915_reset_stats *args = data; 858 struct i915_ctx_hang_stats *hs; 859 int ret; 860 861 if (args->flags || args->pad) 862 return -EINVAL; 863 864 if (args->ctx_id == DEFAULT_CONTEXT_ID && !capable(CAP_SYS_ADMIN)) 865 return -EPERM; 866 867 ret = mutex_lock_interruptible(&dev->struct_mutex); 868 if (ret) 869 return ret; 870 871 hs = i915_gem_context_get_hang_stats(dev, file, args->ctx_id); 872 if (IS_ERR(hs)) { 873 mutex_unlock(&dev->struct_mutex); 874 return PTR_ERR(hs); 875 } 876 877 if (capable(CAP_SYS_ADMIN)) 878 args->reset_count = i915_reset_count(&dev_priv->gpu_error); 879 else 880 args->reset_count = 0; 881 882 args->batch_active = hs->batch_active; 883 args->batch_pending = hs->batch_pending; 884 885 mutex_unlock(&dev->struct_mutex); 886 887 return 0; 888 } 889 890 static int i965_reset_complete(struct drm_device *dev) 891 { 892 u8 gdrst; 893 pci_read_config_byte(dev->pdev, I965_GDRST, &gdrst); 894 return (gdrst & GRDOM_RESET_ENABLE) == 0; 895 } 896 897 static int i965_do_reset(struct drm_device *dev) 898 { 899 int ret; 900 901 /* 902 * Set the domains we want to reset (GRDOM/bits 2 and 3) as 903 * well as the reset bit (GR/bit 0). Setting the GR bit 904 * triggers the reset; when done, the hardware will clear it. 905 */ 906 pci_write_config_byte(dev->pdev, I965_GDRST, 907 GRDOM_RENDER | GRDOM_RESET_ENABLE); 908 ret = wait_for(i965_reset_complete(dev), 500); 909 if (ret) 910 return ret; 911 912 /* We can't reset render&media without also resetting display ... */ 913 pci_write_config_byte(dev->pdev, I965_GDRST, 914 GRDOM_MEDIA | GRDOM_RESET_ENABLE); 915 916 ret = wait_for(i965_reset_complete(dev), 500); 917 if (ret) 918 return ret; 919 920 pci_write_config_byte(dev->pdev, I965_GDRST, 0); 921 922 return 0; 923 } 924 925 static int ironlake_do_reset(struct drm_device *dev) 926 { 927 struct drm_i915_private *dev_priv = dev->dev_private; 928 u32 gdrst; 929 int ret; 930 931 gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR); 932 gdrst &= ~GRDOM_MASK; 933 I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, 934 gdrst | GRDOM_RENDER | GRDOM_RESET_ENABLE); 935 ret = wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500); 936 if (ret) 937 return ret; 938 939 /* We can't reset render&media without also resetting display ... */ 940 gdrst = I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR); 941 gdrst &= ~GRDOM_MASK; 942 I915_WRITE(MCHBAR_MIRROR_BASE + ILK_GDSR, 943 gdrst | GRDOM_MEDIA | GRDOM_RESET_ENABLE); 944 return wait_for(I915_READ(MCHBAR_MIRROR_BASE + ILK_GDSR) & 0x1, 500); 945 } 946 947 static int gen6_do_reset(struct drm_device *dev) 948 { 949 struct drm_i915_private *dev_priv = dev->dev_private; 950 int ret; 951 unsigned long irqflags; 952 953 /* Hold uncore.lock across reset to prevent any register access 954 * with forcewake not set correctly 955 */ 956 spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); 957 958 /* Reset the chip */ 959 960 /* GEN6_GDRST is not in the gt power well, no need to check 961 * for fifo space for the write or forcewake the chip for 962 * the read 963 */ 964 __raw_i915_write32(dev_priv, GEN6_GDRST, GEN6_GRDOM_FULL); 965 966 /* Spin waiting for the device to ack the reset request */ 967 ret = wait_for((__raw_i915_read32(dev_priv, GEN6_GDRST) & GEN6_GRDOM_FULL) == 0, 500); 968 969 intel_uncore_forcewake_reset(dev); 970 971 /* If reset with a user forcewake, try to restore, otherwise turn it off */ 972 if (dev_priv->uncore.forcewake_count) 973 dev_priv->uncore.funcs.force_wake_get(dev_priv, FORCEWAKE_ALL); 974 else 975 dev_priv->uncore.funcs.force_wake_put(dev_priv, FORCEWAKE_ALL); 976 977 /* Restore fifo count */ 978 dev_priv->uncore.fifo_count = __raw_i915_read32(dev_priv, GTFIFOCTL) & GT_FIFO_FREE_ENTRIES_MASK; 979 980 spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); 981 return ret; 982 } 983 984 int intel_gpu_reset(struct drm_device *dev) 985 { 986 switch (INTEL_INFO(dev)->gen) { 987 case 8: 988 case 7: 989 case 6: return gen6_do_reset(dev); 990 case 5: return ironlake_do_reset(dev); 991 case 4: return i965_do_reset(dev); 992 default: return -ENODEV; 993 } 994 } 995 996 void intel_uncore_check_errors(struct drm_device *dev) 997 { 998 struct drm_i915_private *dev_priv = dev->dev_private; 999 1000 if (HAS_FPGA_DBG_UNCLAIMED(dev) && 1001 (__raw_i915_read32(dev_priv, FPGA_DBG) & FPGA_DBG_RM_NOCLAIM)) { 1002 DRM_ERROR("Unclaimed register before interrupt\n"); 1003 __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM); 1004 } 1005 } 1006