1 /* 2 * SPDX-License-Identifier: MIT 3 * 4 * Copyright © 2019 Intel Corporation 5 */ 6 7 #include <drm/i915_drm.h> 8 9 #include "i915_drv.h" 10 #include "intel_gt.h" 11 #include "intel_gt_irq.h" 12 #include "intel_gt_pm_irq.h" 13 #include "intel_rps.h" 14 #include "intel_sideband.h" 15 #ifdef __linux__ 16 #include "../../../platform/x86/intel_ips.h" 17 #endif 18 19 /* 20 * Lock protecting IPS related data structures 21 */ 22 static DEFINE_SPINLOCK(mchdev_lock); 23 24 static struct intel_gt *rps_to_gt(struct intel_rps *rps) 25 { 26 return container_of(rps, struct intel_gt, rps); 27 } 28 29 static struct drm_i915_private *rps_to_i915(struct intel_rps *rps) 30 { 31 return rps_to_gt(rps)->i915; 32 } 33 34 static struct intel_uncore *rps_to_uncore(struct intel_rps *rps) 35 { 36 return rps_to_gt(rps)->uncore; 37 } 38 39 static u32 rps_pm_sanitize_mask(struct intel_rps *rps, u32 mask) 40 { 41 return mask & ~rps->pm_intrmsk_mbz; 42 } 43 44 static inline void set(struct intel_uncore *uncore, i915_reg_t reg, u32 val) 45 { 46 intel_uncore_write_fw(uncore, reg, val); 47 } 48 49 static u32 rps_pm_mask(struct intel_rps *rps, u8 val) 50 { 51 u32 mask = 0; 52 53 /* We use UP_EI_EXPIRED interrupts for both up/down in manual mode */ 54 if (val > rps->min_freq_softlimit) 55 mask |= (GEN6_PM_RP_UP_EI_EXPIRED | 56 GEN6_PM_RP_DOWN_THRESHOLD | 57 GEN6_PM_RP_DOWN_TIMEOUT); 58 59 if (val < rps->max_freq_softlimit) 60 mask |= GEN6_PM_RP_UP_EI_EXPIRED | GEN6_PM_RP_UP_THRESHOLD; 61 62 mask &= READ_ONCE(rps->pm_events); 63 64 return rps_pm_sanitize_mask(rps, ~mask); 65 } 66 67 static void rps_reset_ei(struct intel_rps *rps) 68 { 69 memset(&rps->ei, 0, sizeof(rps->ei)); 70 } 71 72 static void rps_enable_interrupts(struct intel_rps *rps) 73 { 74 struct intel_gt *gt = rps_to_gt(rps); 75 u32 events; 76 77 rps_reset_ei(rps); 78 79 if (IS_VALLEYVIEW(gt->i915)) 80 /* WaGsvRC0ResidencyMethod:vlv */ 81 events = GEN6_PM_RP_UP_EI_EXPIRED; 82 else 83 events = (GEN6_PM_RP_UP_THRESHOLD | 84 GEN6_PM_RP_DOWN_THRESHOLD | 85 GEN6_PM_RP_DOWN_TIMEOUT); 86 WRITE_ONCE(rps->pm_events, events); 87 88 spin_lock_irq(>->irq_lock); 89 gen6_gt_pm_enable_irq(gt, rps->pm_events); 90 spin_unlock_irq(>->irq_lock); 91 92 intel_uncore_write(gt->uncore, 93 GEN6_PMINTRMSK, rps_pm_mask(rps, rps->last_freq)); 94 } 95 96 static void gen6_rps_reset_interrupts(struct intel_rps *rps) 97 { 98 gen6_gt_pm_reset_iir(rps_to_gt(rps), GEN6_PM_RPS_EVENTS); 99 } 100 101 static void gen11_rps_reset_interrupts(struct intel_rps *rps) 102 { 103 while (gen11_gt_reset_one_iir(rps_to_gt(rps), 0, GEN11_GTPM)) 104 ; 105 } 106 107 static void rps_reset_interrupts(struct intel_rps *rps) 108 { 109 struct intel_gt *gt = rps_to_gt(rps); 110 111 spin_lock_irq(>->irq_lock); 112 if (INTEL_GEN(gt->i915) >= 11) 113 gen11_rps_reset_interrupts(rps); 114 else 115 gen6_rps_reset_interrupts(rps); 116 117 rps->pm_iir = 0; 118 spin_unlock_irq(>->irq_lock); 119 } 120 121 static void rps_disable_interrupts(struct intel_rps *rps) 122 { 123 struct intel_gt *gt = rps_to_gt(rps); 124 125 WRITE_ONCE(rps->pm_events, 0); 126 127 intel_uncore_write(gt->uncore, 128 GEN6_PMINTRMSK, rps_pm_sanitize_mask(rps, ~0u)); 129 130 spin_lock_irq(>->irq_lock); 131 gen6_gt_pm_disable_irq(gt, GEN6_PM_RPS_EVENTS); 132 spin_unlock_irq(>->irq_lock); 133 134 intel_synchronize_irq(gt->i915); 135 136 /* 137 * Now that we will not be generating any more work, flush any 138 * outstanding tasks. As we are called on the RPS idle path, 139 * we will reset the GPU to minimum frequencies, so the current 140 * state of the worker can be discarded. 141 */ 142 cancel_work_sync(&rps->work); 143 144 rps_reset_interrupts(rps); 145 } 146 147 static const struct cparams { 148 u16 i; 149 u16 t; 150 u16 m; 151 u16 c; 152 } cparams[] = { 153 { 1, 1333, 301, 28664 }, 154 { 1, 1066, 294, 24460 }, 155 { 1, 800, 294, 25192 }, 156 { 0, 1333, 276, 27605 }, 157 { 0, 1066, 276, 27605 }, 158 { 0, 800, 231, 23784 }, 159 }; 160 161 static void gen5_rps_init(struct intel_rps *rps) 162 { 163 struct drm_i915_private *i915 = rps_to_i915(rps); 164 struct intel_uncore *uncore = rps_to_uncore(rps); 165 u8 fmax, fmin, fstart; 166 u32 rgvmodectl; 167 int c_m, i; 168 169 if (i915->fsb_freq <= 3200) 170 c_m = 0; 171 else if (i915->fsb_freq <= 4800) 172 c_m = 1; 173 else 174 c_m = 2; 175 176 for (i = 0; i < ARRAY_SIZE(cparams); i++) { 177 if (cparams[i].i == c_m && cparams[i].t == i915->mem_freq) { 178 rps->ips.m = cparams[i].m; 179 rps->ips.c = cparams[i].c; 180 break; 181 } 182 } 183 184 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 185 186 /* Set up min, max, and cur for interrupt handling */ 187 fmax = (rgvmodectl & MEMMODE_FMAX_MASK) >> MEMMODE_FMAX_SHIFT; 188 fmin = (rgvmodectl & MEMMODE_FMIN_MASK); 189 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 190 MEMMODE_FSTART_SHIFT; 191 DRM_DEBUG_DRIVER("fmax: %d, fmin: %d, fstart: %d\n", 192 fmax, fmin, fstart); 193 194 rps->min_freq = fmax; 195 rps->max_freq = fmin; 196 197 rps->idle_freq = rps->min_freq; 198 rps->cur_freq = rps->idle_freq; 199 } 200 201 static unsigned long 202 __ips_chipset_val(struct intel_ips *ips) 203 { 204 struct intel_uncore *uncore = 205 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 206 unsigned long now = jiffies_to_msecs(jiffies), dt; 207 unsigned long result; 208 u64 total, delta; 209 210 lockdep_assert_held(&mchdev_lock); 211 212 /* 213 * Prevent division-by-zero if we are asking too fast. 214 * Also, we don't get interesting results if we are polling 215 * faster than once in 10ms, so just return the saved value 216 * in such cases. 217 */ 218 dt = now - ips->last_time1; 219 if (dt <= 10) 220 return ips->chipset_power; 221 222 /* FIXME: handle per-counter overflow */ 223 total = intel_uncore_read(uncore, DMIEC); 224 total += intel_uncore_read(uncore, DDREC); 225 total += intel_uncore_read(uncore, CSIEC); 226 227 delta = total - ips->last_count1; 228 229 result = div_u64(div_u64(ips->m * delta, dt) + ips->c, 10); 230 231 ips->last_count1 = total; 232 ips->last_time1 = now; 233 234 ips->chipset_power = result; 235 236 return result; 237 } 238 239 static unsigned long ips_mch_val(struct intel_uncore *uncore) 240 { 241 unsigned int m, x, b; 242 u32 tsfs; 243 244 tsfs = intel_uncore_read(uncore, TSFS); 245 x = intel_uncore_read8(uncore, TR1); 246 247 b = tsfs & TSFS_INTR_MASK; 248 m = (tsfs & TSFS_SLOPE_MASK) >> TSFS_SLOPE_SHIFT; 249 250 return m * x / 127 - b; 251 } 252 253 static int _pxvid_to_vd(u8 pxvid) 254 { 255 if (pxvid == 0) 256 return 0; 257 258 if (pxvid >= 8 && pxvid < 31) 259 pxvid = 31; 260 261 return (pxvid + 2) * 125; 262 } 263 264 static u32 pvid_to_extvid(struct drm_i915_private *i915, u8 pxvid) 265 { 266 const int vd = _pxvid_to_vd(pxvid); 267 268 if (INTEL_INFO(i915)->is_mobile) 269 return max(vd - 1125, 0); 270 271 return vd; 272 } 273 274 static void __gen5_ips_update(struct intel_ips *ips) 275 { 276 struct intel_uncore *uncore = 277 rps_to_uncore(container_of(ips, struct intel_rps, ips)); 278 u64 now, delta, dt; 279 u32 count; 280 281 lockdep_assert_held(&mchdev_lock); 282 283 now = ktime_get_raw_ns(); 284 dt = now - ips->last_time2; 285 do_div(dt, NSEC_PER_MSEC); 286 287 /* Don't divide by 0 */ 288 if (dt <= 10) 289 return; 290 291 count = intel_uncore_read(uncore, GFXEC); 292 delta = count - ips->last_count2; 293 294 ips->last_count2 = count; 295 ips->last_time2 = now; 296 297 /* More magic constants... */ 298 ips->gfx_power = div_u64(delta * 1181, dt * 10); 299 } 300 301 static void gen5_rps_update(struct intel_rps *rps) 302 { 303 spin_lock_irq(&mchdev_lock); 304 __gen5_ips_update(&rps->ips); 305 spin_unlock_irq(&mchdev_lock); 306 } 307 308 static bool gen5_rps_set(struct intel_rps *rps, u8 val) 309 { 310 struct intel_uncore *uncore = rps_to_uncore(rps); 311 u16 rgvswctl; 312 313 lockdep_assert_held(&mchdev_lock); 314 315 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 316 if (rgvswctl & MEMCTL_CMD_STS) { 317 DRM_DEBUG("gpu busy, RCS change rejected\n"); 318 return false; /* still busy with another command */ 319 } 320 321 /* Invert the frequency bin into an ips delay */ 322 val = rps->max_freq - val; 323 val = rps->min_freq + val; 324 325 rgvswctl = 326 (MEMCTL_CMD_CHFREQ << MEMCTL_CMD_SHIFT) | 327 (val << MEMCTL_FREQ_SHIFT) | 328 MEMCTL_SFCAVM; 329 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 330 intel_uncore_posting_read16(uncore, MEMSWCTL); 331 332 rgvswctl |= MEMCTL_CMD_STS; 333 intel_uncore_write16(uncore, MEMSWCTL, rgvswctl); 334 335 return true; 336 } 337 338 static unsigned long intel_pxfreq(u32 vidfreq) 339 { 340 int div = (vidfreq & 0x3f0000) >> 16; 341 int post = (vidfreq & 0x3000) >> 12; 342 int pre = (vidfreq & 0x7); 343 344 if (!pre) 345 return 0; 346 347 return div * 133333 / (pre << post); 348 } 349 350 static unsigned int init_emon(struct intel_uncore *uncore) 351 { 352 u8 pxw[16]; 353 int i; 354 355 /* Disable to program */ 356 intel_uncore_write(uncore, ECR, 0); 357 intel_uncore_posting_read(uncore, ECR); 358 359 /* Program energy weights for various events */ 360 intel_uncore_write(uncore, SDEW, 0x15040d00); 361 intel_uncore_write(uncore, CSIEW0, 0x007f0000); 362 intel_uncore_write(uncore, CSIEW1, 0x1e220004); 363 intel_uncore_write(uncore, CSIEW2, 0x04000004); 364 365 for (i = 0; i < 5; i++) 366 intel_uncore_write(uncore, PEW(i), 0); 367 for (i = 0; i < 3; i++) 368 intel_uncore_write(uncore, DEW(i), 0); 369 370 /* Program P-state weights to account for frequency power adjustment */ 371 for (i = 0; i < 16; i++) { 372 u32 pxvidfreq = intel_uncore_read(uncore, PXVFREQ(i)); 373 unsigned int freq = intel_pxfreq(pxvidfreq); 374 unsigned int vid = 375 (pxvidfreq & PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 376 unsigned int val; 377 378 val = vid * vid * freq / 1000 * 255; 379 val /= 127 * 127 * 900; 380 381 pxw[i] = val; 382 } 383 /* Render standby states get 0 weight */ 384 pxw[14] = 0; 385 pxw[15] = 0; 386 387 for (i = 0; i < 4; i++) { 388 intel_uncore_write(uncore, PXW(i), 389 pxw[i * 4 + 0] << 24 | 390 pxw[i * 4 + 1] << 16 | 391 pxw[i * 4 + 2] << 8 | 392 pxw[i * 4 + 3] << 0); 393 } 394 395 /* Adjust magic regs to magic values (more experimental results) */ 396 intel_uncore_write(uncore, OGW0, 0); 397 intel_uncore_write(uncore, OGW1, 0); 398 intel_uncore_write(uncore, EG0, 0x00007f00); 399 intel_uncore_write(uncore, EG1, 0x0000000e); 400 intel_uncore_write(uncore, EG2, 0x000e0000); 401 intel_uncore_write(uncore, EG3, 0x68000300); 402 intel_uncore_write(uncore, EG4, 0x42000000); 403 intel_uncore_write(uncore, EG5, 0x00140031); 404 intel_uncore_write(uncore, EG6, 0); 405 intel_uncore_write(uncore, EG7, 0); 406 407 for (i = 0; i < 8; i++) 408 intel_uncore_write(uncore, PXWL(i), 0); 409 410 /* Enable PMON + select events */ 411 intel_uncore_write(uncore, ECR, 0x80000019); 412 413 return intel_uncore_read(uncore, LCFUSE02) & LCFUSE_HIV_MASK; 414 } 415 416 static bool gen5_rps_enable(struct intel_rps *rps) 417 { 418 struct intel_uncore *uncore = rps_to_uncore(rps); 419 u8 fstart, vstart; 420 u32 rgvmodectl; 421 422 spin_lock_irq(&mchdev_lock); 423 424 rgvmodectl = intel_uncore_read(uncore, MEMMODECTL); 425 426 /* Enable temp reporting */ 427 intel_uncore_write16(uncore, PMMISC, 428 intel_uncore_read16(uncore, PMMISC) | MCPPCE_EN); 429 intel_uncore_write16(uncore, TSC1, 430 intel_uncore_read16(uncore, TSC1) | TSE); 431 432 /* 100ms RC evaluation intervals */ 433 intel_uncore_write(uncore, RCUPEI, 100000); 434 intel_uncore_write(uncore, RCDNEI, 100000); 435 436 /* Set max/min thresholds to 90ms and 80ms respectively */ 437 intel_uncore_write(uncore, RCBMAXAVG, 90000); 438 intel_uncore_write(uncore, RCBMINAVG, 80000); 439 440 intel_uncore_write(uncore, MEMIHYST, 1); 441 442 /* Set up min, max, and cur for interrupt handling */ 443 fstart = (rgvmodectl & MEMMODE_FSTART_MASK) >> 444 MEMMODE_FSTART_SHIFT; 445 446 vstart = (intel_uncore_read(uncore, PXVFREQ(fstart)) & 447 PXVFREQ_PX_MASK) >> PXVFREQ_PX_SHIFT; 448 449 intel_uncore_write(uncore, 450 MEMINTREN, 451 MEMINT_CX_SUPR_EN | MEMINT_EVAL_CHG_EN); 452 453 intel_uncore_write(uncore, VIDSTART, vstart); 454 intel_uncore_posting_read(uncore, VIDSTART); 455 456 rgvmodectl |= MEMMODE_SWMODE_EN; 457 intel_uncore_write(uncore, MEMMODECTL, rgvmodectl); 458 459 if (wait_for_atomic((intel_uncore_read(uncore, MEMSWCTL) & 460 MEMCTL_CMD_STS) == 0, 10)) 461 DRM_ERROR("stuck trying to change perf mode\n"); 462 mdelay(1); 463 464 gen5_rps_set(rps, rps->cur_freq); 465 466 rps->ips.last_count1 = intel_uncore_read(uncore, DMIEC); 467 rps->ips.last_count1 += intel_uncore_read(uncore, DDREC); 468 rps->ips.last_count1 += intel_uncore_read(uncore, CSIEC); 469 rps->ips.last_time1 = jiffies_to_msecs(jiffies); 470 471 rps->ips.last_count2 = intel_uncore_read(uncore, GFXEC); 472 rps->ips.last_time2 = ktime_get_raw_ns(); 473 474 spin_unlock_irq(&mchdev_lock); 475 476 rps->ips.corr = init_emon(uncore); 477 478 return true; 479 } 480 481 static void gen5_rps_disable(struct intel_rps *rps) 482 { 483 struct intel_uncore *uncore = rps_to_uncore(rps); 484 u16 rgvswctl; 485 486 spin_lock_irq(&mchdev_lock); 487 488 rgvswctl = intel_uncore_read16(uncore, MEMSWCTL); 489 490 /* Ack interrupts, disable EFC interrupt */ 491 intel_uncore_write(uncore, MEMINTREN, 492 intel_uncore_read(uncore, MEMINTREN) & 493 ~MEMINT_EVAL_CHG_EN); 494 intel_uncore_write(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 495 intel_uncore_write(uncore, DEIER, 496 intel_uncore_read(uncore, DEIER) & ~DE_PCU_EVENT); 497 intel_uncore_write(uncore, DEIIR, DE_PCU_EVENT); 498 intel_uncore_write(uncore, DEIMR, 499 intel_uncore_read(uncore, DEIMR) | DE_PCU_EVENT); 500 501 /* Go back to the starting frequency */ 502 gen5_rps_set(rps, rps->idle_freq); 503 mdelay(1); 504 rgvswctl |= MEMCTL_CMD_STS; 505 intel_uncore_write(uncore, MEMSWCTL, rgvswctl); 506 mdelay(1); 507 508 spin_unlock_irq(&mchdev_lock); 509 } 510 511 static u32 rps_limits(struct intel_rps *rps, u8 val) 512 { 513 u32 limits; 514 515 /* 516 * Only set the down limit when we've reached the lowest level to avoid 517 * getting more interrupts, otherwise leave this clear. This prevents a 518 * race in the hw when coming out of rc6: There's a tiny window where 519 * the hw runs at the minimal clock before selecting the desired 520 * frequency, if the down threshold expires in that window we will not 521 * receive a down interrupt. 522 */ 523 if (INTEL_GEN(rps_to_i915(rps)) >= 9) { 524 limits = rps->max_freq_softlimit << 23; 525 if (val <= rps->min_freq_softlimit) 526 limits |= rps->min_freq_softlimit << 14; 527 } else { 528 limits = rps->max_freq_softlimit << 24; 529 if (val <= rps->min_freq_softlimit) 530 limits |= rps->min_freq_softlimit << 16; 531 } 532 533 return limits; 534 } 535 536 static void rps_set_power(struct intel_rps *rps, int new_power) 537 { 538 struct intel_uncore *uncore = rps_to_uncore(rps); 539 struct drm_i915_private *i915 = rps_to_i915(rps); 540 u32 threshold_up = 0, threshold_down = 0; /* in % */ 541 u32 ei_up = 0, ei_down = 0; 542 543 lockdep_assert_held(&rps->power.mutex); 544 545 if (new_power == rps->power.mode) 546 return; 547 548 /* Note the units here are not exactly 1us, but 1280ns. */ 549 switch (new_power) { 550 case LOW_POWER: 551 /* Upclock if more than 95% busy over 16ms */ 552 ei_up = 16000; 553 threshold_up = 95; 554 555 /* Downclock if less than 85% busy over 32ms */ 556 ei_down = 32000; 557 threshold_down = 85; 558 break; 559 560 case BETWEEN: 561 /* Upclock if more than 90% busy over 13ms */ 562 ei_up = 13000; 563 threshold_up = 90; 564 565 /* Downclock if less than 75% busy over 32ms */ 566 ei_down = 32000; 567 threshold_down = 75; 568 break; 569 570 case HIGH_POWER: 571 /* Upclock if more than 85% busy over 10ms */ 572 ei_up = 10000; 573 threshold_up = 85; 574 575 /* Downclock if less than 60% busy over 32ms */ 576 ei_down = 32000; 577 threshold_down = 60; 578 break; 579 } 580 581 /* When byt can survive without system hang with dynamic 582 * sw freq adjustments, this restriction can be lifted. 583 */ 584 if (IS_VALLEYVIEW(i915)) 585 goto skip_hw_write; 586 587 set(uncore, GEN6_RP_UP_EI, GT_INTERVAL_FROM_US(i915, ei_up)); 588 set(uncore, GEN6_RP_UP_THRESHOLD, 589 GT_INTERVAL_FROM_US(i915, ei_up * threshold_up / 100)); 590 591 set(uncore, GEN6_RP_DOWN_EI, GT_INTERVAL_FROM_US(i915, ei_down)); 592 set(uncore, GEN6_RP_DOWN_THRESHOLD, 593 GT_INTERVAL_FROM_US(i915, ei_down * threshold_down / 100)); 594 595 set(uncore, GEN6_RP_CONTROL, 596 (INTEL_GEN(i915) > 9 ? 0 : GEN6_RP_MEDIA_TURBO) | 597 GEN6_RP_MEDIA_HW_NORMAL_MODE | 598 GEN6_RP_MEDIA_IS_GFX | 599 GEN6_RP_ENABLE | 600 GEN6_RP_UP_BUSY_AVG | 601 GEN6_RP_DOWN_IDLE_AVG); 602 603 skip_hw_write: 604 rps->power.mode = new_power; 605 rps->power.up_threshold = threshold_up; 606 rps->power.down_threshold = threshold_down; 607 } 608 609 static void gen6_rps_set_thresholds(struct intel_rps *rps, u8 val) 610 { 611 int new_power; 612 613 new_power = rps->power.mode; 614 switch (rps->power.mode) { 615 case LOW_POWER: 616 if (val > rps->efficient_freq + 1 && 617 val > rps->cur_freq) 618 new_power = BETWEEN; 619 break; 620 621 case BETWEEN: 622 if (val <= rps->efficient_freq && 623 val < rps->cur_freq) 624 new_power = LOW_POWER; 625 else if (val >= rps->rp0_freq && 626 val > rps->cur_freq) 627 new_power = HIGH_POWER; 628 break; 629 630 case HIGH_POWER: 631 if (val < (rps->rp1_freq + rps->rp0_freq) >> 1 && 632 val < rps->cur_freq) 633 new_power = BETWEEN; 634 break; 635 } 636 /* Max/min bins are special */ 637 if (val <= rps->min_freq_softlimit) 638 new_power = LOW_POWER; 639 if (val >= rps->max_freq_softlimit) 640 new_power = HIGH_POWER; 641 642 mutex_lock(&rps->power.mutex); 643 if (rps->power.interactive) 644 new_power = HIGH_POWER; 645 rps_set_power(rps, new_power); 646 mutex_unlock(&rps->power.mutex); 647 } 648 649 void intel_rps_mark_interactive(struct intel_rps *rps, bool interactive) 650 { 651 mutex_lock(&rps->power.mutex); 652 if (interactive) { 653 if (!rps->power.interactive++ && READ_ONCE(rps->active)) 654 rps_set_power(rps, HIGH_POWER); 655 } else { 656 GEM_BUG_ON(!rps->power.interactive); 657 rps->power.interactive--; 658 } 659 mutex_unlock(&rps->power.mutex); 660 } 661 662 static int gen6_rps_set(struct intel_rps *rps, u8 val) 663 { 664 struct intel_uncore *uncore = rps_to_uncore(rps); 665 struct drm_i915_private *i915 = rps_to_i915(rps); 666 u32 swreq; 667 668 if (INTEL_GEN(i915) >= 9) 669 swreq = GEN9_FREQUENCY(val); 670 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 671 swreq = HSW_FREQUENCY(val); 672 else 673 swreq = (GEN6_FREQUENCY(val) | 674 GEN6_OFFSET(0) | 675 GEN6_AGGRESSIVE_TURBO); 676 set(uncore, GEN6_RPNSWREQ, swreq); 677 678 return 0; 679 } 680 681 static int vlv_rps_set(struct intel_rps *rps, u8 val) 682 { 683 struct drm_i915_private *i915 = rps_to_i915(rps); 684 int err; 685 686 vlv_punit_get(i915); 687 err = vlv_punit_write(i915, PUNIT_REG_GPU_FREQ_REQ, val); 688 vlv_punit_put(i915); 689 690 return err; 691 } 692 693 static int rps_set(struct intel_rps *rps, u8 val, bool update) 694 { 695 struct drm_i915_private *i915 = rps_to_i915(rps); 696 int err; 697 698 if (INTEL_GEN(i915) < 6) 699 return 0; 700 701 if (val == rps->last_freq) 702 return 0; 703 704 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 705 err = vlv_rps_set(rps, val); 706 else 707 err = gen6_rps_set(rps, val); 708 if (err) 709 return err; 710 711 if (update) 712 gen6_rps_set_thresholds(rps, val); 713 rps->last_freq = val; 714 715 return 0; 716 } 717 718 void intel_rps_unpark(struct intel_rps *rps) 719 { 720 u8 freq; 721 722 if (!rps->enabled) 723 return; 724 725 /* 726 * Use the user's desired frequency as a guide, but for better 727 * performance, jump directly to RPe as our starting frequency. 728 */ 729 mutex_lock(&rps->lock); 730 731 WRITE_ONCE(rps->active, true); 732 733 freq = max(rps->cur_freq, rps->efficient_freq), 734 freq = clamp(freq, rps->min_freq_softlimit, rps->max_freq_softlimit); 735 intel_rps_set(rps, freq); 736 737 rps->last_adj = 0; 738 739 mutex_unlock(&rps->lock); 740 741 if (INTEL_GEN(rps_to_i915(rps)) >= 6) 742 rps_enable_interrupts(rps); 743 744 if (IS_GEN(rps_to_i915(rps), 5)) 745 gen5_rps_update(rps); 746 } 747 748 void intel_rps_park(struct intel_rps *rps) 749 { 750 struct drm_i915_private *i915 = rps_to_i915(rps); 751 752 if (!rps->enabled) 753 return; 754 755 if (INTEL_GEN(i915) >= 6) 756 rps_disable_interrupts(rps); 757 758 WRITE_ONCE(rps->active, false); 759 if (rps->last_freq <= rps->idle_freq) 760 return; 761 762 /* 763 * The punit delays the write of the frequency and voltage until it 764 * determines the GPU is awake. During normal usage we don't want to 765 * waste power changing the frequency if the GPU is sleeping (rc6). 766 * However, the GPU and driver is now idle and we do not want to delay 767 * switching to minimum voltage (reducing power whilst idle) as we do 768 * not expect to be woken in the near future and so must flush the 769 * change by waking the device. 770 * 771 * We choose to take the media powerwell (either would do to trick the 772 * punit into committing the voltage change) as that takes a lot less 773 * power than the render powerwell. 774 */ 775 intel_uncore_forcewake_get(rps_to_uncore(rps), FORCEWAKE_MEDIA); 776 rps_set(rps, rps->idle_freq, false); 777 intel_uncore_forcewake_put(rps_to_uncore(rps), FORCEWAKE_MEDIA); 778 779 /* 780 * Since we will try and restart from the previously requested 781 * frequency on unparking, treat this idle point as a downclock 782 * interrupt and reduce the frequency for resume. If we park/unpark 783 * more frequently than the rps worker can run, we will not respond 784 * to any EI and never see a change in frequency. 785 * 786 * (Note we accommodate Cherryview's limitation of only using an 787 * even bin by applying it to all.) 788 */ 789 rps->cur_freq = 790 max_t(int, round_down(rps->cur_freq - 1, 2), rps->min_freq); 791 } 792 793 void intel_rps_boost(struct i915_request *rq) 794 { 795 struct intel_rps *rps = &READ_ONCE(rq->engine)->gt->rps; 796 unsigned long flags; 797 798 if (i915_request_signaled(rq) || !READ_ONCE(rps->active)) 799 return; 800 801 /* Serializes with i915_request_retire() */ 802 spin_lock_irqsave(&rq->lock, flags); 803 if (!i915_request_has_waitboost(rq) && 804 !dma_fence_is_signaled_locked(&rq->fence)) { 805 set_bit(I915_FENCE_FLAG_BOOST, &rq->fence.flags); 806 807 if (!atomic_fetch_inc(&rps->num_waiters) && 808 READ_ONCE(rps->cur_freq) < rps->boost_freq) 809 schedule_work(&rps->work); 810 811 atomic_inc(&rps->boosts); 812 } 813 spin_unlock_irqrestore(&rq->lock, flags); 814 } 815 816 int intel_rps_set(struct intel_rps *rps, u8 val) 817 { 818 int err; 819 820 lockdep_assert_held(&rps->lock); 821 GEM_BUG_ON(val > rps->max_freq); 822 GEM_BUG_ON(val < rps->min_freq); 823 824 if (rps->active) { 825 err = rps_set(rps, val, true); 826 if (err) 827 return err; 828 829 /* 830 * Make sure we continue to get interrupts 831 * until we hit the minimum or maximum frequencies. 832 */ 833 if (INTEL_GEN(rps_to_i915(rps)) >= 6) { 834 struct intel_uncore *uncore = rps_to_uncore(rps); 835 836 set(uncore, 837 GEN6_RP_INTERRUPT_LIMITS, rps_limits(rps, val)); 838 839 set(uncore, GEN6_PMINTRMSK, rps_pm_mask(rps, val)); 840 } 841 } 842 843 rps->cur_freq = val; 844 return 0; 845 } 846 847 static void gen6_rps_init(struct intel_rps *rps) 848 { 849 struct drm_i915_private *i915 = rps_to_i915(rps); 850 struct intel_uncore *uncore = rps_to_uncore(rps); 851 852 /* All of these values are in units of 50MHz */ 853 854 /* static values from HW: RP0 > RP1 > RPn (min_freq) */ 855 if (IS_GEN9_LP(i915)) { 856 u32 rp_state_cap = intel_uncore_read(uncore, BXT_RP_STATE_CAP); 857 858 rps->rp0_freq = (rp_state_cap >> 16) & 0xff; 859 rps->rp1_freq = (rp_state_cap >> 8) & 0xff; 860 rps->min_freq = (rp_state_cap >> 0) & 0xff; 861 } else { 862 u32 rp_state_cap = intel_uncore_read(uncore, GEN6_RP_STATE_CAP); 863 864 rps->rp0_freq = (rp_state_cap >> 0) & 0xff; 865 rps->rp1_freq = (rp_state_cap >> 8) & 0xff; 866 rps->min_freq = (rp_state_cap >> 16) & 0xff; 867 } 868 869 /* hw_max = RP0 until we check for overclocking */ 870 rps->max_freq = rps->rp0_freq; 871 872 rps->efficient_freq = rps->rp1_freq; 873 if (IS_HASWELL(i915) || IS_BROADWELL(i915) || 874 IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) { 875 u32 ddcc_status = 0; 876 877 if (sandybridge_pcode_read(i915, 878 HSW_PCODE_DYNAMIC_DUTY_CYCLE_CONTROL, 879 &ddcc_status, NULL) == 0) 880 rps->efficient_freq = 881 clamp_t(u8, 882 (ddcc_status >> 8) & 0xff, 883 rps->min_freq, 884 rps->max_freq); 885 } 886 887 if (IS_GEN9_BC(i915) || INTEL_GEN(i915) >= 10) { 888 /* Store the frequency values in 16.66 MHZ units, which is 889 * the natural hardware unit for SKL 890 */ 891 rps->rp0_freq *= GEN9_FREQ_SCALER; 892 rps->rp1_freq *= GEN9_FREQ_SCALER; 893 rps->min_freq *= GEN9_FREQ_SCALER; 894 rps->max_freq *= GEN9_FREQ_SCALER; 895 rps->efficient_freq *= GEN9_FREQ_SCALER; 896 } 897 } 898 899 static bool rps_reset(struct intel_rps *rps) 900 { 901 /* force a reset */ 902 rps->power.mode = -1; 903 rps->last_freq = -1; 904 905 if (rps_set(rps, rps->min_freq, true)) { 906 DRM_ERROR("Failed to reset RPS to initial values\n"); 907 return false; 908 } 909 910 rps->cur_freq = rps->min_freq; 911 return true; 912 } 913 914 /* See the Gen9_GT_PM_Programming_Guide doc for the below */ 915 static bool gen9_rps_enable(struct intel_rps *rps) 916 { 917 struct drm_i915_private *i915 = rps_to_i915(rps); 918 struct intel_uncore *uncore = rps_to_uncore(rps); 919 920 /* Program defaults and thresholds for RPS */ 921 if (IS_GEN(i915, 9)) 922 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 923 GEN9_FREQUENCY(rps->rp1_freq)); 924 925 /* 1 second timeout */ 926 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 927 GT_INTERVAL_FROM_US(i915, 1000000)); 928 929 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 0xa); 930 931 return rps_reset(rps); 932 } 933 934 static bool gen8_rps_enable(struct intel_rps *rps) 935 { 936 struct intel_uncore *uncore = rps_to_uncore(rps); 937 938 intel_uncore_write_fw(uncore, GEN6_RC_VIDEO_FREQ, 939 HSW_FREQUENCY(rps->rp1_freq)); 940 941 /* NB: Docs say 1s, and 1000000 - which aren't equivalent */ 942 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 943 100000000 / 128); /* 1 second timeout */ 944 945 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 946 947 return rps_reset(rps); 948 } 949 950 static bool gen6_rps_enable(struct intel_rps *rps) 951 { 952 struct intel_uncore *uncore = rps_to_uncore(rps); 953 954 /* Power down if completely idle for over 50ms */ 955 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 50000); 956 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 957 958 return rps_reset(rps); 959 } 960 961 static int chv_rps_max_freq(struct intel_rps *rps) 962 { 963 struct drm_i915_private *i915 = rps_to_i915(rps); 964 u32 val; 965 966 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 967 968 switch (RUNTIME_INFO(i915)->sseu.eu_total) { 969 case 8: 970 /* (2 * 4) config */ 971 val >>= FB_GFX_FMAX_AT_VMAX_2SS4EU_FUSE_SHIFT; 972 break; 973 case 12: 974 /* (2 * 6) config */ 975 val >>= FB_GFX_FMAX_AT_VMAX_2SS6EU_FUSE_SHIFT; 976 break; 977 case 16: 978 /* (2 * 8) config */ 979 default: 980 /* Setting (2 * 8) Min RP0 for any other combination */ 981 val >>= FB_GFX_FMAX_AT_VMAX_2SS8EU_FUSE_SHIFT; 982 break; 983 } 984 985 return val & FB_GFX_FREQ_FUSE_MASK; 986 } 987 988 static int chv_rps_rpe_freq(struct intel_rps *rps) 989 { 990 struct drm_i915_private *i915 = rps_to_i915(rps); 991 u32 val; 992 993 val = vlv_punit_read(i915, PUNIT_GPU_DUTYCYCLE_REG); 994 val >>= PUNIT_GPU_DUTYCYCLE_RPE_FREQ_SHIFT; 995 996 return val & PUNIT_GPU_DUTYCYCLE_RPE_FREQ_MASK; 997 } 998 999 static int chv_rps_guar_freq(struct intel_rps *rps) 1000 { 1001 struct drm_i915_private *i915 = rps_to_i915(rps); 1002 u32 val; 1003 1004 val = vlv_punit_read(i915, FB_GFX_FMAX_AT_VMAX_FUSE); 1005 1006 return val & FB_GFX_FREQ_FUSE_MASK; 1007 } 1008 1009 static u32 chv_rps_min_freq(struct intel_rps *rps) 1010 { 1011 struct drm_i915_private *i915 = rps_to_i915(rps); 1012 u32 val; 1013 1014 val = vlv_punit_read(i915, FB_GFX_FMIN_AT_VMIN_FUSE); 1015 val >>= FB_GFX_FMIN_AT_VMIN_FUSE_SHIFT; 1016 1017 return val & FB_GFX_FREQ_FUSE_MASK; 1018 } 1019 1020 static bool chv_rps_enable(struct intel_rps *rps) 1021 { 1022 struct intel_uncore *uncore = rps_to_uncore(rps); 1023 struct drm_i915_private *i915 = rps_to_i915(rps); 1024 u32 val; 1025 1026 /* 1: Program defaults and thresholds for RPS*/ 1027 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1028 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1029 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1030 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1031 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1032 1033 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1034 1035 /* 2: Enable RPS */ 1036 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1037 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1038 GEN6_RP_MEDIA_IS_GFX | 1039 GEN6_RP_ENABLE | 1040 GEN6_RP_UP_BUSY_AVG | 1041 GEN6_RP_DOWN_IDLE_AVG); 1042 1043 /* Setting Fixed Bias */ 1044 vlv_punit_get(i915); 1045 1046 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | CHV_BIAS_CPU_50_SOC_50; 1047 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1048 1049 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1050 1051 vlv_punit_put(i915); 1052 1053 /* RPS code assumes GPLL is used */ 1054 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1055 "GPLL not enabled\n"); 1056 1057 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE)); 1058 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); 1059 1060 return rps_reset(rps); 1061 } 1062 1063 static int vlv_rps_guar_freq(struct intel_rps *rps) 1064 { 1065 struct drm_i915_private *i915 = rps_to_i915(rps); 1066 u32 val, rp1; 1067 1068 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1069 1070 rp1 = val & FB_GFX_FGUARANTEED_FREQ_FUSE_MASK; 1071 rp1 >>= FB_GFX_FGUARANTEED_FREQ_FUSE_SHIFT; 1072 1073 return rp1; 1074 } 1075 1076 static int vlv_rps_max_freq(struct intel_rps *rps) 1077 { 1078 struct drm_i915_private *i915 = rps_to_i915(rps); 1079 u32 val, rp0; 1080 1081 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FREQ_FUSE); 1082 1083 rp0 = (val & FB_GFX_MAX_FREQ_FUSE_MASK) >> FB_GFX_MAX_FREQ_FUSE_SHIFT; 1084 /* Clamp to max */ 1085 rp0 = min_t(u32, rp0, 0xea); 1086 1087 return rp0; 1088 } 1089 1090 static int vlv_rps_rpe_freq(struct intel_rps *rps) 1091 { 1092 struct drm_i915_private *i915 = rps_to_i915(rps); 1093 u32 val, rpe; 1094 1095 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_LO); 1096 rpe = (val & FB_FMAX_VMIN_FREQ_LO_MASK) >> FB_FMAX_VMIN_FREQ_LO_SHIFT; 1097 val = vlv_nc_read(i915, IOSF_NC_FB_GFX_FMAX_FUSE_HI); 1098 rpe |= (val & FB_FMAX_VMIN_FREQ_HI_MASK) << 5; 1099 1100 return rpe; 1101 } 1102 1103 static int vlv_rps_min_freq(struct intel_rps *rps) 1104 { 1105 struct drm_i915_private *i915 = rps_to_i915(rps); 1106 u32 val; 1107 1108 val = vlv_punit_read(i915, PUNIT_REG_GPU_LFM) & 0xff; 1109 /* 1110 * According to the BYT Punit GPU turbo HAS 1.1.6.3 the minimum value 1111 * for the minimum frequency in GPLL mode is 0xc1. Contrary to this on 1112 * a BYT-M B0 the above register contains 0xbf. Moreover when setting 1113 * a frequency Punit will not allow values below 0xc0. Clamp it 0xc0 1114 * to make sure it matches what Punit accepts. 1115 */ 1116 return max_t(u32, val, 0xc0); 1117 } 1118 1119 static bool vlv_rps_enable(struct intel_rps *rps) 1120 { 1121 struct intel_uncore *uncore = rps_to_uncore(rps); 1122 struct drm_i915_private *i915 = rps_to_i915(rps); 1123 u32 val; 1124 1125 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_TIMEOUT, 1000000); 1126 intel_uncore_write_fw(uncore, GEN6_RP_UP_THRESHOLD, 59400); 1127 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_THRESHOLD, 245000); 1128 intel_uncore_write_fw(uncore, GEN6_RP_UP_EI, 66000); 1129 intel_uncore_write_fw(uncore, GEN6_RP_DOWN_EI, 350000); 1130 1131 intel_uncore_write_fw(uncore, GEN6_RP_IDLE_HYSTERSIS, 10); 1132 1133 intel_uncore_write_fw(uncore, GEN6_RP_CONTROL, 1134 GEN6_RP_MEDIA_TURBO | 1135 GEN6_RP_MEDIA_HW_NORMAL_MODE | 1136 GEN6_RP_MEDIA_IS_GFX | 1137 GEN6_RP_ENABLE | 1138 GEN6_RP_UP_BUSY_AVG | 1139 GEN6_RP_DOWN_IDLE_CONT); 1140 1141 vlv_punit_get(i915); 1142 1143 /* Setting Fixed Bias */ 1144 val = VLV_OVERRIDE_EN | VLV_SOC_TDP_EN | VLV_BIAS_CPU_125_SOC_875; 1145 vlv_punit_write(i915, VLV_TURBO_SOC_OVERRIDE, val); 1146 1147 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1148 1149 vlv_punit_put(i915); 1150 1151 /* RPS code assumes GPLL is used */ 1152 drm_WARN_ONCE(&i915->drm, (val & GPLLENABLE) == 0, 1153 "GPLL not enabled\n"); 1154 1155 DRM_DEBUG_DRIVER("GPLL enabled? %s\n", yesno(val & GPLLENABLE)); 1156 DRM_DEBUG_DRIVER("GPU status: 0x%08x\n", val); 1157 1158 return rps_reset(rps); 1159 } 1160 1161 static unsigned long __ips_gfx_val(struct intel_ips *ips) 1162 { 1163 struct intel_rps *rps = container_of(ips, typeof(*rps), ips); 1164 struct intel_uncore *uncore = rps_to_uncore(rps); 1165 unsigned long t, corr, state1, corr2, state2; 1166 u32 pxvid, ext_v; 1167 1168 lockdep_assert_held(&mchdev_lock); 1169 1170 pxvid = intel_uncore_read(uncore, PXVFREQ(rps->cur_freq)); 1171 pxvid = (pxvid >> 24) & 0x7f; 1172 ext_v = pvid_to_extvid(rps_to_i915(rps), pxvid); 1173 1174 state1 = ext_v; 1175 1176 /* Revel in the empirically derived constants */ 1177 1178 /* Correction factor in 1/100000 units */ 1179 t = ips_mch_val(uncore); 1180 if (t > 80) 1181 corr = t * 2349 + 135940; 1182 else if (t >= 50) 1183 corr = t * 964 + 29317; 1184 else /* < 50 */ 1185 corr = t * 301 + 1004; 1186 1187 corr = corr * 150142 * state1 / 10000 - 78642; 1188 corr /= 100000; 1189 corr2 = corr * ips->corr; 1190 1191 state2 = corr2 * state1 / 10000; 1192 state2 /= 100; /* convert to mW */ 1193 1194 __gen5_ips_update(ips); 1195 1196 return ips->gfx_power + state2; 1197 } 1198 1199 void intel_rps_enable(struct intel_rps *rps) 1200 { 1201 struct drm_i915_private *i915 = rps_to_i915(rps); 1202 struct intel_uncore *uncore = rps_to_uncore(rps); 1203 1204 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL); 1205 if (IS_CHERRYVIEW(i915)) 1206 rps->enabled = chv_rps_enable(rps); 1207 else if (IS_VALLEYVIEW(i915)) 1208 rps->enabled = vlv_rps_enable(rps); 1209 else if (INTEL_GEN(i915) >= 9) 1210 rps->enabled = gen9_rps_enable(rps); 1211 else if (INTEL_GEN(i915) >= 8) 1212 rps->enabled = gen8_rps_enable(rps); 1213 else if (INTEL_GEN(i915) >= 6) 1214 rps->enabled = gen6_rps_enable(rps); 1215 else if (IS_IRONLAKE_M(i915)) 1216 rps->enabled = gen5_rps_enable(rps); 1217 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL); 1218 if (!rps->enabled) 1219 return; 1220 1221 drm_WARN_ON(&i915->drm, rps->max_freq < rps->min_freq); 1222 drm_WARN_ON(&i915->drm, rps->idle_freq > rps->max_freq); 1223 1224 drm_WARN_ON(&i915->drm, rps->efficient_freq < rps->min_freq); 1225 drm_WARN_ON(&i915->drm, rps->efficient_freq > rps->max_freq); 1226 } 1227 1228 static void gen6_rps_disable(struct intel_rps *rps) 1229 { 1230 set(rps_to_uncore(rps), GEN6_RP_CONTROL, 0); 1231 } 1232 1233 void intel_rps_disable(struct intel_rps *rps) 1234 { 1235 struct drm_i915_private *i915 = rps_to_i915(rps); 1236 1237 rps->enabled = false; 1238 1239 if (INTEL_GEN(i915) >= 6) 1240 gen6_rps_disable(rps); 1241 else if (IS_IRONLAKE_M(i915)) 1242 gen5_rps_disable(rps); 1243 } 1244 1245 static int byt_gpu_freq(struct intel_rps *rps, int val) 1246 { 1247 /* 1248 * N = val - 0xb7 1249 * Slow = Fast = GPLL ref * N 1250 */ 1251 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * (val - 0xb7), 1000); 1252 } 1253 1254 static int byt_freq_opcode(struct intel_rps *rps, int val) 1255 { 1256 return DIV_ROUND_CLOSEST(1000 * val, rps->gpll_ref_freq) + 0xb7; 1257 } 1258 1259 static int chv_gpu_freq(struct intel_rps *rps, int val) 1260 { 1261 /* 1262 * N = val / 2 1263 * CU (slow) = CU2x (fast) / 2 = GPLL ref * N / 2 1264 */ 1265 return DIV_ROUND_CLOSEST(rps->gpll_ref_freq * val, 2 * 2 * 1000); 1266 } 1267 1268 static int chv_freq_opcode(struct intel_rps *rps, int val) 1269 { 1270 /* CHV needs even values */ 1271 return DIV_ROUND_CLOSEST(2 * 1000 * val, rps->gpll_ref_freq) * 2; 1272 } 1273 1274 int intel_gpu_freq(struct intel_rps *rps, int val) 1275 { 1276 struct drm_i915_private *i915 = rps_to_i915(rps); 1277 1278 if (INTEL_GEN(i915) >= 9) 1279 return DIV_ROUND_CLOSEST(val * GT_FREQUENCY_MULTIPLIER, 1280 GEN9_FREQ_SCALER); 1281 else if (IS_CHERRYVIEW(i915)) 1282 return chv_gpu_freq(rps, val); 1283 else if (IS_VALLEYVIEW(i915)) 1284 return byt_gpu_freq(rps, val); 1285 else 1286 return val * GT_FREQUENCY_MULTIPLIER; 1287 } 1288 1289 int intel_freq_opcode(struct intel_rps *rps, int val) 1290 { 1291 struct drm_i915_private *i915 = rps_to_i915(rps); 1292 1293 if (INTEL_GEN(i915) >= 9) 1294 return DIV_ROUND_CLOSEST(val * GEN9_FREQ_SCALER, 1295 GT_FREQUENCY_MULTIPLIER); 1296 else if (IS_CHERRYVIEW(i915)) 1297 return chv_freq_opcode(rps, val); 1298 else if (IS_VALLEYVIEW(i915)) 1299 return byt_freq_opcode(rps, val); 1300 else 1301 return DIV_ROUND_CLOSEST(val, GT_FREQUENCY_MULTIPLIER); 1302 } 1303 1304 static void vlv_init_gpll_ref_freq(struct intel_rps *rps) 1305 { 1306 struct drm_i915_private *i915 = rps_to_i915(rps); 1307 1308 rps->gpll_ref_freq = 1309 vlv_get_cck_clock(i915, "GPLL ref", 1310 CCK_GPLL_CLOCK_CONTROL, 1311 i915->czclk_freq); 1312 1313 DRM_DEBUG_DRIVER("GPLL reference freq: %d kHz\n", rps->gpll_ref_freq); 1314 } 1315 1316 static void vlv_rps_init(struct intel_rps *rps) 1317 { 1318 struct drm_i915_private *i915 = rps_to_i915(rps); 1319 u32 val; 1320 1321 vlv_iosf_sb_get(i915, 1322 BIT(VLV_IOSF_SB_PUNIT) | 1323 BIT(VLV_IOSF_SB_NC) | 1324 BIT(VLV_IOSF_SB_CCK)); 1325 1326 vlv_init_gpll_ref_freq(rps); 1327 1328 val = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1329 switch ((val >> 6) & 3) { 1330 case 0: 1331 case 1: 1332 i915->mem_freq = 800; 1333 break; 1334 case 2: 1335 i915->mem_freq = 1066; 1336 break; 1337 case 3: 1338 i915->mem_freq = 1333; 1339 break; 1340 } 1341 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", i915->mem_freq); 1342 1343 rps->max_freq = vlv_rps_max_freq(rps); 1344 rps->rp0_freq = rps->max_freq; 1345 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", 1346 intel_gpu_freq(rps, rps->max_freq), 1347 rps->max_freq); 1348 1349 rps->efficient_freq = vlv_rps_rpe_freq(rps); 1350 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", 1351 intel_gpu_freq(rps, rps->efficient_freq), 1352 rps->efficient_freq); 1353 1354 rps->rp1_freq = vlv_rps_guar_freq(rps); 1355 DRM_DEBUG_DRIVER("RP1(Guar Freq) GPU freq: %d MHz (%u)\n", 1356 intel_gpu_freq(rps, rps->rp1_freq), 1357 rps->rp1_freq); 1358 1359 rps->min_freq = vlv_rps_min_freq(rps); 1360 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", 1361 intel_gpu_freq(rps, rps->min_freq), 1362 rps->min_freq); 1363 1364 vlv_iosf_sb_put(i915, 1365 BIT(VLV_IOSF_SB_PUNIT) | 1366 BIT(VLV_IOSF_SB_NC) | 1367 BIT(VLV_IOSF_SB_CCK)); 1368 } 1369 1370 static void chv_rps_init(struct intel_rps *rps) 1371 { 1372 struct drm_i915_private *i915 = rps_to_i915(rps); 1373 u32 val; 1374 1375 vlv_iosf_sb_get(i915, 1376 BIT(VLV_IOSF_SB_PUNIT) | 1377 BIT(VLV_IOSF_SB_NC) | 1378 BIT(VLV_IOSF_SB_CCK)); 1379 1380 vlv_init_gpll_ref_freq(rps); 1381 1382 val = vlv_cck_read(i915, CCK_FUSE_REG); 1383 1384 switch ((val >> 2) & 0x7) { 1385 case 3: 1386 i915->mem_freq = 2000; 1387 break; 1388 default: 1389 i915->mem_freq = 1600; 1390 break; 1391 } 1392 DRM_DEBUG_DRIVER("DDR speed: %d MHz\n", i915->mem_freq); 1393 1394 rps->max_freq = chv_rps_max_freq(rps); 1395 rps->rp0_freq = rps->max_freq; 1396 DRM_DEBUG_DRIVER("max GPU freq: %d MHz (%u)\n", 1397 intel_gpu_freq(rps, rps->max_freq), 1398 rps->max_freq); 1399 1400 rps->efficient_freq = chv_rps_rpe_freq(rps); 1401 DRM_DEBUG_DRIVER("RPe GPU freq: %d MHz (%u)\n", 1402 intel_gpu_freq(rps, rps->efficient_freq), 1403 rps->efficient_freq); 1404 1405 rps->rp1_freq = chv_rps_guar_freq(rps); 1406 DRM_DEBUG_DRIVER("RP1(Guar) GPU freq: %d MHz (%u)\n", 1407 intel_gpu_freq(rps, rps->rp1_freq), 1408 rps->rp1_freq); 1409 1410 rps->min_freq = chv_rps_min_freq(rps); 1411 DRM_DEBUG_DRIVER("min GPU freq: %d MHz (%u)\n", 1412 intel_gpu_freq(rps, rps->min_freq), 1413 rps->min_freq); 1414 1415 vlv_iosf_sb_put(i915, 1416 BIT(VLV_IOSF_SB_PUNIT) | 1417 BIT(VLV_IOSF_SB_NC) | 1418 BIT(VLV_IOSF_SB_CCK)); 1419 1420 drm_WARN_ONCE(&i915->drm, (rps->max_freq | rps->efficient_freq | 1421 rps->rp1_freq | rps->min_freq) & 1, 1422 "Odd GPU freq values\n"); 1423 } 1424 1425 static void vlv_c0_read(struct intel_uncore *uncore, struct intel_rps_ei *ei) 1426 { 1427 ei->ktime = ktime_get_raw(); 1428 ei->render_c0 = intel_uncore_read(uncore, VLV_RENDER_C0_COUNT); 1429 ei->media_c0 = intel_uncore_read(uncore, VLV_MEDIA_C0_COUNT); 1430 } 1431 1432 static u32 vlv_wa_c0_ei(struct intel_rps *rps, u32 pm_iir) 1433 { 1434 struct intel_uncore *uncore = rps_to_uncore(rps); 1435 const struct intel_rps_ei *prev = &rps->ei; 1436 struct intel_rps_ei now; 1437 u32 events = 0; 1438 1439 if ((pm_iir & GEN6_PM_RP_UP_EI_EXPIRED) == 0) 1440 return 0; 1441 1442 vlv_c0_read(uncore, &now); 1443 1444 #ifdef __linux__ 1445 if (prev->ktime) { 1446 #else 1447 if (ktime_to_ns(prev->ktime)) { 1448 #endif 1449 u64 time, c0; 1450 u32 render, media; 1451 1452 time = ktime_us_delta(now.ktime, prev->ktime); 1453 1454 time *= rps_to_i915(rps)->czclk_freq; 1455 1456 /* Workload can be split between render + media, 1457 * e.g. SwapBuffers being blitted in X after being rendered in 1458 * mesa. To account for this we need to combine both engines 1459 * into our activity counter. 1460 */ 1461 render = now.render_c0 - prev->render_c0; 1462 media = now.media_c0 - prev->media_c0; 1463 c0 = max(render, media); 1464 c0 *= 1000 * 100 << 8; /* to usecs and scale to threshold% */ 1465 1466 if (c0 > time * rps->power.up_threshold) 1467 events = GEN6_PM_RP_UP_THRESHOLD; 1468 else if (c0 < time * rps->power.down_threshold) 1469 events = GEN6_PM_RP_DOWN_THRESHOLD; 1470 } 1471 1472 rps->ei = now; 1473 return events; 1474 } 1475 1476 static void rps_work(struct work_struct *work) 1477 { 1478 struct intel_rps *rps = container_of(work, typeof(*rps), work); 1479 struct intel_gt *gt = rps_to_gt(rps); 1480 bool client_boost = false; 1481 int new_freq, adj, min, max; 1482 u32 pm_iir = 0; 1483 1484 spin_lock_irq(>->irq_lock); 1485 pm_iir = fetch_and_zero(&rps->pm_iir) & READ_ONCE(rps->pm_events); 1486 client_boost = atomic_read(&rps->num_waiters); 1487 spin_unlock_irq(>->irq_lock); 1488 1489 /* Make sure we didn't queue anything we're not going to process. */ 1490 if (!pm_iir && !client_boost) 1491 goto out; 1492 1493 mutex_lock(&rps->lock); 1494 1495 pm_iir |= vlv_wa_c0_ei(rps, pm_iir); 1496 1497 adj = rps->last_adj; 1498 new_freq = rps->cur_freq; 1499 min = rps->min_freq_softlimit; 1500 max = rps->max_freq_softlimit; 1501 if (client_boost) 1502 max = rps->max_freq; 1503 if (client_boost && new_freq < rps->boost_freq) { 1504 new_freq = rps->boost_freq; 1505 adj = 0; 1506 } else if (pm_iir & GEN6_PM_RP_UP_THRESHOLD) { 1507 if (adj > 0) 1508 adj *= 2; 1509 else /* CHV needs even encode values */ 1510 adj = IS_CHERRYVIEW(gt->i915) ? 2 : 1; 1511 1512 if (new_freq >= rps->max_freq_softlimit) 1513 adj = 0; 1514 } else if (client_boost) { 1515 adj = 0; 1516 } else if (pm_iir & GEN6_PM_RP_DOWN_TIMEOUT) { 1517 if (rps->cur_freq > rps->efficient_freq) 1518 new_freq = rps->efficient_freq; 1519 else if (rps->cur_freq > rps->min_freq_softlimit) 1520 new_freq = rps->min_freq_softlimit; 1521 adj = 0; 1522 } else if (pm_iir & GEN6_PM_RP_DOWN_THRESHOLD) { 1523 if (adj < 0) 1524 adj *= 2; 1525 else /* CHV needs even encode values */ 1526 adj = IS_CHERRYVIEW(gt->i915) ? -2 : -1; 1527 1528 if (new_freq <= rps->min_freq_softlimit) 1529 adj = 0; 1530 } else { /* unknown event */ 1531 adj = 0; 1532 } 1533 1534 rps->last_adj = adj; 1535 1536 /* 1537 * Limit deboosting and boosting to keep ourselves at the extremes 1538 * when in the respective power modes (i.e. slowly decrease frequencies 1539 * while in the HIGH_POWER zone and slowly increase frequencies while 1540 * in the LOW_POWER zone). On idle, we will hit the timeout and drop 1541 * to the next level quickly, and conversely if busy we expect to 1542 * hit a waitboost and rapidly switch into max power. 1543 */ 1544 if ((adj < 0 && rps->power.mode == HIGH_POWER) || 1545 (adj > 0 && rps->power.mode == LOW_POWER)) 1546 rps->last_adj = 0; 1547 1548 /* sysfs frequency interfaces may have snuck in while servicing the 1549 * interrupt 1550 */ 1551 new_freq += adj; 1552 new_freq = clamp_t(int, new_freq, min, max); 1553 1554 if (intel_rps_set(rps, new_freq)) { 1555 DRM_DEBUG_DRIVER("Failed to set new GPU frequency\n"); 1556 rps->last_adj = 0; 1557 } 1558 1559 mutex_unlock(&rps->lock); 1560 1561 out: 1562 spin_lock_irq(>->irq_lock); 1563 gen6_gt_pm_unmask_irq(gt, rps->pm_events); 1564 spin_unlock_irq(>->irq_lock); 1565 } 1566 1567 void gen11_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1568 { 1569 struct intel_gt *gt = rps_to_gt(rps); 1570 const u32 events = rps->pm_events & pm_iir; 1571 1572 lockdep_assert_held(>->irq_lock); 1573 1574 if (unlikely(!events)) 1575 return; 1576 1577 gen6_gt_pm_mask_irq(gt, events); 1578 1579 rps->pm_iir |= events; 1580 schedule_work(&rps->work); 1581 } 1582 1583 void gen6_rps_irq_handler(struct intel_rps *rps, u32 pm_iir) 1584 { 1585 struct intel_gt *gt = rps_to_gt(rps); 1586 u32 events; 1587 1588 events = pm_iir & READ_ONCE(rps->pm_events); 1589 if (events) { 1590 spin_lock(>->irq_lock); 1591 1592 gen6_gt_pm_mask_irq(gt, events); 1593 rps->pm_iir |= events; 1594 1595 schedule_work(&rps->work); 1596 spin_unlock(>->irq_lock); 1597 } 1598 1599 if (INTEL_GEN(gt->i915) >= 8) 1600 return; 1601 1602 if (pm_iir & PM_VEBOX_USER_INTERRUPT) 1603 intel_engine_signal_breadcrumbs(gt->engine[VECS0]); 1604 1605 if (pm_iir & PM_VEBOX_CS_ERROR_INTERRUPT) 1606 DRM_DEBUG("Command parser error, pm_iir 0x%08x\n", pm_iir); 1607 } 1608 1609 void gen5_rps_irq_handler(struct intel_rps *rps) 1610 { 1611 struct intel_uncore *uncore = rps_to_uncore(rps); 1612 u32 busy_up, busy_down, max_avg, min_avg; 1613 u8 new_freq; 1614 1615 spin_lock(&mchdev_lock); 1616 1617 intel_uncore_write16(uncore, 1618 MEMINTRSTS, 1619 intel_uncore_read(uncore, MEMINTRSTS)); 1620 1621 intel_uncore_write16(uncore, MEMINTRSTS, MEMINT_EVAL_CHG); 1622 busy_up = intel_uncore_read(uncore, RCPREVBSYTUPAVG); 1623 busy_down = intel_uncore_read(uncore, RCPREVBSYTDNAVG); 1624 max_avg = intel_uncore_read(uncore, RCBMAXAVG); 1625 min_avg = intel_uncore_read(uncore, RCBMINAVG); 1626 1627 /* Handle RCS change request from hw */ 1628 new_freq = rps->cur_freq; 1629 if (busy_up > max_avg) 1630 new_freq++; 1631 else if (busy_down < min_avg) 1632 new_freq--; 1633 new_freq = clamp(new_freq, 1634 rps->min_freq_softlimit, 1635 rps->max_freq_softlimit); 1636 1637 if (new_freq != rps->cur_freq && gen5_rps_set(rps, new_freq)) 1638 rps->cur_freq = new_freq; 1639 1640 spin_unlock(&mchdev_lock); 1641 } 1642 1643 void intel_rps_init_early(struct intel_rps *rps) 1644 { 1645 rw_init(&rps->lock, "rpslk"); 1646 rw_init(&rps->power.mutex, "rpspwr"); 1647 1648 INIT_WORK(&rps->work, rps_work); 1649 1650 atomic_set(&rps->num_waiters, 0); 1651 } 1652 1653 void intel_rps_init(struct intel_rps *rps) 1654 { 1655 struct drm_i915_private *i915 = rps_to_i915(rps); 1656 1657 if (IS_CHERRYVIEW(i915)) 1658 chv_rps_init(rps); 1659 else if (IS_VALLEYVIEW(i915)) 1660 vlv_rps_init(rps); 1661 else if (INTEL_GEN(i915) >= 6) 1662 gen6_rps_init(rps); 1663 else if (IS_IRONLAKE_M(i915)) 1664 gen5_rps_init(rps); 1665 1666 /* Derive initial user preferences/limits from the hardware limits */ 1667 rps->max_freq_softlimit = rps->max_freq; 1668 rps->min_freq_softlimit = rps->min_freq; 1669 1670 /* After setting max-softlimit, find the overclock max freq */ 1671 if (IS_GEN(i915, 6) || IS_IVYBRIDGE(i915) || IS_HASWELL(i915)) { 1672 u32 params = 0; 1673 1674 sandybridge_pcode_read(i915, GEN6_READ_OC_PARAMS, 1675 ¶ms, NULL); 1676 if (params & BIT(31)) { /* OC supported */ 1677 DRM_DEBUG_DRIVER("Overclocking supported, max: %dMHz, overclock: %dMHz\n", 1678 (rps->max_freq & 0xff) * 50, 1679 (params & 0xff) * 50); 1680 rps->max_freq = params & 0xff; 1681 } 1682 } 1683 1684 /* Finally allow us to boost to max by default */ 1685 rps->boost_freq = rps->max_freq; 1686 rps->idle_freq = rps->min_freq; 1687 rps->cur_freq = rps->idle_freq; 1688 1689 rps->pm_intrmsk_mbz = 0; 1690 1691 /* 1692 * SNB,IVB,HSW can while VLV,CHV may hard hang on looping batchbuffer 1693 * if GEN6_PM_UP_EI_EXPIRED is masked. 1694 * 1695 * TODO: verify if this can be reproduced on VLV,CHV. 1696 */ 1697 if (INTEL_GEN(i915) <= 7) 1698 rps->pm_intrmsk_mbz |= GEN6_PM_RP_UP_EI_EXPIRED; 1699 1700 if (INTEL_GEN(i915) >= 8 && INTEL_GEN(i915) < 11) 1701 rps->pm_intrmsk_mbz |= GEN8_PMINTR_DISABLE_REDIRECT_TO_GUC; 1702 } 1703 1704 u32 intel_rps_get_cagf(struct intel_rps *rps, u32 rpstat) 1705 { 1706 struct drm_i915_private *i915 = rps_to_i915(rps); 1707 u32 cagf; 1708 1709 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) 1710 cagf = (rpstat >> 8) & 0xff; 1711 else if (INTEL_GEN(i915) >= 9) 1712 cagf = (rpstat & GEN9_CAGF_MASK) >> GEN9_CAGF_SHIFT; 1713 else if (IS_HASWELL(i915) || IS_BROADWELL(i915)) 1714 cagf = (rpstat & HSW_CAGF_MASK) >> HSW_CAGF_SHIFT; 1715 else 1716 cagf = (rpstat & GEN6_CAGF_MASK) >> GEN6_CAGF_SHIFT; 1717 1718 return cagf; 1719 } 1720 1721 static u32 read_cagf(struct intel_rps *rps) 1722 { 1723 struct drm_i915_private *i915 = rps_to_i915(rps); 1724 u32 freq; 1725 1726 if (IS_VALLEYVIEW(i915) || IS_CHERRYVIEW(i915)) { 1727 vlv_punit_get(i915); 1728 freq = vlv_punit_read(i915, PUNIT_REG_GPU_FREQ_STS); 1729 vlv_punit_put(i915); 1730 } else { 1731 freq = intel_uncore_read(rps_to_gt(rps)->uncore, GEN6_RPSTAT1); 1732 } 1733 1734 return intel_rps_get_cagf(rps, freq); 1735 } 1736 1737 u32 intel_rps_read_actual_frequency(struct intel_rps *rps) 1738 { 1739 struct intel_runtime_pm *rpm = rps_to_gt(rps)->uncore->rpm; 1740 intel_wakeref_t wakeref; 1741 u32 freq = 0; 1742 1743 with_intel_runtime_pm_if_in_use(rpm, wakeref) 1744 freq = intel_gpu_freq(rps, read_cagf(rps)); 1745 1746 return freq; 1747 } 1748 1749 /* External interface for intel_ips.ko */ 1750 1751 static struct drm_i915_private __rcu *ips_mchdev; 1752 1753 /** 1754 * Tells the intel_ips driver that the i915 driver is now loaded, if 1755 * IPS got loaded first. 1756 * 1757 * This awkward dance is so that neither module has to depend on the 1758 * other in order for IPS to do the appropriate communication of 1759 * GPU turbo limits to i915. 1760 */ 1761 static void 1762 ips_ping_for_i915_load(void) 1763 { 1764 #ifdef __linux__ 1765 void (*link)(void); 1766 1767 link = symbol_get(ips_link_to_i915_driver); 1768 if (link) { 1769 link(); 1770 symbol_put(ips_link_to_i915_driver); 1771 } 1772 #endif 1773 } 1774 1775 void intel_rps_driver_register(struct intel_rps *rps) 1776 { 1777 struct intel_gt *gt = rps_to_gt(rps); 1778 1779 /* 1780 * We only register the i915 ips part with intel-ips once everything is 1781 * set up, to avoid intel-ips sneaking in and reading bogus values. 1782 */ 1783 if (IS_GEN(gt->i915, 5)) { 1784 GEM_BUG_ON(ips_mchdev); 1785 rcu_assign_pointer(ips_mchdev, gt->i915); 1786 ips_ping_for_i915_load(); 1787 } 1788 } 1789 1790 void intel_rps_driver_unregister(struct intel_rps *rps) 1791 { 1792 if (rcu_access_pointer(ips_mchdev) == rps_to_i915(rps)) 1793 rcu_assign_pointer(ips_mchdev, NULL); 1794 } 1795 1796 static struct drm_i915_private *mchdev_get(void) 1797 { 1798 struct drm_i915_private *i915; 1799 1800 rcu_read_lock(); 1801 i915 = rcu_dereference(ips_mchdev); 1802 if (!kref_get_unless_zero(&i915->drm.ref)) 1803 i915 = NULL; 1804 rcu_read_unlock(); 1805 1806 return i915; 1807 } 1808 1809 /** 1810 * i915_read_mch_val - return value for IPS use 1811 * 1812 * Calculate and return a value for the IPS driver to use when deciding whether 1813 * we have thermal and power headroom to increase CPU or GPU power budget. 1814 */ 1815 unsigned long i915_read_mch_val(void) 1816 { 1817 struct drm_i915_private *i915; 1818 unsigned long chipset_val = 0; 1819 unsigned long graphics_val = 0; 1820 intel_wakeref_t wakeref; 1821 1822 i915 = mchdev_get(); 1823 if (!i915) 1824 return 0; 1825 1826 with_intel_runtime_pm(&i915->runtime_pm, wakeref) { 1827 struct intel_ips *ips = &i915->gt.rps.ips; 1828 1829 spin_lock_irq(&mchdev_lock); 1830 chipset_val = __ips_chipset_val(ips); 1831 graphics_val = __ips_gfx_val(ips); 1832 spin_unlock_irq(&mchdev_lock); 1833 } 1834 1835 drm_dev_put(&i915->drm); 1836 return chipset_val + graphics_val; 1837 } 1838 EXPORT_SYMBOL_GPL(i915_read_mch_val); 1839 1840 /** 1841 * i915_gpu_raise - raise GPU frequency limit 1842 * 1843 * Raise the limit; IPS indicates we have thermal headroom. 1844 */ 1845 bool i915_gpu_raise(void) 1846 { 1847 struct drm_i915_private *i915; 1848 struct intel_rps *rps; 1849 1850 i915 = mchdev_get(); 1851 if (!i915) 1852 return false; 1853 1854 rps = &i915->gt.rps; 1855 1856 spin_lock_irq(&mchdev_lock); 1857 if (rps->max_freq_softlimit < rps->max_freq) 1858 rps->max_freq_softlimit++; 1859 spin_unlock_irq(&mchdev_lock); 1860 1861 drm_dev_put(&i915->drm); 1862 return true; 1863 } 1864 EXPORT_SYMBOL_GPL(i915_gpu_raise); 1865 1866 /** 1867 * i915_gpu_lower - lower GPU frequency limit 1868 * 1869 * IPS indicates we're close to a thermal limit, so throttle back the GPU 1870 * frequency maximum. 1871 */ 1872 bool i915_gpu_lower(void) 1873 { 1874 struct drm_i915_private *i915; 1875 struct intel_rps *rps; 1876 1877 i915 = mchdev_get(); 1878 if (!i915) 1879 return false; 1880 1881 rps = &i915->gt.rps; 1882 1883 spin_lock_irq(&mchdev_lock); 1884 if (rps->max_freq_softlimit > rps->min_freq) 1885 rps->max_freq_softlimit--; 1886 spin_unlock_irq(&mchdev_lock); 1887 1888 drm_dev_put(&i915->drm); 1889 return true; 1890 } 1891 EXPORT_SYMBOL_GPL(i915_gpu_lower); 1892 1893 /** 1894 * i915_gpu_busy - indicate GPU business to IPS 1895 * 1896 * Tell the IPS driver whether or not the GPU is busy. 1897 */ 1898 bool i915_gpu_busy(void) 1899 { 1900 struct drm_i915_private *i915; 1901 bool ret; 1902 1903 i915 = mchdev_get(); 1904 if (!i915) 1905 return false; 1906 1907 ret = i915->gt.awake; 1908 1909 drm_dev_put(&i915->drm); 1910 return ret; 1911 } 1912 EXPORT_SYMBOL_GPL(i915_gpu_busy); 1913 1914 /** 1915 * i915_gpu_turbo_disable - disable graphics turbo 1916 * 1917 * Disable graphics turbo by resetting the max frequency and setting the 1918 * current frequency to the default. 1919 */ 1920 bool i915_gpu_turbo_disable(void) 1921 { 1922 struct drm_i915_private *i915; 1923 struct intel_rps *rps; 1924 bool ret; 1925 1926 i915 = mchdev_get(); 1927 if (!i915) 1928 return false; 1929 1930 rps = &i915->gt.rps; 1931 1932 spin_lock_irq(&mchdev_lock); 1933 rps->max_freq_softlimit = rps->min_freq; 1934 ret = gen5_rps_set(&i915->gt.rps, rps->min_freq); 1935 spin_unlock_irq(&mchdev_lock); 1936 1937 drm_dev_put(&i915->drm); 1938 return ret; 1939 } 1940 EXPORT_SYMBOL_GPL(i915_gpu_turbo_disable); 1941