19edbd4a0SFrançois Tigeot /*
29edbd4a0SFrançois Tigeot * Copyright © 2013 Intel Corporation
39edbd4a0SFrançois Tigeot *
49edbd4a0SFrançois Tigeot * Permission is hereby granted, free of charge, to any person obtaining a
59edbd4a0SFrançois Tigeot * copy of this software and associated documentation files (the "Software"),
69edbd4a0SFrançois Tigeot * to deal in the Software without restriction, including without limitation
79edbd4a0SFrançois Tigeot * the rights to use, copy, modify, merge, publish, distribute, sublicense,
89edbd4a0SFrançois Tigeot * and/or sell copies of the Software, and to permit persons to whom the
99edbd4a0SFrançois Tigeot * Software is furnished to do so, subject to the following conditions:
109edbd4a0SFrançois Tigeot *
119edbd4a0SFrançois Tigeot * The above copyright notice and this permission notice (including the next
129edbd4a0SFrançois Tigeot * paragraph) shall be included in all copies or substantial portions of the
139edbd4a0SFrançois Tigeot * Software.
149edbd4a0SFrançois Tigeot *
159edbd4a0SFrançois Tigeot * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
169edbd4a0SFrançois Tigeot * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
179edbd4a0SFrançois Tigeot * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
189edbd4a0SFrançois Tigeot * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
199edbd4a0SFrançois Tigeot * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
209edbd4a0SFrançois Tigeot * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
219edbd4a0SFrançois Tigeot * IN THE SOFTWARE.
229edbd4a0SFrançois Tigeot */
239edbd4a0SFrançois Tigeot
249edbd4a0SFrançois Tigeot #include "i915_drv.h"
259edbd4a0SFrançois Tigeot #include "intel_drv.h"
26477eb7f9SFrançois Tigeot #include "i915_vgpu.h"
279edbd4a0SFrançois Tigeot
28a85cb24fSFrançois Tigeot #include <asm/iosf_mbi.h>
29ff14819bSFrançois Tigeot #include <linux/pm_runtime.h>
30ff14819bSFrançois Tigeot
31352ff8bdSFrançois Tigeot #define FORCEWAKE_ACK_TIMEOUT_MS 50
32*3f2dd94aSFrançois Tigeot #define GT_FIFO_TIMEOUT_MS 10
339edbd4a0SFrançois Tigeot
34aee94f86SFrançois Tigeot #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32((dev_priv__), (reg__))
359edbd4a0SFrançois Tigeot
362c9916cdSFrançois Tigeot static const char * const forcewake_domain_names[] = {
372c9916cdSFrançois Tigeot "render",
382c9916cdSFrançois Tigeot "blitter",
392c9916cdSFrançois Tigeot "media",
402c9916cdSFrançois Tigeot };
412c9916cdSFrançois Tigeot
422c9916cdSFrançois Tigeot const char *
intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)432c9916cdSFrançois Tigeot intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)
442c9916cdSFrançois Tigeot {
45352ff8bdSFrançois Tigeot BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT);
462c9916cdSFrançois Tigeot
472c9916cdSFrançois Tigeot if (id >= 0 && id < FW_DOMAIN_ID_COUNT)
482c9916cdSFrançois Tigeot return forcewake_domain_names[id];
492c9916cdSFrançois Tigeot
502c9916cdSFrançois Tigeot WARN_ON(id);
512c9916cdSFrançois Tigeot
522c9916cdSFrançois Tigeot return "unknown";
532c9916cdSFrançois Tigeot }
542c9916cdSFrançois Tigeot
552c9916cdSFrançois Tigeot static inline void
fw_domain_reset(struct drm_i915_private * i915,const struct intel_uncore_forcewake_domain * d)56a85cb24fSFrançois Tigeot fw_domain_reset(struct drm_i915_private *i915,
57a85cb24fSFrançois Tigeot const struct intel_uncore_forcewake_domain *d)
582c9916cdSFrançois Tigeot {
59a85cb24fSFrançois Tigeot __raw_i915_write32(i915, d->reg_set, i915->uncore.fw_reset);
602c9916cdSFrançois Tigeot }
612c9916cdSFrançois Tigeot
622c9916cdSFrançois Tigeot static inline void
fw_domain_arm_timer(struct intel_uncore_forcewake_domain * d)632c9916cdSFrançois Tigeot fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d)
642c9916cdSFrançois Tigeot {
658621f407SFrançois Tigeot d->wake_count++;
668621f407SFrançois Tigeot hrtimer_start_range_ns(&d->timer,
674be47400SFrançois Tigeot NSEC_PER_MSEC,
688621f407SFrançois Tigeot NSEC_PER_MSEC,
698621f407SFrançois Tigeot HRTIMER_MODE_REL);
702c9916cdSFrançois Tigeot }
712c9916cdSFrançois Tigeot
722c9916cdSFrançois Tigeot static inline void
fw_domain_wait_ack_clear(const struct drm_i915_private * i915,const struct intel_uncore_forcewake_domain * d)73a85cb24fSFrançois Tigeot fw_domain_wait_ack_clear(const struct drm_i915_private *i915,
74a85cb24fSFrançois Tigeot const struct intel_uncore_forcewake_domain *d)
752c9916cdSFrançois Tigeot {
76a85cb24fSFrançois Tigeot if (wait_for_atomic((__raw_i915_read32(i915, d->reg_ack) &
772c9916cdSFrançois Tigeot FORCEWAKE_KERNEL) == 0,
782c9916cdSFrançois Tigeot FORCEWAKE_ACK_TIMEOUT_MS))
792c9916cdSFrançois Tigeot DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n",
802c9916cdSFrançois Tigeot intel_uncore_forcewake_domain_to_str(d->id));
812c9916cdSFrançois Tigeot }
822c9916cdSFrançois Tigeot
832c9916cdSFrançois Tigeot static inline void
fw_domain_get(struct drm_i915_private * i915,const struct intel_uncore_forcewake_domain * d)84a85cb24fSFrançois Tigeot fw_domain_get(struct drm_i915_private *i915,
85a85cb24fSFrançois Tigeot const struct intel_uncore_forcewake_domain *d)
862c9916cdSFrançois Tigeot {
87a85cb24fSFrançois Tigeot __raw_i915_write32(i915, d->reg_set, i915->uncore.fw_set);
882c9916cdSFrançois Tigeot }
892c9916cdSFrançois Tigeot
902c9916cdSFrançois Tigeot static inline void
fw_domain_wait_ack(const struct drm_i915_private * i915,const struct intel_uncore_forcewake_domain * d)91a85cb24fSFrançois Tigeot fw_domain_wait_ack(const struct drm_i915_private *i915,
92a85cb24fSFrançois Tigeot const struct intel_uncore_forcewake_domain *d)
932c9916cdSFrançois Tigeot {
94a85cb24fSFrançois Tigeot if (wait_for_atomic((__raw_i915_read32(i915, d->reg_ack) &
952c9916cdSFrançois Tigeot FORCEWAKE_KERNEL),
962c9916cdSFrançois Tigeot FORCEWAKE_ACK_TIMEOUT_MS))
972c9916cdSFrançois Tigeot DRM_ERROR("%s: timed out waiting for forcewake ack request.\n",
982c9916cdSFrançois Tigeot intel_uncore_forcewake_domain_to_str(d->id));
992c9916cdSFrançois Tigeot }
1002c9916cdSFrançois Tigeot
1012c9916cdSFrançois Tigeot static inline void
fw_domain_put(const struct drm_i915_private * i915,const struct intel_uncore_forcewake_domain * d)102a85cb24fSFrançois Tigeot fw_domain_put(const struct drm_i915_private *i915,
103a85cb24fSFrançois Tigeot const struct intel_uncore_forcewake_domain *d)
1042c9916cdSFrançois Tigeot {
105a85cb24fSFrançois Tigeot __raw_i915_write32(i915, d->reg_set, i915->uncore.fw_clear);
1062c9916cdSFrançois Tigeot }
1072c9916cdSFrançois Tigeot
1082c9916cdSFrançois Tigeot static void
fw_domains_get(struct drm_i915_private * i915,enum forcewake_domains fw_domains)109a85cb24fSFrançois Tigeot fw_domains_get(struct drm_i915_private *i915, enum forcewake_domains fw_domains)
1102c9916cdSFrançois Tigeot {
1112c9916cdSFrançois Tigeot struct intel_uncore_forcewake_domain *d;
112a85cb24fSFrançois Tigeot unsigned int tmp;
1132c9916cdSFrançois Tigeot
114a85cb24fSFrançois Tigeot GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains);
115a85cb24fSFrançois Tigeot
116a85cb24fSFrançois Tigeot for_each_fw_domain_masked(d, fw_domains, i915, tmp) {
117a85cb24fSFrançois Tigeot fw_domain_wait_ack_clear(i915, d);
118a85cb24fSFrançois Tigeot fw_domain_get(i915, d);
1192c9916cdSFrançois Tigeot }
1208621f407SFrançois Tigeot
121a85cb24fSFrançois Tigeot for_each_fw_domain_masked(d, fw_domains, i915, tmp)
122a85cb24fSFrançois Tigeot fw_domain_wait_ack(i915, d);
1234be47400SFrançois Tigeot
124a85cb24fSFrançois Tigeot i915->uncore.fw_domains_active |= fw_domains;
1252c9916cdSFrançois Tigeot }
1262c9916cdSFrançois Tigeot
1272c9916cdSFrançois Tigeot static void
fw_domains_put(struct drm_i915_private * i915,enum forcewake_domains fw_domains)128a85cb24fSFrançois Tigeot fw_domains_put(struct drm_i915_private *i915, enum forcewake_domains fw_domains)
1292c9916cdSFrançois Tigeot {
1302c9916cdSFrançois Tigeot struct intel_uncore_forcewake_domain *d;
131a85cb24fSFrançois Tigeot unsigned int tmp;
1322c9916cdSFrançois Tigeot
133a85cb24fSFrançois Tigeot GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains);
1344be47400SFrançois Tigeot
135a85cb24fSFrançois Tigeot for_each_fw_domain_masked(d, fw_domains, i915, tmp)
136a85cb24fSFrançois Tigeot fw_domain_put(i915, d);
137a85cb24fSFrançois Tigeot
138a85cb24fSFrançois Tigeot i915->uncore.fw_domains_active &= ~fw_domains;
1392c9916cdSFrançois Tigeot }
1402c9916cdSFrançois Tigeot
1412c9916cdSFrançois Tigeot static void
fw_domains_reset(struct drm_i915_private * i915,enum forcewake_domains fw_domains)142a85cb24fSFrançois Tigeot fw_domains_reset(struct drm_i915_private *i915,
143a85cb24fSFrançois Tigeot enum forcewake_domains fw_domains)
1442c9916cdSFrançois Tigeot {
1452c9916cdSFrançois Tigeot struct intel_uncore_forcewake_domain *d;
146a85cb24fSFrançois Tigeot unsigned int tmp;
1472c9916cdSFrançois Tigeot
148a85cb24fSFrançois Tigeot if (!fw_domains)
1492c9916cdSFrançois Tigeot return;
1502c9916cdSFrançois Tigeot
151a85cb24fSFrançois Tigeot GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains);
1522c9916cdSFrançois Tigeot
153a85cb24fSFrançois Tigeot for_each_fw_domain_masked(d, fw_domains, i915, tmp)
154a85cb24fSFrançois Tigeot fw_domain_reset(i915, d);
1552c9916cdSFrançois Tigeot }
1562c9916cdSFrançois Tigeot
__gen6_gt_wait_for_thread_c0(struct drm_i915_private * dev_priv)1579edbd4a0SFrançois Tigeot static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
1589edbd4a0SFrançois Tigeot {
1599edbd4a0SFrançois Tigeot /* w/a for a sporadic read returning 0 by waiting for the GT
1609edbd4a0SFrançois Tigeot * thread to wake up.
1619edbd4a0SFrançois Tigeot */
1622c9916cdSFrançois Tigeot if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) &
1632c9916cdSFrançois Tigeot GEN6_GT_THREAD_STATUS_CORE_MASK) == 0, 500))
1649edbd4a0SFrançois Tigeot DRM_ERROR("GT thread status wait timed out\n");
1659edbd4a0SFrançois Tigeot }
1669edbd4a0SFrançois Tigeot
fw_domains_get_with_thread_status(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)1672c9916cdSFrançois Tigeot static void fw_domains_get_with_thread_status(struct drm_i915_private *dev_priv,
1682c9916cdSFrançois Tigeot enum forcewake_domains fw_domains)
1699edbd4a0SFrançois Tigeot {
1702c9916cdSFrançois Tigeot fw_domains_get(dev_priv, fw_domains);
1719edbd4a0SFrançois Tigeot
1722c9916cdSFrançois Tigeot /* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */
1739edbd4a0SFrançois Tigeot __gen6_gt_wait_for_thread_c0(dev_priv);
1749edbd4a0SFrançois Tigeot }
1759edbd4a0SFrançois Tigeot
fifo_free_entries(struct drm_i915_private * dev_priv)176477eb7f9SFrançois Tigeot static inline u32 fifo_free_entries(struct drm_i915_private *dev_priv)
177477eb7f9SFrançois Tigeot {
178477eb7f9SFrançois Tigeot u32 count = __raw_i915_read32(dev_priv, GTFIFOCTL);
179477eb7f9SFrançois Tigeot
180477eb7f9SFrançois Tigeot return count & GT_FIFO_FREE_ENTRIES_MASK;
181477eb7f9SFrançois Tigeot }
182477eb7f9SFrançois Tigeot
__gen6_gt_wait_for_fifo(struct drm_i915_private * dev_priv)183*3f2dd94aSFrançois Tigeot static void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
1849edbd4a0SFrançois Tigeot {
185*3f2dd94aSFrançois Tigeot u32 n;
1869edbd4a0SFrançois Tigeot
1879edbd4a0SFrançois Tigeot /* On VLV, FIFO will be shared by both SW and HW.
1889edbd4a0SFrançois Tigeot * So, we need to read the FREE_ENTRIES everytime */
1898621f407SFrançois Tigeot if (IS_VALLEYVIEW(dev_priv))
190*3f2dd94aSFrançois Tigeot n = fifo_free_entries(dev_priv);
191*3f2dd94aSFrançois Tigeot else
192*3f2dd94aSFrançois Tigeot n = dev_priv->uncore.fifo_count;
1939edbd4a0SFrançois Tigeot
194*3f2dd94aSFrançois Tigeot if (n <= GT_FIFO_NUM_RESERVED_ENTRIES) {
195*3f2dd94aSFrançois Tigeot if (wait_for_atomic((n = fifo_free_entries(dev_priv)) >
196*3f2dd94aSFrançois Tigeot GT_FIFO_NUM_RESERVED_ENTRIES,
197*3f2dd94aSFrançois Tigeot GT_FIFO_TIMEOUT_MS)) {
198*3f2dd94aSFrançois Tigeot DRM_DEBUG("GT_FIFO timeout, entries: %u\n", n);
199*3f2dd94aSFrançois Tigeot return;
2009edbd4a0SFrançois Tigeot }
2019edbd4a0SFrançois Tigeot }
2029edbd4a0SFrançois Tigeot
203*3f2dd94aSFrançois Tigeot dev_priv->uncore.fifo_count = n - 1;
2049edbd4a0SFrançois Tigeot }
2059edbd4a0SFrançois Tigeot
2068621f407SFrançois Tigeot static enum hrtimer_restart
intel_uncore_fw_release_timer(struct hrtimer * timer)2078621f407SFrançois Tigeot intel_uncore_fw_release_timer(struct hrtimer *timer)
2089edbd4a0SFrançois Tigeot {
2098621f407SFrançois Tigeot struct intel_uncore_forcewake_domain *domain =
2108621f407SFrançois Tigeot container_of(timer, struct intel_uncore_forcewake_domain, timer);
211a85cb24fSFrançois Tigeot struct drm_i915_private *dev_priv =
212a85cb24fSFrançois Tigeot container_of(domain, struct drm_i915_private, uncore.fw_domain[domain->id]);
2135e269720SFrançois Tigeot unsigned long irqflags;
2142c9916cdSFrançois Tigeot
2151e12ee3bSFrançois Tigeot assert_rpm_device_not_suspended(dev_priv);
2162c9916cdSFrançois Tigeot
217*3f2dd94aSFrançois Tigeot if (xchg(&domain->active, false))
218*3f2dd94aSFrançois Tigeot return HRTIMER_RESTART;
219*3f2dd94aSFrançois Tigeot
2201e12ee3bSFrançois Tigeot spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
2212c9916cdSFrançois Tigeot if (WARN_ON(domain->wake_count == 0))
2222c9916cdSFrançois Tigeot domain->wake_count++;
2232c9916cdSFrançois Tigeot
2244be47400SFrançois Tigeot if (--domain->wake_count == 0)
2251e12ee3bSFrançois Tigeot dev_priv->uncore.funcs.force_wake_put(dev_priv, domain->mask);
2262c9916cdSFrançois Tigeot
2271e12ee3bSFrançois Tigeot spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
2288621f407SFrançois Tigeot
2298621f407SFrançois Tigeot return HRTIMER_NORESTART;
2309edbd4a0SFrançois Tigeot }
2319edbd4a0SFrançois Tigeot
intel_uncore_forcewake_reset(struct drm_i915_private * dev_priv,bool restore)232a85cb24fSFrançois Tigeot static void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
2331487f786SFrançois Tigeot bool restore)
2349edbd4a0SFrançois Tigeot {
2355e269720SFrançois Tigeot unsigned long irqflags;
2362c9916cdSFrançois Tigeot struct intel_uncore_forcewake_domain *domain;
2372c9916cdSFrançois Tigeot int retry_count = 100;
2381e12ee3bSFrançois Tigeot enum forcewake_domains fw, active_domains;
239ba55f2f5SFrançois Tigeot
240ba55f2f5SFrançois Tigeot /* Hold uncore.lock across reset to prevent any register access
2412c9916cdSFrançois Tigeot * with forcewake not set correctly. Wait until all pending
2422c9916cdSFrançois Tigeot * timers are run before holding.
243ba55f2f5SFrançois Tigeot */
2442c9916cdSFrançois Tigeot while (1) {
245a85cb24fSFrançois Tigeot unsigned int tmp;
246a85cb24fSFrançois Tigeot
2472c9916cdSFrançois Tigeot active_domains = 0;
248ba55f2f5SFrançois Tigeot
249a85cb24fSFrançois Tigeot for_each_fw_domain(domain, dev_priv, tmp) {
250*3f2dd94aSFrançois Tigeot smp_store_mb(domain->active, false);
2518621f407SFrançois Tigeot if (hrtimer_cancel(&domain->timer) == 0)
2522c9916cdSFrançois Tigeot continue;
253ba55f2f5SFrançois Tigeot
2548621f407SFrançois Tigeot intel_uncore_fw_release_timer(&domain->timer);
2559edbd4a0SFrançois Tigeot }
256ba55f2f5SFrançois Tigeot
2575e269720SFrançois Tigeot spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
2582c9916cdSFrançois Tigeot
259a85cb24fSFrançois Tigeot for_each_fw_domain(domain, dev_priv, tmp) {
2608621f407SFrançois Tigeot if (hrtimer_active(&domain->timer))
2618621f407SFrançois Tigeot active_domains |= domain->mask;
2622c9916cdSFrançois Tigeot }
2632c9916cdSFrançois Tigeot
2642c9916cdSFrançois Tigeot if (active_domains == 0)
2652c9916cdSFrançois Tigeot break;
2662c9916cdSFrançois Tigeot
2672c9916cdSFrançois Tigeot if (--retry_count == 0) {
2682c9916cdSFrançois Tigeot DRM_ERROR("Timed out waiting for forcewake timers to finish\n");
2692c9916cdSFrançois Tigeot break;
2702c9916cdSFrançois Tigeot }
2712c9916cdSFrançois Tigeot
2725e269720SFrançois Tigeot spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
2732c9916cdSFrançois Tigeot cond_resched();
2742c9916cdSFrançois Tigeot }
2752c9916cdSFrançois Tigeot
2762c9916cdSFrançois Tigeot WARN_ON(active_domains);
2772c9916cdSFrançois Tigeot
2781e12ee3bSFrançois Tigeot fw = dev_priv->uncore.fw_domains_active;
2792c9916cdSFrançois Tigeot if (fw)
2802c9916cdSFrançois Tigeot dev_priv->uncore.funcs.force_wake_put(dev_priv, fw);
2812c9916cdSFrançois Tigeot
282a85cb24fSFrançois Tigeot fw_domains_reset(dev_priv, dev_priv->uncore.fw_domains);
2832c9916cdSFrançois Tigeot
2842c9916cdSFrançois Tigeot if (restore) { /* If reset with a user forcewake, try to restore */
285ba55f2f5SFrançois Tigeot if (fw)
286ba55f2f5SFrançois Tigeot dev_priv->uncore.funcs.force_wake_get(dev_priv, fw);
287ba55f2f5SFrançois Tigeot
2881487f786SFrançois Tigeot if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
289ba55f2f5SFrançois Tigeot dev_priv->uncore.fifo_count =
290477eb7f9SFrançois Tigeot fifo_free_entries(dev_priv);
291ba55f2f5SFrançois Tigeot }
292ba55f2f5SFrançois Tigeot
2932c9916cdSFrançois Tigeot if (!restore)
2942c9916cdSFrançois Tigeot assert_forcewakes_inactive(dev_priv);
2952c9916cdSFrançois Tigeot
2965e269720SFrançois Tigeot spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
2979edbd4a0SFrançois Tigeot }
2989edbd4a0SFrançois Tigeot
gen9_edram_size(struct drm_i915_private * dev_priv)2998621f407SFrançois Tigeot static u64 gen9_edram_size(struct drm_i915_private *dev_priv)
3009edbd4a0SFrançois Tigeot {
3018621f407SFrançois Tigeot const unsigned int ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 };
3028621f407SFrançois Tigeot const unsigned int sets[4] = { 1, 1, 2, 2 };
3038621f407SFrançois Tigeot const u32 cap = dev_priv->edram_cap;
3049edbd4a0SFrançois Tigeot
3058621f407SFrançois Tigeot return EDRAM_NUM_BANKS(cap) *
3068621f407SFrançois Tigeot ways[EDRAM_WAYS_IDX(cap)] *
3078621f407SFrançois Tigeot sets[EDRAM_SETS_IDX(cap)] *
3088621f407SFrançois Tigeot 1024 * 1024;
3099edbd4a0SFrançois Tigeot }
3108621f407SFrançois Tigeot
intel_uncore_edram_size(struct drm_i915_private * dev_priv)3118621f407SFrançois Tigeot u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv)
3128621f407SFrançois Tigeot {
3138621f407SFrançois Tigeot if (!HAS_EDRAM(dev_priv))
3148621f407SFrançois Tigeot return 0;
3158621f407SFrançois Tigeot
3168621f407SFrançois Tigeot /* The needed capability bits for size calculation
3178621f407SFrançois Tigeot * are not there with pre gen9 so return 128MB always.
3188621f407SFrançois Tigeot */
3198621f407SFrançois Tigeot if (INTEL_GEN(dev_priv) < 9)
3208621f407SFrançois Tigeot return 128 * 1024 * 1024;
3218621f407SFrançois Tigeot
3228621f407SFrançois Tigeot return gen9_edram_size(dev_priv);
3238621f407SFrançois Tigeot }
3248621f407SFrançois Tigeot
intel_uncore_edram_detect(struct drm_i915_private * dev_priv)3258621f407SFrançois Tigeot static void intel_uncore_edram_detect(struct drm_i915_private *dev_priv)
3268621f407SFrançois Tigeot {
3278621f407SFrançois Tigeot if (IS_HASWELL(dev_priv) ||
3288621f407SFrançois Tigeot IS_BROADWELL(dev_priv) ||
3298621f407SFrançois Tigeot INTEL_GEN(dev_priv) >= 9) {
3308621f407SFrançois Tigeot dev_priv->edram_cap = __raw_i915_read32(dev_priv,
3318621f407SFrançois Tigeot HSW_EDRAM_CAP);
3328621f407SFrançois Tigeot
3338621f407SFrançois Tigeot /* NB: We can't write IDICR yet because we do not have gt funcs
3348621f407SFrançois Tigeot * set up */
3358621f407SFrançois Tigeot } else {
3368621f407SFrançois Tigeot dev_priv->edram_cap = 0;
3378621f407SFrançois Tigeot }
3388621f407SFrançois Tigeot
3398621f407SFrançois Tigeot if (HAS_EDRAM(dev_priv))
3408621f407SFrançois Tigeot DRM_INFO("Found %lluMB of eDRAM\n",
3418621f407SFrançois Tigeot intel_uncore_edram_size(dev_priv) / (1024 * 1024));
3422c9916cdSFrançois Tigeot }
3432c9916cdSFrançois Tigeot
344c0e85e96SFrançois Tigeot static bool
fpga_check_for_unclaimed_mmio(struct drm_i915_private * dev_priv)345c0e85e96SFrançois Tigeot fpga_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
346c0e85e96SFrançois Tigeot {
347c0e85e96SFrançois Tigeot u32 dbg;
348c0e85e96SFrançois Tigeot
349c0e85e96SFrançois Tigeot dbg = __raw_i915_read32(dev_priv, FPGA_DBG);
350c0e85e96SFrançois Tigeot if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM)))
351c0e85e96SFrançois Tigeot return false;
352c0e85e96SFrançois Tigeot
353c0e85e96SFrançois Tigeot __raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
354c0e85e96SFrançois Tigeot
355c0e85e96SFrançois Tigeot return true;
356c0e85e96SFrançois Tigeot }
357c0e85e96SFrançois Tigeot
358c0e85e96SFrançois Tigeot static bool
vlv_check_for_unclaimed_mmio(struct drm_i915_private * dev_priv)359c0e85e96SFrançois Tigeot vlv_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
360c0e85e96SFrançois Tigeot {
361c0e85e96SFrançois Tigeot u32 cer;
362c0e85e96SFrançois Tigeot
363c0e85e96SFrançois Tigeot cer = __raw_i915_read32(dev_priv, CLAIM_ER);
364c0e85e96SFrançois Tigeot if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK))))
365c0e85e96SFrançois Tigeot return false;
366c0e85e96SFrançois Tigeot
367c0e85e96SFrançois Tigeot __raw_i915_write32(dev_priv, CLAIM_ER, CLAIM_ER_CLR);
368c0e85e96SFrançois Tigeot
369c0e85e96SFrançois Tigeot return true;
370c0e85e96SFrançois Tigeot }
371c0e85e96SFrançois Tigeot
372c0e85e96SFrançois Tigeot static bool
gen6_check_for_fifo_debug(struct drm_i915_private * dev_priv)373*3f2dd94aSFrançois Tigeot gen6_check_for_fifo_debug(struct drm_i915_private *dev_priv)
374*3f2dd94aSFrançois Tigeot {
375*3f2dd94aSFrançois Tigeot u32 fifodbg;
376*3f2dd94aSFrançois Tigeot
377*3f2dd94aSFrançois Tigeot fifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
378*3f2dd94aSFrançois Tigeot
379*3f2dd94aSFrançois Tigeot if (unlikely(fifodbg)) {
380*3f2dd94aSFrançois Tigeot DRM_DEBUG_DRIVER("GTFIFODBG = 0x08%x\n", fifodbg);
381*3f2dd94aSFrançois Tigeot __raw_i915_write32(dev_priv, GTFIFODBG, fifodbg);
382*3f2dd94aSFrançois Tigeot }
383*3f2dd94aSFrançois Tigeot
384*3f2dd94aSFrançois Tigeot return fifodbg;
385*3f2dd94aSFrançois Tigeot }
386*3f2dd94aSFrançois Tigeot
387*3f2dd94aSFrançois Tigeot static bool
check_for_unclaimed_mmio(struct drm_i915_private * dev_priv)388c0e85e96SFrançois Tigeot check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
389c0e85e96SFrançois Tigeot {
390*3f2dd94aSFrançois Tigeot bool ret = false;
391*3f2dd94aSFrançois Tigeot
392c0e85e96SFrançois Tigeot if (HAS_FPGA_DBG_UNCLAIMED(dev_priv))
393*3f2dd94aSFrançois Tigeot ret |= fpga_check_for_unclaimed_mmio(dev_priv);
394c0e85e96SFrançois Tigeot
395c0e85e96SFrançois Tigeot if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
396*3f2dd94aSFrançois Tigeot ret |= vlv_check_for_unclaimed_mmio(dev_priv);
397c0e85e96SFrançois Tigeot
398*3f2dd94aSFrançois Tigeot if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
399*3f2dd94aSFrançois Tigeot ret |= gen6_check_for_fifo_debug(dev_priv);
400*3f2dd94aSFrançois Tigeot
401*3f2dd94aSFrançois Tigeot return ret;
402c0e85e96SFrançois Tigeot }
403c0e85e96SFrançois Tigeot
__intel_uncore_early_sanitize(struct drm_i915_private * dev_priv,bool restore_forcewake)4041487f786SFrançois Tigeot static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
4052c9916cdSFrançois Tigeot bool restore_forcewake)
4062c9916cdSFrançois Tigeot {
407c0e85e96SFrançois Tigeot /* clear out unclaimed reg detection bit */
408c0e85e96SFrançois Tigeot if (check_for_unclaimed_mmio(dev_priv))
409c0e85e96SFrançois Tigeot DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n");
4109edbd4a0SFrançois Tigeot
411477eb7f9SFrançois Tigeot /* WaDisableShadowRegForCpd:chv */
4121487f786SFrançois Tigeot if (IS_CHERRYVIEW(dev_priv)) {
413477eb7f9SFrançois Tigeot __raw_i915_write32(dev_priv, GTFIFOCTL,
414477eb7f9SFrançois Tigeot __raw_i915_read32(dev_priv, GTFIFOCTL) |
415477eb7f9SFrançois Tigeot GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL |
416477eb7f9SFrançois Tigeot GT_FIFO_CTL_RC6_POLICY_STALL);
417477eb7f9SFrançois Tigeot }
418477eb7f9SFrançois Tigeot
4191487f786SFrançois Tigeot intel_uncore_forcewake_reset(dev_priv, restore_forcewake);
4209edbd4a0SFrançois Tigeot }
4219edbd4a0SFrançois Tigeot
intel_uncore_suspend(struct drm_i915_private * dev_priv)422a85cb24fSFrançois Tigeot void intel_uncore_suspend(struct drm_i915_private *dev_priv)
4232c9916cdSFrançois Tigeot {
424a85cb24fSFrançois Tigeot iosf_mbi_unregister_pmic_bus_access_notifier(
425a85cb24fSFrançois Tigeot &dev_priv->uncore.pmic_bus_access_nb);
426a85cb24fSFrançois Tigeot intel_uncore_forcewake_reset(dev_priv, false);
427a85cb24fSFrançois Tigeot }
428a85cb24fSFrançois Tigeot
intel_uncore_resume_early(struct drm_i915_private * dev_priv)429a85cb24fSFrançois Tigeot void intel_uncore_resume_early(struct drm_i915_private *dev_priv)
430a85cb24fSFrançois Tigeot {
431a85cb24fSFrançois Tigeot __intel_uncore_early_sanitize(dev_priv, true);
432a85cb24fSFrançois Tigeot iosf_mbi_register_pmic_bus_access_notifier(
433a85cb24fSFrançois Tigeot &dev_priv->uncore.pmic_bus_access_nb);
4341487f786SFrançois Tigeot i915_check_and_clear_faults(dev_priv);
4352c9916cdSFrançois Tigeot }
4362c9916cdSFrançois Tigeot
intel_uncore_runtime_resume(struct drm_i915_private * dev_priv)437*3f2dd94aSFrançois Tigeot void intel_uncore_runtime_resume(struct drm_i915_private *dev_priv)
438*3f2dd94aSFrançois Tigeot {
439*3f2dd94aSFrançois Tigeot iosf_mbi_register_pmic_bus_access_notifier(
440*3f2dd94aSFrançois Tigeot &dev_priv->uncore.pmic_bus_access_nb);
441*3f2dd94aSFrançois Tigeot }
442*3f2dd94aSFrançois Tigeot
intel_uncore_sanitize(struct drm_i915_private * dev_priv)4431487f786SFrançois Tigeot void intel_uncore_sanitize(struct drm_i915_private *dev_priv)
4449edbd4a0SFrançois Tigeot {
445*3f2dd94aSFrançois Tigeot i915_modparams.enable_rc6 =
446*3f2dd94aSFrançois Tigeot sanitize_rc6_option(dev_priv, i915_modparams.enable_rc6);
447c0e85e96SFrançois Tigeot
4489edbd4a0SFrançois Tigeot /* BIOS often leaves RC6 enabled, but disable it for hw init */
44987df8fc6SFrançois Tigeot intel_sanitize_gt_powersave(dev_priv);
4509edbd4a0SFrançois Tigeot }
4519edbd4a0SFrançois Tigeot
__intel_uncore_forcewake_get(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)45219c468b4SFrançois Tigeot static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
45319c468b4SFrançois Tigeot enum forcewake_domains fw_domains)
45419c468b4SFrançois Tigeot {
45519c468b4SFrançois Tigeot struct intel_uncore_forcewake_domain *domain;
456a85cb24fSFrançois Tigeot unsigned int tmp;
45719c468b4SFrançois Tigeot
45819c468b4SFrançois Tigeot fw_domains &= dev_priv->uncore.fw_domains;
45919c468b4SFrançois Tigeot
460*3f2dd94aSFrançois Tigeot for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) {
461*3f2dd94aSFrançois Tigeot if (domain->wake_count++) {
4628621f407SFrançois Tigeot fw_domains &= ~domain->mask;
463*3f2dd94aSFrançois Tigeot domain->active = true;
464*3f2dd94aSFrançois Tigeot }
465*3f2dd94aSFrançois Tigeot }
46619c468b4SFrançois Tigeot
4674be47400SFrançois Tigeot if (fw_domains)
46819c468b4SFrançois Tigeot dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
46919c468b4SFrançois Tigeot }
47019c468b4SFrançois Tigeot
4712c9916cdSFrançois Tigeot /**
4722c9916cdSFrançois Tigeot * intel_uncore_forcewake_get - grab forcewake domain references
4732c9916cdSFrançois Tigeot * @dev_priv: i915 device instance
4742c9916cdSFrançois Tigeot * @fw_domains: forcewake domains to get reference on
4752c9916cdSFrançois Tigeot *
4762c9916cdSFrançois Tigeot * This function can be used get GT's forcewake domain references.
4772c9916cdSFrançois Tigeot * Normal register access will handle the forcewake domains automatically.
4782c9916cdSFrançois Tigeot * However if some sequence requires the GT to not power down a particular
4792c9916cdSFrançois Tigeot * forcewake domains this function should be called at the beginning of the
4802c9916cdSFrançois Tigeot * sequence. And subsequently the reference should be dropped by symmetric
4812c9916cdSFrançois Tigeot * call to intel_unforce_forcewake_put(). Usually caller wants all the domains
4822c9916cdSFrançois Tigeot * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL.
4839edbd4a0SFrançois Tigeot */
intel_uncore_forcewake_get(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)4842c9916cdSFrançois Tigeot void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
4852c9916cdSFrançois Tigeot enum forcewake_domains fw_domains)
4869edbd4a0SFrançois Tigeot {
4875e269720SFrançois Tigeot unsigned long irqflags;
4882c9916cdSFrançois Tigeot
4899edbd4a0SFrançois Tigeot if (!dev_priv->uncore.funcs.force_wake_get)
4909edbd4a0SFrançois Tigeot return;
4919edbd4a0SFrançois Tigeot
492aee94f86SFrançois Tigeot assert_rpm_wakelock_held(dev_priv);
4939edbd4a0SFrançois Tigeot
4945e269720SFrançois Tigeot spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
49519c468b4SFrançois Tigeot __intel_uncore_forcewake_get(dev_priv, fw_domains);
4965e269720SFrançois Tigeot spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
4972c9916cdSFrançois Tigeot }
4982c9916cdSFrançois Tigeot
49919c468b4SFrançois Tigeot /**
500*3f2dd94aSFrançois Tigeot * intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace
501*3f2dd94aSFrançois Tigeot * @dev_priv: i915 device instance
502*3f2dd94aSFrançois Tigeot *
503*3f2dd94aSFrançois Tigeot * This function is a wrapper around intel_uncore_forcewake_get() to acquire
504*3f2dd94aSFrançois Tigeot * the GT powerwell and in the process disable our debugging for the
505*3f2dd94aSFrançois Tigeot * duration of userspace's bypass.
506*3f2dd94aSFrançois Tigeot */
intel_uncore_forcewake_user_get(struct drm_i915_private * dev_priv)507*3f2dd94aSFrançois Tigeot void intel_uncore_forcewake_user_get(struct drm_i915_private *dev_priv)
508*3f2dd94aSFrançois Tigeot {
509*3f2dd94aSFrançois Tigeot spin_lock_irq(&dev_priv->uncore.lock);
510*3f2dd94aSFrançois Tigeot if (!dev_priv->uncore.user_forcewake.count++) {
511*3f2dd94aSFrançois Tigeot intel_uncore_forcewake_get__locked(dev_priv, FORCEWAKE_ALL);
512*3f2dd94aSFrançois Tigeot
513*3f2dd94aSFrançois Tigeot /* Save and disable mmio debugging for the user bypass */
514*3f2dd94aSFrançois Tigeot dev_priv->uncore.user_forcewake.saved_mmio_check =
515*3f2dd94aSFrançois Tigeot dev_priv->uncore.unclaimed_mmio_check;
516*3f2dd94aSFrançois Tigeot dev_priv->uncore.user_forcewake.saved_mmio_debug =
517*3f2dd94aSFrançois Tigeot i915_modparams.mmio_debug;
518*3f2dd94aSFrançois Tigeot
519*3f2dd94aSFrançois Tigeot dev_priv->uncore.unclaimed_mmio_check = 0;
520*3f2dd94aSFrançois Tigeot i915_modparams.mmio_debug = 0;
521*3f2dd94aSFrançois Tigeot }
522*3f2dd94aSFrançois Tigeot spin_unlock_irq(&dev_priv->uncore.lock);
523*3f2dd94aSFrançois Tigeot }
524*3f2dd94aSFrançois Tigeot
525*3f2dd94aSFrançois Tigeot /**
526*3f2dd94aSFrançois Tigeot * intel_uncore_forcewake_user_put - release forcewake on behalf of userspace
527*3f2dd94aSFrançois Tigeot * @dev_priv: i915 device instance
528*3f2dd94aSFrançois Tigeot *
529*3f2dd94aSFrançois Tigeot * This function complements intel_uncore_forcewake_user_get() and releases
530*3f2dd94aSFrançois Tigeot * the GT powerwell taken on behalf of the userspace bypass.
531*3f2dd94aSFrançois Tigeot */
intel_uncore_forcewake_user_put(struct drm_i915_private * dev_priv)532*3f2dd94aSFrançois Tigeot void intel_uncore_forcewake_user_put(struct drm_i915_private *dev_priv)
533*3f2dd94aSFrançois Tigeot {
534*3f2dd94aSFrançois Tigeot spin_lock_irq(&dev_priv->uncore.lock);
535*3f2dd94aSFrançois Tigeot if (!--dev_priv->uncore.user_forcewake.count) {
536*3f2dd94aSFrançois Tigeot if (intel_uncore_unclaimed_mmio(dev_priv))
537*3f2dd94aSFrançois Tigeot dev_info(dev_priv->drm.dev,
538*3f2dd94aSFrançois Tigeot "Invalid mmio detected during user access\n");
539*3f2dd94aSFrançois Tigeot
540*3f2dd94aSFrançois Tigeot dev_priv->uncore.unclaimed_mmio_check =
541*3f2dd94aSFrançois Tigeot dev_priv->uncore.user_forcewake.saved_mmio_check;
542*3f2dd94aSFrançois Tigeot i915_modparams.mmio_debug =
543*3f2dd94aSFrançois Tigeot dev_priv->uncore.user_forcewake.saved_mmio_debug;
544*3f2dd94aSFrançois Tigeot
545*3f2dd94aSFrançois Tigeot intel_uncore_forcewake_put__locked(dev_priv, FORCEWAKE_ALL);
546*3f2dd94aSFrançois Tigeot }
547*3f2dd94aSFrançois Tigeot spin_unlock_irq(&dev_priv->uncore.lock);
548*3f2dd94aSFrançois Tigeot }
549*3f2dd94aSFrançois Tigeot
550*3f2dd94aSFrançois Tigeot /**
55119c468b4SFrançois Tigeot * intel_uncore_forcewake_get__locked - grab forcewake domain references
55219c468b4SFrançois Tigeot * @dev_priv: i915 device instance
55319c468b4SFrançois Tigeot * @fw_domains: forcewake domains to get reference on
55419c468b4SFrançois Tigeot *
55519c468b4SFrançois Tigeot * See intel_uncore_forcewake_get(). This variant places the onus
55619c468b4SFrançois Tigeot * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
55719c468b4SFrançois Tigeot */
intel_uncore_forcewake_get__locked(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)55819c468b4SFrançois Tigeot void intel_uncore_forcewake_get__locked(struct drm_i915_private *dev_priv,
55919c468b4SFrançois Tigeot enum forcewake_domains fw_domains)
56019c468b4SFrançois Tigeot {
561a85cb24fSFrançois Tigeot lockdep_assert_held(&dev_priv->uncore.lock);
5622c9916cdSFrançois Tigeot
56319c468b4SFrançois Tigeot if (!dev_priv->uncore.funcs.force_wake_get)
56419c468b4SFrançois Tigeot return;
56519c468b4SFrançois Tigeot
56619c468b4SFrançois Tigeot __intel_uncore_forcewake_get(dev_priv, fw_domains);
56719c468b4SFrançois Tigeot }
56819c468b4SFrançois Tigeot
__intel_uncore_forcewake_put(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)56919c468b4SFrançois Tigeot static void __intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
57019c468b4SFrançois Tigeot enum forcewake_domains fw_domains)
57119c468b4SFrançois Tigeot {
57219c468b4SFrançois Tigeot struct intel_uncore_forcewake_domain *domain;
573a85cb24fSFrançois Tigeot unsigned int tmp;
57419c468b4SFrançois Tigeot
57519c468b4SFrançois Tigeot fw_domains &= dev_priv->uncore.fw_domains;
57619c468b4SFrançois Tigeot
577a85cb24fSFrançois Tigeot for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) {
57819c468b4SFrançois Tigeot if (WARN_ON(domain->wake_count == 0))
57919c468b4SFrançois Tigeot continue;
58019c468b4SFrançois Tigeot
581*3f2dd94aSFrançois Tigeot if (--domain->wake_count) {
582*3f2dd94aSFrançois Tigeot domain->active = true;
58319c468b4SFrançois Tigeot continue;
584*3f2dd94aSFrançois Tigeot }
58519c468b4SFrançois Tigeot
58619c468b4SFrançois Tigeot fw_domain_arm_timer(domain);
58719c468b4SFrançois Tigeot }
5889edbd4a0SFrançois Tigeot }
5899edbd4a0SFrançois Tigeot
5902c9916cdSFrançois Tigeot /**
5912c9916cdSFrançois Tigeot * intel_uncore_forcewake_put - release a forcewake domain reference
5922c9916cdSFrançois Tigeot * @dev_priv: i915 device instance
5932c9916cdSFrançois Tigeot * @fw_domains: forcewake domains to put references
5942c9916cdSFrançois Tigeot *
5952c9916cdSFrançois Tigeot * This function drops the device-level forcewakes for specified
5962c9916cdSFrançois Tigeot * domains obtained by intel_uncore_forcewake_get().
5979edbd4a0SFrançois Tigeot */
intel_uncore_forcewake_put(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)5982c9916cdSFrançois Tigeot void intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
5992c9916cdSFrançois Tigeot enum forcewake_domains fw_domains)
6009edbd4a0SFrançois Tigeot {
6014db2f768SFrançois Tigeot unsigned long irqflags;
602ba55f2f5SFrançois Tigeot
6039edbd4a0SFrançois Tigeot if (!dev_priv->uncore.funcs.force_wake_put)
6049edbd4a0SFrançois Tigeot return;
6059edbd4a0SFrançois Tigeot
6064db2f768SFrançois Tigeot spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
60719c468b4SFrançois Tigeot __intel_uncore_forcewake_put(dev_priv, fw_domains);
6084db2f768SFrançois Tigeot spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
6099edbd4a0SFrançois Tigeot }
6102c9916cdSFrançois Tigeot
61119c468b4SFrançois Tigeot /**
61219c468b4SFrançois Tigeot * intel_uncore_forcewake_put__locked - grab forcewake domain references
61319c468b4SFrançois Tigeot * @dev_priv: i915 device instance
61419c468b4SFrançois Tigeot * @fw_domains: forcewake domains to get reference on
61519c468b4SFrançois Tigeot *
61619c468b4SFrançois Tigeot * See intel_uncore_forcewake_put(). This variant places the onus
61719c468b4SFrançois Tigeot * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
61819c468b4SFrançois Tigeot */
intel_uncore_forcewake_put__locked(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)61919c468b4SFrançois Tigeot void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv,
62019c468b4SFrançois Tigeot enum forcewake_domains fw_domains)
62119c468b4SFrançois Tigeot {
622a85cb24fSFrançois Tigeot lockdep_assert_held(&dev_priv->uncore.lock);
62319c468b4SFrançois Tigeot
62419c468b4SFrançois Tigeot if (!dev_priv->uncore.funcs.force_wake_put)
62519c468b4SFrançois Tigeot return;
62619c468b4SFrançois Tigeot
62719c468b4SFrançois Tigeot __intel_uncore_forcewake_put(dev_priv, fw_domains);
6289edbd4a0SFrançois Tigeot }
6299edbd4a0SFrançois Tigeot
assert_forcewakes_inactive(struct drm_i915_private * dev_priv)6302c9916cdSFrançois Tigeot void assert_forcewakes_inactive(struct drm_i915_private *dev_priv)
631ba55f2f5SFrançois Tigeot {
632ba55f2f5SFrançois Tigeot if (!dev_priv->uncore.funcs.force_wake_get)
633ba55f2f5SFrançois Tigeot return;
634ba55f2f5SFrançois Tigeot
635*3f2dd94aSFrançois Tigeot WARN(dev_priv->uncore.fw_domains_active,
636*3f2dd94aSFrançois Tigeot "Expected all fw_domains to be inactive, but %08x are still on\n",
637*3f2dd94aSFrançois Tigeot dev_priv->uncore.fw_domains_active);
638*3f2dd94aSFrançois Tigeot }
639*3f2dd94aSFrançois Tigeot
assert_forcewakes_active(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)640*3f2dd94aSFrançois Tigeot void assert_forcewakes_active(struct drm_i915_private *dev_priv,
641*3f2dd94aSFrançois Tigeot enum forcewake_domains fw_domains)
642*3f2dd94aSFrançois Tigeot {
643*3f2dd94aSFrançois Tigeot if (!dev_priv->uncore.funcs.force_wake_get)
644*3f2dd94aSFrançois Tigeot return;
645*3f2dd94aSFrançois Tigeot
646*3f2dd94aSFrançois Tigeot assert_rpm_wakelock_held(dev_priv);
647*3f2dd94aSFrançois Tigeot
648*3f2dd94aSFrançois Tigeot fw_domains &= dev_priv->uncore.fw_domains;
649*3f2dd94aSFrançois Tigeot WARN(fw_domains & ~dev_priv->uncore.fw_domains_active,
650*3f2dd94aSFrançois Tigeot "Expected %08x fw_domains to be active, but %08x are off\n",
651*3f2dd94aSFrançois Tigeot fw_domains, fw_domains & ~dev_priv->uncore.fw_domains_active);
652ba55f2f5SFrançois Tigeot }
653ba55f2f5SFrançois Tigeot
6549edbd4a0SFrançois Tigeot /* We give fast paths for the really cool registers */
655aee94f86SFrançois Tigeot #define NEEDS_FORCE_WAKE(reg) ((reg) < 0x40000)
6569edbd4a0SFrançois Tigeot
6578621f407SFrançois Tigeot #define __gen6_reg_read_fw_domains(offset) \
6588621f407SFrançois Tigeot ({ \
6598621f407SFrançois Tigeot enum forcewake_domains __fwd; \
6608621f407SFrançois Tigeot if (NEEDS_FORCE_WAKE(offset)) \
6618621f407SFrançois Tigeot __fwd = FORCEWAKE_RENDER; \
6628621f407SFrançois Tigeot else \
6638621f407SFrançois Tigeot __fwd = 0; \
6648621f407SFrançois Tigeot __fwd; \
6658621f407SFrançois Tigeot })
6668621f407SFrançois Tigeot
fw_range_cmp(u32 offset,const struct intel_forcewake_range * entry)6671e12ee3bSFrançois Tigeot static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry)
6681e12ee3bSFrançois Tigeot {
6691e12ee3bSFrançois Tigeot if (offset < entry->start)
6701e12ee3bSFrançois Tigeot return -1;
6711e12ee3bSFrançois Tigeot else if (offset > entry->end)
6721e12ee3bSFrançois Tigeot return 1;
6731e12ee3bSFrançois Tigeot else
6741e12ee3bSFrançois Tigeot return 0;
6751e12ee3bSFrançois Tigeot }
67624edb884SFrançois Tigeot
6771e12ee3bSFrançois Tigeot /* Copied and "macroized" from lib/bsearch.c */
6781e12ee3bSFrançois Tigeot #define BSEARCH(key, base, num, cmp) ({ \
6791e12ee3bSFrançois Tigeot unsigned int start__ = 0, end__ = (num); \
6801e12ee3bSFrançois Tigeot typeof(base) result__ = NULL; \
6811e12ee3bSFrançois Tigeot while (start__ < end__) { \
6821e12ee3bSFrançois Tigeot unsigned int mid__ = start__ + (end__ - start__) / 2; \
6831e12ee3bSFrançois Tigeot int ret__ = (cmp)((key), (base) + mid__); \
6841e12ee3bSFrançois Tigeot if (ret__ < 0) { \
6851e12ee3bSFrançois Tigeot end__ = mid__; \
6861e12ee3bSFrançois Tigeot } else if (ret__ > 0) { \
6871e12ee3bSFrançois Tigeot start__ = mid__ + 1; \
6881e12ee3bSFrançois Tigeot } else { \
6891e12ee3bSFrançois Tigeot result__ = (base) + mid__; \
6901e12ee3bSFrançois Tigeot break; \
6911e12ee3bSFrançois Tigeot } \
6921e12ee3bSFrançois Tigeot } \
6931e12ee3bSFrançois Tigeot result__; \
6941e12ee3bSFrançois Tigeot })
695ba55f2f5SFrançois Tigeot
6961e12ee3bSFrançois Tigeot static enum forcewake_domains
find_fw_domain(struct drm_i915_private * dev_priv,u32 offset)6971e12ee3bSFrançois Tigeot find_fw_domain(struct drm_i915_private *dev_priv, u32 offset)
6981e12ee3bSFrançois Tigeot {
6991e12ee3bSFrançois Tigeot const struct intel_forcewake_range *entry;
70024edb884SFrançois Tigeot
7011e12ee3bSFrançois Tigeot entry = BSEARCH(offset,
7021e12ee3bSFrançois Tigeot dev_priv->uncore.fw_domains_table,
7031e12ee3bSFrançois Tigeot dev_priv->uncore.fw_domains_table_entries,
7041e12ee3bSFrançois Tigeot fw_range_cmp);
7051e12ee3bSFrançois Tigeot
706a85cb24fSFrançois Tigeot if (!entry)
707a85cb24fSFrançois Tigeot return 0;
7081e12ee3bSFrançois Tigeot
709a85cb24fSFrançois Tigeot WARN(entry->domains & ~dev_priv->uncore.fw_domains,
710a85cb24fSFrançois Tigeot "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n",
711a85cb24fSFrançois Tigeot entry->domains & ~dev_priv->uncore.fw_domains, offset);
7121e12ee3bSFrançois Tigeot
713a85cb24fSFrançois Tigeot return entry->domains;
7141e12ee3bSFrançois Tigeot }
7151e12ee3bSFrançois Tigeot
7161e12ee3bSFrançois Tigeot #define GEN_FW_RANGE(s, e, d) \
7171e12ee3bSFrançois Tigeot { .start = (s), .end = (e), .domains = (d) }
7181e12ee3bSFrançois Tigeot
7191e12ee3bSFrançois Tigeot #define HAS_FWTABLE(dev_priv) \
720*3f2dd94aSFrançois Tigeot (INTEL_GEN(dev_priv) >= 9 || \
7211e12ee3bSFrançois Tigeot IS_CHERRYVIEW(dev_priv) || \
7221e12ee3bSFrançois Tigeot IS_VALLEYVIEW(dev_priv))
7231e12ee3bSFrançois Tigeot
7241e12ee3bSFrançois Tigeot /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
7251e12ee3bSFrançois Tigeot static const struct intel_forcewake_range __vlv_fw_ranges[] = {
7261e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
7271e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER),
7281e12ee3bSFrançois Tigeot GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER),
7291e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
7301e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA),
7311e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER),
7321e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
7331e12ee3bSFrançois Tigeot };
7341e12ee3bSFrançois Tigeot
7351e12ee3bSFrançois Tigeot #define __fwtable_reg_read_fw_domains(offset) \
7368621f407SFrançois Tigeot ({ \
7378621f407SFrançois Tigeot enum forcewake_domains __fwd = 0; \
7381e12ee3bSFrançois Tigeot if (NEEDS_FORCE_WAKE((offset))) \
7391e12ee3bSFrançois Tigeot __fwd = find_fw_domain(dev_priv, offset); \
7408621f407SFrançois Tigeot __fwd; \
7418621f407SFrançois Tigeot })
7428621f407SFrançois Tigeot
7431e12ee3bSFrançois Tigeot /* *Must* be sorted by offset! See intel_shadow_table_check(). */
7448621f407SFrançois Tigeot static const i915_reg_t gen8_shadowed_regs[] = {
7451e12ee3bSFrançois Tigeot RING_TAIL(RENDER_RING_BASE), /* 0x2000 (base) */
7461e12ee3bSFrançois Tigeot GEN6_RPNSWREQ, /* 0xA008 */
7471e12ee3bSFrançois Tigeot GEN6_RC_VIDEO_FREQ, /* 0xA00C */
7481e12ee3bSFrançois Tigeot RING_TAIL(GEN6_BSD_RING_BASE), /* 0x12000 (base) */
7491e12ee3bSFrançois Tigeot RING_TAIL(VEBOX_RING_BASE), /* 0x1a000 (base) */
7501e12ee3bSFrançois Tigeot RING_TAIL(BLT_RING_BASE), /* 0x22000 (base) */
7518621f407SFrançois Tigeot /* TODO: Other registers are not yet used */
7528621f407SFrançois Tigeot };
7538621f407SFrançois Tigeot
mmio_reg_cmp(u32 key,const i915_reg_t * reg)7541e12ee3bSFrançois Tigeot static int mmio_reg_cmp(u32 key, const i915_reg_t *reg)
7551e12ee3bSFrançois Tigeot {
7561e12ee3bSFrançois Tigeot u32 offset = i915_mmio_reg_offset(*reg);
7571e12ee3bSFrançois Tigeot
7581e12ee3bSFrançois Tigeot if (key < offset)
7591e12ee3bSFrançois Tigeot return -1;
7601e12ee3bSFrançois Tigeot else if (key > offset)
7611e12ee3bSFrançois Tigeot return 1;
7621e12ee3bSFrançois Tigeot else
7631e12ee3bSFrançois Tigeot return 0;
7641e12ee3bSFrançois Tigeot }
7651e12ee3bSFrançois Tigeot
is_gen8_shadowed(u32 offset)7668621f407SFrançois Tigeot static bool is_gen8_shadowed(u32 offset)
7678621f407SFrançois Tigeot {
7681e12ee3bSFrançois Tigeot const i915_reg_t *regs = gen8_shadowed_regs;
7698621f407SFrançois Tigeot
7701e12ee3bSFrançois Tigeot return BSEARCH(offset, regs, ARRAY_SIZE(gen8_shadowed_regs),
7711e12ee3bSFrançois Tigeot mmio_reg_cmp);
7728621f407SFrançois Tigeot }
7738621f407SFrançois Tigeot
7748621f407SFrançois Tigeot #define __gen8_reg_write_fw_domains(offset) \
7758621f407SFrançois Tigeot ({ \
7768621f407SFrançois Tigeot enum forcewake_domains __fwd; \
7778621f407SFrançois Tigeot if (NEEDS_FORCE_WAKE(offset) && !is_gen8_shadowed(offset)) \
7788621f407SFrançois Tigeot __fwd = FORCEWAKE_RENDER; \
7798621f407SFrançois Tigeot else \
7808621f407SFrançois Tigeot __fwd = 0; \
7818621f407SFrançois Tigeot __fwd; \
7828621f407SFrançois Tigeot })
7838621f407SFrançois Tigeot
7841e12ee3bSFrançois Tigeot /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
7851e12ee3bSFrançois Tigeot static const struct intel_forcewake_range __chv_fw_ranges[] = {
7861e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
7871e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
7881e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
7891e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
7901e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
7911e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
7921e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA),
7931e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
7941e12ee3bSFrançois Tigeot GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
7951e12ee3bSFrançois Tigeot GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
7961e12ee3bSFrançois Tigeot GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER),
7971e12ee3bSFrançois Tigeot GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
7981e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
7991e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA),
8001e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA),
8011e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA),
8028621f407SFrançois Tigeot };
8038621f407SFrançois Tigeot
8041e12ee3bSFrançois Tigeot #define __fwtable_reg_write_fw_domains(offset) \
8058621f407SFrançois Tigeot ({ \
8061e12ee3bSFrançois Tigeot enum forcewake_domains __fwd = 0; \
8071e12ee3bSFrançois Tigeot if (NEEDS_FORCE_WAKE((offset)) && !is_gen8_shadowed(offset)) \
8081e12ee3bSFrançois Tigeot __fwd = find_fw_domain(dev_priv, offset); \
8098621f407SFrançois Tigeot __fwd; \
8108621f407SFrançois Tigeot })
8118621f407SFrançois Tigeot
8121e12ee3bSFrançois Tigeot /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
8131e12ee3bSFrançois Tigeot static const struct intel_forcewake_range __gen9_fw_ranges[] = {
8141e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER),
8151e12ee3bSFrançois Tigeot GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
8161e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
8171e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER),
8181e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
8191e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER),
8201e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
8211e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_BLITTER),
8221e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA),
8231e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
8241e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER),
8251e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
8261e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_BLITTER),
8271e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA),
8281e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_BLITTER),
8291e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
8301e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER),
8311e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
8321e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER),
8331e12ee3bSFrançois Tigeot GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
8344be47400SFrançois Tigeot GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_BLITTER),
8351e12ee3bSFrançois Tigeot GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
8361e12ee3bSFrançois Tigeot GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER),
8371e12ee3bSFrançois Tigeot GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
8381e12ee3bSFrançois Tigeot GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_BLITTER),
8391e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
8401e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_BLITTER),
8411e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA),
8421e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_BLITTER),
8431e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER),
8441e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_BLITTER),
8451e12ee3bSFrançois Tigeot GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
8461e12ee3bSFrançois Tigeot };
8471e12ee3bSFrançois Tigeot
8489edbd4a0SFrançois Tigeot static void
ilk_dummy_write(struct drm_i915_private * dev_priv)8499edbd4a0SFrançois Tigeot ilk_dummy_write(struct drm_i915_private *dev_priv)
8509edbd4a0SFrançois Tigeot {
8519edbd4a0SFrançois Tigeot /* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
8529edbd4a0SFrançois Tigeot * the chip from rc6 before touching it for real. MI_MODE is masked,
8539edbd4a0SFrançois Tigeot * hence harmless to write 0 into. */
8549edbd4a0SFrançois Tigeot __raw_i915_write32(dev_priv, MI_MODE, 0);
8559edbd4a0SFrançois Tigeot }
8569edbd4a0SFrançois Tigeot
8579edbd4a0SFrançois Tigeot static void
__unclaimed_reg_debug(struct drm_i915_private * dev_priv,const i915_reg_t reg,const bool read,const bool before)858c0e85e96SFrançois Tigeot __unclaimed_reg_debug(struct drm_i915_private *dev_priv,
859c0e85e96SFrançois Tigeot const i915_reg_t reg,
860c0e85e96SFrançois Tigeot const bool read,
861c0e85e96SFrançois Tigeot const bool before)
8629edbd4a0SFrançois Tigeot {
8631e12ee3bSFrançois Tigeot if (WARN(check_for_unclaimed_mmio(dev_priv) && !before,
8641e12ee3bSFrançois Tigeot "Unclaimed %s register 0x%x\n",
8651e12ee3bSFrançois Tigeot read ? "read from" : "write to",
866c0e85e96SFrançois Tigeot i915_mmio_reg_offset(reg)))
867*3f2dd94aSFrançois Tigeot /* Only report the first N failures */
868*3f2dd94aSFrançois Tigeot i915_modparams.mmio_debug--;
8699edbd4a0SFrançois Tigeot }
8709edbd4a0SFrançois Tigeot
871c0e85e96SFrançois Tigeot static inline void
unclaimed_reg_debug(struct drm_i915_private * dev_priv,const i915_reg_t reg,const bool read,const bool before)872c0e85e96SFrançois Tigeot unclaimed_reg_debug(struct drm_i915_private *dev_priv,
873c0e85e96SFrançois Tigeot const i915_reg_t reg,
874c0e85e96SFrançois Tigeot const bool read,
875c0e85e96SFrançois Tigeot const bool before)
8769edbd4a0SFrançois Tigeot {
877*3f2dd94aSFrançois Tigeot if (likely(!i915_modparams.mmio_debug))
87824edb884SFrançois Tigeot return;
87924edb884SFrançois Tigeot
880c0e85e96SFrançois Tigeot __unclaimed_reg_debug(dev_priv, reg, read, before);
8819edbd4a0SFrançois Tigeot }
8829edbd4a0SFrançois Tigeot
8832c9916cdSFrançois Tigeot #define GEN2_READ_HEADER(x) \
8849edbd4a0SFrançois Tigeot u##x val = 0; \
885aee94f86SFrançois Tigeot assert_rpm_wakelock_held(dev_priv);
8869edbd4a0SFrançois Tigeot
8872c9916cdSFrançois Tigeot #define GEN2_READ_FOOTER \
8889edbd4a0SFrançois Tigeot trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
8899edbd4a0SFrançois Tigeot return val
8909edbd4a0SFrançois Tigeot
8912c9916cdSFrançois Tigeot #define __gen2_read(x) \
8929edbd4a0SFrançois Tigeot static u##x \
893aee94f86SFrançois Tigeot gen2_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
8942c9916cdSFrançois Tigeot GEN2_READ_HEADER(x); \
8959edbd4a0SFrançois Tigeot val = __raw_i915_read##x(dev_priv, reg); \
8962c9916cdSFrançois Tigeot GEN2_READ_FOOTER; \
8979edbd4a0SFrançois Tigeot }
8989edbd4a0SFrançois Tigeot
8999edbd4a0SFrançois Tigeot #define __gen5_read(x) \
9009edbd4a0SFrançois Tigeot static u##x \
901aee94f86SFrançois Tigeot gen5_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
9022c9916cdSFrançois Tigeot GEN2_READ_HEADER(x); \
9039edbd4a0SFrançois Tigeot ilk_dummy_write(dev_priv); \
9049edbd4a0SFrançois Tigeot val = __raw_i915_read##x(dev_priv, reg); \
9052c9916cdSFrançois Tigeot GEN2_READ_FOOTER; \
9062c9916cdSFrançois Tigeot }
9072c9916cdSFrançois Tigeot
9082c9916cdSFrançois Tigeot __gen5_read(8)
9092c9916cdSFrançois Tigeot __gen5_read(16)
9102c9916cdSFrançois Tigeot __gen5_read(32)
9112c9916cdSFrançois Tigeot __gen5_read(64)
9122c9916cdSFrançois Tigeot __gen2_read(8)
9132c9916cdSFrançois Tigeot __gen2_read(16)
9142c9916cdSFrançois Tigeot __gen2_read(32)
9152c9916cdSFrançois Tigeot __gen2_read(64)
9162c9916cdSFrançois Tigeot
9172c9916cdSFrançois Tigeot #undef __gen5_read
9182c9916cdSFrançois Tigeot #undef __gen2_read
9192c9916cdSFrançois Tigeot
9202c9916cdSFrançois Tigeot #undef GEN2_READ_FOOTER
9212c9916cdSFrançois Tigeot #undef GEN2_READ_HEADER
9222c9916cdSFrançois Tigeot
9232c9916cdSFrançois Tigeot #define GEN6_READ_HEADER(x) \
924aee94f86SFrançois Tigeot u32 offset = i915_mmio_reg_offset(reg); \
9255e269720SFrançois Tigeot unsigned long irqflags; \
9262c9916cdSFrançois Tigeot u##x val = 0; \
927aee94f86SFrançois Tigeot assert_rpm_wakelock_held(dev_priv); \
928c0e85e96SFrançois Tigeot spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
929c0e85e96SFrançois Tigeot unclaimed_reg_debug(dev_priv, reg, true, true)
9302c9916cdSFrançois Tigeot
9312c9916cdSFrançois Tigeot #define GEN6_READ_FOOTER \
932c0e85e96SFrançois Tigeot unclaimed_reg_debug(dev_priv, reg, true, false); \
9335e269720SFrançois Tigeot spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
9342c9916cdSFrançois Tigeot trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
9352c9916cdSFrançois Tigeot return val
9362c9916cdSFrançois Tigeot
___force_wake_auto(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)9371e12ee3bSFrançois Tigeot static noinline void ___force_wake_auto(struct drm_i915_private *dev_priv,
9382c9916cdSFrançois Tigeot enum forcewake_domains fw_domains)
9392c9916cdSFrançois Tigeot {
9402c9916cdSFrançois Tigeot struct intel_uncore_forcewake_domain *domain;
941a85cb24fSFrançois Tigeot unsigned int tmp;
9422c9916cdSFrançois Tigeot
943a85cb24fSFrançois Tigeot GEM_BUG_ON(fw_domains & ~dev_priv->uncore.fw_domains);
944a85cb24fSFrançois Tigeot
945a85cb24fSFrançois Tigeot for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp)
9461e12ee3bSFrançois Tigeot fw_domain_arm_timer(domain);
9471e12ee3bSFrançois Tigeot
9481e12ee3bSFrançois Tigeot dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
9491e12ee3bSFrançois Tigeot }
9501e12ee3bSFrançois Tigeot
__force_wake_auto(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)9511e12ee3bSFrançois Tigeot static inline void __force_wake_auto(struct drm_i915_private *dev_priv,
9521e12ee3bSFrançois Tigeot enum forcewake_domains fw_domains)
9531e12ee3bSFrançois Tigeot {
9542c9916cdSFrançois Tigeot if (WARN_ON(!fw_domains))
9552c9916cdSFrançois Tigeot return;
9562c9916cdSFrançois Tigeot
9571e12ee3bSFrançois Tigeot /* Turn on all requested but inactive supported forcewake domains. */
9581e12ee3bSFrançois Tigeot fw_domains &= dev_priv->uncore.fw_domains;
9591e12ee3bSFrançois Tigeot fw_domains &= ~dev_priv->uncore.fw_domains_active;
9602c9916cdSFrançois Tigeot
9612c9916cdSFrançois Tigeot if (fw_domains)
9621e12ee3bSFrançois Tigeot ___force_wake_auto(dev_priv, fw_domains);
9639edbd4a0SFrançois Tigeot }
9649edbd4a0SFrançois Tigeot
965a85cb24fSFrançois Tigeot #define __gen_read(func, x) \
9669edbd4a0SFrançois Tigeot static u##x \
967a85cb24fSFrançois Tigeot func##_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
9688621f407SFrançois Tigeot enum forcewake_domains fw_engine; \
9692c9916cdSFrançois Tigeot GEN6_READ_HEADER(x); \
970a85cb24fSFrançois Tigeot fw_engine = __##func##_reg_read_fw_domains(offset); \
9718621f407SFrançois Tigeot if (fw_engine) \
9728621f407SFrançois Tigeot __force_wake_auto(dev_priv, fw_engine); \
9739edbd4a0SFrançois Tigeot val = __raw_i915_read##x(dev_priv, reg); \
9742c9916cdSFrançois Tigeot GEN6_READ_FOOTER; \
9759edbd4a0SFrançois Tigeot }
976a85cb24fSFrançois Tigeot #define __gen6_read(x) __gen_read(gen6, x)
977a85cb24fSFrançois Tigeot #define __fwtable_read(x) __gen_read(fwtable, x)
9789edbd4a0SFrançois Tigeot
9791e12ee3bSFrançois Tigeot __fwtable_read(8)
9801e12ee3bSFrançois Tigeot __fwtable_read(16)
9811e12ee3bSFrançois Tigeot __fwtable_read(32)
9821e12ee3bSFrançois Tigeot __fwtable_read(64)
9839edbd4a0SFrançois Tigeot __gen6_read(8)
9849edbd4a0SFrançois Tigeot __gen6_read(16)
9859edbd4a0SFrançois Tigeot __gen6_read(32)
9869edbd4a0SFrançois Tigeot __gen6_read(64)
9879edbd4a0SFrançois Tigeot
9881e12ee3bSFrançois Tigeot #undef __fwtable_read
9899edbd4a0SFrançois Tigeot #undef __gen6_read
9902c9916cdSFrançois Tigeot #undef GEN6_READ_FOOTER
9912c9916cdSFrançois Tigeot #undef GEN6_READ_HEADER
9929edbd4a0SFrançois Tigeot
9932c9916cdSFrançois Tigeot #define GEN2_WRITE_HEADER \
9949edbd4a0SFrançois Tigeot trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
995aee94f86SFrançois Tigeot assert_rpm_wakelock_held(dev_priv); \
9969edbd4a0SFrançois Tigeot
9972c9916cdSFrançois Tigeot #define GEN2_WRITE_FOOTER
9989edbd4a0SFrançois Tigeot
9992c9916cdSFrançois Tigeot #define __gen2_write(x) \
10009edbd4a0SFrançois Tigeot static void \
1001aee94f86SFrançois Tigeot gen2_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
10022c9916cdSFrançois Tigeot GEN2_WRITE_HEADER; \
10039edbd4a0SFrançois Tigeot __raw_i915_write##x(dev_priv, reg, val); \
10042c9916cdSFrançois Tigeot GEN2_WRITE_FOOTER; \
10059edbd4a0SFrançois Tigeot }
10069edbd4a0SFrançois Tigeot
10079edbd4a0SFrançois Tigeot #define __gen5_write(x) \
10089edbd4a0SFrançois Tigeot static void \
1009aee94f86SFrançois Tigeot gen5_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
10102c9916cdSFrançois Tigeot GEN2_WRITE_HEADER; \
10119edbd4a0SFrançois Tigeot ilk_dummy_write(dev_priv); \
10129edbd4a0SFrançois Tigeot __raw_i915_write##x(dev_priv, reg, val); \
10132c9916cdSFrançois Tigeot GEN2_WRITE_FOOTER; \
10149edbd4a0SFrançois Tigeot }
10159edbd4a0SFrançois Tigeot
10162c9916cdSFrançois Tigeot __gen5_write(8)
10172c9916cdSFrançois Tigeot __gen5_write(16)
10182c9916cdSFrançois Tigeot __gen5_write(32)
10192c9916cdSFrançois Tigeot __gen2_write(8)
10202c9916cdSFrançois Tigeot __gen2_write(16)
10212c9916cdSFrançois Tigeot __gen2_write(32)
10222c9916cdSFrançois Tigeot
10232c9916cdSFrançois Tigeot #undef __gen5_write
10242c9916cdSFrançois Tigeot #undef __gen2_write
10252c9916cdSFrançois Tigeot
10262c9916cdSFrançois Tigeot #undef GEN2_WRITE_FOOTER
10272c9916cdSFrançois Tigeot #undef GEN2_WRITE_HEADER
10282c9916cdSFrançois Tigeot
10292c9916cdSFrançois Tigeot #define GEN6_WRITE_HEADER \
1030aee94f86SFrançois Tigeot u32 offset = i915_mmio_reg_offset(reg); \
10315e269720SFrançois Tigeot unsigned long irqflags; \
10322c9916cdSFrançois Tigeot trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1033aee94f86SFrançois Tigeot assert_rpm_wakelock_held(dev_priv); \
1034c0e85e96SFrançois Tigeot spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
1035c0e85e96SFrançois Tigeot unclaimed_reg_debug(dev_priv, reg, false, true)
10362c9916cdSFrançois Tigeot
10372c9916cdSFrançois Tigeot #define GEN6_WRITE_FOOTER \
1038c0e85e96SFrançois Tigeot unclaimed_reg_debug(dev_priv, reg, false, false); \
10395e269720SFrançois Tigeot spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)
10402c9916cdSFrançois Tigeot
10419edbd4a0SFrançois Tigeot #define __gen6_write(x) \
10429edbd4a0SFrançois Tigeot static void \
1043aee94f86SFrançois Tigeot gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
10442c9916cdSFrançois Tigeot GEN6_WRITE_HEADER; \
1045*3f2dd94aSFrançois Tigeot if (NEEDS_FORCE_WAKE(offset)) \
1046*3f2dd94aSFrançois Tigeot __gen6_gt_wait_for_fifo(dev_priv); \
10479edbd4a0SFrançois Tigeot __raw_i915_write##x(dev_priv, reg, val); \
10482c9916cdSFrançois Tigeot GEN6_WRITE_FOOTER; \
10499edbd4a0SFrançois Tigeot }
10509edbd4a0SFrançois Tigeot
1051a85cb24fSFrançois Tigeot #define __gen_write(func, x) \
10529edbd4a0SFrançois Tigeot static void \
1053a85cb24fSFrançois Tigeot func##_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
10548621f407SFrançois Tigeot enum forcewake_domains fw_engine; \
10552c9916cdSFrançois Tigeot GEN6_WRITE_HEADER; \
1056a85cb24fSFrançois Tigeot fw_engine = __##func##_reg_write_fw_domains(offset); \
10578621f407SFrançois Tigeot if (fw_engine) \
10588621f407SFrançois Tigeot __force_wake_auto(dev_priv, fw_engine); \
10599edbd4a0SFrançois Tigeot __raw_i915_write##x(dev_priv, reg, val); \
10602c9916cdSFrançois Tigeot GEN6_WRITE_FOOTER; \
10619edbd4a0SFrançois Tigeot }
1062a85cb24fSFrançois Tigeot #define __gen8_write(x) __gen_write(gen8, x)
1063a85cb24fSFrançois Tigeot #define __fwtable_write(x) __gen_write(fwtable, x)
106424edb884SFrançois Tigeot
10651e12ee3bSFrançois Tigeot __fwtable_write(8)
10661e12ee3bSFrançois Tigeot __fwtable_write(16)
10671e12ee3bSFrançois Tigeot __fwtable_write(32)
10689edbd4a0SFrançois Tigeot __gen8_write(8)
10699edbd4a0SFrançois Tigeot __gen8_write(16)
10709edbd4a0SFrançois Tigeot __gen8_write(32)
10719edbd4a0SFrançois Tigeot __gen6_write(8)
10729edbd4a0SFrançois Tigeot __gen6_write(16)
10739edbd4a0SFrançois Tigeot __gen6_write(32)
10749edbd4a0SFrançois Tigeot
10751e12ee3bSFrançois Tigeot #undef __fwtable_write
10769edbd4a0SFrançois Tigeot #undef __gen8_write
10779edbd4a0SFrançois Tigeot #undef __gen6_write
10782c9916cdSFrançois Tigeot #undef GEN6_WRITE_FOOTER
10792c9916cdSFrançois Tigeot #undef GEN6_WRITE_HEADER
10809edbd4a0SFrançois Tigeot
1081*3f2dd94aSFrançois Tigeot #define ASSIGN_WRITE_MMIO_VFUNCS(i915, x) \
10822c9916cdSFrançois Tigeot do { \
1083*3f2dd94aSFrançois Tigeot (i915)->uncore.funcs.mmio_writeb = x##_write8; \
1084*3f2dd94aSFrançois Tigeot (i915)->uncore.funcs.mmio_writew = x##_write16; \
1085*3f2dd94aSFrançois Tigeot (i915)->uncore.funcs.mmio_writel = x##_write32; \
10862c9916cdSFrançois Tigeot } while (0)
10872c9916cdSFrançois Tigeot
1088*3f2dd94aSFrançois Tigeot #define ASSIGN_READ_MMIO_VFUNCS(i915, x) \
10892c9916cdSFrançois Tigeot do { \
1090*3f2dd94aSFrançois Tigeot (i915)->uncore.funcs.mmio_readb = x##_read8; \
1091*3f2dd94aSFrançois Tigeot (i915)->uncore.funcs.mmio_readw = x##_read16; \
1092*3f2dd94aSFrançois Tigeot (i915)->uncore.funcs.mmio_readl = x##_read32; \
1093*3f2dd94aSFrançois Tigeot (i915)->uncore.funcs.mmio_readq = x##_read64; \
10942c9916cdSFrançois Tigeot } while (0)
10952c9916cdSFrançois Tigeot
10962c9916cdSFrançois Tigeot
fw_domain_init(struct drm_i915_private * dev_priv,enum forcewake_domain_id domain_id,i915_reg_t reg_set,i915_reg_t reg_ack)10972c9916cdSFrançois Tigeot static void fw_domain_init(struct drm_i915_private *dev_priv,
10982c9916cdSFrançois Tigeot enum forcewake_domain_id domain_id,
1099aee94f86SFrançois Tigeot i915_reg_t reg_set,
1100aee94f86SFrançois Tigeot i915_reg_t reg_ack)
11012c9916cdSFrançois Tigeot {
11022c9916cdSFrançois Tigeot struct intel_uncore_forcewake_domain *d;
11032c9916cdSFrançois Tigeot
11042c9916cdSFrançois Tigeot if (WARN_ON(domain_id >= FW_DOMAIN_ID_COUNT))
11052c9916cdSFrançois Tigeot return;
11062c9916cdSFrançois Tigeot
11072c9916cdSFrançois Tigeot d = &dev_priv->uncore.fw_domain[domain_id];
11082c9916cdSFrançois Tigeot
11092c9916cdSFrançois Tigeot WARN_ON(d->wake_count);
11102c9916cdSFrançois Tigeot
1111a85cb24fSFrançois Tigeot WARN_ON(!i915_mmio_reg_valid(reg_set));
1112a85cb24fSFrançois Tigeot WARN_ON(!i915_mmio_reg_valid(reg_ack));
1113a85cb24fSFrançois Tigeot
11142c9916cdSFrançois Tigeot d->wake_count = 0;
11152c9916cdSFrançois Tigeot d->reg_set = reg_set;
11162c9916cdSFrançois Tigeot d->reg_ack = reg_ack;
11172c9916cdSFrançois Tigeot
11182c9916cdSFrançois Tigeot d->id = domain_id;
11192c9916cdSFrançois Tigeot
11208621f407SFrançois Tigeot BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER));
11218621f407SFrançois Tigeot BUILD_BUG_ON(FORCEWAKE_BLITTER != (1 << FW_DOMAIN_ID_BLITTER));
11228621f407SFrançois Tigeot BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA));
11238621f407SFrançois Tigeot
1124a85cb24fSFrançois Tigeot d->mask = BIT(domain_id);
11258621f407SFrançois Tigeot
11268621f407SFrançois Tigeot hrtimer_init(&d->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
11278621f407SFrançois Tigeot d->timer.function = intel_uncore_fw_release_timer;
11282c9916cdSFrançois Tigeot
1129a85cb24fSFrançois Tigeot dev_priv->uncore.fw_domains |= BIT(domain_id);
11302c9916cdSFrançois Tigeot
1131a85cb24fSFrançois Tigeot fw_domain_reset(dev_priv, d);
11322c9916cdSFrançois Tigeot }
11332c9916cdSFrançois Tigeot
intel_uncore_fw_domains_init(struct drm_i915_private * dev_priv)11341487f786SFrançois Tigeot static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
11359edbd4a0SFrançois Tigeot {
1136a85cb24fSFrançois Tigeot if (INTEL_GEN(dev_priv) <= 5 || intel_vgpu_active(dev_priv))
11372c9916cdSFrançois Tigeot return;
1138ba55f2f5SFrançois Tigeot
1139a85cb24fSFrançois Tigeot if (IS_GEN6(dev_priv)) {
1140a85cb24fSFrançois Tigeot dev_priv->uncore.fw_reset = 0;
1141a85cb24fSFrançois Tigeot dev_priv->uncore.fw_set = FORCEWAKE_KERNEL;
1142a85cb24fSFrançois Tigeot dev_priv->uncore.fw_clear = 0;
1143a85cb24fSFrançois Tigeot } else {
1144a85cb24fSFrançois Tigeot /* WaRsClearFWBitsAtReset:bdw,skl */
1145a85cb24fSFrançois Tigeot dev_priv->uncore.fw_reset = _MASKED_BIT_DISABLE(0xffff);
1146a85cb24fSFrançois Tigeot dev_priv->uncore.fw_set = _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL);
1147a85cb24fSFrançois Tigeot dev_priv->uncore.fw_clear = _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL);
1148a85cb24fSFrançois Tigeot }
1149a85cb24fSFrançois Tigeot
1150*3f2dd94aSFrançois Tigeot if (INTEL_GEN(dev_priv) >= 9) {
11512c9916cdSFrançois Tigeot dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
11522c9916cdSFrançois Tigeot dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
11532c9916cdSFrançois Tigeot fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
11542c9916cdSFrançois Tigeot FORCEWAKE_RENDER_GEN9,
11552c9916cdSFrançois Tigeot FORCEWAKE_ACK_RENDER_GEN9);
11562c9916cdSFrançois Tigeot fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER,
11572c9916cdSFrançois Tigeot FORCEWAKE_BLITTER_GEN9,
11582c9916cdSFrançois Tigeot FORCEWAKE_ACK_BLITTER_GEN9);
11592c9916cdSFrançois Tigeot fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
11602c9916cdSFrançois Tigeot FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9);
11611487f786SFrançois Tigeot } else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
11622c9916cdSFrançois Tigeot dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
11632c9916cdSFrançois Tigeot dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
11642c9916cdSFrançois Tigeot fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
11652c9916cdSFrançois Tigeot FORCEWAKE_VLV, FORCEWAKE_ACK_VLV);
11662c9916cdSFrançois Tigeot fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
11672c9916cdSFrançois Tigeot FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV);
11681487f786SFrançois Tigeot } else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
11692c9916cdSFrançois Tigeot dev_priv->uncore.funcs.force_wake_get =
11702c9916cdSFrançois Tigeot fw_domains_get_with_thread_status;
11712c9916cdSFrançois Tigeot dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
11722c9916cdSFrançois Tigeot fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
11732c9916cdSFrançois Tigeot FORCEWAKE_MT, FORCEWAKE_ACK_HSW);
11741487f786SFrançois Tigeot } else if (IS_IVYBRIDGE(dev_priv)) {
11759edbd4a0SFrançois Tigeot u32 ecobus;
11769edbd4a0SFrançois Tigeot
11779edbd4a0SFrançois Tigeot /* IVB configs may use multi-threaded forcewake */
11789edbd4a0SFrançois Tigeot
11799edbd4a0SFrançois Tigeot /* A small trick here - if the bios hasn't configured
11809edbd4a0SFrançois Tigeot * MT forcewake, and if the device is in RC6, then
11819edbd4a0SFrançois Tigeot * force_wake_mt_get will not wake the device and the
11829edbd4a0SFrançois Tigeot * ECOBUS read will return zero. Which will be
11839edbd4a0SFrançois Tigeot * (correctly) interpreted by the test below as MT
11849edbd4a0SFrançois Tigeot * forcewake being disabled.
11859edbd4a0SFrançois Tigeot */
11862c9916cdSFrançois Tigeot dev_priv->uncore.funcs.force_wake_get =
11872c9916cdSFrançois Tigeot fw_domains_get_with_thread_status;
1188*3f2dd94aSFrançois Tigeot dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
11892c9916cdSFrançois Tigeot
11902c9916cdSFrançois Tigeot /* We need to init first for ECOBUS access and then
11912c9916cdSFrançois Tigeot * determine later if we want to reinit, in case of MT access is
11922c9916cdSFrançois Tigeot * not working. In this stage we don't know which flavour this
11932c9916cdSFrançois Tigeot * ivb is, so it is better to reset also the gen6 fw registers
11942c9916cdSFrançois Tigeot * before the ecobus check.
11952c9916cdSFrançois Tigeot */
11962c9916cdSFrançois Tigeot
11972c9916cdSFrançois Tigeot __raw_i915_write32(dev_priv, FORCEWAKE, 0);
11982c9916cdSFrançois Tigeot __raw_posting_read(dev_priv, ECOBUS);
11992c9916cdSFrançois Tigeot
12002c9916cdSFrançois Tigeot fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
12012c9916cdSFrançois Tigeot FORCEWAKE_MT, FORCEWAKE_MT_ACK);
12022c9916cdSFrançois Tigeot
1203303bf270SFrançois Tigeot spin_lock_irq(&dev_priv->uncore.lock);
1204a85cb24fSFrançois Tigeot fw_domains_get_with_thread_status(dev_priv, FORCEWAKE_RENDER);
12059edbd4a0SFrançois Tigeot ecobus = __raw_i915_read32(dev_priv, ECOBUS);
1206*3f2dd94aSFrançois Tigeot fw_domains_put(dev_priv, FORCEWAKE_RENDER);
1207303bf270SFrançois Tigeot spin_unlock_irq(&dev_priv->uncore.lock);
12089edbd4a0SFrançois Tigeot
12092c9916cdSFrançois Tigeot if (!(ecobus & FORCEWAKE_MT_ENABLE)) {
12109edbd4a0SFrançois Tigeot DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
12119edbd4a0SFrançois Tigeot DRM_INFO("when using vblank-synced partial screen updates.\n");
12122c9916cdSFrançois Tigeot fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
12132c9916cdSFrançois Tigeot FORCEWAKE, FORCEWAKE_ACK);
12149edbd4a0SFrançois Tigeot }
12151487f786SFrançois Tigeot } else if (IS_GEN6(dev_priv)) {
12169edbd4a0SFrançois Tigeot dev_priv->uncore.funcs.force_wake_get =
12172c9916cdSFrançois Tigeot fw_domains_get_with_thread_status;
1218*3f2dd94aSFrançois Tigeot dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
12192c9916cdSFrançois Tigeot fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
12202c9916cdSFrançois Tigeot FORCEWAKE, FORCEWAKE_ACK);
12219edbd4a0SFrançois Tigeot }
12229edbd4a0SFrançois Tigeot
12232c9916cdSFrançois Tigeot /* All future platforms are expected to require complex power gating */
12242c9916cdSFrançois Tigeot WARN_ON(dev_priv->uncore.fw_domains == 0);
12252c9916cdSFrançois Tigeot }
12262c9916cdSFrançois Tigeot
12271e12ee3bSFrançois Tigeot #define ASSIGN_FW_DOMAINS_TABLE(d) \
12281e12ee3bSFrançois Tigeot { \
12291e12ee3bSFrançois Tigeot dev_priv->uncore.fw_domains_table = \
12301e12ee3bSFrançois Tigeot (struct intel_forcewake_range *)(d); \
12311e12ee3bSFrançois Tigeot dev_priv->uncore.fw_domains_table_entries = ARRAY_SIZE((d)); \
12321e12ee3bSFrançois Tigeot }
12331e12ee3bSFrançois Tigeot
i915_pmic_bus_access_notifier(struct notifier_block * nb,unsigned long action,void * data)1234a85cb24fSFrançois Tigeot static int i915_pmic_bus_access_notifier(struct notifier_block *nb,
1235a85cb24fSFrançois Tigeot unsigned long action, void *data)
1236a85cb24fSFrançois Tigeot {
1237a85cb24fSFrançois Tigeot struct drm_i915_private *dev_priv = container_of(nb,
1238a85cb24fSFrançois Tigeot struct drm_i915_private, uncore.pmic_bus_access_nb);
1239a85cb24fSFrançois Tigeot
1240a85cb24fSFrançois Tigeot switch (action) {
1241a85cb24fSFrançois Tigeot case MBI_PMIC_BUS_ACCESS_BEGIN:
1242a85cb24fSFrançois Tigeot /*
1243a85cb24fSFrançois Tigeot * forcewake all now to make sure that we don't need to do a
1244a85cb24fSFrançois Tigeot * forcewake later which on systems where this notifier gets
1245a85cb24fSFrançois Tigeot * called requires the punit to access to the shared pmic i2c
1246a85cb24fSFrançois Tigeot * bus, which will be busy after this notification, leading to:
1247a85cb24fSFrançois Tigeot * "render: timed out waiting for forcewake ack request."
1248a85cb24fSFrançois Tigeot * errors.
1249*3f2dd94aSFrançois Tigeot *
1250*3f2dd94aSFrançois Tigeot * The notifier is unregistered during intel_runtime_suspend(),
1251*3f2dd94aSFrançois Tigeot * so it's ok to access the HW here without holding a RPM
1252*3f2dd94aSFrançois Tigeot * wake reference -> disable wakeref asserts for the time of
1253*3f2dd94aSFrançois Tigeot * the access.
1254a85cb24fSFrançois Tigeot */
1255*3f2dd94aSFrançois Tigeot disable_rpm_wakeref_asserts(dev_priv);
1256a85cb24fSFrançois Tigeot intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
1257*3f2dd94aSFrançois Tigeot enable_rpm_wakeref_asserts(dev_priv);
1258a85cb24fSFrançois Tigeot break;
1259a85cb24fSFrançois Tigeot case MBI_PMIC_BUS_ACCESS_END:
1260a85cb24fSFrançois Tigeot intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
1261a85cb24fSFrançois Tigeot break;
1262a85cb24fSFrançois Tigeot }
1263a85cb24fSFrançois Tigeot
1264a85cb24fSFrançois Tigeot return NOTIFY_OK;
1265a85cb24fSFrançois Tigeot }
1266a85cb24fSFrançois Tigeot
intel_uncore_init(struct drm_i915_private * dev_priv)12671487f786SFrançois Tigeot void intel_uncore_init(struct drm_i915_private *dev_priv)
12682c9916cdSFrançois Tigeot {
12691487f786SFrançois Tigeot i915_check_vgpu(dev_priv);
1270477eb7f9SFrançois Tigeot
12718621f407SFrançois Tigeot intel_uncore_edram_detect(dev_priv);
12721487f786SFrançois Tigeot intel_uncore_fw_domains_init(dev_priv);
12731487f786SFrançois Tigeot __intel_uncore_early_sanitize(dev_priv, false);
12742c9916cdSFrançois Tigeot
1275c0e85e96SFrançois Tigeot dev_priv->uncore.unclaimed_mmio_check = 1;
1276a85cb24fSFrançois Tigeot dev_priv->uncore.pmic_bus_access_nb.notifier_call =
1277a85cb24fSFrançois Tigeot i915_pmic_bus_access_notifier;
1278c0e85e96SFrançois Tigeot
1279a85cb24fSFrançois Tigeot if (IS_GEN(dev_priv, 2, 4) || intel_vgpu_active(dev_priv)) {
1280*3f2dd94aSFrançois Tigeot ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen2);
1281*3f2dd94aSFrançois Tigeot ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen2);
1282a85cb24fSFrançois Tigeot } else if (IS_GEN5(dev_priv)) {
1283*3f2dd94aSFrançois Tigeot ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen5);
1284*3f2dd94aSFrançois Tigeot ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen5);
1285a85cb24fSFrançois Tigeot } else if (IS_GEN(dev_priv, 6, 7)) {
1286*3f2dd94aSFrançois Tigeot ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen6);
1287a85cb24fSFrançois Tigeot
1288a85cb24fSFrançois Tigeot if (IS_VALLEYVIEW(dev_priv)) {
1289a85cb24fSFrançois Tigeot ASSIGN_FW_DOMAINS_TABLE(__vlv_fw_ranges);
1290*3f2dd94aSFrançois Tigeot ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
1291a85cb24fSFrançois Tigeot } else {
1292*3f2dd94aSFrançois Tigeot ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6);
1293a85cb24fSFrançois Tigeot }
1294a85cb24fSFrançois Tigeot } else if (IS_GEN8(dev_priv)) {
1295a85cb24fSFrançois Tigeot if (IS_CHERRYVIEW(dev_priv)) {
1296a85cb24fSFrançois Tigeot ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges);
1297*3f2dd94aSFrançois Tigeot ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable);
1298*3f2dd94aSFrançois Tigeot ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
1299a85cb24fSFrançois Tigeot
1300a85cb24fSFrançois Tigeot } else {
1301*3f2dd94aSFrançois Tigeot ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen8);
1302*3f2dd94aSFrançois Tigeot ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6);
1303a85cb24fSFrançois Tigeot }
1304a85cb24fSFrançois Tigeot } else {
13051e12ee3bSFrançois Tigeot ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges);
1306*3f2dd94aSFrançois Tigeot ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable);
1307*3f2dd94aSFrançois Tigeot ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
13089edbd4a0SFrançois Tigeot }
13092c9916cdSFrançois Tigeot
1310a85cb24fSFrançois Tigeot iosf_mbi_register_pmic_bus_access_notifier(
1311a85cb24fSFrançois Tigeot &dev_priv->uncore.pmic_bus_access_nb);
1312477eb7f9SFrançois Tigeot
13131487f786SFrançois Tigeot i915_check_and_clear_faults(dev_priv);
13149edbd4a0SFrançois Tigeot }
13159edbd4a0SFrançois Tigeot
intel_uncore_fini(struct drm_i915_private * dev_priv)13161487f786SFrançois Tigeot void intel_uncore_fini(struct drm_i915_private *dev_priv)
13179edbd4a0SFrançois Tigeot {
1318a85cb24fSFrançois Tigeot iosf_mbi_unregister_pmic_bus_access_notifier(
1319a85cb24fSFrançois Tigeot &dev_priv->uncore.pmic_bus_access_nb);
1320a85cb24fSFrançois Tigeot
13219edbd4a0SFrançois Tigeot /* Paranoia: make sure we have disabled everything before we exit. */
13221487f786SFrançois Tigeot intel_uncore_sanitize(dev_priv);
13231487f786SFrançois Tigeot intel_uncore_forcewake_reset(dev_priv, false);
13249edbd4a0SFrançois Tigeot }
13259edbd4a0SFrançois Tigeot
1326*3f2dd94aSFrançois Tigeot static const struct reg_whitelist {
1327*3f2dd94aSFrançois Tigeot i915_reg_t offset_ldw;
1328*3f2dd94aSFrançois Tigeot i915_reg_t offset_udw;
1329*3f2dd94aSFrançois Tigeot u16 gen_mask;
1330*3f2dd94aSFrançois Tigeot u8 size;
1331*3f2dd94aSFrançois Tigeot } reg_read_whitelist[] = { {
1332*3f2dd94aSFrançois Tigeot .offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE),
1333aee94f86SFrançois Tigeot .offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE),
1334*3f2dd94aSFrançois Tigeot .gen_mask = INTEL_GEN_MASK(4, 10),
1335*3f2dd94aSFrançois Tigeot .size = 8
1336*3f2dd94aSFrançois Tigeot } };
13379edbd4a0SFrançois Tigeot
i915_reg_read_ioctl(struct drm_device * dev,void * data,struct drm_file * file)13389edbd4a0SFrançois Tigeot int i915_reg_read_ioctl(struct drm_device *dev,
13399edbd4a0SFrançois Tigeot void *data, struct drm_file *file)
13409edbd4a0SFrançois Tigeot {
1341bf017597SFrançois Tigeot struct drm_i915_private *dev_priv = to_i915(dev);
13429edbd4a0SFrançois Tigeot struct drm_i915_reg_read *reg = data;
1343*3f2dd94aSFrançois Tigeot struct reg_whitelist const *entry;
1344*3f2dd94aSFrançois Tigeot unsigned int flags;
1345*3f2dd94aSFrançois Tigeot int remain;
1346*3f2dd94aSFrançois Tigeot int ret = 0;
13479edbd4a0SFrançois Tigeot
1348*3f2dd94aSFrançois Tigeot entry = reg_read_whitelist;
1349*3f2dd94aSFrançois Tigeot remain = ARRAY_SIZE(reg_read_whitelist);
1350*3f2dd94aSFrançois Tigeot while (remain) {
1351*3f2dd94aSFrançois Tigeot u32 entry_offset = i915_mmio_reg_offset(entry->offset_ldw);
1352*3f2dd94aSFrançois Tigeot
1353*3f2dd94aSFrançois Tigeot GEM_BUG_ON(!is_power_of_2(entry->size));
1354*3f2dd94aSFrançois Tigeot GEM_BUG_ON(entry->size > 8);
1355*3f2dd94aSFrançois Tigeot GEM_BUG_ON(entry_offset & (entry->size - 1));
1356*3f2dd94aSFrançois Tigeot
1357*3f2dd94aSFrançois Tigeot if (INTEL_INFO(dev_priv)->gen_mask & entry->gen_mask &&
1358*3f2dd94aSFrançois Tigeot entry_offset == (reg->offset & -entry->size))
13599edbd4a0SFrançois Tigeot break;
1360*3f2dd94aSFrançois Tigeot entry++;
1361*3f2dd94aSFrançois Tigeot remain--;
13629edbd4a0SFrançois Tigeot }
13639edbd4a0SFrançois Tigeot
1364*3f2dd94aSFrançois Tigeot if (!remain)
13659edbd4a0SFrançois Tigeot return -EINVAL;
13669edbd4a0SFrançois Tigeot
1367*3f2dd94aSFrançois Tigeot flags = reg->offset & (entry->size - 1);
136819c468b4SFrançois Tigeot
1369ba55f2f5SFrançois Tigeot intel_runtime_pm_get(dev_priv);
1370*3f2dd94aSFrançois Tigeot if (entry->size == 8 && flags == I915_REG_READ_8B_WA)
1371*3f2dd94aSFrançois Tigeot reg->val = I915_READ64_2x32(entry->offset_ldw,
1372*3f2dd94aSFrançois Tigeot entry->offset_udw);
1373*3f2dd94aSFrançois Tigeot else if (entry->size == 8 && flags == 0)
1374*3f2dd94aSFrançois Tigeot reg->val = I915_READ64(entry->offset_ldw);
1375*3f2dd94aSFrançois Tigeot else if (entry->size == 4 && flags == 0)
1376*3f2dd94aSFrançois Tigeot reg->val = I915_READ(entry->offset_ldw);
1377*3f2dd94aSFrançois Tigeot else if (entry->size == 2 && flags == 0)
1378*3f2dd94aSFrançois Tigeot reg->val = I915_READ16(entry->offset_ldw);
1379*3f2dd94aSFrançois Tigeot else if (entry->size == 1 && flags == 0)
1380*3f2dd94aSFrançois Tigeot reg->val = I915_READ8(entry->offset_ldw);
1381*3f2dd94aSFrançois Tigeot else
1382ba55f2f5SFrançois Tigeot ret = -EINVAL;
1383ba55f2f5SFrançois Tigeot intel_runtime_pm_put(dev_priv);
1384*3f2dd94aSFrançois Tigeot
1385ba55f2f5SFrançois Tigeot return ret;
13869edbd4a0SFrançois Tigeot }
13879edbd4a0SFrançois Tigeot
gen3_stop_engine(struct intel_engine_cs * engine)1388*3f2dd94aSFrançois Tigeot static void gen3_stop_engine(struct intel_engine_cs *engine)
1389*3f2dd94aSFrançois Tigeot {
1390*3f2dd94aSFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
1391*3f2dd94aSFrançois Tigeot const u32 base = engine->mmio_base;
1392*3f2dd94aSFrançois Tigeot const i915_reg_t mode = RING_MI_MODE(base);
1393*3f2dd94aSFrançois Tigeot
1394*3f2dd94aSFrançois Tigeot I915_WRITE_FW(mode, _MASKED_BIT_ENABLE(STOP_RING));
1395*3f2dd94aSFrançois Tigeot if (intel_wait_for_register_fw(dev_priv,
1396*3f2dd94aSFrançois Tigeot mode,
1397*3f2dd94aSFrançois Tigeot MODE_IDLE,
1398*3f2dd94aSFrançois Tigeot MODE_IDLE,
1399*3f2dd94aSFrançois Tigeot 500))
1400*3f2dd94aSFrançois Tigeot DRM_DEBUG_DRIVER("%s: timed out on STOP_RING\n",
1401*3f2dd94aSFrançois Tigeot engine->name);
1402*3f2dd94aSFrançois Tigeot
1403*3f2dd94aSFrançois Tigeot I915_WRITE_FW(RING_CTL(base), 0);
1404*3f2dd94aSFrançois Tigeot I915_WRITE_FW(RING_HEAD(base), 0);
1405*3f2dd94aSFrançois Tigeot I915_WRITE_FW(RING_TAIL(base), 0);
1406*3f2dd94aSFrançois Tigeot
1407*3f2dd94aSFrançois Tigeot /* Check acts as a post */
1408*3f2dd94aSFrançois Tigeot if (I915_READ_FW(RING_HEAD(base)) != 0)
1409*3f2dd94aSFrançois Tigeot DRM_DEBUG_DRIVER("%s: ring head not parked\n",
1410*3f2dd94aSFrançois Tigeot engine->name);
1411*3f2dd94aSFrançois Tigeot }
1412*3f2dd94aSFrançois Tigeot
i915_stop_engines(struct drm_i915_private * dev_priv,unsigned engine_mask)1413*3f2dd94aSFrançois Tigeot static void i915_stop_engines(struct drm_i915_private *dev_priv,
1414*3f2dd94aSFrançois Tigeot unsigned engine_mask)
1415*3f2dd94aSFrançois Tigeot {
1416*3f2dd94aSFrançois Tigeot struct intel_engine_cs *engine;
1417*3f2dd94aSFrançois Tigeot enum intel_engine_id id;
1418*3f2dd94aSFrançois Tigeot
1419*3f2dd94aSFrançois Tigeot if (INTEL_GEN(dev_priv) < 3)
1420*3f2dd94aSFrançois Tigeot return;
1421*3f2dd94aSFrançois Tigeot
1422*3f2dd94aSFrançois Tigeot for_each_engine_masked(engine, dev_priv, engine_mask, id)
1423*3f2dd94aSFrançois Tigeot gen3_stop_engine(engine);
1424*3f2dd94aSFrançois Tigeot }
1425*3f2dd94aSFrançois Tigeot
i915_reset_complete(struct pci_dev * pdev)1426*3f2dd94aSFrançois Tigeot static bool i915_reset_complete(struct pci_dev *pdev)
14279edbd4a0SFrançois Tigeot {
14289edbd4a0SFrançois Tigeot u8 gdrst;
1429*3f2dd94aSFrançois Tigeot
14301487f786SFrançois Tigeot pci_read_config_byte(pdev, I915_GDRST, &gdrst);
14312c9916cdSFrançois Tigeot return (gdrst & GRDOM_RESET_STATUS) == 0;
14322c9916cdSFrançois Tigeot }
14332c9916cdSFrançois Tigeot
i915_do_reset(struct drm_i915_private * dev_priv,unsigned engine_mask)14341487f786SFrançois Tigeot static int i915_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
14352c9916cdSFrançois Tigeot {
1436303bf270SFrançois Tigeot struct pci_dev *pdev = dev_priv->drm.pdev;
14372c9916cdSFrançois Tigeot
14381487f786SFrançois Tigeot /* assert reset for at least 20 usec */
14391487f786SFrançois Tigeot pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
1440*3f2dd94aSFrançois Tigeot usleep_range(50, 200);
14411487f786SFrançois Tigeot pci_write_config_byte(pdev, I915_GDRST, 0);
14421487f786SFrançois Tigeot
14431487f786SFrançois Tigeot return wait_for(i915_reset_complete(pdev), 500);
14442c9916cdSFrançois Tigeot }
14452c9916cdSFrançois Tigeot
g4x_reset_complete(struct pci_dev * pdev)1446*3f2dd94aSFrançois Tigeot static bool g4x_reset_complete(struct pci_dev *pdev)
14472c9916cdSFrançois Tigeot {
14482c9916cdSFrançois Tigeot u8 gdrst;
1449*3f2dd94aSFrançois Tigeot
14501487f786SFrançois Tigeot pci_read_config_byte(pdev, I915_GDRST, &gdrst);
14519edbd4a0SFrançois Tigeot return (gdrst & GRDOM_RESET_ENABLE) == 0;
14529edbd4a0SFrançois Tigeot }
14539edbd4a0SFrançois Tigeot
g33_do_reset(struct drm_i915_private * dev_priv,unsigned engine_mask)14541487f786SFrançois Tigeot static int g33_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
14559edbd4a0SFrançois Tigeot {
1456303bf270SFrançois Tigeot struct pci_dev *pdev = dev_priv->drm.pdev;
1457*3f2dd94aSFrançois Tigeot
14581487f786SFrançois Tigeot pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
14591487f786SFrançois Tigeot return wait_for(g4x_reset_complete(pdev), 500);
14609edbd4a0SFrançois Tigeot }
14619edbd4a0SFrançois Tigeot
g4x_do_reset(struct drm_i915_private * dev_priv,unsigned engine_mask)14621487f786SFrançois Tigeot static int g4x_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
14639edbd4a0SFrançois Tigeot {
1464303bf270SFrançois Tigeot struct pci_dev *pdev = dev_priv->drm.pdev;
14659edbd4a0SFrançois Tigeot int ret;
14669edbd4a0SFrançois Tigeot
1467ba55f2f5SFrançois Tigeot /* WaVcpClkGateDisableForMediaReset:ctg,elk */
1468*3f2dd94aSFrançois Tigeot I915_WRITE(VDECCLK_GATE_D,
1469*3f2dd94aSFrançois Tigeot I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE);
1470ba55f2f5SFrançois Tigeot POSTING_READ(VDECCLK_GATE_D);
1471ba55f2f5SFrançois Tigeot
14721487f786SFrançois Tigeot pci_write_config_byte(pdev, I915_GDRST,
1473ba55f2f5SFrançois Tigeot GRDOM_MEDIA | GRDOM_RESET_ENABLE);
14741487f786SFrançois Tigeot ret = wait_for(g4x_reset_complete(pdev), 500);
1475*3f2dd94aSFrançois Tigeot if (ret) {
1476*3f2dd94aSFrançois Tigeot DRM_DEBUG_DRIVER("Wait for media reset failed\n");
1477*3f2dd94aSFrançois Tigeot goto out;
1478*3f2dd94aSFrançois Tigeot }
1479ba55f2f5SFrançois Tigeot
1480*3f2dd94aSFrançois Tigeot pci_write_config_byte(pdev, I915_GDRST,
1481*3f2dd94aSFrançois Tigeot GRDOM_RENDER | GRDOM_RESET_ENABLE);
1482*3f2dd94aSFrançois Tigeot ret = wait_for(g4x_reset_complete(pdev), 500);
1483*3f2dd94aSFrançois Tigeot if (ret) {
1484*3f2dd94aSFrançois Tigeot DRM_DEBUG_DRIVER("Wait for render reset failed\n");
1485*3f2dd94aSFrançois Tigeot goto out;
1486*3f2dd94aSFrançois Tigeot }
1487ba55f2f5SFrançois Tigeot
1488*3f2dd94aSFrançois Tigeot out:
14891487f786SFrançois Tigeot pci_write_config_byte(pdev, I915_GDRST, 0);
1490ba55f2f5SFrançois Tigeot
1491*3f2dd94aSFrançois Tigeot I915_WRITE(VDECCLK_GATE_D,
1492*3f2dd94aSFrançois Tigeot I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE);
1493*3f2dd94aSFrançois Tigeot POSTING_READ(VDECCLK_GATE_D);
1494*3f2dd94aSFrançois Tigeot
1495*3f2dd94aSFrançois Tigeot return ret;
1496ba55f2f5SFrançois Tigeot }
1497ba55f2f5SFrançois Tigeot
ironlake_do_reset(struct drm_i915_private * dev_priv,unsigned engine_mask)14981487f786SFrançois Tigeot static int ironlake_do_reset(struct drm_i915_private *dev_priv,
14991487f786SFrançois Tigeot unsigned engine_mask)
1500ba55f2f5SFrançois Tigeot {
1501ba55f2f5SFrançois Tigeot int ret;
1502ba55f2f5SFrançois Tigeot
1503*3f2dd94aSFrançois Tigeot I915_WRITE(ILK_GDSR, ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
15041487f786SFrançois Tigeot ret = intel_wait_for_register(dev_priv,
15051487f786SFrançois Tigeot ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0,
15061487f786SFrançois Tigeot 500);
1507*3f2dd94aSFrançois Tigeot if (ret) {
1508*3f2dd94aSFrançois Tigeot DRM_DEBUG_DRIVER("Wait for render reset failed\n");
1509*3f2dd94aSFrançois Tigeot goto out;
1510*3f2dd94aSFrançois Tigeot }
1511ba55f2f5SFrançois Tigeot
1512*3f2dd94aSFrançois Tigeot I915_WRITE(ILK_GDSR, ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
15131487f786SFrançois Tigeot ret = intel_wait_for_register(dev_priv,
15141487f786SFrançois Tigeot ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0,
15151487f786SFrançois Tigeot 500);
1516*3f2dd94aSFrançois Tigeot if (ret) {
1517*3f2dd94aSFrançois Tigeot DRM_DEBUG_DRIVER("Wait for media reset failed\n");
1518*3f2dd94aSFrançois Tigeot goto out;
1519*3f2dd94aSFrançois Tigeot }
1520ba55f2f5SFrançois Tigeot
1521*3f2dd94aSFrançois Tigeot out:
1522352ff8bdSFrançois Tigeot I915_WRITE(ILK_GDSR, 0);
1523*3f2dd94aSFrançois Tigeot POSTING_READ(ILK_GDSR);
1524*3f2dd94aSFrançois Tigeot return ret;
15259edbd4a0SFrançois Tigeot }
15269edbd4a0SFrançois Tigeot
15278621f407SFrançois Tigeot /* Reset the hardware domains (GENX_GRDOM_*) specified by mask */
gen6_hw_domain_reset(struct drm_i915_private * dev_priv,u32 hw_domain_mask)15288621f407SFrançois Tigeot static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv,
15298621f407SFrançois Tigeot u32 hw_domain_mask)
15309edbd4a0SFrançois Tigeot {
1531*3f2dd94aSFrançois Tigeot int err;
1532*3f2dd94aSFrançois Tigeot
15339edbd4a0SFrançois Tigeot /* GEN6_GDRST is not in the gt power well, no need to check
15349edbd4a0SFrançois Tigeot * for fifo space for the write or forcewake the chip for
15359edbd4a0SFrançois Tigeot * the read
15369edbd4a0SFrançois Tigeot */
15378621f407SFrançois Tigeot __raw_i915_write32(dev_priv, GEN6_GDRST, hw_domain_mask);
15389edbd4a0SFrançois Tigeot
1539*3f2dd94aSFrançois Tigeot /* Wait for the device to ack the reset requests */
1540*3f2dd94aSFrançois Tigeot err = intel_wait_for_register_fw(dev_priv,
15411487f786SFrançois Tigeot GEN6_GDRST, hw_domain_mask, 0,
15421487f786SFrançois Tigeot 500);
1543*3f2dd94aSFrançois Tigeot if (err)
1544*3f2dd94aSFrançois Tigeot DRM_DEBUG_DRIVER("Wait for 0x%08x engines reset failed\n",
1545*3f2dd94aSFrançois Tigeot hw_domain_mask);
1546*3f2dd94aSFrançois Tigeot
1547*3f2dd94aSFrançois Tigeot return err;
15488621f407SFrançois Tigeot }
15498621f407SFrançois Tigeot
15508621f407SFrançois Tigeot /**
15518621f407SFrançois Tigeot * gen6_reset_engines - reset individual engines
15521487f786SFrançois Tigeot * @dev_priv: i915 device
15538621f407SFrançois Tigeot * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset
15548621f407SFrançois Tigeot *
15558621f407SFrançois Tigeot * This function will reset the individual engines that are set in engine_mask.
15568621f407SFrançois Tigeot * If you provide ALL_ENGINES as mask, full global domain reset will be issued.
15578621f407SFrançois Tigeot *
15588621f407SFrançois Tigeot * Note: It is responsibility of the caller to handle the difference between
15598621f407SFrançois Tigeot * asking full domain reset versus reset for all available individual engines.
15608621f407SFrançois Tigeot *
15618621f407SFrançois Tigeot * Returns 0 on success, nonzero on error.
15628621f407SFrançois Tigeot */
gen6_reset_engines(struct drm_i915_private * dev_priv,unsigned engine_mask)15631487f786SFrançois Tigeot static int gen6_reset_engines(struct drm_i915_private *dev_priv,
15641487f786SFrançois Tigeot unsigned engine_mask)
15658621f407SFrançois Tigeot {
15668621f407SFrançois Tigeot struct intel_engine_cs *engine;
15678621f407SFrançois Tigeot const u32 hw_engine_mask[I915_NUM_ENGINES] = {
15688621f407SFrançois Tigeot [RCS] = GEN6_GRDOM_RENDER,
15698621f407SFrançois Tigeot [BCS] = GEN6_GRDOM_BLT,
15708621f407SFrançois Tigeot [VCS] = GEN6_GRDOM_MEDIA,
15718621f407SFrançois Tigeot [VCS2] = GEN8_GRDOM_MEDIA2,
15728621f407SFrançois Tigeot [VECS] = GEN6_GRDOM_VECS,
15738621f407SFrançois Tigeot };
15748621f407SFrançois Tigeot u32 hw_mask;
15758621f407SFrançois Tigeot
15768621f407SFrançois Tigeot if (engine_mask == ALL_ENGINES) {
15778621f407SFrançois Tigeot hw_mask = GEN6_GRDOM_FULL;
15788621f407SFrançois Tigeot } else {
15791e12ee3bSFrançois Tigeot unsigned int tmp;
15801e12ee3bSFrançois Tigeot
15818621f407SFrançois Tigeot hw_mask = 0;
15821e12ee3bSFrançois Tigeot for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
15838621f407SFrançois Tigeot hw_mask |= hw_engine_mask[engine->id];
15848621f407SFrançois Tigeot }
15858621f407SFrançois Tigeot
1586*3f2dd94aSFrançois Tigeot return gen6_hw_domain_reset(dev_priv, hw_mask);
15879edbd4a0SFrançois Tigeot }
15889edbd4a0SFrançois Tigeot
15891487f786SFrançois Tigeot /**
1590*3f2dd94aSFrançois Tigeot * __intel_wait_for_register_fw - wait until register matches expected state
15911487f786SFrançois Tigeot * @dev_priv: the i915 device
15921487f786SFrançois Tigeot * @reg: the register to read
15931487f786SFrançois Tigeot * @mask: mask to apply to register value
15941487f786SFrançois Tigeot * @value: expected value
1595*3f2dd94aSFrançois Tigeot * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
1596*3f2dd94aSFrançois Tigeot * @slow_timeout_ms: slow timeout in millisecond
1597*3f2dd94aSFrançois Tigeot * @out_value: optional placeholder to hold registry value
15981487f786SFrançois Tigeot *
15991487f786SFrançois Tigeot * This routine waits until the target register @reg contains the expected
160087df8fc6SFrançois Tigeot * @value after applying the @mask, i.e. it waits until ::
160187df8fc6SFrançois Tigeot *
160287df8fc6SFrançois Tigeot * (I915_READ_FW(reg) & mask) == value
160387df8fc6SFrançois Tigeot *
1604*3f2dd94aSFrançois Tigeot * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.
1605*3f2dd94aSFrançois Tigeot * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us
1606*3f2dd94aSFrançois Tigeot * must be not larger than 20,0000 microseconds.
16071487f786SFrançois Tigeot *
16081487f786SFrançois Tigeot * Note that this routine assumes the caller holds forcewake asserted, it is
16091487f786SFrançois Tigeot * not suitable for very long waits. See intel_wait_for_register() if you
16101487f786SFrançois Tigeot * wish to wait without holding forcewake for the duration (i.e. you expect
16111487f786SFrançois Tigeot * the wait to be slow).
16121487f786SFrançois Tigeot *
16131487f786SFrançois Tigeot * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
16141487f786SFrançois Tigeot */
__intel_wait_for_register_fw(struct drm_i915_private * dev_priv,i915_reg_t reg,u32 mask,u32 value,unsigned int fast_timeout_us,unsigned int slow_timeout_ms,u32 * out_value)1615*3f2dd94aSFrançois Tigeot int __intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
1616aee94f86SFrançois Tigeot i915_reg_t reg,
1617*3f2dd94aSFrançois Tigeot u32 mask,
1618*3f2dd94aSFrançois Tigeot u32 value,
1619*3f2dd94aSFrançois Tigeot unsigned int fast_timeout_us,
1620*3f2dd94aSFrançois Tigeot unsigned int slow_timeout_ms,
1621*3f2dd94aSFrançois Tigeot u32 *out_value)
1622a05eeebfSFrançois Tigeot {
1623*3f2dd94aSFrançois Tigeot u32 reg_value = 0;
1624*3f2dd94aSFrançois Tigeot #define done (((reg_value = I915_READ_FW(reg)) & mask) == value)
1625*3f2dd94aSFrançois Tigeot int ret;
1626*3f2dd94aSFrançois Tigeot
1627*3f2dd94aSFrançois Tigeot /* Catch any overuse of this function */
1628*3f2dd94aSFrançois Tigeot might_sleep_if(slow_timeout_ms);
1629*3f2dd94aSFrançois Tigeot GEM_BUG_ON(fast_timeout_us > 20000);
1630*3f2dd94aSFrançois Tigeot
1631*3f2dd94aSFrançois Tigeot ret = -ETIMEDOUT;
1632*3f2dd94aSFrançois Tigeot if (fast_timeout_us && fast_timeout_us <= 20000)
1633*3f2dd94aSFrançois Tigeot ret = _wait_for_atomic(done, fast_timeout_us, 0);
1634*3f2dd94aSFrançois Tigeot if (ret && slow_timeout_ms)
1635*3f2dd94aSFrançois Tigeot ret = wait_for(done, slow_timeout_ms);
1636*3f2dd94aSFrançois Tigeot
1637*3f2dd94aSFrançois Tigeot if (out_value)
1638*3f2dd94aSFrançois Tigeot *out_value = reg_value;
1639*3f2dd94aSFrançois Tigeot
16401487f786SFrançois Tigeot return ret;
16411487f786SFrançois Tigeot #undef done
16421487f786SFrançois Tigeot }
16431487f786SFrançois Tigeot
16441487f786SFrançois Tigeot /**
16451487f786SFrançois Tigeot * intel_wait_for_register - wait until register matches expected state
16461487f786SFrançois Tigeot * @dev_priv: the i915 device
16471487f786SFrançois Tigeot * @reg: the register to read
16481487f786SFrançois Tigeot * @mask: mask to apply to register value
16491487f786SFrançois Tigeot * @value: expected value
16501487f786SFrançois Tigeot * @timeout_ms: timeout in millisecond
16511487f786SFrançois Tigeot *
16521487f786SFrançois Tigeot * This routine waits until the target register @reg contains the expected
165387df8fc6SFrançois Tigeot * @value after applying the @mask, i.e. it waits until ::
165487df8fc6SFrançois Tigeot *
165587df8fc6SFrançois Tigeot * (I915_READ(reg) & mask) == value
165687df8fc6SFrançois Tigeot *
16571487f786SFrançois Tigeot * Otherwise, the wait will timeout after @timeout_ms milliseconds.
16581487f786SFrançois Tigeot *
16591487f786SFrançois Tigeot * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
16601487f786SFrançois Tigeot */
intel_wait_for_register(struct drm_i915_private * dev_priv,i915_reg_t reg,u32 mask,u32 value,unsigned int timeout_ms)16611487f786SFrançois Tigeot int intel_wait_for_register(struct drm_i915_private *dev_priv,
16621487f786SFrançois Tigeot i915_reg_t reg,
1663*3f2dd94aSFrançois Tigeot u32 mask,
1664*3f2dd94aSFrançois Tigeot u32 value,
1665*3f2dd94aSFrançois Tigeot unsigned int timeout_ms)
16661487f786SFrançois Tigeot {
16671487f786SFrançois Tigeot unsigned fw =
16681487f786SFrançois Tigeot intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ);
16691487f786SFrançois Tigeot int ret;
16701487f786SFrançois Tigeot
1671*3f2dd94aSFrançois Tigeot might_sleep();
1672*3f2dd94aSFrançois Tigeot
1673*3f2dd94aSFrançois Tigeot spin_lock_irq(&dev_priv->uncore.lock);
1674*3f2dd94aSFrançois Tigeot intel_uncore_forcewake_get__locked(dev_priv, fw);
1675*3f2dd94aSFrançois Tigeot
1676*3f2dd94aSFrançois Tigeot ret = __intel_wait_for_register_fw(dev_priv,
1677*3f2dd94aSFrançois Tigeot reg, mask, value,
1678*3f2dd94aSFrançois Tigeot 2, 0, NULL);
1679*3f2dd94aSFrançois Tigeot
1680*3f2dd94aSFrançois Tigeot intel_uncore_forcewake_put__locked(dev_priv, fw);
1681*3f2dd94aSFrançois Tigeot spin_unlock_irq(&dev_priv->uncore.lock);
1682*3f2dd94aSFrançois Tigeot
16831487f786SFrançois Tigeot if (ret)
16841487f786SFrançois Tigeot ret = wait_for((I915_READ_NOTRACE(reg) & mask) == value,
16851487f786SFrançois Tigeot timeout_ms);
16861487f786SFrançois Tigeot
16871487f786SFrançois Tigeot return ret;
1688a05eeebfSFrançois Tigeot }
1689a05eeebfSFrançois Tigeot
gen8_reset_engine_start(struct intel_engine_cs * engine)1690*3f2dd94aSFrançois Tigeot static int gen8_reset_engine_start(struct intel_engine_cs *engine)
1691a05eeebfSFrançois Tigeot {
16921487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
16938621f407SFrançois Tigeot int ret;
1694a05eeebfSFrançois Tigeot
16958621f407SFrançois Tigeot I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
1696a05eeebfSFrançois Tigeot _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET));
1697a05eeebfSFrançois Tigeot
16981487f786SFrançois Tigeot ret = intel_wait_for_register_fw(dev_priv,
1699a05eeebfSFrançois Tigeot RING_RESET_CTL(engine->mmio_base),
1700a05eeebfSFrançois Tigeot RESET_CTL_READY_TO_RESET,
1701a05eeebfSFrançois Tigeot RESET_CTL_READY_TO_RESET,
17028621f407SFrançois Tigeot 700);
17038621f407SFrançois Tigeot if (ret)
1704a05eeebfSFrançois Tigeot DRM_ERROR("%s: reset request timeout\n", engine->name);
17058621f407SFrançois Tigeot
17068621f407SFrançois Tigeot return ret;
1707a05eeebfSFrançois Tigeot }
1708a05eeebfSFrançois Tigeot
gen8_reset_engine_cancel(struct intel_engine_cs * engine)1709*3f2dd94aSFrançois Tigeot static void gen8_reset_engine_cancel(struct intel_engine_cs *engine)
17108621f407SFrançois Tigeot {
17111487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
17128621f407SFrançois Tigeot
17138621f407SFrançois Tigeot I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
17148621f407SFrançois Tigeot _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET));
17158621f407SFrançois Tigeot }
17168621f407SFrançois Tigeot
gen8_reset_engines(struct drm_i915_private * dev_priv,unsigned engine_mask)17171487f786SFrançois Tigeot static int gen8_reset_engines(struct drm_i915_private *dev_priv,
17181487f786SFrançois Tigeot unsigned engine_mask)
17198621f407SFrançois Tigeot {
17208621f407SFrançois Tigeot struct intel_engine_cs *engine;
17211e12ee3bSFrançois Tigeot unsigned int tmp;
17228621f407SFrançois Tigeot
17231e12ee3bSFrançois Tigeot for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
1724*3f2dd94aSFrançois Tigeot if (gen8_reset_engine_start(engine))
17258621f407SFrançois Tigeot goto not_ready;
17268621f407SFrançois Tigeot
17271487f786SFrançois Tigeot return gen6_reset_engines(dev_priv, engine_mask);
1728a05eeebfSFrançois Tigeot
1729a05eeebfSFrançois Tigeot not_ready:
17301e12ee3bSFrançois Tigeot for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
1731*3f2dd94aSFrançois Tigeot gen8_reset_engine_cancel(engine);
1732a05eeebfSFrançois Tigeot
1733a05eeebfSFrançois Tigeot return -EIO;
1734a05eeebfSFrançois Tigeot }
1735a05eeebfSFrançois Tigeot
17361487f786SFrançois Tigeot typedef int (*reset_func)(struct drm_i915_private *, unsigned engine_mask);
17371487f786SFrançois Tigeot
intel_get_gpu_reset(struct drm_i915_private * dev_priv)17381487f786SFrançois Tigeot static reset_func intel_get_gpu_reset(struct drm_i915_private *dev_priv)
1739a05eeebfSFrançois Tigeot {
1740*3f2dd94aSFrançois Tigeot if (!i915_modparams.reset)
1741a05eeebfSFrançois Tigeot return NULL;
1742a05eeebfSFrançois Tigeot
17431487f786SFrançois Tigeot if (INTEL_INFO(dev_priv)->gen >= 8)
17448621f407SFrançois Tigeot return gen8_reset_engines;
17451487f786SFrançois Tigeot else if (INTEL_INFO(dev_priv)->gen >= 6)
17468621f407SFrançois Tigeot return gen6_reset_engines;
17471487f786SFrançois Tigeot else if (IS_GEN5(dev_priv))
1748a05eeebfSFrançois Tigeot return ironlake_do_reset;
17491487f786SFrançois Tigeot else if (IS_G4X(dev_priv))
1750a05eeebfSFrançois Tigeot return g4x_do_reset;
1751a85cb24fSFrançois Tigeot else if (IS_G33(dev_priv) || IS_PINEVIEW(dev_priv))
1752a05eeebfSFrançois Tigeot return g33_do_reset;
17531487f786SFrançois Tigeot else if (INTEL_INFO(dev_priv)->gen >= 3)
1754a05eeebfSFrançois Tigeot return i915_do_reset;
1755a05eeebfSFrançois Tigeot else
1756a05eeebfSFrançois Tigeot return NULL;
1757a05eeebfSFrançois Tigeot }
1758a05eeebfSFrançois Tigeot
intel_gpu_reset(struct drm_i915_private * dev_priv,unsigned engine_mask)17591487f786SFrançois Tigeot int intel_gpu_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
17609edbd4a0SFrançois Tigeot {
1761*3f2dd94aSFrançois Tigeot reset_func reset = intel_get_gpu_reset(dev_priv);
1762*3f2dd94aSFrançois Tigeot int retry;
1763352ff8bdSFrançois Tigeot int ret;
1764a05eeebfSFrançois Tigeot
1765*3f2dd94aSFrançois Tigeot might_sleep();
1766a05eeebfSFrançois Tigeot
1767352ff8bdSFrançois Tigeot /* If the power well sleeps during the reset, the reset
1768352ff8bdSFrançois Tigeot * request may be dropped and never completes (causing -EIO).
1769352ff8bdSFrançois Tigeot */
1770352ff8bdSFrançois Tigeot intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
1771*3f2dd94aSFrançois Tigeot for (retry = 0; retry < 3; retry++) {
1772*3f2dd94aSFrançois Tigeot
1773*3f2dd94aSFrançois Tigeot /* We stop engines, otherwise we might get failed reset and a
1774*3f2dd94aSFrançois Tigeot * dead gpu (on elk). Also as modern gpu as kbl can suffer
1775*3f2dd94aSFrançois Tigeot * from system hang if batchbuffer is progressing when
1776*3f2dd94aSFrançois Tigeot * the reset is issued, regardless of READY_TO_RESET ack.
1777*3f2dd94aSFrançois Tigeot * Thus assume it is best to stop engines on all gens
1778*3f2dd94aSFrançois Tigeot * where we have a gpu reset.
1779*3f2dd94aSFrançois Tigeot *
1780*3f2dd94aSFrançois Tigeot * WaMediaResetMainRingCleanup:ctg,elk (presumably)
1781*3f2dd94aSFrançois Tigeot *
1782*3f2dd94aSFrançois Tigeot * FIXME: Wa for more modern gens needs to be validated
1783*3f2dd94aSFrançois Tigeot */
1784*3f2dd94aSFrançois Tigeot i915_stop_engines(dev_priv, engine_mask);
1785*3f2dd94aSFrançois Tigeot
1786*3f2dd94aSFrançois Tigeot ret = -ENODEV;
1787*3f2dd94aSFrançois Tigeot if (reset)
17881487f786SFrançois Tigeot ret = reset(dev_priv, engine_mask);
1789*3f2dd94aSFrançois Tigeot if (ret != -ETIMEDOUT)
1790*3f2dd94aSFrançois Tigeot break;
1791*3f2dd94aSFrançois Tigeot
1792*3f2dd94aSFrançois Tigeot cond_resched();
1793*3f2dd94aSFrançois Tigeot }
1794352ff8bdSFrançois Tigeot intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
1795352ff8bdSFrançois Tigeot
1796352ff8bdSFrançois Tigeot return ret;
1797a05eeebfSFrançois Tigeot }
1798a05eeebfSFrançois Tigeot
intel_has_gpu_reset(struct drm_i915_private * dev_priv)17991487f786SFrançois Tigeot bool intel_has_gpu_reset(struct drm_i915_private *dev_priv)
1800a05eeebfSFrançois Tigeot {
18011487f786SFrançois Tigeot return intel_get_gpu_reset(dev_priv) != NULL;
18029edbd4a0SFrançois Tigeot }
18039edbd4a0SFrançois Tigeot
1804*3f2dd94aSFrançois Tigeot /*
1805*3f2dd94aSFrançois Tigeot * When GuC submission is enabled, GuC manages ELSP and can initiate the
1806*3f2dd94aSFrançois Tigeot * engine reset too. For now, fall back to full GPU reset if it is enabled.
1807*3f2dd94aSFrançois Tigeot */
intel_has_reset_engine(struct drm_i915_private * dev_priv)1808*3f2dd94aSFrançois Tigeot bool intel_has_reset_engine(struct drm_i915_private *dev_priv)
1809*3f2dd94aSFrançois Tigeot {
1810*3f2dd94aSFrançois Tigeot return (dev_priv->info.has_reset_engine &&
1811*3f2dd94aSFrançois Tigeot !dev_priv->guc.execbuf_client &&
1812*3f2dd94aSFrançois Tigeot i915_modparams.reset >= 2);
1813*3f2dd94aSFrançois Tigeot }
1814*3f2dd94aSFrançois Tigeot
intel_guc_reset(struct drm_i915_private * dev_priv)18158621f407SFrançois Tigeot int intel_guc_reset(struct drm_i915_private *dev_priv)
18168621f407SFrançois Tigeot {
18178621f407SFrançois Tigeot int ret;
18188621f407SFrançois Tigeot
18191487f786SFrançois Tigeot if (!HAS_GUC(dev_priv))
18208621f407SFrançois Tigeot return -EINVAL;
18218621f407SFrançois Tigeot
18228621f407SFrançois Tigeot intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
18238621f407SFrançois Tigeot ret = gen6_hw_domain_reset(dev_priv, GEN9_GRDOM_GUC);
18248621f407SFrançois Tigeot intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
18258621f407SFrançois Tigeot
18268621f407SFrançois Tigeot return ret;
18278621f407SFrançois Tigeot }
18288621f407SFrançois Tigeot
intel_uncore_unclaimed_mmio(struct drm_i915_private * dev_priv)1829c0e85e96SFrançois Tigeot bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv)
18309edbd4a0SFrançois Tigeot {
1831c0e85e96SFrançois Tigeot return check_for_unclaimed_mmio(dev_priv);
18329edbd4a0SFrançois Tigeot }
1833c0e85e96SFrançois Tigeot
1834c0e85e96SFrançois Tigeot bool
intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private * dev_priv)1835c0e85e96SFrançois Tigeot intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv)
1836c0e85e96SFrançois Tigeot {
1837*3f2dd94aSFrançois Tigeot if (unlikely(i915_modparams.mmio_debug ||
1838c0e85e96SFrançois Tigeot dev_priv->uncore.unclaimed_mmio_check <= 0))
1839c0e85e96SFrançois Tigeot return false;
1840c0e85e96SFrançois Tigeot
1841c0e85e96SFrançois Tigeot if (unlikely(intel_uncore_unclaimed_mmio(dev_priv))) {
1842c0e85e96SFrançois Tigeot DRM_DEBUG("Unclaimed register detected, "
1843c0e85e96SFrançois Tigeot "enabling oneshot unclaimed register reporting. "
1844c0e85e96SFrançois Tigeot "Please use i915.mmio_debug=N for more information.\n");
1845*3f2dd94aSFrançois Tigeot i915_modparams.mmio_debug++;
1846c0e85e96SFrançois Tigeot dev_priv->uncore.unclaimed_mmio_check--;
1847c0e85e96SFrançois Tigeot return true;
1848c0e85e96SFrançois Tigeot }
1849c0e85e96SFrançois Tigeot
1850c0e85e96SFrançois Tigeot return false;
18519edbd4a0SFrançois Tigeot }
18528621f407SFrançois Tigeot
18538621f407SFrançois Tigeot static enum forcewake_domains
intel_uncore_forcewake_for_read(struct drm_i915_private * dev_priv,i915_reg_t reg)18548621f407SFrançois Tigeot intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv,
18558621f407SFrançois Tigeot i915_reg_t reg)
18568621f407SFrançois Tigeot {
18571e12ee3bSFrançois Tigeot u32 offset = i915_mmio_reg_offset(reg);
18588621f407SFrançois Tigeot enum forcewake_domains fw_domains;
18598621f407SFrançois Tigeot
18601e12ee3bSFrançois Tigeot if (HAS_FWTABLE(dev_priv)) {
18611e12ee3bSFrançois Tigeot fw_domains = __fwtable_reg_read_fw_domains(offset);
18621e12ee3bSFrançois Tigeot } else if (INTEL_GEN(dev_priv) >= 6) {
18631e12ee3bSFrançois Tigeot fw_domains = __gen6_reg_read_fw_domains(offset);
18641e12ee3bSFrançois Tigeot } else {
18651e12ee3bSFrançois Tigeot WARN_ON(!IS_GEN(dev_priv, 2, 5));
18661e12ee3bSFrançois Tigeot fw_domains = 0;
18678621f407SFrançois Tigeot }
18688621f407SFrançois Tigeot
18698621f407SFrançois Tigeot WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
18708621f407SFrançois Tigeot
18718621f407SFrançois Tigeot return fw_domains;
18728621f407SFrançois Tigeot }
18738621f407SFrançois Tigeot
18748621f407SFrançois Tigeot static enum forcewake_domains
intel_uncore_forcewake_for_write(struct drm_i915_private * dev_priv,i915_reg_t reg)18758621f407SFrançois Tigeot intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv,
18768621f407SFrançois Tigeot i915_reg_t reg)
18778621f407SFrançois Tigeot {
18781e12ee3bSFrançois Tigeot u32 offset = i915_mmio_reg_offset(reg);
18798621f407SFrançois Tigeot enum forcewake_domains fw_domains;
18808621f407SFrançois Tigeot
18811e12ee3bSFrançois Tigeot if (HAS_FWTABLE(dev_priv) && !IS_VALLEYVIEW(dev_priv)) {
18821e12ee3bSFrançois Tigeot fw_domains = __fwtable_reg_write_fw_domains(offset);
18831e12ee3bSFrançois Tigeot } else if (IS_GEN8(dev_priv)) {
18841e12ee3bSFrançois Tigeot fw_domains = __gen8_reg_write_fw_domains(offset);
18851e12ee3bSFrançois Tigeot } else if (IS_GEN(dev_priv, 6, 7)) {
18868621f407SFrançois Tigeot fw_domains = FORCEWAKE_RENDER;
18871e12ee3bSFrançois Tigeot } else {
18881e12ee3bSFrançois Tigeot WARN_ON(!IS_GEN(dev_priv, 2, 5));
18891e12ee3bSFrançois Tigeot fw_domains = 0;
18908621f407SFrançois Tigeot }
18918621f407SFrançois Tigeot
18928621f407SFrançois Tigeot WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
18938621f407SFrançois Tigeot
18948621f407SFrançois Tigeot return fw_domains;
18958621f407SFrançois Tigeot }
18968621f407SFrançois Tigeot
18978621f407SFrançois Tigeot /**
18988621f407SFrançois Tigeot * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access
18998621f407SFrançois Tigeot * a register
19008621f407SFrançois Tigeot * @dev_priv: pointer to struct drm_i915_private
19018621f407SFrançois Tigeot * @reg: register in question
19028621f407SFrançois Tigeot * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE
19038621f407SFrançois Tigeot *
19048621f407SFrançois Tigeot * Returns a set of forcewake domains required to be taken with for example
19058621f407SFrançois Tigeot * intel_uncore_forcewake_get for the specified register to be accessible in the
19068621f407SFrançois Tigeot * specified mode (read, write or read/write) with raw mmio accessors.
19078621f407SFrançois Tigeot *
19088621f407SFrançois Tigeot * NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the
19098621f407SFrançois Tigeot * callers to do FIFO management on their own or risk losing writes.
19108621f407SFrançois Tigeot */
19118621f407SFrançois Tigeot enum forcewake_domains
intel_uncore_forcewake_for_reg(struct drm_i915_private * dev_priv,i915_reg_t reg,unsigned int op)19128621f407SFrançois Tigeot intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
19138621f407SFrançois Tigeot i915_reg_t reg, unsigned int op)
19148621f407SFrançois Tigeot {
19158621f407SFrançois Tigeot enum forcewake_domains fw_domains = 0;
19168621f407SFrançois Tigeot
19178621f407SFrançois Tigeot WARN_ON(!op);
19188621f407SFrançois Tigeot
19191e12ee3bSFrançois Tigeot if (intel_vgpu_active(dev_priv))
19201e12ee3bSFrançois Tigeot return 0;
19211e12ee3bSFrançois Tigeot
19228621f407SFrançois Tigeot if (op & FW_REG_READ)
19238621f407SFrançois Tigeot fw_domains = intel_uncore_forcewake_for_read(dev_priv, reg);
19248621f407SFrançois Tigeot
19258621f407SFrançois Tigeot if (op & FW_REG_WRITE)
19268621f407SFrançois Tigeot fw_domains |= intel_uncore_forcewake_for_write(dev_priv, reg);
19278621f407SFrançois Tigeot
19288621f407SFrançois Tigeot return fw_domains;
19298621f407SFrançois Tigeot }
1930a85cb24fSFrançois Tigeot
1931a85cb24fSFrançois Tigeot #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1932*3f2dd94aSFrançois Tigeot #include "selftests/mock_uncore.c"
1933a85cb24fSFrançois Tigeot #include "selftests/intel_uncore.c"
1934a85cb24fSFrançois Tigeot #endif
1935