xref: /dflybsd-src/sys/dev/drm/i915/intel_uncore.c (revision 3f2dd94a569761201b5b0a18b2f697f97fe1b9dc)
19edbd4a0SFrançois Tigeot /*
29edbd4a0SFrançois Tigeot  * Copyright © 2013 Intel Corporation
39edbd4a0SFrançois Tigeot  *
49edbd4a0SFrançois Tigeot  * Permission is hereby granted, free of charge, to any person obtaining a
59edbd4a0SFrançois Tigeot  * copy of this software and associated documentation files (the "Software"),
69edbd4a0SFrançois Tigeot  * to deal in the Software without restriction, including without limitation
79edbd4a0SFrançois Tigeot  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
89edbd4a0SFrançois Tigeot  * and/or sell copies of the Software, and to permit persons to whom the
99edbd4a0SFrançois Tigeot  * Software is furnished to do so, subject to the following conditions:
109edbd4a0SFrançois Tigeot  *
119edbd4a0SFrançois Tigeot  * The above copyright notice and this permission notice (including the next
129edbd4a0SFrançois Tigeot  * paragraph) shall be included in all copies or substantial portions of the
139edbd4a0SFrançois Tigeot  * Software.
149edbd4a0SFrançois Tigeot  *
159edbd4a0SFrançois Tigeot  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
169edbd4a0SFrançois Tigeot  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
179edbd4a0SFrançois Tigeot  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
189edbd4a0SFrançois Tigeot  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
199edbd4a0SFrançois Tigeot  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
209edbd4a0SFrançois Tigeot  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
219edbd4a0SFrançois Tigeot  * IN THE SOFTWARE.
229edbd4a0SFrançois Tigeot  */
239edbd4a0SFrançois Tigeot 
249edbd4a0SFrançois Tigeot #include "i915_drv.h"
259edbd4a0SFrançois Tigeot #include "intel_drv.h"
26477eb7f9SFrançois Tigeot #include "i915_vgpu.h"
279edbd4a0SFrançois Tigeot 
28a85cb24fSFrançois Tigeot #include <asm/iosf_mbi.h>
29ff14819bSFrançois Tigeot #include <linux/pm_runtime.h>
30ff14819bSFrançois Tigeot 
31352ff8bdSFrançois Tigeot #define FORCEWAKE_ACK_TIMEOUT_MS 50
32*3f2dd94aSFrançois Tigeot #define GT_FIFO_TIMEOUT_MS	 10
339edbd4a0SFrançois Tigeot 
34aee94f86SFrançois Tigeot #define __raw_posting_read(dev_priv__, reg__) (void)__raw_i915_read32((dev_priv__), (reg__))
359edbd4a0SFrançois Tigeot 
362c9916cdSFrançois Tigeot static const char * const forcewake_domain_names[] = {
372c9916cdSFrançois Tigeot 	"render",
382c9916cdSFrançois Tigeot 	"blitter",
392c9916cdSFrançois Tigeot 	"media",
402c9916cdSFrançois Tigeot };
412c9916cdSFrançois Tigeot 
422c9916cdSFrançois Tigeot const char *
intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)432c9916cdSFrançois Tigeot intel_uncore_forcewake_domain_to_str(const enum forcewake_domain_id id)
442c9916cdSFrançois Tigeot {
45352ff8bdSFrançois Tigeot 	BUILD_BUG_ON(ARRAY_SIZE(forcewake_domain_names) != FW_DOMAIN_ID_COUNT);
462c9916cdSFrançois Tigeot 
472c9916cdSFrançois Tigeot 	if (id >= 0 && id < FW_DOMAIN_ID_COUNT)
482c9916cdSFrançois Tigeot 		return forcewake_domain_names[id];
492c9916cdSFrançois Tigeot 
502c9916cdSFrançois Tigeot 	WARN_ON(id);
512c9916cdSFrançois Tigeot 
522c9916cdSFrançois Tigeot 	return "unknown";
532c9916cdSFrançois Tigeot }
542c9916cdSFrançois Tigeot 
552c9916cdSFrançois Tigeot static inline void
fw_domain_reset(struct drm_i915_private * i915,const struct intel_uncore_forcewake_domain * d)56a85cb24fSFrançois Tigeot fw_domain_reset(struct drm_i915_private *i915,
57a85cb24fSFrançois Tigeot 		const struct intel_uncore_forcewake_domain *d)
582c9916cdSFrançois Tigeot {
59a85cb24fSFrançois Tigeot 	__raw_i915_write32(i915, d->reg_set, i915->uncore.fw_reset);
602c9916cdSFrançois Tigeot }
612c9916cdSFrançois Tigeot 
622c9916cdSFrançois Tigeot static inline void
fw_domain_arm_timer(struct intel_uncore_forcewake_domain * d)632c9916cdSFrançois Tigeot fw_domain_arm_timer(struct intel_uncore_forcewake_domain *d)
642c9916cdSFrançois Tigeot {
658621f407SFrançois Tigeot 	d->wake_count++;
668621f407SFrançois Tigeot 	hrtimer_start_range_ns(&d->timer,
674be47400SFrançois Tigeot 			       NSEC_PER_MSEC,
688621f407SFrançois Tigeot 			       NSEC_PER_MSEC,
698621f407SFrançois Tigeot 			       HRTIMER_MODE_REL);
702c9916cdSFrançois Tigeot }
712c9916cdSFrançois Tigeot 
722c9916cdSFrançois Tigeot static inline void
fw_domain_wait_ack_clear(const struct drm_i915_private * i915,const struct intel_uncore_forcewake_domain * d)73a85cb24fSFrançois Tigeot fw_domain_wait_ack_clear(const struct drm_i915_private *i915,
74a85cb24fSFrançois Tigeot 			 const struct intel_uncore_forcewake_domain *d)
752c9916cdSFrançois Tigeot {
76a85cb24fSFrançois Tigeot 	if (wait_for_atomic((__raw_i915_read32(i915, d->reg_ack) &
772c9916cdSFrançois Tigeot 			     FORCEWAKE_KERNEL) == 0,
782c9916cdSFrançois Tigeot 			    FORCEWAKE_ACK_TIMEOUT_MS))
792c9916cdSFrançois Tigeot 		DRM_ERROR("%s: timed out waiting for forcewake ack to clear.\n",
802c9916cdSFrançois Tigeot 			  intel_uncore_forcewake_domain_to_str(d->id));
812c9916cdSFrançois Tigeot }
822c9916cdSFrançois Tigeot 
832c9916cdSFrançois Tigeot static inline void
fw_domain_get(struct drm_i915_private * i915,const struct intel_uncore_forcewake_domain * d)84a85cb24fSFrançois Tigeot fw_domain_get(struct drm_i915_private *i915,
85a85cb24fSFrançois Tigeot 	      const struct intel_uncore_forcewake_domain *d)
862c9916cdSFrançois Tigeot {
87a85cb24fSFrançois Tigeot 	__raw_i915_write32(i915, d->reg_set, i915->uncore.fw_set);
882c9916cdSFrançois Tigeot }
892c9916cdSFrançois Tigeot 
902c9916cdSFrançois Tigeot static inline void
fw_domain_wait_ack(const struct drm_i915_private * i915,const struct intel_uncore_forcewake_domain * d)91a85cb24fSFrançois Tigeot fw_domain_wait_ack(const struct drm_i915_private *i915,
92a85cb24fSFrançois Tigeot 		   const struct intel_uncore_forcewake_domain *d)
932c9916cdSFrançois Tigeot {
94a85cb24fSFrançois Tigeot 	if (wait_for_atomic((__raw_i915_read32(i915, d->reg_ack) &
952c9916cdSFrançois Tigeot 			     FORCEWAKE_KERNEL),
962c9916cdSFrançois Tigeot 			    FORCEWAKE_ACK_TIMEOUT_MS))
972c9916cdSFrançois Tigeot 		DRM_ERROR("%s: timed out waiting for forcewake ack request.\n",
982c9916cdSFrançois Tigeot 			  intel_uncore_forcewake_domain_to_str(d->id));
992c9916cdSFrançois Tigeot }
1002c9916cdSFrançois Tigeot 
1012c9916cdSFrançois Tigeot static inline void
fw_domain_put(const struct drm_i915_private * i915,const struct intel_uncore_forcewake_domain * d)102a85cb24fSFrançois Tigeot fw_domain_put(const struct drm_i915_private *i915,
103a85cb24fSFrançois Tigeot 	      const struct intel_uncore_forcewake_domain *d)
1042c9916cdSFrançois Tigeot {
105a85cb24fSFrançois Tigeot 	__raw_i915_write32(i915, d->reg_set, i915->uncore.fw_clear);
1062c9916cdSFrançois Tigeot }
1072c9916cdSFrançois Tigeot 
1082c9916cdSFrançois Tigeot static void
fw_domains_get(struct drm_i915_private * i915,enum forcewake_domains fw_domains)109a85cb24fSFrançois Tigeot fw_domains_get(struct drm_i915_private *i915, enum forcewake_domains fw_domains)
1102c9916cdSFrançois Tigeot {
1112c9916cdSFrançois Tigeot 	struct intel_uncore_forcewake_domain *d;
112a85cb24fSFrançois Tigeot 	unsigned int tmp;
1132c9916cdSFrançois Tigeot 
114a85cb24fSFrançois Tigeot 	GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains);
115a85cb24fSFrançois Tigeot 
116a85cb24fSFrançois Tigeot 	for_each_fw_domain_masked(d, fw_domains, i915, tmp) {
117a85cb24fSFrançois Tigeot 		fw_domain_wait_ack_clear(i915, d);
118a85cb24fSFrançois Tigeot 		fw_domain_get(i915, d);
1192c9916cdSFrançois Tigeot 	}
1208621f407SFrançois Tigeot 
121a85cb24fSFrançois Tigeot 	for_each_fw_domain_masked(d, fw_domains, i915, tmp)
122a85cb24fSFrançois Tigeot 		fw_domain_wait_ack(i915, d);
1234be47400SFrançois Tigeot 
124a85cb24fSFrançois Tigeot 	i915->uncore.fw_domains_active |= fw_domains;
1252c9916cdSFrançois Tigeot }
1262c9916cdSFrançois Tigeot 
1272c9916cdSFrançois Tigeot static void
fw_domains_put(struct drm_i915_private * i915,enum forcewake_domains fw_domains)128a85cb24fSFrançois Tigeot fw_domains_put(struct drm_i915_private *i915, enum forcewake_domains fw_domains)
1292c9916cdSFrançois Tigeot {
1302c9916cdSFrançois Tigeot 	struct intel_uncore_forcewake_domain *d;
131a85cb24fSFrançois Tigeot 	unsigned int tmp;
1322c9916cdSFrançois Tigeot 
133a85cb24fSFrançois Tigeot 	GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains);
1344be47400SFrançois Tigeot 
135a85cb24fSFrançois Tigeot 	for_each_fw_domain_masked(d, fw_domains, i915, tmp)
136a85cb24fSFrançois Tigeot 		fw_domain_put(i915, d);
137a85cb24fSFrançois Tigeot 
138a85cb24fSFrançois Tigeot 	i915->uncore.fw_domains_active &= ~fw_domains;
1392c9916cdSFrançois Tigeot }
1402c9916cdSFrançois Tigeot 
1412c9916cdSFrançois Tigeot static void
fw_domains_reset(struct drm_i915_private * i915,enum forcewake_domains fw_domains)142a85cb24fSFrançois Tigeot fw_domains_reset(struct drm_i915_private *i915,
143a85cb24fSFrançois Tigeot 		 enum forcewake_domains fw_domains)
1442c9916cdSFrançois Tigeot {
1452c9916cdSFrançois Tigeot 	struct intel_uncore_forcewake_domain *d;
146a85cb24fSFrançois Tigeot 	unsigned int tmp;
1472c9916cdSFrançois Tigeot 
148a85cb24fSFrançois Tigeot 	if (!fw_domains)
1492c9916cdSFrançois Tigeot 		return;
1502c9916cdSFrançois Tigeot 
151a85cb24fSFrançois Tigeot 	GEM_BUG_ON(fw_domains & ~i915->uncore.fw_domains);
1522c9916cdSFrançois Tigeot 
153a85cb24fSFrançois Tigeot 	for_each_fw_domain_masked(d, fw_domains, i915, tmp)
154a85cb24fSFrançois Tigeot 		fw_domain_reset(i915, d);
1552c9916cdSFrançois Tigeot }
1562c9916cdSFrançois Tigeot 
__gen6_gt_wait_for_thread_c0(struct drm_i915_private * dev_priv)1579edbd4a0SFrançois Tigeot static void __gen6_gt_wait_for_thread_c0(struct drm_i915_private *dev_priv)
1589edbd4a0SFrançois Tigeot {
1599edbd4a0SFrançois Tigeot 	/* w/a for a sporadic read returning 0 by waiting for the GT
1609edbd4a0SFrançois Tigeot 	 * thread to wake up.
1619edbd4a0SFrançois Tigeot 	 */
1622c9916cdSFrançois Tigeot 	if (wait_for_atomic_us((__raw_i915_read32(dev_priv, GEN6_GT_THREAD_STATUS_REG) &
1632c9916cdSFrançois Tigeot 				GEN6_GT_THREAD_STATUS_CORE_MASK) == 0, 500))
1649edbd4a0SFrançois Tigeot 		DRM_ERROR("GT thread status wait timed out\n");
1659edbd4a0SFrançois Tigeot }
1669edbd4a0SFrançois Tigeot 
fw_domains_get_with_thread_status(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)1672c9916cdSFrançois Tigeot static void fw_domains_get_with_thread_status(struct drm_i915_private *dev_priv,
1682c9916cdSFrançois Tigeot 					      enum forcewake_domains fw_domains)
1699edbd4a0SFrançois Tigeot {
1702c9916cdSFrançois Tigeot 	fw_domains_get(dev_priv, fw_domains);
1719edbd4a0SFrançois Tigeot 
1722c9916cdSFrançois Tigeot 	/* WaRsForcewakeWaitTC0:snb,ivb,hsw,bdw,vlv */
1739edbd4a0SFrançois Tigeot 	__gen6_gt_wait_for_thread_c0(dev_priv);
1749edbd4a0SFrançois Tigeot }
1759edbd4a0SFrançois Tigeot 
fifo_free_entries(struct drm_i915_private * dev_priv)176477eb7f9SFrançois Tigeot static inline u32 fifo_free_entries(struct drm_i915_private *dev_priv)
177477eb7f9SFrançois Tigeot {
178477eb7f9SFrançois Tigeot 	u32 count = __raw_i915_read32(dev_priv, GTFIFOCTL);
179477eb7f9SFrançois Tigeot 
180477eb7f9SFrançois Tigeot 	return count & GT_FIFO_FREE_ENTRIES_MASK;
181477eb7f9SFrançois Tigeot }
182477eb7f9SFrançois Tigeot 
__gen6_gt_wait_for_fifo(struct drm_i915_private * dev_priv)183*3f2dd94aSFrançois Tigeot static void __gen6_gt_wait_for_fifo(struct drm_i915_private *dev_priv)
1849edbd4a0SFrançois Tigeot {
185*3f2dd94aSFrançois Tigeot 	u32 n;
1869edbd4a0SFrançois Tigeot 
1879edbd4a0SFrançois Tigeot 	/* On VLV, FIFO will be shared by both SW and HW.
1889edbd4a0SFrançois Tigeot 	 * So, we need to read the FREE_ENTRIES everytime */
1898621f407SFrançois Tigeot 	if (IS_VALLEYVIEW(dev_priv))
190*3f2dd94aSFrançois Tigeot 		n = fifo_free_entries(dev_priv);
191*3f2dd94aSFrançois Tigeot 	else
192*3f2dd94aSFrançois Tigeot 		n = dev_priv->uncore.fifo_count;
1939edbd4a0SFrançois Tigeot 
194*3f2dd94aSFrançois Tigeot 	if (n <= GT_FIFO_NUM_RESERVED_ENTRIES) {
195*3f2dd94aSFrançois Tigeot 		if (wait_for_atomic((n = fifo_free_entries(dev_priv)) >
196*3f2dd94aSFrançois Tigeot 				    GT_FIFO_NUM_RESERVED_ENTRIES,
197*3f2dd94aSFrançois Tigeot 				    GT_FIFO_TIMEOUT_MS)) {
198*3f2dd94aSFrançois Tigeot 			DRM_DEBUG("GT_FIFO timeout, entries: %u\n", n);
199*3f2dd94aSFrançois Tigeot 			return;
2009edbd4a0SFrançois Tigeot 		}
2019edbd4a0SFrançois Tigeot 	}
2029edbd4a0SFrançois Tigeot 
203*3f2dd94aSFrançois Tigeot 	dev_priv->uncore.fifo_count = n - 1;
2049edbd4a0SFrançois Tigeot }
2059edbd4a0SFrançois Tigeot 
2068621f407SFrançois Tigeot static enum hrtimer_restart
intel_uncore_fw_release_timer(struct hrtimer * timer)2078621f407SFrançois Tigeot intel_uncore_fw_release_timer(struct hrtimer *timer)
2089edbd4a0SFrançois Tigeot {
2098621f407SFrançois Tigeot 	struct intel_uncore_forcewake_domain *domain =
2108621f407SFrançois Tigeot 	       container_of(timer, struct intel_uncore_forcewake_domain, timer);
211a85cb24fSFrançois Tigeot 	struct drm_i915_private *dev_priv =
212a85cb24fSFrançois Tigeot 		container_of(domain, struct drm_i915_private, uncore.fw_domain[domain->id]);
2135e269720SFrançois Tigeot 	unsigned long irqflags;
2142c9916cdSFrançois Tigeot 
2151e12ee3bSFrançois Tigeot 	assert_rpm_device_not_suspended(dev_priv);
2162c9916cdSFrançois Tigeot 
217*3f2dd94aSFrançois Tigeot 	if (xchg(&domain->active, false))
218*3f2dd94aSFrançois Tigeot 		return HRTIMER_RESTART;
219*3f2dd94aSFrançois Tigeot 
2201e12ee3bSFrançois Tigeot 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
2212c9916cdSFrançois Tigeot 	if (WARN_ON(domain->wake_count == 0))
2222c9916cdSFrançois Tigeot 		domain->wake_count++;
2232c9916cdSFrançois Tigeot 
2244be47400SFrançois Tigeot 	if (--domain->wake_count == 0)
2251e12ee3bSFrançois Tigeot 		dev_priv->uncore.funcs.force_wake_put(dev_priv, domain->mask);
2262c9916cdSFrançois Tigeot 
2271e12ee3bSFrançois Tigeot 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
2288621f407SFrançois Tigeot 
2298621f407SFrançois Tigeot 	return HRTIMER_NORESTART;
2309edbd4a0SFrançois Tigeot }
2319edbd4a0SFrançois Tigeot 
intel_uncore_forcewake_reset(struct drm_i915_private * dev_priv,bool restore)232a85cb24fSFrançois Tigeot static void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv,
2331487f786SFrançois Tigeot 					 bool restore)
2349edbd4a0SFrançois Tigeot {
2355e269720SFrançois Tigeot 	unsigned long irqflags;
2362c9916cdSFrançois Tigeot 	struct intel_uncore_forcewake_domain *domain;
2372c9916cdSFrançois Tigeot 	int retry_count = 100;
2381e12ee3bSFrançois Tigeot 	enum forcewake_domains fw, active_domains;
239ba55f2f5SFrançois Tigeot 
240ba55f2f5SFrançois Tigeot 	/* Hold uncore.lock across reset to prevent any register access
2412c9916cdSFrançois Tigeot 	 * with forcewake not set correctly. Wait until all pending
2422c9916cdSFrançois Tigeot 	 * timers are run before holding.
243ba55f2f5SFrançois Tigeot 	 */
2442c9916cdSFrançois Tigeot 	while (1) {
245a85cb24fSFrançois Tigeot 		unsigned int tmp;
246a85cb24fSFrançois Tigeot 
2472c9916cdSFrançois Tigeot 		active_domains = 0;
248ba55f2f5SFrançois Tigeot 
249a85cb24fSFrançois Tigeot 		for_each_fw_domain(domain, dev_priv, tmp) {
250*3f2dd94aSFrançois Tigeot 			smp_store_mb(domain->active, false);
2518621f407SFrançois Tigeot 			if (hrtimer_cancel(&domain->timer) == 0)
2522c9916cdSFrançois Tigeot 				continue;
253ba55f2f5SFrançois Tigeot 
2548621f407SFrançois Tigeot 			intel_uncore_fw_release_timer(&domain->timer);
2559edbd4a0SFrançois Tigeot 		}
256ba55f2f5SFrançois Tigeot 
2575e269720SFrançois Tigeot 		spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
2582c9916cdSFrançois Tigeot 
259a85cb24fSFrançois Tigeot 		for_each_fw_domain(domain, dev_priv, tmp) {
2608621f407SFrançois Tigeot 			if (hrtimer_active(&domain->timer))
2618621f407SFrançois Tigeot 				active_domains |= domain->mask;
2622c9916cdSFrançois Tigeot 		}
2632c9916cdSFrançois Tigeot 
2642c9916cdSFrançois Tigeot 		if (active_domains == 0)
2652c9916cdSFrançois Tigeot 			break;
2662c9916cdSFrançois Tigeot 
2672c9916cdSFrançois Tigeot 		if (--retry_count == 0) {
2682c9916cdSFrançois Tigeot 			DRM_ERROR("Timed out waiting for forcewake timers to finish\n");
2692c9916cdSFrançois Tigeot 			break;
2702c9916cdSFrançois Tigeot 		}
2712c9916cdSFrançois Tigeot 
2725e269720SFrançois Tigeot 		spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
2732c9916cdSFrançois Tigeot 		cond_resched();
2742c9916cdSFrançois Tigeot 	}
2752c9916cdSFrançois Tigeot 
2762c9916cdSFrançois Tigeot 	WARN_ON(active_domains);
2772c9916cdSFrançois Tigeot 
2781e12ee3bSFrançois Tigeot 	fw = dev_priv->uncore.fw_domains_active;
2792c9916cdSFrançois Tigeot 	if (fw)
2802c9916cdSFrançois Tigeot 		dev_priv->uncore.funcs.force_wake_put(dev_priv, fw);
2812c9916cdSFrançois Tigeot 
282a85cb24fSFrançois Tigeot 	fw_domains_reset(dev_priv, dev_priv->uncore.fw_domains);
2832c9916cdSFrançois Tigeot 
2842c9916cdSFrançois Tigeot 	if (restore) { /* If reset with a user forcewake, try to restore */
285ba55f2f5SFrançois Tigeot 		if (fw)
286ba55f2f5SFrançois Tigeot 			dev_priv->uncore.funcs.force_wake_get(dev_priv, fw);
287ba55f2f5SFrançois Tigeot 
2881487f786SFrançois Tigeot 		if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
289ba55f2f5SFrançois Tigeot 			dev_priv->uncore.fifo_count =
290477eb7f9SFrançois Tigeot 				fifo_free_entries(dev_priv);
291ba55f2f5SFrançois Tigeot 	}
292ba55f2f5SFrançois Tigeot 
2932c9916cdSFrançois Tigeot 	if (!restore)
2942c9916cdSFrançois Tigeot 		assert_forcewakes_inactive(dev_priv);
2952c9916cdSFrançois Tigeot 
2965e269720SFrançois Tigeot 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
2979edbd4a0SFrançois Tigeot }
2989edbd4a0SFrançois Tigeot 
gen9_edram_size(struct drm_i915_private * dev_priv)2998621f407SFrançois Tigeot static u64 gen9_edram_size(struct drm_i915_private *dev_priv)
3009edbd4a0SFrançois Tigeot {
3018621f407SFrançois Tigeot 	const unsigned int ways[8] = { 4, 8, 12, 16, 16, 16, 16, 16 };
3028621f407SFrançois Tigeot 	const unsigned int sets[4] = { 1, 1, 2, 2 };
3038621f407SFrançois Tigeot 	const u32 cap = dev_priv->edram_cap;
3049edbd4a0SFrançois Tigeot 
3058621f407SFrançois Tigeot 	return EDRAM_NUM_BANKS(cap) *
3068621f407SFrançois Tigeot 		ways[EDRAM_WAYS_IDX(cap)] *
3078621f407SFrançois Tigeot 		sets[EDRAM_SETS_IDX(cap)] *
3088621f407SFrançois Tigeot 		1024 * 1024;
3099edbd4a0SFrançois Tigeot }
3108621f407SFrançois Tigeot 
intel_uncore_edram_size(struct drm_i915_private * dev_priv)3118621f407SFrançois Tigeot u64 intel_uncore_edram_size(struct drm_i915_private *dev_priv)
3128621f407SFrançois Tigeot {
3138621f407SFrançois Tigeot 	if (!HAS_EDRAM(dev_priv))
3148621f407SFrançois Tigeot 		return 0;
3158621f407SFrançois Tigeot 
3168621f407SFrançois Tigeot 	/* The needed capability bits for size calculation
3178621f407SFrançois Tigeot 	 * are not there with pre gen9 so return 128MB always.
3188621f407SFrançois Tigeot 	 */
3198621f407SFrançois Tigeot 	if (INTEL_GEN(dev_priv) < 9)
3208621f407SFrançois Tigeot 		return 128 * 1024 * 1024;
3218621f407SFrançois Tigeot 
3228621f407SFrançois Tigeot 	return gen9_edram_size(dev_priv);
3238621f407SFrançois Tigeot }
3248621f407SFrançois Tigeot 
intel_uncore_edram_detect(struct drm_i915_private * dev_priv)3258621f407SFrançois Tigeot static void intel_uncore_edram_detect(struct drm_i915_private *dev_priv)
3268621f407SFrançois Tigeot {
3278621f407SFrançois Tigeot 	if (IS_HASWELL(dev_priv) ||
3288621f407SFrançois Tigeot 	    IS_BROADWELL(dev_priv) ||
3298621f407SFrançois Tigeot 	    INTEL_GEN(dev_priv) >= 9) {
3308621f407SFrançois Tigeot 		dev_priv->edram_cap = __raw_i915_read32(dev_priv,
3318621f407SFrançois Tigeot 							HSW_EDRAM_CAP);
3328621f407SFrançois Tigeot 
3338621f407SFrançois Tigeot 		/* NB: We can't write IDICR yet because we do not have gt funcs
3348621f407SFrançois Tigeot 		 * set up */
3358621f407SFrançois Tigeot 	} else {
3368621f407SFrançois Tigeot 		dev_priv->edram_cap = 0;
3378621f407SFrançois Tigeot 	}
3388621f407SFrançois Tigeot 
3398621f407SFrançois Tigeot 	if (HAS_EDRAM(dev_priv))
3408621f407SFrançois Tigeot 		DRM_INFO("Found %lluMB of eDRAM\n",
3418621f407SFrançois Tigeot 			 intel_uncore_edram_size(dev_priv) / (1024 * 1024));
3422c9916cdSFrançois Tigeot }
3432c9916cdSFrançois Tigeot 
344c0e85e96SFrançois Tigeot static bool
fpga_check_for_unclaimed_mmio(struct drm_i915_private * dev_priv)345c0e85e96SFrançois Tigeot fpga_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
346c0e85e96SFrançois Tigeot {
347c0e85e96SFrançois Tigeot 	u32 dbg;
348c0e85e96SFrançois Tigeot 
349c0e85e96SFrançois Tigeot 	dbg = __raw_i915_read32(dev_priv, FPGA_DBG);
350c0e85e96SFrançois Tigeot 	if (likely(!(dbg & FPGA_DBG_RM_NOCLAIM)))
351c0e85e96SFrançois Tigeot 		return false;
352c0e85e96SFrançois Tigeot 
353c0e85e96SFrançois Tigeot 	__raw_i915_write32(dev_priv, FPGA_DBG, FPGA_DBG_RM_NOCLAIM);
354c0e85e96SFrançois Tigeot 
355c0e85e96SFrançois Tigeot 	return true;
356c0e85e96SFrançois Tigeot }
357c0e85e96SFrançois Tigeot 
358c0e85e96SFrançois Tigeot static bool
vlv_check_for_unclaimed_mmio(struct drm_i915_private * dev_priv)359c0e85e96SFrançois Tigeot vlv_check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
360c0e85e96SFrançois Tigeot {
361c0e85e96SFrançois Tigeot 	u32 cer;
362c0e85e96SFrançois Tigeot 
363c0e85e96SFrançois Tigeot 	cer = __raw_i915_read32(dev_priv, CLAIM_ER);
364c0e85e96SFrançois Tigeot 	if (likely(!(cer & (CLAIM_ER_OVERFLOW | CLAIM_ER_CTR_MASK))))
365c0e85e96SFrançois Tigeot 		return false;
366c0e85e96SFrançois Tigeot 
367c0e85e96SFrançois Tigeot 	__raw_i915_write32(dev_priv, CLAIM_ER, CLAIM_ER_CLR);
368c0e85e96SFrançois Tigeot 
369c0e85e96SFrançois Tigeot 	return true;
370c0e85e96SFrançois Tigeot }
371c0e85e96SFrançois Tigeot 
372c0e85e96SFrançois Tigeot static bool
gen6_check_for_fifo_debug(struct drm_i915_private * dev_priv)373*3f2dd94aSFrançois Tigeot gen6_check_for_fifo_debug(struct drm_i915_private *dev_priv)
374*3f2dd94aSFrançois Tigeot {
375*3f2dd94aSFrançois Tigeot 	u32 fifodbg;
376*3f2dd94aSFrançois Tigeot 
377*3f2dd94aSFrançois Tigeot 	fifodbg = __raw_i915_read32(dev_priv, GTFIFODBG);
378*3f2dd94aSFrançois Tigeot 
379*3f2dd94aSFrançois Tigeot 	if (unlikely(fifodbg)) {
380*3f2dd94aSFrançois Tigeot 		DRM_DEBUG_DRIVER("GTFIFODBG = 0x08%x\n", fifodbg);
381*3f2dd94aSFrançois Tigeot 		__raw_i915_write32(dev_priv, GTFIFODBG, fifodbg);
382*3f2dd94aSFrançois Tigeot 	}
383*3f2dd94aSFrançois Tigeot 
384*3f2dd94aSFrançois Tigeot 	return fifodbg;
385*3f2dd94aSFrançois Tigeot }
386*3f2dd94aSFrançois Tigeot 
387*3f2dd94aSFrançois Tigeot static bool
check_for_unclaimed_mmio(struct drm_i915_private * dev_priv)388c0e85e96SFrançois Tigeot check_for_unclaimed_mmio(struct drm_i915_private *dev_priv)
389c0e85e96SFrançois Tigeot {
390*3f2dd94aSFrançois Tigeot 	bool ret = false;
391*3f2dd94aSFrançois Tigeot 
392c0e85e96SFrançois Tigeot 	if (HAS_FPGA_DBG_UNCLAIMED(dev_priv))
393*3f2dd94aSFrançois Tigeot 		ret |= fpga_check_for_unclaimed_mmio(dev_priv);
394c0e85e96SFrançois Tigeot 
395c0e85e96SFrançois Tigeot 	if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv))
396*3f2dd94aSFrançois Tigeot 		ret |= vlv_check_for_unclaimed_mmio(dev_priv);
397c0e85e96SFrançois Tigeot 
398*3f2dd94aSFrançois Tigeot 	if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv))
399*3f2dd94aSFrançois Tigeot 		ret |= gen6_check_for_fifo_debug(dev_priv);
400*3f2dd94aSFrançois Tigeot 
401*3f2dd94aSFrançois Tigeot 	return ret;
402c0e85e96SFrançois Tigeot }
403c0e85e96SFrançois Tigeot 
__intel_uncore_early_sanitize(struct drm_i915_private * dev_priv,bool restore_forcewake)4041487f786SFrançois Tigeot static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv,
4052c9916cdSFrançois Tigeot 					  bool restore_forcewake)
4062c9916cdSFrançois Tigeot {
407c0e85e96SFrançois Tigeot 	/* clear out unclaimed reg detection bit */
408c0e85e96SFrançois Tigeot 	if (check_for_unclaimed_mmio(dev_priv))
409c0e85e96SFrançois Tigeot 		DRM_DEBUG("unclaimed mmio detected on uncore init, clearing\n");
4109edbd4a0SFrançois Tigeot 
411477eb7f9SFrançois Tigeot 	/* WaDisableShadowRegForCpd:chv */
4121487f786SFrançois Tigeot 	if (IS_CHERRYVIEW(dev_priv)) {
413477eb7f9SFrançois Tigeot 		__raw_i915_write32(dev_priv, GTFIFOCTL,
414477eb7f9SFrançois Tigeot 				   __raw_i915_read32(dev_priv, GTFIFOCTL) |
415477eb7f9SFrançois Tigeot 				   GT_FIFO_CTL_BLOCK_ALL_POLICY_STALL |
416477eb7f9SFrançois Tigeot 				   GT_FIFO_CTL_RC6_POLICY_STALL);
417477eb7f9SFrançois Tigeot 	}
418477eb7f9SFrançois Tigeot 
4191487f786SFrançois Tigeot 	intel_uncore_forcewake_reset(dev_priv, restore_forcewake);
4209edbd4a0SFrançois Tigeot }
4219edbd4a0SFrançois Tigeot 
intel_uncore_suspend(struct drm_i915_private * dev_priv)422a85cb24fSFrançois Tigeot void intel_uncore_suspend(struct drm_i915_private *dev_priv)
4232c9916cdSFrançois Tigeot {
424a85cb24fSFrançois Tigeot 	iosf_mbi_unregister_pmic_bus_access_notifier(
425a85cb24fSFrançois Tigeot 		&dev_priv->uncore.pmic_bus_access_nb);
426a85cb24fSFrançois Tigeot 	intel_uncore_forcewake_reset(dev_priv, false);
427a85cb24fSFrançois Tigeot }
428a85cb24fSFrançois Tigeot 
intel_uncore_resume_early(struct drm_i915_private * dev_priv)429a85cb24fSFrançois Tigeot void intel_uncore_resume_early(struct drm_i915_private *dev_priv)
430a85cb24fSFrançois Tigeot {
431a85cb24fSFrançois Tigeot 	__intel_uncore_early_sanitize(dev_priv, true);
432a85cb24fSFrançois Tigeot 	iosf_mbi_register_pmic_bus_access_notifier(
433a85cb24fSFrançois Tigeot 		&dev_priv->uncore.pmic_bus_access_nb);
4341487f786SFrançois Tigeot 	i915_check_and_clear_faults(dev_priv);
4352c9916cdSFrançois Tigeot }
4362c9916cdSFrançois Tigeot 
intel_uncore_runtime_resume(struct drm_i915_private * dev_priv)437*3f2dd94aSFrançois Tigeot void intel_uncore_runtime_resume(struct drm_i915_private *dev_priv)
438*3f2dd94aSFrançois Tigeot {
439*3f2dd94aSFrançois Tigeot 	iosf_mbi_register_pmic_bus_access_notifier(
440*3f2dd94aSFrançois Tigeot 		&dev_priv->uncore.pmic_bus_access_nb);
441*3f2dd94aSFrançois Tigeot }
442*3f2dd94aSFrançois Tigeot 
intel_uncore_sanitize(struct drm_i915_private * dev_priv)4431487f786SFrançois Tigeot void intel_uncore_sanitize(struct drm_i915_private *dev_priv)
4449edbd4a0SFrançois Tigeot {
445*3f2dd94aSFrançois Tigeot 	i915_modparams.enable_rc6 =
446*3f2dd94aSFrançois Tigeot 		sanitize_rc6_option(dev_priv, i915_modparams.enable_rc6);
447c0e85e96SFrançois Tigeot 
4489edbd4a0SFrançois Tigeot 	/* BIOS often leaves RC6 enabled, but disable it for hw init */
44987df8fc6SFrançois Tigeot 	intel_sanitize_gt_powersave(dev_priv);
4509edbd4a0SFrançois Tigeot }
4519edbd4a0SFrançois Tigeot 
__intel_uncore_forcewake_get(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)45219c468b4SFrançois Tigeot static void __intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
45319c468b4SFrançois Tigeot 					 enum forcewake_domains fw_domains)
45419c468b4SFrançois Tigeot {
45519c468b4SFrançois Tigeot 	struct intel_uncore_forcewake_domain *domain;
456a85cb24fSFrançois Tigeot 	unsigned int tmp;
45719c468b4SFrançois Tigeot 
45819c468b4SFrançois Tigeot 	fw_domains &= dev_priv->uncore.fw_domains;
45919c468b4SFrançois Tigeot 
460*3f2dd94aSFrançois Tigeot 	for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) {
461*3f2dd94aSFrançois Tigeot 		if (domain->wake_count++) {
4628621f407SFrançois Tigeot 			fw_domains &= ~domain->mask;
463*3f2dd94aSFrançois Tigeot 			domain->active = true;
464*3f2dd94aSFrançois Tigeot 		}
465*3f2dd94aSFrançois Tigeot 	}
46619c468b4SFrançois Tigeot 
4674be47400SFrançois Tigeot 	if (fw_domains)
46819c468b4SFrançois Tigeot 		dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
46919c468b4SFrançois Tigeot }
47019c468b4SFrançois Tigeot 
4712c9916cdSFrançois Tigeot /**
4722c9916cdSFrançois Tigeot  * intel_uncore_forcewake_get - grab forcewake domain references
4732c9916cdSFrançois Tigeot  * @dev_priv: i915 device instance
4742c9916cdSFrançois Tigeot  * @fw_domains: forcewake domains to get reference on
4752c9916cdSFrançois Tigeot  *
4762c9916cdSFrançois Tigeot  * This function can be used get GT's forcewake domain references.
4772c9916cdSFrançois Tigeot  * Normal register access will handle the forcewake domains automatically.
4782c9916cdSFrançois Tigeot  * However if some sequence requires the GT to not power down a particular
4792c9916cdSFrançois Tigeot  * forcewake domains this function should be called at the beginning of the
4802c9916cdSFrançois Tigeot  * sequence. And subsequently the reference should be dropped by symmetric
4812c9916cdSFrançois Tigeot  * call to intel_unforce_forcewake_put(). Usually caller wants all the domains
4822c9916cdSFrançois Tigeot  * to be kept awake so the @fw_domains would be then FORCEWAKE_ALL.
4839edbd4a0SFrançois Tigeot  */
intel_uncore_forcewake_get(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)4842c9916cdSFrançois Tigeot void intel_uncore_forcewake_get(struct drm_i915_private *dev_priv,
4852c9916cdSFrançois Tigeot 				enum forcewake_domains fw_domains)
4869edbd4a0SFrançois Tigeot {
4875e269720SFrançois Tigeot 	unsigned long irqflags;
4882c9916cdSFrançois Tigeot 
4899edbd4a0SFrançois Tigeot 	if (!dev_priv->uncore.funcs.force_wake_get)
4909edbd4a0SFrançois Tigeot 		return;
4919edbd4a0SFrançois Tigeot 
492aee94f86SFrançois Tigeot 	assert_rpm_wakelock_held(dev_priv);
4939edbd4a0SFrançois Tigeot 
4945e269720SFrançois Tigeot 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
49519c468b4SFrançois Tigeot 	__intel_uncore_forcewake_get(dev_priv, fw_domains);
4965e269720SFrançois Tigeot 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
4972c9916cdSFrançois Tigeot }
4982c9916cdSFrançois Tigeot 
49919c468b4SFrançois Tigeot /**
500*3f2dd94aSFrançois Tigeot  * intel_uncore_forcewake_user_get - claim forcewake on behalf of userspace
501*3f2dd94aSFrançois Tigeot  * @dev_priv: i915 device instance
502*3f2dd94aSFrançois Tigeot  *
503*3f2dd94aSFrançois Tigeot  * This function is a wrapper around intel_uncore_forcewake_get() to acquire
504*3f2dd94aSFrançois Tigeot  * the GT powerwell and in the process disable our debugging for the
505*3f2dd94aSFrançois Tigeot  * duration of userspace's bypass.
506*3f2dd94aSFrançois Tigeot  */
intel_uncore_forcewake_user_get(struct drm_i915_private * dev_priv)507*3f2dd94aSFrançois Tigeot void intel_uncore_forcewake_user_get(struct drm_i915_private *dev_priv)
508*3f2dd94aSFrançois Tigeot {
509*3f2dd94aSFrançois Tigeot 	spin_lock_irq(&dev_priv->uncore.lock);
510*3f2dd94aSFrançois Tigeot 	if (!dev_priv->uncore.user_forcewake.count++) {
511*3f2dd94aSFrançois Tigeot 		intel_uncore_forcewake_get__locked(dev_priv, FORCEWAKE_ALL);
512*3f2dd94aSFrançois Tigeot 
513*3f2dd94aSFrançois Tigeot 		/* Save and disable mmio debugging for the user bypass */
514*3f2dd94aSFrançois Tigeot 		dev_priv->uncore.user_forcewake.saved_mmio_check =
515*3f2dd94aSFrançois Tigeot 			dev_priv->uncore.unclaimed_mmio_check;
516*3f2dd94aSFrançois Tigeot 		dev_priv->uncore.user_forcewake.saved_mmio_debug =
517*3f2dd94aSFrançois Tigeot 			i915_modparams.mmio_debug;
518*3f2dd94aSFrançois Tigeot 
519*3f2dd94aSFrançois Tigeot 		dev_priv->uncore.unclaimed_mmio_check = 0;
520*3f2dd94aSFrançois Tigeot 		i915_modparams.mmio_debug = 0;
521*3f2dd94aSFrançois Tigeot 	}
522*3f2dd94aSFrançois Tigeot 	spin_unlock_irq(&dev_priv->uncore.lock);
523*3f2dd94aSFrançois Tigeot }
524*3f2dd94aSFrançois Tigeot 
525*3f2dd94aSFrançois Tigeot /**
526*3f2dd94aSFrançois Tigeot  * intel_uncore_forcewake_user_put - release forcewake on behalf of userspace
527*3f2dd94aSFrançois Tigeot  * @dev_priv: i915 device instance
528*3f2dd94aSFrançois Tigeot  *
529*3f2dd94aSFrançois Tigeot  * This function complements intel_uncore_forcewake_user_get() and releases
530*3f2dd94aSFrançois Tigeot  * the GT powerwell taken on behalf of the userspace bypass.
531*3f2dd94aSFrançois Tigeot  */
intel_uncore_forcewake_user_put(struct drm_i915_private * dev_priv)532*3f2dd94aSFrançois Tigeot void intel_uncore_forcewake_user_put(struct drm_i915_private *dev_priv)
533*3f2dd94aSFrançois Tigeot {
534*3f2dd94aSFrançois Tigeot 	spin_lock_irq(&dev_priv->uncore.lock);
535*3f2dd94aSFrançois Tigeot 	if (!--dev_priv->uncore.user_forcewake.count) {
536*3f2dd94aSFrançois Tigeot 		if (intel_uncore_unclaimed_mmio(dev_priv))
537*3f2dd94aSFrançois Tigeot 			dev_info(dev_priv->drm.dev,
538*3f2dd94aSFrançois Tigeot 				 "Invalid mmio detected during user access\n");
539*3f2dd94aSFrançois Tigeot 
540*3f2dd94aSFrançois Tigeot 		dev_priv->uncore.unclaimed_mmio_check =
541*3f2dd94aSFrançois Tigeot 			dev_priv->uncore.user_forcewake.saved_mmio_check;
542*3f2dd94aSFrançois Tigeot 		i915_modparams.mmio_debug =
543*3f2dd94aSFrançois Tigeot 			dev_priv->uncore.user_forcewake.saved_mmio_debug;
544*3f2dd94aSFrançois Tigeot 
545*3f2dd94aSFrançois Tigeot 		intel_uncore_forcewake_put__locked(dev_priv, FORCEWAKE_ALL);
546*3f2dd94aSFrançois Tigeot 	}
547*3f2dd94aSFrançois Tigeot 	spin_unlock_irq(&dev_priv->uncore.lock);
548*3f2dd94aSFrançois Tigeot }
549*3f2dd94aSFrançois Tigeot 
550*3f2dd94aSFrançois Tigeot /**
55119c468b4SFrançois Tigeot  * intel_uncore_forcewake_get__locked - grab forcewake domain references
55219c468b4SFrançois Tigeot  * @dev_priv: i915 device instance
55319c468b4SFrançois Tigeot  * @fw_domains: forcewake domains to get reference on
55419c468b4SFrançois Tigeot  *
55519c468b4SFrançois Tigeot  * See intel_uncore_forcewake_get(). This variant places the onus
55619c468b4SFrançois Tigeot  * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
55719c468b4SFrançois Tigeot  */
intel_uncore_forcewake_get__locked(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)55819c468b4SFrançois Tigeot void intel_uncore_forcewake_get__locked(struct drm_i915_private *dev_priv,
55919c468b4SFrançois Tigeot 					enum forcewake_domains fw_domains)
56019c468b4SFrançois Tigeot {
561a85cb24fSFrançois Tigeot 	lockdep_assert_held(&dev_priv->uncore.lock);
5622c9916cdSFrançois Tigeot 
56319c468b4SFrançois Tigeot 	if (!dev_priv->uncore.funcs.force_wake_get)
56419c468b4SFrançois Tigeot 		return;
56519c468b4SFrançois Tigeot 
56619c468b4SFrançois Tigeot 	__intel_uncore_forcewake_get(dev_priv, fw_domains);
56719c468b4SFrançois Tigeot }
56819c468b4SFrançois Tigeot 
__intel_uncore_forcewake_put(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)56919c468b4SFrançois Tigeot static void __intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
57019c468b4SFrançois Tigeot 					 enum forcewake_domains fw_domains)
57119c468b4SFrançois Tigeot {
57219c468b4SFrançois Tigeot 	struct intel_uncore_forcewake_domain *domain;
573a85cb24fSFrançois Tigeot 	unsigned int tmp;
57419c468b4SFrançois Tigeot 
57519c468b4SFrançois Tigeot 	fw_domains &= dev_priv->uncore.fw_domains;
57619c468b4SFrançois Tigeot 
577a85cb24fSFrançois Tigeot 	for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp) {
57819c468b4SFrançois Tigeot 		if (WARN_ON(domain->wake_count == 0))
57919c468b4SFrançois Tigeot 			continue;
58019c468b4SFrançois Tigeot 
581*3f2dd94aSFrançois Tigeot 		if (--domain->wake_count) {
582*3f2dd94aSFrançois Tigeot 			domain->active = true;
58319c468b4SFrançois Tigeot 			continue;
584*3f2dd94aSFrançois Tigeot 		}
58519c468b4SFrançois Tigeot 
58619c468b4SFrançois Tigeot 		fw_domain_arm_timer(domain);
58719c468b4SFrançois Tigeot 	}
5889edbd4a0SFrançois Tigeot }
5899edbd4a0SFrançois Tigeot 
5902c9916cdSFrançois Tigeot /**
5912c9916cdSFrançois Tigeot  * intel_uncore_forcewake_put - release a forcewake domain reference
5922c9916cdSFrançois Tigeot  * @dev_priv: i915 device instance
5932c9916cdSFrançois Tigeot  * @fw_domains: forcewake domains to put references
5942c9916cdSFrançois Tigeot  *
5952c9916cdSFrançois Tigeot  * This function drops the device-level forcewakes for specified
5962c9916cdSFrançois Tigeot  * domains obtained by intel_uncore_forcewake_get().
5979edbd4a0SFrançois Tigeot  */
intel_uncore_forcewake_put(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)5982c9916cdSFrançois Tigeot void intel_uncore_forcewake_put(struct drm_i915_private *dev_priv,
5992c9916cdSFrançois Tigeot 				enum forcewake_domains fw_domains)
6009edbd4a0SFrançois Tigeot {
6014db2f768SFrançois Tigeot 	unsigned long irqflags;
602ba55f2f5SFrançois Tigeot 
6039edbd4a0SFrançois Tigeot 	if (!dev_priv->uncore.funcs.force_wake_put)
6049edbd4a0SFrançois Tigeot 		return;
6059edbd4a0SFrançois Tigeot 
6064db2f768SFrançois Tigeot 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags);
60719c468b4SFrançois Tigeot 	__intel_uncore_forcewake_put(dev_priv, fw_domains);
6084db2f768SFrançois Tigeot 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags);
6099edbd4a0SFrançois Tigeot }
6102c9916cdSFrançois Tigeot 
61119c468b4SFrançois Tigeot /**
61219c468b4SFrançois Tigeot  * intel_uncore_forcewake_put__locked - grab forcewake domain references
61319c468b4SFrançois Tigeot  * @dev_priv: i915 device instance
61419c468b4SFrançois Tigeot  * @fw_domains: forcewake domains to get reference on
61519c468b4SFrançois Tigeot  *
61619c468b4SFrançois Tigeot  * See intel_uncore_forcewake_put(). This variant places the onus
61719c468b4SFrançois Tigeot  * on the caller to explicitly handle the dev_priv->uncore.lock spinlock.
61819c468b4SFrançois Tigeot  */
intel_uncore_forcewake_put__locked(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)61919c468b4SFrançois Tigeot void intel_uncore_forcewake_put__locked(struct drm_i915_private *dev_priv,
62019c468b4SFrançois Tigeot 					enum forcewake_domains fw_domains)
62119c468b4SFrançois Tigeot {
622a85cb24fSFrançois Tigeot 	lockdep_assert_held(&dev_priv->uncore.lock);
62319c468b4SFrançois Tigeot 
62419c468b4SFrançois Tigeot 	if (!dev_priv->uncore.funcs.force_wake_put)
62519c468b4SFrançois Tigeot 		return;
62619c468b4SFrançois Tigeot 
62719c468b4SFrançois Tigeot 	__intel_uncore_forcewake_put(dev_priv, fw_domains);
6289edbd4a0SFrançois Tigeot }
6299edbd4a0SFrançois Tigeot 
assert_forcewakes_inactive(struct drm_i915_private * dev_priv)6302c9916cdSFrançois Tigeot void assert_forcewakes_inactive(struct drm_i915_private *dev_priv)
631ba55f2f5SFrançois Tigeot {
632ba55f2f5SFrançois Tigeot 	if (!dev_priv->uncore.funcs.force_wake_get)
633ba55f2f5SFrançois Tigeot 		return;
634ba55f2f5SFrançois Tigeot 
635*3f2dd94aSFrançois Tigeot 	WARN(dev_priv->uncore.fw_domains_active,
636*3f2dd94aSFrançois Tigeot 	     "Expected all fw_domains to be inactive, but %08x are still on\n",
637*3f2dd94aSFrançois Tigeot 	     dev_priv->uncore.fw_domains_active);
638*3f2dd94aSFrançois Tigeot }
639*3f2dd94aSFrançois Tigeot 
assert_forcewakes_active(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)640*3f2dd94aSFrançois Tigeot void assert_forcewakes_active(struct drm_i915_private *dev_priv,
641*3f2dd94aSFrançois Tigeot 			      enum forcewake_domains fw_domains)
642*3f2dd94aSFrançois Tigeot {
643*3f2dd94aSFrançois Tigeot 	if (!dev_priv->uncore.funcs.force_wake_get)
644*3f2dd94aSFrançois Tigeot 		return;
645*3f2dd94aSFrançois Tigeot 
646*3f2dd94aSFrançois Tigeot 	assert_rpm_wakelock_held(dev_priv);
647*3f2dd94aSFrançois Tigeot 
648*3f2dd94aSFrançois Tigeot 	fw_domains &= dev_priv->uncore.fw_domains;
649*3f2dd94aSFrançois Tigeot 	WARN(fw_domains & ~dev_priv->uncore.fw_domains_active,
650*3f2dd94aSFrançois Tigeot 	     "Expected %08x fw_domains to be active, but %08x are off\n",
651*3f2dd94aSFrançois Tigeot 	     fw_domains, fw_domains & ~dev_priv->uncore.fw_domains_active);
652ba55f2f5SFrançois Tigeot }
653ba55f2f5SFrançois Tigeot 
6549edbd4a0SFrançois Tigeot /* We give fast paths for the really cool registers */
655aee94f86SFrançois Tigeot #define NEEDS_FORCE_WAKE(reg) ((reg) < 0x40000)
6569edbd4a0SFrançois Tigeot 
6578621f407SFrançois Tigeot #define __gen6_reg_read_fw_domains(offset) \
6588621f407SFrançois Tigeot ({ \
6598621f407SFrançois Tigeot 	enum forcewake_domains __fwd; \
6608621f407SFrançois Tigeot 	if (NEEDS_FORCE_WAKE(offset)) \
6618621f407SFrançois Tigeot 		__fwd = FORCEWAKE_RENDER; \
6628621f407SFrançois Tigeot 	else \
6638621f407SFrançois Tigeot 		__fwd = 0; \
6648621f407SFrançois Tigeot 	__fwd; \
6658621f407SFrançois Tigeot })
6668621f407SFrançois Tigeot 
fw_range_cmp(u32 offset,const struct intel_forcewake_range * entry)6671e12ee3bSFrançois Tigeot static int fw_range_cmp(u32 offset, const struct intel_forcewake_range *entry)
6681e12ee3bSFrançois Tigeot {
6691e12ee3bSFrançois Tigeot 	if (offset < entry->start)
6701e12ee3bSFrançois Tigeot 		return -1;
6711e12ee3bSFrançois Tigeot 	else if (offset > entry->end)
6721e12ee3bSFrançois Tigeot 		return 1;
6731e12ee3bSFrançois Tigeot 	else
6741e12ee3bSFrançois Tigeot 		return 0;
6751e12ee3bSFrançois Tigeot }
67624edb884SFrançois Tigeot 
6771e12ee3bSFrançois Tigeot /* Copied and "macroized" from lib/bsearch.c */
6781e12ee3bSFrançois Tigeot #define BSEARCH(key, base, num, cmp) ({                                 \
6791e12ee3bSFrançois Tigeot 	unsigned int start__ = 0, end__ = (num);                        \
6801e12ee3bSFrançois Tigeot 	typeof(base) result__ = NULL;                                   \
6811e12ee3bSFrançois Tigeot 	while (start__ < end__) {                                       \
6821e12ee3bSFrançois Tigeot 		unsigned int mid__ = start__ + (end__ - start__) / 2;   \
6831e12ee3bSFrançois Tigeot 		int ret__ = (cmp)((key), (base) + mid__);               \
6841e12ee3bSFrançois Tigeot 		if (ret__ < 0) {                                        \
6851e12ee3bSFrançois Tigeot 			end__ = mid__;                                  \
6861e12ee3bSFrançois Tigeot 		} else if (ret__ > 0) {                                 \
6871e12ee3bSFrançois Tigeot 			start__ = mid__ + 1;                            \
6881e12ee3bSFrançois Tigeot 		} else {                                                \
6891e12ee3bSFrançois Tigeot 			result__ = (base) + mid__;                      \
6901e12ee3bSFrançois Tigeot 			break;                                          \
6911e12ee3bSFrançois Tigeot 		}                                                       \
6921e12ee3bSFrançois Tigeot 	}                                                               \
6931e12ee3bSFrançois Tigeot 	result__;                                                       \
6941e12ee3bSFrançois Tigeot })
695ba55f2f5SFrançois Tigeot 
6961e12ee3bSFrançois Tigeot static enum forcewake_domains
find_fw_domain(struct drm_i915_private * dev_priv,u32 offset)6971e12ee3bSFrançois Tigeot find_fw_domain(struct drm_i915_private *dev_priv, u32 offset)
6981e12ee3bSFrançois Tigeot {
6991e12ee3bSFrançois Tigeot 	const struct intel_forcewake_range *entry;
70024edb884SFrançois Tigeot 
7011e12ee3bSFrançois Tigeot 	entry = BSEARCH(offset,
7021e12ee3bSFrançois Tigeot 			dev_priv->uncore.fw_domains_table,
7031e12ee3bSFrançois Tigeot 			dev_priv->uncore.fw_domains_table_entries,
7041e12ee3bSFrançois Tigeot 			fw_range_cmp);
7051e12ee3bSFrançois Tigeot 
706a85cb24fSFrançois Tigeot 	if (!entry)
707a85cb24fSFrançois Tigeot 		return 0;
7081e12ee3bSFrançois Tigeot 
709a85cb24fSFrançois Tigeot 	WARN(entry->domains & ~dev_priv->uncore.fw_domains,
710a85cb24fSFrançois Tigeot 	     "Uninitialized forcewake domain(s) 0x%x accessed at 0x%x\n",
711a85cb24fSFrançois Tigeot 	     entry->domains & ~dev_priv->uncore.fw_domains, offset);
7121e12ee3bSFrançois Tigeot 
713a85cb24fSFrançois Tigeot 	return entry->domains;
7141e12ee3bSFrançois Tigeot }
7151e12ee3bSFrançois Tigeot 
7161e12ee3bSFrançois Tigeot #define GEN_FW_RANGE(s, e, d) \
7171e12ee3bSFrançois Tigeot 	{ .start = (s), .end = (e), .domains = (d) }
7181e12ee3bSFrançois Tigeot 
7191e12ee3bSFrançois Tigeot #define HAS_FWTABLE(dev_priv) \
720*3f2dd94aSFrançois Tigeot 	(INTEL_GEN(dev_priv) >= 9 || \
7211e12ee3bSFrançois Tigeot 	 IS_CHERRYVIEW(dev_priv) || \
7221e12ee3bSFrançois Tigeot 	 IS_VALLEYVIEW(dev_priv))
7231e12ee3bSFrançois Tigeot 
7241e12ee3bSFrançois Tigeot /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
7251e12ee3bSFrançois Tigeot static const struct intel_forcewake_range __vlv_fw_ranges[] = {
7261e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
7271e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x5000, 0x7fff, FORCEWAKE_RENDER),
7281e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0xb000, 0x11fff, FORCEWAKE_RENDER),
7291e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
7301e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x22000, 0x23fff, FORCEWAKE_MEDIA),
7311e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x2e000, 0x2ffff, FORCEWAKE_RENDER),
7321e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
7331e12ee3bSFrançois Tigeot };
7341e12ee3bSFrançois Tigeot 
7351e12ee3bSFrançois Tigeot #define __fwtable_reg_read_fw_domains(offset) \
7368621f407SFrançois Tigeot ({ \
7378621f407SFrançois Tigeot 	enum forcewake_domains __fwd = 0; \
7381e12ee3bSFrançois Tigeot 	if (NEEDS_FORCE_WAKE((offset))) \
7391e12ee3bSFrançois Tigeot 		__fwd = find_fw_domain(dev_priv, offset); \
7408621f407SFrançois Tigeot 	__fwd; \
7418621f407SFrançois Tigeot })
7428621f407SFrançois Tigeot 
7431e12ee3bSFrançois Tigeot /* *Must* be sorted by offset! See intel_shadow_table_check(). */
7448621f407SFrançois Tigeot static const i915_reg_t gen8_shadowed_regs[] = {
7451e12ee3bSFrançois Tigeot 	RING_TAIL(RENDER_RING_BASE),	/* 0x2000 (base) */
7461e12ee3bSFrançois Tigeot 	GEN6_RPNSWREQ,			/* 0xA008 */
7471e12ee3bSFrançois Tigeot 	GEN6_RC_VIDEO_FREQ,		/* 0xA00C */
7481e12ee3bSFrançois Tigeot 	RING_TAIL(GEN6_BSD_RING_BASE),	/* 0x12000 (base) */
7491e12ee3bSFrançois Tigeot 	RING_TAIL(VEBOX_RING_BASE),	/* 0x1a000 (base) */
7501e12ee3bSFrançois Tigeot 	RING_TAIL(BLT_RING_BASE),	/* 0x22000 (base) */
7518621f407SFrançois Tigeot 	/* TODO: Other registers are not yet used */
7528621f407SFrançois Tigeot };
7538621f407SFrançois Tigeot 
mmio_reg_cmp(u32 key,const i915_reg_t * reg)7541e12ee3bSFrançois Tigeot static int mmio_reg_cmp(u32 key, const i915_reg_t *reg)
7551e12ee3bSFrançois Tigeot {
7561e12ee3bSFrançois Tigeot 	u32 offset = i915_mmio_reg_offset(*reg);
7571e12ee3bSFrançois Tigeot 
7581e12ee3bSFrançois Tigeot 	if (key < offset)
7591e12ee3bSFrançois Tigeot 		return -1;
7601e12ee3bSFrançois Tigeot 	else if (key > offset)
7611e12ee3bSFrançois Tigeot 		return 1;
7621e12ee3bSFrançois Tigeot 	else
7631e12ee3bSFrançois Tigeot 		return 0;
7641e12ee3bSFrançois Tigeot }
7651e12ee3bSFrançois Tigeot 
is_gen8_shadowed(u32 offset)7668621f407SFrançois Tigeot static bool is_gen8_shadowed(u32 offset)
7678621f407SFrançois Tigeot {
7681e12ee3bSFrançois Tigeot 	const i915_reg_t *regs = gen8_shadowed_regs;
7698621f407SFrançois Tigeot 
7701e12ee3bSFrançois Tigeot 	return BSEARCH(offset, regs, ARRAY_SIZE(gen8_shadowed_regs),
7711e12ee3bSFrançois Tigeot 		       mmio_reg_cmp);
7728621f407SFrançois Tigeot }
7738621f407SFrançois Tigeot 
7748621f407SFrançois Tigeot #define __gen8_reg_write_fw_domains(offset) \
7758621f407SFrançois Tigeot ({ \
7768621f407SFrançois Tigeot 	enum forcewake_domains __fwd; \
7778621f407SFrançois Tigeot 	if (NEEDS_FORCE_WAKE(offset) && !is_gen8_shadowed(offset)) \
7788621f407SFrançois Tigeot 		__fwd = FORCEWAKE_RENDER; \
7798621f407SFrançois Tigeot 	else \
7808621f407SFrançois Tigeot 		__fwd = 0; \
7818621f407SFrançois Tigeot 	__fwd; \
7828621f407SFrançois Tigeot })
7838621f407SFrançois Tigeot 
7841e12ee3bSFrançois Tigeot /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
7851e12ee3bSFrançois Tigeot static const struct intel_forcewake_range __chv_fw_ranges[] = {
7861e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x2000, 0x3fff, FORCEWAKE_RENDER),
7871e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x4000, 0x4fff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
7881e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
7891e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x8000, 0x82ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
7901e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
7911e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x8500, 0x85ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
7921e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x8800, 0x88ff, FORCEWAKE_MEDIA),
7931e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x9000, 0xafff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
7941e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
7951e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
7961e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0xe000, 0xe7ff, FORCEWAKE_RENDER),
7971e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0xf000, 0xffff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
7981e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
7991e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x1a000, 0x1bfff, FORCEWAKE_MEDIA),
8001e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x1e800, 0x1e9ff, FORCEWAKE_MEDIA),
8011e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x30000, 0x37fff, FORCEWAKE_MEDIA),
8028621f407SFrançois Tigeot };
8038621f407SFrançois Tigeot 
8041e12ee3bSFrançois Tigeot #define __fwtable_reg_write_fw_domains(offset) \
8058621f407SFrançois Tigeot ({ \
8061e12ee3bSFrançois Tigeot 	enum forcewake_domains __fwd = 0; \
8071e12ee3bSFrançois Tigeot 	if (NEEDS_FORCE_WAKE((offset)) && !is_gen8_shadowed(offset)) \
8081e12ee3bSFrançois Tigeot 		__fwd = find_fw_domain(dev_priv, offset); \
8098621f407SFrançois Tigeot 	__fwd; \
8108621f407SFrançois Tigeot })
8118621f407SFrançois Tigeot 
8121e12ee3bSFrançois Tigeot /* *Must* be sorted by offset ranges! See intel_fw_table_check(). */
8131e12ee3bSFrançois Tigeot static const struct intel_forcewake_range __gen9_fw_ranges[] = {
8141e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x0, 0xaff, FORCEWAKE_BLITTER),
8151e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0xb00, 0x1fff, 0), /* uncore range */
8161e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x2000, 0x26ff, FORCEWAKE_RENDER),
8171e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x2700, 0x2fff, FORCEWAKE_BLITTER),
8181e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x3000, 0x3fff, FORCEWAKE_RENDER),
8191e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x4000, 0x51ff, FORCEWAKE_BLITTER),
8201e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x5200, 0x7fff, FORCEWAKE_RENDER),
8211e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x8000, 0x812f, FORCEWAKE_BLITTER),
8221e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x8130, 0x813f, FORCEWAKE_MEDIA),
8231e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x8140, 0x815f, FORCEWAKE_RENDER),
8241e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x8160, 0x82ff, FORCEWAKE_BLITTER),
8251e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x8300, 0x84ff, FORCEWAKE_RENDER),
8261e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x8500, 0x87ff, FORCEWAKE_BLITTER),
8271e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x8800, 0x89ff, FORCEWAKE_MEDIA),
8281e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x8a00, 0x8bff, FORCEWAKE_BLITTER),
8291e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x8c00, 0x8cff, FORCEWAKE_RENDER),
8301e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x8d00, 0x93ff, FORCEWAKE_BLITTER),
8311e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x9400, 0x97ff, FORCEWAKE_RENDER | FORCEWAKE_MEDIA),
8321e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x9800, 0xafff, FORCEWAKE_BLITTER),
8331e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0xb000, 0xb47f, FORCEWAKE_RENDER),
8344be47400SFrançois Tigeot 	GEN_FW_RANGE(0xb480, 0xcfff, FORCEWAKE_BLITTER),
8351e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0xd000, 0xd7ff, FORCEWAKE_MEDIA),
8361e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0xd800, 0xdfff, FORCEWAKE_BLITTER),
8371e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0xe000, 0xe8ff, FORCEWAKE_RENDER),
8381e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0xe900, 0x11fff, FORCEWAKE_BLITTER),
8391e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x12000, 0x13fff, FORCEWAKE_MEDIA),
8401e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x14000, 0x19fff, FORCEWAKE_BLITTER),
8411e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x1a000, 0x1e9ff, FORCEWAKE_MEDIA),
8421e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x1ea00, 0x243ff, FORCEWAKE_BLITTER),
8431e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x24400, 0x247ff, FORCEWAKE_RENDER),
8441e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x24800, 0x2ffff, FORCEWAKE_BLITTER),
8451e12ee3bSFrançois Tigeot 	GEN_FW_RANGE(0x30000, 0x3ffff, FORCEWAKE_MEDIA),
8461e12ee3bSFrançois Tigeot };
8471e12ee3bSFrançois Tigeot 
8489edbd4a0SFrançois Tigeot static void
ilk_dummy_write(struct drm_i915_private * dev_priv)8499edbd4a0SFrançois Tigeot ilk_dummy_write(struct drm_i915_private *dev_priv)
8509edbd4a0SFrançois Tigeot {
8519edbd4a0SFrançois Tigeot 	/* WaIssueDummyWriteToWakeupFromRC6:ilk Issue a dummy write to wake up
8529edbd4a0SFrançois Tigeot 	 * the chip from rc6 before touching it for real. MI_MODE is masked,
8539edbd4a0SFrançois Tigeot 	 * hence harmless to write 0 into. */
8549edbd4a0SFrançois Tigeot 	__raw_i915_write32(dev_priv, MI_MODE, 0);
8559edbd4a0SFrançois Tigeot }
8569edbd4a0SFrançois Tigeot 
8579edbd4a0SFrançois Tigeot static void
__unclaimed_reg_debug(struct drm_i915_private * dev_priv,const i915_reg_t reg,const bool read,const bool before)858c0e85e96SFrançois Tigeot __unclaimed_reg_debug(struct drm_i915_private *dev_priv,
859c0e85e96SFrançois Tigeot 		      const i915_reg_t reg,
860c0e85e96SFrançois Tigeot 		      const bool read,
861c0e85e96SFrançois Tigeot 		      const bool before)
8629edbd4a0SFrançois Tigeot {
8631e12ee3bSFrançois Tigeot 	if (WARN(check_for_unclaimed_mmio(dev_priv) && !before,
8641e12ee3bSFrançois Tigeot 		 "Unclaimed %s register 0x%x\n",
8651e12ee3bSFrançois Tigeot 		 read ? "read from" : "write to",
866c0e85e96SFrançois Tigeot 		 i915_mmio_reg_offset(reg)))
867*3f2dd94aSFrançois Tigeot 		/* Only report the first N failures */
868*3f2dd94aSFrançois Tigeot 		i915_modparams.mmio_debug--;
8699edbd4a0SFrançois Tigeot }
8709edbd4a0SFrançois Tigeot 
871c0e85e96SFrançois Tigeot static inline void
unclaimed_reg_debug(struct drm_i915_private * dev_priv,const i915_reg_t reg,const bool read,const bool before)872c0e85e96SFrançois Tigeot unclaimed_reg_debug(struct drm_i915_private *dev_priv,
873c0e85e96SFrançois Tigeot 		    const i915_reg_t reg,
874c0e85e96SFrançois Tigeot 		    const bool read,
875c0e85e96SFrançois Tigeot 		    const bool before)
8769edbd4a0SFrançois Tigeot {
877*3f2dd94aSFrançois Tigeot 	if (likely(!i915_modparams.mmio_debug))
87824edb884SFrançois Tigeot 		return;
87924edb884SFrançois Tigeot 
880c0e85e96SFrançois Tigeot 	__unclaimed_reg_debug(dev_priv, reg, read, before);
8819edbd4a0SFrançois Tigeot }
8829edbd4a0SFrançois Tigeot 
8832c9916cdSFrançois Tigeot #define GEN2_READ_HEADER(x) \
8849edbd4a0SFrançois Tigeot 	u##x val = 0; \
885aee94f86SFrançois Tigeot 	assert_rpm_wakelock_held(dev_priv);
8869edbd4a0SFrançois Tigeot 
8872c9916cdSFrançois Tigeot #define GEN2_READ_FOOTER \
8889edbd4a0SFrançois Tigeot 	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
8899edbd4a0SFrançois Tigeot 	return val
8909edbd4a0SFrançois Tigeot 
8912c9916cdSFrançois Tigeot #define __gen2_read(x) \
8929edbd4a0SFrançois Tigeot static u##x \
893aee94f86SFrançois Tigeot gen2_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
8942c9916cdSFrançois Tigeot 	GEN2_READ_HEADER(x); \
8959edbd4a0SFrançois Tigeot 	val = __raw_i915_read##x(dev_priv, reg); \
8962c9916cdSFrançois Tigeot 	GEN2_READ_FOOTER; \
8979edbd4a0SFrançois Tigeot }
8989edbd4a0SFrançois Tigeot 
8999edbd4a0SFrançois Tigeot #define __gen5_read(x) \
9009edbd4a0SFrançois Tigeot static u##x \
901aee94f86SFrançois Tigeot gen5_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
9022c9916cdSFrançois Tigeot 	GEN2_READ_HEADER(x); \
9039edbd4a0SFrançois Tigeot 	ilk_dummy_write(dev_priv); \
9049edbd4a0SFrançois Tigeot 	val = __raw_i915_read##x(dev_priv, reg); \
9052c9916cdSFrançois Tigeot 	GEN2_READ_FOOTER; \
9062c9916cdSFrançois Tigeot }
9072c9916cdSFrançois Tigeot 
9082c9916cdSFrançois Tigeot __gen5_read(8)
9092c9916cdSFrançois Tigeot __gen5_read(16)
9102c9916cdSFrançois Tigeot __gen5_read(32)
9112c9916cdSFrançois Tigeot __gen5_read(64)
9122c9916cdSFrançois Tigeot __gen2_read(8)
9132c9916cdSFrançois Tigeot __gen2_read(16)
9142c9916cdSFrançois Tigeot __gen2_read(32)
9152c9916cdSFrançois Tigeot __gen2_read(64)
9162c9916cdSFrançois Tigeot 
9172c9916cdSFrançois Tigeot #undef __gen5_read
9182c9916cdSFrançois Tigeot #undef __gen2_read
9192c9916cdSFrançois Tigeot 
9202c9916cdSFrançois Tigeot #undef GEN2_READ_FOOTER
9212c9916cdSFrançois Tigeot #undef GEN2_READ_HEADER
9222c9916cdSFrançois Tigeot 
9232c9916cdSFrançois Tigeot #define GEN6_READ_HEADER(x) \
924aee94f86SFrançois Tigeot 	u32 offset = i915_mmio_reg_offset(reg); \
9255e269720SFrançois Tigeot 	unsigned long irqflags; \
9262c9916cdSFrançois Tigeot 	u##x val = 0; \
927aee94f86SFrançois Tigeot 	assert_rpm_wakelock_held(dev_priv); \
928c0e85e96SFrançois Tigeot 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
929c0e85e96SFrançois Tigeot 	unclaimed_reg_debug(dev_priv, reg, true, true)
9302c9916cdSFrançois Tigeot 
9312c9916cdSFrançois Tigeot #define GEN6_READ_FOOTER \
932c0e85e96SFrançois Tigeot 	unclaimed_reg_debug(dev_priv, reg, true, false); \
9335e269720SFrançois Tigeot 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); \
9342c9916cdSFrançois Tigeot 	trace_i915_reg_rw(false, reg, val, sizeof(val), trace); \
9352c9916cdSFrançois Tigeot 	return val
9362c9916cdSFrançois Tigeot 
___force_wake_auto(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)9371e12ee3bSFrançois Tigeot static noinline void ___force_wake_auto(struct drm_i915_private *dev_priv,
9382c9916cdSFrançois Tigeot 					enum forcewake_domains fw_domains)
9392c9916cdSFrançois Tigeot {
9402c9916cdSFrançois Tigeot 	struct intel_uncore_forcewake_domain *domain;
941a85cb24fSFrançois Tigeot 	unsigned int tmp;
9422c9916cdSFrançois Tigeot 
943a85cb24fSFrançois Tigeot 	GEM_BUG_ON(fw_domains & ~dev_priv->uncore.fw_domains);
944a85cb24fSFrançois Tigeot 
945a85cb24fSFrançois Tigeot 	for_each_fw_domain_masked(domain, fw_domains, dev_priv, tmp)
9461e12ee3bSFrançois Tigeot 		fw_domain_arm_timer(domain);
9471e12ee3bSFrançois Tigeot 
9481e12ee3bSFrançois Tigeot 	dev_priv->uncore.funcs.force_wake_get(dev_priv, fw_domains);
9491e12ee3bSFrançois Tigeot }
9501e12ee3bSFrançois Tigeot 
__force_wake_auto(struct drm_i915_private * dev_priv,enum forcewake_domains fw_domains)9511e12ee3bSFrançois Tigeot static inline void __force_wake_auto(struct drm_i915_private *dev_priv,
9521e12ee3bSFrançois Tigeot 				     enum forcewake_domains fw_domains)
9531e12ee3bSFrançois Tigeot {
9542c9916cdSFrançois Tigeot 	if (WARN_ON(!fw_domains))
9552c9916cdSFrançois Tigeot 		return;
9562c9916cdSFrançois Tigeot 
9571e12ee3bSFrançois Tigeot 	/* Turn on all requested but inactive supported forcewake domains. */
9581e12ee3bSFrançois Tigeot 	fw_domains &= dev_priv->uncore.fw_domains;
9591e12ee3bSFrançois Tigeot 	fw_domains &= ~dev_priv->uncore.fw_domains_active;
9602c9916cdSFrançois Tigeot 
9612c9916cdSFrançois Tigeot 	if (fw_domains)
9621e12ee3bSFrançois Tigeot 		___force_wake_auto(dev_priv, fw_domains);
9639edbd4a0SFrançois Tigeot }
9649edbd4a0SFrançois Tigeot 
965a85cb24fSFrançois Tigeot #define __gen_read(func, x) \
9669edbd4a0SFrançois Tigeot static u##x \
967a85cb24fSFrançois Tigeot func##_read##x(struct drm_i915_private *dev_priv, i915_reg_t reg, bool trace) { \
9688621f407SFrançois Tigeot 	enum forcewake_domains fw_engine; \
9692c9916cdSFrançois Tigeot 	GEN6_READ_HEADER(x); \
970a85cb24fSFrançois Tigeot 	fw_engine = __##func##_reg_read_fw_domains(offset); \
9718621f407SFrançois Tigeot 	if (fw_engine) \
9728621f407SFrançois Tigeot 		__force_wake_auto(dev_priv, fw_engine); \
9739edbd4a0SFrançois Tigeot 	val = __raw_i915_read##x(dev_priv, reg); \
9742c9916cdSFrançois Tigeot 	GEN6_READ_FOOTER; \
9759edbd4a0SFrançois Tigeot }
976a85cb24fSFrançois Tigeot #define __gen6_read(x) __gen_read(gen6, x)
977a85cb24fSFrançois Tigeot #define __fwtable_read(x) __gen_read(fwtable, x)
9789edbd4a0SFrançois Tigeot 
9791e12ee3bSFrançois Tigeot __fwtable_read(8)
9801e12ee3bSFrançois Tigeot __fwtable_read(16)
9811e12ee3bSFrançois Tigeot __fwtable_read(32)
9821e12ee3bSFrançois Tigeot __fwtable_read(64)
9839edbd4a0SFrançois Tigeot __gen6_read(8)
9849edbd4a0SFrançois Tigeot __gen6_read(16)
9859edbd4a0SFrançois Tigeot __gen6_read(32)
9869edbd4a0SFrançois Tigeot __gen6_read(64)
9879edbd4a0SFrançois Tigeot 
9881e12ee3bSFrançois Tigeot #undef __fwtable_read
9899edbd4a0SFrançois Tigeot #undef __gen6_read
9902c9916cdSFrançois Tigeot #undef GEN6_READ_FOOTER
9912c9916cdSFrançois Tigeot #undef GEN6_READ_HEADER
9929edbd4a0SFrançois Tigeot 
9932c9916cdSFrançois Tigeot #define GEN2_WRITE_HEADER \
9949edbd4a0SFrançois Tigeot 	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
995aee94f86SFrançois Tigeot 	assert_rpm_wakelock_held(dev_priv); \
9969edbd4a0SFrançois Tigeot 
9972c9916cdSFrançois Tigeot #define GEN2_WRITE_FOOTER
9989edbd4a0SFrançois Tigeot 
9992c9916cdSFrançois Tigeot #define __gen2_write(x) \
10009edbd4a0SFrançois Tigeot static void \
1001aee94f86SFrançois Tigeot gen2_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
10022c9916cdSFrançois Tigeot 	GEN2_WRITE_HEADER; \
10039edbd4a0SFrançois Tigeot 	__raw_i915_write##x(dev_priv, reg, val); \
10042c9916cdSFrançois Tigeot 	GEN2_WRITE_FOOTER; \
10059edbd4a0SFrançois Tigeot }
10069edbd4a0SFrançois Tigeot 
10079edbd4a0SFrançois Tigeot #define __gen5_write(x) \
10089edbd4a0SFrançois Tigeot static void \
1009aee94f86SFrançois Tigeot gen5_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
10102c9916cdSFrançois Tigeot 	GEN2_WRITE_HEADER; \
10119edbd4a0SFrançois Tigeot 	ilk_dummy_write(dev_priv); \
10129edbd4a0SFrançois Tigeot 	__raw_i915_write##x(dev_priv, reg, val); \
10132c9916cdSFrançois Tigeot 	GEN2_WRITE_FOOTER; \
10149edbd4a0SFrançois Tigeot }
10159edbd4a0SFrançois Tigeot 
10162c9916cdSFrançois Tigeot __gen5_write(8)
10172c9916cdSFrançois Tigeot __gen5_write(16)
10182c9916cdSFrançois Tigeot __gen5_write(32)
10192c9916cdSFrançois Tigeot __gen2_write(8)
10202c9916cdSFrançois Tigeot __gen2_write(16)
10212c9916cdSFrançois Tigeot __gen2_write(32)
10222c9916cdSFrançois Tigeot 
10232c9916cdSFrançois Tigeot #undef __gen5_write
10242c9916cdSFrançois Tigeot #undef __gen2_write
10252c9916cdSFrançois Tigeot 
10262c9916cdSFrançois Tigeot #undef GEN2_WRITE_FOOTER
10272c9916cdSFrançois Tigeot #undef GEN2_WRITE_HEADER
10282c9916cdSFrançois Tigeot 
10292c9916cdSFrançois Tigeot #define GEN6_WRITE_HEADER \
1030aee94f86SFrançois Tigeot 	u32 offset = i915_mmio_reg_offset(reg); \
10315e269720SFrançois Tigeot 	unsigned long irqflags; \
10322c9916cdSFrançois Tigeot 	trace_i915_reg_rw(true, reg, val, sizeof(val), trace); \
1033aee94f86SFrançois Tigeot 	assert_rpm_wakelock_held(dev_priv); \
1034c0e85e96SFrançois Tigeot 	spin_lock_irqsave(&dev_priv->uncore.lock, irqflags); \
1035c0e85e96SFrançois Tigeot 	unclaimed_reg_debug(dev_priv, reg, false, true)
10362c9916cdSFrançois Tigeot 
10372c9916cdSFrançois Tigeot #define GEN6_WRITE_FOOTER \
1038c0e85e96SFrançois Tigeot 	unclaimed_reg_debug(dev_priv, reg, false, false); \
10395e269720SFrançois Tigeot 	spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags)
10402c9916cdSFrançois Tigeot 
10419edbd4a0SFrançois Tigeot #define __gen6_write(x) \
10429edbd4a0SFrançois Tigeot static void \
1043aee94f86SFrançois Tigeot gen6_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
10442c9916cdSFrançois Tigeot 	GEN6_WRITE_HEADER; \
1045*3f2dd94aSFrançois Tigeot 	if (NEEDS_FORCE_WAKE(offset)) \
1046*3f2dd94aSFrançois Tigeot 		__gen6_gt_wait_for_fifo(dev_priv); \
10479edbd4a0SFrançois Tigeot 	__raw_i915_write##x(dev_priv, reg, val); \
10482c9916cdSFrançois Tigeot 	GEN6_WRITE_FOOTER; \
10499edbd4a0SFrançois Tigeot }
10509edbd4a0SFrançois Tigeot 
1051a85cb24fSFrançois Tigeot #define __gen_write(func, x) \
10529edbd4a0SFrançois Tigeot static void \
1053a85cb24fSFrançois Tigeot func##_write##x(struct drm_i915_private *dev_priv, i915_reg_t reg, u##x val, bool trace) { \
10548621f407SFrançois Tigeot 	enum forcewake_domains fw_engine; \
10552c9916cdSFrançois Tigeot 	GEN6_WRITE_HEADER; \
1056a85cb24fSFrançois Tigeot 	fw_engine = __##func##_reg_write_fw_domains(offset); \
10578621f407SFrançois Tigeot 	if (fw_engine) \
10588621f407SFrançois Tigeot 		__force_wake_auto(dev_priv, fw_engine); \
10599edbd4a0SFrançois Tigeot 	__raw_i915_write##x(dev_priv, reg, val); \
10602c9916cdSFrançois Tigeot 	GEN6_WRITE_FOOTER; \
10619edbd4a0SFrançois Tigeot }
1062a85cb24fSFrançois Tigeot #define __gen8_write(x) __gen_write(gen8, x)
1063a85cb24fSFrançois Tigeot #define __fwtable_write(x) __gen_write(fwtable, x)
106424edb884SFrançois Tigeot 
10651e12ee3bSFrançois Tigeot __fwtable_write(8)
10661e12ee3bSFrançois Tigeot __fwtable_write(16)
10671e12ee3bSFrançois Tigeot __fwtable_write(32)
10689edbd4a0SFrançois Tigeot __gen8_write(8)
10699edbd4a0SFrançois Tigeot __gen8_write(16)
10709edbd4a0SFrançois Tigeot __gen8_write(32)
10719edbd4a0SFrançois Tigeot __gen6_write(8)
10729edbd4a0SFrançois Tigeot __gen6_write(16)
10739edbd4a0SFrançois Tigeot __gen6_write(32)
10749edbd4a0SFrançois Tigeot 
10751e12ee3bSFrançois Tigeot #undef __fwtable_write
10769edbd4a0SFrançois Tigeot #undef __gen8_write
10779edbd4a0SFrançois Tigeot #undef __gen6_write
10782c9916cdSFrançois Tigeot #undef GEN6_WRITE_FOOTER
10792c9916cdSFrançois Tigeot #undef GEN6_WRITE_HEADER
10809edbd4a0SFrançois Tigeot 
1081*3f2dd94aSFrançois Tigeot #define ASSIGN_WRITE_MMIO_VFUNCS(i915, x) \
10822c9916cdSFrançois Tigeot do { \
1083*3f2dd94aSFrançois Tigeot 	(i915)->uncore.funcs.mmio_writeb = x##_write8; \
1084*3f2dd94aSFrançois Tigeot 	(i915)->uncore.funcs.mmio_writew = x##_write16; \
1085*3f2dd94aSFrançois Tigeot 	(i915)->uncore.funcs.mmio_writel = x##_write32; \
10862c9916cdSFrançois Tigeot } while (0)
10872c9916cdSFrançois Tigeot 
1088*3f2dd94aSFrançois Tigeot #define ASSIGN_READ_MMIO_VFUNCS(i915, x) \
10892c9916cdSFrançois Tigeot do { \
1090*3f2dd94aSFrançois Tigeot 	(i915)->uncore.funcs.mmio_readb = x##_read8; \
1091*3f2dd94aSFrançois Tigeot 	(i915)->uncore.funcs.mmio_readw = x##_read16; \
1092*3f2dd94aSFrançois Tigeot 	(i915)->uncore.funcs.mmio_readl = x##_read32; \
1093*3f2dd94aSFrançois Tigeot 	(i915)->uncore.funcs.mmio_readq = x##_read64; \
10942c9916cdSFrançois Tigeot } while (0)
10952c9916cdSFrançois Tigeot 
10962c9916cdSFrançois Tigeot 
fw_domain_init(struct drm_i915_private * dev_priv,enum forcewake_domain_id domain_id,i915_reg_t reg_set,i915_reg_t reg_ack)10972c9916cdSFrançois Tigeot static void fw_domain_init(struct drm_i915_private *dev_priv,
10982c9916cdSFrançois Tigeot 			   enum forcewake_domain_id domain_id,
1099aee94f86SFrançois Tigeot 			   i915_reg_t reg_set,
1100aee94f86SFrançois Tigeot 			   i915_reg_t reg_ack)
11012c9916cdSFrançois Tigeot {
11022c9916cdSFrançois Tigeot 	struct intel_uncore_forcewake_domain *d;
11032c9916cdSFrançois Tigeot 
11042c9916cdSFrançois Tigeot 	if (WARN_ON(domain_id >= FW_DOMAIN_ID_COUNT))
11052c9916cdSFrançois Tigeot 		return;
11062c9916cdSFrançois Tigeot 
11072c9916cdSFrançois Tigeot 	d = &dev_priv->uncore.fw_domain[domain_id];
11082c9916cdSFrançois Tigeot 
11092c9916cdSFrançois Tigeot 	WARN_ON(d->wake_count);
11102c9916cdSFrançois Tigeot 
1111a85cb24fSFrançois Tigeot 	WARN_ON(!i915_mmio_reg_valid(reg_set));
1112a85cb24fSFrançois Tigeot 	WARN_ON(!i915_mmio_reg_valid(reg_ack));
1113a85cb24fSFrançois Tigeot 
11142c9916cdSFrançois Tigeot 	d->wake_count = 0;
11152c9916cdSFrançois Tigeot 	d->reg_set = reg_set;
11162c9916cdSFrançois Tigeot 	d->reg_ack = reg_ack;
11172c9916cdSFrançois Tigeot 
11182c9916cdSFrançois Tigeot 	d->id = domain_id;
11192c9916cdSFrançois Tigeot 
11208621f407SFrançois Tigeot 	BUILD_BUG_ON(FORCEWAKE_RENDER != (1 << FW_DOMAIN_ID_RENDER));
11218621f407SFrançois Tigeot 	BUILD_BUG_ON(FORCEWAKE_BLITTER != (1 << FW_DOMAIN_ID_BLITTER));
11228621f407SFrançois Tigeot 	BUILD_BUG_ON(FORCEWAKE_MEDIA != (1 << FW_DOMAIN_ID_MEDIA));
11238621f407SFrançois Tigeot 
1124a85cb24fSFrançois Tigeot 	d->mask = BIT(domain_id);
11258621f407SFrançois Tigeot 
11268621f407SFrançois Tigeot 	hrtimer_init(&d->timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
11278621f407SFrançois Tigeot 	d->timer.function = intel_uncore_fw_release_timer;
11282c9916cdSFrançois Tigeot 
1129a85cb24fSFrançois Tigeot 	dev_priv->uncore.fw_domains |= BIT(domain_id);
11302c9916cdSFrançois Tigeot 
1131a85cb24fSFrançois Tigeot 	fw_domain_reset(dev_priv, d);
11322c9916cdSFrançois Tigeot }
11332c9916cdSFrançois Tigeot 
intel_uncore_fw_domains_init(struct drm_i915_private * dev_priv)11341487f786SFrançois Tigeot static void intel_uncore_fw_domains_init(struct drm_i915_private *dev_priv)
11359edbd4a0SFrançois Tigeot {
1136a85cb24fSFrançois Tigeot 	if (INTEL_GEN(dev_priv) <= 5 || intel_vgpu_active(dev_priv))
11372c9916cdSFrançois Tigeot 		return;
1138ba55f2f5SFrançois Tigeot 
1139a85cb24fSFrançois Tigeot 	if (IS_GEN6(dev_priv)) {
1140a85cb24fSFrançois Tigeot 		dev_priv->uncore.fw_reset = 0;
1141a85cb24fSFrançois Tigeot 		dev_priv->uncore.fw_set = FORCEWAKE_KERNEL;
1142a85cb24fSFrançois Tigeot 		dev_priv->uncore.fw_clear = 0;
1143a85cb24fSFrançois Tigeot 	} else {
1144a85cb24fSFrançois Tigeot 		/* WaRsClearFWBitsAtReset:bdw,skl */
1145a85cb24fSFrançois Tigeot 		dev_priv->uncore.fw_reset = _MASKED_BIT_DISABLE(0xffff);
1146a85cb24fSFrançois Tigeot 		dev_priv->uncore.fw_set = _MASKED_BIT_ENABLE(FORCEWAKE_KERNEL);
1147a85cb24fSFrançois Tigeot 		dev_priv->uncore.fw_clear = _MASKED_BIT_DISABLE(FORCEWAKE_KERNEL);
1148a85cb24fSFrançois Tigeot 	}
1149a85cb24fSFrançois Tigeot 
1150*3f2dd94aSFrançois Tigeot 	if (INTEL_GEN(dev_priv) >= 9) {
11512c9916cdSFrançois Tigeot 		dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
11522c9916cdSFrançois Tigeot 		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
11532c9916cdSFrançois Tigeot 		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
11542c9916cdSFrançois Tigeot 			       FORCEWAKE_RENDER_GEN9,
11552c9916cdSFrançois Tigeot 			       FORCEWAKE_ACK_RENDER_GEN9);
11562c9916cdSFrançois Tigeot 		fw_domain_init(dev_priv, FW_DOMAIN_ID_BLITTER,
11572c9916cdSFrançois Tigeot 			       FORCEWAKE_BLITTER_GEN9,
11582c9916cdSFrançois Tigeot 			       FORCEWAKE_ACK_BLITTER_GEN9);
11592c9916cdSFrançois Tigeot 		fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
11602c9916cdSFrançois Tigeot 			       FORCEWAKE_MEDIA_GEN9, FORCEWAKE_ACK_MEDIA_GEN9);
11611487f786SFrançois Tigeot 	} else if (IS_VALLEYVIEW(dev_priv) || IS_CHERRYVIEW(dev_priv)) {
11622c9916cdSFrançois Tigeot 		dev_priv->uncore.funcs.force_wake_get = fw_domains_get;
11632c9916cdSFrançois Tigeot 		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
11642c9916cdSFrançois Tigeot 		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
11652c9916cdSFrançois Tigeot 			       FORCEWAKE_VLV, FORCEWAKE_ACK_VLV);
11662c9916cdSFrançois Tigeot 		fw_domain_init(dev_priv, FW_DOMAIN_ID_MEDIA,
11672c9916cdSFrançois Tigeot 			       FORCEWAKE_MEDIA_VLV, FORCEWAKE_ACK_MEDIA_VLV);
11681487f786SFrançois Tigeot 	} else if (IS_HASWELL(dev_priv) || IS_BROADWELL(dev_priv)) {
11692c9916cdSFrançois Tigeot 		dev_priv->uncore.funcs.force_wake_get =
11702c9916cdSFrançois Tigeot 			fw_domains_get_with_thread_status;
11712c9916cdSFrançois Tigeot 		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
11722c9916cdSFrançois Tigeot 		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
11732c9916cdSFrançois Tigeot 			       FORCEWAKE_MT, FORCEWAKE_ACK_HSW);
11741487f786SFrançois Tigeot 	} else if (IS_IVYBRIDGE(dev_priv)) {
11759edbd4a0SFrançois Tigeot 		u32 ecobus;
11769edbd4a0SFrançois Tigeot 
11779edbd4a0SFrançois Tigeot 		/* IVB configs may use multi-threaded forcewake */
11789edbd4a0SFrançois Tigeot 
11799edbd4a0SFrançois Tigeot 		/* A small trick here - if the bios hasn't configured
11809edbd4a0SFrançois Tigeot 		 * MT forcewake, and if the device is in RC6, then
11819edbd4a0SFrançois Tigeot 		 * force_wake_mt_get will not wake the device and the
11829edbd4a0SFrançois Tigeot 		 * ECOBUS read will return zero. Which will be
11839edbd4a0SFrançois Tigeot 		 * (correctly) interpreted by the test below as MT
11849edbd4a0SFrançois Tigeot 		 * forcewake being disabled.
11859edbd4a0SFrançois Tigeot 		 */
11862c9916cdSFrançois Tigeot 		dev_priv->uncore.funcs.force_wake_get =
11872c9916cdSFrançois Tigeot 			fw_domains_get_with_thread_status;
1188*3f2dd94aSFrançois Tigeot 		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
11892c9916cdSFrançois Tigeot 
11902c9916cdSFrançois Tigeot 		/* We need to init first for ECOBUS access and then
11912c9916cdSFrançois Tigeot 		 * determine later if we want to reinit, in case of MT access is
11922c9916cdSFrançois Tigeot 		 * not working. In this stage we don't know which flavour this
11932c9916cdSFrançois Tigeot 		 * ivb is, so it is better to reset also the gen6 fw registers
11942c9916cdSFrançois Tigeot 		 * before the ecobus check.
11952c9916cdSFrançois Tigeot 		 */
11962c9916cdSFrançois Tigeot 
11972c9916cdSFrançois Tigeot 		__raw_i915_write32(dev_priv, FORCEWAKE, 0);
11982c9916cdSFrançois Tigeot 		__raw_posting_read(dev_priv, ECOBUS);
11992c9916cdSFrançois Tigeot 
12002c9916cdSFrançois Tigeot 		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
12012c9916cdSFrançois Tigeot 			       FORCEWAKE_MT, FORCEWAKE_MT_ACK);
12022c9916cdSFrançois Tigeot 
1203303bf270SFrançois Tigeot 		spin_lock_irq(&dev_priv->uncore.lock);
1204a85cb24fSFrançois Tigeot 		fw_domains_get_with_thread_status(dev_priv, FORCEWAKE_RENDER);
12059edbd4a0SFrançois Tigeot 		ecobus = __raw_i915_read32(dev_priv, ECOBUS);
1206*3f2dd94aSFrançois Tigeot 		fw_domains_put(dev_priv, FORCEWAKE_RENDER);
1207303bf270SFrançois Tigeot 		spin_unlock_irq(&dev_priv->uncore.lock);
12089edbd4a0SFrançois Tigeot 
12092c9916cdSFrançois Tigeot 		if (!(ecobus & FORCEWAKE_MT_ENABLE)) {
12109edbd4a0SFrançois Tigeot 			DRM_INFO("No MT forcewake available on Ivybridge, this can result in issues\n");
12119edbd4a0SFrançois Tigeot 			DRM_INFO("when using vblank-synced partial screen updates.\n");
12122c9916cdSFrançois Tigeot 			fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
12132c9916cdSFrançois Tigeot 				       FORCEWAKE, FORCEWAKE_ACK);
12149edbd4a0SFrançois Tigeot 		}
12151487f786SFrançois Tigeot 	} else if (IS_GEN6(dev_priv)) {
12169edbd4a0SFrançois Tigeot 		dev_priv->uncore.funcs.force_wake_get =
12172c9916cdSFrançois Tigeot 			fw_domains_get_with_thread_status;
1218*3f2dd94aSFrançois Tigeot 		dev_priv->uncore.funcs.force_wake_put = fw_domains_put;
12192c9916cdSFrançois Tigeot 		fw_domain_init(dev_priv, FW_DOMAIN_ID_RENDER,
12202c9916cdSFrançois Tigeot 			       FORCEWAKE, FORCEWAKE_ACK);
12219edbd4a0SFrançois Tigeot 	}
12229edbd4a0SFrançois Tigeot 
12232c9916cdSFrançois Tigeot 	/* All future platforms are expected to require complex power gating */
12242c9916cdSFrançois Tigeot 	WARN_ON(dev_priv->uncore.fw_domains == 0);
12252c9916cdSFrançois Tigeot }
12262c9916cdSFrançois Tigeot 
12271e12ee3bSFrançois Tigeot #define ASSIGN_FW_DOMAINS_TABLE(d) \
12281e12ee3bSFrançois Tigeot { \
12291e12ee3bSFrançois Tigeot 	dev_priv->uncore.fw_domains_table = \
12301e12ee3bSFrançois Tigeot 			(struct intel_forcewake_range *)(d); \
12311e12ee3bSFrançois Tigeot 	dev_priv->uncore.fw_domains_table_entries = ARRAY_SIZE((d)); \
12321e12ee3bSFrançois Tigeot }
12331e12ee3bSFrançois Tigeot 
i915_pmic_bus_access_notifier(struct notifier_block * nb,unsigned long action,void * data)1234a85cb24fSFrançois Tigeot static int i915_pmic_bus_access_notifier(struct notifier_block *nb,
1235a85cb24fSFrançois Tigeot 					 unsigned long action, void *data)
1236a85cb24fSFrançois Tigeot {
1237a85cb24fSFrançois Tigeot 	struct drm_i915_private *dev_priv = container_of(nb,
1238a85cb24fSFrançois Tigeot 			struct drm_i915_private, uncore.pmic_bus_access_nb);
1239a85cb24fSFrançois Tigeot 
1240a85cb24fSFrançois Tigeot 	switch (action) {
1241a85cb24fSFrançois Tigeot 	case MBI_PMIC_BUS_ACCESS_BEGIN:
1242a85cb24fSFrançois Tigeot 		/*
1243a85cb24fSFrançois Tigeot 		 * forcewake all now to make sure that we don't need to do a
1244a85cb24fSFrançois Tigeot 		 * forcewake later which on systems where this notifier gets
1245a85cb24fSFrançois Tigeot 		 * called requires the punit to access to the shared pmic i2c
1246a85cb24fSFrançois Tigeot 		 * bus, which will be busy after this notification, leading to:
1247a85cb24fSFrançois Tigeot 		 * "render: timed out waiting for forcewake ack request."
1248a85cb24fSFrançois Tigeot 		 * errors.
1249*3f2dd94aSFrançois Tigeot 		 *
1250*3f2dd94aSFrançois Tigeot 		 * The notifier is unregistered during intel_runtime_suspend(),
1251*3f2dd94aSFrançois Tigeot 		 * so it's ok to access the HW here without holding a RPM
1252*3f2dd94aSFrançois Tigeot 		 * wake reference -> disable wakeref asserts for the time of
1253*3f2dd94aSFrançois Tigeot 		 * the access.
1254a85cb24fSFrançois Tigeot 		 */
1255*3f2dd94aSFrançois Tigeot 		disable_rpm_wakeref_asserts(dev_priv);
1256a85cb24fSFrançois Tigeot 		intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
1257*3f2dd94aSFrançois Tigeot 		enable_rpm_wakeref_asserts(dev_priv);
1258a85cb24fSFrançois Tigeot 		break;
1259a85cb24fSFrançois Tigeot 	case MBI_PMIC_BUS_ACCESS_END:
1260a85cb24fSFrançois Tigeot 		intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
1261a85cb24fSFrançois Tigeot 		break;
1262a85cb24fSFrançois Tigeot 	}
1263a85cb24fSFrançois Tigeot 
1264a85cb24fSFrançois Tigeot 	return NOTIFY_OK;
1265a85cb24fSFrançois Tigeot }
1266a85cb24fSFrançois Tigeot 
intel_uncore_init(struct drm_i915_private * dev_priv)12671487f786SFrançois Tigeot void intel_uncore_init(struct drm_i915_private *dev_priv)
12682c9916cdSFrançois Tigeot {
12691487f786SFrançois Tigeot 	i915_check_vgpu(dev_priv);
1270477eb7f9SFrançois Tigeot 
12718621f407SFrançois Tigeot 	intel_uncore_edram_detect(dev_priv);
12721487f786SFrançois Tigeot 	intel_uncore_fw_domains_init(dev_priv);
12731487f786SFrançois Tigeot 	__intel_uncore_early_sanitize(dev_priv, false);
12742c9916cdSFrançois Tigeot 
1275c0e85e96SFrançois Tigeot 	dev_priv->uncore.unclaimed_mmio_check = 1;
1276a85cb24fSFrançois Tigeot 	dev_priv->uncore.pmic_bus_access_nb.notifier_call =
1277a85cb24fSFrançois Tigeot 		i915_pmic_bus_access_notifier;
1278c0e85e96SFrançois Tigeot 
1279a85cb24fSFrançois Tigeot 	if (IS_GEN(dev_priv, 2, 4) || intel_vgpu_active(dev_priv)) {
1280*3f2dd94aSFrançois Tigeot 		ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen2);
1281*3f2dd94aSFrançois Tigeot 		ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen2);
1282a85cb24fSFrançois Tigeot 	} else if (IS_GEN5(dev_priv)) {
1283*3f2dd94aSFrançois Tigeot 		ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen5);
1284*3f2dd94aSFrançois Tigeot 		ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen5);
1285a85cb24fSFrançois Tigeot 	} else if (IS_GEN(dev_priv, 6, 7)) {
1286*3f2dd94aSFrançois Tigeot 		ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen6);
1287a85cb24fSFrançois Tigeot 
1288a85cb24fSFrançois Tigeot 		if (IS_VALLEYVIEW(dev_priv)) {
1289a85cb24fSFrançois Tigeot 			ASSIGN_FW_DOMAINS_TABLE(__vlv_fw_ranges);
1290*3f2dd94aSFrançois Tigeot 			ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
1291a85cb24fSFrançois Tigeot 		} else {
1292*3f2dd94aSFrançois Tigeot 			ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6);
1293a85cb24fSFrançois Tigeot 		}
1294a85cb24fSFrançois Tigeot 	} else if (IS_GEN8(dev_priv)) {
1295a85cb24fSFrançois Tigeot 		if (IS_CHERRYVIEW(dev_priv)) {
1296a85cb24fSFrançois Tigeot 			ASSIGN_FW_DOMAINS_TABLE(__chv_fw_ranges);
1297*3f2dd94aSFrançois Tigeot 			ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable);
1298*3f2dd94aSFrançois Tigeot 			ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
1299a85cb24fSFrançois Tigeot 
1300a85cb24fSFrançois Tigeot 		} else {
1301*3f2dd94aSFrançois Tigeot 			ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, gen8);
1302*3f2dd94aSFrançois Tigeot 			ASSIGN_READ_MMIO_VFUNCS(dev_priv, gen6);
1303a85cb24fSFrançois Tigeot 		}
1304a85cb24fSFrançois Tigeot 	} else {
13051e12ee3bSFrançois Tigeot 		ASSIGN_FW_DOMAINS_TABLE(__gen9_fw_ranges);
1306*3f2dd94aSFrançois Tigeot 		ASSIGN_WRITE_MMIO_VFUNCS(dev_priv, fwtable);
1307*3f2dd94aSFrançois Tigeot 		ASSIGN_READ_MMIO_VFUNCS(dev_priv, fwtable);
13089edbd4a0SFrançois Tigeot 	}
13092c9916cdSFrançois Tigeot 
1310a85cb24fSFrançois Tigeot 	iosf_mbi_register_pmic_bus_access_notifier(
1311a85cb24fSFrançois Tigeot 		&dev_priv->uncore.pmic_bus_access_nb);
1312477eb7f9SFrançois Tigeot 
13131487f786SFrançois Tigeot 	i915_check_and_clear_faults(dev_priv);
13149edbd4a0SFrançois Tigeot }
13159edbd4a0SFrançois Tigeot 
intel_uncore_fini(struct drm_i915_private * dev_priv)13161487f786SFrançois Tigeot void intel_uncore_fini(struct drm_i915_private *dev_priv)
13179edbd4a0SFrançois Tigeot {
1318a85cb24fSFrançois Tigeot 	iosf_mbi_unregister_pmic_bus_access_notifier(
1319a85cb24fSFrançois Tigeot 		&dev_priv->uncore.pmic_bus_access_nb);
1320a85cb24fSFrançois Tigeot 
13219edbd4a0SFrançois Tigeot 	/* Paranoia: make sure we have disabled everything before we exit. */
13221487f786SFrançois Tigeot 	intel_uncore_sanitize(dev_priv);
13231487f786SFrançois Tigeot 	intel_uncore_forcewake_reset(dev_priv, false);
13249edbd4a0SFrançois Tigeot }
13259edbd4a0SFrançois Tigeot 
1326*3f2dd94aSFrançois Tigeot static const struct reg_whitelist {
1327*3f2dd94aSFrançois Tigeot 	i915_reg_t offset_ldw;
1328*3f2dd94aSFrançois Tigeot 	i915_reg_t offset_udw;
1329*3f2dd94aSFrançois Tigeot 	u16 gen_mask;
1330*3f2dd94aSFrançois Tigeot 	u8 size;
1331*3f2dd94aSFrançois Tigeot } reg_read_whitelist[] = { {
1332*3f2dd94aSFrançois Tigeot 	.offset_ldw = RING_TIMESTAMP(RENDER_RING_BASE),
1333aee94f86SFrançois Tigeot 	.offset_udw = RING_TIMESTAMP_UDW(RENDER_RING_BASE),
1334*3f2dd94aSFrançois Tigeot 	.gen_mask = INTEL_GEN_MASK(4, 10),
1335*3f2dd94aSFrançois Tigeot 	.size = 8
1336*3f2dd94aSFrançois Tigeot } };
13379edbd4a0SFrançois Tigeot 
i915_reg_read_ioctl(struct drm_device * dev,void * data,struct drm_file * file)13389edbd4a0SFrançois Tigeot int i915_reg_read_ioctl(struct drm_device *dev,
13399edbd4a0SFrançois Tigeot 			void *data, struct drm_file *file)
13409edbd4a0SFrançois Tigeot {
1341bf017597SFrançois Tigeot 	struct drm_i915_private *dev_priv = to_i915(dev);
13429edbd4a0SFrançois Tigeot 	struct drm_i915_reg_read *reg = data;
1343*3f2dd94aSFrançois Tigeot 	struct reg_whitelist const *entry;
1344*3f2dd94aSFrançois Tigeot 	unsigned int flags;
1345*3f2dd94aSFrançois Tigeot 	int remain;
1346*3f2dd94aSFrançois Tigeot 	int ret = 0;
13479edbd4a0SFrançois Tigeot 
1348*3f2dd94aSFrançois Tigeot 	entry = reg_read_whitelist;
1349*3f2dd94aSFrançois Tigeot 	remain = ARRAY_SIZE(reg_read_whitelist);
1350*3f2dd94aSFrançois Tigeot 	while (remain) {
1351*3f2dd94aSFrançois Tigeot 		u32 entry_offset = i915_mmio_reg_offset(entry->offset_ldw);
1352*3f2dd94aSFrançois Tigeot 
1353*3f2dd94aSFrançois Tigeot 		GEM_BUG_ON(!is_power_of_2(entry->size));
1354*3f2dd94aSFrançois Tigeot 		GEM_BUG_ON(entry->size > 8);
1355*3f2dd94aSFrançois Tigeot 		GEM_BUG_ON(entry_offset & (entry->size - 1));
1356*3f2dd94aSFrançois Tigeot 
1357*3f2dd94aSFrançois Tigeot 		if (INTEL_INFO(dev_priv)->gen_mask & entry->gen_mask &&
1358*3f2dd94aSFrançois Tigeot 		    entry_offset == (reg->offset & -entry->size))
13599edbd4a0SFrançois Tigeot 			break;
1360*3f2dd94aSFrançois Tigeot 		entry++;
1361*3f2dd94aSFrançois Tigeot 		remain--;
13629edbd4a0SFrançois Tigeot 	}
13639edbd4a0SFrançois Tigeot 
1364*3f2dd94aSFrançois Tigeot 	if (!remain)
13659edbd4a0SFrançois Tigeot 		return -EINVAL;
13669edbd4a0SFrançois Tigeot 
1367*3f2dd94aSFrançois Tigeot 	flags = reg->offset & (entry->size - 1);
136819c468b4SFrançois Tigeot 
1369ba55f2f5SFrançois Tigeot 	intel_runtime_pm_get(dev_priv);
1370*3f2dd94aSFrançois Tigeot 	if (entry->size == 8 && flags == I915_REG_READ_8B_WA)
1371*3f2dd94aSFrançois Tigeot 		reg->val = I915_READ64_2x32(entry->offset_ldw,
1372*3f2dd94aSFrançois Tigeot 					    entry->offset_udw);
1373*3f2dd94aSFrançois Tigeot 	else if (entry->size == 8 && flags == 0)
1374*3f2dd94aSFrançois Tigeot 		reg->val = I915_READ64(entry->offset_ldw);
1375*3f2dd94aSFrançois Tigeot 	else if (entry->size == 4 && flags == 0)
1376*3f2dd94aSFrançois Tigeot 		reg->val = I915_READ(entry->offset_ldw);
1377*3f2dd94aSFrançois Tigeot 	else if (entry->size == 2 && flags == 0)
1378*3f2dd94aSFrançois Tigeot 		reg->val = I915_READ16(entry->offset_ldw);
1379*3f2dd94aSFrançois Tigeot 	else if (entry->size == 1 && flags == 0)
1380*3f2dd94aSFrançois Tigeot 		reg->val = I915_READ8(entry->offset_ldw);
1381*3f2dd94aSFrançois Tigeot 	else
1382ba55f2f5SFrançois Tigeot 		ret = -EINVAL;
1383ba55f2f5SFrançois Tigeot 	intel_runtime_pm_put(dev_priv);
1384*3f2dd94aSFrançois Tigeot 
1385ba55f2f5SFrançois Tigeot 	return ret;
13869edbd4a0SFrançois Tigeot }
13879edbd4a0SFrançois Tigeot 
gen3_stop_engine(struct intel_engine_cs * engine)1388*3f2dd94aSFrançois Tigeot static void gen3_stop_engine(struct intel_engine_cs *engine)
1389*3f2dd94aSFrançois Tigeot {
1390*3f2dd94aSFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
1391*3f2dd94aSFrançois Tigeot 	const u32 base = engine->mmio_base;
1392*3f2dd94aSFrançois Tigeot 	const i915_reg_t mode = RING_MI_MODE(base);
1393*3f2dd94aSFrançois Tigeot 
1394*3f2dd94aSFrançois Tigeot 	I915_WRITE_FW(mode, _MASKED_BIT_ENABLE(STOP_RING));
1395*3f2dd94aSFrançois Tigeot 	if (intel_wait_for_register_fw(dev_priv,
1396*3f2dd94aSFrançois Tigeot 				       mode,
1397*3f2dd94aSFrançois Tigeot 				       MODE_IDLE,
1398*3f2dd94aSFrançois Tigeot 				       MODE_IDLE,
1399*3f2dd94aSFrançois Tigeot 				       500))
1400*3f2dd94aSFrançois Tigeot 		DRM_DEBUG_DRIVER("%s: timed out on STOP_RING\n",
1401*3f2dd94aSFrançois Tigeot 				 engine->name);
1402*3f2dd94aSFrançois Tigeot 
1403*3f2dd94aSFrançois Tigeot 	I915_WRITE_FW(RING_CTL(base), 0);
1404*3f2dd94aSFrançois Tigeot 	I915_WRITE_FW(RING_HEAD(base), 0);
1405*3f2dd94aSFrançois Tigeot 	I915_WRITE_FW(RING_TAIL(base), 0);
1406*3f2dd94aSFrançois Tigeot 
1407*3f2dd94aSFrançois Tigeot 	/* Check acts as a post */
1408*3f2dd94aSFrançois Tigeot 	if (I915_READ_FW(RING_HEAD(base)) != 0)
1409*3f2dd94aSFrançois Tigeot 		DRM_DEBUG_DRIVER("%s: ring head not parked\n",
1410*3f2dd94aSFrançois Tigeot 				 engine->name);
1411*3f2dd94aSFrançois Tigeot }
1412*3f2dd94aSFrançois Tigeot 
i915_stop_engines(struct drm_i915_private * dev_priv,unsigned engine_mask)1413*3f2dd94aSFrançois Tigeot static void i915_stop_engines(struct drm_i915_private *dev_priv,
1414*3f2dd94aSFrançois Tigeot 			      unsigned engine_mask)
1415*3f2dd94aSFrançois Tigeot {
1416*3f2dd94aSFrançois Tigeot 	struct intel_engine_cs *engine;
1417*3f2dd94aSFrançois Tigeot 	enum intel_engine_id id;
1418*3f2dd94aSFrançois Tigeot 
1419*3f2dd94aSFrançois Tigeot 	if (INTEL_GEN(dev_priv) < 3)
1420*3f2dd94aSFrançois Tigeot 		return;
1421*3f2dd94aSFrançois Tigeot 
1422*3f2dd94aSFrançois Tigeot 	for_each_engine_masked(engine, dev_priv, engine_mask, id)
1423*3f2dd94aSFrançois Tigeot 		gen3_stop_engine(engine);
1424*3f2dd94aSFrançois Tigeot }
1425*3f2dd94aSFrançois Tigeot 
i915_reset_complete(struct pci_dev * pdev)1426*3f2dd94aSFrançois Tigeot static bool i915_reset_complete(struct pci_dev *pdev)
14279edbd4a0SFrançois Tigeot {
14289edbd4a0SFrançois Tigeot 	u8 gdrst;
1429*3f2dd94aSFrançois Tigeot 
14301487f786SFrançois Tigeot 	pci_read_config_byte(pdev, I915_GDRST, &gdrst);
14312c9916cdSFrançois Tigeot 	return (gdrst & GRDOM_RESET_STATUS) == 0;
14322c9916cdSFrançois Tigeot }
14332c9916cdSFrançois Tigeot 
i915_do_reset(struct drm_i915_private * dev_priv,unsigned engine_mask)14341487f786SFrançois Tigeot static int i915_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
14352c9916cdSFrançois Tigeot {
1436303bf270SFrançois Tigeot 	struct pci_dev *pdev = dev_priv->drm.pdev;
14372c9916cdSFrançois Tigeot 
14381487f786SFrançois Tigeot 	/* assert reset for at least 20 usec */
14391487f786SFrançois Tigeot 	pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
1440*3f2dd94aSFrançois Tigeot 	usleep_range(50, 200);
14411487f786SFrançois Tigeot 	pci_write_config_byte(pdev, I915_GDRST, 0);
14421487f786SFrançois Tigeot 
14431487f786SFrançois Tigeot 	return wait_for(i915_reset_complete(pdev), 500);
14442c9916cdSFrançois Tigeot }
14452c9916cdSFrançois Tigeot 
g4x_reset_complete(struct pci_dev * pdev)1446*3f2dd94aSFrançois Tigeot static bool g4x_reset_complete(struct pci_dev *pdev)
14472c9916cdSFrançois Tigeot {
14482c9916cdSFrançois Tigeot 	u8 gdrst;
1449*3f2dd94aSFrançois Tigeot 
14501487f786SFrançois Tigeot 	pci_read_config_byte(pdev, I915_GDRST, &gdrst);
14519edbd4a0SFrançois Tigeot 	return (gdrst & GRDOM_RESET_ENABLE) == 0;
14529edbd4a0SFrançois Tigeot }
14539edbd4a0SFrançois Tigeot 
g33_do_reset(struct drm_i915_private * dev_priv,unsigned engine_mask)14541487f786SFrançois Tigeot static int g33_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
14559edbd4a0SFrançois Tigeot {
1456303bf270SFrançois Tigeot 	struct pci_dev *pdev = dev_priv->drm.pdev;
1457*3f2dd94aSFrançois Tigeot 
14581487f786SFrançois Tigeot 	pci_write_config_byte(pdev, I915_GDRST, GRDOM_RESET_ENABLE);
14591487f786SFrançois Tigeot 	return wait_for(g4x_reset_complete(pdev), 500);
14609edbd4a0SFrançois Tigeot }
14619edbd4a0SFrançois Tigeot 
g4x_do_reset(struct drm_i915_private * dev_priv,unsigned engine_mask)14621487f786SFrançois Tigeot static int g4x_do_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
14639edbd4a0SFrançois Tigeot {
1464303bf270SFrançois Tigeot 	struct pci_dev *pdev = dev_priv->drm.pdev;
14659edbd4a0SFrançois Tigeot 	int ret;
14669edbd4a0SFrançois Tigeot 
1467ba55f2f5SFrançois Tigeot 	/* WaVcpClkGateDisableForMediaReset:ctg,elk */
1468*3f2dd94aSFrançois Tigeot 	I915_WRITE(VDECCLK_GATE_D,
1469*3f2dd94aSFrançois Tigeot 		   I915_READ(VDECCLK_GATE_D) | VCP_UNIT_CLOCK_GATE_DISABLE);
1470ba55f2f5SFrançois Tigeot 	POSTING_READ(VDECCLK_GATE_D);
1471ba55f2f5SFrançois Tigeot 
14721487f786SFrançois Tigeot 	pci_write_config_byte(pdev, I915_GDRST,
1473ba55f2f5SFrançois Tigeot 			      GRDOM_MEDIA | GRDOM_RESET_ENABLE);
14741487f786SFrançois Tigeot 	ret =  wait_for(g4x_reset_complete(pdev), 500);
1475*3f2dd94aSFrançois Tigeot 	if (ret) {
1476*3f2dd94aSFrançois Tigeot 		DRM_DEBUG_DRIVER("Wait for media reset failed\n");
1477*3f2dd94aSFrançois Tigeot 		goto out;
1478*3f2dd94aSFrançois Tigeot 	}
1479ba55f2f5SFrançois Tigeot 
1480*3f2dd94aSFrançois Tigeot 	pci_write_config_byte(pdev, I915_GDRST,
1481*3f2dd94aSFrançois Tigeot 			      GRDOM_RENDER | GRDOM_RESET_ENABLE);
1482*3f2dd94aSFrançois Tigeot 	ret =  wait_for(g4x_reset_complete(pdev), 500);
1483*3f2dd94aSFrançois Tigeot 	if (ret) {
1484*3f2dd94aSFrançois Tigeot 		DRM_DEBUG_DRIVER("Wait for render reset failed\n");
1485*3f2dd94aSFrançois Tigeot 		goto out;
1486*3f2dd94aSFrançois Tigeot 	}
1487ba55f2f5SFrançois Tigeot 
1488*3f2dd94aSFrançois Tigeot out:
14891487f786SFrançois Tigeot 	pci_write_config_byte(pdev, I915_GDRST, 0);
1490ba55f2f5SFrançois Tigeot 
1491*3f2dd94aSFrançois Tigeot 	I915_WRITE(VDECCLK_GATE_D,
1492*3f2dd94aSFrançois Tigeot 		   I915_READ(VDECCLK_GATE_D) & ~VCP_UNIT_CLOCK_GATE_DISABLE);
1493*3f2dd94aSFrançois Tigeot 	POSTING_READ(VDECCLK_GATE_D);
1494*3f2dd94aSFrançois Tigeot 
1495*3f2dd94aSFrançois Tigeot 	return ret;
1496ba55f2f5SFrançois Tigeot }
1497ba55f2f5SFrançois Tigeot 
ironlake_do_reset(struct drm_i915_private * dev_priv,unsigned engine_mask)14981487f786SFrançois Tigeot static int ironlake_do_reset(struct drm_i915_private *dev_priv,
14991487f786SFrançois Tigeot 			     unsigned engine_mask)
1500ba55f2f5SFrançois Tigeot {
1501ba55f2f5SFrançois Tigeot 	int ret;
1502ba55f2f5SFrançois Tigeot 
1503*3f2dd94aSFrançois Tigeot 	I915_WRITE(ILK_GDSR, ILK_GRDOM_RENDER | ILK_GRDOM_RESET_ENABLE);
15041487f786SFrançois Tigeot 	ret = intel_wait_for_register(dev_priv,
15051487f786SFrançois Tigeot 				      ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0,
15061487f786SFrançois Tigeot 				      500);
1507*3f2dd94aSFrançois Tigeot 	if (ret) {
1508*3f2dd94aSFrançois Tigeot 		DRM_DEBUG_DRIVER("Wait for render reset failed\n");
1509*3f2dd94aSFrançois Tigeot 		goto out;
1510*3f2dd94aSFrançois Tigeot 	}
1511ba55f2f5SFrançois Tigeot 
1512*3f2dd94aSFrançois Tigeot 	I915_WRITE(ILK_GDSR, ILK_GRDOM_MEDIA | ILK_GRDOM_RESET_ENABLE);
15131487f786SFrançois Tigeot 	ret = intel_wait_for_register(dev_priv,
15141487f786SFrançois Tigeot 				      ILK_GDSR, ILK_GRDOM_RESET_ENABLE, 0,
15151487f786SFrançois Tigeot 				      500);
1516*3f2dd94aSFrançois Tigeot 	if (ret) {
1517*3f2dd94aSFrançois Tigeot 		DRM_DEBUG_DRIVER("Wait for media reset failed\n");
1518*3f2dd94aSFrançois Tigeot 		goto out;
1519*3f2dd94aSFrançois Tigeot 	}
1520ba55f2f5SFrançois Tigeot 
1521*3f2dd94aSFrançois Tigeot out:
1522352ff8bdSFrançois Tigeot 	I915_WRITE(ILK_GDSR, 0);
1523*3f2dd94aSFrançois Tigeot 	POSTING_READ(ILK_GDSR);
1524*3f2dd94aSFrançois Tigeot 	return ret;
15259edbd4a0SFrançois Tigeot }
15269edbd4a0SFrançois Tigeot 
15278621f407SFrançois Tigeot /* Reset the hardware domains (GENX_GRDOM_*) specified by mask */
gen6_hw_domain_reset(struct drm_i915_private * dev_priv,u32 hw_domain_mask)15288621f407SFrançois Tigeot static int gen6_hw_domain_reset(struct drm_i915_private *dev_priv,
15298621f407SFrançois Tigeot 				u32 hw_domain_mask)
15309edbd4a0SFrançois Tigeot {
1531*3f2dd94aSFrançois Tigeot 	int err;
1532*3f2dd94aSFrançois Tigeot 
15339edbd4a0SFrançois Tigeot 	/* GEN6_GDRST is not in the gt power well, no need to check
15349edbd4a0SFrançois Tigeot 	 * for fifo space for the write or forcewake the chip for
15359edbd4a0SFrançois Tigeot 	 * the read
15369edbd4a0SFrançois Tigeot 	 */
15378621f407SFrançois Tigeot 	__raw_i915_write32(dev_priv, GEN6_GDRST, hw_domain_mask);
15389edbd4a0SFrançois Tigeot 
1539*3f2dd94aSFrançois Tigeot 	/* Wait for the device to ack the reset requests */
1540*3f2dd94aSFrançois Tigeot 	err = intel_wait_for_register_fw(dev_priv,
15411487f786SFrançois Tigeot 					  GEN6_GDRST, hw_domain_mask, 0,
15421487f786SFrançois Tigeot 					  500);
1543*3f2dd94aSFrançois Tigeot 	if (err)
1544*3f2dd94aSFrançois Tigeot 		DRM_DEBUG_DRIVER("Wait for 0x%08x engines reset failed\n",
1545*3f2dd94aSFrançois Tigeot 				 hw_domain_mask);
1546*3f2dd94aSFrançois Tigeot 
1547*3f2dd94aSFrançois Tigeot 	return err;
15488621f407SFrançois Tigeot }
15498621f407SFrançois Tigeot 
15508621f407SFrançois Tigeot /**
15518621f407SFrançois Tigeot  * gen6_reset_engines - reset individual engines
15521487f786SFrançois Tigeot  * @dev_priv: i915 device
15538621f407SFrançois Tigeot  * @engine_mask: mask of intel_ring_flag() engines or ALL_ENGINES for full reset
15548621f407SFrançois Tigeot  *
15558621f407SFrançois Tigeot  * This function will reset the individual engines that are set in engine_mask.
15568621f407SFrançois Tigeot  * If you provide ALL_ENGINES as mask, full global domain reset will be issued.
15578621f407SFrançois Tigeot  *
15588621f407SFrançois Tigeot  * Note: It is responsibility of the caller to handle the difference between
15598621f407SFrançois Tigeot  * asking full domain reset versus reset for all available individual engines.
15608621f407SFrançois Tigeot  *
15618621f407SFrançois Tigeot  * Returns 0 on success, nonzero on error.
15628621f407SFrançois Tigeot  */
gen6_reset_engines(struct drm_i915_private * dev_priv,unsigned engine_mask)15631487f786SFrançois Tigeot static int gen6_reset_engines(struct drm_i915_private *dev_priv,
15641487f786SFrançois Tigeot 			      unsigned engine_mask)
15658621f407SFrançois Tigeot {
15668621f407SFrançois Tigeot 	struct intel_engine_cs *engine;
15678621f407SFrançois Tigeot 	const u32 hw_engine_mask[I915_NUM_ENGINES] = {
15688621f407SFrançois Tigeot 		[RCS] = GEN6_GRDOM_RENDER,
15698621f407SFrançois Tigeot 		[BCS] = GEN6_GRDOM_BLT,
15708621f407SFrançois Tigeot 		[VCS] = GEN6_GRDOM_MEDIA,
15718621f407SFrançois Tigeot 		[VCS2] = GEN8_GRDOM_MEDIA2,
15728621f407SFrançois Tigeot 		[VECS] = GEN6_GRDOM_VECS,
15738621f407SFrançois Tigeot 	};
15748621f407SFrançois Tigeot 	u32 hw_mask;
15758621f407SFrançois Tigeot 
15768621f407SFrançois Tigeot 	if (engine_mask == ALL_ENGINES) {
15778621f407SFrançois Tigeot 		hw_mask = GEN6_GRDOM_FULL;
15788621f407SFrançois Tigeot 	} else {
15791e12ee3bSFrançois Tigeot 		unsigned int tmp;
15801e12ee3bSFrançois Tigeot 
15818621f407SFrançois Tigeot 		hw_mask = 0;
15821e12ee3bSFrançois Tigeot 		for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
15838621f407SFrançois Tigeot 			hw_mask |= hw_engine_mask[engine->id];
15848621f407SFrançois Tigeot 	}
15858621f407SFrançois Tigeot 
1586*3f2dd94aSFrançois Tigeot 	return gen6_hw_domain_reset(dev_priv, hw_mask);
15879edbd4a0SFrançois Tigeot }
15889edbd4a0SFrançois Tigeot 
15891487f786SFrançois Tigeot /**
1590*3f2dd94aSFrançois Tigeot  * __intel_wait_for_register_fw - wait until register matches expected state
15911487f786SFrançois Tigeot  * @dev_priv: the i915 device
15921487f786SFrançois Tigeot  * @reg: the register to read
15931487f786SFrançois Tigeot  * @mask: mask to apply to register value
15941487f786SFrançois Tigeot  * @value: expected value
1595*3f2dd94aSFrançois Tigeot  * @fast_timeout_us: fast timeout in microsecond for atomic/tight wait
1596*3f2dd94aSFrançois Tigeot  * @slow_timeout_ms: slow timeout in millisecond
1597*3f2dd94aSFrançois Tigeot  * @out_value: optional placeholder to hold registry value
15981487f786SFrançois Tigeot  *
15991487f786SFrançois Tigeot  * This routine waits until the target register @reg contains the expected
160087df8fc6SFrançois Tigeot  * @value after applying the @mask, i.e. it waits until ::
160187df8fc6SFrançois Tigeot  *
160287df8fc6SFrançois Tigeot  *     (I915_READ_FW(reg) & mask) == value
160387df8fc6SFrançois Tigeot  *
1604*3f2dd94aSFrançois Tigeot  * Otherwise, the wait will timeout after @slow_timeout_ms milliseconds.
1605*3f2dd94aSFrançois Tigeot  * For atomic context @slow_timeout_ms must be zero and @fast_timeout_us
1606*3f2dd94aSFrançois Tigeot  * must be not larger than 20,0000 microseconds.
16071487f786SFrançois Tigeot  *
16081487f786SFrançois Tigeot  * Note that this routine assumes the caller holds forcewake asserted, it is
16091487f786SFrançois Tigeot  * not suitable for very long waits. See intel_wait_for_register() if you
16101487f786SFrançois Tigeot  * wish to wait without holding forcewake for the duration (i.e. you expect
16111487f786SFrançois Tigeot  * the wait to be slow).
16121487f786SFrançois Tigeot  *
16131487f786SFrançois Tigeot  * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
16141487f786SFrançois Tigeot  */
__intel_wait_for_register_fw(struct drm_i915_private * dev_priv,i915_reg_t reg,u32 mask,u32 value,unsigned int fast_timeout_us,unsigned int slow_timeout_ms,u32 * out_value)1615*3f2dd94aSFrançois Tigeot int __intel_wait_for_register_fw(struct drm_i915_private *dev_priv,
1616aee94f86SFrançois Tigeot 				 i915_reg_t reg,
1617*3f2dd94aSFrançois Tigeot 				 u32 mask,
1618*3f2dd94aSFrançois Tigeot 				 u32 value,
1619*3f2dd94aSFrançois Tigeot 				 unsigned int fast_timeout_us,
1620*3f2dd94aSFrançois Tigeot 				 unsigned int slow_timeout_ms,
1621*3f2dd94aSFrançois Tigeot 				 u32 *out_value)
1622a05eeebfSFrançois Tigeot {
1623*3f2dd94aSFrançois Tigeot 	u32 reg_value = 0;
1624*3f2dd94aSFrançois Tigeot #define done (((reg_value = I915_READ_FW(reg)) & mask) == value)
1625*3f2dd94aSFrançois Tigeot 	int ret;
1626*3f2dd94aSFrançois Tigeot 
1627*3f2dd94aSFrançois Tigeot 	/* Catch any overuse of this function */
1628*3f2dd94aSFrançois Tigeot 	might_sleep_if(slow_timeout_ms);
1629*3f2dd94aSFrançois Tigeot 	GEM_BUG_ON(fast_timeout_us > 20000);
1630*3f2dd94aSFrançois Tigeot 
1631*3f2dd94aSFrançois Tigeot 	ret = -ETIMEDOUT;
1632*3f2dd94aSFrançois Tigeot 	if (fast_timeout_us && fast_timeout_us <= 20000)
1633*3f2dd94aSFrançois Tigeot 		ret = _wait_for_atomic(done, fast_timeout_us, 0);
1634*3f2dd94aSFrançois Tigeot 	if (ret && slow_timeout_ms)
1635*3f2dd94aSFrançois Tigeot 		ret = wait_for(done, slow_timeout_ms);
1636*3f2dd94aSFrançois Tigeot 
1637*3f2dd94aSFrançois Tigeot 	if (out_value)
1638*3f2dd94aSFrançois Tigeot 		*out_value = reg_value;
1639*3f2dd94aSFrançois Tigeot 
16401487f786SFrançois Tigeot 	return ret;
16411487f786SFrançois Tigeot #undef done
16421487f786SFrançois Tigeot }
16431487f786SFrançois Tigeot 
16441487f786SFrançois Tigeot /**
16451487f786SFrançois Tigeot  * intel_wait_for_register - wait until register matches expected state
16461487f786SFrançois Tigeot  * @dev_priv: the i915 device
16471487f786SFrançois Tigeot  * @reg: the register to read
16481487f786SFrançois Tigeot  * @mask: mask to apply to register value
16491487f786SFrançois Tigeot  * @value: expected value
16501487f786SFrançois Tigeot  * @timeout_ms: timeout in millisecond
16511487f786SFrançois Tigeot  *
16521487f786SFrançois Tigeot  * This routine waits until the target register @reg contains the expected
165387df8fc6SFrançois Tigeot  * @value after applying the @mask, i.e. it waits until ::
165487df8fc6SFrançois Tigeot  *
165587df8fc6SFrançois Tigeot  *     (I915_READ(reg) & mask) == value
165687df8fc6SFrançois Tigeot  *
16571487f786SFrançois Tigeot  * Otherwise, the wait will timeout after @timeout_ms milliseconds.
16581487f786SFrançois Tigeot  *
16591487f786SFrançois Tigeot  * Returns 0 if the register matches the desired condition, or -ETIMEOUT.
16601487f786SFrançois Tigeot  */
intel_wait_for_register(struct drm_i915_private * dev_priv,i915_reg_t reg,u32 mask,u32 value,unsigned int timeout_ms)16611487f786SFrançois Tigeot int intel_wait_for_register(struct drm_i915_private *dev_priv,
16621487f786SFrançois Tigeot 			    i915_reg_t reg,
1663*3f2dd94aSFrançois Tigeot 			    u32 mask,
1664*3f2dd94aSFrançois Tigeot 			    u32 value,
1665*3f2dd94aSFrançois Tigeot 			    unsigned int timeout_ms)
16661487f786SFrançois Tigeot {
16671487f786SFrançois Tigeot 	unsigned fw =
16681487f786SFrançois Tigeot 		intel_uncore_forcewake_for_reg(dev_priv, reg, FW_REG_READ);
16691487f786SFrançois Tigeot 	int ret;
16701487f786SFrançois Tigeot 
1671*3f2dd94aSFrançois Tigeot 	might_sleep();
1672*3f2dd94aSFrançois Tigeot 
1673*3f2dd94aSFrançois Tigeot 	spin_lock_irq(&dev_priv->uncore.lock);
1674*3f2dd94aSFrançois Tigeot 	intel_uncore_forcewake_get__locked(dev_priv, fw);
1675*3f2dd94aSFrançois Tigeot 
1676*3f2dd94aSFrançois Tigeot 	ret = __intel_wait_for_register_fw(dev_priv,
1677*3f2dd94aSFrançois Tigeot 					   reg, mask, value,
1678*3f2dd94aSFrançois Tigeot 					   2, 0, NULL);
1679*3f2dd94aSFrançois Tigeot 
1680*3f2dd94aSFrançois Tigeot 	intel_uncore_forcewake_put__locked(dev_priv, fw);
1681*3f2dd94aSFrançois Tigeot 	spin_unlock_irq(&dev_priv->uncore.lock);
1682*3f2dd94aSFrançois Tigeot 
16831487f786SFrançois Tigeot 	if (ret)
16841487f786SFrançois Tigeot 		ret = wait_for((I915_READ_NOTRACE(reg) & mask) == value,
16851487f786SFrançois Tigeot 			       timeout_ms);
16861487f786SFrançois Tigeot 
16871487f786SFrançois Tigeot 	return ret;
1688a05eeebfSFrançois Tigeot }
1689a05eeebfSFrançois Tigeot 
gen8_reset_engine_start(struct intel_engine_cs * engine)1690*3f2dd94aSFrançois Tigeot static int gen8_reset_engine_start(struct intel_engine_cs *engine)
1691a05eeebfSFrançois Tigeot {
16921487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
16938621f407SFrançois Tigeot 	int ret;
1694a05eeebfSFrançois Tigeot 
16958621f407SFrançois Tigeot 	I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
1696a05eeebfSFrançois Tigeot 		      _MASKED_BIT_ENABLE(RESET_CTL_REQUEST_RESET));
1697a05eeebfSFrançois Tigeot 
16981487f786SFrançois Tigeot 	ret = intel_wait_for_register_fw(dev_priv,
1699a05eeebfSFrançois Tigeot 					 RING_RESET_CTL(engine->mmio_base),
1700a05eeebfSFrançois Tigeot 					 RESET_CTL_READY_TO_RESET,
1701a05eeebfSFrançois Tigeot 					 RESET_CTL_READY_TO_RESET,
17028621f407SFrançois Tigeot 					 700);
17038621f407SFrançois Tigeot 	if (ret)
1704a05eeebfSFrançois Tigeot 		DRM_ERROR("%s: reset request timeout\n", engine->name);
17058621f407SFrançois Tigeot 
17068621f407SFrançois Tigeot 	return ret;
1707a05eeebfSFrançois Tigeot }
1708a05eeebfSFrançois Tigeot 
gen8_reset_engine_cancel(struct intel_engine_cs * engine)1709*3f2dd94aSFrançois Tigeot static void gen8_reset_engine_cancel(struct intel_engine_cs *engine)
17108621f407SFrançois Tigeot {
17111487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
17128621f407SFrançois Tigeot 
17138621f407SFrançois Tigeot 	I915_WRITE_FW(RING_RESET_CTL(engine->mmio_base),
17148621f407SFrançois Tigeot 		      _MASKED_BIT_DISABLE(RESET_CTL_REQUEST_RESET));
17158621f407SFrançois Tigeot }
17168621f407SFrançois Tigeot 
gen8_reset_engines(struct drm_i915_private * dev_priv,unsigned engine_mask)17171487f786SFrançois Tigeot static int gen8_reset_engines(struct drm_i915_private *dev_priv,
17181487f786SFrançois Tigeot 			      unsigned engine_mask)
17198621f407SFrançois Tigeot {
17208621f407SFrançois Tigeot 	struct intel_engine_cs *engine;
17211e12ee3bSFrançois Tigeot 	unsigned int tmp;
17228621f407SFrançois Tigeot 
17231e12ee3bSFrançois Tigeot 	for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
1724*3f2dd94aSFrançois Tigeot 		if (gen8_reset_engine_start(engine))
17258621f407SFrançois Tigeot 			goto not_ready;
17268621f407SFrançois Tigeot 
17271487f786SFrançois Tigeot 	return gen6_reset_engines(dev_priv, engine_mask);
1728a05eeebfSFrançois Tigeot 
1729a05eeebfSFrançois Tigeot not_ready:
17301e12ee3bSFrançois Tigeot 	for_each_engine_masked(engine, dev_priv, engine_mask, tmp)
1731*3f2dd94aSFrançois Tigeot 		gen8_reset_engine_cancel(engine);
1732a05eeebfSFrançois Tigeot 
1733a05eeebfSFrançois Tigeot 	return -EIO;
1734a05eeebfSFrançois Tigeot }
1735a05eeebfSFrançois Tigeot 
17361487f786SFrançois Tigeot typedef int (*reset_func)(struct drm_i915_private *, unsigned engine_mask);
17371487f786SFrançois Tigeot 
intel_get_gpu_reset(struct drm_i915_private * dev_priv)17381487f786SFrançois Tigeot static reset_func intel_get_gpu_reset(struct drm_i915_private *dev_priv)
1739a05eeebfSFrançois Tigeot {
1740*3f2dd94aSFrançois Tigeot 	if (!i915_modparams.reset)
1741a05eeebfSFrançois Tigeot 		return NULL;
1742a05eeebfSFrançois Tigeot 
17431487f786SFrançois Tigeot 	if (INTEL_INFO(dev_priv)->gen >= 8)
17448621f407SFrançois Tigeot 		return gen8_reset_engines;
17451487f786SFrançois Tigeot 	else if (INTEL_INFO(dev_priv)->gen >= 6)
17468621f407SFrançois Tigeot 		return gen6_reset_engines;
17471487f786SFrançois Tigeot 	else if (IS_GEN5(dev_priv))
1748a05eeebfSFrançois Tigeot 		return ironlake_do_reset;
17491487f786SFrançois Tigeot 	else if (IS_G4X(dev_priv))
1750a05eeebfSFrançois Tigeot 		return g4x_do_reset;
1751a85cb24fSFrançois Tigeot 	else if (IS_G33(dev_priv) || IS_PINEVIEW(dev_priv))
1752a05eeebfSFrançois Tigeot 		return g33_do_reset;
17531487f786SFrançois Tigeot 	else if (INTEL_INFO(dev_priv)->gen >= 3)
1754a05eeebfSFrançois Tigeot 		return i915_do_reset;
1755a05eeebfSFrançois Tigeot 	else
1756a05eeebfSFrançois Tigeot 		return NULL;
1757a05eeebfSFrançois Tigeot }
1758a05eeebfSFrançois Tigeot 
intel_gpu_reset(struct drm_i915_private * dev_priv,unsigned engine_mask)17591487f786SFrançois Tigeot int intel_gpu_reset(struct drm_i915_private *dev_priv, unsigned engine_mask)
17609edbd4a0SFrançois Tigeot {
1761*3f2dd94aSFrançois Tigeot 	reset_func reset = intel_get_gpu_reset(dev_priv);
1762*3f2dd94aSFrançois Tigeot 	int retry;
1763352ff8bdSFrançois Tigeot 	int ret;
1764a05eeebfSFrançois Tigeot 
1765*3f2dd94aSFrançois Tigeot 	might_sleep();
1766a05eeebfSFrançois Tigeot 
1767352ff8bdSFrançois Tigeot 	/* If the power well sleeps during the reset, the reset
1768352ff8bdSFrançois Tigeot 	 * request may be dropped and never completes (causing -EIO).
1769352ff8bdSFrançois Tigeot 	 */
1770352ff8bdSFrançois Tigeot 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
1771*3f2dd94aSFrançois Tigeot 	for (retry = 0; retry < 3; retry++) {
1772*3f2dd94aSFrançois Tigeot 
1773*3f2dd94aSFrançois Tigeot 		/* We stop engines, otherwise we might get failed reset and a
1774*3f2dd94aSFrançois Tigeot 		 * dead gpu (on elk). Also as modern gpu as kbl can suffer
1775*3f2dd94aSFrançois Tigeot 		 * from system hang if batchbuffer is progressing when
1776*3f2dd94aSFrançois Tigeot 		 * the reset is issued, regardless of READY_TO_RESET ack.
1777*3f2dd94aSFrançois Tigeot 		 * Thus assume it is best to stop engines on all gens
1778*3f2dd94aSFrançois Tigeot 		 * where we have a gpu reset.
1779*3f2dd94aSFrançois Tigeot 		 *
1780*3f2dd94aSFrançois Tigeot 		 * WaMediaResetMainRingCleanup:ctg,elk (presumably)
1781*3f2dd94aSFrançois Tigeot 		 *
1782*3f2dd94aSFrançois Tigeot 		 * FIXME: Wa for more modern gens needs to be validated
1783*3f2dd94aSFrançois Tigeot 		 */
1784*3f2dd94aSFrançois Tigeot 		i915_stop_engines(dev_priv, engine_mask);
1785*3f2dd94aSFrançois Tigeot 
1786*3f2dd94aSFrançois Tigeot 		ret = -ENODEV;
1787*3f2dd94aSFrançois Tigeot 		if (reset)
17881487f786SFrançois Tigeot 			ret = reset(dev_priv, engine_mask);
1789*3f2dd94aSFrançois Tigeot 		if (ret != -ETIMEDOUT)
1790*3f2dd94aSFrançois Tigeot 			break;
1791*3f2dd94aSFrançois Tigeot 
1792*3f2dd94aSFrançois Tigeot 		cond_resched();
1793*3f2dd94aSFrançois Tigeot 	}
1794352ff8bdSFrançois Tigeot 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
1795352ff8bdSFrançois Tigeot 
1796352ff8bdSFrançois Tigeot 	return ret;
1797a05eeebfSFrançois Tigeot }
1798a05eeebfSFrançois Tigeot 
intel_has_gpu_reset(struct drm_i915_private * dev_priv)17991487f786SFrançois Tigeot bool intel_has_gpu_reset(struct drm_i915_private *dev_priv)
1800a05eeebfSFrançois Tigeot {
18011487f786SFrançois Tigeot 	return intel_get_gpu_reset(dev_priv) != NULL;
18029edbd4a0SFrançois Tigeot }
18039edbd4a0SFrançois Tigeot 
1804*3f2dd94aSFrançois Tigeot /*
1805*3f2dd94aSFrançois Tigeot  * When GuC submission is enabled, GuC manages ELSP and can initiate the
1806*3f2dd94aSFrançois Tigeot  * engine reset too. For now, fall back to full GPU reset if it is enabled.
1807*3f2dd94aSFrançois Tigeot  */
intel_has_reset_engine(struct drm_i915_private * dev_priv)1808*3f2dd94aSFrançois Tigeot bool intel_has_reset_engine(struct drm_i915_private *dev_priv)
1809*3f2dd94aSFrançois Tigeot {
1810*3f2dd94aSFrançois Tigeot 	return (dev_priv->info.has_reset_engine &&
1811*3f2dd94aSFrançois Tigeot 		!dev_priv->guc.execbuf_client &&
1812*3f2dd94aSFrançois Tigeot 		i915_modparams.reset >= 2);
1813*3f2dd94aSFrançois Tigeot }
1814*3f2dd94aSFrançois Tigeot 
intel_guc_reset(struct drm_i915_private * dev_priv)18158621f407SFrançois Tigeot int intel_guc_reset(struct drm_i915_private *dev_priv)
18168621f407SFrançois Tigeot {
18178621f407SFrançois Tigeot 	int ret;
18188621f407SFrançois Tigeot 
18191487f786SFrançois Tigeot 	if (!HAS_GUC(dev_priv))
18208621f407SFrançois Tigeot 		return -EINVAL;
18218621f407SFrançois Tigeot 
18228621f407SFrançois Tigeot 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
18238621f407SFrançois Tigeot 	ret = gen6_hw_domain_reset(dev_priv, GEN9_GRDOM_GUC);
18248621f407SFrançois Tigeot 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
18258621f407SFrançois Tigeot 
18268621f407SFrançois Tigeot 	return ret;
18278621f407SFrançois Tigeot }
18288621f407SFrançois Tigeot 
intel_uncore_unclaimed_mmio(struct drm_i915_private * dev_priv)1829c0e85e96SFrançois Tigeot bool intel_uncore_unclaimed_mmio(struct drm_i915_private *dev_priv)
18309edbd4a0SFrançois Tigeot {
1831c0e85e96SFrançois Tigeot 	return check_for_unclaimed_mmio(dev_priv);
18329edbd4a0SFrançois Tigeot }
1833c0e85e96SFrançois Tigeot 
1834c0e85e96SFrançois Tigeot bool
intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private * dev_priv)1835c0e85e96SFrançois Tigeot intel_uncore_arm_unclaimed_mmio_detection(struct drm_i915_private *dev_priv)
1836c0e85e96SFrançois Tigeot {
1837*3f2dd94aSFrançois Tigeot 	if (unlikely(i915_modparams.mmio_debug ||
1838c0e85e96SFrançois Tigeot 		     dev_priv->uncore.unclaimed_mmio_check <= 0))
1839c0e85e96SFrançois Tigeot 		return false;
1840c0e85e96SFrançois Tigeot 
1841c0e85e96SFrançois Tigeot 	if (unlikely(intel_uncore_unclaimed_mmio(dev_priv))) {
1842c0e85e96SFrançois Tigeot 		DRM_DEBUG("Unclaimed register detected, "
1843c0e85e96SFrançois Tigeot 			  "enabling oneshot unclaimed register reporting. "
1844c0e85e96SFrançois Tigeot 			  "Please use i915.mmio_debug=N for more information.\n");
1845*3f2dd94aSFrançois Tigeot 		i915_modparams.mmio_debug++;
1846c0e85e96SFrançois Tigeot 		dev_priv->uncore.unclaimed_mmio_check--;
1847c0e85e96SFrançois Tigeot 		return true;
1848c0e85e96SFrançois Tigeot 	}
1849c0e85e96SFrançois Tigeot 
1850c0e85e96SFrançois Tigeot 	return false;
18519edbd4a0SFrançois Tigeot }
18528621f407SFrançois Tigeot 
18538621f407SFrançois Tigeot static enum forcewake_domains
intel_uncore_forcewake_for_read(struct drm_i915_private * dev_priv,i915_reg_t reg)18548621f407SFrançois Tigeot intel_uncore_forcewake_for_read(struct drm_i915_private *dev_priv,
18558621f407SFrançois Tigeot 				i915_reg_t reg)
18568621f407SFrançois Tigeot {
18571e12ee3bSFrançois Tigeot 	u32 offset = i915_mmio_reg_offset(reg);
18588621f407SFrançois Tigeot 	enum forcewake_domains fw_domains;
18598621f407SFrançois Tigeot 
18601e12ee3bSFrançois Tigeot 	if (HAS_FWTABLE(dev_priv)) {
18611e12ee3bSFrançois Tigeot 		fw_domains = __fwtable_reg_read_fw_domains(offset);
18621e12ee3bSFrançois Tigeot 	} else if (INTEL_GEN(dev_priv) >= 6) {
18631e12ee3bSFrançois Tigeot 		fw_domains = __gen6_reg_read_fw_domains(offset);
18641e12ee3bSFrançois Tigeot 	} else {
18651e12ee3bSFrançois Tigeot 		WARN_ON(!IS_GEN(dev_priv, 2, 5));
18661e12ee3bSFrançois Tigeot 		fw_domains = 0;
18678621f407SFrançois Tigeot 	}
18688621f407SFrançois Tigeot 
18698621f407SFrançois Tigeot 	WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
18708621f407SFrançois Tigeot 
18718621f407SFrançois Tigeot 	return fw_domains;
18728621f407SFrançois Tigeot }
18738621f407SFrançois Tigeot 
18748621f407SFrançois Tigeot static enum forcewake_domains
intel_uncore_forcewake_for_write(struct drm_i915_private * dev_priv,i915_reg_t reg)18758621f407SFrançois Tigeot intel_uncore_forcewake_for_write(struct drm_i915_private *dev_priv,
18768621f407SFrançois Tigeot 				 i915_reg_t reg)
18778621f407SFrançois Tigeot {
18781e12ee3bSFrançois Tigeot 	u32 offset = i915_mmio_reg_offset(reg);
18798621f407SFrançois Tigeot 	enum forcewake_domains fw_domains;
18808621f407SFrançois Tigeot 
18811e12ee3bSFrançois Tigeot 	if (HAS_FWTABLE(dev_priv) && !IS_VALLEYVIEW(dev_priv)) {
18821e12ee3bSFrançois Tigeot 		fw_domains = __fwtable_reg_write_fw_domains(offset);
18831e12ee3bSFrançois Tigeot 	} else if (IS_GEN8(dev_priv)) {
18841e12ee3bSFrançois Tigeot 		fw_domains = __gen8_reg_write_fw_domains(offset);
18851e12ee3bSFrançois Tigeot 	} else if (IS_GEN(dev_priv, 6, 7)) {
18868621f407SFrançois Tigeot 		fw_domains = FORCEWAKE_RENDER;
18871e12ee3bSFrançois Tigeot 	} else {
18881e12ee3bSFrançois Tigeot 		WARN_ON(!IS_GEN(dev_priv, 2, 5));
18891e12ee3bSFrançois Tigeot 		fw_domains = 0;
18908621f407SFrançois Tigeot 	}
18918621f407SFrançois Tigeot 
18928621f407SFrançois Tigeot 	WARN_ON(fw_domains & ~dev_priv->uncore.fw_domains);
18938621f407SFrançois Tigeot 
18948621f407SFrançois Tigeot 	return fw_domains;
18958621f407SFrançois Tigeot }
18968621f407SFrançois Tigeot 
18978621f407SFrançois Tigeot /**
18988621f407SFrançois Tigeot  * intel_uncore_forcewake_for_reg - which forcewake domains are needed to access
18998621f407SFrançois Tigeot  * 				    a register
19008621f407SFrançois Tigeot  * @dev_priv: pointer to struct drm_i915_private
19018621f407SFrançois Tigeot  * @reg: register in question
19028621f407SFrançois Tigeot  * @op: operation bitmask of FW_REG_READ and/or FW_REG_WRITE
19038621f407SFrançois Tigeot  *
19048621f407SFrançois Tigeot  * Returns a set of forcewake domains required to be taken with for example
19058621f407SFrançois Tigeot  * intel_uncore_forcewake_get for the specified register to be accessible in the
19068621f407SFrançois Tigeot  * specified mode (read, write or read/write) with raw mmio accessors.
19078621f407SFrançois Tigeot  *
19088621f407SFrançois Tigeot  * NOTE: On Gen6 and Gen7 write forcewake domain (FORCEWAKE_RENDER) requires the
19098621f407SFrançois Tigeot  * callers to do FIFO management on their own or risk losing writes.
19108621f407SFrançois Tigeot  */
19118621f407SFrançois Tigeot enum forcewake_domains
intel_uncore_forcewake_for_reg(struct drm_i915_private * dev_priv,i915_reg_t reg,unsigned int op)19128621f407SFrançois Tigeot intel_uncore_forcewake_for_reg(struct drm_i915_private *dev_priv,
19138621f407SFrançois Tigeot 			       i915_reg_t reg, unsigned int op)
19148621f407SFrançois Tigeot {
19158621f407SFrançois Tigeot 	enum forcewake_domains fw_domains = 0;
19168621f407SFrançois Tigeot 
19178621f407SFrançois Tigeot 	WARN_ON(!op);
19188621f407SFrançois Tigeot 
19191e12ee3bSFrançois Tigeot 	if (intel_vgpu_active(dev_priv))
19201e12ee3bSFrançois Tigeot 		return 0;
19211e12ee3bSFrançois Tigeot 
19228621f407SFrançois Tigeot 	if (op & FW_REG_READ)
19238621f407SFrançois Tigeot 		fw_domains = intel_uncore_forcewake_for_read(dev_priv, reg);
19248621f407SFrançois Tigeot 
19258621f407SFrançois Tigeot 	if (op & FW_REG_WRITE)
19268621f407SFrançois Tigeot 		fw_domains |= intel_uncore_forcewake_for_write(dev_priv, reg);
19278621f407SFrançois Tigeot 
19288621f407SFrançois Tigeot 	return fw_domains;
19298621f407SFrançois Tigeot }
1930a85cb24fSFrançois Tigeot 
1931a85cb24fSFrançois Tigeot #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1932*3f2dd94aSFrançois Tigeot #include "selftests/mock_uncore.c"
1933a85cb24fSFrançois Tigeot #include "selftests/intel_uncore.c"
1934a85cb24fSFrançois Tigeot #endif
1935