xref: /dflybsd-src/sys/dev/drm/i915/intel_ringbuffer.h (revision 3f2dd94a569761201b5b0a18b2f697f97fe1b9dc)
1e3adcf8fSFrançois Tigeot #ifndef _INTEL_RINGBUFFER_H_
2e3adcf8fSFrançois Tigeot #define _INTEL_RINGBUFFER_H_
3e3adcf8fSFrançois Tigeot 
4ba55f2f5SFrançois Tigeot #include <linux/hashtable.h>
519c468b4SFrançois Tigeot #include "i915_gem_batch_pool.h"
671f41f3eSFrançois Tigeot #include "i915_gem_request.h"
74be47400SFrançois Tigeot #include "i915_gem_timeline.h"
8a85cb24fSFrançois Tigeot #include "i915_selftest.h"
9ba55f2f5SFrançois Tigeot 
10*3f2dd94aSFrançois Tigeot struct drm_printer;
11*3f2dd94aSFrançois Tigeot 
12ba55f2f5SFrançois Tigeot #define I915_CMD_HASH_ORDER 9
13ba55f2f5SFrançois Tigeot 
141b13d190SFrançois Tigeot /* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
151b13d190SFrançois Tigeot  * but keeps the logic simple. Indeed, the whole purpose of this macro is just
161b13d190SFrançois Tigeot  * to give some inclination as to some of the magic values used in the various
171b13d190SFrançois Tigeot  * workarounds!
181b13d190SFrançois Tigeot  */
191b13d190SFrançois Tigeot #define CACHELINE_BYTES 64
20a05eeebfSFrançois Tigeot #define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(uint32_t))
211b13d190SFrançois Tigeot 
22e3adcf8fSFrançois Tigeot struct intel_hw_status_page {
231e12ee3bSFrançois Tigeot 	struct i915_vma *vma;
24f4e1c372SFrançois Tigeot 	u32 *page_addr;
251e12ee3bSFrançois Tigeot 	u32 ggtt_offset;
26e3adcf8fSFrançois Tigeot };
27e3adcf8fSFrançois Tigeot 
2887df8fc6SFrançois Tigeot #define I915_READ_TAIL(engine) I915_READ(RING_TAIL((engine)->mmio_base))
2987df8fc6SFrançois Tigeot #define I915_WRITE_TAIL(engine, val) I915_WRITE(RING_TAIL((engine)->mmio_base), val)
30e3adcf8fSFrançois Tigeot 
3187df8fc6SFrançois Tigeot #define I915_READ_START(engine) I915_READ(RING_START((engine)->mmio_base))
3287df8fc6SFrançois Tigeot #define I915_WRITE_START(engine, val) I915_WRITE(RING_START((engine)->mmio_base), val)
33e3adcf8fSFrançois Tigeot 
3487df8fc6SFrançois Tigeot #define I915_READ_HEAD(engine)  I915_READ(RING_HEAD((engine)->mmio_base))
3587df8fc6SFrançois Tigeot #define I915_WRITE_HEAD(engine, val) I915_WRITE(RING_HEAD((engine)->mmio_base), val)
36e3adcf8fSFrançois Tigeot 
3787df8fc6SFrançois Tigeot #define I915_READ_CTL(engine) I915_READ(RING_CTL((engine)->mmio_base))
3887df8fc6SFrançois Tigeot #define I915_WRITE_CTL(engine, val) I915_WRITE(RING_CTL((engine)->mmio_base), val)
39e3adcf8fSFrançois Tigeot 
4087df8fc6SFrançois Tigeot #define I915_READ_IMR(engine) I915_READ(RING_IMR((engine)->mmio_base))
4187df8fc6SFrançois Tigeot #define I915_WRITE_IMR(engine, val) I915_WRITE(RING_IMR((engine)->mmio_base), val)
42e3adcf8fSFrançois Tigeot 
4387df8fc6SFrançois Tigeot #define I915_READ_MODE(engine) I915_READ(RING_MI_MODE((engine)->mmio_base))
4487df8fc6SFrançois Tigeot #define I915_WRITE_MODE(engine, val) I915_WRITE(RING_MI_MODE((engine)->mmio_base), val)
45ba55f2f5SFrançois Tigeot 
4624edb884SFrançois Tigeot /* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to
4724edb884SFrançois Tigeot  * do the writes, and that must have qw aligned offsets, simply pretend it's 8b.
4824edb884SFrançois Tigeot  */
498621f407SFrançois Tigeot #define gen8_semaphore_seqno_size sizeof(uint64_t)
508621f407SFrançois Tigeot #define GEN8_SEMAPHORE_OFFSET(__from, __to)			     \
518621f407SFrançois Tigeot 	(((__from) * I915_NUM_ENGINES  + (__to)) * gen8_semaphore_seqno_size)
5224edb884SFrançois Tigeot #define GEN8_SIGNAL_OFFSET(__ring, to)			     \
531e12ee3bSFrançois Tigeot 	(dev_priv->semaphore->node.start + \
548621f407SFrançois Tigeot 	 GEN8_SEMAPHORE_OFFSET((__ring)->id, (to)))
5524edb884SFrançois Tigeot #define GEN8_WAIT_OFFSET(__ring, from)			     \
561e12ee3bSFrançois Tigeot 	(dev_priv->semaphore->node.start + \
578621f407SFrançois Tigeot 	 GEN8_SEMAPHORE_OFFSET(from, (__ring)->id))
5824edb884SFrançois Tigeot 
5971f41f3eSFrançois Tigeot enum intel_engine_hangcheck_action {
60a85cb24fSFrançois Tigeot 	ENGINE_IDLE = 0,
61a85cb24fSFrançois Tigeot 	ENGINE_WAIT,
62a85cb24fSFrançois Tigeot 	ENGINE_ACTIVE_SEQNO,
63a85cb24fSFrançois Tigeot 	ENGINE_ACTIVE_HEAD,
64a85cb24fSFrançois Tigeot 	ENGINE_ACTIVE_SUBUNITS,
65a85cb24fSFrançois Tigeot 	ENGINE_WAIT_KICK,
66a85cb24fSFrançois Tigeot 	ENGINE_DEAD,
679edbd4a0SFrançois Tigeot };
685d0b1887SFrançois Tigeot 
69a85cb24fSFrançois Tigeot static inline const char *
hangcheck_action_to_str(const enum intel_engine_hangcheck_action a)70a85cb24fSFrançois Tigeot hangcheck_action_to_str(const enum intel_engine_hangcheck_action a)
71a85cb24fSFrançois Tigeot {
72a85cb24fSFrançois Tigeot 	switch (a) {
73a85cb24fSFrançois Tigeot 	case ENGINE_IDLE:
74a85cb24fSFrançois Tigeot 		return "idle";
75a85cb24fSFrançois Tigeot 	case ENGINE_WAIT:
76a85cb24fSFrançois Tigeot 		return "wait";
77a85cb24fSFrançois Tigeot 	case ENGINE_ACTIVE_SEQNO:
78a85cb24fSFrançois Tigeot 		return "active seqno";
79a85cb24fSFrançois Tigeot 	case ENGINE_ACTIVE_HEAD:
80a85cb24fSFrançois Tigeot 		return "active head";
81a85cb24fSFrançois Tigeot 	case ENGINE_ACTIVE_SUBUNITS:
82a85cb24fSFrançois Tigeot 		return "active subunits";
83a85cb24fSFrançois Tigeot 	case ENGINE_WAIT_KICK:
84a85cb24fSFrançois Tigeot 		return "wait kick";
85a85cb24fSFrançois Tigeot 	case ENGINE_DEAD:
86a85cb24fSFrançois Tigeot 		return "dead";
87a85cb24fSFrançois Tigeot 	}
88a85cb24fSFrançois Tigeot 
89a85cb24fSFrançois Tigeot 	return "unknown";
90a85cb24fSFrançois Tigeot }
91ba55f2f5SFrançois Tigeot 
921e12ee3bSFrançois Tigeot #define I915_MAX_SLICES	3
931e12ee3bSFrançois Tigeot #define I915_MAX_SUBSLICES 3
941e12ee3bSFrançois Tigeot 
951e12ee3bSFrançois Tigeot #define instdone_slice_mask(dev_priv__) \
961e12ee3bSFrançois Tigeot 	(INTEL_GEN(dev_priv__) == 7 ? \
971e12ee3bSFrançois Tigeot 	 1 : INTEL_INFO(dev_priv__)->sseu.slice_mask)
981e12ee3bSFrançois Tigeot 
991e12ee3bSFrançois Tigeot #define instdone_subslice_mask(dev_priv__) \
1001e12ee3bSFrançois Tigeot 	(INTEL_GEN(dev_priv__) == 7 ? \
1011e12ee3bSFrançois Tigeot 	 1 : INTEL_INFO(dev_priv__)->sseu.subslice_mask)
1021e12ee3bSFrançois Tigeot 
1031e12ee3bSFrançois Tigeot #define for_each_instdone_slice_subslice(dev_priv__, slice__, subslice__) \
1041e12ee3bSFrançois Tigeot 	for ((slice__) = 0, (subslice__) = 0; \
1051e12ee3bSFrançois Tigeot 	     (slice__) < I915_MAX_SLICES; \
1061e12ee3bSFrançois Tigeot 	     (subslice__) = ((subslice__) + 1) < I915_MAX_SUBSLICES ? (subslice__) + 1 : 0, \
1071e12ee3bSFrançois Tigeot 	       (slice__) += ((subslice__) == 0)) \
1081e12ee3bSFrançois Tigeot 		for_each_if((BIT(slice__) & instdone_slice_mask(dev_priv__)) && \
1091e12ee3bSFrançois Tigeot 			    (BIT(subslice__) & instdone_subslice_mask(dev_priv__)))
1101e12ee3bSFrançois Tigeot 
1111e12ee3bSFrançois Tigeot struct intel_instdone {
1121e12ee3bSFrançois Tigeot 	u32 instdone;
1131e12ee3bSFrançois Tigeot 	/* The following exist only in the RCS engine */
1141e12ee3bSFrançois Tigeot 	u32 slice_common;
1151e12ee3bSFrançois Tigeot 	u32 sampler[I915_MAX_SLICES][I915_MAX_SUBSLICES];
1161e12ee3bSFrançois Tigeot 	u32 row[I915_MAX_SLICES][I915_MAX_SUBSLICES];
1171e12ee3bSFrançois Tigeot };
1181e12ee3bSFrançois Tigeot 
11971f41f3eSFrançois Tigeot struct intel_engine_hangcheck {
120ba55f2f5SFrançois Tigeot 	u64 acthd;
1215d0b1887SFrançois Tigeot 	u32 seqno;
12271f41f3eSFrançois Tigeot 	enum intel_engine_hangcheck_action action;
123a85cb24fSFrançois Tigeot 	unsigned long action_timestamp;
124ba55f2f5SFrançois Tigeot 	int deadlock;
1251e12ee3bSFrançois Tigeot 	struct intel_instdone instdone;
126*3f2dd94aSFrançois Tigeot 	struct drm_i915_gem_request *active_request;
127a85cb24fSFrançois Tigeot 	bool stalled;
1285d0b1887SFrançois Tigeot };
1295d0b1887SFrançois Tigeot 
13071f41f3eSFrançois Tigeot struct intel_ring {
131c0e85e96SFrançois Tigeot 	struct i915_vma *vma;
1321e12ee3bSFrançois Tigeot 	void *vaddr;
133e3adcf8fSFrançois Tigeot 
13471f41f3eSFrançois Tigeot 	struct list_head request_list;
13571f41f3eSFrançois Tigeot 
13615ac6249SFrançois Tigeot 	u32 head;
13715ac6249SFrançois Tigeot 	u32 tail;
138a85cb24fSFrançois Tigeot 	u32 emit;
139a85cb24fSFrançois Tigeot 
140*3f2dd94aSFrançois Tigeot 	u32 space;
141*3f2dd94aSFrançois Tigeot 	u32 size;
142*3f2dd94aSFrançois Tigeot 	u32 effective_size;
143ba55f2f5SFrançois Tigeot };
144ba55f2f5SFrançois Tigeot 
1451487f786SFrançois Tigeot struct i915_gem_context;
1468621f407SFrançois Tigeot struct drm_i915_reg_table;
1472c9916cdSFrançois Tigeot 
148a05eeebfSFrançois Tigeot /*
149a05eeebfSFrançois Tigeot  * we use a single page to load ctx workarounds so all of these
150a05eeebfSFrançois Tigeot  * values are referred in terms of dwords
151a05eeebfSFrançois Tigeot  *
152a05eeebfSFrançois Tigeot  * struct i915_wa_ctx_bb:
153a05eeebfSFrançois Tigeot  *  offset: specifies batch starting position, also helpful in case
154a05eeebfSFrançois Tigeot  *    if we want to have multiple batches at different offsets based on
155a05eeebfSFrançois Tigeot  *    some criteria. It is not a requirement at the moment but provides
156a05eeebfSFrançois Tigeot  *    an option for future use.
157a05eeebfSFrançois Tigeot  *  size: size of the batch in DWORDS
158a05eeebfSFrançois Tigeot  */
159a05eeebfSFrançois Tigeot struct i915_ctx_workarounds {
160a05eeebfSFrançois Tigeot 	struct i915_wa_ctx_bb {
161a05eeebfSFrançois Tigeot 		u32 offset;
162a05eeebfSFrançois Tigeot 		u32 size;
163a05eeebfSFrançois Tigeot 	} indirect_ctx, per_ctx;
1641e12ee3bSFrançois Tigeot 	struct i915_vma *vma;
165a05eeebfSFrançois Tigeot };
166a05eeebfSFrançois Tigeot 
167303bf270SFrançois Tigeot struct drm_i915_gem_request;
1684be47400SFrançois Tigeot struct intel_render_state;
169303bf270SFrançois Tigeot 
170a85cb24fSFrançois Tigeot /*
171a85cb24fSFrançois Tigeot  * Engine IDs definitions.
172a85cb24fSFrançois Tigeot  * Keep instances of the same type engine together.
173a85cb24fSFrançois Tigeot  */
1748621f407SFrançois Tigeot enum intel_engine_id {
175c0e85e96SFrançois Tigeot 	RCS = 0,
176ba55f2f5SFrançois Tigeot 	BCS,
177c0e85e96SFrançois Tigeot 	VCS,
178a85cb24fSFrançois Tigeot 	VCS2,
179c0e85e96SFrançois Tigeot #define _VCS(n) (VCS + (n))
180a85cb24fSFrançois Tigeot 	VECS
181a85cb24fSFrançois Tigeot };
182a85cb24fSFrançois Tigeot 
183*3f2dd94aSFrançois Tigeot struct i915_priolist {
184*3f2dd94aSFrançois Tigeot 	struct rb_node node;
185*3f2dd94aSFrançois Tigeot 	struct list_head requests;
186*3f2dd94aSFrançois Tigeot 	int priority;
187*3f2dd94aSFrançois Tigeot };
188*3f2dd94aSFrançois Tigeot 
189*3f2dd94aSFrançois Tigeot /**
190*3f2dd94aSFrançois Tigeot  * struct intel_engine_execlists - execlist submission queue and port state
191*3f2dd94aSFrançois Tigeot  *
192*3f2dd94aSFrançois Tigeot  * The struct intel_engine_execlists represents the combined logical state of
193*3f2dd94aSFrançois Tigeot  * driver and the hardware state for execlist mode of submission.
194*3f2dd94aSFrançois Tigeot  */
195*3f2dd94aSFrançois Tigeot struct intel_engine_execlists {
196*3f2dd94aSFrançois Tigeot 	/**
197*3f2dd94aSFrançois Tigeot 	 * @irq_tasklet: softirq tasklet for bottom handler
198*3f2dd94aSFrançois Tigeot 	 */
199*3f2dd94aSFrançois Tigeot 	struct tasklet_struct irq_tasklet;
200*3f2dd94aSFrançois Tigeot 
201*3f2dd94aSFrançois Tigeot 	/**
202*3f2dd94aSFrançois Tigeot 	 * @default_priolist: priority list for I915_PRIORITY_NORMAL
203*3f2dd94aSFrançois Tigeot 	 */
204*3f2dd94aSFrançois Tigeot 	struct i915_priolist default_priolist;
205*3f2dd94aSFrançois Tigeot 
206*3f2dd94aSFrançois Tigeot 	/**
207*3f2dd94aSFrançois Tigeot 	 * @no_priolist: priority lists disabled
208*3f2dd94aSFrançois Tigeot 	 */
209*3f2dd94aSFrançois Tigeot 	bool no_priolist;
210*3f2dd94aSFrançois Tigeot 
211*3f2dd94aSFrançois Tigeot 	/**
212*3f2dd94aSFrançois Tigeot 	 * @port: execlist port states
213*3f2dd94aSFrançois Tigeot 	 *
214*3f2dd94aSFrançois Tigeot 	 * For each hardware ELSP (ExecList Submission Port) we keep
215*3f2dd94aSFrançois Tigeot 	 * track of the last request and the number of times we submitted
216*3f2dd94aSFrançois Tigeot 	 * that port to hw. We then count the number of times the hw reports
217*3f2dd94aSFrançois Tigeot 	 * a context completion or preemption. As only one context can
218*3f2dd94aSFrançois Tigeot 	 * be active on hw, we limit resubmission of context to port[0]. This
219*3f2dd94aSFrançois Tigeot 	 * is called Lite Restore, of the context.
220*3f2dd94aSFrançois Tigeot 	 */
221*3f2dd94aSFrançois Tigeot 	struct execlist_port {
222*3f2dd94aSFrançois Tigeot 		/**
223*3f2dd94aSFrançois Tigeot 		 * @request_count: combined request and submission count
224*3f2dd94aSFrançois Tigeot 		 */
225*3f2dd94aSFrançois Tigeot 		struct drm_i915_gem_request *request_count;
226*3f2dd94aSFrançois Tigeot #define EXECLIST_COUNT_BITS 2
227*3f2dd94aSFrançois Tigeot #define port_request(p) ptr_mask_bits((p)->request_count, EXECLIST_COUNT_BITS)
228*3f2dd94aSFrançois Tigeot #define port_count(p) ptr_unmask_bits((p)->request_count, EXECLIST_COUNT_BITS)
229*3f2dd94aSFrançois Tigeot #define port_pack(rq, count) ptr_pack_bits(rq, count, EXECLIST_COUNT_BITS)
230*3f2dd94aSFrançois Tigeot #define port_unpack(p, count) ptr_unpack_bits((p)->request_count, count, EXECLIST_COUNT_BITS)
231*3f2dd94aSFrançois Tigeot #define port_set(p, packed) ((p)->request_count = (packed))
232*3f2dd94aSFrançois Tigeot #define port_isset(p) ((p)->request_count)
233*3f2dd94aSFrançois Tigeot #define port_index(p, execlists) ((p) - (execlists)->port)
234*3f2dd94aSFrançois Tigeot 
235*3f2dd94aSFrançois Tigeot 		/**
236*3f2dd94aSFrançois Tigeot 		 * @context_id: context ID for port
237*3f2dd94aSFrançois Tigeot 		 */
238*3f2dd94aSFrançois Tigeot 		GEM_DEBUG_DECL(u32 context_id);
239*3f2dd94aSFrançois Tigeot 
240*3f2dd94aSFrançois Tigeot #define EXECLIST_MAX_PORTS 2
241*3f2dd94aSFrançois Tigeot 	} port[EXECLIST_MAX_PORTS];
242*3f2dd94aSFrançois Tigeot 
243*3f2dd94aSFrançois Tigeot 	/**
244*3f2dd94aSFrançois Tigeot 	 * @active: is the HW active? We consider the HW as active after
245*3f2dd94aSFrançois Tigeot 	 * submitting any context for execution and until we have seen the
246*3f2dd94aSFrançois Tigeot 	 * last context completion event. After that, we do not expect any
247*3f2dd94aSFrançois Tigeot 	 * more events until we submit, and so can park the HW.
248*3f2dd94aSFrançois Tigeot 	 *
249*3f2dd94aSFrançois Tigeot 	 * As we have a small number of different sources from which we feed
250*3f2dd94aSFrançois Tigeot 	 * the HW, we track the state of each inside a single bitfield.
251*3f2dd94aSFrançois Tigeot 	 */
252*3f2dd94aSFrançois Tigeot 	unsigned int active;
253*3f2dd94aSFrançois Tigeot #define EXECLISTS_ACTIVE_USER 0
254*3f2dd94aSFrançois Tigeot #define EXECLISTS_ACTIVE_PREEMPT 1
255*3f2dd94aSFrançois Tigeot 
256*3f2dd94aSFrançois Tigeot 	/**
257*3f2dd94aSFrançois Tigeot 	 * @port_mask: number of execlist ports - 1
258*3f2dd94aSFrançois Tigeot 	 */
259*3f2dd94aSFrançois Tigeot 	unsigned int port_mask;
260*3f2dd94aSFrançois Tigeot 
261*3f2dd94aSFrançois Tigeot 	/**
262*3f2dd94aSFrançois Tigeot 	 * @queue: queue of requests, in priority lists
263*3f2dd94aSFrançois Tigeot 	 */
264*3f2dd94aSFrançois Tigeot 	struct rb_root queue;
265*3f2dd94aSFrançois Tigeot 
266*3f2dd94aSFrançois Tigeot 	/**
267*3f2dd94aSFrançois Tigeot 	 * @first: leftmost level in priority @queue
268*3f2dd94aSFrançois Tigeot 	 */
269*3f2dd94aSFrançois Tigeot 	struct rb_node *first;
270*3f2dd94aSFrançois Tigeot 
271*3f2dd94aSFrançois Tigeot 	/**
272*3f2dd94aSFrançois Tigeot 	 * @fw_domains: forcewake domains for irq tasklet
273*3f2dd94aSFrançois Tigeot 	 */
274*3f2dd94aSFrançois Tigeot 	unsigned int fw_domains;
275*3f2dd94aSFrançois Tigeot 
276*3f2dd94aSFrançois Tigeot 	/**
277*3f2dd94aSFrançois Tigeot 	 * @csb_head: context status buffer head
278*3f2dd94aSFrançois Tigeot 	 */
279*3f2dd94aSFrançois Tigeot 	unsigned int csb_head;
280*3f2dd94aSFrançois Tigeot 
281*3f2dd94aSFrançois Tigeot 	/**
282*3f2dd94aSFrançois Tigeot 	 * @csb_use_mmio: access csb through mmio, instead of hwsp
283*3f2dd94aSFrançois Tigeot 	 */
284*3f2dd94aSFrançois Tigeot 	bool csb_use_mmio;
285*3f2dd94aSFrançois Tigeot };
286*3f2dd94aSFrançois Tigeot 
287*3f2dd94aSFrançois Tigeot #define INTEL_ENGINE_CS_MAX_NAME 8
288*3f2dd94aSFrançois Tigeot 
289a85cb24fSFrançois Tigeot struct intel_engine_cs {
290a85cb24fSFrançois Tigeot 	struct drm_i915_private *i915;
291*3f2dd94aSFrançois Tigeot 	char name[INTEL_ENGINE_CS_MAX_NAME];
292a85cb24fSFrançois Tigeot 	enum intel_engine_id id;
293*3f2dd94aSFrançois Tigeot 	unsigned int uabi_id;
294a85cb24fSFrançois Tigeot 	unsigned int hw_id;
295a85cb24fSFrançois Tigeot 	unsigned int guc_id;
296*3f2dd94aSFrançois Tigeot 
297*3f2dd94aSFrançois Tigeot 	u8 class;
298*3f2dd94aSFrançois Tigeot 	u8 instance;
299*3f2dd94aSFrançois Tigeot 	u32 context_size;
300ba55f2f5SFrançois Tigeot 	u32 mmio_base;
30187df8fc6SFrançois Tigeot 	unsigned int irq_shift;
302*3f2dd94aSFrançois Tigeot 
30371f41f3eSFrançois Tigeot 	struct intel_ring *buffer;
3044be47400SFrançois Tigeot 	struct intel_timeline *timeline;
3054be47400SFrançois Tigeot 
3064be47400SFrançois Tigeot 	struct intel_render_state *render_state;
307ba55f2f5SFrançois Tigeot 
308a85cb24fSFrançois Tigeot 	atomic_t irq_count;
309a85cb24fSFrançois Tigeot 	unsigned long irq_posted;
310a85cb24fSFrançois Tigeot #define ENGINE_IRQ_BREADCRUMB 0
311a85cb24fSFrançois Tigeot #define ENGINE_IRQ_EXECLIST 1
312a85cb24fSFrançois Tigeot 
313303bf270SFrançois Tigeot 	/* Rather than have every client wait upon all user interrupts,
314303bf270SFrançois Tigeot 	 * with the herd waking after every interrupt and each doing the
315303bf270SFrançois Tigeot 	 * heavyweight seqno dance, we delegate the task (of being the
316303bf270SFrançois Tigeot 	 * bottom-half of the user interrupt) to the first client. After
317303bf270SFrançois Tigeot 	 * every interrupt, we wake up one client, who does the heavyweight
318303bf270SFrançois Tigeot 	 * coherent seqno read and either goes back to sleep (if incomplete),
319303bf270SFrançois Tigeot 	 * or wakes up all the completed clients in parallel, before then
320303bf270SFrançois Tigeot 	 * transferring the bottom-half status to the next client in the queue.
321303bf270SFrançois Tigeot 	 *
322303bf270SFrançois Tigeot 	 * Compared to walking the entire list of waiters in a single dedicated
323303bf270SFrançois Tigeot 	 * bottom-half, we reduce the latency of the first waiter by avoiding
324303bf270SFrançois Tigeot 	 * a context switch, but incur additional coherent seqno reads when
325303bf270SFrançois Tigeot 	 * following the chain of request breadcrumbs. Since it is most likely
326303bf270SFrançois Tigeot 	 * that we have a single client waiting on each seqno, then reducing
327303bf270SFrançois Tigeot 	 * the overhead of waking that client is much preferred.
328303bf270SFrançois Tigeot 	 */
329303bf270SFrançois Tigeot 	struct intel_breadcrumbs {
330a85cb24fSFrançois Tigeot 		spinlock_t irq_lock; /* protects irq_*; irqsafe */
331a85cb24fSFrançois Tigeot 		struct intel_wait *irq_wait; /* oldest waiter by retirement */
332303bf270SFrançois Tigeot 
333a85cb24fSFrançois Tigeot 		spinlock_t rb_lock; /* protects the rb and wraps irq_lock */
334303bf270SFrançois Tigeot 		struct rb_root waiters; /* sorted by retirement, priority */
335303bf270SFrançois Tigeot 		struct rb_root signals; /* sorted by retirement */
336303bf270SFrançois Tigeot 		struct task_struct *signaler; /* used for fence signalling */
337a85cb24fSFrançois Tigeot 		struct drm_i915_gem_request __rcu *first_signal;
338303bf270SFrançois Tigeot 		struct timer_list fake_irq; /* used after a missed interrupt */
3391e12ee3bSFrançois Tigeot 		struct timer_list hangcheck; /* detect missed interrupts */
3401e12ee3bSFrançois Tigeot 
341a85cb24fSFrançois Tigeot 		unsigned int hangcheck_interrupts;
342303bf270SFrançois Tigeot 
343a85cb24fSFrançois Tigeot 		bool irq_armed : 1;
344303bf270SFrançois Tigeot 		bool irq_enabled : 1;
345a85cb24fSFrançois Tigeot 		I915_SELFTEST_DECLARE(bool mock : 1);
346303bf270SFrançois Tigeot 	} breadcrumbs;
347303bf270SFrançois Tigeot 
34819c468b4SFrançois Tigeot 	/*
34919c468b4SFrançois Tigeot 	 * A pool of objects to use as shadow copies of client batch buffers
35019c468b4SFrançois Tigeot 	 * when the command parser is enabled. Prevents the client from
35119c468b4SFrançois Tigeot 	 * modifying the batch contents after software parsing.
35219c468b4SFrançois Tigeot 	 */
35319c468b4SFrançois Tigeot 	struct i915_gem_batch_pool batch_pool;
35419c468b4SFrançois Tigeot 
355ba55f2f5SFrançois Tigeot 	struct intel_hw_status_page status_page;
356a05eeebfSFrançois Tigeot 	struct i915_ctx_workarounds wa_ctx;
3571e12ee3bSFrançois Tigeot 	struct i915_vma *scratch;
358e3adcf8fSFrançois Tigeot 
359303bf270SFrançois Tigeot 	u32             irq_keep_mask; /* always keep these interrupts */
36015ac6249SFrançois Tigeot 	u32		irq_enable_mask; /* bitmask to enable ring interrupt */
36187df8fc6SFrançois Tigeot 	void		(*irq_enable)(struct intel_engine_cs *engine);
36287df8fc6SFrançois Tigeot 	void		(*irq_disable)(struct intel_engine_cs *engine);
363e3adcf8fSFrançois Tigeot 
36487df8fc6SFrançois Tigeot 	int		(*init_hw)(struct intel_engine_cs *engine);
3651e12ee3bSFrançois Tigeot 	void		(*reset_hw)(struct intel_engine_cs *engine,
3661e12ee3bSFrançois Tigeot 				    struct drm_i915_gem_request *req);
367e3adcf8fSFrançois Tigeot 
368a85cb24fSFrançois Tigeot 	void		(*set_default_submission)(struct intel_engine_cs *engine);
369a85cb24fSFrançois Tigeot 
370*3f2dd94aSFrançois Tigeot 	struct intel_ring *(*context_pin)(struct intel_engine_cs *engine,
371a85cb24fSFrançois Tigeot 					  struct i915_gem_context *ctx);
372a85cb24fSFrançois Tigeot 	void		(*context_unpin)(struct intel_engine_cs *engine,
373a85cb24fSFrançois Tigeot 					 struct i915_gem_context *ctx);
374a85cb24fSFrançois Tigeot 	int		(*request_alloc)(struct drm_i915_gem_request *req);
375a05eeebfSFrançois Tigeot 	int		(*init_context)(struct drm_i915_gem_request *req);
3761b13d190SFrançois Tigeot 
37771f41f3eSFrançois Tigeot 	int		(*emit_flush)(struct drm_i915_gem_request *request,
37871f41f3eSFrançois Tigeot 				      u32 mode);
37971f41f3eSFrançois Tigeot #define EMIT_INVALIDATE	BIT(0)
38071f41f3eSFrançois Tigeot #define EMIT_FLUSH	BIT(1)
38171f41f3eSFrançois Tigeot #define EMIT_BARRIER	(EMIT_INVALIDATE | EMIT_FLUSH)
38271f41f3eSFrançois Tigeot 	int		(*emit_bb_start)(struct drm_i915_gem_request *req,
38371f41f3eSFrançois Tigeot 					 u64 offset, u32 length,
38471f41f3eSFrançois Tigeot 					 unsigned int dispatch_flags);
38571f41f3eSFrançois Tigeot #define I915_DISPATCH_SECURE BIT(0)
38671f41f3eSFrançois Tigeot #define I915_DISPATCH_PINNED BIT(1)
38771f41f3eSFrançois Tigeot #define I915_DISPATCH_RS     BIT(2)
3884be47400SFrançois Tigeot 	void		(*emit_breadcrumb)(struct drm_i915_gem_request *req,
389a85cb24fSFrançois Tigeot 					   u32 *cs);
3904be47400SFrançois Tigeot 	int		emit_breadcrumb_sz;
3911e12ee3bSFrançois Tigeot 
3921e12ee3bSFrançois Tigeot 	/* Pass the request to the hardware queue (e.g. directly into
3931e12ee3bSFrançois Tigeot 	 * the legacy ringbuffer or to the end of an execlist).
3941e12ee3bSFrançois Tigeot 	 *
3951e12ee3bSFrançois Tigeot 	 * This is called from an atomic context with irqs disabled; must
3961e12ee3bSFrançois Tigeot 	 * be irq safe.
3971e12ee3bSFrançois Tigeot 	 */
39871f41f3eSFrançois Tigeot 	void		(*submit_request)(struct drm_i915_gem_request *req);
3991e12ee3bSFrançois Tigeot 
4004be47400SFrançois Tigeot 	/* Call when the priority on a request has changed and it and its
4014be47400SFrançois Tigeot 	 * dependencies may need rescheduling. Note the request itself may
4024be47400SFrançois Tigeot 	 * not be ready to run!
4034be47400SFrançois Tigeot 	 *
4044be47400SFrançois Tigeot 	 * Called under the struct_mutex.
4054be47400SFrançois Tigeot 	 */
4064be47400SFrançois Tigeot 	void		(*schedule)(struct drm_i915_gem_request *request,
4074be47400SFrançois Tigeot 				    int priority);
4084be47400SFrançois Tigeot 
409*3f2dd94aSFrançois Tigeot 	/*
410*3f2dd94aSFrançois Tigeot 	 * Cancel all requests on the hardware, or queued for execution.
411*3f2dd94aSFrançois Tigeot 	 * This should only cancel the ready requests that have been
412*3f2dd94aSFrançois Tigeot 	 * submitted to the engine (via the engine->submit_request callback).
413*3f2dd94aSFrançois Tigeot 	 * This is called when marking the device as wedged.
414*3f2dd94aSFrançois Tigeot 	 */
415*3f2dd94aSFrançois Tigeot 	void		(*cancel_requests)(struct intel_engine_cs *engine);
416*3f2dd94aSFrançois Tigeot 
417b030f26bSFrançois Tigeot 	/* Some chipsets are not quite as coherent as advertised and need
418b030f26bSFrançois Tigeot 	 * an expensive kick to force a true read of the up-to-date seqno.
419b030f26bSFrançois Tigeot 	 * However, the up-to-date seqno is not always required and the last
420b030f26bSFrançois Tigeot 	 * seen value is good enough. Note that the seqno will always be
421b030f26bSFrançois Tigeot 	 * monotonic, even if not coherent.
422b030f26bSFrançois Tigeot 	 */
42387df8fc6SFrançois Tigeot 	void		(*irq_seqno_barrier)(struct intel_engine_cs *engine);
42487df8fc6SFrançois Tigeot 	void		(*cleanup)(struct intel_engine_cs *engine);
425e3adcf8fSFrançois Tigeot 
42624edb884SFrançois Tigeot 	/* GEN8 signal/wait table - never trust comments!
42724edb884SFrançois Tigeot 	 *	  signal to	signal to    signal to   signal to      signal to
42824edb884SFrançois Tigeot 	 *	    RCS		   VCS          BCS        VECS		 VCS2
42924edb884SFrançois Tigeot 	 *      --------------------------------------------------------------------
43024edb884SFrançois Tigeot 	 *  RCS | NOP (0x00) | VCS (0x08) | BCS (0x10) | VECS (0x18) | VCS2 (0x20) |
43124edb884SFrançois Tigeot 	 *	|-------------------------------------------------------------------
43224edb884SFrançois Tigeot 	 *  VCS | RCS (0x28) | NOP (0x30) | BCS (0x38) | VECS (0x40) | VCS2 (0x48) |
43324edb884SFrançois Tigeot 	 *	|-------------------------------------------------------------------
43424edb884SFrançois Tigeot 	 *  BCS | RCS (0x50) | VCS (0x58) | NOP (0x60) | VECS (0x68) | VCS2 (0x70) |
43524edb884SFrançois Tigeot 	 *	|-------------------------------------------------------------------
43624edb884SFrançois Tigeot 	 * VECS | RCS (0x78) | VCS (0x80) | BCS (0x88) |  NOP (0x90) | VCS2 (0x98) |
43724edb884SFrançois Tigeot 	 *	|-------------------------------------------------------------------
43824edb884SFrançois Tigeot 	 * VCS2 | RCS (0xa0) | VCS (0xa8) | BCS (0xb0) | VECS (0xb8) | NOP  (0xc0) |
43924edb884SFrançois Tigeot 	 *	|-------------------------------------------------------------------
44024edb884SFrançois Tigeot 	 *
44124edb884SFrançois Tigeot 	 * Generalization:
44224edb884SFrançois Tigeot 	 *  f(x, y) := (x->id * NUM_RINGS * seqno_size) + (seqno_size * y->id)
44324edb884SFrançois Tigeot 	 *  ie. transpose of g(x, y)
44424edb884SFrançois Tigeot 	 *
44524edb884SFrançois Tigeot 	 *	 sync from	sync from    sync from    sync from	sync from
44624edb884SFrançois Tigeot 	 *	    RCS		   VCS          BCS        VECS		 VCS2
44724edb884SFrançois Tigeot 	 *      --------------------------------------------------------------------
44824edb884SFrançois Tigeot 	 *  RCS | NOP (0x00) | VCS (0x28) | BCS (0x50) | VECS (0x78) | VCS2 (0xa0) |
44924edb884SFrançois Tigeot 	 *	|-------------------------------------------------------------------
45024edb884SFrançois Tigeot 	 *  VCS | RCS (0x08) | NOP (0x30) | BCS (0x58) | VECS (0x80) | VCS2 (0xa8) |
45124edb884SFrançois Tigeot 	 *	|-------------------------------------------------------------------
45224edb884SFrançois Tigeot 	 *  BCS | RCS (0x10) | VCS (0x38) | NOP (0x60) | VECS (0x88) | VCS2 (0xb0) |
45324edb884SFrançois Tigeot 	 *	|-------------------------------------------------------------------
45424edb884SFrançois Tigeot 	 * VECS | RCS (0x18) | VCS (0x40) | BCS (0x68) |  NOP (0x90) | VCS2 (0xb8) |
45524edb884SFrançois Tigeot 	 *	|-------------------------------------------------------------------
45624edb884SFrançois Tigeot 	 * VCS2 | RCS (0x20) | VCS (0x48) | BCS (0x70) | VECS (0x98) |  NOP (0xc0) |
45724edb884SFrançois Tigeot 	 *	|-------------------------------------------------------------------
45824edb884SFrançois Tigeot 	 *
45924edb884SFrançois Tigeot 	 * Generalization:
46024edb884SFrançois Tigeot 	 *  g(x, y) := (y->id * NUM_RINGS * seqno_size) + (seqno_size * x->id)
46124edb884SFrançois Tigeot 	 *  ie. transpose of f(x, y)
46224edb884SFrançois Tigeot 	 */
463ba55f2f5SFrançois Tigeot 	struct {
46424edb884SFrançois Tigeot 		union {
4651e12ee3bSFrançois Tigeot #define GEN6_SEMAPHORE_LAST	VECS_HW
4661e12ee3bSFrançois Tigeot #define GEN6_NUM_SEMAPHORES	(GEN6_SEMAPHORE_LAST + 1)
4671e12ee3bSFrançois Tigeot #define GEN6_SEMAPHORES_MASK	GENMASK(GEN6_SEMAPHORE_LAST, 0)
468ba55f2f5SFrançois Tigeot 			struct {
4695d0b1887SFrançois Tigeot 				/* our mbox written by others */
4701e12ee3bSFrançois Tigeot 				u32		wait[GEN6_NUM_SEMAPHORES];
4715d0b1887SFrançois Tigeot 				/* mboxes this ring signals to */
4721e12ee3bSFrançois Tigeot 				i915_reg_t	signal[GEN6_NUM_SEMAPHORES];
473ba55f2f5SFrançois Tigeot 			} mbox;
4748621f407SFrançois Tigeot 			u64		signal_ggtt[I915_NUM_ENGINES];
47524edb884SFrançois Tigeot 		};
476ba55f2f5SFrançois Tigeot 
477ba55f2f5SFrançois Tigeot 		/* AKA wait() */
47871f41f3eSFrançois Tigeot 		int	(*sync_to)(struct drm_i915_gem_request *req,
47971f41f3eSFrançois Tigeot 				   struct drm_i915_gem_request *signal);
480a85cb24fSFrançois Tigeot 		u32	*(*signal)(struct drm_i915_gem_request *req, u32 *cs);
481ba55f2f5SFrançois Tigeot 	} semaphore;
4825d0b1887SFrançois Tigeot 
483*3f2dd94aSFrançois Tigeot 	struct intel_engine_execlists execlists;
484e3adcf8fSFrançois Tigeot 
485a85cb24fSFrançois Tigeot 	/* Contexts are pinned whilst they are active on the GPU. The last
486a85cb24fSFrançois Tigeot 	 * context executed remains active whilst the GPU is idle - the
487a85cb24fSFrançois Tigeot 	 * switch away and write to the context object only occurs on the
488a85cb24fSFrançois Tigeot 	 * next execution.  Contexts are only unpinned on retirement of the
489a85cb24fSFrançois Tigeot 	 * following request ensuring that we can always write to the object
490a85cb24fSFrançois Tigeot 	 * on the context switch even after idling. Across suspend, we switch
491a85cb24fSFrançois Tigeot 	 * to the kernel context and trash it as the save may not happen
492a85cb24fSFrançois Tigeot 	 * before the hardware is powered down.
493a85cb24fSFrançois Tigeot 	 */
494a85cb24fSFrançois Tigeot 	struct i915_gem_context *last_retired_context;
495a85cb24fSFrançois Tigeot 
496a85cb24fSFrançois Tigeot 	/* We track the current MI_SET_CONTEXT in order to eliminate
497a85cb24fSFrançois Tigeot 	 * redudant context switches. This presumes that requests are not
498a85cb24fSFrançois Tigeot 	 * reordered! Or when they are the tracking is updated along with
499a85cb24fSFrançois Tigeot 	 * the emission of individual requests into the legacy command
500a85cb24fSFrançois Tigeot 	 * stream (ring).
501a85cb24fSFrançois Tigeot 	 */
502a85cb24fSFrançois Tigeot 	struct i915_gem_context *legacy_active_context;
503a85cb24fSFrançois Tigeot 
504a85cb24fSFrançois Tigeot 	/* status_notifier: list of callbacks for context-switch changes */
505a85cb24fSFrançois Tigeot 	struct atomic_notifier_head context_status_notifier;
5065d0b1887SFrançois Tigeot 
50771f41f3eSFrançois Tigeot 	struct intel_engine_hangcheck hangcheck;
50815ac6249SFrançois Tigeot 
509ba55f2f5SFrançois Tigeot 	bool needs_cmd_parser;
510ba55f2f5SFrançois Tigeot 
511ba55f2f5SFrançois Tigeot 	/*
512ba55f2f5SFrançois Tigeot 	 * Table of commands the command parser needs to know about
51371f41f3eSFrançois Tigeot 	 * for this engine.
514ba55f2f5SFrançois Tigeot 	 */
515ba55f2f5SFrançois Tigeot 	DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER);
516ba55f2f5SFrançois Tigeot 
517ba55f2f5SFrançois Tigeot 	/*
518ba55f2f5SFrançois Tigeot 	 * Table of registers allowed in commands that read/write registers.
519ba55f2f5SFrançois Tigeot 	 */
5208621f407SFrançois Tigeot 	const struct drm_i915_reg_table *reg_tables;
5218621f407SFrançois Tigeot 	int reg_table_count;
522ba55f2f5SFrançois Tigeot 
523ba55f2f5SFrançois Tigeot 	/*
524ba55f2f5SFrançois Tigeot 	 * Returns the bitmask for the length field of the specified command.
525ba55f2f5SFrançois Tigeot 	 * Return 0 for an unrecognized/invalid command.
526ba55f2f5SFrançois Tigeot 	 *
52771f41f3eSFrançois Tigeot 	 * If the command parser finds an entry for a command in the engine's
528ba55f2f5SFrançois Tigeot 	 * cmd_tables, it gets the command's length based on the table entry.
52971f41f3eSFrançois Tigeot 	 * If not, it calls this function to determine the per-engine length
53071f41f3eSFrançois Tigeot 	 * field encoding for the command (i.e. different opcode ranges use
53171f41f3eSFrançois Tigeot 	 * certain bits to encode the command length in the header).
532ba55f2f5SFrançois Tigeot 	 */
533ba55f2f5SFrançois Tigeot 	u32 (*get_cmd_length_mask)(u32 cmd_header);
534e3adcf8fSFrançois Tigeot };
535e3adcf8fSFrançois Tigeot 
536*3f2dd94aSFrançois Tigeot static inline void
execlists_set_active(struct intel_engine_execlists * execlists,unsigned int bit)537*3f2dd94aSFrançois Tigeot execlists_set_active(struct intel_engine_execlists *execlists,
538*3f2dd94aSFrançois Tigeot 		     unsigned int bit)
539*3f2dd94aSFrançois Tigeot {
540*3f2dd94aSFrançois Tigeot 	__set_bit(bit, (unsigned long *)&execlists->active);
541*3f2dd94aSFrançois Tigeot }
542*3f2dd94aSFrançois Tigeot 
543*3f2dd94aSFrançois Tigeot static inline void
execlists_clear_active(struct intel_engine_execlists * execlists,unsigned int bit)544*3f2dd94aSFrançois Tigeot execlists_clear_active(struct intel_engine_execlists *execlists,
545*3f2dd94aSFrançois Tigeot 		       unsigned int bit)
546*3f2dd94aSFrançois Tigeot {
547*3f2dd94aSFrançois Tigeot 	__clear_bit(bit, (unsigned long *)&execlists->active);
548*3f2dd94aSFrançois Tigeot }
549*3f2dd94aSFrançois Tigeot 
550*3f2dd94aSFrançois Tigeot static inline bool
execlists_is_active(const struct intel_engine_execlists * execlists,unsigned int bit)551*3f2dd94aSFrançois Tigeot execlists_is_active(const struct intel_engine_execlists *execlists,
552*3f2dd94aSFrançois Tigeot 		    unsigned int bit)
553*3f2dd94aSFrançois Tigeot {
554*3f2dd94aSFrançois Tigeot 	return test_bit(bit, (unsigned long *)&execlists->active);
555*3f2dd94aSFrançois Tigeot }
556*3f2dd94aSFrançois Tigeot 
557*3f2dd94aSFrançois Tigeot static inline unsigned int
execlists_num_ports(const struct intel_engine_execlists * const execlists)558*3f2dd94aSFrançois Tigeot execlists_num_ports(const struct intel_engine_execlists * const execlists)
559*3f2dd94aSFrançois Tigeot {
560*3f2dd94aSFrançois Tigeot 	return execlists->port_mask + 1;
561*3f2dd94aSFrançois Tigeot }
562*3f2dd94aSFrançois Tigeot 
563*3f2dd94aSFrançois Tigeot static inline void
execlists_port_complete(struct intel_engine_execlists * const execlists,struct execlist_port * const port)564*3f2dd94aSFrançois Tigeot execlists_port_complete(struct intel_engine_execlists * const execlists,
565*3f2dd94aSFrançois Tigeot 			struct execlist_port * const port)
566*3f2dd94aSFrançois Tigeot {
567*3f2dd94aSFrançois Tigeot 	const unsigned int m = execlists->port_mask;
568*3f2dd94aSFrançois Tigeot 
569*3f2dd94aSFrançois Tigeot 	GEM_BUG_ON(port_index(port, execlists) != 0);
570*3f2dd94aSFrançois Tigeot 	GEM_BUG_ON(!execlists_is_active(execlists, EXECLISTS_ACTIVE_USER));
571*3f2dd94aSFrançois Tigeot 
572*3f2dd94aSFrançois Tigeot 	memmove(port, port + 1, m * sizeof(struct execlist_port));
573*3f2dd94aSFrançois Tigeot 	memset(port + m, 0, sizeof(struct execlist_port));
574*3f2dd94aSFrançois Tigeot }
575*3f2dd94aSFrançois Tigeot 
576a85cb24fSFrançois Tigeot static inline unsigned int
intel_engine_flag(const struct intel_engine_cs * engine)577303bf270SFrançois Tigeot intel_engine_flag(const struct intel_engine_cs *engine)
578e3adcf8fSFrançois Tigeot {
579a85cb24fSFrançois Tigeot 	return BIT(engine->id);
580352ff8bdSFrançois Tigeot }
581352ff8bdSFrançois Tigeot 
582f4e1c372SFrançois Tigeot static inline u32
intel_read_status_page(struct intel_engine_cs * engine,int reg)5838621f407SFrançois Tigeot intel_read_status_page(struct intel_engine_cs *engine, int reg)
584e3adcf8fSFrançois Tigeot {
585f4e1c372SFrançois Tigeot 	/* Ensure that the compiler doesn't optimize away the load. */
5868621f407SFrançois Tigeot 	return READ_ONCE(engine->status_page.page_addr[reg]);
587e3adcf8fSFrançois Tigeot }
588e3adcf8fSFrançois Tigeot 
589a2fdbec6SFrançois Tigeot static inline void
intel_write_status_page(struct intel_engine_cs * engine,int reg,u32 value)590a85cb24fSFrançois Tigeot intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value)
591a2fdbec6SFrançois Tigeot {
592a85cb24fSFrançois Tigeot 	/* Writing into the status page should be done sparingly. Since
593a85cb24fSFrançois Tigeot 	 * we do when we are uncertain of the device state, we take a bit
594a85cb24fSFrançois Tigeot 	 * of extra paranoia to try and ensure that the HWS takes the value
595a85cb24fSFrançois Tigeot 	 * we give and that it doesn't end up trapped inside the CPU!
596a85cb24fSFrançois Tigeot 	 */
597a85cb24fSFrançois Tigeot 	if (static_cpu_has(X86_FEATURE_CLFLUSH)) {
598a85cb24fSFrançois Tigeot 		mb();
599a85cb24fSFrançois Tigeot 		linux_clflush(&engine->status_page.page_addr[reg]);
6008621f407SFrançois Tigeot 		engine->status_page.page_addr[reg] = value;
601a85cb24fSFrançois Tigeot 		linux_clflush(&engine->status_page.page_addr[reg]);
602a85cb24fSFrançois Tigeot 		mb();
603a85cb24fSFrançois Tigeot 	} else {
604a85cb24fSFrançois Tigeot 		WRITE_ONCE(engine->status_page.page_addr[reg], value);
605a85cb24fSFrançois Tigeot 	}
606a2fdbec6SFrançois Tigeot }
607a2fdbec6SFrançois Tigeot 
608c0e85e96SFrançois Tigeot /*
609f4e1c372SFrançois Tigeot  * Reads a dword out of the status page, which is written to from the command
610f4e1c372SFrançois Tigeot  * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
611f4e1c372SFrançois Tigeot  * MI_STORE_DATA_IMM.
612f4e1c372SFrançois Tigeot  *
613f4e1c372SFrançois Tigeot  * The following dwords have a reserved meaning:
614f4e1c372SFrançois Tigeot  * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes.
615f4e1c372SFrançois Tigeot  * 0x04: ring 0 head pointer
616f4e1c372SFrançois Tigeot  * 0x05: ring 1 head pointer (915-class)
617f4e1c372SFrançois Tigeot  * 0x06: ring 2 head pointer (915-class)
618f4e1c372SFrançois Tigeot  * 0x10-0x1b: Context status DWords (GM45)
619f4e1c372SFrançois Tigeot  * 0x1f: Last written status offset. (GM45)
620477eb7f9SFrançois Tigeot  * 0x20-0x2f: Reserved (Gen6+)
621f4e1c372SFrançois Tigeot  *
622477eb7f9SFrançois Tigeot  * The area from dword 0x30 to 0x3ff is available for driver usage.
623f4e1c372SFrançois Tigeot  */
624477eb7f9SFrançois Tigeot #define I915_GEM_HWS_INDEX		0x30
625c0e85e96SFrançois Tigeot #define I915_GEM_HWS_INDEX_ADDR (I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
626477eb7f9SFrançois Tigeot #define I915_GEM_HWS_SCRATCH_INDEX	0x40
627f4e1c372SFrançois Tigeot #define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
628e3adcf8fSFrançois Tigeot 
629*3f2dd94aSFrançois Tigeot #define I915_HWS_CSB_BUF0_INDEX		0x10
630*3f2dd94aSFrançois Tigeot #define I915_HWS_CSB_WRITE_INDEX	0x1f
631*3f2dd94aSFrançois Tigeot #define CNL_HWS_CSB_WRITE_INDEX		0x2f
632*3f2dd94aSFrançois Tigeot 
63371f41f3eSFrançois Tigeot struct intel_ring *
63471f41f3eSFrançois Tigeot intel_engine_create_ring(struct intel_engine_cs *engine, int size);
635*3f2dd94aSFrançois Tigeot int intel_ring_pin(struct intel_ring *ring,
636*3f2dd94aSFrançois Tigeot 		   struct drm_i915_private *i915,
637*3f2dd94aSFrançois Tigeot 		   unsigned int offset_bias);
638a85cb24fSFrançois Tigeot void intel_ring_reset(struct intel_ring *ring, u32 tail);
639*3f2dd94aSFrançois Tigeot unsigned int intel_ring_update_space(struct intel_ring *ring);
64071f41f3eSFrançois Tigeot void intel_ring_unpin(struct intel_ring *ring);
64171f41f3eSFrançois Tigeot void intel_ring_free(struct intel_ring *ring);
6421b13d190SFrançois Tigeot 
64371f41f3eSFrançois Tigeot void intel_engine_stop(struct intel_engine_cs *engine);
64471f41f3eSFrançois Tigeot void intel_engine_cleanup(struct intel_engine_cs *engine);
645f4e1c372SFrançois Tigeot 
6461e12ee3bSFrançois Tigeot void intel_legacy_submission_resume(struct drm_i915_private *dev_priv);
6471e12ee3bSFrançois Tigeot 
648a05eeebfSFrançois Tigeot int __must_check intel_ring_cacheline_align(struct drm_i915_gem_request *req);
64987df8fc6SFrançois Tigeot 
650*3f2dd94aSFrançois Tigeot u32 __must_check *intel_ring_begin(struct drm_i915_gem_request *req,
651*3f2dd94aSFrançois Tigeot 				   unsigned int n);
65287df8fc6SFrançois Tigeot 
653a85cb24fSFrançois Tigeot static inline void
intel_ring_advance(struct drm_i915_gem_request * req,u32 * cs)654a85cb24fSFrançois Tigeot intel_ring_advance(struct drm_i915_gem_request *req, u32 *cs)
65587df8fc6SFrançois Tigeot {
65671f41f3eSFrançois Tigeot 	/* Dummy function.
65771f41f3eSFrançois Tigeot 	 *
65871f41f3eSFrançois Tigeot 	 * This serves as a placeholder in the code so that the reader
65971f41f3eSFrançois Tigeot 	 * can compare against the preceding intel_ring_begin() and
66071f41f3eSFrançois Tigeot 	 * check that the number of dwords emitted matches the space
66171f41f3eSFrançois Tigeot 	 * reserved for the command packet (i.e. the value passed to
66271f41f3eSFrançois Tigeot 	 * intel_ring_begin()).
66371f41f3eSFrançois Tigeot 	 */
664a85cb24fSFrançois Tigeot 	GEM_BUG_ON((req->ring->vaddr + req->ring->emit) != cs);
66587df8fc6SFrançois Tigeot }
66687df8fc6SFrançois Tigeot 
667a85cb24fSFrançois Tigeot static inline u32
intel_ring_wrap(const struct intel_ring * ring,u32 pos)668a85cb24fSFrançois Tigeot intel_ring_wrap(const struct intel_ring *ring, u32 pos)
669a85cb24fSFrançois Tigeot {
670a85cb24fSFrançois Tigeot 	return pos & (ring->size - 1);
671a85cb24fSFrançois Tigeot }
672a85cb24fSFrançois Tigeot 
673a85cb24fSFrançois Tigeot static inline u32
intel_ring_offset(const struct drm_i915_gem_request * req,void * addr)674a85cb24fSFrançois Tigeot intel_ring_offset(const struct drm_i915_gem_request *req, void *addr)
675aee94f86SFrançois Tigeot {
67671f41f3eSFrançois Tigeot 	/* Don't write ring->size (equivalent to 0) as that hangs some GPUs. */
677a85cb24fSFrançois Tigeot 	u32 offset = addr - req->ring->vaddr;
678a85cb24fSFrançois Tigeot 	GEM_BUG_ON(offset > req->ring->size);
679a85cb24fSFrançois Tigeot 	return intel_ring_wrap(req->ring, offset);
6809edbd4a0SFrançois Tigeot }
68187df8fc6SFrançois Tigeot 
682a85cb24fSFrançois Tigeot static inline void
assert_ring_tail_valid(const struct intel_ring * ring,unsigned int tail)683a85cb24fSFrançois Tigeot assert_ring_tail_valid(const struct intel_ring *ring, unsigned int tail)
684a85cb24fSFrançois Tigeot {
685a85cb24fSFrançois Tigeot 	/* We could combine these into a single tail operation, but keeping
686a85cb24fSFrançois Tigeot 	 * them as seperate tests will help identify the cause should one
687a85cb24fSFrançois Tigeot 	 * ever fire.
688a85cb24fSFrançois Tigeot 	 */
689a85cb24fSFrançois Tigeot 	GEM_BUG_ON(!IS_ALIGNED(tail, 8));
690a85cb24fSFrançois Tigeot 	GEM_BUG_ON(tail >= ring->size);
691*3f2dd94aSFrançois Tigeot 
692*3f2dd94aSFrançois Tigeot 	/*
693*3f2dd94aSFrançois Tigeot 	 * "Ring Buffer Use"
694*3f2dd94aSFrançois Tigeot 	 *	Gen2 BSpec "1. Programming Environment" / 1.4.4.6
695*3f2dd94aSFrançois Tigeot 	 *	Gen3 BSpec "1c Memory Interface Functions" / 2.3.4.5
696*3f2dd94aSFrançois Tigeot 	 *	Gen4+ BSpec "1c Memory Interface and Command Stream" / 5.3.4.5
697*3f2dd94aSFrançois Tigeot 	 * "If the Ring Buffer Head Pointer and the Tail Pointer are on the
698*3f2dd94aSFrançois Tigeot 	 * same cacheline, the Head Pointer must not be greater than the Tail
699*3f2dd94aSFrançois Tigeot 	 * Pointer."
700*3f2dd94aSFrançois Tigeot 	 *
701*3f2dd94aSFrançois Tigeot 	 * We use ring->head as the last known location of the actual RING_HEAD,
702*3f2dd94aSFrançois Tigeot 	 * it may have advanced but in the worst case it is equally the same
703*3f2dd94aSFrançois Tigeot 	 * as ring->head and so we should never program RING_TAIL to advance
704*3f2dd94aSFrançois Tigeot 	 * into the same cacheline as ring->head.
705*3f2dd94aSFrançois Tigeot 	 */
706*3f2dd94aSFrançois Tigeot #define cacheline(a) round_down(a, CACHELINE_BYTES)
707*3f2dd94aSFrançois Tigeot 	GEM_BUG_ON(cacheline(tail) == cacheline(ring->head) &&
708*3f2dd94aSFrançois Tigeot 		   tail < ring->head);
709*3f2dd94aSFrançois Tigeot #undef cacheline
710a85cb24fSFrançois Tigeot }
711a85cb24fSFrançois Tigeot 
712a85cb24fSFrançois Tigeot static inline unsigned int
intel_ring_set_tail(struct intel_ring * ring,unsigned int tail)713a85cb24fSFrançois Tigeot intel_ring_set_tail(struct intel_ring *ring, unsigned int tail)
714a85cb24fSFrançois Tigeot {
715a85cb24fSFrançois Tigeot 	/* Whilst writes to the tail are strictly order, there is no
716a85cb24fSFrançois Tigeot 	 * serialisation between readers and the writers. The tail may be
717a85cb24fSFrançois Tigeot 	 * read by i915_gem_request_retire() just as it is being updated
718a85cb24fSFrançois Tigeot 	 * by execlists, as although the breadcrumb is complete, the context
719a85cb24fSFrançois Tigeot 	 * switch hasn't been seen.
720a85cb24fSFrançois Tigeot 	 */
721a85cb24fSFrançois Tigeot 	assert_ring_tail_valid(ring, tail);
722a85cb24fSFrançois Tigeot 	ring->tail = tail;
723a85cb24fSFrançois Tigeot 	return tail;
724a85cb24fSFrançois Tigeot }
7259edbd4a0SFrançois Tigeot 
7264be47400SFrançois Tigeot void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno);
727e3adcf8fSFrançois Tigeot 
72887df8fc6SFrançois Tigeot void intel_engine_setup_common(struct intel_engine_cs *engine);
72987df8fc6SFrançois Tigeot int intel_engine_init_common(struct intel_engine_cs *engine);
7301e12ee3bSFrançois Tigeot int intel_engine_create_scratch(struct intel_engine_cs *engine, int size);
73171f41f3eSFrançois Tigeot void intel_engine_cleanup_common(struct intel_engine_cs *engine);
73271f41f3eSFrançois Tigeot 
73387df8fc6SFrançois Tigeot int intel_init_render_ring_buffer(struct intel_engine_cs *engine);
73487df8fc6SFrançois Tigeot int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine);
73587df8fc6SFrançois Tigeot int intel_init_blt_ring_buffer(struct intel_engine_cs *engine);
73687df8fc6SFrançois Tigeot int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine);
737e3adcf8fSFrançois Tigeot 
73871f41f3eSFrançois Tigeot u64 intel_engine_get_active_head(struct intel_engine_cs *engine);
7391e12ee3bSFrançois Tigeot u64 intel_engine_get_last_batch_head(struct intel_engine_cs *engine);
7401e12ee3bSFrançois Tigeot 
intel_engine_get_seqno(struct intel_engine_cs * engine)741303bf270SFrançois Tigeot static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine)
742303bf270SFrançois Tigeot {
743303bf270SFrançois Tigeot 	return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
744303bf270SFrançois Tigeot }
745e3adcf8fSFrançois Tigeot 
intel_engine_last_submit(struct intel_engine_cs * engine)7464be47400SFrançois Tigeot static inline u32 intel_engine_last_submit(struct intel_engine_cs *engine)
7474be47400SFrançois Tigeot {
7484be47400SFrançois Tigeot 	/* We are only peeking at the tail of the submit queue (and not the
7494be47400SFrançois Tigeot 	 * queue itself) in order to gain a hint as to the current active
7504be47400SFrançois Tigeot 	 * state of the engine. Callers are not expected to be taking
7514be47400SFrançois Tigeot 	 * engine->timeline->lock, nor are they expected to be concerned
7524be47400SFrançois Tigeot 	 * wtih serialising this hint with anything, so document it as
7534be47400SFrançois Tigeot 	 * a hint and nothing more.
7544be47400SFrançois Tigeot 	 */
755a85cb24fSFrançois Tigeot 	return READ_ONCE(engine->timeline->seqno);
7564be47400SFrançois Tigeot }
7574be47400SFrançois Tigeot 
7588621f407SFrançois Tigeot int init_workarounds_ring(struct intel_engine_cs *engine);
759a85cb24fSFrançois Tigeot int intel_ring_workarounds_emit(struct drm_i915_gem_request *req);
7602c9916cdSFrançois Tigeot 
7611e12ee3bSFrançois Tigeot void intel_engine_get_instdone(struct intel_engine_cs *engine,
7621e12ee3bSFrançois Tigeot 			       struct intel_instdone *instdone);
7631e12ee3bSFrançois Tigeot 
764a05eeebfSFrançois Tigeot /*
765a05eeebfSFrançois Tigeot  * Arbitrary size for largest possible 'add request' sequence. The code paths
766a05eeebfSFrançois Tigeot  * are complex and variable. Empirical measurement shows that the worst case
7671487f786SFrançois Tigeot  * is BDW at 192 bytes (6 + 6 + 36 dwords), then ILK at 136 bytes. However,
7681487f786SFrançois Tigeot  * we need to allocate double the largest single packet within that emission
7691487f786SFrançois Tigeot  * to account for tail wraparound (so 6 + 6 + 72 dwords for BDW).
770a05eeebfSFrançois Tigeot  */
7711487f786SFrançois Tigeot #define MIN_SPACE_FOR_ADD_REQUEST 336
772a05eeebfSFrançois Tigeot 
intel_hws_seqno_address(struct intel_engine_cs * engine)7731487f786SFrançois Tigeot static inline u32 intel_hws_seqno_address(struct intel_engine_cs *engine)
7741487f786SFrançois Tigeot {
7751e12ee3bSFrançois Tigeot 	return engine->status_page.ggtt_offset + I915_GEM_HWS_INDEX_ADDR;
7761487f786SFrançois Tigeot }
777f4e1c372SFrançois Tigeot 
778303bf270SFrançois Tigeot /* intel_breadcrumbs.c -- user interrupt bottom-half for waiters */
779303bf270SFrançois Tigeot int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine);
780303bf270SFrançois Tigeot 
intel_wait_init(struct intel_wait * wait,struct drm_i915_gem_request * rq)781a85cb24fSFrançois Tigeot static inline void intel_wait_init(struct intel_wait *wait,
782a85cb24fSFrançois Tigeot 				   struct drm_i915_gem_request *rq)
783a85cb24fSFrançois Tigeot {
784a85cb24fSFrançois Tigeot 	wait->tsk = current;
785a85cb24fSFrançois Tigeot 	wait->request = rq;
786a85cb24fSFrançois Tigeot }
787a85cb24fSFrançois Tigeot 
intel_wait_init_for_seqno(struct intel_wait * wait,u32 seqno)788a85cb24fSFrançois Tigeot static inline void intel_wait_init_for_seqno(struct intel_wait *wait, u32 seqno)
789303bf270SFrançois Tigeot {
790303bf270SFrançois Tigeot 	wait->tsk = current;
791303bf270SFrançois Tigeot 	wait->seqno = seqno;
792303bf270SFrançois Tigeot }
793303bf270SFrançois Tigeot 
intel_wait_has_seqno(const struct intel_wait * wait)794a85cb24fSFrançois Tigeot static inline bool intel_wait_has_seqno(const struct intel_wait *wait)
795a85cb24fSFrançois Tigeot {
796a85cb24fSFrançois Tigeot 	return wait->seqno;
797a85cb24fSFrançois Tigeot }
798a85cb24fSFrançois Tigeot 
799a85cb24fSFrançois Tigeot static inline bool
intel_wait_update_seqno(struct intel_wait * wait,u32 seqno)800a85cb24fSFrançois Tigeot intel_wait_update_seqno(struct intel_wait *wait, u32 seqno)
801a85cb24fSFrançois Tigeot {
802a85cb24fSFrançois Tigeot 	wait->seqno = seqno;
803a85cb24fSFrançois Tigeot 	return intel_wait_has_seqno(wait);
804a85cb24fSFrançois Tigeot }
805a85cb24fSFrançois Tigeot 
806a85cb24fSFrançois Tigeot static inline bool
intel_wait_update_request(struct intel_wait * wait,const struct drm_i915_gem_request * rq)807a85cb24fSFrançois Tigeot intel_wait_update_request(struct intel_wait *wait,
808a85cb24fSFrançois Tigeot 			  const struct drm_i915_gem_request *rq)
809a85cb24fSFrançois Tigeot {
810a85cb24fSFrançois Tigeot 	return intel_wait_update_seqno(wait, i915_gem_request_global_seqno(rq));
811a85cb24fSFrançois Tigeot }
812a85cb24fSFrançois Tigeot 
813a85cb24fSFrançois Tigeot static inline bool
intel_wait_check_seqno(const struct intel_wait * wait,u32 seqno)814a85cb24fSFrançois Tigeot intel_wait_check_seqno(const struct intel_wait *wait, u32 seqno)
815a85cb24fSFrançois Tigeot {
816a85cb24fSFrançois Tigeot 	return wait->seqno == seqno;
817a85cb24fSFrançois Tigeot }
818a85cb24fSFrançois Tigeot 
819a85cb24fSFrançois Tigeot static inline bool
intel_wait_check_request(const struct intel_wait * wait,const struct drm_i915_gem_request * rq)820a85cb24fSFrançois Tigeot intel_wait_check_request(const struct intel_wait *wait,
821a85cb24fSFrançois Tigeot 			 const struct drm_i915_gem_request *rq)
822a85cb24fSFrançois Tigeot {
823a85cb24fSFrançois Tigeot 	return intel_wait_check_seqno(wait, i915_gem_request_global_seqno(rq));
824a85cb24fSFrançois Tigeot }
825a85cb24fSFrançois Tigeot 
intel_wait_complete(const struct intel_wait * wait)826303bf270SFrançois Tigeot static inline bool intel_wait_complete(const struct intel_wait *wait)
827303bf270SFrançois Tigeot {
828303bf270SFrançois Tigeot 	return RB_EMPTY_NODE(&wait->node);
829303bf270SFrançois Tigeot }
830303bf270SFrançois Tigeot 
831303bf270SFrançois Tigeot bool intel_engine_add_wait(struct intel_engine_cs *engine,
832303bf270SFrançois Tigeot 			   struct intel_wait *wait);
833303bf270SFrançois Tigeot void intel_engine_remove_wait(struct intel_engine_cs *engine,
834303bf270SFrançois Tigeot 			      struct intel_wait *wait);
835*3f2dd94aSFrançois Tigeot void intel_engine_enable_signaling(struct drm_i915_gem_request *request,
836*3f2dd94aSFrançois Tigeot 				   bool wakeup);
837a85cb24fSFrançois Tigeot void intel_engine_cancel_signaling(struct drm_i915_gem_request *request);
838303bf270SFrançois Tigeot 
intel_engine_has_waiter(const struct intel_engine_cs * engine)8391e12ee3bSFrançois Tigeot static inline bool intel_engine_has_waiter(const struct intel_engine_cs *engine)
840303bf270SFrançois Tigeot {
841a85cb24fSFrançois Tigeot 	return READ_ONCE(engine->breadcrumbs.irq_wait);
842303bf270SFrançois Tigeot }
843303bf270SFrançois Tigeot 
844a85cb24fSFrançois Tigeot unsigned int intel_engine_wakeup(struct intel_engine_cs *engine);
845a85cb24fSFrançois Tigeot #define ENGINE_WAKEUP_WAITER BIT(0)
846a85cb24fSFrançois Tigeot #define ENGINE_WAKEUP_ASLEEP BIT(1)
8471e12ee3bSFrançois Tigeot 
848a85cb24fSFrançois Tigeot void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine);
849a85cb24fSFrançois Tigeot void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine);
850303bf270SFrançois Tigeot 
8511e12ee3bSFrançois Tigeot void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine);
852303bf270SFrançois Tigeot void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine);
853a85cb24fSFrançois Tigeot bool intel_breadcrumbs_busy(struct intel_engine_cs *engine);
854a85cb24fSFrançois Tigeot 
gen8_emit_pipe_control(u32 * batch,u32 flags,u32 offset)855a85cb24fSFrançois Tigeot static inline u32 *gen8_emit_pipe_control(u32 *batch, u32 flags, u32 offset)
856a85cb24fSFrançois Tigeot {
857a85cb24fSFrançois Tigeot 	memset(batch, 0, 6 * sizeof(u32));
858a85cb24fSFrançois Tigeot 
859a85cb24fSFrançois Tigeot 	batch[0] = GFX_OP_PIPE_CONTROL(6);
860a85cb24fSFrançois Tigeot 	batch[1] = flags;
861a85cb24fSFrançois Tigeot 	batch[2] = offset;
862a85cb24fSFrançois Tigeot 
863a85cb24fSFrançois Tigeot 	return batch + 6;
864a85cb24fSFrançois Tigeot }
865a85cb24fSFrançois Tigeot 
866a85cb24fSFrançois Tigeot bool intel_engine_is_idle(struct intel_engine_cs *engine);
867a85cb24fSFrançois Tigeot bool intel_engines_are_idle(struct drm_i915_private *dev_priv);
868a85cb24fSFrançois Tigeot 
869*3f2dd94aSFrançois Tigeot void intel_engines_mark_idle(struct drm_i915_private *i915);
870a85cb24fSFrançois Tigeot void intel_engines_reset_default_submission(struct drm_i915_private *i915);
87171f41f3eSFrançois Tigeot 
872*3f2dd94aSFrançois Tigeot bool intel_engine_can_store_dword(struct intel_engine_cs *engine);
873*3f2dd94aSFrançois Tigeot 
874*3f2dd94aSFrançois Tigeot void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *p);
875*3f2dd94aSFrançois Tigeot 
876e3adcf8fSFrançois Tigeot #endif /* _INTEL_RINGBUFFER_H_ */
877