1e3adcf8fSFrançois Tigeot #ifndef _INTEL_RINGBUFFER_H_
2e3adcf8fSFrançois Tigeot #define _INTEL_RINGBUFFER_H_
3e3adcf8fSFrançois Tigeot
4ba55f2f5SFrançois Tigeot #include <linux/hashtable.h>
519c468b4SFrançois Tigeot #include "i915_gem_batch_pool.h"
671f41f3eSFrançois Tigeot #include "i915_gem_request.h"
74be47400SFrançois Tigeot #include "i915_gem_timeline.h"
8a85cb24fSFrançois Tigeot #include "i915_selftest.h"
9ba55f2f5SFrançois Tigeot
10*3f2dd94aSFrançois Tigeot struct drm_printer;
11*3f2dd94aSFrançois Tigeot
12ba55f2f5SFrançois Tigeot #define I915_CMD_HASH_ORDER 9
13ba55f2f5SFrançois Tigeot
141b13d190SFrançois Tigeot /* Early gen2 devices have a cacheline of just 32 bytes, using 64 is overkill,
151b13d190SFrançois Tigeot * but keeps the logic simple. Indeed, the whole purpose of this macro is just
161b13d190SFrançois Tigeot * to give some inclination as to some of the magic values used in the various
171b13d190SFrançois Tigeot * workarounds!
181b13d190SFrançois Tigeot */
191b13d190SFrançois Tigeot #define CACHELINE_BYTES 64
20a05eeebfSFrançois Tigeot #define CACHELINE_DWORDS (CACHELINE_BYTES / sizeof(uint32_t))
211b13d190SFrançois Tigeot
22e3adcf8fSFrançois Tigeot struct intel_hw_status_page {
231e12ee3bSFrançois Tigeot struct i915_vma *vma;
24f4e1c372SFrançois Tigeot u32 *page_addr;
251e12ee3bSFrançois Tigeot u32 ggtt_offset;
26e3adcf8fSFrançois Tigeot };
27e3adcf8fSFrançois Tigeot
2887df8fc6SFrançois Tigeot #define I915_READ_TAIL(engine) I915_READ(RING_TAIL((engine)->mmio_base))
2987df8fc6SFrançois Tigeot #define I915_WRITE_TAIL(engine, val) I915_WRITE(RING_TAIL((engine)->mmio_base), val)
30e3adcf8fSFrançois Tigeot
3187df8fc6SFrançois Tigeot #define I915_READ_START(engine) I915_READ(RING_START((engine)->mmio_base))
3287df8fc6SFrançois Tigeot #define I915_WRITE_START(engine, val) I915_WRITE(RING_START((engine)->mmio_base), val)
33e3adcf8fSFrançois Tigeot
3487df8fc6SFrançois Tigeot #define I915_READ_HEAD(engine) I915_READ(RING_HEAD((engine)->mmio_base))
3587df8fc6SFrançois Tigeot #define I915_WRITE_HEAD(engine, val) I915_WRITE(RING_HEAD((engine)->mmio_base), val)
36e3adcf8fSFrançois Tigeot
3787df8fc6SFrançois Tigeot #define I915_READ_CTL(engine) I915_READ(RING_CTL((engine)->mmio_base))
3887df8fc6SFrançois Tigeot #define I915_WRITE_CTL(engine, val) I915_WRITE(RING_CTL((engine)->mmio_base), val)
39e3adcf8fSFrançois Tigeot
4087df8fc6SFrançois Tigeot #define I915_READ_IMR(engine) I915_READ(RING_IMR((engine)->mmio_base))
4187df8fc6SFrançois Tigeot #define I915_WRITE_IMR(engine, val) I915_WRITE(RING_IMR((engine)->mmio_base), val)
42e3adcf8fSFrançois Tigeot
4387df8fc6SFrançois Tigeot #define I915_READ_MODE(engine) I915_READ(RING_MI_MODE((engine)->mmio_base))
4487df8fc6SFrançois Tigeot #define I915_WRITE_MODE(engine, val) I915_WRITE(RING_MI_MODE((engine)->mmio_base), val)
45ba55f2f5SFrançois Tigeot
4624edb884SFrançois Tigeot /* seqno size is actually only a uint32, but since we plan to use MI_FLUSH_DW to
4724edb884SFrançois Tigeot * do the writes, and that must have qw aligned offsets, simply pretend it's 8b.
4824edb884SFrançois Tigeot */
498621f407SFrançois Tigeot #define gen8_semaphore_seqno_size sizeof(uint64_t)
508621f407SFrançois Tigeot #define GEN8_SEMAPHORE_OFFSET(__from, __to) \
518621f407SFrançois Tigeot (((__from) * I915_NUM_ENGINES + (__to)) * gen8_semaphore_seqno_size)
5224edb884SFrançois Tigeot #define GEN8_SIGNAL_OFFSET(__ring, to) \
531e12ee3bSFrançois Tigeot (dev_priv->semaphore->node.start + \
548621f407SFrançois Tigeot GEN8_SEMAPHORE_OFFSET((__ring)->id, (to)))
5524edb884SFrançois Tigeot #define GEN8_WAIT_OFFSET(__ring, from) \
561e12ee3bSFrançois Tigeot (dev_priv->semaphore->node.start + \
578621f407SFrançois Tigeot GEN8_SEMAPHORE_OFFSET(from, (__ring)->id))
5824edb884SFrançois Tigeot
5971f41f3eSFrançois Tigeot enum intel_engine_hangcheck_action {
60a85cb24fSFrançois Tigeot ENGINE_IDLE = 0,
61a85cb24fSFrançois Tigeot ENGINE_WAIT,
62a85cb24fSFrançois Tigeot ENGINE_ACTIVE_SEQNO,
63a85cb24fSFrançois Tigeot ENGINE_ACTIVE_HEAD,
64a85cb24fSFrançois Tigeot ENGINE_ACTIVE_SUBUNITS,
65a85cb24fSFrançois Tigeot ENGINE_WAIT_KICK,
66a85cb24fSFrançois Tigeot ENGINE_DEAD,
679edbd4a0SFrançois Tigeot };
685d0b1887SFrançois Tigeot
69a85cb24fSFrançois Tigeot static inline const char *
hangcheck_action_to_str(const enum intel_engine_hangcheck_action a)70a85cb24fSFrançois Tigeot hangcheck_action_to_str(const enum intel_engine_hangcheck_action a)
71a85cb24fSFrançois Tigeot {
72a85cb24fSFrançois Tigeot switch (a) {
73a85cb24fSFrançois Tigeot case ENGINE_IDLE:
74a85cb24fSFrançois Tigeot return "idle";
75a85cb24fSFrançois Tigeot case ENGINE_WAIT:
76a85cb24fSFrançois Tigeot return "wait";
77a85cb24fSFrançois Tigeot case ENGINE_ACTIVE_SEQNO:
78a85cb24fSFrançois Tigeot return "active seqno";
79a85cb24fSFrançois Tigeot case ENGINE_ACTIVE_HEAD:
80a85cb24fSFrançois Tigeot return "active head";
81a85cb24fSFrançois Tigeot case ENGINE_ACTIVE_SUBUNITS:
82a85cb24fSFrançois Tigeot return "active subunits";
83a85cb24fSFrançois Tigeot case ENGINE_WAIT_KICK:
84a85cb24fSFrançois Tigeot return "wait kick";
85a85cb24fSFrançois Tigeot case ENGINE_DEAD:
86a85cb24fSFrançois Tigeot return "dead";
87a85cb24fSFrançois Tigeot }
88a85cb24fSFrançois Tigeot
89a85cb24fSFrançois Tigeot return "unknown";
90a85cb24fSFrançois Tigeot }
91ba55f2f5SFrançois Tigeot
921e12ee3bSFrançois Tigeot #define I915_MAX_SLICES 3
931e12ee3bSFrançois Tigeot #define I915_MAX_SUBSLICES 3
941e12ee3bSFrançois Tigeot
951e12ee3bSFrançois Tigeot #define instdone_slice_mask(dev_priv__) \
961e12ee3bSFrançois Tigeot (INTEL_GEN(dev_priv__) == 7 ? \
971e12ee3bSFrançois Tigeot 1 : INTEL_INFO(dev_priv__)->sseu.slice_mask)
981e12ee3bSFrançois Tigeot
991e12ee3bSFrançois Tigeot #define instdone_subslice_mask(dev_priv__) \
1001e12ee3bSFrançois Tigeot (INTEL_GEN(dev_priv__) == 7 ? \
1011e12ee3bSFrançois Tigeot 1 : INTEL_INFO(dev_priv__)->sseu.subslice_mask)
1021e12ee3bSFrançois Tigeot
1031e12ee3bSFrançois Tigeot #define for_each_instdone_slice_subslice(dev_priv__, slice__, subslice__) \
1041e12ee3bSFrançois Tigeot for ((slice__) = 0, (subslice__) = 0; \
1051e12ee3bSFrançois Tigeot (slice__) < I915_MAX_SLICES; \
1061e12ee3bSFrançois Tigeot (subslice__) = ((subslice__) + 1) < I915_MAX_SUBSLICES ? (subslice__) + 1 : 0, \
1071e12ee3bSFrançois Tigeot (slice__) += ((subslice__) == 0)) \
1081e12ee3bSFrançois Tigeot for_each_if((BIT(slice__) & instdone_slice_mask(dev_priv__)) && \
1091e12ee3bSFrançois Tigeot (BIT(subslice__) & instdone_subslice_mask(dev_priv__)))
1101e12ee3bSFrançois Tigeot
1111e12ee3bSFrançois Tigeot struct intel_instdone {
1121e12ee3bSFrançois Tigeot u32 instdone;
1131e12ee3bSFrançois Tigeot /* The following exist only in the RCS engine */
1141e12ee3bSFrançois Tigeot u32 slice_common;
1151e12ee3bSFrançois Tigeot u32 sampler[I915_MAX_SLICES][I915_MAX_SUBSLICES];
1161e12ee3bSFrançois Tigeot u32 row[I915_MAX_SLICES][I915_MAX_SUBSLICES];
1171e12ee3bSFrançois Tigeot };
1181e12ee3bSFrançois Tigeot
11971f41f3eSFrançois Tigeot struct intel_engine_hangcheck {
120ba55f2f5SFrançois Tigeot u64 acthd;
1215d0b1887SFrançois Tigeot u32 seqno;
12271f41f3eSFrançois Tigeot enum intel_engine_hangcheck_action action;
123a85cb24fSFrançois Tigeot unsigned long action_timestamp;
124ba55f2f5SFrançois Tigeot int deadlock;
1251e12ee3bSFrançois Tigeot struct intel_instdone instdone;
126*3f2dd94aSFrançois Tigeot struct drm_i915_gem_request *active_request;
127a85cb24fSFrançois Tigeot bool stalled;
1285d0b1887SFrançois Tigeot };
1295d0b1887SFrançois Tigeot
13071f41f3eSFrançois Tigeot struct intel_ring {
131c0e85e96SFrançois Tigeot struct i915_vma *vma;
1321e12ee3bSFrançois Tigeot void *vaddr;
133e3adcf8fSFrançois Tigeot
13471f41f3eSFrançois Tigeot struct list_head request_list;
13571f41f3eSFrançois Tigeot
13615ac6249SFrançois Tigeot u32 head;
13715ac6249SFrançois Tigeot u32 tail;
138a85cb24fSFrançois Tigeot u32 emit;
139a85cb24fSFrançois Tigeot
140*3f2dd94aSFrançois Tigeot u32 space;
141*3f2dd94aSFrançois Tigeot u32 size;
142*3f2dd94aSFrançois Tigeot u32 effective_size;
143ba55f2f5SFrançois Tigeot };
144ba55f2f5SFrançois Tigeot
1451487f786SFrançois Tigeot struct i915_gem_context;
1468621f407SFrançois Tigeot struct drm_i915_reg_table;
1472c9916cdSFrançois Tigeot
148a05eeebfSFrançois Tigeot /*
149a05eeebfSFrançois Tigeot * we use a single page to load ctx workarounds so all of these
150a05eeebfSFrançois Tigeot * values are referred in terms of dwords
151a05eeebfSFrançois Tigeot *
152a05eeebfSFrançois Tigeot * struct i915_wa_ctx_bb:
153a05eeebfSFrançois Tigeot * offset: specifies batch starting position, also helpful in case
154a05eeebfSFrançois Tigeot * if we want to have multiple batches at different offsets based on
155a05eeebfSFrançois Tigeot * some criteria. It is not a requirement at the moment but provides
156a05eeebfSFrançois Tigeot * an option for future use.
157a05eeebfSFrançois Tigeot * size: size of the batch in DWORDS
158a05eeebfSFrançois Tigeot */
159a05eeebfSFrançois Tigeot struct i915_ctx_workarounds {
160a05eeebfSFrançois Tigeot struct i915_wa_ctx_bb {
161a05eeebfSFrançois Tigeot u32 offset;
162a05eeebfSFrançois Tigeot u32 size;
163a05eeebfSFrançois Tigeot } indirect_ctx, per_ctx;
1641e12ee3bSFrançois Tigeot struct i915_vma *vma;
165a05eeebfSFrançois Tigeot };
166a05eeebfSFrançois Tigeot
167303bf270SFrançois Tigeot struct drm_i915_gem_request;
1684be47400SFrançois Tigeot struct intel_render_state;
169303bf270SFrançois Tigeot
170a85cb24fSFrançois Tigeot /*
171a85cb24fSFrançois Tigeot * Engine IDs definitions.
172a85cb24fSFrançois Tigeot * Keep instances of the same type engine together.
173a85cb24fSFrançois Tigeot */
1748621f407SFrançois Tigeot enum intel_engine_id {
175c0e85e96SFrançois Tigeot RCS = 0,
176ba55f2f5SFrançois Tigeot BCS,
177c0e85e96SFrançois Tigeot VCS,
178a85cb24fSFrançois Tigeot VCS2,
179c0e85e96SFrançois Tigeot #define _VCS(n) (VCS + (n))
180a85cb24fSFrançois Tigeot VECS
181a85cb24fSFrançois Tigeot };
182a85cb24fSFrançois Tigeot
183*3f2dd94aSFrançois Tigeot struct i915_priolist {
184*3f2dd94aSFrançois Tigeot struct rb_node node;
185*3f2dd94aSFrançois Tigeot struct list_head requests;
186*3f2dd94aSFrançois Tigeot int priority;
187*3f2dd94aSFrançois Tigeot };
188*3f2dd94aSFrançois Tigeot
189*3f2dd94aSFrançois Tigeot /**
190*3f2dd94aSFrançois Tigeot * struct intel_engine_execlists - execlist submission queue and port state
191*3f2dd94aSFrançois Tigeot *
192*3f2dd94aSFrançois Tigeot * The struct intel_engine_execlists represents the combined logical state of
193*3f2dd94aSFrançois Tigeot * driver and the hardware state for execlist mode of submission.
194*3f2dd94aSFrançois Tigeot */
195*3f2dd94aSFrançois Tigeot struct intel_engine_execlists {
196*3f2dd94aSFrançois Tigeot /**
197*3f2dd94aSFrançois Tigeot * @irq_tasklet: softirq tasklet for bottom handler
198*3f2dd94aSFrançois Tigeot */
199*3f2dd94aSFrançois Tigeot struct tasklet_struct irq_tasklet;
200*3f2dd94aSFrançois Tigeot
201*3f2dd94aSFrançois Tigeot /**
202*3f2dd94aSFrançois Tigeot * @default_priolist: priority list for I915_PRIORITY_NORMAL
203*3f2dd94aSFrançois Tigeot */
204*3f2dd94aSFrançois Tigeot struct i915_priolist default_priolist;
205*3f2dd94aSFrançois Tigeot
206*3f2dd94aSFrançois Tigeot /**
207*3f2dd94aSFrançois Tigeot * @no_priolist: priority lists disabled
208*3f2dd94aSFrançois Tigeot */
209*3f2dd94aSFrançois Tigeot bool no_priolist;
210*3f2dd94aSFrançois Tigeot
211*3f2dd94aSFrançois Tigeot /**
212*3f2dd94aSFrançois Tigeot * @port: execlist port states
213*3f2dd94aSFrançois Tigeot *
214*3f2dd94aSFrançois Tigeot * For each hardware ELSP (ExecList Submission Port) we keep
215*3f2dd94aSFrançois Tigeot * track of the last request and the number of times we submitted
216*3f2dd94aSFrançois Tigeot * that port to hw. We then count the number of times the hw reports
217*3f2dd94aSFrançois Tigeot * a context completion or preemption. As only one context can
218*3f2dd94aSFrançois Tigeot * be active on hw, we limit resubmission of context to port[0]. This
219*3f2dd94aSFrançois Tigeot * is called Lite Restore, of the context.
220*3f2dd94aSFrançois Tigeot */
221*3f2dd94aSFrançois Tigeot struct execlist_port {
222*3f2dd94aSFrançois Tigeot /**
223*3f2dd94aSFrançois Tigeot * @request_count: combined request and submission count
224*3f2dd94aSFrançois Tigeot */
225*3f2dd94aSFrançois Tigeot struct drm_i915_gem_request *request_count;
226*3f2dd94aSFrançois Tigeot #define EXECLIST_COUNT_BITS 2
227*3f2dd94aSFrançois Tigeot #define port_request(p) ptr_mask_bits((p)->request_count, EXECLIST_COUNT_BITS)
228*3f2dd94aSFrançois Tigeot #define port_count(p) ptr_unmask_bits((p)->request_count, EXECLIST_COUNT_BITS)
229*3f2dd94aSFrançois Tigeot #define port_pack(rq, count) ptr_pack_bits(rq, count, EXECLIST_COUNT_BITS)
230*3f2dd94aSFrançois Tigeot #define port_unpack(p, count) ptr_unpack_bits((p)->request_count, count, EXECLIST_COUNT_BITS)
231*3f2dd94aSFrançois Tigeot #define port_set(p, packed) ((p)->request_count = (packed))
232*3f2dd94aSFrançois Tigeot #define port_isset(p) ((p)->request_count)
233*3f2dd94aSFrançois Tigeot #define port_index(p, execlists) ((p) - (execlists)->port)
234*3f2dd94aSFrançois Tigeot
235*3f2dd94aSFrançois Tigeot /**
236*3f2dd94aSFrançois Tigeot * @context_id: context ID for port
237*3f2dd94aSFrançois Tigeot */
238*3f2dd94aSFrançois Tigeot GEM_DEBUG_DECL(u32 context_id);
239*3f2dd94aSFrançois Tigeot
240*3f2dd94aSFrançois Tigeot #define EXECLIST_MAX_PORTS 2
241*3f2dd94aSFrançois Tigeot } port[EXECLIST_MAX_PORTS];
242*3f2dd94aSFrançois Tigeot
243*3f2dd94aSFrançois Tigeot /**
244*3f2dd94aSFrançois Tigeot * @active: is the HW active? We consider the HW as active after
245*3f2dd94aSFrançois Tigeot * submitting any context for execution and until we have seen the
246*3f2dd94aSFrançois Tigeot * last context completion event. After that, we do not expect any
247*3f2dd94aSFrançois Tigeot * more events until we submit, and so can park the HW.
248*3f2dd94aSFrançois Tigeot *
249*3f2dd94aSFrançois Tigeot * As we have a small number of different sources from which we feed
250*3f2dd94aSFrançois Tigeot * the HW, we track the state of each inside a single bitfield.
251*3f2dd94aSFrançois Tigeot */
252*3f2dd94aSFrançois Tigeot unsigned int active;
253*3f2dd94aSFrançois Tigeot #define EXECLISTS_ACTIVE_USER 0
254*3f2dd94aSFrançois Tigeot #define EXECLISTS_ACTIVE_PREEMPT 1
255*3f2dd94aSFrançois Tigeot
256*3f2dd94aSFrançois Tigeot /**
257*3f2dd94aSFrançois Tigeot * @port_mask: number of execlist ports - 1
258*3f2dd94aSFrançois Tigeot */
259*3f2dd94aSFrançois Tigeot unsigned int port_mask;
260*3f2dd94aSFrançois Tigeot
261*3f2dd94aSFrançois Tigeot /**
262*3f2dd94aSFrançois Tigeot * @queue: queue of requests, in priority lists
263*3f2dd94aSFrançois Tigeot */
264*3f2dd94aSFrançois Tigeot struct rb_root queue;
265*3f2dd94aSFrançois Tigeot
266*3f2dd94aSFrançois Tigeot /**
267*3f2dd94aSFrançois Tigeot * @first: leftmost level in priority @queue
268*3f2dd94aSFrançois Tigeot */
269*3f2dd94aSFrançois Tigeot struct rb_node *first;
270*3f2dd94aSFrançois Tigeot
271*3f2dd94aSFrançois Tigeot /**
272*3f2dd94aSFrançois Tigeot * @fw_domains: forcewake domains for irq tasklet
273*3f2dd94aSFrançois Tigeot */
274*3f2dd94aSFrançois Tigeot unsigned int fw_domains;
275*3f2dd94aSFrançois Tigeot
276*3f2dd94aSFrançois Tigeot /**
277*3f2dd94aSFrançois Tigeot * @csb_head: context status buffer head
278*3f2dd94aSFrançois Tigeot */
279*3f2dd94aSFrançois Tigeot unsigned int csb_head;
280*3f2dd94aSFrançois Tigeot
281*3f2dd94aSFrançois Tigeot /**
282*3f2dd94aSFrançois Tigeot * @csb_use_mmio: access csb through mmio, instead of hwsp
283*3f2dd94aSFrançois Tigeot */
284*3f2dd94aSFrançois Tigeot bool csb_use_mmio;
285*3f2dd94aSFrançois Tigeot };
286*3f2dd94aSFrançois Tigeot
287*3f2dd94aSFrançois Tigeot #define INTEL_ENGINE_CS_MAX_NAME 8
288*3f2dd94aSFrançois Tigeot
289a85cb24fSFrançois Tigeot struct intel_engine_cs {
290a85cb24fSFrançois Tigeot struct drm_i915_private *i915;
291*3f2dd94aSFrançois Tigeot char name[INTEL_ENGINE_CS_MAX_NAME];
292a85cb24fSFrançois Tigeot enum intel_engine_id id;
293*3f2dd94aSFrançois Tigeot unsigned int uabi_id;
294a85cb24fSFrançois Tigeot unsigned int hw_id;
295a85cb24fSFrançois Tigeot unsigned int guc_id;
296*3f2dd94aSFrançois Tigeot
297*3f2dd94aSFrançois Tigeot u8 class;
298*3f2dd94aSFrançois Tigeot u8 instance;
299*3f2dd94aSFrançois Tigeot u32 context_size;
300ba55f2f5SFrançois Tigeot u32 mmio_base;
30187df8fc6SFrançois Tigeot unsigned int irq_shift;
302*3f2dd94aSFrançois Tigeot
30371f41f3eSFrançois Tigeot struct intel_ring *buffer;
3044be47400SFrançois Tigeot struct intel_timeline *timeline;
3054be47400SFrançois Tigeot
3064be47400SFrançois Tigeot struct intel_render_state *render_state;
307ba55f2f5SFrançois Tigeot
308a85cb24fSFrançois Tigeot atomic_t irq_count;
309a85cb24fSFrançois Tigeot unsigned long irq_posted;
310a85cb24fSFrançois Tigeot #define ENGINE_IRQ_BREADCRUMB 0
311a85cb24fSFrançois Tigeot #define ENGINE_IRQ_EXECLIST 1
312a85cb24fSFrançois Tigeot
313303bf270SFrançois Tigeot /* Rather than have every client wait upon all user interrupts,
314303bf270SFrançois Tigeot * with the herd waking after every interrupt and each doing the
315303bf270SFrançois Tigeot * heavyweight seqno dance, we delegate the task (of being the
316303bf270SFrançois Tigeot * bottom-half of the user interrupt) to the first client. After
317303bf270SFrançois Tigeot * every interrupt, we wake up one client, who does the heavyweight
318303bf270SFrançois Tigeot * coherent seqno read and either goes back to sleep (if incomplete),
319303bf270SFrançois Tigeot * or wakes up all the completed clients in parallel, before then
320303bf270SFrançois Tigeot * transferring the bottom-half status to the next client in the queue.
321303bf270SFrançois Tigeot *
322303bf270SFrançois Tigeot * Compared to walking the entire list of waiters in a single dedicated
323303bf270SFrançois Tigeot * bottom-half, we reduce the latency of the first waiter by avoiding
324303bf270SFrançois Tigeot * a context switch, but incur additional coherent seqno reads when
325303bf270SFrançois Tigeot * following the chain of request breadcrumbs. Since it is most likely
326303bf270SFrançois Tigeot * that we have a single client waiting on each seqno, then reducing
327303bf270SFrançois Tigeot * the overhead of waking that client is much preferred.
328303bf270SFrançois Tigeot */
329303bf270SFrançois Tigeot struct intel_breadcrumbs {
330a85cb24fSFrançois Tigeot spinlock_t irq_lock; /* protects irq_*; irqsafe */
331a85cb24fSFrançois Tigeot struct intel_wait *irq_wait; /* oldest waiter by retirement */
332303bf270SFrançois Tigeot
333a85cb24fSFrançois Tigeot spinlock_t rb_lock; /* protects the rb and wraps irq_lock */
334303bf270SFrançois Tigeot struct rb_root waiters; /* sorted by retirement, priority */
335303bf270SFrançois Tigeot struct rb_root signals; /* sorted by retirement */
336303bf270SFrançois Tigeot struct task_struct *signaler; /* used for fence signalling */
337a85cb24fSFrançois Tigeot struct drm_i915_gem_request __rcu *first_signal;
338303bf270SFrançois Tigeot struct timer_list fake_irq; /* used after a missed interrupt */
3391e12ee3bSFrançois Tigeot struct timer_list hangcheck; /* detect missed interrupts */
3401e12ee3bSFrançois Tigeot
341a85cb24fSFrançois Tigeot unsigned int hangcheck_interrupts;
342303bf270SFrançois Tigeot
343a85cb24fSFrançois Tigeot bool irq_armed : 1;
344303bf270SFrançois Tigeot bool irq_enabled : 1;
345a85cb24fSFrançois Tigeot I915_SELFTEST_DECLARE(bool mock : 1);
346303bf270SFrançois Tigeot } breadcrumbs;
347303bf270SFrançois Tigeot
34819c468b4SFrançois Tigeot /*
34919c468b4SFrançois Tigeot * A pool of objects to use as shadow copies of client batch buffers
35019c468b4SFrançois Tigeot * when the command parser is enabled. Prevents the client from
35119c468b4SFrançois Tigeot * modifying the batch contents after software parsing.
35219c468b4SFrançois Tigeot */
35319c468b4SFrançois Tigeot struct i915_gem_batch_pool batch_pool;
35419c468b4SFrançois Tigeot
355ba55f2f5SFrançois Tigeot struct intel_hw_status_page status_page;
356a05eeebfSFrançois Tigeot struct i915_ctx_workarounds wa_ctx;
3571e12ee3bSFrançois Tigeot struct i915_vma *scratch;
358e3adcf8fSFrançois Tigeot
359303bf270SFrançois Tigeot u32 irq_keep_mask; /* always keep these interrupts */
36015ac6249SFrançois Tigeot u32 irq_enable_mask; /* bitmask to enable ring interrupt */
36187df8fc6SFrançois Tigeot void (*irq_enable)(struct intel_engine_cs *engine);
36287df8fc6SFrançois Tigeot void (*irq_disable)(struct intel_engine_cs *engine);
363e3adcf8fSFrançois Tigeot
36487df8fc6SFrançois Tigeot int (*init_hw)(struct intel_engine_cs *engine);
3651e12ee3bSFrançois Tigeot void (*reset_hw)(struct intel_engine_cs *engine,
3661e12ee3bSFrançois Tigeot struct drm_i915_gem_request *req);
367e3adcf8fSFrançois Tigeot
368a85cb24fSFrançois Tigeot void (*set_default_submission)(struct intel_engine_cs *engine);
369a85cb24fSFrançois Tigeot
370*3f2dd94aSFrançois Tigeot struct intel_ring *(*context_pin)(struct intel_engine_cs *engine,
371a85cb24fSFrançois Tigeot struct i915_gem_context *ctx);
372a85cb24fSFrançois Tigeot void (*context_unpin)(struct intel_engine_cs *engine,
373a85cb24fSFrançois Tigeot struct i915_gem_context *ctx);
374a85cb24fSFrançois Tigeot int (*request_alloc)(struct drm_i915_gem_request *req);
375a05eeebfSFrançois Tigeot int (*init_context)(struct drm_i915_gem_request *req);
3761b13d190SFrançois Tigeot
37771f41f3eSFrançois Tigeot int (*emit_flush)(struct drm_i915_gem_request *request,
37871f41f3eSFrançois Tigeot u32 mode);
37971f41f3eSFrançois Tigeot #define EMIT_INVALIDATE BIT(0)
38071f41f3eSFrançois Tigeot #define EMIT_FLUSH BIT(1)
38171f41f3eSFrançois Tigeot #define EMIT_BARRIER (EMIT_INVALIDATE | EMIT_FLUSH)
38271f41f3eSFrançois Tigeot int (*emit_bb_start)(struct drm_i915_gem_request *req,
38371f41f3eSFrançois Tigeot u64 offset, u32 length,
38471f41f3eSFrançois Tigeot unsigned int dispatch_flags);
38571f41f3eSFrançois Tigeot #define I915_DISPATCH_SECURE BIT(0)
38671f41f3eSFrançois Tigeot #define I915_DISPATCH_PINNED BIT(1)
38771f41f3eSFrançois Tigeot #define I915_DISPATCH_RS BIT(2)
3884be47400SFrançois Tigeot void (*emit_breadcrumb)(struct drm_i915_gem_request *req,
389a85cb24fSFrançois Tigeot u32 *cs);
3904be47400SFrançois Tigeot int emit_breadcrumb_sz;
3911e12ee3bSFrançois Tigeot
3921e12ee3bSFrançois Tigeot /* Pass the request to the hardware queue (e.g. directly into
3931e12ee3bSFrançois Tigeot * the legacy ringbuffer or to the end of an execlist).
3941e12ee3bSFrançois Tigeot *
3951e12ee3bSFrançois Tigeot * This is called from an atomic context with irqs disabled; must
3961e12ee3bSFrançois Tigeot * be irq safe.
3971e12ee3bSFrançois Tigeot */
39871f41f3eSFrançois Tigeot void (*submit_request)(struct drm_i915_gem_request *req);
3991e12ee3bSFrançois Tigeot
4004be47400SFrançois Tigeot /* Call when the priority on a request has changed and it and its
4014be47400SFrançois Tigeot * dependencies may need rescheduling. Note the request itself may
4024be47400SFrançois Tigeot * not be ready to run!
4034be47400SFrançois Tigeot *
4044be47400SFrançois Tigeot * Called under the struct_mutex.
4054be47400SFrançois Tigeot */
4064be47400SFrançois Tigeot void (*schedule)(struct drm_i915_gem_request *request,
4074be47400SFrançois Tigeot int priority);
4084be47400SFrançois Tigeot
409*3f2dd94aSFrançois Tigeot /*
410*3f2dd94aSFrançois Tigeot * Cancel all requests on the hardware, or queued for execution.
411*3f2dd94aSFrançois Tigeot * This should only cancel the ready requests that have been
412*3f2dd94aSFrançois Tigeot * submitted to the engine (via the engine->submit_request callback).
413*3f2dd94aSFrançois Tigeot * This is called when marking the device as wedged.
414*3f2dd94aSFrançois Tigeot */
415*3f2dd94aSFrançois Tigeot void (*cancel_requests)(struct intel_engine_cs *engine);
416*3f2dd94aSFrançois Tigeot
417b030f26bSFrançois Tigeot /* Some chipsets are not quite as coherent as advertised and need
418b030f26bSFrançois Tigeot * an expensive kick to force a true read of the up-to-date seqno.
419b030f26bSFrançois Tigeot * However, the up-to-date seqno is not always required and the last
420b030f26bSFrançois Tigeot * seen value is good enough. Note that the seqno will always be
421b030f26bSFrançois Tigeot * monotonic, even if not coherent.
422b030f26bSFrançois Tigeot */
42387df8fc6SFrançois Tigeot void (*irq_seqno_barrier)(struct intel_engine_cs *engine);
42487df8fc6SFrançois Tigeot void (*cleanup)(struct intel_engine_cs *engine);
425e3adcf8fSFrançois Tigeot
42624edb884SFrançois Tigeot /* GEN8 signal/wait table - never trust comments!
42724edb884SFrançois Tigeot * signal to signal to signal to signal to signal to
42824edb884SFrançois Tigeot * RCS VCS BCS VECS VCS2
42924edb884SFrançois Tigeot * --------------------------------------------------------------------
43024edb884SFrançois Tigeot * RCS | NOP (0x00) | VCS (0x08) | BCS (0x10) | VECS (0x18) | VCS2 (0x20) |
43124edb884SFrançois Tigeot * |-------------------------------------------------------------------
43224edb884SFrançois Tigeot * VCS | RCS (0x28) | NOP (0x30) | BCS (0x38) | VECS (0x40) | VCS2 (0x48) |
43324edb884SFrançois Tigeot * |-------------------------------------------------------------------
43424edb884SFrançois Tigeot * BCS | RCS (0x50) | VCS (0x58) | NOP (0x60) | VECS (0x68) | VCS2 (0x70) |
43524edb884SFrançois Tigeot * |-------------------------------------------------------------------
43624edb884SFrançois Tigeot * VECS | RCS (0x78) | VCS (0x80) | BCS (0x88) | NOP (0x90) | VCS2 (0x98) |
43724edb884SFrançois Tigeot * |-------------------------------------------------------------------
43824edb884SFrançois Tigeot * VCS2 | RCS (0xa0) | VCS (0xa8) | BCS (0xb0) | VECS (0xb8) | NOP (0xc0) |
43924edb884SFrançois Tigeot * |-------------------------------------------------------------------
44024edb884SFrançois Tigeot *
44124edb884SFrançois Tigeot * Generalization:
44224edb884SFrançois Tigeot * f(x, y) := (x->id * NUM_RINGS * seqno_size) + (seqno_size * y->id)
44324edb884SFrançois Tigeot * ie. transpose of g(x, y)
44424edb884SFrançois Tigeot *
44524edb884SFrançois Tigeot * sync from sync from sync from sync from sync from
44624edb884SFrançois Tigeot * RCS VCS BCS VECS VCS2
44724edb884SFrançois Tigeot * --------------------------------------------------------------------
44824edb884SFrançois Tigeot * RCS | NOP (0x00) | VCS (0x28) | BCS (0x50) | VECS (0x78) | VCS2 (0xa0) |
44924edb884SFrançois Tigeot * |-------------------------------------------------------------------
45024edb884SFrançois Tigeot * VCS | RCS (0x08) | NOP (0x30) | BCS (0x58) | VECS (0x80) | VCS2 (0xa8) |
45124edb884SFrançois Tigeot * |-------------------------------------------------------------------
45224edb884SFrançois Tigeot * BCS | RCS (0x10) | VCS (0x38) | NOP (0x60) | VECS (0x88) | VCS2 (0xb0) |
45324edb884SFrançois Tigeot * |-------------------------------------------------------------------
45424edb884SFrançois Tigeot * VECS | RCS (0x18) | VCS (0x40) | BCS (0x68) | NOP (0x90) | VCS2 (0xb8) |
45524edb884SFrançois Tigeot * |-------------------------------------------------------------------
45624edb884SFrançois Tigeot * VCS2 | RCS (0x20) | VCS (0x48) | BCS (0x70) | VECS (0x98) | NOP (0xc0) |
45724edb884SFrançois Tigeot * |-------------------------------------------------------------------
45824edb884SFrançois Tigeot *
45924edb884SFrançois Tigeot * Generalization:
46024edb884SFrançois Tigeot * g(x, y) := (y->id * NUM_RINGS * seqno_size) + (seqno_size * x->id)
46124edb884SFrançois Tigeot * ie. transpose of f(x, y)
46224edb884SFrançois Tigeot */
463ba55f2f5SFrançois Tigeot struct {
46424edb884SFrançois Tigeot union {
4651e12ee3bSFrançois Tigeot #define GEN6_SEMAPHORE_LAST VECS_HW
4661e12ee3bSFrançois Tigeot #define GEN6_NUM_SEMAPHORES (GEN6_SEMAPHORE_LAST + 1)
4671e12ee3bSFrançois Tigeot #define GEN6_SEMAPHORES_MASK GENMASK(GEN6_SEMAPHORE_LAST, 0)
468ba55f2f5SFrançois Tigeot struct {
4695d0b1887SFrançois Tigeot /* our mbox written by others */
4701e12ee3bSFrançois Tigeot u32 wait[GEN6_NUM_SEMAPHORES];
4715d0b1887SFrançois Tigeot /* mboxes this ring signals to */
4721e12ee3bSFrançois Tigeot i915_reg_t signal[GEN6_NUM_SEMAPHORES];
473ba55f2f5SFrançois Tigeot } mbox;
4748621f407SFrançois Tigeot u64 signal_ggtt[I915_NUM_ENGINES];
47524edb884SFrançois Tigeot };
476ba55f2f5SFrançois Tigeot
477ba55f2f5SFrançois Tigeot /* AKA wait() */
47871f41f3eSFrançois Tigeot int (*sync_to)(struct drm_i915_gem_request *req,
47971f41f3eSFrançois Tigeot struct drm_i915_gem_request *signal);
480a85cb24fSFrançois Tigeot u32 *(*signal)(struct drm_i915_gem_request *req, u32 *cs);
481ba55f2f5SFrançois Tigeot } semaphore;
4825d0b1887SFrançois Tigeot
483*3f2dd94aSFrançois Tigeot struct intel_engine_execlists execlists;
484e3adcf8fSFrançois Tigeot
485a85cb24fSFrançois Tigeot /* Contexts are pinned whilst they are active on the GPU. The last
486a85cb24fSFrançois Tigeot * context executed remains active whilst the GPU is idle - the
487a85cb24fSFrançois Tigeot * switch away and write to the context object only occurs on the
488a85cb24fSFrançois Tigeot * next execution. Contexts are only unpinned on retirement of the
489a85cb24fSFrançois Tigeot * following request ensuring that we can always write to the object
490a85cb24fSFrançois Tigeot * on the context switch even after idling. Across suspend, we switch
491a85cb24fSFrançois Tigeot * to the kernel context and trash it as the save may not happen
492a85cb24fSFrançois Tigeot * before the hardware is powered down.
493a85cb24fSFrançois Tigeot */
494a85cb24fSFrançois Tigeot struct i915_gem_context *last_retired_context;
495a85cb24fSFrançois Tigeot
496a85cb24fSFrançois Tigeot /* We track the current MI_SET_CONTEXT in order to eliminate
497a85cb24fSFrançois Tigeot * redudant context switches. This presumes that requests are not
498a85cb24fSFrançois Tigeot * reordered! Or when they are the tracking is updated along with
499a85cb24fSFrançois Tigeot * the emission of individual requests into the legacy command
500a85cb24fSFrançois Tigeot * stream (ring).
501a85cb24fSFrançois Tigeot */
502a85cb24fSFrançois Tigeot struct i915_gem_context *legacy_active_context;
503a85cb24fSFrançois Tigeot
504a85cb24fSFrançois Tigeot /* status_notifier: list of callbacks for context-switch changes */
505a85cb24fSFrançois Tigeot struct atomic_notifier_head context_status_notifier;
5065d0b1887SFrançois Tigeot
50771f41f3eSFrançois Tigeot struct intel_engine_hangcheck hangcheck;
50815ac6249SFrançois Tigeot
509ba55f2f5SFrançois Tigeot bool needs_cmd_parser;
510ba55f2f5SFrançois Tigeot
511ba55f2f5SFrançois Tigeot /*
512ba55f2f5SFrançois Tigeot * Table of commands the command parser needs to know about
51371f41f3eSFrançois Tigeot * for this engine.
514ba55f2f5SFrançois Tigeot */
515ba55f2f5SFrançois Tigeot DECLARE_HASHTABLE(cmd_hash, I915_CMD_HASH_ORDER);
516ba55f2f5SFrançois Tigeot
517ba55f2f5SFrançois Tigeot /*
518ba55f2f5SFrançois Tigeot * Table of registers allowed in commands that read/write registers.
519ba55f2f5SFrançois Tigeot */
5208621f407SFrançois Tigeot const struct drm_i915_reg_table *reg_tables;
5218621f407SFrançois Tigeot int reg_table_count;
522ba55f2f5SFrançois Tigeot
523ba55f2f5SFrançois Tigeot /*
524ba55f2f5SFrançois Tigeot * Returns the bitmask for the length field of the specified command.
525ba55f2f5SFrançois Tigeot * Return 0 for an unrecognized/invalid command.
526ba55f2f5SFrançois Tigeot *
52771f41f3eSFrançois Tigeot * If the command parser finds an entry for a command in the engine's
528ba55f2f5SFrançois Tigeot * cmd_tables, it gets the command's length based on the table entry.
52971f41f3eSFrançois Tigeot * If not, it calls this function to determine the per-engine length
53071f41f3eSFrançois Tigeot * field encoding for the command (i.e. different opcode ranges use
53171f41f3eSFrançois Tigeot * certain bits to encode the command length in the header).
532ba55f2f5SFrançois Tigeot */
533ba55f2f5SFrançois Tigeot u32 (*get_cmd_length_mask)(u32 cmd_header);
534e3adcf8fSFrançois Tigeot };
535e3adcf8fSFrançois Tigeot
536*3f2dd94aSFrançois Tigeot static inline void
execlists_set_active(struct intel_engine_execlists * execlists,unsigned int bit)537*3f2dd94aSFrançois Tigeot execlists_set_active(struct intel_engine_execlists *execlists,
538*3f2dd94aSFrançois Tigeot unsigned int bit)
539*3f2dd94aSFrançois Tigeot {
540*3f2dd94aSFrançois Tigeot __set_bit(bit, (unsigned long *)&execlists->active);
541*3f2dd94aSFrançois Tigeot }
542*3f2dd94aSFrançois Tigeot
543*3f2dd94aSFrançois Tigeot static inline void
execlists_clear_active(struct intel_engine_execlists * execlists,unsigned int bit)544*3f2dd94aSFrançois Tigeot execlists_clear_active(struct intel_engine_execlists *execlists,
545*3f2dd94aSFrançois Tigeot unsigned int bit)
546*3f2dd94aSFrançois Tigeot {
547*3f2dd94aSFrançois Tigeot __clear_bit(bit, (unsigned long *)&execlists->active);
548*3f2dd94aSFrançois Tigeot }
549*3f2dd94aSFrançois Tigeot
550*3f2dd94aSFrançois Tigeot static inline bool
execlists_is_active(const struct intel_engine_execlists * execlists,unsigned int bit)551*3f2dd94aSFrançois Tigeot execlists_is_active(const struct intel_engine_execlists *execlists,
552*3f2dd94aSFrançois Tigeot unsigned int bit)
553*3f2dd94aSFrançois Tigeot {
554*3f2dd94aSFrançois Tigeot return test_bit(bit, (unsigned long *)&execlists->active);
555*3f2dd94aSFrançois Tigeot }
556*3f2dd94aSFrançois Tigeot
557*3f2dd94aSFrançois Tigeot static inline unsigned int
execlists_num_ports(const struct intel_engine_execlists * const execlists)558*3f2dd94aSFrançois Tigeot execlists_num_ports(const struct intel_engine_execlists * const execlists)
559*3f2dd94aSFrançois Tigeot {
560*3f2dd94aSFrançois Tigeot return execlists->port_mask + 1;
561*3f2dd94aSFrançois Tigeot }
562*3f2dd94aSFrançois Tigeot
563*3f2dd94aSFrançois Tigeot static inline void
execlists_port_complete(struct intel_engine_execlists * const execlists,struct execlist_port * const port)564*3f2dd94aSFrançois Tigeot execlists_port_complete(struct intel_engine_execlists * const execlists,
565*3f2dd94aSFrançois Tigeot struct execlist_port * const port)
566*3f2dd94aSFrançois Tigeot {
567*3f2dd94aSFrançois Tigeot const unsigned int m = execlists->port_mask;
568*3f2dd94aSFrançois Tigeot
569*3f2dd94aSFrançois Tigeot GEM_BUG_ON(port_index(port, execlists) != 0);
570*3f2dd94aSFrançois Tigeot GEM_BUG_ON(!execlists_is_active(execlists, EXECLISTS_ACTIVE_USER));
571*3f2dd94aSFrançois Tigeot
572*3f2dd94aSFrançois Tigeot memmove(port, port + 1, m * sizeof(struct execlist_port));
573*3f2dd94aSFrançois Tigeot memset(port + m, 0, sizeof(struct execlist_port));
574*3f2dd94aSFrançois Tigeot }
575*3f2dd94aSFrançois Tigeot
576a85cb24fSFrançois Tigeot static inline unsigned int
intel_engine_flag(const struct intel_engine_cs * engine)577303bf270SFrançois Tigeot intel_engine_flag(const struct intel_engine_cs *engine)
578e3adcf8fSFrançois Tigeot {
579a85cb24fSFrançois Tigeot return BIT(engine->id);
580352ff8bdSFrançois Tigeot }
581352ff8bdSFrançois Tigeot
582f4e1c372SFrançois Tigeot static inline u32
intel_read_status_page(struct intel_engine_cs * engine,int reg)5838621f407SFrançois Tigeot intel_read_status_page(struct intel_engine_cs *engine, int reg)
584e3adcf8fSFrançois Tigeot {
585f4e1c372SFrançois Tigeot /* Ensure that the compiler doesn't optimize away the load. */
5868621f407SFrançois Tigeot return READ_ONCE(engine->status_page.page_addr[reg]);
587e3adcf8fSFrançois Tigeot }
588e3adcf8fSFrançois Tigeot
589a2fdbec6SFrançois Tigeot static inline void
intel_write_status_page(struct intel_engine_cs * engine,int reg,u32 value)590a85cb24fSFrançois Tigeot intel_write_status_page(struct intel_engine_cs *engine, int reg, u32 value)
591a2fdbec6SFrançois Tigeot {
592a85cb24fSFrançois Tigeot /* Writing into the status page should be done sparingly. Since
593a85cb24fSFrançois Tigeot * we do when we are uncertain of the device state, we take a bit
594a85cb24fSFrançois Tigeot * of extra paranoia to try and ensure that the HWS takes the value
595a85cb24fSFrançois Tigeot * we give and that it doesn't end up trapped inside the CPU!
596a85cb24fSFrançois Tigeot */
597a85cb24fSFrançois Tigeot if (static_cpu_has(X86_FEATURE_CLFLUSH)) {
598a85cb24fSFrançois Tigeot mb();
599a85cb24fSFrançois Tigeot linux_clflush(&engine->status_page.page_addr[reg]);
6008621f407SFrançois Tigeot engine->status_page.page_addr[reg] = value;
601a85cb24fSFrançois Tigeot linux_clflush(&engine->status_page.page_addr[reg]);
602a85cb24fSFrançois Tigeot mb();
603a85cb24fSFrançois Tigeot } else {
604a85cb24fSFrançois Tigeot WRITE_ONCE(engine->status_page.page_addr[reg], value);
605a85cb24fSFrançois Tigeot }
606a2fdbec6SFrançois Tigeot }
607a2fdbec6SFrançois Tigeot
608c0e85e96SFrançois Tigeot /*
609f4e1c372SFrançois Tigeot * Reads a dword out of the status page, which is written to from the command
610f4e1c372SFrançois Tigeot * queue by automatic updates, MI_REPORT_HEAD, MI_STORE_DATA_INDEX, or
611f4e1c372SFrançois Tigeot * MI_STORE_DATA_IMM.
612f4e1c372SFrançois Tigeot *
613f4e1c372SFrançois Tigeot * The following dwords have a reserved meaning:
614f4e1c372SFrançois Tigeot * 0x00: ISR copy, updated when an ISR bit not set in the HWSTAM changes.
615f4e1c372SFrançois Tigeot * 0x04: ring 0 head pointer
616f4e1c372SFrançois Tigeot * 0x05: ring 1 head pointer (915-class)
617f4e1c372SFrançois Tigeot * 0x06: ring 2 head pointer (915-class)
618f4e1c372SFrançois Tigeot * 0x10-0x1b: Context status DWords (GM45)
619f4e1c372SFrançois Tigeot * 0x1f: Last written status offset. (GM45)
620477eb7f9SFrançois Tigeot * 0x20-0x2f: Reserved (Gen6+)
621f4e1c372SFrançois Tigeot *
622477eb7f9SFrançois Tigeot * The area from dword 0x30 to 0x3ff is available for driver usage.
623f4e1c372SFrançois Tigeot */
624477eb7f9SFrançois Tigeot #define I915_GEM_HWS_INDEX 0x30
625c0e85e96SFrançois Tigeot #define I915_GEM_HWS_INDEX_ADDR (I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
626477eb7f9SFrançois Tigeot #define I915_GEM_HWS_SCRATCH_INDEX 0x40
627f4e1c372SFrançois Tigeot #define I915_GEM_HWS_SCRATCH_ADDR (I915_GEM_HWS_SCRATCH_INDEX << MI_STORE_DWORD_INDEX_SHIFT)
628e3adcf8fSFrançois Tigeot
629*3f2dd94aSFrançois Tigeot #define I915_HWS_CSB_BUF0_INDEX 0x10
630*3f2dd94aSFrançois Tigeot #define I915_HWS_CSB_WRITE_INDEX 0x1f
631*3f2dd94aSFrançois Tigeot #define CNL_HWS_CSB_WRITE_INDEX 0x2f
632*3f2dd94aSFrançois Tigeot
63371f41f3eSFrançois Tigeot struct intel_ring *
63471f41f3eSFrançois Tigeot intel_engine_create_ring(struct intel_engine_cs *engine, int size);
635*3f2dd94aSFrançois Tigeot int intel_ring_pin(struct intel_ring *ring,
636*3f2dd94aSFrançois Tigeot struct drm_i915_private *i915,
637*3f2dd94aSFrançois Tigeot unsigned int offset_bias);
638a85cb24fSFrançois Tigeot void intel_ring_reset(struct intel_ring *ring, u32 tail);
639*3f2dd94aSFrançois Tigeot unsigned int intel_ring_update_space(struct intel_ring *ring);
64071f41f3eSFrançois Tigeot void intel_ring_unpin(struct intel_ring *ring);
64171f41f3eSFrançois Tigeot void intel_ring_free(struct intel_ring *ring);
6421b13d190SFrançois Tigeot
64371f41f3eSFrançois Tigeot void intel_engine_stop(struct intel_engine_cs *engine);
64471f41f3eSFrançois Tigeot void intel_engine_cleanup(struct intel_engine_cs *engine);
645f4e1c372SFrançois Tigeot
6461e12ee3bSFrançois Tigeot void intel_legacy_submission_resume(struct drm_i915_private *dev_priv);
6471e12ee3bSFrançois Tigeot
648a05eeebfSFrançois Tigeot int __must_check intel_ring_cacheline_align(struct drm_i915_gem_request *req);
64987df8fc6SFrançois Tigeot
650*3f2dd94aSFrançois Tigeot u32 __must_check *intel_ring_begin(struct drm_i915_gem_request *req,
651*3f2dd94aSFrançois Tigeot unsigned int n);
65287df8fc6SFrançois Tigeot
653a85cb24fSFrançois Tigeot static inline void
intel_ring_advance(struct drm_i915_gem_request * req,u32 * cs)654a85cb24fSFrançois Tigeot intel_ring_advance(struct drm_i915_gem_request *req, u32 *cs)
65587df8fc6SFrançois Tigeot {
65671f41f3eSFrançois Tigeot /* Dummy function.
65771f41f3eSFrançois Tigeot *
65871f41f3eSFrançois Tigeot * This serves as a placeholder in the code so that the reader
65971f41f3eSFrançois Tigeot * can compare against the preceding intel_ring_begin() and
66071f41f3eSFrançois Tigeot * check that the number of dwords emitted matches the space
66171f41f3eSFrançois Tigeot * reserved for the command packet (i.e. the value passed to
66271f41f3eSFrançois Tigeot * intel_ring_begin()).
66371f41f3eSFrançois Tigeot */
664a85cb24fSFrançois Tigeot GEM_BUG_ON((req->ring->vaddr + req->ring->emit) != cs);
66587df8fc6SFrançois Tigeot }
66687df8fc6SFrançois Tigeot
667a85cb24fSFrançois Tigeot static inline u32
intel_ring_wrap(const struct intel_ring * ring,u32 pos)668a85cb24fSFrançois Tigeot intel_ring_wrap(const struct intel_ring *ring, u32 pos)
669a85cb24fSFrançois Tigeot {
670a85cb24fSFrançois Tigeot return pos & (ring->size - 1);
671a85cb24fSFrançois Tigeot }
672a85cb24fSFrançois Tigeot
673a85cb24fSFrançois Tigeot static inline u32
intel_ring_offset(const struct drm_i915_gem_request * req,void * addr)674a85cb24fSFrançois Tigeot intel_ring_offset(const struct drm_i915_gem_request *req, void *addr)
675aee94f86SFrançois Tigeot {
67671f41f3eSFrançois Tigeot /* Don't write ring->size (equivalent to 0) as that hangs some GPUs. */
677a85cb24fSFrançois Tigeot u32 offset = addr - req->ring->vaddr;
678a85cb24fSFrançois Tigeot GEM_BUG_ON(offset > req->ring->size);
679a85cb24fSFrançois Tigeot return intel_ring_wrap(req->ring, offset);
6809edbd4a0SFrançois Tigeot }
68187df8fc6SFrançois Tigeot
682a85cb24fSFrançois Tigeot static inline void
assert_ring_tail_valid(const struct intel_ring * ring,unsigned int tail)683a85cb24fSFrançois Tigeot assert_ring_tail_valid(const struct intel_ring *ring, unsigned int tail)
684a85cb24fSFrançois Tigeot {
685a85cb24fSFrançois Tigeot /* We could combine these into a single tail operation, but keeping
686a85cb24fSFrançois Tigeot * them as seperate tests will help identify the cause should one
687a85cb24fSFrançois Tigeot * ever fire.
688a85cb24fSFrançois Tigeot */
689a85cb24fSFrançois Tigeot GEM_BUG_ON(!IS_ALIGNED(tail, 8));
690a85cb24fSFrançois Tigeot GEM_BUG_ON(tail >= ring->size);
691*3f2dd94aSFrançois Tigeot
692*3f2dd94aSFrançois Tigeot /*
693*3f2dd94aSFrançois Tigeot * "Ring Buffer Use"
694*3f2dd94aSFrançois Tigeot * Gen2 BSpec "1. Programming Environment" / 1.4.4.6
695*3f2dd94aSFrançois Tigeot * Gen3 BSpec "1c Memory Interface Functions" / 2.3.4.5
696*3f2dd94aSFrançois Tigeot * Gen4+ BSpec "1c Memory Interface and Command Stream" / 5.3.4.5
697*3f2dd94aSFrançois Tigeot * "If the Ring Buffer Head Pointer and the Tail Pointer are on the
698*3f2dd94aSFrançois Tigeot * same cacheline, the Head Pointer must not be greater than the Tail
699*3f2dd94aSFrançois Tigeot * Pointer."
700*3f2dd94aSFrançois Tigeot *
701*3f2dd94aSFrançois Tigeot * We use ring->head as the last known location of the actual RING_HEAD,
702*3f2dd94aSFrançois Tigeot * it may have advanced but in the worst case it is equally the same
703*3f2dd94aSFrançois Tigeot * as ring->head and so we should never program RING_TAIL to advance
704*3f2dd94aSFrançois Tigeot * into the same cacheline as ring->head.
705*3f2dd94aSFrançois Tigeot */
706*3f2dd94aSFrançois Tigeot #define cacheline(a) round_down(a, CACHELINE_BYTES)
707*3f2dd94aSFrançois Tigeot GEM_BUG_ON(cacheline(tail) == cacheline(ring->head) &&
708*3f2dd94aSFrançois Tigeot tail < ring->head);
709*3f2dd94aSFrançois Tigeot #undef cacheline
710a85cb24fSFrançois Tigeot }
711a85cb24fSFrançois Tigeot
712a85cb24fSFrançois Tigeot static inline unsigned int
intel_ring_set_tail(struct intel_ring * ring,unsigned int tail)713a85cb24fSFrançois Tigeot intel_ring_set_tail(struct intel_ring *ring, unsigned int tail)
714a85cb24fSFrançois Tigeot {
715a85cb24fSFrançois Tigeot /* Whilst writes to the tail are strictly order, there is no
716a85cb24fSFrançois Tigeot * serialisation between readers and the writers. The tail may be
717a85cb24fSFrançois Tigeot * read by i915_gem_request_retire() just as it is being updated
718a85cb24fSFrançois Tigeot * by execlists, as although the breadcrumb is complete, the context
719a85cb24fSFrançois Tigeot * switch hasn't been seen.
720a85cb24fSFrançois Tigeot */
721a85cb24fSFrançois Tigeot assert_ring_tail_valid(ring, tail);
722a85cb24fSFrançois Tigeot ring->tail = tail;
723a85cb24fSFrançois Tigeot return tail;
724a85cb24fSFrançois Tigeot }
7259edbd4a0SFrançois Tigeot
7264be47400SFrançois Tigeot void intel_engine_init_global_seqno(struct intel_engine_cs *engine, u32 seqno);
727e3adcf8fSFrançois Tigeot
72887df8fc6SFrançois Tigeot void intel_engine_setup_common(struct intel_engine_cs *engine);
72987df8fc6SFrançois Tigeot int intel_engine_init_common(struct intel_engine_cs *engine);
7301e12ee3bSFrançois Tigeot int intel_engine_create_scratch(struct intel_engine_cs *engine, int size);
73171f41f3eSFrançois Tigeot void intel_engine_cleanup_common(struct intel_engine_cs *engine);
73271f41f3eSFrançois Tigeot
73387df8fc6SFrançois Tigeot int intel_init_render_ring_buffer(struct intel_engine_cs *engine);
73487df8fc6SFrançois Tigeot int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine);
73587df8fc6SFrançois Tigeot int intel_init_blt_ring_buffer(struct intel_engine_cs *engine);
73687df8fc6SFrançois Tigeot int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine);
737e3adcf8fSFrançois Tigeot
73871f41f3eSFrançois Tigeot u64 intel_engine_get_active_head(struct intel_engine_cs *engine);
7391e12ee3bSFrançois Tigeot u64 intel_engine_get_last_batch_head(struct intel_engine_cs *engine);
7401e12ee3bSFrançois Tigeot
intel_engine_get_seqno(struct intel_engine_cs * engine)741303bf270SFrançois Tigeot static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine)
742303bf270SFrançois Tigeot {
743303bf270SFrançois Tigeot return intel_read_status_page(engine, I915_GEM_HWS_INDEX);
744303bf270SFrançois Tigeot }
745e3adcf8fSFrançois Tigeot
intel_engine_last_submit(struct intel_engine_cs * engine)7464be47400SFrançois Tigeot static inline u32 intel_engine_last_submit(struct intel_engine_cs *engine)
7474be47400SFrançois Tigeot {
7484be47400SFrançois Tigeot /* We are only peeking at the tail of the submit queue (and not the
7494be47400SFrançois Tigeot * queue itself) in order to gain a hint as to the current active
7504be47400SFrançois Tigeot * state of the engine. Callers are not expected to be taking
7514be47400SFrançois Tigeot * engine->timeline->lock, nor are they expected to be concerned
7524be47400SFrançois Tigeot * wtih serialising this hint with anything, so document it as
7534be47400SFrançois Tigeot * a hint and nothing more.
7544be47400SFrançois Tigeot */
755a85cb24fSFrançois Tigeot return READ_ONCE(engine->timeline->seqno);
7564be47400SFrançois Tigeot }
7574be47400SFrançois Tigeot
7588621f407SFrançois Tigeot int init_workarounds_ring(struct intel_engine_cs *engine);
759a85cb24fSFrançois Tigeot int intel_ring_workarounds_emit(struct drm_i915_gem_request *req);
7602c9916cdSFrançois Tigeot
7611e12ee3bSFrançois Tigeot void intel_engine_get_instdone(struct intel_engine_cs *engine,
7621e12ee3bSFrançois Tigeot struct intel_instdone *instdone);
7631e12ee3bSFrançois Tigeot
764a05eeebfSFrançois Tigeot /*
765a05eeebfSFrançois Tigeot * Arbitrary size for largest possible 'add request' sequence. The code paths
766a05eeebfSFrançois Tigeot * are complex and variable. Empirical measurement shows that the worst case
7671487f786SFrançois Tigeot * is BDW at 192 bytes (6 + 6 + 36 dwords), then ILK at 136 bytes. However,
7681487f786SFrançois Tigeot * we need to allocate double the largest single packet within that emission
7691487f786SFrançois Tigeot * to account for tail wraparound (so 6 + 6 + 72 dwords for BDW).
770a05eeebfSFrançois Tigeot */
7711487f786SFrançois Tigeot #define MIN_SPACE_FOR_ADD_REQUEST 336
772a05eeebfSFrançois Tigeot
intel_hws_seqno_address(struct intel_engine_cs * engine)7731487f786SFrançois Tigeot static inline u32 intel_hws_seqno_address(struct intel_engine_cs *engine)
7741487f786SFrançois Tigeot {
7751e12ee3bSFrançois Tigeot return engine->status_page.ggtt_offset + I915_GEM_HWS_INDEX_ADDR;
7761487f786SFrançois Tigeot }
777f4e1c372SFrançois Tigeot
778303bf270SFrançois Tigeot /* intel_breadcrumbs.c -- user interrupt bottom-half for waiters */
779303bf270SFrançois Tigeot int intel_engine_init_breadcrumbs(struct intel_engine_cs *engine);
780303bf270SFrançois Tigeot
intel_wait_init(struct intel_wait * wait,struct drm_i915_gem_request * rq)781a85cb24fSFrançois Tigeot static inline void intel_wait_init(struct intel_wait *wait,
782a85cb24fSFrançois Tigeot struct drm_i915_gem_request *rq)
783a85cb24fSFrançois Tigeot {
784a85cb24fSFrançois Tigeot wait->tsk = current;
785a85cb24fSFrançois Tigeot wait->request = rq;
786a85cb24fSFrançois Tigeot }
787a85cb24fSFrançois Tigeot
intel_wait_init_for_seqno(struct intel_wait * wait,u32 seqno)788a85cb24fSFrançois Tigeot static inline void intel_wait_init_for_seqno(struct intel_wait *wait, u32 seqno)
789303bf270SFrançois Tigeot {
790303bf270SFrançois Tigeot wait->tsk = current;
791303bf270SFrançois Tigeot wait->seqno = seqno;
792303bf270SFrançois Tigeot }
793303bf270SFrançois Tigeot
intel_wait_has_seqno(const struct intel_wait * wait)794a85cb24fSFrançois Tigeot static inline bool intel_wait_has_seqno(const struct intel_wait *wait)
795a85cb24fSFrançois Tigeot {
796a85cb24fSFrançois Tigeot return wait->seqno;
797a85cb24fSFrançois Tigeot }
798a85cb24fSFrançois Tigeot
799a85cb24fSFrançois Tigeot static inline bool
intel_wait_update_seqno(struct intel_wait * wait,u32 seqno)800a85cb24fSFrançois Tigeot intel_wait_update_seqno(struct intel_wait *wait, u32 seqno)
801a85cb24fSFrançois Tigeot {
802a85cb24fSFrançois Tigeot wait->seqno = seqno;
803a85cb24fSFrançois Tigeot return intel_wait_has_seqno(wait);
804a85cb24fSFrançois Tigeot }
805a85cb24fSFrançois Tigeot
806a85cb24fSFrançois Tigeot static inline bool
intel_wait_update_request(struct intel_wait * wait,const struct drm_i915_gem_request * rq)807a85cb24fSFrançois Tigeot intel_wait_update_request(struct intel_wait *wait,
808a85cb24fSFrançois Tigeot const struct drm_i915_gem_request *rq)
809a85cb24fSFrançois Tigeot {
810a85cb24fSFrançois Tigeot return intel_wait_update_seqno(wait, i915_gem_request_global_seqno(rq));
811a85cb24fSFrançois Tigeot }
812a85cb24fSFrançois Tigeot
813a85cb24fSFrançois Tigeot static inline bool
intel_wait_check_seqno(const struct intel_wait * wait,u32 seqno)814a85cb24fSFrançois Tigeot intel_wait_check_seqno(const struct intel_wait *wait, u32 seqno)
815a85cb24fSFrançois Tigeot {
816a85cb24fSFrançois Tigeot return wait->seqno == seqno;
817a85cb24fSFrançois Tigeot }
818a85cb24fSFrançois Tigeot
819a85cb24fSFrançois Tigeot static inline bool
intel_wait_check_request(const struct intel_wait * wait,const struct drm_i915_gem_request * rq)820a85cb24fSFrançois Tigeot intel_wait_check_request(const struct intel_wait *wait,
821a85cb24fSFrançois Tigeot const struct drm_i915_gem_request *rq)
822a85cb24fSFrançois Tigeot {
823a85cb24fSFrançois Tigeot return intel_wait_check_seqno(wait, i915_gem_request_global_seqno(rq));
824a85cb24fSFrançois Tigeot }
825a85cb24fSFrançois Tigeot
intel_wait_complete(const struct intel_wait * wait)826303bf270SFrançois Tigeot static inline bool intel_wait_complete(const struct intel_wait *wait)
827303bf270SFrançois Tigeot {
828303bf270SFrançois Tigeot return RB_EMPTY_NODE(&wait->node);
829303bf270SFrançois Tigeot }
830303bf270SFrançois Tigeot
831303bf270SFrançois Tigeot bool intel_engine_add_wait(struct intel_engine_cs *engine,
832303bf270SFrançois Tigeot struct intel_wait *wait);
833303bf270SFrançois Tigeot void intel_engine_remove_wait(struct intel_engine_cs *engine,
834303bf270SFrançois Tigeot struct intel_wait *wait);
835*3f2dd94aSFrançois Tigeot void intel_engine_enable_signaling(struct drm_i915_gem_request *request,
836*3f2dd94aSFrançois Tigeot bool wakeup);
837a85cb24fSFrançois Tigeot void intel_engine_cancel_signaling(struct drm_i915_gem_request *request);
838303bf270SFrançois Tigeot
intel_engine_has_waiter(const struct intel_engine_cs * engine)8391e12ee3bSFrançois Tigeot static inline bool intel_engine_has_waiter(const struct intel_engine_cs *engine)
840303bf270SFrançois Tigeot {
841a85cb24fSFrançois Tigeot return READ_ONCE(engine->breadcrumbs.irq_wait);
842303bf270SFrançois Tigeot }
843303bf270SFrançois Tigeot
844a85cb24fSFrançois Tigeot unsigned int intel_engine_wakeup(struct intel_engine_cs *engine);
845a85cb24fSFrançois Tigeot #define ENGINE_WAKEUP_WAITER BIT(0)
846a85cb24fSFrançois Tigeot #define ENGINE_WAKEUP_ASLEEP BIT(1)
8471e12ee3bSFrançois Tigeot
848a85cb24fSFrançois Tigeot void __intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine);
849a85cb24fSFrançois Tigeot void intel_engine_disarm_breadcrumbs(struct intel_engine_cs *engine);
850303bf270SFrançois Tigeot
8511e12ee3bSFrançois Tigeot void intel_engine_reset_breadcrumbs(struct intel_engine_cs *engine);
852303bf270SFrançois Tigeot void intel_engine_fini_breadcrumbs(struct intel_engine_cs *engine);
853a85cb24fSFrançois Tigeot bool intel_breadcrumbs_busy(struct intel_engine_cs *engine);
854a85cb24fSFrançois Tigeot
gen8_emit_pipe_control(u32 * batch,u32 flags,u32 offset)855a85cb24fSFrançois Tigeot static inline u32 *gen8_emit_pipe_control(u32 *batch, u32 flags, u32 offset)
856a85cb24fSFrançois Tigeot {
857a85cb24fSFrançois Tigeot memset(batch, 0, 6 * sizeof(u32));
858a85cb24fSFrançois Tigeot
859a85cb24fSFrançois Tigeot batch[0] = GFX_OP_PIPE_CONTROL(6);
860a85cb24fSFrançois Tigeot batch[1] = flags;
861a85cb24fSFrançois Tigeot batch[2] = offset;
862a85cb24fSFrançois Tigeot
863a85cb24fSFrançois Tigeot return batch + 6;
864a85cb24fSFrançois Tigeot }
865a85cb24fSFrançois Tigeot
866a85cb24fSFrançois Tigeot bool intel_engine_is_idle(struct intel_engine_cs *engine);
867a85cb24fSFrançois Tigeot bool intel_engines_are_idle(struct drm_i915_private *dev_priv);
868a85cb24fSFrançois Tigeot
869*3f2dd94aSFrançois Tigeot void intel_engines_mark_idle(struct drm_i915_private *i915);
870a85cb24fSFrançois Tigeot void intel_engines_reset_default_submission(struct drm_i915_private *i915);
87171f41f3eSFrançois Tigeot
872*3f2dd94aSFrançois Tigeot bool intel_engine_can_store_dword(struct intel_engine_cs *engine);
873*3f2dd94aSFrançois Tigeot
874*3f2dd94aSFrançois Tigeot void intel_engine_dump(struct intel_engine_cs *engine, struct drm_printer *p);
875*3f2dd94aSFrançois Tigeot
876e3adcf8fSFrançois Tigeot #endif /* _INTEL_RINGBUFFER_H_ */
877