xref: /dflybsd-src/sys/dev/drm/i915/intel_ringbuffer.c (revision 3f2dd94a569761201b5b0a18b2f697f97fe1b9dc)
1e3adcf8fSFrançois Tigeot /*
2e3adcf8fSFrançois Tigeot  * Copyright © 2008-2010 Intel Corporation
3e3adcf8fSFrançois Tigeot  *
4e3adcf8fSFrançois Tigeot  * Permission is hereby granted, free of charge, to any person obtaining a
5e3adcf8fSFrançois Tigeot  * copy of this software and associated documentation files (the "Software"),
6e3adcf8fSFrançois Tigeot  * to deal in the Software without restriction, including without limitation
7e3adcf8fSFrançois Tigeot  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8e3adcf8fSFrançois Tigeot  * and/or sell copies of the Software, and to permit persons to whom the
9e3adcf8fSFrançois Tigeot  * Software is furnished to do so, subject to the following conditions:
10e3adcf8fSFrançois Tigeot  *
11e3adcf8fSFrançois Tigeot  * The above copyright notice and this permission notice (including the next
12e3adcf8fSFrançois Tigeot  * paragraph) shall be included in all copies or substantial portions of the
13e3adcf8fSFrançois Tigeot  * Software.
14e3adcf8fSFrançois Tigeot  *
15e3adcf8fSFrançois Tigeot  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16e3adcf8fSFrançois Tigeot  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17e3adcf8fSFrançois Tigeot  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18e3adcf8fSFrançois Tigeot  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19e3adcf8fSFrançois Tigeot  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20e3adcf8fSFrançois Tigeot  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21e3adcf8fSFrançois Tigeot  * IN THE SOFTWARE.
22e3adcf8fSFrançois Tigeot  *
23e3adcf8fSFrançois Tigeot  * Authors:
24e3adcf8fSFrançois Tigeot  *    Eric Anholt <eric@anholt.net>
25e3adcf8fSFrançois Tigeot  *    Zou Nan hai <nanhai.zou@intel.com>
26e3adcf8fSFrançois Tigeot  *    Xiang Hai hao<haihao.xiang@intel.com>
27e3adcf8fSFrançois Tigeot  *
28e3adcf8fSFrançois Tigeot  */
29e3adcf8fSFrançois Tigeot 
30aee94f86SFrançois Tigeot #include <linux/log2.h>
3118e26a6dSFrançois Tigeot #include <drm/drmP.h>
32e3adcf8fSFrançois Tigeot #include "i915_drv.h"
33a2fdbec6SFrançois Tigeot #include <drm/i915_drm.h>
34a2fdbec6SFrançois Tigeot #include "i915_trace.h"
35e3adcf8fSFrançois Tigeot #include "intel_drv.h"
36e3adcf8fSFrançois Tigeot 
371487f786SFrançois Tigeot /* Rough estimate of the typical request size, performing a flush,
381487f786SFrançois Tigeot  * set-context and then emitting the batch.
391487f786SFrançois Tigeot  */
401487f786SFrançois Tigeot #define LEGACY_REQUEST_SIZE 200
411487f786SFrançois Tigeot 
__intel_ring_space(unsigned int head,unsigned int tail,unsigned int size)42*3f2dd94aSFrançois Tigeot static unsigned int __intel_ring_space(unsigned int head,
43*3f2dd94aSFrançois Tigeot 				       unsigned int tail,
44*3f2dd94aSFrançois Tigeot 				       unsigned int size)
45e3adcf8fSFrançois Tigeot {
46*3f2dd94aSFrançois Tigeot 	/*
47*3f2dd94aSFrançois Tigeot 	 * "If the Ring Buffer Head Pointer and the Tail Pointer are on the
48*3f2dd94aSFrançois Tigeot 	 * same cacheline, the Head Pointer must not be greater than the Tail
49*3f2dd94aSFrançois Tigeot 	 * Pointer."
50*3f2dd94aSFrançois Tigeot 	 */
51*3f2dd94aSFrançois Tigeot 	GEM_BUG_ON(!is_power_of_2(size));
52*3f2dd94aSFrançois Tigeot 	return (head - tail - CACHELINE_BYTES) & (size - 1);
532c9916cdSFrançois Tigeot }
542c9916cdSFrançois Tigeot 
intel_ring_update_space(struct intel_ring * ring)55*3f2dd94aSFrançois Tigeot unsigned int intel_ring_update_space(struct intel_ring *ring)
562c9916cdSFrançois Tigeot {
57*3f2dd94aSFrançois Tigeot 	unsigned int space;
58*3f2dd94aSFrançois Tigeot 
59*3f2dd94aSFrançois Tigeot 	space = __intel_ring_space(ring->head, ring->emit, ring->size);
60*3f2dd94aSFrançois Tigeot 
61*3f2dd94aSFrançois Tigeot 	ring->space = space;
62*3f2dd94aSFrançois Tigeot 	return space;
639edbd4a0SFrançois Tigeot }
649edbd4a0SFrançois Tigeot 
65e3adcf8fSFrançois Tigeot static int
gen2_render_ring_flush(struct drm_i915_gem_request * req,u32 mode)6671f41f3eSFrançois Tigeot gen2_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
67686a02f1SFrançois Tigeot {
68a85cb24fSFrançois Tigeot 	u32 cmd, *cs;
69686a02f1SFrançois Tigeot 
70686a02f1SFrançois Tigeot 	cmd = MI_FLUSH;
71686a02f1SFrançois Tigeot 
7271f41f3eSFrançois Tigeot 	if (mode & EMIT_INVALIDATE)
73686a02f1SFrançois Tigeot 		cmd |= MI_READ_FLUSH;
74686a02f1SFrançois Tigeot 
75a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 2);
76a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
77a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
78686a02f1SFrançois Tigeot 
79a85cb24fSFrançois Tigeot 	*cs++ = cmd;
80a85cb24fSFrançois Tigeot 	*cs++ = MI_NOOP;
81a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
82686a02f1SFrançois Tigeot 
83686a02f1SFrançois Tigeot 	return 0;
84686a02f1SFrançois Tigeot }
85686a02f1SFrançois Tigeot 
86686a02f1SFrançois Tigeot static int
gen4_render_ring_flush(struct drm_i915_gem_request * req,u32 mode)8771f41f3eSFrançois Tigeot gen4_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
88e3adcf8fSFrançois Tigeot {
89a85cb24fSFrançois Tigeot 	u32 cmd, *cs;
90e3adcf8fSFrançois Tigeot 
91e3adcf8fSFrançois Tigeot 	/*
92e3adcf8fSFrançois Tigeot 	 * read/write caches:
93e3adcf8fSFrançois Tigeot 	 *
94e3adcf8fSFrançois Tigeot 	 * I915_GEM_DOMAIN_RENDER is always invalidated, but is
95e3adcf8fSFrançois Tigeot 	 * only flushed if MI_NO_WRITE_FLUSH is unset.  On 965, it is
96e3adcf8fSFrançois Tigeot 	 * also flushed at 2d versus 3d pipeline switches.
97e3adcf8fSFrançois Tigeot 	 *
98e3adcf8fSFrançois Tigeot 	 * read-only caches:
99e3adcf8fSFrançois Tigeot 	 *
100e3adcf8fSFrançois Tigeot 	 * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
101e3adcf8fSFrançois Tigeot 	 * MI_READ_FLUSH is set, and is always flushed on 965.
102e3adcf8fSFrançois Tigeot 	 *
103e3adcf8fSFrançois Tigeot 	 * I915_GEM_DOMAIN_COMMAND may not exist?
104e3adcf8fSFrançois Tigeot 	 *
105e3adcf8fSFrançois Tigeot 	 * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
106e3adcf8fSFrançois Tigeot 	 * invalidated when MI_EXE_FLUSH is set.
107e3adcf8fSFrançois Tigeot 	 *
108e3adcf8fSFrançois Tigeot 	 * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
109e3adcf8fSFrançois Tigeot 	 * invalidated with every MI_FLUSH.
110e3adcf8fSFrançois Tigeot 	 *
111e3adcf8fSFrançois Tigeot 	 * TLBs:
112e3adcf8fSFrançois Tigeot 	 *
113e3adcf8fSFrançois Tigeot 	 * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
114e3adcf8fSFrançois Tigeot 	 * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
115e3adcf8fSFrançois Tigeot 	 * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
116e3adcf8fSFrançois Tigeot 	 * are flushed at any MI_FLUSH.
117e3adcf8fSFrançois Tigeot 	 */
118e3adcf8fSFrançois Tigeot 
11971f41f3eSFrançois Tigeot 	cmd = MI_FLUSH;
12071f41f3eSFrançois Tigeot 	if (mode & EMIT_INVALIDATE) {
121e3adcf8fSFrançois Tigeot 		cmd |= MI_EXE_FLUSH;
12271f41f3eSFrançois Tigeot 		if (IS_G4X(req->i915) || IS_GEN5(req->i915))
123e3adcf8fSFrançois Tigeot 			cmd |= MI_INVALIDATE_ISP;
12471f41f3eSFrançois Tigeot 	}
125e3adcf8fSFrançois Tigeot 
126a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 2);
127a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
128a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
129e3adcf8fSFrançois Tigeot 
130a85cb24fSFrançois Tigeot 	*cs++ = cmd;
131a85cb24fSFrançois Tigeot 	*cs++ = MI_NOOP;
132a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
133e3adcf8fSFrançois Tigeot 
134e3adcf8fSFrançois Tigeot 	return 0;
135e3adcf8fSFrançois Tigeot }
136e3adcf8fSFrançois Tigeot 
137e3adcf8fSFrançois Tigeot /**
138e3adcf8fSFrançois Tigeot  * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
139e3adcf8fSFrançois Tigeot  * implementing two workarounds on gen6.  From section 1.4.7.1
140e3adcf8fSFrançois Tigeot  * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
141e3adcf8fSFrançois Tigeot  *
142e3adcf8fSFrançois Tigeot  * [DevSNB-C+{W/A}] Before any depth stall flush (including those
143e3adcf8fSFrançois Tigeot  * produced by non-pipelined state commands), software needs to first
144e3adcf8fSFrançois Tigeot  * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
145e3adcf8fSFrançois Tigeot  * 0.
146e3adcf8fSFrançois Tigeot  *
147e3adcf8fSFrançois Tigeot  * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
148e3adcf8fSFrançois Tigeot  * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
149e3adcf8fSFrançois Tigeot  *
150e3adcf8fSFrançois Tigeot  * And the workaround for these two requires this workaround first:
151e3adcf8fSFrançois Tigeot  *
152e3adcf8fSFrançois Tigeot  * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
153e3adcf8fSFrançois Tigeot  * BEFORE the pipe-control with a post-sync op and no write-cache
154e3adcf8fSFrançois Tigeot  * flushes.
155e3adcf8fSFrançois Tigeot  *
156e3adcf8fSFrançois Tigeot  * And this last workaround is tricky because of the requirements on
157e3adcf8fSFrançois Tigeot  * that bit.  From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
158e3adcf8fSFrançois Tigeot  * volume 2 part 1:
159e3adcf8fSFrançois Tigeot  *
160e3adcf8fSFrançois Tigeot  *     "1 of the following must also be set:
161e3adcf8fSFrançois Tigeot  *      - Render Target Cache Flush Enable ([12] of DW1)
162e3adcf8fSFrançois Tigeot  *      - Depth Cache Flush Enable ([0] of DW1)
163e3adcf8fSFrançois Tigeot  *      - Stall at Pixel Scoreboard ([1] of DW1)
164e3adcf8fSFrançois Tigeot  *      - Depth Stall ([13] of DW1)
165e3adcf8fSFrançois Tigeot  *      - Post-Sync Operation ([13] of DW1)
166e3adcf8fSFrançois Tigeot  *      - Notify Enable ([8] of DW1)"
167e3adcf8fSFrançois Tigeot  *
168e3adcf8fSFrançois Tigeot  * The cache flushes require the workaround flush that triggered this
169e3adcf8fSFrançois Tigeot  * one, so we can't use it.  Depth stall would trigger the same.
170e3adcf8fSFrançois Tigeot  * Post-sync nonzero is what triggered this second workaround, so we
171e3adcf8fSFrançois Tigeot  * can't use that one either.  Notify enable is IRQs, which aren't
172e3adcf8fSFrançois Tigeot  * really our business.  That leaves only stall at scoreboard.
173e3adcf8fSFrançois Tigeot  */
174e3adcf8fSFrançois Tigeot static int
intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request * req)175a05eeebfSFrançois Tigeot intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req)
176e3adcf8fSFrançois Tigeot {
17771f41f3eSFrançois Tigeot 	u32 scratch_addr =
1781e12ee3bSFrançois Tigeot 		i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
179a85cb24fSFrançois Tigeot 	u32 *cs;
180e3adcf8fSFrançois Tigeot 
181a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 6);
182a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
183a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
184e3adcf8fSFrançois Tigeot 
185a85cb24fSFrançois Tigeot 	*cs++ = GFX_OP_PIPE_CONTROL(5);
186a85cb24fSFrançois Tigeot 	*cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;
187a85cb24fSFrançois Tigeot 	*cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;
188a85cb24fSFrançois Tigeot 	*cs++ = 0; /* low dword */
189a85cb24fSFrançois Tigeot 	*cs++ = 0; /* high dword */
190a85cb24fSFrançois Tigeot 	*cs++ = MI_NOOP;
191a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
192e3adcf8fSFrançois Tigeot 
193a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 6);
194a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
195a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
196e3adcf8fSFrançois Tigeot 
197a85cb24fSFrançois Tigeot 	*cs++ = GFX_OP_PIPE_CONTROL(5);
198a85cb24fSFrançois Tigeot 	*cs++ = PIPE_CONTROL_QW_WRITE;
199a85cb24fSFrançois Tigeot 	*cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;
200a85cb24fSFrançois Tigeot 	*cs++ = 0;
201a85cb24fSFrançois Tigeot 	*cs++ = 0;
202a85cb24fSFrançois Tigeot 	*cs++ = MI_NOOP;
203a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
204e3adcf8fSFrançois Tigeot 
205e3adcf8fSFrançois Tigeot 	return 0;
206e3adcf8fSFrançois Tigeot }
207e3adcf8fSFrançois Tigeot 
208e3adcf8fSFrançois Tigeot static int
gen6_render_ring_flush(struct drm_i915_gem_request * req,u32 mode)20971f41f3eSFrançois Tigeot gen6_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
210e3adcf8fSFrançois Tigeot {
21171f41f3eSFrançois Tigeot 	u32 scratch_addr =
2121e12ee3bSFrançois Tigeot 		i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
213a85cb24fSFrançois Tigeot 	u32 *cs, flags = 0;
214e3adcf8fSFrançois Tigeot 	int ret;
215e3adcf8fSFrançois Tigeot 
216e3adcf8fSFrançois Tigeot 	/* Force SNB workarounds for PIPE_CONTROL flushes */
217a05eeebfSFrançois Tigeot 	ret = intel_emit_post_sync_nonzero_flush(req);
218686a02f1SFrançois Tigeot 	if (ret)
219686a02f1SFrançois Tigeot 		return ret;
220e3adcf8fSFrançois Tigeot 
221e3adcf8fSFrançois Tigeot 	/* Just flush everything.  Experiments have shown that reducing the
222e3adcf8fSFrançois Tigeot 	 * number of bits based on the write domains has little performance
223e3adcf8fSFrançois Tigeot 	 * impact.
224e3adcf8fSFrançois Tigeot 	 */
22571f41f3eSFrançois Tigeot 	if (mode & EMIT_FLUSH) {
226e3adcf8fSFrançois Tigeot 		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
227b5c29a34SFrançois Tigeot 		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
228b5c29a34SFrançois Tigeot 		/*
229b5c29a34SFrançois Tigeot 		 * Ensure that any following seqno writes only happen
230b5c29a34SFrançois Tigeot 		 * when the render cache is indeed flushed.
231b5c29a34SFrançois Tigeot 		 */
232b5c29a34SFrançois Tigeot 		flags |= PIPE_CONTROL_CS_STALL;
233b5c29a34SFrançois Tigeot 	}
23471f41f3eSFrançois Tigeot 	if (mode & EMIT_INVALIDATE) {
235686a02f1SFrançois Tigeot 		flags |= PIPE_CONTROL_TLB_INVALIDATE;
236e3adcf8fSFrançois Tigeot 		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
237e3adcf8fSFrançois Tigeot 		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
238e3adcf8fSFrançois Tigeot 		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
239e3adcf8fSFrançois Tigeot 		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
240e3adcf8fSFrançois Tigeot 		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
241686a02f1SFrançois Tigeot 		/*
242b5c29a34SFrançois Tigeot 		 * TLB invalidate requires a post-sync write.
243686a02f1SFrançois Tigeot 		 */
244b5c29a34SFrançois Tigeot 		flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
245b5c29a34SFrançois Tigeot 	}
246e3adcf8fSFrançois Tigeot 
247a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 4);
248a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
249a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
250e3adcf8fSFrançois Tigeot 
251a85cb24fSFrançois Tigeot 	*cs++ = GFX_OP_PIPE_CONTROL(4);
252a85cb24fSFrançois Tigeot 	*cs++ = flags;
253a85cb24fSFrançois Tigeot 	*cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;
254a85cb24fSFrançois Tigeot 	*cs++ = 0;
255a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
256b5c29a34SFrançois Tigeot 
257b5c29a34SFrançois Tigeot 	return 0;
258b5c29a34SFrançois Tigeot }
259b5c29a34SFrançois Tigeot 
260b5c29a34SFrançois Tigeot static int
gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request * req)261a05eeebfSFrançois Tigeot gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request *req)
262b5c29a34SFrançois Tigeot {
263a85cb24fSFrançois Tigeot 	u32 *cs;
264b5c29a34SFrançois Tigeot 
265a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 4);
266a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
267a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
268b5c29a34SFrançois Tigeot 
269a85cb24fSFrançois Tigeot 	*cs++ = GFX_OP_PIPE_CONTROL(4);
270a85cb24fSFrançois Tigeot 	*cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;
271a85cb24fSFrançois Tigeot 	*cs++ = 0;
272a85cb24fSFrançois Tigeot 	*cs++ = 0;
273a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
274b5c29a34SFrançois Tigeot 
275b5c29a34SFrançois Tigeot 	return 0;
276b5c29a34SFrançois Tigeot }
277b5c29a34SFrançois Tigeot 
278b5c29a34SFrançois Tigeot static int
gen7_render_ring_flush(struct drm_i915_gem_request * req,u32 mode)27971f41f3eSFrançois Tigeot gen7_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
280b5c29a34SFrançois Tigeot {
28171f41f3eSFrançois Tigeot 	u32 scratch_addr =
2821e12ee3bSFrançois Tigeot 		i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
283a85cb24fSFrançois Tigeot 	u32 *cs, flags = 0;
284b5c29a34SFrançois Tigeot 
285b5c29a34SFrançois Tigeot 	/*
286b5c29a34SFrançois Tigeot 	 * Ensure that any following seqno writes only happen when the render
287b5c29a34SFrançois Tigeot 	 * cache is indeed flushed.
288b5c29a34SFrançois Tigeot 	 *
289b5c29a34SFrançois Tigeot 	 * Workaround: 4th PIPE_CONTROL command (except the ones with only
290b5c29a34SFrançois Tigeot 	 * read-cache invalidate bits set) must have the CS_STALL bit set. We
291b5c29a34SFrançois Tigeot 	 * don't try to be clever and just set it unconditionally.
292b5c29a34SFrançois Tigeot 	 */
293b5c29a34SFrançois Tigeot 	flags |= PIPE_CONTROL_CS_STALL;
294b5c29a34SFrançois Tigeot 
295b5c29a34SFrançois Tigeot 	/* Just flush everything.  Experiments have shown that reducing the
296b5c29a34SFrançois Tigeot 	 * number of bits based on the write domains has little performance
297b5c29a34SFrançois Tigeot 	 * impact.
298b5c29a34SFrançois Tigeot 	 */
29971f41f3eSFrançois Tigeot 	if (mode & EMIT_FLUSH) {
300b5c29a34SFrançois Tigeot 		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
301b5c29a34SFrançois Tigeot 		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
302aee94f86SFrançois Tigeot 		flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
303b49c8cf9SFrançois Tigeot 		flags |= PIPE_CONTROL_FLUSH_ENABLE;
304b5c29a34SFrançois Tigeot 	}
30571f41f3eSFrançois Tigeot 	if (mode & EMIT_INVALIDATE) {
306b5c29a34SFrançois Tigeot 		flags |= PIPE_CONTROL_TLB_INVALIDATE;
307b5c29a34SFrançois Tigeot 		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
308b5c29a34SFrançois Tigeot 		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
309b5c29a34SFrançois Tigeot 		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
310b5c29a34SFrançois Tigeot 		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
311b5c29a34SFrançois Tigeot 		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
3122c9916cdSFrançois Tigeot 		flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR;
313b5c29a34SFrançois Tigeot 		/*
314b5c29a34SFrançois Tigeot 		 * TLB invalidate requires a post-sync write.
315b5c29a34SFrançois Tigeot 		 */
316b5c29a34SFrançois Tigeot 		flags |= PIPE_CONTROL_QW_WRITE;
317a2fdbec6SFrançois Tigeot 		flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
318b5c29a34SFrançois Tigeot 
3190dbf0ea8SMatthew Dillon 		flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD;
3200dbf0ea8SMatthew Dillon 
321b5c29a34SFrançois Tigeot 		/* Workaround: we must issue a pipe_control with CS-stall bit
322b5c29a34SFrançois Tigeot 		 * set before a pipe_control command that has the state cache
323b5c29a34SFrançois Tigeot 		 * invalidate bit set. */
324a05eeebfSFrançois Tigeot 		gen7_render_ring_cs_stall_wa(req);
325b5c29a34SFrançois Tigeot 	}
326b5c29a34SFrançois Tigeot 
327a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 4);
328a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
329a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
330b5c29a34SFrançois Tigeot 
331a85cb24fSFrançois Tigeot 	*cs++ = GFX_OP_PIPE_CONTROL(4);
332a85cb24fSFrançois Tigeot 	*cs++ = flags;
333a85cb24fSFrançois Tigeot 	*cs++ = scratch_addr;
334a85cb24fSFrançois Tigeot 	*cs++ = 0;
335a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
33624edb884SFrançois Tigeot 
33724edb884SFrançois Tigeot 	return 0;
33824edb884SFrançois Tigeot }
33924edb884SFrançois Tigeot 
34024edb884SFrançois Tigeot static int
gen8_render_ring_flush(struct drm_i915_gem_request * req,u32 mode)34171f41f3eSFrançois Tigeot gen8_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
3429edbd4a0SFrançois Tigeot {
343a85cb24fSFrançois Tigeot 	u32 flags;
344a85cb24fSFrançois Tigeot 	u32 *cs;
3459edbd4a0SFrançois Tigeot 
346a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, mode & EMIT_INVALIDATE ? 12 : 6);
347a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
348a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
349a85cb24fSFrançois Tigeot 
350a85cb24fSFrançois Tigeot 	flags = PIPE_CONTROL_CS_STALL;
3519edbd4a0SFrançois Tigeot 
35271f41f3eSFrançois Tigeot 	if (mode & EMIT_FLUSH) {
3539edbd4a0SFrançois Tigeot 		flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
3549edbd4a0SFrançois Tigeot 		flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
355aee94f86SFrançois Tigeot 		flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
356b49c8cf9SFrançois Tigeot 		flags |= PIPE_CONTROL_FLUSH_ENABLE;
3579edbd4a0SFrançois Tigeot 	}
35871f41f3eSFrançois Tigeot 	if (mode & EMIT_INVALIDATE) {
3599edbd4a0SFrançois Tigeot 		flags |= PIPE_CONTROL_TLB_INVALIDATE;
3609edbd4a0SFrançois Tigeot 		flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
3619edbd4a0SFrançois Tigeot 		flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
3629edbd4a0SFrançois Tigeot 		flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
3639edbd4a0SFrançois Tigeot 		flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
3649edbd4a0SFrançois Tigeot 		flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
3659edbd4a0SFrançois Tigeot 		flags |= PIPE_CONTROL_QW_WRITE;
3669edbd4a0SFrançois Tigeot 		flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
3679edbd4a0SFrançois Tigeot 
36824edb884SFrançois Tigeot 		/* WaCsStallBeforeStateCacheInvalidate:bdw,chv */
369a85cb24fSFrançois Tigeot 		cs = gen8_emit_pipe_control(cs,
37024edb884SFrançois Tigeot 					    PIPE_CONTROL_CS_STALL |
37124edb884SFrançois Tigeot 					    PIPE_CONTROL_STALL_AT_SCOREBOARD,
37224edb884SFrançois Tigeot 					    0);
37324edb884SFrançois Tigeot 	}
3749edbd4a0SFrançois Tigeot 
375a85cb24fSFrançois Tigeot 	cs = gen8_emit_pipe_control(cs, flags,
376a85cb24fSFrançois Tigeot 				    i915_ggtt_offset(req->engine->scratch) +
377a85cb24fSFrançois Tigeot 				    2 * CACHELINE_BYTES);
378a85cb24fSFrançois Tigeot 
379a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
380a85cb24fSFrançois Tigeot 
381a85cb24fSFrançois Tigeot 	return 0;
3829edbd4a0SFrançois Tigeot }
3839edbd4a0SFrançois Tigeot 
ring_setup_phys_status_page(struct intel_engine_cs * engine)3848621f407SFrançois Tigeot static void ring_setup_phys_status_page(struct intel_engine_cs *engine)
3855d0b1887SFrançois Tigeot {
3861487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
3875d0b1887SFrançois Tigeot 	u32 addr;
3885d0b1887SFrançois Tigeot 
3895d0b1887SFrançois Tigeot 	addr = dev_priv->status_page_dmah->busaddr;
3901487f786SFrançois Tigeot 	if (INTEL_GEN(dev_priv) >= 4)
3915d0b1887SFrançois Tigeot 		addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
3925d0b1887SFrançois Tigeot 	I915_WRITE(HWS_PGA, addr);
3935d0b1887SFrançois Tigeot }
3945d0b1887SFrançois Tigeot 
intel_ring_setup_status_page(struct intel_engine_cs * engine)3958621f407SFrançois Tigeot static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
396477eb7f9SFrançois Tigeot {
3971487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
398aee94f86SFrançois Tigeot 	i915_reg_t mmio;
399477eb7f9SFrançois Tigeot 
400477eb7f9SFrançois Tigeot 	/* The ring status page addresses are no longer next to the rest of
401477eb7f9SFrançois Tigeot 	 * the ring registers as of gen7.
402477eb7f9SFrançois Tigeot 	 */
4031487f786SFrançois Tigeot 	if (IS_GEN7(dev_priv)) {
4048621f407SFrançois Tigeot 		switch (engine->id) {
405*3f2dd94aSFrançois Tigeot 		/*
406*3f2dd94aSFrançois Tigeot 		 * No more rings exist on Gen7. Default case is only to shut up
407*3f2dd94aSFrançois Tigeot 		 * gcc switch check warning.
408*3f2dd94aSFrançois Tigeot 		 */
409*3f2dd94aSFrançois Tigeot 		default:
410*3f2dd94aSFrançois Tigeot 			GEM_BUG_ON(engine->id);
411477eb7f9SFrançois Tigeot 		case RCS:
412477eb7f9SFrançois Tigeot 			mmio = RENDER_HWS_PGA_GEN7;
413477eb7f9SFrançois Tigeot 			break;
414477eb7f9SFrançois Tigeot 		case BCS:
415477eb7f9SFrançois Tigeot 			mmio = BLT_HWS_PGA_GEN7;
416477eb7f9SFrançois Tigeot 			break;
417477eb7f9SFrançois Tigeot 		case VCS:
418477eb7f9SFrançois Tigeot 			mmio = BSD_HWS_PGA_GEN7;
419477eb7f9SFrançois Tigeot 			break;
420477eb7f9SFrançois Tigeot 		case VECS:
421477eb7f9SFrançois Tigeot 			mmio = VEBOX_HWS_PGA_GEN7;
422477eb7f9SFrançois Tigeot 			break;
423477eb7f9SFrançois Tigeot 		}
4241487f786SFrançois Tigeot 	} else if (IS_GEN6(dev_priv)) {
4258621f407SFrançois Tigeot 		mmio = RING_HWS_PGA_GEN6(engine->mmio_base);
426477eb7f9SFrançois Tigeot 	} else {
427477eb7f9SFrançois Tigeot 		/* XXX: gen8 returns to sanity */
4288621f407SFrançois Tigeot 		mmio = RING_HWS_PGA(engine->mmio_base);
429477eb7f9SFrançois Tigeot 	}
430477eb7f9SFrançois Tigeot 
431*3f2dd94aSFrançois Tigeot 	if (INTEL_GEN(dev_priv) >= 6)
432*3f2dd94aSFrançois Tigeot 		I915_WRITE(RING_HWSTAM(engine->mmio_base), 0xffffffff);
433*3f2dd94aSFrançois Tigeot 
4341e12ee3bSFrançois Tigeot 	I915_WRITE(mmio, engine->status_page.ggtt_offset);
435477eb7f9SFrançois Tigeot 	POSTING_READ(mmio);
436477eb7f9SFrançois Tigeot 
437477eb7f9SFrançois Tigeot 	/*
438477eb7f9SFrançois Tigeot 	 * Flush the TLB for this page
439477eb7f9SFrançois Tigeot 	 *
440477eb7f9SFrançois Tigeot 	 * FIXME: These two bits have disappeared on gen8, so a question
441477eb7f9SFrançois Tigeot 	 * arises: do we still need this and if so how should we go about
442477eb7f9SFrançois Tigeot 	 * invalidating the TLB?
443477eb7f9SFrançois Tigeot 	 */
4441487f786SFrançois Tigeot 	if (IS_GEN(dev_priv, 6, 7)) {
4458621f407SFrançois Tigeot 		i915_reg_t reg = RING_INSTPM(engine->mmio_base);
446477eb7f9SFrançois Tigeot 
447477eb7f9SFrançois Tigeot 		/* ring should be idle before issuing a sync flush*/
4488621f407SFrançois Tigeot 		WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0);
449477eb7f9SFrançois Tigeot 
450477eb7f9SFrançois Tigeot 		I915_WRITE(reg,
451477eb7f9SFrançois Tigeot 			   _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
452477eb7f9SFrançois Tigeot 					      INSTPM_SYNC_FLUSH));
4531487f786SFrançois Tigeot 		if (intel_wait_for_register(dev_priv,
4541487f786SFrançois Tigeot 					    reg, INSTPM_SYNC_FLUSH, 0,
455477eb7f9SFrançois Tigeot 					    1000))
456477eb7f9SFrançois Tigeot 			DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
4578621f407SFrançois Tigeot 				  engine->name);
458477eb7f9SFrançois Tigeot 	}
459477eb7f9SFrançois Tigeot }
460477eb7f9SFrançois Tigeot 
stop_ring(struct intel_engine_cs * engine)4618621f407SFrançois Tigeot static bool stop_ring(struct intel_engine_cs *engine)
462e3adcf8fSFrançois Tigeot {
4631487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
464e3adcf8fSFrançois Tigeot 
4651e12ee3bSFrançois Tigeot 	if (INTEL_GEN(dev_priv) > 2) {
4668621f407SFrançois Tigeot 		I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING));
4671487f786SFrançois Tigeot 		if (intel_wait_for_register(dev_priv,
4681487f786SFrançois Tigeot 					    RING_MI_MODE(engine->mmio_base),
4691487f786SFrançois Tigeot 					    MODE_IDLE,
4701487f786SFrançois Tigeot 					    MODE_IDLE,
4711487f786SFrançois Tigeot 					    1000)) {
4728621f407SFrançois Tigeot 			DRM_ERROR("%s : timed out trying to stop ring\n",
4738621f407SFrançois Tigeot 				  engine->name);
4741b13d190SFrançois Tigeot 			/* Sometimes we observe that the idle flag is not
4751b13d190SFrançois Tigeot 			 * set even though the ring is empty. So double
4761b13d190SFrançois Tigeot 			 * check before giving up.
4771b13d190SFrançois Tigeot 			 */
4788621f407SFrançois Tigeot 			if (I915_READ_HEAD(engine) != I915_READ_TAIL(engine))
479ba55f2f5SFrançois Tigeot 				return false;
480ba55f2f5SFrançois Tigeot 		}
481ba55f2f5SFrançois Tigeot 	}
482686a02f1SFrançois Tigeot 
4838621f407SFrançois Tigeot 	I915_WRITE_CTL(engine, 0);
4848621f407SFrançois Tigeot 	I915_WRITE_HEAD(engine, 0);
48571f41f3eSFrançois Tigeot 	I915_WRITE_TAIL(engine, 0);
486e3adcf8fSFrançois Tigeot 
4878621f407SFrançois Tigeot 	return (I915_READ_HEAD(engine) & HEAD_ADDR) == 0;
488ba55f2f5SFrançois Tigeot }
489ba55f2f5SFrançois Tigeot 
init_ring_common(struct intel_engine_cs * engine)4908621f407SFrançois Tigeot static int init_ring_common(struct intel_engine_cs *engine)
4918621f407SFrançois Tigeot {
4921487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
49371f41f3eSFrançois Tigeot 	struct intel_ring *ring = engine->buffer;
494ba55f2f5SFrançois Tigeot 	int ret = 0;
495ba55f2f5SFrançois Tigeot 
4962c9916cdSFrançois Tigeot 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
497ba55f2f5SFrançois Tigeot 
4988621f407SFrançois Tigeot 	if (!stop_ring(engine)) {
499ba55f2f5SFrançois Tigeot 		/* G45 ring initialization often fails to reset head to zero */
500b5c29a34SFrançois Tigeot 		DRM_DEBUG_KMS("%s head not reset to zero "
501e3adcf8fSFrançois Tigeot 			      "ctl %08x head %08x tail %08x start %08x\n",
5028621f407SFrançois Tigeot 			      engine->name,
5038621f407SFrançois Tigeot 			      I915_READ_CTL(engine),
5048621f407SFrançois Tigeot 			      I915_READ_HEAD(engine),
5058621f407SFrançois Tigeot 			      I915_READ_TAIL(engine),
5068621f407SFrançois Tigeot 			      I915_READ_START(engine));
507e3adcf8fSFrançois Tigeot 
5088621f407SFrançois Tigeot 		if (!stop_ring(engine)) {
509e3adcf8fSFrançois Tigeot 			DRM_ERROR("failed to set %s head to zero "
510e3adcf8fSFrançois Tigeot 				  "ctl %08x head %08x tail %08x start %08x\n",
5118621f407SFrançois Tigeot 				  engine->name,
5128621f407SFrançois Tigeot 				  I915_READ_CTL(engine),
5138621f407SFrançois Tigeot 				  I915_READ_HEAD(engine),
5148621f407SFrançois Tigeot 				  I915_READ_TAIL(engine),
5158621f407SFrançois Tigeot 				  I915_READ_START(engine));
516686a02f1SFrançois Tigeot 			ret = -EIO;
517686a02f1SFrançois Tigeot 			goto out;
518e3adcf8fSFrançois Tigeot 		}
519ba55f2f5SFrançois Tigeot 	}
520ba55f2f5SFrançois Tigeot 
5211e12ee3bSFrançois Tigeot 	if (HWS_NEEDS_PHYSICAL(dev_priv))
5228621f407SFrançois Tigeot 		ring_setup_phys_status_page(engine);
5231e12ee3bSFrançois Tigeot 	else
5241e12ee3bSFrançois Tigeot 		intel_ring_setup_status_page(engine);
5251e12ee3bSFrançois Tigeot 
5261e12ee3bSFrançois Tigeot 	intel_engine_reset_breadcrumbs(engine);
527ba55f2f5SFrançois Tigeot 
5280f370975SMatthew Dillon 	/* Enforce ordering by reading HEAD register back */
5298621f407SFrançois Tigeot 	I915_READ_HEAD(engine);
5300f370975SMatthew Dillon 
531ba55f2f5SFrançois Tigeot 	/* Initialize the ring. This must happen _after_ we've cleared the ring
532ba55f2f5SFrançois Tigeot 	 * registers with the above sequence (the readback of the HEAD registers
533ba55f2f5SFrançois Tigeot 	 * also enforces ordering), otherwise the hw might lose the new ring
534ba55f2f5SFrançois Tigeot 	 * register values. */
5351e12ee3bSFrançois Tigeot 	I915_WRITE_START(engine, i915_ggtt_offset(ring->vma));
5361b13d190SFrançois Tigeot 
5371b13d190SFrançois Tigeot 	/* WaClearRingBufHeadRegAtInit:ctg,elk */
5388621f407SFrançois Tigeot 	if (I915_READ_HEAD(engine))
5391b13d190SFrançois Tigeot 		DRM_DEBUG("%s initialization failed [head=%08x], fudging\n",
5408621f407SFrançois Tigeot 			  engine->name, I915_READ_HEAD(engine));
5411b13d190SFrançois Tigeot 
5421e12ee3bSFrançois Tigeot 	intel_ring_update_space(ring);
5431e12ee3bSFrançois Tigeot 	I915_WRITE_HEAD(engine, ring->head);
5441e12ee3bSFrançois Tigeot 	I915_WRITE_TAIL(engine, ring->tail);
5451e12ee3bSFrançois Tigeot 	(void)I915_READ_TAIL(engine);
5461e12ee3bSFrançois Tigeot 
5471e12ee3bSFrançois Tigeot 	I915_WRITE_CTL(engine, RING_CTL_SIZE(ring->size) | RING_VALID);
548ba55f2f5SFrançois Tigeot 
549ba55f2f5SFrançois Tigeot 	/* If the head is still not zero, the ring is dead */
550*3f2dd94aSFrançois Tigeot 	if (intel_wait_for_register(dev_priv, RING_CTL(engine->mmio_base),
5511e12ee3bSFrançois Tigeot 				    RING_VALID, RING_VALID,
5521e12ee3bSFrançois Tigeot 				    50)) {
553ba55f2f5SFrançois Tigeot 		DRM_ERROR("%s initialization failed "
5541e12ee3bSFrançois Tigeot 			  "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n",
5558621f407SFrançois Tigeot 			  engine->name,
5568621f407SFrançois Tigeot 			  I915_READ_CTL(engine),
5578621f407SFrançois Tigeot 			  I915_READ_CTL(engine) & RING_VALID,
5581e12ee3bSFrançois Tigeot 			  I915_READ_HEAD(engine), ring->head,
5591e12ee3bSFrançois Tigeot 			  I915_READ_TAIL(engine), ring->tail,
5608621f407SFrançois Tigeot 			  I915_READ_START(engine),
5611e12ee3bSFrançois Tigeot 			  i915_ggtt_offset(ring->vma));
562ba55f2f5SFrançois Tigeot 		ret = -EIO;
563ba55f2f5SFrançois Tigeot 		goto out;
564ba55f2f5SFrançois Tigeot 	}
565e3adcf8fSFrançois Tigeot 
5668621f407SFrançois Tigeot 	intel_engine_init_hangcheck(engine);
5675d0b1887SFrançois Tigeot 
568*3f2dd94aSFrançois Tigeot 	if (INTEL_GEN(dev_priv) > 2)
569*3f2dd94aSFrançois Tigeot 		I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING));
570*3f2dd94aSFrançois Tigeot 
571686a02f1SFrançois Tigeot out:
5722c9916cdSFrançois Tigeot 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
573686a02f1SFrançois Tigeot 
574686a02f1SFrançois Tigeot 	return ret;
575e3adcf8fSFrançois Tigeot }
576e3adcf8fSFrançois Tigeot 
reset_ring_common(struct intel_engine_cs * engine,struct drm_i915_gem_request * request)5771e12ee3bSFrançois Tigeot static void reset_ring_common(struct intel_engine_cs *engine,
5781e12ee3bSFrançois Tigeot 			      struct drm_i915_gem_request *request)
5791b13d190SFrançois Tigeot {
580*3f2dd94aSFrançois Tigeot 	/*
581*3f2dd94aSFrançois Tigeot 	 * RC6 must be prevented until the reset is complete and the engine
582*3f2dd94aSFrançois Tigeot 	 * reinitialised. If it occurs in the middle of this sequence, the
583*3f2dd94aSFrançois Tigeot 	 * state written to/loaded from the power context is ill-defined (e.g.
584*3f2dd94aSFrançois Tigeot 	 * the PP_BASE_DIR may be lost).
585*3f2dd94aSFrançois Tigeot 	 */
586*3f2dd94aSFrançois Tigeot 	assert_forcewakes_active(engine->i915, FORCEWAKE_ALL);
587*3f2dd94aSFrançois Tigeot 
588*3f2dd94aSFrançois Tigeot 	/*
589*3f2dd94aSFrançois Tigeot 	 * Try to restore the logical GPU state to match the continuation
590a85cb24fSFrançois Tigeot 	 * of the request queue. If we skip the context/PD restore, then
591a85cb24fSFrançois Tigeot 	 * the next request may try to execute assuming that its context
592a85cb24fSFrançois Tigeot 	 * is valid and loaded on the GPU and so may try to access invalid
593a85cb24fSFrançois Tigeot 	 * memory, prompting repeated GPU hangs.
594a85cb24fSFrançois Tigeot 	 *
595a85cb24fSFrançois Tigeot 	 * If the request was guilty, we still restore the logical state
596a85cb24fSFrançois Tigeot 	 * in case the next request requires it (e.g. the aliasing ppgtt),
597a85cb24fSFrançois Tigeot 	 * but skip over the hung batch.
598a85cb24fSFrançois Tigeot 	 *
599a85cb24fSFrançois Tigeot 	 * If the request was innocent, we try to replay the request with
600a85cb24fSFrançois Tigeot 	 * the restored context.
601a85cb24fSFrançois Tigeot 	 */
602a85cb24fSFrançois Tigeot 	if (request) {
603a85cb24fSFrançois Tigeot 		struct drm_i915_private *dev_priv = request->i915;
604a85cb24fSFrançois Tigeot 		struct intel_context *ce = &request->ctx->engine[engine->id];
605a85cb24fSFrançois Tigeot 		struct i915_hw_ppgtt *ppgtt;
6061b13d190SFrançois Tigeot 
607a85cb24fSFrançois Tigeot 		/* FIXME consider gen8 reset */
608a85cb24fSFrançois Tigeot 
609a85cb24fSFrançois Tigeot 		if (ce->state) {
610a85cb24fSFrançois Tigeot 			I915_WRITE(CCID,
611a85cb24fSFrançois Tigeot 				   i915_ggtt_offset(ce->state) |
612a85cb24fSFrançois Tigeot 				   BIT(8) /* must be set! */ |
613a85cb24fSFrançois Tigeot 				   CCID_EXTENDED_STATE_SAVE |
614a85cb24fSFrançois Tigeot 				   CCID_EXTENDED_STATE_RESTORE |
615a85cb24fSFrançois Tigeot 				   CCID_EN);
616e3adcf8fSFrançois Tigeot 		}
617e3adcf8fSFrançois Tigeot 
618a85cb24fSFrançois Tigeot 		ppgtt = request->ctx->ppgtt ?: engine->i915->mm.aliasing_ppgtt;
619a85cb24fSFrançois Tigeot 		if (ppgtt) {
620a85cb24fSFrançois Tigeot 			u32 pd_offset = ppgtt->pd.base.ggtt_offset << 10;
6211b13d190SFrançois Tigeot 
622a85cb24fSFrançois Tigeot 			I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G);
623a85cb24fSFrançois Tigeot 			I915_WRITE(RING_PP_DIR_BASE(engine), pd_offset);
6241b13d190SFrançois Tigeot 
625a85cb24fSFrançois Tigeot 			/* Wait for the PD reload to complete */
626a85cb24fSFrançois Tigeot 			if (intel_wait_for_register(dev_priv,
627a85cb24fSFrançois Tigeot 						    RING_PP_DIR_BASE(engine),
628a85cb24fSFrançois Tigeot 						    BIT(0), 0,
629a85cb24fSFrançois Tigeot 						    10))
630a85cb24fSFrançois Tigeot 				DRM_ERROR("Wait for reload of ppgtt page-directory timed out\n");
6311b13d190SFrançois Tigeot 
632a85cb24fSFrançois Tigeot 			ppgtt->pd_dirty_rings &= ~intel_engine_flag(engine);
6332c9916cdSFrançois Tigeot 		}
6342c9916cdSFrançois Tigeot 
635a85cb24fSFrançois Tigeot 		/* If the rq hung, jump to its breadcrumb and skip the batch */
636a85cb24fSFrançois Tigeot 		if (request->fence.error == -EIO)
637a85cb24fSFrançois Tigeot 			request->ring->head = request->postfix;
638a85cb24fSFrançois Tigeot 	} else {
639a85cb24fSFrançois Tigeot 		engine->legacy_active_context = NULL;
640a85cb24fSFrançois Tigeot 	}
6412c9916cdSFrançois Tigeot }
6422c9916cdSFrançois Tigeot 
intel_rcs_ctx_init(struct drm_i915_gem_request * req)643a05eeebfSFrançois Tigeot static int intel_rcs_ctx_init(struct drm_i915_gem_request *req)
6442c9916cdSFrançois Tigeot {
6452c9916cdSFrançois Tigeot 	int ret;
6462c9916cdSFrançois Tigeot 
647a05eeebfSFrançois Tigeot 	ret = intel_ring_workarounds_emit(req);
6482c9916cdSFrançois Tigeot 	if (ret != 0)
6492c9916cdSFrançois Tigeot 		return ret;
6502c9916cdSFrançois Tigeot 
6514be47400SFrançois Tigeot 	ret = i915_gem_render_state_emit(req);
6522c9916cdSFrançois Tigeot 	if (ret)
6532c9916cdSFrançois Tigeot 		return ret;
654c0e85e96SFrançois Tigeot 
655c0e85e96SFrançois Tigeot 	return 0;
6562c9916cdSFrançois Tigeot }
6572c9916cdSFrançois Tigeot 
init_render_ring(struct intel_engine_cs * engine)6588621f407SFrançois Tigeot static int init_render_ring(struct intel_engine_cs *engine)
659e3adcf8fSFrançois Tigeot {
6601487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
6618621f407SFrançois Tigeot 	int ret = init_ring_common(engine);
66224edb884SFrançois Tigeot 	if (ret)
66324edb884SFrançois Tigeot 		return ret;
664e3adcf8fSFrançois Tigeot 
665ba55f2f5SFrançois Tigeot 	/* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
6661487f786SFrançois Tigeot 	if (IS_GEN(dev_priv, 4, 6))
667f4e1c372SFrançois Tigeot 		I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
668f4e1c372SFrançois Tigeot 
669f4e1c372SFrançois Tigeot 	/* We need to disable the AsyncFlip performance optimisations in order
670f4e1c372SFrançois Tigeot 	 * to use MI_WAIT_FOR_EVENT within the CS. It should already be
671f4e1c372SFrançois Tigeot 	 * programmed to '1' on all products.
6725d0b1887SFrançois Tigeot 	 *
673a05eeebfSFrançois Tigeot 	 * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
674f4e1c372SFrançois Tigeot 	 */
6751487f786SFrançois Tigeot 	if (IS_GEN(dev_priv, 6, 7))
676f4e1c372SFrançois Tigeot 		I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
677f4e1c372SFrançois Tigeot 
678f4e1c372SFrançois Tigeot 	/* Required for the hardware to program scanline values for waiting */
679ba55f2f5SFrançois Tigeot 	/* WaEnableFlushTlbInvalidationMode:snb */
6801487f786SFrançois Tigeot 	if (IS_GEN6(dev_priv))
681f4e1c372SFrançois Tigeot 		I915_WRITE(GFX_MODE,
682ba55f2f5SFrançois Tigeot 			   _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
683f4e1c372SFrançois Tigeot 
684ba55f2f5SFrançois Tigeot 	/* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
6851487f786SFrançois Tigeot 	if (IS_GEN7(dev_priv))
686e3adcf8fSFrançois Tigeot 		I915_WRITE(GFX_MODE_GEN7,
687ba55f2f5SFrançois Tigeot 			   _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
688f4e1c372SFrançois Tigeot 			   _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
689e3adcf8fSFrançois Tigeot 
6901487f786SFrançois Tigeot 	if (IS_GEN6(dev_priv)) {
691e3adcf8fSFrançois Tigeot 		/* From the Sandybridge PRM, volume 1 part 3, page 24:
692e3adcf8fSFrançois Tigeot 		 * "If this bit is set, STCunit will have LRA as replacement
693e3adcf8fSFrançois Tigeot 		 *  policy. [...] This bit must be reset.  LRA replacement
694e3adcf8fSFrançois Tigeot 		 *  policy is not supported."
695e3adcf8fSFrançois Tigeot 		 */
696e3adcf8fSFrançois Tigeot 		I915_WRITE(CACHE_MODE_0,
697f4e1c372SFrançois Tigeot 			   _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
698e3adcf8fSFrançois Tigeot 	}
699e3adcf8fSFrançois Tigeot 
7001487f786SFrançois Tigeot 	if (IS_GEN(dev_priv, 6, 7))
701f4e1c372SFrançois Tigeot 		I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
702f4e1c372SFrançois Tigeot 
703303bf270SFrançois Tigeot 	if (INTEL_INFO(dev_priv)->gen >= 6)
704303bf270SFrançois Tigeot 		I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
705e3adcf8fSFrançois Tigeot 
7068621f407SFrançois Tigeot 	return init_workarounds_ring(engine);
707e3adcf8fSFrançois Tigeot }
708e3adcf8fSFrançois Tigeot 
render_ring_cleanup(struct intel_engine_cs * engine)7098621f407SFrançois Tigeot static void render_ring_cleanup(struct intel_engine_cs *engine)
710e3adcf8fSFrançois Tigeot {
7111487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
71224edb884SFrançois Tigeot 
7131e12ee3bSFrançois Tigeot 	i915_vma_unpin_and_release(&dev_priv->semaphore);
714e3adcf8fSFrançois Tigeot }
715e3adcf8fSFrançois Tigeot 
gen8_rcs_signal(struct drm_i915_gem_request * req,u32 * cs)716a85cb24fSFrançois Tigeot static u32 *gen8_rcs_signal(struct drm_i915_gem_request *req, u32 *cs)
71724edb884SFrançois Tigeot {
71871f41f3eSFrançois Tigeot 	struct drm_i915_private *dev_priv = req->i915;
71924edb884SFrançois Tigeot 	struct intel_engine_cs *waiter;
7208621f407SFrançois Tigeot 	enum intel_engine_id id;
72124edb884SFrançois Tigeot 
7221e12ee3bSFrançois Tigeot 	for_each_engine(waiter, dev_priv, id) {
72371f41f3eSFrançois Tigeot 		u64 gtt_offset = req->engine->semaphore.signal_ggtt[id];
72424edb884SFrançois Tigeot 		if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
72524edb884SFrançois Tigeot 			continue;
72624edb884SFrançois Tigeot 
727a85cb24fSFrançois Tigeot 		*cs++ = GFX_OP_PIPE_CONTROL(6);
728a85cb24fSFrançois Tigeot 		*cs++ = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_QW_WRITE |
729a85cb24fSFrançois Tigeot 			PIPE_CONTROL_CS_STALL;
730a85cb24fSFrançois Tigeot 		*cs++ = lower_32_bits(gtt_offset);
731a85cb24fSFrançois Tigeot 		*cs++ = upper_32_bits(gtt_offset);
732a85cb24fSFrançois Tigeot 		*cs++ = req->global_seqno;
733a85cb24fSFrançois Tigeot 		*cs++ = 0;
734a85cb24fSFrançois Tigeot 		*cs++ = MI_SEMAPHORE_SIGNAL |
735a85cb24fSFrançois Tigeot 			MI_SEMAPHORE_TARGET(waiter->hw_id);
736a85cb24fSFrançois Tigeot 		*cs++ = 0;
73724edb884SFrançois Tigeot 	}
73824edb884SFrançois Tigeot 
739a85cb24fSFrançois Tigeot 	return cs;
7404be47400SFrançois Tigeot }
7414be47400SFrançois Tigeot 
gen8_xcs_signal(struct drm_i915_gem_request * req,u32 * cs)742a85cb24fSFrançois Tigeot static u32 *gen8_xcs_signal(struct drm_i915_gem_request *req, u32 *cs)
74324edb884SFrançois Tigeot {
74471f41f3eSFrançois Tigeot 	struct drm_i915_private *dev_priv = req->i915;
74524edb884SFrançois Tigeot 	struct intel_engine_cs *waiter;
7468621f407SFrançois Tigeot 	enum intel_engine_id id;
74724edb884SFrançois Tigeot 
7481e12ee3bSFrançois Tigeot 	for_each_engine(waiter, dev_priv, id) {
74971f41f3eSFrançois Tigeot 		u64 gtt_offset = req->engine->semaphore.signal_ggtt[id];
75024edb884SFrançois Tigeot 		if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
75124edb884SFrançois Tigeot 			continue;
75224edb884SFrançois Tigeot 
753a85cb24fSFrançois Tigeot 		*cs++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW;
754a85cb24fSFrançois Tigeot 		*cs++ = lower_32_bits(gtt_offset) | MI_FLUSH_DW_USE_GTT;
755a85cb24fSFrançois Tigeot 		*cs++ = upper_32_bits(gtt_offset);
756a85cb24fSFrançois Tigeot 		*cs++ = req->global_seqno;
757a85cb24fSFrançois Tigeot 		*cs++ = MI_SEMAPHORE_SIGNAL |
758a85cb24fSFrançois Tigeot 			MI_SEMAPHORE_TARGET(waiter->hw_id);
759a85cb24fSFrançois Tigeot 		*cs++ = 0;
76024edb884SFrançois Tigeot 	}
76124edb884SFrançois Tigeot 
762a85cb24fSFrançois Tigeot 	return cs;
7634be47400SFrançois Tigeot }
7644be47400SFrançois Tigeot 
gen6_signal(struct drm_i915_gem_request * req,u32 * cs)765a85cb24fSFrançois Tigeot static u32 *gen6_signal(struct drm_i915_gem_request *req, u32 *cs)
766e3adcf8fSFrançois Tigeot {
76771f41f3eSFrançois Tigeot 	struct drm_i915_private *dev_priv = req->i915;
7681e12ee3bSFrançois Tigeot 	struct intel_engine_cs *engine;
7698621f407SFrançois Tigeot 	enum intel_engine_id id;
7704be47400SFrançois Tigeot 	int num_rings = 0;
771ba55f2f5SFrançois Tigeot 
7721e12ee3bSFrançois Tigeot 	for_each_engine(engine, dev_priv, id) {
7731e12ee3bSFrançois Tigeot 		i915_reg_t mbox_reg;
774aee94f86SFrançois Tigeot 
7751e12ee3bSFrançois Tigeot 		if (!(BIT(engine->hw_id) & GEN6_SEMAPHORES_MASK))
7761e12ee3bSFrançois Tigeot 			continue;
7771e12ee3bSFrançois Tigeot 
7781e12ee3bSFrançois Tigeot 		mbox_reg = req->engine->semaphore.mbox.signal[engine->hw_id];
779aee94f86SFrançois Tigeot 		if (i915_mmio_reg_valid(mbox_reg)) {
780a85cb24fSFrançois Tigeot 			*cs++ = MI_LOAD_REGISTER_IMM(1);
781a85cb24fSFrançois Tigeot 			*cs++ = i915_mmio_reg_offset(mbox_reg);
782a85cb24fSFrançois Tigeot 			*cs++ = req->global_seqno;
7834be47400SFrançois Tigeot 			num_rings++;
784ba55f2f5SFrançois Tigeot 		}
785ba55f2f5SFrançois Tigeot 	}
7864be47400SFrançois Tigeot 	if (num_rings & 1)
787a85cb24fSFrançois Tigeot 		*cs++ = MI_NOOP;
788ba55f2f5SFrançois Tigeot 
789a85cb24fSFrançois Tigeot 	return cs;
79071f41f3eSFrançois Tigeot }
79171f41f3eSFrançois Tigeot 
cancel_requests(struct intel_engine_cs * engine)792*3f2dd94aSFrançois Tigeot static void cancel_requests(struct intel_engine_cs *engine)
793*3f2dd94aSFrançois Tigeot {
794*3f2dd94aSFrançois Tigeot 	struct drm_i915_gem_request *request;
795*3f2dd94aSFrançois Tigeot 	unsigned long flags;
796*3f2dd94aSFrançois Tigeot 
797*3f2dd94aSFrançois Tigeot 	spin_lock_irqsave(&engine->timeline->lock, flags);
798*3f2dd94aSFrançois Tigeot 
799*3f2dd94aSFrançois Tigeot 	/* Mark all submitted requests as skipped. */
800*3f2dd94aSFrançois Tigeot 	list_for_each_entry(request, &engine->timeline->requests, link) {
801*3f2dd94aSFrançois Tigeot 		GEM_BUG_ON(!request->global_seqno);
802*3f2dd94aSFrançois Tigeot 		if (!i915_gem_request_completed(request))
803*3f2dd94aSFrançois Tigeot 			dma_fence_set_error(&request->fence, -EIO);
804*3f2dd94aSFrançois Tigeot 	}
805*3f2dd94aSFrançois Tigeot 	/* Remaining _unready_ requests will be nop'ed when submitted */
806*3f2dd94aSFrançois Tigeot 
807*3f2dd94aSFrançois Tigeot 	spin_unlock_irqrestore(&engine->timeline->lock, flags);
808*3f2dd94aSFrançois Tigeot }
809*3f2dd94aSFrançois Tigeot 
i9xx_submit_request(struct drm_i915_gem_request * request)81071f41f3eSFrançois Tigeot static void i9xx_submit_request(struct drm_i915_gem_request *request)
81171f41f3eSFrançois Tigeot {
81271f41f3eSFrançois Tigeot 	struct drm_i915_private *dev_priv = request->i915;
81371f41f3eSFrançois Tigeot 
8144be47400SFrançois Tigeot 	i915_gem_request_submit(request);
8154be47400SFrançois Tigeot 
816a85cb24fSFrançois Tigeot 	I915_WRITE_TAIL(request->engine,
817a85cb24fSFrançois Tigeot 			intel_ring_set_tail(request->ring, request->tail));
81871f41f3eSFrançois Tigeot }
81971f41f3eSFrançois Tigeot 
i9xx_emit_breadcrumb(struct drm_i915_gem_request * req,u32 * cs)820a85cb24fSFrançois Tigeot static void i9xx_emit_breadcrumb(struct drm_i915_gem_request *req, u32 *cs)
82171f41f3eSFrançois Tigeot {
822a85cb24fSFrançois Tigeot 	*cs++ = MI_STORE_DWORD_INDEX;
823a85cb24fSFrançois Tigeot 	*cs++ = I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT;
824a85cb24fSFrançois Tigeot 	*cs++ = req->global_seqno;
825a85cb24fSFrançois Tigeot 	*cs++ = MI_USER_INTERRUPT;
82671f41f3eSFrançois Tigeot 
827a85cb24fSFrançois Tigeot 	req->tail = intel_ring_offset(req, cs);
828a85cb24fSFrançois Tigeot 	assert_ring_tail_valid(req->ring, req->tail);
829e3adcf8fSFrançois Tigeot }
830e3adcf8fSFrançois Tigeot 
8314be47400SFrançois Tigeot static const int i9xx_emit_breadcrumb_sz = 4;
8324be47400SFrançois Tigeot 
833e3adcf8fSFrançois Tigeot /**
8344be47400SFrançois Tigeot  * gen6_sema_emit_breadcrumb - Update the semaphore mailbox registers
835e3adcf8fSFrançois Tigeot  *
836a05eeebfSFrançois Tigeot  * @request - request to write to the ring
837e3adcf8fSFrançois Tigeot  *
838e3adcf8fSFrançois Tigeot  * Update the mailbox registers in the *other* rings with the current seqno.
839e3adcf8fSFrançois Tigeot  * This acts like a signal in the canonical semaphore.
840e3adcf8fSFrançois Tigeot  */
gen6_sema_emit_breadcrumb(struct drm_i915_gem_request * req,u32 * cs)841a85cb24fSFrançois Tigeot static void gen6_sema_emit_breadcrumb(struct drm_i915_gem_request *req, u32 *cs)
842e3adcf8fSFrançois Tigeot {
8434be47400SFrançois Tigeot 	return i9xx_emit_breadcrumb(req,
844a85cb24fSFrançois Tigeot 				    req->engine->semaphore.signal(req, cs));
845e3adcf8fSFrançois Tigeot }
846e3adcf8fSFrançois Tigeot 
gen8_render_emit_breadcrumb(struct drm_i915_gem_request * req,u32 * cs)8474be47400SFrançois Tigeot static void gen8_render_emit_breadcrumb(struct drm_i915_gem_request *req,
848a85cb24fSFrançois Tigeot 					u32 *cs)
8491487f786SFrançois Tigeot {
8501487f786SFrançois Tigeot 	struct intel_engine_cs *engine = req->engine;
8511487f786SFrançois Tigeot 
8524be47400SFrançois Tigeot 	if (engine->semaphore.signal)
853a85cb24fSFrançois Tigeot 		cs = engine->semaphore.signal(req, cs);
85471f41f3eSFrançois Tigeot 
855a85cb24fSFrançois Tigeot 	*cs++ = GFX_OP_PIPE_CONTROL(6);
856a85cb24fSFrançois Tigeot 	*cs++ = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_CS_STALL |
857a85cb24fSFrançois Tigeot 		PIPE_CONTROL_QW_WRITE;
858a85cb24fSFrançois Tigeot 	*cs++ = intel_hws_seqno_address(engine);
859a85cb24fSFrançois Tigeot 	*cs++ = 0;
860a85cb24fSFrançois Tigeot 	*cs++ = req->global_seqno;
8611487f786SFrançois Tigeot 	/* We're thrashing one dword of HWS. */
862a85cb24fSFrançois Tigeot 	*cs++ = 0;
863a85cb24fSFrançois Tigeot 	*cs++ = MI_USER_INTERRUPT;
864a85cb24fSFrançois Tigeot 	*cs++ = MI_NOOP;
86571f41f3eSFrançois Tigeot 
866a85cb24fSFrançois Tigeot 	req->tail = intel_ring_offset(req, cs);
867a85cb24fSFrançois Tigeot 	assert_ring_tail_valid(req->ring, req->tail);
8681487f786SFrançois Tigeot }
8691487f786SFrançois Tigeot 
8704be47400SFrançois Tigeot static const int gen8_render_emit_breadcrumb_sz = 8;
8714be47400SFrançois Tigeot 
872e3adcf8fSFrançois Tigeot /**
873e3adcf8fSFrançois Tigeot  * intel_ring_sync - sync the waiter to the signaller on seqno
874e3adcf8fSFrançois Tigeot  *
875e3adcf8fSFrançois Tigeot  * @waiter - ring that is waiting
876e3adcf8fSFrançois Tigeot  * @signaller - ring which has, or will signal
877e3adcf8fSFrançois Tigeot  * @seqno - seqno which the waiter will block on
878e3adcf8fSFrançois Tigeot  */
87924edb884SFrançois Tigeot 
88024edb884SFrançois Tigeot static int
gen8_ring_sync_to(struct drm_i915_gem_request * req,struct drm_i915_gem_request * signal)88171f41f3eSFrançois Tigeot gen8_ring_sync_to(struct drm_i915_gem_request *req,
88271f41f3eSFrançois Tigeot 		  struct drm_i915_gem_request *signal)
88324edb884SFrançois Tigeot {
88471f41f3eSFrançois Tigeot 	struct drm_i915_private *dev_priv = req->i915;
88571f41f3eSFrançois Tigeot 	u64 offset = GEN8_WAIT_OFFSET(req->engine, signal->engine->id);
8861487f786SFrançois Tigeot 	struct i915_hw_ppgtt *ppgtt;
887a85cb24fSFrançois Tigeot 	u32 *cs;
88824edb884SFrançois Tigeot 
889a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 4);
890a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
891a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
89224edb884SFrançois Tigeot 
893a85cb24fSFrançois Tigeot 	*cs++ = MI_SEMAPHORE_WAIT | MI_SEMAPHORE_GLOBAL_GTT |
894a85cb24fSFrançois Tigeot 		MI_SEMAPHORE_SAD_GTE_SDD;
895a85cb24fSFrançois Tigeot 	*cs++ = signal->global_seqno;
896a85cb24fSFrançois Tigeot 	*cs++ = lower_32_bits(offset);
897a85cb24fSFrançois Tigeot 	*cs++ = upper_32_bits(offset);
898a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
8991487f786SFrançois Tigeot 
9001487f786SFrançois Tigeot 	/* When the !RCS engines idle waiting upon a semaphore, they lose their
9011487f786SFrançois Tigeot 	 * pagetables and we must reload them before executing the batch.
9021487f786SFrançois Tigeot 	 * We do this on the i915_switch_context() following the wait and
9031487f786SFrançois Tigeot 	 * before the dispatch.
9041487f786SFrançois Tigeot 	 */
90571f41f3eSFrançois Tigeot 	ppgtt = req->ctx->ppgtt;
90671f41f3eSFrançois Tigeot 	if (ppgtt && req->engine->id != RCS)
90771f41f3eSFrançois Tigeot 		ppgtt->pd_dirty_rings |= intel_engine_flag(req->engine);
90824edb884SFrançois Tigeot 	return 0;
90924edb884SFrançois Tigeot }
91024edb884SFrançois Tigeot 
911e3adcf8fSFrançois Tigeot static int
gen6_ring_sync_to(struct drm_i915_gem_request * req,struct drm_i915_gem_request * signal)91271f41f3eSFrançois Tigeot gen6_ring_sync_to(struct drm_i915_gem_request *req,
91371f41f3eSFrançois Tigeot 		  struct drm_i915_gem_request *signal)
914e3adcf8fSFrançois Tigeot {
915e3adcf8fSFrançois Tigeot 	u32 dw1 = MI_SEMAPHORE_MBOX |
916e3adcf8fSFrançois Tigeot 		  MI_SEMAPHORE_COMPARE |
917e3adcf8fSFrançois Tigeot 		  MI_SEMAPHORE_REGISTER;
9181e12ee3bSFrançois Tigeot 	u32 wait_mbox = signal->engine->semaphore.mbox.wait[req->engine->hw_id];
919a85cb24fSFrançois Tigeot 	u32 *cs;
920e3adcf8fSFrançois Tigeot 
92171f41f3eSFrançois Tigeot 	WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
92271f41f3eSFrançois Tigeot 
923a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 4);
924a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
925a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
92671f41f3eSFrançois Tigeot 
927a85cb24fSFrançois Tigeot 	*cs++ = dw1 | wait_mbox;
928686a02f1SFrançois Tigeot 	/* Throughout all of the GEM code, seqno passed implies our current
929686a02f1SFrançois Tigeot 	 * seqno is >= the last seqno executed. However for hardware the
930686a02f1SFrançois Tigeot 	 * comparison is strictly greater than.
931686a02f1SFrançois Tigeot 	 */
932a85cb24fSFrançois Tigeot 	*cs++ = signal->global_seqno - 1;
933a85cb24fSFrançois Tigeot 	*cs++ = 0;
934a85cb24fSFrançois Tigeot 	*cs++ = MI_NOOP;
935a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
936e3adcf8fSFrançois Tigeot 
937e3adcf8fSFrançois Tigeot 	return 0;
938e3adcf8fSFrançois Tigeot }
939e3adcf8fSFrançois Tigeot 
940303bf270SFrançois Tigeot static void
gen5_seqno_barrier(struct intel_engine_cs * engine)94187df8fc6SFrançois Tigeot gen5_seqno_barrier(struct intel_engine_cs *engine)
942e3adcf8fSFrançois Tigeot {
943303bf270SFrançois Tigeot 	/* MI_STORE are internally buffered by the GPU and not flushed
944303bf270SFrançois Tigeot 	 * either by MI_FLUSH or SyncFlush or any other combination of
945303bf270SFrançois Tigeot 	 * MI commands.
946e3adcf8fSFrançois Tigeot 	 *
947303bf270SFrançois Tigeot 	 * "Only the submission of the store operation is guaranteed.
948303bf270SFrançois Tigeot 	 * The write result will be complete (coherent) some time later
949303bf270SFrançois Tigeot 	 * (this is practically a finite period but there is no guaranteed
950303bf270SFrançois Tigeot 	 * latency)."
951303bf270SFrançois Tigeot 	 *
952303bf270SFrançois Tigeot 	 * Empirically, we observe that we need a delay of at least 75us to
953303bf270SFrançois Tigeot 	 * be sure that the seqno write is visible by the CPU.
954e3adcf8fSFrançois Tigeot 	 */
955303bf270SFrançois Tigeot 	usleep_range(125, 250);
956e3adcf8fSFrançois Tigeot }
957e3adcf8fSFrançois Tigeot 
9588621f407SFrançois Tigeot static void
gen6_seqno_barrier(struct intel_engine_cs * engine)9598621f407SFrançois Tigeot gen6_seqno_barrier(struct intel_engine_cs *engine)
960e3adcf8fSFrançois Tigeot {
9611487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
9628621f407SFrançois Tigeot 
963e3adcf8fSFrançois Tigeot 	/* Workaround to force correct ordering between irq and seqno writes on
964e3adcf8fSFrançois Tigeot 	 * ivb (and maybe also on snb) by reading from a CS register (like
9658621f407SFrançois Tigeot 	 * ACTHD) before reading the status page.
9668621f407SFrançois Tigeot 	 *
9678621f407SFrançois Tigeot 	 * Note that this effectively stalls the read by the time it takes to
9688621f407SFrançois Tigeot 	 * do a memory transaction, which more or less ensures that the write
9698621f407SFrançois Tigeot 	 * from the GPU has sufficient time to invalidate the CPU cacheline.
9708621f407SFrançois Tigeot 	 * Alternatively we could delay the interrupt from the CS ring to give
9718621f407SFrançois Tigeot 	 * the write time to land, but that would incur a delay after every
9728621f407SFrançois Tigeot 	 * batch i.e. much more frequent than a delay when waiting for the
9738621f407SFrançois Tigeot 	 * interrupt (with the same net latency).
9748621f407SFrançois Tigeot 	 *
9758621f407SFrançois Tigeot 	 * Also note that to prevent whole machine hangs on gen7, we have to
9768621f407SFrançois Tigeot 	 * take the spinlock to guard against concurrent cacheline access.
9778621f407SFrançois Tigeot 	 */
9788621f407SFrançois Tigeot 	spin_lock_irq(&dev_priv->uncore.lock);
9798621f407SFrançois Tigeot 	POSTING_READ_FW(RING_ACTHD(engine->mmio_base));
9808621f407SFrançois Tigeot 	spin_unlock_irq(&dev_priv->uncore.lock);
981e3adcf8fSFrançois Tigeot }
982e3adcf8fSFrançois Tigeot 
983303bf270SFrançois Tigeot static void
gen5_irq_enable(struct intel_engine_cs * engine)984303bf270SFrançois Tigeot gen5_irq_enable(struct intel_engine_cs *engine)
985e3adcf8fSFrançois Tigeot {
986303bf270SFrançois Tigeot 	gen5_enable_gt_irq(engine->i915, engine->irq_enable_mask);
987e3adcf8fSFrançois Tigeot }
988e3adcf8fSFrançois Tigeot 
989a2fdbec6SFrançois Tigeot static void
gen5_irq_disable(struct intel_engine_cs * engine)990303bf270SFrançois Tigeot gen5_irq_disable(struct intel_engine_cs *engine)
991a2fdbec6SFrançois Tigeot {
992303bf270SFrançois Tigeot 	gen5_disable_gt_irq(engine->i915, engine->irq_enable_mask);
993e3adcf8fSFrançois Tigeot }
994e3adcf8fSFrançois Tigeot 
995a2fdbec6SFrançois Tigeot static void
i9xx_irq_enable(struct intel_engine_cs * engine)996303bf270SFrançois Tigeot i9xx_irq_enable(struct intel_engine_cs *engine)
997e3adcf8fSFrançois Tigeot {
9981487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
999e3adcf8fSFrançois Tigeot 
10008621f407SFrançois Tigeot 	dev_priv->irq_mask &= ~engine->irq_enable_mask;
1001686a02f1SFrançois Tigeot 	I915_WRITE(IMR, dev_priv->irq_mask);
1002303bf270SFrançois Tigeot 	POSTING_READ_FW(RING_IMR(engine->mmio_base));
1003686a02f1SFrançois Tigeot }
1004686a02f1SFrançois Tigeot 
1005686a02f1SFrançois Tigeot static void
i9xx_irq_disable(struct intel_engine_cs * engine)1006303bf270SFrançois Tigeot i9xx_irq_disable(struct intel_engine_cs *engine)
1007686a02f1SFrançois Tigeot {
10081487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
1009686a02f1SFrançois Tigeot 
10108621f407SFrançois Tigeot 	dev_priv->irq_mask |= engine->irq_enable_mask;
1011686a02f1SFrançois Tigeot 	I915_WRITE(IMR, dev_priv->irq_mask);
1012686a02f1SFrançois Tigeot }
1013686a02f1SFrançois Tigeot 
1014686a02f1SFrançois Tigeot static void
i8xx_irq_enable(struct intel_engine_cs * engine)1015303bf270SFrançois Tigeot i8xx_irq_enable(struct intel_engine_cs *engine)
1016686a02f1SFrançois Tigeot {
10171487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
1018686a02f1SFrançois Tigeot 
1019303bf270SFrançois Tigeot 	dev_priv->irq_mask &= ~engine->irq_enable_mask;
1020303bf270SFrançois Tigeot 	I915_WRITE16(IMR, dev_priv->irq_mask);
1021303bf270SFrançois Tigeot 	POSTING_READ16(RING_IMR(engine->mmio_base));
1022303bf270SFrançois Tigeot }
1023303bf270SFrançois Tigeot 
1024303bf270SFrançois Tigeot static void
i8xx_irq_disable(struct intel_engine_cs * engine)1025303bf270SFrançois Tigeot i8xx_irq_disable(struct intel_engine_cs *engine)
1026303bf270SFrançois Tigeot {
1027303bf270SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
1028303bf270SFrançois Tigeot 
10298621f407SFrançois Tigeot 	dev_priv->irq_mask |= engine->irq_enable_mask;
1030686a02f1SFrançois Tigeot 	I915_WRITE16(IMR, dev_priv->irq_mask);
1031e3adcf8fSFrançois Tigeot }
1032e3adcf8fSFrançois Tigeot 
1033e3adcf8fSFrançois Tigeot static int
bsd_ring_flush(struct drm_i915_gem_request * req,u32 mode)103471f41f3eSFrançois Tigeot bsd_ring_flush(struct drm_i915_gem_request *req, u32 mode)
1035e3adcf8fSFrançois Tigeot {
1036a85cb24fSFrançois Tigeot 	u32 *cs;
1037e3adcf8fSFrançois Tigeot 
1038a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 2);
1039a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
1040a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
1041e3adcf8fSFrançois Tigeot 
1042a85cb24fSFrançois Tigeot 	*cs++ = MI_FLUSH;
1043a85cb24fSFrançois Tigeot 	*cs++ = MI_NOOP;
1044a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
1045e3adcf8fSFrançois Tigeot 	return 0;
1046e3adcf8fSFrançois Tigeot }
1047e3adcf8fSFrançois Tigeot 
1048303bf270SFrançois Tigeot static void
gen6_irq_enable(struct intel_engine_cs * engine)1049303bf270SFrançois Tigeot gen6_irq_enable(struct intel_engine_cs *engine)
1050e3adcf8fSFrançois Tigeot {
10511487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
1052e3adcf8fSFrançois Tigeot 
10538621f407SFrançois Tigeot 	I915_WRITE_IMR(engine,
10548621f407SFrançois Tigeot 		       ~(engine->irq_enable_mask |
1055303bf270SFrançois Tigeot 			 engine->irq_keep_mask));
10568621f407SFrançois Tigeot 	gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
1057e3adcf8fSFrançois Tigeot }
1058e3adcf8fSFrançois Tigeot 
1059e3adcf8fSFrançois Tigeot static void
gen6_irq_disable(struct intel_engine_cs * engine)1060303bf270SFrançois Tigeot gen6_irq_disable(struct intel_engine_cs *engine)
1061e3adcf8fSFrançois Tigeot {
10621487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
1063e3adcf8fSFrançois Tigeot 
1064303bf270SFrançois Tigeot 	I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
10658621f407SFrançois Tigeot 	gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
1066e3adcf8fSFrançois Tigeot }
1067e3adcf8fSFrançois Tigeot 
1068303bf270SFrançois Tigeot static void
hsw_vebox_irq_enable(struct intel_engine_cs * engine)1069303bf270SFrançois Tigeot hsw_vebox_irq_enable(struct intel_engine_cs *engine)
10705d0b1887SFrançois Tigeot {
10711487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
10725d0b1887SFrançois Tigeot 
10738621f407SFrançois Tigeot 	I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
10744be47400SFrançois Tigeot 	gen6_unmask_pm_irq(dev_priv, engine->irq_enable_mask);
10755d0b1887SFrançois Tigeot }
10765d0b1887SFrançois Tigeot 
10775d0b1887SFrançois Tigeot static void
hsw_vebox_irq_disable(struct intel_engine_cs * engine)1078303bf270SFrançois Tigeot hsw_vebox_irq_disable(struct intel_engine_cs *engine)
10795d0b1887SFrançois Tigeot {
10801487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
10815d0b1887SFrançois Tigeot 
10828621f407SFrançois Tigeot 	I915_WRITE_IMR(engine, ~0);
10834be47400SFrançois Tigeot 	gen6_mask_pm_irq(dev_priv, engine->irq_enable_mask);
10845d0b1887SFrançois Tigeot }
10859edbd4a0SFrançois Tigeot 
1086303bf270SFrançois Tigeot static void
gen8_irq_enable(struct intel_engine_cs * engine)1087303bf270SFrançois Tigeot gen8_irq_enable(struct intel_engine_cs *engine)
10889edbd4a0SFrançois Tigeot {
10891487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
10909edbd4a0SFrançois Tigeot 
10918621f407SFrançois Tigeot 	I915_WRITE_IMR(engine,
10928621f407SFrançois Tigeot 		       ~(engine->irq_enable_mask |
1093303bf270SFrançois Tigeot 			 engine->irq_keep_mask));
1094303bf270SFrançois Tigeot 	POSTING_READ_FW(RING_IMR(engine->mmio_base));
10959edbd4a0SFrançois Tigeot }
10969edbd4a0SFrançois Tigeot 
10979edbd4a0SFrançois Tigeot static void
gen8_irq_disable(struct intel_engine_cs * engine)1098303bf270SFrançois Tigeot gen8_irq_disable(struct intel_engine_cs *engine)
10999edbd4a0SFrançois Tigeot {
11001487f786SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
11019edbd4a0SFrançois Tigeot 
1102303bf270SFrançois Tigeot 	I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
11035d0b1887SFrançois Tigeot }
11045d0b1887SFrançois Tigeot 
1105e3adcf8fSFrançois Tigeot static int
i965_emit_bb_start(struct drm_i915_gem_request * req,u64 offset,u32 length,unsigned int dispatch_flags)110671f41f3eSFrançois Tigeot i965_emit_bb_start(struct drm_i915_gem_request *req,
1107ba55f2f5SFrançois Tigeot 		   u64 offset, u32 length,
110871f41f3eSFrançois Tigeot 		   unsigned int dispatch_flags)
1109e3adcf8fSFrançois Tigeot {
1110a85cb24fSFrançois Tigeot 	u32 *cs;
1111e3adcf8fSFrançois Tigeot 
1112a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 2);
1113a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
1114a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
1115e3adcf8fSFrançois Tigeot 
1116a85cb24fSFrançois Tigeot 	*cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT | (dispatch_flags &
1117a85cb24fSFrançois Tigeot 		I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965);
1118a85cb24fSFrançois Tigeot 	*cs++ = offset;
1119a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
1120e3adcf8fSFrançois Tigeot 
1121e3adcf8fSFrançois Tigeot 	return 0;
1122e3adcf8fSFrançois Tigeot }
1123e3adcf8fSFrançois Tigeot 
1124b5c29a34SFrançois Tigeot /* Just userspace ABI convention to limit the wa batch bo to a resonable size */
1125b5c29a34SFrançois Tigeot #define I830_BATCH_LIMIT (256*1024)
112624edb884SFrançois Tigeot #define I830_TLB_ENTRIES (2)
112724edb884SFrançois Tigeot #define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
1128e3adcf8fSFrançois Tigeot static int
i830_emit_bb_start(struct drm_i915_gem_request * req,u64 offset,u32 len,unsigned int dispatch_flags)112971f41f3eSFrançois Tigeot i830_emit_bb_start(struct drm_i915_gem_request *req,
1130ba55f2f5SFrançois Tigeot 		   u64 offset, u32 len,
113171f41f3eSFrançois Tigeot 		   unsigned int dispatch_flags)
1132e3adcf8fSFrançois Tigeot {
1133a85cb24fSFrançois Tigeot 	u32 *cs, cs_offset = i915_ggtt_offset(req->engine->scratch);
1134e3adcf8fSFrançois Tigeot 
1135a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 6);
1136a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
1137a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
113824edb884SFrançois Tigeot 
113924edb884SFrançois Tigeot 	/* Evict the invalid PTE TLBs */
1140a85cb24fSFrançois Tigeot 	*cs++ = COLOR_BLT_CMD | BLT_WRITE_RGBA;
1141a85cb24fSFrançois Tigeot 	*cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096;
1142a85cb24fSFrançois Tigeot 	*cs++ = I830_TLB_ENTRIES << 16 | 4; /* load each page */
1143a85cb24fSFrançois Tigeot 	*cs++ = cs_offset;
1144a85cb24fSFrançois Tigeot 	*cs++ = 0xdeadbeef;
1145a85cb24fSFrançois Tigeot 	*cs++ = MI_NOOP;
1146a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
114724edb884SFrançois Tigeot 
1148477eb7f9SFrançois Tigeot 	if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) {
114924edb884SFrançois Tigeot 		if (len > I830_BATCH_LIMIT)
115024edb884SFrançois Tigeot 			return -ENOSPC;
115124edb884SFrançois Tigeot 
1152a85cb24fSFrançois Tigeot 		cs = intel_ring_begin(req, 6 + 2);
1153a85cb24fSFrançois Tigeot 		if (IS_ERR(cs))
1154a85cb24fSFrançois Tigeot 			return PTR_ERR(cs);
115524edb884SFrançois Tigeot 
115624edb884SFrançois Tigeot 		/* Blit the batch (which has now all relocs applied) to the
115724edb884SFrançois Tigeot 		 * stable batch scratch bo area (so that the CS never
115824edb884SFrançois Tigeot 		 * stumbles over its tlb invalidation bug) ...
115924edb884SFrançois Tigeot 		 */
1160a85cb24fSFrançois Tigeot 		*cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA;
1161a85cb24fSFrançois Tigeot 		*cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096;
1162a85cb24fSFrançois Tigeot 		*cs++ = DIV_ROUND_UP(len, 4096) << 16 | 4096;
1163a85cb24fSFrançois Tigeot 		*cs++ = cs_offset;
1164a85cb24fSFrançois Tigeot 		*cs++ = 4096;
1165a85cb24fSFrançois Tigeot 		*cs++ = offset;
116624edb884SFrançois Tigeot 
1167a85cb24fSFrançois Tigeot 		*cs++ = MI_FLUSH;
1168a85cb24fSFrançois Tigeot 		*cs++ = MI_NOOP;
1169a85cb24fSFrançois Tigeot 		intel_ring_advance(req, cs);
117024edb884SFrançois Tigeot 
117124edb884SFrançois Tigeot 		/* ... and execute it. */
117224edb884SFrançois Tigeot 		offset = cs_offset;
117324edb884SFrançois Tigeot 	}
117424edb884SFrançois Tigeot 
1175a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 2);
1176a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
1177a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
1178e3adcf8fSFrançois Tigeot 
1179a85cb24fSFrançois Tigeot 	*cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
1180a85cb24fSFrançois Tigeot 	*cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 :
1181a85cb24fSFrançois Tigeot 		MI_BATCH_NON_SECURE);
1182a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
1183686a02f1SFrançois Tigeot 
1184686a02f1SFrançois Tigeot 	return 0;
1185686a02f1SFrançois Tigeot }
1186686a02f1SFrançois Tigeot 
1187686a02f1SFrançois Tigeot static int
i915_emit_bb_start(struct drm_i915_gem_request * req,u64 offset,u32 len,unsigned int dispatch_flags)118871f41f3eSFrançois Tigeot i915_emit_bb_start(struct drm_i915_gem_request *req,
1189ba55f2f5SFrançois Tigeot 		   u64 offset, u32 len,
119071f41f3eSFrançois Tigeot 		   unsigned int dispatch_flags)
1191686a02f1SFrançois Tigeot {
1192a85cb24fSFrançois Tigeot 	u32 *cs;
1193686a02f1SFrançois Tigeot 
1194a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 2);
1195a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
1196a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
1197e3adcf8fSFrançois Tigeot 
1198a85cb24fSFrançois Tigeot 	*cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
1199a85cb24fSFrançois Tigeot 	*cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 :
1200a85cb24fSFrançois Tigeot 		MI_BATCH_NON_SECURE);
1201a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
1202e3adcf8fSFrançois Tigeot 
1203e3adcf8fSFrançois Tigeot 	return 0;
1204e3adcf8fSFrançois Tigeot }
1205e3adcf8fSFrançois Tigeot 
1206*3f2dd94aSFrançois Tigeot 
1207*3f2dd94aSFrançois Tigeot 
intel_ring_pin(struct intel_ring * ring,struct drm_i915_private * i915,unsigned int offset_bias)1208*3f2dd94aSFrançois Tigeot int intel_ring_pin(struct intel_ring *ring,
1209*3f2dd94aSFrançois Tigeot 		   struct drm_i915_private *i915,
1210*3f2dd94aSFrançois Tigeot 		   unsigned int offset_bias)
1211c0e85e96SFrançois Tigeot {
1212*3f2dd94aSFrançois Tigeot 	enum i915_map_type map = HAS_LLC(i915) ? I915_MAP_WB : I915_MAP_WC;
12131e12ee3bSFrançois Tigeot 	struct i915_vma *vma = ring->vma;
1214*3f2dd94aSFrançois Tigeot 	unsigned int flags;
12158621f407SFrançois Tigeot 	void *addr;
12162c9916cdSFrançois Tigeot 	int ret;
12172c9916cdSFrançois Tigeot 
12181e12ee3bSFrançois Tigeot 	GEM_BUG_ON(ring->vaddr);
12191e12ee3bSFrançois Tigeot 
12201e12ee3bSFrançois Tigeot 
1221a85cb24fSFrançois Tigeot 	flags = PIN_GLOBAL;
1222a85cb24fSFrançois Tigeot 	if (offset_bias)
1223a85cb24fSFrançois Tigeot 		flags |= PIN_OFFSET_BIAS | offset_bias;
12241e12ee3bSFrançois Tigeot 	if (vma->obj->stolen)
12251e12ee3bSFrançois Tigeot 		flags |= PIN_MAPPABLE;
12261e12ee3bSFrançois Tigeot 
12271e12ee3bSFrançois Tigeot 	if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
12281e12ee3bSFrançois Tigeot 		if (flags & PIN_MAPPABLE || map == I915_MAP_WC)
12291e12ee3bSFrançois Tigeot 			ret = i915_gem_object_set_to_gtt_domain(vma->obj, true);
12301e12ee3bSFrançois Tigeot 		else
12311e12ee3bSFrançois Tigeot 			ret = i915_gem_object_set_to_cpu_domain(vma->obj, true);
12321e12ee3bSFrançois Tigeot 		if (unlikely(ret))
12331e12ee3bSFrançois Tigeot 			return ret;
12341e12ee3bSFrançois Tigeot 	}
12351e12ee3bSFrançois Tigeot 
12361e12ee3bSFrançois Tigeot 	ret = i915_vma_pin(vma, 0, PAGE_SIZE, flags);
12371e12ee3bSFrançois Tigeot 	if (unlikely(ret))
1238aee94f86SFrançois Tigeot 		return ret;
1239aee94f86SFrançois Tigeot 
12401e12ee3bSFrançois Tigeot 	if (i915_vma_is_map_and_fenceable(vma))
12411e12ee3bSFrançois Tigeot 		addr = (void __force *)i915_vma_pin_iomap(vma);
12421e12ee3bSFrançois Tigeot 	else
12431e12ee3bSFrançois Tigeot 		addr = i915_gem_object_pin_map(vma->obj, map);
12441e12ee3bSFrançois Tigeot 	if (IS_ERR(addr))
12451e12ee3bSFrançois Tigeot 		goto err;
12462c9916cdSFrançois Tigeot 
1247*3f2dd94aSFrançois Tigeot 	vma->obj->pin_global++;
1248*3f2dd94aSFrançois Tigeot 
124971f41f3eSFrançois Tigeot 	ring->vaddr = addr;
12502c9916cdSFrançois Tigeot 	return 0;
12518621f407SFrançois Tigeot 
12521e12ee3bSFrançois Tigeot err:
12531e12ee3bSFrançois Tigeot 	i915_vma_unpin(vma);
12541e12ee3bSFrançois Tigeot 	return PTR_ERR(addr);
12552c9916cdSFrançois Tigeot }
12562c9916cdSFrançois Tigeot 
intel_ring_reset(struct intel_ring * ring,u32 tail)1257a85cb24fSFrançois Tigeot void intel_ring_reset(struct intel_ring *ring, u32 tail)
1258a85cb24fSFrançois Tigeot {
1259a85cb24fSFrançois Tigeot 	GEM_BUG_ON(!list_empty(&ring->request_list));
1260a85cb24fSFrançois Tigeot 	ring->tail = tail;
1261a85cb24fSFrançois Tigeot 	ring->head = tail;
1262a85cb24fSFrançois Tigeot 	ring->emit = tail;
1263a85cb24fSFrançois Tigeot 	intel_ring_update_space(ring);
1264a85cb24fSFrançois Tigeot }
1265a85cb24fSFrançois Tigeot 
intel_ring_unpin(struct intel_ring * ring)126671f41f3eSFrançois Tigeot void intel_ring_unpin(struct intel_ring *ring)
1267e3adcf8fSFrançois Tigeot {
126871f41f3eSFrançois Tigeot 	GEM_BUG_ON(!ring->vma);
126971f41f3eSFrançois Tigeot 	GEM_BUG_ON(!ring->vaddr);
127071f41f3eSFrançois Tigeot 
1271a85cb24fSFrançois Tigeot 	/* Discard any unused bytes beyond that submitted to hw. */
1272a85cb24fSFrançois Tigeot 	intel_ring_reset(ring, ring->tail);
1273a85cb24fSFrançois Tigeot 
12741e12ee3bSFrançois Tigeot 	if (i915_vma_is_map_and_fenceable(ring->vma))
127571f41f3eSFrançois Tigeot 		i915_vma_unpin_iomap(ring->vma);
12761e12ee3bSFrançois Tigeot 	else
12771e12ee3bSFrançois Tigeot 		i915_gem_object_unpin_map(ring->vma->obj);
127871f41f3eSFrançois Tigeot 	ring->vaddr = NULL;
127971f41f3eSFrançois Tigeot 
1280*3f2dd94aSFrançois Tigeot 	ring->vma->obj->pin_global--;
12811e12ee3bSFrançois Tigeot 	i915_vma_unpin(ring->vma);
128271f41f3eSFrançois Tigeot }
128371f41f3eSFrançois Tigeot 
12841e12ee3bSFrançois Tigeot static struct i915_vma *
intel_ring_create_vma(struct drm_i915_private * dev_priv,int size)12851e12ee3bSFrançois Tigeot intel_ring_create_vma(struct drm_i915_private *dev_priv, int size)
128624edb884SFrançois Tigeot {
1287e3adcf8fSFrançois Tigeot 	struct drm_i915_gem_object *obj;
12881e12ee3bSFrançois Tigeot 	struct i915_vma *vma;
1289e3adcf8fSFrançois Tigeot 
1290a85cb24fSFrançois Tigeot 	obj = i915_gem_object_create_stolen(dev_priv, size);
12911e12ee3bSFrançois Tigeot 	if (!obj)
1292*3f2dd94aSFrançois Tigeot 		obj = i915_gem_object_create_internal(dev_priv, size);
12931487f786SFrançois Tigeot 	if (IS_ERR(obj))
12941e12ee3bSFrançois Tigeot 		return ERR_CAST(obj);
1295e3adcf8fSFrançois Tigeot 
129624edb884SFrançois Tigeot 	/* mark ring buffers as read-only from GPU side by default */
129724edb884SFrançois Tigeot 	obj->gt_ro = 1;
129824edb884SFrançois Tigeot 
1299a85cb24fSFrançois Tigeot 	vma = i915_vma_instance(obj, &dev_priv->ggtt.base, NULL);
13001e12ee3bSFrançois Tigeot 	if (IS_ERR(vma))
13011e12ee3bSFrançois Tigeot 		goto err;
1302ba55f2f5SFrançois Tigeot 
13031e12ee3bSFrançois Tigeot 	return vma;
13041e12ee3bSFrançois Tigeot 
13051e12ee3bSFrançois Tigeot err:
13061e12ee3bSFrançois Tigeot 	i915_gem_object_put(obj);
13071e12ee3bSFrançois Tigeot 	return vma;
1308ba55f2f5SFrançois Tigeot }
1309ba55f2f5SFrançois Tigeot 
131071f41f3eSFrançois Tigeot struct intel_ring *
intel_engine_create_ring(struct intel_engine_cs * engine,int size)131171f41f3eSFrançois Tigeot intel_engine_create_ring(struct intel_engine_cs *engine, int size)
1312352ff8bdSFrançois Tigeot {
131371f41f3eSFrançois Tigeot 	struct intel_ring *ring;
13141e12ee3bSFrançois Tigeot 	struct i915_vma *vma;
1315352ff8bdSFrançois Tigeot 
131671f41f3eSFrançois Tigeot 	GEM_BUG_ON(!is_power_of_2(size));
13171e12ee3bSFrançois Tigeot 	GEM_BUG_ON(RING_CTL_SIZE(size) & ~RING_NR_PAGES);
131871f41f3eSFrançois Tigeot 
1319352ff8bdSFrançois Tigeot 	ring = kzalloc(sizeof(*ring), GFP_KERNEL);
13201e12ee3bSFrançois Tigeot 	if (!ring)
1321352ff8bdSFrançois Tigeot 		return ERR_PTR(-ENOMEM);
1322352ff8bdSFrançois Tigeot 
132371f41f3eSFrançois Tigeot 	INIT_LIST_HEAD(&ring->request_list);
132471f41f3eSFrançois Tigeot 
1325352ff8bdSFrançois Tigeot 	ring->size = size;
1326352ff8bdSFrançois Tigeot 	/* Workaround an erratum on the i830 which causes a hang if
1327352ff8bdSFrançois Tigeot 	 * the TAIL pointer points to within the last 2 cachelines
1328352ff8bdSFrançois Tigeot 	 * of the buffer.
1329352ff8bdSFrançois Tigeot 	 */
1330352ff8bdSFrançois Tigeot 	ring->effective_size = size;
1331a85cb24fSFrançois Tigeot 	if (IS_I830(engine->i915) || IS_I845G(engine->i915))
1332352ff8bdSFrançois Tigeot 		ring->effective_size -= 2 * CACHELINE_BYTES;
1333352ff8bdSFrançois Tigeot 
1334352ff8bdSFrançois Tigeot 	intel_ring_update_space(ring);
1335352ff8bdSFrançois Tigeot 
13361e12ee3bSFrançois Tigeot 	vma = intel_ring_create_vma(engine->i915, size);
13371e12ee3bSFrançois Tigeot 	if (IS_ERR(vma)) {
1338352ff8bdSFrançois Tigeot 		kfree(ring);
13391e12ee3bSFrançois Tigeot 		return ERR_CAST(vma);
1340352ff8bdSFrançois Tigeot 	}
13411e12ee3bSFrançois Tigeot 	ring->vma = vma;
1342352ff8bdSFrançois Tigeot 
1343352ff8bdSFrançois Tigeot 	return ring;
1344352ff8bdSFrançois Tigeot }
1345352ff8bdSFrançois Tigeot 
1346352ff8bdSFrançois Tigeot void
intel_ring_free(struct intel_ring * ring)134771f41f3eSFrançois Tigeot intel_ring_free(struct intel_ring *ring)
1348352ff8bdSFrançois Tigeot {
13494be47400SFrançois Tigeot 	struct drm_i915_gem_object *obj = ring->vma->obj;
13504be47400SFrançois Tigeot 
13514be47400SFrançois Tigeot 	i915_vma_close(ring->vma);
13524be47400SFrançois Tigeot 	__i915_gem_object_release_unless_active(obj);
13534be47400SFrançois Tigeot 
1354352ff8bdSFrançois Tigeot 	kfree(ring);
1355352ff8bdSFrançois Tigeot }
1356352ff8bdSFrançois Tigeot 
context_pin(struct i915_gem_context * ctx)1357a85cb24fSFrançois Tigeot static int context_pin(struct i915_gem_context *ctx)
1358a85cb24fSFrançois Tigeot {
1359a85cb24fSFrançois Tigeot 	struct i915_vma *vma = ctx->engine[RCS].state;
1360a85cb24fSFrançois Tigeot 	int ret;
1361a85cb24fSFrançois Tigeot 
1362a85cb24fSFrançois Tigeot 	/* Clear this page out of any CPU caches for coherent swap-in/out.
1363a85cb24fSFrançois Tigeot 	 * We only want to do this on the first bind so that we do not stall
1364a85cb24fSFrançois Tigeot 	 * on an active context (which by nature is already on the GPU).
1365a85cb24fSFrançois Tigeot 	 */
1366a85cb24fSFrançois Tigeot 	if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
1367a85cb24fSFrançois Tigeot 		ret = i915_gem_object_set_to_gtt_domain(vma->obj, false);
1368a85cb24fSFrançois Tigeot 		if (ret)
1369a85cb24fSFrançois Tigeot 			return ret;
1370a85cb24fSFrançois Tigeot 	}
1371a85cb24fSFrançois Tigeot 
1372a85cb24fSFrançois Tigeot 	return i915_vma_pin(vma, 0, I915_GTT_MIN_ALIGNMENT,
1373a85cb24fSFrançois Tigeot 			    PIN_GLOBAL | PIN_HIGH);
1374a85cb24fSFrançois Tigeot }
1375a85cb24fSFrançois Tigeot 
1376*3f2dd94aSFrançois Tigeot static struct i915_vma *
alloc_context_vma(struct intel_engine_cs * engine)1377*3f2dd94aSFrançois Tigeot alloc_context_vma(struct intel_engine_cs *engine)
1378*3f2dd94aSFrançois Tigeot {
1379*3f2dd94aSFrançois Tigeot 	struct drm_i915_private *i915 = engine->i915;
1380*3f2dd94aSFrançois Tigeot 	struct drm_i915_gem_object *obj;
1381*3f2dd94aSFrançois Tigeot 	struct i915_vma *vma;
1382*3f2dd94aSFrançois Tigeot 
1383*3f2dd94aSFrançois Tigeot 	obj = i915_gem_object_create(i915, engine->context_size);
1384*3f2dd94aSFrançois Tigeot 	if (IS_ERR(obj))
1385*3f2dd94aSFrançois Tigeot 		return ERR_CAST(obj);
1386*3f2dd94aSFrançois Tigeot 
1387*3f2dd94aSFrançois Tigeot 	/*
1388*3f2dd94aSFrançois Tigeot 	 * Try to make the context utilize L3 as well as LLC.
1389*3f2dd94aSFrançois Tigeot 	 *
1390*3f2dd94aSFrançois Tigeot 	 * On VLV we don't have L3 controls in the PTEs so we
1391*3f2dd94aSFrançois Tigeot 	 * shouldn't touch the cache level, especially as that
1392*3f2dd94aSFrançois Tigeot 	 * would make the object snooped which might have a
1393*3f2dd94aSFrançois Tigeot 	 * negative performance impact.
1394*3f2dd94aSFrançois Tigeot 	 *
1395*3f2dd94aSFrançois Tigeot 	 * Snooping is required on non-llc platforms in execlist
1396*3f2dd94aSFrançois Tigeot 	 * mode, but since all GGTT accesses use PAT entry 0 we
1397*3f2dd94aSFrançois Tigeot 	 * get snooping anyway regardless of cache_level.
1398*3f2dd94aSFrançois Tigeot 	 *
1399*3f2dd94aSFrançois Tigeot 	 * This is only applicable for Ivy Bridge devices since
1400*3f2dd94aSFrançois Tigeot 	 * later platforms don't have L3 control bits in the PTE.
1401*3f2dd94aSFrançois Tigeot 	 */
1402*3f2dd94aSFrançois Tigeot 	if (IS_IVYBRIDGE(i915)) {
1403*3f2dd94aSFrançois Tigeot 		/* Ignore any error, regard it as a simple optimisation */
1404*3f2dd94aSFrançois Tigeot 		i915_gem_object_set_cache_level(obj, I915_CACHE_L3_LLC);
1405*3f2dd94aSFrançois Tigeot 	}
1406*3f2dd94aSFrançois Tigeot 
1407*3f2dd94aSFrançois Tigeot 	vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
1408*3f2dd94aSFrançois Tigeot 	if (IS_ERR(vma))
1409*3f2dd94aSFrançois Tigeot 		i915_gem_object_put(obj);
1410*3f2dd94aSFrançois Tigeot 
1411*3f2dd94aSFrançois Tigeot 	return vma;
1412*3f2dd94aSFrançois Tigeot }
1413*3f2dd94aSFrançois Tigeot 
1414*3f2dd94aSFrançois Tigeot static struct intel_ring *
intel_ring_context_pin(struct intel_engine_cs * engine,struct i915_gem_context * ctx)1415*3f2dd94aSFrançois Tigeot intel_ring_context_pin(struct intel_engine_cs *engine,
1416a85cb24fSFrançois Tigeot 		       struct i915_gem_context *ctx)
14171487f786SFrançois Tigeot {
14181487f786SFrançois Tigeot 	struct intel_context *ce = &ctx->engine[engine->id];
14191487f786SFrançois Tigeot 	int ret;
14201487f786SFrançois Tigeot 
1421303bf270SFrançois Tigeot 	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
14221487f786SFrançois Tigeot 
1423*3f2dd94aSFrançois Tigeot 	if (likely(ce->pin_count++))
1424*3f2dd94aSFrançois Tigeot 		goto out;
1425a85cb24fSFrançois Tigeot 	GEM_BUG_ON(!ce->pin_count); /* no overflow please! */
14261487f786SFrançois Tigeot 
1427*3f2dd94aSFrançois Tigeot 	if (!ce->state && engine->context_size) {
1428*3f2dd94aSFrançois Tigeot 		struct i915_vma *vma;
1429*3f2dd94aSFrançois Tigeot 
1430*3f2dd94aSFrançois Tigeot 		vma = alloc_context_vma(engine);
1431*3f2dd94aSFrançois Tigeot 		if (IS_ERR(vma)) {
1432*3f2dd94aSFrançois Tigeot 			ret = PTR_ERR(vma);
1433*3f2dd94aSFrançois Tigeot 			goto err;
1434*3f2dd94aSFrançois Tigeot 		}
1435*3f2dd94aSFrançois Tigeot 
1436*3f2dd94aSFrançois Tigeot 		ce->state = vma;
1437*3f2dd94aSFrançois Tigeot 	}
1438*3f2dd94aSFrançois Tigeot 
14391487f786SFrançois Tigeot 	if (ce->state) {
1440a85cb24fSFrançois Tigeot 		ret = context_pin(ctx);
1441a85cb24fSFrançois Tigeot 		if (ret)
1442*3f2dd94aSFrançois Tigeot 			goto err;
1443a85cb24fSFrançois Tigeot 
1444a85cb24fSFrançois Tigeot 		ce->state->obj->mm.dirty = true;
1445*3f2dd94aSFrançois Tigeot 		ce->state->obj->pin_global++;
14464be47400SFrançois Tigeot 	}
14471487f786SFrançois Tigeot 
14481487f786SFrançois Tigeot 	/* The kernel context is only used as a placeholder for flushing the
14491487f786SFrançois Tigeot 	 * active context. It is never used for submitting user rendering and
14501487f786SFrançois Tigeot 	 * as such never requires the golden render context, and so we can skip
14511487f786SFrançois Tigeot 	 * emitting it when we switch to the kernel context. This is required
14521487f786SFrançois Tigeot 	 * as during eviction we cannot allocate and pin the renderstate in
14531487f786SFrançois Tigeot 	 * order to initialise the context.
14541487f786SFrançois Tigeot 	 */
1455a85cb24fSFrançois Tigeot 	if (i915_gem_context_is_kernel(ctx))
14561487f786SFrançois Tigeot 		ce->initialised = true;
14571487f786SFrançois Tigeot 
145887df8fc6SFrançois Tigeot 	i915_gem_context_get(ctx);
14591487f786SFrançois Tigeot 
1460*3f2dd94aSFrançois Tigeot out:
1461*3f2dd94aSFrançois Tigeot 	/* One ringbuffer to rule them all */
1462*3f2dd94aSFrançois Tigeot 	return engine->buffer;
1463*3f2dd94aSFrançois Tigeot 
1464*3f2dd94aSFrançois Tigeot err:
14651487f786SFrançois Tigeot 	ce->pin_count = 0;
1466*3f2dd94aSFrançois Tigeot 	return ERR_PTR(ret);
14671487f786SFrançois Tigeot }
14681487f786SFrançois Tigeot 
intel_ring_context_unpin(struct intel_engine_cs * engine,struct i915_gem_context * ctx)1469a85cb24fSFrançois Tigeot static void intel_ring_context_unpin(struct intel_engine_cs *engine,
1470a85cb24fSFrançois Tigeot 				     struct i915_gem_context *ctx)
14711487f786SFrançois Tigeot {
14721487f786SFrançois Tigeot 	struct intel_context *ce = &ctx->engine[engine->id];
14731487f786SFrançois Tigeot 
1474303bf270SFrançois Tigeot 	lockdep_assert_held(&ctx->i915->drm.struct_mutex);
1475a85cb24fSFrançois Tigeot 	GEM_BUG_ON(ce->pin_count == 0);
14761487f786SFrançois Tigeot 
14771487f786SFrançois Tigeot 	if (--ce->pin_count)
14781487f786SFrançois Tigeot 		return;
14791487f786SFrançois Tigeot 
1480*3f2dd94aSFrançois Tigeot 	if (ce->state) {
1481*3f2dd94aSFrançois Tigeot 		ce->state->obj->pin_global--;
14821e12ee3bSFrançois Tigeot 		i915_vma_unpin(ce->state);
1483*3f2dd94aSFrançois Tigeot 	}
14841487f786SFrançois Tigeot 
148587df8fc6SFrançois Tigeot 	i915_gem_context_put(ctx);
14861487f786SFrançois Tigeot }
14871487f786SFrançois Tigeot 
intel_init_ring_buffer(struct intel_engine_cs * engine)148887df8fc6SFrançois Tigeot static int intel_init_ring_buffer(struct intel_engine_cs *engine)
1489ba55f2f5SFrançois Tigeot {
149071f41f3eSFrançois Tigeot 	struct intel_ring *ring;
1491*3f2dd94aSFrançois Tigeot 	int err;
14922c9916cdSFrançois Tigeot 
149387df8fc6SFrançois Tigeot 	intel_engine_setup_common(engine);
149487df8fc6SFrançois Tigeot 
1495*3f2dd94aSFrançois Tigeot 	err = intel_engine_init_common(engine);
1496*3f2dd94aSFrançois Tigeot 	if (err)
1497*3f2dd94aSFrançois Tigeot 		goto err;
1498ba55f2f5SFrançois Tigeot 
149971f41f3eSFrançois Tigeot 	ring = intel_engine_create_ring(engine, 32 * PAGE_SIZE);
150071f41f3eSFrançois Tigeot 	if (IS_ERR(ring)) {
1501*3f2dd94aSFrançois Tigeot 		err = PTR_ERR(ring);
1502*3f2dd94aSFrançois Tigeot 		goto err;
1503ba55f2f5SFrançois Tigeot 	}
1504ba55f2f5SFrançois Tigeot 
1505a85cb24fSFrançois Tigeot 	/* Ring wraparound at offset 0 sometimes hangs. No idea why. */
1506*3f2dd94aSFrançois Tigeot 	err = intel_ring_pin(ring, engine->i915, I915_GTT_PAGE_SIZE);
1507*3f2dd94aSFrançois Tigeot 	if (err)
1508*3f2dd94aSFrançois Tigeot 		goto err_ring;
1509*3f2dd94aSFrançois Tigeot 
1510*3f2dd94aSFrançois Tigeot 	GEM_BUG_ON(engine->buffer);
15111e12ee3bSFrançois Tigeot 	engine->buffer = ring;
1512e3adcf8fSFrançois Tigeot 
1513e3adcf8fSFrançois Tigeot 	return 0;
1514e3adcf8fSFrançois Tigeot 
1515*3f2dd94aSFrançois Tigeot err_ring:
1516*3f2dd94aSFrançois Tigeot 	intel_ring_free(ring);
1517*3f2dd94aSFrançois Tigeot err:
1518*3f2dd94aSFrançois Tigeot 	intel_engine_cleanup_common(engine);
1519*3f2dd94aSFrançois Tigeot 	return err;
1520e3adcf8fSFrançois Tigeot }
1521e3adcf8fSFrançois Tigeot 
intel_engine_cleanup(struct intel_engine_cs * engine)152271f41f3eSFrançois Tigeot void intel_engine_cleanup(struct intel_engine_cs *engine)
1523e3adcf8fSFrançois Tigeot {
1524*3f2dd94aSFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
1525e3adcf8fSFrançois Tigeot 
15261e12ee3bSFrançois Tigeot 	WARN_ON(INTEL_GEN(dev_priv) > 2 &&
15271e12ee3bSFrançois Tigeot 		(I915_READ_MODE(engine) & MODE_IDLE) == 0);
1528b030f26bSFrançois Tigeot 
152971f41f3eSFrançois Tigeot 	intel_ring_unpin(engine->buffer);
153071f41f3eSFrançois Tigeot 	intel_ring_free(engine->buffer);
1531e3adcf8fSFrançois Tigeot 
15328621f407SFrançois Tigeot 	if (engine->cleanup)
15338621f407SFrançois Tigeot 		engine->cleanup(engine);
1534e3adcf8fSFrançois Tigeot 
153571f41f3eSFrançois Tigeot 	intel_engine_cleanup_common(engine);
15361487f786SFrançois Tigeot 
15371e12ee3bSFrançois Tigeot 	dev_priv->engine[engine->id] = NULL;
15381e12ee3bSFrançois Tigeot 	kfree(engine);
15391e12ee3bSFrançois Tigeot }
15401e12ee3bSFrançois Tigeot 
intel_legacy_submission_resume(struct drm_i915_private * dev_priv)15411e12ee3bSFrançois Tigeot void intel_legacy_submission_resume(struct drm_i915_private *dev_priv)
15421e12ee3bSFrançois Tigeot {
15431e12ee3bSFrançois Tigeot 	struct intel_engine_cs *engine;
15441e12ee3bSFrançois Tigeot 	enum intel_engine_id id;
15451e12ee3bSFrançois Tigeot 
1546a85cb24fSFrançois Tigeot 	/* Restart from the beginning of the rings for convenience */
1547a85cb24fSFrançois Tigeot 	for_each_engine(engine, dev_priv, id)
1548a85cb24fSFrançois Tigeot 		intel_ring_reset(engine->buffer, 0);
1549e3adcf8fSFrançois Tigeot }
1550e3adcf8fSFrançois Tigeot 
ring_request_alloc(struct drm_i915_gem_request * request)1551a85cb24fSFrançois Tigeot static int ring_request_alloc(struct drm_i915_gem_request *request)
1552b5c29a34SFrançois Tigeot {
1553a85cb24fSFrançois Tigeot 	u32 *cs;
1554a85cb24fSFrançois Tigeot 
1555a85cb24fSFrançois Tigeot 	GEM_BUG_ON(!request->ctx->engine[request->engine->id].pin_count);
15569edbd4a0SFrançois Tigeot 
15571487f786SFrançois Tigeot 	/* Flush enough space to reduce the likelihood of waiting after
15581487f786SFrançois Tigeot 	 * we start building the request - in which case we will just
15591487f786SFrançois Tigeot 	 * have to repeat work.
1560a05eeebfSFrançois Tigeot 	 */
15611487f786SFrançois Tigeot 	request->reserved_space += LEGACY_REQUEST_SIZE;
1562a2fdbec6SFrançois Tigeot 
1563a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(request, 0);
1564a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
1565a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
1566a05eeebfSFrançois Tigeot 
15671487f786SFrançois Tigeot 	request->reserved_space -= LEGACY_REQUEST_SIZE;
15681487f786SFrançois Tigeot 	return 0;
1569a05eeebfSFrançois Tigeot }
1570a05eeebfSFrançois Tigeot 
wait_for_space(struct drm_i915_gem_request * req,unsigned int bytes)1571*3f2dd94aSFrançois Tigeot static noinline int wait_for_space(struct drm_i915_gem_request *req,
1572*3f2dd94aSFrançois Tigeot 				   unsigned int bytes)
1573a05eeebfSFrançois Tigeot {
157471f41f3eSFrançois Tigeot 	struct intel_ring *ring = req->ring;
15758621f407SFrançois Tigeot 	struct drm_i915_gem_request *target;
15764be47400SFrançois Tigeot 	long timeout;
15774be47400SFrançois Tigeot 
15784be47400SFrançois Tigeot 	lockdep_assert_held(&req->i915->drm.struct_mutex);
15798621f407SFrançois Tigeot 
1580*3f2dd94aSFrançois Tigeot 	if (intel_ring_update_space(ring) >= bytes)
15818621f407SFrançois Tigeot 		return 0;
15828621f407SFrançois Tigeot 
15838621f407SFrançois Tigeot 	/*
15848621f407SFrançois Tigeot 	 * Space is reserved in the ringbuffer for finalising the request,
15858621f407SFrançois Tigeot 	 * as that cannot be allowed to fail. During request finalisation,
15868621f407SFrançois Tigeot 	 * reserved_space is set to 0 to stop the overallocation and the
15878621f407SFrançois Tigeot 	 * assumption is that then we never need to wait (which has the
15888621f407SFrançois Tigeot 	 * risk of failing with EINTR).
15898621f407SFrançois Tigeot 	 *
15908621f407SFrançois Tigeot 	 * See also i915_gem_request_alloc() and i915_add_request().
15918621f407SFrançois Tigeot 	 */
15921487f786SFrançois Tigeot 	GEM_BUG_ON(!req->reserved_space);
15938621f407SFrançois Tigeot 
159471f41f3eSFrançois Tigeot 	list_for_each_entry(target, &ring->request_list, ring_link) {
15958621f407SFrançois Tigeot 		/* Would completion of this request free enough space? */
1596*3f2dd94aSFrançois Tigeot 		if (bytes <= __intel_ring_space(target->postfix,
1597*3f2dd94aSFrançois Tigeot 						ring->emit, ring->size))
15988621f407SFrançois Tigeot 			break;
15998621f407SFrançois Tigeot 	}
16008621f407SFrançois Tigeot 
160171f41f3eSFrançois Tigeot 	if (WARN_ON(&target->ring_link == &ring->request_list))
16028621f407SFrançois Tigeot 		return -ENOSPC;
16038621f407SFrançois Tigeot 
16044be47400SFrançois Tigeot 	timeout = i915_wait_request(target,
16051e12ee3bSFrançois Tigeot 				    I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
16064be47400SFrançois Tigeot 				    MAX_SCHEDULE_TIMEOUT);
16074be47400SFrançois Tigeot 	if (timeout < 0)
16084be47400SFrançois Tigeot 		return timeout;
160971f41f3eSFrançois Tigeot 
161071f41f3eSFrançois Tigeot 	i915_gem_request_retire_upto(target);
161171f41f3eSFrançois Tigeot 
161271f41f3eSFrançois Tigeot 	intel_ring_update_space(ring);
161371f41f3eSFrançois Tigeot 	GEM_BUG_ON(ring->space < bytes);
161471f41f3eSFrançois Tigeot 	return 0;
16158621f407SFrançois Tigeot }
16168621f407SFrançois Tigeot 
intel_ring_begin(struct drm_i915_gem_request * req,unsigned int num_dwords)1617*3f2dd94aSFrançois Tigeot u32 *intel_ring_begin(struct drm_i915_gem_request *req,
1618*3f2dd94aSFrançois Tigeot 		      unsigned int num_dwords)
16198621f407SFrançois Tigeot {
162071f41f3eSFrançois Tigeot 	struct intel_ring *ring = req->ring;
1621*3f2dd94aSFrançois Tigeot 	const unsigned int remain_usable = ring->effective_size - ring->emit;
1622*3f2dd94aSFrançois Tigeot 	const unsigned int bytes = num_dwords * sizeof(u32);
1623*3f2dd94aSFrançois Tigeot 	unsigned int need_wrap = 0;
1624*3f2dd94aSFrançois Tigeot 	unsigned int total_bytes;
1625a85cb24fSFrançois Tigeot 	u32 *cs;
1626a05eeebfSFrançois Tigeot 
1627*3f2dd94aSFrançois Tigeot 	/* Packets must be qword aligned. */
1628*3f2dd94aSFrançois Tigeot 	GEM_BUG_ON(num_dwords & 1);
1629a05eeebfSFrançois Tigeot 
1630*3f2dd94aSFrançois Tigeot 	total_bytes = bytes + req->reserved_space;
1631*3f2dd94aSFrançois Tigeot 	GEM_BUG_ON(total_bytes > ring->effective_size);
1632*3f2dd94aSFrançois Tigeot 
1633*3f2dd94aSFrançois Tigeot 	if (unlikely(total_bytes > remain_usable)) {
1634*3f2dd94aSFrançois Tigeot 		const int remain_actual = ring->size - ring->emit;
1635*3f2dd94aSFrançois Tigeot 
1636*3f2dd94aSFrançois Tigeot 		if (bytes > remain_usable) {
1637a05eeebfSFrançois Tigeot 			/*
1638*3f2dd94aSFrançois Tigeot 			 * Not enough space for the basic request. So need to
1639*3f2dd94aSFrançois Tigeot 			 * flush out the remainder and then wait for
1640*3f2dd94aSFrançois Tigeot 			 * base + reserved.
1641a05eeebfSFrançois Tigeot 			 */
1642*3f2dd94aSFrançois Tigeot 			total_bytes += remain_actual;
1643*3f2dd94aSFrançois Tigeot 			need_wrap = remain_actual | 1;
1644*3f2dd94aSFrançois Tigeot 		} else  {
1645a05eeebfSFrançois Tigeot 			/*
1646a05eeebfSFrançois Tigeot 			 * The base request will fit but the reserved space
1647*3f2dd94aSFrançois Tigeot 			 * falls off the end. So we don't need an immediate
1648*3f2dd94aSFrançois Tigeot 			 * wrap and only need to effectively wait for the
1649*3f2dd94aSFrançois Tigeot 			 * reserved size from the start of ringbuffer.
1650a05eeebfSFrançois Tigeot 			 */
1651*3f2dd94aSFrançois Tigeot 			total_bytes = req->reserved_space + remain_actual;
1652*3f2dd94aSFrançois Tigeot 		}
1653a05eeebfSFrançois Tigeot 	}
1654a05eeebfSFrançois Tigeot 
1655*3f2dd94aSFrançois Tigeot 	if (unlikely(total_bytes > ring->space)) {
1656*3f2dd94aSFrançois Tigeot 		int ret = wait_for_space(req, total_bytes);
1657a2fdbec6SFrançois Tigeot 		if (unlikely(ret))
1658a85cb24fSFrançois Tigeot 			return ERR_PTR(ret);
1659a2fdbec6SFrançois Tigeot 	}
1660a2fdbec6SFrançois Tigeot 
16618621f407SFrançois Tigeot 	if (unlikely(need_wrap)) {
1662*3f2dd94aSFrançois Tigeot 		need_wrap &= ~1;
1663*3f2dd94aSFrançois Tigeot 		GEM_BUG_ON(need_wrap > ring->space);
1664*3f2dd94aSFrançois Tigeot 		GEM_BUG_ON(ring->emit + need_wrap > ring->size);
16658621f407SFrançois Tigeot 
16668621f407SFrançois Tigeot 		/* Fill the tail with MI_NOOP */
1667*3f2dd94aSFrançois Tigeot 		memset(ring->vaddr + ring->emit, 0, need_wrap);
1668a85cb24fSFrançois Tigeot 		ring->emit = 0;
1669*3f2dd94aSFrançois Tigeot 		ring->space -= need_wrap;
1670a2fdbec6SFrançois Tigeot 	}
1671a2fdbec6SFrançois Tigeot 
1672a85cb24fSFrançois Tigeot 	GEM_BUG_ON(ring->emit > ring->size - bytes);
1673*3f2dd94aSFrançois Tigeot 	GEM_BUG_ON(ring->space < bytes);
1674a85cb24fSFrançois Tigeot 	cs = ring->vaddr + ring->emit;
1675*3f2dd94aSFrançois Tigeot 	GEM_DEBUG_EXEC(memset(cs, POISON_INUSE, bytes));
1676a85cb24fSFrançois Tigeot 	ring->emit += bytes;
167771f41f3eSFrançois Tigeot 	ring->space -= bytes;
1678a85cb24fSFrançois Tigeot 
1679a85cb24fSFrançois Tigeot 	return cs;
16809edbd4a0SFrançois Tigeot }
16819edbd4a0SFrançois Tigeot 
16829edbd4a0SFrançois Tigeot /* Align the ring tail to a cacheline boundary */
intel_ring_cacheline_align(struct drm_i915_gem_request * req)1683a05eeebfSFrançois Tigeot int intel_ring_cacheline_align(struct drm_i915_gem_request *req)
16849edbd4a0SFrançois Tigeot {
168571f41f3eSFrançois Tigeot 	int num_dwords =
1686a85cb24fSFrançois Tigeot 		(req->ring->emit & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
1687a85cb24fSFrançois Tigeot 	u32 *cs;
16889edbd4a0SFrançois Tigeot 
16899edbd4a0SFrançois Tigeot 	if (num_dwords == 0)
16909edbd4a0SFrançois Tigeot 		return 0;
16919edbd4a0SFrançois Tigeot 
1692ba55f2f5SFrançois Tigeot 	num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
1693a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, num_dwords);
1694a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
1695a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
16969edbd4a0SFrançois Tigeot 
16979edbd4a0SFrançois Tigeot 	while (num_dwords--)
1698a85cb24fSFrançois Tigeot 		*cs++ = MI_NOOP;
16999edbd4a0SFrançois Tigeot 
1700a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
17019edbd4a0SFrançois Tigeot 
17029edbd4a0SFrançois Tigeot 	return 0;
1703e3adcf8fSFrançois Tigeot }
1704e3adcf8fSFrançois Tigeot 
gen6_bsd_submit_request(struct drm_i915_gem_request * request)170571f41f3eSFrançois Tigeot static void gen6_bsd_submit_request(struct drm_i915_gem_request *request)
1706e3adcf8fSFrançois Tigeot {
170771f41f3eSFrançois Tigeot 	struct drm_i915_private *dev_priv = request->i915;
17081487f786SFrançois Tigeot 
17091487f786SFrançois Tigeot 	intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
1710e3adcf8fSFrançois Tigeot 
1711e3adcf8fSFrançois Tigeot        /* Every tail move must follow the sequence below */
1712f4e1c372SFrançois Tigeot 
1713f4e1c372SFrançois Tigeot 	/* Disable notification that the ring is IDLE. The GT
1714f4e1c372SFrançois Tigeot 	 * will then assume that it is busy and bring it out of rc6.
1715f4e1c372SFrançois Tigeot 	 */
17161487f786SFrançois Tigeot 	I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL,
1717f4e1c372SFrançois Tigeot 		      _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1718e3adcf8fSFrançois Tigeot 
1719f4e1c372SFrançois Tigeot 	/* Clear the context id. Here be magic! */
17201487f786SFrançois Tigeot 	I915_WRITE64_FW(GEN6_BSD_RNCID, 0x0);
1721e3adcf8fSFrançois Tigeot 
1722f4e1c372SFrançois Tigeot 	/* Wait for the ring not to be idle, i.e. for it to wake up. */
1723*3f2dd94aSFrançois Tigeot 	if (__intel_wait_for_register_fw(dev_priv,
17241487f786SFrançois Tigeot 					 GEN6_BSD_SLEEP_PSMI_CONTROL,
17251487f786SFrançois Tigeot 					 GEN6_BSD_SLEEP_INDICATOR,
17261487f786SFrançois Tigeot 					 0,
1727*3f2dd94aSFrançois Tigeot 					 1000, 0, NULL))
1728f4e1c372SFrançois Tigeot 		DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
1729f4e1c372SFrançois Tigeot 
1730f4e1c372SFrançois Tigeot 	/* Now that the ring is fully powered up, update the tail */
173171f41f3eSFrançois Tigeot 	i9xx_submit_request(request);
1732f4e1c372SFrançois Tigeot 
1733f4e1c372SFrançois Tigeot 	/* Let the ring send IDLE messages to the GT again,
1734f4e1c372SFrançois Tigeot 	 * and so let it sleep to conserve power when idle.
1735f4e1c372SFrançois Tigeot 	 */
17361487f786SFrançois Tigeot 	I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL,
1737f4e1c372SFrançois Tigeot 		      _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
17381487f786SFrançois Tigeot 
17391487f786SFrançois Tigeot 	intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
1740e3adcf8fSFrançois Tigeot }
1741e3adcf8fSFrançois Tigeot 
gen6_bsd_ring_flush(struct drm_i915_gem_request * req,u32 mode)174271f41f3eSFrançois Tigeot static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req, u32 mode)
1743e3adcf8fSFrançois Tigeot {
1744a85cb24fSFrançois Tigeot 	u32 cmd, *cs;
1745e3adcf8fSFrançois Tigeot 
1746a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 4);
1747a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
1748a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
1749e3adcf8fSFrançois Tigeot 
1750e3adcf8fSFrançois Tigeot 	cmd = MI_FLUSH_DW;
17511487f786SFrançois Tigeot 	if (INTEL_GEN(req->i915) >= 8)
17529edbd4a0SFrançois Tigeot 		cmd += 1;
17532c9916cdSFrançois Tigeot 
17542c9916cdSFrançois Tigeot 	/* We always require a command barrier so that subsequent
17552c9916cdSFrançois Tigeot 	 * commands, such as breadcrumb interrupts, are strictly ordered
17562c9916cdSFrançois Tigeot 	 * wrt the contents of the write cache being flushed to memory
17572c9916cdSFrançois Tigeot 	 * (and thus being coherent from the CPU).
17582c9916cdSFrançois Tigeot 	 */
17592c9916cdSFrançois Tigeot 	cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
17602c9916cdSFrançois Tigeot 
1761b5c29a34SFrançois Tigeot 	/*
1762b5c29a34SFrançois Tigeot 	 * Bspec vol 1c.5 - video engine command streamer:
1763b5c29a34SFrançois Tigeot 	 * "If ENABLED, all TLBs will be invalidated once the flush
1764b5c29a34SFrançois Tigeot 	 * operation is complete. This bit is only valid when the
1765b5c29a34SFrançois Tigeot 	 * Post-Sync Operation field is a value of 1h or 3h."
1766b5c29a34SFrançois Tigeot 	 */
176771f41f3eSFrançois Tigeot 	if (mode & EMIT_INVALIDATE)
17682c9916cdSFrançois Tigeot 		cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
17692c9916cdSFrançois Tigeot 
1770a85cb24fSFrançois Tigeot 	*cs++ = cmd;
1771a85cb24fSFrançois Tigeot 	*cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
17721487f786SFrançois Tigeot 	if (INTEL_GEN(req->i915) >= 8) {
1773a85cb24fSFrançois Tigeot 		*cs++ = 0; /* upper addr */
1774a85cb24fSFrançois Tigeot 		*cs++ = 0; /* value */
17759edbd4a0SFrançois Tigeot 	} else  {
1776a85cb24fSFrançois Tigeot 		*cs++ = 0;
1777a85cb24fSFrançois Tigeot 		*cs++ = MI_NOOP;
17789edbd4a0SFrançois Tigeot 	}
1779a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
17809edbd4a0SFrançois Tigeot 	return 0;
17819edbd4a0SFrançois Tigeot }
17829edbd4a0SFrançois Tigeot 
17839edbd4a0SFrançois Tigeot static int
gen8_emit_bb_start(struct drm_i915_gem_request * req,u64 offset,u32 len,unsigned int dispatch_flags)178471f41f3eSFrançois Tigeot gen8_emit_bb_start(struct drm_i915_gem_request *req,
1785ba55f2f5SFrançois Tigeot 		   u64 offset, u32 len,
178671f41f3eSFrançois Tigeot 		   unsigned int dispatch_flags)
17879edbd4a0SFrançois Tigeot {
178871f41f3eSFrançois Tigeot 	bool ppgtt = USES_PPGTT(req->i915) &&
1789477eb7f9SFrançois Tigeot 			!(dispatch_flags & I915_DISPATCH_SECURE);
1790a85cb24fSFrançois Tigeot 	u32 *cs;
17919edbd4a0SFrançois Tigeot 
1792a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 4);
1793a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
1794a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
17959edbd4a0SFrançois Tigeot 
17969edbd4a0SFrançois Tigeot 	/* FIXME(BDW): Address space and security selectors. */
1797a85cb24fSFrançois Tigeot 	*cs++ = MI_BATCH_BUFFER_START_GEN8 | (ppgtt << 8) | (dispatch_flags &
1798a85cb24fSFrançois Tigeot 		I915_DISPATCH_RS ? MI_BATCH_RESOURCE_STREAMER : 0);
1799a85cb24fSFrançois Tigeot 	*cs++ = lower_32_bits(offset);
1800a85cb24fSFrançois Tigeot 	*cs++ = upper_32_bits(offset);
1801a85cb24fSFrançois Tigeot 	*cs++ = MI_NOOP;
1802a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
18039edbd4a0SFrançois Tigeot 
1804e3adcf8fSFrançois Tigeot 	return 0;
1805e3adcf8fSFrançois Tigeot }
1806e3adcf8fSFrançois Tigeot 
1807e3adcf8fSFrançois Tigeot static int
hsw_emit_bb_start(struct drm_i915_gem_request * req,u64 offset,u32 len,unsigned int dispatch_flags)180871f41f3eSFrançois Tigeot hsw_emit_bb_start(struct drm_i915_gem_request *req,
1809ba55f2f5SFrançois Tigeot 		  u64 offset, u32 len,
181071f41f3eSFrançois Tigeot 		  unsigned int dispatch_flags)
1811e3adcf8fSFrançois Tigeot {
1812a85cb24fSFrançois Tigeot 	u32 *cs;
1813e3adcf8fSFrançois Tigeot 
1814a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 2);
1815a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
1816a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
1817e3adcf8fSFrançois Tigeot 
1818a85cb24fSFrançois Tigeot 	*cs++ = MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ?
1819a05eeebfSFrançois Tigeot 		0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW) |
1820a05eeebfSFrançois Tigeot 		(dispatch_flags & I915_DISPATCH_RS ?
1821a85cb24fSFrançois Tigeot 		MI_BATCH_RESOURCE_STREAMER : 0);
1822b5c29a34SFrançois Tigeot 	/* bit0-7 is the length on GEN6+ */
1823a85cb24fSFrançois Tigeot 	*cs++ = offset;
1824a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
1825b5c29a34SFrançois Tigeot 
1826b5c29a34SFrançois Tigeot 	return 0;
1827b5c29a34SFrançois Tigeot }
1828b5c29a34SFrançois Tigeot 
1829b5c29a34SFrançois Tigeot static int
gen6_emit_bb_start(struct drm_i915_gem_request * req,u64 offset,u32 len,unsigned int dispatch_flags)183071f41f3eSFrançois Tigeot gen6_emit_bb_start(struct drm_i915_gem_request *req,
1831ba55f2f5SFrançois Tigeot 		   u64 offset, u32 len,
183271f41f3eSFrançois Tigeot 		   unsigned int dispatch_flags)
1833b5c29a34SFrançois Tigeot {
1834a85cb24fSFrançois Tigeot 	u32 *cs;
1835b5c29a34SFrançois Tigeot 
1836a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 2);
1837a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
1838a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
1839b5c29a34SFrançois Tigeot 
1840a85cb24fSFrançois Tigeot 	*cs++ = MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ?
1841a85cb24fSFrançois Tigeot 		0 : MI_BATCH_NON_SECURE_I965);
1842e3adcf8fSFrançois Tigeot 	/* bit0-7 is the length on GEN6+ */
1843a85cb24fSFrançois Tigeot 	*cs++ = offset;
1844a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
1845e3adcf8fSFrançois Tigeot 
1846e3adcf8fSFrançois Tigeot 	return 0;
1847e3adcf8fSFrançois Tigeot }
1848e3adcf8fSFrançois Tigeot 
1849e3adcf8fSFrançois Tigeot /* Blitter support (SandyBridge+) */
1850e3adcf8fSFrançois Tigeot 
gen6_ring_flush(struct drm_i915_gem_request * req,u32 mode)185171f41f3eSFrançois Tigeot static int gen6_ring_flush(struct drm_i915_gem_request *req, u32 mode)
1852e3adcf8fSFrançois Tigeot {
1853a85cb24fSFrançois Tigeot 	u32 cmd, *cs;
1854e3adcf8fSFrançois Tigeot 
1855a85cb24fSFrançois Tigeot 	cs = intel_ring_begin(req, 4);
1856a85cb24fSFrançois Tigeot 	if (IS_ERR(cs))
1857a85cb24fSFrançois Tigeot 		return PTR_ERR(cs);
1858e3adcf8fSFrançois Tigeot 
1859e3adcf8fSFrançois Tigeot 	cmd = MI_FLUSH_DW;
18601487f786SFrançois Tigeot 	if (INTEL_GEN(req->i915) >= 8)
18619edbd4a0SFrançois Tigeot 		cmd += 1;
18622c9916cdSFrançois Tigeot 
18632c9916cdSFrançois Tigeot 	/* We always require a command barrier so that subsequent
18642c9916cdSFrançois Tigeot 	 * commands, such as breadcrumb interrupts, are strictly ordered
18652c9916cdSFrançois Tigeot 	 * wrt the contents of the write cache being flushed to memory
18662c9916cdSFrançois Tigeot 	 * (and thus being coherent from the CPU).
18672c9916cdSFrançois Tigeot 	 */
18682c9916cdSFrançois Tigeot 	cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
18692c9916cdSFrançois Tigeot 
1870b5c29a34SFrançois Tigeot 	/*
1871b5c29a34SFrançois Tigeot 	 * Bspec vol 1c.3 - blitter engine command streamer:
1872b5c29a34SFrançois Tigeot 	 * "If ENABLED, all TLBs will be invalidated once the flush
1873b5c29a34SFrançois Tigeot 	 * operation is complete. This bit is only valid when the
1874b5c29a34SFrançois Tigeot 	 * Post-Sync Operation field is a value of 1h or 3h."
1875b5c29a34SFrançois Tigeot 	 */
187671f41f3eSFrançois Tigeot 	if (mode & EMIT_INVALIDATE)
18772c9916cdSFrançois Tigeot 		cmd |= MI_INVALIDATE_TLB;
1878a85cb24fSFrançois Tigeot 	*cs++ = cmd;
1879a85cb24fSFrançois Tigeot 	*cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
18801487f786SFrançois Tigeot 	if (INTEL_GEN(req->i915) >= 8) {
1881a85cb24fSFrançois Tigeot 		*cs++ = 0; /* upper addr */
1882a85cb24fSFrançois Tigeot 		*cs++ = 0; /* value */
18839edbd4a0SFrançois Tigeot 	} else  {
1884a85cb24fSFrançois Tigeot 		*cs++ = 0;
1885a85cb24fSFrançois Tigeot 		*cs++ = MI_NOOP;
18869edbd4a0SFrançois Tigeot 	}
1887a85cb24fSFrançois Tigeot 	intel_ring_advance(req, cs);
18885d0b1887SFrançois Tigeot 
1889e3adcf8fSFrançois Tigeot 	return 0;
1890e3adcf8fSFrançois Tigeot }
1891e3adcf8fSFrançois Tigeot 
intel_ring_init_semaphores(struct drm_i915_private * dev_priv,struct intel_engine_cs * engine)18921487f786SFrançois Tigeot static void intel_ring_init_semaphores(struct drm_i915_private *dev_priv,
18931487f786SFrançois Tigeot 				       struct intel_engine_cs *engine)
18941487f786SFrançois Tigeot {
18951487f786SFrançois Tigeot 	struct drm_i915_gem_object *obj;
18961487f786SFrançois Tigeot 	int ret, i;
18971487f786SFrançois Tigeot 
1898*3f2dd94aSFrançois Tigeot 	if (!i915_modparams.semaphores)
18991487f786SFrançois Tigeot 		return;
19001487f786SFrançois Tigeot 
19011e12ee3bSFrançois Tigeot 	if (INTEL_GEN(dev_priv) >= 8 && !dev_priv->semaphore) {
19021e12ee3bSFrançois Tigeot 		struct i915_vma *vma;
19031487f786SFrançois Tigeot 
1904a85cb24fSFrançois Tigeot 		obj = i915_gem_object_create(dev_priv, PAGE_SIZE);
19051e12ee3bSFrançois Tigeot 		if (IS_ERR(obj))
19061e12ee3bSFrançois Tigeot 			goto err;
19071e12ee3bSFrançois Tigeot 
1908a85cb24fSFrançois Tigeot 		vma = i915_vma_instance(obj, &dev_priv->ggtt.base, NULL);
19091e12ee3bSFrançois Tigeot 		if (IS_ERR(vma))
19101e12ee3bSFrançois Tigeot 			goto err_obj;
19111e12ee3bSFrançois Tigeot 
19121e12ee3bSFrançois Tigeot 		ret = i915_gem_object_set_to_gtt_domain(obj, false);
19131e12ee3bSFrançois Tigeot 		if (ret)
19141e12ee3bSFrançois Tigeot 			goto err_obj;
19151e12ee3bSFrançois Tigeot 
19161e12ee3bSFrançois Tigeot 		ret = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
19171e12ee3bSFrançois Tigeot 		if (ret)
19181e12ee3bSFrançois Tigeot 			goto err_obj;
19191e12ee3bSFrançois Tigeot 
19201e12ee3bSFrançois Tigeot 		dev_priv->semaphore = vma;
19211e12ee3bSFrançois Tigeot 	}
19221487f786SFrançois Tigeot 
19231487f786SFrançois Tigeot 	if (INTEL_GEN(dev_priv) >= 8) {
19241e12ee3bSFrançois Tigeot 		u32 offset = i915_ggtt_offset(dev_priv->semaphore);
19251487f786SFrançois Tigeot 
192671f41f3eSFrançois Tigeot 		engine->semaphore.sync_to = gen8_ring_sync_to;
19271487f786SFrançois Tigeot 		engine->semaphore.signal = gen8_xcs_signal;
19281487f786SFrançois Tigeot 
19291487f786SFrançois Tigeot 		for (i = 0; i < I915_NUM_ENGINES; i++) {
19301e12ee3bSFrançois Tigeot 			u32 ring_offset;
19311487f786SFrançois Tigeot 
19321487f786SFrançois Tigeot 			if (i != engine->id)
19331487f786SFrançois Tigeot 				ring_offset = offset + GEN8_SEMAPHORE_OFFSET(engine->id, i);
19341487f786SFrançois Tigeot 			else
19351487f786SFrançois Tigeot 				ring_offset = MI_SEMAPHORE_SYNC_INVALID;
19361487f786SFrançois Tigeot 
19371487f786SFrançois Tigeot 			engine->semaphore.signal_ggtt[i] = ring_offset;
19381487f786SFrançois Tigeot 		}
19391487f786SFrançois Tigeot 	} else if (INTEL_GEN(dev_priv) >= 6) {
194071f41f3eSFrançois Tigeot 		engine->semaphore.sync_to = gen6_ring_sync_to;
19411487f786SFrançois Tigeot 		engine->semaphore.signal = gen6_signal;
19421487f786SFrançois Tigeot 
19431487f786SFrançois Tigeot 		/*
19441487f786SFrançois Tigeot 		 * The current semaphore is only applied on pre-gen8
19451487f786SFrançois Tigeot 		 * platform.  And there is no VCS2 ring on the pre-gen8
19461487f786SFrançois Tigeot 		 * platform. So the semaphore between RCS and VCS2 is
19471487f786SFrançois Tigeot 		 * initialized as INVALID.  Gen8 will initialize the
19481487f786SFrançois Tigeot 		 * sema between VCS2 and RCS later.
19491487f786SFrançois Tigeot 		 */
19501e12ee3bSFrançois Tigeot 		for (i = 0; i < GEN6_NUM_SEMAPHORES; i++) {
19511487f786SFrançois Tigeot 			static const struct {
19521487f786SFrançois Tigeot 				u32 wait_mbox;
19531487f786SFrançois Tigeot 				i915_reg_t mbox_reg;
19541e12ee3bSFrançois Tigeot 			} sem_data[GEN6_NUM_SEMAPHORES][GEN6_NUM_SEMAPHORES] = {
19551e12ee3bSFrançois Tigeot 				[RCS_HW] = {
19561e12ee3bSFrançois Tigeot 					[VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_RV,  .mbox_reg = GEN6_VRSYNC },
19571e12ee3bSFrançois Tigeot 					[BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_RB,  .mbox_reg = GEN6_BRSYNC },
19581e12ee3bSFrançois Tigeot 					[VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_RVE, .mbox_reg = GEN6_VERSYNC },
19591487f786SFrançois Tigeot 				},
19601e12ee3bSFrançois Tigeot 				[VCS_HW] = {
19611e12ee3bSFrançois Tigeot 					[RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VR,  .mbox_reg = GEN6_RVSYNC },
19621e12ee3bSFrançois Tigeot 					[BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VB,  .mbox_reg = GEN6_BVSYNC },
19631e12ee3bSFrançois Tigeot 					[VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VVE, .mbox_reg = GEN6_VEVSYNC },
19641487f786SFrançois Tigeot 				},
19651e12ee3bSFrançois Tigeot 				[BCS_HW] = {
19661e12ee3bSFrançois Tigeot 					[RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_BR,  .mbox_reg = GEN6_RBSYNC },
19671e12ee3bSFrançois Tigeot 					[VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_BV,  .mbox_reg = GEN6_VBSYNC },
19681e12ee3bSFrançois Tigeot 					[VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_BVE, .mbox_reg = GEN6_VEBSYNC },
19691487f786SFrançois Tigeot 				},
19701e12ee3bSFrançois Tigeot 				[VECS_HW] = {
19711e12ee3bSFrançois Tigeot 					[RCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VER, .mbox_reg = GEN6_RVESYNC },
19721e12ee3bSFrançois Tigeot 					[VCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VEV, .mbox_reg = GEN6_VVESYNC },
19731e12ee3bSFrançois Tigeot 					[BCS_HW] =  { .wait_mbox = MI_SEMAPHORE_SYNC_VEB, .mbox_reg = GEN6_BVESYNC },
19741487f786SFrançois Tigeot 				},
19751487f786SFrançois Tigeot 			};
19761487f786SFrançois Tigeot 			u32 wait_mbox;
19771487f786SFrançois Tigeot 			i915_reg_t mbox_reg;
19781487f786SFrançois Tigeot 
19791e12ee3bSFrançois Tigeot 			if (i == engine->hw_id) {
19801487f786SFrançois Tigeot 				wait_mbox = MI_SEMAPHORE_SYNC_INVALID;
19811487f786SFrançois Tigeot 				mbox_reg = GEN6_NOSYNC;
19821487f786SFrançois Tigeot 			} else {
19831e12ee3bSFrançois Tigeot 				wait_mbox = sem_data[engine->hw_id][i].wait_mbox;
19841e12ee3bSFrançois Tigeot 				mbox_reg = sem_data[engine->hw_id][i].mbox_reg;
19851487f786SFrançois Tigeot 			}
19861487f786SFrançois Tigeot 
19871487f786SFrançois Tigeot 			engine->semaphore.mbox.wait[i] = wait_mbox;
19881487f786SFrançois Tigeot 			engine->semaphore.mbox.signal[i] = mbox_reg;
19891487f786SFrançois Tigeot 		}
19901487f786SFrançois Tigeot 	}
19911e12ee3bSFrançois Tigeot 
19921e12ee3bSFrançois Tigeot 	return;
19931e12ee3bSFrançois Tigeot 
19941e12ee3bSFrançois Tigeot err_obj:
19951e12ee3bSFrançois Tigeot 	i915_gem_object_put(obj);
19961e12ee3bSFrançois Tigeot err:
19971e12ee3bSFrançois Tigeot 	DRM_DEBUG_DRIVER("Failed to allocate space for semaphores, disabling\n");
1998*3f2dd94aSFrançois Tigeot 	i915_modparams.semaphores = 0;
19991487f786SFrançois Tigeot }
20001487f786SFrançois Tigeot 
intel_ring_init_irq(struct drm_i915_private * dev_priv,struct intel_engine_cs * engine)20011487f786SFrançois Tigeot static void intel_ring_init_irq(struct drm_i915_private *dev_priv,
20021487f786SFrançois Tigeot 				struct intel_engine_cs *engine)
20031487f786SFrançois Tigeot {
200487df8fc6SFrançois Tigeot 	engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << engine->irq_shift;
200587df8fc6SFrançois Tigeot 
20061487f786SFrançois Tigeot 	if (INTEL_GEN(dev_priv) >= 8) {
2007303bf270SFrançois Tigeot 		engine->irq_enable = gen8_irq_enable;
2008303bf270SFrançois Tigeot 		engine->irq_disable = gen8_irq_disable;
20091487f786SFrançois Tigeot 		engine->irq_seqno_barrier = gen6_seqno_barrier;
20101487f786SFrançois Tigeot 	} else if (INTEL_GEN(dev_priv) >= 6) {
2011303bf270SFrançois Tigeot 		engine->irq_enable = gen6_irq_enable;
2012303bf270SFrançois Tigeot 		engine->irq_disable = gen6_irq_disable;
20131487f786SFrançois Tigeot 		engine->irq_seqno_barrier = gen6_seqno_barrier;
20141487f786SFrançois Tigeot 	} else if (INTEL_GEN(dev_priv) >= 5) {
2015303bf270SFrançois Tigeot 		engine->irq_enable = gen5_irq_enable;
2016303bf270SFrançois Tigeot 		engine->irq_disable = gen5_irq_disable;
2017303bf270SFrançois Tigeot 		engine->irq_seqno_barrier = gen5_seqno_barrier;
20181487f786SFrançois Tigeot 	} else if (INTEL_GEN(dev_priv) >= 3) {
2019303bf270SFrançois Tigeot 		engine->irq_enable = i9xx_irq_enable;
2020303bf270SFrançois Tigeot 		engine->irq_disable = i9xx_irq_disable;
20211487f786SFrançois Tigeot 	} else {
2022303bf270SFrançois Tigeot 		engine->irq_enable = i8xx_irq_enable;
2023303bf270SFrançois Tigeot 		engine->irq_disable = i8xx_irq_disable;
20241487f786SFrançois Tigeot 	}
20251487f786SFrançois Tigeot }
20261487f786SFrançois Tigeot 
i9xx_set_default_submission(struct intel_engine_cs * engine)2027a85cb24fSFrançois Tigeot static void i9xx_set_default_submission(struct intel_engine_cs *engine)
2028a85cb24fSFrançois Tigeot {
2029a85cb24fSFrançois Tigeot 	engine->submit_request = i9xx_submit_request;
2030*3f2dd94aSFrançois Tigeot 	engine->cancel_requests = cancel_requests;
2031a85cb24fSFrançois Tigeot }
2032a85cb24fSFrançois Tigeot 
gen6_bsd_set_default_submission(struct intel_engine_cs * engine)2033a85cb24fSFrançois Tigeot static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine)
2034a85cb24fSFrançois Tigeot {
2035a85cb24fSFrançois Tigeot 	engine->submit_request = gen6_bsd_submit_request;
2036*3f2dd94aSFrançois Tigeot 	engine->cancel_requests = cancel_requests;
2037a85cb24fSFrançois Tigeot }
2038a85cb24fSFrançois Tigeot 
intel_ring_default_vfuncs(struct drm_i915_private * dev_priv,struct intel_engine_cs * engine)20391487f786SFrançois Tigeot static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv,
20401487f786SFrançois Tigeot 				      struct intel_engine_cs *engine)
20411487f786SFrançois Tigeot {
20421487f786SFrançois Tigeot 	intel_ring_init_irq(dev_priv, engine);
20431487f786SFrançois Tigeot 	intel_ring_init_semaphores(dev_priv, engine);
204471f41f3eSFrançois Tigeot 
204571f41f3eSFrançois Tigeot 	engine->init_hw = init_ring_common;
20461e12ee3bSFrançois Tigeot 	engine->reset_hw = reset_ring_common;
204771f41f3eSFrançois Tigeot 
2048a85cb24fSFrançois Tigeot 	engine->context_pin = intel_ring_context_pin;
2049a85cb24fSFrançois Tigeot 	engine->context_unpin = intel_ring_context_unpin;
2050a85cb24fSFrançois Tigeot 
2051a85cb24fSFrançois Tigeot 	engine->request_alloc = ring_request_alloc;
2052a85cb24fSFrançois Tigeot 
20534be47400SFrançois Tigeot 	engine->emit_breadcrumb = i9xx_emit_breadcrumb;
20544be47400SFrançois Tigeot 	engine->emit_breadcrumb_sz = i9xx_emit_breadcrumb_sz;
2055*3f2dd94aSFrançois Tigeot 	if (i915_modparams.semaphores) {
20564be47400SFrançois Tigeot 		int num_rings;
20574be47400SFrançois Tigeot 
20584be47400SFrançois Tigeot 		engine->emit_breadcrumb = gen6_sema_emit_breadcrumb;
20594be47400SFrançois Tigeot 
2060*3f2dd94aSFrançois Tigeot 		num_rings = INTEL_INFO(dev_priv)->num_rings - 1;
20614be47400SFrançois Tigeot 		if (INTEL_GEN(dev_priv) >= 8) {
20624be47400SFrançois Tigeot 			engine->emit_breadcrumb_sz += num_rings * 6;
20634be47400SFrançois Tigeot 		} else {
20644be47400SFrançois Tigeot 			engine->emit_breadcrumb_sz += num_rings * 3;
20654be47400SFrançois Tigeot 			if (num_rings & 1)
20664be47400SFrançois Tigeot 				engine->emit_breadcrumb_sz++;
20674be47400SFrançois Tigeot 		}
20684be47400SFrançois Tigeot 	}
2069a85cb24fSFrançois Tigeot 
2070a85cb24fSFrançois Tigeot 	engine->set_default_submission = i9xx_set_default_submission;
207171f41f3eSFrançois Tigeot 
207271f41f3eSFrançois Tigeot 	if (INTEL_GEN(dev_priv) >= 8)
207371f41f3eSFrançois Tigeot 		engine->emit_bb_start = gen8_emit_bb_start;
207471f41f3eSFrançois Tigeot 	else if (INTEL_GEN(dev_priv) >= 6)
207571f41f3eSFrançois Tigeot 		engine->emit_bb_start = gen6_emit_bb_start;
207671f41f3eSFrançois Tigeot 	else if (INTEL_GEN(dev_priv) >= 4)
207771f41f3eSFrançois Tigeot 		engine->emit_bb_start = i965_emit_bb_start;
2078a85cb24fSFrançois Tigeot 	else if (IS_I830(dev_priv) || IS_I845G(dev_priv))
207971f41f3eSFrançois Tigeot 		engine->emit_bb_start = i830_emit_bb_start;
208071f41f3eSFrançois Tigeot 	else
208171f41f3eSFrançois Tigeot 		engine->emit_bb_start = i915_emit_bb_start;
20821487f786SFrançois Tigeot }
20831487f786SFrançois Tigeot 
intel_init_render_ring_buffer(struct intel_engine_cs * engine)208487df8fc6SFrançois Tigeot int intel_init_render_ring_buffer(struct intel_engine_cs *engine)
2085e3adcf8fSFrançois Tigeot {
208687df8fc6SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
208724edb884SFrançois Tigeot 	int ret;
2088e3adcf8fSFrançois Tigeot 
20891487f786SFrançois Tigeot 	intel_ring_default_vfuncs(dev_priv, engine);
20902c9916cdSFrançois Tigeot 
2091303bf270SFrançois Tigeot 	if (HAS_L3_DPF(dev_priv))
2092303bf270SFrançois Tigeot 		engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2093303bf270SFrançois Tigeot 
20941487f786SFrançois Tigeot 	if (INTEL_GEN(dev_priv) >= 8) {
20958621f407SFrançois Tigeot 		engine->init_context = intel_rcs_ctx_init;
20964be47400SFrançois Tigeot 		engine->emit_breadcrumb = gen8_render_emit_breadcrumb;
20974be47400SFrançois Tigeot 		engine->emit_breadcrumb_sz = gen8_render_emit_breadcrumb_sz;
209871f41f3eSFrançois Tigeot 		engine->emit_flush = gen8_render_ring_flush;
2099*3f2dd94aSFrançois Tigeot 		if (i915_modparams.semaphores) {
21004be47400SFrançois Tigeot 			int num_rings;
21014be47400SFrançois Tigeot 
21028621f407SFrançois Tigeot 			engine->semaphore.signal = gen8_rcs_signal;
21034be47400SFrançois Tigeot 
2104*3f2dd94aSFrançois Tigeot 			num_rings = INTEL_INFO(dev_priv)->num_rings - 1;
2105a85cb24fSFrançois Tigeot 			engine->emit_breadcrumb_sz += num_rings * 8;
21064be47400SFrançois Tigeot 		}
21071487f786SFrançois Tigeot 	} else if (INTEL_GEN(dev_priv) >= 6) {
21088621f407SFrançois Tigeot 		engine->init_context = intel_rcs_ctx_init;
210971f41f3eSFrançois Tigeot 		engine->emit_flush = gen7_render_ring_flush;
21101487f786SFrançois Tigeot 		if (IS_GEN6(dev_priv))
211171f41f3eSFrançois Tigeot 			engine->emit_flush = gen6_render_ring_flush;
21121487f786SFrançois Tigeot 	} else if (IS_GEN5(dev_priv)) {
211371f41f3eSFrançois Tigeot 		engine->emit_flush = gen4_render_ring_flush;
2114686a02f1SFrançois Tigeot 	} else {
21151487f786SFrançois Tigeot 		if (INTEL_GEN(dev_priv) < 4)
211671f41f3eSFrançois Tigeot 			engine->emit_flush = gen2_render_ring_flush;
2117686a02f1SFrançois Tigeot 		else
211871f41f3eSFrançois Tigeot 			engine->emit_flush = gen4_render_ring_flush;
21198621f407SFrançois Tigeot 		engine->irq_enable_mask = I915_USER_INTERRUPT;
2120686a02f1SFrançois Tigeot 	}
212124edb884SFrançois Tigeot 
21221487f786SFrançois Tigeot 	if (IS_HASWELL(dev_priv))
212371f41f3eSFrançois Tigeot 		engine->emit_bb_start = hsw_emit_bb_start;
21241487f786SFrançois Tigeot 
21258621f407SFrançois Tigeot 	engine->init_hw = init_render_ring;
21268621f407SFrançois Tigeot 	engine->cleanup = render_ring_cleanup;
2127e3adcf8fSFrançois Tigeot 
212887df8fc6SFrançois Tigeot 	ret = intel_init_ring_buffer(engine);
2129b5c29a34SFrançois Tigeot 	if (ret)
21302c9916cdSFrançois Tigeot 		return ret;
21312c9916cdSFrançois Tigeot 
2132303bf270SFrançois Tigeot 	if (INTEL_GEN(dev_priv) >= 6) {
2133a85cb24fSFrançois Tigeot 		ret = intel_engine_create_scratch(engine, PAGE_SIZE);
2134303bf270SFrançois Tigeot 		if (ret)
2135303bf270SFrançois Tigeot 			return ret;
2136303bf270SFrançois Tigeot 	} else if (HAS_BROKEN_CS_TLB(dev_priv)) {
21371e12ee3bSFrançois Tigeot 		ret = intel_engine_create_scratch(engine, I830_WA_SIZE);
21382c9916cdSFrançois Tigeot 		if (ret)
21392c9916cdSFrançois Tigeot 			return ret;
2140b5c29a34SFrançois Tigeot 	}
2141b5c29a34SFrançois Tigeot 
2142e3adcf8fSFrançois Tigeot 	return 0;
2143e3adcf8fSFrançois Tigeot }
2144e3adcf8fSFrançois Tigeot 
intel_init_bsd_ring_buffer(struct intel_engine_cs * engine)214587df8fc6SFrançois Tigeot int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine)
2146e3adcf8fSFrançois Tigeot {
214787df8fc6SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
2148686a02f1SFrançois Tigeot 
21491487f786SFrançois Tigeot 	intel_ring_default_vfuncs(dev_priv, engine);
21501487f786SFrançois Tigeot 
21511487f786SFrançois Tigeot 	if (INTEL_GEN(dev_priv) >= 6) {
2152686a02f1SFrançois Tigeot 		/* gen6 bsd needs a special wa for tail updates */
21531487f786SFrançois Tigeot 		if (IS_GEN6(dev_priv))
2154a85cb24fSFrançois Tigeot 			engine->set_default_submission = gen6_bsd_set_default_submission;
215571f41f3eSFrançois Tigeot 		engine->emit_flush = gen6_bsd_ring_flush;
215687df8fc6SFrançois Tigeot 		if (INTEL_GEN(dev_priv) < 8)
21578621f407SFrançois Tigeot 			engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
2158686a02f1SFrançois Tigeot 	} else {
21598621f407SFrançois Tigeot 		engine->mmio_base = BSD_RING_BASE;
216071f41f3eSFrançois Tigeot 		engine->emit_flush = bsd_ring_flush;
21611487f786SFrançois Tigeot 		if (IS_GEN5(dev_priv))
21628621f407SFrançois Tigeot 			engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
21631487f786SFrançois Tigeot 		else
21648621f407SFrançois Tigeot 			engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;
2165686a02f1SFrançois Tigeot 	}
2166e3adcf8fSFrançois Tigeot 
216787df8fc6SFrançois Tigeot 	return intel_init_ring_buffer(engine);
2168e3adcf8fSFrançois Tigeot }
2169e3adcf8fSFrançois Tigeot 
intel_init_blt_ring_buffer(struct intel_engine_cs * engine)217087df8fc6SFrançois Tigeot int intel_init_blt_ring_buffer(struct intel_engine_cs *engine)
2171e3adcf8fSFrançois Tigeot {
217287df8fc6SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
21731487f786SFrançois Tigeot 
21741487f786SFrançois Tigeot 	intel_ring_default_vfuncs(dev_priv, engine);
21751487f786SFrançois Tigeot 
217671f41f3eSFrançois Tigeot 	engine->emit_flush = gen6_ring_flush;
217787df8fc6SFrançois Tigeot 	if (INTEL_GEN(dev_priv) < 8)
21788621f407SFrançois Tigeot 		engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
21795d0b1887SFrançois Tigeot 
218087df8fc6SFrançois Tigeot 	return intel_init_ring_buffer(engine);
21815d0b1887SFrançois Tigeot }
21825d0b1887SFrançois Tigeot 
intel_init_vebox_ring_buffer(struct intel_engine_cs * engine)218387df8fc6SFrançois Tigeot int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine)
21845d0b1887SFrançois Tigeot {
218587df8fc6SFrançois Tigeot 	struct drm_i915_private *dev_priv = engine->i915;
21869edbd4a0SFrançois Tigeot 
21871487f786SFrançois Tigeot 	intel_ring_default_vfuncs(dev_priv, engine);
21881487f786SFrançois Tigeot 
218971f41f3eSFrançois Tigeot 	engine->emit_flush = gen6_ring_flush;
21901487f786SFrançois Tigeot 
219187df8fc6SFrançois Tigeot 	if (INTEL_GEN(dev_priv) < 8) {
21928621f407SFrançois Tigeot 		engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
2193303bf270SFrançois Tigeot 		engine->irq_enable = hsw_vebox_irq_enable;
2194303bf270SFrançois Tigeot 		engine->irq_disable = hsw_vebox_irq_disable;
219524edb884SFrançois Tigeot 	}
2196e3adcf8fSFrançois Tigeot 
219787df8fc6SFrançois Tigeot 	return intel_init_ring_buffer(engine);
2198e3adcf8fSFrançois Tigeot }
2199