1e3adcf8fSFrançois Tigeot /*
2e3adcf8fSFrançois Tigeot * Copyright © 2008-2010 Intel Corporation
3e3adcf8fSFrançois Tigeot *
4e3adcf8fSFrançois Tigeot * Permission is hereby granted, free of charge, to any person obtaining a
5e3adcf8fSFrançois Tigeot * copy of this software and associated documentation files (the "Software"),
6e3adcf8fSFrançois Tigeot * to deal in the Software without restriction, including without limitation
7e3adcf8fSFrançois Tigeot * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8e3adcf8fSFrançois Tigeot * and/or sell copies of the Software, and to permit persons to whom the
9e3adcf8fSFrançois Tigeot * Software is furnished to do so, subject to the following conditions:
10e3adcf8fSFrançois Tigeot *
11e3adcf8fSFrançois Tigeot * The above copyright notice and this permission notice (including the next
12e3adcf8fSFrançois Tigeot * paragraph) shall be included in all copies or substantial portions of the
13e3adcf8fSFrançois Tigeot * Software.
14e3adcf8fSFrançois Tigeot *
15e3adcf8fSFrançois Tigeot * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16e3adcf8fSFrançois Tigeot * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17e3adcf8fSFrançois Tigeot * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18e3adcf8fSFrançois Tigeot * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19e3adcf8fSFrançois Tigeot * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20e3adcf8fSFrançois Tigeot * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21e3adcf8fSFrançois Tigeot * IN THE SOFTWARE.
22e3adcf8fSFrançois Tigeot *
23e3adcf8fSFrançois Tigeot * Authors:
24e3adcf8fSFrançois Tigeot * Eric Anholt <eric@anholt.net>
25e3adcf8fSFrançois Tigeot * Zou Nan hai <nanhai.zou@intel.com>
26e3adcf8fSFrançois Tigeot * Xiang Hai hao<haihao.xiang@intel.com>
27e3adcf8fSFrançois Tigeot *
28e3adcf8fSFrançois Tigeot */
29e3adcf8fSFrançois Tigeot
30aee94f86SFrançois Tigeot #include <linux/log2.h>
3118e26a6dSFrançois Tigeot #include <drm/drmP.h>
32e3adcf8fSFrançois Tigeot #include "i915_drv.h"
33a2fdbec6SFrançois Tigeot #include <drm/i915_drm.h>
34a2fdbec6SFrançois Tigeot #include "i915_trace.h"
35e3adcf8fSFrançois Tigeot #include "intel_drv.h"
36e3adcf8fSFrançois Tigeot
371487f786SFrançois Tigeot /* Rough estimate of the typical request size, performing a flush,
381487f786SFrançois Tigeot * set-context and then emitting the batch.
391487f786SFrançois Tigeot */
401487f786SFrançois Tigeot #define LEGACY_REQUEST_SIZE 200
411487f786SFrançois Tigeot
__intel_ring_space(unsigned int head,unsigned int tail,unsigned int size)42*3f2dd94aSFrançois Tigeot static unsigned int __intel_ring_space(unsigned int head,
43*3f2dd94aSFrançois Tigeot unsigned int tail,
44*3f2dd94aSFrançois Tigeot unsigned int size)
45e3adcf8fSFrançois Tigeot {
46*3f2dd94aSFrançois Tigeot /*
47*3f2dd94aSFrançois Tigeot * "If the Ring Buffer Head Pointer and the Tail Pointer are on the
48*3f2dd94aSFrançois Tigeot * same cacheline, the Head Pointer must not be greater than the Tail
49*3f2dd94aSFrançois Tigeot * Pointer."
50*3f2dd94aSFrançois Tigeot */
51*3f2dd94aSFrançois Tigeot GEM_BUG_ON(!is_power_of_2(size));
52*3f2dd94aSFrançois Tigeot return (head - tail - CACHELINE_BYTES) & (size - 1);
532c9916cdSFrançois Tigeot }
542c9916cdSFrançois Tigeot
intel_ring_update_space(struct intel_ring * ring)55*3f2dd94aSFrançois Tigeot unsigned int intel_ring_update_space(struct intel_ring *ring)
562c9916cdSFrançois Tigeot {
57*3f2dd94aSFrançois Tigeot unsigned int space;
58*3f2dd94aSFrançois Tigeot
59*3f2dd94aSFrançois Tigeot space = __intel_ring_space(ring->head, ring->emit, ring->size);
60*3f2dd94aSFrançois Tigeot
61*3f2dd94aSFrançois Tigeot ring->space = space;
62*3f2dd94aSFrançois Tigeot return space;
639edbd4a0SFrançois Tigeot }
649edbd4a0SFrançois Tigeot
65e3adcf8fSFrançois Tigeot static int
gen2_render_ring_flush(struct drm_i915_gem_request * req,u32 mode)6671f41f3eSFrançois Tigeot gen2_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
67686a02f1SFrançois Tigeot {
68a85cb24fSFrançois Tigeot u32 cmd, *cs;
69686a02f1SFrançois Tigeot
70686a02f1SFrançois Tigeot cmd = MI_FLUSH;
71686a02f1SFrançois Tigeot
7271f41f3eSFrançois Tigeot if (mode & EMIT_INVALIDATE)
73686a02f1SFrançois Tigeot cmd |= MI_READ_FLUSH;
74686a02f1SFrançois Tigeot
75a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 2);
76a85cb24fSFrançois Tigeot if (IS_ERR(cs))
77a85cb24fSFrançois Tigeot return PTR_ERR(cs);
78686a02f1SFrançois Tigeot
79a85cb24fSFrançois Tigeot *cs++ = cmd;
80a85cb24fSFrançois Tigeot *cs++ = MI_NOOP;
81a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
82686a02f1SFrançois Tigeot
83686a02f1SFrançois Tigeot return 0;
84686a02f1SFrançois Tigeot }
85686a02f1SFrançois Tigeot
86686a02f1SFrançois Tigeot static int
gen4_render_ring_flush(struct drm_i915_gem_request * req,u32 mode)8771f41f3eSFrançois Tigeot gen4_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
88e3adcf8fSFrançois Tigeot {
89a85cb24fSFrançois Tigeot u32 cmd, *cs;
90e3adcf8fSFrançois Tigeot
91e3adcf8fSFrançois Tigeot /*
92e3adcf8fSFrançois Tigeot * read/write caches:
93e3adcf8fSFrançois Tigeot *
94e3adcf8fSFrançois Tigeot * I915_GEM_DOMAIN_RENDER is always invalidated, but is
95e3adcf8fSFrançois Tigeot * only flushed if MI_NO_WRITE_FLUSH is unset. On 965, it is
96e3adcf8fSFrançois Tigeot * also flushed at 2d versus 3d pipeline switches.
97e3adcf8fSFrançois Tigeot *
98e3adcf8fSFrançois Tigeot * read-only caches:
99e3adcf8fSFrançois Tigeot *
100e3adcf8fSFrançois Tigeot * I915_GEM_DOMAIN_SAMPLER is flushed on pre-965 if
101e3adcf8fSFrançois Tigeot * MI_READ_FLUSH is set, and is always flushed on 965.
102e3adcf8fSFrançois Tigeot *
103e3adcf8fSFrançois Tigeot * I915_GEM_DOMAIN_COMMAND may not exist?
104e3adcf8fSFrançois Tigeot *
105e3adcf8fSFrançois Tigeot * I915_GEM_DOMAIN_INSTRUCTION, which exists on 965, is
106e3adcf8fSFrançois Tigeot * invalidated when MI_EXE_FLUSH is set.
107e3adcf8fSFrançois Tigeot *
108e3adcf8fSFrançois Tigeot * I915_GEM_DOMAIN_VERTEX, which exists on 965, is
109e3adcf8fSFrançois Tigeot * invalidated with every MI_FLUSH.
110e3adcf8fSFrançois Tigeot *
111e3adcf8fSFrançois Tigeot * TLBs:
112e3adcf8fSFrançois Tigeot *
113e3adcf8fSFrançois Tigeot * On 965, TLBs associated with I915_GEM_DOMAIN_COMMAND
114e3adcf8fSFrançois Tigeot * and I915_GEM_DOMAIN_CPU in are invalidated at PTE write and
115e3adcf8fSFrançois Tigeot * I915_GEM_DOMAIN_RENDER and I915_GEM_DOMAIN_SAMPLER
116e3adcf8fSFrançois Tigeot * are flushed at any MI_FLUSH.
117e3adcf8fSFrançois Tigeot */
118e3adcf8fSFrançois Tigeot
11971f41f3eSFrançois Tigeot cmd = MI_FLUSH;
12071f41f3eSFrançois Tigeot if (mode & EMIT_INVALIDATE) {
121e3adcf8fSFrançois Tigeot cmd |= MI_EXE_FLUSH;
12271f41f3eSFrançois Tigeot if (IS_G4X(req->i915) || IS_GEN5(req->i915))
123e3adcf8fSFrançois Tigeot cmd |= MI_INVALIDATE_ISP;
12471f41f3eSFrançois Tigeot }
125e3adcf8fSFrançois Tigeot
126a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 2);
127a85cb24fSFrançois Tigeot if (IS_ERR(cs))
128a85cb24fSFrançois Tigeot return PTR_ERR(cs);
129e3adcf8fSFrançois Tigeot
130a85cb24fSFrançois Tigeot *cs++ = cmd;
131a85cb24fSFrançois Tigeot *cs++ = MI_NOOP;
132a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
133e3adcf8fSFrançois Tigeot
134e3adcf8fSFrançois Tigeot return 0;
135e3adcf8fSFrançois Tigeot }
136e3adcf8fSFrançois Tigeot
137e3adcf8fSFrançois Tigeot /**
138e3adcf8fSFrançois Tigeot * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
139e3adcf8fSFrançois Tigeot * implementing two workarounds on gen6. From section 1.4.7.1
140e3adcf8fSFrançois Tigeot * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
141e3adcf8fSFrançois Tigeot *
142e3adcf8fSFrançois Tigeot * [DevSNB-C+{W/A}] Before any depth stall flush (including those
143e3adcf8fSFrançois Tigeot * produced by non-pipelined state commands), software needs to first
144e3adcf8fSFrançois Tigeot * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
145e3adcf8fSFrançois Tigeot * 0.
146e3adcf8fSFrançois Tigeot *
147e3adcf8fSFrançois Tigeot * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
148e3adcf8fSFrançois Tigeot * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
149e3adcf8fSFrançois Tigeot *
150e3adcf8fSFrançois Tigeot * And the workaround for these two requires this workaround first:
151e3adcf8fSFrançois Tigeot *
152e3adcf8fSFrançois Tigeot * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
153e3adcf8fSFrançois Tigeot * BEFORE the pipe-control with a post-sync op and no write-cache
154e3adcf8fSFrançois Tigeot * flushes.
155e3adcf8fSFrançois Tigeot *
156e3adcf8fSFrançois Tigeot * And this last workaround is tricky because of the requirements on
157e3adcf8fSFrançois Tigeot * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
158e3adcf8fSFrançois Tigeot * volume 2 part 1:
159e3adcf8fSFrançois Tigeot *
160e3adcf8fSFrançois Tigeot * "1 of the following must also be set:
161e3adcf8fSFrançois Tigeot * - Render Target Cache Flush Enable ([12] of DW1)
162e3adcf8fSFrançois Tigeot * - Depth Cache Flush Enable ([0] of DW1)
163e3adcf8fSFrançois Tigeot * - Stall at Pixel Scoreboard ([1] of DW1)
164e3adcf8fSFrançois Tigeot * - Depth Stall ([13] of DW1)
165e3adcf8fSFrançois Tigeot * - Post-Sync Operation ([13] of DW1)
166e3adcf8fSFrançois Tigeot * - Notify Enable ([8] of DW1)"
167e3adcf8fSFrançois Tigeot *
168e3adcf8fSFrançois Tigeot * The cache flushes require the workaround flush that triggered this
169e3adcf8fSFrançois Tigeot * one, so we can't use it. Depth stall would trigger the same.
170e3adcf8fSFrançois Tigeot * Post-sync nonzero is what triggered this second workaround, so we
171e3adcf8fSFrançois Tigeot * can't use that one either. Notify enable is IRQs, which aren't
172e3adcf8fSFrançois Tigeot * really our business. That leaves only stall at scoreboard.
173e3adcf8fSFrançois Tigeot */
174e3adcf8fSFrançois Tigeot static int
intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request * req)175a05eeebfSFrançois Tigeot intel_emit_post_sync_nonzero_flush(struct drm_i915_gem_request *req)
176e3adcf8fSFrançois Tigeot {
17771f41f3eSFrançois Tigeot u32 scratch_addr =
1781e12ee3bSFrançois Tigeot i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
179a85cb24fSFrançois Tigeot u32 *cs;
180e3adcf8fSFrançois Tigeot
181a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 6);
182a85cb24fSFrançois Tigeot if (IS_ERR(cs))
183a85cb24fSFrançois Tigeot return PTR_ERR(cs);
184e3adcf8fSFrançois Tigeot
185a85cb24fSFrançois Tigeot *cs++ = GFX_OP_PIPE_CONTROL(5);
186a85cb24fSFrançois Tigeot *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;
187a85cb24fSFrançois Tigeot *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;
188a85cb24fSFrançois Tigeot *cs++ = 0; /* low dword */
189a85cb24fSFrançois Tigeot *cs++ = 0; /* high dword */
190a85cb24fSFrançois Tigeot *cs++ = MI_NOOP;
191a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
192e3adcf8fSFrançois Tigeot
193a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 6);
194a85cb24fSFrançois Tigeot if (IS_ERR(cs))
195a85cb24fSFrançois Tigeot return PTR_ERR(cs);
196e3adcf8fSFrançois Tigeot
197a85cb24fSFrançois Tigeot *cs++ = GFX_OP_PIPE_CONTROL(5);
198a85cb24fSFrançois Tigeot *cs++ = PIPE_CONTROL_QW_WRITE;
199a85cb24fSFrançois Tigeot *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;
200a85cb24fSFrançois Tigeot *cs++ = 0;
201a85cb24fSFrançois Tigeot *cs++ = 0;
202a85cb24fSFrançois Tigeot *cs++ = MI_NOOP;
203a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
204e3adcf8fSFrançois Tigeot
205e3adcf8fSFrançois Tigeot return 0;
206e3adcf8fSFrançois Tigeot }
207e3adcf8fSFrançois Tigeot
208e3adcf8fSFrançois Tigeot static int
gen6_render_ring_flush(struct drm_i915_gem_request * req,u32 mode)20971f41f3eSFrançois Tigeot gen6_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
210e3adcf8fSFrançois Tigeot {
21171f41f3eSFrançois Tigeot u32 scratch_addr =
2121e12ee3bSFrançois Tigeot i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
213a85cb24fSFrançois Tigeot u32 *cs, flags = 0;
214e3adcf8fSFrançois Tigeot int ret;
215e3adcf8fSFrançois Tigeot
216e3adcf8fSFrançois Tigeot /* Force SNB workarounds for PIPE_CONTROL flushes */
217a05eeebfSFrançois Tigeot ret = intel_emit_post_sync_nonzero_flush(req);
218686a02f1SFrançois Tigeot if (ret)
219686a02f1SFrançois Tigeot return ret;
220e3adcf8fSFrançois Tigeot
221e3adcf8fSFrançois Tigeot /* Just flush everything. Experiments have shown that reducing the
222e3adcf8fSFrançois Tigeot * number of bits based on the write domains has little performance
223e3adcf8fSFrançois Tigeot * impact.
224e3adcf8fSFrançois Tigeot */
22571f41f3eSFrançois Tigeot if (mode & EMIT_FLUSH) {
226e3adcf8fSFrançois Tigeot flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
227b5c29a34SFrançois Tigeot flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
228b5c29a34SFrançois Tigeot /*
229b5c29a34SFrançois Tigeot * Ensure that any following seqno writes only happen
230b5c29a34SFrançois Tigeot * when the render cache is indeed flushed.
231b5c29a34SFrançois Tigeot */
232b5c29a34SFrançois Tigeot flags |= PIPE_CONTROL_CS_STALL;
233b5c29a34SFrançois Tigeot }
23471f41f3eSFrançois Tigeot if (mode & EMIT_INVALIDATE) {
235686a02f1SFrançois Tigeot flags |= PIPE_CONTROL_TLB_INVALIDATE;
236e3adcf8fSFrançois Tigeot flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
237e3adcf8fSFrançois Tigeot flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
238e3adcf8fSFrançois Tigeot flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
239e3adcf8fSFrançois Tigeot flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
240e3adcf8fSFrançois Tigeot flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
241686a02f1SFrançois Tigeot /*
242b5c29a34SFrançois Tigeot * TLB invalidate requires a post-sync write.
243686a02f1SFrançois Tigeot */
244b5c29a34SFrançois Tigeot flags |= PIPE_CONTROL_QW_WRITE | PIPE_CONTROL_CS_STALL;
245b5c29a34SFrançois Tigeot }
246e3adcf8fSFrançois Tigeot
247a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 4);
248a85cb24fSFrançois Tigeot if (IS_ERR(cs))
249a85cb24fSFrançois Tigeot return PTR_ERR(cs);
250e3adcf8fSFrançois Tigeot
251a85cb24fSFrançois Tigeot *cs++ = GFX_OP_PIPE_CONTROL(4);
252a85cb24fSFrançois Tigeot *cs++ = flags;
253a85cb24fSFrançois Tigeot *cs++ = scratch_addr | PIPE_CONTROL_GLOBAL_GTT;
254a85cb24fSFrançois Tigeot *cs++ = 0;
255a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
256b5c29a34SFrançois Tigeot
257b5c29a34SFrançois Tigeot return 0;
258b5c29a34SFrançois Tigeot }
259b5c29a34SFrançois Tigeot
260b5c29a34SFrançois Tigeot static int
gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request * req)261a05eeebfSFrançois Tigeot gen7_render_ring_cs_stall_wa(struct drm_i915_gem_request *req)
262b5c29a34SFrançois Tigeot {
263a85cb24fSFrançois Tigeot u32 *cs;
264b5c29a34SFrançois Tigeot
265a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 4);
266a85cb24fSFrançois Tigeot if (IS_ERR(cs))
267a85cb24fSFrançois Tigeot return PTR_ERR(cs);
268b5c29a34SFrançois Tigeot
269a85cb24fSFrançois Tigeot *cs++ = GFX_OP_PIPE_CONTROL(4);
270a85cb24fSFrançois Tigeot *cs++ = PIPE_CONTROL_CS_STALL | PIPE_CONTROL_STALL_AT_SCOREBOARD;
271a85cb24fSFrançois Tigeot *cs++ = 0;
272a85cb24fSFrançois Tigeot *cs++ = 0;
273a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
274b5c29a34SFrançois Tigeot
275b5c29a34SFrançois Tigeot return 0;
276b5c29a34SFrançois Tigeot }
277b5c29a34SFrançois Tigeot
278b5c29a34SFrançois Tigeot static int
gen7_render_ring_flush(struct drm_i915_gem_request * req,u32 mode)27971f41f3eSFrançois Tigeot gen7_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
280b5c29a34SFrançois Tigeot {
28171f41f3eSFrançois Tigeot u32 scratch_addr =
2821e12ee3bSFrançois Tigeot i915_ggtt_offset(req->engine->scratch) + 2 * CACHELINE_BYTES;
283a85cb24fSFrançois Tigeot u32 *cs, flags = 0;
284b5c29a34SFrançois Tigeot
285b5c29a34SFrançois Tigeot /*
286b5c29a34SFrançois Tigeot * Ensure that any following seqno writes only happen when the render
287b5c29a34SFrançois Tigeot * cache is indeed flushed.
288b5c29a34SFrançois Tigeot *
289b5c29a34SFrançois Tigeot * Workaround: 4th PIPE_CONTROL command (except the ones with only
290b5c29a34SFrançois Tigeot * read-cache invalidate bits set) must have the CS_STALL bit set. We
291b5c29a34SFrançois Tigeot * don't try to be clever and just set it unconditionally.
292b5c29a34SFrançois Tigeot */
293b5c29a34SFrançois Tigeot flags |= PIPE_CONTROL_CS_STALL;
294b5c29a34SFrançois Tigeot
295b5c29a34SFrançois Tigeot /* Just flush everything. Experiments have shown that reducing the
296b5c29a34SFrançois Tigeot * number of bits based on the write domains has little performance
297b5c29a34SFrançois Tigeot * impact.
298b5c29a34SFrançois Tigeot */
29971f41f3eSFrançois Tigeot if (mode & EMIT_FLUSH) {
300b5c29a34SFrançois Tigeot flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
301b5c29a34SFrançois Tigeot flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
302aee94f86SFrançois Tigeot flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
303b49c8cf9SFrançois Tigeot flags |= PIPE_CONTROL_FLUSH_ENABLE;
304b5c29a34SFrançois Tigeot }
30571f41f3eSFrançois Tigeot if (mode & EMIT_INVALIDATE) {
306b5c29a34SFrançois Tigeot flags |= PIPE_CONTROL_TLB_INVALIDATE;
307b5c29a34SFrançois Tigeot flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
308b5c29a34SFrançois Tigeot flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
309b5c29a34SFrançois Tigeot flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
310b5c29a34SFrançois Tigeot flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
311b5c29a34SFrançois Tigeot flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
3122c9916cdSFrançois Tigeot flags |= PIPE_CONTROL_MEDIA_STATE_CLEAR;
313b5c29a34SFrançois Tigeot /*
314b5c29a34SFrançois Tigeot * TLB invalidate requires a post-sync write.
315b5c29a34SFrançois Tigeot */
316b5c29a34SFrançois Tigeot flags |= PIPE_CONTROL_QW_WRITE;
317a2fdbec6SFrançois Tigeot flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
318b5c29a34SFrançois Tigeot
3190dbf0ea8SMatthew Dillon flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD;
3200dbf0ea8SMatthew Dillon
321b5c29a34SFrançois Tigeot /* Workaround: we must issue a pipe_control with CS-stall bit
322b5c29a34SFrançois Tigeot * set before a pipe_control command that has the state cache
323b5c29a34SFrançois Tigeot * invalidate bit set. */
324a05eeebfSFrançois Tigeot gen7_render_ring_cs_stall_wa(req);
325b5c29a34SFrançois Tigeot }
326b5c29a34SFrançois Tigeot
327a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 4);
328a85cb24fSFrançois Tigeot if (IS_ERR(cs))
329a85cb24fSFrançois Tigeot return PTR_ERR(cs);
330b5c29a34SFrançois Tigeot
331a85cb24fSFrançois Tigeot *cs++ = GFX_OP_PIPE_CONTROL(4);
332a85cb24fSFrançois Tigeot *cs++ = flags;
333a85cb24fSFrançois Tigeot *cs++ = scratch_addr;
334a85cb24fSFrançois Tigeot *cs++ = 0;
335a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
33624edb884SFrançois Tigeot
33724edb884SFrançois Tigeot return 0;
33824edb884SFrançois Tigeot }
33924edb884SFrançois Tigeot
34024edb884SFrançois Tigeot static int
gen8_render_ring_flush(struct drm_i915_gem_request * req,u32 mode)34171f41f3eSFrançois Tigeot gen8_render_ring_flush(struct drm_i915_gem_request *req, u32 mode)
3429edbd4a0SFrançois Tigeot {
343a85cb24fSFrançois Tigeot u32 flags;
344a85cb24fSFrançois Tigeot u32 *cs;
3459edbd4a0SFrançois Tigeot
346a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, mode & EMIT_INVALIDATE ? 12 : 6);
347a85cb24fSFrançois Tigeot if (IS_ERR(cs))
348a85cb24fSFrançois Tigeot return PTR_ERR(cs);
349a85cb24fSFrançois Tigeot
350a85cb24fSFrançois Tigeot flags = PIPE_CONTROL_CS_STALL;
3519edbd4a0SFrançois Tigeot
35271f41f3eSFrançois Tigeot if (mode & EMIT_FLUSH) {
3539edbd4a0SFrançois Tigeot flags |= PIPE_CONTROL_RENDER_TARGET_CACHE_FLUSH;
3549edbd4a0SFrançois Tigeot flags |= PIPE_CONTROL_DEPTH_CACHE_FLUSH;
355aee94f86SFrançois Tigeot flags |= PIPE_CONTROL_DC_FLUSH_ENABLE;
356b49c8cf9SFrançois Tigeot flags |= PIPE_CONTROL_FLUSH_ENABLE;
3579edbd4a0SFrançois Tigeot }
35871f41f3eSFrançois Tigeot if (mode & EMIT_INVALIDATE) {
3599edbd4a0SFrançois Tigeot flags |= PIPE_CONTROL_TLB_INVALIDATE;
3609edbd4a0SFrançois Tigeot flags |= PIPE_CONTROL_INSTRUCTION_CACHE_INVALIDATE;
3619edbd4a0SFrançois Tigeot flags |= PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE;
3629edbd4a0SFrançois Tigeot flags |= PIPE_CONTROL_VF_CACHE_INVALIDATE;
3639edbd4a0SFrançois Tigeot flags |= PIPE_CONTROL_CONST_CACHE_INVALIDATE;
3649edbd4a0SFrançois Tigeot flags |= PIPE_CONTROL_STATE_CACHE_INVALIDATE;
3659edbd4a0SFrançois Tigeot flags |= PIPE_CONTROL_QW_WRITE;
3669edbd4a0SFrançois Tigeot flags |= PIPE_CONTROL_GLOBAL_GTT_IVB;
3679edbd4a0SFrançois Tigeot
36824edb884SFrançois Tigeot /* WaCsStallBeforeStateCacheInvalidate:bdw,chv */
369a85cb24fSFrançois Tigeot cs = gen8_emit_pipe_control(cs,
37024edb884SFrançois Tigeot PIPE_CONTROL_CS_STALL |
37124edb884SFrançois Tigeot PIPE_CONTROL_STALL_AT_SCOREBOARD,
37224edb884SFrançois Tigeot 0);
37324edb884SFrançois Tigeot }
3749edbd4a0SFrançois Tigeot
375a85cb24fSFrançois Tigeot cs = gen8_emit_pipe_control(cs, flags,
376a85cb24fSFrançois Tigeot i915_ggtt_offset(req->engine->scratch) +
377a85cb24fSFrançois Tigeot 2 * CACHELINE_BYTES);
378a85cb24fSFrançois Tigeot
379a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
380a85cb24fSFrançois Tigeot
381a85cb24fSFrançois Tigeot return 0;
3829edbd4a0SFrançois Tigeot }
3839edbd4a0SFrançois Tigeot
ring_setup_phys_status_page(struct intel_engine_cs * engine)3848621f407SFrançois Tigeot static void ring_setup_phys_status_page(struct intel_engine_cs *engine)
3855d0b1887SFrançois Tigeot {
3861487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
3875d0b1887SFrançois Tigeot u32 addr;
3885d0b1887SFrançois Tigeot
3895d0b1887SFrançois Tigeot addr = dev_priv->status_page_dmah->busaddr;
3901487f786SFrançois Tigeot if (INTEL_GEN(dev_priv) >= 4)
3915d0b1887SFrançois Tigeot addr |= (dev_priv->status_page_dmah->busaddr >> 28) & 0xf0;
3925d0b1887SFrançois Tigeot I915_WRITE(HWS_PGA, addr);
3935d0b1887SFrançois Tigeot }
3945d0b1887SFrançois Tigeot
intel_ring_setup_status_page(struct intel_engine_cs * engine)3958621f407SFrançois Tigeot static void intel_ring_setup_status_page(struct intel_engine_cs *engine)
396477eb7f9SFrançois Tigeot {
3971487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
398aee94f86SFrançois Tigeot i915_reg_t mmio;
399477eb7f9SFrançois Tigeot
400477eb7f9SFrançois Tigeot /* The ring status page addresses are no longer next to the rest of
401477eb7f9SFrançois Tigeot * the ring registers as of gen7.
402477eb7f9SFrançois Tigeot */
4031487f786SFrançois Tigeot if (IS_GEN7(dev_priv)) {
4048621f407SFrançois Tigeot switch (engine->id) {
405*3f2dd94aSFrançois Tigeot /*
406*3f2dd94aSFrançois Tigeot * No more rings exist on Gen7. Default case is only to shut up
407*3f2dd94aSFrançois Tigeot * gcc switch check warning.
408*3f2dd94aSFrançois Tigeot */
409*3f2dd94aSFrançois Tigeot default:
410*3f2dd94aSFrançois Tigeot GEM_BUG_ON(engine->id);
411477eb7f9SFrançois Tigeot case RCS:
412477eb7f9SFrançois Tigeot mmio = RENDER_HWS_PGA_GEN7;
413477eb7f9SFrançois Tigeot break;
414477eb7f9SFrançois Tigeot case BCS:
415477eb7f9SFrançois Tigeot mmio = BLT_HWS_PGA_GEN7;
416477eb7f9SFrançois Tigeot break;
417477eb7f9SFrançois Tigeot case VCS:
418477eb7f9SFrançois Tigeot mmio = BSD_HWS_PGA_GEN7;
419477eb7f9SFrançois Tigeot break;
420477eb7f9SFrançois Tigeot case VECS:
421477eb7f9SFrançois Tigeot mmio = VEBOX_HWS_PGA_GEN7;
422477eb7f9SFrançois Tigeot break;
423477eb7f9SFrançois Tigeot }
4241487f786SFrançois Tigeot } else if (IS_GEN6(dev_priv)) {
4258621f407SFrançois Tigeot mmio = RING_HWS_PGA_GEN6(engine->mmio_base);
426477eb7f9SFrançois Tigeot } else {
427477eb7f9SFrançois Tigeot /* XXX: gen8 returns to sanity */
4288621f407SFrançois Tigeot mmio = RING_HWS_PGA(engine->mmio_base);
429477eb7f9SFrançois Tigeot }
430477eb7f9SFrançois Tigeot
431*3f2dd94aSFrançois Tigeot if (INTEL_GEN(dev_priv) >= 6)
432*3f2dd94aSFrançois Tigeot I915_WRITE(RING_HWSTAM(engine->mmio_base), 0xffffffff);
433*3f2dd94aSFrançois Tigeot
4341e12ee3bSFrançois Tigeot I915_WRITE(mmio, engine->status_page.ggtt_offset);
435477eb7f9SFrançois Tigeot POSTING_READ(mmio);
436477eb7f9SFrançois Tigeot
437477eb7f9SFrançois Tigeot /*
438477eb7f9SFrançois Tigeot * Flush the TLB for this page
439477eb7f9SFrançois Tigeot *
440477eb7f9SFrançois Tigeot * FIXME: These two bits have disappeared on gen8, so a question
441477eb7f9SFrançois Tigeot * arises: do we still need this and if so how should we go about
442477eb7f9SFrançois Tigeot * invalidating the TLB?
443477eb7f9SFrançois Tigeot */
4441487f786SFrançois Tigeot if (IS_GEN(dev_priv, 6, 7)) {
4458621f407SFrançois Tigeot i915_reg_t reg = RING_INSTPM(engine->mmio_base);
446477eb7f9SFrançois Tigeot
447477eb7f9SFrançois Tigeot /* ring should be idle before issuing a sync flush*/
4488621f407SFrançois Tigeot WARN_ON((I915_READ_MODE(engine) & MODE_IDLE) == 0);
449477eb7f9SFrançois Tigeot
450477eb7f9SFrançois Tigeot I915_WRITE(reg,
451477eb7f9SFrançois Tigeot _MASKED_BIT_ENABLE(INSTPM_TLB_INVALIDATE |
452477eb7f9SFrançois Tigeot INSTPM_SYNC_FLUSH));
4531487f786SFrançois Tigeot if (intel_wait_for_register(dev_priv,
4541487f786SFrançois Tigeot reg, INSTPM_SYNC_FLUSH, 0,
455477eb7f9SFrançois Tigeot 1000))
456477eb7f9SFrançois Tigeot DRM_ERROR("%s: wait for SyncFlush to complete for TLB invalidation timed out\n",
4578621f407SFrançois Tigeot engine->name);
458477eb7f9SFrançois Tigeot }
459477eb7f9SFrançois Tigeot }
460477eb7f9SFrançois Tigeot
stop_ring(struct intel_engine_cs * engine)4618621f407SFrançois Tigeot static bool stop_ring(struct intel_engine_cs *engine)
462e3adcf8fSFrançois Tigeot {
4631487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
464e3adcf8fSFrançois Tigeot
4651e12ee3bSFrançois Tigeot if (INTEL_GEN(dev_priv) > 2) {
4668621f407SFrançois Tigeot I915_WRITE_MODE(engine, _MASKED_BIT_ENABLE(STOP_RING));
4671487f786SFrançois Tigeot if (intel_wait_for_register(dev_priv,
4681487f786SFrançois Tigeot RING_MI_MODE(engine->mmio_base),
4691487f786SFrançois Tigeot MODE_IDLE,
4701487f786SFrançois Tigeot MODE_IDLE,
4711487f786SFrançois Tigeot 1000)) {
4728621f407SFrançois Tigeot DRM_ERROR("%s : timed out trying to stop ring\n",
4738621f407SFrançois Tigeot engine->name);
4741b13d190SFrançois Tigeot /* Sometimes we observe that the idle flag is not
4751b13d190SFrançois Tigeot * set even though the ring is empty. So double
4761b13d190SFrançois Tigeot * check before giving up.
4771b13d190SFrançois Tigeot */
4788621f407SFrançois Tigeot if (I915_READ_HEAD(engine) != I915_READ_TAIL(engine))
479ba55f2f5SFrançois Tigeot return false;
480ba55f2f5SFrançois Tigeot }
481ba55f2f5SFrançois Tigeot }
482686a02f1SFrançois Tigeot
4838621f407SFrançois Tigeot I915_WRITE_CTL(engine, 0);
4848621f407SFrançois Tigeot I915_WRITE_HEAD(engine, 0);
48571f41f3eSFrançois Tigeot I915_WRITE_TAIL(engine, 0);
486e3adcf8fSFrançois Tigeot
4878621f407SFrançois Tigeot return (I915_READ_HEAD(engine) & HEAD_ADDR) == 0;
488ba55f2f5SFrançois Tigeot }
489ba55f2f5SFrançois Tigeot
init_ring_common(struct intel_engine_cs * engine)4908621f407SFrançois Tigeot static int init_ring_common(struct intel_engine_cs *engine)
4918621f407SFrançois Tigeot {
4921487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
49371f41f3eSFrançois Tigeot struct intel_ring *ring = engine->buffer;
494ba55f2f5SFrançois Tigeot int ret = 0;
495ba55f2f5SFrançois Tigeot
4962c9916cdSFrançois Tigeot intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
497ba55f2f5SFrançois Tigeot
4988621f407SFrançois Tigeot if (!stop_ring(engine)) {
499ba55f2f5SFrançois Tigeot /* G45 ring initialization often fails to reset head to zero */
500b5c29a34SFrançois Tigeot DRM_DEBUG_KMS("%s head not reset to zero "
501e3adcf8fSFrançois Tigeot "ctl %08x head %08x tail %08x start %08x\n",
5028621f407SFrançois Tigeot engine->name,
5038621f407SFrançois Tigeot I915_READ_CTL(engine),
5048621f407SFrançois Tigeot I915_READ_HEAD(engine),
5058621f407SFrançois Tigeot I915_READ_TAIL(engine),
5068621f407SFrançois Tigeot I915_READ_START(engine));
507e3adcf8fSFrançois Tigeot
5088621f407SFrançois Tigeot if (!stop_ring(engine)) {
509e3adcf8fSFrançois Tigeot DRM_ERROR("failed to set %s head to zero "
510e3adcf8fSFrançois Tigeot "ctl %08x head %08x tail %08x start %08x\n",
5118621f407SFrançois Tigeot engine->name,
5128621f407SFrançois Tigeot I915_READ_CTL(engine),
5138621f407SFrançois Tigeot I915_READ_HEAD(engine),
5148621f407SFrançois Tigeot I915_READ_TAIL(engine),
5158621f407SFrançois Tigeot I915_READ_START(engine));
516686a02f1SFrançois Tigeot ret = -EIO;
517686a02f1SFrançois Tigeot goto out;
518e3adcf8fSFrançois Tigeot }
519ba55f2f5SFrançois Tigeot }
520ba55f2f5SFrançois Tigeot
5211e12ee3bSFrançois Tigeot if (HWS_NEEDS_PHYSICAL(dev_priv))
5228621f407SFrançois Tigeot ring_setup_phys_status_page(engine);
5231e12ee3bSFrançois Tigeot else
5241e12ee3bSFrançois Tigeot intel_ring_setup_status_page(engine);
5251e12ee3bSFrançois Tigeot
5261e12ee3bSFrançois Tigeot intel_engine_reset_breadcrumbs(engine);
527ba55f2f5SFrançois Tigeot
5280f370975SMatthew Dillon /* Enforce ordering by reading HEAD register back */
5298621f407SFrançois Tigeot I915_READ_HEAD(engine);
5300f370975SMatthew Dillon
531ba55f2f5SFrançois Tigeot /* Initialize the ring. This must happen _after_ we've cleared the ring
532ba55f2f5SFrançois Tigeot * registers with the above sequence (the readback of the HEAD registers
533ba55f2f5SFrançois Tigeot * also enforces ordering), otherwise the hw might lose the new ring
534ba55f2f5SFrançois Tigeot * register values. */
5351e12ee3bSFrançois Tigeot I915_WRITE_START(engine, i915_ggtt_offset(ring->vma));
5361b13d190SFrançois Tigeot
5371b13d190SFrançois Tigeot /* WaClearRingBufHeadRegAtInit:ctg,elk */
5388621f407SFrançois Tigeot if (I915_READ_HEAD(engine))
5391b13d190SFrançois Tigeot DRM_DEBUG("%s initialization failed [head=%08x], fudging\n",
5408621f407SFrançois Tigeot engine->name, I915_READ_HEAD(engine));
5411b13d190SFrançois Tigeot
5421e12ee3bSFrançois Tigeot intel_ring_update_space(ring);
5431e12ee3bSFrançois Tigeot I915_WRITE_HEAD(engine, ring->head);
5441e12ee3bSFrançois Tigeot I915_WRITE_TAIL(engine, ring->tail);
5451e12ee3bSFrançois Tigeot (void)I915_READ_TAIL(engine);
5461e12ee3bSFrançois Tigeot
5471e12ee3bSFrançois Tigeot I915_WRITE_CTL(engine, RING_CTL_SIZE(ring->size) | RING_VALID);
548ba55f2f5SFrançois Tigeot
549ba55f2f5SFrançois Tigeot /* If the head is still not zero, the ring is dead */
550*3f2dd94aSFrançois Tigeot if (intel_wait_for_register(dev_priv, RING_CTL(engine->mmio_base),
5511e12ee3bSFrançois Tigeot RING_VALID, RING_VALID,
5521e12ee3bSFrançois Tigeot 50)) {
553ba55f2f5SFrançois Tigeot DRM_ERROR("%s initialization failed "
5541e12ee3bSFrançois Tigeot "ctl %08x (valid? %d) head %08x [%08x] tail %08x [%08x] start %08x [expected %08x]\n",
5558621f407SFrançois Tigeot engine->name,
5568621f407SFrançois Tigeot I915_READ_CTL(engine),
5578621f407SFrançois Tigeot I915_READ_CTL(engine) & RING_VALID,
5581e12ee3bSFrançois Tigeot I915_READ_HEAD(engine), ring->head,
5591e12ee3bSFrançois Tigeot I915_READ_TAIL(engine), ring->tail,
5608621f407SFrançois Tigeot I915_READ_START(engine),
5611e12ee3bSFrançois Tigeot i915_ggtt_offset(ring->vma));
562ba55f2f5SFrançois Tigeot ret = -EIO;
563ba55f2f5SFrançois Tigeot goto out;
564ba55f2f5SFrançois Tigeot }
565e3adcf8fSFrançois Tigeot
5668621f407SFrançois Tigeot intel_engine_init_hangcheck(engine);
5675d0b1887SFrançois Tigeot
568*3f2dd94aSFrançois Tigeot if (INTEL_GEN(dev_priv) > 2)
569*3f2dd94aSFrançois Tigeot I915_WRITE_MODE(engine, _MASKED_BIT_DISABLE(STOP_RING));
570*3f2dd94aSFrançois Tigeot
571686a02f1SFrançois Tigeot out:
5722c9916cdSFrançois Tigeot intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
573686a02f1SFrançois Tigeot
574686a02f1SFrançois Tigeot return ret;
575e3adcf8fSFrançois Tigeot }
576e3adcf8fSFrançois Tigeot
reset_ring_common(struct intel_engine_cs * engine,struct drm_i915_gem_request * request)5771e12ee3bSFrançois Tigeot static void reset_ring_common(struct intel_engine_cs *engine,
5781e12ee3bSFrançois Tigeot struct drm_i915_gem_request *request)
5791b13d190SFrançois Tigeot {
580*3f2dd94aSFrançois Tigeot /*
581*3f2dd94aSFrançois Tigeot * RC6 must be prevented until the reset is complete and the engine
582*3f2dd94aSFrançois Tigeot * reinitialised. If it occurs in the middle of this sequence, the
583*3f2dd94aSFrançois Tigeot * state written to/loaded from the power context is ill-defined (e.g.
584*3f2dd94aSFrançois Tigeot * the PP_BASE_DIR may be lost).
585*3f2dd94aSFrançois Tigeot */
586*3f2dd94aSFrançois Tigeot assert_forcewakes_active(engine->i915, FORCEWAKE_ALL);
587*3f2dd94aSFrançois Tigeot
588*3f2dd94aSFrançois Tigeot /*
589*3f2dd94aSFrançois Tigeot * Try to restore the logical GPU state to match the continuation
590a85cb24fSFrançois Tigeot * of the request queue. If we skip the context/PD restore, then
591a85cb24fSFrançois Tigeot * the next request may try to execute assuming that its context
592a85cb24fSFrançois Tigeot * is valid and loaded on the GPU and so may try to access invalid
593a85cb24fSFrançois Tigeot * memory, prompting repeated GPU hangs.
594a85cb24fSFrançois Tigeot *
595a85cb24fSFrançois Tigeot * If the request was guilty, we still restore the logical state
596a85cb24fSFrançois Tigeot * in case the next request requires it (e.g. the aliasing ppgtt),
597a85cb24fSFrançois Tigeot * but skip over the hung batch.
598a85cb24fSFrançois Tigeot *
599a85cb24fSFrançois Tigeot * If the request was innocent, we try to replay the request with
600a85cb24fSFrançois Tigeot * the restored context.
601a85cb24fSFrançois Tigeot */
602a85cb24fSFrançois Tigeot if (request) {
603a85cb24fSFrançois Tigeot struct drm_i915_private *dev_priv = request->i915;
604a85cb24fSFrançois Tigeot struct intel_context *ce = &request->ctx->engine[engine->id];
605a85cb24fSFrançois Tigeot struct i915_hw_ppgtt *ppgtt;
6061b13d190SFrançois Tigeot
607a85cb24fSFrançois Tigeot /* FIXME consider gen8 reset */
608a85cb24fSFrançois Tigeot
609a85cb24fSFrançois Tigeot if (ce->state) {
610a85cb24fSFrançois Tigeot I915_WRITE(CCID,
611a85cb24fSFrançois Tigeot i915_ggtt_offset(ce->state) |
612a85cb24fSFrançois Tigeot BIT(8) /* must be set! */ |
613a85cb24fSFrançois Tigeot CCID_EXTENDED_STATE_SAVE |
614a85cb24fSFrançois Tigeot CCID_EXTENDED_STATE_RESTORE |
615a85cb24fSFrançois Tigeot CCID_EN);
616e3adcf8fSFrançois Tigeot }
617e3adcf8fSFrançois Tigeot
618a85cb24fSFrançois Tigeot ppgtt = request->ctx->ppgtt ?: engine->i915->mm.aliasing_ppgtt;
619a85cb24fSFrançois Tigeot if (ppgtt) {
620a85cb24fSFrançois Tigeot u32 pd_offset = ppgtt->pd.base.ggtt_offset << 10;
6211b13d190SFrançois Tigeot
622a85cb24fSFrançois Tigeot I915_WRITE(RING_PP_DIR_DCLV(engine), PP_DIR_DCLV_2G);
623a85cb24fSFrançois Tigeot I915_WRITE(RING_PP_DIR_BASE(engine), pd_offset);
6241b13d190SFrançois Tigeot
625a85cb24fSFrançois Tigeot /* Wait for the PD reload to complete */
626a85cb24fSFrançois Tigeot if (intel_wait_for_register(dev_priv,
627a85cb24fSFrançois Tigeot RING_PP_DIR_BASE(engine),
628a85cb24fSFrançois Tigeot BIT(0), 0,
629a85cb24fSFrançois Tigeot 10))
630a85cb24fSFrançois Tigeot DRM_ERROR("Wait for reload of ppgtt page-directory timed out\n");
6311b13d190SFrançois Tigeot
632a85cb24fSFrançois Tigeot ppgtt->pd_dirty_rings &= ~intel_engine_flag(engine);
6332c9916cdSFrançois Tigeot }
6342c9916cdSFrançois Tigeot
635a85cb24fSFrançois Tigeot /* If the rq hung, jump to its breadcrumb and skip the batch */
636a85cb24fSFrançois Tigeot if (request->fence.error == -EIO)
637a85cb24fSFrançois Tigeot request->ring->head = request->postfix;
638a85cb24fSFrançois Tigeot } else {
639a85cb24fSFrançois Tigeot engine->legacy_active_context = NULL;
640a85cb24fSFrançois Tigeot }
6412c9916cdSFrançois Tigeot }
6422c9916cdSFrançois Tigeot
intel_rcs_ctx_init(struct drm_i915_gem_request * req)643a05eeebfSFrançois Tigeot static int intel_rcs_ctx_init(struct drm_i915_gem_request *req)
6442c9916cdSFrançois Tigeot {
6452c9916cdSFrançois Tigeot int ret;
6462c9916cdSFrançois Tigeot
647a05eeebfSFrançois Tigeot ret = intel_ring_workarounds_emit(req);
6482c9916cdSFrançois Tigeot if (ret != 0)
6492c9916cdSFrançois Tigeot return ret;
6502c9916cdSFrançois Tigeot
6514be47400SFrançois Tigeot ret = i915_gem_render_state_emit(req);
6522c9916cdSFrançois Tigeot if (ret)
6532c9916cdSFrançois Tigeot return ret;
654c0e85e96SFrançois Tigeot
655c0e85e96SFrançois Tigeot return 0;
6562c9916cdSFrançois Tigeot }
6572c9916cdSFrançois Tigeot
init_render_ring(struct intel_engine_cs * engine)6588621f407SFrançois Tigeot static int init_render_ring(struct intel_engine_cs *engine)
659e3adcf8fSFrançois Tigeot {
6601487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
6618621f407SFrançois Tigeot int ret = init_ring_common(engine);
66224edb884SFrançois Tigeot if (ret)
66324edb884SFrançois Tigeot return ret;
664e3adcf8fSFrançois Tigeot
665ba55f2f5SFrançois Tigeot /* WaTimedSingleVertexDispatch:cl,bw,ctg,elk,ilk,snb */
6661487f786SFrançois Tigeot if (IS_GEN(dev_priv, 4, 6))
667f4e1c372SFrançois Tigeot I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(VS_TIMER_DISPATCH));
668f4e1c372SFrançois Tigeot
669f4e1c372SFrançois Tigeot /* We need to disable the AsyncFlip performance optimisations in order
670f4e1c372SFrançois Tigeot * to use MI_WAIT_FOR_EVENT within the CS. It should already be
671f4e1c372SFrançois Tigeot * programmed to '1' on all products.
6725d0b1887SFrançois Tigeot *
673a05eeebfSFrançois Tigeot * WaDisableAsyncFlipPerfMode:snb,ivb,hsw,vlv
674f4e1c372SFrançois Tigeot */
6751487f786SFrançois Tigeot if (IS_GEN(dev_priv, 6, 7))
676f4e1c372SFrançois Tigeot I915_WRITE(MI_MODE, _MASKED_BIT_ENABLE(ASYNC_FLIP_PERF_DISABLE));
677f4e1c372SFrançois Tigeot
678f4e1c372SFrançois Tigeot /* Required for the hardware to program scanline values for waiting */
679ba55f2f5SFrançois Tigeot /* WaEnableFlushTlbInvalidationMode:snb */
6801487f786SFrançois Tigeot if (IS_GEN6(dev_priv))
681f4e1c372SFrançois Tigeot I915_WRITE(GFX_MODE,
682ba55f2f5SFrançois Tigeot _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT));
683f4e1c372SFrançois Tigeot
684ba55f2f5SFrançois Tigeot /* WaBCSVCSTlbInvalidationMode:ivb,vlv,hsw */
6851487f786SFrançois Tigeot if (IS_GEN7(dev_priv))
686e3adcf8fSFrançois Tigeot I915_WRITE(GFX_MODE_GEN7,
687ba55f2f5SFrançois Tigeot _MASKED_BIT_ENABLE(GFX_TLB_INVALIDATE_EXPLICIT) |
688f4e1c372SFrançois Tigeot _MASKED_BIT_ENABLE(GFX_REPLAY_MODE));
689e3adcf8fSFrançois Tigeot
6901487f786SFrançois Tigeot if (IS_GEN6(dev_priv)) {
691e3adcf8fSFrançois Tigeot /* From the Sandybridge PRM, volume 1 part 3, page 24:
692e3adcf8fSFrançois Tigeot * "If this bit is set, STCunit will have LRA as replacement
693e3adcf8fSFrançois Tigeot * policy. [...] This bit must be reset. LRA replacement
694e3adcf8fSFrançois Tigeot * policy is not supported."
695e3adcf8fSFrançois Tigeot */
696e3adcf8fSFrançois Tigeot I915_WRITE(CACHE_MODE_0,
697f4e1c372SFrançois Tigeot _MASKED_BIT_DISABLE(CM0_STC_EVICT_DISABLE_LRA_SNB));
698e3adcf8fSFrançois Tigeot }
699e3adcf8fSFrançois Tigeot
7001487f786SFrançois Tigeot if (IS_GEN(dev_priv, 6, 7))
701f4e1c372SFrançois Tigeot I915_WRITE(INSTPM, _MASKED_BIT_ENABLE(INSTPM_FORCE_ORDERING));
702f4e1c372SFrançois Tigeot
703303bf270SFrançois Tigeot if (INTEL_INFO(dev_priv)->gen >= 6)
704303bf270SFrançois Tigeot I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
705e3adcf8fSFrançois Tigeot
7068621f407SFrançois Tigeot return init_workarounds_ring(engine);
707e3adcf8fSFrançois Tigeot }
708e3adcf8fSFrançois Tigeot
render_ring_cleanup(struct intel_engine_cs * engine)7098621f407SFrançois Tigeot static void render_ring_cleanup(struct intel_engine_cs *engine)
710e3adcf8fSFrançois Tigeot {
7111487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
71224edb884SFrançois Tigeot
7131e12ee3bSFrançois Tigeot i915_vma_unpin_and_release(&dev_priv->semaphore);
714e3adcf8fSFrançois Tigeot }
715e3adcf8fSFrançois Tigeot
gen8_rcs_signal(struct drm_i915_gem_request * req,u32 * cs)716a85cb24fSFrançois Tigeot static u32 *gen8_rcs_signal(struct drm_i915_gem_request *req, u32 *cs)
71724edb884SFrançois Tigeot {
71871f41f3eSFrançois Tigeot struct drm_i915_private *dev_priv = req->i915;
71924edb884SFrançois Tigeot struct intel_engine_cs *waiter;
7208621f407SFrançois Tigeot enum intel_engine_id id;
72124edb884SFrançois Tigeot
7221e12ee3bSFrançois Tigeot for_each_engine(waiter, dev_priv, id) {
72371f41f3eSFrançois Tigeot u64 gtt_offset = req->engine->semaphore.signal_ggtt[id];
72424edb884SFrançois Tigeot if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
72524edb884SFrançois Tigeot continue;
72624edb884SFrançois Tigeot
727a85cb24fSFrançois Tigeot *cs++ = GFX_OP_PIPE_CONTROL(6);
728a85cb24fSFrançois Tigeot *cs++ = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_QW_WRITE |
729a85cb24fSFrançois Tigeot PIPE_CONTROL_CS_STALL;
730a85cb24fSFrançois Tigeot *cs++ = lower_32_bits(gtt_offset);
731a85cb24fSFrançois Tigeot *cs++ = upper_32_bits(gtt_offset);
732a85cb24fSFrançois Tigeot *cs++ = req->global_seqno;
733a85cb24fSFrançois Tigeot *cs++ = 0;
734a85cb24fSFrançois Tigeot *cs++ = MI_SEMAPHORE_SIGNAL |
735a85cb24fSFrançois Tigeot MI_SEMAPHORE_TARGET(waiter->hw_id);
736a85cb24fSFrançois Tigeot *cs++ = 0;
73724edb884SFrançois Tigeot }
73824edb884SFrançois Tigeot
739a85cb24fSFrançois Tigeot return cs;
7404be47400SFrançois Tigeot }
7414be47400SFrançois Tigeot
gen8_xcs_signal(struct drm_i915_gem_request * req,u32 * cs)742a85cb24fSFrançois Tigeot static u32 *gen8_xcs_signal(struct drm_i915_gem_request *req, u32 *cs)
74324edb884SFrançois Tigeot {
74471f41f3eSFrançois Tigeot struct drm_i915_private *dev_priv = req->i915;
74524edb884SFrançois Tigeot struct intel_engine_cs *waiter;
7468621f407SFrançois Tigeot enum intel_engine_id id;
74724edb884SFrançois Tigeot
7481e12ee3bSFrançois Tigeot for_each_engine(waiter, dev_priv, id) {
74971f41f3eSFrançois Tigeot u64 gtt_offset = req->engine->semaphore.signal_ggtt[id];
75024edb884SFrançois Tigeot if (gtt_offset == MI_SEMAPHORE_SYNC_INVALID)
75124edb884SFrançois Tigeot continue;
75224edb884SFrançois Tigeot
753a85cb24fSFrançois Tigeot *cs++ = (MI_FLUSH_DW + 1) | MI_FLUSH_DW_OP_STOREDW;
754a85cb24fSFrançois Tigeot *cs++ = lower_32_bits(gtt_offset) | MI_FLUSH_DW_USE_GTT;
755a85cb24fSFrançois Tigeot *cs++ = upper_32_bits(gtt_offset);
756a85cb24fSFrançois Tigeot *cs++ = req->global_seqno;
757a85cb24fSFrançois Tigeot *cs++ = MI_SEMAPHORE_SIGNAL |
758a85cb24fSFrançois Tigeot MI_SEMAPHORE_TARGET(waiter->hw_id);
759a85cb24fSFrançois Tigeot *cs++ = 0;
76024edb884SFrançois Tigeot }
76124edb884SFrançois Tigeot
762a85cb24fSFrançois Tigeot return cs;
7634be47400SFrançois Tigeot }
7644be47400SFrançois Tigeot
gen6_signal(struct drm_i915_gem_request * req,u32 * cs)765a85cb24fSFrançois Tigeot static u32 *gen6_signal(struct drm_i915_gem_request *req, u32 *cs)
766e3adcf8fSFrançois Tigeot {
76771f41f3eSFrançois Tigeot struct drm_i915_private *dev_priv = req->i915;
7681e12ee3bSFrançois Tigeot struct intel_engine_cs *engine;
7698621f407SFrançois Tigeot enum intel_engine_id id;
7704be47400SFrançois Tigeot int num_rings = 0;
771ba55f2f5SFrançois Tigeot
7721e12ee3bSFrançois Tigeot for_each_engine(engine, dev_priv, id) {
7731e12ee3bSFrançois Tigeot i915_reg_t mbox_reg;
774aee94f86SFrançois Tigeot
7751e12ee3bSFrançois Tigeot if (!(BIT(engine->hw_id) & GEN6_SEMAPHORES_MASK))
7761e12ee3bSFrançois Tigeot continue;
7771e12ee3bSFrançois Tigeot
7781e12ee3bSFrançois Tigeot mbox_reg = req->engine->semaphore.mbox.signal[engine->hw_id];
779aee94f86SFrançois Tigeot if (i915_mmio_reg_valid(mbox_reg)) {
780a85cb24fSFrançois Tigeot *cs++ = MI_LOAD_REGISTER_IMM(1);
781a85cb24fSFrançois Tigeot *cs++ = i915_mmio_reg_offset(mbox_reg);
782a85cb24fSFrançois Tigeot *cs++ = req->global_seqno;
7834be47400SFrançois Tigeot num_rings++;
784ba55f2f5SFrançois Tigeot }
785ba55f2f5SFrançois Tigeot }
7864be47400SFrançois Tigeot if (num_rings & 1)
787a85cb24fSFrançois Tigeot *cs++ = MI_NOOP;
788ba55f2f5SFrançois Tigeot
789a85cb24fSFrançois Tigeot return cs;
79071f41f3eSFrançois Tigeot }
79171f41f3eSFrançois Tigeot
cancel_requests(struct intel_engine_cs * engine)792*3f2dd94aSFrançois Tigeot static void cancel_requests(struct intel_engine_cs *engine)
793*3f2dd94aSFrançois Tigeot {
794*3f2dd94aSFrançois Tigeot struct drm_i915_gem_request *request;
795*3f2dd94aSFrançois Tigeot unsigned long flags;
796*3f2dd94aSFrançois Tigeot
797*3f2dd94aSFrançois Tigeot spin_lock_irqsave(&engine->timeline->lock, flags);
798*3f2dd94aSFrançois Tigeot
799*3f2dd94aSFrançois Tigeot /* Mark all submitted requests as skipped. */
800*3f2dd94aSFrançois Tigeot list_for_each_entry(request, &engine->timeline->requests, link) {
801*3f2dd94aSFrançois Tigeot GEM_BUG_ON(!request->global_seqno);
802*3f2dd94aSFrançois Tigeot if (!i915_gem_request_completed(request))
803*3f2dd94aSFrançois Tigeot dma_fence_set_error(&request->fence, -EIO);
804*3f2dd94aSFrançois Tigeot }
805*3f2dd94aSFrançois Tigeot /* Remaining _unready_ requests will be nop'ed when submitted */
806*3f2dd94aSFrançois Tigeot
807*3f2dd94aSFrançois Tigeot spin_unlock_irqrestore(&engine->timeline->lock, flags);
808*3f2dd94aSFrançois Tigeot }
809*3f2dd94aSFrançois Tigeot
i9xx_submit_request(struct drm_i915_gem_request * request)81071f41f3eSFrançois Tigeot static void i9xx_submit_request(struct drm_i915_gem_request *request)
81171f41f3eSFrançois Tigeot {
81271f41f3eSFrançois Tigeot struct drm_i915_private *dev_priv = request->i915;
81371f41f3eSFrançois Tigeot
8144be47400SFrançois Tigeot i915_gem_request_submit(request);
8154be47400SFrançois Tigeot
816a85cb24fSFrançois Tigeot I915_WRITE_TAIL(request->engine,
817a85cb24fSFrançois Tigeot intel_ring_set_tail(request->ring, request->tail));
81871f41f3eSFrançois Tigeot }
81971f41f3eSFrançois Tigeot
i9xx_emit_breadcrumb(struct drm_i915_gem_request * req,u32 * cs)820a85cb24fSFrançois Tigeot static void i9xx_emit_breadcrumb(struct drm_i915_gem_request *req, u32 *cs)
82171f41f3eSFrançois Tigeot {
822a85cb24fSFrançois Tigeot *cs++ = MI_STORE_DWORD_INDEX;
823a85cb24fSFrançois Tigeot *cs++ = I915_GEM_HWS_INDEX << MI_STORE_DWORD_INDEX_SHIFT;
824a85cb24fSFrançois Tigeot *cs++ = req->global_seqno;
825a85cb24fSFrançois Tigeot *cs++ = MI_USER_INTERRUPT;
82671f41f3eSFrançois Tigeot
827a85cb24fSFrançois Tigeot req->tail = intel_ring_offset(req, cs);
828a85cb24fSFrançois Tigeot assert_ring_tail_valid(req->ring, req->tail);
829e3adcf8fSFrançois Tigeot }
830e3adcf8fSFrançois Tigeot
8314be47400SFrançois Tigeot static const int i9xx_emit_breadcrumb_sz = 4;
8324be47400SFrançois Tigeot
833e3adcf8fSFrançois Tigeot /**
8344be47400SFrançois Tigeot * gen6_sema_emit_breadcrumb - Update the semaphore mailbox registers
835e3adcf8fSFrançois Tigeot *
836a05eeebfSFrançois Tigeot * @request - request to write to the ring
837e3adcf8fSFrançois Tigeot *
838e3adcf8fSFrançois Tigeot * Update the mailbox registers in the *other* rings with the current seqno.
839e3adcf8fSFrançois Tigeot * This acts like a signal in the canonical semaphore.
840e3adcf8fSFrançois Tigeot */
gen6_sema_emit_breadcrumb(struct drm_i915_gem_request * req,u32 * cs)841a85cb24fSFrançois Tigeot static void gen6_sema_emit_breadcrumb(struct drm_i915_gem_request *req, u32 *cs)
842e3adcf8fSFrançois Tigeot {
8434be47400SFrançois Tigeot return i9xx_emit_breadcrumb(req,
844a85cb24fSFrançois Tigeot req->engine->semaphore.signal(req, cs));
845e3adcf8fSFrançois Tigeot }
846e3adcf8fSFrançois Tigeot
gen8_render_emit_breadcrumb(struct drm_i915_gem_request * req,u32 * cs)8474be47400SFrançois Tigeot static void gen8_render_emit_breadcrumb(struct drm_i915_gem_request *req,
848a85cb24fSFrançois Tigeot u32 *cs)
8491487f786SFrançois Tigeot {
8501487f786SFrançois Tigeot struct intel_engine_cs *engine = req->engine;
8511487f786SFrançois Tigeot
8524be47400SFrançois Tigeot if (engine->semaphore.signal)
853a85cb24fSFrançois Tigeot cs = engine->semaphore.signal(req, cs);
85471f41f3eSFrançois Tigeot
855a85cb24fSFrançois Tigeot *cs++ = GFX_OP_PIPE_CONTROL(6);
856a85cb24fSFrançois Tigeot *cs++ = PIPE_CONTROL_GLOBAL_GTT_IVB | PIPE_CONTROL_CS_STALL |
857a85cb24fSFrançois Tigeot PIPE_CONTROL_QW_WRITE;
858a85cb24fSFrançois Tigeot *cs++ = intel_hws_seqno_address(engine);
859a85cb24fSFrançois Tigeot *cs++ = 0;
860a85cb24fSFrançois Tigeot *cs++ = req->global_seqno;
8611487f786SFrançois Tigeot /* We're thrashing one dword of HWS. */
862a85cb24fSFrançois Tigeot *cs++ = 0;
863a85cb24fSFrançois Tigeot *cs++ = MI_USER_INTERRUPT;
864a85cb24fSFrançois Tigeot *cs++ = MI_NOOP;
86571f41f3eSFrançois Tigeot
866a85cb24fSFrançois Tigeot req->tail = intel_ring_offset(req, cs);
867a85cb24fSFrançois Tigeot assert_ring_tail_valid(req->ring, req->tail);
8681487f786SFrançois Tigeot }
8691487f786SFrançois Tigeot
8704be47400SFrançois Tigeot static const int gen8_render_emit_breadcrumb_sz = 8;
8714be47400SFrançois Tigeot
872e3adcf8fSFrançois Tigeot /**
873e3adcf8fSFrançois Tigeot * intel_ring_sync - sync the waiter to the signaller on seqno
874e3adcf8fSFrançois Tigeot *
875e3adcf8fSFrançois Tigeot * @waiter - ring that is waiting
876e3adcf8fSFrançois Tigeot * @signaller - ring which has, or will signal
877e3adcf8fSFrançois Tigeot * @seqno - seqno which the waiter will block on
878e3adcf8fSFrançois Tigeot */
87924edb884SFrançois Tigeot
88024edb884SFrançois Tigeot static int
gen8_ring_sync_to(struct drm_i915_gem_request * req,struct drm_i915_gem_request * signal)88171f41f3eSFrançois Tigeot gen8_ring_sync_to(struct drm_i915_gem_request *req,
88271f41f3eSFrançois Tigeot struct drm_i915_gem_request *signal)
88324edb884SFrançois Tigeot {
88471f41f3eSFrançois Tigeot struct drm_i915_private *dev_priv = req->i915;
88571f41f3eSFrançois Tigeot u64 offset = GEN8_WAIT_OFFSET(req->engine, signal->engine->id);
8861487f786SFrançois Tigeot struct i915_hw_ppgtt *ppgtt;
887a85cb24fSFrançois Tigeot u32 *cs;
88824edb884SFrançois Tigeot
889a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 4);
890a85cb24fSFrançois Tigeot if (IS_ERR(cs))
891a85cb24fSFrançois Tigeot return PTR_ERR(cs);
89224edb884SFrançois Tigeot
893a85cb24fSFrançois Tigeot *cs++ = MI_SEMAPHORE_WAIT | MI_SEMAPHORE_GLOBAL_GTT |
894a85cb24fSFrançois Tigeot MI_SEMAPHORE_SAD_GTE_SDD;
895a85cb24fSFrançois Tigeot *cs++ = signal->global_seqno;
896a85cb24fSFrançois Tigeot *cs++ = lower_32_bits(offset);
897a85cb24fSFrançois Tigeot *cs++ = upper_32_bits(offset);
898a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
8991487f786SFrançois Tigeot
9001487f786SFrançois Tigeot /* When the !RCS engines idle waiting upon a semaphore, they lose their
9011487f786SFrançois Tigeot * pagetables and we must reload them before executing the batch.
9021487f786SFrançois Tigeot * We do this on the i915_switch_context() following the wait and
9031487f786SFrançois Tigeot * before the dispatch.
9041487f786SFrançois Tigeot */
90571f41f3eSFrançois Tigeot ppgtt = req->ctx->ppgtt;
90671f41f3eSFrançois Tigeot if (ppgtt && req->engine->id != RCS)
90771f41f3eSFrançois Tigeot ppgtt->pd_dirty_rings |= intel_engine_flag(req->engine);
90824edb884SFrançois Tigeot return 0;
90924edb884SFrançois Tigeot }
91024edb884SFrançois Tigeot
911e3adcf8fSFrançois Tigeot static int
gen6_ring_sync_to(struct drm_i915_gem_request * req,struct drm_i915_gem_request * signal)91271f41f3eSFrançois Tigeot gen6_ring_sync_to(struct drm_i915_gem_request *req,
91371f41f3eSFrançois Tigeot struct drm_i915_gem_request *signal)
914e3adcf8fSFrançois Tigeot {
915e3adcf8fSFrançois Tigeot u32 dw1 = MI_SEMAPHORE_MBOX |
916e3adcf8fSFrançois Tigeot MI_SEMAPHORE_COMPARE |
917e3adcf8fSFrançois Tigeot MI_SEMAPHORE_REGISTER;
9181e12ee3bSFrançois Tigeot u32 wait_mbox = signal->engine->semaphore.mbox.wait[req->engine->hw_id];
919a85cb24fSFrançois Tigeot u32 *cs;
920e3adcf8fSFrançois Tigeot
92171f41f3eSFrançois Tigeot WARN_ON(wait_mbox == MI_SEMAPHORE_SYNC_INVALID);
92271f41f3eSFrançois Tigeot
923a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 4);
924a85cb24fSFrançois Tigeot if (IS_ERR(cs))
925a85cb24fSFrançois Tigeot return PTR_ERR(cs);
92671f41f3eSFrançois Tigeot
927a85cb24fSFrançois Tigeot *cs++ = dw1 | wait_mbox;
928686a02f1SFrançois Tigeot /* Throughout all of the GEM code, seqno passed implies our current
929686a02f1SFrançois Tigeot * seqno is >= the last seqno executed. However for hardware the
930686a02f1SFrançois Tigeot * comparison is strictly greater than.
931686a02f1SFrançois Tigeot */
932a85cb24fSFrançois Tigeot *cs++ = signal->global_seqno - 1;
933a85cb24fSFrançois Tigeot *cs++ = 0;
934a85cb24fSFrançois Tigeot *cs++ = MI_NOOP;
935a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
936e3adcf8fSFrançois Tigeot
937e3adcf8fSFrançois Tigeot return 0;
938e3adcf8fSFrançois Tigeot }
939e3adcf8fSFrançois Tigeot
940303bf270SFrançois Tigeot static void
gen5_seqno_barrier(struct intel_engine_cs * engine)94187df8fc6SFrançois Tigeot gen5_seqno_barrier(struct intel_engine_cs *engine)
942e3adcf8fSFrançois Tigeot {
943303bf270SFrançois Tigeot /* MI_STORE are internally buffered by the GPU and not flushed
944303bf270SFrançois Tigeot * either by MI_FLUSH or SyncFlush or any other combination of
945303bf270SFrançois Tigeot * MI commands.
946e3adcf8fSFrançois Tigeot *
947303bf270SFrançois Tigeot * "Only the submission of the store operation is guaranteed.
948303bf270SFrançois Tigeot * The write result will be complete (coherent) some time later
949303bf270SFrançois Tigeot * (this is practically a finite period but there is no guaranteed
950303bf270SFrançois Tigeot * latency)."
951303bf270SFrançois Tigeot *
952303bf270SFrançois Tigeot * Empirically, we observe that we need a delay of at least 75us to
953303bf270SFrançois Tigeot * be sure that the seqno write is visible by the CPU.
954e3adcf8fSFrançois Tigeot */
955303bf270SFrançois Tigeot usleep_range(125, 250);
956e3adcf8fSFrançois Tigeot }
957e3adcf8fSFrançois Tigeot
9588621f407SFrançois Tigeot static void
gen6_seqno_barrier(struct intel_engine_cs * engine)9598621f407SFrançois Tigeot gen6_seqno_barrier(struct intel_engine_cs *engine)
960e3adcf8fSFrançois Tigeot {
9611487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
9628621f407SFrançois Tigeot
963e3adcf8fSFrançois Tigeot /* Workaround to force correct ordering between irq and seqno writes on
964e3adcf8fSFrançois Tigeot * ivb (and maybe also on snb) by reading from a CS register (like
9658621f407SFrançois Tigeot * ACTHD) before reading the status page.
9668621f407SFrançois Tigeot *
9678621f407SFrançois Tigeot * Note that this effectively stalls the read by the time it takes to
9688621f407SFrançois Tigeot * do a memory transaction, which more or less ensures that the write
9698621f407SFrançois Tigeot * from the GPU has sufficient time to invalidate the CPU cacheline.
9708621f407SFrançois Tigeot * Alternatively we could delay the interrupt from the CS ring to give
9718621f407SFrançois Tigeot * the write time to land, but that would incur a delay after every
9728621f407SFrançois Tigeot * batch i.e. much more frequent than a delay when waiting for the
9738621f407SFrançois Tigeot * interrupt (with the same net latency).
9748621f407SFrançois Tigeot *
9758621f407SFrançois Tigeot * Also note that to prevent whole machine hangs on gen7, we have to
9768621f407SFrançois Tigeot * take the spinlock to guard against concurrent cacheline access.
9778621f407SFrançois Tigeot */
9788621f407SFrançois Tigeot spin_lock_irq(&dev_priv->uncore.lock);
9798621f407SFrançois Tigeot POSTING_READ_FW(RING_ACTHD(engine->mmio_base));
9808621f407SFrançois Tigeot spin_unlock_irq(&dev_priv->uncore.lock);
981e3adcf8fSFrançois Tigeot }
982e3adcf8fSFrançois Tigeot
983303bf270SFrançois Tigeot static void
gen5_irq_enable(struct intel_engine_cs * engine)984303bf270SFrançois Tigeot gen5_irq_enable(struct intel_engine_cs *engine)
985e3adcf8fSFrançois Tigeot {
986303bf270SFrançois Tigeot gen5_enable_gt_irq(engine->i915, engine->irq_enable_mask);
987e3adcf8fSFrançois Tigeot }
988e3adcf8fSFrançois Tigeot
989a2fdbec6SFrançois Tigeot static void
gen5_irq_disable(struct intel_engine_cs * engine)990303bf270SFrançois Tigeot gen5_irq_disable(struct intel_engine_cs *engine)
991a2fdbec6SFrançois Tigeot {
992303bf270SFrançois Tigeot gen5_disable_gt_irq(engine->i915, engine->irq_enable_mask);
993e3adcf8fSFrançois Tigeot }
994e3adcf8fSFrançois Tigeot
995a2fdbec6SFrançois Tigeot static void
i9xx_irq_enable(struct intel_engine_cs * engine)996303bf270SFrançois Tigeot i9xx_irq_enable(struct intel_engine_cs *engine)
997e3adcf8fSFrançois Tigeot {
9981487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
999e3adcf8fSFrançois Tigeot
10008621f407SFrançois Tigeot dev_priv->irq_mask &= ~engine->irq_enable_mask;
1001686a02f1SFrançois Tigeot I915_WRITE(IMR, dev_priv->irq_mask);
1002303bf270SFrançois Tigeot POSTING_READ_FW(RING_IMR(engine->mmio_base));
1003686a02f1SFrançois Tigeot }
1004686a02f1SFrançois Tigeot
1005686a02f1SFrançois Tigeot static void
i9xx_irq_disable(struct intel_engine_cs * engine)1006303bf270SFrançois Tigeot i9xx_irq_disable(struct intel_engine_cs *engine)
1007686a02f1SFrançois Tigeot {
10081487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
1009686a02f1SFrançois Tigeot
10108621f407SFrançois Tigeot dev_priv->irq_mask |= engine->irq_enable_mask;
1011686a02f1SFrançois Tigeot I915_WRITE(IMR, dev_priv->irq_mask);
1012686a02f1SFrançois Tigeot }
1013686a02f1SFrançois Tigeot
1014686a02f1SFrançois Tigeot static void
i8xx_irq_enable(struct intel_engine_cs * engine)1015303bf270SFrançois Tigeot i8xx_irq_enable(struct intel_engine_cs *engine)
1016686a02f1SFrançois Tigeot {
10171487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
1018686a02f1SFrançois Tigeot
1019303bf270SFrançois Tigeot dev_priv->irq_mask &= ~engine->irq_enable_mask;
1020303bf270SFrançois Tigeot I915_WRITE16(IMR, dev_priv->irq_mask);
1021303bf270SFrançois Tigeot POSTING_READ16(RING_IMR(engine->mmio_base));
1022303bf270SFrançois Tigeot }
1023303bf270SFrançois Tigeot
1024303bf270SFrançois Tigeot static void
i8xx_irq_disable(struct intel_engine_cs * engine)1025303bf270SFrançois Tigeot i8xx_irq_disable(struct intel_engine_cs *engine)
1026303bf270SFrançois Tigeot {
1027303bf270SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
1028303bf270SFrançois Tigeot
10298621f407SFrançois Tigeot dev_priv->irq_mask |= engine->irq_enable_mask;
1030686a02f1SFrançois Tigeot I915_WRITE16(IMR, dev_priv->irq_mask);
1031e3adcf8fSFrançois Tigeot }
1032e3adcf8fSFrançois Tigeot
1033e3adcf8fSFrançois Tigeot static int
bsd_ring_flush(struct drm_i915_gem_request * req,u32 mode)103471f41f3eSFrançois Tigeot bsd_ring_flush(struct drm_i915_gem_request *req, u32 mode)
1035e3adcf8fSFrançois Tigeot {
1036a85cb24fSFrançois Tigeot u32 *cs;
1037e3adcf8fSFrançois Tigeot
1038a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 2);
1039a85cb24fSFrançois Tigeot if (IS_ERR(cs))
1040a85cb24fSFrançois Tigeot return PTR_ERR(cs);
1041e3adcf8fSFrançois Tigeot
1042a85cb24fSFrançois Tigeot *cs++ = MI_FLUSH;
1043a85cb24fSFrançois Tigeot *cs++ = MI_NOOP;
1044a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
1045e3adcf8fSFrançois Tigeot return 0;
1046e3adcf8fSFrançois Tigeot }
1047e3adcf8fSFrançois Tigeot
1048303bf270SFrançois Tigeot static void
gen6_irq_enable(struct intel_engine_cs * engine)1049303bf270SFrançois Tigeot gen6_irq_enable(struct intel_engine_cs *engine)
1050e3adcf8fSFrançois Tigeot {
10511487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
1052e3adcf8fSFrançois Tigeot
10538621f407SFrançois Tigeot I915_WRITE_IMR(engine,
10548621f407SFrançois Tigeot ~(engine->irq_enable_mask |
1055303bf270SFrançois Tigeot engine->irq_keep_mask));
10568621f407SFrançois Tigeot gen5_enable_gt_irq(dev_priv, engine->irq_enable_mask);
1057e3adcf8fSFrançois Tigeot }
1058e3adcf8fSFrançois Tigeot
1059e3adcf8fSFrançois Tigeot static void
gen6_irq_disable(struct intel_engine_cs * engine)1060303bf270SFrançois Tigeot gen6_irq_disable(struct intel_engine_cs *engine)
1061e3adcf8fSFrançois Tigeot {
10621487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
1063e3adcf8fSFrançois Tigeot
1064303bf270SFrançois Tigeot I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
10658621f407SFrançois Tigeot gen5_disable_gt_irq(dev_priv, engine->irq_enable_mask);
1066e3adcf8fSFrançois Tigeot }
1067e3adcf8fSFrançois Tigeot
1068303bf270SFrançois Tigeot static void
hsw_vebox_irq_enable(struct intel_engine_cs * engine)1069303bf270SFrançois Tigeot hsw_vebox_irq_enable(struct intel_engine_cs *engine)
10705d0b1887SFrançois Tigeot {
10711487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
10725d0b1887SFrançois Tigeot
10738621f407SFrançois Tigeot I915_WRITE_IMR(engine, ~engine->irq_enable_mask);
10744be47400SFrançois Tigeot gen6_unmask_pm_irq(dev_priv, engine->irq_enable_mask);
10755d0b1887SFrançois Tigeot }
10765d0b1887SFrançois Tigeot
10775d0b1887SFrançois Tigeot static void
hsw_vebox_irq_disable(struct intel_engine_cs * engine)1078303bf270SFrançois Tigeot hsw_vebox_irq_disable(struct intel_engine_cs *engine)
10795d0b1887SFrançois Tigeot {
10801487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
10815d0b1887SFrançois Tigeot
10828621f407SFrançois Tigeot I915_WRITE_IMR(engine, ~0);
10834be47400SFrançois Tigeot gen6_mask_pm_irq(dev_priv, engine->irq_enable_mask);
10845d0b1887SFrançois Tigeot }
10859edbd4a0SFrançois Tigeot
1086303bf270SFrançois Tigeot static void
gen8_irq_enable(struct intel_engine_cs * engine)1087303bf270SFrançois Tigeot gen8_irq_enable(struct intel_engine_cs *engine)
10889edbd4a0SFrançois Tigeot {
10891487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
10909edbd4a0SFrançois Tigeot
10918621f407SFrançois Tigeot I915_WRITE_IMR(engine,
10928621f407SFrançois Tigeot ~(engine->irq_enable_mask |
1093303bf270SFrançois Tigeot engine->irq_keep_mask));
1094303bf270SFrançois Tigeot POSTING_READ_FW(RING_IMR(engine->mmio_base));
10959edbd4a0SFrançois Tigeot }
10969edbd4a0SFrançois Tigeot
10979edbd4a0SFrançois Tigeot static void
gen8_irq_disable(struct intel_engine_cs * engine)1098303bf270SFrançois Tigeot gen8_irq_disable(struct intel_engine_cs *engine)
10999edbd4a0SFrançois Tigeot {
11001487f786SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
11019edbd4a0SFrançois Tigeot
1102303bf270SFrançois Tigeot I915_WRITE_IMR(engine, ~engine->irq_keep_mask);
11035d0b1887SFrançois Tigeot }
11045d0b1887SFrançois Tigeot
1105e3adcf8fSFrançois Tigeot static int
i965_emit_bb_start(struct drm_i915_gem_request * req,u64 offset,u32 length,unsigned int dispatch_flags)110671f41f3eSFrançois Tigeot i965_emit_bb_start(struct drm_i915_gem_request *req,
1107ba55f2f5SFrançois Tigeot u64 offset, u32 length,
110871f41f3eSFrançois Tigeot unsigned int dispatch_flags)
1109e3adcf8fSFrançois Tigeot {
1110a85cb24fSFrançois Tigeot u32 *cs;
1111e3adcf8fSFrançois Tigeot
1112a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 2);
1113a85cb24fSFrançois Tigeot if (IS_ERR(cs))
1114a85cb24fSFrançois Tigeot return PTR_ERR(cs);
1115e3adcf8fSFrançois Tigeot
1116a85cb24fSFrançois Tigeot *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT | (dispatch_flags &
1117a85cb24fSFrançois Tigeot I915_DISPATCH_SECURE ? 0 : MI_BATCH_NON_SECURE_I965);
1118a85cb24fSFrançois Tigeot *cs++ = offset;
1119a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
1120e3adcf8fSFrançois Tigeot
1121e3adcf8fSFrançois Tigeot return 0;
1122e3adcf8fSFrançois Tigeot }
1123e3adcf8fSFrançois Tigeot
1124b5c29a34SFrançois Tigeot /* Just userspace ABI convention to limit the wa batch bo to a resonable size */
1125b5c29a34SFrançois Tigeot #define I830_BATCH_LIMIT (256*1024)
112624edb884SFrançois Tigeot #define I830_TLB_ENTRIES (2)
112724edb884SFrançois Tigeot #define I830_WA_SIZE max(I830_TLB_ENTRIES*4096, I830_BATCH_LIMIT)
1128e3adcf8fSFrançois Tigeot static int
i830_emit_bb_start(struct drm_i915_gem_request * req,u64 offset,u32 len,unsigned int dispatch_flags)112971f41f3eSFrançois Tigeot i830_emit_bb_start(struct drm_i915_gem_request *req,
1130ba55f2f5SFrançois Tigeot u64 offset, u32 len,
113171f41f3eSFrançois Tigeot unsigned int dispatch_flags)
1132e3adcf8fSFrançois Tigeot {
1133a85cb24fSFrançois Tigeot u32 *cs, cs_offset = i915_ggtt_offset(req->engine->scratch);
1134e3adcf8fSFrançois Tigeot
1135a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 6);
1136a85cb24fSFrançois Tigeot if (IS_ERR(cs))
1137a85cb24fSFrançois Tigeot return PTR_ERR(cs);
113824edb884SFrançois Tigeot
113924edb884SFrançois Tigeot /* Evict the invalid PTE TLBs */
1140a85cb24fSFrançois Tigeot *cs++ = COLOR_BLT_CMD | BLT_WRITE_RGBA;
1141a85cb24fSFrançois Tigeot *cs++ = BLT_DEPTH_32 | BLT_ROP_COLOR_COPY | 4096;
1142a85cb24fSFrançois Tigeot *cs++ = I830_TLB_ENTRIES << 16 | 4; /* load each page */
1143a85cb24fSFrançois Tigeot *cs++ = cs_offset;
1144a85cb24fSFrançois Tigeot *cs++ = 0xdeadbeef;
1145a85cb24fSFrançois Tigeot *cs++ = MI_NOOP;
1146a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
114724edb884SFrançois Tigeot
1148477eb7f9SFrançois Tigeot if ((dispatch_flags & I915_DISPATCH_PINNED) == 0) {
114924edb884SFrançois Tigeot if (len > I830_BATCH_LIMIT)
115024edb884SFrançois Tigeot return -ENOSPC;
115124edb884SFrançois Tigeot
1152a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 6 + 2);
1153a85cb24fSFrançois Tigeot if (IS_ERR(cs))
1154a85cb24fSFrançois Tigeot return PTR_ERR(cs);
115524edb884SFrançois Tigeot
115624edb884SFrançois Tigeot /* Blit the batch (which has now all relocs applied) to the
115724edb884SFrançois Tigeot * stable batch scratch bo area (so that the CS never
115824edb884SFrançois Tigeot * stumbles over its tlb invalidation bug) ...
115924edb884SFrançois Tigeot */
1160a85cb24fSFrançois Tigeot *cs++ = SRC_COPY_BLT_CMD | BLT_WRITE_RGBA;
1161a85cb24fSFrançois Tigeot *cs++ = BLT_DEPTH_32 | BLT_ROP_SRC_COPY | 4096;
1162a85cb24fSFrançois Tigeot *cs++ = DIV_ROUND_UP(len, 4096) << 16 | 4096;
1163a85cb24fSFrançois Tigeot *cs++ = cs_offset;
1164a85cb24fSFrançois Tigeot *cs++ = 4096;
1165a85cb24fSFrançois Tigeot *cs++ = offset;
116624edb884SFrançois Tigeot
1167a85cb24fSFrançois Tigeot *cs++ = MI_FLUSH;
1168a85cb24fSFrançois Tigeot *cs++ = MI_NOOP;
1169a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
117024edb884SFrançois Tigeot
117124edb884SFrançois Tigeot /* ... and execute it. */
117224edb884SFrançois Tigeot offset = cs_offset;
117324edb884SFrançois Tigeot }
117424edb884SFrançois Tigeot
1175a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 2);
1176a85cb24fSFrançois Tigeot if (IS_ERR(cs))
1177a85cb24fSFrançois Tigeot return PTR_ERR(cs);
1178e3adcf8fSFrançois Tigeot
1179a85cb24fSFrançois Tigeot *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
1180a85cb24fSFrançois Tigeot *cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 :
1181a85cb24fSFrançois Tigeot MI_BATCH_NON_SECURE);
1182a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
1183686a02f1SFrançois Tigeot
1184686a02f1SFrançois Tigeot return 0;
1185686a02f1SFrançois Tigeot }
1186686a02f1SFrançois Tigeot
1187686a02f1SFrançois Tigeot static int
i915_emit_bb_start(struct drm_i915_gem_request * req,u64 offset,u32 len,unsigned int dispatch_flags)118871f41f3eSFrançois Tigeot i915_emit_bb_start(struct drm_i915_gem_request *req,
1189ba55f2f5SFrançois Tigeot u64 offset, u32 len,
119071f41f3eSFrançois Tigeot unsigned int dispatch_flags)
1191686a02f1SFrançois Tigeot {
1192a85cb24fSFrançois Tigeot u32 *cs;
1193686a02f1SFrançois Tigeot
1194a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 2);
1195a85cb24fSFrançois Tigeot if (IS_ERR(cs))
1196a85cb24fSFrançois Tigeot return PTR_ERR(cs);
1197e3adcf8fSFrançois Tigeot
1198a85cb24fSFrançois Tigeot *cs++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
1199a85cb24fSFrançois Tigeot *cs++ = offset | (dispatch_flags & I915_DISPATCH_SECURE ? 0 :
1200a85cb24fSFrançois Tigeot MI_BATCH_NON_SECURE);
1201a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
1202e3adcf8fSFrançois Tigeot
1203e3adcf8fSFrançois Tigeot return 0;
1204e3adcf8fSFrançois Tigeot }
1205e3adcf8fSFrançois Tigeot
1206*3f2dd94aSFrançois Tigeot
1207*3f2dd94aSFrançois Tigeot
intel_ring_pin(struct intel_ring * ring,struct drm_i915_private * i915,unsigned int offset_bias)1208*3f2dd94aSFrançois Tigeot int intel_ring_pin(struct intel_ring *ring,
1209*3f2dd94aSFrançois Tigeot struct drm_i915_private *i915,
1210*3f2dd94aSFrançois Tigeot unsigned int offset_bias)
1211c0e85e96SFrançois Tigeot {
1212*3f2dd94aSFrançois Tigeot enum i915_map_type map = HAS_LLC(i915) ? I915_MAP_WB : I915_MAP_WC;
12131e12ee3bSFrançois Tigeot struct i915_vma *vma = ring->vma;
1214*3f2dd94aSFrançois Tigeot unsigned int flags;
12158621f407SFrançois Tigeot void *addr;
12162c9916cdSFrançois Tigeot int ret;
12172c9916cdSFrançois Tigeot
12181e12ee3bSFrançois Tigeot GEM_BUG_ON(ring->vaddr);
12191e12ee3bSFrançois Tigeot
12201e12ee3bSFrançois Tigeot
1221a85cb24fSFrançois Tigeot flags = PIN_GLOBAL;
1222a85cb24fSFrançois Tigeot if (offset_bias)
1223a85cb24fSFrançois Tigeot flags |= PIN_OFFSET_BIAS | offset_bias;
12241e12ee3bSFrançois Tigeot if (vma->obj->stolen)
12251e12ee3bSFrançois Tigeot flags |= PIN_MAPPABLE;
12261e12ee3bSFrançois Tigeot
12271e12ee3bSFrançois Tigeot if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
12281e12ee3bSFrançois Tigeot if (flags & PIN_MAPPABLE || map == I915_MAP_WC)
12291e12ee3bSFrançois Tigeot ret = i915_gem_object_set_to_gtt_domain(vma->obj, true);
12301e12ee3bSFrançois Tigeot else
12311e12ee3bSFrançois Tigeot ret = i915_gem_object_set_to_cpu_domain(vma->obj, true);
12321e12ee3bSFrançois Tigeot if (unlikely(ret))
12331e12ee3bSFrançois Tigeot return ret;
12341e12ee3bSFrançois Tigeot }
12351e12ee3bSFrançois Tigeot
12361e12ee3bSFrançois Tigeot ret = i915_vma_pin(vma, 0, PAGE_SIZE, flags);
12371e12ee3bSFrançois Tigeot if (unlikely(ret))
1238aee94f86SFrançois Tigeot return ret;
1239aee94f86SFrançois Tigeot
12401e12ee3bSFrançois Tigeot if (i915_vma_is_map_and_fenceable(vma))
12411e12ee3bSFrançois Tigeot addr = (void __force *)i915_vma_pin_iomap(vma);
12421e12ee3bSFrançois Tigeot else
12431e12ee3bSFrançois Tigeot addr = i915_gem_object_pin_map(vma->obj, map);
12441e12ee3bSFrançois Tigeot if (IS_ERR(addr))
12451e12ee3bSFrançois Tigeot goto err;
12462c9916cdSFrançois Tigeot
1247*3f2dd94aSFrançois Tigeot vma->obj->pin_global++;
1248*3f2dd94aSFrançois Tigeot
124971f41f3eSFrançois Tigeot ring->vaddr = addr;
12502c9916cdSFrançois Tigeot return 0;
12518621f407SFrançois Tigeot
12521e12ee3bSFrançois Tigeot err:
12531e12ee3bSFrançois Tigeot i915_vma_unpin(vma);
12541e12ee3bSFrançois Tigeot return PTR_ERR(addr);
12552c9916cdSFrançois Tigeot }
12562c9916cdSFrançois Tigeot
intel_ring_reset(struct intel_ring * ring,u32 tail)1257a85cb24fSFrançois Tigeot void intel_ring_reset(struct intel_ring *ring, u32 tail)
1258a85cb24fSFrançois Tigeot {
1259a85cb24fSFrançois Tigeot GEM_BUG_ON(!list_empty(&ring->request_list));
1260a85cb24fSFrançois Tigeot ring->tail = tail;
1261a85cb24fSFrançois Tigeot ring->head = tail;
1262a85cb24fSFrançois Tigeot ring->emit = tail;
1263a85cb24fSFrançois Tigeot intel_ring_update_space(ring);
1264a85cb24fSFrançois Tigeot }
1265a85cb24fSFrançois Tigeot
intel_ring_unpin(struct intel_ring * ring)126671f41f3eSFrançois Tigeot void intel_ring_unpin(struct intel_ring *ring)
1267e3adcf8fSFrançois Tigeot {
126871f41f3eSFrançois Tigeot GEM_BUG_ON(!ring->vma);
126971f41f3eSFrançois Tigeot GEM_BUG_ON(!ring->vaddr);
127071f41f3eSFrançois Tigeot
1271a85cb24fSFrançois Tigeot /* Discard any unused bytes beyond that submitted to hw. */
1272a85cb24fSFrançois Tigeot intel_ring_reset(ring, ring->tail);
1273a85cb24fSFrançois Tigeot
12741e12ee3bSFrançois Tigeot if (i915_vma_is_map_and_fenceable(ring->vma))
127571f41f3eSFrançois Tigeot i915_vma_unpin_iomap(ring->vma);
12761e12ee3bSFrançois Tigeot else
12771e12ee3bSFrançois Tigeot i915_gem_object_unpin_map(ring->vma->obj);
127871f41f3eSFrançois Tigeot ring->vaddr = NULL;
127971f41f3eSFrançois Tigeot
1280*3f2dd94aSFrançois Tigeot ring->vma->obj->pin_global--;
12811e12ee3bSFrançois Tigeot i915_vma_unpin(ring->vma);
128271f41f3eSFrançois Tigeot }
128371f41f3eSFrançois Tigeot
12841e12ee3bSFrançois Tigeot static struct i915_vma *
intel_ring_create_vma(struct drm_i915_private * dev_priv,int size)12851e12ee3bSFrançois Tigeot intel_ring_create_vma(struct drm_i915_private *dev_priv, int size)
128624edb884SFrançois Tigeot {
1287e3adcf8fSFrançois Tigeot struct drm_i915_gem_object *obj;
12881e12ee3bSFrançois Tigeot struct i915_vma *vma;
1289e3adcf8fSFrançois Tigeot
1290a85cb24fSFrançois Tigeot obj = i915_gem_object_create_stolen(dev_priv, size);
12911e12ee3bSFrançois Tigeot if (!obj)
1292*3f2dd94aSFrançois Tigeot obj = i915_gem_object_create_internal(dev_priv, size);
12931487f786SFrançois Tigeot if (IS_ERR(obj))
12941e12ee3bSFrançois Tigeot return ERR_CAST(obj);
1295e3adcf8fSFrançois Tigeot
129624edb884SFrançois Tigeot /* mark ring buffers as read-only from GPU side by default */
129724edb884SFrançois Tigeot obj->gt_ro = 1;
129824edb884SFrançois Tigeot
1299a85cb24fSFrançois Tigeot vma = i915_vma_instance(obj, &dev_priv->ggtt.base, NULL);
13001e12ee3bSFrançois Tigeot if (IS_ERR(vma))
13011e12ee3bSFrançois Tigeot goto err;
1302ba55f2f5SFrançois Tigeot
13031e12ee3bSFrançois Tigeot return vma;
13041e12ee3bSFrançois Tigeot
13051e12ee3bSFrançois Tigeot err:
13061e12ee3bSFrançois Tigeot i915_gem_object_put(obj);
13071e12ee3bSFrançois Tigeot return vma;
1308ba55f2f5SFrançois Tigeot }
1309ba55f2f5SFrançois Tigeot
131071f41f3eSFrançois Tigeot struct intel_ring *
intel_engine_create_ring(struct intel_engine_cs * engine,int size)131171f41f3eSFrançois Tigeot intel_engine_create_ring(struct intel_engine_cs *engine, int size)
1312352ff8bdSFrançois Tigeot {
131371f41f3eSFrançois Tigeot struct intel_ring *ring;
13141e12ee3bSFrançois Tigeot struct i915_vma *vma;
1315352ff8bdSFrançois Tigeot
131671f41f3eSFrançois Tigeot GEM_BUG_ON(!is_power_of_2(size));
13171e12ee3bSFrançois Tigeot GEM_BUG_ON(RING_CTL_SIZE(size) & ~RING_NR_PAGES);
131871f41f3eSFrançois Tigeot
1319352ff8bdSFrançois Tigeot ring = kzalloc(sizeof(*ring), GFP_KERNEL);
13201e12ee3bSFrançois Tigeot if (!ring)
1321352ff8bdSFrançois Tigeot return ERR_PTR(-ENOMEM);
1322352ff8bdSFrançois Tigeot
132371f41f3eSFrançois Tigeot INIT_LIST_HEAD(&ring->request_list);
132471f41f3eSFrançois Tigeot
1325352ff8bdSFrançois Tigeot ring->size = size;
1326352ff8bdSFrançois Tigeot /* Workaround an erratum on the i830 which causes a hang if
1327352ff8bdSFrançois Tigeot * the TAIL pointer points to within the last 2 cachelines
1328352ff8bdSFrançois Tigeot * of the buffer.
1329352ff8bdSFrançois Tigeot */
1330352ff8bdSFrançois Tigeot ring->effective_size = size;
1331a85cb24fSFrançois Tigeot if (IS_I830(engine->i915) || IS_I845G(engine->i915))
1332352ff8bdSFrançois Tigeot ring->effective_size -= 2 * CACHELINE_BYTES;
1333352ff8bdSFrançois Tigeot
1334352ff8bdSFrançois Tigeot intel_ring_update_space(ring);
1335352ff8bdSFrançois Tigeot
13361e12ee3bSFrançois Tigeot vma = intel_ring_create_vma(engine->i915, size);
13371e12ee3bSFrançois Tigeot if (IS_ERR(vma)) {
1338352ff8bdSFrançois Tigeot kfree(ring);
13391e12ee3bSFrançois Tigeot return ERR_CAST(vma);
1340352ff8bdSFrançois Tigeot }
13411e12ee3bSFrançois Tigeot ring->vma = vma;
1342352ff8bdSFrançois Tigeot
1343352ff8bdSFrançois Tigeot return ring;
1344352ff8bdSFrançois Tigeot }
1345352ff8bdSFrançois Tigeot
1346352ff8bdSFrançois Tigeot void
intel_ring_free(struct intel_ring * ring)134771f41f3eSFrançois Tigeot intel_ring_free(struct intel_ring *ring)
1348352ff8bdSFrançois Tigeot {
13494be47400SFrançois Tigeot struct drm_i915_gem_object *obj = ring->vma->obj;
13504be47400SFrançois Tigeot
13514be47400SFrançois Tigeot i915_vma_close(ring->vma);
13524be47400SFrançois Tigeot __i915_gem_object_release_unless_active(obj);
13534be47400SFrançois Tigeot
1354352ff8bdSFrançois Tigeot kfree(ring);
1355352ff8bdSFrançois Tigeot }
1356352ff8bdSFrançois Tigeot
context_pin(struct i915_gem_context * ctx)1357a85cb24fSFrançois Tigeot static int context_pin(struct i915_gem_context *ctx)
1358a85cb24fSFrançois Tigeot {
1359a85cb24fSFrançois Tigeot struct i915_vma *vma = ctx->engine[RCS].state;
1360a85cb24fSFrançois Tigeot int ret;
1361a85cb24fSFrançois Tigeot
1362a85cb24fSFrançois Tigeot /* Clear this page out of any CPU caches for coherent swap-in/out.
1363a85cb24fSFrançois Tigeot * We only want to do this on the first bind so that we do not stall
1364a85cb24fSFrançois Tigeot * on an active context (which by nature is already on the GPU).
1365a85cb24fSFrançois Tigeot */
1366a85cb24fSFrançois Tigeot if (!(vma->flags & I915_VMA_GLOBAL_BIND)) {
1367a85cb24fSFrançois Tigeot ret = i915_gem_object_set_to_gtt_domain(vma->obj, false);
1368a85cb24fSFrançois Tigeot if (ret)
1369a85cb24fSFrançois Tigeot return ret;
1370a85cb24fSFrançois Tigeot }
1371a85cb24fSFrançois Tigeot
1372a85cb24fSFrançois Tigeot return i915_vma_pin(vma, 0, I915_GTT_MIN_ALIGNMENT,
1373a85cb24fSFrançois Tigeot PIN_GLOBAL | PIN_HIGH);
1374a85cb24fSFrançois Tigeot }
1375a85cb24fSFrançois Tigeot
1376*3f2dd94aSFrançois Tigeot static struct i915_vma *
alloc_context_vma(struct intel_engine_cs * engine)1377*3f2dd94aSFrançois Tigeot alloc_context_vma(struct intel_engine_cs *engine)
1378*3f2dd94aSFrançois Tigeot {
1379*3f2dd94aSFrançois Tigeot struct drm_i915_private *i915 = engine->i915;
1380*3f2dd94aSFrançois Tigeot struct drm_i915_gem_object *obj;
1381*3f2dd94aSFrançois Tigeot struct i915_vma *vma;
1382*3f2dd94aSFrançois Tigeot
1383*3f2dd94aSFrançois Tigeot obj = i915_gem_object_create(i915, engine->context_size);
1384*3f2dd94aSFrançois Tigeot if (IS_ERR(obj))
1385*3f2dd94aSFrançois Tigeot return ERR_CAST(obj);
1386*3f2dd94aSFrançois Tigeot
1387*3f2dd94aSFrançois Tigeot /*
1388*3f2dd94aSFrançois Tigeot * Try to make the context utilize L3 as well as LLC.
1389*3f2dd94aSFrançois Tigeot *
1390*3f2dd94aSFrançois Tigeot * On VLV we don't have L3 controls in the PTEs so we
1391*3f2dd94aSFrançois Tigeot * shouldn't touch the cache level, especially as that
1392*3f2dd94aSFrançois Tigeot * would make the object snooped which might have a
1393*3f2dd94aSFrançois Tigeot * negative performance impact.
1394*3f2dd94aSFrançois Tigeot *
1395*3f2dd94aSFrançois Tigeot * Snooping is required on non-llc platforms in execlist
1396*3f2dd94aSFrançois Tigeot * mode, but since all GGTT accesses use PAT entry 0 we
1397*3f2dd94aSFrançois Tigeot * get snooping anyway regardless of cache_level.
1398*3f2dd94aSFrançois Tigeot *
1399*3f2dd94aSFrançois Tigeot * This is only applicable for Ivy Bridge devices since
1400*3f2dd94aSFrançois Tigeot * later platforms don't have L3 control bits in the PTE.
1401*3f2dd94aSFrançois Tigeot */
1402*3f2dd94aSFrançois Tigeot if (IS_IVYBRIDGE(i915)) {
1403*3f2dd94aSFrançois Tigeot /* Ignore any error, regard it as a simple optimisation */
1404*3f2dd94aSFrançois Tigeot i915_gem_object_set_cache_level(obj, I915_CACHE_L3_LLC);
1405*3f2dd94aSFrançois Tigeot }
1406*3f2dd94aSFrançois Tigeot
1407*3f2dd94aSFrançois Tigeot vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
1408*3f2dd94aSFrançois Tigeot if (IS_ERR(vma))
1409*3f2dd94aSFrançois Tigeot i915_gem_object_put(obj);
1410*3f2dd94aSFrançois Tigeot
1411*3f2dd94aSFrançois Tigeot return vma;
1412*3f2dd94aSFrançois Tigeot }
1413*3f2dd94aSFrançois Tigeot
1414*3f2dd94aSFrançois Tigeot static struct intel_ring *
intel_ring_context_pin(struct intel_engine_cs * engine,struct i915_gem_context * ctx)1415*3f2dd94aSFrançois Tigeot intel_ring_context_pin(struct intel_engine_cs *engine,
1416a85cb24fSFrançois Tigeot struct i915_gem_context *ctx)
14171487f786SFrançois Tigeot {
14181487f786SFrançois Tigeot struct intel_context *ce = &ctx->engine[engine->id];
14191487f786SFrançois Tigeot int ret;
14201487f786SFrançois Tigeot
1421303bf270SFrançois Tigeot lockdep_assert_held(&ctx->i915->drm.struct_mutex);
14221487f786SFrançois Tigeot
1423*3f2dd94aSFrançois Tigeot if (likely(ce->pin_count++))
1424*3f2dd94aSFrançois Tigeot goto out;
1425a85cb24fSFrançois Tigeot GEM_BUG_ON(!ce->pin_count); /* no overflow please! */
14261487f786SFrançois Tigeot
1427*3f2dd94aSFrançois Tigeot if (!ce->state && engine->context_size) {
1428*3f2dd94aSFrançois Tigeot struct i915_vma *vma;
1429*3f2dd94aSFrançois Tigeot
1430*3f2dd94aSFrançois Tigeot vma = alloc_context_vma(engine);
1431*3f2dd94aSFrançois Tigeot if (IS_ERR(vma)) {
1432*3f2dd94aSFrançois Tigeot ret = PTR_ERR(vma);
1433*3f2dd94aSFrançois Tigeot goto err;
1434*3f2dd94aSFrançois Tigeot }
1435*3f2dd94aSFrançois Tigeot
1436*3f2dd94aSFrançois Tigeot ce->state = vma;
1437*3f2dd94aSFrançois Tigeot }
1438*3f2dd94aSFrançois Tigeot
14391487f786SFrançois Tigeot if (ce->state) {
1440a85cb24fSFrançois Tigeot ret = context_pin(ctx);
1441a85cb24fSFrançois Tigeot if (ret)
1442*3f2dd94aSFrançois Tigeot goto err;
1443a85cb24fSFrançois Tigeot
1444a85cb24fSFrançois Tigeot ce->state->obj->mm.dirty = true;
1445*3f2dd94aSFrançois Tigeot ce->state->obj->pin_global++;
14464be47400SFrançois Tigeot }
14471487f786SFrançois Tigeot
14481487f786SFrançois Tigeot /* The kernel context is only used as a placeholder for flushing the
14491487f786SFrançois Tigeot * active context. It is never used for submitting user rendering and
14501487f786SFrançois Tigeot * as such never requires the golden render context, and so we can skip
14511487f786SFrançois Tigeot * emitting it when we switch to the kernel context. This is required
14521487f786SFrançois Tigeot * as during eviction we cannot allocate and pin the renderstate in
14531487f786SFrançois Tigeot * order to initialise the context.
14541487f786SFrançois Tigeot */
1455a85cb24fSFrançois Tigeot if (i915_gem_context_is_kernel(ctx))
14561487f786SFrançois Tigeot ce->initialised = true;
14571487f786SFrançois Tigeot
145887df8fc6SFrançois Tigeot i915_gem_context_get(ctx);
14591487f786SFrançois Tigeot
1460*3f2dd94aSFrançois Tigeot out:
1461*3f2dd94aSFrançois Tigeot /* One ringbuffer to rule them all */
1462*3f2dd94aSFrançois Tigeot return engine->buffer;
1463*3f2dd94aSFrançois Tigeot
1464*3f2dd94aSFrançois Tigeot err:
14651487f786SFrançois Tigeot ce->pin_count = 0;
1466*3f2dd94aSFrançois Tigeot return ERR_PTR(ret);
14671487f786SFrançois Tigeot }
14681487f786SFrançois Tigeot
intel_ring_context_unpin(struct intel_engine_cs * engine,struct i915_gem_context * ctx)1469a85cb24fSFrançois Tigeot static void intel_ring_context_unpin(struct intel_engine_cs *engine,
1470a85cb24fSFrançois Tigeot struct i915_gem_context *ctx)
14711487f786SFrançois Tigeot {
14721487f786SFrançois Tigeot struct intel_context *ce = &ctx->engine[engine->id];
14731487f786SFrançois Tigeot
1474303bf270SFrançois Tigeot lockdep_assert_held(&ctx->i915->drm.struct_mutex);
1475a85cb24fSFrançois Tigeot GEM_BUG_ON(ce->pin_count == 0);
14761487f786SFrançois Tigeot
14771487f786SFrançois Tigeot if (--ce->pin_count)
14781487f786SFrançois Tigeot return;
14791487f786SFrançois Tigeot
1480*3f2dd94aSFrançois Tigeot if (ce->state) {
1481*3f2dd94aSFrançois Tigeot ce->state->obj->pin_global--;
14821e12ee3bSFrançois Tigeot i915_vma_unpin(ce->state);
1483*3f2dd94aSFrançois Tigeot }
14841487f786SFrançois Tigeot
148587df8fc6SFrançois Tigeot i915_gem_context_put(ctx);
14861487f786SFrançois Tigeot }
14871487f786SFrançois Tigeot
intel_init_ring_buffer(struct intel_engine_cs * engine)148887df8fc6SFrançois Tigeot static int intel_init_ring_buffer(struct intel_engine_cs *engine)
1489ba55f2f5SFrançois Tigeot {
149071f41f3eSFrançois Tigeot struct intel_ring *ring;
1491*3f2dd94aSFrançois Tigeot int err;
14922c9916cdSFrançois Tigeot
149387df8fc6SFrançois Tigeot intel_engine_setup_common(engine);
149487df8fc6SFrançois Tigeot
1495*3f2dd94aSFrançois Tigeot err = intel_engine_init_common(engine);
1496*3f2dd94aSFrançois Tigeot if (err)
1497*3f2dd94aSFrançois Tigeot goto err;
1498ba55f2f5SFrançois Tigeot
149971f41f3eSFrançois Tigeot ring = intel_engine_create_ring(engine, 32 * PAGE_SIZE);
150071f41f3eSFrançois Tigeot if (IS_ERR(ring)) {
1501*3f2dd94aSFrançois Tigeot err = PTR_ERR(ring);
1502*3f2dd94aSFrançois Tigeot goto err;
1503ba55f2f5SFrançois Tigeot }
1504ba55f2f5SFrançois Tigeot
1505a85cb24fSFrançois Tigeot /* Ring wraparound at offset 0 sometimes hangs. No idea why. */
1506*3f2dd94aSFrançois Tigeot err = intel_ring_pin(ring, engine->i915, I915_GTT_PAGE_SIZE);
1507*3f2dd94aSFrançois Tigeot if (err)
1508*3f2dd94aSFrançois Tigeot goto err_ring;
1509*3f2dd94aSFrançois Tigeot
1510*3f2dd94aSFrançois Tigeot GEM_BUG_ON(engine->buffer);
15111e12ee3bSFrançois Tigeot engine->buffer = ring;
1512e3adcf8fSFrançois Tigeot
1513e3adcf8fSFrançois Tigeot return 0;
1514e3adcf8fSFrançois Tigeot
1515*3f2dd94aSFrançois Tigeot err_ring:
1516*3f2dd94aSFrançois Tigeot intel_ring_free(ring);
1517*3f2dd94aSFrançois Tigeot err:
1518*3f2dd94aSFrançois Tigeot intel_engine_cleanup_common(engine);
1519*3f2dd94aSFrançois Tigeot return err;
1520e3adcf8fSFrançois Tigeot }
1521e3adcf8fSFrançois Tigeot
intel_engine_cleanup(struct intel_engine_cs * engine)152271f41f3eSFrançois Tigeot void intel_engine_cleanup(struct intel_engine_cs *engine)
1523e3adcf8fSFrançois Tigeot {
1524*3f2dd94aSFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
1525e3adcf8fSFrançois Tigeot
15261e12ee3bSFrançois Tigeot WARN_ON(INTEL_GEN(dev_priv) > 2 &&
15271e12ee3bSFrançois Tigeot (I915_READ_MODE(engine) & MODE_IDLE) == 0);
1528b030f26bSFrançois Tigeot
152971f41f3eSFrançois Tigeot intel_ring_unpin(engine->buffer);
153071f41f3eSFrançois Tigeot intel_ring_free(engine->buffer);
1531e3adcf8fSFrançois Tigeot
15328621f407SFrançois Tigeot if (engine->cleanup)
15338621f407SFrançois Tigeot engine->cleanup(engine);
1534e3adcf8fSFrançois Tigeot
153571f41f3eSFrançois Tigeot intel_engine_cleanup_common(engine);
15361487f786SFrançois Tigeot
15371e12ee3bSFrançois Tigeot dev_priv->engine[engine->id] = NULL;
15381e12ee3bSFrançois Tigeot kfree(engine);
15391e12ee3bSFrançois Tigeot }
15401e12ee3bSFrançois Tigeot
intel_legacy_submission_resume(struct drm_i915_private * dev_priv)15411e12ee3bSFrançois Tigeot void intel_legacy_submission_resume(struct drm_i915_private *dev_priv)
15421e12ee3bSFrançois Tigeot {
15431e12ee3bSFrançois Tigeot struct intel_engine_cs *engine;
15441e12ee3bSFrançois Tigeot enum intel_engine_id id;
15451e12ee3bSFrançois Tigeot
1546a85cb24fSFrançois Tigeot /* Restart from the beginning of the rings for convenience */
1547a85cb24fSFrançois Tigeot for_each_engine(engine, dev_priv, id)
1548a85cb24fSFrançois Tigeot intel_ring_reset(engine->buffer, 0);
1549e3adcf8fSFrançois Tigeot }
1550e3adcf8fSFrançois Tigeot
ring_request_alloc(struct drm_i915_gem_request * request)1551a85cb24fSFrançois Tigeot static int ring_request_alloc(struct drm_i915_gem_request *request)
1552b5c29a34SFrançois Tigeot {
1553a85cb24fSFrançois Tigeot u32 *cs;
1554a85cb24fSFrançois Tigeot
1555a85cb24fSFrançois Tigeot GEM_BUG_ON(!request->ctx->engine[request->engine->id].pin_count);
15569edbd4a0SFrançois Tigeot
15571487f786SFrançois Tigeot /* Flush enough space to reduce the likelihood of waiting after
15581487f786SFrançois Tigeot * we start building the request - in which case we will just
15591487f786SFrançois Tigeot * have to repeat work.
1560a05eeebfSFrançois Tigeot */
15611487f786SFrançois Tigeot request->reserved_space += LEGACY_REQUEST_SIZE;
1562a2fdbec6SFrançois Tigeot
1563a85cb24fSFrançois Tigeot cs = intel_ring_begin(request, 0);
1564a85cb24fSFrançois Tigeot if (IS_ERR(cs))
1565a85cb24fSFrançois Tigeot return PTR_ERR(cs);
1566a05eeebfSFrançois Tigeot
15671487f786SFrançois Tigeot request->reserved_space -= LEGACY_REQUEST_SIZE;
15681487f786SFrançois Tigeot return 0;
1569a05eeebfSFrançois Tigeot }
1570a05eeebfSFrançois Tigeot
wait_for_space(struct drm_i915_gem_request * req,unsigned int bytes)1571*3f2dd94aSFrançois Tigeot static noinline int wait_for_space(struct drm_i915_gem_request *req,
1572*3f2dd94aSFrançois Tigeot unsigned int bytes)
1573a05eeebfSFrançois Tigeot {
157471f41f3eSFrançois Tigeot struct intel_ring *ring = req->ring;
15758621f407SFrançois Tigeot struct drm_i915_gem_request *target;
15764be47400SFrançois Tigeot long timeout;
15774be47400SFrançois Tigeot
15784be47400SFrançois Tigeot lockdep_assert_held(&req->i915->drm.struct_mutex);
15798621f407SFrançois Tigeot
1580*3f2dd94aSFrançois Tigeot if (intel_ring_update_space(ring) >= bytes)
15818621f407SFrançois Tigeot return 0;
15828621f407SFrançois Tigeot
15838621f407SFrançois Tigeot /*
15848621f407SFrançois Tigeot * Space is reserved in the ringbuffer for finalising the request,
15858621f407SFrançois Tigeot * as that cannot be allowed to fail. During request finalisation,
15868621f407SFrançois Tigeot * reserved_space is set to 0 to stop the overallocation and the
15878621f407SFrançois Tigeot * assumption is that then we never need to wait (which has the
15888621f407SFrançois Tigeot * risk of failing with EINTR).
15898621f407SFrançois Tigeot *
15908621f407SFrançois Tigeot * See also i915_gem_request_alloc() and i915_add_request().
15918621f407SFrançois Tigeot */
15921487f786SFrançois Tigeot GEM_BUG_ON(!req->reserved_space);
15938621f407SFrançois Tigeot
159471f41f3eSFrançois Tigeot list_for_each_entry(target, &ring->request_list, ring_link) {
15958621f407SFrançois Tigeot /* Would completion of this request free enough space? */
1596*3f2dd94aSFrançois Tigeot if (bytes <= __intel_ring_space(target->postfix,
1597*3f2dd94aSFrançois Tigeot ring->emit, ring->size))
15988621f407SFrançois Tigeot break;
15998621f407SFrançois Tigeot }
16008621f407SFrançois Tigeot
160171f41f3eSFrançois Tigeot if (WARN_ON(&target->ring_link == &ring->request_list))
16028621f407SFrançois Tigeot return -ENOSPC;
16038621f407SFrançois Tigeot
16044be47400SFrançois Tigeot timeout = i915_wait_request(target,
16051e12ee3bSFrançois Tigeot I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
16064be47400SFrançois Tigeot MAX_SCHEDULE_TIMEOUT);
16074be47400SFrançois Tigeot if (timeout < 0)
16084be47400SFrançois Tigeot return timeout;
160971f41f3eSFrançois Tigeot
161071f41f3eSFrançois Tigeot i915_gem_request_retire_upto(target);
161171f41f3eSFrançois Tigeot
161271f41f3eSFrançois Tigeot intel_ring_update_space(ring);
161371f41f3eSFrançois Tigeot GEM_BUG_ON(ring->space < bytes);
161471f41f3eSFrançois Tigeot return 0;
16158621f407SFrançois Tigeot }
16168621f407SFrançois Tigeot
intel_ring_begin(struct drm_i915_gem_request * req,unsigned int num_dwords)1617*3f2dd94aSFrançois Tigeot u32 *intel_ring_begin(struct drm_i915_gem_request *req,
1618*3f2dd94aSFrançois Tigeot unsigned int num_dwords)
16198621f407SFrançois Tigeot {
162071f41f3eSFrançois Tigeot struct intel_ring *ring = req->ring;
1621*3f2dd94aSFrançois Tigeot const unsigned int remain_usable = ring->effective_size - ring->emit;
1622*3f2dd94aSFrançois Tigeot const unsigned int bytes = num_dwords * sizeof(u32);
1623*3f2dd94aSFrançois Tigeot unsigned int need_wrap = 0;
1624*3f2dd94aSFrançois Tigeot unsigned int total_bytes;
1625a85cb24fSFrançois Tigeot u32 *cs;
1626a05eeebfSFrançois Tigeot
1627*3f2dd94aSFrançois Tigeot /* Packets must be qword aligned. */
1628*3f2dd94aSFrançois Tigeot GEM_BUG_ON(num_dwords & 1);
1629a05eeebfSFrançois Tigeot
1630*3f2dd94aSFrançois Tigeot total_bytes = bytes + req->reserved_space;
1631*3f2dd94aSFrançois Tigeot GEM_BUG_ON(total_bytes > ring->effective_size);
1632*3f2dd94aSFrançois Tigeot
1633*3f2dd94aSFrançois Tigeot if (unlikely(total_bytes > remain_usable)) {
1634*3f2dd94aSFrançois Tigeot const int remain_actual = ring->size - ring->emit;
1635*3f2dd94aSFrançois Tigeot
1636*3f2dd94aSFrançois Tigeot if (bytes > remain_usable) {
1637a05eeebfSFrançois Tigeot /*
1638*3f2dd94aSFrançois Tigeot * Not enough space for the basic request. So need to
1639*3f2dd94aSFrançois Tigeot * flush out the remainder and then wait for
1640*3f2dd94aSFrançois Tigeot * base + reserved.
1641a05eeebfSFrançois Tigeot */
1642*3f2dd94aSFrançois Tigeot total_bytes += remain_actual;
1643*3f2dd94aSFrançois Tigeot need_wrap = remain_actual | 1;
1644*3f2dd94aSFrançois Tigeot } else {
1645a05eeebfSFrançois Tigeot /*
1646a05eeebfSFrançois Tigeot * The base request will fit but the reserved space
1647*3f2dd94aSFrançois Tigeot * falls off the end. So we don't need an immediate
1648*3f2dd94aSFrançois Tigeot * wrap and only need to effectively wait for the
1649*3f2dd94aSFrançois Tigeot * reserved size from the start of ringbuffer.
1650a05eeebfSFrançois Tigeot */
1651*3f2dd94aSFrançois Tigeot total_bytes = req->reserved_space + remain_actual;
1652*3f2dd94aSFrançois Tigeot }
1653a05eeebfSFrançois Tigeot }
1654a05eeebfSFrançois Tigeot
1655*3f2dd94aSFrançois Tigeot if (unlikely(total_bytes > ring->space)) {
1656*3f2dd94aSFrançois Tigeot int ret = wait_for_space(req, total_bytes);
1657a2fdbec6SFrançois Tigeot if (unlikely(ret))
1658a85cb24fSFrançois Tigeot return ERR_PTR(ret);
1659a2fdbec6SFrançois Tigeot }
1660a2fdbec6SFrançois Tigeot
16618621f407SFrançois Tigeot if (unlikely(need_wrap)) {
1662*3f2dd94aSFrançois Tigeot need_wrap &= ~1;
1663*3f2dd94aSFrançois Tigeot GEM_BUG_ON(need_wrap > ring->space);
1664*3f2dd94aSFrançois Tigeot GEM_BUG_ON(ring->emit + need_wrap > ring->size);
16658621f407SFrançois Tigeot
16668621f407SFrançois Tigeot /* Fill the tail with MI_NOOP */
1667*3f2dd94aSFrançois Tigeot memset(ring->vaddr + ring->emit, 0, need_wrap);
1668a85cb24fSFrançois Tigeot ring->emit = 0;
1669*3f2dd94aSFrançois Tigeot ring->space -= need_wrap;
1670a2fdbec6SFrançois Tigeot }
1671a2fdbec6SFrançois Tigeot
1672a85cb24fSFrançois Tigeot GEM_BUG_ON(ring->emit > ring->size - bytes);
1673*3f2dd94aSFrançois Tigeot GEM_BUG_ON(ring->space < bytes);
1674a85cb24fSFrançois Tigeot cs = ring->vaddr + ring->emit;
1675*3f2dd94aSFrançois Tigeot GEM_DEBUG_EXEC(memset(cs, POISON_INUSE, bytes));
1676a85cb24fSFrançois Tigeot ring->emit += bytes;
167771f41f3eSFrançois Tigeot ring->space -= bytes;
1678a85cb24fSFrançois Tigeot
1679a85cb24fSFrançois Tigeot return cs;
16809edbd4a0SFrançois Tigeot }
16819edbd4a0SFrançois Tigeot
16829edbd4a0SFrançois Tigeot /* Align the ring tail to a cacheline boundary */
intel_ring_cacheline_align(struct drm_i915_gem_request * req)1683a05eeebfSFrançois Tigeot int intel_ring_cacheline_align(struct drm_i915_gem_request *req)
16849edbd4a0SFrançois Tigeot {
168571f41f3eSFrançois Tigeot int num_dwords =
1686a85cb24fSFrançois Tigeot (req->ring->emit & (CACHELINE_BYTES - 1)) / sizeof(uint32_t);
1687a85cb24fSFrançois Tigeot u32 *cs;
16889edbd4a0SFrançois Tigeot
16899edbd4a0SFrançois Tigeot if (num_dwords == 0)
16909edbd4a0SFrançois Tigeot return 0;
16919edbd4a0SFrançois Tigeot
1692ba55f2f5SFrançois Tigeot num_dwords = CACHELINE_BYTES / sizeof(uint32_t) - num_dwords;
1693a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, num_dwords);
1694a85cb24fSFrançois Tigeot if (IS_ERR(cs))
1695a85cb24fSFrançois Tigeot return PTR_ERR(cs);
16969edbd4a0SFrançois Tigeot
16979edbd4a0SFrançois Tigeot while (num_dwords--)
1698a85cb24fSFrançois Tigeot *cs++ = MI_NOOP;
16999edbd4a0SFrançois Tigeot
1700a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
17019edbd4a0SFrançois Tigeot
17029edbd4a0SFrançois Tigeot return 0;
1703e3adcf8fSFrançois Tigeot }
1704e3adcf8fSFrançois Tigeot
gen6_bsd_submit_request(struct drm_i915_gem_request * request)170571f41f3eSFrançois Tigeot static void gen6_bsd_submit_request(struct drm_i915_gem_request *request)
1706e3adcf8fSFrançois Tigeot {
170771f41f3eSFrançois Tigeot struct drm_i915_private *dev_priv = request->i915;
17081487f786SFrançois Tigeot
17091487f786SFrançois Tigeot intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
1710e3adcf8fSFrançois Tigeot
1711e3adcf8fSFrançois Tigeot /* Every tail move must follow the sequence below */
1712f4e1c372SFrançois Tigeot
1713f4e1c372SFrançois Tigeot /* Disable notification that the ring is IDLE. The GT
1714f4e1c372SFrançois Tigeot * will then assume that it is busy and bring it out of rc6.
1715f4e1c372SFrançois Tigeot */
17161487f786SFrançois Tigeot I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL,
1717f4e1c372SFrançois Tigeot _MASKED_BIT_ENABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
1718e3adcf8fSFrançois Tigeot
1719f4e1c372SFrançois Tigeot /* Clear the context id. Here be magic! */
17201487f786SFrançois Tigeot I915_WRITE64_FW(GEN6_BSD_RNCID, 0x0);
1721e3adcf8fSFrançois Tigeot
1722f4e1c372SFrançois Tigeot /* Wait for the ring not to be idle, i.e. for it to wake up. */
1723*3f2dd94aSFrançois Tigeot if (__intel_wait_for_register_fw(dev_priv,
17241487f786SFrançois Tigeot GEN6_BSD_SLEEP_PSMI_CONTROL,
17251487f786SFrançois Tigeot GEN6_BSD_SLEEP_INDICATOR,
17261487f786SFrançois Tigeot 0,
1727*3f2dd94aSFrançois Tigeot 1000, 0, NULL))
1728f4e1c372SFrançois Tigeot DRM_ERROR("timed out waiting for the BSD ring to wake up\n");
1729f4e1c372SFrançois Tigeot
1730f4e1c372SFrançois Tigeot /* Now that the ring is fully powered up, update the tail */
173171f41f3eSFrançois Tigeot i9xx_submit_request(request);
1732f4e1c372SFrançois Tigeot
1733f4e1c372SFrançois Tigeot /* Let the ring send IDLE messages to the GT again,
1734f4e1c372SFrançois Tigeot * and so let it sleep to conserve power when idle.
1735f4e1c372SFrançois Tigeot */
17361487f786SFrançois Tigeot I915_WRITE_FW(GEN6_BSD_SLEEP_PSMI_CONTROL,
1737f4e1c372SFrançois Tigeot _MASKED_BIT_DISABLE(GEN6_BSD_SLEEP_MSG_DISABLE));
17381487f786SFrançois Tigeot
17391487f786SFrançois Tigeot intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
1740e3adcf8fSFrançois Tigeot }
1741e3adcf8fSFrançois Tigeot
gen6_bsd_ring_flush(struct drm_i915_gem_request * req,u32 mode)174271f41f3eSFrançois Tigeot static int gen6_bsd_ring_flush(struct drm_i915_gem_request *req, u32 mode)
1743e3adcf8fSFrançois Tigeot {
1744a85cb24fSFrançois Tigeot u32 cmd, *cs;
1745e3adcf8fSFrançois Tigeot
1746a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 4);
1747a85cb24fSFrançois Tigeot if (IS_ERR(cs))
1748a85cb24fSFrançois Tigeot return PTR_ERR(cs);
1749e3adcf8fSFrançois Tigeot
1750e3adcf8fSFrançois Tigeot cmd = MI_FLUSH_DW;
17511487f786SFrançois Tigeot if (INTEL_GEN(req->i915) >= 8)
17529edbd4a0SFrançois Tigeot cmd += 1;
17532c9916cdSFrançois Tigeot
17542c9916cdSFrançois Tigeot /* We always require a command barrier so that subsequent
17552c9916cdSFrançois Tigeot * commands, such as breadcrumb interrupts, are strictly ordered
17562c9916cdSFrançois Tigeot * wrt the contents of the write cache being flushed to memory
17572c9916cdSFrançois Tigeot * (and thus being coherent from the CPU).
17582c9916cdSFrançois Tigeot */
17592c9916cdSFrançois Tigeot cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
17602c9916cdSFrançois Tigeot
1761b5c29a34SFrançois Tigeot /*
1762b5c29a34SFrançois Tigeot * Bspec vol 1c.5 - video engine command streamer:
1763b5c29a34SFrançois Tigeot * "If ENABLED, all TLBs will be invalidated once the flush
1764b5c29a34SFrançois Tigeot * operation is complete. This bit is only valid when the
1765b5c29a34SFrançois Tigeot * Post-Sync Operation field is a value of 1h or 3h."
1766b5c29a34SFrançois Tigeot */
176771f41f3eSFrançois Tigeot if (mode & EMIT_INVALIDATE)
17682c9916cdSFrançois Tigeot cmd |= MI_INVALIDATE_TLB | MI_INVALIDATE_BSD;
17692c9916cdSFrançois Tigeot
1770a85cb24fSFrançois Tigeot *cs++ = cmd;
1771a85cb24fSFrançois Tigeot *cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
17721487f786SFrançois Tigeot if (INTEL_GEN(req->i915) >= 8) {
1773a85cb24fSFrançois Tigeot *cs++ = 0; /* upper addr */
1774a85cb24fSFrançois Tigeot *cs++ = 0; /* value */
17759edbd4a0SFrançois Tigeot } else {
1776a85cb24fSFrançois Tigeot *cs++ = 0;
1777a85cb24fSFrançois Tigeot *cs++ = MI_NOOP;
17789edbd4a0SFrançois Tigeot }
1779a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
17809edbd4a0SFrançois Tigeot return 0;
17819edbd4a0SFrançois Tigeot }
17829edbd4a0SFrançois Tigeot
17839edbd4a0SFrançois Tigeot static int
gen8_emit_bb_start(struct drm_i915_gem_request * req,u64 offset,u32 len,unsigned int dispatch_flags)178471f41f3eSFrançois Tigeot gen8_emit_bb_start(struct drm_i915_gem_request *req,
1785ba55f2f5SFrançois Tigeot u64 offset, u32 len,
178671f41f3eSFrançois Tigeot unsigned int dispatch_flags)
17879edbd4a0SFrançois Tigeot {
178871f41f3eSFrançois Tigeot bool ppgtt = USES_PPGTT(req->i915) &&
1789477eb7f9SFrançois Tigeot !(dispatch_flags & I915_DISPATCH_SECURE);
1790a85cb24fSFrançois Tigeot u32 *cs;
17919edbd4a0SFrançois Tigeot
1792a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 4);
1793a85cb24fSFrançois Tigeot if (IS_ERR(cs))
1794a85cb24fSFrançois Tigeot return PTR_ERR(cs);
17959edbd4a0SFrançois Tigeot
17969edbd4a0SFrançois Tigeot /* FIXME(BDW): Address space and security selectors. */
1797a85cb24fSFrançois Tigeot *cs++ = MI_BATCH_BUFFER_START_GEN8 | (ppgtt << 8) | (dispatch_flags &
1798a85cb24fSFrançois Tigeot I915_DISPATCH_RS ? MI_BATCH_RESOURCE_STREAMER : 0);
1799a85cb24fSFrançois Tigeot *cs++ = lower_32_bits(offset);
1800a85cb24fSFrançois Tigeot *cs++ = upper_32_bits(offset);
1801a85cb24fSFrançois Tigeot *cs++ = MI_NOOP;
1802a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
18039edbd4a0SFrançois Tigeot
1804e3adcf8fSFrançois Tigeot return 0;
1805e3adcf8fSFrançois Tigeot }
1806e3adcf8fSFrançois Tigeot
1807e3adcf8fSFrançois Tigeot static int
hsw_emit_bb_start(struct drm_i915_gem_request * req,u64 offset,u32 len,unsigned int dispatch_flags)180871f41f3eSFrançois Tigeot hsw_emit_bb_start(struct drm_i915_gem_request *req,
1809ba55f2f5SFrançois Tigeot u64 offset, u32 len,
181071f41f3eSFrançois Tigeot unsigned int dispatch_flags)
1811e3adcf8fSFrançois Tigeot {
1812a85cb24fSFrançois Tigeot u32 *cs;
1813e3adcf8fSFrançois Tigeot
1814a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 2);
1815a85cb24fSFrançois Tigeot if (IS_ERR(cs))
1816a85cb24fSFrançois Tigeot return PTR_ERR(cs);
1817e3adcf8fSFrançois Tigeot
1818a85cb24fSFrançois Tigeot *cs++ = MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ?
1819a05eeebfSFrançois Tigeot 0 : MI_BATCH_PPGTT_HSW | MI_BATCH_NON_SECURE_HSW) |
1820a05eeebfSFrançois Tigeot (dispatch_flags & I915_DISPATCH_RS ?
1821a85cb24fSFrançois Tigeot MI_BATCH_RESOURCE_STREAMER : 0);
1822b5c29a34SFrançois Tigeot /* bit0-7 is the length on GEN6+ */
1823a85cb24fSFrançois Tigeot *cs++ = offset;
1824a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
1825b5c29a34SFrançois Tigeot
1826b5c29a34SFrançois Tigeot return 0;
1827b5c29a34SFrançois Tigeot }
1828b5c29a34SFrançois Tigeot
1829b5c29a34SFrançois Tigeot static int
gen6_emit_bb_start(struct drm_i915_gem_request * req,u64 offset,u32 len,unsigned int dispatch_flags)183071f41f3eSFrançois Tigeot gen6_emit_bb_start(struct drm_i915_gem_request *req,
1831ba55f2f5SFrançois Tigeot u64 offset, u32 len,
183271f41f3eSFrançois Tigeot unsigned int dispatch_flags)
1833b5c29a34SFrançois Tigeot {
1834a85cb24fSFrançois Tigeot u32 *cs;
1835b5c29a34SFrançois Tigeot
1836a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 2);
1837a85cb24fSFrançois Tigeot if (IS_ERR(cs))
1838a85cb24fSFrançois Tigeot return PTR_ERR(cs);
1839b5c29a34SFrançois Tigeot
1840a85cb24fSFrançois Tigeot *cs++ = MI_BATCH_BUFFER_START | (dispatch_flags & I915_DISPATCH_SECURE ?
1841a85cb24fSFrançois Tigeot 0 : MI_BATCH_NON_SECURE_I965);
1842e3adcf8fSFrançois Tigeot /* bit0-7 is the length on GEN6+ */
1843a85cb24fSFrançois Tigeot *cs++ = offset;
1844a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
1845e3adcf8fSFrançois Tigeot
1846e3adcf8fSFrançois Tigeot return 0;
1847e3adcf8fSFrançois Tigeot }
1848e3adcf8fSFrançois Tigeot
1849e3adcf8fSFrançois Tigeot /* Blitter support (SandyBridge+) */
1850e3adcf8fSFrançois Tigeot
gen6_ring_flush(struct drm_i915_gem_request * req,u32 mode)185171f41f3eSFrançois Tigeot static int gen6_ring_flush(struct drm_i915_gem_request *req, u32 mode)
1852e3adcf8fSFrançois Tigeot {
1853a85cb24fSFrançois Tigeot u32 cmd, *cs;
1854e3adcf8fSFrançois Tigeot
1855a85cb24fSFrançois Tigeot cs = intel_ring_begin(req, 4);
1856a85cb24fSFrançois Tigeot if (IS_ERR(cs))
1857a85cb24fSFrançois Tigeot return PTR_ERR(cs);
1858e3adcf8fSFrançois Tigeot
1859e3adcf8fSFrançois Tigeot cmd = MI_FLUSH_DW;
18601487f786SFrançois Tigeot if (INTEL_GEN(req->i915) >= 8)
18619edbd4a0SFrançois Tigeot cmd += 1;
18622c9916cdSFrançois Tigeot
18632c9916cdSFrançois Tigeot /* We always require a command barrier so that subsequent
18642c9916cdSFrançois Tigeot * commands, such as breadcrumb interrupts, are strictly ordered
18652c9916cdSFrançois Tigeot * wrt the contents of the write cache being flushed to memory
18662c9916cdSFrançois Tigeot * (and thus being coherent from the CPU).
18672c9916cdSFrançois Tigeot */
18682c9916cdSFrançois Tigeot cmd |= MI_FLUSH_DW_STORE_INDEX | MI_FLUSH_DW_OP_STOREDW;
18692c9916cdSFrançois Tigeot
1870b5c29a34SFrançois Tigeot /*
1871b5c29a34SFrançois Tigeot * Bspec vol 1c.3 - blitter engine command streamer:
1872b5c29a34SFrançois Tigeot * "If ENABLED, all TLBs will be invalidated once the flush
1873b5c29a34SFrançois Tigeot * operation is complete. This bit is only valid when the
1874b5c29a34SFrançois Tigeot * Post-Sync Operation field is a value of 1h or 3h."
1875b5c29a34SFrançois Tigeot */
187671f41f3eSFrançois Tigeot if (mode & EMIT_INVALIDATE)
18772c9916cdSFrançois Tigeot cmd |= MI_INVALIDATE_TLB;
1878a85cb24fSFrançois Tigeot *cs++ = cmd;
1879a85cb24fSFrançois Tigeot *cs++ = I915_GEM_HWS_SCRATCH_ADDR | MI_FLUSH_DW_USE_GTT;
18801487f786SFrançois Tigeot if (INTEL_GEN(req->i915) >= 8) {
1881a85cb24fSFrançois Tigeot *cs++ = 0; /* upper addr */
1882a85cb24fSFrançois Tigeot *cs++ = 0; /* value */
18839edbd4a0SFrançois Tigeot } else {
1884a85cb24fSFrançois Tigeot *cs++ = 0;
1885a85cb24fSFrançois Tigeot *cs++ = MI_NOOP;
18869edbd4a0SFrançois Tigeot }
1887a85cb24fSFrançois Tigeot intel_ring_advance(req, cs);
18885d0b1887SFrançois Tigeot
1889e3adcf8fSFrançois Tigeot return 0;
1890e3adcf8fSFrançois Tigeot }
1891e3adcf8fSFrançois Tigeot
intel_ring_init_semaphores(struct drm_i915_private * dev_priv,struct intel_engine_cs * engine)18921487f786SFrançois Tigeot static void intel_ring_init_semaphores(struct drm_i915_private *dev_priv,
18931487f786SFrançois Tigeot struct intel_engine_cs *engine)
18941487f786SFrançois Tigeot {
18951487f786SFrançois Tigeot struct drm_i915_gem_object *obj;
18961487f786SFrançois Tigeot int ret, i;
18971487f786SFrançois Tigeot
1898*3f2dd94aSFrançois Tigeot if (!i915_modparams.semaphores)
18991487f786SFrançois Tigeot return;
19001487f786SFrançois Tigeot
19011e12ee3bSFrançois Tigeot if (INTEL_GEN(dev_priv) >= 8 && !dev_priv->semaphore) {
19021e12ee3bSFrançois Tigeot struct i915_vma *vma;
19031487f786SFrançois Tigeot
1904a85cb24fSFrançois Tigeot obj = i915_gem_object_create(dev_priv, PAGE_SIZE);
19051e12ee3bSFrançois Tigeot if (IS_ERR(obj))
19061e12ee3bSFrançois Tigeot goto err;
19071e12ee3bSFrançois Tigeot
1908a85cb24fSFrançois Tigeot vma = i915_vma_instance(obj, &dev_priv->ggtt.base, NULL);
19091e12ee3bSFrançois Tigeot if (IS_ERR(vma))
19101e12ee3bSFrançois Tigeot goto err_obj;
19111e12ee3bSFrançois Tigeot
19121e12ee3bSFrançois Tigeot ret = i915_gem_object_set_to_gtt_domain(obj, false);
19131e12ee3bSFrançois Tigeot if (ret)
19141e12ee3bSFrançois Tigeot goto err_obj;
19151e12ee3bSFrançois Tigeot
19161e12ee3bSFrançois Tigeot ret = i915_vma_pin(vma, 0, 0, PIN_GLOBAL | PIN_HIGH);
19171e12ee3bSFrançois Tigeot if (ret)
19181e12ee3bSFrançois Tigeot goto err_obj;
19191e12ee3bSFrançois Tigeot
19201e12ee3bSFrançois Tigeot dev_priv->semaphore = vma;
19211e12ee3bSFrançois Tigeot }
19221487f786SFrançois Tigeot
19231487f786SFrançois Tigeot if (INTEL_GEN(dev_priv) >= 8) {
19241e12ee3bSFrançois Tigeot u32 offset = i915_ggtt_offset(dev_priv->semaphore);
19251487f786SFrançois Tigeot
192671f41f3eSFrançois Tigeot engine->semaphore.sync_to = gen8_ring_sync_to;
19271487f786SFrançois Tigeot engine->semaphore.signal = gen8_xcs_signal;
19281487f786SFrançois Tigeot
19291487f786SFrançois Tigeot for (i = 0; i < I915_NUM_ENGINES; i++) {
19301e12ee3bSFrançois Tigeot u32 ring_offset;
19311487f786SFrançois Tigeot
19321487f786SFrançois Tigeot if (i != engine->id)
19331487f786SFrançois Tigeot ring_offset = offset + GEN8_SEMAPHORE_OFFSET(engine->id, i);
19341487f786SFrançois Tigeot else
19351487f786SFrançois Tigeot ring_offset = MI_SEMAPHORE_SYNC_INVALID;
19361487f786SFrançois Tigeot
19371487f786SFrançois Tigeot engine->semaphore.signal_ggtt[i] = ring_offset;
19381487f786SFrançois Tigeot }
19391487f786SFrançois Tigeot } else if (INTEL_GEN(dev_priv) >= 6) {
194071f41f3eSFrançois Tigeot engine->semaphore.sync_to = gen6_ring_sync_to;
19411487f786SFrançois Tigeot engine->semaphore.signal = gen6_signal;
19421487f786SFrançois Tigeot
19431487f786SFrançois Tigeot /*
19441487f786SFrançois Tigeot * The current semaphore is only applied on pre-gen8
19451487f786SFrançois Tigeot * platform. And there is no VCS2 ring on the pre-gen8
19461487f786SFrançois Tigeot * platform. So the semaphore between RCS and VCS2 is
19471487f786SFrançois Tigeot * initialized as INVALID. Gen8 will initialize the
19481487f786SFrançois Tigeot * sema between VCS2 and RCS later.
19491487f786SFrançois Tigeot */
19501e12ee3bSFrançois Tigeot for (i = 0; i < GEN6_NUM_SEMAPHORES; i++) {
19511487f786SFrançois Tigeot static const struct {
19521487f786SFrançois Tigeot u32 wait_mbox;
19531487f786SFrançois Tigeot i915_reg_t mbox_reg;
19541e12ee3bSFrançois Tigeot } sem_data[GEN6_NUM_SEMAPHORES][GEN6_NUM_SEMAPHORES] = {
19551e12ee3bSFrançois Tigeot [RCS_HW] = {
19561e12ee3bSFrançois Tigeot [VCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_RV, .mbox_reg = GEN6_VRSYNC },
19571e12ee3bSFrançois Tigeot [BCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_RB, .mbox_reg = GEN6_BRSYNC },
19581e12ee3bSFrançois Tigeot [VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_RVE, .mbox_reg = GEN6_VERSYNC },
19591487f786SFrançois Tigeot },
19601e12ee3bSFrançois Tigeot [VCS_HW] = {
19611e12ee3bSFrançois Tigeot [RCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VR, .mbox_reg = GEN6_RVSYNC },
19621e12ee3bSFrançois Tigeot [BCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VB, .mbox_reg = GEN6_BVSYNC },
19631e12ee3bSFrançois Tigeot [VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VVE, .mbox_reg = GEN6_VEVSYNC },
19641487f786SFrançois Tigeot },
19651e12ee3bSFrançois Tigeot [BCS_HW] = {
19661e12ee3bSFrançois Tigeot [RCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_BR, .mbox_reg = GEN6_RBSYNC },
19671e12ee3bSFrançois Tigeot [VCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_BV, .mbox_reg = GEN6_VBSYNC },
19681e12ee3bSFrançois Tigeot [VECS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_BVE, .mbox_reg = GEN6_VEBSYNC },
19691487f786SFrançois Tigeot },
19701e12ee3bSFrançois Tigeot [VECS_HW] = {
19711e12ee3bSFrançois Tigeot [RCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VER, .mbox_reg = GEN6_RVESYNC },
19721e12ee3bSFrançois Tigeot [VCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VEV, .mbox_reg = GEN6_VVESYNC },
19731e12ee3bSFrançois Tigeot [BCS_HW] = { .wait_mbox = MI_SEMAPHORE_SYNC_VEB, .mbox_reg = GEN6_BVESYNC },
19741487f786SFrançois Tigeot },
19751487f786SFrançois Tigeot };
19761487f786SFrançois Tigeot u32 wait_mbox;
19771487f786SFrançois Tigeot i915_reg_t mbox_reg;
19781487f786SFrançois Tigeot
19791e12ee3bSFrançois Tigeot if (i == engine->hw_id) {
19801487f786SFrançois Tigeot wait_mbox = MI_SEMAPHORE_SYNC_INVALID;
19811487f786SFrançois Tigeot mbox_reg = GEN6_NOSYNC;
19821487f786SFrançois Tigeot } else {
19831e12ee3bSFrançois Tigeot wait_mbox = sem_data[engine->hw_id][i].wait_mbox;
19841e12ee3bSFrançois Tigeot mbox_reg = sem_data[engine->hw_id][i].mbox_reg;
19851487f786SFrançois Tigeot }
19861487f786SFrançois Tigeot
19871487f786SFrançois Tigeot engine->semaphore.mbox.wait[i] = wait_mbox;
19881487f786SFrançois Tigeot engine->semaphore.mbox.signal[i] = mbox_reg;
19891487f786SFrançois Tigeot }
19901487f786SFrançois Tigeot }
19911e12ee3bSFrançois Tigeot
19921e12ee3bSFrançois Tigeot return;
19931e12ee3bSFrançois Tigeot
19941e12ee3bSFrançois Tigeot err_obj:
19951e12ee3bSFrançois Tigeot i915_gem_object_put(obj);
19961e12ee3bSFrançois Tigeot err:
19971e12ee3bSFrançois Tigeot DRM_DEBUG_DRIVER("Failed to allocate space for semaphores, disabling\n");
1998*3f2dd94aSFrançois Tigeot i915_modparams.semaphores = 0;
19991487f786SFrançois Tigeot }
20001487f786SFrançois Tigeot
intel_ring_init_irq(struct drm_i915_private * dev_priv,struct intel_engine_cs * engine)20011487f786SFrançois Tigeot static void intel_ring_init_irq(struct drm_i915_private *dev_priv,
20021487f786SFrançois Tigeot struct intel_engine_cs *engine)
20031487f786SFrançois Tigeot {
200487df8fc6SFrançois Tigeot engine->irq_enable_mask = GT_RENDER_USER_INTERRUPT << engine->irq_shift;
200587df8fc6SFrançois Tigeot
20061487f786SFrançois Tigeot if (INTEL_GEN(dev_priv) >= 8) {
2007303bf270SFrançois Tigeot engine->irq_enable = gen8_irq_enable;
2008303bf270SFrançois Tigeot engine->irq_disable = gen8_irq_disable;
20091487f786SFrançois Tigeot engine->irq_seqno_barrier = gen6_seqno_barrier;
20101487f786SFrançois Tigeot } else if (INTEL_GEN(dev_priv) >= 6) {
2011303bf270SFrançois Tigeot engine->irq_enable = gen6_irq_enable;
2012303bf270SFrançois Tigeot engine->irq_disable = gen6_irq_disable;
20131487f786SFrançois Tigeot engine->irq_seqno_barrier = gen6_seqno_barrier;
20141487f786SFrançois Tigeot } else if (INTEL_GEN(dev_priv) >= 5) {
2015303bf270SFrançois Tigeot engine->irq_enable = gen5_irq_enable;
2016303bf270SFrançois Tigeot engine->irq_disable = gen5_irq_disable;
2017303bf270SFrançois Tigeot engine->irq_seqno_barrier = gen5_seqno_barrier;
20181487f786SFrançois Tigeot } else if (INTEL_GEN(dev_priv) >= 3) {
2019303bf270SFrançois Tigeot engine->irq_enable = i9xx_irq_enable;
2020303bf270SFrançois Tigeot engine->irq_disable = i9xx_irq_disable;
20211487f786SFrançois Tigeot } else {
2022303bf270SFrançois Tigeot engine->irq_enable = i8xx_irq_enable;
2023303bf270SFrançois Tigeot engine->irq_disable = i8xx_irq_disable;
20241487f786SFrançois Tigeot }
20251487f786SFrançois Tigeot }
20261487f786SFrançois Tigeot
i9xx_set_default_submission(struct intel_engine_cs * engine)2027a85cb24fSFrançois Tigeot static void i9xx_set_default_submission(struct intel_engine_cs *engine)
2028a85cb24fSFrançois Tigeot {
2029a85cb24fSFrançois Tigeot engine->submit_request = i9xx_submit_request;
2030*3f2dd94aSFrançois Tigeot engine->cancel_requests = cancel_requests;
2031a85cb24fSFrançois Tigeot }
2032a85cb24fSFrançois Tigeot
gen6_bsd_set_default_submission(struct intel_engine_cs * engine)2033a85cb24fSFrançois Tigeot static void gen6_bsd_set_default_submission(struct intel_engine_cs *engine)
2034a85cb24fSFrançois Tigeot {
2035a85cb24fSFrançois Tigeot engine->submit_request = gen6_bsd_submit_request;
2036*3f2dd94aSFrançois Tigeot engine->cancel_requests = cancel_requests;
2037a85cb24fSFrançois Tigeot }
2038a85cb24fSFrançois Tigeot
intel_ring_default_vfuncs(struct drm_i915_private * dev_priv,struct intel_engine_cs * engine)20391487f786SFrançois Tigeot static void intel_ring_default_vfuncs(struct drm_i915_private *dev_priv,
20401487f786SFrançois Tigeot struct intel_engine_cs *engine)
20411487f786SFrançois Tigeot {
20421487f786SFrançois Tigeot intel_ring_init_irq(dev_priv, engine);
20431487f786SFrançois Tigeot intel_ring_init_semaphores(dev_priv, engine);
204471f41f3eSFrançois Tigeot
204571f41f3eSFrançois Tigeot engine->init_hw = init_ring_common;
20461e12ee3bSFrançois Tigeot engine->reset_hw = reset_ring_common;
204771f41f3eSFrançois Tigeot
2048a85cb24fSFrançois Tigeot engine->context_pin = intel_ring_context_pin;
2049a85cb24fSFrançois Tigeot engine->context_unpin = intel_ring_context_unpin;
2050a85cb24fSFrançois Tigeot
2051a85cb24fSFrançois Tigeot engine->request_alloc = ring_request_alloc;
2052a85cb24fSFrançois Tigeot
20534be47400SFrançois Tigeot engine->emit_breadcrumb = i9xx_emit_breadcrumb;
20544be47400SFrançois Tigeot engine->emit_breadcrumb_sz = i9xx_emit_breadcrumb_sz;
2055*3f2dd94aSFrançois Tigeot if (i915_modparams.semaphores) {
20564be47400SFrançois Tigeot int num_rings;
20574be47400SFrançois Tigeot
20584be47400SFrançois Tigeot engine->emit_breadcrumb = gen6_sema_emit_breadcrumb;
20594be47400SFrançois Tigeot
2060*3f2dd94aSFrançois Tigeot num_rings = INTEL_INFO(dev_priv)->num_rings - 1;
20614be47400SFrançois Tigeot if (INTEL_GEN(dev_priv) >= 8) {
20624be47400SFrançois Tigeot engine->emit_breadcrumb_sz += num_rings * 6;
20634be47400SFrançois Tigeot } else {
20644be47400SFrançois Tigeot engine->emit_breadcrumb_sz += num_rings * 3;
20654be47400SFrançois Tigeot if (num_rings & 1)
20664be47400SFrançois Tigeot engine->emit_breadcrumb_sz++;
20674be47400SFrançois Tigeot }
20684be47400SFrançois Tigeot }
2069a85cb24fSFrançois Tigeot
2070a85cb24fSFrançois Tigeot engine->set_default_submission = i9xx_set_default_submission;
207171f41f3eSFrançois Tigeot
207271f41f3eSFrançois Tigeot if (INTEL_GEN(dev_priv) >= 8)
207371f41f3eSFrançois Tigeot engine->emit_bb_start = gen8_emit_bb_start;
207471f41f3eSFrançois Tigeot else if (INTEL_GEN(dev_priv) >= 6)
207571f41f3eSFrançois Tigeot engine->emit_bb_start = gen6_emit_bb_start;
207671f41f3eSFrançois Tigeot else if (INTEL_GEN(dev_priv) >= 4)
207771f41f3eSFrançois Tigeot engine->emit_bb_start = i965_emit_bb_start;
2078a85cb24fSFrançois Tigeot else if (IS_I830(dev_priv) || IS_I845G(dev_priv))
207971f41f3eSFrançois Tigeot engine->emit_bb_start = i830_emit_bb_start;
208071f41f3eSFrançois Tigeot else
208171f41f3eSFrançois Tigeot engine->emit_bb_start = i915_emit_bb_start;
20821487f786SFrançois Tigeot }
20831487f786SFrançois Tigeot
intel_init_render_ring_buffer(struct intel_engine_cs * engine)208487df8fc6SFrançois Tigeot int intel_init_render_ring_buffer(struct intel_engine_cs *engine)
2085e3adcf8fSFrançois Tigeot {
208687df8fc6SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
208724edb884SFrançois Tigeot int ret;
2088e3adcf8fSFrançois Tigeot
20891487f786SFrançois Tigeot intel_ring_default_vfuncs(dev_priv, engine);
20902c9916cdSFrançois Tigeot
2091303bf270SFrançois Tigeot if (HAS_L3_DPF(dev_priv))
2092303bf270SFrançois Tigeot engine->irq_keep_mask = GT_RENDER_L3_PARITY_ERROR_INTERRUPT;
2093303bf270SFrançois Tigeot
20941487f786SFrançois Tigeot if (INTEL_GEN(dev_priv) >= 8) {
20958621f407SFrançois Tigeot engine->init_context = intel_rcs_ctx_init;
20964be47400SFrançois Tigeot engine->emit_breadcrumb = gen8_render_emit_breadcrumb;
20974be47400SFrançois Tigeot engine->emit_breadcrumb_sz = gen8_render_emit_breadcrumb_sz;
209871f41f3eSFrançois Tigeot engine->emit_flush = gen8_render_ring_flush;
2099*3f2dd94aSFrançois Tigeot if (i915_modparams.semaphores) {
21004be47400SFrançois Tigeot int num_rings;
21014be47400SFrançois Tigeot
21028621f407SFrançois Tigeot engine->semaphore.signal = gen8_rcs_signal;
21034be47400SFrançois Tigeot
2104*3f2dd94aSFrançois Tigeot num_rings = INTEL_INFO(dev_priv)->num_rings - 1;
2105a85cb24fSFrançois Tigeot engine->emit_breadcrumb_sz += num_rings * 8;
21064be47400SFrançois Tigeot }
21071487f786SFrançois Tigeot } else if (INTEL_GEN(dev_priv) >= 6) {
21088621f407SFrançois Tigeot engine->init_context = intel_rcs_ctx_init;
210971f41f3eSFrançois Tigeot engine->emit_flush = gen7_render_ring_flush;
21101487f786SFrançois Tigeot if (IS_GEN6(dev_priv))
211171f41f3eSFrançois Tigeot engine->emit_flush = gen6_render_ring_flush;
21121487f786SFrançois Tigeot } else if (IS_GEN5(dev_priv)) {
211371f41f3eSFrançois Tigeot engine->emit_flush = gen4_render_ring_flush;
2114686a02f1SFrançois Tigeot } else {
21151487f786SFrançois Tigeot if (INTEL_GEN(dev_priv) < 4)
211671f41f3eSFrançois Tigeot engine->emit_flush = gen2_render_ring_flush;
2117686a02f1SFrançois Tigeot else
211871f41f3eSFrançois Tigeot engine->emit_flush = gen4_render_ring_flush;
21198621f407SFrançois Tigeot engine->irq_enable_mask = I915_USER_INTERRUPT;
2120686a02f1SFrançois Tigeot }
212124edb884SFrançois Tigeot
21221487f786SFrançois Tigeot if (IS_HASWELL(dev_priv))
212371f41f3eSFrançois Tigeot engine->emit_bb_start = hsw_emit_bb_start;
21241487f786SFrançois Tigeot
21258621f407SFrançois Tigeot engine->init_hw = init_render_ring;
21268621f407SFrançois Tigeot engine->cleanup = render_ring_cleanup;
2127e3adcf8fSFrançois Tigeot
212887df8fc6SFrançois Tigeot ret = intel_init_ring_buffer(engine);
2129b5c29a34SFrançois Tigeot if (ret)
21302c9916cdSFrançois Tigeot return ret;
21312c9916cdSFrançois Tigeot
2132303bf270SFrançois Tigeot if (INTEL_GEN(dev_priv) >= 6) {
2133a85cb24fSFrançois Tigeot ret = intel_engine_create_scratch(engine, PAGE_SIZE);
2134303bf270SFrançois Tigeot if (ret)
2135303bf270SFrançois Tigeot return ret;
2136303bf270SFrançois Tigeot } else if (HAS_BROKEN_CS_TLB(dev_priv)) {
21371e12ee3bSFrançois Tigeot ret = intel_engine_create_scratch(engine, I830_WA_SIZE);
21382c9916cdSFrançois Tigeot if (ret)
21392c9916cdSFrançois Tigeot return ret;
2140b5c29a34SFrançois Tigeot }
2141b5c29a34SFrançois Tigeot
2142e3adcf8fSFrançois Tigeot return 0;
2143e3adcf8fSFrançois Tigeot }
2144e3adcf8fSFrançois Tigeot
intel_init_bsd_ring_buffer(struct intel_engine_cs * engine)214587df8fc6SFrançois Tigeot int intel_init_bsd_ring_buffer(struct intel_engine_cs *engine)
2146e3adcf8fSFrançois Tigeot {
214787df8fc6SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
2148686a02f1SFrançois Tigeot
21491487f786SFrançois Tigeot intel_ring_default_vfuncs(dev_priv, engine);
21501487f786SFrançois Tigeot
21511487f786SFrançois Tigeot if (INTEL_GEN(dev_priv) >= 6) {
2152686a02f1SFrançois Tigeot /* gen6 bsd needs a special wa for tail updates */
21531487f786SFrançois Tigeot if (IS_GEN6(dev_priv))
2154a85cb24fSFrançois Tigeot engine->set_default_submission = gen6_bsd_set_default_submission;
215571f41f3eSFrançois Tigeot engine->emit_flush = gen6_bsd_ring_flush;
215687df8fc6SFrançois Tigeot if (INTEL_GEN(dev_priv) < 8)
21578621f407SFrançois Tigeot engine->irq_enable_mask = GT_BSD_USER_INTERRUPT;
2158686a02f1SFrançois Tigeot } else {
21598621f407SFrançois Tigeot engine->mmio_base = BSD_RING_BASE;
216071f41f3eSFrançois Tigeot engine->emit_flush = bsd_ring_flush;
21611487f786SFrançois Tigeot if (IS_GEN5(dev_priv))
21628621f407SFrançois Tigeot engine->irq_enable_mask = ILK_BSD_USER_INTERRUPT;
21631487f786SFrançois Tigeot else
21648621f407SFrançois Tigeot engine->irq_enable_mask = I915_BSD_USER_INTERRUPT;
2165686a02f1SFrançois Tigeot }
2166e3adcf8fSFrançois Tigeot
216787df8fc6SFrançois Tigeot return intel_init_ring_buffer(engine);
2168e3adcf8fSFrançois Tigeot }
2169e3adcf8fSFrançois Tigeot
intel_init_blt_ring_buffer(struct intel_engine_cs * engine)217087df8fc6SFrançois Tigeot int intel_init_blt_ring_buffer(struct intel_engine_cs *engine)
2171e3adcf8fSFrançois Tigeot {
217287df8fc6SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
21731487f786SFrançois Tigeot
21741487f786SFrançois Tigeot intel_ring_default_vfuncs(dev_priv, engine);
21751487f786SFrançois Tigeot
217671f41f3eSFrançois Tigeot engine->emit_flush = gen6_ring_flush;
217787df8fc6SFrançois Tigeot if (INTEL_GEN(dev_priv) < 8)
21788621f407SFrançois Tigeot engine->irq_enable_mask = GT_BLT_USER_INTERRUPT;
21795d0b1887SFrançois Tigeot
218087df8fc6SFrançois Tigeot return intel_init_ring_buffer(engine);
21815d0b1887SFrançois Tigeot }
21825d0b1887SFrançois Tigeot
intel_init_vebox_ring_buffer(struct intel_engine_cs * engine)218387df8fc6SFrançois Tigeot int intel_init_vebox_ring_buffer(struct intel_engine_cs *engine)
21845d0b1887SFrançois Tigeot {
218587df8fc6SFrançois Tigeot struct drm_i915_private *dev_priv = engine->i915;
21869edbd4a0SFrançois Tigeot
21871487f786SFrançois Tigeot intel_ring_default_vfuncs(dev_priv, engine);
21881487f786SFrançois Tigeot
218971f41f3eSFrançois Tigeot engine->emit_flush = gen6_ring_flush;
21901487f786SFrançois Tigeot
219187df8fc6SFrançois Tigeot if (INTEL_GEN(dev_priv) < 8) {
21928621f407SFrançois Tigeot engine->irq_enable_mask = PM_VEBOX_USER_INTERRUPT;
2193303bf270SFrançois Tigeot engine->irq_enable = hsw_vebox_irq_enable;
2194303bf270SFrançois Tigeot engine->irq_disable = hsw_vebox_irq_disable;
219524edb884SFrançois Tigeot }
2196e3adcf8fSFrançois Tigeot
219787df8fc6SFrançois Tigeot return intel_init_ring_buffer(engine);
2198e3adcf8fSFrançois Tigeot }
2199