187df8fc6SFrançois Tigeot /*
287df8fc6SFrançois Tigeot * Copyright © 2008-2015 Intel Corporation
387df8fc6SFrançois Tigeot *
487df8fc6SFrançois Tigeot * Permission is hereby granted, free of charge, to any person obtaining a
587df8fc6SFrançois Tigeot * copy of this software and associated documentation files (the "Software"),
687df8fc6SFrançois Tigeot * to deal in the Software without restriction, including without limitation
787df8fc6SFrançois Tigeot * the rights to use, copy, modify, merge, publish, distribute, sublicense,
887df8fc6SFrançois Tigeot * and/or sell copies of the Software, and to permit persons to whom the
987df8fc6SFrançois Tigeot * Software is furnished to do so, subject to the following conditions:
1087df8fc6SFrançois Tigeot *
1187df8fc6SFrançois Tigeot * The above copyright notice and this permission notice (including the next
1287df8fc6SFrançois Tigeot * paragraph) shall be included in all copies or substantial portions of the
1387df8fc6SFrançois Tigeot * Software.
1487df8fc6SFrançois Tigeot *
1587df8fc6SFrançois Tigeot * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
1687df8fc6SFrançois Tigeot * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
1787df8fc6SFrançois Tigeot * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
1887df8fc6SFrançois Tigeot * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
1987df8fc6SFrançois Tigeot * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
2087df8fc6SFrançois Tigeot * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
2187df8fc6SFrançois Tigeot * IN THE SOFTWARE.
2287df8fc6SFrançois Tigeot *
2387df8fc6SFrançois Tigeot */
2487df8fc6SFrançois Tigeot
2587df8fc6SFrançois Tigeot #ifndef I915_GEM_REQUEST_H
2687df8fc6SFrançois Tigeot #define I915_GEM_REQUEST_H
2787df8fc6SFrançois Tigeot
286559babbSFrançois Tigeot #include <linux/dma-fence.h>
2987df8fc6SFrançois Tigeot
3087df8fc6SFrançois Tigeot #include "i915_gem.h"
311e12ee3bSFrançois Tigeot #include "i915_sw_fence.h"
3287df8fc6SFrançois Tigeot
33*3f2dd94aSFrançois Tigeot #include <uapi/drm/i915_drm.h>
34*3f2dd94aSFrançois Tigeot
354be47400SFrançois Tigeot struct drm_file;
364be47400SFrançois Tigeot struct drm_i915_gem_object;
37a85cb24fSFrançois Tigeot struct drm_i915_gem_request;
384be47400SFrançois Tigeot
3971f41f3eSFrançois Tigeot struct intel_wait {
4071f41f3eSFrançois Tigeot struct rb_node node;
4171f41f3eSFrançois Tigeot struct task_struct *tsk;
42a85cb24fSFrançois Tigeot struct drm_i915_gem_request *request;
4371f41f3eSFrançois Tigeot u32 seqno;
4471f41f3eSFrançois Tigeot };
4571f41f3eSFrançois Tigeot
4671f41f3eSFrançois Tigeot struct intel_signal_node {
4771f41f3eSFrançois Tigeot struct rb_node node;
4871f41f3eSFrançois Tigeot struct intel_wait wait;
4971f41f3eSFrançois Tigeot };
5071f41f3eSFrançois Tigeot
514be47400SFrançois Tigeot struct i915_dependency {
524be47400SFrançois Tigeot struct i915_priotree *signaler;
534be47400SFrançois Tigeot struct list_head signal_link;
544be47400SFrançois Tigeot struct list_head wait_link;
554be47400SFrançois Tigeot struct list_head dfs_link;
564be47400SFrançois Tigeot unsigned long flags;
574be47400SFrançois Tigeot #define I915_DEPENDENCY_ALLOC BIT(0)
584be47400SFrançois Tigeot };
594be47400SFrançois Tigeot
604be47400SFrançois Tigeot /* Requests exist in a complex web of interdependencies. Each request
614be47400SFrançois Tigeot * has to wait for some other request to complete before it is ready to be run
624be47400SFrançois Tigeot * (e.g. we have to wait until the pixels have been rendering into a texture
634be47400SFrançois Tigeot * before we can copy from it). We track the readiness of a request in terms
644be47400SFrançois Tigeot * of fences, but we also need to keep the dependency tree for the lifetime
654be47400SFrançois Tigeot * of the request (beyond the life of an individual fence). We use the tree
664be47400SFrançois Tigeot * at various points to reorder the requests whilst keeping the requests
674be47400SFrançois Tigeot * in order with respect to their various dependencies.
684be47400SFrançois Tigeot */
694be47400SFrançois Tigeot struct i915_priotree {
704be47400SFrançois Tigeot struct list_head signalers_list; /* those before us, we depend upon */
714be47400SFrançois Tigeot struct list_head waiters_list; /* those after us, they depend upon us */
72*3f2dd94aSFrançois Tigeot struct list_head link;
734be47400SFrançois Tigeot int priority;
74*3f2dd94aSFrançois Tigeot };
75*3f2dd94aSFrançois Tigeot
76*3f2dd94aSFrançois Tigeot enum {
77*3f2dd94aSFrançois Tigeot I915_PRIORITY_MIN = I915_CONTEXT_MIN_USER_PRIORITY - 1,
78*3f2dd94aSFrançois Tigeot I915_PRIORITY_NORMAL = I915_CONTEXT_DEFAULT_PRIORITY,
79*3f2dd94aSFrançois Tigeot I915_PRIORITY_MAX = I915_CONTEXT_MAX_USER_PRIORITY + 1,
80*3f2dd94aSFrançois Tigeot
81*3f2dd94aSFrançois Tigeot I915_PRIORITY_INVALID = INT_MIN
82*3f2dd94aSFrançois Tigeot };
83*3f2dd94aSFrançois Tigeot
84*3f2dd94aSFrançois Tigeot struct i915_gem_capture_list {
85*3f2dd94aSFrançois Tigeot struct i915_gem_capture_list *next;
86*3f2dd94aSFrançois Tigeot struct i915_vma *vma;
874be47400SFrançois Tigeot };
884be47400SFrançois Tigeot
8987df8fc6SFrançois Tigeot /**
9087df8fc6SFrançois Tigeot * Request queue structure.
9187df8fc6SFrançois Tigeot *
9287df8fc6SFrançois Tigeot * The request queue allows us to note sequence numbers that have been emitted
9387df8fc6SFrançois Tigeot * and may be associated with active buffers to be retired.
9487df8fc6SFrançois Tigeot *
9587df8fc6SFrançois Tigeot * By keeping this list, we can avoid having to do questionable sequence
9687df8fc6SFrançois Tigeot * number comparisons on buffer last_read|write_seqno. It also allows an
9787df8fc6SFrançois Tigeot * emission time to be associated with the request for tracking how far ahead
9887df8fc6SFrançois Tigeot * of the GPU the submission is.
9987df8fc6SFrançois Tigeot *
10071f41f3eSFrançois Tigeot * When modifying this structure be very aware that we perform a lockless
10171f41f3eSFrançois Tigeot * RCU lookup of it that may race against reallocation of the struct
10271f41f3eSFrançois Tigeot * from the slab freelist. We intentionally do not zero the structure on
10371f41f3eSFrançois Tigeot * allocation so that the lookup can use the dangling pointers (and is
10471f41f3eSFrançois Tigeot * cogniscent that those pointers may be wrong). Instead, everything that
10571f41f3eSFrançois Tigeot * needs to be initialised must be done so explicitly.
10671f41f3eSFrançois Tigeot *
10787df8fc6SFrançois Tigeot * The requests are reference counted.
10887df8fc6SFrançois Tigeot */
10987df8fc6SFrançois Tigeot struct drm_i915_gem_request {
1106559babbSFrançois Tigeot struct dma_fence fence;
1116559babbSFrançois Tigeot spinlock_t lock;
11287df8fc6SFrançois Tigeot
11387df8fc6SFrançois Tigeot /** On Which ring this request was generated */
11487df8fc6SFrançois Tigeot struct drm_i915_private *i915;
11587df8fc6SFrançois Tigeot
11687df8fc6SFrançois Tigeot /**
11787df8fc6SFrançois Tigeot * Context and ring buffer related to this request
11887df8fc6SFrançois Tigeot * Contexts are refcounted, so when this request is associated with a
11987df8fc6SFrançois Tigeot * context, we must increment the context's refcount, to guarantee that
12087df8fc6SFrançois Tigeot * it persists while any request is linked to it. Requests themselves
12187df8fc6SFrançois Tigeot * are also refcounted, so the request will only be freed when the last
12287df8fc6SFrançois Tigeot * reference to it is dismissed, and the code in
12387df8fc6SFrançois Tigeot * i915_gem_request_free() will then decrement the refcount on the
12487df8fc6SFrançois Tigeot * context.
12587df8fc6SFrançois Tigeot */
12687df8fc6SFrançois Tigeot struct i915_gem_context *ctx;
12787df8fc6SFrançois Tigeot struct intel_engine_cs *engine;
12871f41f3eSFrançois Tigeot struct intel_ring *ring;
1294be47400SFrançois Tigeot struct intel_timeline *timeline;
13087df8fc6SFrançois Tigeot struct intel_signal_node signaling;
13187df8fc6SFrançois Tigeot
1324be47400SFrançois Tigeot /* Fences for the various phases in the request's lifetime.
1334be47400SFrançois Tigeot *
1344be47400SFrançois Tigeot * The submit fence is used to await upon all of the request's
1354be47400SFrançois Tigeot * dependencies. When it is signaled, the request is ready to run.
1364be47400SFrançois Tigeot * It is used by the driver to then queue the request for execution.
1374be47400SFrançois Tigeot */
1381e12ee3bSFrançois Tigeot struct i915_sw_fence submit;
139*3f2dd94aSFrançois Tigeot wait_queue_entry_t submitq;
140a85cb24fSFrançois Tigeot wait_queue_head_t execute;
1414be47400SFrançois Tigeot
1424be47400SFrançois Tigeot /* A list of everyone we wait upon, and everyone who waits upon us.
1434be47400SFrançois Tigeot * Even though we will not be submitted to the hardware before the
1444be47400SFrançois Tigeot * submit fence is signaled (it waits for all external events as well
1454be47400SFrançois Tigeot * as our own requests), the scheduler still needs to know the
1464be47400SFrançois Tigeot * dependency tree for the lifetime of the request (from execbuf
1474be47400SFrançois Tigeot * to retirement), i.e. bidirectional dependency information for the
1484be47400SFrançois Tigeot * request not tied to individual fences.
1494be47400SFrançois Tigeot */
1504be47400SFrançois Tigeot struct i915_priotree priotree;
1514be47400SFrançois Tigeot struct i915_dependency dep;
1524be47400SFrançois Tigeot
153a85cb24fSFrançois Tigeot /** GEM sequence number associated with this request on the
154a85cb24fSFrançois Tigeot * global execution timeline. It is zero when the request is not
155a85cb24fSFrançois Tigeot * on the HW queue (i.e. not on the engine timeline list).
156a85cb24fSFrançois Tigeot * Its value is guarded by the timeline spinlock.
15787df8fc6SFrançois Tigeot */
158a85cb24fSFrançois Tigeot u32 global_seqno;
15987df8fc6SFrançois Tigeot
1601e12ee3bSFrançois Tigeot /** Position in the ring of the start of the request */
16187df8fc6SFrançois Tigeot u32 head;
16287df8fc6SFrançois Tigeot
16387df8fc6SFrançois Tigeot /**
1641e12ee3bSFrançois Tigeot * Position in the ring of the start of the postfix.
1651e12ee3bSFrançois Tigeot * This is required to calculate the maximum available ring space
1661e12ee3bSFrançois Tigeot * without overwriting the postfix.
16787df8fc6SFrançois Tigeot */
16887df8fc6SFrançois Tigeot u32 postfix;
16987df8fc6SFrançois Tigeot
1701e12ee3bSFrançois Tigeot /** Position in the ring of the end of the whole request */
17187df8fc6SFrançois Tigeot u32 tail;
17287df8fc6SFrançois Tigeot
1731e12ee3bSFrançois Tigeot /** Position in the ring of the end of any workarounds after the tail */
1741e12ee3bSFrançois Tigeot u32 wa_tail;
1751e12ee3bSFrançois Tigeot
1761e12ee3bSFrançois Tigeot /** Preallocate space in the ring for the emitting the request */
17787df8fc6SFrançois Tigeot u32 reserved_space;
17887df8fc6SFrançois Tigeot
17987df8fc6SFrançois Tigeot /** Batch buffer related to this request if any (used for
18087df8fc6SFrançois Tigeot * error state dump only).
18187df8fc6SFrançois Tigeot */
1821e12ee3bSFrançois Tigeot struct i915_vma *batch;
183*3f2dd94aSFrançois Tigeot /** Additional buffers requested by userspace to be captured upon
184*3f2dd94aSFrançois Tigeot * a GPU hang. The vma/obj on this list are protected by their
185*3f2dd94aSFrançois Tigeot * active reference - all objects on this list must also be
186*3f2dd94aSFrançois Tigeot * on the active_list (of their final request).
187*3f2dd94aSFrançois Tigeot */
188*3f2dd94aSFrançois Tigeot struct i915_gem_capture_list *capture_list;
18971f41f3eSFrançois Tigeot struct list_head active_list;
19087df8fc6SFrançois Tigeot
19187df8fc6SFrançois Tigeot /** Time at which this request was emitted, in jiffies. */
19287df8fc6SFrançois Tigeot unsigned long emitted_jiffies;
19387df8fc6SFrançois Tigeot
194*3f2dd94aSFrançois Tigeot bool waitboost;
195*3f2dd94aSFrançois Tigeot
19671f41f3eSFrançois Tigeot /** engine->request_list entry for this request */
19771f41f3eSFrançois Tigeot struct list_head link;
19871f41f3eSFrançois Tigeot
19971f41f3eSFrançois Tigeot /** ring->request_list entry for this request */
20071f41f3eSFrançois Tigeot struct list_head ring_link;
20187df8fc6SFrançois Tigeot
20287df8fc6SFrançois Tigeot struct drm_i915_file_private *file_priv;
20387df8fc6SFrançois Tigeot /** file_priv list entry for this request */
204a85cb24fSFrançois Tigeot struct list_head client_link;
20587df8fc6SFrançois Tigeot };
20687df8fc6SFrançois Tigeot
2076559babbSFrançois Tigeot extern const struct dma_fence_ops i915_fence_ops;
20887df8fc6SFrançois Tigeot
dma_fence_is_i915(const struct dma_fence * fence)2094be47400SFrançois Tigeot static inline bool dma_fence_is_i915(const struct dma_fence *fence)
21087df8fc6SFrançois Tigeot {
21187df8fc6SFrançois Tigeot return fence->ops == &i915_fence_ops;
21287df8fc6SFrançois Tigeot }
21387df8fc6SFrançois Tigeot
21487df8fc6SFrançois Tigeot struct drm_i915_gem_request * __must_check
21587df8fc6SFrançois Tigeot i915_gem_request_alloc(struct intel_engine_cs *engine,
21687df8fc6SFrançois Tigeot struct i915_gem_context *ctx);
21787df8fc6SFrançois Tigeot void i915_gem_request_retire_upto(struct drm_i915_gem_request *req);
21887df8fc6SFrançois Tigeot
21987df8fc6SFrançois Tigeot static inline struct drm_i915_gem_request *
to_request(struct dma_fence * fence)2206559babbSFrançois Tigeot to_request(struct dma_fence *fence)
22187df8fc6SFrançois Tigeot {
22287df8fc6SFrançois Tigeot /* We assume that NULL fence/request are interoperable */
22387df8fc6SFrançois Tigeot BUILD_BUG_ON(offsetof(struct drm_i915_gem_request, fence) != 0);
2244be47400SFrançois Tigeot GEM_BUG_ON(fence && !dma_fence_is_i915(fence));
22587df8fc6SFrançois Tigeot return container_of(fence, struct drm_i915_gem_request, fence);
22687df8fc6SFrançois Tigeot }
22787df8fc6SFrançois Tigeot
22887df8fc6SFrançois Tigeot static inline struct drm_i915_gem_request *
i915_gem_request_get(struct drm_i915_gem_request * req)22987df8fc6SFrançois Tigeot i915_gem_request_get(struct drm_i915_gem_request *req)
23087df8fc6SFrançois Tigeot {
2316559babbSFrançois Tigeot return to_request(dma_fence_get(&req->fence));
23287df8fc6SFrançois Tigeot }
23387df8fc6SFrançois Tigeot
23471f41f3eSFrançois Tigeot static inline struct drm_i915_gem_request *
i915_gem_request_get_rcu(struct drm_i915_gem_request * req)23571f41f3eSFrançois Tigeot i915_gem_request_get_rcu(struct drm_i915_gem_request *req)
23671f41f3eSFrançois Tigeot {
2376559babbSFrançois Tigeot return to_request(dma_fence_get_rcu(&req->fence));
23871f41f3eSFrançois Tigeot }
23971f41f3eSFrançois Tigeot
24087df8fc6SFrançois Tigeot static inline void
i915_gem_request_put(struct drm_i915_gem_request * req)24187df8fc6SFrançois Tigeot i915_gem_request_put(struct drm_i915_gem_request *req)
24287df8fc6SFrançois Tigeot {
2436559babbSFrançois Tigeot dma_fence_put(&req->fence);
24487df8fc6SFrançois Tigeot }
24587df8fc6SFrançois Tigeot
i915_gem_request_assign(struct drm_i915_gem_request ** pdst,struct drm_i915_gem_request * src)24687df8fc6SFrançois Tigeot static inline void i915_gem_request_assign(struct drm_i915_gem_request **pdst,
24787df8fc6SFrançois Tigeot struct drm_i915_gem_request *src)
24887df8fc6SFrançois Tigeot {
24987df8fc6SFrançois Tigeot if (src)
25087df8fc6SFrançois Tigeot i915_gem_request_get(src);
25187df8fc6SFrançois Tigeot
25287df8fc6SFrançois Tigeot if (*pdst)
25387df8fc6SFrançois Tigeot i915_gem_request_put(*pdst);
25487df8fc6SFrançois Tigeot
25587df8fc6SFrançois Tigeot *pdst = src;
25687df8fc6SFrançois Tigeot }
25787df8fc6SFrançois Tigeot
258a85cb24fSFrançois Tigeot /**
259a85cb24fSFrançois Tigeot * i915_gem_request_global_seqno - report the current global seqno
260a85cb24fSFrançois Tigeot * @request - the request
261a85cb24fSFrançois Tigeot *
262a85cb24fSFrançois Tigeot * A request is assigned a global seqno only when it is on the hardware
263a85cb24fSFrançois Tigeot * execution queue. The global seqno can be used to maintain a list of
264a85cb24fSFrançois Tigeot * requests on the same engine in retirement order, for example for
265a85cb24fSFrançois Tigeot * constructing a priority queue for waiting. Prior to its execution, or
266a85cb24fSFrançois Tigeot * if it is subsequently removed in the event of preemption, its global
267a85cb24fSFrançois Tigeot * seqno is zero. As both insertion and removal from the execution queue
268a85cb24fSFrançois Tigeot * may operate in IRQ context, it is not guarded by the usual struct_mutex
269a85cb24fSFrançois Tigeot * BKL. Instead those relying on the global seqno must be prepared for its
270a85cb24fSFrançois Tigeot * value to change between reads. Only when the request is complete can
271a85cb24fSFrançois Tigeot * the global seqno be stable (due to the memory barriers on submitting
272a85cb24fSFrançois Tigeot * the commands to the hardware to write the breadcrumb, if the HWS shows
273a85cb24fSFrançois Tigeot * that it has passed the global seqno and the global seqno is unchanged
274a85cb24fSFrançois Tigeot * after the read, it is indeed complete).
275a85cb24fSFrançois Tigeot */
276a85cb24fSFrançois Tigeot static u32
i915_gem_request_global_seqno(const struct drm_i915_gem_request * request)277a85cb24fSFrançois Tigeot i915_gem_request_global_seqno(const struct drm_i915_gem_request *request)
278a85cb24fSFrançois Tigeot {
279a85cb24fSFrançois Tigeot return READ_ONCE(request->global_seqno);
280a85cb24fSFrançois Tigeot }
281a85cb24fSFrançois Tigeot
2821e12ee3bSFrançois Tigeot int
2831e12ee3bSFrançois Tigeot i915_gem_request_await_object(struct drm_i915_gem_request *to,
2841e12ee3bSFrançois Tigeot struct drm_i915_gem_object *obj,
2851e12ee3bSFrançois Tigeot bool write);
2864be47400SFrançois Tigeot int i915_gem_request_await_dma_fence(struct drm_i915_gem_request *req,
2874be47400SFrançois Tigeot struct dma_fence *fence);
2881e12ee3bSFrançois Tigeot
2891e12ee3bSFrançois Tigeot void __i915_add_request(struct drm_i915_gem_request *req, bool flush_caches);
29087df8fc6SFrançois Tigeot #define i915_add_request(req) \
2911e12ee3bSFrançois Tigeot __i915_add_request(req, false)
29287df8fc6SFrançois Tigeot
2934be47400SFrançois Tigeot void __i915_gem_request_submit(struct drm_i915_gem_request *request);
2944be47400SFrançois Tigeot void i915_gem_request_submit(struct drm_i915_gem_request *request);
2954be47400SFrançois Tigeot
296a85cb24fSFrançois Tigeot void __i915_gem_request_unsubmit(struct drm_i915_gem_request *request);
297a85cb24fSFrançois Tigeot void i915_gem_request_unsubmit(struct drm_i915_gem_request *request);
298a85cb24fSFrançois Tigeot
29987df8fc6SFrançois Tigeot struct intel_rps_client;
30087df8fc6SFrançois Tigeot #define NO_WAITBOOST ERR_PTR(-1)
30187df8fc6SFrançois Tigeot #define IS_RPS_CLIENT(p) (!IS_ERR(p))
30287df8fc6SFrançois Tigeot #define IS_RPS_USER(p) (!IS_ERR_OR_NULL(p))
30387df8fc6SFrançois Tigeot
3044be47400SFrançois Tigeot long i915_wait_request(struct drm_i915_gem_request *req,
3051e12ee3bSFrançois Tigeot unsigned int flags,
3064be47400SFrançois Tigeot long timeout)
30771f41f3eSFrançois Tigeot __attribute__((nonnull(1)));
3081e12ee3bSFrançois Tigeot #define I915_WAIT_INTERRUPTIBLE BIT(0)
3091e12ee3bSFrançois Tigeot #define I915_WAIT_LOCKED BIT(1) /* struct_mutex held, handle GPU reset */
3104be47400SFrançois Tigeot #define I915_WAIT_ALL BIT(2) /* used by i915_gem_object_wait() */
31187df8fc6SFrançois Tigeot
31271f41f3eSFrançois Tigeot static inline u32 intel_engine_get_seqno(struct intel_engine_cs *engine);
31387df8fc6SFrançois Tigeot
31487df8fc6SFrançois Tigeot /**
31587df8fc6SFrançois Tigeot * Returns true if seq1 is later than seq2.
31687df8fc6SFrançois Tigeot */
i915_seqno_passed(u32 seq1,u32 seq2)31787df8fc6SFrançois Tigeot static inline bool i915_seqno_passed(u32 seq1, u32 seq2)
31887df8fc6SFrançois Tigeot {
31987df8fc6SFrançois Tigeot return (s32)(seq1 - seq2) >= 0;
32087df8fc6SFrançois Tigeot }
32187df8fc6SFrançois Tigeot
32287df8fc6SFrançois Tigeot static inline bool
__i915_gem_request_completed(const struct drm_i915_gem_request * req,u32 seqno)323a85cb24fSFrançois Tigeot __i915_gem_request_completed(const struct drm_i915_gem_request *req, u32 seqno)
3244be47400SFrançois Tigeot {
325a85cb24fSFrançois Tigeot GEM_BUG_ON(!seqno);
326a85cb24fSFrançois Tigeot return i915_seqno_passed(intel_engine_get_seqno(req->engine), seqno) &&
327a85cb24fSFrançois Tigeot seqno == i915_gem_request_global_seqno(req);
3284be47400SFrançois Tigeot }
3294be47400SFrançois Tigeot
3304be47400SFrançois Tigeot static inline bool
i915_gem_request_completed(const struct drm_i915_gem_request * req)33187df8fc6SFrançois Tigeot i915_gem_request_completed(const struct drm_i915_gem_request *req)
33287df8fc6SFrançois Tigeot {
333a85cb24fSFrançois Tigeot u32 seqno;
334a85cb24fSFrançois Tigeot
335a85cb24fSFrançois Tigeot seqno = i915_gem_request_global_seqno(req);
336a85cb24fSFrançois Tigeot if (!seqno)
3374be47400SFrançois Tigeot return false;
3384be47400SFrançois Tigeot
339a85cb24fSFrançois Tigeot return __i915_gem_request_completed(req, seqno);
34087df8fc6SFrançois Tigeot }
34187df8fc6SFrançois Tigeot
34271f41f3eSFrançois Tigeot /* We treat requests as fences. This is not be to confused with our
34371f41f3eSFrançois Tigeot * "fence registers" but pipeline synchronisation objects ala GL_ARB_sync.
34471f41f3eSFrançois Tigeot * We use the fences to synchronize access from the CPU with activity on the
34571f41f3eSFrançois Tigeot * GPU, for example, we should not rewrite an object's PTE whilst the GPU
34671f41f3eSFrançois Tigeot * is reading them. We also track fences at a higher level to provide
34771f41f3eSFrançois Tigeot * implicit synchronisation around GEM objects, e.g. set-domain will wait
34871f41f3eSFrançois Tigeot * for outstanding GPU rendering before marking the object ready for CPU
34971f41f3eSFrançois Tigeot * access, or a pageflip will wait until the GPU is complete before showing
35071f41f3eSFrançois Tigeot * the frame on the scanout.
35171f41f3eSFrançois Tigeot *
35271f41f3eSFrançois Tigeot * In order to use a fence, the object must track the fence it needs to
35371f41f3eSFrançois Tigeot * serialise with. For example, GEM objects want to track both read and
35471f41f3eSFrançois Tigeot * write access so that we can perform concurrent read operations between
35571f41f3eSFrançois Tigeot * the CPU and GPU engines, as well as waiting for all rendering to
35671f41f3eSFrançois Tigeot * complete, or waiting for the last GPU user of a "fence register". The
35771f41f3eSFrançois Tigeot * object then embeds a #i915_gem_active to track the most recent (in
35871f41f3eSFrançois Tigeot * retirement order) request relevant for the desired mode of access.
35971f41f3eSFrançois Tigeot * The #i915_gem_active is updated with i915_gem_active_set() to track the
36071f41f3eSFrançois Tigeot * most recent fence request, typically this is done as part of
36171f41f3eSFrançois Tigeot * i915_vma_move_to_active().
36271f41f3eSFrançois Tigeot *
36371f41f3eSFrançois Tigeot * When the #i915_gem_active completes (is retired), it will
36471f41f3eSFrançois Tigeot * signal its completion to the owner through a callback as well as mark
36571f41f3eSFrançois Tigeot * itself as idle (i915_gem_active.request == NULL). The owner
36671f41f3eSFrançois Tigeot * can then perform any action, such as delayed freeing of an active
36771f41f3eSFrançois Tigeot * resource including itself.
36871f41f3eSFrançois Tigeot */
36971f41f3eSFrançois Tigeot struct i915_gem_active;
37071f41f3eSFrançois Tigeot
37171f41f3eSFrançois Tigeot typedef void (*i915_gem_retire_fn)(struct i915_gem_active *,
37271f41f3eSFrançois Tigeot struct drm_i915_gem_request *);
37371f41f3eSFrançois Tigeot
37471f41f3eSFrançois Tigeot struct i915_gem_active {
37571f41f3eSFrançois Tigeot struct drm_i915_gem_request __rcu *request;
37671f41f3eSFrançois Tigeot struct list_head link;
37771f41f3eSFrançois Tigeot i915_gem_retire_fn retire;
37871f41f3eSFrançois Tigeot };
37971f41f3eSFrançois Tigeot
38071f41f3eSFrançois Tigeot void i915_gem_retire_noop(struct i915_gem_active *,
38171f41f3eSFrançois Tigeot struct drm_i915_gem_request *request);
38271f41f3eSFrançois Tigeot
38371f41f3eSFrançois Tigeot /**
38471f41f3eSFrançois Tigeot * init_request_active - prepares the activity tracker for use
38571f41f3eSFrançois Tigeot * @active - the active tracker
38671f41f3eSFrançois Tigeot * @func - a callback when then the tracker is retired (becomes idle),
38771f41f3eSFrançois Tigeot * can be NULL
38871f41f3eSFrançois Tigeot *
38971f41f3eSFrançois Tigeot * init_request_active() prepares the embedded @active struct for use as
39071f41f3eSFrançois Tigeot * an activity tracker, that is for tracking the last known active request
39171f41f3eSFrançois Tigeot * associated with it. When the last request becomes idle, when it is retired
39271f41f3eSFrançois Tigeot * after completion, the optional callback @func is invoked.
39371f41f3eSFrançois Tigeot */
39471f41f3eSFrançois Tigeot static inline void
init_request_active(struct i915_gem_active * active,i915_gem_retire_fn retire)39571f41f3eSFrançois Tigeot init_request_active(struct i915_gem_active *active,
39671f41f3eSFrançois Tigeot i915_gem_retire_fn retire)
39771f41f3eSFrançois Tigeot {
39871f41f3eSFrançois Tigeot INIT_LIST_HEAD(&active->link);
39971f41f3eSFrançois Tigeot active->retire = retire ?: i915_gem_retire_noop;
40071f41f3eSFrançois Tigeot }
40171f41f3eSFrançois Tigeot
40271f41f3eSFrançois Tigeot /**
40371f41f3eSFrançois Tigeot * i915_gem_active_set - updates the tracker to watch the current request
40471f41f3eSFrançois Tigeot * @active - the active tracker
40571f41f3eSFrançois Tigeot * @request - the request to watch
40671f41f3eSFrançois Tigeot *
40771f41f3eSFrançois Tigeot * i915_gem_active_set() watches the given @request for completion. Whilst
40871f41f3eSFrançois Tigeot * that @request is busy, the @active reports busy. When that @request is
40971f41f3eSFrançois Tigeot * retired, the @active tracker is updated to report idle.
41071f41f3eSFrançois Tigeot */
41171f41f3eSFrançois Tigeot static inline void
i915_gem_active_set(struct i915_gem_active * active,struct drm_i915_gem_request * request)41271f41f3eSFrançois Tigeot i915_gem_active_set(struct i915_gem_active *active,
41371f41f3eSFrançois Tigeot struct drm_i915_gem_request *request)
41471f41f3eSFrançois Tigeot {
41571f41f3eSFrançois Tigeot list_move(&active->link, &request->active_list);
41671f41f3eSFrançois Tigeot rcu_assign_pointer(active->request, request);
41771f41f3eSFrançois Tigeot }
41871f41f3eSFrançois Tigeot
4194be47400SFrançois Tigeot /**
4204be47400SFrançois Tigeot * i915_gem_active_set_retire_fn - updates the retirement callback
4214be47400SFrançois Tigeot * @active - the active tracker
4224be47400SFrançois Tigeot * @fn - the routine called when the request is retired
4234be47400SFrançois Tigeot * @mutex - struct_mutex used to guard retirements
4244be47400SFrançois Tigeot *
4254be47400SFrançois Tigeot * i915_gem_active_set_retire_fn() updates the function pointer that
4264be47400SFrançois Tigeot * is called when the final request associated with the @active tracker
4274be47400SFrançois Tigeot * is retired.
4284be47400SFrançois Tigeot */
4294be47400SFrançois Tigeot static inline void
i915_gem_active_set_retire_fn(struct i915_gem_active * active,i915_gem_retire_fn fn,struct lock * mutex)4304be47400SFrançois Tigeot i915_gem_active_set_retire_fn(struct i915_gem_active *active,
4314be47400SFrançois Tigeot i915_gem_retire_fn fn,
4324be47400SFrançois Tigeot struct lock *mutex)
4334be47400SFrançois Tigeot {
4344be47400SFrançois Tigeot lockdep_assert_held(mutex);
4354be47400SFrançois Tigeot active->retire = fn ?: i915_gem_retire_noop;
4364be47400SFrançois Tigeot }
4374be47400SFrançois Tigeot
43871f41f3eSFrançois Tigeot static inline struct drm_i915_gem_request *
__i915_gem_active_peek(const struct i915_gem_active * active)43971f41f3eSFrançois Tigeot __i915_gem_active_peek(const struct i915_gem_active *active)
44071f41f3eSFrançois Tigeot {
44171f41f3eSFrançois Tigeot /* Inside the error capture (running with the driver in an unknown
44271f41f3eSFrançois Tigeot * state), we want to bend the rules slightly (a lot).
44371f41f3eSFrançois Tigeot *
44471f41f3eSFrançois Tigeot * Work is in progress to make it safer, in the meantime this keeps
44571f41f3eSFrançois Tigeot * the known issue from spamming the logs.
44671f41f3eSFrançois Tigeot */
44771f41f3eSFrançois Tigeot return rcu_dereference_protected(active->request, 1);
44871f41f3eSFrançois Tigeot }
44971f41f3eSFrançois Tigeot
45071f41f3eSFrançois Tigeot /**
45171f41f3eSFrançois Tigeot * i915_gem_active_raw - return the active request
45271f41f3eSFrançois Tigeot * @active - the active tracker
45371f41f3eSFrançois Tigeot *
45471f41f3eSFrançois Tigeot * i915_gem_active_raw() returns the current request being tracked, or NULL.
45571f41f3eSFrançois Tigeot * It does not obtain a reference on the request for the caller, so the caller
45671f41f3eSFrançois Tigeot * must hold struct_mutex.
45771f41f3eSFrançois Tigeot */
45871f41f3eSFrançois Tigeot static inline struct drm_i915_gem_request *
i915_gem_active_raw(const struct i915_gem_active * active,struct lock * mutex)45971f41f3eSFrançois Tigeot i915_gem_active_raw(const struct i915_gem_active *active, struct lock *mutex)
46071f41f3eSFrançois Tigeot {
46171f41f3eSFrançois Tigeot return rcu_dereference_protected(active->request,
46271f41f3eSFrançois Tigeot lockdep_is_held(mutex));
46371f41f3eSFrançois Tigeot }
46471f41f3eSFrançois Tigeot
46571f41f3eSFrançois Tigeot /**
46671f41f3eSFrançois Tigeot * i915_gem_active_peek - report the active request being monitored
46771f41f3eSFrançois Tigeot * @active - the active tracker
46871f41f3eSFrançois Tigeot *
46971f41f3eSFrançois Tigeot * i915_gem_active_peek() returns the current request being tracked if
47071f41f3eSFrançois Tigeot * still active, or NULL. It does not obtain a reference on the request
47171f41f3eSFrançois Tigeot * for the caller, so the caller must hold struct_mutex.
47271f41f3eSFrançois Tigeot */
47371f41f3eSFrançois Tigeot static inline struct drm_i915_gem_request *
i915_gem_active_peek(const struct i915_gem_active * active,struct lock * mutex)47471f41f3eSFrançois Tigeot i915_gem_active_peek(const struct i915_gem_active *active, struct lock *mutex)
47571f41f3eSFrançois Tigeot {
47671f41f3eSFrançois Tigeot struct drm_i915_gem_request *request;
47771f41f3eSFrançois Tigeot
47871f41f3eSFrançois Tigeot request = i915_gem_active_raw(active, mutex);
47971f41f3eSFrançois Tigeot if (!request || i915_gem_request_completed(request))
48071f41f3eSFrançois Tigeot return NULL;
48171f41f3eSFrançois Tigeot
48271f41f3eSFrançois Tigeot return request;
48371f41f3eSFrançois Tigeot }
48471f41f3eSFrançois Tigeot
48571f41f3eSFrançois Tigeot /**
48671f41f3eSFrançois Tigeot * i915_gem_active_get - return a reference to the active request
48771f41f3eSFrançois Tigeot * @active - the active tracker
48871f41f3eSFrançois Tigeot *
48971f41f3eSFrançois Tigeot * i915_gem_active_get() returns a reference to the active request, or NULL
49071f41f3eSFrançois Tigeot * if the active tracker is idle. The caller must hold struct_mutex.
49171f41f3eSFrançois Tigeot */
49271f41f3eSFrançois Tigeot static inline struct drm_i915_gem_request *
i915_gem_active_get(const struct i915_gem_active * active,struct lock * mutex)49371f41f3eSFrançois Tigeot i915_gem_active_get(const struct i915_gem_active *active, struct lock *mutex)
49471f41f3eSFrançois Tigeot {
49571f41f3eSFrançois Tigeot return i915_gem_request_get(i915_gem_active_peek(active, mutex));
49671f41f3eSFrançois Tigeot }
49771f41f3eSFrançois Tigeot
49871f41f3eSFrançois Tigeot /**
49971f41f3eSFrançois Tigeot * __i915_gem_active_get_rcu - return a reference to the active request
50071f41f3eSFrançois Tigeot * @active - the active tracker
50171f41f3eSFrançois Tigeot *
50271f41f3eSFrançois Tigeot * __i915_gem_active_get() returns a reference to the active request, or NULL
50371f41f3eSFrançois Tigeot * if the active tracker is idle. The caller must hold the RCU read lock, but
50471f41f3eSFrançois Tigeot * the returned pointer is safe to use outside of RCU.
50571f41f3eSFrançois Tigeot */
50671f41f3eSFrançois Tigeot static inline struct drm_i915_gem_request *
__i915_gem_active_get_rcu(const struct i915_gem_active * active)50771f41f3eSFrançois Tigeot __i915_gem_active_get_rcu(const struct i915_gem_active *active)
50871f41f3eSFrançois Tigeot {
50971f41f3eSFrançois Tigeot /* Performing a lockless retrieval of the active request is super
510a85cb24fSFrançois Tigeot * tricky. SLAB_TYPESAFE_BY_RCU merely guarantees that the backing
51171f41f3eSFrançois Tigeot * slab of request objects will not be freed whilst we hold the
51271f41f3eSFrançois Tigeot * RCU read lock. It does not guarantee that the request itself
51371f41f3eSFrançois Tigeot * will not be freed and then *reused*. Viz,
51471f41f3eSFrançois Tigeot *
51571f41f3eSFrançois Tigeot * Thread A Thread B
51671f41f3eSFrançois Tigeot *
51771f41f3eSFrançois Tigeot * req = active.request
51871f41f3eSFrançois Tigeot * retire(req) -> free(req);
51971f41f3eSFrançois Tigeot * (req is now first on the slab freelist)
52071f41f3eSFrançois Tigeot * active.request = NULL
52171f41f3eSFrançois Tigeot *
52271f41f3eSFrançois Tigeot * req = new submission on a new object
52371f41f3eSFrançois Tigeot * ref(req)
52471f41f3eSFrançois Tigeot *
52571f41f3eSFrançois Tigeot * To prevent the request from being reused whilst the caller
52671f41f3eSFrançois Tigeot * uses it, we take a reference like normal. Whilst acquiring
52771f41f3eSFrançois Tigeot * the reference we check that it is not in a destroyed state
52871f41f3eSFrançois Tigeot * (refcnt == 0). That prevents the request being reallocated
52971f41f3eSFrançois Tigeot * whilst the caller holds on to it. To check that the request
53071f41f3eSFrançois Tigeot * was not reallocated as we acquired the reference we have to
53171f41f3eSFrançois Tigeot * check that our request remains the active request across
53271f41f3eSFrançois Tigeot * the lookup, in the same manner as a seqlock. The visibility
53371f41f3eSFrançois Tigeot * of the pointer versus the reference counting is controlled
53471f41f3eSFrançois Tigeot * by using RCU barriers (rcu_dereference and rcu_assign_pointer).
53571f41f3eSFrançois Tigeot *
53671f41f3eSFrançois Tigeot * In the middle of all that, we inspect whether the request is
53771f41f3eSFrançois Tigeot * complete. Retiring is lazy so the request may be completed long
53871f41f3eSFrançois Tigeot * before the active tracker is updated. Querying whether the
53971f41f3eSFrançois Tigeot * request is complete is far cheaper (as it involves no locked
54071f41f3eSFrançois Tigeot * instructions setting cachelines to exclusive) than acquiring
54171f41f3eSFrançois Tigeot * the reference, so we do it first. The RCU read lock ensures the
54271f41f3eSFrançois Tigeot * pointer dereference is valid, but does not ensure that the
54371f41f3eSFrançois Tigeot * seqno nor HWS is the right one! However, if the request was
54471f41f3eSFrançois Tigeot * reallocated, that means the active tracker's request was complete.
54571f41f3eSFrançois Tigeot * If the new request is also complete, then both are and we can
54671f41f3eSFrançois Tigeot * just report the active tracker is idle. If the new request is
54771f41f3eSFrançois Tigeot * incomplete, then we acquire a reference on it and check that
54871f41f3eSFrançois Tigeot * it remained the active request.
54971f41f3eSFrançois Tigeot *
55071f41f3eSFrançois Tigeot * It is then imperative that we do not zero the request on
55171f41f3eSFrançois Tigeot * reallocation, so that we can chase the dangling pointers!
55271f41f3eSFrançois Tigeot * See i915_gem_request_alloc().
55371f41f3eSFrançois Tigeot */
55471f41f3eSFrançois Tigeot do {
55571f41f3eSFrançois Tigeot struct drm_i915_gem_request *request;
55671f41f3eSFrançois Tigeot
55771f41f3eSFrançois Tigeot request = rcu_dereference(active->request);
55871f41f3eSFrançois Tigeot if (!request || i915_gem_request_completed(request))
55971f41f3eSFrançois Tigeot return NULL;
56071f41f3eSFrançois Tigeot
5611e12ee3bSFrançois Tigeot /* An especially silly compiler could decide to recompute the
5621e12ee3bSFrançois Tigeot * result of i915_gem_request_completed, more specifically
5631e12ee3bSFrançois Tigeot * re-emit the load for request->fence.seqno. A race would catch
5641e12ee3bSFrançois Tigeot * a later seqno value, which could flip the result from true to
5651e12ee3bSFrançois Tigeot * false. Which means part of the instructions below might not
5661e12ee3bSFrançois Tigeot * be executed, while later on instructions are executed. Due to
5671e12ee3bSFrançois Tigeot * barriers within the refcounting the inconsistency can't reach
5681e12ee3bSFrançois Tigeot * past the call to i915_gem_request_get_rcu, but not executing
5691e12ee3bSFrançois Tigeot * that while still executing i915_gem_request_put() creates
5701e12ee3bSFrançois Tigeot * havoc enough. Prevent this with a compiler barrier.
5711e12ee3bSFrançois Tigeot */
5721e12ee3bSFrançois Tigeot barrier();
5731e12ee3bSFrançois Tigeot
57471f41f3eSFrançois Tigeot request = i915_gem_request_get_rcu(request);
57571f41f3eSFrançois Tigeot
57671f41f3eSFrançois Tigeot /* What stops the following rcu_access_pointer() from occurring
57771f41f3eSFrançois Tigeot * before the above i915_gem_request_get_rcu()? If we were
57871f41f3eSFrançois Tigeot * to read the value before pausing to get the reference to
57971f41f3eSFrançois Tigeot * the request, we may not notice a change in the active
58071f41f3eSFrançois Tigeot * tracker.
58171f41f3eSFrançois Tigeot *
58271f41f3eSFrançois Tigeot * The rcu_access_pointer() is a mere compiler barrier, which
58371f41f3eSFrançois Tigeot * means both the CPU and compiler are free to perform the
58471f41f3eSFrançois Tigeot * memory read without constraint. The compiler only has to
58571f41f3eSFrançois Tigeot * ensure that any operations after the rcu_access_pointer()
58671f41f3eSFrançois Tigeot * occur afterwards in program order. This means the read may
58771f41f3eSFrançois Tigeot * be performed earlier by an out-of-order CPU, or adventurous
58871f41f3eSFrançois Tigeot * compiler.
58971f41f3eSFrançois Tigeot *
59071f41f3eSFrançois Tigeot * The atomic operation at the heart of
5916559babbSFrançois Tigeot * i915_gem_request_get_rcu(), see dma_fence_get_rcu(), is
59271f41f3eSFrançois Tigeot * atomic_inc_not_zero() which is only a full memory barrier
59371f41f3eSFrançois Tigeot * when successful. That is, if i915_gem_request_get_rcu()
59471f41f3eSFrançois Tigeot * returns the request (and so with the reference counted
59571f41f3eSFrançois Tigeot * incremented) then the following read for rcu_access_pointer()
59671f41f3eSFrançois Tigeot * must occur after the atomic operation and so confirm
59771f41f3eSFrançois Tigeot * that this request is the one currently being tracked.
59871f41f3eSFrançois Tigeot *
59971f41f3eSFrançois Tigeot * The corresponding write barrier is part of
60071f41f3eSFrançois Tigeot * rcu_assign_pointer().
60171f41f3eSFrançois Tigeot */
60271f41f3eSFrançois Tigeot if (!request || request == rcu_access_pointer(active->request))
60371f41f3eSFrançois Tigeot return rcu_pointer_handoff(request);
60471f41f3eSFrançois Tigeot
60571f41f3eSFrançois Tigeot i915_gem_request_put(request);
60671f41f3eSFrançois Tigeot } while (1);
60771f41f3eSFrançois Tigeot }
60871f41f3eSFrançois Tigeot
60971f41f3eSFrançois Tigeot /**
61071f41f3eSFrançois Tigeot * i915_gem_active_get_unlocked - return a reference to the active request
61171f41f3eSFrançois Tigeot * @active - the active tracker
61271f41f3eSFrançois Tigeot *
61371f41f3eSFrançois Tigeot * i915_gem_active_get_unlocked() returns a reference to the active request,
61471f41f3eSFrançois Tigeot * or NULL if the active tracker is idle. The reference is obtained under RCU,
61571f41f3eSFrançois Tigeot * so no locking is required by the caller.
61671f41f3eSFrançois Tigeot *
61771f41f3eSFrançois Tigeot * The reference should be freed with i915_gem_request_put().
61871f41f3eSFrançois Tigeot */
61971f41f3eSFrançois Tigeot static inline struct drm_i915_gem_request *
i915_gem_active_get_unlocked(const struct i915_gem_active * active)62071f41f3eSFrançois Tigeot i915_gem_active_get_unlocked(const struct i915_gem_active *active)
62171f41f3eSFrançois Tigeot {
62271f41f3eSFrançois Tigeot struct drm_i915_gem_request *request;
62371f41f3eSFrançois Tigeot
62471f41f3eSFrançois Tigeot rcu_read_lock();
62571f41f3eSFrançois Tigeot request = __i915_gem_active_get_rcu(active);
62671f41f3eSFrançois Tigeot rcu_read_unlock();
62771f41f3eSFrançois Tigeot
62871f41f3eSFrançois Tigeot return request;
62971f41f3eSFrançois Tigeot }
63071f41f3eSFrançois Tigeot
63171f41f3eSFrançois Tigeot /**
63271f41f3eSFrançois Tigeot * i915_gem_active_isset - report whether the active tracker is assigned
63371f41f3eSFrançois Tigeot * @active - the active tracker
63471f41f3eSFrançois Tigeot *
63571f41f3eSFrançois Tigeot * i915_gem_active_isset() returns true if the active tracker is currently
63671f41f3eSFrançois Tigeot * assigned to a request. Due to the lazy retiring, that request may be idle
63771f41f3eSFrançois Tigeot * and this may report stale information.
63871f41f3eSFrançois Tigeot */
63971f41f3eSFrançois Tigeot static inline bool
i915_gem_active_isset(const struct i915_gem_active * active)64071f41f3eSFrançois Tigeot i915_gem_active_isset(const struct i915_gem_active *active)
64171f41f3eSFrançois Tigeot {
64271f41f3eSFrançois Tigeot return rcu_access_pointer(active->request);
64371f41f3eSFrançois Tigeot }
64471f41f3eSFrançois Tigeot
64571f41f3eSFrançois Tigeot /**
64671f41f3eSFrançois Tigeot * i915_gem_active_wait - waits until the request is completed
64771f41f3eSFrançois Tigeot * @active - the active request on which to wait
6481e12ee3bSFrançois Tigeot * @flags - how to wait
64971f41f3eSFrançois Tigeot * @timeout - how long to wait at most
65071f41f3eSFrançois Tigeot * @rps - userspace client to charge for a waitboost
65171f41f3eSFrançois Tigeot *
6524be47400SFrançois Tigeot * i915_gem_active_wait() waits until the request is completed before
65371f41f3eSFrançois Tigeot * returning, without requiring any locks to be held. Note that it does not
65471f41f3eSFrançois Tigeot * retire any requests before returning.
65571f41f3eSFrançois Tigeot *
65671f41f3eSFrançois Tigeot * This function relies on RCU in order to acquire the reference to the active
65771f41f3eSFrançois Tigeot * request without holding any locks. See __i915_gem_active_get_rcu() for the
65871f41f3eSFrançois Tigeot * glory details on how that is managed. Once the reference is acquired, we
65971f41f3eSFrançois Tigeot * can then wait upon the request, and afterwards release our reference,
66071f41f3eSFrançois Tigeot * free of any locking.
66171f41f3eSFrançois Tigeot *
66271f41f3eSFrançois Tigeot * This function wraps i915_wait_request(), see it for the full details on
66371f41f3eSFrançois Tigeot * the arguments.
66471f41f3eSFrançois Tigeot *
66571f41f3eSFrançois Tigeot * Returns 0 if successful, or a negative error code.
66671f41f3eSFrançois Tigeot */
66771f41f3eSFrançois Tigeot static inline int
i915_gem_active_wait(const struct i915_gem_active * active,unsigned int flags)6684be47400SFrançois Tigeot i915_gem_active_wait(const struct i915_gem_active *active, unsigned int flags)
66971f41f3eSFrançois Tigeot {
67071f41f3eSFrançois Tigeot struct drm_i915_gem_request *request;
6714be47400SFrançois Tigeot long ret = 0;
67271f41f3eSFrançois Tigeot
67371f41f3eSFrançois Tigeot request = i915_gem_active_get_unlocked(active);
67471f41f3eSFrançois Tigeot if (request) {
6754be47400SFrançois Tigeot ret = i915_wait_request(request, flags, MAX_SCHEDULE_TIMEOUT);
67671f41f3eSFrançois Tigeot i915_gem_request_put(request);
67771f41f3eSFrançois Tigeot }
67871f41f3eSFrançois Tigeot
6794be47400SFrançois Tigeot return ret < 0 ? ret : 0;
68071f41f3eSFrançois Tigeot }
68171f41f3eSFrançois Tigeot
68271f41f3eSFrançois Tigeot /**
68371f41f3eSFrançois Tigeot * i915_gem_active_retire - waits until the request is retired
68471f41f3eSFrançois Tigeot * @active - the active request on which to wait
68571f41f3eSFrançois Tigeot *
68671f41f3eSFrançois Tigeot * i915_gem_active_retire() waits until the request is completed,
68771f41f3eSFrançois Tigeot * and then ensures that at least the retirement handler for this
68871f41f3eSFrançois Tigeot * @active tracker is called before returning. If the @active
68971f41f3eSFrançois Tigeot * tracker is idle, the function returns immediately.
69071f41f3eSFrançois Tigeot */
69171f41f3eSFrançois Tigeot static inline int __must_check
i915_gem_active_retire(struct i915_gem_active * active,struct lock * mutex)69271f41f3eSFrançois Tigeot i915_gem_active_retire(struct i915_gem_active *active,
69371f41f3eSFrançois Tigeot struct lock *mutex)
69471f41f3eSFrançois Tigeot {
69571f41f3eSFrançois Tigeot struct drm_i915_gem_request *request;
6964be47400SFrançois Tigeot long ret;
69771f41f3eSFrançois Tigeot
69871f41f3eSFrançois Tigeot request = i915_gem_active_raw(active, mutex);
69971f41f3eSFrançois Tigeot if (!request)
70071f41f3eSFrançois Tigeot return 0;
70171f41f3eSFrançois Tigeot
7021e12ee3bSFrançois Tigeot ret = i915_wait_request(request,
7031e12ee3bSFrançois Tigeot I915_WAIT_INTERRUPTIBLE | I915_WAIT_LOCKED,
7044be47400SFrançois Tigeot MAX_SCHEDULE_TIMEOUT);
7054be47400SFrançois Tigeot if (ret < 0)
70671f41f3eSFrançois Tigeot return ret;
70771f41f3eSFrançois Tigeot
70871f41f3eSFrançois Tigeot list_del_init(&active->link);
70971f41f3eSFrançois Tigeot RCU_INIT_POINTER(active->request, NULL);
71071f41f3eSFrançois Tigeot
71171f41f3eSFrançois Tigeot active->retire(active, request);
71271f41f3eSFrançois Tigeot
71371f41f3eSFrançois Tigeot return 0;
71471f41f3eSFrançois Tigeot }
71571f41f3eSFrançois Tigeot
71671f41f3eSFrançois Tigeot #define for_each_active(mask, idx) \
71771f41f3eSFrançois Tigeot for (; mask ? idx = ffs(mask) - 1, 1 : 0; mask &= ~BIT(idx))
71871f41f3eSFrançois Tigeot
71987df8fc6SFrançois Tigeot #endif /* I915_GEM_REQUEST_H */
720