1 /* $NetBSD: linux_dma_fence.c,v 1.42 2022/09/01 09:37:06 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2018 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Taylor R. Campbell.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #include <sys/cdefs.h>
33 __KERNEL_RCSID(0, "$NetBSD: linux_dma_fence.c,v 1.42 2022/09/01 09:37:06 riastradh Exp $");
34
35 #include <sys/atomic.h>
36 #include <sys/condvar.h>
37 #include <sys/lock.h>
38 #include <sys/queue.h>
39 #include <sys/sdt.h>
40
41 #include <linux/atomic.h>
42 #include <linux/dma-fence.h>
43 #include <linux/errno.h>
44 #include <linux/kref.h>
45 #include <linux/sched.h>
46 #include <linux/spinlock.h>
47
48 #define FENCE_MAGIC_GOOD 0x607ba424048c37e5ULL
49 #define FENCE_MAGIC_BAD 0x7641ca721344505fULL
50
51 SDT_PROBE_DEFINE1(sdt, drm, fence, init,
52 "struct dma_fence *"/*fence*/);
53 SDT_PROBE_DEFINE1(sdt, drm, fence, reset,
54 "struct dma_fence *"/*fence*/);
55 SDT_PROBE_DEFINE1(sdt, drm, fence, release,
56 "struct dma_fence *"/*fence*/);
57 SDT_PROBE_DEFINE1(sdt, drm, fence, free,
58 "struct dma_fence *"/*fence*/);
59 SDT_PROBE_DEFINE1(sdt, drm, fence, destroy,
60 "struct dma_fence *"/*fence*/);
61
62 SDT_PROBE_DEFINE1(sdt, drm, fence, enable_signaling,
63 "struct dma_fence *"/*fence*/);
64 SDT_PROBE_DEFINE2(sdt, drm, fence, add_callback,
65 "struct dma_fence *"/*fence*/,
66 "struct dma_fence_callback *"/*callback*/);
67 SDT_PROBE_DEFINE2(sdt, drm, fence, remove_callback,
68 "struct dma_fence *"/*fence*/,
69 "struct dma_fence_callback *"/*callback*/);
70 SDT_PROBE_DEFINE2(sdt, drm, fence, callback,
71 "struct dma_fence *"/*fence*/,
72 "struct dma_fence_callback *"/*callback*/);
73 SDT_PROBE_DEFINE1(sdt, drm, fence, test,
74 "struct dma_fence *"/*fence*/);
75 SDT_PROBE_DEFINE2(sdt, drm, fence, set_error,
76 "struct dma_fence *"/*fence*/,
77 "int"/*error*/);
78 SDT_PROBE_DEFINE1(sdt, drm, fence, signal,
79 "struct dma_fence *"/*fence*/);
80
81 SDT_PROBE_DEFINE3(sdt, drm, fence, wait_start,
82 "struct dma_fence *"/*fence*/,
83 "bool"/*intr*/,
84 "long"/*timeout*/);
85 SDT_PROBE_DEFINE2(sdt, drm, fence, wait_done,
86 "struct dma_fence *"/*fence*/,
87 "long"/*ret*/);
88
89 /*
90 * linux_dma_fence_trace
91 *
92 * True if we print DMA_FENCE_TRACE messages, false if not. These
93 * are extremely noisy, too much even for AB_VERBOSE and AB_DEBUG
94 * in boothowto.
95 */
96 int linux_dma_fence_trace = 0;
97
98 static struct {
99 spinlock_t lock;
100 struct dma_fence fence;
101 } dma_fence_stub __cacheline_aligned;
102
dma_fence_stub_name(struct dma_fence * f)103 static const char *dma_fence_stub_name(struct dma_fence *f)
104 {
105
106 KASSERT(f == &dma_fence_stub.fence);
107 return "stub";
108 }
109
110 static void
dma_fence_stub_release(struct dma_fence * f)111 dma_fence_stub_release(struct dma_fence *f)
112 {
113
114 KASSERT(f == &dma_fence_stub.fence);
115 dma_fence_destroy(f);
116 }
117
118 static const struct dma_fence_ops dma_fence_stub_ops = {
119 .get_driver_name = dma_fence_stub_name,
120 .get_timeline_name = dma_fence_stub_name,
121 .release = dma_fence_stub_release,
122 };
123
124 /*
125 * linux_dma_fences_init(), linux_dma_fences_fini()
126 *
127 * Set up and tear down module state.
128 */
129 void
linux_dma_fences_init(void)130 linux_dma_fences_init(void)
131 {
132 int error __diagused;
133
134 spin_lock_init(&dma_fence_stub.lock);
135 dma_fence_init(&dma_fence_stub.fence, &dma_fence_stub_ops,
136 &dma_fence_stub.lock, /*context*/0, /*seqno*/0);
137 error = dma_fence_signal(&dma_fence_stub.fence);
138 KASSERTMSG(error == 0, "error=%d", error);
139 }
140
141 void
linux_dma_fences_fini(void)142 linux_dma_fences_fini(void)
143 {
144
145 dma_fence_put(&dma_fence_stub.fence);
146 spin_lock_destroy(&dma_fence_stub.lock);
147 }
148
149 /*
150 * dma_fence_referenced_p(fence)
151 *
152 * True if fence has a positive reference count. True after
153 * dma_fence_init; after the last dma_fence_put, this becomes
154 * false. The fence must have been initialized and must not have
155 * been destroyed.
156 */
157 static inline bool __diagused
dma_fence_referenced_p(struct dma_fence * fence)158 dma_fence_referenced_p(struct dma_fence *fence)
159 {
160
161 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence);
162 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence);
163
164 return kref_referenced_p(&fence->refcount);
165 }
166
167 /*
168 * dma_fence_init(fence, ops, lock, context, seqno)
169 *
170 * Initialize fence. Caller should call dma_fence_destroy when
171 * done, after all references have been released.
172 */
173 void
dma_fence_init(struct dma_fence * fence,const struct dma_fence_ops * ops,spinlock_t * lock,uint64_t context,uint64_t seqno)174 dma_fence_init(struct dma_fence *fence, const struct dma_fence_ops *ops,
175 spinlock_t *lock, uint64_t context, uint64_t seqno)
176 {
177
178 kref_init(&fence->refcount);
179 fence->lock = lock;
180 fence->flags = 0;
181 fence->context = context;
182 fence->seqno = seqno;
183 fence->ops = ops;
184 fence->error = 0;
185 TAILQ_INIT(&fence->f_callbacks);
186 cv_init(&fence->f_cv, "dmafence");
187
188 #ifdef DIAGNOSTIC
189 fence->f_magic = FENCE_MAGIC_GOOD;
190 #endif
191
192 SDT_PROBE1(sdt, drm, fence, init, fence);
193 }
194
195 /*
196 * dma_fence_reset(fence)
197 *
198 * Ensure fence is in a quiescent state. Allowed either for newly
199 * initialized or freed fences, but not fences with more than one
200 * reference.
201 *
202 * XXX extension to Linux API
203 */
204 void
dma_fence_reset(struct dma_fence * fence,const struct dma_fence_ops * ops,spinlock_t * lock,uint64_t context,uint64_t seqno)205 dma_fence_reset(struct dma_fence *fence, const struct dma_fence_ops *ops,
206 spinlock_t *lock, uint64_t context, uint64_t seqno)
207 {
208
209 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence);
210 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence);
211 KASSERT(kref_read(&fence->refcount) == 0 ||
212 kref_read(&fence->refcount) == 1);
213 KASSERT(TAILQ_EMPTY(&fence->f_callbacks));
214 KASSERT(fence->lock == lock);
215 KASSERT(fence->ops == ops);
216
217 kref_init(&fence->refcount);
218 fence->flags = 0;
219 fence->context = context;
220 fence->seqno = seqno;
221 fence->error = 0;
222
223 SDT_PROBE1(sdt, drm, fence, reset, fence);
224 }
225
226 /*
227 * dma_fence_destroy(fence)
228 *
229 * Clean up memory initialized with dma_fence_init. This is meant
230 * to be used after a fence release callback.
231 *
232 * XXX extension to Linux API
233 */
234 void
dma_fence_destroy(struct dma_fence * fence)235 dma_fence_destroy(struct dma_fence *fence)
236 {
237
238 KASSERT(!dma_fence_referenced_p(fence));
239
240 SDT_PROBE1(sdt, drm, fence, destroy, fence);
241
242 #ifdef DIAGNOSTIC
243 fence->f_magic = FENCE_MAGIC_BAD;
244 #endif
245
246 KASSERT(TAILQ_EMPTY(&fence->f_callbacks));
247 cv_destroy(&fence->f_cv);
248 }
249
250 static void
dma_fence_free_cb(struct rcu_head * rcu)251 dma_fence_free_cb(struct rcu_head *rcu)
252 {
253 struct dma_fence *fence = container_of(rcu, struct dma_fence, rcu);
254
255 KASSERT(!dma_fence_referenced_p(fence));
256
257 dma_fence_destroy(fence);
258 kfree(fence);
259 }
260
261 /*
262 * dma_fence_free(fence)
263 *
264 * Schedule fence to be destroyed and then freed with kfree after
265 * any pending RCU read sections on all CPUs have completed.
266 * Caller must guarantee all references have been released. This
267 * is meant to be used after a fence release callback.
268 *
269 * NOTE: Callers assume kfree will be used. We don't even use
270 * kmalloc to allocate these -- caller is expected to allocate
271 * memory with kmalloc to be initialized with dma_fence_init.
272 */
273 void
dma_fence_free(struct dma_fence * fence)274 dma_fence_free(struct dma_fence *fence)
275 {
276
277 KASSERT(!dma_fence_referenced_p(fence));
278
279 SDT_PROBE1(sdt, drm, fence, free, fence);
280
281 call_rcu(&fence->rcu, &dma_fence_free_cb);
282 }
283
284 /*
285 * dma_fence_context_alloc(n)
286 *
287 * Return the first of a contiguous sequence of unique
288 * identifiers, at least until the system wraps around.
289 */
290 uint64_t
dma_fence_context_alloc(unsigned n)291 dma_fence_context_alloc(unsigned n)
292 {
293 static struct {
294 volatile unsigned lock;
295 uint64_t context;
296 } S;
297 uint64_t c;
298
299 while (__predict_false(atomic_swap_uint(&S.lock, 1)))
300 SPINLOCK_BACKOFF_HOOK;
301 membar_acquire();
302 c = S.context;
303 S.context += n;
304 atomic_store_release(&S.lock, 0);
305
306 return c;
307 }
308
309 /*
310 * __dma_fence_is_later(a, b, ops)
311 *
312 * True if sequence number a is later than sequence number b,
313 * according to the given fence ops.
314 *
315 * - For fence ops with 64-bit sequence numbers, this is simply
316 * defined to be a > b as unsigned 64-bit integers.
317 *
318 * - For fence ops with 32-bit sequence numbers, this is defined
319 * to mean that the 32-bit unsigned difference a - b is less
320 * than INT_MAX.
321 */
322 bool
__dma_fence_is_later(uint64_t a,uint64_t b,const struct dma_fence_ops * ops)323 __dma_fence_is_later(uint64_t a, uint64_t b, const struct dma_fence_ops *ops)
324 {
325
326 if (ops->use_64bit_seqno)
327 return a > b;
328 else
329 return (unsigned)a - (unsigned)b < INT_MAX;
330 }
331
332 /*
333 * dma_fence_is_later(a, b)
334 *
335 * True if the sequence number of fence a is later than the
336 * sequence number of fence b. Since sequence numbers wrap
337 * around, we define this to mean that the sequence number of
338 * fence a is no more than INT_MAX past the sequence number of
339 * fence b.
340 *
341 * The two fences must have the context. Whether sequence numbers
342 * are 32-bit is determined by a.
343 */
344 bool
dma_fence_is_later(struct dma_fence * a,struct dma_fence * b)345 dma_fence_is_later(struct dma_fence *a, struct dma_fence *b)
346 {
347
348 KASSERTMSG(a->f_magic != FENCE_MAGIC_BAD, "fence %p", a);
349 KASSERTMSG(a->f_magic == FENCE_MAGIC_GOOD, "fence %p", a);
350 KASSERTMSG(b->f_magic != FENCE_MAGIC_BAD, "fence %p", b);
351 KASSERTMSG(b->f_magic == FENCE_MAGIC_GOOD, "fence %p", b);
352 KASSERTMSG(a->context == b->context, "incommensurate fences"
353 ": %"PRIu64" @ %p =/= %"PRIu64" @ %p",
354 a->context, a, b->context, b);
355
356 return __dma_fence_is_later(a->seqno, b->seqno, a->ops);
357 }
358
359 /*
360 * dma_fence_get_stub()
361 *
362 * Return a dma fence that is always already signalled.
363 */
364 struct dma_fence *
dma_fence_get_stub(void)365 dma_fence_get_stub(void)
366 {
367
368 return dma_fence_get(&dma_fence_stub.fence);
369 }
370
371 /*
372 * dma_fence_get(fence)
373 *
374 * Acquire a reference to fence and return it, or return NULL if
375 * fence is NULL. The fence, if nonnull, must not be being
376 * destroyed.
377 */
378 struct dma_fence *
dma_fence_get(struct dma_fence * fence)379 dma_fence_get(struct dma_fence *fence)
380 {
381
382 if (fence == NULL)
383 return NULL;
384
385 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence);
386 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence);
387
388 kref_get(&fence->refcount);
389 return fence;
390 }
391
392 /*
393 * dma_fence_get_rcu(fence)
394 *
395 * Attempt to acquire a reference to a fence that may be about to
396 * be destroyed, during a read section. Return the fence on
397 * success, or NULL on failure. The fence must be nonnull.
398 */
399 struct dma_fence *
dma_fence_get_rcu(struct dma_fence * fence)400 dma_fence_get_rcu(struct dma_fence *fence)
401 {
402
403 __insn_barrier();
404 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence);
405 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence);
406 if (!kref_get_unless_zero(&fence->refcount))
407 return NULL;
408 return fence;
409 }
410
411 /*
412 * dma_fence_get_rcu_safe(fencep)
413 *
414 * Attempt to acquire a reference to the fence *fencep, which may
415 * be about to be destroyed, during a read section. If the value
416 * of *fencep changes after we read *fencep but before we
417 * increment its reference count, retry. Return *fencep on
418 * success, or NULL on failure.
419 */
420 struct dma_fence *
dma_fence_get_rcu_safe(struct dma_fence * volatile const * fencep)421 dma_fence_get_rcu_safe(struct dma_fence *volatile const *fencep)
422 {
423 struct dma_fence *fence;
424
425 retry:
426 /*
427 * Load the fence, ensuring we observe the fully initialized
428 * content.
429 */
430 if ((fence = atomic_load_consume(fencep)) == NULL)
431 return NULL;
432
433 /* Try to acquire a reference. If we can't, try again. */
434 if (!dma_fence_get_rcu(fence))
435 goto retry;
436
437 /*
438 * Confirm that it's still the same fence. If not, release it
439 * and retry.
440 */
441 if (fence != atomic_load_relaxed(fencep)) {
442 dma_fence_put(fence);
443 goto retry;
444 }
445
446 /* Success! */
447 KASSERT(dma_fence_referenced_p(fence));
448 return fence;
449 }
450
451 static void
dma_fence_release(struct kref * refcount)452 dma_fence_release(struct kref *refcount)
453 {
454 struct dma_fence *fence = container_of(refcount, struct dma_fence,
455 refcount);
456
457 KASSERTMSG(TAILQ_EMPTY(&fence->f_callbacks),
458 "fence %p has pending callbacks", fence);
459 KASSERT(!dma_fence_referenced_p(fence));
460
461 SDT_PROBE1(sdt, drm, fence, release, fence);
462
463 if (fence->ops->release)
464 (*fence->ops->release)(fence);
465 else
466 dma_fence_free(fence);
467 }
468
469 /*
470 * dma_fence_put(fence)
471 *
472 * Release a reference to fence. If this was the last one, call
473 * the fence's release callback.
474 */
475 void
dma_fence_put(struct dma_fence * fence)476 dma_fence_put(struct dma_fence *fence)
477 {
478
479 if (fence == NULL)
480 return;
481 KASSERT(dma_fence_referenced_p(fence));
482 kref_put(&fence->refcount, &dma_fence_release);
483 }
484
485 /*
486 * dma_fence_ensure_signal_enabled(fence)
487 *
488 * Internal subroutine. If the fence was already signalled,
489 * return -ENOENT. Otherwise, if the enable signalling callback
490 * has not been called yet, call it. If fails, signal the fence
491 * and return -ENOENT. If it succeeds, or if it had already been
492 * called, return zero to indicate success.
493 *
494 * Caller must hold the fence's lock.
495 */
496 static int
dma_fence_ensure_signal_enabled(struct dma_fence * fence)497 dma_fence_ensure_signal_enabled(struct dma_fence *fence)
498 {
499 bool already_enabled;
500
501 KASSERT(dma_fence_referenced_p(fence));
502 KASSERT(spin_is_locked(fence->lock));
503
504 /* Determine whether signalling was enabled, and enable it. */
505 already_enabled = test_and_set_bit(DMA_FENCE_FLAG_ENABLE_SIGNAL_BIT,
506 &fence->flags);
507
508 /* If the fence was already signalled, fail with -ENOENT. */
509 if (fence->flags & (1u << DMA_FENCE_FLAG_SIGNALED_BIT))
510 return -ENOENT;
511
512 /*
513 * Otherwise, if it wasn't enabled yet, try to enable
514 * signalling.
515 */
516 if (!already_enabled && fence->ops->enable_signaling) {
517 SDT_PROBE1(sdt, drm, fence, enable_signaling, fence);
518 if (!(*fence->ops->enable_signaling)(fence)) {
519 /* If it failed, signal and return -ENOENT. */
520 dma_fence_signal_locked(fence);
521 return -ENOENT;
522 }
523 }
524
525 /* Success! */
526 return 0;
527 }
528
529 /*
530 * dma_fence_add_callback(fence, fcb, fn)
531 *
532 * If fence has been signalled, return -ENOENT. If the enable
533 * signalling callback hasn't been called yet, call it; if it
534 * fails, return -ENOENT. Otherwise, arrange to call fn(fence,
535 * fcb) when it is signalled, and return 0.
536 *
537 * The fence uses memory allocated by the caller in fcb from the
538 * time of dma_fence_add_callback either to the time of
539 * dma_fence_remove_callback, or just before calling fn.
540 */
541 int
dma_fence_add_callback(struct dma_fence * fence,struct dma_fence_cb * fcb,dma_fence_func_t fn)542 dma_fence_add_callback(struct dma_fence *fence, struct dma_fence_cb *fcb,
543 dma_fence_func_t fn)
544 {
545 int ret;
546
547 KASSERT(dma_fence_referenced_p(fence));
548
549 /* Optimistically try to skip the lock if it's already signalled. */
550 if (atomic_load_relaxed(&fence->flags) &
551 (1u << DMA_FENCE_FLAG_SIGNALED_BIT)) {
552 ret = -ENOENT;
553 goto out0;
554 }
555
556 /* Acquire the lock. */
557 spin_lock(fence->lock);
558
559 /* Ensure signalling is enabled, or fail if we can't. */
560 ret = dma_fence_ensure_signal_enabled(fence);
561 if (ret)
562 goto out1;
563
564 /* Insert the callback. */
565 SDT_PROBE2(sdt, drm, fence, add_callback, fence, fcb);
566 fcb->func = fn;
567 TAILQ_INSERT_TAIL(&fence->f_callbacks, fcb, fcb_entry);
568 fcb->fcb_onqueue = true;
569 ret = 0;
570
571 /* Release the lock and we're done. */
572 out1: spin_unlock(fence->lock);
573 out0: if (ret) {
574 fcb->func = NULL;
575 fcb->fcb_onqueue = false;
576 }
577 return ret;
578 }
579
580 /*
581 * dma_fence_remove_callback(fence, fcb)
582 *
583 * Remove the callback fcb from fence. Return true if it was
584 * removed from the list, or false if it had already run and so
585 * was no longer queued anyway. Caller must have already called
586 * dma_fence_add_callback(fence, fcb).
587 */
588 bool
dma_fence_remove_callback(struct dma_fence * fence,struct dma_fence_cb * fcb)589 dma_fence_remove_callback(struct dma_fence *fence, struct dma_fence_cb *fcb)
590 {
591 bool onqueue;
592
593 KASSERT(dma_fence_referenced_p(fence));
594
595 spin_lock(fence->lock);
596 onqueue = fcb->fcb_onqueue;
597 if (onqueue) {
598 SDT_PROBE2(sdt, drm, fence, remove_callback, fence, fcb);
599 TAILQ_REMOVE(&fence->f_callbacks, fcb, fcb_entry);
600 fcb->fcb_onqueue = false;
601 }
602 spin_unlock(fence->lock);
603
604 return onqueue;
605 }
606
607 /*
608 * dma_fence_enable_sw_signaling(fence)
609 *
610 * If it hasn't been called yet and the fence hasn't been
611 * signalled yet, call the fence's enable_sw_signaling callback.
612 * If when that happens, the callback indicates failure by
613 * returning false, signal the fence.
614 */
615 void
dma_fence_enable_sw_signaling(struct dma_fence * fence)616 dma_fence_enable_sw_signaling(struct dma_fence *fence)
617 {
618
619 KASSERT(dma_fence_referenced_p(fence));
620
621 spin_lock(fence->lock);
622 if ((fence->flags & (1u << DMA_FENCE_FLAG_SIGNALED_BIT)) == 0)
623 (void)dma_fence_ensure_signal_enabled(fence);
624 spin_unlock(fence->lock);
625 }
626
627 /*
628 * dma_fence_is_signaled(fence)
629 *
630 * Test whether the fence has been signalled. If it has been
631 * signalled by dma_fence_signal(_locked), return true. If the
632 * signalled callback returns true indicating that some implicit
633 * external condition has changed, call the callbacks as if with
634 * dma_fence_signal.
635 */
636 bool
dma_fence_is_signaled(struct dma_fence * fence)637 dma_fence_is_signaled(struct dma_fence *fence)
638 {
639 bool signaled;
640
641 KASSERT(dma_fence_referenced_p(fence));
642
643 spin_lock(fence->lock);
644 signaled = dma_fence_is_signaled_locked(fence);
645 spin_unlock(fence->lock);
646
647 return signaled;
648 }
649
650 /*
651 * dma_fence_is_signaled_locked(fence)
652 *
653 * Test whether the fence has been signalled. Like
654 * dma_fence_is_signaleed, but caller already holds the fence's lock.
655 */
656 bool
dma_fence_is_signaled_locked(struct dma_fence * fence)657 dma_fence_is_signaled_locked(struct dma_fence *fence)
658 {
659
660 KASSERT(dma_fence_referenced_p(fence));
661 KASSERT(spin_is_locked(fence->lock));
662
663 /* Check whether we already set the signalled bit. */
664 if (fence->flags & (1u << DMA_FENCE_FLAG_SIGNALED_BIT))
665 return true;
666
667 /* If there's a signalled callback, test it. */
668 if (fence->ops->signaled) {
669 SDT_PROBE1(sdt, drm, fence, test, fence);
670 if ((*fence->ops->signaled)(fence)) {
671 /*
672 * It's been signalled implicitly by some
673 * external phenomonen. Act as though someone
674 * has called dma_fence_signal.
675 */
676 dma_fence_signal_locked(fence);
677 return true;
678 }
679 }
680
681 return false;
682 }
683
684 /*
685 * dma_fence_set_error(fence, error)
686 *
687 * Set an error code prior to dma_fence_signal for use by a
688 * waiter to learn about success or failure of the fence.
689 */
690 void
dma_fence_set_error(struct dma_fence * fence,int error)691 dma_fence_set_error(struct dma_fence *fence, int error)
692 {
693
694 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence);
695 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence);
696 KASSERT((atomic_load_relaxed(&fence->flags) &
697 (1u << DMA_FENCE_FLAG_SIGNALED_BIT)) == 0);
698 KASSERTMSG(error >= -ELAST, "%d", error);
699 KASSERTMSG(error < 0, "%d", error);
700
701 SDT_PROBE2(sdt, drm, fence, set_error, fence, error);
702 fence->error = error;
703 }
704
705 /*
706 * dma_fence_get_status(fence)
707 *
708 * Return 0 if fence has yet to be signalled, 1 if it has been
709 * signalled without error, or negative error code if
710 * dma_fence_set_error was used.
711 */
712 int
dma_fence_get_status(struct dma_fence * fence)713 dma_fence_get_status(struct dma_fence *fence)
714 {
715 int ret;
716
717 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence);
718 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence);
719
720 spin_lock(fence->lock);
721 if (!dma_fence_is_signaled_locked(fence)) {
722 ret = 0;
723 } else if (fence->error) {
724 ret = fence->error;
725 KASSERTMSG(ret < 0, "%d", ret);
726 } else {
727 ret = 1;
728 }
729 spin_unlock(fence->lock);
730
731 return ret;
732 }
733
734 /*
735 * dma_fence_signal(fence)
736 *
737 * Signal the fence. If it has already been signalled, return
738 * -EINVAL. If it has not been signalled, call the enable
739 * signalling callback if it hasn't been called yet, and remove
740 * each registered callback from the queue and call it; then
741 * return 0.
742 */
743 int
dma_fence_signal(struct dma_fence * fence)744 dma_fence_signal(struct dma_fence *fence)
745 {
746 int ret;
747
748 KASSERT(dma_fence_referenced_p(fence));
749
750 spin_lock(fence->lock);
751 ret = dma_fence_signal_locked(fence);
752 spin_unlock(fence->lock);
753
754 return ret;
755 }
756
757 /*
758 * dma_fence_signal_locked(fence)
759 *
760 * Signal the fence. Like dma_fence_signal, but caller already
761 * holds the fence's lock.
762 */
763 int
dma_fence_signal_locked(struct dma_fence * fence)764 dma_fence_signal_locked(struct dma_fence *fence)
765 {
766 struct dma_fence_cb *fcb, *next;
767
768 KASSERT(dma_fence_referenced_p(fence));
769 KASSERT(spin_is_locked(fence->lock));
770
771 /* If it's been signalled, fail; otherwise set the signalled bit. */
772 if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
773 return -EINVAL;
774
775 SDT_PROBE1(sdt, drm, fence, signal, fence);
776
777 /* Set the timestamp. */
778 fence->timestamp = ktime_get();
779 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
780
781 /* Wake waiters. */
782 cv_broadcast(&fence->f_cv);
783
784 /* Remove and call the callbacks. */
785 TAILQ_FOREACH_SAFE(fcb, &fence->f_callbacks, fcb_entry, next) {
786 SDT_PROBE2(sdt, drm, fence, callback, fence, fcb);
787 TAILQ_REMOVE(&fence->f_callbacks, fcb, fcb_entry);
788 fcb->fcb_onqueue = false;
789 (*fcb->func)(fence, fcb);
790 }
791
792 /* Success! */
793 return 0;
794 }
795
796 struct wait_any {
797 struct dma_fence_cb fcb;
798 struct wait_any1 {
799 kmutex_t lock;
800 kcondvar_t cv;
801 struct wait_any *cb;
802 bool done;
803 } *common;
804 };
805
806 static void
wait_any_cb(struct dma_fence * fence,struct dma_fence_cb * fcb)807 wait_any_cb(struct dma_fence *fence, struct dma_fence_cb *fcb)
808 {
809 struct wait_any *cb = container_of(fcb, struct wait_any, fcb);
810
811 KASSERT(dma_fence_referenced_p(fence));
812
813 mutex_enter(&cb->common->lock);
814 cb->common->done = true;
815 cv_broadcast(&cb->common->cv);
816 mutex_exit(&cb->common->lock);
817 }
818
819 /*
820 * dma_fence_wait_any_timeout(fence, nfences, intr, timeout, ip)
821 *
822 * Wait for any of fences[0], fences[1], fences[2], ...,
823 * fences[nfences-1] to be signalled. If ip is nonnull, set *ip
824 * to the index of the first one.
825 *
826 * Return -ERESTARTSYS if interrupted, 0 on timeout, or time
827 * remaining (at least 1) on success.
828 */
829 long
dma_fence_wait_any_timeout(struct dma_fence ** fences,uint32_t nfences,bool intr,long timeout,uint32_t * ip)830 dma_fence_wait_any_timeout(struct dma_fence **fences, uint32_t nfences,
831 bool intr, long timeout, uint32_t *ip)
832 {
833 struct wait_any1 common;
834 struct wait_any *cb;
835 uint32_t i, j;
836 int start, end;
837 long ret = 0;
838
839 KASSERTMSG(timeout >= 0, "timeout %ld", timeout);
840 KASSERTMSG(timeout <= MAX_SCHEDULE_TIMEOUT, "timeout %ld", timeout);
841
842 /* Optimistically check whether any are signalled. */
843 for (i = 0; i < nfences; i++) {
844 KASSERT(dma_fence_referenced_p(fences[i]));
845 if (dma_fence_is_signaled(fences[i])) {
846 if (ip)
847 *ip = i;
848 return MAX(1, timeout);
849 }
850 }
851
852 /*
853 * If timeout is zero, we're just polling, so stop here as if
854 * we timed out instantly.
855 */
856 if (timeout == 0)
857 return 0;
858
859 /* Allocate an array of callback records. */
860 cb = kcalloc(nfences, sizeof(cb[0]), GFP_KERNEL);
861 if (cb == NULL)
862 return -ENOMEM;
863
864 /* Initialize a mutex and condvar for the common wait. */
865 mutex_init(&common.lock, MUTEX_DEFAULT, IPL_VM);
866 cv_init(&common.cv, "fence");
867 common.cb = cb;
868 common.done = false;
869
870 /*
871 * Add a callback to each of the fences, or stop if already
872 * signalled.
873 */
874 for (i = 0; i < nfences; i++) {
875 cb[i].common = &common;
876 KASSERT(dma_fence_referenced_p(fences[i]));
877 ret = dma_fence_add_callback(fences[i], &cb[i].fcb,
878 &wait_any_cb);
879 if (ret) {
880 KASSERT(ret == -ENOENT);
881 if (ip)
882 *ip = i;
883 ret = MAX(1, timeout);
884 goto out;
885 }
886 }
887
888 /*
889 * None of them was ready immediately. Wait for one of the
890 * callbacks to notify us when it is done.
891 */
892 mutex_enter(&common.lock);
893 while (!common.done) {
894 /* Wait for the time remaining. */
895 start = getticks();
896 if (intr) {
897 if (timeout != MAX_SCHEDULE_TIMEOUT) {
898 ret = -cv_timedwait_sig(&common.cv,
899 &common.lock, MIN(timeout, /* paranoia */
900 MAX_SCHEDULE_TIMEOUT));
901 } else {
902 ret = -cv_wait_sig(&common.cv, &common.lock);
903 }
904 } else {
905 if (timeout != MAX_SCHEDULE_TIMEOUT) {
906 ret = -cv_timedwait(&common.cv,
907 &common.lock, MIN(timeout, /* paranoia */
908 MAX_SCHEDULE_TIMEOUT));
909 } else {
910 cv_wait(&common.cv, &common.lock);
911 ret = 0;
912 }
913 }
914 end = getticks();
915
916 /* Deduct from time remaining. If none left, time out. */
917 if (timeout != MAX_SCHEDULE_TIMEOUT) {
918 timeout -= MIN(timeout,
919 (unsigned)end - (unsigned)start);
920 if (timeout == 0)
921 ret = -EWOULDBLOCK;
922 }
923
924 /* If the wait failed, give up. */
925 if (ret)
926 break;
927 }
928 mutex_exit(&common.lock);
929
930 /*
931 * Massage the return code if nonzero:
932 * - if we were interrupted, return -ERESTARTSYS;
933 * - if we timed out, return 0.
934 * No other failure is possible. On success, ret=0 but we
935 * check again below to verify anyway.
936 */
937 if (ret) {
938 KASSERTMSG((ret == -EINTR || ret == -ERESTART ||
939 ret == -EWOULDBLOCK), "ret=%ld", ret);
940 if (ret == -EINTR || ret == -ERESTART) {
941 ret = -ERESTARTSYS;
942 } else if (ret == -EWOULDBLOCK) {
943 KASSERT(timeout != MAX_SCHEDULE_TIMEOUT);
944 ret = 0; /* timed out */
945 }
946 }
947
948 KASSERT(ret != -ERESTART); /* would be confused with time left */
949
950 /*
951 * Test whether any of the fences has been signalled. If they
952 * have, return success.
953 */
954 for (j = 0; j < nfences; j++) {
955 if (dma_fence_is_signaled(fences[i])) {
956 if (ip)
957 *ip = j;
958 ret = MAX(1, timeout);
959 goto out;
960 }
961 }
962
963 /*
964 * If user passed MAX_SCHEDULE_TIMEOUT, we can't return 0
965 * meaning timed out because we're supposed to wait forever.
966 */
967 KASSERT(timeout == MAX_SCHEDULE_TIMEOUT ? ret != 0 : 1);
968
969 out: while (i --> 0)
970 (void)dma_fence_remove_callback(fences[i], &cb[i].fcb);
971 cv_destroy(&common.cv);
972 mutex_destroy(&common.lock);
973 kfree(cb);
974 return ret;
975 }
976
977 /*
978 * dma_fence_wait_timeout(fence, intr, timeout)
979 *
980 * Wait until fence is signalled; or until interrupt, if intr is
981 * true; or until timeout, if positive. Return -ERESTARTSYS if
982 * interrupted, negative error code on any other error, zero on
983 * timeout, or positive number of ticks remaining if the fence is
984 * signalled before the timeout. Works by calling the fence wait
985 * callback.
986 *
987 * The timeout must be nonnegative and at most
988 * MAX_SCHEDULE_TIMEOUT, which means wait indefinitely.
989 */
990 long
dma_fence_wait_timeout(struct dma_fence * fence,bool intr,long timeout)991 dma_fence_wait_timeout(struct dma_fence *fence, bool intr, long timeout)
992 {
993 long ret;
994
995 KASSERT(dma_fence_referenced_p(fence));
996 KASSERTMSG(timeout >= 0, "timeout %ld", timeout);
997 KASSERTMSG(timeout <= MAX_SCHEDULE_TIMEOUT, "timeout %ld", timeout);
998
999 SDT_PROBE3(sdt, drm, fence, wait_start, fence, intr, timeout);
1000 if (fence->ops->wait)
1001 ret = (*fence->ops->wait)(fence, intr, timeout);
1002 else
1003 ret = dma_fence_default_wait(fence, intr, timeout);
1004 SDT_PROBE2(sdt, drm, fence, wait_done, fence, ret);
1005
1006 return ret;
1007 }
1008
1009 /*
1010 * dma_fence_wait(fence, intr)
1011 *
1012 * Wait until fence is signalled; or until interrupt, if intr is
1013 * true. Return -ERESTARTSYS if interrupted, negative error code
1014 * on any other error, zero on sucess. Works by calling the fence
1015 * wait callback with MAX_SCHEDULE_TIMEOUT.
1016 */
1017 long
dma_fence_wait(struct dma_fence * fence,bool intr)1018 dma_fence_wait(struct dma_fence *fence, bool intr)
1019 {
1020 long ret;
1021
1022 KASSERT(dma_fence_referenced_p(fence));
1023
1024 ret = dma_fence_wait_timeout(fence, intr, MAX_SCHEDULE_TIMEOUT);
1025 KASSERT(ret != 0);
1026 KASSERTMSG(ret == -ERESTARTSYS || ret == MAX_SCHEDULE_TIMEOUT,
1027 "ret=%ld", ret);
1028
1029 return (ret < 0 ? ret : 0);
1030 }
1031
1032 /*
1033 * dma_fence_default_wait(fence, intr, timeout)
1034 *
1035 * Default implementation of fence wait callback using a condition
1036 * variable. If the fence is already signalled, return timeout,
1037 * or 1 if timeout is zero meaning poll. If the enable signalling
1038 * callback hasn't been called, call it, and if it fails, act as
1039 * if the fence had been signalled. Otherwise, wait on the
1040 * internal condvar. If timeout is MAX_SCHEDULE_TIMEOUT, wait
1041 * indefinitely.
1042 */
1043 long
dma_fence_default_wait(struct dma_fence * fence,bool intr,long timeout)1044 dma_fence_default_wait(struct dma_fence *fence, bool intr, long timeout)
1045 {
1046 int starttime = 0, now = 0, deadline = 0; /* XXXGCC */
1047 kmutex_t *lock = &fence->lock->sl_lock;
1048 long ret = 0;
1049
1050 KASSERT(dma_fence_referenced_p(fence));
1051 KASSERTMSG(timeout >= 0, "timeout %ld", timeout);
1052 KASSERTMSG(timeout <= MAX_SCHEDULE_TIMEOUT, "timeout %ld", timeout);
1053
1054 /* Optimistically try to skip the lock if it's already signalled. */
1055 if (atomic_load_relaxed(&fence->flags) &
1056 (1u << DMA_FENCE_FLAG_SIGNALED_BIT))
1057 return MAX(1, timeout);
1058
1059 /* Acquire the lock. */
1060 spin_lock(fence->lock);
1061
1062 /* Ensure signalling is enabled, or stop if already completed. */
1063 if (dma_fence_ensure_signal_enabled(fence) != 0) {
1064 ret = MAX(1, timeout);
1065 goto out;
1066 }
1067
1068 /* If merely polling, stop here. */
1069 if (timeout == 0) {
1070 ret = 0;
1071 goto out;
1072 }
1073
1074 /* Find out what our deadline is so we can handle spurious wakeup. */
1075 if (timeout < MAX_SCHEDULE_TIMEOUT) {
1076 now = getticks();
1077 starttime = now;
1078 deadline = starttime + timeout;
1079 }
1080
1081 /* Wait until the signalled bit is set. */
1082 while (!(fence->flags & (1u << DMA_FENCE_FLAG_SIGNALED_BIT))) {
1083 /*
1084 * If there's a timeout and we've passed the deadline,
1085 * give up.
1086 */
1087 if (timeout < MAX_SCHEDULE_TIMEOUT) {
1088 now = getticks();
1089 if (deadline <= now) {
1090 ret = -EWOULDBLOCK;
1091 break;
1092 }
1093 }
1094
1095 /* Wait for the time remaining. */
1096 if (intr) {
1097 if (timeout < MAX_SCHEDULE_TIMEOUT) {
1098 ret = -cv_timedwait_sig(&fence->f_cv, lock,
1099 deadline - now);
1100 } else {
1101 ret = -cv_wait_sig(&fence->f_cv, lock);
1102 }
1103 } else {
1104 if (timeout < MAX_SCHEDULE_TIMEOUT) {
1105 ret = -cv_timedwait(&fence->f_cv, lock,
1106 deadline - now);
1107 } else {
1108 cv_wait(&fence->f_cv, lock);
1109 ret = 0;
1110 }
1111 }
1112
1113 /* If the wait failed, give up. */
1114 if (ret)
1115 break;
1116 }
1117
1118 /*
1119 * Massage the return code if nonzero:
1120 * - if we were interrupted, return -ERESTARTSYS;
1121 * - if we timed out, return 0.
1122 * No other failure is possible. On success, ret=0 but we
1123 * check again below to verify anyway.
1124 */
1125 if (ret) {
1126 KASSERTMSG((ret == -EINTR || ret == -ERESTART ||
1127 ret == -EWOULDBLOCK), "ret=%ld", ret);
1128 if (ret == -EINTR || ret == -ERESTART) {
1129 ret = -ERESTARTSYS;
1130 } else if (ret == -EWOULDBLOCK) {
1131 KASSERT(timeout < MAX_SCHEDULE_TIMEOUT);
1132 ret = 0; /* timed out */
1133 }
1134 }
1135
1136 KASSERT(ret != -ERESTART); /* would be confused with time left */
1137
1138 /* Check again in case it was signalled after a wait. */
1139 if (fence->flags & (1u << DMA_FENCE_FLAG_SIGNALED_BIT)) {
1140 if (timeout < MAX_SCHEDULE_TIMEOUT)
1141 ret = MAX(1, deadline - now);
1142 else
1143 ret = MAX_SCHEDULE_TIMEOUT;
1144 }
1145
1146 out: /* All done. Release the lock. */
1147 spin_unlock(fence->lock);
1148 return ret;
1149 }
1150
1151 /*
1152 * __dma_fence_signal(fence)
1153 *
1154 * Set fence's signalled bit, without waking waiters yet. Return
1155 * true if it was newly set, false if it was already set.
1156 */
1157 bool
__dma_fence_signal(struct dma_fence * fence)1158 __dma_fence_signal(struct dma_fence *fence)
1159 {
1160
1161 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence);
1162 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence);
1163
1164 if (test_and_set_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
1165 return false;
1166
1167 return true;
1168 }
1169
1170 /*
1171 * __dma_fence_signal_wake(fence)
1172 *
1173 * Set fence's timestamp and wake fence's waiters. Caller must
1174 * have previously called __dma_fence_signal and it must have
1175 * previously returned true.
1176 */
1177 void
__dma_fence_signal_wake(struct dma_fence * fence,ktime_t timestamp)1178 __dma_fence_signal_wake(struct dma_fence *fence, ktime_t timestamp)
1179 {
1180 struct dma_fence_cb *fcb, *next;
1181
1182 KASSERTMSG(fence->f_magic != FENCE_MAGIC_BAD, "fence %p", fence);
1183 KASSERTMSG(fence->f_magic == FENCE_MAGIC_GOOD, "fence %p", fence);
1184
1185 spin_lock(fence->lock);
1186
1187 KASSERT(fence->flags & DMA_FENCE_FLAG_SIGNALED_BIT);
1188
1189 SDT_PROBE1(sdt, drm, fence, signal, fence);
1190
1191 /* Set the timestamp. */
1192 fence->timestamp = timestamp;
1193 set_bit(DMA_FENCE_FLAG_TIMESTAMP_BIT, &fence->flags);
1194
1195 /* Wake waiters. */
1196 cv_broadcast(&fence->f_cv);
1197
1198 /* Remove and call the callbacks. */
1199 TAILQ_FOREACH_SAFE(fcb, &fence->f_callbacks, fcb_entry, next) {
1200 TAILQ_REMOVE(&fence->f_callbacks, fcb, fcb_entry);
1201 fcb->fcb_onqueue = false;
1202 (*fcb->func)(fence, fcb);
1203 }
1204
1205 spin_unlock(fence->lock);
1206 }
1207