xref: /netbsd-src/sys/external/bsd/drm2/linux/linux_wait_bit.c (revision 02f8a9bf395be12d7e80f6ade34bb5c7c7ef5a55)
1*02f8a9bfSriastradh /*	$NetBSD: linux_wait_bit.c,v 1.5 2021/12/19 12:36:09 riastradh Exp $	*/
27319b3e1Sriastradh 
37319b3e1Sriastradh /*-
47319b3e1Sriastradh  * Copyright (c) 2018 The NetBSD Foundation, Inc.
57319b3e1Sriastradh  * All rights reserved.
67319b3e1Sriastradh  *
77319b3e1Sriastradh  * This code is derived from software contributed to The NetBSD Foundation
87319b3e1Sriastradh  * by Taylor R. Campbell.
97319b3e1Sriastradh  *
107319b3e1Sriastradh  * Redistribution and use in source and binary forms, with or without
117319b3e1Sriastradh  * modification, are permitted provided that the following conditions
127319b3e1Sriastradh  * are met:
137319b3e1Sriastradh  * 1. Redistributions of source code must retain the above copyright
147319b3e1Sriastradh  *    notice, this list of conditions and the following disclaimer.
157319b3e1Sriastradh  * 2. Redistributions in binary form must reproduce the above copyright
167319b3e1Sriastradh  *    notice, this list of conditions and the following disclaimer in the
177319b3e1Sriastradh  *    documentation and/or other materials provided with the distribution.
187319b3e1Sriastradh  *
197319b3e1Sriastradh  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
207319b3e1Sriastradh  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
217319b3e1Sriastradh  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
227319b3e1Sriastradh  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
237319b3e1Sriastradh  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
247319b3e1Sriastradh  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
257319b3e1Sriastradh  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
267319b3e1Sriastradh  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
277319b3e1Sriastradh  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
287319b3e1Sriastradh  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
297319b3e1Sriastradh  * POSSIBILITY OF SUCH DAMAGE.
307319b3e1Sriastradh  */
317319b3e1Sriastradh 
327319b3e1Sriastradh #include <sys/cdefs.h>
33*02f8a9bfSriastradh __KERNEL_RCSID(0, "$NetBSD: linux_wait_bit.c,v 1.5 2021/12/19 12:36:09 riastradh Exp $");
347319b3e1Sriastradh 
357319b3e1Sriastradh #include <sys/param.h>
367319b3e1Sriastradh #include <sys/types.h>
377319b3e1Sriastradh #include <sys/bitops.h>
387319b3e1Sriastradh #include <sys/condvar.h>
397319b3e1Sriastradh #include <sys/mutex.h>
407319b3e1Sriastradh #include <sys/systm.h>
417319b3e1Sriastradh 
427319b3e1Sriastradh #include <linux/bitops.h>
437319b3e1Sriastradh #include <linux/sched.h>
447319b3e1Sriastradh #include <linux/wait_bit.h>
457319b3e1Sriastradh 
467319b3e1Sriastradh static struct {
477319b3e1Sriastradh 	struct waitbitentry {
487319b3e1Sriastradh 		kmutex_t	lock;
497319b3e1Sriastradh 		kcondvar_t	cv;
507319b3e1Sriastradh 	}		ent;
517319b3e1Sriastradh 	char		pad[CACHE_LINE_SIZE - sizeof(struct waitbitentry)];
527319b3e1Sriastradh } waitbittab[PAGE_SIZE/CACHE_LINE_SIZE] __cacheline_aligned;
537319b3e1Sriastradh CTASSERT(sizeof(waitbittab) == PAGE_SIZE);
547319b3e1Sriastradh CTASSERT(sizeof(waitbittab[0]) == CACHE_LINE_SIZE);
557319b3e1Sriastradh 
567319b3e1Sriastradh int
linux_wait_bit_init(void)577319b3e1Sriastradh linux_wait_bit_init(void)
587319b3e1Sriastradh {
597319b3e1Sriastradh 	size_t i;
607319b3e1Sriastradh 
617319b3e1Sriastradh 	for (i = 0; i < __arraycount(waitbittab); i++) {
627319b3e1Sriastradh 		mutex_init(&waitbittab[i].ent.lock, MUTEX_DEFAULT, IPL_VM);
637319b3e1Sriastradh 		cv_init(&waitbittab[i].ent.cv, "waitbit");
647319b3e1Sriastradh 	}
657319b3e1Sriastradh 
667319b3e1Sriastradh 	return 0;
677319b3e1Sriastradh }
687319b3e1Sriastradh 
697319b3e1Sriastradh void
linux_wait_bit_fini(void)707319b3e1Sriastradh linux_wait_bit_fini(void)
717319b3e1Sriastradh {
727319b3e1Sriastradh 	size_t i;
737319b3e1Sriastradh 
747319b3e1Sriastradh 	for (i = 0; i < __arraycount(waitbittab); i++) {
757319b3e1Sriastradh 		cv_destroy(&waitbittab[i].ent.cv);
767319b3e1Sriastradh 		mutex_destroy(&waitbittab[i].ent.lock);
777319b3e1Sriastradh 	}
787319b3e1Sriastradh }
797319b3e1Sriastradh 
807319b3e1Sriastradh static inline size_t
wait_bit_hash(const volatile unsigned long * bitmap,unsigned bit)817319b3e1Sriastradh wait_bit_hash(const volatile unsigned long *bitmap, unsigned bit)
827319b3e1Sriastradh {
837319b3e1Sriastradh 	/* Try to avoid cache line collisions.  */
847319b3e1Sriastradh 	const volatile unsigned long *word = bitmap + bit/(NBBY*sizeof(*word));
857319b3e1Sriastradh 
867319b3e1Sriastradh 	return ((uintptr_t)word >> ilog2(CACHE_LINE_SIZE)) %
877319b3e1Sriastradh 	    __arraycount(waitbittab);
887319b3e1Sriastradh }
897319b3e1Sriastradh 
907319b3e1Sriastradh static struct waitbitentry *
wait_bit_enter(const volatile unsigned long * bitmap,unsigned bit)917319b3e1Sriastradh wait_bit_enter(const volatile unsigned long *bitmap, unsigned bit)
927319b3e1Sriastradh {
937319b3e1Sriastradh 	struct waitbitentry *wbe = &waitbittab[wait_bit_hash(bitmap, bit)].ent;
947319b3e1Sriastradh 
957319b3e1Sriastradh 	mutex_enter(&wbe->lock);
967319b3e1Sriastradh 
977319b3e1Sriastradh 	return wbe;
987319b3e1Sriastradh }
997319b3e1Sriastradh 
1007319b3e1Sriastradh static void
wait_bit_exit(struct waitbitentry * wbe)1017319b3e1Sriastradh wait_bit_exit(struct waitbitentry *wbe)
1027319b3e1Sriastradh {
1037319b3e1Sriastradh 
1047319b3e1Sriastradh 	mutex_exit(&wbe->lock);
1057319b3e1Sriastradh }
1067319b3e1Sriastradh 
107*02f8a9bfSriastradh /*
108*02f8a9bfSriastradh  * clear_and_wake_up_bit(bit, bitmap)
109*02f8a9bfSriastradh  *
110*02f8a9bfSriastradh  *	Clear the specified bit in the bitmap and wake any waiters in
111*02f8a9bfSriastradh  *	wait_on_bit or wait_on_bit_timeout that were waiting for it to
112*02f8a9bfSriastradh  *	clear.
113*02f8a9bfSriastradh  */
114b29600b0Sriastradh void
clear_and_wake_up_bit(int bit,volatile unsigned long * bitmap)115b29600b0Sriastradh clear_and_wake_up_bit(int bit, volatile unsigned long *bitmap)
116b29600b0Sriastradh {
117b29600b0Sriastradh 	struct waitbitentry *wbe;
118b29600b0Sriastradh 
119b29600b0Sriastradh 	wbe = wait_bit_enter(bitmap, bit);
120b29600b0Sriastradh 	clear_bit(bit, bitmap);
121b29600b0Sriastradh 	cv_broadcast(&wbe->cv);
122b29600b0Sriastradh 	wait_bit_exit(wbe);
123b29600b0Sriastradh }
124b29600b0Sriastradh 
125*02f8a9bfSriastradh /*
126*02f8a9bfSriastradh  * wait_on_bit(bitmap, bit, flags)
127*02f8a9bfSriastradh  *
128*02f8a9bfSriastradh  *	Wait for the specified bit in bitmap to be cleared.  Returns 0
129*02f8a9bfSriastradh  *	on success, -EINTR on signal, unless flags has
130*02f8a9bfSriastradh  *	TASK_UNINTERRUPTIBLE set.
131*02f8a9bfSriastradh  */
1327319b3e1Sriastradh int
wait_on_bit(const volatile unsigned long * bitmap,unsigned bit,int flags)133d602960fSriastradh wait_on_bit(const volatile unsigned long *bitmap, unsigned bit, int flags)
134d602960fSriastradh {
135d602960fSriastradh 	struct waitbitentry *wbe;
136d602960fSriastradh 	int error, ret;
137d602960fSriastradh 
138*02f8a9bfSriastradh 	if (test_bit(bit, bitmap) == 0)
139d602960fSriastradh 		return 0;
140d602960fSriastradh 
141d602960fSriastradh 	wbe = wait_bit_enter(bitmap, bit);
142d602960fSriastradh 
143*02f8a9bfSriastradh 	while (test_bit(bit, bitmap)) {
144d602960fSriastradh 		if (flags & TASK_UNINTERRUPTIBLE) {
145d602960fSriastradh 			cv_wait(&wbe->cv, &wbe->lock);
146d602960fSriastradh 		} else {
147d602960fSriastradh 			error = cv_wait_sig(&wbe->cv, &wbe->lock);
148*02f8a9bfSriastradh 			if (error) {
149*02f8a9bfSriastradh 				/* cv_wait_sig can only fail on signal.  */
150*02f8a9bfSriastradh 				KASSERTMSG(error == EINTR || error == ERESTART,
151*02f8a9bfSriastradh 				    "error=%d", error);
152*02f8a9bfSriastradh 				ret = -EINTR;
153d602960fSriastradh 				goto out;
154d602960fSriastradh 			}
155d602960fSriastradh 		}
156*02f8a9bfSriastradh 	}
157d602960fSriastradh 
158*02f8a9bfSriastradh 	/* Bit is clear.  Return zero on success.   */
159*02f8a9bfSriastradh 	KASSERT(test_bit(bit, bitmap) == 0);
160d602960fSriastradh 	ret = 0;
161d602960fSriastradh 
162*02f8a9bfSriastradh out:	KASSERT(test_bit(bit, bitmap) == 0 || ret != 0);
163*02f8a9bfSriastradh 	wait_bit_exit(wbe);
164d602960fSriastradh 	return ret;
165d602960fSriastradh }
166d602960fSriastradh 
167*02f8a9bfSriastradh /*
168*02f8a9bfSriastradh  * wait_on_bit_timeout(bitmap, bit, flags, timeout)
169*02f8a9bfSriastradh  *
170*02f8a9bfSriastradh  *	Wait for the specified bit in bitmap to be cleared.  Returns 0
171*02f8a9bfSriastradh  *	on success, -EINTR on signal unless flags has
172*02f8a9bfSriastradh  *	TASK_UNINTERRUPTIBLE set, or -EAGAIN on timeout.
173*02f8a9bfSriastradh  */
174d602960fSriastradh int
wait_on_bit_timeout(const volatile unsigned long * bitmap,unsigned bit,int flags,unsigned long timeout)1757319b3e1Sriastradh wait_on_bit_timeout(const volatile unsigned long *bitmap, unsigned bit,
1767319b3e1Sriastradh     int flags, unsigned long timeout)
1777319b3e1Sriastradh {
1787319b3e1Sriastradh 	struct waitbitentry *wbe;
1797319b3e1Sriastradh 	int error, ret;
1807319b3e1Sriastradh 
181*02f8a9bfSriastradh 	if (test_bit(bit, bitmap) == 0)
182*02f8a9bfSriastradh 		return 0;
1837319b3e1Sriastradh 
1847319b3e1Sriastradh 	wbe = wait_bit_enter(bitmap, bit);
1857319b3e1Sriastradh 
186*02f8a9bfSriastradh 	while (test_bit(bit, bitmap)) {
1877319b3e1Sriastradh 		unsigned starttime, endtime;
1887319b3e1Sriastradh 
189*02f8a9bfSriastradh 		if (timeout == 0) {
190*02f8a9bfSriastradh 			ret = -EAGAIN;
1917319b3e1Sriastradh 			goto out;
1927319b3e1Sriastradh 		}
1937319b3e1Sriastradh 
194*02f8a9bfSriastradh 		starttime = getticks();
195*02f8a9bfSriastradh 		if (flags & TASK_UNINTERRUPTIBLE) {
196*02f8a9bfSriastradh 			error = cv_timedwait(&wbe->cv, &wbe->lock,
197*02f8a9bfSriastradh 			    MIN(timeout, INT_MAX/2));
198*02f8a9bfSriastradh 		} else {
199*02f8a9bfSriastradh 			error = cv_timedwait_sig(&wbe->cv, &wbe->lock,
200*02f8a9bfSriastradh 			    MIN(timeout, INT_MAX/2));
201*02f8a9bfSriastradh 		}
202*02f8a9bfSriastradh 		endtime = getticks();
203*02f8a9bfSriastradh 
204*02f8a9bfSriastradh 		/*
205*02f8a9bfSriastradh 		 * If we were interrupted or timed out, massage the
206*02f8a9bfSriastradh 		 * error return and stop here.
207*02f8a9bfSriastradh 		 */
208*02f8a9bfSriastradh 		if (error) {
209*02f8a9bfSriastradh 			KASSERTMSG((error == EINTR || error == ERESTART ||
210*02f8a9bfSriastradh 				error == EWOULDBLOCK), "error=%d", error);
21199922f3fSriastradh 			if (error == EINTR || error == ERESTART) {
212*02f8a9bfSriastradh 				ret = -EINTR;
213*02f8a9bfSriastradh 			} else if (error == EWOULDBLOCK) {
214*02f8a9bfSriastradh 				ret = -EAGAIN;
215*02f8a9bfSriastradh 			} else {
216*02f8a9bfSriastradh 				panic("invalid error=%d", error);
217*02f8a9bfSriastradh 			}
2187319b3e1Sriastradh 			goto out;
2197319b3e1Sriastradh 		}
2207319b3e1Sriastradh 
2217319b3e1Sriastradh 		/* Otherwise, debit the time spent.  */
222*02f8a9bfSriastradh 		timeout -= MIN(timeout, (endtime - starttime));
2237319b3e1Sriastradh 	}
224*02f8a9bfSriastradh 
225*02f8a9bfSriastradh 	/* Bit is clear.  Return zero on success.  */
226*02f8a9bfSriastradh 	KASSERT(test_bit(bit, bitmap) == 0);
2277319b3e1Sriastradh 	ret = timeout;
2287319b3e1Sriastradh 
229*02f8a9bfSriastradh out:	KASSERT(test_bit(bit, bitmap) == 0 || ret != 0);
230*02f8a9bfSriastradh 	wait_bit_exit(wbe);
2317319b3e1Sriastradh 	return ret;
2327319b3e1Sriastradh }
233