1*02f8a9bfSriastradh /* $NetBSD: linux_wait_bit.c,v 1.5 2021/12/19 12:36:09 riastradh Exp $ */
27319b3e1Sriastradh
37319b3e1Sriastradh /*-
47319b3e1Sriastradh * Copyright (c) 2018 The NetBSD Foundation, Inc.
57319b3e1Sriastradh * All rights reserved.
67319b3e1Sriastradh *
77319b3e1Sriastradh * This code is derived from software contributed to The NetBSD Foundation
87319b3e1Sriastradh * by Taylor R. Campbell.
97319b3e1Sriastradh *
107319b3e1Sriastradh * Redistribution and use in source and binary forms, with or without
117319b3e1Sriastradh * modification, are permitted provided that the following conditions
127319b3e1Sriastradh * are met:
137319b3e1Sriastradh * 1. Redistributions of source code must retain the above copyright
147319b3e1Sriastradh * notice, this list of conditions and the following disclaimer.
157319b3e1Sriastradh * 2. Redistributions in binary form must reproduce the above copyright
167319b3e1Sriastradh * notice, this list of conditions and the following disclaimer in the
177319b3e1Sriastradh * documentation and/or other materials provided with the distribution.
187319b3e1Sriastradh *
197319b3e1Sriastradh * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
207319b3e1Sriastradh * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
217319b3e1Sriastradh * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
227319b3e1Sriastradh * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
237319b3e1Sriastradh * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
247319b3e1Sriastradh * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
257319b3e1Sriastradh * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
267319b3e1Sriastradh * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
277319b3e1Sriastradh * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
287319b3e1Sriastradh * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
297319b3e1Sriastradh * POSSIBILITY OF SUCH DAMAGE.
307319b3e1Sriastradh */
317319b3e1Sriastradh
327319b3e1Sriastradh #include <sys/cdefs.h>
33*02f8a9bfSriastradh __KERNEL_RCSID(0, "$NetBSD: linux_wait_bit.c,v 1.5 2021/12/19 12:36:09 riastradh Exp $");
347319b3e1Sriastradh
357319b3e1Sriastradh #include <sys/param.h>
367319b3e1Sriastradh #include <sys/types.h>
377319b3e1Sriastradh #include <sys/bitops.h>
387319b3e1Sriastradh #include <sys/condvar.h>
397319b3e1Sriastradh #include <sys/mutex.h>
407319b3e1Sriastradh #include <sys/systm.h>
417319b3e1Sriastradh
427319b3e1Sriastradh #include <linux/bitops.h>
437319b3e1Sriastradh #include <linux/sched.h>
447319b3e1Sriastradh #include <linux/wait_bit.h>
457319b3e1Sriastradh
467319b3e1Sriastradh static struct {
477319b3e1Sriastradh struct waitbitentry {
487319b3e1Sriastradh kmutex_t lock;
497319b3e1Sriastradh kcondvar_t cv;
507319b3e1Sriastradh } ent;
517319b3e1Sriastradh char pad[CACHE_LINE_SIZE - sizeof(struct waitbitentry)];
527319b3e1Sriastradh } waitbittab[PAGE_SIZE/CACHE_LINE_SIZE] __cacheline_aligned;
537319b3e1Sriastradh CTASSERT(sizeof(waitbittab) == PAGE_SIZE);
547319b3e1Sriastradh CTASSERT(sizeof(waitbittab[0]) == CACHE_LINE_SIZE);
557319b3e1Sriastradh
567319b3e1Sriastradh int
linux_wait_bit_init(void)577319b3e1Sriastradh linux_wait_bit_init(void)
587319b3e1Sriastradh {
597319b3e1Sriastradh size_t i;
607319b3e1Sriastradh
617319b3e1Sriastradh for (i = 0; i < __arraycount(waitbittab); i++) {
627319b3e1Sriastradh mutex_init(&waitbittab[i].ent.lock, MUTEX_DEFAULT, IPL_VM);
637319b3e1Sriastradh cv_init(&waitbittab[i].ent.cv, "waitbit");
647319b3e1Sriastradh }
657319b3e1Sriastradh
667319b3e1Sriastradh return 0;
677319b3e1Sriastradh }
687319b3e1Sriastradh
697319b3e1Sriastradh void
linux_wait_bit_fini(void)707319b3e1Sriastradh linux_wait_bit_fini(void)
717319b3e1Sriastradh {
727319b3e1Sriastradh size_t i;
737319b3e1Sriastradh
747319b3e1Sriastradh for (i = 0; i < __arraycount(waitbittab); i++) {
757319b3e1Sriastradh cv_destroy(&waitbittab[i].ent.cv);
767319b3e1Sriastradh mutex_destroy(&waitbittab[i].ent.lock);
777319b3e1Sriastradh }
787319b3e1Sriastradh }
797319b3e1Sriastradh
807319b3e1Sriastradh static inline size_t
wait_bit_hash(const volatile unsigned long * bitmap,unsigned bit)817319b3e1Sriastradh wait_bit_hash(const volatile unsigned long *bitmap, unsigned bit)
827319b3e1Sriastradh {
837319b3e1Sriastradh /* Try to avoid cache line collisions. */
847319b3e1Sriastradh const volatile unsigned long *word = bitmap + bit/(NBBY*sizeof(*word));
857319b3e1Sriastradh
867319b3e1Sriastradh return ((uintptr_t)word >> ilog2(CACHE_LINE_SIZE)) %
877319b3e1Sriastradh __arraycount(waitbittab);
887319b3e1Sriastradh }
897319b3e1Sriastradh
907319b3e1Sriastradh static struct waitbitentry *
wait_bit_enter(const volatile unsigned long * bitmap,unsigned bit)917319b3e1Sriastradh wait_bit_enter(const volatile unsigned long *bitmap, unsigned bit)
927319b3e1Sriastradh {
937319b3e1Sriastradh struct waitbitentry *wbe = &waitbittab[wait_bit_hash(bitmap, bit)].ent;
947319b3e1Sriastradh
957319b3e1Sriastradh mutex_enter(&wbe->lock);
967319b3e1Sriastradh
977319b3e1Sriastradh return wbe;
987319b3e1Sriastradh }
997319b3e1Sriastradh
1007319b3e1Sriastradh static void
wait_bit_exit(struct waitbitentry * wbe)1017319b3e1Sriastradh wait_bit_exit(struct waitbitentry *wbe)
1027319b3e1Sriastradh {
1037319b3e1Sriastradh
1047319b3e1Sriastradh mutex_exit(&wbe->lock);
1057319b3e1Sriastradh }
1067319b3e1Sriastradh
107*02f8a9bfSriastradh /*
108*02f8a9bfSriastradh * clear_and_wake_up_bit(bit, bitmap)
109*02f8a9bfSriastradh *
110*02f8a9bfSriastradh * Clear the specified bit in the bitmap and wake any waiters in
111*02f8a9bfSriastradh * wait_on_bit or wait_on_bit_timeout that were waiting for it to
112*02f8a9bfSriastradh * clear.
113*02f8a9bfSriastradh */
114b29600b0Sriastradh void
clear_and_wake_up_bit(int bit,volatile unsigned long * bitmap)115b29600b0Sriastradh clear_and_wake_up_bit(int bit, volatile unsigned long *bitmap)
116b29600b0Sriastradh {
117b29600b0Sriastradh struct waitbitentry *wbe;
118b29600b0Sriastradh
119b29600b0Sriastradh wbe = wait_bit_enter(bitmap, bit);
120b29600b0Sriastradh clear_bit(bit, bitmap);
121b29600b0Sriastradh cv_broadcast(&wbe->cv);
122b29600b0Sriastradh wait_bit_exit(wbe);
123b29600b0Sriastradh }
124b29600b0Sriastradh
125*02f8a9bfSriastradh /*
126*02f8a9bfSriastradh * wait_on_bit(bitmap, bit, flags)
127*02f8a9bfSriastradh *
128*02f8a9bfSriastradh * Wait for the specified bit in bitmap to be cleared. Returns 0
129*02f8a9bfSriastradh * on success, -EINTR on signal, unless flags has
130*02f8a9bfSriastradh * TASK_UNINTERRUPTIBLE set.
131*02f8a9bfSriastradh */
1327319b3e1Sriastradh int
wait_on_bit(const volatile unsigned long * bitmap,unsigned bit,int flags)133d602960fSriastradh wait_on_bit(const volatile unsigned long *bitmap, unsigned bit, int flags)
134d602960fSriastradh {
135d602960fSriastradh struct waitbitentry *wbe;
136d602960fSriastradh int error, ret;
137d602960fSriastradh
138*02f8a9bfSriastradh if (test_bit(bit, bitmap) == 0)
139d602960fSriastradh return 0;
140d602960fSriastradh
141d602960fSriastradh wbe = wait_bit_enter(bitmap, bit);
142d602960fSriastradh
143*02f8a9bfSriastradh while (test_bit(bit, bitmap)) {
144d602960fSriastradh if (flags & TASK_UNINTERRUPTIBLE) {
145d602960fSriastradh cv_wait(&wbe->cv, &wbe->lock);
146d602960fSriastradh } else {
147d602960fSriastradh error = cv_wait_sig(&wbe->cv, &wbe->lock);
148*02f8a9bfSriastradh if (error) {
149*02f8a9bfSriastradh /* cv_wait_sig can only fail on signal. */
150*02f8a9bfSriastradh KASSERTMSG(error == EINTR || error == ERESTART,
151*02f8a9bfSriastradh "error=%d", error);
152*02f8a9bfSriastradh ret = -EINTR;
153d602960fSriastradh goto out;
154d602960fSriastradh }
155d602960fSriastradh }
156*02f8a9bfSriastradh }
157d602960fSriastradh
158*02f8a9bfSriastradh /* Bit is clear. Return zero on success. */
159*02f8a9bfSriastradh KASSERT(test_bit(bit, bitmap) == 0);
160d602960fSriastradh ret = 0;
161d602960fSriastradh
162*02f8a9bfSriastradh out: KASSERT(test_bit(bit, bitmap) == 0 || ret != 0);
163*02f8a9bfSriastradh wait_bit_exit(wbe);
164d602960fSriastradh return ret;
165d602960fSriastradh }
166d602960fSriastradh
167*02f8a9bfSriastradh /*
168*02f8a9bfSriastradh * wait_on_bit_timeout(bitmap, bit, flags, timeout)
169*02f8a9bfSriastradh *
170*02f8a9bfSriastradh * Wait for the specified bit in bitmap to be cleared. Returns 0
171*02f8a9bfSriastradh * on success, -EINTR on signal unless flags has
172*02f8a9bfSriastradh * TASK_UNINTERRUPTIBLE set, or -EAGAIN on timeout.
173*02f8a9bfSriastradh */
174d602960fSriastradh int
wait_on_bit_timeout(const volatile unsigned long * bitmap,unsigned bit,int flags,unsigned long timeout)1757319b3e1Sriastradh wait_on_bit_timeout(const volatile unsigned long *bitmap, unsigned bit,
1767319b3e1Sriastradh int flags, unsigned long timeout)
1777319b3e1Sriastradh {
1787319b3e1Sriastradh struct waitbitentry *wbe;
1797319b3e1Sriastradh int error, ret;
1807319b3e1Sriastradh
181*02f8a9bfSriastradh if (test_bit(bit, bitmap) == 0)
182*02f8a9bfSriastradh return 0;
1837319b3e1Sriastradh
1847319b3e1Sriastradh wbe = wait_bit_enter(bitmap, bit);
1857319b3e1Sriastradh
186*02f8a9bfSriastradh while (test_bit(bit, bitmap)) {
1877319b3e1Sriastradh unsigned starttime, endtime;
1887319b3e1Sriastradh
189*02f8a9bfSriastradh if (timeout == 0) {
190*02f8a9bfSriastradh ret = -EAGAIN;
1917319b3e1Sriastradh goto out;
1927319b3e1Sriastradh }
1937319b3e1Sriastradh
194*02f8a9bfSriastradh starttime = getticks();
195*02f8a9bfSriastradh if (flags & TASK_UNINTERRUPTIBLE) {
196*02f8a9bfSriastradh error = cv_timedwait(&wbe->cv, &wbe->lock,
197*02f8a9bfSriastradh MIN(timeout, INT_MAX/2));
198*02f8a9bfSriastradh } else {
199*02f8a9bfSriastradh error = cv_timedwait_sig(&wbe->cv, &wbe->lock,
200*02f8a9bfSriastradh MIN(timeout, INT_MAX/2));
201*02f8a9bfSriastradh }
202*02f8a9bfSriastradh endtime = getticks();
203*02f8a9bfSriastradh
204*02f8a9bfSriastradh /*
205*02f8a9bfSriastradh * If we were interrupted or timed out, massage the
206*02f8a9bfSriastradh * error return and stop here.
207*02f8a9bfSriastradh */
208*02f8a9bfSriastradh if (error) {
209*02f8a9bfSriastradh KASSERTMSG((error == EINTR || error == ERESTART ||
210*02f8a9bfSriastradh error == EWOULDBLOCK), "error=%d", error);
21199922f3fSriastradh if (error == EINTR || error == ERESTART) {
212*02f8a9bfSriastradh ret = -EINTR;
213*02f8a9bfSriastradh } else if (error == EWOULDBLOCK) {
214*02f8a9bfSriastradh ret = -EAGAIN;
215*02f8a9bfSriastradh } else {
216*02f8a9bfSriastradh panic("invalid error=%d", error);
217*02f8a9bfSriastradh }
2187319b3e1Sriastradh goto out;
2197319b3e1Sriastradh }
2207319b3e1Sriastradh
2217319b3e1Sriastradh /* Otherwise, debit the time spent. */
222*02f8a9bfSriastradh timeout -= MIN(timeout, (endtime - starttime));
2237319b3e1Sriastradh }
224*02f8a9bfSriastradh
225*02f8a9bfSriastradh /* Bit is clear. Return zero on success. */
226*02f8a9bfSriastradh KASSERT(test_bit(bit, bitmap) == 0);
2277319b3e1Sriastradh ret = timeout;
2287319b3e1Sriastradh
229*02f8a9bfSriastradh out: KASSERT(test_bit(bit, bitmap) == 0 || ret != 0);
230*02f8a9bfSriastradh wait_bit_exit(wbe);
2317319b3e1Sriastradh return ret;
2327319b3e1Sriastradh }
233