1 /* $NetBSD: linux_wait_bit.c,v 1.5 2021/12/19 12:36:09 riastradh Exp $ */ 2 3 /*- 4 * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Taylor R. Campbell. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __KERNEL_RCSID(0, "$NetBSD: linux_wait_bit.c,v 1.5 2021/12/19 12:36:09 riastradh Exp $"); 34 35 #include <sys/param.h> 36 #include <sys/types.h> 37 #include <sys/bitops.h> 38 #include <sys/condvar.h> 39 #include <sys/mutex.h> 40 #include <sys/systm.h> 41 42 #include <linux/bitops.h> 43 #include <linux/sched.h> 44 #include <linux/wait_bit.h> 45 46 static struct { 47 struct waitbitentry { 48 kmutex_t lock; 49 kcondvar_t cv; 50 } ent; 51 char pad[CACHE_LINE_SIZE - sizeof(struct waitbitentry)]; 52 } waitbittab[PAGE_SIZE/CACHE_LINE_SIZE] __cacheline_aligned; 53 CTASSERT(sizeof(waitbittab) == PAGE_SIZE); 54 CTASSERT(sizeof(waitbittab[0]) == CACHE_LINE_SIZE); 55 56 int 57 linux_wait_bit_init(void) 58 { 59 size_t i; 60 61 for (i = 0; i < __arraycount(waitbittab); i++) { 62 mutex_init(&waitbittab[i].ent.lock, MUTEX_DEFAULT, IPL_VM); 63 cv_init(&waitbittab[i].ent.cv, "waitbit"); 64 } 65 66 return 0; 67 } 68 69 void 70 linux_wait_bit_fini(void) 71 { 72 size_t i; 73 74 for (i = 0; i < __arraycount(waitbittab); i++) { 75 cv_destroy(&waitbittab[i].ent.cv); 76 mutex_destroy(&waitbittab[i].ent.lock); 77 } 78 } 79 80 static inline size_t 81 wait_bit_hash(const volatile unsigned long *bitmap, unsigned bit) 82 { 83 /* Try to avoid cache line collisions. */ 84 const volatile unsigned long *word = bitmap + bit/(NBBY*sizeof(*word)); 85 86 return ((uintptr_t)word >> ilog2(CACHE_LINE_SIZE)) % 87 __arraycount(waitbittab); 88 } 89 90 static struct waitbitentry * 91 wait_bit_enter(const volatile unsigned long *bitmap, unsigned bit) 92 { 93 struct waitbitentry *wbe = &waitbittab[wait_bit_hash(bitmap, bit)].ent; 94 95 mutex_enter(&wbe->lock); 96 97 return wbe; 98 } 99 100 static void 101 wait_bit_exit(struct waitbitentry *wbe) 102 { 103 104 mutex_exit(&wbe->lock); 105 } 106 107 /* 108 * clear_and_wake_up_bit(bit, bitmap) 109 * 110 * Clear the specified bit in the bitmap and wake any waiters in 111 * wait_on_bit or wait_on_bit_timeout that were waiting for it to 112 * clear. 113 */ 114 void 115 clear_and_wake_up_bit(int bit, volatile unsigned long *bitmap) 116 { 117 struct waitbitentry *wbe; 118 119 wbe = wait_bit_enter(bitmap, bit); 120 clear_bit(bit, bitmap); 121 cv_broadcast(&wbe->cv); 122 wait_bit_exit(wbe); 123 } 124 125 /* 126 * wait_on_bit(bitmap, bit, flags) 127 * 128 * Wait for the specified bit in bitmap to be cleared. Returns 0 129 * on success, -EINTR on signal, unless flags has 130 * TASK_UNINTERRUPTIBLE set. 131 */ 132 int 133 wait_on_bit(const volatile unsigned long *bitmap, unsigned bit, int flags) 134 { 135 struct waitbitentry *wbe; 136 int error, ret; 137 138 if (test_bit(bit, bitmap) == 0) 139 return 0; 140 141 wbe = wait_bit_enter(bitmap, bit); 142 143 while (test_bit(bit, bitmap)) { 144 if (flags & TASK_UNINTERRUPTIBLE) { 145 cv_wait(&wbe->cv, &wbe->lock); 146 } else { 147 error = cv_wait_sig(&wbe->cv, &wbe->lock); 148 if (error) { 149 /* cv_wait_sig can only fail on signal. */ 150 KASSERTMSG(error == EINTR || error == ERESTART, 151 "error=%d", error); 152 ret = -EINTR; 153 goto out; 154 } 155 } 156 } 157 158 /* Bit is clear. Return zero on success. */ 159 KASSERT(test_bit(bit, bitmap) == 0); 160 ret = 0; 161 162 out: KASSERT(test_bit(bit, bitmap) == 0 || ret != 0); 163 wait_bit_exit(wbe); 164 return ret; 165 } 166 167 /* 168 * wait_on_bit_timeout(bitmap, bit, flags, timeout) 169 * 170 * Wait for the specified bit in bitmap to be cleared. Returns 0 171 * on success, -EINTR on signal unless flags has 172 * TASK_UNINTERRUPTIBLE set, or -EAGAIN on timeout. 173 */ 174 int 175 wait_on_bit_timeout(const volatile unsigned long *bitmap, unsigned bit, 176 int flags, unsigned long timeout) 177 { 178 struct waitbitentry *wbe; 179 int error, ret; 180 181 if (test_bit(bit, bitmap) == 0) 182 return 0; 183 184 wbe = wait_bit_enter(bitmap, bit); 185 186 while (test_bit(bit, bitmap)) { 187 unsigned starttime, endtime; 188 189 if (timeout == 0) { 190 ret = -EAGAIN; 191 goto out; 192 } 193 194 starttime = getticks(); 195 if (flags & TASK_UNINTERRUPTIBLE) { 196 error = cv_timedwait(&wbe->cv, &wbe->lock, 197 MIN(timeout, INT_MAX/2)); 198 } else { 199 error = cv_timedwait_sig(&wbe->cv, &wbe->lock, 200 MIN(timeout, INT_MAX/2)); 201 } 202 endtime = getticks(); 203 204 /* 205 * If we were interrupted or timed out, massage the 206 * error return and stop here. 207 */ 208 if (error) { 209 KASSERTMSG((error == EINTR || error == ERESTART || 210 error == EWOULDBLOCK), "error=%d", error); 211 if (error == EINTR || error == ERESTART) { 212 ret = -EINTR; 213 } else if (error == EWOULDBLOCK) { 214 ret = -EAGAIN; 215 } else { 216 panic("invalid error=%d", error); 217 } 218 goto out; 219 } 220 221 /* Otherwise, debit the time spent. */ 222 timeout -= MIN(timeout, (endtime - starttime)); 223 } 224 225 /* Bit is clear. Return zero on success. */ 226 KASSERT(test_bit(bit, bitmap) == 0); 227 ret = timeout; 228 229 out: KASSERT(test_bit(bit, bitmap) == 0 || ret != 0); 230 wait_bit_exit(wbe); 231 return ret; 232 } 233