1 /* $OpenBSD: kern_rwlock.c,v 1.22 2014/07/09 13:32:00 guenther Exp $ */ 2 3 /* 4 * Copyright (c) 2002, 2003 Artur Grabowski <art@openbsd.org> 5 * Copyright (c) 2011 Thordur Bjornsson <thib@secnorth.net> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/param.h> 21 #include <sys/systm.h> 22 #include <sys/proc.h> 23 #include <sys/rwlock.h> 24 #include <sys/limits.h> 25 26 #include <machine/lock.h> 27 28 /* XXX - temporary measure until proc0 is properly aligned */ 29 #define RW_PROC(p) (((long)p) & ~RWLOCK_MASK) 30 31 /* 32 * Magic wand for lock operations. Every operation checks if certain 33 * flags are set and if they aren't, it increments the lock with some 34 * value (that might need some computing in a few cases). If the operation 35 * fails, we need to set certain flags while waiting for the lock. 36 * 37 * RW_WRITE The lock must be completely empty. We increment it with 38 * RWLOCK_WRLOCK and the proc pointer of the holder. 39 * Sets RWLOCK_WAIT|RWLOCK_WRWANT while waiting. 40 * RW_READ RWLOCK_WRLOCK|RWLOCK_WRWANT may not be set. We increment 41 * with RWLOCK_READ_INCR. RWLOCK_WAIT while waiting. 42 */ 43 static const struct rwlock_op { 44 unsigned long inc; 45 unsigned long check; 46 unsigned long wait_set; 47 long proc_mult; 48 int wait_prio; 49 } rw_ops[] = { 50 { /* RW_WRITE */ 51 RWLOCK_WRLOCK, 52 ULONG_MAX, 53 RWLOCK_WAIT | RWLOCK_WRWANT, 54 1, 55 PLOCK - 4 56 }, 57 { /* RW_READ */ 58 RWLOCK_READ_INCR, 59 RWLOCK_WRLOCK, 60 RWLOCK_WAIT, 61 0, 62 PLOCK 63 }, 64 { /* Sparse Entry. */ 65 0, 66 }, 67 { /* RW_DOWNGRADE */ 68 RWLOCK_READ_INCR - RWLOCK_WRLOCK, 69 0, 70 0, 71 -1, 72 PLOCK 73 }, 74 }; 75 76 #ifndef __HAVE_MD_RWLOCK 77 /* 78 * Simple cases that should be in MD code and atomic. 79 */ 80 void 81 rw_enter_read(struct rwlock *rwl) 82 { 83 unsigned long owner = rwl->rwl_owner; 84 85 if (__predict_false((owner & RWLOCK_WRLOCK) || 86 rw_cas(&rwl->rwl_owner, owner, owner + RWLOCK_READ_INCR))) 87 rw_enter(rwl, RW_READ); 88 } 89 90 void 91 rw_enter_write(struct rwlock *rwl) 92 { 93 struct proc *p = curproc; 94 95 if (__predict_false(rw_cas(&rwl->rwl_owner, 0, 96 RW_PROC(p) | RWLOCK_WRLOCK))) 97 rw_enter(rwl, RW_WRITE); 98 } 99 100 void 101 rw_exit_read(struct rwlock *rwl) 102 { 103 unsigned long owner = rwl->rwl_owner; 104 105 rw_assert_rdlock(rwl); 106 107 if (__predict_false((owner & RWLOCK_WAIT) || 108 rw_cas(&rwl->rwl_owner, owner, owner - RWLOCK_READ_INCR))) 109 rw_exit(rwl); 110 } 111 112 void 113 rw_exit_write(struct rwlock *rwl) 114 { 115 unsigned long owner = rwl->rwl_owner; 116 117 rw_assert_wrlock(rwl); 118 119 if (__predict_false((owner & RWLOCK_WAIT) || 120 rw_cas(&rwl->rwl_owner, owner, 0))) 121 rw_exit(rwl); 122 } 123 124 #ifndef rw_cas 125 int 126 rw_cas(volatile unsigned long *p, unsigned long o, unsigned long n) 127 { 128 if (*p != o) 129 return (1); 130 *p = n; 131 132 return (0); 133 } 134 #endif 135 136 #endif 137 138 #ifdef DIAGNOSTIC 139 /* 140 * Put the diagnostic functions here to keep the main code free 141 * from ifdef clutter. 142 */ 143 static void 144 rw_enter_diag(struct rwlock *rwl, int flags) 145 { 146 switch (flags & RW_OPMASK) { 147 case RW_WRITE: 148 case RW_READ: 149 if (RW_PROC(curproc) == RW_PROC(rwl->rwl_owner)) 150 panic("rw_enter: %s locking against myself", 151 rwl->rwl_name); 152 break; 153 case RW_DOWNGRADE: 154 /* 155 * If we're downgrading, we must hold the write lock. 156 */ 157 if ((rwl->rwl_owner & RWLOCK_WRLOCK) == 0) 158 panic("rw_enter: %s downgrade of non-write lock", 159 rwl->rwl_name); 160 if (RW_PROC(curproc) != RW_PROC(rwl->rwl_owner)) 161 panic("rw_enter: %s downgrade, not holder", 162 rwl->rwl_name); 163 break; 164 165 default: 166 panic("rw_enter: unknown op 0x%x", flags); 167 } 168 } 169 170 #else 171 #define rw_enter_diag(r, f) 172 #endif 173 174 void 175 rw_init(struct rwlock *rwl, const char *name) 176 { 177 rwl->rwl_owner = 0; 178 rwl->rwl_name = name; 179 } 180 181 int 182 rw_enter(struct rwlock *rwl, int flags) 183 { 184 const struct rwlock_op *op; 185 struct sleep_state sls; 186 unsigned long inc, o; 187 int error; 188 189 op = &rw_ops[(flags & RW_OPMASK) - 1]; 190 191 inc = op->inc + RW_PROC(curproc) * op->proc_mult; 192 retry: 193 while (__predict_false(((o = rwl->rwl_owner) & op->check) != 0)) { 194 unsigned long set = o | op->wait_set; 195 int do_sleep; 196 197 rw_enter_diag(rwl, flags); 198 199 if (flags & RW_NOSLEEP) 200 return (EBUSY); 201 202 sleep_setup(&sls, rwl, op->wait_prio, rwl->rwl_name); 203 if (flags & RW_INTR) 204 sleep_setup_signal(&sls, op->wait_prio | PCATCH); 205 206 do_sleep = !rw_cas(&rwl->rwl_owner, o, set); 207 208 sleep_finish(&sls, do_sleep); 209 if ((flags & RW_INTR) && 210 (error = sleep_finish_signal(&sls)) != 0) 211 return (error); 212 if (flags & RW_SLEEPFAIL) 213 return (EAGAIN); 214 } 215 216 if (__predict_false(rw_cas(&rwl->rwl_owner, o, o + inc))) 217 goto retry; 218 219 /* 220 * If old lock had RWLOCK_WAIT and RWLOCK_WRLOCK set, it means we 221 * downgraded a write lock and had possible read waiter, wake them 222 * to let them retry the lock. 223 */ 224 if (__predict_false((o & (RWLOCK_WRLOCK|RWLOCK_WAIT)) == 225 (RWLOCK_WRLOCK|RWLOCK_WAIT))) 226 wakeup(rwl); 227 228 return (0); 229 } 230 231 void 232 rw_exit(struct rwlock *rwl) 233 { 234 unsigned long owner = rwl->rwl_owner; 235 int wrlock = owner & RWLOCK_WRLOCK; 236 unsigned long set; 237 238 if (wrlock) 239 rw_assert_wrlock(rwl); 240 else 241 rw_assert_rdlock(rwl); 242 243 do { 244 owner = rwl->rwl_owner; 245 if (wrlock) 246 set = 0; 247 else 248 set = (owner - RWLOCK_READ_INCR) & 249 ~(RWLOCK_WAIT|RWLOCK_WRWANT); 250 } while (rw_cas(&rwl->rwl_owner, owner, set)); 251 252 if (owner & RWLOCK_WAIT) 253 wakeup(rwl); 254 } 255 256 int 257 rw_status(struct rwlock *rwl) 258 { 259 if (rwl->rwl_owner & RWLOCK_WRLOCK) { 260 if (RW_PROC(curproc) == RW_PROC(rwl->rwl_owner)) 261 return RW_WRITE; 262 else 263 return RW_WRITE_OTHER; 264 } 265 if (rwl->rwl_owner) 266 return RW_READ; 267 return (0); 268 } 269 270 #ifdef DIAGNOSTIC 271 void 272 rw_assert_wrlock(struct rwlock *rwl) 273 { 274 if (!(rwl->rwl_owner & RWLOCK_WRLOCK)) 275 panic("%s: lock not held", rwl->rwl_name); 276 277 if (RWLOCK_OWNER(rwl) != (struct proc *)RW_PROC(curproc)) 278 panic("%s: lock not held by this process", rwl->rwl_name); 279 } 280 281 void 282 rw_assert_rdlock(struct rwlock *rwl) 283 { 284 if (!RWLOCK_OWNER(rwl) || (rwl->rwl_owner & RWLOCK_WRLOCK)) 285 panic("%s: lock not shared", rwl->rwl_name); 286 } 287 288 void 289 rw_assert_unlocked(struct rwlock *rwl) 290 { 291 if (rwl->rwl_owner != 0L) 292 panic("%s: lock held", rwl->rwl_name); 293 } 294 #endif 295 296 /* recursive rwlocks; */ 297 void 298 rrw_init(struct rrwlock *rrwl, char *name) 299 { 300 memset(rrwl, 0, sizeof(struct rrwlock)); 301 rw_init(&rrwl->rrwl_lock, name); 302 } 303 304 int 305 rrw_enter(struct rrwlock *rrwl, int flags) 306 { 307 int rv; 308 309 if (RWLOCK_OWNER(&rrwl->rrwl_lock) == 310 (struct proc *)RW_PROC(curproc)) { 311 if (flags & RW_RECURSEFAIL) 312 return (EDEADLK); 313 else { 314 rrwl->rrwl_wcnt++; 315 return (0); 316 } 317 } 318 319 rv = rw_enter(&rrwl->rrwl_lock, flags); 320 if (rv == 0) 321 rrwl->rrwl_wcnt = 1; 322 323 return (rv); 324 } 325 326 void 327 rrw_exit(struct rrwlock *rrwl) 328 { 329 330 if (RWLOCK_OWNER(&rrwl->rrwl_lock) == 331 (struct proc *)RW_PROC(curproc)) { 332 KASSERT(rrwl->rrwl_wcnt > 0); 333 rrwl->rrwl_wcnt--; 334 if (rrwl->rrwl_wcnt != 0) 335 return; 336 } 337 338 rw_exit(&rrwl->rrwl_lock); 339 } 340 341 int 342 rrw_status(struct rrwlock *rrwl) 343 { 344 return (rw_status(&rrwl->rrwl_lock)); 345 } 346