1 /* $OpenBSD: kern_rwlock.c,v 1.27 2015/03/14 07:33:42 jsg Exp $ */ 2 3 /* 4 * Copyright (c) 2002, 2003 Artur Grabowski <art@openbsd.org> 5 * Copyright (c) 2011 Thordur Bjornsson <thib@secnorth.net> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/param.h> 21 #include <sys/systm.h> 22 #include <sys/proc.h> 23 #include <sys/rwlock.h> 24 #include <sys/limits.h> 25 #include <sys/atomic.h> 26 27 /* XXX - temporary measure until proc0 is properly aligned */ 28 #define RW_PROC(p) (((long)p) & ~RWLOCK_MASK) 29 30 #ifdef MULTIPROCESSOR 31 #define rw_cas(p, o, n) (atomic_cas_ulong(p, o, n) != o) 32 #else 33 static inline int 34 rw_cas(volatile unsigned long *p, unsigned long o, unsigned long n) 35 { 36 if (*p != o) 37 return (1); 38 *p = n; 39 40 return (0); 41 } 42 #endif 43 44 /* 45 * Magic wand for lock operations. Every operation checks if certain 46 * flags are set and if they aren't, it increments the lock with some 47 * value (that might need some computing in a few cases). If the operation 48 * fails, we need to set certain flags while waiting for the lock. 49 * 50 * RW_WRITE The lock must be completely empty. We increment it with 51 * RWLOCK_WRLOCK and the proc pointer of the holder. 52 * Sets RWLOCK_WAIT|RWLOCK_WRWANT while waiting. 53 * RW_READ RWLOCK_WRLOCK|RWLOCK_WRWANT may not be set. We increment 54 * with RWLOCK_READ_INCR. RWLOCK_WAIT while waiting. 55 */ 56 static const struct rwlock_op { 57 unsigned long inc; 58 unsigned long check; 59 unsigned long wait_set; 60 long proc_mult; 61 int wait_prio; 62 } rw_ops[] = { 63 { /* RW_WRITE */ 64 RWLOCK_WRLOCK, 65 ULONG_MAX, 66 RWLOCK_WAIT | RWLOCK_WRWANT, 67 1, 68 PLOCK - 4 69 }, 70 { /* RW_READ */ 71 RWLOCK_READ_INCR, 72 RWLOCK_WRLOCK, 73 RWLOCK_WAIT, 74 0, 75 PLOCK 76 }, 77 { /* Sparse Entry. */ 78 0, 79 }, 80 { /* RW_DOWNGRADE */ 81 RWLOCK_READ_INCR - RWLOCK_WRLOCK, 82 0, 83 0, 84 -1, 85 PLOCK 86 }, 87 }; 88 89 void 90 rw_enter_read(struct rwlock *rwl) 91 { 92 unsigned long owner = rwl->rwl_owner; 93 94 if (__predict_false((owner & RWLOCK_WRLOCK) || 95 rw_cas(&rwl->rwl_owner, owner, owner + RWLOCK_READ_INCR))) 96 rw_enter(rwl, RW_READ); 97 else 98 membar_enter(); 99 } 100 101 void 102 rw_enter_write(struct rwlock *rwl) 103 { 104 struct proc *p = curproc; 105 106 if (__predict_false(rw_cas(&rwl->rwl_owner, 0, 107 RW_PROC(p) | RWLOCK_WRLOCK))) 108 rw_enter(rwl, RW_WRITE); 109 else 110 membar_enter(); 111 } 112 113 void 114 rw_exit_read(struct rwlock *rwl) 115 { 116 unsigned long owner = rwl->rwl_owner; 117 118 rw_assert_rdlock(rwl); 119 120 membar_exit(); 121 if (__predict_false((owner & RWLOCK_WAIT) || 122 rw_cas(&rwl->rwl_owner, owner, owner - RWLOCK_READ_INCR))) 123 rw_exit(rwl); 124 } 125 126 void 127 rw_exit_write(struct rwlock *rwl) 128 { 129 unsigned long owner = rwl->rwl_owner; 130 131 rw_assert_wrlock(rwl); 132 133 membar_exit(); 134 if (__predict_false((owner & RWLOCK_WAIT) || 135 rw_cas(&rwl->rwl_owner, owner, 0))) 136 rw_exit(rwl); 137 } 138 139 #ifdef DIAGNOSTIC 140 /* 141 * Put the diagnostic functions here to keep the main code free 142 * from ifdef clutter. 143 */ 144 static void 145 rw_enter_diag(struct rwlock *rwl, int flags) 146 { 147 switch (flags & RW_OPMASK) { 148 case RW_WRITE: 149 case RW_READ: 150 if (RW_PROC(curproc) == RW_PROC(rwl->rwl_owner)) 151 panic("rw_enter: %s locking against myself", 152 rwl->rwl_name); 153 break; 154 case RW_DOWNGRADE: 155 /* 156 * If we're downgrading, we must hold the write lock. 157 */ 158 if ((rwl->rwl_owner & RWLOCK_WRLOCK) == 0) 159 panic("rw_enter: %s downgrade of non-write lock", 160 rwl->rwl_name); 161 if (RW_PROC(curproc) != RW_PROC(rwl->rwl_owner)) 162 panic("rw_enter: %s downgrade, not holder", 163 rwl->rwl_name); 164 break; 165 166 default: 167 panic("rw_enter: unknown op 0x%x", flags); 168 } 169 } 170 171 #else 172 #define rw_enter_diag(r, f) 173 #endif 174 175 void 176 rw_init(struct rwlock *rwl, const char *name) 177 { 178 rwl->rwl_owner = 0; 179 rwl->rwl_name = name; 180 } 181 182 int 183 rw_enter(struct rwlock *rwl, int flags) 184 { 185 const struct rwlock_op *op; 186 struct sleep_state sls; 187 unsigned long inc, o; 188 int error; 189 190 op = &rw_ops[(flags & RW_OPMASK) - 1]; 191 192 inc = op->inc + RW_PROC(curproc) * op->proc_mult; 193 retry: 194 while (__predict_false(((o = rwl->rwl_owner) & op->check) != 0)) { 195 unsigned long set = o | op->wait_set; 196 int do_sleep; 197 198 rw_enter_diag(rwl, flags); 199 200 if (flags & RW_NOSLEEP) 201 return (EBUSY); 202 203 sleep_setup(&sls, rwl, op->wait_prio, rwl->rwl_name); 204 if (flags & RW_INTR) 205 sleep_setup_signal(&sls, op->wait_prio | PCATCH); 206 207 do_sleep = !rw_cas(&rwl->rwl_owner, o, set); 208 209 sleep_finish(&sls, do_sleep); 210 if ((flags & RW_INTR) && 211 (error = sleep_finish_signal(&sls)) != 0) 212 return (error); 213 if (flags & RW_SLEEPFAIL) 214 return (EAGAIN); 215 } 216 217 if (__predict_false(rw_cas(&rwl->rwl_owner, o, o + inc))) 218 goto retry; 219 membar_enter(); 220 221 /* 222 * If old lock had RWLOCK_WAIT and RWLOCK_WRLOCK set, it means we 223 * downgraded a write lock and had possible read waiter, wake them 224 * to let them retry the lock. 225 */ 226 if (__predict_false((o & (RWLOCK_WRLOCK|RWLOCK_WAIT)) == 227 (RWLOCK_WRLOCK|RWLOCK_WAIT))) 228 wakeup(rwl); 229 230 return (0); 231 } 232 233 void 234 rw_exit(struct rwlock *rwl) 235 { 236 unsigned long owner = rwl->rwl_owner; 237 int wrlock = owner & RWLOCK_WRLOCK; 238 unsigned long set; 239 240 if (wrlock) 241 rw_assert_wrlock(rwl); 242 else 243 rw_assert_rdlock(rwl); 244 245 membar_exit(); 246 do { 247 owner = rwl->rwl_owner; 248 if (wrlock) 249 set = 0; 250 else 251 set = (owner - RWLOCK_READ_INCR) & 252 ~(RWLOCK_WAIT|RWLOCK_WRWANT); 253 } while (rw_cas(&rwl->rwl_owner, owner, set)); 254 255 if (owner & RWLOCK_WAIT) 256 wakeup(rwl); 257 } 258 259 int 260 rw_status(struct rwlock *rwl) 261 { 262 if (rwl->rwl_owner & RWLOCK_WRLOCK) { 263 if (RW_PROC(curproc) == RW_PROC(rwl->rwl_owner)) 264 return RW_WRITE; 265 else 266 return RW_WRITE_OTHER; 267 } 268 if (rwl->rwl_owner) 269 return RW_READ; 270 return (0); 271 } 272 273 #ifdef DIAGNOSTIC 274 void 275 rw_assert_wrlock(struct rwlock *rwl) 276 { 277 if (!(rwl->rwl_owner & RWLOCK_WRLOCK)) 278 panic("%s: lock not held", rwl->rwl_name); 279 280 if (RWLOCK_OWNER(rwl) != (struct proc *)RW_PROC(curproc)) 281 panic("%s: lock not held by this process", rwl->rwl_name); 282 } 283 284 void 285 rw_assert_rdlock(struct rwlock *rwl) 286 { 287 if (!RWLOCK_OWNER(rwl) || (rwl->rwl_owner & RWLOCK_WRLOCK)) 288 panic("%s: lock not shared", rwl->rwl_name); 289 } 290 291 void 292 rw_assert_unlocked(struct rwlock *rwl) 293 { 294 if (rwl->rwl_owner != 0L) 295 panic("%s: lock held", rwl->rwl_name); 296 } 297 #endif 298 299 /* recursive rwlocks; */ 300 void 301 rrw_init(struct rrwlock *rrwl, char *name) 302 { 303 memset(rrwl, 0, sizeof(struct rrwlock)); 304 rw_init(&rrwl->rrwl_lock, name); 305 } 306 307 int 308 rrw_enter(struct rrwlock *rrwl, int flags) 309 { 310 int rv; 311 312 if (RWLOCK_OWNER(&rrwl->rrwl_lock) == 313 (struct proc *)RW_PROC(curproc)) { 314 if (flags & RW_RECURSEFAIL) 315 return (EDEADLK); 316 else { 317 rrwl->rrwl_wcnt++; 318 return (0); 319 } 320 } 321 322 rv = rw_enter(&rrwl->rrwl_lock, flags); 323 if (rv == 0) 324 rrwl->rrwl_wcnt = 1; 325 326 return (rv); 327 } 328 329 void 330 rrw_exit(struct rrwlock *rrwl) 331 { 332 333 if (RWLOCK_OWNER(&rrwl->rrwl_lock) == 334 (struct proc *)RW_PROC(curproc)) { 335 KASSERT(rrwl->rrwl_wcnt > 0); 336 rrwl->rrwl_wcnt--; 337 if (rrwl->rrwl_wcnt != 0) 338 return; 339 } 340 341 rw_exit(&rrwl->rrwl_lock); 342 } 343 344 int 345 rrw_status(struct rrwlock *rrwl) 346 { 347 return (rw_status(&rrwl->rrwl_lock)); 348 } 349