1 /* $NetBSD: kern_rwlock.c,v 1.54 2019/05/09 05:00:31 ozaki-r Exp $ */ 2 3 /*- 4 * Copyright (c) 2002, 2006, 2007, 2008, 2009 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe and Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Kernel reader/writer lock implementation, modeled after those 34 * found in Solaris, a description of which can be found in: 35 * 36 * Solaris Internals: Core Kernel Architecture, Jim Mauro and 37 * Richard McDougall. 38 */ 39 40 #include <sys/cdefs.h> 41 __KERNEL_RCSID(0, "$NetBSD: kern_rwlock.c,v 1.54 2019/05/09 05:00:31 ozaki-r Exp $"); 42 43 #define __RWLOCK_PRIVATE 44 45 #include <sys/param.h> 46 #include <sys/proc.h> 47 #include <sys/rwlock.h> 48 #include <sys/sched.h> 49 #include <sys/sleepq.h> 50 #include <sys/systm.h> 51 #include <sys/lockdebug.h> 52 #include <sys/cpu.h> 53 #include <sys/atomic.h> 54 #include <sys/lock.h> 55 #include <sys/pserialize.h> 56 57 #include <dev/lockstat.h> 58 59 /* 60 * LOCKDEBUG 61 */ 62 63 #if defined(LOCKDEBUG) 64 65 #define RW_WANTLOCK(rw, op) \ 66 LOCKDEBUG_WANTLOCK(RW_DEBUG_P(rw), (rw), \ 67 (uintptr_t)__builtin_return_address(0), op == RW_READER); 68 #define RW_LOCKED(rw, op) \ 69 LOCKDEBUG_LOCKED(RW_DEBUG_P(rw), (rw), NULL, \ 70 (uintptr_t)__builtin_return_address(0), op == RW_READER); 71 #define RW_UNLOCKED(rw, op) \ 72 LOCKDEBUG_UNLOCKED(RW_DEBUG_P(rw), (rw), \ 73 (uintptr_t)__builtin_return_address(0), op == RW_READER); 74 #define RW_DASSERT(rw, cond) \ 75 do { \ 76 if (__predict_false(!(cond))) \ 77 rw_abort(__func__, __LINE__, rw, "assertion failed: " #cond);\ 78 } while (/* CONSTCOND */ 0); 79 80 #else /* LOCKDEBUG */ 81 82 #define RW_WANTLOCK(rw, op) /* nothing */ 83 #define RW_LOCKED(rw, op) /* nothing */ 84 #define RW_UNLOCKED(rw, op) /* nothing */ 85 #define RW_DASSERT(rw, cond) /* nothing */ 86 87 #endif /* LOCKDEBUG */ 88 89 /* 90 * DIAGNOSTIC 91 */ 92 93 #if defined(DIAGNOSTIC) 94 95 #define RW_ASSERT(rw, cond) \ 96 do { \ 97 if (__predict_false(!(cond))) \ 98 rw_abort(__func__, __LINE__, rw, "assertion failed: " #cond);\ 99 } while (/* CONSTCOND */ 0) 100 101 #else 102 103 #define RW_ASSERT(rw, cond) /* nothing */ 104 105 #endif /* DIAGNOSTIC */ 106 107 #define RW_SETDEBUG(rw, on) ((rw)->rw_owner |= (on) ? 0 : RW_NODEBUG) 108 #define RW_DEBUG_P(rw) (((rw)->rw_owner & RW_NODEBUG) == 0) 109 #if defined(LOCKDEBUG) 110 #define RW_INHERITDEBUG(n, o) (n) |= (o) & RW_NODEBUG 111 #else /* defined(LOCKDEBUG) */ 112 #define RW_INHERITDEBUG(n, o) /* nothing */ 113 #endif /* defined(LOCKDEBUG) */ 114 115 static void rw_abort(const char *, size_t, krwlock_t *, const char *); 116 static void rw_dump(const volatile void *, lockop_printer_t); 117 static lwp_t *rw_owner(wchan_t); 118 119 static inline uintptr_t 120 rw_cas(krwlock_t *rw, uintptr_t o, uintptr_t n) 121 { 122 123 RW_INHERITDEBUG(n, o); 124 return (uintptr_t)atomic_cas_ptr((volatile void *)&rw->rw_owner, 125 (void *)o, (void *)n); 126 } 127 128 static inline void 129 rw_swap(krwlock_t *rw, uintptr_t o, uintptr_t n) 130 { 131 132 RW_INHERITDEBUG(n, o); 133 n = (uintptr_t)atomic_swap_ptr((volatile void *)&rw->rw_owner, 134 (void *)n); 135 RW_DASSERT(rw, n == o); 136 } 137 138 /* 139 * For platforms that do not provide stubs, or for the LOCKDEBUG case. 140 */ 141 #ifdef LOCKDEBUG 142 #undef __HAVE_RW_STUBS 143 #endif 144 145 #ifndef __HAVE_RW_STUBS 146 __strong_alias(rw_enter,rw_vector_enter); 147 __strong_alias(rw_exit,rw_vector_exit); 148 __strong_alias(rw_tryenter,rw_vector_tryenter); 149 #endif 150 151 lockops_t rwlock_lockops = { 152 .lo_name = "Reader / writer lock", 153 .lo_type = LOCKOPS_SLEEP, 154 .lo_dump = rw_dump, 155 }; 156 157 syncobj_t rw_syncobj = { 158 .sobj_flag = SOBJ_SLEEPQ_SORTED, 159 .sobj_unsleep = turnstile_unsleep, 160 .sobj_changepri = turnstile_changepri, 161 .sobj_lendpri = sleepq_lendpri, 162 .sobj_owner = rw_owner, 163 }; 164 165 /* 166 * rw_dump: 167 * 168 * Dump the contents of a rwlock structure. 169 */ 170 static void 171 rw_dump(const volatile void *cookie, lockop_printer_t pr) 172 { 173 const volatile krwlock_t *rw = cookie; 174 175 pr("owner/count : %#018lx flags : %#018x\n", 176 (long)RW_OWNER(rw), (int)RW_FLAGS(rw)); 177 } 178 179 /* 180 * rw_abort: 181 * 182 * Dump information about an error and panic the system. This 183 * generates a lot of machine code in the DIAGNOSTIC case, so 184 * we ask the compiler to not inline it. 185 */ 186 static void __noinline 187 rw_abort(const char *func, size_t line, krwlock_t *rw, const char *msg) 188 { 189 190 if (panicstr != NULL) 191 return; 192 193 LOCKDEBUG_ABORT(func, line, rw, &rwlock_lockops, msg); 194 } 195 196 /* 197 * rw_init: 198 * 199 * Initialize a rwlock for use. 200 */ 201 void _rw_init(krwlock_t *, uintptr_t); 202 void 203 _rw_init(krwlock_t *rw, uintptr_t return_address) 204 { 205 bool dodebug; 206 207 memset(rw, 0, sizeof(*rw)); 208 209 dodebug = LOCKDEBUG_ALLOC(rw, &rwlock_lockops, return_address); 210 RW_SETDEBUG(rw, dodebug); 211 } 212 213 void 214 rw_init(krwlock_t *rw) 215 { 216 217 _rw_init(rw, (uintptr_t)__builtin_return_address(0)); 218 } 219 220 /* 221 * rw_destroy: 222 * 223 * Tear down a rwlock. 224 */ 225 void 226 rw_destroy(krwlock_t *rw) 227 { 228 229 RW_ASSERT(rw, (rw->rw_owner & ~RW_NODEBUG) == 0); 230 LOCKDEBUG_FREE(RW_DEBUG_P(rw), rw); 231 } 232 233 /* 234 * rw_oncpu: 235 * 236 * Return true if an rwlock owner is running on a CPU in the system. 237 * If the target is waiting on the kernel big lock, then we must 238 * release it. This is necessary to avoid deadlock. 239 */ 240 static bool 241 rw_oncpu(uintptr_t owner) 242 { 243 #ifdef MULTIPROCESSOR 244 struct cpu_info *ci; 245 lwp_t *l; 246 247 KASSERT(kpreempt_disabled()); 248 249 if ((owner & (RW_WRITE_LOCKED|RW_HAS_WAITERS)) != RW_WRITE_LOCKED) { 250 return false; 251 } 252 253 /* 254 * See lwp_dtor() why dereference of the LWP pointer is safe. 255 * We must have kernel preemption disabled for that. 256 */ 257 l = (lwp_t *)(owner & RW_THREAD); 258 ci = l->l_cpu; 259 260 if (ci && ci->ci_curlwp == l) { 261 /* Target is running; do we need to block? */ 262 return (ci->ci_biglock_wanted != l); 263 } 264 #endif 265 /* Not running. It may be safe to block now. */ 266 return false; 267 } 268 269 /* 270 * rw_vector_enter: 271 * 272 * Acquire a rwlock. 273 */ 274 void 275 rw_vector_enter(krwlock_t *rw, const krw_t op) 276 { 277 uintptr_t owner, incr, need_wait, set_wait, curthread, next; 278 turnstile_t *ts; 279 int queue; 280 lwp_t *l; 281 LOCKSTAT_TIMER(slptime); 282 LOCKSTAT_TIMER(slpcnt); 283 LOCKSTAT_TIMER(spintime); 284 LOCKSTAT_COUNTER(spincnt); 285 LOCKSTAT_FLAG(lsflag); 286 287 l = curlwp; 288 curthread = (uintptr_t)l; 289 290 RW_ASSERT(rw, !cpu_intr_p()); 291 RW_ASSERT(rw, curthread != 0); 292 RW_WANTLOCK(rw, op); 293 294 if (panicstr == NULL) { 295 KDASSERT(pserialize_not_in_read_section()); 296 LOCKDEBUG_BARRIER(&kernel_lock, 1); 297 } 298 299 /* 300 * We play a slight trick here. If we're a reader, we want 301 * increment the read count. If we're a writer, we want to 302 * set the owner field and the WRITE_LOCKED bit. 303 * 304 * In the latter case, we expect those bits to be zero, 305 * therefore we can use an add operation to set them, which 306 * means an add operation for both cases. 307 */ 308 if (__predict_true(op == RW_READER)) { 309 incr = RW_READ_INCR; 310 set_wait = RW_HAS_WAITERS; 311 need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED; 312 queue = TS_READER_Q; 313 } else { 314 RW_DASSERT(rw, op == RW_WRITER); 315 incr = curthread | RW_WRITE_LOCKED; 316 set_wait = RW_HAS_WAITERS | RW_WRITE_WANTED; 317 need_wait = RW_WRITE_LOCKED | RW_THREAD; 318 queue = TS_WRITER_Q; 319 } 320 321 LOCKSTAT_ENTER(lsflag); 322 323 KPREEMPT_DISABLE(curlwp); 324 for (owner = rw->rw_owner; ;) { 325 /* 326 * Read the lock owner field. If the need-to-wait 327 * indicator is clear, then try to acquire the lock. 328 */ 329 if ((owner & need_wait) == 0) { 330 next = rw_cas(rw, owner, (owner + incr) & 331 ~RW_WRITE_WANTED); 332 if (__predict_true(next == owner)) { 333 /* Got it! */ 334 membar_enter(); 335 break; 336 } 337 338 /* 339 * Didn't get it -- spin around again (we'll 340 * probably sleep on the next iteration). 341 */ 342 owner = next; 343 continue; 344 } 345 if (__predict_false(panicstr != NULL)) { 346 KPREEMPT_ENABLE(curlwp); 347 return; 348 } 349 if (__predict_false(RW_OWNER(rw) == curthread)) { 350 rw_abort(__func__, __LINE__, rw, 351 "locking against myself"); 352 } 353 /* 354 * If the lock owner is running on another CPU, and 355 * there are no existing waiters, then spin. 356 */ 357 if (rw_oncpu(owner)) { 358 LOCKSTAT_START_TIMER(lsflag, spintime); 359 u_int count = SPINLOCK_BACKOFF_MIN; 360 do { 361 KPREEMPT_ENABLE(curlwp); 362 SPINLOCK_BACKOFF(count); 363 KPREEMPT_DISABLE(curlwp); 364 owner = rw->rw_owner; 365 } while (rw_oncpu(owner)); 366 LOCKSTAT_STOP_TIMER(lsflag, spintime); 367 LOCKSTAT_COUNT(spincnt, 1); 368 if ((owner & need_wait) == 0) 369 continue; 370 } 371 372 /* 373 * Grab the turnstile chain lock. Once we have that, we 374 * can adjust the waiter bits and sleep queue. 375 */ 376 ts = turnstile_lookup(rw); 377 378 /* 379 * Mark the rwlock as having waiters. If the set fails, 380 * then we may not need to sleep and should spin again. 381 * Reload rw_owner because turnstile_lookup() may have 382 * spun on the turnstile chain lock. 383 */ 384 owner = rw->rw_owner; 385 if ((owner & need_wait) == 0 || rw_oncpu(owner)) { 386 turnstile_exit(rw); 387 continue; 388 } 389 next = rw_cas(rw, owner, owner | set_wait); 390 if (__predict_false(next != owner)) { 391 turnstile_exit(rw); 392 owner = next; 393 continue; 394 } 395 396 LOCKSTAT_START_TIMER(lsflag, slptime); 397 turnstile_block(ts, queue, rw, &rw_syncobj); 398 LOCKSTAT_STOP_TIMER(lsflag, slptime); 399 LOCKSTAT_COUNT(slpcnt, 1); 400 401 /* 402 * No need for a memory barrier because of context switch. 403 * If not handed the lock, then spin again. 404 */ 405 if (op == RW_READER || (rw->rw_owner & RW_THREAD) == curthread) 406 break; 407 408 owner = rw->rw_owner; 409 } 410 KPREEMPT_ENABLE(curlwp); 411 412 LOCKSTAT_EVENT(lsflag, rw, LB_RWLOCK | 413 (op == RW_WRITER ? LB_SLEEP1 : LB_SLEEP2), slpcnt, slptime); 414 LOCKSTAT_EVENT(lsflag, rw, LB_RWLOCK | LB_SPIN, spincnt, spintime); 415 LOCKSTAT_EXIT(lsflag); 416 417 RW_DASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) || 418 (op == RW_READER && RW_COUNT(rw) != 0)); 419 RW_LOCKED(rw, op); 420 } 421 422 /* 423 * rw_vector_exit: 424 * 425 * Release a rwlock. 426 */ 427 void 428 rw_vector_exit(krwlock_t *rw) 429 { 430 uintptr_t curthread, owner, decr, newown, next; 431 turnstile_t *ts; 432 int rcnt, wcnt; 433 lwp_t *l; 434 435 curthread = (uintptr_t)curlwp; 436 RW_ASSERT(rw, curthread != 0); 437 438 if (__predict_false(panicstr != NULL)) 439 return; 440 441 /* 442 * Again, we use a trick. Since we used an add operation to 443 * set the required lock bits, we can use a subtract to clear 444 * them, which makes the read-release and write-release path 445 * the same. 446 */ 447 owner = rw->rw_owner; 448 if (__predict_false((owner & RW_WRITE_LOCKED) != 0)) { 449 RW_UNLOCKED(rw, RW_WRITER); 450 RW_ASSERT(rw, RW_OWNER(rw) == curthread); 451 decr = curthread | RW_WRITE_LOCKED; 452 } else { 453 RW_UNLOCKED(rw, RW_READER); 454 RW_ASSERT(rw, RW_COUNT(rw) != 0); 455 decr = RW_READ_INCR; 456 } 457 458 /* 459 * Compute what we expect the new value of the lock to be. Only 460 * proceed to do direct handoff if there are waiters, and if the 461 * lock would become unowned. 462 */ 463 membar_exit(); 464 for (;;) { 465 newown = (owner - decr); 466 if ((newown & (RW_THREAD | RW_HAS_WAITERS)) == RW_HAS_WAITERS) 467 break; 468 next = rw_cas(rw, owner, newown); 469 if (__predict_true(next == owner)) 470 return; 471 owner = next; 472 } 473 474 /* 475 * Grab the turnstile chain lock. This gets the interlock 476 * on the sleep queue. Once we have that, we can adjust the 477 * waiter bits. 478 */ 479 ts = turnstile_lookup(rw); 480 owner = rw->rw_owner; 481 RW_DASSERT(rw, ts != NULL); 482 RW_DASSERT(rw, (owner & RW_HAS_WAITERS) != 0); 483 484 wcnt = TS_WAITERS(ts, TS_WRITER_Q); 485 rcnt = TS_WAITERS(ts, TS_READER_Q); 486 487 /* 488 * Give the lock away. 489 * 490 * If we are releasing a write lock, then prefer to wake all 491 * outstanding readers. Otherwise, wake one writer if there 492 * are outstanding readers, or all writers if there are no 493 * pending readers. If waking one specific writer, the writer 494 * is handed the lock here. If waking multiple writers, we 495 * set WRITE_WANTED to block out new readers, and let them 496 * do the work of acquiring the lock in rw_vector_enter(). 497 */ 498 if (rcnt == 0 || decr == RW_READ_INCR) { 499 RW_DASSERT(rw, wcnt != 0); 500 RW_DASSERT(rw, (owner & RW_WRITE_WANTED) != 0); 501 502 if (rcnt != 0) { 503 /* Give the lock to the longest waiting writer. */ 504 l = TS_FIRST(ts, TS_WRITER_Q); 505 newown = (uintptr_t)l | RW_WRITE_LOCKED | RW_HAS_WAITERS; 506 if (wcnt > 1) 507 newown |= RW_WRITE_WANTED; 508 rw_swap(rw, owner, newown); 509 turnstile_wakeup(ts, TS_WRITER_Q, 1, l); 510 } else { 511 /* Wake all writers and let them fight it out. */ 512 rw_swap(rw, owner, RW_WRITE_WANTED); 513 turnstile_wakeup(ts, TS_WRITER_Q, wcnt, NULL); 514 } 515 } else { 516 RW_DASSERT(rw, rcnt != 0); 517 518 /* 519 * Give the lock to all blocked readers. If there 520 * is a writer waiting, new readers that arrive 521 * after the release will be blocked out. 522 */ 523 newown = rcnt << RW_READ_COUNT_SHIFT; 524 if (wcnt != 0) 525 newown |= RW_HAS_WAITERS | RW_WRITE_WANTED; 526 527 /* Wake up all sleeping readers. */ 528 rw_swap(rw, owner, newown); 529 turnstile_wakeup(ts, TS_READER_Q, rcnt, NULL); 530 } 531 } 532 533 /* 534 * rw_vector_tryenter: 535 * 536 * Try to acquire a rwlock. 537 */ 538 int 539 rw_vector_tryenter(krwlock_t *rw, const krw_t op) 540 { 541 uintptr_t curthread, owner, incr, need_wait, next; 542 543 curthread = (uintptr_t)curlwp; 544 545 RW_ASSERT(rw, curthread != 0); 546 547 if (op == RW_READER) { 548 incr = RW_READ_INCR; 549 need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED; 550 } else { 551 RW_DASSERT(rw, op == RW_WRITER); 552 incr = curthread | RW_WRITE_LOCKED; 553 need_wait = RW_WRITE_LOCKED | RW_THREAD; 554 } 555 556 for (owner = rw->rw_owner;; owner = next) { 557 owner = rw->rw_owner; 558 if (__predict_false((owner & need_wait) != 0)) 559 return 0; 560 next = rw_cas(rw, owner, owner + incr); 561 if (__predict_true(next == owner)) { 562 /* Got it! */ 563 membar_enter(); 564 break; 565 } 566 } 567 568 RW_WANTLOCK(rw, op); 569 RW_LOCKED(rw, op); 570 RW_DASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) || 571 (op == RW_READER && RW_COUNT(rw) != 0)); 572 573 return 1; 574 } 575 576 /* 577 * rw_downgrade: 578 * 579 * Downgrade a write lock to a read lock. 580 */ 581 void 582 rw_downgrade(krwlock_t *rw) 583 { 584 uintptr_t owner, curthread, newown, next; 585 turnstile_t *ts; 586 int rcnt, wcnt; 587 588 curthread = (uintptr_t)curlwp; 589 RW_ASSERT(rw, curthread != 0); 590 RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) != 0); 591 RW_ASSERT(rw, RW_OWNER(rw) == curthread); 592 RW_UNLOCKED(rw, RW_WRITER); 593 #if !defined(DIAGNOSTIC) 594 __USE(curthread); 595 #endif 596 597 598 membar_producer(); 599 owner = rw->rw_owner; 600 if ((owner & RW_HAS_WAITERS) == 0) { 601 /* 602 * There are no waiters, so we can do this the easy way. 603 * Try swapping us down to one read hold. If it fails, the 604 * lock condition has changed and we most likely now have 605 * waiters. 606 */ 607 next = rw_cas(rw, owner, RW_READ_INCR); 608 if (__predict_true(next == owner)) { 609 RW_LOCKED(rw, RW_READER); 610 RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) == 0); 611 RW_DASSERT(rw, RW_COUNT(rw) != 0); 612 return; 613 } 614 owner = next; 615 } 616 617 /* 618 * Grab the turnstile chain lock. This gets the interlock 619 * on the sleep queue. Once we have that, we can adjust the 620 * waiter bits. 621 */ 622 for (;; owner = next) { 623 ts = turnstile_lookup(rw); 624 RW_DASSERT(rw, ts != NULL); 625 626 rcnt = TS_WAITERS(ts, TS_READER_Q); 627 wcnt = TS_WAITERS(ts, TS_WRITER_Q); 628 629 /* 630 * If there are no readers, just preserve the waiters 631 * bits, swap us down to one read hold and return. 632 */ 633 if (rcnt == 0) { 634 RW_DASSERT(rw, wcnt != 0); 635 RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_WANTED) != 0); 636 RW_DASSERT(rw, (rw->rw_owner & RW_HAS_WAITERS) != 0); 637 638 newown = RW_READ_INCR | RW_HAS_WAITERS | RW_WRITE_WANTED; 639 next = rw_cas(rw, owner, newown); 640 turnstile_exit(rw); 641 if (__predict_true(next == owner)) 642 break; 643 } else { 644 /* 645 * Give the lock to all blocked readers. We may 646 * retain one read hold if downgrading. If there 647 * is a writer waiting, new readers will be blocked 648 * out. 649 */ 650 newown = (rcnt << RW_READ_COUNT_SHIFT) + RW_READ_INCR; 651 if (wcnt != 0) 652 newown |= RW_HAS_WAITERS | RW_WRITE_WANTED; 653 654 next = rw_cas(rw, owner, newown); 655 if (__predict_true(next == owner)) { 656 /* Wake up all sleeping readers. */ 657 turnstile_wakeup(ts, TS_READER_Q, rcnt, NULL); 658 break; 659 } 660 turnstile_exit(rw); 661 } 662 } 663 664 RW_WANTLOCK(rw, RW_READER); 665 RW_LOCKED(rw, RW_READER); 666 RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) == 0); 667 RW_DASSERT(rw, RW_COUNT(rw) != 0); 668 } 669 670 /* 671 * rw_tryupgrade: 672 * 673 * Try to upgrade a read lock to a write lock. We must be the 674 * only reader. 675 */ 676 int 677 rw_tryupgrade(krwlock_t *rw) 678 { 679 uintptr_t owner, curthread, newown, next; 680 681 curthread = (uintptr_t)curlwp; 682 RW_ASSERT(rw, curthread != 0); 683 RW_ASSERT(rw, rw_read_held(rw)); 684 685 for (owner = rw->rw_owner;; owner = next) { 686 RW_ASSERT(rw, (owner & RW_WRITE_LOCKED) == 0); 687 if (__predict_false((owner & RW_THREAD) != RW_READ_INCR)) { 688 RW_ASSERT(rw, (owner & RW_THREAD) != 0); 689 return 0; 690 } 691 newown = curthread | RW_WRITE_LOCKED | (owner & ~RW_THREAD); 692 next = rw_cas(rw, owner, newown); 693 if (__predict_true(next == owner)) { 694 membar_producer(); 695 break; 696 } 697 } 698 699 RW_UNLOCKED(rw, RW_READER); 700 RW_WANTLOCK(rw, RW_WRITER); 701 RW_LOCKED(rw, RW_WRITER); 702 RW_DASSERT(rw, rw->rw_owner & RW_WRITE_LOCKED); 703 RW_DASSERT(rw, RW_OWNER(rw) == curthread); 704 705 return 1; 706 } 707 708 /* 709 * rw_read_held: 710 * 711 * Returns true if the rwlock is held for reading. Must only be 712 * used for diagnostic assertions, and never be used to make 713 * decisions about how to use a rwlock. 714 */ 715 int 716 rw_read_held(krwlock_t *rw) 717 { 718 uintptr_t owner; 719 720 if (panicstr != NULL) 721 return 1; 722 if (rw == NULL) 723 return 0; 724 owner = rw->rw_owner; 725 return (owner & RW_WRITE_LOCKED) == 0 && (owner & RW_THREAD) != 0; 726 } 727 728 /* 729 * rw_write_held: 730 * 731 * Returns true if the rwlock is held for writing. Must only be 732 * used for diagnostic assertions, and never be used to make 733 * decisions about how to use a rwlock. 734 */ 735 int 736 rw_write_held(krwlock_t *rw) 737 { 738 739 if (panicstr != NULL) 740 return 1; 741 if (rw == NULL) 742 return 0; 743 return (rw->rw_owner & (RW_WRITE_LOCKED | RW_THREAD)) == 744 (RW_WRITE_LOCKED | (uintptr_t)curlwp); 745 } 746 747 /* 748 * rw_lock_held: 749 * 750 * Returns true if the rwlock is held for reading or writing. Must 751 * only be used for diagnostic assertions, and never be used to make 752 * decisions about how to use a rwlock. 753 */ 754 int 755 rw_lock_held(krwlock_t *rw) 756 { 757 758 if (panicstr != NULL) 759 return 1; 760 if (rw == NULL) 761 return 0; 762 return (rw->rw_owner & RW_THREAD) != 0; 763 } 764 765 /* 766 * rw_owner: 767 * 768 * Return the current owner of an RW lock, but only if it is write 769 * held. Used for priority inheritance. 770 */ 771 static lwp_t * 772 rw_owner(wchan_t obj) 773 { 774 krwlock_t *rw = (void *)(uintptr_t)obj; /* discard qualifiers */ 775 uintptr_t owner = rw->rw_owner; 776 777 if ((owner & RW_WRITE_LOCKED) == 0) 778 return NULL; 779 780 return (void *)(owner & RW_THREAD); 781 } 782