1 /* $NetBSD: kern_rwlock.c,v 1.18 2008/01/28 19:58:32 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2002, 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe and Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. All advertising materials mentioning features or use of this software 19 * must display the following acknowledgement: 20 * This product includes software developed by the NetBSD 21 * Foundation, Inc. and its contributors. 22 * 4. Neither the name of The NetBSD Foundation nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 28 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 29 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 30 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 31 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 32 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 33 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 34 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 35 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 */ 38 39 /* 40 * Kernel reader/writer lock implementation, modeled after those 41 * found in Solaris, a description of which can be found in: 42 * 43 * Solaris Internals: Core Kernel Architecture, Jim Mauro and 44 * Richard McDougall. 45 */ 46 47 #include <sys/cdefs.h> 48 __KERNEL_RCSID(0, "$NetBSD: kern_rwlock.c,v 1.18 2008/01/28 19:58:32 ad Exp $"); 49 50 #include "opt_multiprocessor.h" 51 52 #define __RWLOCK_PRIVATE 53 54 #include <sys/param.h> 55 #include <sys/proc.h> 56 #include <sys/rwlock.h> 57 #include <sys/sched.h> 58 #include <sys/sleepq.h> 59 #include <sys/systm.h> 60 #include <sys/lockdebug.h> 61 #include <sys/cpu.h> 62 #include <sys/atomic.h> 63 #include <sys/lock.h> 64 65 #include <dev/lockstat.h> 66 67 /* 68 * LOCKDEBUG 69 */ 70 71 #if defined(LOCKDEBUG) 72 73 #define RW_WANTLOCK(rw, op) \ 74 LOCKDEBUG_WANTLOCK(RW_DEBUG_P(rw), (rw), \ 75 (uintptr_t)__builtin_return_address(0), op == RW_READER); 76 #define RW_LOCKED(rw, op) \ 77 LOCKDEBUG_LOCKED(RW_DEBUG_P(rw), (rw), \ 78 (uintptr_t)__builtin_return_address(0), op == RW_READER); 79 #define RW_UNLOCKED(rw, op) \ 80 LOCKDEBUG_UNLOCKED(RW_DEBUG_P(rw), (rw), \ 81 (uintptr_t)__builtin_return_address(0), op == RW_READER); 82 #define RW_DASSERT(rw, cond) \ 83 do { \ 84 if (!(cond)) \ 85 rw_abort(rw, __func__, "assertion failed: " #cond); \ 86 } while (/* CONSTCOND */ 0); 87 88 #else /* LOCKDEBUG */ 89 90 #define RW_WANTLOCK(rw, op) /* nothing */ 91 #define RW_LOCKED(rw, op) /* nothing */ 92 #define RW_UNLOCKED(rw, op) /* nothing */ 93 #define RW_DASSERT(rw, cond) /* nothing */ 94 95 #endif /* LOCKDEBUG */ 96 97 /* 98 * DIAGNOSTIC 99 */ 100 101 #if defined(DIAGNOSTIC) 102 103 #define RW_ASSERT(rw, cond) \ 104 do { \ 105 if (!(cond)) \ 106 rw_abort(rw, __func__, "assertion failed: " #cond); \ 107 } while (/* CONSTCOND */ 0) 108 109 #else 110 111 #define RW_ASSERT(rw, cond) /* nothing */ 112 113 #endif /* DIAGNOSTIC */ 114 115 /* 116 * For platforms that use 'simple' RW locks. 117 */ 118 #ifdef __HAVE_SIMPLE_RW_LOCKS 119 #define RW_ACQUIRE(rw, old, new) RW_CAS1(&(rw)->rw_owner, old, new) 120 #define RW_RELEASE(rw, old, new) RW_CAS1(&(rw)->rw_owner, old, new) 121 #define RW_SETDEBUG(rw, on) ((rw)->rw_owner |= (on) ? RW_DEBUG : 0) 122 #define RW_DEBUG_P(rw) (((rw)->rw_owner & RW_DEBUG) != 0) 123 #if defined(LOCKDEBUG) 124 #define RW_INHERITDEBUG(new, old) (new) |= (old) & RW_DEBUG 125 #else /* defined(LOCKDEBUG) */ 126 #define RW_INHERITDEBUG(new, old) /* nothing */ 127 #endif /* defined(LOCKDEBUG) */ 128 129 static inline int 130 RW_CAS1(volatile uintptr_t *ptr, uintptr_t old, uintptr_t new) 131 { 132 133 RW_INHERITDEBUG(new, old); 134 return RW_CAS(ptr, old, new); 135 } 136 137 static inline int 138 RW_SET_WAITERS(krwlock_t *rw, uintptr_t need, uintptr_t set) 139 { 140 uintptr_t old; 141 142 if (((old = rw->rw_owner) & need) == 0) 143 return 0; 144 return RW_CAS(&rw->rw_owner, old, old | set); 145 } 146 #endif /* __HAVE_SIMPLE_RW_LOCKS */ 147 148 /* 149 * For platforms that do not provide stubs, or for the LOCKDEBUG case. 150 */ 151 #ifdef LOCKDEBUG 152 #undef __HAVE_RW_STUBS 153 #endif 154 155 #ifndef __HAVE_RW_STUBS 156 __strong_alias(rw_enter,rw_vector_enter); 157 __strong_alias(rw_exit,rw_vector_exit); 158 __strong_alias(rw_tryenter,rw_vector_tryenter); 159 #endif 160 161 static void rw_dump(volatile void *); 162 static lwp_t *rw_owner(wchan_t); 163 164 lockops_t rwlock_lockops = { 165 "Reader / writer lock", 166 1, 167 rw_dump 168 }; 169 170 syncobj_t rw_syncobj = { 171 SOBJ_SLEEPQ_SORTED, 172 turnstile_unsleep, 173 turnstile_changepri, 174 sleepq_lendpri, 175 rw_owner, 176 }; 177 178 /* 179 * rw_dump: 180 * 181 * Dump the contents of a rwlock structure. 182 */ 183 static void 184 rw_dump(volatile void *cookie) 185 { 186 volatile krwlock_t *rw = cookie; 187 188 printf_nolog("owner/count : %#018lx flags : %#018x\n", 189 (long)RW_OWNER(rw), (int)RW_FLAGS(rw)); 190 } 191 192 /* 193 * rw_abort: 194 * 195 * Dump information about an error and panic the system. This 196 * generates a lot of machine code in the DIAGNOSTIC case, so 197 * we ask the compiler to not inline it. 198 */ 199 #if __GNUC_PREREQ__(3, 0) 200 __attribute ((noinline)) 201 #endif 202 static void 203 rw_abort(krwlock_t *rw, const char *func, const char *msg) 204 { 205 206 if (panicstr != NULL) 207 return; 208 209 LOCKDEBUG_ABORT(rw, &rwlock_lockops, func, msg); 210 } 211 212 /* 213 * rw_init: 214 * 215 * Initialize a rwlock for use. 216 */ 217 void 218 rw_init(krwlock_t *rw) 219 { 220 bool dodebug; 221 222 memset(rw, 0, sizeof(*rw)); 223 224 dodebug = LOCKDEBUG_ALLOC(rw, &rwlock_lockops, 225 (uintptr_t)__builtin_return_address(0)); 226 RW_SETDEBUG(rw, dodebug); 227 } 228 229 /* 230 * rw_destroy: 231 * 232 * Tear down a rwlock. 233 */ 234 void 235 rw_destroy(krwlock_t *rw) 236 { 237 238 RW_ASSERT(rw, (rw->rw_owner & ~RW_DEBUG) == 0); 239 LOCKDEBUG_FREE(RW_DEBUG_P(rw), rw); 240 } 241 242 /* 243 * rw_vector_enter: 244 * 245 * Acquire a rwlock. 246 */ 247 void 248 rw_vector_enter(krwlock_t *rw, const krw_t op) 249 { 250 uintptr_t owner, incr, need_wait, set_wait, curthread; 251 turnstile_t *ts; 252 int queue; 253 lwp_t *l; 254 LOCKSTAT_TIMER(slptime); 255 LOCKSTAT_FLAG(lsflag); 256 257 l = curlwp; 258 curthread = (uintptr_t)l; 259 260 RW_ASSERT(rw, !cpu_intr_p()); 261 RW_ASSERT(rw, curthread != 0); 262 RW_WANTLOCK(rw, op); 263 264 if (panicstr == NULL) { 265 LOCKDEBUG_BARRIER(&kernel_lock, 1); 266 } 267 268 /* 269 * We play a slight trick here. If we're a reader, we want 270 * increment the read count. If we're a writer, we want to 271 * set the owner field and whe WRITE_LOCKED bit. 272 * 273 * In the latter case, we expect those bits to be zero, 274 * therefore we can use an add operation to set them, which 275 * means an add operation for both cases. 276 */ 277 if (__predict_true(op == RW_READER)) { 278 incr = RW_READ_INCR; 279 set_wait = RW_HAS_WAITERS; 280 need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED; 281 queue = TS_READER_Q; 282 } else { 283 RW_DASSERT(rw, op == RW_WRITER); 284 incr = curthread | RW_WRITE_LOCKED; 285 set_wait = RW_HAS_WAITERS | RW_WRITE_WANTED; 286 need_wait = RW_WRITE_LOCKED | RW_THREAD; 287 queue = TS_WRITER_Q; 288 } 289 290 LOCKSTAT_ENTER(lsflag); 291 292 for (;;) { 293 /* 294 * Read the lock owner field. If the need-to-wait 295 * indicator is clear, then try to acquire the lock. 296 */ 297 owner = rw->rw_owner; 298 if ((owner & need_wait) == 0) { 299 if (RW_ACQUIRE(rw, owner, owner + incr)) { 300 /* Got it! */ 301 break; 302 } 303 304 /* 305 * Didn't get it -- spin around again (we'll 306 * probably sleep on the next iteration). 307 */ 308 continue; 309 } 310 311 if (panicstr != NULL) 312 return; 313 if (RW_OWNER(rw) == curthread) 314 rw_abort(rw, __func__, "locking against myself"); 315 316 /* 317 * Grab the turnstile chain lock. Once we have that, we 318 * can adjust the waiter bits and sleep queue. 319 */ 320 ts = turnstile_lookup(rw); 321 322 /* 323 * Mark the rwlock as having waiters. If the set fails, 324 * then we may not need to sleep and should spin again. 325 */ 326 if (!RW_SET_WAITERS(rw, need_wait, set_wait)) { 327 turnstile_exit(rw); 328 continue; 329 } 330 331 LOCKSTAT_START_TIMER(lsflag, slptime); 332 333 turnstile_block(ts, queue, rw, &rw_syncobj); 334 335 /* If we wake up and arrive here, we've been handed the lock. */ 336 RW_RECEIVE(rw); 337 338 LOCKSTAT_STOP_TIMER(lsflag, slptime); 339 LOCKSTAT_EVENT(lsflag, rw, 340 LB_RWLOCK | (op == RW_WRITER ? LB_SLEEP1 : LB_SLEEP2), 341 1, slptime); 342 343 break; 344 } 345 346 LOCKSTAT_EXIT(lsflag); 347 348 RW_DASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) || 349 (op == RW_READER && RW_COUNT(rw) != 0)); 350 RW_LOCKED(rw, op); 351 } 352 353 /* 354 * rw_vector_exit: 355 * 356 * Release a rwlock. 357 */ 358 void 359 rw_vector_exit(krwlock_t *rw) 360 { 361 uintptr_t curthread, owner, decr, new; 362 turnstile_t *ts; 363 int rcnt, wcnt; 364 lwp_t *l; 365 366 curthread = (uintptr_t)curlwp; 367 RW_ASSERT(rw, curthread != 0); 368 369 if (panicstr != NULL) 370 return; 371 372 /* 373 * Again, we use a trick. Since we used an add operation to 374 * set the required lock bits, we can use a subtract to clear 375 * them, which makes the read-release and write-release path 376 * the same. 377 */ 378 owner = rw->rw_owner; 379 if (__predict_false((owner & RW_WRITE_LOCKED) != 0)) { 380 RW_UNLOCKED(rw, RW_WRITER); 381 RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) != 0); 382 RW_ASSERT(rw, RW_OWNER(rw) == curthread); 383 decr = curthread | RW_WRITE_LOCKED; 384 } else { 385 RW_UNLOCKED(rw, RW_READER); 386 RW_ASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) == 0); 387 RW_ASSERT(rw, RW_COUNT(rw) != 0); 388 decr = RW_READ_INCR; 389 } 390 391 /* 392 * Compute what we expect the new value of the lock to be. Only 393 * proceed to do direct handoff if there are waiters, and if the 394 * lock would become unowned. 395 */ 396 for (;; owner = rw->rw_owner) { 397 new = (owner - decr); 398 if ((new & (RW_THREAD | RW_HAS_WAITERS)) == RW_HAS_WAITERS) 399 break; 400 if (RW_RELEASE(rw, owner, new)) 401 return; 402 } 403 404 for (;;) { 405 /* 406 * Grab the turnstile chain lock. This gets the interlock 407 * on the sleep queue. Once we have that, we can adjust the 408 * waiter bits. 409 */ 410 ts = turnstile_lookup(rw); 411 RW_DASSERT(rw, ts != NULL); 412 RW_DASSERT(rw, (rw->rw_owner & RW_HAS_WAITERS) != 0); 413 414 owner = rw->rw_owner; 415 wcnt = TS_WAITERS(ts, TS_WRITER_Q); 416 rcnt = TS_WAITERS(ts, TS_READER_Q); 417 418 /* 419 * Give the lock away. 420 * 421 * If we are releasing a write lock, then wake all 422 * outstanding readers. If we are releasing a read 423 * lock, then wake one writer. 424 */ 425 if (rcnt == 0 || (decr == RW_READ_INCR && wcnt != 0)) { 426 RW_DASSERT(rw, wcnt != 0); 427 RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_WANTED) != 0); 428 429 /* 430 * Give the lock to the longest waiting 431 * writer. 432 */ 433 l = TS_FIRST(ts, TS_WRITER_Q); 434 new = (uintptr_t)l | RW_WRITE_LOCKED; 435 436 if (wcnt > 1) 437 new |= RW_HAS_WAITERS | RW_WRITE_WANTED; 438 else if (rcnt != 0) 439 new |= RW_HAS_WAITERS; 440 441 RW_GIVE(rw); 442 if (!RW_RELEASE(rw, owner, new)) { 443 /* Oops, try again. */ 444 turnstile_exit(rw); 445 continue; 446 } 447 448 /* Wake the writer. */ 449 turnstile_wakeup(ts, TS_WRITER_Q, 1, l); 450 } else { 451 RW_DASSERT(rw, rcnt != 0); 452 453 /* 454 * Give the lock to all blocked readers. If there 455 * is a writer waiting, new readers that arrive 456 * after the release will be blocked out. 457 */ 458 new = rcnt << RW_READ_COUNT_SHIFT; 459 if (wcnt != 0) 460 new |= RW_HAS_WAITERS | RW_WRITE_WANTED; 461 462 RW_GIVE(rw); 463 if (!RW_RELEASE(rw, owner, new)) { 464 /* Oops, try again. */ 465 turnstile_exit(rw); 466 continue; 467 } 468 469 /* Wake up all sleeping readers. */ 470 turnstile_wakeup(ts, TS_READER_Q, rcnt, NULL); 471 } 472 473 break; 474 } 475 } 476 477 /* 478 * rw_vector_tryenter: 479 * 480 * Try to acquire a rwlock. 481 */ 482 int 483 rw_vector_tryenter(krwlock_t *rw, const krw_t op) 484 { 485 uintptr_t curthread, owner, incr, need_wait; 486 487 curthread = (uintptr_t)curlwp; 488 489 RW_ASSERT(rw, curthread != 0); 490 491 if (op == RW_READER) { 492 incr = RW_READ_INCR; 493 need_wait = RW_WRITE_LOCKED | RW_WRITE_WANTED; 494 } else { 495 RW_DASSERT(rw, op == RW_WRITER); 496 incr = curthread | RW_WRITE_LOCKED; 497 need_wait = RW_WRITE_LOCKED | RW_THREAD; 498 } 499 500 for (;;) { 501 owner = rw->rw_owner; 502 if ((owner & need_wait) == 0) { 503 if (RW_ACQUIRE(rw, owner, owner + incr)) { 504 /* Got it! */ 505 break; 506 } 507 continue; 508 } 509 return 0; 510 } 511 512 RW_WANTLOCK(rw, op); 513 RW_LOCKED(rw, op); 514 RW_DASSERT(rw, (op != RW_READER && RW_OWNER(rw) == curthread) || 515 (op == RW_READER && RW_COUNT(rw) != 0)); 516 517 return 1; 518 } 519 520 /* 521 * rw_downgrade: 522 * 523 * Downgrade a write lock to a read lock. 524 */ 525 void 526 rw_downgrade(krwlock_t *rw) 527 { 528 uintptr_t owner, curthread, new; 529 turnstile_t *ts; 530 int rcnt, wcnt; 531 532 curthread = (uintptr_t)curlwp; 533 RW_ASSERT(rw, curthread != 0); 534 RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) != 0); 535 RW_ASSERT(rw, RW_OWNER(rw) == curthread); 536 RW_UNLOCKED(rw, RW_WRITER); 537 538 owner = rw->rw_owner; 539 if ((owner & RW_HAS_WAITERS) == 0) { 540 /* 541 * There are no waiters, so we can do this the easy way. 542 * Try swapping us down to one read hold. If it fails, the 543 * lock condition has changed and we most likely now have 544 * waiters. 545 */ 546 if (RW_RELEASE(rw, owner, RW_READ_INCR)) { 547 RW_LOCKED(rw, RW_READER); 548 RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) == 0); 549 RW_DASSERT(rw, RW_COUNT(rw) != 0); 550 return; 551 } 552 } 553 554 /* 555 * Grab the turnstile chain lock. This gets the interlock 556 * on the sleep queue. Once we have that, we can adjust the 557 * waiter bits. 558 */ 559 for (;;) { 560 ts = turnstile_lookup(rw); 561 RW_DASSERT(rw, ts != NULL); 562 563 owner = rw->rw_owner; 564 rcnt = TS_WAITERS(ts, TS_READER_Q); 565 wcnt = TS_WAITERS(ts, TS_WRITER_Q); 566 567 /* 568 * If there are no readers, just preserve the waiters 569 * bits, swap us down to one read hold and return. 570 */ 571 if (rcnt == 0) { 572 RW_DASSERT(rw, wcnt != 0); 573 RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_WANTED) != 0); 574 RW_DASSERT(rw, (rw->rw_owner & RW_HAS_WAITERS) != 0); 575 576 new = RW_READ_INCR | RW_HAS_WAITERS | RW_WRITE_WANTED; 577 if (!RW_RELEASE(rw, owner, new)) { 578 /* Oops, try again. */ 579 turnstile_exit(ts); 580 continue; 581 } 582 break; 583 } 584 585 /* 586 * Give the lock to all blocked readers. We may 587 * retain one read hold if downgrading. If there 588 * is a writer waiting, new readers will be blocked 589 * out. 590 */ 591 new = (rcnt << RW_READ_COUNT_SHIFT) + RW_READ_INCR; 592 if (wcnt != 0) 593 new |= RW_HAS_WAITERS | RW_WRITE_WANTED; 594 595 RW_GIVE(rw); 596 if (!RW_RELEASE(rw, owner, new)) { 597 /* Oops, try again. */ 598 turnstile_exit(rw); 599 continue; 600 } 601 602 /* Wake up all sleeping readers. */ 603 turnstile_wakeup(ts, TS_READER_Q, rcnt, NULL); 604 break; 605 } 606 607 RW_LOCKED(rw, RW_READER); 608 RW_DASSERT(rw, (rw->rw_owner & RW_WRITE_LOCKED) == 0); 609 RW_DASSERT(rw, RW_COUNT(rw) != 0); 610 } 611 612 /* 613 * rw_tryupgrade: 614 * 615 * Try to upgrade a read lock to a write lock. We must be the 616 * only reader. 617 */ 618 int 619 rw_tryupgrade(krwlock_t *rw) 620 { 621 uintptr_t owner, curthread, new; 622 623 curthread = (uintptr_t)curlwp; 624 RW_ASSERT(rw, curthread != 0); 625 RW_WANTLOCK(rw, RW_WRITER); 626 627 for (;;) { 628 owner = rw->rw_owner; 629 RW_ASSERT(rw, (owner & RW_WRITE_LOCKED) == 0); 630 if ((owner & RW_THREAD) != RW_READ_INCR) { 631 RW_ASSERT(rw, (owner & RW_THREAD) != 0); 632 return 0; 633 } 634 new = curthread | RW_WRITE_LOCKED | (owner & ~RW_THREAD); 635 if (RW_ACQUIRE(rw, owner, new)) 636 break; 637 } 638 639 RW_UNLOCKED(rw, RW_READER); 640 RW_LOCKED(rw, RW_WRITER); 641 RW_DASSERT(rw, rw->rw_owner & RW_WRITE_LOCKED); 642 RW_DASSERT(rw, RW_OWNER(rw) == curthread); 643 644 return 1; 645 } 646 647 /* 648 * rw_read_held: 649 * 650 * Returns true if the rwlock is held for reading. Must only be 651 * used for diagnostic assertions, and never be used to make 652 * decisions about how to use a rwlock. 653 */ 654 int 655 rw_read_held(krwlock_t *rw) 656 { 657 uintptr_t owner; 658 659 if (panicstr != NULL) 660 return 1; 661 662 owner = rw->rw_owner; 663 return (owner & RW_WRITE_LOCKED) == 0 && (owner & RW_THREAD) != 0; 664 } 665 666 /* 667 * rw_write_held: 668 * 669 * Returns true if the rwlock is held for writing. Must only be 670 * used for diagnostic assertions, and never be used to make 671 * decisions about how to use a rwlock. 672 */ 673 int 674 rw_write_held(krwlock_t *rw) 675 { 676 677 if (panicstr != NULL) 678 return 1; 679 680 return (rw->rw_owner & (RW_WRITE_LOCKED | RW_THREAD)) == 681 (RW_WRITE_LOCKED | (uintptr_t)curlwp); 682 } 683 684 /* 685 * rw_lock_held: 686 * 687 * Returns true if the rwlock is held for reading or writing. Must 688 * only be used for diagnostic assertions, and never be used to make 689 * decisions about how to use a rwlock. 690 */ 691 int 692 rw_lock_held(krwlock_t *rw) 693 { 694 695 if (panicstr != NULL) 696 return 1; 697 698 return (rw->rw_owner & RW_THREAD) != 0; 699 } 700 701 /* 702 * rw_owner: 703 * 704 * Return the current owner of an RW lock, but only if it is write 705 * held. Used for priority inheritance. 706 */ 707 static lwp_t * 708 rw_owner(wchan_t obj) 709 { 710 krwlock_t *rw = (void *)(uintptr_t)obj; /* discard qualifiers */ 711 uintptr_t owner = rw->rw_owner; 712 713 if ((owner & RW_WRITE_LOCKED) == 0) 714 return NULL; 715 716 return (void *)(owner & RW_THREAD); 717 } 718