1 /* $NetBSD: kern_mutex.c,v 1.53 2012/02/25 22:32:44 rmind Exp $ */ 2 3 /*- 4 * Copyright (c) 2002, 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe and Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Kernel mutex implementation, modeled after those found in Solaris, 34 * a description of which can be found in: 35 * 36 * Solaris Internals: Core Kernel Architecture, Jim Mauro and 37 * Richard McDougall. 38 */ 39 40 #define __MUTEX_PRIVATE 41 42 #include <sys/cdefs.h> 43 __KERNEL_RCSID(0, "$NetBSD: kern_mutex.c,v 1.53 2012/02/25 22:32:44 rmind Exp $"); 44 45 #include <sys/param.h> 46 #include <sys/atomic.h> 47 #include <sys/proc.h> 48 #include <sys/mutex.h> 49 #include <sys/sched.h> 50 #include <sys/sleepq.h> 51 #include <sys/systm.h> 52 #include <sys/lockdebug.h> 53 #include <sys/kernel.h> 54 #include <sys/intr.h> 55 #include <sys/lock.h> 56 #include <sys/types.h> 57 58 #include <dev/lockstat.h> 59 60 #include <machine/lock.h> 61 62 /* 63 * When not running a debug kernel, spin mutexes are not much 64 * more than an splraiseipl() and splx() pair. 65 */ 66 67 #if defined(DIAGNOSTIC) || defined(MULTIPROCESSOR) || defined(LOCKDEBUG) 68 #define FULL 69 #endif 70 71 /* 72 * Debugging support. 73 */ 74 75 #define MUTEX_WANTLOCK(mtx) \ 76 LOCKDEBUG_WANTLOCK(MUTEX_DEBUG_P(mtx), (mtx), \ 77 (uintptr_t)__builtin_return_address(0), false, false) 78 #define MUTEX_LOCKED(mtx) \ 79 LOCKDEBUG_LOCKED(MUTEX_DEBUG_P(mtx), (mtx), NULL, \ 80 (uintptr_t)__builtin_return_address(0), 0) 81 #define MUTEX_UNLOCKED(mtx) \ 82 LOCKDEBUG_UNLOCKED(MUTEX_DEBUG_P(mtx), (mtx), \ 83 (uintptr_t)__builtin_return_address(0), 0) 84 #define MUTEX_ABORT(mtx, msg) \ 85 mutex_abort(mtx, __func__, msg) 86 87 #if defined(LOCKDEBUG) 88 89 #define MUTEX_DASSERT(mtx, cond) \ 90 do { \ 91 if (!(cond)) \ 92 MUTEX_ABORT(mtx, "assertion failed: " #cond); \ 93 } while (/* CONSTCOND */ 0); 94 95 #else /* LOCKDEBUG */ 96 97 #define MUTEX_DASSERT(mtx, cond) /* nothing */ 98 99 #endif /* LOCKDEBUG */ 100 101 #if defined(DIAGNOSTIC) 102 103 #define MUTEX_ASSERT(mtx, cond) \ 104 do { \ 105 if (!(cond)) \ 106 MUTEX_ABORT(mtx, "assertion failed: " #cond); \ 107 } while (/* CONSTCOND */ 0) 108 109 #else /* DIAGNOSTIC */ 110 111 #define MUTEX_ASSERT(mtx, cond) /* nothing */ 112 113 #endif /* DIAGNOSTIC */ 114 115 /* 116 * Spin mutex SPL save / restore. 117 */ 118 119 #define MUTEX_SPIN_SPLRAISE(mtx) \ 120 do { \ 121 struct cpu_info *x__ci; \ 122 int x__cnt, s; \ 123 s = splraiseipl(mtx->mtx_ipl); \ 124 x__ci = curcpu(); \ 125 x__cnt = x__ci->ci_mtx_count--; \ 126 __insn_barrier(); \ 127 if (x__cnt == 0) \ 128 x__ci->ci_mtx_oldspl = (s); \ 129 } while (/* CONSTCOND */ 0) 130 131 #define MUTEX_SPIN_SPLRESTORE(mtx) \ 132 do { \ 133 struct cpu_info *x__ci = curcpu(); \ 134 int s = x__ci->ci_mtx_oldspl; \ 135 __insn_barrier(); \ 136 if (++(x__ci->ci_mtx_count) == 0) \ 137 splx(s); \ 138 } while (/* CONSTCOND */ 0) 139 140 /* 141 * For architectures that provide 'simple' mutexes: they provide a 142 * CAS function that is either MP-safe, or does not need to be MP 143 * safe. Adaptive mutexes on these architectures do not require an 144 * additional interlock. 145 */ 146 147 #ifdef __HAVE_SIMPLE_MUTEXES 148 149 #define MUTEX_OWNER(owner) \ 150 (owner & MUTEX_THREAD) 151 #define MUTEX_HAS_WAITERS(mtx) \ 152 (((int)(mtx)->mtx_owner & MUTEX_BIT_WAITERS) != 0) 153 154 #define MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug) \ 155 if (!dodebug) \ 156 (mtx)->mtx_owner |= MUTEX_BIT_NODEBUG; \ 157 do { \ 158 } while (/* CONSTCOND */ 0); 159 160 #define MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl) \ 161 do { \ 162 (mtx)->mtx_owner = MUTEX_BIT_SPIN; \ 163 if (!dodebug) \ 164 (mtx)->mtx_owner |= MUTEX_BIT_NODEBUG; \ 165 (mtx)->mtx_ipl = makeiplcookie((ipl)); \ 166 __cpu_simple_lock_init(&(mtx)->mtx_lock); \ 167 } while (/* CONSTCOND */ 0) 168 169 #define MUTEX_DESTROY(mtx) \ 170 do { \ 171 (mtx)->mtx_owner = MUTEX_THREAD; \ 172 } while (/* CONSTCOND */ 0); 173 174 #define MUTEX_SPIN_P(mtx) \ 175 (((mtx)->mtx_owner & MUTEX_BIT_SPIN) != 0) 176 #define MUTEX_ADAPTIVE_P(mtx) \ 177 (((mtx)->mtx_owner & MUTEX_BIT_SPIN) == 0) 178 179 #define MUTEX_DEBUG_P(mtx) (((mtx)->mtx_owner & MUTEX_BIT_NODEBUG) == 0) 180 #if defined(LOCKDEBUG) 181 #define MUTEX_OWNED(owner) (((owner) & ~MUTEX_BIT_NODEBUG) != 0) 182 #define MUTEX_INHERITDEBUG(new, old) (new) |= (old) & MUTEX_BIT_NODEBUG 183 #else /* defined(LOCKDEBUG) */ 184 #define MUTEX_OWNED(owner) ((owner) != 0) 185 #define MUTEX_INHERITDEBUG(new, old) /* nothing */ 186 #endif /* defined(LOCKDEBUG) */ 187 188 static inline int 189 MUTEX_ACQUIRE(kmutex_t *mtx, uintptr_t curthread) 190 { 191 int rv; 192 uintptr_t old = 0; 193 uintptr_t new = curthread; 194 195 MUTEX_INHERITDEBUG(old, mtx->mtx_owner); 196 MUTEX_INHERITDEBUG(new, old); 197 rv = MUTEX_CAS(&mtx->mtx_owner, old, new); 198 MUTEX_RECEIVE(mtx); 199 return rv; 200 } 201 202 static inline int 203 MUTEX_SET_WAITERS(kmutex_t *mtx, uintptr_t owner) 204 { 205 int rv; 206 rv = MUTEX_CAS(&mtx->mtx_owner, owner, owner | MUTEX_BIT_WAITERS); 207 MUTEX_RECEIVE(mtx); 208 return rv; 209 } 210 211 static inline void 212 MUTEX_RELEASE(kmutex_t *mtx) 213 { 214 uintptr_t new; 215 216 MUTEX_GIVE(mtx); 217 new = 0; 218 MUTEX_INHERITDEBUG(new, mtx->mtx_owner); 219 mtx->mtx_owner = new; 220 } 221 222 static inline void 223 MUTEX_CLEAR_WAITERS(kmutex_t *mtx) 224 { 225 /* nothing */ 226 } 227 #endif /* __HAVE_SIMPLE_MUTEXES */ 228 229 /* 230 * Patch in stubs via strong alias where they are not available. 231 */ 232 233 #if defined(LOCKDEBUG) 234 #undef __HAVE_MUTEX_STUBS 235 #undef __HAVE_SPIN_MUTEX_STUBS 236 #endif 237 238 #ifndef __HAVE_MUTEX_STUBS 239 __strong_alias(mutex_enter,mutex_vector_enter); 240 __strong_alias(mutex_exit,mutex_vector_exit); 241 #endif 242 243 #ifndef __HAVE_SPIN_MUTEX_STUBS 244 __strong_alias(mutex_spin_enter,mutex_vector_enter); 245 __strong_alias(mutex_spin_exit,mutex_vector_exit); 246 #endif 247 248 static void mutex_abort(kmutex_t *, const char *, const char *); 249 static void mutex_dump(volatile void *); 250 251 lockops_t mutex_spin_lockops = { 252 "Mutex", 253 LOCKOPS_SPIN, 254 mutex_dump 255 }; 256 257 lockops_t mutex_adaptive_lockops = { 258 "Mutex", 259 LOCKOPS_SLEEP, 260 mutex_dump 261 }; 262 263 syncobj_t mutex_syncobj = { 264 SOBJ_SLEEPQ_SORTED, 265 turnstile_unsleep, 266 turnstile_changepri, 267 sleepq_lendpri, 268 (void *)mutex_owner, 269 }; 270 271 /* 272 * mutex_dump: 273 * 274 * Dump the contents of a mutex structure. 275 */ 276 void 277 mutex_dump(volatile void *cookie) 278 { 279 volatile kmutex_t *mtx = cookie; 280 281 printf_nolog("owner field : %#018lx wait/spin: %16d/%d\n", 282 (long)MUTEX_OWNER(mtx->mtx_owner), MUTEX_HAS_WAITERS(mtx), 283 MUTEX_SPIN_P(mtx)); 284 } 285 286 /* 287 * mutex_abort: 288 * 289 * Dump information about an error and panic the system. This 290 * generates a lot of machine code in the DIAGNOSTIC case, so 291 * we ask the compiler to not inline it. 292 */ 293 void __noinline 294 mutex_abort(kmutex_t *mtx, const char *func, const char *msg) 295 { 296 297 LOCKDEBUG_ABORT(mtx, (MUTEX_SPIN_P(mtx) ? 298 &mutex_spin_lockops : &mutex_adaptive_lockops), func, msg); 299 } 300 301 /* 302 * mutex_init: 303 * 304 * Initialize a mutex for use. Note that adaptive mutexes are in 305 * essence spin mutexes that can sleep to avoid deadlock and wasting 306 * CPU time. We can't easily provide a type of mutex that always 307 * sleeps - see comments in mutex_vector_enter() about releasing 308 * mutexes unlocked. 309 */ 310 void 311 mutex_init(kmutex_t *mtx, kmutex_type_t type, int ipl) 312 { 313 bool dodebug; 314 315 memset(mtx, 0, sizeof(*mtx)); 316 317 switch (type) { 318 case MUTEX_ADAPTIVE: 319 KASSERT(ipl == IPL_NONE); 320 break; 321 case MUTEX_DEFAULT: 322 case MUTEX_DRIVER: 323 if (ipl == IPL_NONE || ipl == IPL_SOFTCLOCK || 324 ipl == IPL_SOFTBIO || ipl == IPL_SOFTNET || 325 ipl == IPL_SOFTSERIAL) { 326 type = MUTEX_ADAPTIVE; 327 } else { 328 type = MUTEX_SPIN; 329 } 330 break; 331 default: 332 break; 333 } 334 335 switch (type) { 336 case MUTEX_NODEBUG: 337 dodebug = LOCKDEBUG_ALLOC(mtx, NULL, 338 (uintptr_t)__builtin_return_address(0)); 339 MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl); 340 break; 341 case MUTEX_ADAPTIVE: 342 dodebug = LOCKDEBUG_ALLOC(mtx, &mutex_adaptive_lockops, 343 (uintptr_t)__builtin_return_address(0)); 344 MUTEX_INITIALIZE_ADAPTIVE(mtx, dodebug); 345 break; 346 case MUTEX_SPIN: 347 dodebug = LOCKDEBUG_ALLOC(mtx, &mutex_spin_lockops, 348 (uintptr_t)__builtin_return_address(0)); 349 MUTEX_INITIALIZE_SPIN(mtx, dodebug, ipl); 350 break; 351 default: 352 panic("mutex_init: impossible type"); 353 break; 354 } 355 } 356 357 /* 358 * mutex_destroy: 359 * 360 * Tear down a mutex. 361 */ 362 void 363 mutex_destroy(kmutex_t *mtx) 364 { 365 366 if (MUTEX_ADAPTIVE_P(mtx)) { 367 MUTEX_ASSERT(mtx, !MUTEX_OWNED(mtx->mtx_owner) && 368 !MUTEX_HAS_WAITERS(mtx)); 369 } else { 370 MUTEX_ASSERT(mtx, !__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock)); 371 } 372 373 LOCKDEBUG_FREE(MUTEX_DEBUG_P(mtx), mtx); 374 MUTEX_DESTROY(mtx); 375 } 376 377 #ifdef MULTIPROCESSOR 378 /* 379 * mutex_oncpu: 380 * 381 * Return true if an adaptive mutex owner is running on a CPU in the 382 * system. If the target is waiting on the kernel big lock, then we 383 * must release it. This is necessary to avoid deadlock. 384 */ 385 static bool 386 mutex_oncpu(uintptr_t owner) 387 { 388 struct cpu_info *ci; 389 lwp_t *l; 390 391 KASSERT(kpreempt_disabled()); 392 393 if (!MUTEX_OWNED(owner)) { 394 return false; 395 } 396 397 /* 398 * See lwp_dtor() why dereference of the LWP pointer is safe. 399 * We must have kernel preemption disabled for that. 400 */ 401 l = (lwp_t *)MUTEX_OWNER(owner); 402 ci = l->l_cpu; 403 404 if (ci && ci->ci_curlwp == l) { 405 /* Target is running; do we need to block? */ 406 return (ci->ci_biglock_wanted != l); 407 } 408 409 /* Not running. It may be safe to block now. */ 410 return false; 411 } 412 #endif /* MULTIPROCESSOR */ 413 414 /* 415 * mutex_vector_enter: 416 * 417 * Support routine for mutex_enter() that must handle all cases. In 418 * the LOCKDEBUG case, mutex_enter() is always aliased here, even if 419 * fast-path stubs are available. If an mutex_spin_enter() stub is 420 * not available, then it is also aliased directly here. 421 */ 422 void 423 mutex_vector_enter(kmutex_t *mtx) 424 { 425 uintptr_t owner, curthread; 426 turnstile_t *ts; 427 #ifdef MULTIPROCESSOR 428 u_int count; 429 #endif 430 LOCKSTAT_COUNTER(spincnt); 431 LOCKSTAT_COUNTER(slpcnt); 432 LOCKSTAT_TIMER(spintime); 433 LOCKSTAT_TIMER(slptime); 434 LOCKSTAT_FLAG(lsflag); 435 436 /* 437 * Handle spin mutexes. 438 */ 439 if (MUTEX_SPIN_P(mtx)) { 440 #if defined(LOCKDEBUG) && defined(MULTIPROCESSOR) 441 u_int spins = 0; 442 #endif 443 MUTEX_SPIN_SPLRAISE(mtx); 444 MUTEX_WANTLOCK(mtx); 445 #ifdef FULL 446 if (__cpu_simple_lock_try(&mtx->mtx_lock)) { 447 MUTEX_LOCKED(mtx); 448 return; 449 } 450 #if !defined(MULTIPROCESSOR) 451 MUTEX_ABORT(mtx, "locking against myself"); 452 #else /* !MULTIPROCESSOR */ 453 454 LOCKSTAT_ENTER(lsflag); 455 LOCKSTAT_START_TIMER(lsflag, spintime); 456 count = SPINLOCK_BACKOFF_MIN; 457 458 /* 459 * Spin testing the lock word and do exponential backoff 460 * to reduce cache line ping-ponging between CPUs. 461 */ 462 do { 463 if (panicstr != NULL) 464 break; 465 while (__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock)) { 466 SPINLOCK_BACKOFF(count); 467 #ifdef LOCKDEBUG 468 if (SPINLOCK_SPINOUT(spins)) 469 MUTEX_ABORT(mtx, "spinout"); 470 #endif /* LOCKDEBUG */ 471 } 472 } while (!__cpu_simple_lock_try(&mtx->mtx_lock)); 473 474 if (count != SPINLOCK_BACKOFF_MIN) { 475 LOCKSTAT_STOP_TIMER(lsflag, spintime); 476 LOCKSTAT_EVENT(lsflag, mtx, 477 LB_SPIN_MUTEX | LB_SPIN, 1, spintime); 478 } 479 LOCKSTAT_EXIT(lsflag); 480 #endif /* !MULTIPROCESSOR */ 481 #endif /* FULL */ 482 MUTEX_LOCKED(mtx); 483 return; 484 } 485 486 curthread = (uintptr_t)curlwp; 487 488 MUTEX_DASSERT(mtx, MUTEX_ADAPTIVE_P(mtx)); 489 MUTEX_ASSERT(mtx, curthread != 0); 490 MUTEX_WANTLOCK(mtx); 491 492 if (panicstr == NULL) { 493 LOCKDEBUG_BARRIER(&kernel_lock, 1); 494 } 495 496 LOCKSTAT_ENTER(lsflag); 497 498 /* 499 * Adaptive mutex; spin trying to acquire the mutex. If we 500 * determine that the owner is not running on a processor, 501 * then we stop spinning, and sleep instead. 502 */ 503 KPREEMPT_DISABLE(curlwp); 504 for (owner = mtx->mtx_owner;;) { 505 if (!MUTEX_OWNED(owner)) { 506 /* 507 * Mutex owner clear could mean two things: 508 * 509 * * The mutex has been released. 510 * * The owner field hasn't been set yet. 511 * 512 * Try to acquire it again. If that fails, 513 * we'll just loop again. 514 */ 515 if (MUTEX_ACQUIRE(mtx, curthread)) 516 break; 517 owner = mtx->mtx_owner; 518 continue; 519 } 520 if (__predict_false(panicstr != NULL)) { 521 kpreempt_enable(); 522 return; 523 } 524 if (__predict_false(MUTEX_OWNER(owner) == curthread)) { 525 MUTEX_ABORT(mtx, "locking against myself"); 526 } 527 #ifdef MULTIPROCESSOR 528 /* 529 * Check to see if the owner is running on a processor. 530 * If so, then we should just spin, as the owner will 531 * likely release the lock very soon. 532 */ 533 if (mutex_oncpu(owner)) { 534 LOCKSTAT_START_TIMER(lsflag, spintime); 535 count = SPINLOCK_BACKOFF_MIN; 536 do { 537 KPREEMPT_ENABLE(curlwp); 538 SPINLOCK_BACKOFF(count); 539 KPREEMPT_DISABLE(curlwp); 540 owner = mtx->mtx_owner; 541 } while (mutex_oncpu(owner)); 542 LOCKSTAT_STOP_TIMER(lsflag, spintime); 543 LOCKSTAT_COUNT(spincnt, 1); 544 if (!MUTEX_OWNED(owner)) 545 continue; 546 } 547 #endif 548 549 ts = turnstile_lookup(mtx); 550 551 /* 552 * Once we have the turnstile chain interlock, mark the 553 * mutex has having waiters. If that fails, spin again: 554 * chances are that the mutex has been released. 555 */ 556 if (!MUTEX_SET_WAITERS(mtx, owner)) { 557 turnstile_exit(mtx); 558 owner = mtx->mtx_owner; 559 continue; 560 } 561 562 #ifdef MULTIPROCESSOR 563 /* 564 * mutex_exit() is permitted to release the mutex without 565 * any interlocking instructions, and the following can 566 * occur as a result: 567 * 568 * CPU 1: MUTEX_SET_WAITERS() CPU2: mutex_exit() 569 * ---------------------------- ---------------------------- 570 * .. acquire cache line 571 * .. test for waiters 572 * acquire cache line <- lose cache line 573 * lock cache line .. 574 * verify mutex is held .. 575 * set waiters .. 576 * unlock cache line .. 577 * lose cache line -> acquire cache line 578 * .. clear lock word, waiters 579 * return success 580 * 581 * There is another race that can occur: a third CPU could 582 * acquire the mutex as soon as it is released. Since 583 * adaptive mutexes are primarily spin mutexes, this is not 584 * something that we need to worry about too much. What we 585 * do need to ensure is that the waiters bit gets set. 586 * 587 * To allow the unlocked release, we need to make some 588 * assumptions here: 589 * 590 * o Release is the only non-atomic/unlocked operation 591 * that can be performed on the mutex. (It must still 592 * be atomic on the local CPU, e.g. in case interrupted 593 * or preempted). 594 * 595 * o At any given time, MUTEX_SET_WAITERS() can only ever 596 * be in progress on one CPU in the system - guaranteed 597 * by the turnstile chain lock. 598 * 599 * o No other operations other than MUTEX_SET_WAITERS() 600 * and release can modify a mutex with a non-zero 601 * owner field. 602 * 603 * o The result of a successful MUTEX_SET_WAITERS() call 604 * is an unbuffered write that is immediately visible 605 * to all other processors in the system. 606 * 607 * o If the holding LWP switches away, it posts a store 608 * fence before changing curlwp, ensuring that any 609 * overwrite of the mutex waiters flag by mutex_exit() 610 * completes before the modification of curlwp becomes 611 * visible to this CPU. 612 * 613 * o mi_switch() posts a store fence before setting curlwp 614 * and before resuming execution of an LWP. 615 * 616 * o _kernel_lock() posts a store fence before setting 617 * curcpu()->ci_biglock_wanted, and after clearing it. 618 * This ensures that any overwrite of the mutex waiters 619 * flag by mutex_exit() completes before the modification 620 * of ci_biglock_wanted becomes visible. 621 * 622 * We now post a read memory barrier (after setting the 623 * waiters field) and check the lock holder's status again. 624 * Some of the possible outcomes (not an exhaustive list): 625 * 626 * 1. The on-CPU check returns true: the holding LWP is 627 * running again. The lock may be released soon and 628 * we should spin. Importantly, we can't trust the 629 * value of the waiters flag. 630 * 631 * 2. The on-CPU check returns false: the holding LWP is 632 * not running. We now have the opportunity to check 633 * if mutex_exit() has blatted the modifications made 634 * by MUTEX_SET_WAITERS(). 635 * 636 * 3. The on-CPU check returns false: the holding LWP may 637 * or may not be running. It has context switched at 638 * some point during our check. Again, we have the 639 * chance to see if the waiters bit is still set or 640 * has been overwritten. 641 * 642 * 4. The on-CPU check returns false: the holding LWP is 643 * running on a CPU, but wants the big lock. It's OK 644 * to check the waiters field in this case. 645 * 646 * 5. The has-waiters check fails: the mutex has been 647 * released, the waiters flag cleared and another LWP 648 * now owns the mutex. 649 * 650 * 6. The has-waiters check fails: the mutex has been 651 * released. 652 * 653 * If the waiters bit is not set it's unsafe to go asleep, 654 * as we might never be awoken. 655 */ 656 if ((membar_consumer(), mutex_oncpu(owner)) || 657 (membar_consumer(), !MUTEX_HAS_WAITERS(mtx))) { 658 turnstile_exit(mtx); 659 owner = mtx->mtx_owner; 660 continue; 661 } 662 #endif /* MULTIPROCESSOR */ 663 664 LOCKSTAT_START_TIMER(lsflag, slptime); 665 666 turnstile_block(ts, TS_WRITER_Q, mtx, &mutex_syncobj); 667 668 LOCKSTAT_STOP_TIMER(lsflag, slptime); 669 LOCKSTAT_COUNT(slpcnt, 1); 670 671 owner = mtx->mtx_owner; 672 } 673 KPREEMPT_ENABLE(curlwp); 674 675 LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SLEEP1, 676 slpcnt, slptime); 677 LOCKSTAT_EVENT(lsflag, mtx, LB_ADAPTIVE_MUTEX | LB_SPIN, 678 spincnt, spintime); 679 LOCKSTAT_EXIT(lsflag); 680 681 MUTEX_DASSERT(mtx, MUTEX_OWNER(mtx->mtx_owner) == curthread); 682 MUTEX_LOCKED(mtx); 683 } 684 685 /* 686 * mutex_vector_exit: 687 * 688 * Support routine for mutex_exit() that handles all cases. 689 */ 690 void 691 mutex_vector_exit(kmutex_t *mtx) 692 { 693 turnstile_t *ts; 694 uintptr_t curthread; 695 696 if (MUTEX_SPIN_P(mtx)) { 697 #ifdef FULL 698 if (__predict_false(!__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock))) { 699 if (panicstr != NULL) 700 return; 701 MUTEX_ABORT(mtx, "exiting unheld spin mutex"); 702 } 703 MUTEX_UNLOCKED(mtx); 704 __cpu_simple_unlock(&mtx->mtx_lock); 705 #endif 706 MUTEX_SPIN_SPLRESTORE(mtx); 707 return; 708 } 709 710 if (__predict_false((uintptr_t)panicstr | cold)) { 711 MUTEX_UNLOCKED(mtx); 712 MUTEX_RELEASE(mtx); 713 return; 714 } 715 716 curthread = (uintptr_t)curlwp; 717 MUTEX_DASSERT(mtx, curthread != 0); 718 MUTEX_ASSERT(mtx, MUTEX_OWNER(mtx->mtx_owner) == curthread); 719 MUTEX_UNLOCKED(mtx); 720 721 #ifdef LOCKDEBUG 722 /* 723 * Avoid having to take the turnstile chain lock every time 724 * around. Raise the priority level to splhigh() in order 725 * to disable preemption and so make the following atomic. 726 */ 727 { 728 int s = splhigh(); 729 if (!MUTEX_HAS_WAITERS(mtx)) { 730 MUTEX_RELEASE(mtx); 731 splx(s); 732 return; 733 } 734 splx(s); 735 } 736 #endif 737 738 /* 739 * Get this lock's turnstile. This gets the interlock on 740 * the sleep queue. Once we have that, we can clear the 741 * lock. If there was no turnstile for the lock, there 742 * were no waiters remaining. 743 */ 744 ts = turnstile_lookup(mtx); 745 746 if (ts == NULL) { 747 MUTEX_RELEASE(mtx); 748 turnstile_exit(mtx); 749 } else { 750 MUTEX_RELEASE(mtx); 751 turnstile_wakeup(ts, TS_WRITER_Q, 752 TS_WAITERS(ts, TS_WRITER_Q), NULL); 753 } 754 } 755 756 #ifndef __HAVE_SIMPLE_MUTEXES 757 /* 758 * mutex_wakeup: 759 * 760 * Support routine for mutex_exit() that wakes up all waiters. 761 * We assume that the mutex has been released, but it need not 762 * be. 763 */ 764 void 765 mutex_wakeup(kmutex_t *mtx) 766 { 767 turnstile_t *ts; 768 769 ts = turnstile_lookup(mtx); 770 if (ts == NULL) { 771 turnstile_exit(mtx); 772 return; 773 } 774 MUTEX_CLEAR_WAITERS(mtx); 775 turnstile_wakeup(ts, TS_WRITER_Q, TS_WAITERS(ts, TS_WRITER_Q), NULL); 776 } 777 #endif /* !__HAVE_SIMPLE_MUTEXES */ 778 779 /* 780 * mutex_owned: 781 * 782 * Return true if the current LWP (adaptive) or CPU (spin) 783 * holds the mutex. 784 */ 785 int 786 mutex_owned(kmutex_t *mtx) 787 { 788 789 if (mtx == NULL) 790 return 0; 791 if (MUTEX_ADAPTIVE_P(mtx)) 792 return MUTEX_OWNER(mtx->mtx_owner) == (uintptr_t)curlwp; 793 #ifdef FULL 794 return __SIMPLELOCK_LOCKED_P(&mtx->mtx_lock); 795 #else 796 return 1; 797 #endif 798 } 799 800 /* 801 * mutex_owner: 802 * 803 * Return the current owner of an adaptive mutex. Used for 804 * priority inheritance. 805 */ 806 lwp_t * 807 mutex_owner(kmutex_t *mtx) 808 { 809 810 MUTEX_ASSERT(mtx, MUTEX_ADAPTIVE_P(mtx)); 811 return (struct lwp *)MUTEX_OWNER(mtx->mtx_owner); 812 } 813 814 /* 815 * mutex_tryenter: 816 * 817 * Try to acquire the mutex; return non-zero if we did. 818 */ 819 int 820 mutex_tryenter(kmutex_t *mtx) 821 { 822 uintptr_t curthread; 823 824 /* 825 * Handle spin mutexes. 826 */ 827 if (MUTEX_SPIN_P(mtx)) { 828 MUTEX_SPIN_SPLRAISE(mtx); 829 #ifdef FULL 830 if (__cpu_simple_lock_try(&mtx->mtx_lock)) { 831 MUTEX_WANTLOCK(mtx); 832 MUTEX_LOCKED(mtx); 833 return 1; 834 } 835 MUTEX_SPIN_SPLRESTORE(mtx); 836 #else 837 MUTEX_WANTLOCK(mtx); 838 MUTEX_LOCKED(mtx); 839 return 1; 840 #endif 841 } else { 842 curthread = (uintptr_t)curlwp; 843 MUTEX_ASSERT(mtx, curthread != 0); 844 if (MUTEX_ACQUIRE(mtx, curthread)) { 845 MUTEX_WANTLOCK(mtx); 846 MUTEX_LOCKED(mtx); 847 MUTEX_DASSERT(mtx, 848 MUTEX_OWNER(mtx->mtx_owner) == curthread); 849 return 1; 850 } 851 } 852 853 return 0; 854 } 855 856 #if defined(__HAVE_SPIN_MUTEX_STUBS) || defined(FULL) 857 /* 858 * mutex_spin_retry: 859 * 860 * Support routine for mutex_spin_enter(). Assumes that the caller 861 * has already raised the SPL, and adjusted counters. 862 */ 863 void 864 mutex_spin_retry(kmutex_t *mtx) 865 { 866 #ifdef MULTIPROCESSOR 867 u_int count; 868 LOCKSTAT_TIMER(spintime); 869 LOCKSTAT_FLAG(lsflag); 870 #ifdef LOCKDEBUG 871 u_int spins = 0; 872 #endif /* LOCKDEBUG */ 873 874 MUTEX_WANTLOCK(mtx); 875 876 LOCKSTAT_ENTER(lsflag); 877 LOCKSTAT_START_TIMER(lsflag, spintime); 878 count = SPINLOCK_BACKOFF_MIN; 879 880 /* 881 * Spin testing the lock word and do exponential backoff 882 * to reduce cache line ping-ponging between CPUs. 883 */ 884 do { 885 if (panicstr != NULL) 886 break; 887 while (__SIMPLELOCK_LOCKED_P(&mtx->mtx_lock)) { 888 SPINLOCK_BACKOFF(count); 889 #ifdef LOCKDEBUG 890 if (SPINLOCK_SPINOUT(spins)) 891 MUTEX_ABORT(mtx, "spinout"); 892 #endif /* LOCKDEBUG */ 893 } 894 } while (!__cpu_simple_lock_try(&mtx->mtx_lock)); 895 896 LOCKSTAT_STOP_TIMER(lsflag, spintime); 897 LOCKSTAT_EVENT(lsflag, mtx, LB_SPIN_MUTEX | LB_SPIN, 1, spintime); 898 LOCKSTAT_EXIT(lsflag); 899 900 MUTEX_LOCKED(mtx); 901 #else /* MULTIPROCESSOR */ 902 MUTEX_ABORT(mtx, "locking against myself"); 903 #endif /* MULTIPROCESSOR */ 904 } 905 #endif /* defined(__HAVE_SPIN_MUTEX_STUBS) || defined(FULL) */ 906