1 /* $NetBSD: kern_lock.c,v 1.83 2004/08/04 10:37:08 yamt Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * This code is derived from software contributed to The NetBSD Foundation 12 * by Ross Harvey. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. All advertising materials mentioning features or use of this software 23 * must display the following acknowledgement: 24 * This product includes software developed by the NetBSD 25 * Foundation, Inc. and its contributors. 26 * 4. Neither the name of The NetBSD Foundation nor the names of its 27 * contributors may be used to endorse or promote products derived 28 * from this software without specific prior written permission. 29 * 30 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 31 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 32 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 33 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 34 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 37 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 38 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 * POSSIBILITY OF SUCH DAMAGE. 41 */ 42 43 /* 44 * Copyright (c) 1995 45 * The Regents of the University of California. All rights reserved. 46 * 47 * This code contains ideas from software contributed to Berkeley by 48 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 49 * System project at Carnegie-Mellon University. 50 * 51 * Redistribution and use in source and binary forms, with or without 52 * modification, are permitted provided that the following conditions 53 * are met: 54 * 1. Redistributions of source code must retain the above copyright 55 * notice, this list of conditions and the following disclaimer. 56 * 2. Redistributions in binary form must reproduce the above copyright 57 * notice, this list of conditions and the following disclaimer in the 58 * documentation and/or other materials provided with the distribution. 59 * 3. Neither the name of the University nor the names of its contributors 60 * may be used to endorse or promote products derived from this software 61 * without specific prior written permission. 62 * 63 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 64 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 65 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 66 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 67 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 68 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 69 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 70 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 71 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 72 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 73 * SUCH DAMAGE. 74 * 75 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 76 */ 77 78 #include <sys/cdefs.h> 79 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.83 2004/08/04 10:37:08 yamt Exp $"); 80 81 #include "opt_multiprocessor.h" 82 #include "opt_lockdebug.h" 83 #include "opt_ddb.h" 84 85 #include <sys/param.h> 86 #include <sys/proc.h> 87 #include <sys/lock.h> 88 #include <sys/systm.h> 89 #include <machine/cpu.h> 90 91 #if defined(LOCKDEBUG) 92 #include <sys/syslog.h> 93 /* 94 * note that stdarg.h and the ansi style va_start macro is used for both 95 * ansi and traditional c compiles. 96 * XXX: this requires that stdarg.h define: va_alist and va_dcl 97 */ 98 #include <machine/stdarg.h> 99 100 void lock_printf(const char *fmt, ...) 101 __attribute__((__format__(__printf__,1,2))); 102 103 static int acquire(__volatile struct lock **, int *, int, int, int); 104 105 int lock_debug_syslog = 0; /* defaults to printf, but can be patched */ 106 107 #ifdef DDB 108 #include <ddb/ddbvar.h> 109 #include <machine/db_machdep.h> 110 #include <ddb/db_command.h> 111 #include <ddb/db_interface.h> 112 #endif 113 #endif 114 115 /* 116 * Locking primitives implementation. 117 * Locks provide shared/exclusive synchronization. 118 */ 119 120 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */ 121 #if defined(MULTIPROCESSOR) /* { */ 122 #define COUNT_CPU(cpu_id, x) \ 123 curcpu()->ci_spin_locks += (x) 124 #else 125 u_long spin_locks; 126 #define COUNT_CPU(cpu_id, x) spin_locks += (x) 127 #endif /* MULTIPROCESSOR */ /* } */ 128 129 #define COUNT(lkp, l, cpu_id, x) \ 130 do { \ 131 if ((lkp)->lk_flags & LK_SPIN) \ 132 COUNT_CPU((cpu_id), (x)); \ 133 else \ 134 (l)->l_locks += (x); \ 135 } while (/*CONSTCOND*/0) 136 #else 137 #define COUNT(lkp, p, cpu_id, x) 138 #define COUNT_CPU(cpu_id, x) 139 #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */ 140 141 #ifndef SPINLOCK_SPIN_HOOK /* from <machine/lock.h> */ 142 #define SPINLOCK_SPIN_HOOK /* nothing */ 143 #endif 144 145 #define INTERLOCK_ACQUIRE(lkp, flags, s) \ 146 do { \ 147 if ((flags) & LK_SPIN) \ 148 s = spllock(); \ 149 simple_lock(&(lkp)->lk_interlock); \ 150 } while (/*CONSTCOND*/ 0) 151 152 #define INTERLOCK_RELEASE(lkp, flags, s) \ 153 do { \ 154 simple_unlock(&(lkp)->lk_interlock); \ 155 if ((flags) & LK_SPIN) \ 156 splx(s); \ 157 } while (/*CONSTCOND*/ 0) 158 159 #ifdef DDB /* { */ 160 #ifdef MULTIPROCESSOR 161 int simple_lock_debugger = 1; /* more serious on MP */ 162 #else 163 int simple_lock_debugger = 0; 164 #endif 165 #define SLOCK_DEBUGGER() if (simple_lock_debugger) Debugger() 166 #define SLOCK_TRACE() \ 167 db_stack_trace_print((db_expr_t)__builtin_frame_address(0), \ 168 TRUE, 65535, "", lock_printf); 169 #else 170 #define SLOCK_DEBUGGER() /* nothing */ 171 #define SLOCK_TRACE() /* nothing */ 172 #endif /* } */ 173 174 #if defined(LOCKDEBUG) 175 #if defined(DDB) 176 #define SPINLOCK_SPINCHECK_DEBUGGER Debugger() 177 #else 178 #define SPINLOCK_SPINCHECK_DEBUGGER /* nothing */ 179 #endif 180 181 #define SPINLOCK_SPINCHECK_DECL \ 182 /* 32-bits of count -- wrap constitutes a "spinout" */ \ 183 uint32_t __spinc = 0 184 185 #define SPINLOCK_SPINCHECK \ 186 do { \ 187 if (++__spinc == 0) { \ 188 lock_printf("LK_SPIN spinout, excl %d, share %d\n", \ 189 lkp->lk_exclusivecount, lkp->lk_sharecount); \ 190 if (lkp->lk_exclusivecount) \ 191 lock_printf("held by CPU %lu\n", \ 192 (u_long) lkp->lk_cpu); \ 193 if (lkp->lk_lock_file) \ 194 lock_printf("last locked at %s:%d\n", \ 195 lkp->lk_lock_file, lkp->lk_lock_line); \ 196 if (lkp->lk_unlock_file) \ 197 lock_printf("last unlocked at %s:%d\n", \ 198 lkp->lk_unlock_file, lkp->lk_unlock_line); \ 199 SLOCK_TRACE(); \ 200 SPINLOCK_SPINCHECK_DEBUGGER; \ 201 } \ 202 } while (/*CONSTCOND*/ 0) 203 #else 204 #define SPINLOCK_SPINCHECK_DECL /* nothing */ 205 #define SPINLOCK_SPINCHECK /* nothing */ 206 #endif /* LOCKDEBUG && DDB */ 207 208 /* 209 * Acquire a resource. 210 */ 211 static int 212 acquire(__volatile struct lock **lkpp, int *s, int extflags, 213 int drain, int wanted) 214 { 215 int error; 216 __volatile struct lock *lkp = *lkpp; 217 218 KASSERT(drain || (wanted & LK_WAIT_NONZERO) == 0); 219 220 if (extflags & LK_SPIN) { 221 int interlocked; 222 223 SPINLOCK_SPINCHECK_DECL; 224 225 if (!drain) { 226 lkp->lk_waitcount++; 227 lkp->lk_flags |= LK_WAIT_NONZERO; 228 } 229 for (interlocked = 1;;) { 230 SPINLOCK_SPINCHECK; 231 if ((lkp->lk_flags & wanted) != 0) { 232 if (interlocked) { 233 INTERLOCK_RELEASE(lkp, LK_SPIN, *s); 234 interlocked = 0; 235 } 236 SPINLOCK_SPIN_HOOK; 237 } else if (interlocked) { 238 break; 239 } else { 240 INTERLOCK_ACQUIRE(lkp, LK_SPIN, *s); 241 interlocked = 1; 242 } 243 } 244 if (!drain) { 245 lkp->lk_waitcount--; 246 if (lkp->lk_waitcount == 0) 247 lkp->lk_flags &= ~LK_WAIT_NONZERO; 248 } 249 KASSERT((lkp->lk_flags & wanted) == 0); 250 error = 0; /* sanity */ 251 } else { 252 for (error = 0; (lkp->lk_flags & wanted) != 0; ) { 253 if (drain) 254 lkp->lk_flags |= LK_WAITDRAIN; 255 else { 256 lkp->lk_waitcount++; 257 lkp->lk_flags |= LK_WAIT_NONZERO; 258 } 259 /* XXX Cast away volatile. */ 260 error = ltsleep(drain ? 261 (void *)&lkp->lk_flags : 262 (void *)lkp, lkp->lk_prio, 263 lkp->lk_wmesg, lkp->lk_timo, &lkp->lk_interlock); 264 if (!drain) { 265 lkp->lk_waitcount--; 266 if (lkp->lk_waitcount == 0) 267 lkp->lk_flags &= ~LK_WAIT_NONZERO; 268 } 269 if (error) 270 break; 271 if (extflags & LK_SLEEPFAIL) { 272 error = ENOLCK; 273 break; 274 } 275 if (lkp->lk_newlock != NULL) { 276 simple_lock(&lkp->lk_newlock->lk_interlock); 277 simple_unlock(&lkp->lk_interlock); 278 if (lkp->lk_waitcount == 0) 279 wakeup((void *)&lkp->lk_newlock); 280 *lkpp = lkp = lkp->lk_newlock; 281 } 282 } 283 } 284 285 return error; 286 } 287 288 #define SETHOLDER(lkp, pid, lid, cpu_id) \ 289 do { \ 290 if ((lkp)->lk_flags & LK_SPIN) \ 291 (lkp)->lk_cpu = cpu_id; \ 292 else { \ 293 (lkp)->lk_lockholder = pid; \ 294 (lkp)->lk_locklwp = lid; \ 295 } \ 296 } while (/*CONSTCOND*/0) 297 298 #define WEHOLDIT(lkp, pid, lid, cpu_id) \ 299 (((lkp)->lk_flags & LK_SPIN) != 0 ? \ 300 ((lkp)->lk_cpu == (cpu_id)) : \ 301 ((lkp)->lk_lockholder == (pid) && (lkp)->lk_locklwp == (lid))) 302 303 #define WAKEUP_WAITER(lkp) \ 304 do { \ 305 if (((lkp)->lk_flags & (LK_SPIN | LK_WAIT_NONZERO)) == \ 306 LK_WAIT_NONZERO) { \ 307 /* XXX Cast away volatile. */ \ 308 wakeup((void *)(lkp)); \ 309 } \ 310 } while (/*CONSTCOND*/0) 311 312 #if defined(LOCKDEBUG) /* { */ 313 #if defined(MULTIPROCESSOR) /* { */ 314 struct simplelock spinlock_list_slock = SIMPLELOCK_INITIALIZER; 315 316 #define SPINLOCK_LIST_LOCK() \ 317 __cpu_simple_lock(&spinlock_list_slock.lock_data) 318 319 #define SPINLOCK_LIST_UNLOCK() \ 320 __cpu_simple_unlock(&spinlock_list_slock.lock_data) 321 #else 322 #define SPINLOCK_LIST_LOCK() /* nothing */ 323 324 #define SPINLOCK_LIST_UNLOCK() /* nothing */ 325 #endif /* MULTIPROCESSOR */ /* } */ 326 327 TAILQ_HEAD(, lock) spinlock_list = 328 TAILQ_HEAD_INITIALIZER(spinlock_list); 329 330 #define HAVEIT(lkp) \ 331 do { \ 332 if ((lkp)->lk_flags & LK_SPIN) { \ 333 int s = spllock(); \ 334 SPINLOCK_LIST_LOCK(); \ 335 /* XXX Cast away volatile. */ \ 336 TAILQ_INSERT_TAIL(&spinlock_list, (struct lock *)(lkp), \ 337 lk_list); \ 338 SPINLOCK_LIST_UNLOCK(); \ 339 splx(s); \ 340 } \ 341 } while (/*CONSTCOND*/0) 342 343 #define DONTHAVEIT(lkp) \ 344 do { \ 345 if ((lkp)->lk_flags & LK_SPIN) { \ 346 int s = spllock(); \ 347 SPINLOCK_LIST_LOCK(); \ 348 /* XXX Cast away volatile. */ \ 349 TAILQ_REMOVE(&spinlock_list, (struct lock *)(lkp), \ 350 lk_list); \ 351 SPINLOCK_LIST_UNLOCK(); \ 352 splx(s); \ 353 } \ 354 } while (/*CONSTCOND*/0) 355 #else 356 #define HAVEIT(lkp) /* nothing */ 357 358 #define DONTHAVEIT(lkp) /* nothing */ 359 #endif /* LOCKDEBUG */ /* } */ 360 361 #if defined(LOCKDEBUG) 362 /* 363 * Lock debug printing routine; can be configured to print to console 364 * or log to syslog. 365 */ 366 void 367 lock_printf(const char *fmt, ...) 368 { 369 char b[150]; 370 va_list ap; 371 372 va_start(ap, fmt); 373 if (lock_debug_syslog) 374 vlog(LOG_DEBUG, fmt, ap); 375 else { 376 vsnprintf(b, sizeof(b), fmt, ap); 377 printf_nolog("%s", b); 378 } 379 va_end(ap); 380 } 381 #endif /* LOCKDEBUG */ 382 383 /* 384 * Transfer any waiting processes from one lock to another. 385 */ 386 void 387 transferlockers(struct lock *from, struct lock *to) 388 { 389 390 KASSERT(from != to); 391 KASSERT((from->lk_flags & LK_WAITDRAIN) == 0); 392 if (from->lk_waitcount == 0) 393 return; 394 from->lk_newlock = to; 395 wakeup((void *)from); 396 tsleep((void *)&from->lk_newlock, from->lk_prio, "lkxfer", 0); 397 from->lk_newlock = NULL; 398 from->lk_flags &= ~(LK_WANT_EXCL | LK_WANT_UPGRADE); 399 KASSERT(from->lk_waitcount == 0); 400 } 401 402 403 /* 404 * Initialize a lock; required before use. 405 */ 406 void 407 lockinit(struct lock *lkp, int prio, const char *wmesg, int timo, int flags) 408 { 409 410 memset(lkp, 0, sizeof(struct lock)); 411 simple_lock_init(&lkp->lk_interlock); 412 lkp->lk_flags = flags & LK_EXTFLG_MASK; 413 if (flags & LK_SPIN) 414 lkp->lk_cpu = LK_NOCPU; 415 else { 416 lkp->lk_lockholder = LK_NOPROC; 417 lkp->lk_newlock = NULL; 418 lkp->lk_prio = prio; 419 lkp->lk_timo = timo; 420 } 421 lkp->lk_wmesg = wmesg; /* just a name for spin locks */ 422 #if defined(LOCKDEBUG) 423 lkp->lk_lock_file = NULL; 424 lkp->lk_unlock_file = NULL; 425 #endif 426 } 427 428 /* 429 * Determine the status of a lock. 430 */ 431 int 432 lockstatus(struct lock *lkp) 433 { 434 int s = 0; /* XXX: gcc */ 435 int lock_type = 0; 436 struct lwp *l = curlwp; /* XXX */ 437 pid_t pid; 438 lwpid_t lid; 439 cpuid_t cpu_id; 440 441 if ((lkp->lk_flags & LK_SPIN) || l == NULL) { 442 cpu_id = cpu_number(); 443 pid = LK_KERNPROC; 444 lid = 0; 445 } else { 446 cpu_id = LK_NOCPU; 447 pid = l->l_proc->p_pid; 448 lid = l->l_lid; 449 } 450 451 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 452 if (lkp->lk_exclusivecount != 0) { 453 if (WEHOLDIT(lkp, pid, lid, cpu_id)) 454 lock_type = LK_EXCLUSIVE; 455 else 456 lock_type = LK_EXCLOTHER; 457 } else if (lkp->lk_sharecount != 0) 458 lock_type = LK_SHARED; 459 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 460 return (lock_type); 461 } 462 463 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) 464 /* 465 * Make sure no spin locks are held by a CPU that is about 466 * to context switch. 467 */ 468 void 469 spinlock_switchcheck(void) 470 { 471 u_long cnt; 472 int s; 473 474 s = spllock(); 475 #if defined(MULTIPROCESSOR) 476 cnt = curcpu()->ci_spin_locks; 477 #else 478 cnt = spin_locks; 479 #endif 480 splx(s); 481 482 if (cnt != 0) 483 panic("spinlock_switchcheck: CPU %lu has %lu spin locks", 484 (u_long) cpu_number(), cnt); 485 } 486 #endif /* LOCKDEBUG || DIAGNOSTIC */ 487 488 /* 489 * Locks and IPLs (interrupt priority levels): 490 * 491 * Locks which may be taken from interrupt context must be handled 492 * very carefully; you must spl to the highest IPL where the lock 493 * is needed before acquiring the lock. 494 * 495 * It is also important to avoid deadlock, since certain (very high 496 * priority) interrupts are often needed to keep the system as a whole 497 * from deadlocking, and must not be blocked while you are spinning 498 * waiting for a lower-priority lock. 499 * 500 * In addition, the lock-debugging hooks themselves need to use locks! 501 * 502 * A raw __cpu_simple_lock may be used from interrupts are long as it 503 * is acquired and held at a single IPL. 504 * 505 * A simple_lock (which is a __cpu_simple_lock wrapped with some 506 * debugging hooks) may be used at or below spllock(), which is 507 * typically at or just below splhigh() (i.e. blocks everything 508 * but certain machine-dependent extremely high priority interrupts). 509 * 510 * spinlockmgr spinlocks should be used at or below splsched(). 511 * 512 * Some platforms may have interrupts of higher priority than splsched(), 513 * including hard serial interrupts, inter-processor interrupts, and 514 * kernel debugger traps. 515 */ 516 517 /* 518 * XXX XXX kludge around another kludge.. 519 * 520 * vfs_shutdown() may be called from interrupt context, either as a result 521 * of a panic, or from the debugger. It proceeds to call 522 * sys_sync(&proc0, ...), pretending its running on behalf of proc0 523 * 524 * We would like to make an attempt to sync the filesystems in this case, so 525 * if this happens, we treat attempts to acquire locks specially. 526 * All locks are acquired on behalf of proc0. 527 * 528 * If we've already paniced, we don't block waiting for locks, but 529 * just barge right ahead since we're already going down in flames. 530 */ 531 532 /* 533 * Set, change, or release a lock. 534 * 535 * Shared requests increment the shared count. Exclusive requests set the 536 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 537 * accepted shared locks and shared-to-exclusive upgrades to go away. 538 */ 539 int 540 #if defined(LOCKDEBUG) 541 _lockmgr(__volatile struct lock *lkp, u_int flags, 542 struct simplelock *interlkp, const char *file, int line) 543 #else 544 lockmgr(__volatile struct lock *lkp, u_int flags, 545 struct simplelock *interlkp) 546 #endif 547 { 548 int error; 549 pid_t pid; 550 lwpid_t lid; 551 int extflags; 552 cpuid_t cpu_id; 553 struct lwp *l = curlwp; 554 int lock_shutdown_noblock = 0; 555 int s = 0; 556 557 error = 0; 558 559 /* LK_RETRY is for vn_lock, not for lockmgr. */ 560 KASSERT((flags & LK_RETRY) == 0); 561 562 INTERLOCK_ACQUIRE(lkp, lkp->lk_flags, s); 563 if (flags & LK_INTERLOCK) 564 simple_unlock(interlkp); 565 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 566 567 #ifdef DIAGNOSTIC /* { */ 568 /* 569 * Don't allow spins on sleep locks and don't allow sleeps 570 * on spin locks. 571 */ 572 if ((flags ^ lkp->lk_flags) & LK_SPIN) 573 panic("lockmgr: sleep/spin mismatch"); 574 #endif /* } */ 575 576 if (extflags & LK_SPIN) { 577 pid = LK_KERNPROC; 578 lid = 0; 579 } else { 580 if (l == NULL) { 581 if (!doing_shutdown) { 582 panic("lockmgr: no context"); 583 } else { 584 l = &lwp0; 585 if (panicstr && (!(flags & LK_NOWAIT))) { 586 flags |= LK_NOWAIT; 587 lock_shutdown_noblock = 1; 588 } 589 } 590 } 591 lid = l->l_lid; 592 pid = l->l_proc->p_pid; 593 } 594 cpu_id = cpu_number(); 595 596 /* 597 * Once a lock has drained, the LK_DRAINING flag is set and an 598 * exclusive lock is returned. The only valid operation thereafter 599 * is a single release of that exclusive lock. This final release 600 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any 601 * further requests of any sort will result in a panic. The bits 602 * selected for these two flags are chosen so that they will be set 603 * in memory that is freed (freed memory is filled with 0xdeadbeef). 604 * The final release is permitted to give a new lease on life to 605 * the lock by specifying LK_REENABLE. 606 */ 607 if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) { 608 #ifdef DIAGNOSTIC /* { */ 609 if (lkp->lk_flags & LK_DRAINED) 610 panic("lockmgr: using decommissioned lock"); 611 if ((flags & LK_TYPE_MASK) != LK_RELEASE || 612 WEHOLDIT(lkp, pid, lid, cpu_id) == 0) 613 panic("lockmgr: non-release on draining lock: %d", 614 flags & LK_TYPE_MASK); 615 #endif /* DIAGNOSTIC */ /* } */ 616 lkp->lk_flags &= ~LK_DRAINING; 617 if ((flags & LK_REENABLE) == 0) 618 lkp->lk_flags |= LK_DRAINED; 619 } 620 621 switch (flags & LK_TYPE_MASK) { 622 623 case LK_SHARED: 624 if (WEHOLDIT(lkp, pid, lid, cpu_id) == 0) { 625 /* 626 * If just polling, check to see if we will block. 627 */ 628 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 629 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) { 630 error = EBUSY; 631 break; 632 } 633 /* 634 * Wait for exclusive locks and upgrades to clear. 635 */ 636 error = acquire(&lkp, &s, extflags, 0, 637 LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE); 638 if (error) 639 break; 640 lkp->lk_sharecount++; 641 lkp->lk_flags |= LK_SHARE_NONZERO; 642 COUNT(lkp, l, cpu_id, 1); 643 break; 644 } 645 /* 646 * We hold an exclusive lock, so downgrade it to shared. 647 * An alternative would be to fail with EDEADLK. 648 */ 649 lkp->lk_sharecount++; 650 lkp->lk_flags |= LK_SHARE_NONZERO; 651 COUNT(lkp, l, cpu_id, 1); 652 /* fall into downgrade */ 653 654 case LK_DOWNGRADE: 655 if (WEHOLDIT(lkp, pid, lid, cpu_id) == 0 || 656 lkp->lk_exclusivecount == 0) 657 panic("lockmgr: not holding exclusive lock"); 658 lkp->lk_sharecount += lkp->lk_exclusivecount; 659 lkp->lk_flags |= LK_SHARE_NONZERO; 660 lkp->lk_exclusivecount = 0; 661 lkp->lk_recurselevel = 0; 662 lkp->lk_flags &= ~LK_HAVE_EXCL; 663 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 664 #if defined(LOCKDEBUG) 665 lkp->lk_unlock_file = file; 666 lkp->lk_unlock_line = line; 667 #endif 668 DONTHAVEIT(lkp); 669 WAKEUP_WAITER(lkp); 670 break; 671 672 case LK_EXCLUPGRADE: 673 /* 674 * If another process is ahead of us to get an upgrade, 675 * then we want to fail rather than have an intervening 676 * exclusive access. 677 */ 678 if (lkp->lk_flags & LK_WANT_UPGRADE) { 679 lkp->lk_sharecount--; 680 if (lkp->lk_sharecount == 0) 681 lkp->lk_flags &= ~LK_SHARE_NONZERO; 682 COUNT(lkp, l, cpu_id, -1); 683 error = EBUSY; 684 break; 685 } 686 /* fall into normal upgrade */ 687 688 case LK_UPGRADE: 689 /* 690 * Upgrade a shared lock to an exclusive one. If another 691 * shared lock has already requested an upgrade to an 692 * exclusive lock, our shared lock is released and an 693 * exclusive lock is requested (which will be granted 694 * after the upgrade). If we return an error, the file 695 * will always be unlocked. 696 */ 697 if (WEHOLDIT(lkp, pid, lid, cpu_id) || lkp->lk_sharecount <= 0) 698 panic("lockmgr: upgrade exclusive lock"); 699 lkp->lk_sharecount--; 700 if (lkp->lk_sharecount == 0) 701 lkp->lk_flags &= ~LK_SHARE_NONZERO; 702 COUNT(lkp, l, cpu_id, -1); 703 /* 704 * If we are just polling, check to see if we will block. 705 */ 706 if ((extflags & LK_NOWAIT) && 707 ((lkp->lk_flags & LK_WANT_UPGRADE) || 708 lkp->lk_sharecount > 1)) { 709 error = EBUSY; 710 break; 711 } 712 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) { 713 /* 714 * We are first shared lock to request an upgrade, so 715 * request upgrade and wait for the shared count to 716 * drop to zero, then take exclusive lock. 717 */ 718 lkp->lk_flags |= LK_WANT_UPGRADE; 719 error = acquire(&lkp, &s, extflags, 0, LK_SHARE_NONZERO); 720 lkp->lk_flags &= ~LK_WANT_UPGRADE; 721 if (error) { 722 WAKEUP_WAITER(lkp); 723 break; 724 } 725 lkp->lk_flags |= LK_HAVE_EXCL; 726 SETHOLDER(lkp, pid, lid, cpu_id); 727 #if defined(LOCKDEBUG) 728 lkp->lk_lock_file = file; 729 lkp->lk_lock_line = line; 730 #endif 731 HAVEIT(lkp); 732 if (lkp->lk_exclusivecount != 0) 733 panic("lockmgr: non-zero exclusive count"); 734 lkp->lk_exclusivecount = 1; 735 if (extflags & LK_SETRECURSE) 736 lkp->lk_recurselevel = 1; 737 COUNT(lkp, l, cpu_id, 1); 738 break; 739 } 740 /* 741 * Someone else has requested upgrade. Release our shared 742 * lock, awaken upgrade requestor if we are the last shared 743 * lock, then request an exclusive lock. 744 */ 745 if (lkp->lk_sharecount == 0) 746 WAKEUP_WAITER(lkp); 747 /* fall into exclusive request */ 748 749 case LK_EXCLUSIVE: 750 if (WEHOLDIT(lkp, pid, lid, cpu_id)) { 751 /* 752 * Recursive lock. 753 */ 754 if ((extflags & LK_CANRECURSE) == 0 && 755 lkp->lk_recurselevel == 0) { 756 if (extflags & LK_RECURSEFAIL) { 757 error = EDEADLK; 758 break; 759 } else 760 panic("lockmgr: locking against myself"); 761 } 762 lkp->lk_exclusivecount++; 763 if (extflags & LK_SETRECURSE && 764 lkp->lk_recurselevel == 0) 765 lkp->lk_recurselevel = lkp->lk_exclusivecount; 766 COUNT(lkp, l, cpu_id, 1); 767 break; 768 } 769 /* 770 * If we are just polling, check to see if we will sleep. 771 */ 772 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 773 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 774 LK_SHARE_NONZERO))) { 775 error = EBUSY; 776 break; 777 } 778 /* 779 * Try to acquire the want_exclusive flag. 780 */ 781 error = acquire(&lkp, &s, extflags, 0, 782 LK_HAVE_EXCL | LK_WANT_EXCL); 783 if (error) 784 break; 785 lkp->lk_flags |= LK_WANT_EXCL; 786 /* 787 * Wait for shared locks and upgrades to finish. 788 */ 789 error = acquire(&lkp, &s, extflags, 0, 790 LK_HAVE_EXCL | LK_WANT_UPGRADE | LK_SHARE_NONZERO); 791 lkp->lk_flags &= ~LK_WANT_EXCL; 792 if (error) { 793 WAKEUP_WAITER(lkp); 794 break; 795 } 796 lkp->lk_flags |= LK_HAVE_EXCL; 797 SETHOLDER(lkp, pid, lid, cpu_id); 798 #if defined(LOCKDEBUG) 799 lkp->lk_lock_file = file; 800 lkp->lk_lock_line = line; 801 #endif 802 HAVEIT(lkp); 803 if (lkp->lk_exclusivecount != 0) 804 panic("lockmgr: non-zero exclusive count"); 805 lkp->lk_exclusivecount = 1; 806 if (extflags & LK_SETRECURSE) 807 lkp->lk_recurselevel = 1; 808 COUNT(lkp, l, cpu_id, 1); 809 break; 810 811 case LK_RELEASE: 812 if (lkp->lk_exclusivecount != 0) { 813 if (WEHOLDIT(lkp, pid, lid, cpu_id) == 0) { 814 if (lkp->lk_flags & LK_SPIN) { 815 panic("lockmgr: processor %lu, not " 816 "exclusive lock holder %lu " 817 "unlocking", cpu_id, lkp->lk_cpu); 818 } else { 819 panic("lockmgr: pid %d, not " 820 "exclusive lock holder %d " 821 "unlocking", pid, 822 lkp->lk_lockholder); 823 } 824 } 825 if (lkp->lk_exclusivecount == lkp->lk_recurselevel) 826 lkp->lk_recurselevel = 0; 827 lkp->lk_exclusivecount--; 828 COUNT(lkp, l, cpu_id, -1); 829 if (lkp->lk_exclusivecount == 0) { 830 lkp->lk_flags &= ~LK_HAVE_EXCL; 831 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 832 #if defined(LOCKDEBUG) 833 lkp->lk_unlock_file = file; 834 lkp->lk_unlock_line = line; 835 #endif 836 DONTHAVEIT(lkp); 837 } 838 } else if (lkp->lk_sharecount != 0) { 839 lkp->lk_sharecount--; 840 if (lkp->lk_sharecount == 0) 841 lkp->lk_flags &= ~LK_SHARE_NONZERO; 842 COUNT(lkp, l, cpu_id, -1); 843 } 844 #ifdef DIAGNOSTIC 845 else 846 panic("lockmgr: release of unlocked lock!"); 847 #endif 848 WAKEUP_WAITER(lkp); 849 break; 850 851 case LK_DRAIN: 852 /* 853 * Check that we do not already hold the lock, as it can 854 * never drain if we do. Unfortunately, we have no way to 855 * check for holding a shared lock, but at least we can 856 * check for an exclusive one. 857 */ 858 if (WEHOLDIT(lkp, pid, lid, cpu_id)) 859 panic("lockmgr: draining against myself"); 860 /* 861 * If we are just polling, check to see if we will sleep. 862 */ 863 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 864 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 865 LK_SHARE_NONZERO | LK_WAIT_NONZERO))) { 866 error = EBUSY; 867 break; 868 } 869 error = acquire(&lkp, &s, extflags, 1, 870 LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 871 LK_SHARE_NONZERO | LK_WAIT_NONZERO); 872 if (error) 873 break; 874 lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL; 875 SETHOLDER(lkp, pid, lid, cpu_id); 876 #if defined(LOCKDEBUG) 877 lkp->lk_lock_file = file; 878 lkp->lk_lock_line = line; 879 #endif 880 HAVEIT(lkp); 881 lkp->lk_exclusivecount = 1; 882 /* XXX unlikely that we'd want this */ 883 if (extflags & LK_SETRECURSE) 884 lkp->lk_recurselevel = 1; 885 COUNT(lkp, l, cpu_id, 1); 886 break; 887 888 default: 889 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 890 panic("lockmgr: unknown locktype request %d", 891 flags & LK_TYPE_MASK); 892 /* NOTREACHED */ 893 } 894 if ((lkp->lk_flags & (LK_WAITDRAIN|LK_SPIN)) == LK_WAITDRAIN && 895 ((lkp->lk_flags & 896 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE | 897 LK_SHARE_NONZERO | LK_WAIT_NONZERO)) == 0)) { 898 lkp->lk_flags &= ~LK_WAITDRAIN; 899 wakeup((void *)&lkp->lk_flags); 900 } 901 /* 902 * Note that this panic will be a recursive panic, since 903 * we only set lock_shutdown_noblock above if panicstr != NULL. 904 */ 905 if (error && lock_shutdown_noblock) 906 panic("lockmgr: deadlock (see previous panic)"); 907 908 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 909 return (error); 910 } 911 912 /* 913 * For a recursive spinlock held one or more times by the current CPU, 914 * release all N locks, and return N. 915 * Intended for use in mi_switch() shortly before context switching. 916 */ 917 918 int 919 #if defined(LOCKDEBUG) 920 _spinlock_release_all(__volatile struct lock *lkp, const char *file, int line) 921 #else 922 spinlock_release_all(__volatile struct lock *lkp) 923 #endif 924 { 925 int s, count; 926 cpuid_t cpu_id; 927 928 KASSERT(lkp->lk_flags & LK_SPIN); 929 930 INTERLOCK_ACQUIRE(lkp, LK_SPIN, s); 931 932 cpu_id = cpu_number(); 933 count = lkp->lk_exclusivecount; 934 935 if (count != 0) { 936 #ifdef DIAGNOSTIC 937 if (WEHOLDIT(lkp, 0, 0, cpu_id) == 0) { 938 panic("spinlock_release_all: processor %lu, not " 939 "exclusive lock holder %lu " 940 "unlocking", (long)cpu_id, lkp->lk_cpu); 941 } 942 #endif 943 lkp->lk_recurselevel = 0; 944 lkp->lk_exclusivecount = 0; 945 COUNT_CPU(cpu_id, -count); 946 lkp->lk_flags &= ~LK_HAVE_EXCL; 947 SETHOLDER(lkp, LK_NOPROC, 0, LK_NOCPU); 948 #if defined(LOCKDEBUG) 949 lkp->lk_unlock_file = file; 950 lkp->lk_unlock_line = line; 951 #endif 952 DONTHAVEIT(lkp); 953 } 954 #ifdef DIAGNOSTIC 955 else if (lkp->lk_sharecount != 0) 956 panic("spinlock_release_all: release of shared lock!"); 957 else 958 panic("spinlock_release_all: release of unlocked lock!"); 959 #endif 960 INTERLOCK_RELEASE(lkp, LK_SPIN, s); 961 962 return (count); 963 } 964 965 /* 966 * For a recursive spinlock held one or more times by the current CPU, 967 * release all N locks, and return N. 968 * Intended for use in mi_switch() right after resuming execution. 969 */ 970 971 void 972 #if defined(LOCKDEBUG) 973 _spinlock_acquire_count(__volatile struct lock *lkp, int count, 974 const char *file, int line) 975 #else 976 spinlock_acquire_count(__volatile struct lock *lkp, int count) 977 #endif 978 { 979 int s, error; 980 cpuid_t cpu_id; 981 982 KASSERT(lkp->lk_flags & LK_SPIN); 983 984 INTERLOCK_ACQUIRE(lkp, LK_SPIN, s); 985 986 cpu_id = cpu_number(); 987 988 #ifdef DIAGNOSTIC 989 if (WEHOLDIT(lkp, LK_NOPROC, 0, cpu_id)) 990 panic("spinlock_acquire_count: processor %lu already holds lock", (long)cpu_id); 991 #endif 992 /* 993 * Try to acquire the want_exclusive flag. 994 */ 995 error = acquire(&lkp, &s, LK_SPIN, 0, LK_HAVE_EXCL | LK_WANT_EXCL); 996 lkp->lk_flags |= LK_WANT_EXCL; 997 /* 998 * Wait for shared locks and upgrades to finish. 999 */ 1000 error = acquire(&lkp, &s, LK_SPIN, 0, 1001 LK_HAVE_EXCL | LK_SHARE_NONZERO | LK_WANT_UPGRADE); 1002 lkp->lk_flags &= ~LK_WANT_EXCL; 1003 lkp->lk_flags |= LK_HAVE_EXCL; 1004 SETHOLDER(lkp, LK_NOPROC, 0, cpu_id); 1005 #if defined(LOCKDEBUG) 1006 lkp->lk_lock_file = file; 1007 lkp->lk_lock_line = line; 1008 #endif 1009 HAVEIT(lkp); 1010 if (lkp->lk_exclusivecount != 0) 1011 panic("lockmgr: non-zero exclusive count"); 1012 lkp->lk_exclusivecount = count; 1013 lkp->lk_recurselevel = 1; 1014 COUNT_CPU(cpu_id, count); 1015 1016 INTERLOCK_RELEASE(lkp, lkp->lk_flags, s); 1017 } 1018 1019 1020 1021 /* 1022 * Print out information about state of a lock. Used by VOP_PRINT 1023 * routines to display ststus about contained locks. 1024 */ 1025 void 1026 lockmgr_printinfo(__volatile struct lock *lkp) 1027 { 1028 1029 if (lkp->lk_sharecount) 1030 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 1031 lkp->lk_sharecount); 1032 else if (lkp->lk_flags & LK_HAVE_EXCL) { 1033 printf(" lock type %s: EXCL (count %d) by ", 1034 lkp->lk_wmesg, lkp->lk_exclusivecount); 1035 if (lkp->lk_flags & LK_SPIN) 1036 printf("processor %lu", lkp->lk_cpu); 1037 else 1038 printf("pid %d.%d", lkp->lk_lockholder, 1039 lkp->lk_locklwp); 1040 } else 1041 printf(" not locked"); 1042 if ((lkp->lk_flags & LK_SPIN) == 0 && lkp->lk_waitcount > 0) 1043 printf(" with %d pending", lkp->lk_waitcount); 1044 } 1045 1046 #if defined(LOCKDEBUG) /* { */ 1047 TAILQ_HEAD(, simplelock) simplelock_list = 1048 TAILQ_HEAD_INITIALIZER(simplelock_list); 1049 1050 #if defined(MULTIPROCESSOR) /* { */ 1051 struct simplelock simplelock_list_slock = SIMPLELOCK_INITIALIZER; 1052 1053 #define SLOCK_LIST_LOCK() \ 1054 __cpu_simple_lock(&simplelock_list_slock.lock_data) 1055 1056 #define SLOCK_LIST_UNLOCK() \ 1057 __cpu_simple_unlock(&simplelock_list_slock.lock_data) 1058 1059 #define SLOCK_COUNT(x) \ 1060 curcpu()->ci_simple_locks += (x) 1061 #else 1062 u_long simple_locks; 1063 1064 #define SLOCK_LIST_LOCK() /* nothing */ 1065 1066 #define SLOCK_LIST_UNLOCK() /* nothing */ 1067 1068 #define SLOCK_COUNT(x) simple_locks += (x) 1069 #endif /* MULTIPROCESSOR */ /* } */ 1070 1071 #ifdef MULTIPROCESSOR 1072 #define SLOCK_MP() lock_printf("on CPU %ld\n", \ 1073 (u_long) cpu_number()) 1074 #else 1075 #define SLOCK_MP() /* nothing */ 1076 #endif 1077 1078 #define SLOCK_WHERE(str, alp, id, l) \ 1079 do { \ 1080 lock_printf("\n"); \ 1081 lock_printf(str); \ 1082 lock_printf("lock: %p, currently at: %s:%d\n", (alp), (id), (l)); \ 1083 SLOCK_MP(); \ 1084 if ((alp)->lock_file != NULL) \ 1085 lock_printf("last locked: %s:%d\n", (alp)->lock_file, \ 1086 (alp)->lock_line); \ 1087 if ((alp)->unlock_file != NULL) \ 1088 lock_printf("last unlocked: %s:%d\n", (alp)->unlock_file, \ 1089 (alp)->unlock_line); \ 1090 SLOCK_TRACE() \ 1091 SLOCK_DEBUGGER(); \ 1092 } while (/*CONSTCOND*/0) 1093 1094 /* 1095 * Simple lock functions so that the debugger can see from whence 1096 * they are being called. 1097 */ 1098 void 1099 simple_lock_init(struct simplelock *alp) 1100 { 1101 1102 #if defined(MULTIPROCESSOR) /* { */ 1103 __cpu_simple_lock_init(&alp->lock_data); 1104 #else 1105 alp->lock_data = __SIMPLELOCK_UNLOCKED; 1106 #endif /* } */ 1107 alp->lock_file = NULL; 1108 alp->lock_line = 0; 1109 alp->unlock_file = NULL; 1110 alp->unlock_line = 0; 1111 alp->lock_holder = LK_NOCPU; 1112 } 1113 1114 void 1115 _simple_lock(__volatile struct simplelock *alp, const char *id, int l) 1116 { 1117 cpuid_t cpu_id = cpu_number(); 1118 int s; 1119 1120 s = spllock(); 1121 1122 /* 1123 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 1124 * don't take any action, and just fall into the normal spin case. 1125 */ 1126 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1127 #if defined(MULTIPROCESSOR) /* { */ 1128 if (alp->lock_holder == cpu_id) { 1129 SLOCK_WHERE("simple_lock: locking against myself\n", 1130 alp, id, l); 1131 goto out; 1132 } 1133 #else 1134 SLOCK_WHERE("simple_lock: lock held\n", alp, id, l); 1135 goto out; 1136 #endif /* MULTIPROCESSOR */ /* } */ 1137 } 1138 1139 #if defined(MULTIPROCESSOR) /* { */ 1140 /* Acquire the lock before modifying any fields. */ 1141 splx(s); 1142 __cpu_simple_lock(&alp->lock_data); 1143 s = spllock(); 1144 #else 1145 alp->lock_data = __SIMPLELOCK_LOCKED; 1146 #endif /* } */ 1147 1148 if (alp->lock_holder != LK_NOCPU) { 1149 SLOCK_WHERE("simple_lock: uninitialized lock\n", 1150 alp, id, l); 1151 } 1152 alp->lock_file = id; 1153 alp->lock_line = l; 1154 alp->lock_holder = cpu_id; 1155 1156 SLOCK_LIST_LOCK(); 1157 /* XXX Cast away volatile */ 1158 TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list); 1159 SLOCK_LIST_UNLOCK(); 1160 1161 SLOCK_COUNT(1); 1162 1163 out: 1164 splx(s); 1165 } 1166 1167 int 1168 _simple_lock_held(__volatile struct simplelock *alp) 1169 { 1170 #if defined(MULTIPROCESSOR) || defined(DIAGNOSTIC) 1171 cpuid_t cpu_id = cpu_number(); 1172 #endif 1173 int s, locked = 0; 1174 1175 s = spllock(); 1176 1177 #if defined(MULTIPROCESSOR) 1178 if (__cpu_simple_lock_try(&alp->lock_data) == 0) 1179 locked = (alp->lock_holder == cpu_id); 1180 else 1181 __cpu_simple_unlock(&alp->lock_data); 1182 #else 1183 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1184 locked = 1; 1185 KASSERT(alp->lock_holder == cpu_id); 1186 } 1187 #endif 1188 1189 splx(s); 1190 1191 return (locked); 1192 } 1193 1194 int 1195 _simple_lock_try(__volatile struct simplelock *alp, const char *id, int l) 1196 { 1197 cpuid_t cpu_id = cpu_number(); 1198 int s, rv = 0; 1199 1200 s = spllock(); 1201 1202 /* 1203 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 1204 * don't take any action. 1205 */ 1206 #if defined(MULTIPROCESSOR) /* { */ 1207 if ((rv = __cpu_simple_lock_try(&alp->lock_data)) == 0) { 1208 if (alp->lock_holder == cpu_id) 1209 SLOCK_WHERE("simple_lock_try: locking against myself\n", 1210 alp, id, l); 1211 goto out; 1212 } 1213 #else 1214 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 1215 SLOCK_WHERE("simple_lock_try: lock held\n", alp, id, l); 1216 goto out; 1217 } 1218 alp->lock_data = __SIMPLELOCK_LOCKED; 1219 #endif /* MULTIPROCESSOR */ /* } */ 1220 1221 /* 1222 * At this point, we have acquired the lock. 1223 */ 1224 1225 rv = 1; 1226 1227 alp->lock_file = id; 1228 alp->lock_line = l; 1229 alp->lock_holder = cpu_id; 1230 1231 SLOCK_LIST_LOCK(); 1232 /* XXX Cast away volatile. */ 1233 TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list); 1234 SLOCK_LIST_UNLOCK(); 1235 1236 SLOCK_COUNT(1); 1237 1238 out: 1239 splx(s); 1240 return (rv); 1241 } 1242 1243 void 1244 _simple_unlock(__volatile struct simplelock *alp, const char *id, int l) 1245 { 1246 int s; 1247 1248 s = spllock(); 1249 1250 /* 1251 * MULTIPROCESSOR case: This is `safe' because we think we hold 1252 * the lock, and if we don't, we don't take any action. 1253 */ 1254 if (alp->lock_data == __SIMPLELOCK_UNLOCKED) { 1255 SLOCK_WHERE("simple_unlock: lock not held\n", 1256 alp, id, l); 1257 goto out; 1258 } 1259 1260 SLOCK_LIST_LOCK(); 1261 TAILQ_REMOVE(&simplelock_list, alp, list); 1262 SLOCK_LIST_UNLOCK(); 1263 1264 SLOCK_COUNT(-1); 1265 1266 alp->list.tqe_next = NULL; /* sanity */ 1267 alp->list.tqe_prev = NULL; /* sanity */ 1268 1269 alp->unlock_file = id; 1270 alp->unlock_line = l; 1271 1272 #if defined(MULTIPROCESSOR) /* { */ 1273 alp->lock_holder = LK_NOCPU; 1274 /* Now that we've modified all fields, release the lock. */ 1275 __cpu_simple_unlock(&alp->lock_data); 1276 #else 1277 alp->lock_data = __SIMPLELOCK_UNLOCKED; 1278 KASSERT(alp->lock_holder == cpu_number()); 1279 alp->lock_holder = LK_NOCPU; 1280 #endif /* } */ 1281 1282 out: 1283 splx(s); 1284 } 1285 1286 void 1287 simple_lock_dump(void) 1288 { 1289 struct simplelock *alp; 1290 int s; 1291 1292 s = spllock(); 1293 SLOCK_LIST_LOCK(); 1294 lock_printf("all simple locks:\n"); 1295 TAILQ_FOREACH(alp, &simplelock_list, list) { 1296 lock_printf("%p CPU %lu %s:%d\n", alp, alp->lock_holder, 1297 alp->lock_file, alp->lock_line); 1298 } 1299 SLOCK_LIST_UNLOCK(); 1300 splx(s); 1301 } 1302 1303 void 1304 simple_lock_freecheck(void *start, void *end) 1305 { 1306 struct simplelock *alp; 1307 int s; 1308 1309 s = spllock(); 1310 SLOCK_LIST_LOCK(); 1311 TAILQ_FOREACH(alp, &simplelock_list, list) { 1312 if ((void *)alp >= start && (void *)alp < end) { 1313 lock_printf("freeing simple_lock %p CPU %lu %s:%d\n", 1314 alp, alp->lock_holder, alp->lock_file, 1315 alp->lock_line); 1316 SLOCK_DEBUGGER(); 1317 } 1318 } 1319 SLOCK_LIST_UNLOCK(); 1320 splx(s); 1321 } 1322 1323 /* 1324 * We must be holding exactly one lock: the sched_lock. 1325 */ 1326 1327 void 1328 simple_lock_switchcheck(void) 1329 { 1330 1331 simple_lock_only_held(&sched_lock, "switching"); 1332 } 1333 1334 void 1335 simple_lock_only_held(volatile struct simplelock *lp, const char *where) 1336 { 1337 struct simplelock *alp; 1338 cpuid_t cpu_id = cpu_number(); 1339 int s; 1340 1341 if (lp) { 1342 LOCK_ASSERT(simple_lock_held(lp)); 1343 } 1344 s = spllock(); 1345 SLOCK_LIST_LOCK(); 1346 TAILQ_FOREACH(alp, &simplelock_list, list) { 1347 if (alp == lp) 1348 continue; 1349 if (alp->lock_holder == cpu_id) 1350 break; 1351 } 1352 SLOCK_LIST_UNLOCK(); 1353 splx(s); 1354 1355 if (alp != NULL) { 1356 lock_printf("\n%s with held simple_lock %p " 1357 "CPU %lu %s:%d\n", 1358 where, alp, alp->lock_holder, alp->lock_file, 1359 alp->lock_line); 1360 SLOCK_TRACE(); 1361 SLOCK_DEBUGGER(); 1362 } 1363 } 1364 #endif /* LOCKDEBUG */ /* } */ 1365 1366 #if defined(MULTIPROCESSOR) 1367 /* 1368 * Functions for manipulating the kernel_lock. We put them here 1369 * so that they show up in profiles. 1370 */ 1371 1372 struct lock kernel_lock; 1373 1374 void 1375 _kernel_lock_init(void) 1376 { 1377 1378 spinlockinit(&kernel_lock, "klock", 0); 1379 } 1380 1381 /* 1382 * Acquire/release the kernel lock. Intended for use in the scheduler 1383 * and the lower half of the kernel. 1384 */ 1385 void 1386 _kernel_lock(int flag) 1387 { 1388 1389 SCHED_ASSERT_UNLOCKED(); 1390 spinlockmgr(&kernel_lock, flag, 0); 1391 } 1392 1393 void 1394 _kernel_unlock(void) 1395 { 1396 1397 spinlockmgr(&kernel_lock, LK_RELEASE, 0); 1398 } 1399 1400 /* 1401 * Acquire/release the kernel_lock on behalf of a process. Intended for 1402 * use in the top half of the kernel. 1403 */ 1404 void 1405 _kernel_proc_lock(struct lwp *l) 1406 { 1407 1408 SCHED_ASSERT_UNLOCKED(); 1409 spinlockmgr(&kernel_lock, LK_EXCLUSIVE, 0); 1410 } 1411 1412 void 1413 _kernel_proc_unlock(struct lwp *l) 1414 { 1415 1416 spinlockmgr(&kernel_lock, LK_RELEASE, 0); 1417 } 1418 1419 int 1420 _kernel_lock_release_all() 1421 { 1422 int hold_count; 1423 1424 if (lockstatus(&kernel_lock) == LK_EXCLUSIVE) 1425 hold_count = spinlock_release_all(&kernel_lock); 1426 else 1427 hold_count = 0; 1428 1429 return hold_count; 1430 } 1431 1432 void 1433 _kernel_lock_acquire_count(int hold_count) 1434 { 1435 1436 if (hold_count != 0) 1437 spinlock_acquire_count(&kernel_lock, hold_count); 1438 } 1439 #endif /* MULTIPROCESSOR */ 1440