1 /* $NetBSD: kern_lock.c,v 1.32 2000/06/10 18:44:43 sommerfeld Exp $ */ 2 3 /*- 4 * Copyright (c) 1999, 2000 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * This code is derived from software contributed to The NetBSD Foundation 12 * by Ross Harvey. 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 3. All advertising materials mentioning features or use of this software 23 * must display the following acknowledgement: 24 * This product includes software developed by the NetBSD 25 * Foundation, Inc. and its contributors. 26 * 4. Neither the name of The NetBSD Foundation nor the names of its 27 * contributors may be used to endorse or promote products derived 28 * from this software without specific prior written permission. 29 * 30 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 31 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 32 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 33 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 34 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 35 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 36 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 37 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 38 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 39 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 40 * POSSIBILITY OF SUCH DAMAGE. 41 */ 42 43 /* 44 * Copyright (c) 1995 45 * The Regents of the University of California. All rights reserved. 46 * 47 * This code contains ideas from software contributed to Berkeley by 48 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 49 * System project at Carnegie-Mellon University. 50 * 51 * Redistribution and use in source and binary forms, with or without 52 * modification, are permitted provided that the following conditions 53 * are met: 54 * 1. Redistributions of source code must retain the above copyright 55 * notice, this list of conditions and the following disclaimer. 56 * 2. Redistributions in binary form must reproduce the above copyright 57 * notice, this list of conditions and the following disclaimer in the 58 * documentation and/or other materials provided with the distribution. 59 * 3. All advertising materials mentioning features or use of this software 60 * must display the following acknowledgement: 61 * This product includes software developed by the University of 62 * California, Berkeley and its contributors. 63 * 4. Neither the name of the University nor the names of its contributors 64 * may be used to endorse or promote products derived from this software 65 * without specific prior written permission. 66 * 67 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 68 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 69 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 70 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 71 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 72 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 73 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 74 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 75 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 76 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 77 * SUCH DAMAGE. 78 * 79 * @(#)kern_lock.c 8.18 (Berkeley) 5/21/95 80 */ 81 82 #include "opt_multiprocessor.h" 83 #include "opt_lockdebug.h" 84 #include "opt_ddb.h" 85 86 #include <sys/param.h> 87 #include <sys/proc.h> 88 #include <sys/lock.h> 89 #include <sys/systm.h> 90 #include <machine/cpu.h> 91 92 #if defined(__HAVE_ATOMIC_OPERATIONS) 93 #include <machine/atomic.h> 94 #endif 95 96 #if defined(LOCKDEBUG) 97 #include <sys/syslog.h> 98 /* 99 * note that stdarg.h and the ansi style va_start macro is used for both 100 * ansi and traditional c compiles. 101 * XXX: this requires that stdarg.h define: va_alist and va_dcl 102 */ 103 #include <machine/stdarg.h> 104 105 void lock_printf __P((const char *fmt, ...)); 106 107 int lock_debug_syslog = 0; /* defaults to printf, but can be patched */ 108 #endif 109 110 /* 111 * Locking primitives implementation. 112 * Locks provide shared/exclusive sychronization. 113 */ 114 115 #if defined(LOCKDEBUG) || defined(DIAGNOSTIC) /* { */ 116 #if defined(MULTIPROCESSOR) /* { */ 117 #if defined(__HAVE_ATOMIC_OPERATIONS) /* { */ 118 #define COUNT_CPU(cpu_id, x) \ 119 atomic_add_ulong(&curcpu()->ci_spin_locks, (x)) 120 #else 121 #define COUNT_CPU(cpu_id, x) /* not safe */ 122 #endif /* __HAVE_ATOMIC_OPERATIONS */ /* } */ 123 #else 124 u_long spin_locks; 125 #define COUNT_CPU(cpu_id, x) spin_locks += (x) 126 #endif /* MULTIPROCESSOR */ /* } */ 127 128 #define COUNT(lkp, p, cpu_id, x) \ 129 do { \ 130 if ((lkp)->lk_flags & LK_SPIN) \ 131 COUNT_CPU((cpu_id), (x)); \ 132 else \ 133 (p)->p_locks += (x); \ 134 } while (/*CONSTCOND*/0) 135 #else 136 #define COUNT(lkp, p, cpu_id, x) 137 #endif /* LOCKDEBUG || DIAGNOSTIC */ /* } */ 138 139 /* 140 * Acquire a resource. 141 */ 142 #define ACQUIRE(lkp, error, extflags, drain, wanted) \ 143 if ((extflags) & LK_SPIN) { \ 144 int interlocked; \ 145 \ 146 if ((drain) == 0) \ 147 (lkp)->lk_waitcount++; \ 148 for (interlocked = 1;;) { \ 149 if (wanted) { \ 150 if (interlocked) { \ 151 simple_unlock(&(lkp)->lk_interlock); \ 152 interlocked = 0; \ 153 } \ 154 } else if (interlocked) { \ 155 break; \ 156 } else { \ 157 simple_lock(&(lkp)->lk_interlock); \ 158 interlocked = 1; \ 159 } \ 160 } \ 161 if ((drain) == 0) \ 162 (lkp)->lk_waitcount--; \ 163 KASSERT((wanted) == 0); \ 164 error = 0; /* sanity */ \ 165 } else { \ 166 for (error = 0; wanted; ) { \ 167 if ((drain)) \ 168 (lkp)->lk_flags |= LK_WAITDRAIN; \ 169 else \ 170 (lkp)->lk_waitcount++; \ 171 /* XXX Cast away volatile. */ \ 172 error = ltsleep((drain) ? &(lkp)->lk_flags : \ 173 (void *)(lkp), (lkp)->lk_prio, \ 174 (lkp)->lk_wmesg, (lkp)->lk_timo, \ 175 &(lkp)->lk_interlock); \ 176 if ((drain) == 0) \ 177 (lkp)->lk_waitcount--; \ 178 if (error) \ 179 break; \ 180 if ((extflags) & LK_SLEEPFAIL) { \ 181 error = ENOLCK; \ 182 break; \ 183 } \ 184 } \ 185 } 186 187 #define SETHOLDER(lkp, pid, cpu_id) \ 188 do { \ 189 if ((lkp)->lk_flags & LK_SPIN) \ 190 (lkp)->lk_cpu = cpu_id; \ 191 else \ 192 (lkp)->lk_lockholder = pid; \ 193 } while (/*CONSTCOND*/0) 194 195 #define WEHOLDIT(lkp, pid, cpu_id) \ 196 (((lkp)->lk_flags & LK_SPIN) != 0 ? \ 197 ((lkp)->lk_cpu == (cpu_id)) : ((lkp)->lk_lockholder == (pid))) 198 199 #define WAKEUP_WAITER(lkp) \ 200 do { \ 201 if (((lkp)->lk_flags & LK_SPIN) == 0 && (lkp)->lk_waitcount) { \ 202 /* XXX Cast away volatile. */ \ 203 wakeup_one((void *)(lkp)); \ 204 } \ 205 } while (/*CONSTCOND*/0) 206 207 #if defined(LOCKDEBUG) /* { */ 208 #if defined(MULTIPROCESSOR) /* { */ 209 struct simplelock spinlock_list_slock = SIMPLELOCK_INITIALIZER; 210 211 #define SPINLOCK_LIST_LOCK() \ 212 __cpu_simple_lock(&spinlock_list_slock.lock_data) 213 214 #define SPINLOCK_LIST_UNLOCK() \ 215 __cpu_simple_unlock(&spinlock_list_slock.lock_data) 216 #else 217 #define SPINLOCK_LIST_LOCK() /* nothing */ 218 219 #define SPINLOCK_LIST_UNLOCK() /* nothing */ 220 #endif /* MULTIPROCESSOR */ /* } */ 221 222 TAILQ_HEAD(, lock) spinlock_list = 223 TAILQ_HEAD_INITIALIZER(spinlock_list); 224 225 #define HAVEIT(lkp) \ 226 do { \ 227 if ((lkp)->lk_flags & LK_SPIN) { \ 228 int s = splhigh(); \ 229 SPINLOCK_LIST_LOCK(); \ 230 /* XXX Cast away volatile. */ \ 231 TAILQ_INSERT_TAIL(&spinlock_list, (struct lock *)(lkp), \ 232 lk_list); \ 233 SPINLOCK_LIST_UNLOCK(); \ 234 splx(s); \ 235 } \ 236 } while (/*CONSTCOND*/0) 237 238 #define DONTHAVEIT(lkp) \ 239 do { \ 240 if ((lkp)->lk_flags & LK_SPIN) { \ 241 int s = splhigh(); \ 242 SPINLOCK_LIST_LOCK(); \ 243 /* XXX Cast away volatile. */ \ 244 TAILQ_REMOVE(&spinlock_list, (struct lock *)(lkp), \ 245 lk_list); \ 246 SPINLOCK_LIST_UNLOCK(); \ 247 splx(s); \ 248 } \ 249 } while (/*CONSTCOND*/0) 250 #else 251 #define HAVEIT(lkp) /* nothing */ 252 253 #define DONTHAVEIT(lkp) /* nothing */ 254 #endif /* LOCKDEBUG */ /* } */ 255 256 #if defined(LOCKDEBUG) 257 /* 258 * Lock debug printing routine; can be configured to print to console 259 * or log to syslog. 260 */ 261 void 262 #ifdef __STDC__ 263 lock_printf(const char *fmt, ...) 264 #else 265 lock_printf(fmt, va_alist) 266 char *fmt; 267 va_dcl 268 #endif 269 { 270 va_list ap; 271 272 va_start(ap, fmt); 273 if (lock_debug_syslog) 274 vlog(LOG_DEBUG, fmt, ap); 275 else 276 vprintf(fmt, ap); 277 va_end(ap); 278 } 279 #endif /* LOCKDEBUG */ 280 281 /* 282 * Initialize a lock; required before use. 283 */ 284 void 285 lockinit(lkp, prio, wmesg, timo, flags) 286 struct lock *lkp; 287 int prio; 288 const char *wmesg; 289 int timo; 290 int flags; 291 { 292 293 memset(lkp, 0, sizeof(struct lock)); 294 simple_lock_init(&lkp->lk_interlock); 295 lkp->lk_flags = flags & LK_EXTFLG_MASK; 296 if (flags & LK_SPIN) 297 lkp->lk_cpu = LK_NOCPU; 298 else { 299 lkp->lk_lockholder = LK_NOPROC; 300 lkp->lk_prio = prio; 301 lkp->lk_timo = timo; 302 } 303 lkp->lk_wmesg = wmesg; /* just a name for spin locks */ 304 } 305 306 /* 307 * Determine the status of a lock. 308 */ 309 int 310 lockstatus(lkp) 311 struct lock *lkp; 312 { 313 int lock_type = 0; 314 315 simple_lock(&lkp->lk_interlock); 316 if (lkp->lk_exclusivecount != 0) 317 lock_type = LK_EXCLUSIVE; 318 else if (lkp->lk_sharecount != 0) 319 lock_type = LK_SHARED; 320 simple_unlock(&lkp->lk_interlock); 321 return (lock_type); 322 } 323 324 /* 325 * XXX XXX kludge around another kludge.. 326 * 327 * vfs_shutdown() may be called from interrupt context, either as a result 328 * of a panic, or from the debugger. It proceeds to call 329 * sys_sync(&proc0, ...), pretending its running on behalf of proc0 330 * 331 * We would like to make an attempt to sync the filesystems in this case, so 332 * if this happens, we treat attempts to acquire locks specially. 333 * All locks are acquired on behalf of proc0. 334 * 335 * If we've already paniced, we don't block waiting for locks, but 336 * just barge right ahead since we're already going down in flames. 337 */ 338 339 /* 340 * Set, change, or release a lock. 341 * 342 * Shared requests increment the shared count. Exclusive requests set the 343 * LK_WANT_EXCL flag (preventing further shared locks), and wait for already 344 * accepted shared locks and shared-to-exclusive upgrades to go away. 345 */ 346 int 347 lockmgr(lkp, flags, interlkp) 348 __volatile struct lock *lkp; 349 u_int flags; 350 struct simplelock *interlkp; 351 { 352 int error; 353 pid_t pid; 354 int extflags; 355 cpuid_t cpu_id; 356 struct proc *p = curproc; 357 int lock_shutdown_noblock = 0; 358 359 error = 0; 360 361 simple_lock(&lkp->lk_interlock); 362 if (flags & LK_INTERLOCK) 363 simple_unlock(interlkp); 364 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 365 366 #ifdef DIAGNOSTIC /* { */ 367 /* 368 * Don't allow spins on sleep locks and don't allow sleeps 369 * on spin locks. 370 */ 371 if ((flags ^ lkp->lk_flags) & LK_SPIN) 372 panic("lockmgr: sleep/spin mismatch\n"); 373 #endif /* } */ 374 375 if (extflags & LK_SPIN) 376 pid = LK_KERNPROC; 377 else { 378 if (p == NULL) { 379 if (!doing_shutdown) { 380 #ifdef DIAGNOSTIC 381 panic("lockmgr: no context"); 382 #endif 383 } else { 384 p = &proc0; 385 if (panicstr && (!(flags & LK_NOWAIT))) { 386 flags |= LK_NOWAIT; 387 lock_shutdown_noblock = 1; 388 } 389 } 390 } 391 pid = p->p_pid; 392 } 393 cpu_id = cpu_number(); 394 395 /* 396 * Once a lock has drained, the LK_DRAINING flag is set and an 397 * exclusive lock is returned. The only valid operation thereafter 398 * is a single release of that exclusive lock. This final release 399 * clears the LK_DRAINING flag and sets the LK_DRAINED flag. Any 400 * further requests of any sort will result in a panic. The bits 401 * selected for these two flags are chosen so that they will be set 402 * in memory that is freed (freed memory is filled with 0xdeadbeef). 403 * The final release is permitted to give a new lease on life to 404 * the lock by specifying LK_REENABLE. 405 */ 406 if (lkp->lk_flags & (LK_DRAINING|LK_DRAINED)) { 407 #ifdef DIAGNOSTIC /* { */ 408 if (lkp->lk_flags & LK_DRAINED) 409 panic("lockmgr: using decommissioned lock"); 410 if ((flags & LK_TYPE_MASK) != LK_RELEASE || 411 WEHOLDIT(lkp, pid, cpu_id) == 0) 412 panic("lockmgr: non-release on draining lock: %d\n", 413 flags & LK_TYPE_MASK); 414 #endif /* DIAGNOSTIC */ /* } */ 415 lkp->lk_flags &= ~LK_DRAINING; 416 if ((flags & LK_REENABLE) == 0) 417 lkp->lk_flags |= LK_DRAINED; 418 } 419 420 switch (flags & LK_TYPE_MASK) { 421 422 case LK_SHARED: 423 if (WEHOLDIT(lkp, pid, cpu_id) == 0) { 424 /* 425 * If just polling, check to see if we will block. 426 */ 427 if ((extflags & LK_NOWAIT) && (lkp->lk_flags & 428 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE))) { 429 error = EBUSY; 430 break; 431 } 432 /* 433 * Wait for exclusive locks and upgrades to clear. 434 */ 435 ACQUIRE(lkp, error, extflags, 0, lkp->lk_flags & 436 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)); 437 if (error) 438 break; 439 lkp->lk_sharecount++; 440 COUNT(lkp, p, cpu_id, 1); 441 break; 442 } 443 /* 444 * We hold an exclusive lock, so downgrade it to shared. 445 * An alternative would be to fail with EDEADLK. 446 */ 447 lkp->lk_sharecount++; 448 COUNT(lkp, p, cpu_id, 1); 449 /* fall into downgrade */ 450 451 case LK_DOWNGRADE: 452 if (WEHOLDIT(lkp, pid, cpu_id) == 0 || 453 lkp->lk_exclusivecount == 0) 454 panic("lockmgr: not holding exclusive lock"); 455 lkp->lk_sharecount += lkp->lk_exclusivecount; 456 lkp->lk_exclusivecount = 0; 457 lkp->lk_recurselevel = 0; 458 lkp->lk_flags &= ~LK_HAVE_EXCL; 459 SETHOLDER(lkp, LK_NOPROC, LK_NOCPU); 460 DONTHAVEIT(lkp); 461 WAKEUP_WAITER(lkp); 462 break; 463 464 case LK_EXCLUPGRADE: 465 /* 466 * If another process is ahead of us to get an upgrade, 467 * then we want to fail rather than have an intervening 468 * exclusive access. 469 */ 470 if (lkp->lk_flags & LK_WANT_UPGRADE) { 471 lkp->lk_sharecount--; 472 COUNT(lkp, p, cpu_id, -1); 473 error = EBUSY; 474 break; 475 } 476 /* fall into normal upgrade */ 477 478 case LK_UPGRADE: 479 /* 480 * Upgrade a shared lock to an exclusive one. If another 481 * shared lock has already requested an upgrade to an 482 * exclusive lock, our shared lock is released and an 483 * exclusive lock is requested (which will be granted 484 * after the upgrade). If we return an error, the file 485 * will always be unlocked. 486 */ 487 if (WEHOLDIT(lkp, pid, cpu_id) || lkp->lk_sharecount <= 0) 488 panic("lockmgr: upgrade exclusive lock"); 489 lkp->lk_sharecount--; 490 COUNT(lkp, p, cpu_id, -1); 491 /* 492 * If we are just polling, check to see if we will block. 493 */ 494 if ((extflags & LK_NOWAIT) && 495 ((lkp->lk_flags & LK_WANT_UPGRADE) || 496 lkp->lk_sharecount > 1)) { 497 error = EBUSY; 498 break; 499 } 500 if ((lkp->lk_flags & LK_WANT_UPGRADE) == 0) { 501 /* 502 * We are first shared lock to request an upgrade, so 503 * request upgrade and wait for the shared count to 504 * drop to zero, then take exclusive lock. 505 */ 506 lkp->lk_flags |= LK_WANT_UPGRADE; 507 ACQUIRE(lkp, error, extflags, 0, lkp->lk_sharecount); 508 lkp->lk_flags &= ~LK_WANT_UPGRADE; 509 if (error) 510 break; 511 lkp->lk_flags |= LK_HAVE_EXCL; 512 SETHOLDER(lkp, pid, cpu_id); 513 HAVEIT(lkp); 514 if (lkp->lk_exclusivecount != 0) 515 panic("lockmgr: non-zero exclusive count"); 516 lkp->lk_exclusivecount = 1; 517 if (extflags & LK_SETRECURSE) 518 lkp->lk_recurselevel = 1; 519 COUNT(lkp, p, cpu_id, 1); 520 break; 521 } 522 /* 523 * Someone else has requested upgrade. Release our shared 524 * lock, awaken upgrade requestor if we are the last shared 525 * lock, then request an exclusive lock. 526 */ 527 if (lkp->lk_sharecount == 0) 528 WAKEUP_WAITER(lkp); 529 /* fall into exclusive request */ 530 531 case LK_EXCLUSIVE: 532 if (WEHOLDIT(lkp, pid, cpu_id)) { 533 /* 534 * Recursive lock. 535 */ 536 if ((extflags & LK_CANRECURSE) == 0 && 537 lkp->lk_recurselevel == 0) { 538 if (extflags & LK_RECURSEFAIL) { 539 error = EDEADLK; 540 break; 541 } else 542 panic("lockmgr: locking against myself"); 543 } 544 lkp->lk_exclusivecount++; 545 if (extflags & LK_SETRECURSE && 546 lkp->lk_recurselevel == 0) 547 lkp->lk_recurselevel = lkp->lk_exclusivecount; 548 COUNT(lkp, p, cpu_id, 1); 549 break; 550 } 551 /* 552 * If we are just polling, check to see if we will sleep. 553 */ 554 if ((extflags & LK_NOWAIT) && ((lkp->lk_flags & 555 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 556 lkp->lk_sharecount != 0)) { 557 error = EBUSY; 558 break; 559 } 560 /* 561 * Try to acquire the want_exclusive flag. 562 */ 563 ACQUIRE(lkp, error, extflags, 0, lkp->lk_flags & 564 (LK_HAVE_EXCL | LK_WANT_EXCL)); 565 if (error) 566 break; 567 lkp->lk_flags |= LK_WANT_EXCL; 568 /* 569 * Wait for shared locks and upgrades to finish. 570 */ 571 ACQUIRE(lkp, error, extflags, 0, lkp->lk_sharecount != 0 || 572 (lkp->lk_flags & LK_WANT_UPGRADE)); 573 lkp->lk_flags &= ~LK_WANT_EXCL; 574 if (error) 575 break; 576 lkp->lk_flags |= LK_HAVE_EXCL; 577 SETHOLDER(lkp, pid, cpu_id); 578 HAVEIT(lkp); 579 if (lkp->lk_exclusivecount != 0) 580 panic("lockmgr: non-zero exclusive count"); 581 lkp->lk_exclusivecount = 1; 582 if (extflags & LK_SETRECURSE) 583 lkp->lk_recurselevel = 1; 584 COUNT(lkp, p, cpu_id, 1); 585 break; 586 587 case LK_RELEASE: 588 if (lkp->lk_exclusivecount != 0) { 589 if (WEHOLDIT(lkp, pid, cpu_id) == 0) { 590 if (lkp->lk_flags & LK_SPIN) { 591 panic("lockmgr: processor %lu, not " 592 "exclusive lock holder %lu " 593 "unlocking", cpu_id, lkp->lk_cpu); 594 } else { 595 panic("lockmgr: pid %d, not " 596 "exclusive lock holder %d " 597 "unlocking", pid, 598 lkp->lk_lockholder); 599 } 600 } 601 if (lkp->lk_exclusivecount == lkp->lk_recurselevel) 602 lkp->lk_recurselevel = 0; 603 lkp->lk_exclusivecount--; 604 COUNT(lkp, p, cpu_id, -1); 605 if (lkp->lk_exclusivecount == 0) { 606 lkp->lk_flags &= ~LK_HAVE_EXCL; 607 SETHOLDER(lkp, LK_NOPROC, LK_NOCPU); 608 DONTHAVEIT(lkp); 609 } 610 } else if (lkp->lk_sharecount != 0) { 611 lkp->lk_sharecount--; 612 COUNT(lkp, p, cpu_id, -1); 613 } 614 WAKEUP_WAITER(lkp); 615 break; 616 617 case LK_DRAIN: 618 /* 619 * Check that we do not already hold the lock, as it can 620 * never drain if we do. Unfortunately, we have no way to 621 * check for holding a shared lock, but at least we can 622 * check for an exclusive one. 623 */ 624 if (WEHOLDIT(lkp, pid, cpu_id)) 625 panic("lockmgr: draining against myself"); 626 /* 627 * If we are just polling, check to see if we will sleep. 628 */ 629 if ((extflags & LK_NOWAIT) && ((lkp->lk_flags & 630 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 631 lkp->lk_sharecount != 0 || lkp->lk_waitcount != 0)) { 632 error = EBUSY; 633 break; 634 } 635 ACQUIRE(lkp, error, extflags, 1, 636 ((lkp->lk_flags & 637 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) || 638 lkp->lk_sharecount != 0 || 639 lkp->lk_waitcount != 0)); 640 if (error) 641 break; 642 lkp->lk_flags |= LK_DRAINING | LK_HAVE_EXCL; 643 SETHOLDER(lkp, pid, cpu_id); 644 HAVEIT(lkp); 645 lkp->lk_exclusivecount = 1; 646 /* XXX unlikely that we'd want this */ 647 if (extflags & LK_SETRECURSE) 648 lkp->lk_recurselevel = 1; 649 COUNT(lkp, p, cpu_id, 1); 650 break; 651 652 default: 653 simple_unlock(&lkp->lk_interlock); 654 panic("lockmgr: unknown locktype request %d", 655 flags & LK_TYPE_MASK); 656 /* NOTREACHED */ 657 } 658 if ((lkp->lk_flags & (LK_WAITDRAIN|LK_SPIN)) == LK_WAITDRAIN && 659 ((lkp->lk_flags & 660 (LK_HAVE_EXCL | LK_WANT_EXCL | LK_WANT_UPGRADE)) == 0 && 661 lkp->lk_sharecount == 0 && lkp->lk_waitcount == 0)) { 662 lkp->lk_flags &= ~LK_WAITDRAIN; 663 wakeup_one((void *)&lkp->lk_flags); 664 } 665 /* 666 * Note that this panic will be a recursive panic, since 667 * we only set lock_shutdown_noblock above if panicstr != NULL. 668 */ 669 if (error && lock_shutdown_noblock) 670 panic("lockmgr: deadlock (see previous panic)"); 671 672 simple_unlock(&lkp->lk_interlock); 673 return (error); 674 } 675 676 /* 677 * Print out information about state of a lock. Used by VOP_PRINT 678 * routines to display ststus about contained locks. 679 */ 680 void 681 lockmgr_printinfo(lkp) 682 __volatile struct lock *lkp; 683 { 684 685 if (lkp->lk_sharecount) 686 printf(" lock type %s: SHARED (count %d)", lkp->lk_wmesg, 687 lkp->lk_sharecount); 688 else if (lkp->lk_flags & LK_HAVE_EXCL) { 689 printf(" lock type %s: EXCL (count %d) by ", 690 lkp->lk_wmesg, lkp->lk_exclusivecount); 691 if (lkp->lk_flags & LK_SPIN) 692 printf("processor %lu", lkp->lk_cpu); 693 else 694 printf("pid %d", lkp->lk_lockholder); 695 } else 696 printf(" not locked"); 697 if ((lkp->lk_flags & LK_SPIN) == 0 && lkp->lk_waitcount > 0) 698 printf(" with %d pending", lkp->lk_waitcount); 699 } 700 701 #if defined(LOCKDEBUG) /* { */ 702 TAILQ_HEAD(, simplelock) simplelock_list = 703 TAILQ_HEAD_INITIALIZER(simplelock_list); 704 705 #if defined(MULTIPROCESSOR) /* { */ 706 struct simplelock simplelock_list_slock = SIMPLELOCK_INITIALIZER; 707 708 #define SLOCK_LIST_LOCK() \ 709 __cpu_simple_lock(&simplelock_list_slock.lock_data) 710 711 #define SLOCK_LIST_UNLOCK() \ 712 __cpu_simple_unlock(&simplelock_list_slock.lock_data) 713 714 #if defined(__HAVE_ATOMIC_OPERATIONS) /* { */ 715 #define SLOCK_COUNT(x) \ 716 atomic_add_ulong(&curcpu()->ci_simple_locks, (x)) 717 #else 718 #define SLOCK_COUNT(x) /* not safe */ 719 #endif /* __HAVE_ATOMIC_OPERATIONS */ /* } */ 720 #else 721 u_long simple_locks; 722 723 #define SLOCK_LIST_LOCK() /* nothing */ 724 725 #define SLOCK_LIST_UNLOCK() /* nothing */ 726 727 #define SLOCK_COUNT(x) simple_locks += (x) 728 #endif /* MULTIPROCESSOR */ /* } */ 729 730 #ifdef DDB /* { */ 731 int simple_lock_debugger = 0; 732 #define SLOCK_DEBUGGER() if (simple_lock_debugger) Debugger() 733 #else 734 #define SLOCK_DEBUGGER() /* nothing */ 735 #endif /* } */ 736 737 #ifdef MULTIPROCESSOR 738 #define SLOCK_MP() lock_printf("on cpu %d\n", cpu_number()) 739 #else 740 #define SLOCK_MP() /* nothing */ 741 #endif 742 743 #define SLOCK_WHERE(str, alp, id, l) \ 744 do { \ 745 lock_printf(str); \ 746 lock_printf("lock: %p, currently at: %s:%d\n", (alp), (id), (l)); \ 747 SLOCK_MP(); \ 748 if ((alp)->lock_file != NULL) \ 749 lock_printf("last locked: %s:%d\n", (alp)->lock_file, \ 750 (alp)->lock_line); \ 751 if ((alp)->unlock_file != NULL) \ 752 lock_printf("last unlocked: %s:%d\n", (alp)->unlock_file, \ 753 (alp)->unlock_line); \ 754 SLOCK_DEBUGGER(); \ 755 } while (/*CONSTCOND*/0) 756 757 /* 758 * Simple lock functions so that the debugger can see from whence 759 * they are being called. 760 */ 761 void 762 simple_lock_init(alp) 763 struct simplelock *alp; 764 { 765 766 #if defined(MULTIPROCESSOR) /* { */ 767 __cpu_simple_lock_init(&alp->lock_data); 768 #else 769 alp->lock_data = __SIMPLELOCK_UNLOCKED; 770 #endif /* } */ 771 alp->lock_file = NULL; 772 alp->lock_line = 0; 773 alp->unlock_file = NULL; 774 alp->unlock_line = 0; 775 alp->lock_holder = 0; 776 } 777 778 void 779 _simple_lock(alp, id, l) 780 __volatile struct simplelock *alp; 781 const char *id; 782 int l; 783 { 784 cpuid_t cpu_id = cpu_number(); 785 int s; 786 787 s = splhigh(); 788 789 /* 790 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 791 * don't take any action, and just fall into the normal spin case. 792 */ 793 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 794 #if defined(MULTIPROCESSOR) /* { */ 795 if (alp->lock_holder == cpu_id) { 796 SLOCK_WHERE("simple_lock: locking against myself\n", 797 alp, id, l); 798 goto out; 799 } 800 #else 801 SLOCK_WHERE("simple_lock: lock held\n", alp, id, l); 802 goto out; 803 #endif /* MULTIPROCESSOR */ /* } */ 804 } 805 806 #if defined(MULTIPROCESSOR) /* { */ 807 /* Acquire the lock before modifying any fields. */ 808 __cpu_simple_lock(&alp->lock_data); 809 #else 810 alp->lock_data = __SIMPLELOCK_LOCKED; 811 #endif /* } */ 812 813 alp->lock_file = id; 814 alp->lock_line = l; 815 alp->lock_holder = cpu_id; 816 817 SLOCK_LIST_LOCK(); 818 /* XXX Cast away volatile */ 819 TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list); 820 SLOCK_LIST_UNLOCK(); 821 822 SLOCK_COUNT(1); 823 824 out: 825 splx(s); 826 } 827 828 int 829 _simple_lock_try(alp, id, l) 830 __volatile struct simplelock *alp; 831 const char *id; 832 int l; 833 { 834 cpuid_t cpu_id = cpu_number(); 835 int s, rv = 0; 836 837 s = splhigh(); 838 839 /* 840 * MULTIPROCESSOR case: This is `safe' since if it's not us, we 841 * don't take any action. 842 */ 843 #if defined(MULTIPROCESSOR) /* { */ 844 if ((rv = __cpu_simple_lock_try(&alp->lock_data)) == 0) { 845 if (alp->lock_holder == cpu_id) 846 SLOCK_WHERE("simple_lock_try: locking against myself\n", 847 alp, id, l); 848 goto out; 849 } 850 #else 851 if (alp->lock_data == __SIMPLELOCK_LOCKED) { 852 SLOCK_WHERE("simple_lock_try: lock held\n", alp, id, l); 853 goto out; 854 } 855 alp->lock_data = __SIMPLELOCK_LOCKED; 856 #endif /* MULTIPROCESSOR */ /* } */ 857 858 /* 859 * At this point, we have acquired the lock. 860 */ 861 862 rv = 1; 863 864 alp->lock_file = id; 865 alp->lock_line = l; 866 alp->lock_holder = cpu_id; 867 868 SLOCK_LIST_LOCK(); 869 /* XXX Cast away volatile. */ 870 TAILQ_INSERT_TAIL(&simplelock_list, (struct simplelock *)alp, list); 871 SLOCK_LIST_UNLOCK(); 872 873 SLOCK_COUNT(1); 874 875 out: 876 splx(s); 877 return (rv); 878 } 879 880 void 881 _simple_unlock(alp, id, l) 882 __volatile struct simplelock *alp; 883 const char *id; 884 int l; 885 { 886 int s; 887 888 s = splhigh(); 889 890 /* 891 * MULTIPROCESSOR case: This is `safe' because we think we hold 892 * the lock, and if we don't, we don't take any action. 893 */ 894 if (alp->lock_data == __SIMPLELOCK_UNLOCKED) { 895 SLOCK_WHERE("simple_unlock: lock not held\n", 896 alp, id, l); 897 goto out; 898 } 899 900 SLOCK_LIST_LOCK(); 901 TAILQ_REMOVE(&simplelock_list, alp, list); 902 SLOCK_LIST_UNLOCK(); 903 904 SLOCK_COUNT(-1); 905 906 alp->list.tqe_next = NULL; /* sanity */ 907 alp->list.tqe_prev = NULL; /* sanity */ 908 909 alp->unlock_file = id; 910 alp->unlock_line = l; 911 912 #if defined(MULTIPROCESSOR) /* { */ 913 alp->lock_holder = LK_NOCPU; 914 /* Now that we've modified all fields, release the lock. */ 915 __cpu_simple_unlock(&alp->lock_data); 916 #else 917 alp->lock_data = __SIMPLELOCK_UNLOCKED; 918 #endif /* } */ 919 920 out: 921 splx(s); 922 } 923 924 void 925 simple_lock_dump() 926 { 927 struct simplelock *alp; 928 int s; 929 930 s = splhigh(); 931 SLOCK_LIST_LOCK(); 932 lock_printf("all simple locks:\n"); 933 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 934 alp = TAILQ_NEXT(alp, list)) { 935 lock_printf("%p CPU %lu %s:%d\n", alp, alp->lock_holder, 936 alp->lock_file, alp->lock_line); 937 } 938 SLOCK_LIST_UNLOCK(); 939 splx(s); 940 } 941 942 void 943 simple_lock_freecheck(start, end) 944 void *start, *end; 945 { 946 struct simplelock *alp; 947 int s; 948 949 s = splhigh(); 950 SLOCK_LIST_LOCK(); 951 for (alp = TAILQ_FIRST(&simplelock_list); alp != NULL; 952 alp = TAILQ_NEXT(alp, list)) { 953 if ((void *)alp >= start && (void *)alp < end) { 954 lock_printf("freeing simple_lock %p CPU %lu %s:%d\n", 955 alp, alp->lock_holder, alp->lock_file, 956 alp->lock_line); 957 SLOCK_DEBUGGER(); 958 } 959 } 960 SLOCK_LIST_UNLOCK(); 961 splx(s); 962 } 963 #endif /* LOCKDEBUG */ /* } */ 964