1 /* 2 * Copyright (c) 1995 3 * The Regents of the University of California. All rights reserved. 4 * Copyright (C) 1997 5 * John S. Dyson. All rights reserved. 6 * Copyright (C) 2013-2014 7 * Matthew Dillon, All rights reserved. 8 * 9 * This code contains ideas from software contributed to Berkeley by 10 * Avadis Tevanian, Jr., Michael Wayne Young, and the Mach Operating 11 * System project at Carnegie-Mellon University. 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 3. Neither the name of the University nor the names of its contributors 22 * may be used to endorse or promote products derived from this software 23 * without specific prior written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 35 * SUCH DAMAGE. 36 */ 37 38 #include "opt_lint.h" 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/kernel.h> 43 #include <sys/proc.h> 44 #include <sys/lock.h> 45 #include <sys/sysctl.h> 46 #include <sys/spinlock.h> 47 #include <sys/thread2.h> 48 #include <sys/spinlock2.h> 49 50 static void undo_upreq(struct lock *lkp); 51 52 #ifdef DEBUG_CANCEL_LOCKS 53 54 static int sysctl_cancel_lock(SYSCTL_HANDLER_ARGS); 55 static int sysctl_cancel_test(SYSCTL_HANDLER_ARGS); 56 57 static struct lock cancel_lk; 58 LOCK_SYSINIT(cancellk, &cancel_lk, "cancel", 0); 59 SYSCTL_PROC(_kern, OID_AUTO, cancel_lock, CTLTYPE_INT|CTLFLAG_RW, 0, 0, 60 sysctl_cancel_lock, "I", "test cancelable locks"); 61 SYSCTL_PROC(_kern, OID_AUTO, cancel_test, CTLTYPE_INT|CTLFLAG_RW, 0, 0, 62 sysctl_cancel_test, "I", "test cancelable locks"); 63 64 #endif 65 66 /* 67 * Locking primitives implementation. 68 * Locks provide shared/exclusive sychronization. 69 */ 70 71 #ifdef DEBUG_LOCKS 72 #define COUNT(td, x) (td)->td_locks += (x) 73 #else 74 #define COUNT(td, x) 75 #endif 76 77 /* 78 * Set, change, or release a lock. 79 */ 80 int 81 #ifndef DEBUG_LOCKS 82 lockmgr(struct lock *lkp, u_int flags) 83 #else 84 debuglockmgr(struct lock *lkp, u_int flags, 85 const char *name, const char *file, int line) 86 #endif 87 { 88 thread_t td; 89 thread_t otd; 90 int error; 91 int extflags; 92 int count; 93 int pflags; 94 int wflags; 95 int timo; 96 #ifdef DEBUG_LOCKS 97 int i; 98 #endif 99 100 error = 0; 101 102 if (mycpu->gd_intr_nesting_level && 103 (flags & LK_NOWAIT) == 0 && 104 (flags & LK_TYPE_MASK) != LK_RELEASE && 105 panic_cpu_gd != mycpu 106 ) { 107 108 #ifndef DEBUG_LOCKS 109 panic("lockmgr %s from %p: called from interrupt, ipi, " 110 "or hard code section", 111 lkp->lk_wmesg, ((int **)&lkp)[-1]); 112 #else 113 panic("lockmgr %s from %s:%d: called from interrupt, ipi, " 114 "or hard code section", 115 lkp->lk_wmesg, file, line); 116 #endif 117 } 118 119 #ifdef DEBUG_LOCKS 120 if (mycpu->gd_spinlocks && ((flags & LK_NOWAIT) == 0)) { 121 panic("lockmgr %s from %s:%d: called with %d spinlocks held", 122 lkp->lk_wmesg, file, line, mycpu->gd_spinlocks); 123 } 124 #endif 125 126 extflags = (flags | lkp->lk_flags) & LK_EXTFLG_MASK; 127 td = curthread; 128 129 again: 130 count = lkp->lk_count; 131 cpu_ccfence(); 132 133 switch (flags & LK_TYPE_MASK) { 134 case LK_SHARED: 135 /* 136 * Shared lock critical path case 137 */ 138 if ((count & (LKC_EXREQ|LKC_UPREQ|LKC_EXCL)) == 0) { 139 if (atomic_cmpset_int(&lkp->lk_count, 140 count, count + 1)) { 141 COUNT(td, 1); 142 break; 143 } 144 goto again; 145 } 146 147 /* 148 * If the caller already holds the lock exclusively then 149 * we silently obtain another count on the exclusive lock. 150 * 151 * WARNING! The old FreeBSD behavior was to downgrade, 152 * but this creates a problem when recursions 153 * return to the caller and the caller expects 154 * its original exclusive lock to remain exclusively 155 * locked. 156 */ 157 if (lkp->lk_lockholder == td) { 158 KKASSERT(count & LKC_EXCL); 159 if ((extflags & LK_CANRECURSE) == 0) { 160 if (extflags & LK_NOWAIT) { 161 error = EBUSY; 162 break; 163 } 164 panic("lockmgr: locking against myself"); 165 } 166 atomic_add_int(&lkp->lk_count, 1); 167 COUNT(td, 1); 168 break; 169 } 170 171 /* 172 * Slow path 173 */ 174 pflags = (extflags & LK_PCATCH) ? PCATCH : 0; 175 timo = (extflags & LK_TIMELOCK) ? lkp->lk_timo : 0; 176 wflags = (td->td_flags & TDF_DEADLKTREAT) ? 177 LKC_EXCL : (LKC_EXCL|LKC_EXREQ|LKC_UPREQ); 178 179 /* 180 * Block while the lock is held exclusively or, conditionally, 181 * if other threads are tring to obtain an exclusive lock or 182 * upgrade to one. 183 */ 184 if (count & wflags) { 185 if (extflags & LK_CANCELABLE) { 186 if (count & LKC_CANCEL) { 187 error = ENOLCK; 188 break; 189 } 190 } 191 if (extflags & LK_NOWAIT) { 192 error = EBUSY; 193 break; 194 } 195 tsleep_interlock(lkp, pflags); 196 if (!atomic_cmpset_int(&lkp->lk_count, count, 197 count | LKC_SHREQ)) { 198 goto again; 199 } 200 201 mycpu->gd_cnt.v_lock_name[0] = 'S'; 202 strncpy(mycpu->gd_cnt.v_lock_name + 1, 203 lkp->lk_wmesg, 204 sizeof(mycpu->gd_cnt.v_lock_name) - 2); 205 ++mycpu->gd_cnt.v_lock_colls; 206 207 error = tsleep(lkp, pflags | PINTERLOCKED, 208 lkp->lk_wmesg, timo); 209 if (error) 210 break; 211 if (extflags & LK_SLEEPFAIL) { 212 error = ENOLCK; 213 break; 214 } 215 goto again; 216 } 217 218 /* 219 * Otherwise we can bump the count 220 */ 221 if (atomic_cmpset_int(&lkp->lk_count, count, count + 1)) { 222 COUNT(td, 1); 223 break; 224 } 225 goto again; 226 227 case LK_EXCLUSIVE: 228 /* 229 * Exclusive lock critical path. 230 */ 231 if (count == 0) { 232 if (atomic_cmpset_int(&lkp->lk_count, count, 233 LKC_EXCL | (count + 1))) { 234 lkp->lk_lockholder = td; 235 COUNT(td, 1); 236 break; 237 } 238 goto again; 239 } 240 241 /* 242 * Recursive lock if we already hold it exclusively. 243 */ 244 if (lkp->lk_lockholder == td) { 245 KKASSERT(count & LKC_EXCL); 246 if ((extflags & LK_CANRECURSE) == 0) { 247 if (extflags & LK_NOWAIT) { 248 error = EBUSY; 249 break; 250 } 251 panic("lockmgr: locking against myself"); 252 } 253 atomic_add_int(&lkp->lk_count, 1); 254 COUNT(td, 1); 255 break; 256 } 257 258 /* 259 * We will block, handle LK_NOWAIT 260 */ 261 if (extflags & LK_NOWAIT) { 262 error = EBUSY; 263 break; 264 } 265 if (extflags & LK_CANCELABLE) { 266 if (count & LKC_CANCEL) { 267 error = ENOLCK; 268 break; 269 } 270 } 271 272 /* 273 * Wait until we can obtain the exclusive lock. EXREQ is 274 * automatically cleared when all current holders release 275 * so if we abort the operation we can safely leave it set. 276 * There might be other exclusive requesters. 277 */ 278 pflags = (extflags & LK_PCATCH) ? PCATCH : 0; 279 timo = (extflags & LK_TIMELOCK) ? lkp->lk_timo : 0; 280 281 tsleep_interlock(lkp, pflags); 282 if (!atomic_cmpset_int(&lkp->lk_count, count, 283 count | LKC_EXREQ)) { 284 goto again; 285 } 286 287 mycpu->gd_cnt.v_lock_name[0] = 'X'; 288 strncpy(mycpu->gd_cnt.v_lock_name + 1, 289 lkp->lk_wmesg, 290 sizeof(mycpu->gd_cnt.v_lock_name) - 2); 291 ++mycpu->gd_cnt.v_lock_colls; 292 293 error = tsleep(lkp, pflags | PINTERLOCKED, 294 lkp->lk_wmesg, timo); 295 if (error) 296 break; 297 if (extflags & LK_SLEEPFAIL) { 298 error = ENOLCK; 299 break; 300 } 301 goto again; 302 303 case LK_DOWNGRADE: 304 /* 305 * Downgrade an exclusive lock into a shared lock. All 306 * counts on a recursive exclusive lock become shared. 307 * 308 * This function always succeeds. 309 */ 310 if (lkp->lk_lockholder != td || 311 (count & (LKC_EXCL|LKC_MASK)) != (LKC_EXCL|1)) { 312 panic("lockmgr: not holding exclusive lock"); 313 } 314 315 #ifdef DEBUG_LOCKS 316 for (i = 0; i < LOCKMGR_DEBUG_ARRAY_SIZE; i++) { 317 if (td->td_lockmgr_stack[i] == lkp && 318 td->td_lockmgr_stack_id[i] > 0 319 ) { 320 td->td_lockmgr_stack_id[i]--; 321 break; 322 } 323 } 324 #endif 325 /* 326 * NOTE! Must NULL-out lockholder before releasing LKC_EXCL. 327 */ 328 otd = lkp->lk_lockholder; 329 lkp->lk_lockholder = NULL; 330 if (atomic_cmpset_int(&lkp->lk_count, count, 331 count & ~(LKC_EXCL|LKC_SHREQ))) { 332 if (count & LKC_SHREQ) 333 wakeup(lkp); 334 break; 335 } 336 lkp->lk_lockholder = otd; 337 goto again; 338 339 case LK_EXCLUPGRADE: 340 /* 341 * Upgrade from a single shared lock to an exclusive lock. 342 * 343 * If another process is ahead of us to get an upgrade, 344 * then we want to fail rather than have an intervening 345 * exclusive access. The shared lock is released on 346 * failure. 347 */ 348 if (count & LKC_UPREQ) { 349 flags = LK_RELEASE; 350 error = EBUSY; 351 goto again; 352 } 353 /* fall through into normal upgrade */ 354 355 case LK_UPGRADE: 356 /* 357 * Upgrade a shared lock to an exclusive one. This can cause 358 * the lock to be temporarily released and stolen by other 359 * threads. LK_SLEEPFAIL or LK_NOWAIT may be used to detect 360 * this case, or use LK_EXCLUPGRADE. 361 * 362 * If the lock is already exclusively owned by us, this 363 * operation is a NOP. 364 * 365 * If we return an error (even NOWAIT), the current lock will 366 * be released. 367 * 368 * Start with the critical path. 369 */ 370 if ((count & (LKC_UPREQ|LKC_EXCL|LKC_MASK)) == 1) { 371 if (atomic_cmpset_int(&lkp->lk_count, count, 372 count | LKC_EXCL)) { 373 lkp->lk_lockholder = td; 374 break; 375 } 376 goto again; 377 } 378 379 /* 380 * If we already hold the lock exclusively this operation 381 * succeeds and is a NOP. 382 */ 383 if (count & LKC_EXCL) { 384 if (lkp->lk_lockholder == td) 385 break; 386 panic("lockmgr: upgrade unowned lock"); 387 } 388 if ((count & LKC_MASK) == 0) 389 panic("lockmgr: upgrade unowned lock"); 390 391 /* 392 * We cannot upgrade without blocking at this point. 393 */ 394 if (extflags & LK_NOWAIT) { 395 flags = LK_RELEASE; 396 error = EBUSY; 397 goto again; 398 } 399 if (extflags & LK_CANCELABLE) { 400 if (count & LKC_CANCEL) { 401 error = ENOLCK; 402 break; 403 } 404 } 405 406 /* 407 * Release the shared lock and request the upgrade. 408 */ 409 pflags = (extflags & LK_PCATCH) ? PCATCH : 0; 410 timo = (extflags & LK_TIMELOCK) ? lkp->lk_timo : 0; 411 tsleep_interlock(lkp, pflags); 412 wflags = (count & LKC_UPREQ) ? LKC_EXREQ : LKC_UPREQ; 413 414 /* 415 * If someone else owns UPREQ and this transition would 416 * allow it to be granted, we have to grant it. Otherwise 417 * we release the shared lock. 418 */ 419 if ((count & (LKC_UPREQ|LKC_MASK)) == (LKC_UPREQ | 1)) { 420 wflags |= LKC_EXCL | LKC_UPGRANT; 421 wflags |= count; 422 wflags &= ~LKC_UPREQ; 423 } else { 424 wflags |= (count - 1); 425 } 426 427 if (atomic_cmpset_int(&lkp->lk_count, count, wflags)) { 428 COUNT(td, -1); 429 430 /* 431 * Must wakeup the thread granted the upgrade. 432 */ 433 if ((count & (LKC_UPREQ|LKC_MASK)) == (LKC_UPREQ | 1)) 434 wakeup(lkp); 435 436 mycpu->gd_cnt.v_lock_name[0] = 'U'; 437 strncpy(mycpu->gd_cnt.v_lock_name + 1, 438 lkp->lk_wmesg, 439 sizeof(mycpu->gd_cnt.v_lock_name) - 2); 440 ++mycpu->gd_cnt.v_lock_colls; 441 442 error = tsleep(lkp, pflags | PINTERLOCKED, 443 lkp->lk_wmesg, timo); 444 if (error) 445 break; 446 if (extflags & LK_SLEEPFAIL) { 447 error = ENOLCK; 448 break; 449 } 450 451 /* 452 * Refactor to either LK_EXCLUSIVE or LK_WAITUPGRADE, 453 * depending on whether we were able to acquire the 454 * LKC_UPREQ bit. 455 */ 456 if (count & LKC_UPREQ) 457 flags = LK_EXCLUSIVE; /* someone else */ 458 else 459 flags = LK_WAITUPGRADE; /* we own the bit */ 460 } 461 goto again; 462 463 case LK_WAITUPGRADE: 464 /* 465 * We own the LKC_UPREQ bit, wait until we are granted the 466 * exclusive lock (LKC_UPGRANT is set). 467 * 468 * IF THE OPERATION FAILS (tsleep error tsleep+LK_SLEEPFAIL), 469 * we have to undo the upgrade request and clean up any lock 470 * that might have been granted via a race. 471 */ 472 if (count & LKC_UPGRANT) { 473 if (atomic_cmpset_int(&lkp->lk_count, count, 474 count & ~LKC_UPGRANT)) { 475 lkp->lk_lockholder = td; 476 KKASSERT(count & LKC_EXCL); 477 break; 478 } 479 /* retry */ 480 } else if ((count & LKC_CANCEL) && (extflags & LK_CANCELABLE)) { 481 undo_upreq(lkp); 482 error = ENOLCK; 483 break; 484 } else { 485 pflags = (extflags & LK_PCATCH) ? PCATCH : 0; 486 timo = (extflags & LK_TIMELOCK) ? lkp->lk_timo : 0; 487 tsleep_interlock(lkp, pflags); 488 if (atomic_cmpset_int(&lkp->lk_count, count, count)) { 489 490 mycpu->gd_cnt.v_lock_name[0] = 'U'; 491 strncpy(mycpu->gd_cnt.v_lock_name + 1, 492 lkp->lk_wmesg, 493 sizeof(mycpu->gd_cnt.v_lock_name) - 2); 494 ++mycpu->gd_cnt.v_lock_colls; 495 496 error = tsleep(lkp, pflags | PINTERLOCKED, 497 lkp->lk_wmesg, timo); 498 if (error) { 499 undo_upreq(lkp); 500 break; 501 } 502 if (extflags & LK_SLEEPFAIL) { 503 error = ENOLCK; 504 undo_upreq(lkp); 505 break; 506 } 507 } 508 /* retry */ 509 } 510 goto again; 511 512 case LK_RELEASE: 513 /* 514 * Release the currently held lock. If releasing the current 515 * lock as part of an error return, error will ALREADY be 516 * non-zero. 517 * 518 * When releasing the last lock we automatically transition 519 * LKC_UPREQ to LKC_EXCL|1. 520 * 521 * WARNING! We cannot detect when there are multiple exclusive 522 * requests pending. We clear EXREQ unconditionally 523 * on the 1->0 transition so it is possible for 524 * shared requests to race the next exclusive 525 * request. 526 * 527 * Always succeeds. 528 */ 529 if ((count & LKC_MASK) == 0) 530 panic("lockmgr: LK_RELEASE: no lock held"); 531 532 if (count & LKC_EXCL) { 533 if (lkp->lk_lockholder != LK_KERNTHREAD && 534 lkp->lk_lockholder != td) { 535 panic("lockmgr: pid %d, not exlusive " 536 "lock holder thr %p/%p unlocking", 537 (td->td_proc ? td->td_proc->p_pid : -1), 538 td, lkp->lk_lockholder); 539 } 540 if ((count & (LKC_UPREQ|LKC_MASK)) == 1) { 541 /* 542 * Last exclusive count is being released 543 */ 544 otd = lkp->lk_lockholder; 545 lkp->lk_lockholder = NULL; 546 if (!atomic_cmpset_int(&lkp->lk_count, count, 547 (count - 1) & 548 ~(LKC_EXCL | LKC_EXREQ | 549 LKC_SHREQ| LKC_CANCEL))) { 550 lkp->lk_lockholder = otd; 551 goto again; 552 } 553 if (count & (LKC_EXREQ|LKC_SHREQ)) 554 wakeup(lkp); 555 /* success */ 556 } else if ((count & (LKC_UPREQ|LKC_MASK)) == 557 (LKC_UPREQ | 1)) { 558 /* 559 * Last exclusive count is being released but 560 * an upgrade request is present, automatically 561 * grant an exclusive state to the owner of 562 * the upgrade request. 563 */ 564 otd = lkp->lk_lockholder; 565 lkp->lk_lockholder = NULL; 566 if (!atomic_cmpset_int(&lkp->lk_count, count, 567 (count & ~LKC_UPREQ) | 568 LKC_UPGRANT)) { 569 lkp->lk_lockholder = otd; 570 } 571 wakeup(lkp); 572 /* success */ 573 } else { 574 otd = lkp->lk_lockholder; 575 if (!atomic_cmpset_int(&lkp->lk_count, count, 576 count - 1)) { 577 goto again; 578 } 579 /* success */ 580 } 581 /* success */ 582 if (otd != LK_KERNTHREAD) 583 COUNT(td, -1); 584 } else { 585 if ((count & (LKC_UPREQ|LKC_MASK)) == 1) { 586 /* 587 * Last shared count is being released. 588 */ 589 if (!atomic_cmpset_int(&lkp->lk_count, count, 590 (count - 1) & 591 ~(LKC_EXREQ | LKC_SHREQ | 592 LKC_CANCEL))) { 593 goto again; 594 } 595 if (count & (LKC_EXREQ|LKC_SHREQ)) 596 wakeup(lkp); 597 /* success */ 598 } else if ((count & (LKC_UPREQ|LKC_MASK)) == 599 (LKC_UPREQ | 1)) { 600 /* 601 * Last shared count is being released but 602 * an upgrade request is present, automatically 603 * grant an exclusive state to the owner of 604 * the upgrade request. Masked count 605 * remains 1. 606 */ 607 if (!atomic_cmpset_int(&lkp->lk_count, count, 608 (count & ~(LKC_UPREQ | 609 LKC_CANCEL)) | 610 LKC_EXCL | LKC_UPGRANT)) { 611 goto again; 612 } 613 wakeup(lkp); 614 } else { 615 if (!atomic_cmpset_int(&lkp->lk_count, count, 616 count - 1)) { 617 goto again; 618 } 619 } 620 /* success */ 621 COUNT(td, -1); 622 } 623 break; 624 625 case LK_CANCEL_BEG: 626 /* 627 * Start canceling blocked requestors or later requestors. 628 * requestors must use CANCELABLE. Don't waste time issuing 629 * a wakeup if nobody is pending. 630 */ 631 KKASSERT((count & LKC_CANCEL) == 0); /* disallowed case */ 632 KKASSERT((count & LKC_MASK) != 0); /* issue w/lock held */ 633 if (!atomic_cmpset_int(&lkp->lk_count, 634 count, count | LKC_CANCEL)) { 635 goto again; 636 } 637 if (count & (LKC_EXREQ|LKC_SHREQ|LKC_UPREQ)) { 638 wakeup(lkp); 639 } 640 break; 641 642 case LK_CANCEL_END: 643 atomic_clear_int(&lkp->lk_count, LKC_CANCEL); 644 break; 645 646 default: 647 panic("lockmgr: unknown locktype request %d", 648 flags & LK_TYPE_MASK); 649 /* NOTREACHED */ 650 } 651 return (error); 652 } 653 654 /* 655 * Undo an upgrade request 656 */ 657 static 658 void 659 undo_upreq(struct lock *lkp) 660 { 661 int count; 662 663 for (;;) { 664 count = lkp->lk_count; 665 cpu_ccfence(); 666 if (count & LKC_UPGRANT) { 667 /* 668 * UPREQ was shifted to UPGRANT. We own UPGRANT now, 669 * another thread might own UPREQ. Clear UPGRANT 670 * and release the granted lock. 671 */ 672 if (atomic_cmpset_int(&lkp->lk_count, count, 673 count & ~LKC_UPGRANT)) { 674 lockmgr(lkp, LK_RELEASE); 675 break; 676 } 677 } else if (count & LKC_EXCL) { 678 /* 679 * Clear the UPREQ we still own. Nobody to wakeup 680 * here because there is an existing exclusive 681 * holder. 682 */ 683 KKASSERT(count & LKC_UPREQ); 684 KKASSERT((count & LKC_MASK) > 0); 685 if (atomic_cmpset_int(&lkp->lk_count, count, 686 count & ~LKC_UPREQ)) { 687 wakeup(lkp); 688 break; 689 } 690 } else if (count & LKC_EXREQ) { 691 /* 692 * Clear the UPREQ we still own. We cannot wakeup any 693 * shared waiters because there is an exclusive 694 * request pending. 695 */ 696 KKASSERT(count & LKC_UPREQ); 697 KKASSERT((count & LKC_MASK) > 0); 698 if (atomic_cmpset_int(&lkp->lk_count, count, 699 count & ~LKC_UPREQ)) { 700 break; 701 } 702 } else { 703 /* 704 * Clear the UPREQ we still own. Wakeup any shared 705 * waiters. 706 */ 707 KKASSERT(count & LKC_UPREQ); 708 KKASSERT((count & LKC_MASK) > 0); 709 if (atomic_cmpset_int(&lkp->lk_count, count, 710 count & 711 ~(LKC_UPREQ | LKC_SHREQ))) { 712 if (count & LKC_SHREQ) 713 wakeup(lkp); 714 break; 715 } 716 } 717 /* retry */ 718 } 719 } 720 721 void 722 lockmgr_kernproc(struct lock *lp) 723 { 724 struct thread *td __debugvar = curthread; 725 726 if (lp->lk_lockholder != LK_KERNTHREAD) { 727 KASSERT(lp->lk_lockholder == td, 728 ("lockmgr_kernproc: lock not owned by curthread %p: %p", 729 td, lp->lk_lockholder)); 730 lp->lk_lockholder = LK_KERNTHREAD; 731 COUNT(td, -1); 732 } 733 } 734 735 /* 736 * Initialize a lock; required before use. 737 */ 738 void 739 lockinit(struct lock *lkp, const char *wmesg, int timo, int flags) 740 { 741 lkp->lk_flags = (flags & LK_EXTFLG_MASK); 742 lkp->lk_count = 0; 743 lkp->lk_wmesg = wmesg; 744 lkp->lk_timo = timo; 745 lkp->lk_lockholder = LK_NOTHREAD; 746 } 747 748 /* 749 * Reinitialize a lock that is being reused for a different purpose, but 750 * which may have pending (blocked) threads sitting on it. The caller 751 * must already hold the interlock. 752 */ 753 void 754 lockreinit(struct lock *lkp, const char *wmesg, int timo, int flags) 755 { 756 lkp->lk_wmesg = wmesg; 757 lkp->lk_timo = timo; 758 } 759 760 /* 761 * De-initialize a lock. The structure must no longer be used by anyone. 762 */ 763 void 764 lockuninit(struct lock *lkp) 765 { 766 KKASSERT((lkp->lk_count & (LKC_EXREQ|LKC_SHREQ|LKC_UPREQ)) == 0); 767 } 768 769 /* 770 * Determine the status of a lock. 771 */ 772 int 773 lockstatus(struct lock *lkp, struct thread *td) 774 { 775 int lock_type = 0; 776 int count; 777 778 count = lkp->lk_count; 779 cpu_ccfence(); 780 781 if (count & LKC_EXCL) { 782 if (td == NULL || lkp->lk_lockholder == td) 783 lock_type = LK_EXCLUSIVE; 784 else 785 lock_type = LK_EXCLOTHER; 786 } else if (count & LKC_MASK) { 787 lock_type = LK_SHARED; 788 } 789 return (lock_type); 790 } 791 792 /* 793 * Return non-zero if the caller owns the lock shared or exclusive. 794 * We can only guess re: shared locks. 795 */ 796 int 797 lockowned(struct lock *lkp) 798 { 799 thread_t td = curthread; 800 int count; 801 802 count = lkp->lk_count; 803 cpu_ccfence(); 804 805 if (count & LKC_EXCL) 806 return(lkp->lk_lockholder == td); 807 else 808 return((count & LKC_MASK) != 0); 809 } 810 811 /* 812 * Determine the number of holders of a lock. 813 * 814 * The non-blocking version can usually be used for assertions. 815 */ 816 int 817 lockcount(struct lock *lkp) 818 { 819 return(lkp->lk_count & LKC_MASK); 820 } 821 822 int 823 lockcountnb(struct lock *lkp) 824 { 825 return(lkp->lk_count & LKC_MASK); 826 } 827 828 /* 829 * Print out information about state of a lock. Used by VOP_PRINT 830 * routines to display status about contained locks. 831 */ 832 void 833 lockmgr_printinfo(struct lock *lkp) 834 { 835 struct thread *td = lkp->lk_lockholder; 836 struct proc *p; 837 int count; 838 839 count = lkp->lk_count; 840 cpu_ccfence(); 841 842 if (td && td != LK_KERNTHREAD && td != LK_NOTHREAD) 843 p = td->td_proc; 844 else 845 p = NULL; 846 847 if (count & LKC_EXCL) { 848 kprintf(" lock type %s: EXCLUS (count %08x) by td %p pid %d", 849 lkp->lk_wmesg, count, td, 850 p ? p->p_pid : -99); 851 } else if (count & LKC_MASK) { 852 kprintf(" lock type %s: SHARED (count %08x)", 853 lkp->lk_wmesg, count); 854 } else { 855 kprintf(" lock type %s: NOTHELD", lkp->lk_wmesg); 856 } 857 if (count & (LKC_EXREQ|LKC_SHREQ)) 858 kprintf(" with waiters\n"); 859 else 860 kprintf("\n"); 861 } 862 863 void 864 lock_sysinit(struct lock_args *arg) 865 { 866 lockinit(arg->la_lock, arg->la_desc, 0, arg->la_flags); 867 } 868 869 #ifdef DEBUG_CANCEL_LOCKS 870 871 static 872 int 873 sysctl_cancel_lock(SYSCTL_HANDLER_ARGS) 874 { 875 int error; 876 877 if (req->newptr) { 878 SYSCTL_XUNLOCK(); 879 lockmgr(&cancel_lk, LK_EXCLUSIVE); 880 kprintf("x"); 881 error = tsleep(&error, PCATCH, "canmas", hz * 5); 882 lockmgr(&cancel_lk, LK_CANCEL_BEG); 883 kprintf("y"); 884 error = tsleep(&error, PCATCH, "canmas", hz * 5); 885 kprintf("z"); 886 lockmgr(&cancel_lk, LK_RELEASE); 887 SYSCTL_XLOCK(); 888 SYSCTL_OUT(req, &error, sizeof(error)); 889 } 890 error = 0; 891 892 return error; 893 } 894 895 static 896 int 897 sysctl_cancel_test(SYSCTL_HANDLER_ARGS) 898 { 899 int error; 900 901 if (req->newptr) { 902 error = lockmgr(&cancel_lk, LK_EXCLUSIVE|LK_CANCELABLE); 903 if (error == 0) 904 lockmgr(&cancel_lk, LK_RELEASE); 905 SYSCTL_OUT(req, &error, sizeof(error)); 906 kprintf("test %d\n", error); 907 } 908 909 return 0; 910 } 911 912 #endif 913