1 /* $NetBSD: subr_lockdebug.c,v 1.64 2018/03/19 08:41:21 ozaki-r Exp $ */ 2 3 /*- 4 * Copyright (c) 2006, 2007, 2008 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Basic lock debugging code shared among lock primitives. 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: subr_lockdebug.c,v 1.64 2018/03/19 08:41:21 ozaki-r Exp $"); 38 39 #ifdef _KERNEL_OPT 40 #include "opt_ddb.h" 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/proc.h> 45 #include <sys/systm.h> 46 #include <sys/kernel.h> 47 #include <sys/kmem.h> 48 #include <sys/lockdebug.h> 49 #include <sys/sleepq.h> 50 #include <sys/cpu.h> 51 #include <sys/atomic.h> 52 #include <sys/lock.h> 53 #include <sys/rbtree.h> 54 #include <sys/ksyms.h> 55 56 #include <machine/lock.h> 57 58 unsigned int ld_panic; 59 60 #ifdef LOCKDEBUG 61 62 #define LD_BATCH_SHIFT 9 63 #define LD_BATCH (1 << LD_BATCH_SHIFT) 64 #define LD_BATCH_MASK (LD_BATCH - 1) 65 #define LD_MAX_LOCKS 1048576 66 #define LD_SLOP 16 67 68 #define LD_LOCKED 0x01 69 #define LD_SLEEPER 0x02 70 71 #define LD_WRITE_LOCK 0x80000000 72 73 typedef struct lockdebug { 74 struct rb_node ld_rb_node; 75 __cpu_simple_lock_t ld_spinlock; 76 _TAILQ_ENTRY(struct lockdebug, volatile) ld_chain; 77 _TAILQ_ENTRY(struct lockdebug, volatile) ld_achain; 78 volatile void *ld_lock; 79 lockops_t *ld_lockops; 80 struct lwp *ld_lwp; 81 uintptr_t ld_locked; 82 uintptr_t ld_unlocked; 83 uintptr_t ld_initaddr; 84 uint16_t ld_shares; 85 uint16_t ld_cpu; 86 uint8_t ld_flags; 87 uint8_t ld_shwant; /* advisory */ 88 uint8_t ld_exwant; /* advisory */ 89 uint8_t ld_unused; 90 } volatile lockdebug_t; 91 92 typedef _TAILQ_HEAD(lockdebuglist, struct lockdebug, volatile) lockdebuglist_t; 93 94 __cpu_simple_lock_t ld_mod_lk; 95 lockdebuglist_t ld_free = TAILQ_HEAD_INITIALIZER(ld_free); 96 lockdebuglist_t ld_all = TAILQ_HEAD_INITIALIZER(ld_all); 97 int ld_nfree; 98 int ld_freeptr; 99 int ld_recurse; 100 bool ld_nomore; 101 lockdebug_t ld_prime[LD_BATCH]; 102 103 static void lockdebug_abort1(const char *, size_t, lockdebug_t *, int, 104 const char *, bool); 105 static int lockdebug_more(int); 106 static void lockdebug_init(void); 107 static void lockdebug_dump(lockdebug_t *, void (*)(const char *, ...) 108 __printflike(1, 2)); 109 110 static signed int 111 ld_rbto_compare_nodes(void *ctx, const void *n1, const void *n2) 112 { 113 const lockdebug_t *ld1 = n1; 114 const lockdebug_t *ld2 = n2; 115 const uintptr_t a = (uintptr_t)ld1->ld_lock; 116 const uintptr_t b = (uintptr_t)ld2->ld_lock; 117 118 if (a < b) 119 return -1; 120 if (a > b) 121 return 1; 122 return 0; 123 } 124 125 static signed int 126 ld_rbto_compare_key(void *ctx, const void *n, const void *key) 127 { 128 const lockdebug_t *ld = n; 129 const uintptr_t a = (uintptr_t)ld->ld_lock; 130 const uintptr_t b = (uintptr_t)key; 131 132 if (a < b) 133 return -1; 134 if (a > b) 135 return 1; 136 return 0; 137 } 138 139 static rb_tree_t ld_rb_tree; 140 141 static const rb_tree_ops_t ld_rb_tree_ops = { 142 .rbto_compare_nodes = ld_rbto_compare_nodes, 143 .rbto_compare_key = ld_rbto_compare_key, 144 .rbto_node_offset = offsetof(lockdebug_t, ld_rb_node), 145 .rbto_context = NULL 146 }; 147 148 static inline lockdebug_t * 149 lockdebug_lookup1(const volatile void *lock) 150 { 151 lockdebug_t *ld; 152 struct cpu_info *ci; 153 154 ci = curcpu(); 155 __cpu_simple_lock(&ci->ci_data.cpu_ld_lock); 156 ld = rb_tree_find_node(&ld_rb_tree, (void *)(intptr_t)lock); 157 __cpu_simple_unlock(&ci->ci_data.cpu_ld_lock); 158 if (ld == NULL) { 159 return NULL; 160 } 161 __cpu_simple_lock(&ld->ld_spinlock); 162 163 return ld; 164 } 165 166 static void 167 lockdebug_lock_cpus(void) 168 { 169 CPU_INFO_ITERATOR cii; 170 struct cpu_info *ci; 171 172 for (CPU_INFO_FOREACH(cii, ci)) { 173 __cpu_simple_lock(&ci->ci_data.cpu_ld_lock); 174 } 175 } 176 177 static void 178 lockdebug_unlock_cpus(void) 179 { 180 CPU_INFO_ITERATOR cii; 181 struct cpu_info *ci; 182 183 for (CPU_INFO_FOREACH(cii, ci)) { 184 __cpu_simple_unlock(&ci->ci_data.cpu_ld_lock); 185 } 186 } 187 188 /* 189 * lockdebug_lookup: 190 * 191 * Find a lockdebug structure by a pointer to a lock and return it locked. 192 */ 193 static inline lockdebug_t * 194 lockdebug_lookup(const char *func, size_t line, const volatile void *lock, 195 uintptr_t where) 196 { 197 lockdebug_t *ld; 198 199 ld = lockdebug_lookup1(lock); 200 if (__predict_false(ld == NULL)) { 201 panic("%s,%zu: uninitialized lock (lock=%p, from=%08" 202 PRIxPTR ")", func, line, lock, where); 203 } 204 return ld; 205 } 206 207 /* 208 * lockdebug_init: 209 * 210 * Initialize the lockdebug system. Allocate an initial pool of 211 * lockdebug structures before the VM system is up and running. 212 */ 213 static void 214 lockdebug_init(void) 215 { 216 lockdebug_t *ld; 217 int i; 218 219 TAILQ_INIT(&curcpu()->ci_data.cpu_ld_locks); 220 TAILQ_INIT(&curlwp->l_ld_locks); 221 __cpu_simple_lock_init(&curcpu()->ci_data.cpu_ld_lock); 222 __cpu_simple_lock_init(&ld_mod_lk); 223 224 rb_tree_init(&ld_rb_tree, &ld_rb_tree_ops); 225 226 ld = ld_prime; 227 for (i = 1, ld++; i < LD_BATCH; i++, ld++) { 228 __cpu_simple_lock_init(&ld->ld_spinlock); 229 TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain); 230 TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain); 231 } 232 ld_freeptr = 1; 233 ld_nfree = LD_BATCH - 1; 234 } 235 236 /* 237 * lockdebug_alloc: 238 * 239 * A lock is being initialized, so allocate an associated debug 240 * structure. 241 */ 242 bool 243 lockdebug_alloc(const char *func, size_t line, volatile void *lock, 244 lockops_t *lo, uintptr_t initaddr) 245 { 246 struct cpu_info *ci; 247 lockdebug_t *ld; 248 int s; 249 250 if (__predict_false(lo == NULL || panicstr != NULL || ld_panic)) 251 return false; 252 if (__predict_false(ld_freeptr == 0)) 253 lockdebug_init(); 254 255 s = splhigh(); 256 __cpu_simple_lock(&ld_mod_lk); 257 if (__predict_false((ld = lockdebug_lookup1(lock)) != NULL)) { 258 __cpu_simple_unlock(&ld_mod_lk); 259 lockdebug_abort1(func, line, ld, s, "already initialized", 260 true); 261 return false; 262 } 263 264 /* 265 * Pinch a new debug structure. We may recurse because we call 266 * kmem_alloc(), which may need to initialize new locks somewhere 267 * down the path. If not recursing, we try to maintain at least 268 * LD_SLOP structures free, which should hopefully be enough to 269 * satisfy kmem_alloc(). If we can't provide a structure, not to 270 * worry: we'll just mark the lock as not having an ID. 271 */ 272 ci = curcpu(); 273 ci->ci_lkdebug_recurse++; 274 if (TAILQ_EMPTY(&ld_free)) { 275 if (ci->ci_lkdebug_recurse > 1 || ld_nomore) { 276 ci->ci_lkdebug_recurse--; 277 __cpu_simple_unlock(&ld_mod_lk); 278 splx(s); 279 return false; 280 } 281 s = lockdebug_more(s); 282 } else if (ci->ci_lkdebug_recurse == 1 && ld_nfree < LD_SLOP) { 283 s = lockdebug_more(s); 284 } 285 if (__predict_false((ld = TAILQ_FIRST(&ld_free)) == NULL)) { 286 __cpu_simple_unlock(&ld_mod_lk); 287 splx(s); 288 return false; 289 } 290 TAILQ_REMOVE(&ld_free, ld, ld_chain); 291 ld_nfree--; 292 ci->ci_lkdebug_recurse--; 293 294 if (__predict_false(ld->ld_lock != NULL)) { 295 panic("%s,%zu: corrupt table ld %p", func, line, ld); 296 } 297 298 /* Initialise the structure. */ 299 ld->ld_lock = lock; 300 ld->ld_lockops = lo; 301 ld->ld_locked = 0; 302 ld->ld_unlocked = 0; 303 ld->ld_lwp = NULL; 304 ld->ld_initaddr = initaddr; 305 ld->ld_flags = (lo->lo_type == LOCKOPS_SLEEP ? LD_SLEEPER : 0); 306 lockdebug_lock_cpus(); 307 (void)rb_tree_insert_node(&ld_rb_tree, __UNVOLATILE(ld)); 308 lockdebug_unlock_cpus(); 309 __cpu_simple_unlock(&ld_mod_lk); 310 311 splx(s); 312 return true; 313 } 314 315 /* 316 * lockdebug_free: 317 * 318 * A lock is being destroyed, so release debugging resources. 319 */ 320 void 321 lockdebug_free(const char *func, size_t line, volatile void *lock) 322 { 323 lockdebug_t *ld; 324 int s; 325 326 if (__predict_false(panicstr != NULL || ld_panic)) 327 return; 328 329 s = splhigh(); 330 __cpu_simple_lock(&ld_mod_lk); 331 ld = lockdebug_lookup(func, line, lock, 332 (uintptr_t) __builtin_return_address(0)); 333 if (__predict_false(ld == NULL)) { 334 __cpu_simple_unlock(&ld_mod_lk); 335 panic("%s,%zu: destroying uninitialized object %p" 336 "(ld_lock=%p)", func, line, lock, ld->ld_lock); 337 return; 338 } 339 if (__predict_false((ld->ld_flags & LD_LOCKED) != 0 || 340 ld->ld_shares != 0)) { 341 __cpu_simple_unlock(&ld_mod_lk); 342 lockdebug_abort1(func, line, ld, s, "is locked or in use", 343 true); 344 return; 345 } 346 lockdebug_lock_cpus(); 347 rb_tree_remove_node(&ld_rb_tree, __UNVOLATILE(ld)); 348 lockdebug_unlock_cpus(); 349 ld->ld_lock = NULL; 350 TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain); 351 ld_nfree++; 352 __cpu_simple_unlock(&ld->ld_spinlock); 353 __cpu_simple_unlock(&ld_mod_lk); 354 splx(s); 355 } 356 357 /* 358 * lockdebug_more: 359 * 360 * Allocate a batch of debug structures and add to the free list. 361 * Must be called with ld_mod_lk held. 362 */ 363 static int 364 lockdebug_more(int s) 365 { 366 lockdebug_t *ld; 367 void *block; 368 int i, base, m; 369 370 /* 371 * Can't call kmem_alloc() if in interrupt context. XXX We could 372 * deadlock, because we don't know which locks the caller holds. 373 */ 374 if (cpu_intr_p() || cpu_softintr_p()) { 375 return s; 376 } 377 378 while (ld_nfree < LD_SLOP) { 379 __cpu_simple_unlock(&ld_mod_lk); 380 splx(s); 381 block = kmem_zalloc(LD_BATCH * sizeof(lockdebug_t), KM_SLEEP); 382 s = splhigh(); 383 __cpu_simple_lock(&ld_mod_lk); 384 385 if (ld_nfree > LD_SLOP) { 386 /* Somebody beat us to it. */ 387 __cpu_simple_unlock(&ld_mod_lk); 388 splx(s); 389 kmem_free(block, LD_BATCH * sizeof(lockdebug_t)); 390 s = splhigh(); 391 __cpu_simple_lock(&ld_mod_lk); 392 continue; 393 } 394 395 base = ld_freeptr; 396 ld_nfree += LD_BATCH; 397 ld = block; 398 base <<= LD_BATCH_SHIFT; 399 m = min(LD_MAX_LOCKS, base + LD_BATCH); 400 401 if (m == LD_MAX_LOCKS) 402 ld_nomore = true; 403 404 for (i = base; i < m; i++, ld++) { 405 __cpu_simple_lock_init(&ld->ld_spinlock); 406 TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain); 407 TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain); 408 } 409 410 membar_producer(); 411 } 412 413 return s; 414 } 415 416 /* 417 * lockdebug_wantlock: 418 * 419 * Process the preamble to a lock acquire. The "shared" 420 * parameter controls which ld_{ex,sh}want counter is 421 * updated; a negative value of shared updates neither. 422 */ 423 void 424 lockdebug_wantlock(const char *func, size_t line, 425 const volatile void *lock, uintptr_t where, int shared) 426 { 427 struct lwp *l = curlwp; 428 lockdebug_t *ld; 429 bool recurse; 430 int s; 431 432 (void)shared; 433 recurse = false; 434 435 if (__predict_false(panicstr != NULL || ld_panic)) 436 return; 437 438 s = splhigh(); 439 if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) { 440 splx(s); 441 return; 442 } 443 if ((ld->ld_flags & LD_LOCKED) != 0 || ld->ld_shares != 0) { 444 if ((ld->ld_flags & LD_SLEEPER) != 0) { 445 if (ld->ld_lwp == l) 446 recurse = true; 447 } else if (ld->ld_cpu == (uint16_t)cpu_index(curcpu())) 448 recurse = true; 449 } 450 if (cpu_intr_p()) { 451 if (__predict_false((ld->ld_flags & LD_SLEEPER) != 0)) { 452 lockdebug_abort1(func, line, ld, s, 453 "acquiring sleep lock from interrupt context", 454 true); 455 return; 456 } 457 } 458 if (shared > 0) 459 ld->ld_shwant++; 460 else if (shared == 0) 461 ld->ld_exwant++; 462 if (__predict_false(recurse)) { 463 lockdebug_abort1(func, line, ld, s, "locking against myself", 464 true); 465 return; 466 } 467 __cpu_simple_unlock(&ld->ld_spinlock); 468 splx(s); 469 } 470 471 /* 472 * lockdebug_locked: 473 * 474 * Process a lock acquire operation. 475 */ 476 void 477 lockdebug_locked(const char *func, size_t line, 478 volatile void *lock, void *cvlock, uintptr_t where, int shared) 479 { 480 struct lwp *l = curlwp; 481 lockdebug_t *ld; 482 int s; 483 484 if (__predict_false(panicstr != NULL || ld_panic)) 485 return; 486 487 s = splhigh(); 488 if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) { 489 splx(s); 490 return; 491 } 492 if (cvlock) { 493 KASSERT(ld->ld_lockops->lo_type == LOCKOPS_CV); 494 if (lock == (void *)&lbolt) { 495 /* nothing */ 496 } else if (ld->ld_shares++ == 0) { 497 ld->ld_locked = (uintptr_t)cvlock; 498 } else if (__predict_false(cvlock != (void *)ld->ld_locked)) { 499 lockdebug_abort1(func, line, ld, s, 500 "multiple locks used with condition variable", 501 true); 502 return; 503 } 504 } else if (shared) { 505 l->l_shlocks++; 506 ld->ld_locked = where; 507 ld->ld_shares++; 508 ld->ld_shwant--; 509 } else { 510 if (__predict_false((ld->ld_flags & LD_LOCKED) != 0)) { 511 lockdebug_abort1(func, line, ld, s, "already locked", 512 true); 513 return; 514 } 515 ld->ld_flags |= LD_LOCKED; 516 ld->ld_locked = where; 517 ld->ld_exwant--; 518 if ((ld->ld_flags & LD_SLEEPER) != 0) { 519 TAILQ_INSERT_TAIL(&l->l_ld_locks, ld, ld_chain); 520 } else { 521 TAILQ_INSERT_TAIL(&curcpu()->ci_data.cpu_ld_locks, 522 ld, ld_chain); 523 } 524 } 525 ld->ld_cpu = (uint16_t)cpu_index(curcpu()); 526 ld->ld_lwp = l; 527 __cpu_simple_unlock(&ld->ld_spinlock); 528 splx(s); 529 } 530 531 /* 532 * lockdebug_unlocked: 533 * 534 * Process a lock release operation. 535 */ 536 void 537 lockdebug_unlocked(const char *func, size_t line, 538 volatile void *lock, uintptr_t where, int shared) 539 { 540 struct lwp *l = curlwp; 541 lockdebug_t *ld; 542 int s; 543 544 if (__predict_false(panicstr != NULL || ld_panic)) 545 return; 546 547 s = splhigh(); 548 if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) { 549 splx(s); 550 return; 551 } 552 if (ld->ld_lockops->lo_type == LOCKOPS_CV) { 553 if (lock == (void *)&lbolt) { 554 /* nothing */ 555 } else { 556 ld->ld_shares--; 557 } 558 } else if (shared) { 559 if (__predict_false(l->l_shlocks == 0)) { 560 lockdebug_abort1(func, line, ld, s, 561 "no shared locks held by LWP", true); 562 return; 563 } 564 if (__predict_false(ld->ld_shares == 0)) { 565 lockdebug_abort1(func, line, ld, s, 566 "no shared holds on this lock", true); 567 return; 568 } 569 l->l_shlocks--; 570 ld->ld_shares--; 571 if (ld->ld_lwp == l) { 572 ld->ld_unlocked = where; 573 ld->ld_lwp = NULL; 574 } 575 if (ld->ld_cpu == (uint16_t)cpu_index(curcpu())) 576 ld->ld_cpu = (uint16_t)-1; 577 } else { 578 if (__predict_false((ld->ld_flags & LD_LOCKED) == 0)) { 579 lockdebug_abort1(func, line, ld, s, "not locked", true); 580 return; 581 } 582 583 if ((ld->ld_flags & LD_SLEEPER) != 0) { 584 if (__predict_false(ld->ld_lwp != curlwp)) { 585 lockdebug_abort1(func, line, ld, s, 586 "not held by current LWP", true); 587 return; 588 } 589 TAILQ_REMOVE(&l->l_ld_locks, ld, ld_chain); 590 } else { 591 uint16_t idx = (uint16_t)cpu_index(curcpu()); 592 if (__predict_false(ld->ld_cpu != idx)) { 593 lockdebug_abort1(func, line, ld, s, 594 "not held by current CPU", true); 595 return; 596 } 597 TAILQ_REMOVE(&curcpu()->ci_data.cpu_ld_locks, ld, 598 ld_chain); 599 } 600 ld->ld_flags &= ~LD_LOCKED; 601 ld->ld_unlocked = where; 602 ld->ld_lwp = NULL; 603 } 604 __cpu_simple_unlock(&ld->ld_spinlock); 605 splx(s); 606 } 607 608 /* 609 * lockdebug_wakeup: 610 * 611 * Process a wakeup on a condition variable. 612 */ 613 void 614 lockdebug_wakeup(const char *func, size_t line, volatile void *lock, 615 uintptr_t where) 616 { 617 lockdebug_t *ld; 618 int s; 619 620 if (__predict_false(panicstr != NULL || ld_panic || lock == (void *)&lbolt)) 621 return; 622 623 s = splhigh(); 624 /* Find the CV... */ 625 if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) { 626 splx(s); 627 return; 628 } 629 /* 630 * If it has any waiters, ensure that they are using the 631 * same interlock. 632 */ 633 if (__predict_false(ld->ld_shares != 0 && 634 !mutex_owned((kmutex_t *)ld->ld_locked))) { 635 lockdebug_abort1(func, line, ld, s, "interlocking mutex not " 636 "held during wakeup", true); 637 return; 638 } 639 __cpu_simple_unlock(&ld->ld_spinlock); 640 splx(s); 641 } 642 643 /* 644 * lockdebug_barrier: 645 * 646 * Panic if we hold more than one specified spin lock, and optionally, 647 * if we hold sleep locks. 648 */ 649 void 650 lockdebug_barrier(const char *func, size_t line, volatile void *spinlock, 651 int slplocks) 652 { 653 struct lwp *l = curlwp; 654 lockdebug_t *ld; 655 int s; 656 657 if (__predict_false(panicstr != NULL || ld_panic)) 658 return; 659 660 s = splhigh(); 661 if ((l->l_pflag & LP_INTR) == 0) { 662 TAILQ_FOREACH(ld, &curcpu()->ci_data.cpu_ld_locks, ld_chain) { 663 if (ld->ld_lock == spinlock) { 664 continue; 665 } 666 __cpu_simple_lock(&ld->ld_spinlock); 667 lockdebug_abort1(func, line, ld, s, 668 "spin lock held", true); 669 return; 670 } 671 } 672 if (slplocks) { 673 splx(s); 674 return; 675 } 676 ld = TAILQ_FIRST(&l->l_ld_locks); 677 if (__predict_false(ld != NULL)) { 678 __cpu_simple_lock(&ld->ld_spinlock); 679 lockdebug_abort1(func, line, ld, s, "sleep lock held", true); 680 return; 681 } 682 splx(s); 683 if (l->l_shlocks != 0) { 684 TAILQ_FOREACH(ld, &ld_all, ld_achain) { 685 if (ld->ld_lockops->lo_type == LOCKOPS_CV) 686 continue; 687 if (ld->ld_lwp == l) 688 lockdebug_dump(ld, printf); 689 } 690 panic("%s,%zu: holding %d shared locks", func, line, 691 l->l_shlocks); 692 } 693 } 694 695 /* 696 * lockdebug_mem_check: 697 * 698 * Check for in-use locks within a memory region that is 699 * being freed. 700 */ 701 void 702 lockdebug_mem_check(const char *func, size_t line, void *base, size_t sz) 703 { 704 lockdebug_t *ld; 705 struct cpu_info *ci; 706 int s; 707 708 if (__predict_false(panicstr != NULL || ld_panic)) 709 return; 710 711 s = splhigh(); 712 ci = curcpu(); 713 __cpu_simple_lock(&ci->ci_data.cpu_ld_lock); 714 ld = (lockdebug_t *)rb_tree_find_node_geq(&ld_rb_tree, base); 715 if (ld != NULL) { 716 const uintptr_t lock = (uintptr_t)ld->ld_lock; 717 718 if (__predict_false((uintptr_t)base > lock)) 719 panic("%s,%zu: corrupt tree ld=%p, base=%p, sz=%zu", 720 func, line, ld, base, sz); 721 if (lock >= (uintptr_t)base + sz) 722 ld = NULL; 723 } 724 __cpu_simple_unlock(&ci->ci_data.cpu_ld_lock); 725 if (__predict_false(ld != NULL)) { 726 __cpu_simple_lock(&ld->ld_spinlock); 727 lockdebug_abort1(func, line, ld, s, 728 "allocation contains active lock", !cold); 729 return; 730 } 731 splx(s); 732 } 733 734 /* 735 * lockdebug_dump: 736 * 737 * Dump information about a lock on panic, or for DDB. 738 */ 739 static void 740 lockdebug_dump(lockdebug_t *ld, void (*pr)(const char *, ...) 741 __printflike(1, 2)) 742 { 743 int sleeper = (ld->ld_flags & LD_SLEEPER); 744 745 (*pr)( 746 "lock address : %#018lx type : %18s\n" 747 "initialized : %#018lx", 748 (long)ld->ld_lock, (sleeper ? "sleep/adaptive" : "spin"), 749 (long)ld->ld_initaddr); 750 751 if (ld->ld_lockops->lo_type == LOCKOPS_CV) { 752 (*pr)(" interlock: %#018lx\n", (long)ld->ld_locked); 753 } else { 754 (*pr)("\n" 755 "shared holds : %18u exclusive: %18u\n" 756 "shares wanted: %18u exclusive: %18u\n" 757 "current cpu : %18u last held: %18u\n" 758 "current lwp : %#018lx last held: %#018lx\n" 759 "last locked%c : %#018lx unlocked%c: %#018lx\n", 760 (unsigned)ld->ld_shares, ((ld->ld_flags & LD_LOCKED) != 0), 761 (unsigned)ld->ld_shwant, (unsigned)ld->ld_exwant, 762 (unsigned)cpu_index(curcpu()), (unsigned)ld->ld_cpu, 763 (long)curlwp, (long)ld->ld_lwp, 764 ((ld->ld_flags & LD_LOCKED) ? '*' : ' '), 765 (long)ld->ld_locked, 766 ((ld->ld_flags & LD_LOCKED) ? ' ' : '*'), 767 (long)ld->ld_unlocked); 768 } 769 770 if (ld->ld_lockops->lo_dump != NULL) 771 (*ld->ld_lockops->lo_dump)(ld->ld_lock); 772 773 if (sleeper) { 774 (*pr)("\n"); 775 turnstile_print(ld->ld_lock, pr); 776 } 777 } 778 779 /* 780 * lockdebug_abort1: 781 * 782 * An error has been trapped - dump lock info and panic. 783 */ 784 static void 785 lockdebug_abort1(const char *func, size_t line, lockdebug_t *ld, int s, 786 const char *msg, bool dopanic) 787 { 788 789 /* 790 * Don't make the situation worse if the system is already going 791 * down in flames. Once a panic is triggered, lockdebug state 792 * becomes stale and cannot be trusted. 793 */ 794 if (atomic_inc_uint_nv(&ld_panic) != 1) { 795 __cpu_simple_unlock(&ld->ld_spinlock); 796 splx(s); 797 return; 798 } 799 800 printf_nolog("%s error: %s,%zu: %s\n\n", ld->ld_lockops->lo_name, 801 func, line, msg); 802 lockdebug_dump(ld, printf_nolog); 803 __cpu_simple_unlock(&ld->ld_spinlock); 804 splx(s); 805 printf_nolog("\n"); 806 if (dopanic) 807 panic("LOCKDEBUG: %s error: %s,%zu: %s", 808 ld->ld_lockops->lo_name, func, line, msg); 809 } 810 811 #endif /* LOCKDEBUG */ 812 813 /* 814 * lockdebug_lock_print: 815 * 816 * Handle the DDB 'show lock' command. 817 */ 818 #ifdef DDB 819 #include <machine/db_machdep.h> 820 #include <ddb/db_interface.h> 821 822 void 823 lockdebug_lock_print(void *addr, void (*pr)(const char *, ...)) 824 { 825 #ifdef LOCKDEBUG 826 lockdebug_t *ld; 827 828 TAILQ_FOREACH(ld, &ld_all, ld_achain) { 829 if (ld->ld_lock == NULL) 830 continue; 831 if (addr == NULL || ld->ld_lock == addr) { 832 lockdebug_dump(ld, pr); 833 if (addr != NULL) 834 return; 835 } 836 } 837 if (addr != NULL) { 838 (*pr)("Sorry, no record of a lock with address %p found.\n", 839 addr); 840 } 841 #else 842 (*pr)("Sorry, kernel not built with the LOCKDEBUG option.\n"); 843 #endif /* LOCKDEBUG */ 844 } 845 846 #ifdef LOCKDEBUG 847 static void 848 lockdebug_show_all_locks_lwp(void (*pr)(const char *, ...), bool show_trace) 849 { 850 struct proc *p; 851 852 LIST_FOREACH(p, &allproc, p_list) { 853 struct lwp *l; 854 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 855 lockdebug_t *ld; 856 const char *sym; 857 int i = 0; 858 if (TAILQ_EMPTY(&l->l_ld_locks)) 859 continue; 860 (*pr)("Locks held by an LWP (%s):\n", 861 l->l_name ? l->l_name : p->p_comm); 862 TAILQ_FOREACH(ld, &l->l_ld_locks, ld_chain) { 863 ksyms_getname(NULL, &sym, 864 (vaddr_t)ld->ld_initaddr, 865 KSYMS_CLOSEST|KSYMS_PROC|KSYMS_ANY); 866 (*pr)("Lock %d (initialized at %s)\n", i++, sym); 867 lockdebug_dump(ld, pr); 868 } 869 if (show_trace) { 870 db_stack_trace_print((db_expr_t)(intptr_t)l, 871 true, 872 32 /* Limit just in case */, 873 "a", pr); 874 } 875 (*pr)("\n"); 876 } 877 } 878 } 879 880 static void 881 lockdebug_show_all_locks_cpu(void (*pr)(const char *, ...), bool show_trace) 882 { 883 lockdebug_t *ld; 884 CPU_INFO_ITERATOR cii; 885 struct cpu_info *ci; 886 const char *sym; 887 888 for (CPU_INFO_FOREACH(cii, ci)) { 889 int i = 0; 890 if (TAILQ_EMPTY(&ci->ci_data.cpu_ld_locks)) 891 continue; 892 (*pr)("Locks held on CPU %u:\n", ci->ci_index); 893 TAILQ_FOREACH(ld, &ci->ci_data.cpu_ld_locks, ld_chain) { 894 ksyms_getname(NULL, &sym, 895 (vaddr_t)ld->ld_initaddr, 896 KSYMS_CLOSEST|KSYMS_PROC|KSYMS_ANY); 897 (*pr)("Lock %d (initialized at %s)\n", i++, sym); 898 lockdebug_dump(ld, pr); 899 if (show_trace) { 900 db_stack_trace_print( 901 (db_expr_t)(intptr_t)ci->ci_curlwp, 902 true, 903 32 /* Limit just in case */, 904 "a", pr); 905 } 906 (*pr)("\n"); 907 } 908 } 909 } 910 #endif /* LOCKDEBUG */ 911 912 void 913 lockdebug_show_all_locks(void (*pr)(const char *, ...), const char *modif) 914 { 915 #ifdef LOCKDEBUG 916 bool show_trace = false; 917 if (modif[0] == 't') 918 show_trace = true; 919 920 (*pr)("[Locks tracked through LWPs]\n"); 921 lockdebug_show_all_locks_lwp(pr, show_trace); 922 (*pr)("\n"); 923 924 (*pr)("[Locks tracked through CPUs]\n"); 925 lockdebug_show_all_locks_cpu(pr, show_trace); 926 (*pr)("\n"); 927 #else 928 (*pr)("Sorry, kernel not built with the LOCKDEBUG option.\n"); 929 #endif /* LOCKDEBUG */ 930 } 931 932 void 933 lockdebug_show_lockstats(void (*pr)(const char *, ...)) 934 { 935 #ifdef LOCKDEBUG 936 lockdebug_t *ld; 937 void *_ld; 938 uint32_t n_null = 0; 939 uint32_t n_spin_mutex = 0; 940 uint32_t n_adaptive_mutex = 0; 941 uint32_t n_rwlock = 0; 942 uint32_t n_cv = 0; 943 uint32_t n_others = 0; 944 945 RB_TREE_FOREACH(_ld, &ld_rb_tree) { 946 ld = _ld; 947 if (ld->ld_lock == NULL) { 948 n_null++; 949 continue; 950 } 951 if (ld->ld_lockops->lo_type == LOCKOPS_CV) { 952 n_cv++; 953 continue; 954 } 955 if (ld->ld_lockops->lo_name[0] == 'M') { 956 if (ld->ld_lockops->lo_type == LOCKOPS_SLEEP) 957 n_adaptive_mutex++; 958 else 959 n_spin_mutex++; 960 continue; 961 } 962 if (ld->ld_lockops->lo_name[0] == 'R') { 963 n_rwlock++; 964 continue; 965 } 966 n_others++; 967 } 968 (*pr)( 969 "condvar: %u\n" 970 "spin mutex: %u\n" 971 "adaptive mutex: %u\n" 972 "rwlock: %u\n" 973 "null locks: %u\n" 974 "others: %u\n", 975 n_cv, n_spin_mutex, n_adaptive_mutex, n_rwlock, 976 n_null, n_others); 977 #else 978 (*pr)("Sorry, kernel not built with the LOCKDEBUG option.\n"); 979 #endif /* LOCKDEBUG */ 980 } 981 #endif /* DDB */ 982 983 /* 984 * lockdebug_abort: 985 * 986 * An error has been trapped - dump lock info and call panic(). 987 */ 988 void 989 lockdebug_abort(const char *func, size_t line, const volatile void *lock, 990 lockops_t *ops, const char *msg) 991 { 992 #ifdef LOCKDEBUG 993 lockdebug_t *ld; 994 int s; 995 996 s = splhigh(); 997 if ((ld = lockdebug_lookup(func, line, lock, 998 (uintptr_t) __builtin_return_address(0))) != NULL) { 999 lockdebug_abort1(func, line, ld, s, msg, true); 1000 return; 1001 } 1002 splx(s); 1003 #endif /* LOCKDEBUG */ 1004 1005 /* 1006 * Complain first on the occurrance only. Otherwise proceeed to 1007 * panic where we will `rendezvous' with other CPUs if the machine 1008 * is going down in flames. 1009 */ 1010 if (atomic_inc_uint_nv(&ld_panic) == 1) { 1011 printf_nolog("%s error: %s,%zu: %s\n\n" 1012 "lock address : %#018lx\n" 1013 "current cpu : %18d\n" 1014 "current lwp : %#018lx\n", 1015 ops->lo_name, func, line, msg, (long)lock, 1016 (int)cpu_index(curcpu()), (long)curlwp); 1017 (*ops->lo_dump)(lock); 1018 printf_nolog("\n"); 1019 } 1020 1021 panic("lock error: %s: %s,%zu: %s: lock %p cpu %d lwp %p", 1022 ops->lo_name, func, line, msg, lock, cpu_index(curcpu()), curlwp); 1023 } 1024