1 /* $NetBSD: subr_lockdebug.c,v 1.74 2020/01/21 20:31:57 ad Exp $ */ 2 3 /*- 4 * Copyright (c) 2006, 2007, 2008, 2020 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Andrew Doran. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Basic lock debugging code shared among lock primitives. 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: subr_lockdebug.c,v 1.74 2020/01/21 20:31:57 ad Exp $"); 38 39 #ifdef _KERNEL_OPT 40 #include "opt_ddb.h" 41 #endif 42 43 #include <sys/param.h> 44 #include <sys/proc.h> 45 #include <sys/systm.h> 46 #include <sys/kernel.h> 47 #include <sys/kmem.h> 48 #include <sys/lockdebug.h> 49 #include <sys/sleepq.h> 50 #include <sys/cpu.h> 51 #include <sys/atomic.h> 52 #include <sys/lock.h> 53 #include <sys/rbtree.h> 54 #include <sys/ksyms.h> 55 56 #include <machine/lock.h> 57 58 unsigned int ld_panic; 59 60 #ifdef LOCKDEBUG 61 62 #ifdef __ia64__ 63 #define LD_BATCH_SHIFT 16 64 #else 65 #define LD_BATCH_SHIFT 9 66 #endif 67 #define LD_BATCH (1 << LD_BATCH_SHIFT) 68 #define LD_BATCH_MASK (LD_BATCH - 1) 69 #define LD_MAX_LOCKS 1048576 70 #define LD_SLOP 16 71 72 #define LD_LOCKED 0x01 73 #define LD_SLEEPER 0x02 74 75 #define LD_WRITE_LOCK 0x80000000 76 77 typedef struct lockdebug { 78 struct rb_node ld_rb_node; 79 __cpu_simple_lock_t ld_spinlock; 80 _TAILQ_ENTRY(struct lockdebug, volatile) ld_chain; 81 _TAILQ_ENTRY(struct lockdebug, volatile) ld_achain; 82 volatile void *ld_lock; 83 lockops_t *ld_lockops; 84 struct lwp *ld_lwp; 85 uintptr_t ld_locked; 86 uintptr_t ld_unlocked; 87 uintptr_t ld_initaddr; 88 uint16_t ld_shares; 89 uint16_t ld_cpu; 90 uint8_t ld_flags; 91 uint8_t ld_shwant; /* advisory */ 92 uint8_t ld_exwant; /* advisory */ 93 uint8_t ld_unused; 94 } volatile lockdebug_t; 95 96 typedef _TAILQ_HEAD(lockdebuglist, struct lockdebug, volatile) lockdebuglist_t; 97 98 __cpu_simple_lock_t ld_mod_lk; 99 lockdebuglist_t ld_free = TAILQ_HEAD_INITIALIZER(ld_free); 100 lockdebuglist_t ld_all = TAILQ_HEAD_INITIALIZER(ld_all); 101 int ld_nfree; 102 int ld_freeptr; 103 int ld_recurse; 104 bool ld_nomore; 105 lockdebug_t ld_prime[LD_BATCH]; 106 107 static void lockdebug_abort1(const char *, size_t, lockdebug_t *, int, 108 const char *, bool); 109 static int lockdebug_more(int); 110 static void lockdebug_init(void); 111 static void lockdebug_dump(lwp_t *, lockdebug_t *, 112 void (*)(const char *, ...) 113 __printflike(1, 2)); 114 115 static signed int 116 ld_rbto_compare_nodes(void *ctx, const void *n1, const void *n2) 117 { 118 const lockdebug_t *ld1 = n1; 119 const lockdebug_t *ld2 = n2; 120 const uintptr_t a = (uintptr_t)ld1->ld_lock; 121 const uintptr_t b = (uintptr_t)ld2->ld_lock; 122 123 if (a < b) 124 return -1; 125 if (a > b) 126 return 1; 127 return 0; 128 } 129 130 static signed int 131 ld_rbto_compare_key(void *ctx, const void *n, const void *key) 132 { 133 const lockdebug_t *ld = n; 134 const uintptr_t a = (uintptr_t)ld->ld_lock; 135 const uintptr_t b = (uintptr_t)key; 136 137 if (a < b) 138 return -1; 139 if (a > b) 140 return 1; 141 return 0; 142 } 143 144 static rb_tree_t ld_rb_tree; 145 146 static const rb_tree_ops_t ld_rb_tree_ops = { 147 .rbto_compare_nodes = ld_rbto_compare_nodes, 148 .rbto_compare_key = ld_rbto_compare_key, 149 .rbto_node_offset = offsetof(lockdebug_t, ld_rb_node), 150 .rbto_context = NULL 151 }; 152 153 static inline lockdebug_t * 154 lockdebug_lookup1(const volatile void *lock) 155 { 156 lockdebug_t *ld; 157 struct cpu_info *ci; 158 159 ci = curcpu(); 160 __cpu_simple_lock(&ci->ci_data.cpu_ld_lock); 161 ld = rb_tree_find_node(&ld_rb_tree, (void *)(intptr_t)lock); 162 __cpu_simple_unlock(&ci->ci_data.cpu_ld_lock); 163 if (ld == NULL) { 164 return NULL; 165 } 166 __cpu_simple_lock(&ld->ld_spinlock); 167 168 return ld; 169 } 170 171 static void 172 lockdebug_lock_cpus(void) 173 { 174 CPU_INFO_ITERATOR cii; 175 struct cpu_info *ci; 176 177 for (CPU_INFO_FOREACH(cii, ci)) { 178 __cpu_simple_lock(&ci->ci_data.cpu_ld_lock); 179 } 180 } 181 182 static void 183 lockdebug_unlock_cpus(void) 184 { 185 CPU_INFO_ITERATOR cii; 186 struct cpu_info *ci; 187 188 for (CPU_INFO_FOREACH(cii, ci)) { 189 __cpu_simple_unlock(&ci->ci_data.cpu_ld_lock); 190 } 191 } 192 193 /* 194 * lockdebug_lookup: 195 * 196 * Find a lockdebug structure by a pointer to a lock and return it locked. 197 */ 198 static inline lockdebug_t * 199 lockdebug_lookup(const char *func, size_t line, const volatile void *lock, 200 uintptr_t where) 201 { 202 lockdebug_t *ld; 203 204 ld = lockdebug_lookup1(lock); 205 if (__predict_false(ld == NULL)) { 206 panic("%s,%zu: uninitialized lock (lock=%p, from=%08" 207 PRIxPTR ")", func, line, lock, where); 208 } 209 return ld; 210 } 211 212 /* 213 * lockdebug_init: 214 * 215 * Initialize the lockdebug system. Allocate an initial pool of 216 * lockdebug structures before the VM system is up and running. 217 */ 218 static void 219 lockdebug_init(void) 220 { 221 lockdebug_t *ld; 222 int i; 223 224 TAILQ_INIT(&curcpu()->ci_data.cpu_ld_locks); 225 TAILQ_INIT(&curlwp->l_ld_locks); 226 __cpu_simple_lock_init(&curcpu()->ci_data.cpu_ld_lock); 227 __cpu_simple_lock_init(&ld_mod_lk); 228 229 rb_tree_init(&ld_rb_tree, &ld_rb_tree_ops); 230 231 ld = ld_prime; 232 for (i = 1, ld++; i < LD_BATCH; i++, ld++) { 233 __cpu_simple_lock_init(&ld->ld_spinlock); 234 TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain); 235 TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain); 236 } 237 ld_freeptr = 1; 238 ld_nfree = LD_BATCH - 1; 239 } 240 241 /* 242 * lockdebug_alloc: 243 * 244 * A lock is being initialized, so allocate an associated debug 245 * structure. 246 */ 247 bool 248 lockdebug_alloc(const char *func, size_t line, volatile void *lock, 249 lockops_t *lo, uintptr_t initaddr) 250 { 251 struct cpu_info *ci; 252 lockdebug_t *ld; 253 int s; 254 255 if (__predict_false(lo == NULL || panicstr != NULL || ld_panic)) 256 return false; 257 if (__predict_false(ld_freeptr == 0)) 258 lockdebug_init(); 259 260 s = splhigh(); 261 __cpu_simple_lock(&ld_mod_lk); 262 if (__predict_false((ld = lockdebug_lookup1(lock)) != NULL)) { 263 __cpu_simple_unlock(&ld_mod_lk); 264 lockdebug_abort1(func, line, ld, s, "already initialized", 265 true); 266 return false; 267 } 268 269 /* 270 * Pinch a new debug structure. We may recurse because we call 271 * kmem_alloc(), which may need to initialize new locks somewhere 272 * down the path. If not recursing, we try to maintain at least 273 * LD_SLOP structures free, which should hopefully be enough to 274 * satisfy kmem_alloc(). If we can't provide a structure, not to 275 * worry: we'll just mark the lock as not having an ID. 276 */ 277 ci = curcpu(); 278 ci->ci_lkdebug_recurse++; 279 if (TAILQ_EMPTY(&ld_free)) { 280 if (ci->ci_lkdebug_recurse > 1 || ld_nomore) { 281 ci->ci_lkdebug_recurse--; 282 __cpu_simple_unlock(&ld_mod_lk); 283 splx(s); 284 return false; 285 } 286 s = lockdebug_more(s); 287 } else if (ci->ci_lkdebug_recurse == 1 && ld_nfree < LD_SLOP) { 288 s = lockdebug_more(s); 289 } 290 if (__predict_false((ld = TAILQ_FIRST(&ld_free)) == NULL)) { 291 __cpu_simple_unlock(&ld_mod_lk); 292 splx(s); 293 return false; 294 } 295 TAILQ_REMOVE(&ld_free, ld, ld_chain); 296 ld_nfree--; 297 ci->ci_lkdebug_recurse--; 298 299 if (__predict_false(ld->ld_lock != NULL)) { 300 panic("%s,%zu: corrupt table ld %p", func, line, ld); 301 } 302 303 /* Initialise the structure. */ 304 ld->ld_lock = lock; 305 ld->ld_lockops = lo; 306 ld->ld_locked = 0; 307 ld->ld_unlocked = 0; 308 ld->ld_lwp = NULL; 309 ld->ld_initaddr = initaddr; 310 ld->ld_flags = (lo->lo_type == LOCKOPS_SLEEP ? LD_SLEEPER : 0); 311 lockdebug_lock_cpus(); 312 (void)rb_tree_insert_node(&ld_rb_tree, __UNVOLATILE(ld)); 313 lockdebug_unlock_cpus(); 314 __cpu_simple_unlock(&ld_mod_lk); 315 316 splx(s); 317 return true; 318 } 319 320 /* 321 * lockdebug_free: 322 * 323 * A lock is being destroyed, so release debugging resources. 324 */ 325 void 326 lockdebug_free(const char *func, size_t line, volatile void *lock) 327 { 328 lockdebug_t *ld; 329 int s; 330 331 if (__predict_false(panicstr != NULL || ld_panic)) 332 return; 333 334 s = splhigh(); 335 __cpu_simple_lock(&ld_mod_lk); 336 ld = lockdebug_lookup(func, line, lock, 337 (uintptr_t) __builtin_return_address(0)); 338 if (__predict_false(ld == NULL)) { 339 __cpu_simple_unlock(&ld_mod_lk); 340 panic("%s,%zu: destroying uninitialized object %p" 341 "(ld_lock=%p)", func, line, lock, ld->ld_lock); 342 return; 343 } 344 if (__predict_false((ld->ld_flags & LD_LOCKED) != 0 || 345 ld->ld_shares != 0)) { 346 __cpu_simple_unlock(&ld_mod_lk); 347 lockdebug_abort1(func, line, ld, s, "is locked or in use", 348 true); 349 return; 350 } 351 lockdebug_lock_cpus(); 352 rb_tree_remove_node(&ld_rb_tree, __UNVOLATILE(ld)); 353 lockdebug_unlock_cpus(); 354 ld->ld_lock = NULL; 355 TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain); 356 ld_nfree++; 357 __cpu_simple_unlock(&ld->ld_spinlock); 358 __cpu_simple_unlock(&ld_mod_lk); 359 splx(s); 360 } 361 362 /* 363 * lockdebug_more: 364 * 365 * Allocate a batch of debug structures and add to the free list. 366 * Must be called with ld_mod_lk held. 367 */ 368 static int 369 lockdebug_more(int s) 370 { 371 lockdebug_t *ld; 372 void *block; 373 int i, base, m; 374 375 /* 376 * Can't call kmem_alloc() if in interrupt context. XXX We could 377 * deadlock, because we don't know which locks the caller holds. 378 */ 379 if (cpu_intr_p() || cpu_softintr_p()) { 380 return s; 381 } 382 383 while (ld_nfree < LD_SLOP) { 384 __cpu_simple_unlock(&ld_mod_lk); 385 splx(s); 386 block = kmem_zalloc(LD_BATCH * sizeof(lockdebug_t), KM_SLEEP); 387 s = splhigh(); 388 __cpu_simple_lock(&ld_mod_lk); 389 390 if (ld_nfree > LD_SLOP) { 391 /* Somebody beat us to it. */ 392 __cpu_simple_unlock(&ld_mod_lk); 393 splx(s); 394 kmem_free(block, LD_BATCH * sizeof(lockdebug_t)); 395 s = splhigh(); 396 __cpu_simple_lock(&ld_mod_lk); 397 continue; 398 } 399 400 base = ld_freeptr; 401 ld_nfree += LD_BATCH; 402 ld = block; 403 base <<= LD_BATCH_SHIFT; 404 m = uimin(LD_MAX_LOCKS, base + LD_BATCH); 405 406 if (m == LD_MAX_LOCKS) 407 ld_nomore = true; 408 409 for (i = base; i < m; i++, ld++) { 410 __cpu_simple_lock_init(&ld->ld_spinlock); 411 TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain); 412 TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain); 413 } 414 415 membar_producer(); 416 } 417 418 return s; 419 } 420 421 /* 422 * lockdebug_wantlock: 423 * 424 * Process the preamble to a lock acquire. The "shared" 425 * parameter controls which ld_{ex,sh}want counter is 426 * updated; a negative value of shared updates neither. 427 */ 428 void 429 lockdebug_wantlock(const char *func, size_t line, 430 const volatile void *lock, uintptr_t where, int shared) 431 { 432 struct lwp *l = curlwp; 433 lockdebug_t *ld; 434 bool recurse; 435 int s; 436 437 (void)shared; 438 recurse = false; 439 440 if (__predict_false(panicstr != NULL || ld_panic)) 441 return; 442 443 s = splhigh(); 444 if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) { 445 splx(s); 446 return; 447 } 448 if ((ld->ld_flags & LD_LOCKED) != 0 || ld->ld_shares != 0) { 449 if ((ld->ld_flags & LD_SLEEPER) != 0) { 450 if (ld->ld_lwp == l) 451 recurse = true; 452 } else if (ld->ld_cpu == (uint16_t)cpu_index(curcpu())) 453 recurse = true; 454 } 455 if (cpu_intr_p()) { 456 if (__predict_false((ld->ld_flags & LD_SLEEPER) != 0)) { 457 lockdebug_abort1(func, line, ld, s, 458 "acquiring sleep lock from interrupt context", 459 true); 460 return; 461 } 462 } 463 if (shared > 0) 464 ld->ld_shwant++; 465 else if (shared == 0) 466 ld->ld_exwant++; 467 if (__predict_false(recurse)) { 468 lockdebug_abort1(func, line, ld, s, "locking against myself", 469 true); 470 return; 471 } 472 if (l->l_ld_wanted == NULL) { 473 l->l_ld_wanted = ld; 474 } 475 __cpu_simple_unlock(&ld->ld_spinlock); 476 splx(s); 477 } 478 479 /* 480 * lockdebug_locked: 481 * 482 * Process a lock acquire operation. 483 */ 484 void 485 lockdebug_locked(const char *func, size_t line, 486 volatile void *lock, void *cvlock, uintptr_t where, int shared) 487 { 488 struct lwp *l = curlwp; 489 lockdebug_t *ld; 490 int s; 491 492 if (__predict_false(panicstr != NULL || ld_panic)) 493 return; 494 495 s = splhigh(); 496 if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) { 497 splx(s); 498 return; 499 } 500 if (cvlock) { 501 KASSERT(ld->ld_lockops->lo_type == LOCKOPS_CV); 502 if (lock == (void *)&lbolt) { 503 /* nothing */ 504 } else if (ld->ld_shares++ == 0) { 505 ld->ld_locked = (uintptr_t)cvlock; 506 } else if (__predict_false(cvlock != (void *)ld->ld_locked)) { 507 lockdebug_abort1(func, line, ld, s, 508 "multiple locks used with condition variable", 509 true); 510 return; 511 } 512 } else if (shared) { 513 l->l_shlocks++; 514 ld->ld_locked = where; 515 ld->ld_shares++; 516 ld->ld_shwant--; 517 } else { 518 if (__predict_false((ld->ld_flags & LD_LOCKED) != 0)) { 519 lockdebug_abort1(func, line, ld, s, "already locked", 520 true); 521 return; 522 } 523 ld->ld_flags |= LD_LOCKED; 524 ld->ld_locked = where; 525 ld->ld_exwant--; 526 if ((ld->ld_flags & LD_SLEEPER) != 0) { 527 TAILQ_INSERT_TAIL(&l->l_ld_locks, ld, ld_chain); 528 } else { 529 TAILQ_INSERT_TAIL(&curcpu()->ci_data.cpu_ld_locks, 530 ld, ld_chain); 531 } 532 } 533 ld->ld_cpu = (uint16_t)cpu_index(curcpu()); 534 ld->ld_lwp = l; 535 __cpu_simple_unlock(&ld->ld_spinlock); 536 if (l->l_ld_wanted == ld) { 537 l->l_ld_wanted = NULL; 538 } 539 splx(s); 540 } 541 542 /* 543 * lockdebug_unlocked: 544 * 545 * Process a lock release operation. 546 */ 547 void 548 lockdebug_unlocked(const char *func, size_t line, 549 volatile void *lock, uintptr_t where, int shared) 550 { 551 struct lwp *l = curlwp; 552 lockdebug_t *ld; 553 int s; 554 555 if (__predict_false(panicstr != NULL || ld_panic)) 556 return; 557 558 s = splhigh(); 559 if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) { 560 splx(s); 561 return; 562 } 563 if (ld->ld_lockops->lo_type == LOCKOPS_CV) { 564 if (lock == (void *)&lbolt) { 565 /* nothing */ 566 } else { 567 ld->ld_shares--; 568 } 569 } else if (shared) { 570 if (__predict_false(l->l_shlocks == 0)) { 571 lockdebug_abort1(func, line, ld, s, 572 "no shared locks held by LWP", true); 573 return; 574 } 575 if (__predict_false(ld->ld_shares == 0)) { 576 lockdebug_abort1(func, line, ld, s, 577 "no shared holds on this lock", true); 578 return; 579 } 580 l->l_shlocks--; 581 ld->ld_shares--; 582 if (ld->ld_lwp == l) { 583 ld->ld_unlocked = where; 584 ld->ld_lwp = NULL; 585 } 586 if (ld->ld_cpu == (uint16_t)cpu_index(curcpu())) 587 ld->ld_cpu = (uint16_t)-1; 588 } else { 589 if (__predict_false((ld->ld_flags & LD_LOCKED) == 0)) { 590 lockdebug_abort1(func, line, ld, s, "not locked", true); 591 return; 592 } 593 594 if ((ld->ld_flags & LD_SLEEPER) != 0) { 595 if (__predict_false(ld->ld_lwp != curlwp)) { 596 lockdebug_abort1(func, line, ld, s, 597 "not held by current LWP", true); 598 return; 599 } 600 TAILQ_REMOVE(&l->l_ld_locks, ld, ld_chain); 601 } else { 602 uint16_t idx = (uint16_t)cpu_index(curcpu()); 603 if (__predict_false(ld->ld_cpu != idx)) { 604 lockdebug_abort1(func, line, ld, s, 605 "not held by current CPU", true); 606 return; 607 } 608 TAILQ_REMOVE(&curcpu()->ci_data.cpu_ld_locks, ld, 609 ld_chain); 610 } 611 ld->ld_flags &= ~LD_LOCKED; 612 ld->ld_unlocked = where; 613 ld->ld_lwp = NULL; 614 } 615 __cpu_simple_unlock(&ld->ld_spinlock); 616 splx(s); 617 } 618 619 /* 620 * lockdebug_wakeup: 621 * 622 * Process a wakeup on a condition variable. 623 */ 624 void 625 lockdebug_wakeup(const char *func, size_t line, volatile void *lock, 626 uintptr_t where) 627 { 628 lockdebug_t *ld; 629 int s; 630 631 if (__predict_false(panicstr != NULL || ld_panic || lock == (void *)&lbolt)) 632 return; 633 634 s = splhigh(); 635 /* Find the CV... */ 636 if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) { 637 splx(s); 638 return; 639 } 640 /* 641 * If it has any waiters, ensure that they are using the 642 * same interlock. 643 */ 644 if (__predict_false(ld->ld_shares != 0 && 645 !mutex_owned((kmutex_t *)ld->ld_locked))) { 646 lockdebug_abort1(func, line, ld, s, "interlocking mutex not " 647 "held during wakeup", true); 648 return; 649 } 650 __cpu_simple_unlock(&ld->ld_spinlock); 651 splx(s); 652 } 653 654 /* 655 * lockdebug_barrier: 656 * 657 * Panic if we hold more than one specified lock, and optionally, if we 658 * hold any sleep locks. 659 */ 660 void 661 lockdebug_barrier(const char *func, size_t line, volatile void *onelock, 662 int slplocks) 663 { 664 struct lwp *l = curlwp; 665 lockdebug_t *ld; 666 int s; 667 668 if (__predict_false(panicstr != NULL || ld_panic)) 669 return; 670 671 s = splhigh(); 672 if ((l->l_pflag & LP_INTR) == 0) { 673 TAILQ_FOREACH(ld, &curcpu()->ci_data.cpu_ld_locks, ld_chain) { 674 if (ld->ld_lock == onelock) { 675 continue; 676 } 677 __cpu_simple_lock(&ld->ld_spinlock); 678 lockdebug_abort1(func, line, ld, s, 679 "spin lock held", true); 680 return; 681 } 682 } 683 if (slplocks) { 684 splx(s); 685 return; 686 } 687 ld = TAILQ_FIRST(&l->l_ld_locks); 688 if (__predict_false(ld != NULL && ld->ld_lock != onelock)) { 689 __cpu_simple_lock(&ld->ld_spinlock); 690 lockdebug_abort1(func, line, ld, s, "sleep lock held", true); 691 return; 692 } 693 splx(s); 694 if (l->l_shlocks != 0) { 695 TAILQ_FOREACH(ld, &ld_all, ld_achain) { 696 if (ld->ld_lock == onelock) { 697 continue; 698 } 699 if (ld->ld_lockops->lo_type == LOCKOPS_CV) 700 continue; 701 if (ld->ld_lwp == l) 702 lockdebug_dump(l, ld, printf); 703 } 704 panic("%s,%zu: holding %d shared locks", func, line, 705 l->l_shlocks); 706 } 707 } 708 709 /* 710 * lockdebug_mem_check: 711 * 712 * Check for in-use locks within a memory region that is 713 * being freed. 714 */ 715 void 716 lockdebug_mem_check(const char *func, size_t line, void *base, size_t sz) 717 { 718 lockdebug_t *ld; 719 struct cpu_info *ci; 720 int s; 721 722 if (__predict_false(panicstr != NULL || ld_panic)) 723 return; 724 725 s = splhigh(); 726 ci = curcpu(); 727 __cpu_simple_lock(&ci->ci_data.cpu_ld_lock); 728 ld = (lockdebug_t *)rb_tree_find_node_geq(&ld_rb_tree, base); 729 if (ld != NULL) { 730 const uintptr_t lock = (uintptr_t)ld->ld_lock; 731 732 if (__predict_false((uintptr_t)base > lock)) 733 panic("%s,%zu: corrupt tree ld=%p, base=%p, sz=%zu", 734 func, line, ld, base, sz); 735 if (lock >= (uintptr_t)base + sz) 736 ld = NULL; 737 } 738 __cpu_simple_unlock(&ci->ci_data.cpu_ld_lock); 739 if (__predict_false(ld != NULL)) { 740 __cpu_simple_lock(&ld->ld_spinlock); 741 lockdebug_abort1(func, line, ld, s, 742 "allocation contains active lock", !cold); 743 return; 744 } 745 splx(s); 746 } 747 748 /* 749 * lockdebug_dump: 750 * 751 * Dump information about a lock on panic, or for DDB. 752 */ 753 static void 754 lockdebug_dump(lwp_t *l, lockdebug_t *ld, void (*pr)(const char *, ...) 755 __printflike(1, 2)) 756 { 757 int sleeper = (ld->ld_flags & LD_SLEEPER); 758 759 (*pr)( 760 "lock address : %#018lx type : %18s\n" 761 "initialized : %#018lx", 762 (long)ld->ld_lock, (sleeper ? "sleep/adaptive" : "spin"), 763 (long)ld->ld_initaddr); 764 765 if (ld->ld_lockops->lo_type == LOCKOPS_CV) { 766 (*pr)(" interlock: %#018lx\n", (long)ld->ld_locked); 767 } else { 768 (*pr)("\n" 769 "shared holds : %18u exclusive: %18u\n" 770 "shares wanted: %18u exclusive: %18u\n" 771 "relevant cpu : %18u last held: %18u\n" 772 "relevant lwp : %#018lx last held: %#018lx\n" 773 "last locked%c : %#018lx unlocked%c: %#018lx\n", 774 (unsigned)ld->ld_shares, ((ld->ld_flags & LD_LOCKED) != 0), 775 (unsigned)ld->ld_shwant, (unsigned)ld->ld_exwant, 776 (unsigned)cpu_index(l->l_cpu), (unsigned)ld->ld_cpu, 777 (long)l, (long)ld->ld_lwp, 778 ((ld->ld_flags & LD_LOCKED) ? '*' : ' '), 779 (long)ld->ld_locked, 780 ((ld->ld_flags & LD_LOCKED) ? ' ' : '*'), 781 (long)ld->ld_unlocked); 782 } 783 784 if (ld->ld_lockops->lo_dump != NULL) 785 (*ld->ld_lockops->lo_dump)(ld->ld_lock, pr); 786 787 if (sleeper) { 788 turnstile_print(ld->ld_lock, pr); 789 } 790 } 791 792 /* 793 * lockdebug_abort1: 794 * 795 * An error has been trapped - dump lock info and panic. 796 */ 797 static void 798 lockdebug_abort1(const char *func, size_t line, lockdebug_t *ld, int s, 799 const char *msg, bool dopanic) 800 { 801 802 /* 803 * Don't make the situation worse if the system is already going 804 * down in flames. Once a panic is triggered, lockdebug state 805 * becomes stale and cannot be trusted. 806 */ 807 if (atomic_inc_uint_nv(&ld_panic) != 1) { 808 __cpu_simple_unlock(&ld->ld_spinlock); 809 splx(s); 810 return; 811 } 812 813 printf_nolog("%s error: %s,%zu: %s\n\n", ld->ld_lockops->lo_name, 814 func, line, msg); 815 lockdebug_dump(curlwp, ld, printf_nolog); 816 __cpu_simple_unlock(&ld->ld_spinlock); 817 splx(s); 818 printf_nolog("\n"); 819 if (dopanic) 820 panic("LOCKDEBUG: %s error: %s,%zu: %s", 821 ld->ld_lockops->lo_name, func, line, msg); 822 } 823 824 #endif /* LOCKDEBUG */ 825 826 /* 827 * lockdebug_lock_print: 828 * 829 * Handle the DDB 'show lock' command. 830 */ 831 #ifdef DDB 832 #include <machine/db_machdep.h> 833 #include <ddb/db_interface.h> 834 835 void 836 lockdebug_lock_print(void *addr, 837 void (*pr)(const char *, ...) __printflike(1, 2)) 838 { 839 #ifdef LOCKDEBUG 840 lockdebug_t *ld; 841 842 TAILQ_FOREACH(ld, &ld_all, ld_achain) { 843 if (ld->ld_lock == NULL) 844 continue; 845 if (addr == NULL || ld->ld_lock == addr) { 846 lockdebug_dump(curlwp, ld, pr); 847 if (addr != NULL) 848 return; 849 } 850 } 851 if (addr != NULL) { 852 (*pr)("Sorry, no record of a lock with address %p found.\n", 853 addr); 854 } 855 #else 856 (*pr)("Sorry, kernel not built with the LOCKDEBUG option.\n"); 857 #endif /* LOCKDEBUG */ 858 } 859 860 #ifdef LOCKDEBUG 861 static void 862 lockdebug_show_one(lwp_t *l, lockdebug_t *ld, int i, 863 void (*pr)(const char *, ...) __printflike(1, 2)) 864 { 865 const char *sym; 866 867 ksyms_getname(NULL, &sym, (vaddr_t)ld->ld_initaddr, 868 KSYMS_CLOSEST|KSYMS_PROC|KSYMS_ANY); 869 (*pr)("* Lock %d (initialized at %s)\n", i++, sym); 870 lockdebug_dump(l, ld, pr); 871 } 872 873 static void 874 lockdebug_show_trace(const void *ptr, 875 void (*pr)(const char *, ...) __printflike(1, 2)) 876 { 877 db_stack_trace_print((db_expr_t)(intptr_t)ptr, true, 32, "a", pr); 878 } 879 880 static void 881 lockdebug_show_all_locks_lwp(void (*pr)(const char *, ...) __printflike(1, 2), 882 bool show_trace) 883 { 884 struct proc *p; 885 886 LIST_FOREACH(p, &allproc, p_list) { 887 struct lwp *l; 888 LIST_FOREACH(l, &p->p_lwps, l_sibling) { 889 lockdebug_t *ld; 890 int i = 0; 891 if (TAILQ_EMPTY(&l->l_ld_locks) && 892 l->l_ld_wanted == NULL) { 893 continue; 894 } 895 (*pr)("\n****** LWP %d.%d (%s) @ %p, l_stat=%d\n", 896 p->p_pid, l->l_lid, 897 l->l_name ? l->l_name : p->p_comm, l, l->l_stat); 898 if (!TAILQ_EMPTY(&l->l_ld_locks)) { 899 (*pr)("\n*** Locks held: \n"); 900 TAILQ_FOREACH(ld, &l->l_ld_locks, ld_chain) { 901 (*pr)("\n"); 902 lockdebug_show_one(l, ld, i++, pr); 903 } 904 } else { 905 (*pr)("\n*** Locks held: none\n"); 906 } 907 908 if (l->l_ld_wanted != NULL) { 909 (*pr)("\n*** Locks wanted: \n\n"); 910 lockdebug_show_one(l, l->l_ld_wanted, 0, pr); 911 } else { 912 (*pr)("\n*** Locks wanted: none\n"); 913 } 914 if (show_trace) { 915 (*pr)("\n*** Traceback: \n\n"); 916 lockdebug_show_trace(l, pr); 917 (*pr)("\n"); 918 } 919 } 920 } 921 } 922 923 static void 924 lockdebug_show_all_locks_cpu(void (*pr)(const char *, ...) __printflike(1, 2), 925 bool show_trace) 926 { 927 lockdebug_t *ld; 928 CPU_INFO_ITERATOR cii; 929 struct cpu_info *ci; 930 931 for (CPU_INFO_FOREACH(cii, ci)) { 932 int i = 0; 933 if (TAILQ_EMPTY(&ci->ci_data.cpu_ld_locks)) 934 continue; 935 (*pr)("\n******* Locks held on %s:\n", cpu_name(ci)); 936 TAILQ_FOREACH(ld, &ci->ci_data.cpu_ld_locks, ld_chain) { 937 (*pr)("\n"); 938 #ifdef MULTIPROCESSOR 939 lockdebug_show_one(ci->ci_curlwp, ld, i++, pr); 940 if (show_trace) 941 lockdebug_show_trace(ci->ci_curlwp, pr); 942 #else 943 lockdebug_show_one(curlwp, ld, i++, pr); 944 if (show_trace) 945 lockdebug_show_trace(curlwp, pr); 946 #endif 947 } 948 } 949 } 950 #endif /* LOCKDEBUG */ 951 952 void 953 lockdebug_show_all_locks(void (*pr)(const char *, ...) __printflike(1, 2), 954 const char *modif) 955 { 956 #ifdef LOCKDEBUG 957 bool show_trace = false; 958 if (modif[0] == 't') 959 show_trace = true; 960 961 (*pr)("[Locks tracked through LWPs]\n"); 962 lockdebug_show_all_locks_lwp(pr, show_trace); 963 (*pr)("\n"); 964 965 (*pr)("[Locks tracked through CPUs]\n"); 966 lockdebug_show_all_locks_cpu(pr, show_trace); 967 (*pr)("\n"); 968 #else 969 (*pr)("Sorry, kernel not built with the LOCKDEBUG option.\n"); 970 #endif /* LOCKDEBUG */ 971 } 972 973 void 974 lockdebug_show_lockstats(void (*pr)(const char *, ...) __printflike(1, 2)) 975 { 976 #ifdef LOCKDEBUG 977 lockdebug_t *ld; 978 void *_ld; 979 uint32_t n_null = 0; 980 uint32_t n_spin_mutex = 0; 981 uint32_t n_adaptive_mutex = 0; 982 uint32_t n_rwlock = 0; 983 uint32_t n_cv = 0; 984 uint32_t n_others = 0; 985 986 RB_TREE_FOREACH(_ld, &ld_rb_tree) { 987 ld = _ld; 988 if (ld->ld_lock == NULL) { 989 n_null++; 990 continue; 991 } 992 if (ld->ld_lockops->lo_type == LOCKOPS_CV) { 993 n_cv++; 994 continue; 995 } 996 if (ld->ld_lockops->lo_name[0] == 'M') { 997 if (ld->ld_lockops->lo_type == LOCKOPS_SLEEP) 998 n_adaptive_mutex++; 999 else 1000 n_spin_mutex++; 1001 continue; 1002 } 1003 if (ld->ld_lockops->lo_name[0] == 'R') { 1004 n_rwlock++; 1005 continue; 1006 } 1007 n_others++; 1008 } 1009 (*pr)( 1010 "condvar: %u\n" 1011 "spin mutex: %u\n" 1012 "adaptive mutex: %u\n" 1013 "rwlock: %u\n" 1014 "null locks: %u\n" 1015 "others: %u\n", 1016 n_cv, n_spin_mutex, n_adaptive_mutex, n_rwlock, 1017 n_null, n_others); 1018 #else 1019 (*pr)("Sorry, kernel not built with the LOCKDEBUG option.\n"); 1020 #endif /* LOCKDEBUG */ 1021 } 1022 #endif /* DDB */ 1023 1024 /* 1025 * lockdebug_dismiss: 1026 * 1027 * The system is rebooting, and potentially from an unsafe 1028 * place so avoid any future aborts. 1029 */ 1030 void 1031 lockdebug_dismiss(void) 1032 { 1033 1034 atomic_inc_uint_nv(&ld_panic); 1035 } 1036 1037 /* 1038 * lockdebug_abort: 1039 * 1040 * An error has been trapped - dump lock info and call panic(). 1041 */ 1042 void 1043 lockdebug_abort(const char *func, size_t line, const volatile void *lock, 1044 lockops_t *ops, const char *msg) 1045 { 1046 #ifdef LOCKDEBUG 1047 lockdebug_t *ld; 1048 int s; 1049 1050 s = splhigh(); 1051 if ((ld = lockdebug_lookup(func, line, lock, 1052 (uintptr_t) __builtin_return_address(0))) != NULL) { 1053 lockdebug_abort1(func, line, ld, s, msg, true); 1054 return; 1055 } 1056 splx(s); 1057 #endif /* LOCKDEBUG */ 1058 1059 /* 1060 * Don't make the situation worse if the system is already going 1061 * down in flames. Once a panic is triggered, lockdebug state 1062 * becomes stale and cannot be trusted. 1063 */ 1064 if (atomic_inc_uint_nv(&ld_panic) > 1) 1065 return; 1066 1067 printf_nolog("%s error: %s,%zu: %s\n\n" 1068 "lock address : %#018lx\n" 1069 "current cpu : %18d\n" 1070 "current lwp : %#018lx\n", 1071 ops->lo_name, func, line, msg, (long)lock, 1072 (int)cpu_index(curcpu()), (long)curlwp); 1073 (*ops->lo_dump)(lock, printf_nolog); 1074 printf_nolog("\n"); 1075 1076 panic("lock error: %s: %s,%zu: %s: lock %p cpu %d lwp %p", 1077 ops->lo_name, func, line, msg, lock, cpu_index(curcpu()), curlwp); 1078 } 1079