149 	const lockdebug_t *ld = n;  in ld_rbto_compare_key()  local
150 	const uintptr_t a = (uintptr_t)ld->ld_lock;  in ld_rbto_compare_key()
172 	lockdebug_t *ld;  in lockdebug_lookup1()  local
177 	ld = rb_tree_find_node(&ld_rb_tree, (void *)(intptr_t)lock);  in lockdebug_lookup1()
179 	if (ld == NULL) {  in lockdebug_lookup1()
182 	__cpu_simple_lock(&ld->ld_spinlock);  in lockdebug_lookup1()
184 	return ld;  in lockdebug_lookup1()
218 	lockdebug_t *ld;  in lockdebug_lookup()  local
221 	ld = lockdebug_lookup1(lock);  in lockdebug_lookup()
224 	if (__predict_false(ld == NULL)) {  in lockdebug_lookup()
228 	return ld;  in lockdebug_lookup()
240 	lockdebug_t *ld;  in lockdebug_init()  local
250 	ld = ld_prime;  in lockdebug_init()
251 	for (i = 1, ld++; i < LD_BATCH; i++, ld++) {  in lockdebug_init()
252 		__cpu_simple_lock_init(&ld->ld_spinlock);  in lockdebug_init()
253 		TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);  in lockdebug_init()
254 		TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain);  in lockdebug_init()
271 	lockdebug_t *ld;  in lockdebug_alloc()  local
281 	if (__predict_false((ld = lockdebug_lookup1(lock)) != NULL)) {  in lockdebug_alloc()
283 		lockdebug_abort1(func, line, ld, s, "already initialized",  in lockdebug_alloc()
309 	if (__predict_false((ld = TAILQ_FIRST(&ld_free)) == NULL)) {  in lockdebug_alloc()
314 	TAILQ_REMOVE(&ld_free, ld, ld_chain);  in lockdebug_alloc()
318 	if (__predict_false(ld->ld_lock != NULL)) {  in lockdebug_alloc()
319 		panic("%s,%zu: corrupt table ld %p", func, line, ld);  in lockdebug_alloc()
323 	ld->ld_lock = lock;  in lockdebug_alloc()
324 	ld->ld_lockops = lo;  in lockdebug_alloc()
325 	ld->ld_locked = 0;  in lockdebug_alloc()
326 	ld->ld_unlocked = 0;  in lockdebug_alloc()
327 	ld->ld_lwp = NULL;  in lockdebug_alloc()
328 	ld->ld_initaddr = initaddr;  in lockdebug_alloc()
329 	ld->ld_flags = (lo->lo_type == LOCKOPS_SLEEP ? LD_SLEEPER : 0);  in lockdebug_alloc()
331 	(void)rb_tree_insert_node(&ld_rb_tree, __UNVOLATILE(ld));  in lockdebug_alloc()
347 	lockdebug_t *ld;  in lockdebug_free()  local
355 	ld = lockdebug_lookup(func, line, lock,  in lockdebug_free()
357 	if (__predict_false(ld == NULL)) {  in lockdebug_free()
360 		    "(ld_lock=%p)", func, line, lock, ld->ld_lock);  in lockdebug_free()
363 	if (__predict_false((ld->ld_flags & LD_LOCKED) != 0 ||  in lockdebug_free()
364 	    ld->ld_shares != 0)) {  in lockdebug_free()
366 		lockdebug_abort1(func, line, ld, s, "is locked or in use",  in lockdebug_free()
371 	rb_tree_remove_node(&ld_rb_tree, __UNVOLATILE(ld));  in lockdebug_free()
373 	ld->ld_lock = NULL;  in lockdebug_free()
374 	TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);  in lockdebug_free()
376 	__cpu_simple_unlock(&ld->ld_spinlock);  in lockdebug_free()
390 	lockdebug_t *ld;  in lockdebug_more()  local
421 		ld = block;  in lockdebug_more()
428 		for (i = base; i < m; i++, ld++) {  in lockdebug_more()
429 			__cpu_simple_lock_init(&ld->ld_spinlock);  in lockdebug_more()
430 			TAILQ_INSERT_TAIL(&ld_free, ld, ld_chain);  in lockdebug_more()
431 			TAILQ_INSERT_TAIL(&ld_all, ld, ld_achain);  in lockdebug_more()
452 	lockdebug_t *ld;  in lockdebug_wantlock()  local
463 	if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) {  in lockdebug_wantlock()
467 	if ((ld->ld_flags & LD_LOCKED) != 0 || ld->ld_shares != 0) {  in lockdebug_wantlock()
468 		if ((ld->ld_flags & LD_SLEEPER) != 0) {  in lockdebug_wantlock()
469 			if (ld->ld_lwp == l)  in lockdebug_wantlock()
471 		} else if (ld->ld_cpu == (uint16_t)cpu_index(curcpu()))  in lockdebug_wantlock()
475 		if (__predict_false((ld->ld_flags & LD_SLEEPER) != 0)) {  in lockdebug_wantlock()
476 			lockdebug_abort1(func, line, ld, s,  in lockdebug_wantlock()
483 		ld->ld_shwant++;  in lockdebug_wantlock()
485 		ld->ld_exwant++;  in lockdebug_wantlock()
487 		lockdebug_abort1(func, line, ld, s, "locking against myself",  in lockdebug_wantlock()
492 		l->l_ld_wanted = ld;  in lockdebug_wantlock()
494 	__cpu_simple_unlock(&ld->ld_spinlock);  in lockdebug_wantlock()
508 	lockdebug_t *ld;  in lockdebug_locked()  local
515 	if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) {  in lockdebug_locked()
521 		ld->ld_locked = where;  in lockdebug_locked()
522 		ld->ld_shares++;  in lockdebug_locked()
523 		ld->ld_shwant--;  in lockdebug_locked()
525 		if (__predict_false((ld->ld_flags & LD_LOCKED) != 0)) {  in lockdebug_locked()
526 			lockdebug_abort1(func, line, ld, s, "already locked",  in lockdebug_locked()
530 		ld->ld_flags |= LD_LOCKED;  in lockdebug_locked()
531 		ld->ld_locked = where;  in lockdebug_locked()
532 		ld->ld_exwant--;  in lockdebug_locked()
533 		if ((ld->ld_flags & LD_SLEEPER) != 0) {  in lockdebug_locked()
534 			TAILQ_INSERT_TAIL(&l->l_ld_locks, ld, ld_chain);  in lockdebug_locked()
537 			    ld, ld_chain);  in lockdebug_locked()
540 	ld->ld_cpu = (uint16_t)cpu_index(curcpu());  in lockdebug_locked()
541 	ld->ld_lwp = l;  in lockdebug_locked()
542 	__cpu_simple_unlock(&ld->ld_spinlock);  in lockdebug_locked()
543 	if (l->l_ld_wanted == ld) {  in lockdebug_locked()
559 	lockdebug_t *ld;  in lockdebug_unlocked()  local
566 	if ((ld = lockdebug_lookup(func, line, lock, where)) == NULL) {  in lockdebug_unlocked()
572 			lockdebug_abort1(func, line, ld, s,  in lockdebug_unlocked()
576 		if (__predict_false(ld->ld_shares == 0)) {  in lockdebug_unlocked()
577 			lockdebug_abort1(func, line, ld, s,  in lockdebug_unlocked()
582 		ld->ld_shares--;  in lockdebug_unlocked()
583 		if (ld->ld_lwp == l) {  in lockdebug_unlocked()
584 			ld->ld_unlocked = where;  in lockdebug_unlocked()
585 			ld->ld_lwp = NULL;  in lockdebug_unlocked()
587 		if (ld->ld_cpu == (uint16_t)cpu_index(curcpu()))  in lockdebug_unlocked()
588 			ld->ld_cpu = (uint16_t)-1;  in lockdebug_unlocked()
590 		if (__predict_false((ld->ld_flags & LD_LOCKED) == 0)) {  in lockdebug_unlocked()
591 			lockdebug_abort1(func, line, ld, s, "not locked", true);  in lockdebug_unlocked()
595 		if ((ld->ld_flags & LD_SLEEPER) != 0) {  in lockdebug_unlocked()
596 			if (__predict_false(ld->ld_lwp != curlwp)) {  in lockdebug_unlocked()
597 				lockdebug_abort1(func, line, ld, s,  in lockdebug_unlocked()
601 			TAILQ_REMOVE(&l->l_ld_locks, ld, ld_chain);  in lockdebug_unlocked()
604 			if (__predict_false(ld->ld_cpu != idx)) {  in lockdebug_unlocked()
605 				lockdebug_abort1(func, line, ld, s,  in lockdebug_unlocked()
609 			TAILQ_REMOVE(&curcpu()->ci_data.cpu_ld_locks, ld,  in lockdebug_unlocked()
612 		ld->ld_flags &= ~LD_LOCKED;  in lockdebug_unlocked()
613 		ld->ld_unlocked = where;  in lockdebug_unlocked()
614 		ld->ld_lwp = NULL;  in lockdebug_unlocked()
616 	__cpu_simple_unlock(&ld->ld_spinlock);  in lockdebug_unlocked()
631 	lockdebug_t *ld;  in lockdebug_barrier()  local
639 		TAILQ_FOREACH(ld, &curcpu()->ci_data.cpu_ld_locks, ld_chain) {  in lockdebug_barrier()
640 			if (ld->ld_lock == onelock) {  in lockdebug_barrier()
643 			__cpu_simple_lock(&ld->ld_spinlock);  in lockdebug_barrier()
644 			lockdebug_abort1(func, line, ld, s,  in lockdebug_barrier()
653 	ld = TAILQ_FIRST(&l->l_ld_locks);  in lockdebug_barrier()
654 	if (__predict_false(ld != NULL && ld->ld_lock != onelock)) {  in lockdebug_barrier()
655 		__cpu_simple_lock(&ld->ld_spinlock);  in lockdebug_barrier()
656 		lockdebug_abort1(func, line, ld, s, "sleep lock held", true);  in lockdebug_barrier()
661 		TAILQ_FOREACH(ld, &ld_all, ld_achain) {  in lockdebug_barrier()
662 			if (ld->ld_lock == onelock) {  in lockdebug_barrier()
665 			if (ld->ld_lwp == l)  in lockdebug_barrier()
666 				lockdebug_dump(l, ld, printf);  in lockdebug_barrier()
682 	lockdebug_t *ld;  in lockdebug_mem_check()  local
694 	ld = (lockdebug_t *)rb_tree_find_node_geq(&ld_rb_tree, base);  in lockdebug_mem_check()
695 	if (ld != NULL) {  in lockdebug_mem_check()
696 		const uintptr_t lock = (uintptr_t)ld->ld_lock;  in lockdebug_mem_check()
700 			    func, line, ld, base, sz);  in lockdebug_mem_check()
702 			ld = NULL;  in lockdebug_mem_check()
705 	if (__predict_false(ld != NULL)) {  in lockdebug_mem_check()
706 		__cpu_simple_lock(&ld->ld_spinlock);  in lockdebug_mem_check()
707 		lockdebug_abort1(func, line, ld, s,  in lockdebug_mem_check()
724 lockdebug_dump(lwp_t *l, lockdebug_t *ld, void (*pr)(const char *, ...)
727 	int sleeper = (ld->ld_flags & LD_SLEEPER);
728 	lockops_t *lo = ld->ld_lockops;
732 	db_symstr(locksym, sizeof(locksym), (db_expr_t)(intptr_t)ld->ld_lock,
734 	db_symstr(initsym, sizeof(initsym), (db_expr_t)ld->ld_initaddr,
736 	db_symstr(lockedsym, sizeof(lockedsym), (db_expr_t)ld->ld_locked,
738 	db_symstr(unlockedsym, sizeof(unlockedsym), (db_expr_t)ld->ld_unlocked,
742 	    (unsigned long)ld->ld_lock);
744 	    (unsigned long)ld->ld_initaddr);
746 	    (unsigned long)ld->ld_locked);
748 	    (unsigned long)ld->ld_unlocked);
761 	db_read_bytes((db_addr_t)ld->ld_lockops, sizeof(los), (char *)lo);
770 	    (unsigned)ld->ld_shares, ((ld->ld_flags & LD_LOCKED) != 0),
771 	    (unsigned)ld->ld_shwant, (unsigned)ld->ld_exwant,
772 	    (unsigned)cpu_index(l->l_cpu), (unsigned)ld->ld_cpu,
773 	    (long)l, (long)ld->ld_lwp,
774 	    ((ld->ld_flags & LD_LOCKED) ? '*' : ' '),
776 	    ((ld->ld_flags & LD_LOCKED) ? ' ' : '*'),
781 		(*lo->lo_dump)(ld->ld_lock, pr);
784 		turnstile_print(ld->ld_lock, pr);
796 lockdebug_abort1(const char *func, size_t line, lockdebug_t *ld, int s,  in lockdebug_abort1()  argument
806 		__cpu_simple_unlock(&ld->ld_spinlock);  in lockdebug_abort1()
811 	printf("%s error: %s,%zu: %s\n\n", ld->ld_lockops->lo_name,  in lockdebug_abort1()
813 	lockdebug_dump(curlwp, ld, printf);  in lockdebug_abort1()
814 	__cpu_simple_unlock(&ld->ld_spinlock);  in lockdebug_abort1()
819 		    ld->ld_lockops->lo_name, func, line, msg);  in lockdebug_abort1()
836 	lockdebug_t *ld, lds;  variable
838 	TAILQ_FOREACH(ld, &ld_all, ld_achain) {
839 		db_read_bytes((db_addr_t)ld, sizeof(lds), __UNVOLATILE(&lds));
840 		ld = &lds;
841 		if (ld->ld_lock == NULL)
843 		if (addr == NULL || ld->ld_lock == addr) {
844 			lockdebug_dump(curlwp, ld, pr);
868 lockdebug_show_one(lwp_t *l, lockdebug_t *ld, int i,
874 	db_symstr(sym, sizeof(sym), (db_expr_t)ld->ld_initaddr, DB_STGY_PROC);
876 	snprintf(sym, sizeof(sym), "%p", (void *)ld->ld_initaddr);
879 	lockdebug_dump(l, ld, pr);
899 			lockdebug_t *ld;  variable
910 				TAILQ_FOREACH(ld, &l->l_ld_locks, ld_chain) {
912 					lockdebug_show_one(l, ld, i++, pr);
937 	lockdebug_t *ld;  variable
946 		TAILQ_FOREACH(ld, &ci->ci_data.cpu_ld_locks, ld_chain) {
949 			lockdebug_show_one(ci->ci_curlwp, ld, i++, pr);
953 			lockdebug_show_one(curlwp, ld, i++, pr);
989 	lockdebug_t *ld;  variable
998 		ld = _ld;
999 		if (ld->ld_lock == NULL) {
1003 		if (ld->ld_lockops->lo_name[0] == 'M') {
1004 			if (ld->ld_lockops->lo_type == LOCKOPS_SLEEP)
1010 		if (ld->ld_lockops->lo_name[0] == 'R') {
1055 	lockdebug_t *ld;  in lockdebug_abort()  local
1059 	if ((ld = lockdebug_lookup(func, line, lock,  in lockdebug_abort()
1061 		lockdebug_abort1(func, line, ld, s, msg, true);  in lockdebug_abort()