xref: /netbsd-src/sys/kern/kern_lock.c (revision 8feb0f0b7eaff0608f8350bbfa3098827b4bb91b)
1 /*	$NetBSD: kern_lock.c,v 1.182 2023/01/27 09:28:41 ozaki-r Exp $	*/
2 
3 /*-
4  * Copyright (c) 2002, 2006, 2007, 2008, 2009, 2020 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9  * NASA Ames Research Center, and by Andrew Doran.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30  * POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 #include <sys/cdefs.h>
34 __KERNEL_RCSID(0, "$NetBSD: kern_lock.c,v 1.182 2023/01/27 09:28:41 ozaki-r Exp $");
35 
36 #ifdef _KERNEL_OPT
37 #include "opt_lockdebug.h"
38 #endif
39 
40 #include <sys/param.h>
41 #include <sys/proc.h>
42 #include <sys/lock.h>
43 #include <sys/systm.h>
44 #include <sys/kernel.h>
45 #include <sys/lockdebug.h>
46 #include <sys/cpu.h>
47 #include <sys/syslog.h>
48 #include <sys/atomic.h>
49 #include <sys/lwp.h>
50 #include <sys/pserialize.h>
51 
52 #if defined(DIAGNOSTIC) && !defined(LOCKDEBUG)
53 #include <sys/ksyms.h>
54 #endif
55 
56 #include <machine/lock.h>
57 
58 #include <dev/lockstat.h>
59 
60 #define	RETURN_ADDRESS	(uintptr_t)__builtin_return_address(0)
61 
62 bool	kernel_lock_dodebug;
63 
64 __cpu_simple_lock_t kernel_lock[CACHE_LINE_SIZE / sizeof(__cpu_simple_lock_t)]
65     __cacheline_aligned;
66 
67 void
68 assert_sleepable(void)
69 {
70 	const char *reason;
71 	uint64_t pctr;
72 	bool idle;
73 
74 	if (__predict_false(panicstr != NULL)) {
75 		return;
76 	}
77 
78 	LOCKDEBUG_BARRIER(kernel_lock, 1);
79 
80 	/*
81 	 * Avoid disabling/re-enabling preemption here since this
82 	 * routine may be called in delicate situations.
83 	 */
84 	do {
85 		pctr = lwp_pctr();
86 		__insn_barrier();
87 		idle = CURCPU_IDLE_P();
88 		__insn_barrier();
89 	} while (pctr != lwp_pctr());
90 
91 	reason = NULL;
92 	if (idle && !cold) {
93 		reason = "idle";
94 	}
95 	if (cpu_intr_p()) {
96 		reason = "interrupt";
97 	}
98 	if (cpu_softintr_p()) {
99 		reason = "softint";
100 	}
101 	if (!pserialize_not_in_read_section()) {
102 		reason = "pserialize";
103 	}
104 
105 	if (reason) {
106 		panic("%s: %s caller=%p", __func__, reason,
107 		    (void *)RETURN_ADDRESS);
108 	}
109 }
110 
111 /*
112  * Functions for manipulating the kernel_lock.  We put them here
113  * so that they show up in profiles.
114  */
115 
116 #define	_KERNEL_LOCK_ABORT(msg)						\
117     LOCKDEBUG_ABORT(__func__, __LINE__, kernel_lock, &_kernel_lock_ops, msg)
118 
119 #ifdef LOCKDEBUG
120 #define	_KERNEL_LOCK_ASSERT(cond)					\
121 do {									\
122 	if (!(cond))							\
123 		_KERNEL_LOCK_ABORT("assertion failed: " #cond);		\
124 } while (/* CONSTCOND */ 0)
125 #else
126 #define	_KERNEL_LOCK_ASSERT(cond)	/* nothing */
127 #endif
128 
129 static void	_kernel_lock_dump(const volatile void *, lockop_printer_t);
130 
131 lockops_t _kernel_lock_ops = {
132 	.lo_name = "Kernel lock",
133 	.lo_type = LOCKOPS_SPIN,
134 	.lo_dump = _kernel_lock_dump,
135 };
136 
137 #ifdef LOCKDEBUG
138 
139 #include <ddb/ddb.h>
140 
141 static void
142 kernel_lock_trace_ipi(void *cookie)
143 {
144 
145 	printf("%s[%d %s]: hogging kernel lock\n", cpu_name(curcpu()),
146 	    curlwp->l_lid,
147 	    curlwp->l_name ? curlwp->l_name : curproc->p_comm);
148 	db_stacktrace();
149 }
150 
151 #endif
152 
153 /*
154  * Initialize the kernel lock.
155  */
156 void
157 kernel_lock_init(void)
158 {
159 
160 	__cpu_simple_lock_init(kernel_lock);
161 	kernel_lock_dodebug = LOCKDEBUG_ALLOC(kernel_lock, &_kernel_lock_ops,
162 	    RETURN_ADDRESS);
163 }
164 CTASSERT(CACHE_LINE_SIZE >= sizeof(__cpu_simple_lock_t));
165 
166 /*
167  * Print debugging information about the kernel lock.
168  */
169 static void
170 _kernel_lock_dump(const volatile void *junk, lockop_printer_t pr)
171 {
172 	struct cpu_info *ci = curcpu();
173 
174 	(void)junk;
175 
176 	pr("curcpu holds : %18d wanted by: %#018lx\n",
177 	    ci->ci_biglock_count, (long)ci->ci_biglock_wanted);
178 }
179 
180 /*
181  * Acquire 'nlocks' holds on the kernel lock.
182  *
183  * Although it may not look it, this is one of the most central, intricate
184  * routines in the kernel, and tons of code elsewhere depends on its exact
185  * behaviour.  If you change something in here, expect it to bite you in the
186  * rear.
187  */
188 void
189 _kernel_lock(int nlocks)
190 {
191 	struct cpu_info *ci;
192 	LOCKSTAT_TIMER(spintime);
193 	LOCKSTAT_FLAG(lsflag);
194 	struct lwp *owant;
195 #ifdef LOCKDEBUG
196 	static struct cpu_info *kernel_lock_holder;
197 	u_int spins = 0;
198 	u_int starttime = getticks();
199 #endif
200 	int s;
201 	struct lwp *l = curlwp;
202 
203 	_KERNEL_LOCK_ASSERT(nlocks > 0);
204 
205 	s = splvm();
206 	ci = curcpu();
207 	if (ci->ci_biglock_count != 0) {
208 		_KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock));
209 		ci->ci_biglock_count += nlocks;
210 		l->l_blcnt += nlocks;
211 		splx(s);
212 		return;
213 	}
214 
215 	_KERNEL_LOCK_ASSERT(l->l_blcnt == 0);
216 	LOCKDEBUG_WANTLOCK(kernel_lock_dodebug, kernel_lock, RETURN_ADDRESS,
217 	    0);
218 
219 	if (__predict_true(__cpu_simple_lock_try(kernel_lock))) {
220 #ifdef LOCKDEBUG
221 		kernel_lock_holder = curcpu();
222 #endif
223 		ci->ci_biglock_count = nlocks;
224 		l->l_blcnt = nlocks;
225 		LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL,
226 		    RETURN_ADDRESS, 0);
227 		splx(s);
228 		return;
229 	}
230 
231 	/*
232 	 * To remove the ordering constraint between adaptive mutexes
233 	 * and kernel_lock we must make it appear as if this thread is
234 	 * blocking.  For non-interlocked mutex release, a store fence
235 	 * is required to ensure that the result of any mutex_exit()
236 	 * by the current LWP becomes visible on the bus before the set
237 	 * of ci->ci_biglock_wanted becomes visible.
238 	 */
239 	membar_producer();
240 	owant = ci->ci_biglock_wanted;
241 	ci->ci_biglock_wanted = l;
242 #if defined(DIAGNOSTIC) && !defined(LOCKDEBUG)
243 	l->l_ld_wanted = __builtin_return_address(0);
244 #endif
245 
246 	/*
247 	 * Spin until we acquire the lock.  Once we have it, record the
248 	 * time spent with lockstat.
249 	 */
250 	LOCKSTAT_ENTER(lsflag);
251 	LOCKSTAT_START_TIMER(lsflag, spintime);
252 
253 	do {
254 		splx(s);
255 		while (__SIMPLELOCK_LOCKED_P(kernel_lock)) {
256 #ifdef LOCKDEBUG
257 			if (SPINLOCK_SPINOUT(spins) && start_init_exec &&
258 			    (getticks() - starttime) > 10*hz) {
259 				ipi_msg_t msg = {
260 					.func = kernel_lock_trace_ipi,
261 				};
262 				kpreempt_disable();
263 				ipi_unicast(&msg, kernel_lock_holder);
264 				ipi_wait(&msg);
265 				kpreempt_enable();
266 				_KERNEL_LOCK_ABORT("spinout");
267 			}
268 #endif
269 			SPINLOCK_BACKOFF_HOOK;
270 			SPINLOCK_SPIN_HOOK;
271 		}
272 		s = splvm();
273 	} while (!__cpu_simple_lock_try(kernel_lock));
274 
275 	ci->ci_biglock_count = nlocks;
276 	l->l_blcnt = nlocks;
277 	LOCKSTAT_STOP_TIMER(lsflag, spintime);
278 	LOCKDEBUG_LOCKED(kernel_lock_dodebug, kernel_lock, NULL,
279 	    RETURN_ADDRESS, 0);
280 	if (owant == NULL) {
281 		LOCKSTAT_EVENT_RA(lsflag, kernel_lock,
282 		    LB_KERNEL_LOCK | LB_SPIN, 1, spintime, RETURN_ADDRESS);
283 	}
284 	LOCKSTAT_EXIT(lsflag);
285 	splx(s);
286 
287 	/*
288 	 * Now that we have kernel_lock, reset ci_biglock_wanted.  This
289 	 * store must be unbuffered (immediately visible on the bus) in
290 	 * order for non-interlocked mutex release to work correctly.
291 	 * It must be visible before a mutex_exit() can execute on this
292 	 * processor.
293 	 *
294 	 * Note: only where CAS is available in hardware will this be
295 	 * an unbuffered write, but non-interlocked release cannot be
296 	 * done on CPUs without CAS in hardware.
297 	 */
298 	(void)atomic_swap_ptr(&ci->ci_biglock_wanted, owant);
299 
300 	/*
301 	 * Issue a memory barrier as we have acquired a lock.  This also
302 	 * prevents stores from a following mutex_exit() being reordered
303 	 * to occur before our store to ci_biglock_wanted above.
304 	 */
305 #ifndef __HAVE_ATOMIC_AS_MEMBAR
306 	membar_enter();
307 #endif
308 
309 #ifdef LOCKDEBUG
310 	kernel_lock_holder = curcpu();
311 #endif
312 }
313 
314 /*
315  * Release 'nlocks' holds on the kernel lock.  If 'nlocks' is zero, release
316  * all holds.
317  */
318 void
319 _kernel_unlock(int nlocks, int *countp)
320 {
321 	struct cpu_info *ci;
322 	u_int olocks;
323 	int s;
324 	struct lwp *l = curlwp;
325 
326 	_KERNEL_LOCK_ASSERT(nlocks < 2);
327 
328 	olocks = l->l_blcnt;
329 
330 	if (olocks == 0) {
331 		_KERNEL_LOCK_ASSERT(nlocks <= 0);
332 		if (countp != NULL)
333 			*countp = 0;
334 		return;
335 	}
336 
337 	_KERNEL_LOCK_ASSERT(__SIMPLELOCK_LOCKED_P(kernel_lock));
338 
339 	if (nlocks == 0)
340 		nlocks = olocks;
341 	else if (nlocks == -1) {
342 		nlocks = 1;
343 		_KERNEL_LOCK_ASSERT(olocks == 1);
344 	}
345 	s = splvm();
346 	ci = curcpu();
347 	_KERNEL_LOCK_ASSERT(ci->ci_biglock_count >= l->l_blcnt);
348 	if (ci->ci_biglock_count == nlocks) {
349 		LOCKDEBUG_UNLOCKED(kernel_lock_dodebug, kernel_lock,
350 		    RETURN_ADDRESS, 0);
351 		ci->ci_biglock_count = 0;
352 		__cpu_simple_unlock(kernel_lock);
353 		l->l_blcnt -= nlocks;
354 		splx(s);
355 		if (l->l_dopreempt)
356 			kpreempt(0);
357 	} else {
358 		ci->ci_biglock_count -= nlocks;
359 		l->l_blcnt -= nlocks;
360 		splx(s);
361 	}
362 
363 	if (countp != NULL)
364 		*countp = olocks;
365 }
366 
367 bool
368 _kernel_locked_p(void)
369 {
370 	return __SIMPLELOCK_LOCKED_P(kernel_lock);
371 }
372