xref: /netbsd-src/sys/dev/lockstat.c (revision ccd9df534e375a4366c5b55f23782053c7a98d82) 
1 /*	$NetBSD: lockstat.c,v 1.30 2022/04/08 10:17:54 andvar Exp $	*/
2 
3 /*-
4  * Copyright (c) 2006, 2007, 2019 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Andrew Doran.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 /*
33  * Lock statistics driver, providing kernel support for the lockstat(8)
34  * command.
35  *
36  * We use a global lock word (lockstat_lock) to track device opens.
37  * Only one thread can hold the device at a time, providing a global lock.
38  *
39  * XXX Timings for contention on sleep locks are currently incorrect.
40  * XXX Convert this to use timecounters!
41  */
42 
43 #include <sys/cdefs.h>
44 __KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.30 2022/04/08 10:17:54 andvar Exp $");
45 
46 #include <sys/types.h>
47 #include <sys/param.h>
48 
49 #include <sys/atomic.h>
50 #include <sys/conf.h>
51 #include <sys/cpu.h>
52 #include <sys/kernel.h>
53 #include <sys/kmem.h>
54 #include <sys/lock.h>
55 #include <sys/proc.h>
56 #include <sys/resourcevar.h>
57 #include <sys/syslog.h>
58 #include <sys/systm.h>
59 #include <sys/xcall.h>
60 
61 #include <dev/lockstat.h>
62 
63 #include "ioconf.h"
64 
65 #ifndef __HAVE_CPU_COUNTER
66 #error CPU counters not available
67 #endif
68 
69 #if LONG_BIT == 64
70 #define	LOCKSTAT_HASH_SHIFT	3
71 #elif LONG_BIT == 32
72 #define	LOCKSTAT_HASH_SHIFT	2
73 #endif
74 
75 #define	LOCKSTAT_MINBUFS	1000
76 #define	LOCKSTAT_DEFBUFS	20000
77 #define	LOCKSTAT_MAXBUFS	1000000
78 
79 #define	LOCKSTAT_HASH_SIZE	128
80 #define	LOCKSTAT_HASH_MASK	(LOCKSTAT_HASH_SIZE - 1)
81 #define	LOCKSTAT_HASH(key)	\
82 	((key >> LOCKSTAT_HASH_SHIFT) & LOCKSTAT_HASH_MASK)
83 
84 typedef struct lscpu {
85 	SLIST_HEAD(, lsbuf)	lc_free;
86 	u_int			lc_overflow;
87 	LIST_HEAD(lslist, lsbuf) lc_hash[LOCKSTAT_HASH_SIZE];
88 } lscpu_t;
89 
90 typedef struct lslist lslist_t;
91 
92 void	lockstat_start(lsenable_t *);
93 int	lockstat_alloc(lsenable_t *);
94 void	lockstat_init_tables(lsenable_t *);
95 int	lockstat_stop(lsdisable_t *);
96 void	lockstat_free(void);
97 
98 dev_type_open(lockstat_open);
99 dev_type_close(lockstat_close);
100 dev_type_read(lockstat_read);
101 dev_type_ioctl(lockstat_ioctl);
102 
103 volatile u_int	lockstat_enabled;
104 volatile u_int	lockstat_dev_enabled;
105 __cpu_simple_lock_t lockstat_enabled_lock;
106 uintptr_t	lockstat_csstart;
107 uintptr_t	lockstat_csend;
108 uintptr_t	lockstat_csmask;
109 uintptr_t	lockstat_lamask;
110 uintptr_t	lockstat_lockstart;
111 uintptr_t	lockstat_lockend;
112 __cpu_simple_lock_t lockstat_lock;
113 lwp_t		*lockstat_lwp;
114 lsbuf_t		*lockstat_baseb;
115 size_t		lockstat_sizeb;
116 int		lockstat_busy;
117 struct timespec	lockstat_stime;
118 
119 #ifdef KDTRACE_HOOKS
120 volatile u_int lockstat_dtrace_enabled;
121 CTASSERT(LB_NEVENT <= 3);
122 CTASSERT(LB_NLOCK <= (7 << LB_LOCK_SHIFT));
123 void
124 lockstat_probe_stub(uint32_t id, uintptr_t lock, uintptr_t callsite,
125     uintptr_t flags, uintptr_t count, uintptr_t cycles)
126 {
127 }
128 
129 uint32_t	lockstat_probemap[LS_NPROBES];
130 void		(*lockstat_probe_func)(uint32_t, uintptr_t, uintptr_t,
131 		    uintptr_t, uintptr_t, uintptr_t) = &lockstat_probe_stub;
132 #endif
133 
134 const struct cdevsw lockstat_cdevsw = {
135 	.d_open = lockstat_open,
136 	.d_close = lockstat_close,
137 	.d_read = lockstat_read,
138 	.d_write = nowrite,
139 	.d_ioctl = lockstat_ioctl,
140 	.d_stop = nostop,
141 	.d_tty = notty,
142 	.d_poll = nopoll,
143 	.d_mmap = nommap,
144 	.d_kqfilter = nokqfilter,
145 	.d_discard = nodiscard,
146 	.d_flag = D_OTHER | D_MPSAFE
147 };
148 
149 /*
150  * Called when the pseudo-driver is attached.
151  */
152 void
153 lockstatattach(int nunits)
154 {
155 
156 	(void)nunits;
157 
158 	__cpu_simple_lock_init(&lockstat_lock);
159 	__cpu_simple_lock_init(&lockstat_enabled_lock);
160 }
161 
162 /*
163  * Prepare the per-CPU tables for use, or clear down tables when tracing is
164  * stopped.
165  */
166 void
167 lockstat_init_tables(lsenable_t *le)
168 {
169 	int i, per, slop, cpuno;
170 	CPU_INFO_ITERATOR cii;
171 	struct cpu_info *ci;
172 	lscpu_t *lc;
173 	lsbuf_t *lb;
174 
175 	/* coverity[assert_side_effect] */
176 	KASSERT(!lockstat_dev_enabled);
177 
178 	for (CPU_INFO_FOREACH(cii, ci)) {
179 		if (ci->ci_lockstat != NULL) {
180 			kmem_free(ci->ci_lockstat, sizeof(lscpu_t));
181 			ci->ci_lockstat = NULL;
182 		}
183 	}
184 
185 	if (le == NULL)
186 		return;
187 
188 	lb = lockstat_baseb;
189 	per = le->le_nbufs / ncpu;
190 	slop = le->le_nbufs - (per * ncpu);
191 	cpuno = 0;
192 	for (CPU_INFO_FOREACH(cii, ci)) {
193 		lc = kmem_alloc(sizeof(*lc), KM_SLEEP);
194 		lc->lc_overflow = 0;
195 		ci->ci_lockstat = lc;
196 
197 		SLIST_INIT(&lc->lc_free);
198 		for (i = 0; i < LOCKSTAT_HASH_SIZE; i++)
199 			LIST_INIT(&lc->lc_hash[i]);
200 
201 		for (i = per; i != 0; i--, lb++) {
202 			lb->lb_cpu = (uint16_t)cpuno;
203 			SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
204 		}
205 		if (--slop > 0) {
206 			lb->lb_cpu = (uint16_t)cpuno;
207 			SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
208 			lb++;
209 		}
210 		cpuno++;
211 	}
212 }
213 
214 /*
215  * Start collecting lock statistics.
216  */
217 void
218 lockstat_start(lsenable_t *le)
219 {
220 
221 	/* coverity[assert_side_effect] */
222 	KASSERT(!lockstat_dev_enabled);
223 
224 	lockstat_init_tables(le);
225 
226 	if ((le->le_flags & LE_CALLSITE) != 0)
227 		lockstat_csmask = (uintptr_t)-1LL;
228 	else
229 		lockstat_csmask = 0;
230 
231 	if ((le->le_flags & LE_LOCK) != 0)
232 		lockstat_lamask = (uintptr_t)-1LL;
233 	else
234 		lockstat_lamask = 0;
235 
236 	lockstat_csstart = le->le_csstart;
237 	lockstat_csend = le->le_csend;
238 	lockstat_lockstart = le->le_lockstart;
239 	lockstat_lockstart = le->le_lockstart;
240 	lockstat_lockend = le->le_lockend;
241 
242 	/*
243 	 * Ensure everything is initialized on all CPUs, by issuing a
244 	 * null xcall with the side effect of a release barrier on this
245 	 * CPU and an acquire barrier on all other CPUs, before they
246 	 * can witness any flags set in lockstat_dev_enabled -- this
247 	 * way we don't need to add any barriers in lockstat_event.
248 	 */
249 	xc_barrier(0);
250 
251 	/*
252 	 * Start timing after the xcall, so we don't spuriously count
253 	 * xcall communication time, but before flipping the switch, so
254 	 * we don't dirty sample with locks taken in the timecounter.
255 	 */
256 	getnanotime(&lockstat_stime);
257 
258 	LOCKSTAT_ENABLED_UPDATE_BEGIN();
259 	atomic_store_relaxed(&lockstat_dev_enabled, le->le_mask);
260 	LOCKSTAT_ENABLED_UPDATE_END();
261 }
262 
263 /*
264  * Stop collecting lock statistics.
265  */
266 int
267 lockstat_stop(lsdisable_t *ld)
268 {
269 	CPU_INFO_ITERATOR cii;
270 	struct cpu_info *ci;
271 	u_int cpuno, overflow;
272 	struct timespec ts;
273 	int error;
274 	lwp_t *l;
275 
276 	/* coverity[assert_side_effect] */
277 	KASSERT(lockstat_dev_enabled);
278 
279 	/*
280 	 * Disable and wait for other CPUs to exit lockstat_event().
281 	 */
282 	LOCKSTAT_ENABLED_UPDATE_BEGIN();
283 	atomic_store_relaxed(&lockstat_dev_enabled, 0);
284 	LOCKSTAT_ENABLED_UPDATE_END();
285 	getnanotime(&ts);
286 	xc_barrier(0);
287 
288 	/*
289 	 * Did we run out of buffers while tracing?
290 	 */
291 	overflow = 0;
292 	for (CPU_INFO_FOREACH(cii, ci))
293 		overflow += ((lscpu_t *)ci->ci_lockstat)->lc_overflow;
294 
295 	if (overflow != 0) {
296 		error = EOVERFLOW;
297 		log(LOG_NOTICE, "lockstat: %d buffer allocations failed\n",
298 		    overflow);
299 	} else
300 		error = 0;
301 
302 	lockstat_init_tables(NULL);
303 
304 	/* Run through all LWPs and clear the slate for the next run. */
305 	mutex_enter(&proc_lock);
306 	LIST_FOREACH(l, &alllwp, l_list) {
307 		l->l_pfailaddr = 0;
308 		l->l_pfailtime = 0;
309 		l->l_pfaillock = 0;
310 	}
311 	mutex_exit(&proc_lock);
312 
313 	if (ld == NULL)
314 		return error;
315 
316 	/*
317 	 * Fill out the disable struct for the caller.
318 	 */
319 	timespecsub(&ts, &lockstat_stime, &ld->ld_time);
320 	ld->ld_size = lockstat_sizeb;
321 
322 	cpuno = 0;
323 	for (CPU_INFO_FOREACH(cii, ci)) {
324 		if (cpuno >= sizeof(ld->ld_freq) / sizeof(ld->ld_freq[0])) {
325 			log(LOG_WARNING, "lockstat: too many CPUs\n");
326 			break;
327 		}
328 		ld->ld_freq[cpuno++] = cpu_frequency(ci);
329 	}
330 
331 	return error;
332 }
333 
334 /*
335  * Allocate buffers for lockstat_start().
336  */
337 int
338 lockstat_alloc(lsenable_t *le)
339 {
340 	lsbuf_t *lb;
341 	size_t sz;
342 
343 	/* coverity[assert_side_effect] */
344 	KASSERT(!lockstat_dev_enabled);
345 	lockstat_free();
346 
347 	sz = sizeof(*lb) * le->le_nbufs;
348 
349 	lb = kmem_zalloc(sz, KM_SLEEP);
350 
351 	/* coverity[assert_side_effect] */
352 	KASSERT(!lockstat_dev_enabled);
353 	KASSERT(lockstat_baseb == NULL);
354 	lockstat_sizeb = sz;
355 	lockstat_baseb = lb;
356 
357 	return (0);
358 }
359 
360 /*
361  * Free allocated buffers after tracing has stopped.
362  */
363 void
364 lockstat_free(void)
365 {
366 
367 	/* coverity[assert_side_effect] */
368 	KASSERT(!lockstat_dev_enabled);
369 
370 	if (lockstat_baseb != NULL) {
371 		kmem_free(lockstat_baseb, lockstat_sizeb);
372 		lockstat_baseb = NULL;
373 	}
374 }
375 
376 /*
377  * Main entry point from lock primitives.
378  */
379 void
380 lockstat_event(uintptr_t lock, uintptr_t callsite, u_int flags, u_int count,
381 	       uint64_t cycles)
382 {
383 	lslist_t *ll;
384 	lscpu_t *lc;
385 	lsbuf_t *lb;
386 	u_int event;
387 	int s;
388 
389 #ifdef KDTRACE_HOOKS
390 	uint32_t id;
391 	CTASSERT((LS_NPROBES & (LS_NPROBES - 1)) == 0);
392 	if ((id = atomic_load_relaxed(&lockstat_probemap[LS_COMPRESS(flags)]))
393 	    != 0)
394 		(*lockstat_probe_func)(id, lock, callsite, flags, count,
395 		    cycles);
396 #endif
397 
398 	if ((flags & atomic_load_relaxed(&lockstat_dev_enabled)) != flags ||
399 	    count == 0)
400 		return;
401 	if (lock < lockstat_lockstart || lock > lockstat_lockend)
402 		return;
403 	if (callsite < lockstat_csstart || callsite > lockstat_csend)
404 		return;
405 
406 	callsite &= lockstat_csmask;
407 	lock &= lockstat_lamask;
408 
409 	/*
410 	 * Find the table for this lock+callsite pair, and try to locate a
411 	 * buffer with the same key.
412 	 */
413 	s = splhigh();
414 	lc = curcpu()->ci_lockstat;
415 	ll = &lc->lc_hash[LOCKSTAT_HASH(lock ^ callsite)];
416 	event = (flags & LB_EVENT_MASK) - 1;
417 
418 	LIST_FOREACH(lb, ll, lb_chain.list) {
419 		if (lb->lb_lock == lock && lb->lb_callsite == callsite)
420 			break;
421 	}
422 
423 	if (lb != NULL) {
424 		/*
425 		 * We found a record.  Move it to the front of the list, as
426 		 * we're likely to hit it again soon.
427 		 */
428 		if (lb != LIST_FIRST(ll)) {
429 			LIST_REMOVE(lb, lb_chain.list);
430 			LIST_INSERT_HEAD(ll, lb, lb_chain.list);
431 		}
432 		lb->lb_counts[event] += count;
433 		lb->lb_times[event] += cycles;
434 	} else if ((lb = SLIST_FIRST(&lc->lc_free)) != NULL) {
435 		/*
436 		 * Pinch a new buffer and fill it out.
437 		 */
438 		SLIST_REMOVE_HEAD(&lc->lc_free, lb_chain.slist);
439 		LIST_INSERT_HEAD(ll, lb, lb_chain.list);
440 		lb->lb_flags = (uint16_t)flags;
441 		lb->lb_lock = lock;
442 		lb->lb_callsite = callsite;
443 		lb->lb_counts[event] = count;
444 		lb->lb_times[event] = cycles;
445 	} else {
446 		/*
447 		 * We didn't find a buffer and there were none free.
448 		 * lockstat_stop() will notice later on and report the
449 		 * error.
450 		 */
451 		 lc->lc_overflow++;
452 	}
453 
454 	splx(s);
455 }
456 
457 /*
458  * Accept an open() on /dev/lockstat.
459  */
460 int
461 lockstat_open(dev_t dev, int flag, int mode, lwp_t *l)
462 {
463 
464 	if (!__cpu_simple_lock_try(&lockstat_lock))
465 		return EBUSY;
466 	lockstat_lwp = curlwp;
467 	return 0;
468 }
469 
470 /*
471  * Accept the last close() on /dev/lockstat.
472  */
473 int
474 lockstat_close(dev_t dev, int flag, int mode, lwp_t *l)
475 {
476 
477 	lockstat_lwp = NULL;
478 	if (lockstat_dev_enabled) {
479 		lockstat_stop(NULL);
480 		lockstat_free();
481 	}
482 	__cpu_simple_unlock(&lockstat_lock);
483 	return 0;
484 }
485 
486 /*
487  * Handle control operations.
488  */
489 int
490 lockstat_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l)
491 {
492 	lsenable_t *le;
493 	int error;
494 
495 	if (lockstat_lwp != curlwp)
496 		return EBUSY;
497 
498 	switch (cmd) {
499 	case IOC_LOCKSTAT_GVERSION:
500 		*(int *)data = LS_VERSION;
501 		error = 0;
502 		break;
503 
504 	case IOC_LOCKSTAT_ENABLE:
505 		le = (lsenable_t *)data;
506 
507 		if (!cpu_hascounter()) {
508 			error = ENODEV;
509 			break;
510 		}
511 		if (atomic_load_relaxed(&lockstat_dev_enabled)) {
512 			error = EBUSY;
513 			break;
514 		}
515 
516 		/*
517 		 * Sanitize the arguments passed in and set up filtering.
518 		 */
519 		if (le->le_nbufs == 0) {
520 			le->le_nbufs = MIN(LOCKSTAT_DEFBUFS * ncpu,
521 			    LOCKSTAT_MAXBUFS);
522 		} else if (le->le_nbufs > LOCKSTAT_MAXBUFS ||
523 		    le->le_nbufs < LOCKSTAT_MINBUFS) {
524 			error = EINVAL;
525 			break;
526 		}
527 		if ((le->le_flags & LE_ONE_CALLSITE) == 0) {
528 			le->le_csstart = 0;
529 			le->le_csend = le->le_csstart - 1;
530 		}
531 		if ((le->le_flags & LE_ONE_LOCK) == 0) {
532 			le->le_lockstart = 0;
533 			le->le_lockend = le->le_lockstart - 1;
534 		}
535 		if ((le->le_mask & LB_EVENT_MASK) == 0)
536 			return EINVAL;
537 		if ((le->le_mask & LB_LOCK_MASK) == 0)
538 			return EINVAL;
539 
540 		/*
541 		 * Start tracing.
542 		 */
543 		if ((error = lockstat_alloc(le)) == 0)
544 			lockstat_start(le);
545 		break;
546 
547 	case IOC_LOCKSTAT_DISABLE:
548 		if (!atomic_load_relaxed(&lockstat_dev_enabled))
549 			error = EINVAL;
550 		else
551 			error = lockstat_stop((lsdisable_t *)data);
552 		break;
553 
554 	default:
555 		error = ENOTTY;
556 		break;
557 	}
558 
559 	return error;
560 }
561 
562 /*
563  * Copy buffers out to user-space.
564  */
565 int
566 lockstat_read(dev_t dev, struct uio *uio, int flag)
567 {
568 
569 	if (curlwp != lockstat_lwp || lockstat_dev_enabled)
570 		return EBUSY;
571 	return uiomove(lockstat_baseb, lockstat_sizeb, uio);
572 }
573