xref: /netbsd-src/sys/dev/lockstat.c (revision 8b0f9554ff8762542c4defc4f70e1eb76fb508fa) 
1 /*	$NetBSD: lockstat.c,v 1.11 2007/11/06 18:02:43 ad Exp $	*/
2 
3 /*-
4  * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Andrew Doran.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 3. All advertising materials mentioning features or use of this software
19  *    must display the following acknowledgement:
20  *	This product includes software developed by the NetBSD
21  *	Foundation, Inc. and its contributors.
22  * 4. Neither the name of The NetBSD Foundation nor the names of its
23  *    contributors may be used to endorse or promote products derived
24  *    from this software without specific prior written permission.
25  *
26  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36  * POSSIBILITY OF SUCH DAMAGE.
37  */
38 
39 /*
40  * Lock statistics driver, providing kernel support for the lockstat(8)
41  * command.
42  *
43  * We use a global lock word (lockstat_lock) to track device opens.
44  * Only one thread can hold the device at a time, providing a global lock.
45  *
46  * XXX Timings for contention on sleep locks are currently incorrect.
47  */
48 
49 #include <sys/cdefs.h>
50 __KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.11 2007/11/06 18:02:43 ad Exp $");
51 
52 #include <sys/types.h>
53 #include <sys/param.h>
54 #include <sys/lock.h>
55 #include <sys/proc.h>
56 #include <sys/resourcevar.h>
57 #include <sys/systm.h>
58 #include <sys/kernel.h>
59 #include <sys/kmem.h>
60 #include <sys/conf.h>
61 #include <sys/syslog.h>
62 
63 #include <dev/lockstat.h>
64 
65 #ifndef __HAVE_CPU_COUNTER
66 #error CPU counters not available
67 #endif
68 
69 #if LONG_BIT == 64
70 #define	LOCKSTAT_HASH_SHIFT	3
71 #elif LONG_BIT == 32
72 #define	LOCKSTAT_HASH_SHIFT	2
73 #endif
74 
75 #define	LOCKSTAT_MINBUFS	1000
76 #define	LOCKSTAT_DEFBUFS	10000
77 #define	LOCKSTAT_MAXBUFS	50000
78 
79 #define	LOCKSTAT_HASH_SIZE	128
80 #define	LOCKSTAT_HASH_MASK	(LOCKSTAT_HASH_SIZE - 1)
81 #define	LOCKSTAT_HASH(key)	\
82 	((key >> LOCKSTAT_HASH_SHIFT) & LOCKSTAT_HASH_MASK)
83 
84 typedef struct lscpu {
85 	SLIST_HEAD(, lsbuf)	lc_free;
86 	u_int			lc_overflow;
87 	LIST_HEAD(lslist, lsbuf) lc_hash[LOCKSTAT_HASH_SIZE];
88 } lscpu_t;
89 
90 typedef struct lslist lslist_t;
91 
92 void	lockstatattach(int);
93 void	lockstat_start(lsenable_t *);
94 int	lockstat_alloc(lsenable_t *);
95 void	lockstat_init_tables(lsenable_t *);
96 int	lockstat_stop(lsdisable_t *);
97 void	lockstat_free(void);
98 
99 dev_type_open(lockstat_open);
100 dev_type_close(lockstat_close);
101 dev_type_read(lockstat_read);
102 dev_type_ioctl(lockstat_ioctl);
103 
104 volatile u_int	lockstat_enabled;
105 uintptr_t	lockstat_csstart;
106 uintptr_t	lockstat_csend;
107 uintptr_t	lockstat_csmask;
108 uintptr_t	lockstat_lamask;
109 uintptr_t	lockstat_lockstart;
110 uintptr_t	lockstat_lockend;
111 __cpu_simple_lock_t lockstat_lock;
112 lwp_t		*lockstat_lwp;
113 lsbuf_t		*lockstat_baseb;
114 size_t		lockstat_sizeb;
115 int		lockstat_busy;
116 struct timespec	lockstat_stime;
117 
118 const struct cdevsw lockstat_cdevsw = {
119 	lockstat_open, lockstat_close, lockstat_read, nowrite, lockstat_ioctl,
120 	nostop, notty, nopoll, nommap, nokqfilter, D_OTHER | D_MPSAFE
121 };
122 
123 /*
124  * Called when the pseudo-driver is attached.
125  */
126 void
127 lockstatattach(int nunits)
128 {
129 
130 	(void)nunits;
131 
132 	__cpu_simple_lock_init(&lockstat_lock);
133 }
134 
135 /*
136  * Prepare the per-CPU tables for use, or clear down tables when tracing is
137  * stopped.
138  */
139 void
140 lockstat_init_tables(lsenable_t *le)
141 {
142 	int i, per, slop, cpuno;
143 	CPU_INFO_ITERATOR cii;
144 	struct cpu_info *ci;
145 	lscpu_t *lc;
146 	lsbuf_t *lb;
147 
148 	KASSERT(!lockstat_enabled);
149 
150 	for (CPU_INFO_FOREACH(cii, ci)) {
151 		if (ci->ci_lockstat != NULL) {
152 			kmem_free(ci->ci_lockstat, sizeof(lscpu_t));
153 			ci->ci_lockstat = NULL;
154 		}
155 	}
156 
157 	if (le == NULL)
158 		return;
159 
160 	lb = lockstat_baseb;
161 	per = le->le_nbufs / ncpu;
162 	slop = le->le_nbufs - (per * ncpu);
163 	cpuno = 0;
164 	for (CPU_INFO_FOREACH(cii, ci)) {
165 		lc = kmem_alloc(sizeof(*lc), KM_SLEEP);
166 		lc->lc_overflow = 0;
167 		ci->ci_lockstat = lc;
168 
169 		SLIST_INIT(&lc->lc_free);
170 		for (i = 0; i < LOCKSTAT_HASH_SIZE; i++)
171 			LIST_INIT(&lc->lc_hash[i]);
172 
173 		for (i = per; i != 0; i--, lb++) {
174 			lb->lb_cpu = (uint16_t)cpuno;
175 			SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
176 		}
177 		if (--slop > 0) {
178 			lb->lb_cpu = (uint16_t)cpuno;
179 			SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
180 			lb++;
181 		}
182 		cpuno++;
183 	}
184 }
185 
186 /*
187  * Start collecting lock statistics.
188  */
189 void
190 lockstat_start(lsenable_t *le)
191 {
192 
193 	KASSERT(!lockstat_enabled);
194 
195 	lockstat_init_tables(le);
196 
197 	if ((le->le_flags & LE_CALLSITE) != 0)
198 		lockstat_csmask = (uintptr_t)-1LL;
199 	else
200 		lockstat_csmask = 0;
201 
202 	if ((le->le_flags & LE_LOCK) != 0)
203 		lockstat_lamask = (uintptr_t)-1LL;
204 	else
205 		lockstat_lamask = 0;
206 
207 	lockstat_csstart = le->le_csstart;
208 	lockstat_csend = le->le_csend;
209 	lockstat_lockstart = le->le_lockstart;
210 	lockstat_lockstart = le->le_lockstart;
211 	lockstat_lockend = le->le_lockend;
212 	mb_memory();
213 	getnanotime(&lockstat_stime);
214 	lockstat_enabled = le->le_mask;
215 	mb_write();
216 }
217 
218 /*
219  * Stop collecting lock statistics.
220  */
221 int
222 lockstat_stop(lsdisable_t *ld)
223 {
224 	CPU_INFO_ITERATOR cii;
225 	struct cpu_info *ci;
226 	u_int cpuno, overflow;
227 	struct timespec ts;
228 	int error;
229 
230 	KASSERT(lockstat_enabled);
231 
232 	/*
233 	 * Set enabled false, force a write barrier, and wait for other CPUs
234 	 * to exit lockstat_event().
235 	 */
236 	lockstat_enabled = 0;
237 	mb_write();
238 	getnanotime(&ts);
239 	tsleep(&lockstat_stop, PPAUSE, "lockstat", mstohz(10));
240 
241 	/*
242 	 * Did we run out of buffers while tracing?
243 	 */
244 	overflow = 0;
245 	for (CPU_INFO_FOREACH(cii, ci))
246 		overflow += ((lscpu_t *)ci->ci_lockstat)->lc_overflow;
247 
248 	if (overflow != 0) {
249 		error = EOVERFLOW;
250 		log(LOG_NOTICE, "lockstat: %d buffer allocations failed\n",
251 		    overflow);
252 	} else
253 		error = 0;
254 
255 	lockstat_init_tables(NULL);
256 
257 	if (ld == NULL)
258 		return error;
259 
260 	/*
261 	 * Fill out the disable struct for the caller.
262 	 */
263 	timespecsub(&ts, &lockstat_stime, &ld->ld_time);
264 	ld->ld_size = lockstat_sizeb;
265 
266 	cpuno = 0;
267 	for (CPU_INFO_FOREACH(cii, ci)) {
268 		if (cpuno > sizeof(ld->ld_freq) / sizeof(ld->ld_freq[0])) {
269 			log(LOG_WARNING, "lockstat: too many CPUs\n");
270 			break;
271 		}
272 		ld->ld_freq[cpuno++] = cpu_frequency(ci);
273 	}
274 
275 	return error;
276 }
277 
278 /*
279  * Allocate buffers for lockstat_start().
280  */
281 int
282 lockstat_alloc(lsenable_t *le)
283 {
284 	lsbuf_t *lb;
285 	size_t sz;
286 
287 	KASSERT(!lockstat_enabled);
288 	lockstat_free();
289 
290 	sz = sizeof(*lb) * le->le_nbufs;
291 
292 	lb = kmem_zalloc(sz, KM_SLEEP);
293 	if (lb == NULL)
294 		return (ENOMEM);
295 
296 	KASSERT(!lockstat_enabled);
297 	KASSERT(lockstat_baseb == NULL);
298 	lockstat_sizeb = sz;
299 	lockstat_baseb = lb;
300 
301 	return (0);
302 }
303 
304 /*
305  * Free allocated buffers after tracing has stopped.
306  */
307 void
308 lockstat_free(void)
309 {
310 
311 	KASSERT(!lockstat_enabled);
312 
313 	if (lockstat_baseb != NULL) {
314 		kmem_free(lockstat_baseb, lockstat_sizeb);
315 		lockstat_baseb = NULL;
316 	}
317 }
318 
319 /*
320  * Main entry point from lock primatives.
321  */
322 void
323 lockstat_event(uintptr_t lock, uintptr_t callsite, u_int flags, u_int count,
324 	       uint64_t cycles)
325 {
326 	lslist_t *ll;
327 	lscpu_t *lc;
328 	lsbuf_t *lb;
329 	u_int event;
330 	int s;
331 
332 	if ((flags & lockstat_enabled) != flags || count == 0)
333 		return;
334 	if (lock < lockstat_lockstart || lock > lockstat_lockend)
335 		return;
336 	if (callsite < lockstat_csstart || callsite > lockstat_csend)
337 		return;
338 
339 	callsite &= lockstat_csmask;
340 	lock &= lockstat_lamask;
341 
342 	/*
343 	 * Find the table for this lock+callsite pair, and try to locate a
344 	 * buffer with the same key.
345 	 */
346 	s = splhigh();
347 	lc = curcpu()->ci_lockstat;
348 	ll = &lc->lc_hash[LOCKSTAT_HASH(lock ^ callsite)];
349 	event = (flags & LB_EVENT_MASK) - 1;
350 
351 	LIST_FOREACH(lb, ll, lb_chain.list) {
352 		if (lb->lb_lock == lock && lb->lb_callsite == callsite)
353 			break;
354 	}
355 
356 	if (lb != NULL) {
357 		/*
358 		 * We found a record.  Move it to the front of the list, as
359 		 * we're likely to hit it again soon.
360 		 */
361 		if (lb != LIST_FIRST(ll)) {
362 			LIST_REMOVE(lb, lb_chain.list);
363 			LIST_INSERT_HEAD(ll, lb, lb_chain.list);
364 		}
365 		lb->lb_counts[event] += count;
366 		lb->lb_times[event] += cycles;
367 	} else if ((lb = SLIST_FIRST(&lc->lc_free)) != NULL) {
368 		/*
369 		 * Pinch a new buffer and fill it out.
370 		 */
371 		SLIST_REMOVE_HEAD(&lc->lc_free, lb_chain.slist);
372 		LIST_INSERT_HEAD(ll, lb, lb_chain.list);
373 		lb->lb_flags = (uint16_t)flags;
374 		lb->lb_lock = lock;
375 		lb->lb_callsite = callsite;
376 		lb->lb_counts[event] = count;
377 		lb->lb_times[event] = cycles;
378 	} else {
379 		/*
380 		 * We didn't find a buffer and there were none free.
381 		 * lockstat_stop() will notice later on and report the
382 		 * error.
383 		 */
384 		 lc->lc_overflow++;
385 	}
386 
387 	splx(s);
388 }
389 
390 /*
391  * Accept an open() on /dev/lockstat.
392  */
393 int
394 lockstat_open(dev_t dev, int flag, int mode, lwp_t *l)
395 {
396 
397 	if (!__cpu_simple_lock_try(&lockstat_lock))
398 		return EBUSY;
399 	lockstat_lwp = curlwp;
400 	return 0;
401 }
402 
403 /*
404  * Accept the last close() on /dev/lockstat.
405  */
406 int
407 lockstat_close(dev_t dev, int flag, int mode, lwp_t *l)
408 {
409 
410 	lockstat_lwp = NULL;
411 	__cpu_simple_unlock(&lockstat_lock);
412 	return 0;
413 }
414 
415 /*
416  * Handle control operations.
417  */
418 int
419 lockstat_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l)
420 {
421 	lsenable_t *le;
422 	int error;
423 
424 	if (lockstat_lwp != curlwp)
425 		return EBUSY;
426 
427 	switch (cmd) {
428 	case IOC_LOCKSTAT_GVERSION:
429 		*(int *)data = LS_VERSION;
430 		error = 0;
431 		break;
432 
433 	case IOC_LOCKSTAT_ENABLE:
434 		le = (lsenable_t *)data;
435 
436 		if (!cpu_hascounter()) {
437 			error = ENODEV;
438 			break;
439 		}
440 		if (lockstat_enabled) {
441 			error = EBUSY;
442 			break;
443 		}
444 
445 		/*
446 		 * Sanitize the arguments passed in and set up filtering.
447 		 */
448 		if (le->le_nbufs == 0)
449 			le->le_nbufs = LOCKSTAT_DEFBUFS;
450 		else if (le->le_nbufs > LOCKSTAT_MAXBUFS ||
451 		    le->le_nbufs < LOCKSTAT_MINBUFS) {
452 			error = EINVAL;
453 			break;
454 		}
455 		if ((le->le_flags & LE_ONE_CALLSITE) == 0) {
456 			le->le_csstart = 0;
457 			le->le_csend = le->le_csstart - 1;
458 		}
459 		if ((le->le_flags & LE_ONE_LOCK) == 0) {
460 			le->le_lockstart = 0;
461 			le->le_lockend = le->le_lockstart - 1;
462 		}
463 		if ((le->le_mask & LB_EVENT_MASK) == 0)
464 			return EINVAL;
465 		if ((le->le_mask & LB_LOCK_MASK) == 0)
466 			return EINVAL;
467 
468 		/*
469 		 * Start tracing.
470 		 */
471 		if ((error = lockstat_alloc(le)) == 0)
472 			lockstat_start(le);
473 		break;
474 
475 	case IOC_LOCKSTAT_DISABLE:
476 		if (!lockstat_enabled)
477 			error = EINVAL;
478 		else
479 			error = lockstat_stop((lsdisable_t *)data);
480 		break;
481 
482 	default:
483 		error = ENOTTY;
484 		break;
485 	}
486 
487 	return error;
488 }
489 
490 /*
491  * Copy buffers out to user-space.
492  */
493 int
494 lockstat_read(dev_t dev, struct uio *uio, int flag)
495 {
496 
497 	if (curlwp != lockstat_lwp || lockstat_enabled)
498 		return EBUSY;
499 	return uiomove(lockstat_baseb, lockstat_sizeb, uio);
500 }
501