xref: /netbsd-src/sys/dev/lockstat.c (revision 267197ec1eebfcb9810ea27a89625b6ddf68e3e7) 
1 /*	$NetBSD: lockstat.c,v 1.13 2008/01/04 21:17:48 ad Exp $	*/
2 
3 /*-
4  * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Andrew Doran.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 3. All advertising materials mentioning features or use of this software
19  *    must display the following acknowledgement:
20  *	This product includes software developed by the NetBSD
21  *	Foundation, Inc. and its contributors.
22  * 4. Neither the name of The NetBSD Foundation nor the names of its
23  *    contributors may be used to endorse or promote products derived
24  *    from this software without specific prior written permission.
25  *
26  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36  * POSSIBILITY OF SUCH DAMAGE.
37  */
38 
39 /*
40  * Lock statistics driver, providing kernel support for the lockstat(8)
41  * command.
42  *
43  * We use a global lock word (lockstat_lock) to track device opens.
44  * Only one thread can hold the device at a time, providing a global lock.
45  *
46  * XXX Timings for contention on sleep locks are currently incorrect.
47  */
48 
49 #include <sys/cdefs.h>
50 __KERNEL_RCSID(0, "$NetBSD: lockstat.c,v 1.13 2008/01/04 21:17:48 ad Exp $");
51 
52 #include <sys/types.h>
53 #include <sys/param.h>
54 #include <sys/proc.h>
55 #include <sys/resourcevar.h>
56 #include <sys/systm.h>
57 #include <sys/kernel.h>
58 #include <sys/kmem.h>
59 #include <sys/conf.h>
60 #include <sys/syslog.h>
61 #include <sys/atomic.h>
62 
63 #include <dev/lockstat.h>
64 
65 #include <machine/lock.h>
66 
67 #ifndef __HAVE_CPU_COUNTER
68 #error CPU counters not available
69 #endif
70 
71 #if LONG_BIT == 64
72 #define	LOCKSTAT_HASH_SHIFT	3
73 #elif LONG_BIT == 32
74 #define	LOCKSTAT_HASH_SHIFT	2
75 #endif
76 
77 #define	LOCKSTAT_MINBUFS	1000
78 #define	LOCKSTAT_DEFBUFS	10000
79 #define	LOCKSTAT_MAXBUFS	50000
80 
81 #define	LOCKSTAT_HASH_SIZE	128
82 #define	LOCKSTAT_HASH_MASK	(LOCKSTAT_HASH_SIZE - 1)
83 #define	LOCKSTAT_HASH(key)	\
84 	((key >> LOCKSTAT_HASH_SHIFT) & LOCKSTAT_HASH_MASK)
85 
86 typedef struct lscpu {
87 	SLIST_HEAD(, lsbuf)	lc_free;
88 	u_int			lc_overflow;
89 	LIST_HEAD(lslist, lsbuf) lc_hash[LOCKSTAT_HASH_SIZE];
90 } lscpu_t;
91 
92 typedef struct lslist lslist_t;
93 
94 void	lockstatattach(int);
95 void	lockstat_start(lsenable_t *);
96 int	lockstat_alloc(lsenable_t *);
97 void	lockstat_init_tables(lsenable_t *);
98 int	lockstat_stop(lsdisable_t *);
99 void	lockstat_free(void);
100 
101 dev_type_open(lockstat_open);
102 dev_type_close(lockstat_close);
103 dev_type_read(lockstat_read);
104 dev_type_ioctl(lockstat_ioctl);
105 
106 volatile u_int	lockstat_enabled;
107 uintptr_t	lockstat_csstart;
108 uintptr_t	lockstat_csend;
109 uintptr_t	lockstat_csmask;
110 uintptr_t	lockstat_lamask;
111 uintptr_t	lockstat_lockstart;
112 uintptr_t	lockstat_lockend;
113 __cpu_simple_lock_t lockstat_lock;
114 lwp_t		*lockstat_lwp;
115 lsbuf_t		*lockstat_baseb;
116 size_t		lockstat_sizeb;
117 int		lockstat_busy;
118 struct timespec	lockstat_stime;
119 
120 const struct cdevsw lockstat_cdevsw = {
121 	lockstat_open, lockstat_close, lockstat_read, nowrite, lockstat_ioctl,
122 	nostop, notty, nopoll, nommap, nokqfilter, D_OTHER | D_MPSAFE
123 };
124 
125 /*
126  * Called when the pseudo-driver is attached.
127  */
128 void
129 lockstatattach(int nunits)
130 {
131 
132 	(void)nunits;
133 
134 	__cpu_simple_lock_init(&lockstat_lock);
135 }
136 
137 /*
138  * Prepare the per-CPU tables for use, or clear down tables when tracing is
139  * stopped.
140  */
141 void
142 lockstat_init_tables(lsenable_t *le)
143 {
144 	int i, per, slop, cpuno;
145 	CPU_INFO_ITERATOR cii;
146 	struct cpu_info *ci;
147 	lscpu_t *lc;
148 	lsbuf_t *lb;
149 
150 	KASSERT(!lockstat_enabled);
151 
152 	for (CPU_INFO_FOREACH(cii, ci)) {
153 		if (ci->ci_lockstat != NULL) {
154 			kmem_free(ci->ci_lockstat, sizeof(lscpu_t));
155 			ci->ci_lockstat = NULL;
156 		}
157 	}
158 
159 	if (le == NULL)
160 		return;
161 
162 	lb = lockstat_baseb;
163 	per = le->le_nbufs / ncpu;
164 	slop = le->le_nbufs - (per * ncpu);
165 	cpuno = 0;
166 	for (CPU_INFO_FOREACH(cii, ci)) {
167 		lc = kmem_alloc(sizeof(*lc), KM_SLEEP);
168 		lc->lc_overflow = 0;
169 		ci->ci_lockstat = lc;
170 
171 		SLIST_INIT(&lc->lc_free);
172 		for (i = 0; i < LOCKSTAT_HASH_SIZE; i++)
173 			LIST_INIT(&lc->lc_hash[i]);
174 
175 		for (i = per; i != 0; i--, lb++) {
176 			lb->lb_cpu = (uint16_t)cpuno;
177 			SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
178 		}
179 		if (--slop > 0) {
180 			lb->lb_cpu = (uint16_t)cpuno;
181 			SLIST_INSERT_HEAD(&lc->lc_free, lb, lb_chain.slist);
182 			lb++;
183 		}
184 		cpuno++;
185 	}
186 }
187 
188 /*
189  * Start collecting lock statistics.
190  */
191 void
192 lockstat_start(lsenable_t *le)
193 {
194 
195 	KASSERT(!lockstat_enabled);
196 
197 	lockstat_init_tables(le);
198 
199 	if ((le->le_flags & LE_CALLSITE) != 0)
200 		lockstat_csmask = (uintptr_t)-1LL;
201 	else
202 		lockstat_csmask = 0;
203 
204 	if ((le->le_flags & LE_LOCK) != 0)
205 		lockstat_lamask = (uintptr_t)-1LL;
206 	else
207 		lockstat_lamask = 0;
208 
209 	lockstat_csstart = le->le_csstart;
210 	lockstat_csend = le->le_csend;
211 	lockstat_lockstart = le->le_lockstart;
212 	lockstat_lockstart = le->le_lockstart;
213 	lockstat_lockend = le->le_lockend;
214 	membar_sync();
215 	getnanotime(&lockstat_stime);
216 	lockstat_enabled = le->le_mask;
217 	membar_producer();
218 }
219 
220 /*
221  * Stop collecting lock statistics.
222  */
223 int
224 lockstat_stop(lsdisable_t *ld)
225 {
226 	CPU_INFO_ITERATOR cii;
227 	struct cpu_info *ci;
228 	u_int cpuno, overflow;
229 	struct timespec ts;
230 	int error;
231 
232 	KASSERT(lockstat_enabled);
233 
234 	/*
235 	 * Set enabled false, force a write barrier, and wait for other CPUs
236 	 * to exit lockstat_event().
237 	 */
238 	lockstat_enabled = 0;
239 	membar_producer();
240 	getnanotime(&ts);
241 	tsleep(&lockstat_stop, PPAUSE, "lockstat", mstohz(10));
242 
243 	/*
244 	 * Did we run out of buffers while tracing?
245 	 */
246 	overflow = 0;
247 	for (CPU_INFO_FOREACH(cii, ci))
248 		overflow += ((lscpu_t *)ci->ci_lockstat)->lc_overflow;
249 
250 	if (overflow != 0) {
251 		error = EOVERFLOW;
252 		log(LOG_NOTICE, "lockstat: %d buffer allocations failed\n",
253 		    overflow);
254 	} else
255 		error = 0;
256 
257 	lockstat_init_tables(NULL);
258 
259 	if (ld == NULL)
260 		return error;
261 
262 	/*
263 	 * Fill out the disable struct for the caller.
264 	 */
265 	timespecsub(&ts, &lockstat_stime, &ld->ld_time);
266 	ld->ld_size = lockstat_sizeb;
267 
268 	cpuno = 0;
269 	for (CPU_INFO_FOREACH(cii, ci)) {
270 		if (cpuno > sizeof(ld->ld_freq) / sizeof(ld->ld_freq[0])) {
271 			log(LOG_WARNING, "lockstat: too many CPUs\n");
272 			break;
273 		}
274 		ld->ld_freq[cpuno++] = cpu_frequency(ci);
275 	}
276 
277 	return error;
278 }
279 
280 /*
281  * Allocate buffers for lockstat_start().
282  */
283 int
284 lockstat_alloc(lsenable_t *le)
285 {
286 	lsbuf_t *lb;
287 	size_t sz;
288 
289 	KASSERT(!lockstat_enabled);
290 	lockstat_free();
291 
292 	sz = sizeof(*lb) * le->le_nbufs;
293 
294 	lb = kmem_zalloc(sz, KM_SLEEP);
295 	if (lb == NULL)
296 		return (ENOMEM);
297 
298 	KASSERT(!lockstat_enabled);
299 	KASSERT(lockstat_baseb == NULL);
300 	lockstat_sizeb = sz;
301 	lockstat_baseb = lb;
302 
303 	return (0);
304 }
305 
306 /*
307  * Free allocated buffers after tracing has stopped.
308  */
309 void
310 lockstat_free(void)
311 {
312 
313 	KASSERT(!lockstat_enabled);
314 
315 	if (lockstat_baseb != NULL) {
316 		kmem_free(lockstat_baseb, lockstat_sizeb);
317 		lockstat_baseb = NULL;
318 	}
319 }
320 
321 /*
322  * Main entry point from lock primatives.
323  */
324 void
325 lockstat_event(uintptr_t lock, uintptr_t callsite, u_int flags, u_int count,
326 	       uint64_t cycles)
327 {
328 	lslist_t *ll;
329 	lscpu_t *lc;
330 	lsbuf_t *lb;
331 	u_int event;
332 	int s;
333 
334 	if ((flags & lockstat_enabled) != flags || count == 0)
335 		return;
336 	if (lock < lockstat_lockstart || lock > lockstat_lockend)
337 		return;
338 	if (callsite < lockstat_csstart || callsite > lockstat_csend)
339 		return;
340 
341 	callsite &= lockstat_csmask;
342 	lock &= lockstat_lamask;
343 
344 	/*
345 	 * Find the table for this lock+callsite pair, and try to locate a
346 	 * buffer with the same key.
347 	 */
348 	s = splhigh();
349 	lc = curcpu()->ci_lockstat;
350 	ll = &lc->lc_hash[LOCKSTAT_HASH(lock ^ callsite)];
351 	event = (flags & LB_EVENT_MASK) - 1;
352 
353 	LIST_FOREACH(lb, ll, lb_chain.list) {
354 		if (lb->lb_lock == lock && lb->lb_callsite == callsite)
355 			break;
356 	}
357 
358 	if (lb != NULL) {
359 		/*
360 		 * We found a record.  Move it to the front of the list, as
361 		 * we're likely to hit it again soon.
362 		 */
363 		if (lb != LIST_FIRST(ll)) {
364 			LIST_REMOVE(lb, lb_chain.list);
365 			LIST_INSERT_HEAD(ll, lb, lb_chain.list);
366 		}
367 		lb->lb_counts[event] += count;
368 		lb->lb_times[event] += cycles;
369 	} else if ((lb = SLIST_FIRST(&lc->lc_free)) != NULL) {
370 		/*
371 		 * Pinch a new buffer and fill it out.
372 		 */
373 		SLIST_REMOVE_HEAD(&lc->lc_free, lb_chain.slist);
374 		LIST_INSERT_HEAD(ll, lb, lb_chain.list);
375 		lb->lb_flags = (uint16_t)flags;
376 		lb->lb_lock = lock;
377 		lb->lb_callsite = callsite;
378 		lb->lb_counts[event] = count;
379 		lb->lb_times[event] = cycles;
380 	} else {
381 		/*
382 		 * We didn't find a buffer and there were none free.
383 		 * lockstat_stop() will notice later on and report the
384 		 * error.
385 		 */
386 		 lc->lc_overflow++;
387 	}
388 
389 	splx(s);
390 }
391 
392 /*
393  * Accept an open() on /dev/lockstat.
394  */
395 int
396 lockstat_open(dev_t dev, int flag, int mode, lwp_t *l)
397 {
398 
399 	if (!__cpu_simple_lock_try(&lockstat_lock))
400 		return EBUSY;
401 	lockstat_lwp = curlwp;
402 	return 0;
403 }
404 
405 /*
406  * Accept the last close() on /dev/lockstat.
407  */
408 int
409 lockstat_close(dev_t dev, int flag, int mode, lwp_t *l)
410 {
411 
412 	lockstat_lwp = NULL;
413 	__cpu_simple_unlock(&lockstat_lock);
414 	return 0;
415 }
416 
417 /*
418  * Handle control operations.
419  */
420 int
421 lockstat_ioctl(dev_t dev, u_long cmd, void *data, int flag, lwp_t *l)
422 {
423 	lsenable_t *le;
424 	int error;
425 
426 	if (lockstat_lwp != curlwp)
427 		return EBUSY;
428 
429 	switch (cmd) {
430 	case IOC_LOCKSTAT_GVERSION:
431 		*(int *)data = LS_VERSION;
432 		error = 0;
433 		break;
434 
435 	case IOC_LOCKSTAT_ENABLE:
436 		le = (lsenable_t *)data;
437 
438 		if (!cpu_hascounter()) {
439 			error = ENODEV;
440 			break;
441 		}
442 		if (lockstat_enabled) {
443 			error = EBUSY;
444 			break;
445 		}
446 
447 		/*
448 		 * Sanitize the arguments passed in and set up filtering.
449 		 */
450 		if (le->le_nbufs == 0)
451 			le->le_nbufs = LOCKSTAT_DEFBUFS;
452 		else if (le->le_nbufs > LOCKSTAT_MAXBUFS ||
453 		    le->le_nbufs < LOCKSTAT_MINBUFS) {
454 			error = EINVAL;
455 			break;
456 		}
457 		if ((le->le_flags & LE_ONE_CALLSITE) == 0) {
458 			le->le_csstart = 0;
459 			le->le_csend = le->le_csstart - 1;
460 		}
461 		if ((le->le_flags & LE_ONE_LOCK) == 0) {
462 			le->le_lockstart = 0;
463 			le->le_lockend = le->le_lockstart - 1;
464 		}
465 		if ((le->le_mask & LB_EVENT_MASK) == 0)
466 			return EINVAL;
467 		if ((le->le_mask & LB_LOCK_MASK) == 0)
468 			return EINVAL;
469 
470 		/*
471 		 * Start tracing.
472 		 */
473 		if ((error = lockstat_alloc(le)) == 0)
474 			lockstat_start(le);
475 		break;
476 
477 	case IOC_LOCKSTAT_DISABLE:
478 		if (!lockstat_enabled)
479 			error = EINVAL;
480 		else
481 			error = lockstat_stop((lsdisable_t *)data);
482 		break;
483 
484 	default:
485 		error = ENOTTY;
486 		break;
487 	}
488 
489 	return error;
490 }
491 
492 /*
493  * Copy buffers out to user-space.
494  */
495 int
496 lockstat_read(dev_t dev, struct uio *uio, int flag)
497 {
498 
499 	if (curlwp != lockstat_lwp || lockstat_enabled)
500 		return EBUSY;
501 	return uiomove(lockstat_baseb, lockstat_sizeb, uio);
502 }
503