xref: /netbsd-src/usr.sbin/lockstat/main.c (revision 4bfc10355ca5ccd94d950ad6f7092be3470193fa)
1 /*	$NetBSD: main.c,v 1.16 2009/03/21 13:02:19 ad Exp $	*/
2 
3 /*-
4  * Copyright (c) 2006, 2007, 2009 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Andrew Doran.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 #include <sys/cdefs.h>
33 #ifndef lint
34 __RCSID("$NetBSD: main.c,v 1.16 2009/03/21 13:02:19 ad Exp $");
35 #endif /* not lint */
36 
37 #include <sys/types.h>
38 #include <sys/param.h>
39 #include <sys/time.h>
40 #include <sys/fcntl.h>
41 #include <sys/ioctl.h>
42 #include <sys/wait.h>
43 #include <sys/signal.h>
44 #include <sys/sysctl.h>
45 
46 #include <dev/lockstat.h>
47 
48 #include <stdio.h>
49 #include <stdlib.h>
50 #include <string.h>
51 #include <limits.h>
52 #include <unistd.h>
53 #include <err.h>
54 #include <paths.h>
55 #include <util.h>
56 #include <ctype.h>
57 #include <errno.h>
58 #include <stdbool.h>
59 
60 #include "extern.h"
61 
62 #define	_PATH_DEV_LOCKSTAT	"/dev/lockstat"
63 
64 #define	MILLI	1000.0
65 #define	MICRO	1000000.0
66 #define	NANO	1000000000.0
67 #define	PICO	1000000000000.0
68 
69 TAILQ_HEAD(lock_head, lockstruct);
70 typedef struct lock_head locklist_t;
71 TAILQ_HEAD(buf_head, lsbuf);
72 typedef struct buf_head buflist_t;
73 
74 typedef struct lockstruct {
75 	TAILQ_ENTRY(lockstruct)	chain;
76 	buflist_t		bufs;
77 	buflist_t		tosort;
78 	uintptr_t		lock;
79  	double			time;
80 	uint32_t		count;
81 	u_int			flags;
82 	u_int			nbufs;
83 	char			name[NAME_SIZE];
84 } lock_t;
85 
86 typedef struct name {
87 	const char	*name;
88 	int		mask;
89 } name_t;
90 
91 const name_t locknames[] = {
92 	{ "adaptive_mutex", LB_ADAPTIVE_MUTEX },
93 	{ "spin_mutex", LB_SPIN_MUTEX },
94 	{ "rwlock", LB_RWLOCK },
95 	{ "kernel_lock", LB_KERNEL_LOCK },
96 	{ "preemption", LB_NOPREEMPT },
97 	{ "misc", LB_MISC },
98 	{ NULL, 0 }
99 };
100 
101 const name_t eventnames[] = {
102 	{ "spin", LB_SPIN },
103 	{ "sleep_exclusive", LB_SLEEP1 },
104 	{ "sleep_shared", LB_SLEEP2 },
105 	{ NULL, 0 },
106 };
107 
108 const name_t alltypes[] = {
109 	{ "Adaptive mutex spin", LB_ADAPTIVE_MUTEX | LB_SPIN },
110 	{ "Adaptive mutex sleep", LB_ADAPTIVE_MUTEX | LB_SLEEP1 },
111 	{ "Spin mutex spin", LB_SPIN_MUTEX | LB_SPIN },
112 	{ "RW lock sleep (writer)", LB_RWLOCK | LB_SLEEP1 },
113 	{ "RW lock sleep (reader)", LB_RWLOCK | LB_SLEEP2 },
114 	{ "RW lock spin", LB_RWLOCK | LB_SPIN },
115 	{ "Kernel lock spin", LB_KERNEL_LOCK | LB_SPIN },
116 	{ "Kernel preemption defer", LB_NOPREEMPT | LB_SPIN },
117 	{ "Miscellaneous wait", LB_MISC | LB_SPIN },
118 	{ NULL, 0 }
119 };
120 
121 const name_t xtypes[] = {
122 	{ "Spin", LB_SPIN },
123 	{ "Sleep (writer)", LB_SLEEP1 },
124 	{ "Sleep (reader)", LB_SLEEP2 },
125 	{ NULL, 0 }
126 };
127 
128 locklist_t	locklist;
129 locklist_t	freelist;
130 locklist_t	sortlist;
131 
132 lsbuf_t		*bufs;
133 lsdisable_t	ld;
134 bool		lflag;
135 bool		fflag;
136 int		nbufs;
137 bool		cflag;
138 bool		xflag;
139 int		lsfd;
140 int		displayed;
141 int		bin64;
142 double		tscale;
143 double		cscale;
144 double		cpuscale[sizeof(ld.ld_freq) / sizeof(ld.ld_freq[0])];
145 FILE		*outfp;
146 
147 void	findsym(findsym_t, char *, uintptr_t *, uintptr_t *, bool);
148 void	spawn(int, char **);
149 void	display(int, const char *name);
150 void	listnames(const name_t *);
151 void	collapse(bool, bool);
152 int	matchname(const name_t *, char *);
153 void	makelists(int, int);
154 void	nullsig(int);
155 void	usage(void);
156 int	ncpu(void);
157 lock_t	*morelocks(void);
158 
159 int
160 main(int argc, char **argv)
161 {
162 	int eventtype, locktype, ch, nlfd, fd, i;
163 	bool sflag, pflag, mflag, Mflag;
164 	const char *nlistf, *outf;
165 	char *lockname, *funcname;
166 	const name_t *name;
167 	lsenable_t le;
168 	double ms;
169 	char *p;
170 
171 	nlistf = NULL;
172 	outf = NULL;
173 	lockname = NULL;
174 	funcname = NULL;
175 	eventtype = -1;
176 	locktype = -1;
177 	nbufs = 0;
178 	sflag = false;
179 	pflag = false;
180 	mflag = false;
181 	Mflag = false;
182 
183 	while ((ch = getopt(argc, argv, "E:F:L:MN:T:b:ceflmo:pstx")) != -1)
184 		switch (ch) {
185 		case 'E':
186 			eventtype = matchname(eventnames, optarg);
187 			break;
188 		case 'F':
189 			funcname = optarg;
190 			break;
191 		case 'L':
192 			lockname = optarg;
193 			break;
194 		case 'N':
195 			nlistf = optarg;
196 			break;
197 		case 'T':
198 			locktype = matchname(locknames, optarg);
199 			break;
200 		case 'b':
201 			nbufs = (int)strtol(optarg, &p, 0);
202 			if (!isdigit((u_int)*optarg) || *p != '\0')
203 				usage();
204 			break;
205 		case 'c':
206 			cflag = true;
207 			break;
208 		case 'e':
209 			listnames(eventnames);
210 			break;
211 		case 'f':
212 			fflag = true;
213 			break;
214 		case 'l':
215 			lflag = true;
216 			break;
217 		case 'm':
218 			mflag = true;
219 			break;
220 		case 'M':
221 			Mflag = true;
222 			break;
223 		case 'o':
224 			outf = optarg;
225 			break;
226 		case 'p':
227 			pflag = true;
228 			break;
229 		case 's':
230 			sflag = true;
231 			break;
232 		case 't':
233 			listnames(locknames);
234 			break;
235 		case 'x':
236 			xflag = true;
237 			break;
238 		default:
239 			usage();
240 		}
241 	argc -= optind;
242 	argv += optind;
243 
244 	if (*argv == NULL)
245 		usage();
246 
247 	if (outf) {
248 		fd = open(outf, O_WRONLY | O_CREAT | O_TRUNC, 0600);
249 		if (fd == -1)
250 			err(EXIT_FAILURE, "opening %s", outf);
251 		outfp = fdopen(fd, "w");
252 	} else
253 		outfp = stdout;
254 
255 	/*
256 	 * Find the name list for resolving symbol names, and load it into
257 	 * memory.
258 	 */
259 	if (nlistf == NULL) {
260 		nlfd = open(_PATH_KSYMS, O_RDONLY);
261 		nlistf = getbootfile();
262 	} else
263 		nlfd = -1;
264 	if (nlfd == -1) {
265 		if ((nlfd = open(nlistf, O_RDONLY)) < 0)
266 			err(EXIT_FAILURE, "cannot open " _PATH_KSYMS " or %s",
267 			    nlistf);
268 	}
269 	if (loadsym32(nlfd) != 0) {
270 		if (loadsym64(nlfd) != 0)
271 			errx(EXIT_FAILURE, "unable to load symbol table");
272 		bin64 = 1;
273 	}
274 	close(nlfd);
275 
276 	memset(&le, 0, sizeof(le));
277 	le.le_nbufs = nbufs;
278 
279 	/*
280 	 * Set up initial filtering.
281 	 */
282 	if (lockname != NULL) {
283 		findsym(LOCK_BYNAME, lockname, &le.le_lockstart,
284 		    &le.le_lockend, true);
285 		le.le_flags |= LE_ONE_LOCK;
286 	}
287 	if (!lflag)
288 		le.le_flags |= LE_CALLSITE;
289 	if (!fflag)
290 		le.le_flags |= LE_LOCK;
291 	if (funcname != NULL) {
292 		if (lflag)
293 			usage();
294 		findsym(FUNC_BYNAME, funcname, &le.le_csstart, &le.le_csend, true);
295 		le.le_flags |= LE_ONE_CALLSITE;
296 	}
297 	le.le_mask = (eventtype & LB_EVENT_MASK) | (locktype & LB_LOCK_MASK);
298 
299 	/*
300 	 * Start tracing.
301 	 */
302 	if ((lsfd = open(_PATH_DEV_LOCKSTAT, O_RDONLY)) < 0)
303 		err(EXIT_FAILURE, "cannot open " _PATH_DEV_LOCKSTAT);
304 	if (ioctl(lsfd, IOC_LOCKSTAT_GVERSION, &ch) < 0)
305 		err(EXIT_FAILURE, "ioctl");
306 	if (ch != LS_VERSION)
307 		errx(EXIT_FAILURE,
308 		    "incompatible lockstat interface version (%d, kernel %d)",
309 			LS_VERSION, ch);
310 	if (ioctl(lsfd, IOC_LOCKSTAT_ENABLE, &le))
311 		err(EXIT_FAILURE, "cannot enable tracing");
312 
313 	/*
314 	 * Execute the traced program.
315 	 */
316 	spawn(argc, argv);
317 
318 	/*
319 	 * Stop tracing, and read the trace buffers from the kernel.
320 	 */
321 	if (ioctl(lsfd, IOC_LOCKSTAT_DISABLE, &ld) == -1) {
322 		if (errno == EOVERFLOW) {
323 			warnx("overflowed available kernel trace buffers");
324 			exit(EXIT_FAILURE);
325 		}
326 		err(EXIT_FAILURE, "cannot disable tracing");
327 	}
328 	if ((bufs = malloc(ld.ld_size)) == NULL)
329 		err(EXIT_FAILURE, "cannot allocate memory for user buffers");
330 	if (read(lsfd, bufs, ld.ld_size) != ld.ld_size)
331 		err(EXIT_FAILURE, "reading from " _PATH_DEV_LOCKSTAT);
332 	if (close(lsfd))
333 		err(EXIT_FAILURE, "close(" _PATH_DEV_LOCKSTAT ")");
334 
335 	/*
336 	 * Figure out how to scale the results.  For internal use we convert
337 	 * all times from CPU frequency based to picoseconds, and values are
338 	 * eventually displayed in ms.
339 	 */
340 	for (i = 0; i < sizeof(ld.ld_freq) / sizeof(ld.ld_freq[0]); i++)
341 		if (ld.ld_freq[i] != 0)
342 			cpuscale[i] = PICO / ld.ld_freq[i];
343 	ms = ld.ld_time.tv_sec * MILLI + ld.ld_time.tv_nsec / MICRO;
344 	if (pflag)
345 		cscale = 1.0 / ncpu();
346 	else
347 		cscale = 1.0;
348 	cscale *= (sflag ? MILLI / ms : 1.0);
349 	tscale = cscale / NANO;
350 	nbufs = (int)(ld.ld_size / sizeof(lsbuf_t));
351 
352 	TAILQ_INIT(&locklist);
353 	TAILQ_INIT(&sortlist);
354 	TAILQ_INIT(&freelist);
355 
356 	if ((mflag | Mflag) != 0)
357 		collapse(mflag, Mflag);
358 
359 	/*
360 	 * Display the results.
361 	 */
362 	fprintf(outfp, "Elapsed time: %.2f seconds.", ms / MILLI);
363 	if (sflag || pflag) {
364 		fprintf(outfp, " Displaying ");
365 		if (pflag)
366 			fprintf(outfp, "per-CPU ");
367 		if (sflag)
368 			fprintf(outfp, "per-second ");
369 		fprintf(outfp, "averages.");
370 	}
371 	putc('\n', outfp);
372 
373 	for (name = xflag ? xtypes : alltypes; name->name != NULL; name++) {
374 		if (eventtype != -1 &&
375 		    (name->mask & LB_EVENT_MASK) != eventtype)
376 			continue;
377 		if (locktype != -1 &&
378 		    (name->mask & LB_LOCK_MASK) != locktype)
379 			continue;
380 		display(name->mask, name->name);
381 	}
382 
383 	if (displayed == 0)
384 		fprintf(outfp, "None of the selected events were recorded.\n");
385 	exit(EXIT_SUCCESS);
386 }
387 
388 void
389 usage(void)
390 {
391 
392 	fprintf(stderr,
393 	    "%s: usage:\n"
394 	    "%s [options] <command>\n\n"
395 	    "-b nbuf\t\tset number of event buffers to allocate\n"
396 	    "-c\t\treport percentage of total events by count, not time\n"
397 	    "-E event\t\tdisplay only one type of event\n"
398 	    "-e\t\tlist event types\n"
399 	    "-F func\t\tlimit trace to one function\n"
400 	    "-f\t\ttrace only by function\n"
401 	    "-L lock\t\tlimit trace to one lock (name, or address)\n"
402 	    "-l\t\ttrace only by lock\n"
403 	    "-M\t\tmerge lock addresses within unique objects\n"
404 	    "-m\t\tmerge call sites within unique functions\n"
405 	    "-N nlist\tspecify name list file\n"
406 	    "-o file\t\tsend output to named file, not stdout\n"
407 	    "-p\t\tshow average count/time per CPU, not total\n"
408 	    "-s\t\tshow average count/time per second, not total\n"
409 	    "-T type\t\tdisplay only one type of lock\n"
410 	    "-t\t\tlist lock types\n"
411 	    "-x\t\tdon't differentiate event types\n",
412 	    getprogname(), getprogname());
413 
414 	exit(EXIT_FAILURE);
415 }
416 
417 void
418 nullsig(int junk)
419 {
420 
421 	(void)junk;
422 }
423 
424 void
425 listnames(const name_t *name)
426 {
427 
428 	for (; name->name != NULL; name++)
429 		printf("%s\n", name->name);
430 
431 	exit(EXIT_SUCCESS);
432 }
433 
434 int
435 matchname(const name_t *name, char *string)
436 {
437 	int empty, mask;
438 	char *sp;
439 
440 	empty = 1;
441 	mask = 0;
442 
443 	while ((sp = strsep(&string, ",")) != NULL) {
444 		if (*sp == '\0')
445 			usage();
446 
447 		for (; name->name != NULL; name++) {
448 			if (strcasecmp(name->name, sp) == 0) {
449 				mask |= name->mask;
450 				break;
451 			}
452 		}
453 		if (name->name == NULL)
454 			errx(EXIT_FAILURE, "unknown identifier `%s'", sp);
455 		empty = 0;
456 	}
457 
458 	if (empty)
459 		usage();
460 
461 	return mask;
462 }
463 
464 /*
465  * Return the number of CPUs in the running system.
466  */
467 int
468 ncpu(void)
469 {
470 	int rv, mib[2];
471 	size_t varlen;
472 
473 	mib[0] = CTL_HW;
474 	mib[1] = HW_NCPU;
475 	varlen = sizeof(rv);
476 	if (sysctl(mib, 2, &rv, &varlen, NULL, (size_t)0) < 0)
477 		rv = 1;
478 
479 	return (rv);
480 }
481 
482 /*
483  * Call into the ELF parser and look up a symbol by name or by address.
484  */
485 void
486 findsym(findsym_t find, char *name, uintptr_t *start, uintptr_t *end, bool chg)
487 {
488 	uintptr_t tend, sa, ea;
489 	char *p;
490 	int rv;
491 
492 	if (!chg) {
493 		sa = *start;
494 		start = &sa;
495 		end = &ea;
496 	}
497 
498 	if (end == NULL)
499 		end = &tend;
500 
501 	if (find == LOCK_BYNAME) {
502 		if (isdigit((u_int)name[0])) {
503 			*start = (uintptr_t)strtoul(name, &p, 0);
504 			if (*p == '\0')
505 				return;
506 		}
507 	}
508 
509 	if (bin64)
510 		rv = findsym64(find, name, start, end);
511 	else
512 		rv = findsym32(find, name, start, end);
513 
514 	if (find == FUNC_BYNAME || find == LOCK_BYNAME) {
515 		if (rv == -1)
516 			errx(EXIT_FAILURE, "unable to find symbol `%s'", name);
517 		return;
518 	}
519 
520 	if (rv == -1)
521 		snprintf(name, NAME_SIZE, "%016lx", (long)*start);
522 }
523 
524 /*
525  * Fork off the child process and wait for it to complete.  We trap SIGINT
526  * so that the caller can use Ctrl-C to stop tracing early and still get
527  * useful results.
528  */
529 void
530 spawn(int argc, char **argv)
531 {
532 	pid_t pid;
533 
534 	switch (pid = fork()) {
535 	case 0:
536 		close(lsfd);
537 		if (execvp(argv[0], argv) == -1)
538 			err(EXIT_FAILURE, "cannot exec");
539 		break;
540 	case -1:
541 		err(EXIT_FAILURE, "cannot fork to exec");
542 		break;
543 	default:
544 		signal(SIGINT, nullsig);
545 		wait(NULL);
546 		signal(SIGINT, SIG_DFL);
547 		break;
548 	}
549 }
550 
551 /*
552  * Allocate a new block of lock_t structures.
553  */
554 lock_t *
555 morelocks(void)
556 {
557 	const static int batch = 32;
558 	lock_t *l, *lp, *max;
559 
560 	l = (lock_t *)malloc(sizeof(*l) * batch);
561 
562 	for (lp = l, max = l + batch; lp < max; lp++)
563 		TAILQ_INSERT_TAIL(&freelist, lp, chain);
564 
565 	return l;
566 }
567 
568 /*
569  * Collapse addresses from unique objects.
570  */
571 void
572 collapse(bool func, bool lock)
573 {
574 	lsbuf_t *lb, *max;
575 
576 	for (lb = bufs, max = bufs + nbufs; lb < max; lb++) {
577 		if (func && lb->lb_callsite != 0) {
578 			findsym(FUNC_BYADDR, NULL, &lb->lb_callsite, NULL,
579 			    true);
580 		}
581 		if (lock && lb->lb_lock != 0) {
582 			findsym(LOCK_BYADDR, NULL, &lb->lb_lock, NULL,
583 			    true);
584 		}
585 	}
586 }
587 
588 /*
589  * From the kernel supplied data, construct two dimensional lists of locks
590  * and event buffers, indexed by lock type and sorted by event type.
591  */
592 void
593 makelists(int mask, int event)
594 {
595 	lsbuf_t *lb, *lb2, *max;
596 	lock_t *l, *l2;
597 	int type;
598 
599 	/*
600 	 * Recycle lock_t structures from the last run.
601 	 */
602 	while ((l = TAILQ_FIRST(&locklist)) != NULL) {
603 		TAILQ_REMOVE(&locklist, l, chain);
604 		TAILQ_INSERT_HEAD(&freelist, l, chain);
605 	}
606 
607 	type = mask & LB_LOCK_MASK;
608 
609 	for (lb = bufs, max = bufs + nbufs; lb < max; lb++) {
610 		if (!xflag && (lb->lb_flags & LB_LOCK_MASK) != type)
611 			continue;
612 		if (lb->lb_counts[event] == 0)
613 			continue;
614 
615 		/*
616 		 * Look for a record descibing this lock, and allocate a
617 		 * new one if needed.
618 		 */
619 		TAILQ_FOREACH(l, &sortlist, chain) {
620 			if (l->lock == lb->lb_lock)
621 				break;
622 		}
623 		if (l == NULL) {
624 			if ((l = TAILQ_FIRST(&freelist)) == NULL)
625 				l = morelocks();
626 			TAILQ_REMOVE(&freelist, l, chain);
627 			l->flags = lb->lb_flags;
628 			l->lock = lb->lb_lock;
629 			l->nbufs = 0;
630 			l->name[0] = '\0';
631 			l->count = 0;
632 			l->time = 0;
633 			TAILQ_INIT(&l->tosort);
634 			TAILQ_INIT(&l->bufs);
635 			TAILQ_INSERT_TAIL(&sortlist, l, chain);
636 		}
637 
638 		/*
639 		 * Scale the time values per buffer and summarise
640 		 * times+counts per lock.
641 		 */
642 		lb->lb_times[event] *= cpuscale[lb->lb_cpu];
643 		l->count += lb->lb_counts[event];
644 		l->time += lb->lb_times[event];
645 
646 		/*
647 		 * Merge same lock+callsite pairs from multiple CPUs
648 		 * together.
649 		 */
650 		TAILQ_FOREACH(lb2, &l->tosort, lb_chain.tailq) {
651 			if (lb->lb_callsite == lb2->lb_callsite)
652 				break;
653 		}
654 		if (lb2 != NULL) {
655 			lb2->lb_counts[event] += lb->lb_counts[event];
656 			lb2->lb_times[event] += lb->lb_times[event];
657 		} else {
658 			TAILQ_INSERT_HEAD(&l->tosort, lb, lb_chain.tailq);
659 			l->nbufs++;
660 		}
661 	}
662 
663 	/*
664 	 * Now sort the lists.
665 	 */
666 	while ((l = TAILQ_FIRST(&sortlist)) != NULL) {
667 		TAILQ_REMOVE(&sortlist, l, chain);
668 
669 		/*
670 		 * Sort the buffers into the per-lock list.
671 		 */
672 		while ((lb = TAILQ_FIRST(&l->tosort)) != NULL) {
673 			TAILQ_REMOVE(&l->tosort, lb, lb_chain.tailq);
674 
675 			lb2 = TAILQ_FIRST(&l->bufs);
676 			while (lb2 != NULL) {
677 				if (cflag) {
678 					if (lb->lb_counts[event] >
679 					    lb2->lb_counts[event])
680 						break;
681 				} else if (lb->lb_times[event] >
682 				    lb2->lb_times[event])
683 					break;
684 				lb2 = TAILQ_NEXT(lb2, lb_chain.tailq);
685 			}
686 			if (lb2 == NULL)
687 				TAILQ_INSERT_TAIL(&l->bufs, lb,
688 				    lb_chain.tailq);
689 			else
690 				TAILQ_INSERT_BEFORE(lb2, lb, lb_chain.tailq);
691 		}
692 
693 		/*
694 		 * Sort this lock into the per-type list, based on the
695 		 * totals per lock.
696 		 */
697 		l2 = TAILQ_FIRST(&locklist);
698 		while (l2 != NULL) {
699 			if (cflag) {
700 				if (l->count > l2->count)
701 					break;
702 			} else if (l->time > l2->time)
703 				break;
704 			l2 = TAILQ_NEXT(l2, chain);
705 		}
706 		if (l2 == NULL)
707 			TAILQ_INSERT_TAIL(&locklist, l, chain);
708 		else
709 			TAILQ_INSERT_BEFORE(l2, l, chain);
710 	}
711 }
712 
713 /*
714  * Display a summary table for one lock type / event type pair.
715  */
716 void
717 display(int mask, const char *name)
718 {
719 	lock_t *l;
720 	lsbuf_t *lb;
721 	double pcscale, metric;
722 	char fname[NAME_SIZE];
723 	int event;
724 
725 	event = (mask & LB_EVENT_MASK) - 1;
726 	makelists(mask, event);
727 
728 	if (TAILQ_EMPTY(&locklist))
729 		return;
730 
731 	fprintf(outfp, "\n-- %s\n\n"
732 	    "Total%%  Count   Time/ms          Lock                       Caller\n"
733 	    "------ ------- --------- ---------------------- ------------------------------\n",
734 	    name);
735 
736 	/*
737 	 * Sum up all events for this type of lock + event.
738 	 */
739 	pcscale = 0;
740 	TAILQ_FOREACH(l, &locklist, chain) {
741 		if (cflag)
742 			pcscale += l->count;
743 		else
744 			pcscale += l->time;
745 		displayed++;
746 	}
747 	if (pcscale == 0)
748 		pcscale = 100;
749 	else
750 		pcscale = (100.0 / pcscale);
751 
752 	/*
753 	 * For each lock, print a summary total, followed by a breakdown by
754 	 * caller.
755 	 */
756 	TAILQ_FOREACH(l, &locklist, chain) {
757 		if (cflag)
758 			metric = l->count;
759 		else
760 			metric = l->time;
761 		metric *= pcscale;
762 
763 		if (l->name[0] == '\0')
764 			findsym(LOCK_BYADDR, l->name, &l->lock, NULL, false);
765 
766 		if (lflag || l->nbufs > 1)
767 			fprintf(outfp, "%6.2f %7d %9.2f %-22s <all>\n",
768 			    metric, (int)(l->count * cscale),
769 			    l->time * tscale, l->name);
770 
771 		if (lflag)
772 			continue;
773 
774 		TAILQ_FOREACH(lb, &l->bufs, lb_chain.tailq) {
775 			if (cflag)
776 				metric = lb->lb_counts[event];
777 			else
778 				metric = lb->lb_times[event];
779 			metric *= pcscale;
780 
781 			findsym(FUNC_BYADDR, fname, &lb->lb_callsite, NULL,
782 			    false);
783 			fprintf(outfp, "%6.2f %7d %9.2f %-22s %s\n",
784 			    metric, (int)(lb->lb_counts[event] * cscale),
785 			    lb->lb_times[event] * tscale, l->name, fname);
786 		}
787 	}
788 }
789