xref: /netbsd-src/usr.sbin/lockstat/main.c (revision 8b0f9554ff8762542c4defc4f70e1eb76fb508fa)
1 /*	$NetBSD: main.c,v 1.10 2007/07/15 21:24:46 wiz Exp $	*/
2 
3 /*-
4  * Copyright (c) 2006, 2007 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Andrew Doran.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  * 3. All advertising materials mentioning features or use of this software
19  *    must display the following acknowledgement:
20  *	This product includes software developed by the NetBSD
21  *	Foundation, Inc. and its contributors.
22  * 4. Neither the name of The NetBSD Foundation nor the names of its
23  *    contributors may be used to endorse or promote products derived
24  *    from this software without specific prior written permission.
25  *
26  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
27  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
28  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
30  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
31  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
32  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
33  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
34  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
35  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
36  * POSSIBILITY OF SUCH DAMAGE.
37  */
38 
39 /*
40  * TODO:
41  *
42  * - Tracking of times for sleep locks is broken.
43  * - Need better analysis and tracking of events.
44  * - Shouldn't have to parse the namelist here.  We should use something like
45  *   FreeBSD's libelf.
46  * - The way the namelist is searched sucks, is it worth doing something
47  *   better?
48  */
49 
50 #include <sys/cdefs.h>
51 #ifndef lint
52 __RCSID("$NetBSD: main.c,v 1.10 2007/07/15 21:24:46 wiz Exp $");
53 #endif /* not lint */
54 
55 #include <sys/types.h>
56 #include <sys/param.h>
57 #include <sys/time.h>
58 #include <sys/fcntl.h>
59 #include <sys/ioctl.h>
60 #include <sys/wait.h>
61 #include <sys/signal.h>
62 #include <sys/sysctl.h>
63 
64 #include <dev/lockstat.h>
65 
66 #include <stdio.h>
67 #include <stdlib.h>
68 #include <string.h>
69 #include <limits.h>
70 #include <unistd.h>
71 #include <err.h>
72 #include <paths.h>
73 #include <util.h>
74 #include <ctype.h>
75 #include <errno.h>
76 #include <stdbool.h>
77 
78 #include "extern.h"
79 
80 #define	_PATH_DEV_LOCKSTAT	"/dev/lockstat"
81 
82 #define	MILLI	1000.0
83 #define	MICRO	1000000.0
84 #define	NANO	1000000000.0
85 #define	PICO	1000000000000.0
86 
87 TAILQ_HEAD(lock_head, lockstruct);
88 typedef struct lock_head locklist_t;
89 TAILQ_HEAD(buf_head, lsbuf);
90 typedef struct buf_head buflist_t;
91 
92 typedef struct lockstruct {
93 	TAILQ_ENTRY(lockstruct)	chain;
94 	buflist_t		bufs;
95 	buflist_t		tosort;
96 	uintptr_t		lock;
97  	double			time;
98 	uint32_t		count;
99 	u_int			flags;
100 	u_int			nbufs;
101 	char			name[NAME_SIZE];
102 } lock_t;
103 
104 typedef struct name {
105 	const char	*name;
106 	int		mask;
107 } name_t;
108 
109 const name_t locknames[] = {
110 	{ "adaptive_mutex", LB_ADAPTIVE_MUTEX },
111 	{ "spin_mutex", LB_SPIN_MUTEX },
112 	{ "lockmgr", LB_LOCKMGR },
113 	{ "rwlock", LB_RWLOCK },
114 	{ "kernel_lock", LB_KERNEL_LOCK },
115 	{ NULL, 0 }
116 };
117 
118 const name_t eventnames[] = {
119 	{ "spin", LB_SPIN },
120 	{ "sleep_exclusive", LB_SLEEP1 },
121 	{ "sleep_shared", LB_SLEEP2 },
122 	{ NULL, 0 },
123 };
124 
125 const name_t alltypes[] = {
126 	{ "Adaptive mutex spin", LB_ADAPTIVE_MUTEX | LB_SPIN },
127 	{ "Adaptive mutex sleep", LB_ADAPTIVE_MUTEX | LB_SLEEP1 },
128 	{ "Spin mutex spin", LB_SPIN_MUTEX | LB_SPIN },
129 	{ "lockmgr sleep", LB_LOCKMGR | LB_SLEEP1 },
130 	{ "RW lock sleep (writer)", LB_RWLOCK | LB_SLEEP1 },
131 	{ "RW lock sleep (reader)", LB_RWLOCK | LB_SLEEP2 },
132 	{ "Kernel lock spin", LB_KERNEL_LOCK | LB_SPIN },
133 	{ NULL, 0 }
134 };
135 
136 locklist_t	locklist;
137 locklist_t	freelist;
138 locklist_t	sortlist;
139 
140 lsbuf_t		*bufs;
141 lsdisable_t	ld;
142 bool		lflag;
143 bool		fflag;
144 int		nbufs;
145 bool		cflag;
146 int		lsfd;
147 int		displayed;
148 int		bin64;
149 double		tscale;
150 double		cscale;
151 double		cpuscale[sizeof(ld.ld_freq) / sizeof(ld.ld_freq[0])];
152 FILE		*outfp;
153 
154 void	findsym(findsym_t, char *, uintptr_t *, uintptr_t *, bool);
155 void	spawn(int, char **);
156 void	display(int, const char *name);
157 void	listnames(const name_t *);
158 void	collapse(bool, bool);
159 int	matchname(const name_t *, char *);
160 void	makelists(int, int);
161 void	nullsig(int);
162 void	usage(void);
163 int	ncpu(void);
164 lock_t	*morelocks(void);
165 
166 int
167 main(int argc, char **argv)
168 {
169 	int eventtype, locktype, ch, nlfd, fd, i;
170 	bool sflag, pflag, mflag, Mflag;
171 	const char *nlistf, *outf;
172 	char *lockname, *funcname;
173 	const name_t *name;
174 	lsenable_t le;
175 	double ms;
176 	char *p;
177 
178 	nlistf = NULL;
179 	outf = NULL;
180 	lockname = NULL;
181 	funcname = NULL;
182 	eventtype = -1;
183 	locktype = -1;
184 	nbufs = 0;
185 	sflag = false;
186 	pflag = false;
187 	mflag = false;
188 	Mflag = false;
189 
190 	while ((ch = getopt(argc, argv, "E:F:L:MN:T:b:ceflmo:pst")) != -1)
191 		switch (ch) {
192 		case 'E':
193 			eventtype = matchname(eventnames, optarg);
194 			break;
195 		case 'F':
196 			funcname = optarg;
197 			break;
198 		case 'L':
199 			lockname = optarg;
200 			break;
201 		case 'N':
202 			nlistf = optarg;
203 			break;
204 		case 'T':
205 			locktype = matchname(locknames, optarg);
206 			break;
207 		case 'b':
208 			nbufs = (int)strtol(optarg, &p, 0);
209 			if (!isdigit((u_int)*optarg) || *p != '\0')
210 				usage();
211 			break;
212 		case 'c':
213 			cflag = true;
214 			break;
215 		case 'e':
216 			listnames(eventnames);
217 			break;
218 		case 'f':
219 			fflag = true;
220 			break;
221 		case 'l':
222 			lflag = true;
223 			break;
224 		case 'm':
225 			mflag = true;
226 			break;
227 		case 'M':
228 			Mflag = true;
229 			break;
230 		case 'o':
231 			outf = optarg;
232 			break;
233 		case 'p':
234 			pflag = true;
235 			break;
236 		case 's':
237 			sflag = true;
238 			break;
239 		case 't':
240 			listnames(locknames);
241 			break;
242 		default:
243 			usage();
244 		}
245 	argc -= optind;
246 	argv += optind;
247 
248 	if (*argv == NULL)
249 		usage();
250 
251 	if (outf) {
252 		fd = open(outf, O_WRONLY | O_CREAT | O_TRUNC, 0600);
253 		if (fd == -1)
254 			err(EXIT_FAILURE, "opening %s", outf);
255 		outfp = fdopen(fd, "w");
256 	} else
257 		outfp = stdout;
258 
259 	/*
260 	 * Find the name list for resolving symbol names, and load it into
261 	 * memory.
262 	 */
263 	if (nlistf == NULL) {
264 		nlfd = open(_PATH_KSYMS, O_RDONLY);
265 		nlistf = getbootfile();
266 	} else
267 		nlfd = -1;
268 	if (nlfd == -1) {
269 		if ((nlfd = open(nlistf, O_RDONLY)) < 0)
270 			err(EXIT_FAILURE, "cannot open " _PATH_KSYMS " or %s",
271 			    nlistf);
272 	}
273 	if (loadsym32(nlfd) != 0) {
274 		if (loadsym64(nlfd) != 0)
275 			errx(EXIT_FAILURE, "unable to load symbol table");
276 		bin64 = 1;
277 	}
278 	close(nlfd);
279 
280 	memset(&le, 0, sizeof(le));
281 	le.le_nbufs = nbufs;
282 
283 	/*
284 	 * Set up initial filtering.
285 	 */
286 	if (lockname != NULL) {
287 		findsym(LOCK_BYNAME, lockname, &le.le_lockstart,
288 		    &le.le_lockend, true);
289 		le.le_flags |= LE_ONE_LOCK;
290 	}
291 	if (!lflag)
292 		le.le_flags |= LE_CALLSITE;
293 	if (!fflag)
294 		le.le_flags |= LE_LOCK;
295 	if (funcname != NULL) {
296 		if (lflag)
297 			usage();
298 		findsym(FUNC_BYNAME, funcname, &le.le_csstart, &le.le_csend, true);
299 		le.le_flags |= LE_ONE_CALLSITE;
300 	}
301 	le.le_mask = (eventtype & LB_EVENT_MASK) | (locktype & LB_LOCK_MASK);
302 
303 	/*
304 	 * Start tracing.
305 	 */
306 	if ((lsfd = open(_PATH_DEV_LOCKSTAT, O_RDONLY)) < 0)
307 		err(EXIT_FAILURE, "cannot open " _PATH_DEV_LOCKSTAT);
308 	if (ioctl(lsfd, IOC_LOCKSTAT_GVERSION, &ch) < 0)
309 		err(EXIT_FAILURE, "ioctl");
310 	if (ch != LS_VERSION)
311 		errx(EXIT_FAILURE,
312 		    "incompatible lockstat interface version (%d, kernel %d)",
313 			LS_VERSION, ch);
314 	if (ioctl(lsfd, IOC_LOCKSTAT_ENABLE, &le))
315 		err(EXIT_FAILURE, "cannot enable tracing");
316 
317 	/*
318 	 * Execute the traced program.
319 	 */
320 	spawn(argc, argv);
321 
322 	/*
323 	 * Stop tracing, and read the trace buffers from the kernel.
324 	 */
325 	if (ioctl(lsfd, IOC_LOCKSTAT_DISABLE, &ld) == -1) {
326 		if (errno == EOVERFLOW) {
327 			warnx("overflowed available kernel trace buffers");
328 			exit(EXIT_FAILURE);
329 		}
330 		err(EXIT_FAILURE, "cannot disable tracing");
331 	}
332 	if ((bufs = malloc(ld.ld_size)) == NULL)
333 		err(EXIT_FAILURE, "cannot allocate memory for user buffers");
334 	if (read(lsfd, bufs, ld.ld_size) != ld.ld_size)
335 		err(EXIT_FAILURE, "reading from " _PATH_DEV_LOCKSTAT);
336 	if (close(lsfd))
337 		err(EXIT_FAILURE, "close(" _PATH_DEV_LOCKSTAT ")");
338 
339 	/*
340 	 * Figure out how to scale the results.  For internal use we convert
341 	 * all times from CPU frequency based to picoseconds, and values are
342 	 * eventually displayed in ms.
343 	 */
344 	for (i = 0; i < sizeof(ld.ld_freq) / sizeof(ld.ld_freq[0]); i++)
345 		if (ld.ld_freq[i] != 0)
346 			cpuscale[i] = PICO / ld.ld_freq[i];
347 	ms = ld.ld_time.tv_sec * MILLI + ld.ld_time.tv_nsec / MICRO;
348 	if (pflag)
349 		cscale = 1.0 / ncpu();
350 	else
351 		cscale = 1.0;
352 	cscale *= (sflag ? MILLI / ms : 1.0);
353 	tscale = cscale / NANO;
354 	nbufs = (int)(ld.ld_size / sizeof(lsbuf_t));
355 
356 	TAILQ_INIT(&locklist);
357 	TAILQ_INIT(&sortlist);
358 	TAILQ_INIT(&freelist);
359 
360 	if ((mflag | Mflag) != 0)
361 		collapse(mflag, Mflag);
362 
363 	/*
364 	 * Display the results.
365 	 */
366 	fprintf(outfp, "Elapsed time: %.2f seconds.", ms / MILLI);
367 	if (sflag || pflag) {
368 		fprintf(outfp, " Displaying ");
369 		if (pflag)
370 			fprintf(outfp, "per-CPU ");
371 		if (sflag)
372 			fprintf(outfp, "per-second ");
373 		fprintf(outfp, "averages.");
374 	}
375 	putc('\n', outfp);
376 
377 	for (name = alltypes; name->name != NULL; name++) {
378 		if (eventtype != -1 &&
379 		    (name->mask & LB_EVENT_MASK) != eventtype)
380 			continue;
381 		if (locktype != -1 &&
382 		    (name->mask & LB_LOCK_MASK) != locktype)
383 			continue;
384 
385 		display(name->mask, name->name);
386 	}
387 
388 	if (displayed == 0)
389 		fprintf(outfp, "None of the selected events were recorded.\n");
390 	exit(EXIT_SUCCESS);
391 }
392 
393 void
394 usage(void)
395 {
396 
397 	fprintf(stderr,
398 	    "%s: usage:\n"
399 	    "%s [options] <command>\n\n"
400 	    "-b nbuf\t\tset number of event buffers to allocate\n"
401 	    "-c\t\treport percentage of total events by count, not time\n"
402 	    "-E event\t\tdisplay only one type of event\n"
403 	    "-e\t\tlist event types\n"
404 	    "-F func\t\tlimit trace to one function\n"
405 	    "-f\t\ttrace only by function\n"
406 	    "-L lock\t\tlimit trace to one lock (name, or address)\n"
407 	    "-l\t\ttrace only by lock\n"
408 	    "-M\t\tmerge lock addresses within unique objects\n"
409 	    "-m\t\tmerge call sites within unique functions\n"
410 	    "-N nlist\tspecify name list file\n"
411 	    "-o file\t\tsend output to named file, not stdout\n"
412 	    "-p\t\tshow average count/time per CPU, not total\n"
413 	    "-s\t\tshow average count/time per second, not total\n"
414 	    "-T type\t\tdisplay only one type of lock\n"
415 	    "-t\t\tlist lock types\n",
416 	    getprogname(), getprogname());
417 
418 	exit(EXIT_FAILURE);
419 }
420 
421 void
422 nullsig(int junk)
423 {
424 
425 	(void)junk;
426 }
427 
428 void
429 listnames(const name_t *name)
430 {
431 
432 	for (; name->name != NULL; name++)
433 		printf("%s\n", name->name);
434 
435 	exit(EXIT_SUCCESS);
436 }
437 
438 int
439 matchname(const name_t *name, char *string)
440 {
441 	int empty, mask;
442 	char *sp;
443 
444 	empty = 1;
445 	mask = 0;
446 
447 	while ((sp = strsep(&string, ",")) != NULL) {
448 		if (*sp == '\0')
449 			usage();
450 
451 		for (; name->name != NULL; name++) {
452 			if (strcasecmp(name->name, sp) == 0) {
453 				mask |= name->mask;
454 				break;
455 			}
456 		}
457 		if (name->name == NULL)
458 			errx(EXIT_FAILURE, "unknown identifier `%s'", sp);
459 		empty = 0;
460 	}
461 
462 	if (empty)
463 		usage();
464 
465 	return mask;
466 }
467 
468 /*
469  * Return the number of CPUs in the running system.
470  */
471 int
472 ncpu(void)
473 {
474 	int rv, mib[2];
475 	size_t varlen;
476 
477 	mib[0] = CTL_HW;
478 	mib[1] = HW_NCPU;
479 	varlen = sizeof(rv);
480 	if (sysctl(mib, 2, &rv, &varlen, NULL, (size_t)0) < 0)
481 		rv = 1;
482 
483 	return (rv);
484 }
485 
486 /*
487  * Call into the ELF parser and look up a symbol by name or by address.
488  */
489 void
490 findsym(findsym_t find, char *name, uintptr_t *start, uintptr_t *end, bool chg)
491 {
492 	uintptr_t tend, sa, ea;
493 	char *p;
494 	int rv;
495 
496 	if (!chg) {
497 		sa = *start;
498 		start = &sa;
499 		end = &ea;
500 	}
501 
502 	if (end == NULL)
503 		end = &tend;
504 
505 	if (find == LOCK_BYNAME) {
506 		if (isdigit((u_int)name[0])) {
507 			*start = (uintptr_t)strtoul(name, &p, 0);
508 			if (*p == '\0')
509 				return;
510 		}
511 	}
512 
513 	if (bin64)
514 		rv = findsym64(find, name, start, end);
515 	else
516 		rv = findsym32(find, name, start, end);
517 
518 	if (find == FUNC_BYNAME || find == LOCK_BYNAME) {
519 		if (rv == -1)
520 			errx(EXIT_FAILURE, "unable to find symbol `%s'", name);
521 		return;
522 	}
523 
524 	if (rv == -1)
525 		snprintf(name, NAME_SIZE, "%016lx", (long)*start);
526 }
527 
528 /*
529  * Fork off the child process and wait for it to complete.  We trap SIGINT
530  * so that the caller can use Ctrl-C to stop tracing early and still get
531  * useful results.
532  */
533 void
534 spawn(int argc, char **argv)
535 {
536 	pid_t pid;
537 
538 	switch (pid = fork()) {
539 	case 0:
540 		close(lsfd);
541 		if (execvp(argv[0], argv) == -1)
542 			err(EXIT_FAILURE, "cannot exec");
543 		break;
544 	case -1:
545 		err(EXIT_FAILURE, "cannot fork to exec");
546 		break;
547 	default:
548 		signal(SIGINT, nullsig);
549 		wait(NULL);
550 		signal(SIGINT, SIG_DFL);
551 		break;
552 	}
553 }
554 
555 /*
556  * Allocate a new block of lock_t structures.
557  */
558 lock_t *
559 morelocks(void)
560 {
561 	const static int batch = 32;
562 	lock_t *l, *lp, *max;
563 
564 	l = (lock_t *)malloc(sizeof(*l) * batch);
565 
566 	for (lp = l, max = l + batch; lp < max; lp++)
567 		TAILQ_INSERT_TAIL(&freelist, lp, chain);
568 
569 	return l;
570 }
571 
572 /*
573  * Collapse addresses from unique objects.
574  */
575 void
576 collapse(bool func, bool lock)
577 {
578 	lsbuf_t *lb, *max;
579 
580 	for (lb = bufs, max = bufs + nbufs; lb < max; lb++) {
581 		if (func && lb->lb_callsite != 0) {
582 			findsym(FUNC_BYADDR, NULL, &lb->lb_callsite, NULL,
583 			    true);
584 		}
585 		if (lock && lb->lb_lock != 0) {
586 			findsym(LOCK_BYADDR, NULL, &lb->lb_lock, NULL,
587 			    true);
588 		}
589 	}
590 }
591 
592 /*
593  * From the kernel supplied data, construct two dimensional lists of locks
594  * and event buffers, indexed by lock type and sorted by event type.
595  */
596 void
597 makelists(int mask, int event)
598 {
599 	lsbuf_t *lb, *lb2, *max;
600 	lock_t *l, *l2;
601 	int type;
602 
603 	/*
604 	 * Recycle lock_t structures from the last run.
605 	 */
606 	while ((l = TAILQ_FIRST(&locklist)) != NULL) {
607 		TAILQ_REMOVE(&locklist, l, chain);
608 		TAILQ_INSERT_HEAD(&freelist, l, chain);
609 	}
610 
611 	type = mask & LB_LOCK_MASK;
612 
613 	for (lb = bufs, max = bufs + nbufs; lb < max; lb++) {
614 		if ((lb->lb_flags & LB_LOCK_MASK) != type ||
615 		    lb->lb_counts[event] == 0)
616 			continue;
617 
618 		/*
619 		 * Look for a record descibing this lock, and allocate a
620 		 * new one if needed.
621 		 */
622 		TAILQ_FOREACH(l, &sortlist, chain) {
623 			if (l->lock == lb->lb_lock)
624 				break;
625 		}
626 		if (l == NULL) {
627 			if ((l = TAILQ_FIRST(&freelist)) == NULL)
628 				l = morelocks();
629 			TAILQ_REMOVE(&freelist, l, chain);
630 			l->flags = lb->lb_flags;
631 			l->lock = lb->lb_lock;
632 			l->nbufs = 0;
633 			l->name[0] = '\0';
634 			l->count = 0;
635 			l->time = 0;
636 			TAILQ_INIT(&l->tosort);
637 			TAILQ_INIT(&l->bufs);
638 			TAILQ_INSERT_TAIL(&sortlist, l, chain);
639 		}
640 
641 		/*
642 		 * Scale the time values per buffer and summarise
643 		 * times+counts per lock.
644 		 */
645 		lb->lb_times[event] *= cpuscale[lb->lb_cpu];
646 		l->count += lb->lb_counts[event];
647 		l->time += lb->lb_times[event];
648 
649 		/*
650 		 * Merge same lock+callsite pairs from multiple CPUs
651 		 * together.
652 		 */
653 		TAILQ_FOREACH(lb2, &l->tosort, lb_chain.tailq) {
654 			if (lb->lb_callsite == lb2->lb_callsite)
655 				break;
656 		}
657 		if (lb2 != NULL) {
658 			lb2->lb_counts[event] += lb->lb_counts[event];
659 			lb2->lb_times[event] += lb->lb_times[event];
660 		} else {
661 			TAILQ_INSERT_HEAD(&l->tosort, lb, lb_chain.tailq);
662 			l->nbufs++;
663 		}
664 	}
665 
666 	/*
667 	 * Now sort the lists.
668 	 */
669 	while ((l = TAILQ_FIRST(&sortlist)) != NULL) {
670 		TAILQ_REMOVE(&sortlist, l, chain);
671 
672 		/*
673 		 * Sort the buffers into the per-lock list.
674 		 */
675 		while ((lb = TAILQ_FIRST(&l->tosort)) != NULL) {
676 			TAILQ_REMOVE(&l->tosort, lb, lb_chain.tailq);
677 
678 			lb2 = TAILQ_FIRST(&l->bufs);
679 			while (lb2 != NULL) {
680 				if (cflag) {
681 					if (lb->lb_counts[event] >
682 					    lb2->lb_counts[event])
683 						break;
684 				} else if (lb->lb_times[event] >
685 				    lb2->lb_times[event])
686 					break;
687 				lb2 = TAILQ_NEXT(lb2, lb_chain.tailq);
688 			}
689 			if (lb2 == NULL)
690 				TAILQ_INSERT_TAIL(&l->bufs, lb,
691 				    lb_chain.tailq);
692 			else
693 				TAILQ_INSERT_BEFORE(lb2, lb, lb_chain.tailq);
694 		}
695 
696 		/*
697 		 * Sort this lock into the per-type list, based on the
698 		 * totals per lock.
699 		 */
700 		l2 = TAILQ_FIRST(&locklist);
701 		while (l2 != NULL) {
702 			if (cflag) {
703 				if (l->count > l2->count)
704 					break;
705 			} else if (l->time > l2->time)
706 				break;
707 			l2 = TAILQ_NEXT(l2, chain);
708 		}
709 		if (l2 == NULL)
710 			TAILQ_INSERT_TAIL(&locklist, l, chain);
711 		else
712 			TAILQ_INSERT_BEFORE(l2, l, chain);
713 	}
714 }
715 
716 /*
717  * Display a summary table for one lock type / event type pair.
718  */
719 void
720 display(int mask, const char *name)
721 {
722 	lock_t *l;
723 	lsbuf_t *lb;
724 	double pcscale, metric;
725 	char fname[NAME_SIZE];
726 	int event;
727 
728 	event = (mask & LB_EVENT_MASK) - 1;
729 	makelists(mask, event);
730 
731 	if (TAILQ_EMPTY(&locklist))
732 		return;
733 
734 	fprintf(outfp, "\n-- %s\n\n"
735 	    "Total%%  Count   Time/ms          Lock                       Caller\n"
736 	    "------ ------- --------- ---------------------- ------------------------------\n",
737 	    name);
738 
739 	/*
740 	 * Sum up all events for this type of lock + event.
741 	 */
742 	pcscale = 0;
743 	TAILQ_FOREACH(l, &locklist, chain) {
744 		if (cflag)
745 			pcscale += l->count;
746 		else
747 			pcscale += l->time;
748 		displayed++;
749 	}
750 	if (pcscale == 0)
751 		pcscale = 100;
752 	else
753 		pcscale = (100.0 / pcscale);
754 
755 	/*
756 	 * For each lock, print a summary total, followed by a breakdown by
757 	 * caller.
758 	 */
759 	TAILQ_FOREACH(l, &locklist, chain) {
760 		if (cflag)
761 			metric = l->count;
762 		else
763 			metric = l->time;
764 		metric *= pcscale;
765 
766 		if (l->name[0] == '\0')
767 			findsym(LOCK_BYADDR, l->name, &l->lock, NULL, false);
768 
769 		if (lflag || l->nbufs > 1)
770 			fprintf(outfp, "%6.2f %7d %9.2f %-22s <all>\n",
771 			    metric, (int)(l->count * cscale),
772 			    l->time * tscale, l->name);
773 
774 		if (lflag)
775 			continue;
776 
777 		TAILQ_FOREACH(lb, &l->bufs, lb_chain.tailq) {
778 			if (cflag)
779 				metric = lb->lb_counts[event];
780 			else
781 				metric = lb->lb_times[event];
782 			metric *= pcscale;
783 
784 			findsym(FUNC_BYADDR, fname, &lb->lb_callsite, NULL,
785 			    false);
786 			fprintf(outfp, "%6.2f %7d %9.2f %-22s %s\n",
787 			    metric, (int)(lb->lb_counts[event] * cscale),
788 			    lb->lb_times[event] * tscale, l->name, fname);
789 		}
790 	}
791 }
792