xref: /spdk/app/trace/trace.cpp (revision 4e527910925f8d001001e96f1719d015a7a51a94)
1 /*-
2  *   BSD LICENSE
3  *
4  *   Copyright (c) Intel Corporation.
5  *   All rights reserved.
6  *
7  *   Redistribution and use in source and binary forms, with or without
8  *   modification, are permitted provided that the following conditions
9  *   are met:
10  *
11  *     * Redistributions of source code must retain the above copyright
12  *       notice, this list of conditions and the following disclaimer.
13  *     * Redistributions in binary form must reproduce the above copyright
14  *       notice, this list of conditions and the following disclaimer in
15  *       the documentation and/or other materials provided with the
16  *       distribution.
17  *     * Neither the name of Intel Corporation nor the names of its
18  *       contributors may be used to endorse or promote products derived
19  *       from this software without specific prior written permission.
20  *
21  *   THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22  *   "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23  *   LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24  *   A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25  *   OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26  *   SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27  *   LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28  *   DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29  *   THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30  *   (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31  *   OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32  */
33 
34 #include "spdk/stdinc.h"
35 #include "spdk/env.h"
36 #include "spdk/json.h"
37 #include "spdk/likely.h"
38 #include "spdk/string.h"
39 #include "spdk/util.h"
40 
41 #include <map>
42 
43 extern "C" {
44 #include "spdk/trace.h"
45 #include "spdk/util.h"
46 }
47 
48 static struct spdk_trace_histories *g_histories;
49 static struct spdk_json_write_ctx *g_json;
50 static bool g_print_tsc = false;
51 
52 /* This is a bit ugly, but we don't want to include env_dpdk in the app, while spdk_util, which we
53  * do need, uses some of the functions implemented there.  We're not actually using the functions
54  * that depend on those, so just define them as no-ops to allow the app to link.
55  */
56 extern "C" {
57 	void *
58 	spdk_realloc(void *buf, size_t size, size_t align)
59 	{
60 		assert(false);
61 
62 		return NULL;
63 	}
64 
65 	void
66 	spdk_free(void *buf)
67 	{
68 		assert(false);
69 	}
70 } /* extern "C" */
71 
72 static void usage(void);
73 
74 struct entry_key {
75 	entry_key(uint16_t _lcore, uint64_t _tsc) : lcore(_lcore), tsc(_tsc) {}
76 	uint16_t lcore;
77 	uint64_t tsc;
78 };
79 
80 class compare_entry_key
81 {
82 public:
83 	bool operator()(const entry_key &first, const entry_key &second) const
84 	{
85 		if (first.tsc == second.tsc) {
86 			return first.lcore < second.lcore;
87 		} else {
88 			return first.tsc < second.tsc;
89 		}
90 	}
91 };
92 
93 typedef std::map<entry_key, spdk_trace_entry *, compare_entry_key> entry_map;
94 
95 entry_map g_entry_map;
96 
97 struct object_stats {
98 
99 	std::map<uint64_t, uint64_t> start;
100 	std::map<uint64_t, uint64_t> index;
101 	std::map<uint64_t, uint64_t> size;
102 	std::map<uint64_t, uint64_t> tpoint_id;
103 	uint64_t counter;
104 
105 	object_stats() : start(), index(), size(), tpoint_id(), counter(0) {}
106 };
107 
108 struct argument_context {
109 	struct spdk_trace_entry		*entry;
110 	struct spdk_trace_entry_buffer	*buffer;
111 	uint16_t			lcore;
112 	size_t				offset;
113 
114 	argument_context(struct spdk_trace_entry *entry,
115 			 uint16_t lcore) : entry(entry), lcore(lcore)
116 	{
117 		buffer = (struct spdk_trace_entry_buffer *)entry;
118 
119 		/* The first argument resides within the spdk_trace_entry structure, so the initial
120 		 * offset needs to be adjusted to the start of the spdk_trace_entry.args array
121 		 */
122 		offset = offsetof(struct spdk_trace_entry, args) -
123 			 offsetof(struct spdk_trace_entry_buffer, data);
124 	}
125 };
126 
127 struct object_stats g_stats[SPDK_TRACE_MAX_OBJECT];
128 
129 static char *g_exe_name;
130 
131 static uint64_t g_tsc_rate;
132 static uint64_t g_first_tsc = 0x0;
133 
134 static float
135 get_us_from_tsc(uint64_t tsc, uint64_t tsc_rate)
136 {
137 	return ((float)tsc) * 1000 * 1000 / tsc_rate;
138 }
139 
140 static const char *
141 format_argname(const char *name)
142 {
143 	static char namebuf[16];
144 
145 	snprintf(namebuf, sizeof(namebuf), "%s: ", name);
146 	return namebuf;
147 }
148 
149 static void
150 print_ptr(const char *arg_string, uint64_t arg)
151 {
152 	printf("%-7.7s0x%-14jx ", format_argname(arg_string), arg);
153 }
154 
155 static void
156 print_uint64(const char *arg_string, uint64_t arg)
157 {
158 	/*
159 	 * Print arg as signed, since -1 is a common value especially
160 	 *  for FLUSH WRITEBUF when writev() returns -1 due to full
161 	 *  socket buffer.
162 	 */
163 	printf("%-7.7s%-16jd ", format_argname(arg_string), arg);
164 }
165 
166 static void
167 print_string(const char *arg_string, const char *arg)
168 {
169 	printf("%-7.7s%-16.16s ", format_argname(arg_string), arg);
170 }
171 
172 static void
173 print_size(uint32_t size)
174 {
175 	if (size > 0) {
176 		printf("size: %6u ", size);
177 	} else {
178 		printf("%13s", " ");
179 	}
180 }
181 
182 static void
183 print_object_id(uint8_t type, uint64_t id)
184 {
185 	printf("id:    %c%-15jd ", g_histories->flags.object[type].id_prefix, id);
186 }
187 
188 static void
189 print_float(const char *arg_string, float arg)
190 {
191 	printf("%-7s%-16.3f ", format_argname(arg_string), arg);
192 }
193 
194 static struct spdk_trace_entry_buffer *
195 get_next_buffer(struct spdk_trace_entry_buffer *buf, uint16_t lcore)
196 {
197 	struct spdk_trace_history *history;
198 
199 	history = spdk_get_per_lcore_history(g_histories, lcore);
200 	assert(history);
201 
202 	if (spdk_unlikely((void *)buf == &history->entries[history->num_entries - 1])) {
203 		return (struct spdk_trace_entry_buffer *)&history->entries[0];
204 	} else {
205 		return buf + 1;
206 	}
207 }
208 
209 static bool
210 build_arg(struct argument_context *argctx, const struct spdk_trace_argument *arg,
211 	  void *buf, size_t bufsize)
212 {
213 	struct spdk_trace_entry *entry = argctx->entry;
214 	struct spdk_trace_entry_buffer *buffer = argctx->buffer;
215 	size_t curlen, argoff;
216 
217 	argoff = 0;
218 	while (argoff < arg->size) {
219 		if (argctx->offset == sizeof(buffer->data)) {
220 			buffer = get_next_buffer(buffer, argctx->lcore);
221 			if (spdk_unlikely(buffer->tpoint_id != SPDK_TRACE_MAX_TPOINT_ID ||
222 					  buffer->tsc != entry->tsc)) {
223 				return false;
224 			}
225 
226 			argctx->offset = 0;
227 			argctx->buffer = buffer;
228 		}
229 
230 		curlen = spdk_min(sizeof(buffer->data) - argctx->offset, arg->size - argoff);
231 		if (argoff < bufsize) {
232 			memcpy((uint8_t *)buf + argoff, &buffer->data[argctx->offset],
233 			       spdk_min(curlen, bufsize - argoff));
234 		}
235 
236 		argctx->offset += curlen;
237 		argoff += curlen;
238 	}
239 
240 	return true;
241 }
242 
243 static void
244 print_arg(struct argument_context *argctx, const struct spdk_trace_argument *arg)
245 {
246 	uint64_t value = 0;
247 	char strbuf[256];
248 
249 	switch (arg->type) {
250 	case SPDK_TRACE_ARG_TYPE_PTR:
251 		if (build_arg(argctx, arg, &value, sizeof(value))) {
252 			print_ptr(arg->name, value);
253 		}
254 		break;
255 	case SPDK_TRACE_ARG_TYPE_INT:
256 		if (build_arg(argctx, arg, &value, sizeof(value))) {
257 			print_uint64(arg->name, value);
258 		}
259 		break;
260 	case SPDK_TRACE_ARG_TYPE_STR:
261 		assert((size_t)arg->size <= sizeof(strbuf));
262 		if (build_arg(argctx, arg, strbuf, sizeof(strbuf))) {
263 			print_string(arg->name, strbuf);
264 		}
265 		break;
266 	}
267 }
268 
269 static void
270 print_arg_json(struct argument_context *argctx, const struct spdk_trace_argument *arg)
271 {
272 	uint64_t value = 0;
273 	char strbuf[256];
274 
275 	switch (arg->type) {
276 	case SPDK_TRACE_ARG_TYPE_PTR:
277 	case SPDK_TRACE_ARG_TYPE_INT:
278 		if (build_arg(argctx, arg, &value, sizeof(value))) {
279 			spdk_json_write_uint64(g_json, value);
280 		} else {
281 			spdk_json_write_null(g_json);
282 		}
283 		break;
284 	case SPDK_TRACE_ARG_TYPE_STR:
285 		assert((size_t)arg->size <= sizeof(strbuf));
286 		if (build_arg(argctx, arg, strbuf, sizeof(strbuf))) {
287 			spdk_json_write_string(g_json, strbuf);
288 		} else {
289 			spdk_json_write_null(g_json);
290 		}
291 		break;
292 	}
293 }
294 
295 static void
296 print_event(struct spdk_trace_entry *e, uint64_t tsc_rate,
297 	    uint64_t tsc_offset, uint16_t lcore)
298 {
299 	struct spdk_trace_tpoint	*d;
300 	struct object_stats		*stats;
301 	float				us;
302 	size_t				i;
303 
304 	d = &g_histories->flags.tpoint[e->tpoint_id];
305 	stats = &g_stats[d->object_type];
306 	us = get_us_from_tsc(e->tsc - tsc_offset, tsc_rate);
307 
308 	printf("%2d: %10.3f ", lcore, us);
309 	if (g_print_tsc) {
310 		printf("(%9ju) ", e->tsc - tsc_offset);
311 	}
312 	if (g_histories->flags.owner[d->owner_type].id_prefix) {
313 		printf("%c%02d ", g_histories->flags.owner[d->owner_type].id_prefix, e->poller_id);
314 	} else {
315 		printf("%4s", " ");
316 	}
317 
318 	printf("%-*s ", (int)sizeof(d->name), d->name);
319 	print_size(e->size);
320 
321 	if (d->new_object) {
322 		print_object_id(d->object_type, stats->index[e->object_id]);
323 	} else if (d->object_type != OBJECT_NONE) {
324 		if (stats->start.find(e->object_id) != stats->start.end()) {
325 			us = get_us_from_tsc(e->tsc - stats->start[e->object_id],
326 					     tsc_rate);
327 			print_object_id(d->object_type, stats->index[e->object_id]);
328 			print_float("time", us);
329 		} else {
330 			printf("id:    N/A");
331 		}
332 	} else if (e->object_id != 0) {
333 		print_ptr("object", e->object_id);
334 	}
335 
336 	struct argument_context argctx(e, lcore);
337 	for (i = 0; i < d->num_args; ++i) {
338 		print_arg(&argctx, &d->args[i]);
339 	}
340 	printf("\n");
341 }
342 
343 static void
344 print_event_json(struct spdk_trace_entry *e, uint64_t tsc_rate,
345 		 uint64_t tsc_offset, uint16_t lcore)
346 {
347 	struct spdk_trace_tpoint *d;
348 	struct object_stats *stats;
349 	size_t i;
350 
351 	d = &g_histories->flags.tpoint[e->tpoint_id];
352 	stats = &g_stats[d->object_type];
353 
354 	spdk_json_write_object_begin(g_json);
355 	spdk_json_write_named_uint64(g_json, "lcore", lcore);
356 	spdk_json_write_named_uint64(g_json, "tpoint", e->tpoint_id);
357 	spdk_json_write_named_uint64(g_json, "tsc", e->tsc);
358 
359 	if (g_histories->flags.owner[d->owner_type].id_prefix) {
360 		spdk_json_write_named_string_fmt(g_json, "poller", "%c%02d",
361 						 g_histories->flags.owner[d->owner_type].id_prefix,
362 						 e->poller_id);
363 	}
364 	if (e->size != 0) {
365 		spdk_json_write_named_uint32(g_json, "size", e->size);
366 	}
367 	if (d->new_object || d->object_type != OBJECT_NONE || e->object_id != 0) {
368 		char object_type;
369 
370 		spdk_json_write_named_object_begin(g_json, "object");
371 		if (d->new_object) {
372 			object_type =  g_histories->flags.object[d->object_type].id_prefix;
373 			spdk_json_write_named_string_fmt(g_json, "id", "%c%" PRIu64, object_type,
374 							 stats->index[e->object_id]);
375 		} else if (d->object_type != OBJECT_NONE) {
376 			object_type =  g_histories->flags.object[d->object_type].id_prefix;
377 			if (stats->start.find(e->object_id) != stats->start.end()) {
378 				spdk_json_write_named_string_fmt(g_json, "id", "%c%" PRIu64,
379 								 object_type,
380 								 stats->index[e->object_id]);
381 				spdk_json_write_named_uint64(g_json, "time",
382 							     e->tsc - stats->start[e->object_id]);
383 			}
384 		}
385 		spdk_json_write_named_uint64(g_json, "value", e->object_id);
386 		spdk_json_write_object_end(g_json);
387 	}
388 	if (d->num_args > 0) {
389 		struct argument_context argctx(e, lcore);
390 
391 		spdk_json_write_named_array_begin(g_json, "args");
392 		for (i = 0; i < d->num_args; ++i) {
393 			print_arg_json(&argctx, &d->args[i]);
394 		}
395 		spdk_json_write_array_end(g_json);
396 	}
397 
398 	spdk_json_write_object_end(g_json);
399 }
400 
401 static void
402 process_event(struct spdk_trace_entry *e, uint64_t tsc_rate,
403 	      uint64_t tsc_offset, uint16_t lcore)
404 {
405 	struct spdk_trace_tpoint	*d;
406 	struct object_stats		*stats;
407 
408 	d = &g_histories->flags.tpoint[e->tpoint_id];
409 	stats = &g_stats[d->object_type];
410 
411 	if (d->new_object) {
412 		stats->index[e->object_id] = stats->counter++;
413 		stats->tpoint_id[e->object_id] = e->tpoint_id;
414 		stats->start[e->object_id] = e->tsc;
415 		stats->size[e->object_id] = e->size;
416 	}
417 
418 	if (g_json == NULL) {
419 		print_event(e, tsc_rate, tsc_offset, lcore);
420 	} else {
421 		print_event_json(e, tsc_rate, tsc_offset, lcore);
422 	}
423 }
424 
425 static int
426 populate_events(struct spdk_trace_history *history, int num_entries)
427 {
428 	int i, num_entries_filled;
429 	struct spdk_trace_entry *e;
430 	int first, last, lcore;
431 
432 	lcore = history->lcore;
433 
434 	e = history->entries;
435 
436 	num_entries_filled = num_entries;
437 	while (e[num_entries_filled - 1].tsc == 0) {
438 		num_entries_filled--;
439 	}
440 
441 	if (num_entries == num_entries_filled) {
442 		first = last = 0;
443 		for (i = 1; i < num_entries; i++) {
444 			if (e[i].tsc < e[first].tsc) {
445 				first = i;
446 			}
447 			if (e[i].tsc > e[last].tsc) {
448 				last = i;
449 			}
450 		}
451 	} else {
452 		first = 0;
453 		last = num_entries_filled - 1;
454 	}
455 
456 	/*
457 	 * We keep track of the highest first TSC out of all reactors.
458 	 *  We will ignore any events that occured before this TSC on any
459 	 *  other reactors.  This will ensure we only print data for the
460 	 *  subset of time where we have data across all reactors.
461 	 */
462 	if (e[first].tsc > g_first_tsc) {
463 		g_first_tsc = e[first].tsc;
464 	}
465 
466 	i = first;
467 	while (1) {
468 		if (e[i].tpoint_id != SPDK_TRACE_MAX_TPOINT_ID) {
469 			g_entry_map[entry_key(lcore, e[i].tsc)] = &e[i];
470 		}
471 		if (i == last) {
472 			break;
473 		}
474 		i++;
475 		if (i == num_entries_filled) {
476 			i = 0;
477 		}
478 	}
479 
480 	return (0);
481 }
482 
483 static void
484 print_tpoint_definitions(void)
485 {
486 	struct spdk_trace_tpoint *tpoint;
487 	size_t i, j;
488 
489 	/* We only care about these when printing JSON */
490 	if (!g_json) {
491 		return;
492 	}
493 
494 	spdk_json_write_named_uint64(g_json, "tsc_rate", g_tsc_rate);
495 	spdk_json_write_named_array_begin(g_json, "tpoints");
496 
497 	for (i = 0; i < SPDK_COUNTOF(g_histories->flags.tpoint); ++i) {
498 		tpoint = &g_histories->flags.tpoint[i];
499 		if (tpoint->tpoint_id == 0) {
500 			continue;
501 		}
502 
503 		spdk_json_write_object_begin(g_json);
504 		spdk_json_write_named_string(g_json, "name", tpoint->name);
505 		spdk_json_write_named_uint32(g_json, "id", tpoint->tpoint_id);
506 		spdk_json_write_named_bool(g_json, "new_object", tpoint->new_object);
507 
508 		spdk_json_write_named_array_begin(g_json, "args");
509 		for (j = 0; j < tpoint->num_args; ++j) {
510 			spdk_json_write_object_begin(g_json);
511 			spdk_json_write_named_string(g_json, "name", tpoint->args[j].name);
512 			spdk_json_write_named_uint32(g_json, "type", tpoint->args[j].type);
513 			spdk_json_write_named_uint32(g_json, "size", tpoint->args[j].size);
514 			spdk_json_write_object_end(g_json);
515 		}
516 		spdk_json_write_array_end(g_json);
517 		spdk_json_write_object_end(g_json);
518 	}
519 
520 	spdk_json_write_array_end(g_json);
521 }
522 
523 static int
524 print_json(void *cb_ctx, const void *data, size_t size)
525 {
526 	ssize_t rc;
527 
528 	while (size > 0) {
529 		rc = write(STDOUT_FILENO, data, size);
530 		if (rc < 0) {
531 			fprintf(stderr, "%s: %s\n", g_exe_name, spdk_strerror(errno));
532 			abort();
533 		}
534 
535 		size -= rc;
536 	}
537 
538 	return 0;
539 }
540 
541 static void usage(void)
542 {
543 	fprintf(stderr, "usage:\n");
544 	fprintf(stderr, "   %s <option> <lcore#>\n", g_exe_name);
545 	fprintf(stderr, "                 '-c' to display single lcore history\n");
546 	fprintf(stderr, "                 '-t' to display TSC offset for each event\n");
547 	fprintf(stderr, "                 '-s' to specify spdk_trace shm name for a\n");
548 	fprintf(stderr, "                      currently running process\n");
549 	fprintf(stderr, "                 '-i' to specify the shared memory ID\n");
550 	fprintf(stderr, "                 '-p' to specify the trace PID\n");
551 	fprintf(stderr, "                      (If -s is specified, then one of\n");
552 	fprintf(stderr, "                       -i or -p must be specified)\n");
553 	fprintf(stderr, "                 '-f' to specify a tracepoint file name\n");
554 	fprintf(stderr, "                      (-s and -f are mutually exclusive)\n");
555 	fprintf(stderr, "                 '-j' to use JSON to format the output\n");
556 }
557 
558 int main(int argc, char **argv)
559 {
560 	void			*history_ptr;
561 	struct spdk_trace_history *history;
562 	int			fd, i, rc;
563 	int			lcore = SPDK_TRACE_MAX_LCORE;
564 	uint64_t		tsc_offset;
565 	const char		*app_name = NULL;
566 	const char		*file_name = NULL;
567 	int			op;
568 	char			shm_name[64];
569 	int			shm_id = -1, shm_pid = -1;
570 	uint64_t		trace_histories_size;
571 	struct stat		_stat;
572 	bool			json = false;
573 
574 	g_exe_name = argv[0];
575 	while ((op = getopt(argc, argv, "c:f:i:jp:s:t")) != -1) {
576 		switch (op) {
577 		case 'c':
578 			lcore = atoi(optarg);
579 			if (lcore > SPDK_TRACE_MAX_LCORE) {
580 				fprintf(stderr, "Selected lcore: %d "
581 					"exceeds maximum %d\n", lcore,
582 					SPDK_TRACE_MAX_LCORE);
583 				exit(1);
584 			}
585 			break;
586 		case 'i':
587 			shm_id = atoi(optarg);
588 			break;
589 		case 'p':
590 			shm_pid = atoi(optarg);
591 			break;
592 		case 's':
593 			app_name = optarg;
594 			break;
595 		case 'f':
596 			file_name = optarg;
597 			break;
598 		case 't':
599 			g_print_tsc = true;
600 			break;
601 		case 'j':
602 			json = true;
603 			break;
604 		default:
605 			usage();
606 			exit(1);
607 		}
608 	}
609 
610 	if (file_name != NULL && app_name != NULL) {
611 		fprintf(stderr, "-f and -s are mutually exclusive\n");
612 		usage();
613 		exit(1);
614 	}
615 
616 	if (file_name == NULL && app_name == NULL) {
617 		fprintf(stderr, "One of -f and -s must be specified\n");
618 		usage();
619 		exit(1);
620 	}
621 
622 	if (json) {
623 		g_json = spdk_json_write_begin(print_json, NULL, 0);
624 		if (g_json == NULL) {
625 			fprintf(stderr, "Failed to allocate JSON write context\n");
626 			exit(1);
627 		}
628 	}
629 
630 	if (file_name) {
631 		fd = open(file_name, O_RDONLY);
632 	} else {
633 		if (shm_id >= 0) {
634 			snprintf(shm_name, sizeof(shm_name), "/%s_trace.%d", app_name, shm_id);
635 		} else {
636 			snprintf(shm_name, sizeof(shm_name), "/%s_trace.pid%d", app_name, shm_pid);
637 		}
638 		fd = shm_open(shm_name, O_RDONLY, 0600);
639 		file_name = shm_name;
640 	}
641 	if (fd < 0) {
642 		fprintf(stderr, "Could not open %s.\n", file_name);
643 		usage();
644 		exit(-1);
645 	}
646 
647 	rc = fstat(fd, &_stat);
648 	if (rc < 0) {
649 		fprintf(stderr, "Could not get size of %s.\n", file_name);
650 		usage();
651 		exit(-1);
652 	}
653 	if ((size_t)_stat.st_size < sizeof(*g_histories)) {
654 		fprintf(stderr, "%s is not a valid trace file\n", file_name);
655 		usage();
656 		exit(-1);
657 	}
658 
659 	/* Map the header of trace file */
660 	history_ptr = mmap(NULL, sizeof(*g_histories), PROT_READ, MAP_SHARED, fd, 0);
661 	if (history_ptr == MAP_FAILED) {
662 		fprintf(stderr, "Could not mmap %s.\n", file_name);
663 		usage();
664 		exit(-1);
665 	}
666 
667 	g_histories = (struct spdk_trace_histories *)history_ptr;
668 
669 	g_tsc_rate = g_histories->flags.tsc_rate;
670 	if (g_tsc_rate == 0) {
671 		fprintf(stderr, "Invalid tsc_rate %ju\n", g_tsc_rate);
672 		usage();
673 		exit(-1);
674 	}
675 
676 	if (!g_json) {
677 		printf("TSC Rate: %ju\n", g_tsc_rate);
678 	}
679 
680 	/* Remap the entire trace file */
681 	trace_histories_size = spdk_get_trace_histories_size(g_histories);
682 	munmap(history_ptr, sizeof(*g_histories));
683 	if ((size_t)_stat.st_size < trace_histories_size) {
684 		fprintf(stderr, "%s is not a valid trace file\n", file_name);
685 		usage();
686 		exit(-1);
687 	}
688 	history_ptr = mmap(NULL, trace_histories_size, PROT_READ, MAP_SHARED, fd, 0);
689 	if (history_ptr == MAP_FAILED) {
690 		fprintf(stderr, "Could not mmap %s.\n", file_name);
691 		usage();
692 		exit(-1);
693 	}
694 
695 	g_histories = (struct spdk_trace_histories *)history_ptr;
696 
697 	if (lcore == SPDK_TRACE_MAX_LCORE) {
698 		for (i = 0; i < SPDK_TRACE_MAX_LCORE; i++) {
699 			history = spdk_get_per_lcore_history(g_histories, i);
700 			if (history->num_entries == 0 || history->entries[0].tsc == 0) {
701 				continue;
702 			}
703 
704 			if (!g_json && history->num_entries) {
705 				printf("Trace Size of lcore (%d): %ju\n", i, history->num_entries);
706 			}
707 
708 			populate_events(history, history->num_entries);
709 		}
710 	} else {
711 		history = spdk_get_per_lcore_history(g_histories, lcore);
712 		if (history->num_entries > 0 && history->entries[0].tsc != 0) {
713 			if (!g_json && history->num_entries) {
714 				printf("Trace Size of lcore (%d): %ju\n", lcore, history->num_entries);
715 			}
716 
717 			populate_events(history, history->num_entries);
718 		}
719 	}
720 
721 	if (g_json != NULL) {
722 		spdk_json_write_object_begin(g_json);
723 		print_tpoint_definitions();
724 		spdk_json_write_named_array_begin(g_json, "entries");
725 	}
726 
727 	tsc_offset = g_first_tsc;
728 	for (entry_map::iterator it = g_entry_map.begin(); it != g_entry_map.end(); it++) {
729 		if (it->first.tsc < g_first_tsc) {
730 			continue;
731 		}
732 		process_event(it->second, g_tsc_rate, tsc_offset, it->first.lcore);
733 	}
734 
735 	if (g_json != NULL) {
736 		spdk_json_write_array_end(g_json);
737 		spdk_json_write_object_end(g_json);
738 		spdk_json_write_end(g_json);
739 	}
740 
741 	munmap(history_ptr, trace_histories_size);
742 	close(fd);
743 
744 	return (0);
745 }
746