2  * Copyright (c) 2014-2019, Intel Corporation
32 #include "intel-pt.h"
50 		return -pte_internal;  in pt_time_query_tsc()
52 	*tsc = time->tsc;  in pt_time_query_tsc()
55 		*lost_mtc = time->lost_mtc;  in pt_time_query_tsc()
57 		*lost_cyc = time->lost_cyc;  in pt_time_query_tsc()
59 	if (!time->have_tsc)  in pt_time_query_tsc()
60 		return -pte_no_time;  in pt_time_query_tsc()
68 		return -pte_internal;  in pt_time_query_cbr()
70 	if (!time->have_cbr)  in pt_time_query_cbr()
71 		return -pte_no_cbr;  in pt_time_query_cbr()
73 	*cbr = time->cbr;  in pt_time_query_cbr()
80  * We adjust a single wrap-around but fail if the distance is bigger than that.
85 			     uint32_t last_ctc, const struct pt_config *config)  in pt_time_ctc_delta()  argument
87 	if (!config || !ctc_delta)  in pt_time_ctc_delta()
88 		return -pte_internal;  in pt_time_ctc_delta()
90 	/* Correct a single wrap-around.  If we lost enough MTCs to wrap  in pt_time_ctc_delta()
94 		ctc += 1u << (config->mtc_freq + pt_pl_mtc_bit_size);  in pt_time_ctc_delta()
100 			return -pte_bad_packet;  in pt_time_ctc_delta()
103 	*ctc_delta = ctc - last_ctc;  in pt_time_ctc_delta()
112 			  const struct pt_config *config)  in pt_time_ctc_fc()  argument
116 	if (!fc || !config)  in pt_time_ctc_fc()
117 		return -pte_internal;  in pt_time_ctc_fc()
119 	eax = config->cpuid_0x15_eax;  in pt_time_ctc_fc()
120 	ebx = config->cpuid_0x15_ebx;  in pt_time_ctc_fc()
124 		return -pte_bad_config;  in pt_time_ctc_fc()
132 		       const struct pt_config *config)  in pt_time_update_tsc()  argument
134 	(void) config;  in pt_time_update_tsc()
137 		return -pte_internal;  in pt_time_update_tsc()
139 	time->have_tsc = 1;  in pt_time_update_tsc()
140 	time->have_tma = 0;  in pt_time_update_tsc()
141 	time->have_mtc = 0;  in pt_time_update_tsc()
142 	time->tsc = time->base = packet->tsc;  in pt_time_update_tsc()
143 	time->ctc = 0;  in pt_time_update_tsc()
144 	time->fc = 0ull;  in pt_time_update_tsc()
147 	time->lost_mtc = 0;  in pt_time_update_tsc()
148 	time->lost_cyc = 0;  in pt_time_update_tsc()
155 		       const struct pt_config *config)  in pt_time_update_cbr()  argument
159 	(void) config;  in pt_time_update_cbr()
162 		return -pte_internal;  in pt_time_update_cbr()
164 	cbr = packet->ratio;  in pt_time_update_cbr()
166 		return -pte_bad_packet;  in pt_time_update_cbr()
168 	time->have_cbr = 1;  in pt_time_update_cbr()
169 	time->cbr = cbr;  in pt_time_update_cbr()
176 		       const struct pt_config *config)  in pt_time_update_tma()  argument
181 	if (!time || !packet || !config)  in pt_time_update_tma()
182 		return -pte_internal;  in pt_time_update_tma()
185 	if (!time->have_tsc)  in pt_time_update_tma()
186 		return -pte_bad_context;  in pt_time_update_tma()
189 	if (time->have_tma)  in pt_time_update_tma()
190 		return -pte_bad_context;  in pt_time_update_tma()
192 	/* We're ignoring MTC between TSC and TMA. */  in pt_time_update_tma()
193 	if (time->have_mtc)  in pt_time_update_tma()
194 		return -pte_internal;  in pt_time_update_tma()
196 	ctc = packet->ctc;  in pt_time_update_tma()
197 	fc = packet->fc;  in pt_time_update_tma()
199 	mtc_freq = config->mtc_freq;  in pt_time_update_tma()
202 	/* A mask for the relevant CTC bits ignoring high-order bits that are  in pt_time_update_tma()
205 	ctc_mask = (1u << mtc_hi) - 1u;  in pt_time_update_tma()
207 	time->have_tma = 1;  in pt_time_update_tma()
208 	time->base -= fc;  in pt_time_update_tma()
209 	time->fc += fc;  in pt_time_update_tma()
226 		time->have_mtc = 1;  in pt_time_update_tma()
229 	time->ctc = time->ctc_cyc = ctc & ctc_mask;  in pt_time_update_tma()
236 		       const struct pt_config *config)  in pt_time_update_mtc()  argument
243 	if (!time || !packet || !config)  in pt_time_update_mtc()
244 		return -pte_internal;  in pt_time_update_mtc()
246 	have_tsc = time->have_tsc;  in pt_time_update_mtc()
247 	have_tma = time->have_tma;  in pt_time_update_mtc()
248 	have_mtc = time->have_mtc;  in pt_time_update_mtc()
252 	 * Later MTCs will ensure that no time is lost - provided TMA provides  in pt_time_update_mtc()
259 	base = time->base;  in pt_time_update_mtc()
260 	last_ctc = time->ctc;  in pt_time_update_mtc()
261 	mtc_freq = config->mtc_freq;  in pt_time_update_mtc()
263 	ctc = (uint32_t) packet->ctc << mtc_freq;  in pt_time_update_mtc()
266 	if (time->fc || time->lost_cyc || !have_mtc)  in pt_time_update_mtc()
267 		time->ctc_cyc = ctc;  in pt_time_update_mtc()
270 	time->have_mtc = 1;  in pt_time_update_mtc()
271 	time->fc = 0ull;  in pt_time_update_mtc()
272 	time->ctc = ctc;  in pt_time_update_mtc()
275 	time->lost_cyc = 0;  in pt_time_update_mtc()
294 		/* The TMA's CTC value didn't provide enough bits - otherwise,  in pt_time_update_mtc()
298 			return -pte_internal;  in pt_time_update_mtc()
306 		/* We estimate the high-order CTC bits that are not provided by  in pt_time_update_mtc()
318 			ctc_hi -= 1u << pt_pl_tma_ctc_bit_size;  in pt_time_update_mtc()
325 	errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);  in pt_time_update_mtc()
327 		time->lost_mtc += 1;  in pt_time_update_mtc()
331 	errcode = pt_time_ctc_fc(&tsc, ctc_delta, config);  in pt_time_update_mtc()
336 	time->tsc = time->base = base;  in pt_time_update_mtc()
343  * CYC packets measure the Fast Counter since the last CYC(-eligible) packet.
360 			      const struct pt_config *config, uint64_t fcr)  in pt_time_adjust_cyc()  argument
366 	if (!time || !config || !fcr)  in pt_time_adjust_cyc()
367 		return -pte_internal;  in pt_time_adjust_cyc()
369 	last_ctc = time->ctc_cyc;  in pt_time_adjust_cyc()
370 	ctc = time->ctc;  in pt_time_adjust_cyc()
384 	errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);  in pt_time_adjust_cyc()
388 	errcode = pt_time_ctc_fc(&fc, ctc_delta, config);  in pt_time_adjust_cyc()
404 	*cyc = total_cyc - old_cyc;  in pt_time_adjust_cyc()
410 		       const struct pt_config *config, uint64_t fcr)  in pt_time_update_cyc()  argument
414 	if (!time || !packet || !config)  in pt_time_update_cyc()
415 		return -pte_internal;  in pt_time_update_cyc()
418 		time->lost_cyc += 1;  in pt_time_update_cyc()
422 	cyc = packet->value;  in pt_time_update_cyc()
423 	fc = time->fc;  in pt_time_update_cyc()
427 		errcode = pt_time_adjust_cyc(&cyc, time, config, fcr);  in pt_time_update_cyc()
434 	time->fc = fc;  in pt_time_update_cyc()
435 	time->tsc = time->base + fc;  in pt_time_update_cyc()
447 	tcal->min_fcr = UINT64_MAX;  in pt_tcal_init()
455 	return (tcal->min_fcr <= tcal->max_fcr);  in pt_tcal_have_fcr()
461 		return -pte_internal;  in pt_tcal_fcr()
464 		return -pte_no_time;  in pt_tcal_fcr()
466 	*fcr = tcal->fcr;  in pt_tcal_fcr()
474 		return -pte_internal;  in pt_tcal_set_fcr()
476 	tcal->fcr = fcr;  in pt_tcal_set_fcr()
478 	if (fcr < tcal->min_fcr)  in pt_tcal_set_fcr()
479 		tcal->min_fcr = fcr;  in pt_tcal_set_fcr()
481 	if (fcr > tcal->max_fcr)  in pt_tcal_set_fcr()
482 		tcal->max_fcr = fcr;  in pt_tcal_set_fcr()
488 		       const struct pt_config *config)  in pt_tcal_update_psb()  argument
490 	if (!tcal || !config)  in pt_tcal_update_psb()
491 		return -pte_internal;  in pt_tcal_update_psb()
493 	if (config->errata.skl168)  in pt_tcal_update_psb()
494 		tcal->check_skl168 = 1;  in pt_tcal_update_psb()
501 		      const struct pt_config *config)  in pt_tcal_update_tsc()  argument
503 	(void) config;  in pt_tcal_update_tsc()
506 		return -pte_internal;  in pt_tcal_update_tsc()
512 	tcal->tsc = packet->tsc;  in pt_tcal_update_tsc()
513 	tcal->cyc_tsc = 0ull;  in pt_tcal_update_tsc()
520 		      const struct pt_config *config)  in pt_tcal_header_tsc()  argument
524 	(void) config;  in pt_tcal_header_tsc()
527 		return -pte_internal;  in pt_tcal_header_tsc()
529 	last_tsc = tcal->tsc;  in pt_tcal_header_tsc()
530 	cyc = tcal->cyc_tsc;  in pt_tcal_header_tsc()
532 	tsc = packet->tsc;  in pt_tcal_header_tsc()
534 	tcal->tsc = tsc;  in pt_tcal_header_tsc()
535 	tcal->cyc_tsc = 0ull;  in pt_tcal_header_tsc()
541 	if (tcal->have_mtc)  in pt_tcal_header_tsc()
544 	/* Correct a single wrap-around. */  in pt_tcal_header_tsc()
549 			return -pte_bad_packet;  in pt_tcal_header_tsc()
552 	tsc_delta = tsc - last_tsc;  in pt_tcal_header_tsc()
556 	 * Since we're only collecting the CYCs between two TSC, we shouldn't  in pt_tcal_header_tsc()
560 		return -pte_internal;  in pt_tcal_header_tsc()
569 		      const struct pt_config *config)  in pt_tcal_update_cbr()  argument
576 	return pt_tcal_header_cbr(tcal, packet, config);  in pt_tcal_update_cbr()
581 		      const struct pt_config *config)  in pt_tcal_header_cbr()  argument
585 	if (!tcal || !packet || !config)  in pt_tcal_header_cbr()
586 		return -pte_internal;  in pt_tcal_header_cbr()
588 	p1 = config->nom_freq;  in pt_tcal_header_cbr()
593 	cbr = packet->ratio;  in pt_tcal_header_cbr()
595 		return -pte_bad_packet;  in pt_tcal_header_cbr()
604 		      const struct pt_config *config)  in pt_tcal_update_tma()  argument
608 	(void) config;  in pt_tcal_update_tma()
616 		      const struct pt_config *config)  in pt_tcal_update_mtc()  argument
622 	if (!tcal || !packet || !config)  in pt_tcal_update_mtc()
623 		return -pte_internal;  in pt_tcal_update_mtc()
625 	last_ctc = tcal->ctc;  in pt_tcal_update_mtc()
626 	have_mtc = tcal->have_mtc;  in pt_tcal_update_mtc()
627 	cyc = tcal->cyc_mtc;  in pt_tcal_update_mtc()
628 	check_skl168 = tcal->check_skl168;  in pt_tcal_update_mtc()
631 	tcal->check_skl168 = 0;  in pt_tcal_update_mtc()
633 	ctc = (uint32_t) packet->ctc << config->mtc_freq;  in pt_tcal_update_mtc()
637 		tcal->cyc_mtc = 0ull;  in pt_tcal_update_mtc()
638 		tcal->ctc = ctc;  in pt_tcal_update_mtc()
639 		tcal->have_mtc = 1;  in pt_tcal_update_mtc()
651 	tcal->have_mtc = 1;  in pt_tcal_update_mtc()
652 	tcal->cyc_mtc = 0ull;  in pt_tcal_update_mtc()
653 	tcal->ctc = ctc;  in pt_tcal_update_mtc()
656 	tcal->lost_mtc += 1;  in pt_tcal_update_mtc()
658 	errcode = pt_time_ctc_delta(&ctc_delta, ctc, last_ctc, config);  in pt_tcal_update_mtc()
662 	errcode = pt_time_ctc_fc(&fc, ctc_delta, config);  in pt_tcal_update_mtc()
668 	 * Since we're only collecting the CYCs between two MTC, we shouldn't  in pt_tcal_update_mtc()
672 		return -pte_internal;  in pt_tcal_update_mtc()
686 		if ((1u << config->mtc_freq) < ctc_delta)  in pt_tcal_update_mtc()
702 		if ((config->mtc_freq < 10) && pt_tcal_have_fcr(tcal)) {  in pt_tcal_update_mtc()
708 			dfc = (tcal->fcr * (cyc + 0xf00)) >> pt_tcal_fcr_shr;  in pt_tcal_update_mtc()
724 	tcal->lost_mtc = 0;  in pt_tcal_update_mtc()
731 		      const struct pt_config *config)  in pt_tcal_update_cyc()  argument
735 	(void) config;  in pt_tcal_update_cyc()
738 		return -pte_internal;  in pt_tcal_update_cyc()
740 	cyc = packet->value;  in pt_tcal_update_cyc()
741 	tcal->cyc_mtc += cyc;  in pt_tcal_update_cyc()
742 	tcal->cyc_tsc += cyc;  in pt_tcal_update_cyc()
748 		       const struct pt_config *config)  in pt_tcal_update_ovf()  argument
750 	if (!tcal || !config)  in pt_tcal_update_ovf()
751 		return -pte_internal;  in pt_tcal_update_ovf()
753 	tcal->tsc = 0ull;  in pt_tcal_update_ovf()
754 	tcal->cyc_tsc = 0ull;  in pt_tcal_update_ovf()
755 	tcal->cyc_mtc = 0ull;  in pt_tcal_update_ovf()
756 	tcal->ctc = 0;  in pt_tcal_update_ovf()
757 	tcal->have_mtc = 0;  in pt_tcal_update_ovf()