xref: /netbsd-src/sys/dev/tprof/tprof_x86_intel.c (revision 7d62b00eb9ad855ffcd7da46b41e23feb5476fac)
1 /*	$NetBSD: tprof_x86_intel.c,v 1.5 2022/12/01 00:32:52 ryo Exp $	*/
2 
3 /*
4  * Copyright (c) 2018 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Maxime Villard.
9  *
10  * Redistribution and use in source and binary forms, with or without
11  * modification, are permitted provided that the following conditions
12  * are met:
13  * 1. Redistributions of source code must retain the above copyright
14  *    notice, this list of conditions and the following disclaimer.
15  * 2. Redistributions in binary form must reproduce the above copyright
16  *    notice, this list of conditions and the following disclaimer in the
17  *    documentation and/or other materials provided with the distribution.
18  *
19  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29  * POSSIBILITY OF SUCH DAMAGE.
30  */
31 
32 /*
33  * Copyright (c)2008,2009 YAMAMOTO Takashi,
34  * All rights reserved.
35  *
36  * Redistribution and use in source and binary forms, with or without
37  * modification, are permitted provided that the following conditions
38  * are met:
39  * 1. Redistributions of source code must retain the above copyright
40  *    notice, this list of conditions and the following disclaimer.
41  * 2. Redistributions in binary form must reproduce the above copyright
42  *    notice, this list of conditions and the following disclaimer in the
43  *    documentation and/or other materials provided with the distribution.
44  *
45  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
46  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
47  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
48  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
49  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
50  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
51  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
52  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
53  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
54  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
55  * SUCH DAMAGE.
56  */
57 
58 #include <sys/cdefs.h>
59 __KERNEL_RCSID(0, "$NetBSD: tprof_x86_intel.c,v 1.5 2022/12/01 00:32:52 ryo Exp $");
60 
61 #include <sys/param.h>
62 #include <sys/systm.h>
63 #include <sys/kernel.h>
64 #include <sys/module.h>
65 
66 #include <sys/cpu.h>
67 #include <sys/percpu.h>
68 #include <sys/xcall.h>
69 
70 #include <dev/tprof/tprof.h>
71 
72 #include <uvm/uvm.h>		/* VM_MIN_KERNEL_ADDRESS */
73 
74 #include <x86/nmi.h>
75 
76 #include <machine/cpufunc.h>
77 #include <machine/cputypes.h>	/* CPUVENDOR_* */
78 #include <machine/cpuvar.h>	/* cpu_vendor */
79 #include <machine/i82489reg.h>
80 #include <machine/i82489var.h>
81 
82 #define	NCTRS	4	/* XXX */
83 static u_int counter_bitwidth;
84 
85 #define	PERFEVTSEL(i)		(MSR_EVNTSEL0 + (i))
86 #define	PERFCTR(i)		(MSR_PERFCTR0 + (i))
87 
88 #define	PERFEVTSEL_EVENT_SELECT	__BITS(0, 7)
89 #define	PERFEVTSEL_UNIT_MASK	__BITS(8, 15)
90 #define	PERFEVTSEL_USR		__BIT(16)
91 #define	PERFEVTSEL_OS		__BIT(17)
92 #define	PERFEVTSEL_E		__BIT(18)
93 #define	PERFEVTSEL_PC		__BIT(19)
94 #define	PERFEVTSEL_INT		__BIT(20)
95 #define	PERFEVTSEL_EN		__BIT(22)
96 #define	PERFEVTSEL_INV		__BIT(23)
97 #define	PERFEVTSEL_COUNTER_MASK	__BITS(24, 31)
98 
99 static uint32_t intel_lapic_saved[MAXCPUS];
100 static nmi_handler_t *intel_nmi_handle;
101 
102 static uint32_t
103 tprof_intel_ncounters(void)
104 {
105 	return NCTRS;
106 }
107 
108 static u_int
109 tprof_intel_counter_bitwidth(u_int counter)
110 {
111 	return counter_bitwidth;
112 }
113 
114 static inline void
115 tprof_intel_counter_write(u_int counter, uint64_t val)
116 {
117 	wrmsr(PERFCTR(counter), val);
118 }
119 
120 static inline uint64_t
121 tprof_intel_counter_read(u_int counter)
122 {
123 	return rdmsr(PERFCTR(counter));
124 }
125 
126 static void
127 tprof_intel_configure_event(u_int counter, const tprof_param_t *param)
128 {
129 	uint64_t evtval;
130 
131 	evtval =
132 	    __SHIFTIN(param->p_event, PERFEVTSEL_EVENT_SELECT) |
133 	    __SHIFTIN(param->p_unit, PERFEVTSEL_UNIT_MASK) |
134 	    ((param->p_flags & TPROF_PARAM_USER) ? PERFEVTSEL_USR : 0) |
135 	    ((param->p_flags & TPROF_PARAM_KERN) ? PERFEVTSEL_OS : 0) |
136 	    PERFEVTSEL_INT;
137 	wrmsr(PERFEVTSEL(counter), evtval);
138 
139 	/* reset the counter */
140 	tprof_intel_counter_write(counter, param->p_value);
141 }
142 
143 static void
144 tprof_intel_start(tprof_countermask_t runmask)
145 {
146 	int bit;
147 
148 	while ((bit = ffs(runmask)) != 0) {
149 		bit--;
150 		CLR(runmask, __BIT(bit));
151 		wrmsr(PERFEVTSEL(bit), rdmsr(PERFEVTSEL(bit)) | PERFEVTSEL_EN);
152 	}
153 }
154 
155 static void
156 tprof_intel_stop(tprof_countermask_t stopmask)
157 {
158 	int bit;
159 
160 	while ((bit = ffs(stopmask)) != 0) {
161 		bit--;
162 		CLR(stopmask, __BIT(bit));
163 		wrmsr(PERFEVTSEL(bit), rdmsr(PERFEVTSEL(bit)) & ~PERFEVTSEL_EN);
164 	}
165 }
166 
167 static int
168 tprof_intel_nmi(const struct trapframe *tf, void *arg)
169 {
170 	tprof_backend_softc_t *sc = arg;
171 	tprof_frame_info_t tfi;
172 	uint32_t pcint;
173 	int bit;
174 
175 	uint64_t *counters_offset =
176 	    percpu_getptr_remote(sc->sc_ctr_offset_percpu, curcpu());
177 	tprof_countermask_t mask = sc->sc_ctr_ovf_mask;
178 	while ((bit = ffs(mask)) != 0) {
179 		bit--;
180 		CLR(mask, __BIT(bit));
181 
182 		/* If the highest bit is non zero, then it's not for us. */
183 		uint64_t ctr = tprof_intel_counter_read(bit);
184 		if ((ctr & __BIT(counter_bitwidth - 1)) != 0)
185 			continue;	/* not overflowed */
186 
187 		if (ISSET(sc->sc_ctr_prof_mask, __BIT(bit))) {
188 			/* account for the counter, and reset */
189 			tprof_intel_counter_write(bit,
190 			    sc->sc_count[bit].ctr_counter_reset_val);
191 			counters_offset[bit] +=
192 			    sc->sc_count[bit].ctr_counter_val + ctr;
193 
194 			/* record a sample */
195 #if defined(__x86_64__)
196 			tfi.tfi_pc = tf->tf_rip;
197 #else
198 			tfi.tfi_pc = tf->tf_eip;
199 #endif
200 			tfi.tfi_counter = bit;
201 			tfi.tfi_inkernel = tfi.tfi_pc >= VM_MIN_KERNEL_ADDRESS;
202 			tprof_sample(NULL, &tfi);
203 		} else {
204 			/* not profiled, but require to consider overflow */
205 			counters_offset[bit] += __BIT(counter_bitwidth);
206 		}
207 	}
208 
209 	/* unmask PMI */
210 	pcint = lapic_readreg(LAPIC_LVT_PCINT);
211 	KASSERT((pcint & LAPIC_LVT_MASKED) != 0);
212 	lapic_writereg(LAPIC_LVT_PCINT, pcint & ~LAPIC_LVT_MASKED);
213 
214 	return 1;
215 }
216 
217 static uint64_t
218 tprof_intel_counter_estimate_freq(u_int counter)
219 {
220 	return curcpu()->ci_data.cpu_cc_freq;
221 }
222 
223 static uint32_t
224 tprof_intel_ident(void)
225 {
226 	uint32_t descs[4];
227 
228 	if (cpu_vendor != CPUVENDOR_INTEL) {
229 		return TPROF_IDENT_NONE;
230 	}
231 
232 	if (cpuid_level < 0x0A) {
233 		return TPROF_IDENT_NONE;
234 	}
235 	x86_cpuid(0x0A, descs);
236 	if ((descs[0] & CPUID_PERF_VERSION) == 0) {
237 		return TPROF_IDENT_NONE;
238 	}
239 	if ((descs[0] & CPUID_PERF_NGPPC) == 0) {
240 		return TPROF_IDENT_NONE;
241 	}
242 
243 	counter_bitwidth = __SHIFTOUT(descs[0], CPUID_PERF_NBWGPPC);
244 
245 	return TPROF_IDENT_INTEL_GENERIC;
246 }
247 
248 static void
249 tprof_intel_establish_cpu(void *arg1, void *arg2)
250 {
251 	struct cpu_info * const ci = curcpu();
252 
253 	intel_lapic_saved[cpu_index(ci)] = lapic_readreg(LAPIC_LVT_PCINT);
254 	lapic_writereg(LAPIC_LVT_PCINT, LAPIC_DLMODE_NMI);
255 }
256 
257 static void
258 tprof_intel_disestablish_cpu(void *arg1, void *arg2)
259 {
260 	struct cpu_info * const ci = curcpu();
261 
262 	lapic_writereg(LAPIC_LVT_PCINT, intel_lapic_saved[cpu_index(ci)]);
263 }
264 
265 static int
266 tprof_intel_establish(tprof_backend_softc_t *sc)
267 {
268 	uint64_t xc;
269 
270 	if (tprof_intel_ident() == TPROF_IDENT_NONE) {
271 		return ENOTSUP;
272 	}
273 
274 	KASSERT(intel_nmi_handle == NULL);
275 	intel_nmi_handle = nmi_establish(tprof_intel_nmi, sc);
276 
277 	xc = xc_broadcast(0, tprof_intel_establish_cpu, sc, NULL);
278 	xc_wait(xc);
279 
280 	return 0;
281 }
282 
283 static void
284 tprof_intel_disestablish(tprof_backend_softc_t *sc)
285 {
286 	uint64_t xc;
287 
288 	xc = xc_broadcast(0, tprof_intel_disestablish_cpu, sc, NULL);
289 	xc_wait(xc);
290 
291 	KASSERT(intel_nmi_handle != NULL);
292 	nmi_disestablish(intel_nmi_handle);
293 	intel_nmi_handle = NULL;
294 }
295 
296 const tprof_backend_ops_t tprof_intel_ops = {
297 	.tbo_ident = tprof_intel_ident,
298 	.tbo_ncounters = tprof_intel_ncounters,
299 	.tbo_counter_bitwidth = tprof_intel_counter_bitwidth,
300 	.tbo_counter_read = tprof_intel_counter_read,
301 	.tbo_counter_estimate_freq = tprof_intel_counter_estimate_freq,
302 	.tbo_valid_event = NULL,
303 	.tbo_configure_event = tprof_intel_configure_event,
304 	.tbo_start = tprof_intel_start,
305 	.tbo_stop = tprof_intel_stop,
306 	.tbo_establish = tprof_intel_establish,
307 	.tbo_disestablish = tprof_intel_disestablish,
308 };
309