1 /* $NetBSD: tprof_x86_amd.c,v 1.8 2023/04/11 10:07:12 msaitoh Exp $ */
2
3 /*
4 * Copyright (c) 2018 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Maxime Villard.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 /*
33 * Copyright (c)2008,2009 YAMAMOTO Takashi,
34 * All rights reserved.
35 *
36 * Redistribution and use in source and binary forms, with or without
37 * modification, are permitted provided that the following conditions
38 * are met:
39 * 1. Redistributions of source code must retain the above copyright
40 * notice, this list of conditions and the following disclaimer.
41 * 2. Redistributions in binary form must reproduce the above copyright
42 * notice, this list of conditions and the following disclaimer in the
43 * documentation and/or other materials provided with the distribution.
44 *
45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
55 * SUCH DAMAGE.
56 */
57
58 #include <sys/cdefs.h>
59 __KERNEL_RCSID(0, "$NetBSD: tprof_x86_amd.c,v 1.8 2023/04/11 10:07:12 msaitoh Exp $");
60
61 #include <sys/param.h>
62 #include <sys/systm.h>
63 #include <sys/kernel.h>
64 #include <sys/module.h>
65
66 #include <sys/cpu.h>
67 #include <sys/percpu.h>
68 #include <sys/xcall.h>
69
70 #include <dev/tprof/tprof.h>
71
72 #include <uvm/uvm.h> /* VM_MIN_KERNEL_ADDRESS */
73
74 #include <x86/nmi.h>
75
76 #include <machine/cpufunc.h>
77 #include <machine/cputypes.h> /* CPUVENDOR_* */
78 #include <machine/cpuvar.h> /* cpu_vendor */
79 #include <machine/i82489reg.h>
80 #include <machine/i82489var.h>
81
82 #define NCTRS 4
83 #define COUNTER_BITWIDTH 48
84
85 #define PERFEVTSEL(i) (0xc0010000 + (i))
86 #define PERFCTR(i) (0xc0010004 + (i))
87
88 #define PESR_EVENT_MASK_LO __BITS(0, 7)
89 #define PESR_UNIT_MASK __BITS(8, 15)
90 #define PESR_USR __BIT(16)
91 #define PESR_OS __BIT(17)
92 #define PESR_E __BIT(18)
93 #define PESR_PC __BIT(19)
94 #define PESR_INT __BIT(20)
95 /* bit 21 reserved */
96 #define PESR_EN __BIT(22)
97 #define PESR_INV __BIT(23)
98 #define PESR_COUNTER_MASK __BITS(24, 31)
99 #define PESR_EVENT_MASK_HI __BITS(32, 35)
100 /* bit 36-39 reserved */
101 #define PESR_GO __BIT(40)
102 #define PESR_HO __BIT(41)
103 /* bit 42-63 reserved */
104
105 /*
106 * Documents:
107 * http://support.amd.com/TechDocs/32559.pdf
108 * http://developer.amd.com/wordpress/media/2012/10/Basic_Performance_Measurements.pdf
109 */
110
111 static uint32_t amd_lapic_saved[MAXCPUS];
112 static nmi_handler_t *amd_nmi_handle;
113
114 static uint32_t
tprof_amd_ncounters(void)115 tprof_amd_ncounters(void)
116 {
117 return NCTRS;
118 }
119
120 static u_int
tprof_amd_counter_bitwidth(u_int counter)121 tprof_amd_counter_bitwidth(u_int counter)
122 {
123 return COUNTER_BITWIDTH;
124 }
125
126 static inline void
tprof_amd_counter_write(u_int counter,uint64_t val)127 tprof_amd_counter_write(u_int counter, uint64_t val)
128 {
129 wrmsr(PERFCTR(counter), val);
130 }
131
132 static inline uint64_t
tprof_amd_counter_read(u_int counter)133 tprof_amd_counter_read(u_int counter)
134 {
135 return rdmsr(PERFCTR(counter));
136 }
137
138 static void
tprof_amd_configure_event(u_int counter,const tprof_param_t * param)139 tprof_amd_configure_event(u_int counter, const tprof_param_t *param)
140 {
141 uint64_t pesr;
142 uint64_t event_lo;
143 uint64_t event_hi;
144
145 event_hi = param->p_event >> 8;
146 event_lo = param->p_event & 0xff;
147 pesr =
148 ((param->p_flags & TPROF_PARAM_USER) ? PESR_USR : 0) |
149 ((param->p_flags & TPROF_PARAM_KERN) ? PESR_OS : 0) |
150 PESR_INT |
151 __SHIFTIN(event_lo, PESR_EVENT_MASK_LO) |
152 __SHIFTIN(event_hi, PESR_EVENT_MASK_HI) |
153 __SHIFTIN(0, PESR_COUNTER_MASK) |
154 __SHIFTIN(param->p_unit, PESR_UNIT_MASK);
155 wrmsr(PERFEVTSEL(counter), pesr);
156
157 /* Reset the counter */
158 tprof_amd_counter_write(counter, param->p_value);
159 }
160
161 static void
tprof_amd_start(tprof_countermask_t runmask)162 tprof_amd_start(tprof_countermask_t runmask)
163 {
164 int bit;
165
166 while ((bit = ffs(runmask)) != 0) {
167 bit--;
168 CLR(runmask, __BIT(bit));
169 wrmsr(PERFEVTSEL(bit), rdmsr(PERFEVTSEL(bit)) | PESR_EN);
170 }
171 }
172
173 static void
tprof_amd_stop(tprof_countermask_t stopmask)174 tprof_amd_stop(tprof_countermask_t stopmask)
175 {
176 int bit;
177
178 while ((bit = ffs(stopmask)) != 0) {
179 bit--;
180 CLR(stopmask, __BIT(bit));
181 wrmsr(PERFEVTSEL(bit), rdmsr(PERFEVTSEL(bit)) & ~PESR_EN);
182 }
183 }
184
185 static int
tprof_amd_nmi(const struct trapframe * tf,void * arg)186 tprof_amd_nmi(const struct trapframe *tf, void *arg)
187 {
188 tprof_backend_softc_t *sc = arg;
189 tprof_frame_info_t tfi;
190 int bit;
191
192 uint64_t *counters_offset =
193 percpu_getptr_remote(sc->sc_ctr_offset_percpu, curcpu());
194 tprof_countermask_t mask = sc->sc_ctr_ovf_mask;
195 while ((bit = ffs(mask)) != 0) {
196 bit--;
197 CLR(mask, __BIT(bit));
198
199 /* If the highest bit is non zero, then it's not for us. */
200 uint64_t ctr = tprof_amd_counter_read(bit);
201 if ((ctr & __BIT(COUNTER_BITWIDTH - 1)) != 0)
202 continue; /* not overflowed */
203
204 if (ISSET(sc->sc_ctr_prof_mask, __BIT(bit))) {
205 /* Account for the counter, and reset */
206 tprof_amd_counter_write(bit,
207 sc->sc_count[bit].ctr_counter_reset_val);
208 counters_offset[bit] +=
209 sc->sc_count[bit].ctr_counter_val + ctr;
210
211 /* Record a sample */
212 #if defined(__x86_64__)
213 tfi.tfi_pc = tf->tf_rip;
214 #else
215 tfi.tfi_pc = tf->tf_eip;
216 #endif
217 tfi.tfi_counter = bit;
218 tfi.tfi_inkernel = tfi.tfi_pc >= VM_MIN_KERNEL_ADDRESS;
219 tprof_sample(NULL, &tfi);
220 } else {
221 /* Not profiled, but require to consider overflow */
222 counters_offset[bit] += __BIT(COUNTER_BITWIDTH);
223 }
224 }
225
226 return 1;
227 }
228
229 static uint64_t
tprof_amd_counter_estimate_freq(u_int counter)230 tprof_amd_counter_estimate_freq(u_int counter)
231 {
232 return curcpu()->ci_data.cpu_cc_freq;
233 }
234
235 static uint32_t
tprof_amd_ident(void)236 tprof_amd_ident(void)
237 {
238 struct cpu_info *ci = curcpu();
239
240 if (cpu_vendor != CPUVENDOR_AMD)
241 return TPROF_IDENT_NONE;
242
243 switch (CPUID_TO_FAMILY(ci->ci_signature)) {
244 case 0x10:
245 case 0x15:
246 case 0x17:
247 case 0x19:
248 return TPROF_IDENT_AMD_GENERIC;
249 }
250
251 return TPROF_IDENT_NONE;
252 }
253
254 static void
tprof_amd_establish_cpu(void * arg1,void * arg2)255 tprof_amd_establish_cpu(void *arg1, void *arg2)
256 {
257 struct cpu_info * const ci = curcpu();
258
259 amd_lapic_saved[cpu_index(ci)] = lapic_readreg(LAPIC_LVT_PCINT);
260 lapic_writereg(LAPIC_LVT_PCINT, LAPIC_DLMODE_NMI);
261 }
262
263 static void
tprof_amd_disestablish_cpu(void * arg1,void * arg2)264 tprof_amd_disestablish_cpu(void *arg1, void *arg2)
265 {
266 struct cpu_info * const ci = curcpu();
267
268 lapic_writereg(LAPIC_LVT_PCINT, amd_lapic_saved[cpu_index(ci)]);
269 }
270
271 static int
tprof_amd_establish(tprof_backend_softc_t * sc)272 tprof_amd_establish(tprof_backend_softc_t *sc)
273 {
274 uint64_t xc;
275
276 if (tprof_amd_ident() == TPROF_IDENT_NONE)
277 return ENOTSUP;
278
279 KASSERT(amd_nmi_handle == NULL);
280 amd_nmi_handle = nmi_establish(tprof_amd_nmi, sc);
281
282 xc = xc_broadcast(0, tprof_amd_establish_cpu, sc, NULL);
283 xc_wait(xc);
284
285 return 0;
286 }
287
288 static void
tprof_amd_disestablish(tprof_backend_softc_t * sc)289 tprof_amd_disestablish(tprof_backend_softc_t *sc)
290 {
291 uint64_t xc;
292
293 xc = xc_broadcast(0, tprof_amd_disestablish_cpu, sc, NULL);
294 xc_wait(xc);
295
296 KASSERT(amd_nmi_handle != NULL);
297 nmi_disestablish(amd_nmi_handle);
298 amd_nmi_handle = NULL;
299 }
300
301 const tprof_backend_ops_t tprof_amd_ops = {
302 .tbo_ident = tprof_amd_ident,
303 .tbo_ncounters = tprof_amd_ncounters,
304 .tbo_counter_bitwidth = tprof_amd_counter_bitwidth,
305 .tbo_counter_read = tprof_amd_counter_read,
306 .tbo_counter_estimate_freq = tprof_amd_counter_estimate_freq,
307 .tbo_valid_event = NULL,
308 .tbo_configure_event = tprof_amd_configure_event,
309 .tbo_start = tprof_amd_start,
310 .tbo_stop = tprof_amd_stop,
311 .tbo_establish = tprof_amd_establish,
312 .tbo_disestablish = tprof_amd_disestablish,
313 };
314