1 /* $NetBSD: tprof_armv7.c,v 1.13 2023/04/11 10:07:12 msaitoh Exp $ */
2
3 /*-
4 * Copyright (c) 2018 Jared McNeill <jmcneill@invisible.ca>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(0, "$NetBSD: tprof_armv7.c,v 1.13 2023/04/11 10:07:12 msaitoh Exp $");
31
32 #include <sys/param.h>
33 #include <sys/bus.h>
34 #include <sys/cpu.h>
35 #include <sys/percpu.h>
36 #include <sys/xcall.h>
37
38 #include <dev/tprof/tprof.h>
39
40 #include <arm/armreg.h>
41 #include <arm/locore.h>
42
43 #include <dev/tprof/tprof_armv7.h>
44
45 static uint16_t cortexa9_events[] = {
46 0x40, 0x41, 0x42,
47 0x50, 0x51,
48 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68,
49 0x6e,
50 0x70, 0x71, 0x72, 0x73, 0x74,
51 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86,
52 0x8a, 0x8b,
53 0x90, 0x91, 0x92, 0x93,
54 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5
55 };
56
57 static bool
armv7_pmu_event_implemented(uint16_t event)58 armv7_pmu_event_implemented(uint16_t event)
59 {
60 if (CPU_ID_CORTEX_A9_P(curcpu()->ci_midr)) {
61 /* Cortex-A9 with PMUv1 lacks PMCEID0/1 */
62 u_int n;
63
64 /* Events specific to the Cortex-A9 */
65 for (n = 0; n < __arraycount(cortexa9_events); n++) {
66 if (cortexa9_events[n] == event) {
67 return true;
68 }
69 }
70 /* Supported architectural events */
71 if (event != 0x08 && event != 0x0e && event < 0x1e) {
72 return true;
73 }
74 } else {
75 /* PMUv2 */
76 uint32_t eid[2];
77
78 if (event >= 64) {
79 return false;
80 }
81
82 eid[0] = armreg_pmceid0_read();
83 eid[1] = armreg_pmceid1_read();
84
85 const u_int idx = event / 32;
86 const u_int bit = event % 32;
87
88 if (eid[idx] & __BIT(bit)) {
89 return true;
90 }
91 }
92
93 return false;
94 }
95
96 static void
armv7_pmu_set_pmevtyper(u_int counter,uint64_t val)97 armv7_pmu_set_pmevtyper(u_int counter, uint64_t val)
98 {
99 armreg_pmselr_write(counter);
100 isb();
101 armreg_pmxevtyper_write(val);
102 }
103
104 static inline void
armv7_pmu_set_pmevcntr(u_int counter,uint32_t val)105 armv7_pmu_set_pmevcntr(u_int counter, uint32_t val)
106 {
107 armreg_pmselr_write(counter);
108 isb();
109 armreg_pmxevcntr_write(val);
110 }
111
112 static inline uint64_t
armv7_pmu_get_pmevcntr(u_int counter)113 armv7_pmu_get_pmevcntr(u_int counter)
114 {
115 armreg_pmselr_write(counter);
116 isb();
117 return armreg_pmxevcntr_read();
118 }
119
120 /* Read and write at once */
121 static inline uint64_t
armv7_pmu_getset_pmevcntr(u_int counter,uint64_t val)122 armv7_pmu_getset_pmevcntr(u_int counter, uint64_t val)
123 {
124 uint64_t c;
125
126 armreg_pmselr_write(counter);
127 isb();
128 c = armreg_pmxevcntr_read();
129 armreg_pmxevcntr_write(val);
130 return c;
131 }
132
133 static uint32_t
armv7_pmu_ncounters(void)134 armv7_pmu_ncounters(void)
135 {
136 return __SHIFTOUT(armreg_pmcr_read(), PMCR_N);
137 }
138
139 static u_int
armv7_pmu_counter_bitwidth(u_int counter)140 armv7_pmu_counter_bitwidth(u_int counter)
141 {
142 return 32;
143 }
144
145 static uint64_t
armv7_pmu_counter_estimate_freq(u_int counter)146 armv7_pmu_counter_estimate_freq(u_int counter)
147 {
148 uint64_t cpufreq = curcpu()->ci_data.cpu_cc_freq;
149
150 if (ISSET(armreg_pmcr_read(), PMCR_D))
151 cpufreq /= 64;
152 return cpufreq;
153 }
154
155 static int
armv7_pmu_valid_event(u_int counter,const tprof_param_t * param)156 armv7_pmu_valid_event(u_int counter, const tprof_param_t *param)
157 {
158 if (!armv7_pmu_event_implemented(param->p_event)) {
159 printf("%s: event %#" PRIx64 " not implemented on this CPU\n",
160 __func__, param->p_event);
161 return EINVAL;
162 }
163 return 0;
164 }
165
166 static void
armv7_pmu_configure_event(u_int counter,const tprof_param_t * param)167 armv7_pmu_configure_event(u_int counter, const tprof_param_t *param)
168 {
169 /* Disable event counter */
170 armreg_pmcntenclr_write(__BIT(counter) & PMCNTEN_P);
171
172 /* Disable overflow interrupts */
173 armreg_pmintenclr_write(__BIT(counter) & PMINTEN_P);
174
175 /* Configure event counter */
176 uint32_t pmevtyper = __SHIFTIN(param->p_event, PMEVTYPER_EVTCOUNT);
177 if (!ISSET(param->p_flags, TPROF_PARAM_USER))
178 pmevtyper |= PMEVTYPER_U;
179 if (!ISSET(param->p_flags, TPROF_PARAM_KERN))
180 pmevtyper |= PMEVTYPER_P;
181 armv7_pmu_set_pmevtyper(counter, pmevtyper);
182
183 /*
184 * Enable overflow interrupts.
185 * Whether profiled or not, the counter width of armv7 is 32 bits,
186 * so overflow handling is required anyway.
187 */
188 armreg_pmintenset_write(__BIT(counter) & PMINTEN_P);
189
190 /* Clear overflow flag */
191 armreg_pmovsr_write(__BIT(counter) & PMOVS_P);
192
193 /* Reset the counter */
194 armv7_pmu_set_pmevcntr(counter, param->p_value);
195 }
196
197 static void
armv7_pmu_start(tprof_countermask_t runmask)198 armv7_pmu_start(tprof_countermask_t runmask)
199 {
200 /* Enable event counters */
201 armreg_pmcntenset_write(runmask & PMCNTEN_P);
202
203 /*
204 * PMCR.E is shared with PMCCNTR and event counters.
205 * It is set here in case PMCCNTR is not used in the system.
206 */
207 armreg_pmcr_write(armreg_pmcr_read() | PMCR_E);
208 }
209
210 static void
armv7_pmu_stop(tprof_countermask_t stopmask)211 armv7_pmu_stop(tprof_countermask_t stopmask)
212 {
213 /* Disable event counter */
214 armreg_pmcntenclr_write(stopmask & PMCNTEN_P);
215 }
216
217 /* XXX: argument of armv8_pmu_intr() */
218 extern struct tprof_backend *tprof_backend;
219 static void *pmu_intr_arg;
220
221 int
armv7_pmu_intr(void * priv)222 armv7_pmu_intr(void *priv)
223 {
224 const struct trapframe * const tf = priv;
225 tprof_backend_softc_t *sc = pmu_intr_arg;
226 tprof_frame_info_t tfi;
227 int bit;
228 const uint32_t pmovs = armreg_pmovsr_read();
229
230 uint64_t *counters_offset =
231 percpu_getptr_remote(sc->sc_ctr_offset_percpu, curcpu());
232 uint32_t mask = pmovs;
233 while ((bit = ffs(mask)) != 0) {
234 bit--;
235 CLR(mask, __BIT(bit));
236
237 if (ISSET(sc->sc_ctr_prof_mask, __BIT(bit))) {
238 /* Account for the counter, and reset */
239 uint64_t ctr = armv7_pmu_getset_pmevcntr(bit,
240 sc->sc_count[bit].ctr_counter_reset_val);
241 counters_offset[bit] +=
242 sc->sc_count[bit].ctr_counter_val + ctr;
243
244 /* Record a sample */
245 tfi.tfi_pc = tf->tf_pc;
246 tfi.tfi_counter = bit;
247 tfi.tfi_inkernel =
248 tfi.tfi_pc >= VM_MIN_KERNEL_ADDRESS &&
249 tfi.tfi_pc < VM_MAX_KERNEL_ADDRESS;
250 tprof_sample(NULL, &tfi);
251 } else if (ISSET(sc->sc_ctr_ovf_mask, __BIT(bit))) {
252 /* Counter has overflowed */
253 counters_offset[bit] += __BIT(32);
254 }
255 }
256 armreg_pmovsr_write(pmovs);
257
258 return 1;
259 }
260
261 static uint32_t
armv7_pmu_ident(void)262 armv7_pmu_ident(void)
263 {
264 return TPROF_IDENT_ARMV7_GENERIC;
265 }
266
267 static const tprof_backend_ops_t tprof_armv7_pmu_ops = {
268 .tbo_ident = armv7_pmu_ident,
269 .tbo_ncounters = armv7_pmu_ncounters,
270 .tbo_counter_bitwidth = armv7_pmu_counter_bitwidth,
271 .tbo_counter_read = armv7_pmu_get_pmevcntr,
272 .tbo_counter_estimate_freq = armv7_pmu_counter_estimate_freq,
273 .tbo_valid_event = armv7_pmu_valid_event,
274 .tbo_configure_event = armv7_pmu_configure_event,
275 .tbo_start = armv7_pmu_start,
276 .tbo_stop = armv7_pmu_stop,
277 .tbo_establish = NULL,
278 .tbo_disestablish = NULL,
279 };
280
281 static void
armv7_pmu_init_cpu(void * arg1,void * arg2)282 armv7_pmu_init_cpu(void *arg1, void *arg2)
283 {
284 /* Disable user mode access to performance monitors */
285 armreg_pmuserenr_write(0);
286
287 /* Disable interrupts */
288 armreg_pmintenclr_write(PMINTEN_P);
289
290 /* Disable counters */
291 armreg_pmcntenclr_write(PMCNTEN_P);
292 }
293
294 int
armv7_pmu_init(void)295 armv7_pmu_init(void)
296 {
297 int error, ncounters;
298
299 ncounters = armv7_pmu_ncounters();
300 if (ncounters == 0)
301 return ENOTSUP;
302
303 uint64_t xc = xc_broadcast(0, armv7_pmu_init_cpu, NULL, NULL);
304 xc_wait(xc);
305
306 error = tprof_backend_register("tprof_armv7", &tprof_armv7_pmu_ops,
307 TPROF_BACKEND_VERSION);
308 if (error == 0) {
309 /* XXX: for argument of armv7_pmu_intr() */
310 pmu_intr_arg = tprof_backend;
311 }
312
313 return error;
314 }
315