1 /* $NetBSD: tprof_armv8.c,v 1.20 2023/04/11 10:07:12 msaitoh Exp $ */
2
3 /*-
4 * Copyright (c) 2018 Jared McNeill <jmcneill@invisible.ca>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(0, "$NetBSD: tprof_armv8.c,v 1.20 2023/04/11 10:07:12 msaitoh Exp $");
31
32 #include <sys/param.h>
33 #include <sys/bus.h>
34 #include <sys/cpu.h>
35 #include <sys/percpu.h>
36 #include <sys/xcall.h>
37
38 #include <dev/tprof/tprof.h>
39
40 #include <arm/armreg.h>
41 #include <arm/cpufunc.h>
42
43 #include <dev/tprof/tprof_armv8.h>
44
45 static u_int counter_bitwidth;
46
47 /*
48 * armv8 can handle up to 31 event counters,
49 * PMCR_EL0.N counters are actually available.
50 */
51
52 static bool
armv8_pmu_event_implemented(uint16_t event)53 armv8_pmu_event_implemented(uint16_t event)
54 {
55 uint64_t eid[2];
56
57 if (event >= 64)
58 return false;
59
60 eid[0] = reg_pmceid0_el0_read();
61 eid[1] = reg_pmceid1_el0_read();
62
63 /* The low 32bits of PMCEID[01]_EL0 contain the common events 0 to n */
64 const u_int idx = event / 32;
65 const u_int bit = event % 32;
66
67 if (eid[idx] & __BIT(bit))
68 return true;
69
70 return false;
71 }
72
73 static void
armv8_pmu_set_pmevtyper(u_int counter,uint64_t val)74 armv8_pmu_set_pmevtyper(u_int counter, uint64_t val)
75 {
76 reg_pmselr_el0_write(counter);
77 isb();
78 reg_pmxevtyper_el0_write(val);
79 }
80
81 static inline void
armv8_pmu_set_pmevcntr(u_int counter,uint64_t val)82 armv8_pmu_set_pmevcntr(u_int counter, uint64_t val)
83 {
84 reg_pmselr_el0_write(counter);
85 isb();
86 reg_pmxevcntr_el0_write(val);
87 }
88
89 static inline uint64_t
armv8_pmu_get_pmevcntr(u_int counter)90 armv8_pmu_get_pmevcntr(u_int counter)
91 {
92 reg_pmselr_el0_write(counter);
93 isb();
94 return reg_pmxevcntr_el0_read();
95 }
96
97 /* Read and write at once */
98 static inline uint64_t
armv8_pmu_getset_pmevcntr(u_int counter,uint64_t val)99 armv8_pmu_getset_pmevcntr(u_int counter, uint64_t val)
100 {
101 uint64_t c;
102
103 reg_pmselr_el0_write(counter);
104 isb();
105 c = reg_pmxevcntr_el0_read();
106 reg_pmxevcntr_el0_write(val);
107 return c;
108 }
109
110 static uint32_t
armv8_pmu_ncounters(void)111 armv8_pmu_ncounters(void)
112 {
113 return __SHIFTOUT(reg_pmcr_el0_read(), PMCR_N);
114 }
115
116 static u_int
armv8_pmu_counter_bitwidth(u_int counter)117 armv8_pmu_counter_bitwidth(u_int counter)
118 {
119 return counter_bitwidth;
120 }
121
122 static uint64_t
armv8_pmu_counter_estimate_freq(u_int counter)123 armv8_pmu_counter_estimate_freq(u_int counter)
124 {
125 return curcpu()->ci_data.cpu_cc_freq;
126 }
127
128 static int
armv8_pmu_valid_event(u_int counter,const tprof_param_t * param)129 armv8_pmu_valid_event(u_int counter, const tprof_param_t *param)
130 {
131 if (!armv8_pmu_event_implemented(param->p_event)) {
132 printf("%s: event %#" PRIx64 " not implemented on this CPU\n",
133 __func__, param->p_event);
134 return EINVAL;
135 }
136 return 0;
137 }
138
139 static void
armv8_pmu_configure_event(u_int counter,const tprof_param_t * param)140 armv8_pmu_configure_event(u_int counter, const tprof_param_t *param)
141 {
142 /* Disable event counter */
143 reg_pmcntenclr_el0_write(__BIT(counter) & PMCNTEN_P);
144
145 /* Disable overflow interrupts */
146 reg_pmintenclr_el1_write(__BIT(counter) & PMINTEN_P);
147
148 /* Configure event counter */
149 uint64_t pmevtyper = __SHIFTIN(param->p_event, PMEVTYPER_EVTCOUNT);
150 if (!ISSET(param->p_flags, TPROF_PARAM_USER))
151 pmevtyper |= PMEVTYPER_U;
152 if (!ISSET(param->p_flags, TPROF_PARAM_KERN))
153 pmevtyper |= PMEVTYPER_P;
154 armv8_pmu_set_pmevtyper(counter, pmevtyper);
155
156 if (ISSET(param->p_flags, TPROF_PARAM_PROFILE) ||
157 counter_bitwidth != 64) {
158 /* Enable overflow interrupts */
159 reg_pmintenset_el1_write(__BIT(counter) & PMINTEN_P);
160 }
161
162 /* Clear overflow flag */
163 reg_pmovsclr_el0_write(__BIT(counter) & PMOVS_P);
164
165 /* Reset the counter */
166 armv8_pmu_set_pmevcntr(counter, param->p_value);
167 }
168
169 static void
armv8_pmu_start(tprof_countermask_t runmask)170 armv8_pmu_start(tprof_countermask_t runmask)
171 {
172 /* Enable event counters */
173 reg_pmcntenset_el0_write(runmask & PMCNTEN_P);
174
175 /*
176 * PMCR.E is shared with PMCCNTR_EL0 and event counters.
177 * It is set here in case PMCCNTR_EL0 is not used in the system.
178 */
179 reg_pmcr_el0_write(reg_pmcr_el0_read() | PMCR_E);
180 }
181
182 static void
armv8_pmu_stop(tprof_countermask_t stopmask)183 armv8_pmu_stop(tprof_countermask_t stopmask)
184 {
185 /* Disable event counter */
186 reg_pmcntenclr_el0_write(stopmask & PMCNTEN_P);
187 }
188
189 /* XXX: argument of armv8_pmu_intr() */
190 extern struct tprof_backend *tprof_backend;
191 static void *pmu_intr_arg;
192
193 int
armv8_pmu_intr(void * priv)194 armv8_pmu_intr(void *priv)
195 {
196 const struct trapframe * const tf = priv;
197 tprof_backend_softc_t *sc = pmu_intr_arg;
198 tprof_frame_info_t tfi;
199 int bit;
200 const uint32_t pmovs = reg_pmovsset_el0_read();
201
202 uint64_t *counters_offset =
203 percpu_getptr_remote(sc->sc_ctr_offset_percpu, curcpu());
204 uint32_t mask = pmovs;
205 while ((bit = ffs(mask)) != 0) {
206 bit--;
207 CLR(mask, __BIT(bit));
208
209 if (ISSET(sc->sc_ctr_prof_mask, __BIT(bit))) {
210 /* Account for the counter, and reset */
211 uint64_t ctr = armv8_pmu_getset_pmevcntr(bit,
212 sc->sc_count[bit].ctr_counter_reset_val);
213 counters_offset[bit] +=
214 sc->sc_count[bit].ctr_counter_val + ctr;
215
216 /* Record a sample */
217 tfi.tfi_pc = tf->tf_pc;
218 tfi.tfi_counter = bit;
219 tfi.tfi_inkernel =
220 tfi.tfi_pc >= VM_MIN_KERNEL_ADDRESS &&
221 tfi.tfi_pc < VM_MAX_KERNEL_ADDRESS;
222 tprof_sample(NULL, &tfi);
223 } else if (ISSET(sc->sc_ctr_ovf_mask, __BIT(bit))) {
224 /* Counter has overflowed */
225 counters_offset[bit] += __BIT(32);
226 }
227 }
228 reg_pmovsclr_el0_write(pmovs);
229
230 return 1;
231 }
232
233 static uint32_t
armv8_pmu_ident(void)234 armv8_pmu_ident(void)
235 {
236 return TPROF_IDENT_ARMV8_GENERIC;
237 }
238
239 static const tprof_backend_ops_t tprof_armv8_pmu_ops = {
240 .tbo_ident = armv8_pmu_ident,
241 .tbo_ncounters = armv8_pmu_ncounters,
242 .tbo_counter_bitwidth = armv8_pmu_counter_bitwidth,
243 .tbo_counter_read = armv8_pmu_get_pmevcntr,
244 .tbo_counter_estimate_freq = armv8_pmu_counter_estimate_freq,
245 .tbo_valid_event = armv8_pmu_valid_event,
246 .tbo_configure_event = armv8_pmu_configure_event,
247 .tbo_start = armv8_pmu_start,
248 .tbo_stop = armv8_pmu_stop,
249 .tbo_establish = NULL,
250 .tbo_disestablish = NULL,
251 };
252
253 static void
armv8_pmu_init_cpu(void * arg1,void * arg2)254 armv8_pmu_init_cpu(void *arg1, void *arg2)
255 {
256 /* Disable EL0 access to performance monitors */
257 reg_pmuserenr_el0_write(0);
258
259 /* Disable interrupts */
260 reg_pmintenclr_el1_write(PMINTEN_P);
261
262 /* Disable event counters */
263 reg_pmcntenclr_el0_write(PMCNTEN_P);
264 }
265
266 bool
armv8_pmu_detect(void)267 armv8_pmu_detect(void)
268 {
269 const uint64_t dfr0 = reg_id_aa64dfr0_el1_read();
270 const u_int pmuver = __SHIFTOUT(dfr0, ID_AA64DFR0_EL1_PMUVER);
271
272 return pmuver != ID_AA64DFR0_EL1_PMUVER_NONE &&
273 pmuver != ID_AA64DFR0_EL1_PMUVER_IMPL;
274 }
275
276 int
armv8_pmu_init(void)277 armv8_pmu_init(void)
278 {
279 int error, ncounters;
280
281 KASSERT(armv8_pmu_detect());
282
283 ncounters = armv8_pmu_ncounters();
284 if (ncounters == 0)
285 return ENOTSUP;
286
287 /* Is 64bit event counter available? */
288 const uint64_t dfr0 = reg_id_aa64dfr0_el1_read();
289 const u_int pmuver = __SHIFTOUT(dfr0, ID_AA64DFR0_EL1_PMUVER);
290 if (pmuver >= ID_AA64DFR0_EL1_PMUVER_V3P5 &&
291 ISSET(reg_pmcr_el0_read(), PMCR_LP))
292 counter_bitwidth = 64;
293 else
294 counter_bitwidth = 32;
295
296 uint64_t xc = xc_broadcast(0, armv8_pmu_init_cpu, NULL, NULL);
297 xc_wait(xc);
298
299 error = tprof_backend_register("tprof_armv8", &tprof_armv8_pmu_ops,
300 TPROF_BACKEND_VERSION);
301 if (error == 0) {
302 /* XXX: for argument of armv8_pmu_intr() */
303 pmu_intr_arg = tprof_backend;
304 }
305
306 return error;
307 }
308