1 /* $NetBSD: tprof_armv7.c,v 1.13 2023/04/11 10:07:12 msaitoh Exp $ */ 2 3 /*- 4 * Copyright (c) 2018 Jared McNeill <jmcneill@invisible.ca> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(0, "$NetBSD: tprof_armv7.c,v 1.13 2023/04/11 10:07:12 msaitoh Exp $"); 31 32 #include <sys/param.h> 33 #include <sys/bus.h> 34 #include <sys/cpu.h> 35 #include <sys/percpu.h> 36 #include <sys/xcall.h> 37 38 #include <dev/tprof/tprof.h> 39 40 #include <arm/armreg.h> 41 #include <arm/locore.h> 42 43 #include <dev/tprof/tprof_armv7.h> 44 45 static uint16_t cortexa9_events[] = { 46 0x40, 0x41, 0x42, 47 0x50, 0x51, 48 0x60, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 49 0x6e, 50 0x70, 0x71, 0x72, 0x73, 0x74, 51 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 52 0x8a, 0x8b, 53 0x90, 0x91, 0x92, 0x93, 54 0xa0, 0xa1, 0xa2, 0xa3, 0xa4, 0xa5 55 }; 56 57 static bool 58 armv7_pmu_event_implemented(uint16_t event) 59 { 60 if (CPU_ID_CORTEX_A9_P(curcpu()->ci_midr)) { 61 /* Cortex-A9 with PMUv1 lacks PMCEID0/1 */ 62 u_int n; 63 64 /* Events specific to the Cortex-A9 */ 65 for (n = 0; n < __arraycount(cortexa9_events); n++) { 66 if (cortexa9_events[n] == event) { 67 return true; 68 } 69 } 70 /* Supported architectural events */ 71 if (event != 0x08 && event != 0x0e && event < 0x1e) { 72 return true; 73 } 74 } else { 75 /* PMUv2 */ 76 uint32_t eid[2]; 77 78 if (event >= 64) { 79 return false; 80 } 81 82 eid[0] = armreg_pmceid0_read(); 83 eid[1] = armreg_pmceid1_read(); 84 85 const u_int idx = event / 32; 86 const u_int bit = event % 32; 87 88 if (eid[idx] & __BIT(bit)) { 89 return true; 90 } 91 } 92 93 return false; 94 } 95 96 static void 97 armv7_pmu_set_pmevtyper(u_int counter, uint64_t val) 98 { 99 armreg_pmselr_write(counter); 100 isb(); 101 armreg_pmxevtyper_write(val); 102 } 103 104 static inline void 105 armv7_pmu_set_pmevcntr(u_int counter, uint32_t val) 106 { 107 armreg_pmselr_write(counter); 108 isb(); 109 armreg_pmxevcntr_write(val); 110 } 111 112 static inline uint64_t 113 armv7_pmu_get_pmevcntr(u_int counter) 114 { 115 armreg_pmselr_write(counter); 116 isb(); 117 return armreg_pmxevcntr_read(); 118 } 119 120 /* Read and write at once */ 121 static inline uint64_t 122 armv7_pmu_getset_pmevcntr(u_int counter, uint64_t val) 123 { 124 uint64_t c; 125 126 armreg_pmselr_write(counter); 127 isb(); 128 c = armreg_pmxevcntr_read(); 129 armreg_pmxevcntr_write(val); 130 return c; 131 } 132 133 static uint32_t 134 armv7_pmu_ncounters(void) 135 { 136 return __SHIFTOUT(armreg_pmcr_read(), PMCR_N); 137 } 138 139 static u_int 140 armv7_pmu_counter_bitwidth(u_int counter) 141 { 142 return 32; 143 } 144 145 static uint64_t 146 armv7_pmu_counter_estimate_freq(u_int counter) 147 { 148 uint64_t cpufreq = curcpu()->ci_data.cpu_cc_freq; 149 150 if (ISSET(armreg_pmcr_read(), PMCR_D)) 151 cpufreq /= 64; 152 return cpufreq; 153 } 154 155 static int 156 armv7_pmu_valid_event(u_int counter, const tprof_param_t *param) 157 { 158 if (!armv7_pmu_event_implemented(param->p_event)) { 159 printf("%s: event %#" PRIx64 " not implemented on this CPU\n", 160 __func__, param->p_event); 161 return EINVAL; 162 } 163 return 0; 164 } 165 166 static void 167 armv7_pmu_configure_event(u_int counter, const tprof_param_t *param) 168 { 169 /* Disable event counter */ 170 armreg_pmcntenclr_write(__BIT(counter) & PMCNTEN_P); 171 172 /* Disable overflow interrupts */ 173 armreg_pmintenclr_write(__BIT(counter) & PMINTEN_P); 174 175 /* Configure event counter */ 176 uint32_t pmevtyper = __SHIFTIN(param->p_event, PMEVTYPER_EVTCOUNT); 177 if (!ISSET(param->p_flags, TPROF_PARAM_USER)) 178 pmevtyper |= PMEVTYPER_U; 179 if (!ISSET(param->p_flags, TPROF_PARAM_KERN)) 180 pmevtyper |= PMEVTYPER_P; 181 armv7_pmu_set_pmevtyper(counter, pmevtyper); 182 183 /* 184 * Enable overflow interrupts. 185 * Whether profiled or not, the counter width of armv7 is 32 bits, 186 * so overflow handling is required anyway. 187 */ 188 armreg_pmintenset_write(__BIT(counter) & PMINTEN_P); 189 190 /* Clear overflow flag */ 191 armreg_pmovsr_write(__BIT(counter) & PMOVS_P); 192 193 /* Reset the counter */ 194 armv7_pmu_set_pmevcntr(counter, param->p_value); 195 } 196 197 static void 198 armv7_pmu_start(tprof_countermask_t runmask) 199 { 200 /* Enable event counters */ 201 armreg_pmcntenset_write(runmask & PMCNTEN_P); 202 203 /* 204 * PMCR.E is shared with PMCCNTR and event counters. 205 * It is set here in case PMCCNTR is not used in the system. 206 */ 207 armreg_pmcr_write(armreg_pmcr_read() | PMCR_E); 208 } 209 210 static void 211 armv7_pmu_stop(tprof_countermask_t stopmask) 212 { 213 /* Disable event counter */ 214 armreg_pmcntenclr_write(stopmask & PMCNTEN_P); 215 } 216 217 /* XXX: argument of armv8_pmu_intr() */ 218 extern struct tprof_backend *tprof_backend; 219 static void *pmu_intr_arg; 220 221 int 222 armv7_pmu_intr(void *priv) 223 { 224 const struct trapframe * const tf = priv; 225 tprof_backend_softc_t *sc = pmu_intr_arg; 226 tprof_frame_info_t tfi; 227 int bit; 228 const uint32_t pmovs = armreg_pmovsr_read(); 229 230 uint64_t *counters_offset = 231 percpu_getptr_remote(sc->sc_ctr_offset_percpu, curcpu()); 232 uint32_t mask = pmovs; 233 while ((bit = ffs(mask)) != 0) { 234 bit--; 235 CLR(mask, __BIT(bit)); 236 237 if (ISSET(sc->sc_ctr_prof_mask, __BIT(bit))) { 238 /* Account for the counter, and reset */ 239 uint64_t ctr = armv7_pmu_getset_pmevcntr(bit, 240 sc->sc_count[bit].ctr_counter_reset_val); 241 counters_offset[bit] += 242 sc->sc_count[bit].ctr_counter_val + ctr; 243 244 /* Record a sample */ 245 tfi.tfi_pc = tf->tf_pc; 246 tfi.tfi_counter = bit; 247 tfi.tfi_inkernel = 248 tfi.tfi_pc >= VM_MIN_KERNEL_ADDRESS && 249 tfi.tfi_pc < VM_MAX_KERNEL_ADDRESS; 250 tprof_sample(NULL, &tfi); 251 } else if (ISSET(sc->sc_ctr_ovf_mask, __BIT(bit))) { 252 /* Counter has overflowed */ 253 counters_offset[bit] += __BIT(32); 254 } 255 } 256 armreg_pmovsr_write(pmovs); 257 258 return 1; 259 } 260 261 static uint32_t 262 armv7_pmu_ident(void) 263 { 264 return TPROF_IDENT_ARMV7_GENERIC; 265 } 266 267 static const tprof_backend_ops_t tprof_armv7_pmu_ops = { 268 .tbo_ident = armv7_pmu_ident, 269 .tbo_ncounters = armv7_pmu_ncounters, 270 .tbo_counter_bitwidth = armv7_pmu_counter_bitwidth, 271 .tbo_counter_read = armv7_pmu_get_pmevcntr, 272 .tbo_counter_estimate_freq = armv7_pmu_counter_estimate_freq, 273 .tbo_valid_event = armv7_pmu_valid_event, 274 .tbo_configure_event = armv7_pmu_configure_event, 275 .tbo_start = armv7_pmu_start, 276 .tbo_stop = armv7_pmu_stop, 277 .tbo_establish = NULL, 278 .tbo_disestablish = NULL, 279 }; 280 281 static void 282 armv7_pmu_init_cpu(void *arg1, void *arg2) 283 { 284 /* Disable user mode access to performance monitors */ 285 armreg_pmuserenr_write(0); 286 287 /* Disable interrupts */ 288 armreg_pmintenclr_write(PMINTEN_P); 289 290 /* Disable counters */ 291 armreg_pmcntenclr_write(PMCNTEN_P); 292 } 293 294 int 295 armv7_pmu_init(void) 296 { 297 int error, ncounters; 298 299 ncounters = armv7_pmu_ncounters(); 300 if (ncounters == 0) 301 return ENOTSUP; 302 303 uint64_t xc = xc_broadcast(0, armv7_pmu_init_cpu, NULL, NULL); 304 xc_wait(xc); 305 306 error = tprof_backend_register("tprof_armv7", &tprof_armv7_pmu_ops, 307 TPROF_BACKEND_VERSION); 308 if (error == 0) { 309 /* XXX: for argument of armv7_pmu_intr() */ 310 pmu_intr_arg = tprof_backend; 311 } 312 313 return error; 314 } 315