1 /* $NetBSD: tprof_x86_intel.c,v 1.4 2022/05/26 13:02:04 msaitoh Exp $ */ 2 3 /* 4 * Copyright (c) 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 /* 33 * Copyright (c)2008,2009 YAMAMOTO Takashi, 34 * All rights reserved. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 45 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 46 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 47 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 48 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 49 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 50 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 51 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 52 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 53 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 54 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 55 * SUCH DAMAGE. 56 */ 57 58 #include <sys/cdefs.h> 59 __KERNEL_RCSID(0, "$NetBSD: tprof_x86_intel.c,v 1.4 2022/05/26 13:02:04 msaitoh Exp $"); 60 61 #include <sys/param.h> 62 #include <sys/systm.h> 63 #include <sys/device.h> 64 #include <sys/kernel.h> 65 #include <sys/module.h> 66 67 #include <sys/cpu.h> 68 #include <sys/xcall.h> 69 70 #include <dev/tprof/tprof.h> 71 72 #include <uvm/uvm.h> /* VM_MIN_KERNEL_ADDRESS */ 73 74 #include <x86/nmi.h> 75 76 #include <machine/cpufunc.h> 77 #include <machine/cputypes.h> /* CPUVENDOR_* */ 78 #include <machine/cpuvar.h> /* cpu_vendor */ 79 #include <machine/i82489reg.h> 80 #include <machine/i82489var.h> 81 82 #define PERFEVTSEL_EVENT_SELECT __BITS(0, 7) 83 #define PERFEVTSEL_UNIT_MASK __BITS(8, 15) 84 #define PERFEVTSEL_USR __BIT(16) 85 #define PERFEVTSEL_OS __BIT(17) 86 #define PERFEVTSEL_E __BIT(18) 87 #define PERFEVTSEL_PC __BIT(19) 88 #define PERFEVTSEL_INT __BIT(20) 89 #define PERFEVTSEL_EN __BIT(22) 90 #define PERFEVTSEL_INV __BIT(23) 91 #define PERFEVTSEL_COUNTER_MASK __BITS(24, 31) 92 93 static uint64_t counter_bitwidth; 94 static uint64_t counter_val = 5000000; 95 static uint64_t counter_reset_val; 96 97 static uint32_t intel_lapic_saved[MAXCPUS]; 98 static nmi_handler_t *intel_nmi_handle; 99 static tprof_param_t intel_param; 100 101 static void 102 tprof_intel_start_cpu(void *arg1, void *arg2) 103 { 104 struct cpu_info * const ci = curcpu(); 105 uint64_t evtval; 106 107 evtval = 108 __SHIFTIN(intel_param.p_event, PERFEVTSEL_EVENT_SELECT) | 109 __SHIFTIN(intel_param.p_unit, PERFEVTSEL_UNIT_MASK) | 110 ((intel_param.p_flags & TPROF_PARAM_USER) ? PERFEVTSEL_USR : 0) | 111 ((intel_param.p_flags & TPROF_PARAM_KERN) ? PERFEVTSEL_OS : 0) | 112 PERFEVTSEL_INT | 113 PERFEVTSEL_EN; 114 115 wrmsr(MSR_PERFCTR0, counter_reset_val); 116 wrmsr(MSR_EVNTSEL0, evtval); 117 118 intel_lapic_saved[cpu_index(ci)] = lapic_readreg(LAPIC_LVT_PCINT); 119 lapic_writereg(LAPIC_LVT_PCINT, LAPIC_DLMODE_NMI); 120 } 121 122 static void 123 tprof_intel_stop_cpu(void *arg1, void *arg2) 124 { 125 struct cpu_info * const ci = curcpu(); 126 127 wrmsr(MSR_EVNTSEL0, 0); 128 wrmsr(MSR_PERFCTR0, 0); 129 130 lapic_writereg(LAPIC_LVT_PCINT, intel_lapic_saved[cpu_index(ci)]); 131 } 132 133 static int 134 tprof_intel_nmi(const struct trapframe *tf, void *dummy) 135 { 136 uint32_t pcint; 137 uint64_t ctr; 138 tprof_frame_info_t tfi; 139 140 KASSERT(dummy == NULL); 141 142 ctr = rdmsr(MSR_PERFCTR0); 143 /* If the highest bit is non zero, then it's not for us. */ 144 if ((ctr & __BIT(counter_bitwidth-1)) != 0) { 145 return 0; 146 } 147 148 /* record a sample */ 149 #if defined(__x86_64__) 150 tfi.tfi_pc = tf->tf_rip; 151 #else 152 tfi.tfi_pc = tf->tf_eip; 153 #endif 154 tfi.tfi_inkernel = tfi.tfi_pc >= VM_MIN_KERNEL_ADDRESS; 155 tprof_sample(NULL, &tfi); 156 157 /* reset counter */ 158 wrmsr(MSR_PERFCTR0, counter_reset_val); 159 160 /* unmask PMI */ 161 pcint = lapic_readreg(LAPIC_LVT_PCINT); 162 KASSERT((pcint & LAPIC_LVT_MASKED) != 0); 163 lapic_writereg(LAPIC_LVT_PCINT, pcint & ~LAPIC_LVT_MASKED); 164 165 return 1; 166 } 167 168 static uint64_t 169 tprof_intel_estimate_freq(void) 170 { 171 uint64_t cpufreq = curcpu()->ci_data.cpu_cc_freq; 172 uint64_t freq = 10000; 173 174 counter_val = cpufreq / freq; 175 if (counter_val == 0) { 176 counter_val = UINT64_C(4000000000) / freq; 177 } 178 return freq; 179 } 180 181 static uint32_t 182 tprof_intel_ident(void) 183 { 184 uint32_t descs[4]; 185 186 if (cpu_vendor != CPUVENDOR_INTEL) { 187 return TPROF_IDENT_NONE; 188 } 189 190 if (cpuid_level < 0x0A) { 191 return TPROF_IDENT_NONE; 192 } 193 x86_cpuid(0x0A, descs); 194 if ((descs[0] & CPUID_PERF_VERSION) == 0) { 195 return TPROF_IDENT_NONE; 196 } 197 if ((descs[0] & CPUID_PERF_NGPPC) == 0) { 198 return TPROF_IDENT_NONE; 199 } 200 201 counter_bitwidth = __SHIFTOUT(descs[0], CPUID_PERF_NBWGPPC); 202 203 return TPROF_IDENT_INTEL_GENERIC; 204 } 205 206 static int 207 tprof_intel_start(const tprof_param_t *param) 208 { 209 uint64_t xc; 210 211 if (tprof_intel_ident() == TPROF_IDENT_NONE) { 212 return ENOTSUP; 213 } 214 215 KASSERT(intel_nmi_handle == NULL); 216 intel_nmi_handle = nmi_establish(tprof_intel_nmi, NULL); 217 218 counter_reset_val = - counter_val + 1; 219 memcpy(&intel_param, param, sizeof(*param)); 220 221 xc = xc_broadcast(0, tprof_intel_start_cpu, NULL, NULL); 222 xc_wait(xc); 223 224 return 0; 225 } 226 227 static void 228 tprof_intel_stop(const tprof_param_t *param) 229 { 230 uint64_t xc; 231 232 xc = xc_broadcast(0, tprof_intel_stop_cpu, NULL, NULL); 233 xc_wait(xc); 234 235 KASSERT(intel_nmi_handle != NULL); 236 nmi_disestablish(intel_nmi_handle); 237 intel_nmi_handle = NULL; 238 } 239 240 const tprof_backend_ops_t tprof_intel_ops = { 241 .tbo_estimate_freq = tprof_intel_estimate_freq, 242 .tbo_ident = tprof_intel_ident, 243 .tbo_start = tprof_intel_start, 244 .tbo_stop = tprof_intel_stop, 245 }; 246