Lines Matching +full:cpu +full:- +full:core
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
30 * Intel Core PMCs.
42 #include <machine/cpu.h>
65 #define EV_IS_ARCH_NOTSUPP -1
108 core_pcpu_noop(struct pmc_mdep *md, int cpu)
111 (void) cpu;
116 core_pcpu_init(struct pmc_mdep *md, int cpu)
123 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
124 ("[iaf,%d] insane cpu number %d", __LINE__, cpu));
126 PMCDBG1(MDP,INI,1,"core-init cpu=%d", cpu);
128 core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
129 npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
132 npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
137 core_pcpu[cpu] = cc;
138 pc = pmc_pcpu[cpu];
141 ("[core,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu));
143 for (n = 0, phw = cc->pc_corepmcs; n < npmc; n++, phw++) {
144 phw->phw_state = PMC_PHW_FLAG_IS_ENABLED |
145 PMC_PHW_CPU_TO_STATE(cpu) |
147 phw->phw_pmc = NULL;
148 pc->pc_hwpmcs[n + core_ri] = phw;
160 core_pcpu_fini(struct pmc_mdep *md, int cpu)
166 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
167 ("[core,%d] insane cpu number (%d)", __LINE__, cpu));
169 PMCDBG1(MDP,INI,1,"core-pcpu-fini cpu=%d", cpu);
171 if ((cc = core_pcpu[cpu]) == NULL)
174 core_pcpu[cpu] = NULL;
176 pc = pmc_pcpu[cpu];
178 KASSERT(pc != NULL, ("[core,%d] NULL per-cpu %d state", __LINE__,
179 cpu));
181 npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_num;
182 core_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP].pcd_ri;
189 npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF].pcd_num;
193 pc->pc_hwpmcs[n + core_ri] = NULL;
209 if ((v & (1ULL << (core_iaf_width - 1))) == 0)
211 v &= (1ULL << core_iaf_width) - 1;
212 return (1ULL << core_iaf_width) - v;
218 return (1ULL << core_iaf_width) - rlc;
222 iaf_allocate_pmc(int cpu, int ri, struct pmc *pm,
230 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
231 ("[core,%d] illegal CPU %d", __LINE__, cpu));
233 PMCDBG2(MDP,ALL,1, "iaf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps);
238 if (a->pm_class != PMC_CLASS_IAF)
241 if ((a->pm_flags & PMC_F_EV_PMU) == 0)
244 iap = &a->pm_md.pm_iap;
245 config = iap->pm_iap_config;
293 caps = a->pm_caps;
303 pm->pm_md.pm_iaf.pm_iaf_ctrl = (flags << (ri * 4));
305 PMCDBG1(MDP,ALL,2, "iaf-allocate config=0x%jx",
306 (uintmax_t) pm->pm_md.pm_iaf.pm_iaf_ctrl);
312 iaf_config_pmc(int cpu, int ri, struct pmc *pm)
314 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
315 ("[core,%d] illegal CPU %d", __LINE__, cpu));
318 ("[core,%d] illegal row-index %d", __LINE__, ri));
320 PMCDBG3(MDP,CFG,1, "iaf-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
322 KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__,
323 cpu));
325 core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc = pm;
331 iaf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
335 phw = &core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri];
337 snprintf(pi->pm_name, sizeof(pi->pm_name), "IAF-%d", ri);
338 pi->pm_class = PMC_CLASS_IAF;
340 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
341 pi->pm_enabled = TRUE;
342 *ppmc = phw->phw_pmc;
344 pi->pm_enabled = FALSE;
352 iaf_get_config(int cpu, int ri, struct pmc **ppm)
354 *ppm = core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc;
371 iaf_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)
375 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
376 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
378 ("[core,%d] illegal row-index %d", __LINE__, ri));
385 *v = tmp & ((1ULL << core_iaf_width) - 1);
387 PMCDBG4(MDP,REA,1, "iaf-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
394 iaf_release_pmc(int cpu, int ri, struct pmc *pmc)
396 PMCDBG3(MDP,REL,1, "iaf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc);
398 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
399 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
401 ("[core,%d] illegal row-index %d", __LINE__, ri));
403 KASSERT(core_pcpu[cpu]->pc_corepmcs[ri + core_iaf_ri].phw_pmc == NULL,
404 ("[core,%d] PHW pmc non-NULL", __LINE__));
407 if (pmc_alloc_refs-- == 1 && pmc_tsx_force_abort_set) {
417 iaf_start_pmc(int cpu, int ri, struct pmc *pm)
421 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
422 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
424 ("[core,%d] illegal row-index %d", __LINE__, ri));
426 PMCDBG2(MDP,STA,1,"iaf-start cpu=%d ri=%d", cpu, ri);
428 cc = core_pcpu[cpu];
429 cc->pc_iafctrl |= pm->pm_md.pm_iaf.pm_iaf_ctrl;
430 wrmsr(IAF_CTRL, cc->pc_iafctrl);
432 cc->pc_globalctrl |= (1ULL << (ri + IAF_OFFSET));
433 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
436 cc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
437 cc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
443 iaf_stop_pmc(int cpu, int ri, struct pmc *pm)
447 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
448 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
450 ("[core,%d] illegal row-index %d", __LINE__, ri));
452 PMCDBG2(MDP,STA,1,"iaf-stop cpu=%d ri=%d", cpu, ri);
454 cc = core_pcpu[cpu];
456 cc->pc_iafctrl &= ~(IAF_MASK << (ri * 4));
457 wrmsr(IAF_CTRL, cc->pc_iafctrl);
462 cc->pc_iafctrl, (uint32_t) rdmsr(IAF_CTRL),
463 cc->pc_globalctrl, rdmsr(IA_GLOBAL_CTRL));
469 iaf_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)
473 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
474 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
476 ("[core,%d] illegal row-index %d", __LINE__, ri));
478 cc = core_pcpu[cpu];
484 wrmsr(IAF_CTRL, cc->pc_iafctrl & ~(IAF_MASK << (ri * 4)));
486 wrmsr(IAF_CTR0 + ri, v & ((1ULL << core_iaf_width) - 1));
489 wrmsr(IAF_CTRL, cc->pc_iafctrl);
491 PMCDBG6(MDP,WRI,1, "iaf-write cpu=%d ri=%d msr=0x%x v=%jx iafctrl=%jx "
492 "pmc=%jx", cpu, ri, IAF_RI_TO_MSR(ri), v,
507 PMCDBG0(MDP,INI,1, "iaf-initialize");
509 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAF];
511 pcd->pcd_caps = IAF_PMC_CAPS;
512 pcd->pcd_class = PMC_CLASS_IAF;
513 pcd->pcd_num = npmc;
514 pcd->pcd_ri = md->pmd_npmc;
515 pcd->pcd_width = pmcwidth;
517 pcd->pcd_allocate_pmc = iaf_allocate_pmc;
518 pcd->pcd_config_pmc = iaf_config_pmc;
519 pcd->pcd_describe = iaf_describe;
520 pcd->pcd_get_config = iaf_get_config;
521 pcd->pcd_get_msr = iaf_get_msr;
522 pcd->pcd_pcpu_fini = core_pcpu_noop;
523 pcd->pcd_pcpu_init = core_pcpu_noop;
524 pcd->pcd_read_pmc = iaf_read_pmc;
525 pcd->pcd_release_pmc = iaf_release_pmc;
526 pcd->pcd_start_pmc = iaf_start_pmc;
527 pcd->pcd_stop_pmc = iaf_stop_pmc;
528 pcd->pcd_write_pmc = iaf_write_pmc;
530 md->pmd_npmc += npmc;
538 #define IAP_M_CORE (1 << 0) /* Core specificity */
546 #define IAP_F_CORE (0x3 << 14) /* Core specificity */
564 if ((v & (1ULL << (core_iap_width - 1))) == 0)
566 v &= (1ULL << core_iap_width) - 1;
567 return (1ULL << core_iap_width) - v;
573 return (1ULL << core_iap_width) - rlc;
582 * We treat a Core (i.e., Intel architecture v1) PMC as has
586 return ((v & (1ULL << (core_iap_width - 1))) == 0);
712 iap_allocate_pmc(int cpu, int ri, struct pmc *pm,
718 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
719 ("[core,%d] illegal CPU %d", __LINE__, cpu));
721 ("[core,%d] illegal row-index value %d", __LINE__, ri));
723 if (a->pm_class != PMC_CLASS_IAP)
726 if ((a->pm_flags & PMC_F_EV_PMU) == 0)
729 iap = &a->pm_md.pm_iap;
730 ev = IAP_EVSEL_GET(iap->pm_iap_config);
773 pm->pm_md.pm_iap.pm_iap_evsel = iap->pm_iap_config;
778 iap_config_pmc(int cpu, int ri, struct pmc *pm)
780 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
781 ("[core,%d] illegal CPU %d", __LINE__, cpu));
784 ("[core,%d] illegal row-index %d", __LINE__, ri));
786 PMCDBG3(MDP,CFG,1, "iap-config cpu=%d ri=%d pm=%p", cpu, ri, pm);
788 KASSERT(core_pcpu[cpu] != NULL, ("[core,%d] null per-cpu %d", __LINE__,
789 cpu));
791 core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc = pm;
797 iap_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)
801 phw = &core_pcpu[cpu]->pc_corepmcs[ri];
803 snprintf(pi->pm_name, sizeof(pi->pm_name), "IAP-%d", ri);
804 pi->pm_class = PMC_CLASS_IAP;
806 if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {
807 pi->pm_enabled = TRUE;
808 *ppmc = phw->phw_pmc;
810 pi->pm_enabled = FALSE;
818 iap_get_config(int cpu, int ri, struct pmc **ppm)
820 *ppm = core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc;
837 iap_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)
841 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
842 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
844 ("[core,%d] illegal row-index %d", __LINE__, ri));
850 *v = tmp & ((1ULL << core_iap_width) - 1);
852 PMCDBG4(MDP,REA,1, "iap-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,
859 iap_release_pmc(int cpu, int ri, struct pmc *pm)
863 PMCDBG3(MDP,REL,1, "iap-release cpu=%d ri=%d pm=%p", cpu, ri,
866 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
867 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
869 ("[core,%d] illegal row-index %d", __LINE__, ri));
871 KASSERT(core_pcpu[cpu]->pc_corepmcs[ri].phw_pmc
872 == NULL, ("[core,%d] PHW pmc non-NULL", __LINE__));
878 iap_start_pmc(int cpu, int ri, struct pmc *pm)
883 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
884 ("[core,%d] illegal CPU value %d", __LINE__, cpu));
886 ("[core,%d] illegal row-index %d", __LINE__, ri));
888 cc = core_pcpu[cpu];
890 PMCDBG2(MDP,STA,1, "iap-start cpu=%d ri=%d", cpu, ri);
892 evsel = pm->pm_md.pm_iap.pm_iap_evsel;
894 PMCDBG4(MDP,STA,2, "iap-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x",
895 cpu, ri, IAP_EVSEL0 + ri, evsel);
901 wrmsr(IA_OFFCORE_RSP0, pm->pm_md.pm_iap.pm_iap_rsp);
904 wrmsr(IA_OFFCORE_RSP1, pm->pm_md.pm_iap.pm_iap_rsp);
913 cc->pc_globalctrl |= (1ULL << ri);
914 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
921 iap_stop_pmc(int cpu, int ri, struct pmc *pm __unused)
924 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
925 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
927 ("[core,%d] illegal row index %d", __LINE__, ri));
929 PMCDBG2(MDP,STO,1, "iap-stop cpu=%d ri=%d", cpu, ri);
939 iap_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)
942 KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),
943 ("[core,%d] illegal cpu value %d", __LINE__, cpu));
945 ("[core,%d] illegal row index %d", __LINE__, ri));
950 v &= (1ULL << core_iap_width) - 1;
952 PMCDBG4(MDP,WRI,1, "iap-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,
973 PMCDBG0(MDP,INI,1, "iap-initialize");
978 pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_IAP];
980 pcd->pcd_caps = IAP_PMC_CAPS;
981 pcd->pcd_class = PMC_CLASS_IAP;
982 pcd->pcd_num = npmc;
983 pcd->pcd_ri = md->pmd_npmc;
984 pcd->pcd_width = pmcwidth;
986 pcd->pcd_allocate_pmc = iap_allocate_pmc;
987 pcd->pcd_config_pmc = iap_config_pmc;
988 pcd->pcd_describe = iap_describe;
989 pcd->pcd_get_config = iap_get_config;
990 pcd->pcd_get_msr = iap_get_msr;
991 pcd->pcd_pcpu_fini = core_pcpu_fini;
992 pcd->pcd_pcpu_init = core_pcpu_init;
993 pcd->pcd_read_pmc = iap_read_pmc;
994 pcd->pcd_release_pmc = iap_release_pmc;
995 pcd->pcd_start_pmc = iap_start_pmc;
996 pcd->pcd_stop_pmc = iap_stop_pmc;
997 pcd->pcd_write_pmc = iap_write_pmc;
999 md->pmd_npmc += npmc;
1010 PMCDBG3(MDP,INT, 1, "cpu=%d tf=%p um=%d", curcpu, (void *) tf,
1018 if ((pm = cc->pc_corepmcs[ri].phw_pmc) == NULL ||
1027 if (pm->pm_state != PMC_STATE_RUNNING)
1032 v = pm->pm_sc.pm_reloadcount;
1039 wrmsr(IAP_EVSEL0 + ri, pm->pm_md.pm_iap.pm_iap_evsel);
1045 wrmsr(IAP_EVSEL0 + ri, pm->pm_md.pm_iap.pm_iap_evsel | IAP_EN);
1062 int error, found_interrupt = 0, n, cpu;
1068 cpu = curcpu;
1069 PMCDBG3(MDP,INT, 1, "cpu=%d tf=0x%p um=%d", cpu, (void *) tf,
1079 PMCDBG2(MDP,INT, 1, "cpu=%d intrstatus=%jx", cpu,
1088 cc = core_pcpu[cpu];
1089 KASSERT(cc != NULL, ("[core,%d] null pcpu", __LINE__));
1102 pm = cc->pc_corepmcs[n + core_iaf_ri].phw_pmc;
1103 if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING ||
1111 v = iaf_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount);
1116 PMCDBG4(MDP,INT, 1, "iaf-intr cpu=%d error=%d v=%jx(%jx)", curcpu,
1129 pm = cc->pc_corepmcs[n].phw_pmc;
1130 if (pm == NULL || pm->pm_state != PMC_STATE_RUNNING ||
1138 v = iap_reload_count_to_perfctr_value(pm->pm_sc.pm_reloadcount);
1140 PMCDBG3(MDP,INT, 1, "iap-intr cpu=%d error=%d v=%jx", cpu, error,
1156 * Reenable all non-stalled PMCs.
1160 cc->pc_globalctrl &= ~intrdisable;
1161 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
1164 cc->pc_globalctrl &= ~intrdisable;
1165 wrmsr(IA_GLOBAL_CTRL, cc->pc_globalctrl);
1170 PMCDBG4(MDP, INT, 1, "cpu=%d fixedctrl=%jx globalctrl=%jx status=%jx",
1171 cpu, (uintmax_t) rdmsr(IAF_CTRL),
1186 core_cputype = md->pmd_cputype;
1190 PMCDBG3(MDP,INI,1,"core-init cputype=%d ncpu=%d version=%d",
1205 "core-init full-width write supported");
1206 core_iap_wroffset = IAP_A_PMC0 - IAP_PMC0;
1209 "core-init full-width write NOT supported");
1211 PMCDBG0(MDP, INI, 1, "core-init pdcm not supported");
1221 core_pmcmask |= ((1ULL << core_iap_npmc) - 1);
1224 flags = cpuid[CORE_CPUID_EBX] & ((1 << nflags) - 1);
1237 core_pmcmask |= ((1ULL << core_iaf_npmc) - 1) << IAF_OFFSET;
1240 PMCDBG2(MDP,INI,1,"core-init pmcmask=0x%jx iafri=%d", core_pmcmask,
1250 md->pmd_intr = core2_intr;
1252 md->pmd_intr = core_intr;
1260 PMCDBG0(MDP,INI,1, "core-finalize");
1264 ("[core,%d] non-null pcpu cpu %d", __LINE__, i));