1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause
41 #include <machine/cpu.h>
80 uncore_pcpu_noop(struct pmc_mdep *md, int cpu)  in uncore_pcpu_noop()  argument
83 	(void) cpu;  in uncore_pcpu_noop()
88 uncore_pcpu_init(struct pmc_mdep *md, int cpu)  in uncore_pcpu_init()  argument
95 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in uncore_pcpu_init()
96 	    ("[ucf,%d] insane cpu number %d", __LINE__, cpu));  in uncore_pcpu_init()
98 	PMCDBG1(MDP,INI,1,"uncore-init cpu=%d", cpu);  in uncore_pcpu_init()
100 	uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri;  in uncore_pcpu_init()
101 	npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num;  in uncore_pcpu_init()
102 	npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num;  in uncore_pcpu_init()
107 	uncore_pcpu[cpu] = cc;  in uncore_pcpu_init()
108 	pc = pmc_pcpu[cpu];  in uncore_pcpu_init()
111 	    ("[uncore,%d] NULL per-cpu structures cpu=%d", __LINE__, cpu));  in uncore_pcpu_init()
113 	for (n = 0, phw = cc->pc_uncorepmcs; n < npmc; n++, phw++) {  in uncore_pcpu_init()
114 		phw->phw_state 	  = PMC_PHW_FLAG_IS_ENABLED |  in uncore_pcpu_init()
115 		    PMC_PHW_CPU_TO_STATE(cpu) |  in uncore_pcpu_init()
117 		phw->phw_pmc	  = NULL;  in uncore_pcpu_init()
118 		pc->pc_hwpmcs[n + uncore_ri]  = phw;  in uncore_pcpu_init()
125 uncore_pcpu_fini(struct pmc_mdep *md, int cpu)  in uncore_pcpu_fini()  argument
131 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in uncore_pcpu_fini()
132 	    ("[uncore,%d] insane cpu number (%d)", __LINE__, cpu));  in uncore_pcpu_fini()
134 	PMCDBG1(MDP,INI,1,"uncore-pcpu-fini cpu=%d", cpu);  in uncore_pcpu_fini()
136 	if ((cc = uncore_pcpu[cpu]) == NULL)  in uncore_pcpu_fini()
139 	uncore_pcpu[cpu] = NULL;  in uncore_pcpu_fini()
141 	pc = pmc_pcpu[cpu];  in uncore_pcpu_fini()
143 	KASSERT(pc != NULL, ("[uncore,%d] NULL per-cpu %d state", __LINE__,  in uncore_pcpu_fini()
144 		cpu));  in uncore_pcpu_fini()
146 	npmc = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_num;  in uncore_pcpu_fini()
147 	uncore_ri = md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP].pcd_ri;  in uncore_pcpu_fini()
153 	npmc += md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF].pcd_num;  in uncore_pcpu_fini()
156 		pc->pc_hwpmcs[n + uncore_ri] = NULL;  in uncore_pcpu_fini()
172 	if ((v & (1ULL << (uncore_ucf_width - 1))) == 0)  in ucf_perfctr_value_to_reload_count()
174 	v &= (1ULL << uncore_ucf_width) - 1;  in ucf_perfctr_value_to_reload_count()
175 	return (1ULL << uncore_ucf_width) - v;  in ucf_perfctr_value_to_reload_count()
181 	return (1ULL << uncore_ucf_width) - rlc;  in ucf_reload_count_to_perfctr_value()
185 ucf_allocate_pmc(int cpu, int ri, struct pmc *pm,  in ucf_allocate_pmc()  argument
190 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in ucf_allocate_pmc()
191 	    ("[uncore,%d] illegal CPU %d", __LINE__, cpu));  in ucf_allocate_pmc()
193 	PMCDBG2(MDP,ALL,1, "ucf-allocate ri=%d reqcaps=0x%x", ri, pm->pm_caps);  in ucf_allocate_pmc()
198 	if (a->pm_class != PMC_CLASS_UCF)  in ucf_allocate_pmc()
201 	if ((a->pm_flags & PMC_F_EV_PMU) == 0)  in ucf_allocate_pmc()
206 	pm->pm_md.pm_ucf.pm_ucf_ctrl = (flags << (ri * 4));  in ucf_allocate_pmc()
208 	PMCDBG1(MDP,ALL,2, "ucf-allocate config=0x%jx",  in ucf_allocate_pmc()
209 	    (uintmax_t) pm->pm_md.pm_ucf.pm_ucf_ctrl);  in ucf_allocate_pmc()
215 ucf_config_pmc(int cpu, int ri, struct pmc *pm)  in ucf_config_pmc()  argument
217 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in ucf_config_pmc()
218 	    ("[uncore,%d] illegal CPU %d", __LINE__, cpu));  in ucf_config_pmc()
221 	    ("[uncore,%d] illegal row-index %d", __LINE__, ri));  in ucf_config_pmc()
223 	PMCDBG3(MDP,CFG,1, "ucf-config cpu=%d ri=%d pm=%p", cpu, ri, pm);  in ucf_config_pmc()
225 	KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__,  in ucf_config_pmc()
226 	    cpu));  in ucf_config_pmc()
228 	uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc = pm;  in ucf_config_pmc()
234 ucf_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)  in ucf_describe()  argument
238 	phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri];  in ucf_describe()
240 	snprintf(pi->pm_name, sizeof(pi->pm_name), "UCF-%d", ri);  in ucf_describe()
241 	pi->pm_class = PMC_CLASS_UCF;  in ucf_describe()
243 	if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {  in ucf_describe()
244 		pi->pm_enabled = TRUE;  in ucf_describe()
245 		*ppmc          = phw->phw_pmc;  in ucf_describe()
247 		pi->pm_enabled = FALSE;  in ucf_describe()
255 ucf_get_config(int cpu, int ri, struct pmc **ppm)  in ucf_get_config()  argument
257 	*ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc;  in ucf_get_config()
263 ucf_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)  in ucf_read_pmc()  argument
267 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in ucf_read_pmc()
268 	    ("[uncore,%d] illegal cpu value %d", __LINE__, cpu));  in ucf_read_pmc()
270 	    ("[uncore,%d] illegal row-index %d", __LINE__, ri));  in ucf_read_pmc()
279 	PMCDBG3(MDP,REA,1, "ucf-read cpu=%d ri=%d -> v=%jx", cpu, ri, *v);  in ucf_read_pmc()
285 ucf_release_pmc(int cpu, int ri, struct pmc *pmc)  in ucf_release_pmc()  argument
287 	PMCDBG3(MDP,REL,1, "ucf-release cpu=%d ri=%d pm=%p", cpu, ri, pmc);  in ucf_release_pmc()
289 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in ucf_release_pmc()
290 	    ("[uncore,%d] illegal CPU value %d", __LINE__, cpu));  in ucf_release_pmc()
292 	    ("[uncore,%d] illegal row-index %d", __LINE__, ri));  in ucf_release_pmc()
294 	KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri + uncore_ucf_ri].phw_pmc == NULL,  in ucf_release_pmc()
295 	    ("[uncore,%d] PHW pmc non-NULL", __LINE__));  in ucf_release_pmc()
301 ucf_start_pmc(int cpu, int ri, struct pmc *pm)  in ucf_start_pmc()  argument
305 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in ucf_start_pmc()
306 	    ("[uncore,%d] illegal CPU value %d", __LINE__, cpu));  in ucf_start_pmc()
308 	    ("[uncore,%d] illegal row-index %d", __LINE__, ri));  in ucf_start_pmc()
310 	PMCDBG2(MDP,STA,1,"ucf-start cpu=%d ri=%d", cpu, ri);  in ucf_start_pmc()
312 	ucfc = uncore_pcpu[cpu];  in ucf_start_pmc()
313 	ucfc->pc_ucfctrl |= pm->pm_md.pm_ucf.pm_ucf_ctrl;  in ucf_start_pmc()
315 	wrmsr(UCF_CTRL, ucfc->pc_ucfctrl);  in ucf_start_pmc()
317 	ucfc->pc_globalctrl |= (1ULL << (ri + SELECTOFF(uncore_cputype)));  in ucf_start_pmc()
318 	wrmsr(UC_GLOBAL_CTRL, ucfc->pc_globalctrl);  in ucf_start_pmc()
321 	    ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL),  in ucf_start_pmc()
322 	    ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL));  in ucf_start_pmc()
328 ucf_stop_pmc(int cpu, int ri, struct pmc *pm __unused)  in ucf_stop_pmc()  argument
333 	PMCDBG2(MDP,STO,1,"ucf-stop cpu=%d ri=%d", cpu, ri);  in ucf_stop_pmc()
335 	ucfc = uncore_pcpu[cpu];  in ucf_stop_pmc()
337 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in ucf_stop_pmc()
338 	    ("[uncore,%d] illegal CPU value %d", __LINE__, cpu));  in ucf_stop_pmc()
340 	    ("[uncore,%d] illegal row-index %d", __LINE__, ri));  in ucf_stop_pmc()
344 	ucfc->pc_ucfctrl &= ~fc;  in ucf_stop_pmc()
346 	PMCDBG1(MDP,STO,1,"ucf-stop ucfctrl=%x", ucfc->pc_ucfctrl);  in ucf_stop_pmc()
347 	wrmsr(UCF_CTRL, ucfc->pc_ucfctrl);  in ucf_stop_pmc()
352 	    ucfc->pc_ucfctrl, (uint32_t) rdmsr(UCF_CTRL),  in ucf_stop_pmc()
353 	    ucfc->pc_globalctrl, rdmsr(UC_GLOBAL_CTRL));  in ucf_stop_pmc()
359 ucf_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)  in ucf_write_pmc()  argument
363 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in ucf_write_pmc()
364 	    ("[uncore,%d] illegal cpu value %d", __LINE__, cpu));  in ucf_write_pmc()
366 	    ("[uncore,%d] illegal row-index %d", __LINE__, ri));  in ucf_write_pmc()
368 	cc = uncore_pcpu[cpu];  in ucf_write_pmc()
375 	wrmsr(UCF_CTRL, cc->pc_ucfctrl);  in ucf_write_pmc()
377 	PMCDBG4(MDP,WRI,1, "ucf-write cpu=%d ri=%d v=%jx ucfctrl=%jx ",  in ucf_write_pmc()
378 	    cpu, ri, v, (uintmax_t) rdmsr(UCF_CTRL));  in ucf_write_pmc()
391 	PMCDBG0(MDP,INI,1, "ucf-initialize");  in ucf_initialize()
393 	pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCF];  in ucf_initialize()
395 	pcd->pcd_caps	= UCF_PMC_CAPS;  in ucf_initialize()
396 	pcd->pcd_class	= PMC_CLASS_UCF;  in ucf_initialize()
397 	pcd->pcd_num	= npmc;  in ucf_initialize()
398 	pcd->pcd_ri	= md->pmd_npmc;  in ucf_initialize()
399 	pcd->pcd_width	= pmcwidth;  in ucf_initialize()
401 	pcd->pcd_allocate_pmc	= ucf_allocate_pmc;  in ucf_initialize()
402 	pcd->pcd_config_pmc	= ucf_config_pmc;  in ucf_initialize()
403 	pcd->pcd_describe	= ucf_describe;  in ucf_initialize()
404 	pcd->pcd_get_config	= ucf_get_config;  in ucf_initialize()
405 	pcd->pcd_get_msr	= NULL;  in ucf_initialize()
406 	pcd->pcd_pcpu_fini	= uncore_pcpu_noop;  in ucf_initialize()
407 	pcd->pcd_pcpu_init	= uncore_pcpu_noop;  in ucf_initialize()
408 	pcd->pcd_read_pmc	= ucf_read_pmc;  in ucf_initialize()
409 	pcd->pcd_release_pmc	= ucf_release_pmc;  in ucf_initialize()
410 	pcd->pcd_start_pmc	= ucf_start_pmc;  in ucf_initialize()
411 	pcd->pcd_stop_pmc	= ucf_stop_pmc;  in ucf_initialize()
412 	pcd->pcd_write_pmc	= ucf_write_pmc;  in ucf_initialize()
414 	md->pmd_npmc	       += npmc;  in ucf_initialize()
440 #define	UCP_F_I7	(1 << 0)	/* CPU: Core i7 */
441 #define	UCP_F_WM	(1 << 1)	/* CPU: Westmere */
442 #define	UCP_F_SB	(1 << 2)	/* CPU: Sandy Bridge */
443 #define	UCP_F_HW	(1 << 3)	/* CPU: Haswell */
454 	v &= (1ULL << uncore_ucp_width) - 1;  in ucp_perfctr_value_to_reload_count()
455 	return (1ULL << uncore_ucp_width) - v;  in ucp_perfctr_value_to_reload_count()
461 	return (1ULL << uncore_ucp_width) - rlc;  in ucp_reload_count_to_perfctr_value()
489 ucp_allocate_pmc(int cpu, int ri, struct pmc *pm,  in ucp_allocate_pmc()  argument
495 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in ucp_allocate_pmc()
496 	    ("[uncore,%d] illegal CPU %d", __LINE__, cpu));  in ucp_allocate_pmc()
498 	    ("[uncore,%d] illegal row-index value %d", __LINE__, ri));  in ucp_allocate_pmc()
500 	if (a->pm_class != PMC_CLASS_UCP)  in ucp_allocate_pmc()
503 	if ((a->pm_flags & PMC_F_EV_PMU) == 0)  in ucp_allocate_pmc()
506 	ucp = &a->pm_md.pm_ucp;  in ucp_allocate_pmc()
507 	ev = UCP_EVSEL(ucp->pm_ucp_config);  in ucp_allocate_pmc()
518 	pm->pm_md.pm_ucp.pm_ucp_evsel = ucp->pm_ucp_config | UCP_EN;  in ucp_allocate_pmc()
524 ucp_config_pmc(int cpu, int ri, struct pmc *pm)  in ucp_config_pmc()  argument
526 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in ucp_config_pmc()
527 	    ("[uncore,%d] illegal CPU %d", __LINE__, cpu));  in ucp_config_pmc()
530 	    ("[uncore,%d] illegal row-index %d", __LINE__, ri));  in ucp_config_pmc()
532 	PMCDBG3(MDP,CFG,1, "ucp-config cpu=%d ri=%d pm=%p", cpu, ri, pm);  in ucp_config_pmc()
534 	KASSERT(uncore_pcpu[cpu] != NULL, ("[uncore,%d] null per-cpu %d", __LINE__,  in ucp_config_pmc()
535 	    cpu));  in ucp_config_pmc()
537 	uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc = pm;  in ucp_config_pmc()
543 ucp_describe(int cpu, int ri, struct pmc_info *pi, struct pmc **ppmc)  in ucp_describe()  argument
547 	phw = &uncore_pcpu[cpu]->pc_uncorepmcs[ri];  in ucp_describe()
549 	snprintf(pi->pm_name, sizeof(pi->pm_name), "UCP-%d", ri);  in ucp_describe()
550 	pi->pm_class = PMC_CLASS_UCP;  in ucp_describe()
552 	if (phw->phw_state & PMC_PHW_FLAG_IS_ENABLED) {  in ucp_describe()
553 		pi->pm_enabled = TRUE;  in ucp_describe()
554 		*ppmc          = phw->phw_pmc;  in ucp_describe()
556 		pi->pm_enabled = FALSE;  in ucp_describe()
564 ucp_get_config(int cpu, int ri, struct pmc **ppm)  in ucp_get_config()  argument
566 	*ppm = uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc;  in ucp_get_config()
572 ucp_read_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t *v)  in ucp_read_pmc()  argument
576 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in ucp_read_pmc()
577 	    ("[uncore,%d] illegal cpu value %d", __LINE__, cpu));  in ucp_read_pmc()
579 	    ("[uncore,%d] illegal row-index %d", __LINE__, ri));  in ucp_read_pmc()
587 	PMCDBG4(MDP,REA,1, "ucp-read cpu=%d ri=%d msr=0x%x -> v=%jx", cpu, ri,  in ucp_read_pmc()
594 ucp_release_pmc(int cpu, int ri, struct pmc *pm)  in ucp_release_pmc()  argument
598 	PMCDBG3(MDP,REL,1, "ucp-release cpu=%d ri=%d pm=%p", cpu, ri,  in ucp_release_pmc()
601 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in ucp_release_pmc()
602 	    ("[uncore,%d] illegal CPU value %d", __LINE__, cpu));  in ucp_release_pmc()
604 	    ("[uncore,%d] illegal row-index %d", __LINE__, ri));  in ucp_release_pmc()
606 	KASSERT(uncore_pcpu[cpu]->pc_uncorepmcs[ri].phw_pmc  in ucp_release_pmc()
607 	    == NULL, ("[uncore,%d] PHW pmc non-NULL", __LINE__));  in ucp_release_pmc()
613 ucp_start_pmc(int cpu, int ri, struct pmc *pm)  in ucp_start_pmc()  argument
618 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in ucp_start_pmc()
619 	    ("[uncore,%d] illegal CPU value %d", __LINE__, cpu));  in ucp_start_pmc()
621 	    ("[uncore,%d] illegal row-index %d", __LINE__, ri));  in ucp_start_pmc()
623 	cc = uncore_pcpu[cpu];  in ucp_start_pmc()
625 	PMCDBG2(MDP,STA,1, "ucp-start cpu=%d ri=%d", cpu, ri);  in ucp_start_pmc()
627 	evsel = pm->pm_md.pm_ucp.pm_ucp_evsel;  in ucp_start_pmc()
630 	    "ucp-start/2 cpu=%d ri=%d evselmsr=0x%x evsel=0x%x",  in ucp_start_pmc()
631 	    cpu, ri, SELECTSEL(uncore_cputype) + ri, evsel);  in ucp_start_pmc()
635 	cc->pc_globalctrl |= (1ULL << ri);  in ucp_start_pmc()
636 	wrmsr(UC_GLOBAL_CTRL, cc->pc_globalctrl);  in ucp_start_pmc()
642 ucp_stop_pmc(int cpu, int ri, struct pmc *pm __unused)  in ucp_stop_pmc()  argument
645 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in ucp_stop_pmc()
646 	    ("[uncore,%d] illegal cpu value %d", __LINE__, cpu));  in ucp_stop_pmc()
650 	PMCDBG2(MDP,STO,1, "ucp-stop cpu=%d ri=%d", cpu, ri);  in ucp_stop_pmc()
661 ucp_write_pmc(int cpu, int ri, struct pmc *pm, pmc_value_t v)  in ucp_write_pmc()  argument
664 	KASSERT(cpu >= 0 && cpu < pmc_cpu_max(),  in ucp_write_pmc()
665 	    ("[uncore,%d] illegal cpu value %d", __LINE__, cpu));  in ucp_write_pmc()
669 	PMCDBG4(MDP,WRI,1, "ucp-write cpu=%d ri=%d msr=0x%x v=%jx", cpu, ri,  in ucp_write_pmc()
693 	PMCDBG0(MDP,INI,1, "ucp-initialize");  in ucp_initialize()
695 	pcd = &md->pmd_classdep[PMC_MDEP_CLASS_INDEX_UCP];  in ucp_initialize()
697 	pcd->pcd_caps	= UCP_PMC_CAPS;  in ucp_initialize()
698 	pcd->pcd_class	= PMC_CLASS_UCP;  in ucp_initialize()
699 	pcd->pcd_num	= npmc;  in ucp_initialize()
700 	pcd->pcd_ri	= md->pmd_npmc;  in ucp_initialize()
701 	pcd->pcd_width	= pmcwidth;  in ucp_initialize()
703 	pcd->pcd_allocate_pmc	= ucp_allocate_pmc;  in ucp_initialize()
704 	pcd->pcd_config_pmc	= ucp_config_pmc;  in ucp_initialize()
705 	pcd->pcd_describe	= ucp_describe;  in ucp_initialize()
706 	pcd->pcd_get_config	= ucp_get_config;  in ucp_initialize()
707 	pcd->pcd_get_msr	= NULL;  in ucp_initialize()
708 	pcd->pcd_pcpu_fini	= uncore_pcpu_fini;  in ucp_initialize()
709 	pcd->pcd_pcpu_init	= uncore_pcpu_init;  in ucp_initialize()
710 	pcd->pcd_read_pmc	= ucp_read_pmc;  in ucp_initialize()
711 	pcd->pcd_release_pmc	= ucp_release_pmc;  in ucp_initialize()
712 	pcd->pcd_start_pmc	= ucp_start_pmc;  in ucp_initialize()
713 	pcd->pcd_stop_pmc	= ucp_stop_pmc;  in ucp_initialize()
714 	pcd->pcd_write_pmc	= ucp_write_pmc;  in ucp_initialize()
716 	md->pmd_npmc	       += npmc;  in ucp_initialize()
722 	uncore_cputype = md->pmd_cputype;  in pmc_uncore_initialize()
732 	uncore_pmcmask |= ((1ULL << uncore_ucp_npmc) - 1);  in pmc_uncore_initialize()
744 	uncore_pmcmask |= ((1ULL << uncore_ucf_npmc) - 1) << SELECTOFF(uncore_cputype);  in pmc_uncore_initialize()
746 	PMCDBG2(MDP,INI,1,"uncore-init pmcmask=0x%jx ucfri=%d", uncore_pmcmask,  in pmc_uncore_initialize()
758 	PMCDBG0(MDP,INI,1, "uncore-finalize");  in pmc_uncore_finalize()
762 		    ("[uncore,%d] non-null pcpu cpu %d", __LINE__, i));  in pmc_uncore_finalize()