211 static void	cpu_boot_secondary(struct cpu_info *ci);
212 static void	cpu_start_secondary(struct cpu_info *ci);
248 cpu_pcpuarea_init(struct cpu_info *ci)  in cpu_pcpuarea_init()  argument
258 	base = (vaddr_t)&pcpuarea->ent[cpu_index(ci)];  in cpu_pcpuarea_init()
277 cpu_vm_init(struct cpu_info *ci)  in cpu_vm_init()  argument
291 		cai = &ci->ci_cinfo[i];  in cpu_vm_init()
331 	aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors);  in cpu_vm_init()
334 	pmap_tlb_cpu_init(ci);  in cpu_vm_init()
336 	pmap_vpage_cpu_init(ci);  in cpu_vm_init()
345 	struct cpu_info *ci;  in cpu_attach()  local
377 		    sizeof(*ci) + CACHE_LINE_SIZE - 1, 0,  in cpu_attach()
379 		ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);  in cpu_attach()
380 		ci->ci_curldt = -1;  in cpu_attach()
384 		ci = &cpu_info_primary;  in cpu_attach()
403 	ci->ci_self = ci;  in cpu_attach()
404 	sc->sc_info = ci;  in cpu_attach()
405 	ci->ci_dev = self;  in cpu_attach()
406 	ci->ci_acpiid = caa->cpu_id;  in cpu_attach()
407 	ci->ci_cpuid = caa->cpu_number;  in cpu_attach()
408 	ci->ci_func = caa->cpu_func;  in cpu_attach()
409 	ci->ci_kfpu_spl = -1;  in cpu_attach()
413 	cpu_vm_init(ci);  in cpu_attach()
418 		error = mi_cpu_attach(ci);  in cpu_attach()
425 		cpu_pcpuarea_init(ci);  in cpu_attach()
427 		cpu_init_tss(ci);  in cpu_attach()
429 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);  in cpu_attach()
437 	cpu_svs_init(ci);  in cpu_attach()
441 	ci->ci_pmap = pmap_kernel();  in cpu_attach()
442 	ci->ci_tlbstate = TLBSTATE_STALE;  in cpu_attach()
456 		atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);  in cpu_attach()
457 		cpu_intr_init(ci);  in cpu_attach()
458 		tsc_setfunc(ci);  in cpu_attach()
459 		cpu_get_tsc_freq(ci);  in cpu_attach()
460 		cpu_init(ci);  in cpu_attach()
462 		cpu_set_tss_gates(ci);  in cpu_attach()
464 		pmap_cpu_init_late(ci);  in cpu_attach()
474 		kcsan_cpu_init(ci);  in cpu_attach()
482 		atomic_or_32(&ci->ci_flags, CPUF_SP);  in cpu_attach()
483 		cpu_identify(ci);  in cpu_attach()
487 		xen_hvm_init_cpu(ci);  in cpu_attach()
492 		atomic_or_32(&ci->ci_flags, CPUF_BSP);  in cpu_attach()
493 		cpu_identify(ci);  in cpu_attach()
497 		xen_hvm_init_cpu(ci);  in cpu_attach()
506 		cpu_intr_init(ci);  in cpu_attach()
507 		idt_vec_init_cpu_md(&ci->ci_idtvec, cpu_index(ci));  in cpu_attach()
508 		gdt_alloc_cpu(ci);  in cpu_attach()
510 		cpu_set_tss_gates(ci);  in cpu_attach()
512 		pmap_cpu_init_late(ci);  in cpu_attach()
513 		cpu_start_secondary(ci);  in cpu_attach()
514 		if (ci->ci_flags & CPUF_PRESENT) {  in cpu_attach()
517 			cpu_identify(ci);  in cpu_attach()
522 			tmp->ci_next = ci;  in cpu_attach()
531 	pat_init(ci);  in cpu_attach()
538 		struct lwp *l = ci->ci_data.cpu_idlelwp;  in cpu_attach()
572 	struct cpu_info *ci = sc->sc_info;  in cpu_rescan()  local
581 	if (ci == NULL)  in cpu_rescan()
585 	cfaa.ci = ci;  in cpu_rescan()
588 		if (ci->ci_frequency == NULL) {  in cpu_rescan()
590 			ci->ci_frequency =  in cpu_rescan()
595 		if (ci->ci_padlock == NULL) {  in cpu_rescan()
597 			ci->ci_padlock =  in cpu_rescan()
602 		if (ci->ci_temperature == NULL) {  in cpu_rescan()
604 			ci->ci_temperature =  in cpu_rescan()
609 		if (ci->ci_vm == NULL) {  in cpu_rescan()
611 			ci->ci_vm =  in cpu_rescan()
624 	struct cpu_info *ci = sc->sc_info;  in cpu_childdetached()  local
626 	if (ci->ci_frequency == child)  in cpu_childdetached()
627 		ci->ci_frequency = NULL;  in cpu_childdetached()
629 	if (ci->ci_padlock == child)  in cpu_childdetached()
630 		ci->ci_padlock = NULL;  in cpu_childdetached()
632 	if (ci->ci_temperature == child)  in cpu_childdetached()
633 		ci->ci_temperature = NULL;  in cpu_childdetached()
635 	if (ci->ci_vm == child)  in cpu_childdetached()
636 		ci->ci_vm = NULL;  in cpu_childdetached()
644 cpu_init(struct cpu_info *ci)  in cpu_init()  argument
701 		ci->ci_feat_val[1] = descs[2];  in cpu_init()
704 	if (CPU_IS_PRIMARY(ci) &&  in cpu_init()
718 		if ((ci->ci_flags & CPUF_AP) == 0)  in cpu_init()
720 		mtrr_init_cpu(ci);  in cpu_init()
724 	if (strcmp((char *)(ci->ci_vendor), "AuthenticAMD") == 0) {  in cpu_init()
728 		if (CPUID_TO_FAMILY(ci->ci_signature) == 5) {  in cpu_init()
729 			if (CPUID_TO_MODEL(ci->ci_signature) > 8 ||  in cpu_init()
730 			    (CPUID_TO_MODEL(ci->ci_signature) == 8 &&  in cpu_init()
731 			     CPUID_TO_STEPPING(ci->ci_signature) >= 7)) {  in cpu_init()
734 				mtrr_init_cpu(ci);  in cpu_init()
741 	if (ci != &cpu_info_primary) {  in cpu_init()
743 		atomic_or_32(&ci->ci_flags, CPUF_RUNNING);  in cpu_init()
744 		tsc_sync_ap(ci);  in cpu_init()
746 		atomic_or_32(&ci->ci_flags, CPUF_RUNNING);  in cpu_init()
754 	struct cpu_info *ci;  in cpu_boot_secondary_processors()  local
769 		ci = cpu_lookup(i);  in cpu_boot_secondary_processors()
770 		if (ci == NULL)  in cpu_boot_secondary_processors()
772 		if (ci->ci_data.cpu_idlelwp == NULL)  in cpu_boot_secondary_processors()
774 		if ((ci->ci_flags & CPUF_PRESENT) == 0)  in cpu_boot_secondary_processors()
776 		if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))  in cpu_boot_secondary_processors()
778 		cpu_boot_secondary(ci);  in cpu_boot_secondary_processors()
779 		kcpuset_set(cpus, cpu_index(ci));  in cpu_boot_secondary_processors()
793 cpu_init_idle_lwp(struct cpu_info *ci)  in cpu_init_idle_lwp()  argument
795 	struct lwp *l = ci->ci_data.cpu_idlelwp;  in cpu_init_idle_lwp()
804 	struct cpu_info *ci;  in cpu_init_idle_lwps()  local
808 		ci = cpu_lookup(i);  in cpu_init_idle_lwps()
809 		if (ci == NULL)  in cpu_init_idle_lwps()
811 		if (ci->ci_data.cpu_idlelwp == NULL)  in cpu_init_idle_lwps()
813 		if ((ci->ci_flags & CPUF_PRESENT) == 0)  in cpu_init_idle_lwps()
815 		cpu_init_idle_lwp(ci);  in cpu_init_idle_lwps()
821 cpu_start_secondary(struct cpu_info *ci)  in cpu_start_secondary()  argument
832 	atomic_or_32(&ci->ci_flags, CPUF_AP);  in cpu_start_secondary()
833 	ci->ci_curlwp = ci->ci_data.cpu_idlelwp;  in cpu_start_secondary()
834 	if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) {  in cpu_start_secondary()
843 	cpu_starting = ci;  in cpu_start_secondary()
844 	for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {  in cpu_start_secondary()
848 	if ((ci->ci_flags & CPUF_PRESENT) == 0) {  in cpu_start_secondary()
849 		aprint_error_dev(ci->ci_dev, "failed to become ready\n");  in cpu_start_secondary()
863 		tsc_sync_bp(ci);  in cpu_start_secondary()
867 	CPU_START_CLEANUP(ci);  in cpu_start_secondary()
872 cpu_boot_secondary(struct cpu_info *ci)  in cpu_boot_secondary()  argument
878 	atomic_or_32(&ci->ci_flags, CPUF_GO);  in cpu_boot_secondary()
879 	for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {  in cpu_boot_secondary()
882 	if ((ci->ci_flags & CPUF_RUNNING) == 0) {  in cpu_boot_secondary()
883 		aprint_error_dev(ci->ci_dev, "failed to start\n");  in cpu_boot_secondary()
890 		drift = ci->ci_data.cpu_cc_skew;  in cpu_boot_secondary()
893 		tsc_sync_bp(ci);  in cpu_boot_secondary()
895 		drift -= ci->ci_data.cpu_cc_skew;  in cpu_boot_secondary()
896 		aprint_debug_dev(ci->ci_dev, "TSC skew=%lld drift=%lld\n",  in cpu_boot_secondary()
897 		    (long long)ci->ci_data.cpu_cc_skew, (long long)drift);  in cpu_boot_secondary()
911 	struct cpu_info *ci = (struct cpu_info *)v;  in cpu_hatch()  local
943 	cpu_init_msrs(ci, true);  in cpu_hatch()
945 	cpu_probe(ci);  in cpu_hatch()
946 	cpu_speculation_init(ci);  in cpu_hatch()
948 	hyperv_init_cpu(ci);  in cpu_hatch()
951 	ci->ci_data.cpu_cc_freq = cpu_info_primary.ci_data.cpu_cc_freq;  in cpu_hatch()
952 	/* cpu_get_tsc_freq(ci); */  in cpu_hatch()
954 	KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0);  in cpu_hatch()
960 	atomic_or_32(&ci->ci_flags, CPUF_PRESENT);  in cpu_hatch()
961 	tsc_sync_ap(ci);  in cpu_hatch()
978 	while ((ci->ci_flags & CPUF_GO) == 0) {  in cpu_hatch()
980 			x86_monitor(&ci->ci_flags, 0, 0);  in cpu_hatch()
981 			if ((ci->ci_flags & CPUF_GO) != 0) {  in cpu_hatch()
1003 	KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);  in cpu_hatch()
1006 	pd_entry_t * l3_pd = ci->ci_pae_l3_pdir;  in cpu_hatch()
1010 	lcr3(ci->ci_pae_l3_pdirpa);  in cpu_hatch()
1017 	pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp);  in cpu_hatch()
1020 	cpu_init_idt(ci);  in cpu_hatch()
1021 	gdt_init_cpu(ci);  in cpu_hatch()
1027 	fpuinit(ci);  in cpu_hatch()
1029 	ltr(ci->ci_tss_sel);  in cpu_hatch()
1036 	cpu_init(ci);  in cpu_hatch()
1038 	xen_hvm_init_cpu(ci);  in cpu_hatch()
1041 	cpu_get_tsc_freq(ci);  in cpu_hatch()
1051 	aprint_debug_dev(ci->ci_dev, "running\n");  in cpu_hatch()
1053 	kcsan_cpu_init(ci);  in cpu_hatch()
1071 	struct cpu_info *ci;  in cpu_debug_dump()  local
1081 	for (CPU_INFO_FOREACH(cii, ci)) {  in cpu_debug_dump()
1083 		    ci,  in cpu_debug_dump()
1084 		    ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),  in cpu_debug_dump()
1085 		    (long)ci->ci_cpuid,  in cpu_debug_dump()
1086 		    ci->ci_flags, ci->ci_ipis, ci->ci_ilevel,  in cpu_debug_dump()
1087 		    ci->ci_curlwp);  in cpu_debug_dump()
1135 mp_cpu_start(struct cpu_info *ci, paddr_t target)  in mp_cpu_start()  argument
1175 	if (ci->ci_flags & CPUF_AP) {  in mp_cpu_start()
1176 		error = x86_ipi_init(ci->ci_cpuid);  in mp_cpu_start()
1178 			aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n",  in mp_cpu_start()
1184 		error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE);  in mp_cpu_start()
1186 			aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n",  in mp_cpu_start()
1192 		error = x86_ipi_startup(ci->ci_cpuid, target / PAGE_SIZE);  in mp_cpu_start()
1194 			aprint_error_dev(ci->ci_dev, "%s: IPI not taken (3)\n",  in mp_cpu_start()
1208 mp_cpu_start_cleanup(struct cpu_info *ci)  in mp_cpu_start_cleanup()  argument
1225  * cpu_init_msrs(ci, full)
1228  *	CPU, whose struct cpu_info pointer is ci, for:
1249 cpu_init_msrs(struct cpu_info *ci, bool full)  in cpu_init_msrs()  argument
1294 	 *   curlwp for access to the CPU-local area, so set it to ci.  in cpu_init_msrs()
1303 		wrmsr(MSR_GSBASE, (uint64_t)ci);  in cpu_init_msrs()
1329 	struct cpu_info *ci = sc->sc_info;  in cpu_stop()  local
1332 	KASSERT((ci->ci_flags & CPUF_PRESENT) != 0);  in cpu_stop()
1334 	if (CPU_IS_PRIMARY(ci))  in cpu_stop()
1337 	if (ci->ci_data.cpu_idlelwp == NULL)  in cpu_stop()
1340 	sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE);  in cpu_stop()
1344 		err = cpu_setstate(ci, false);  in cpu_stop()
1358 	struct cpu_info *ci = sc->sc_info;  in cpu_suspend()  local
1360 	if ((ci->ci_flags & CPUF_PRESENT) == 0)  in cpu_suspend()
1363 		cpufreq_suspend(ci);  in cpu_suspend()
1373 	struct cpu_info *ci = sc->sc_info;  in cpu_resume()  local
1376 	if ((ci->ci_flags & CPUF_PRESENT) == 0)  in cpu_resume()
1379 	if (CPU_IS_PRIMARY(ci))  in cpu_resume()
1382 	if (ci->ci_data.cpu_idlelwp == NULL)  in cpu_resume()
1387 		err = cpu_setstate(ci, true);  in cpu_resume()
1395 	cpufreq_resume(ci);  in cpu_resume()
1404 	struct cpu_info *ci = sc->sc_info;  in cpu_shutdown()  local
1406 	if ((ci->ci_flags & CPUF_BSP) != 0)  in cpu_shutdown()
1409 	if ((ci->ci_flags & CPUF_PRESENT) == 0)  in cpu_shutdown()
1415 /* Get the TSC frequency and set it to ci->ci_data.cpu_cc_freq. */
1417 cpu_get_tsc_freq(struct cpu_info *ci)  in cpu_get_tsc_freq()  argument
1422 	if (CPU_IS_PRIMARY(ci) && cpu_hascounter()) {  in cpu_get_tsc_freq()
1430 		if (ci->ci_data.cpu_cc_freq == 0)  in cpu_get_tsc_freq()
1431 			freq = freq_from_cpuid = cpu_tsc_freq_cpuid(ci);  in cpu_get_tsc_freq()
1433 			aprint_debug_dev(ci->ci_dev, "TSC freq "  in cpu_get_tsc_freq()
1439 				aprint_debug_dev(ci->ci_dev, "TSC freq "  in cpu_get_tsc_freq()
1476 			aprint_debug_dev(ci->ci_dev, "TSC freq "  in cpu_get_tsc_freq()
1479 		if (ci->ci_data.cpu_cc_freq != 0) {  in cpu_get_tsc_freq()
1480 			freq_from_cpuid = cpu_tsc_freq_cpuid(ci);  in cpu_get_tsc_freq()
1483 				aprint_verbose_dev(ci->ci_dev, "TSC freq "  in cpu_get_tsc_freq()
1490 	ci->ci_data.cpu_cc_freq = freq;  in cpu_get_tsc_freq()
1496 	struct cpu_info *ci = curcpu();  in x86_cpu_idle_mwait()  local
1498 	KASSERT(ci->ci_ilevel == IPL_NONE);  in x86_cpu_idle_mwait()
1500 	x86_monitor(&ci->ci_want_resched, 0, 0);  in x86_cpu_idle_mwait()
1501 	if (__predict_false(ci->ci_want_resched)) {  in x86_cpu_idle_mwait()
1510 	struct cpu_info *ci = curcpu();  in x86_cpu_idle_halt()  local
1512 	KASSERT(ci->ci_ilevel == IPL_NONE);  in x86_cpu_idle_halt()
1515 	if (!__predict_false(ci->ci_want_resched)) {  in x86_cpu_idle_halt()
1538 	struct cpu_info *ci = curcpu();  in cpu_load_pmap()  local
1540 	pd_entry_t *l3_pd = ci->ci_pae_l3_pdir;  in cpu_load_pmap()
1580 cpu_kick(struct cpu_info *ci)  in cpu_kick()  argument
1582 	x86_send_ipi(ci, X86_IPI_AST);  in cpu_kick()