183 static void    	cpu_boot_secondary(struct cpu_info *ci);
184 static void    	cpu_start_secondary(struct cpu_info *ci);
199 	struct cpu_info *ci;  in cpu_attach()  local
214 		ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,  in cpu_attach()
216 		ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);  in cpu_attach()
217 		ci->ci_curldt = -1;  in cpu_attach()
223 		tmp->ci_next = ci;  in cpu_attach()
225 		ci = &phycpu_info_primary;  in cpu_attach()
228 	ci->ci_self = ci;  in cpu_attach()
229 	sc->sc_info = ci;  in cpu_attach()
231 	ci->ci_dev = self;  in cpu_attach()
232 	ci->ci_acpiid = caa->cpu_id;  in cpu_attach()
233 	ci->ci_cpuid = caa->cpu_number;  in cpu_attach()
234 	ci->ci_vcpu = NULL;  in cpu_attach()
235 	ci->ci_index = nphycpu++;  in cpu_attach()
236 	ci->ci_kfpu_spl = -1;  in cpu_attach()
255 	struct cpu_info *ci = sc->sc_info;  in cpu_rescan()  local
258 	cfaa.ci = ci;  in cpu_rescan()
262 		if (ci->ci_frequency == NULL) {  in cpu_rescan()
264 			ci->ci_frequency =  in cpu_rescan()
277 	struct cpu_info *ci = sc->sc_info;  in cpu_childdetached()  local
279 	if (ci->ci_frequency == child)  in cpu_childdetached()
280 		ci->ci_frequency = NULL;  in cpu_childdetached()
320 vcpu_is_up(struct cpu_info *ci)  in vcpu_is_up()  argument
322 	KASSERT(ci != NULL);  in vcpu_is_up()
323 	return HYPERVISOR_vcpu_op(VCPUOP_is_up, ci->ci_vcpuid, NULL);  in vcpu_is_up()
327 cpu_vm_init(struct cpu_info *ci)  in cpu_vm_init()  argument
335 		cai = &ci->ci_cinfo[i];  in cpu_vm_init()
355 	aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors);  in cpu_vm_init()
357 	pmap_tlb_cpu_init(ci);  in cpu_vm_init()
359 	pmap_vpage_cpu_init(ci);  in cpu_vm_init()
368 	struct cpu_info *ci;  in cpu_attach_common()  local
381 		ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1,  in cpu_attach_common()
383 		ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE);  in cpu_attach_common()
384 		memset(ci, 0, sizeof(*ci));  in cpu_attach_common()
385 		cpu_init_tss(ci);  in cpu_attach_common()
389 		ci = &cpu_info_primary;  in cpu_attach_common()
392 	ci->ci_self = ci;  in cpu_attach_common()
393 	sc->sc_info = ci;  in cpu_attach_common()
394 	ci->ci_dev = self;  in cpu_attach_common()
395 	ci->ci_cpuid = cpunum;  in cpu_attach_common()
396 	ci->ci_vcpuid = cpunum;  in cpu_attach_common()
397 	ci->ci_kfpu_spl = -1;  in cpu_attach_common()
401 	ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum];  in cpu_attach_common()
403 	KASSERT(ci->ci_func == 0);  in cpu_attach_common()
404 	ci->ci_func = caa->cpu_func;  in cpu_attach_common()
408 	cpu_vm_init(ci);  in cpu_attach_common()
413 		error = mi_cpu_attach(ci);  in cpu_attach_common()
415 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);  in cpu_attach_common()
423 		KASSERT(ci->ci_data.cpu_idlelwp != NULL);  in cpu_attach_common()
426 	KASSERT(ci->ci_cpuid == ci->ci_index);  in cpu_attach_common()
429 	ci->ci_xen_current_user_pgd = 0;  in cpu_attach_common()
431 	mutex_init(&ci->ci_kpm_mtx, MUTEX_DEFAULT, IPL_VM);  in cpu_attach_common()
433 	ci->ci_pmap = pmap_kernel();  in cpu_attach_common()
434 	ci->ci_tlbstate = TLBSTATE_STALE;  in cpu_attach_common()
441 		atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY);  in cpu_attach_common()
443 		cpu_intr_init(ci);  in cpu_attach_common()
444 		cpu_get_tsc_freq(ci);  in cpu_attach_common()
445 		cpu_init(ci);  in cpu_attach_common()
446 		pmap_cpu_init_late(ci);  in cpu_attach_common()
460 		atomic_or_32(&ci->ci_flags, CPUF_SP);  in cpu_attach_common()
461 		cpu_identify(ci);  in cpu_attach_common()
466 		atomic_or_32(&ci->ci_flags, CPUF_BSP);  in cpu_attach_common()
467 		cpu_identify(ci);  in cpu_attach_common()
472 		atomic_or_32(&ci->ci_flags, CPUF_AP);  in cpu_attach_common()
480 		cpu_intr_init(ci);  in cpu_attach_common()
483 		idt_vec_init_cpu_md(&ci->ci_idtvec, cpu_index(ci));  in cpu_attach_common()
486 		gdt_alloc_cpu(ci);  in cpu_attach_common()
488 		pmap_cpu_init_late(ci);  in cpu_attach_common()
489 		cpu_start_secondary(ci);  in cpu_attach_common()
491 		if (ci->ci_flags & CPUF_PRESENT) {  in cpu_attach_common()
494 			cpu_identify(ci);  in cpu_attach_common()
499 			tmp->ci_next = ci;  in cpu_attach_common()
502 		aprint_error_dev(ci->ci_dev, "not started\n");  in cpu_attach_common()
512 		struct lwp *l = ci->ci_data.cpu_idlelwp;  in cpu_attach_common()
534 cpu_init(struct cpu_info *ci)  in cpu_init()  argument
574 		ci->ci_feat_val[1] = descs[2];  in cpu_init()
582 	atomic_or_32(&ci->ci_flags, CPUF_RUNNING);  in cpu_init()
591 	struct cpu_info *ci;  in cpu_boot_secondary_processors()  local
598 		ci = cpu_lookup(i);  in cpu_boot_secondary_processors()
599 		if (ci == NULL)  in cpu_boot_secondary_processors()
601 		if (ci->ci_data.cpu_idlelwp == NULL)  in cpu_boot_secondary_processors()
603 		if ((ci->ci_flags & CPUF_PRESENT) == 0)  in cpu_boot_secondary_processors()
605 		if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY))  in cpu_boot_secondary_processors()
607 		cpu_boot_secondary(ci);  in cpu_boot_secondary_processors()
608 		kcpuset_set(cpus, cpu_index(ci));  in cpu_boot_secondary_processors()
618 cpu_init_idle_lwp(struct cpu_info *ci)  in cpu_init_idle_lwp()  argument
620 	struct lwp *l = ci->ci_data.cpu_idlelwp;  in cpu_init_idle_lwp()
629 	struct cpu_info *ci;  in cpu_init_idle_lwps()  local
633 		ci = cpu_lookup(i);  in cpu_init_idle_lwps()
634 		if (ci == NULL)  in cpu_init_idle_lwps()
636 		if (ci->ci_data.cpu_idlelwp == NULL)  in cpu_init_idle_lwps()
638 		if ((ci->ci_flags & CPUF_PRESENT) == 0)  in cpu_init_idle_lwps()
640 		cpu_init_idle_lwp(ci);  in cpu_init_idle_lwps()
645 cpu_start_secondary(struct cpu_info *ci)  in cpu_start_secondary()  argument
649 	aprint_debug_dev(ci->ci_dev, "starting\n");  in cpu_start_secondary()
651 	ci->ci_curlwp = ci->ci_data.cpu_idlelwp;  in cpu_start_secondary()
653 	if (CPU_STARTUP(ci, (vaddr_t) cpu_hatch) != 0) {  in cpu_start_secondary()
660 	for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) {  in cpu_start_secondary()
663 	if ((ci->ci_flags & CPUF_PRESENT) == 0) {  in cpu_start_secondary()
664 		aprint_error_dev(ci->ci_dev, "failed to become ready\n");  in cpu_start_secondary()
671 	CPU_START_CLEANUP(ci);  in cpu_start_secondary()
675 cpu_boot_secondary(struct cpu_info *ci)  in cpu_boot_secondary()  argument
678 	atomic_or_32(&ci->ci_flags, CPUF_GO);  in cpu_boot_secondary()
679 	for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) {  in cpu_boot_secondary()
682 	if ((ci->ci_flags & CPUF_RUNNING) == 0) {  in cpu_boot_secondary()
683 		aprint_error_dev(ci->ci_dev, "CPU failed to start\n");  in cpu_boot_secondary()
702 	struct cpu_info *ci = (struct cpu_info *)v;  in cpu_hatch()  local
707 	cpu_init_msrs(ci, true);  in cpu_hatch()
708 	cpu_init_idt(ci);  in cpu_hatch()
709 	gdt_init_cpu(ci);  in cpu_hatch()
711 	cpu_probe(ci);  in cpu_hatch()
713 	atomic_or_32(&ci->ci_flags, CPUF_PRESENT);  in cpu_hatch()
715 	while ((ci->ci_flags & CPUF_GO) == 0) {  in cpu_hatch()
725 	KASSERT((ci->ci_flags & CPUF_RUNNING) == 0);  in cpu_hatch()
727 	KASSERT(ci->ci_curlwp == ci->ci_data.cpu_idlelwp);  in cpu_hatch()
728 	KASSERT(curlwp == ci->ci_data.cpu_idlelwp);  in cpu_hatch()
737 	fpuinit(ci);  in cpu_hatch()
742 	cpu_init(ci);  in cpu_hatch()
743 	cpu_get_tsc_freq(ci);  in cpu_hatch()
749 	aprint_debug_dev(ci->ci_dev, "running\n");  in cpu_hatch()
751 	KASSERT(ci->ci_curlwp == ci->ci_data.cpu_idlelwp);  in cpu_hatch()
767 	struct cpu_info *ci;  in cpu_debug_dump()  local
771 	for (CPU_INFO_FOREACH(cii, ci)) {  in cpu_debug_dump()
773 		    ci,  in cpu_debug_dump()
774 		    ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev),  in cpu_debug_dump()
775 		    (long)ci->ci_vcpuid,  in cpu_debug_dump()
776 		    ci->ci_flags, ci->ci_ipis,  in cpu_debug_dump()
777 		    ci->ci_curlwp);  in cpu_debug_dump()
815 xen_init_amd64_vcpuctxt(struct cpu_info *ci, struct vcpu_guest_context *initctx,  in xen_init_amd64_vcpuctxt()  argument
828 	KASSERT(ci != NULL);  in xen_init_amd64_vcpuctxt()
829 	KASSERT(ci != &cpu_info_primary);  in xen_init_amd64_vcpuctxt()
838 	gdt_prepframes(frames, (vaddr_t)ci->ci_gdt, gdt_ents);  in xen_init_amd64_vcpuctxt()
842 	l = ci->ci_data.cpu_idlelwp;  in xen_init_amd64_vcpuctxt()
849 	vci = ci->ci_vcpu;  in xen_init_amd64_vcpuctxt()
863 	initctx->user_regs.rdi = (uint64_t) ci; /* targetrip(ci); */  in xen_init_amd64_vcpuctxt()
894 	initctx->ctrlreg[3] = xen_pfn_to_cr3(x86_btop(xpmap_ptom(ci->ci_kpm_pdirpa)));  in xen_init_amd64_vcpuctxt()
909 xen_init_i386_vcpuctxt(struct cpu_info *ci, struct vcpu_guest_context *initctx,  in xen_init_i386_vcpuctxt()  argument
922 	KASSERT(ci != NULL);  in xen_init_i386_vcpuctxt()
923 	KASSERT(ci != &cpu_info_primary);  in xen_init_i386_vcpuctxt()
932 	gdt_prepframes(frames, (vaddr_t)ci->ci_gdt, gdt_ents);  in xen_init_i386_vcpuctxt()
939 	l = ci->ci_data.cpu_idlelwp;  in xen_init_i386_vcpuctxt()
946 	vci = ci->ci_vcpu;  in xen_init_i386_vcpuctxt()
963 		/* targeteip(ci); */  in xen_init_i386_vcpuctxt()
965 		arg[1] = (uint32_t)ci; /* arg1 */  in xen_init_i386_vcpuctxt()
992 	initctx->ctrlreg[3] = xen_pfn_to_cr3(x86_btop(xpmap_ptom(ci->ci_pae_l3_pdirpa)));  in xen_init_i386_vcpuctxt()
1006 mp_cpu_start(struct cpu_info *ci, vaddr_t target)  in mp_cpu_start()  argument
1011 	KASSERT(ci != NULL);  in mp_cpu_start()
1012 	KASSERT(ci != &cpu_info_primary);  in mp_cpu_start()
1013 	KASSERT(ci->ci_flags & CPUF_AP);  in mp_cpu_start()
1018 	xen_init_amd64_vcpuctxt(ci, vcpuctx, (void (*)(struct cpu_info *))target);  in mp_cpu_start()
1020 	xen_init_i386_vcpuctxt(ci, vcpuctx, (void (*)(struct cpu_info *))target);  in mp_cpu_start()
1023 	/* Initialise the given vcpu to execute cpu_hatch(ci); */  in mp_cpu_start()
1024 	if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_initialise, ci->ci_vcpuid, vcpuctx))) {  in mp_cpu_start()
1032 	if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_down, ci->ci_vcpuid, NULL))) {  in mp_cpu_start()
1037 	if ((hyperror = HYPERVISOR_vcpu_op(VCPUOP_up, ci->ci_vcpuid, NULL))) {  in mp_cpu_start()
1042 	if (!vcpu_is_up(ci)) {  in mp_cpu_start()
1054 mp_cpu_start_cleanup(struct cpu_info *ci)  in mp_cpu_start_cleanup()  argument
1056 	if (vcpu_is_up(ci)) {  in mp_cpu_start_cleanup()
1057 		aprint_debug_dev(ci->ci_dev, "is started.\n");  in mp_cpu_start_cleanup()
1059 		aprint_error_dev(ci->ci_dev, "did not start up.\n");  in mp_cpu_start_cleanup()
1064 cpu_init_msrs(struct cpu_info *ci, bool full)  in cpu_init_msrs()  argument
1069 		HYPERVISOR_set_segment_base(SEGBASE_GS_KERNEL, (uint64_t)ci);  in cpu_init_msrs()
1085 cpu_get_tsc_freq(struct cpu_info *ci)  in cpu_get_tsc_freq()  argument
1088 	const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time;  in cpu_get_tsc_freq()
1099 	ci->ci_data.cpu_cc_freq = freq;  in cpu_get_tsc_freq()
1108 	struct cpu_info *ci = curcpu();  in cpu_load_pmap()  local
1109 	cpuid_t cid = cpu_index(ci);  in cpu_load_pmap()
1115 	mutex_enter(&ci->ci_kpm_mtx);  in cpu_load_pmap()
1123 	l4_pd_ma = xpmap_ptom_masked(ci->ci_kpm_pdirpa);  in cpu_load_pmap()
1130 	KASSERT(pmap == ci->ci_pmap);  in cpu_load_pmap()
1135 		if (ci->ci_kpm_pdir[i] != new_pgd[i]) {  in cpu_load_pmap()
1142 	ci->ci_xen_current_user_pgd = pmap_pdirpa(pmap, 0);  in cpu_load_pmap()
1144 	paddr_t l3_pd = xpmap_ptom_masked(ci->ci_pae_l3_pdirpa);  in cpu_load_pmap()
1158 	mutex_exit(&ci->ci_kpm_mtx);  in cpu_load_pmap()
1181 pmap_cpu_init_late(struct cpu_info *ci)  in pmap_cpu_init_late()  argument
1194 		ci->ci_normal_pdes[i] = normal_pdes[i];  in pmap_cpu_init_late()
1198 	if (ci == &cpu_info_primary)  in pmap_cpu_init_late()
1201 	KASSERT(ci != NULL);  in pmap_cpu_init_late()
1204 	cpu_alloc_l3_page(ci);  in pmap_cpu_init_late()
1205 	KASSERT(ci->ci_pae_l3_pdirpa != 0);  in pmap_cpu_init_late()
1209 		ci->ci_pae_l3_pdir[i] =  in pmap_cpu_init_late()
1214 	ci->ci_kpm_pdir = (pd_entry_t *)uvm_km_alloc(kernel_map, PAGE_SIZE, 0,  in pmap_cpu_init_late()
1217 	if (ci->ci_kpm_pdir == NULL) {  in pmap_cpu_init_late()
1219 		    __func__, cpu_index(ci));  in pmap_cpu_init_late()
1221 	ci->ci_kpm_pdirpa = vtophys((vaddr_t)ci->ci_kpm_pdir);  in pmap_cpu_init_late()
1222 	KASSERT(ci->ci_kpm_pdirpa != 0);  in pmap_cpu_init_late()
1228 	memcpy(ci->ci_kpm_pdir, pmap_kernel()->pm_pdir, PAGE_SIZE);  in pmap_cpu_init_late()
1231 	ci->ci_kpm_pdir[PDIR_SLOT_PTE] = xpmap_ptom_masked(ci->ci_kpm_pdirpa)  in pmap_cpu_init_late()
1235 	memcpy(ci->ci_kpm_pdir, pmap_kernel()->pm_pdir + PDIR_SLOT_KERN,  in pmap_cpu_init_late()
1240 	pmap_protect(pmap_kernel(), (vaddr_t)ci->ci_kpm_pdir,  in pmap_cpu_init_late()
1241 	    (vaddr_t)ci->ci_kpm_pdir + PAGE_SIZE, VM_PROT_READ);  in pmap_cpu_init_late()
1245 	xpq_queue_pin_l4_table(xpmap_ptom_masked(ci->ci_kpm_pdirpa));  in pmap_cpu_init_late()
1251 	ci->ci_pae_l3_pdir[3] = xpmap_ptom_masked(ci->ci_kpm_pdirpa) | PTE_P;  in pmap_cpu_init_late()
1254 	pmap_protect(pmap_kernel(), (vaddr_t)ci->ci_pae_l3_pdir,  in pmap_cpu_init_late()
1255 	    (vaddr_t)ci->ci_pae_l3_pdir + PAGE_SIZE, VM_PROT_READ);  in pmap_cpu_init_late()
1258 	xpq_queue_pin_l3_table(xpmap_ptom_masked(ci->ci_pae_l3_pdirpa));  in pmap_cpu_init_late()
1277 cpu_kick(struct cpu_info *ci)  in cpu_kick()  argument
1279 	(void)xen_send_ipi(ci, XEN_IPI_AST);  in cpu_kick()