1 /* $NetBSD: cpu.c,v 1.49 2010/08/20 06:34:33 jruoho Exp $ */ 2 /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp */ 3 4 /*- 5 * Copyright (c) 2000 The NetBSD Foundation, Inc. 6 * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi, 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to The NetBSD Foundation 10 * by RedBack Networks Inc. 11 * 12 * Author: Bill Sommerfeld 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 25 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 26 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 27 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33 * POSSIBILITY OF SUCH DAMAGE. 34 */ 35 36 /* 37 * Copyright (c) 1999 Stefan Grefen 38 * 39 * Redistribution and use in source and binary forms, with or without 40 * modification, are permitted provided that the following conditions 41 * are met: 42 * 1. Redistributions of source code must retain the above copyright 43 * notice, this list of conditions and the following disclaimer. 44 * 2. Redistributions in binary form must reproduce the above copyright 45 * notice, this list of conditions and the following disclaimer in the 46 * documentation and/or other materials provided with the distribution. 47 * 3. All advertising materials mentioning features or use of this software 48 * must display the following acknowledgement: 49 * This product includes software developed by the NetBSD 50 * Foundation, Inc. and its contributors. 51 * 4. Neither the name of The NetBSD Foundation nor the names of its 52 * contributors may be used to endorse or promote products derived 53 * from this software without specific prior written permission. 54 * 55 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 56 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 57 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 58 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 59 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 60 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 61 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 62 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 63 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 64 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 65 * SUCH DAMAGE. 66 */ 67 68 #include <sys/cdefs.h> 69 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.49 2010/08/20 06:34:33 jruoho Exp $"); 70 71 #include "opt_ddb.h" 72 #include "opt_multiprocessor.h" 73 #include "opt_mpbios.h" /* for MPDEBUG */ 74 #include "opt_mtrr.h" 75 #include "opt_xen.h" 76 77 #include "lapic.h" 78 #include "ioapic.h" 79 80 #include <sys/param.h> 81 #include <sys/proc.h> 82 #include <sys/systm.h> 83 #include <sys/device.h> 84 #include <sys/kmem.h> 85 #include <sys/cpu.h> 86 #include <sys/atomic.h> 87 #include <sys/reboot.h> 88 89 #include <uvm/uvm_extern.h> 90 91 #include <machine/cpufunc.h> 92 #include <machine/cpuvar.h> 93 #include <machine/pmap.h> 94 #include <machine/vmparam.h> 95 #include <machine/mpbiosvar.h> 96 #include <machine/pcb.h> 97 #include <machine/specialreg.h> 98 #include <machine/segments.h> 99 #include <machine/gdt.h> 100 #include <machine/mtrr.h> 101 #include <machine/pio.h> 102 103 #include <xen/vcpuvar.h> 104 105 #if NLAPIC > 0 106 #include <machine/apicvar.h> 107 #include <machine/i82489reg.h> 108 #include <machine/i82489var.h> 109 #endif 110 111 #include <dev/ic/mc146818reg.h> 112 #include <dev/isa/isareg.h> 113 114 #if MAXCPUS > 32 115 #error cpu_info contains 32bit bitmasks 116 #endif 117 118 int cpu_match(device_t, cfdata_t, void *); 119 void cpu_attach(device_t, device_t, void *); 120 int vcpu_match(device_t, cfdata_t, void *); 121 void vcpu_attach(device_t, device_t, void *); 122 void cpu_attach_common(device_t, device_t, void *); 123 void cpu_offline_md(void); 124 125 struct cpu_softc { 126 device_t sc_dev; /* device tree glue */ 127 struct cpu_info *sc_info; /* pointer to CPU info */ 128 bool sc_wasonline; 129 }; 130 131 int mp_cpu_start(struct cpu_info *, paddr_t); 132 void mp_cpu_start_cleanup(struct cpu_info *); 133 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 134 mp_cpu_start_cleanup }; 135 136 CFATTACH_DECL_NEW(cpu, sizeof(struct cpu_softc), 137 cpu_match, cpu_attach, NULL, NULL); 138 CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc), 139 vcpu_match, vcpu_attach, NULL, NULL); 140 141 /* 142 * Statically-allocated CPU info for the primary CPU (or the only 143 * CPU, on uniprocessors). The CPU info list is initialized to 144 * point at it. 145 */ 146 #ifdef TRAPLOG 147 #include <machine/tlog.h> 148 struct tlog tlog_primary; 149 #endif 150 struct cpu_info cpu_info_primary __aligned(CACHE_LINE_SIZE) = { 151 .ci_dev = 0, 152 .ci_self = &cpu_info_primary, 153 .ci_idepth = -1, 154 .ci_curlwp = &lwp0, 155 .ci_curldt = -1, 156 #ifdef TRAPLOG 157 .ci_tlog = &tlog_primary, 158 #endif 159 160 }; 161 struct cpu_info phycpu_info_primary __aligned(CACHE_LINE_SIZE) = { 162 .ci_dev = 0, 163 .ci_self = &phycpu_info_primary, 164 }; 165 166 struct cpu_info *cpu_info_list = &cpu_info_primary; 167 struct cpu_info *phycpu_info_list = &phycpu_info_primary; 168 169 static void cpu_set_tss_gates(struct cpu_info *ci); 170 171 uint32_t cpus_attached = 0; 172 uint32_t cpus_running = 0; 173 174 uint32_t phycpus_attached = 0; 175 uint32_t phycpus_running = 0; 176 177 uint32_t cpu_feature[5]; /* X86 CPUID feature bits 178 * [0] basic features %edx 179 * [1] basic features %ecx 180 * [2] extended features %edx 181 * [3] extended features %ecx 182 * [4] VIA padlock features 183 */ 184 185 bool x86_mp_online; 186 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 187 188 #if defined(MULTIPROCESSOR) 189 void cpu_hatch(void *); 190 static void cpu_boot_secondary(struct cpu_info *ci); 191 static void cpu_start_secondary(struct cpu_info *ci); 192 static void cpu_copy_trampoline(void); 193 194 /* 195 * Runs once per boot once multiprocessor goo has been detected and 196 * the local APIC on the boot processor has been mapped. 197 * 198 * Called from lapic_boot_init() (from mpbios_scan()). 199 */ 200 void 201 cpu_init_first(void) 202 { 203 204 cpu_info_primary.ci_cpuid = lapic_cpu_number(); 205 cpu_copy_trampoline(); 206 } 207 #endif /* MULTIPROCESSOR */ 208 209 int 210 cpu_match(device_t parent, cfdata_t match, void *aux) 211 { 212 213 return 1; 214 } 215 216 void 217 cpu_attach(device_t parent, device_t self, void *aux) 218 { 219 struct cpu_softc *sc = device_private(self); 220 struct cpu_attach_args *caa = aux; 221 struct cpu_info *ci; 222 uintptr_t ptr; 223 static bool again = false; 224 225 sc->sc_dev = self; 226 227 if (phycpus_attached == ~0) { 228 aprint_error(": increase MAXCPUS\n"); 229 return; 230 } 231 232 /* 233 * If we're an Application Processor, allocate a cpu_info 234 * structure, otherwise use the primary's. 235 */ 236 if (caa->cpu_role == CPU_ROLE_AP) { 237 if ((boothowto & RB_MD1) != 0) { 238 aprint_error(": multiprocessor boot disabled\n"); 239 if (!pmf_device_register(self, NULL, NULL)) 240 aprint_error_dev(self, 241 "couldn't establish power handler\n"); 242 return; 243 } 244 aprint_naive(": Application Processor\n"); 245 ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 246 KM_SLEEP); 247 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 248 ci->ci_curldt = -1; 249 } else { 250 aprint_naive(": %s Processor\n", 251 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 252 ci = &phycpu_info_primary; 253 } 254 255 ci->ci_self = ci; 256 sc->sc_info = ci; 257 258 ci->ci_dev = self; 259 ci->ci_cpuid = caa->cpu_number; 260 ci->ci_vcpu = NULL; 261 262 /* 263 * Boot processor may not be attached first, but the below 264 * must be done to allow booting other processors. 265 */ 266 if (!again) { 267 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 268 /* Basic init */ 269 again = true; 270 } 271 272 printf(": "); 273 switch (caa->cpu_role) { 274 case CPU_ROLE_SP: 275 printf("(uniprocessor)\n"); 276 atomic_or_32(&ci->ci_flags, CPUF_SP); 277 break; 278 279 case CPU_ROLE_BP: 280 printf("(boot processor)\n"); 281 atomic_or_32(&ci->ci_flags, CPUF_BSP); 282 break; 283 284 case CPU_ROLE_AP: 285 /* 286 * report on an AP 287 */ 288 printf("(application processor)\n"); 289 if (ci->ci_flags & CPUF_PRESENT) { 290 struct cpu_info *tmp; 291 292 tmp = phycpu_info_list; 293 while (tmp->ci_next) 294 tmp = tmp->ci_next; 295 296 tmp->ci_next = ci; 297 } 298 break; 299 300 default: 301 panic("unknown processor type??\n"); 302 } 303 304 atomic_or_32(&phycpus_attached, ci->ci_cpumask); 305 306 return; 307 } 308 309 int 310 vcpu_match(device_t parent, cfdata_t match, void *aux) 311 { 312 struct vcpu_attach_args *vcaa = aux; 313 314 if (strcmp(vcaa->vcaa_name, match->cf_name) == 0) 315 return 1; 316 return 0; 317 } 318 319 void 320 vcpu_attach(device_t parent, device_t self, void *aux) 321 { 322 struct vcpu_attach_args *vcaa = aux; 323 324 cpu_attach_common(parent, self, &vcaa->vcaa_caa); 325 } 326 327 static void 328 cpu_vm_init(struct cpu_info *ci) 329 { 330 int ncolors = 2, i; 331 332 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 333 struct x86_cache_info *cai; 334 int tcolors; 335 336 cai = &ci->ci_cinfo[i]; 337 338 tcolors = atop(cai->cai_totalsize); 339 switch(cai->cai_associativity) { 340 case 0xff: 341 tcolors = 1; /* fully associative */ 342 break; 343 case 0: 344 case 1: 345 break; 346 default: 347 tcolors /= cai->cai_associativity; 348 } 349 ncolors = max(ncolors, tcolors); 350 } 351 352 /* 353 * Knowing the size of the largest cache on this CPU, re-color 354 * our pages. 355 */ 356 if (ncolors <= uvmexp.ncolors) 357 return; 358 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 359 uvm_page_recolor(ncolors); 360 } 361 362 void 363 cpu_attach_common(device_t parent, device_t self, void *aux) 364 { 365 struct cpu_softc *sc = device_private(self); 366 struct cpu_attach_args *caa = aux; 367 struct cpu_info *ci; 368 uintptr_t ptr; 369 int cpunum = caa->cpu_number; 370 static bool again = false; 371 372 sc->sc_dev = self; 373 374 /* 375 * If we're an Application Processor, allocate a cpu_info 376 * structure, otherwise use the primary's. 377 */ 378 if (caa->cpu_role == CPU_ROLE_AP) { 379 aprint_naive(": Application Processor\n"); 380 ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 381 KM_SLEEP); 382 ci = (struct cpu_info *)roundup2(ptr, CACHE_LINE_SIZE); 383 memset(ci, 0, sizeof(*ci)); 384 #ifdef TRAPLOG 385 ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP); 386 #endif 387 } else { 388 aprint_naive(": %s Processor\n", 389 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 390 ci = &cpu_info_primary; 391 #if NLAPIC > 0 392 if (cpunum != lapic_cpu_number()) { 393 /* XXX should be done earlier */ 394 uint32_t reg; 395 aprint_verbose("\n"); 396 aprint_verbose_dev(self, "running CPU at apic %d" 397 " instead of at expected %d", lapic_cpu_number(), 398 cpunum); 399 reg = i82489_readreg(LAPIC_ID); 400 i82489_writereg(LAPIC_ID, (reg & ~LAPIC_ID_MASK) | 401 (cpunum << LAPIC_ID_SHIFT)); 402 } 403 if (cpunum != lapic_cpu_number()) { 404 aprint_error_dev(self, "unable to reset apic id\n"); 405 } 406 #endif 407 } 408 409 ci->ci_self = ci; 410 sc->sc_info = ci; 411 ci->ci_dev = self; 412 ci->ci_cpuid = cpunum; 413 414 KASSERT(HYPERVISOR_shared_info != NULL); 415 ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum]; 416 417 ci->ci_func = caa->cpu_func; 418 419 /* Must be called before mi_cpu_attach(). */ 420 cpu_vm_init(ci); 421 422 if (caa->cpu_role == CPU_ROLE_AP) { 423 int error; 424 425 error = mi_cpu_attach(ci); 426 if (error != 0) { 427 aprint_normal("\n"); 428 aprint_error_dev(self, 429 "mi_cpu_attach failed with %d\n", error); 430 return; 431 } 432 } else { 433 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 434 } 435 436 ci->ci_cpumask = (1 << cpu_index(ci)); 437 pmap_reference(pmap_kernel()); 438 ci->ci_pmap = pmap_kernel(); 439 ci->ci_tlbstate = TLBSTATE_STALE; 440 441 /* 442 * Boot processor may not be attached first, but the below 443 * must be done to allow booting other processors. 444 */ 445 if (!again) { 446 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_PRIMARY); 447 /* Basic init. */ 448 cpu_intr_init(ci); 449 cpu_get_tsc_freq(ci); 450 cpu_init(ci); 451 cpu_set_tss_gates(ci); 452 pmap_cpu_init_late(ci); 453 #if NLAPIC > 0 454 if (caa->cpu_role != CPU_ROLE_SP) { 455 /* Enable lapic. */ 456 lapic_enable(); 457 lapic_set_lvt(); 458 lapic_calibrate_timer(); 459 } 460 #endif 461 /* Make sure DELAY() is initialized. */ 462 DELAY(1); 463 again = true; 464 } 465 466 /* further PCB init done later. */ 467 468 switch (caa->cpu_role) { 469 case CPU_ROLE_SP: 470 atomic_or_32(&ci->ci_flags, CPUF_SP); 471 cpu_identify(ci); 472 #if 0 473 x86_errata(); 474 #endif 475 x86_cpu_idle_init(); 476 break; 477 478 case CPU_ROLE_BP: 479 atomic_or_32(&ci->ci_flags, CPUF_BSP); 480 cpu_identify(ci); 481 cpu_init(ci); 482 #if 0 483 x86_errata(); 484 #endif 485 x86_cpu_idle_init(); 486 break; 487 488 case CPU_ROLE_AP: 489 /* 490 * report on an AP 491 */ 492 493 #if defined(MULTIPROCESSOR) 494 cpu_intr_init(ci); 495 gdt_alloc_cpu(ci); 496 cpu_set_tss_gates(ci); 497 pmap_cpu_init_early(ci); 498 pmap_cpu_init_late(ci); 499 cpu_start_secondary(ci); 500 if (ci->ci_flags & CPUF_PRESENT) { 501 struct cpu_info *tmp; 502 503 identifycpu(ci); 504 tmp = cpu_info_list; 505 while (tmp->ci_next) 506 tmp = tmp->ci_next; 507 508 tmp->ci_next = ci; 509 } 510 #else 511 aprint_error_dev(self, "not started\n"); 512 #endif 513 break; 514 515 default: 516 aprint_normal("\n"); 517 panic("unknown processor type??\n"); 518 } 519 520 pat_init(ci); 521 atomic_or_32(&cpus_attached, ci->ci_cpumask); 522 523 #if 0 524 if (!pmf_device_register(self, cpu_suspend, cpu_resume)) 525 aprint_error_dev(self, "couldn't establish power handler\n"); 526 #endif 527 528 #if defined(MULTIPROCESSOR) 529 if (mp_verbose) { 530 struct lwp *l = ci->ci_data.cpu_idlelwp; 531 struct pcb *pcb = lwp_getpcb(l); 532 533 aprint_verbose_dev(self, 534 "idle lwp at %p, idle sp at 0x%p\n", 535 l, 536 #ifdef i386 537 (void *)pcb->pcb_esp 538 #else 539 (void *)pcb->pcb_rsp 540 #endif 541 ); 542 543 } 544 #endif 545 } 546 547 /* 548 * Initialize the processor appropriately. 549 */ 550 551 void 552 cpu_init(struct cpu_info *ci) 553 { 554 555 /* 556 * On a P6 or above, enable global TLB caching if the 557 * hardware supports it. 558 */ 559 if (cpu_feature[0] & CPUID_PGE) 560 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 561 562 #ifdef XXXMTRR 563 /* 564 * On a P6 or above, initialize MTRR's if the hardware supports them. 565 */ 566 if (cpu_feature[0] & CPUID_MTRR) { 567 if ((ci->ci_flags & CPUF_AP) == 0) 568 i686_mtrr_init_first(); 569 mtrr_init_cpu(ci); 570 } 571 #endif 572 /* 573 * If we have FXSAVE/FXRESTOR, use them. 574 */ 575 if (cpu_feature[0] & CPUID_FXSR) { 576 lcr4(rcr4() | CR4_OSFXSR); 577 578 /* 579 * If we have SSE/SSE2, enable XMM exceptions. 580 */ 581 if (cpu_feature[0] & (CPUID_SSE|CPUID_SSE2)) 582 lcr4(rcr4() | CR4_OSXMMEXCPT); 583 } 584 585 #ifdef __x86_64__ 586 /* No user PGD mapped for this CPU yet */ 587 ci->ci_xen_current_user_pgd = 0; 588 #endif 589 590 atomic_or_32(&cpus_running, ci->ci_cpumask); 591 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 592 } 593 594 595 #ifdef MULTIPROCESSOR 596 void 597 cpu_boot_secondary_processors(void) 598 { 599 struct cpu_info *ci; 600 u_long i; 601 602 for (i = 0; i < maxcpus; i++) { 603 ci = cpu_lookup(i); 604 if (ci == NULL) 605 continue; 606 if (ci->ci_data.cpu_idlelwp == NULL) 607 continue; 608 if ((ci->ci_flags & CPUF_PRESENT) == 0) 609 continue; 610 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 611 continue; 612 cpu_boot_secondary(ci); 613 } 614 615 x86_mp_online = true; 616 } 617 618 static void 619 cpu_init_idle_lwp(struct cpu_info *ci) 620 { 621 struct lwp *l = ci->ci_data.cpu_idlelwp; 622 struct pcb *pcb = lwp_getpcb(l); 623 624 pcb->pcb_cr0 = rcr0(); 625 } 626 627 void 628 cpu_init_idle_lwps(void) 629 { 630 struct cpu_info *ci; 631 u_long i; 632 633 for (i = 0; i < maxcpus; i++) { 634 ci = cpu_lookup(i); 635 if (ci == NULL) 636 continue; 637 if (ci->ci_data.cpu_idlelwp == NULL) 638 continue; 639 if ((ci->ci_flags & CPUF_PRESENT) == 0) 640 continue; 641 cpu_init_idle_lwp(ci); 642 } 643 } 644 645 void 646 cpu_start_secondary(struct cpu_info *ci) 647 { 648 int i; 649 struct pmap *kpm = pmap_kernel(); 650 extern uint32_t mp_pdirpa; 651 652 mp_pdirpa = kpm->pm_pdirpa; /* XXX move elsewhere, not per CPU. */ 653 654 atomic_or_32(&ci->ci_flags, CPUF_AP); 655 656 aprint_debug_dev(ci->ci_dev, "starting\n"); 657 658 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 659 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) 660 return; 661 662 /* 663 * wait for it to become ready 664 */ 665 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 666 #ifdef MPDEBUG 667 extern int cpu_trace[3]; 668 static int otrace[3]; 669 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) { 670 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n", 671 cpu_trace[0], cpu_trace[1], cpu_trace[2]); 672 memcpy(otrace, cpu_trace, sizeof(otrace)); 673 } 674 #endif 675 delay(10); 676 } 677 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 678 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 679 #if defined(MPDEBUG) && defined(DDB) 680 printf("dropping into debugger; continue from here to resume boot\n"); 681 Debugger(); 682 #endif 683 } 684 685 CPU_START_CLEANUP(ci); 686 } 687 688 void 689 cpu_boot_secondary(struct cpu_info *ci) 690 { 691 int i; 692 693 atomic_or_32(&ci->ci_flags, CPUF_GO); 694 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 695 delay(10); 696 } 697 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 698 aprint_error_dev(ci->ci_dev, "CPU failed to start\n"); 699 #if defined(MPDEBUG) && defined(DDB) 700 printf("dropping into debugger; continue from here to resume boot\n"); 701 Debugger(); 702 #endif 703 } 704 } 705 706 /* 707 * The CPU ends up here when its ready to run 708 * This is called from code in mptramp.s; at this point, we are running 709 * in the idle pcb/idle stack of the new CPU. When this function returns, 710 * this processor will enter the idle loop and start looking for work. 711 * 712 * XXX should share some of this with init386 in machdep.c 713 */ 714 void 715 cpu_hatch(void *v) 716 { 717 struct cpu_info *ci = (struct cpu_info *)v; 718 struct pcb *pcb; 719 int s, i; 720 721 cpu_probe(ci); 722 723 cpu_feature[0] &= ~CPUID_FEAT_BLACKLIST; 724 cpu_feature[2] &= ~CPUID_FEAT_EXT_BLACKLIST; 725 726 cpu_init_msrs(ci, true); 727 728 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 729 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 730 while ((ci->ci_flags & CPUF_GO) == 0) { 731 /* Don't use delay, boot CPU may be patching the text. */ 732 for (i = 10000; i != 0; i--) 733 x86_pause(); 734 } 735 736 /* Because the text may have been patched in x86_patch(). */ 737 wbinvd(); 738 x86_flush(); 739 740 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 741 742 pcb = lwp_getpcb(curlwp); 743 lcr3(pmap_kernel()->pm_pdirpa); 744 pcb->pcb_cr3 = pmap_kernel()->pm_pdirpa; 745 pcb = lwp_getpcb(ci->ci_data.cpu_idlelwp); 746 lcr0(pcb->pcb_cr0); 747 748 cpu_init_idt(); 749 gdt_init_cpu(ci); 750 lapic_enable(); 751 lapic_set_lvt(); 752 lapic_initclocks(); 753 754 #ifdef i386 755 npxinit(ci); 756 #else 757 fpuinit(ci); 758 #endif 759 760 lldt(GSEL(GLDT_SEL, SEL_KPL)); 761 ltr(ci->ci_tss_sel); 762 763 cpu_init(ci); 764 cpu_get_tsc_freq(ci); 765 766 s = splhigh(); 767 #ifdef i386 768 lapic_tpr = 0; 769 #else 770 lcr8(0); 771 #endif 772 x86_enable_intr(); 773 splx(s); 774 #if 0 775 x86_errata(); 776 #endif 777 778 aprint_debug_dev(ci->ci_dev, "CPU %ld running\n", 779 (long)ci->ci_cpuid); 780 } 781 782 #if defined(DDB) 783 784 #include <ddb/db_output.h> 785 #include <machine/db_machdep.h> 786 787 /* 788 * Dump CPU information from ddb. 789 */ 790 void 791 cpu_debug_dump(void) 792 { 793 struct cpu_info *ci; 794 CPU_INFO_ITERATOR cii; 795 796 db_printf("addr dev id flags ipis curlwp fpcurlwp\n"); 797 for (CPU_INFO_FOREACH(cii, ci)) { 798 db_printf("%p %s %ld %x %x %10p %10p\n", 799 ci, 800 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 801 (long)ci->ci_cpuid, 802 ci->ci_flags, ci->ci_ipis, 803 ci->ci_curlwp, 804 ci->ci_fpcurlwp); 805 } 806 } 807 #endif /* DDB */ 808 809 static void 810 cpu_copy_trampoline(void) 811 { 812 /* 813 * Copy boot code. 814 */ 815 extern u_char cpu_spinup_trampoline[]; 816 extern u_char cpu_spinup_trampoline_end[]; 817 818 vaddr_t mp_trampoline_vaddr; 819 820 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 821 UVM_KMF_VAONLY); 822 823 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 824 VM_PROT_READ | VM_PROT_WRITE, 0); 825 pmap_update(pmap_kernel()); 826 memcpy((void *)mp_trampoline_vaddr, 827 cpu_spinup_trampoline, 828 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 829 830 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 831 pmap_update(pmap_kernel()); 832 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 833 } 834 835 #endif /* MULTIPROCESSOR */ 836 837 #ifdef i386 838 #if 0 839 static void 840 tss_init(struct i386tss *tss, void *stack, void *func) 841 { 842 memset(tss, 0, sizeof *tss); 843 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 844 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 845 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 846 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 847 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 848 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 849 tss->tss_cr3 = pmap_kernel()->pm_pdirpa; 850 tss->tss_esp = (int)((char *)stack + USPACE - 16); 851 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 852 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 853 tss->__tss_eip = (int)func; 854 } 855 #endif 856 857 /* XXX */ 858 #define IDTVEC(name) __CONCAT(X, name) 859 typedef void (vector)(void); 860 extern vector IDTVEC(tss_trap08); 861 #ifdef DDB 862 extern vector Xintrddbipi; 863 extern int ddb_vec; 864 #endif 865 866 static void 867 cpu_set_tss_gates(struct cpu_info *ci) 868 { 869 #if 0 870 struct segment_descriptor sd; 871 872 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 873 UVM_KMF_WIRED); 874 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 875 IDTVEC(tss_trap08)); 876 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 877 SDT_SYS386TSS, SEL_KPL, 0, 0); 878 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 879 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 880 GSEL(GTRAPTSS_SEL, SEL_KPL)); 881 #endif 882 883 #if defined(DDB) && defined(MULTIPROCESSOR) 884 /* 885 * Set up separate handler for the DDB IPI, so that it doesn't 886 * stomp on a possibly corrupted stack. 887 * 888 * XXX overwriting the gate set in db_machine_init. 889 * Should rearrange the code so that it's set only once. 890 */ 891 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 892 UVM_KMF_WIRED); 893 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, 894 Xintrddbipi); 895 896 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 897 SDT_SYS386TSS, SEL_KPL, 0, 0); 898 ci->ci_gdt[GIPITSS_SEL].sd = sd; 899 900 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 901 GSEL(GIPITSS_SEL, SEL_KPL)); 902 #endif 903 } 904 #else 905 static void 906 cpu_set_tss_gates(struct cpu_info *ci) 907 { 908 909 } 910 #endif /* i386 */ 911 912 int 913 mp_cpu_start(struct cpu_info *ci, paddr_t target) 914 { 915 #if 0 916 #if NLAPIC > 0 917 int error; 918 #endif 919 unsigned short dwordptr[2]; 920 921 /* 922 * Bootstrap code must be addressable in real mode 923 * and it must be page aligned. 924 */ 925 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 926 927 /* 928 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 929 */ 930 931 outb(IO_RTC, NVRAM_RESET); 932 outb(IO_RTC+1, NVRAM_RESET_JUMP); 933 934 /* 935 * "and the warm reset vector (DWORD based at 40:67) to point 936 * to the AP startup code ..." 937 */ 938 939 dwordptr[0] = 0; 940 dwordptr[1] = target >> 4; 941 942 pmap_kenter_pa (0, 0, VM_PROT_READ|VM_PROT_WRITE, 0); 943 pmap_update(pmap_kernel()); 944 945 memcpy ((uint8_t *) 0x467, dwordptr, 4); 946 947 pmap_kremove (0, PAGE_SIZE); 948 pmap_update(pmap_kernel()); 949 950 #if NLAPIC > 0 951 /* 952 * ... prior to executing the following sequence:" 953 */ 954 955 if (ci->ci_flags & CPUF_AP) { 956 if ((error = x86_ipi_init(ci->ci_cpuid)) != 0) 957 return error; 958 959 delay(10000); 960 961 if (cpu_feature & CPUID_APIC) { 962 error = x86_ipi_init(ci->ci_cpuid); 963 if (error != 0) { 964 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 965 __func__); 966 return error; 967 } 968 969 delay(10000); 970 971 error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid, 972 LAPIC_DLMODE_STARTUP); 973 if (error != 0) { 974 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 975 __func__); 976 return error; 977 } 978 delay(200); 979 980 error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid, 981 LAPIC_DLMODE_STARTUP); 982 if (error != 0) { 983 aprint_error_dev(ci->ci_dev, "%s: IPI not taken ((3)\n", 984 __func__); 985 return error; 986 } 987 delay(200); 988 } 989 } 990 #endif 991 #endif /* 0 */ 992 return 0; 993 } 994 995 void 996 mp_cpu_start_cleanup(struct cpu_info *ci) 997 { 998 #if 0 999 /* 1000 * Ensure the NVRAM reset byte contains something vaguely sane. 1001 */ 1002 1003 outb(IO_RTC, NVRAM_RESET); 1004 outb(IO_RTC+1, NVRAM_RESET_RST); 1005 #endif 1006 } 1007 1008 void 1009 cpu_init_msrs(struct cpu_info *ci, bool full) 1010 { 1011 #ifdef __x86_64__ 1012 if (full) { 1013 HYPERVISOR_set_segment_base (SEGBASE_FS, 0); 1014 HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci); 1015 HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0); 1016 } 1017 #endif /* __x86_64__ */ 1018 1019 if (cpu_feature[2] & CPUID_NOX) 1020 wrmsr(MSR_EFER, rdmsr(MSR_EFER) | EFER_NXE); 1021 } 1022 1023 void 1024 cpu_offline_md(void) 1025 { 1026 int s; 1027 1028 s = splhigh(); 1029 #ifdef __i386__ 1030 npxsave_cpu(true); 1031 #else 1032 fpusave_cpu(true); 1033 #endif 1034 splx(s); 1035 } 1036 1037 #if 0 1038 /* XXX joerg restructure and restart CPUs individually */ 1039 static bool 1040 cpu_suspend(device_t dv, const pmf_qual_t *qual) 1041 { 1042 struct cpu_softc *sc = device_private(dv); 1043 struct cpu_info *ci = sc->sc_info; 1044 int err; 1045 1046 if (ci->ci_flags & CPUF_PRIMARY) 1047 return true; 1048 if (ci->ci_data.cpu_idlelwp == NULL) 1049 return true; 1050 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1051 return true; 1052 1053 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1054 1055 if (sc->sc_wasonline) { 1056 mutex_enter(&cpu_lock); 1057 err = cpu_setstate(ci, false); 1058 mutex_exit(&cpu_lock); 1059 1060 if (err) 1061 return false; 1062 } 1063 1064 return true; 1065 } 1066 1067 static bool 1068 cpu_resume(device_t dv, const pmf_qual_t *qual) 1069 { 1070 struct cpu_softc *sc = device_private(dv); 1071 struct cpu_info *ci = sc->sc_info; 1072 int err = 0; 1073 1074 if (ci->ci_flags & CPUF_PRIMARY) 1075 return true; 1076 if (ci->ci_data.cpu_idlelwp == NULL) 1077 return true; 1078 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1079 return true; 1080 1081 if (sc->sc_wasonline) { 1082 mutex_enter(&cpu_lock); 1083 err = cpu_setstate(ci, true); 1084 mutex_exit(&cpu_lock); 1085 } 1086 1087 return err == 0; 1088 } 1089 #endif 1090 1091 void 1092 cpu_get_tsc_freq(struct cpu_info *ci) 1093 { 1094 const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time; 1095 delay(1000000); 1096 uint64_t freq = 1000000000ULL << 32; 1097 freq = freq / (uint64_t)tinfo->tsc_to_system_mul; 1098 if ( tinfo->tsc_shift < 0 ) 1099 freq = freq << -tinfo->tsc_shift; 1100 else 1101 freq = freq >> tinfo->tsc_shift; 1102 ci->ci_data.cpu_cc_freq = freq; 1103 } 1104 1105 void 1106 x86_cpu_idle_xen(void) 1107 { 1108 struct cpu_info *ci = curcpu(); 1109 1110 KASSERT(ci->ci_ilevel == IPL_NONE); 1111 1112 x86_disable_intr(); 1113 if (!__predict_false(ci->ci_want_resched)) { 1114 idle_block(); 1115 } else { 1116 x86_enable_intr(); 1117 } 1118 } 1119 1120 /* 1121 * Loads pmap for the current CPU. 1122 */ 1123 void 1124 cpu_load_pmap(struct pmap *pmap) 1125 { 1126 #ifdef i386 1127 #ifdef PAE 1128 int i, s; 1129 struct cpu_info *ci; 1130 1131 s = splvm(); /* just to be safe */ 1132 ci = curcpu(); 1133 paddr_t l3_pd = xpmap_ptom_masked(ci->ci_pae_l3_pdirpa); 1134 /* don't update the kernel L3 slot */ 1135 for (i = 0 ; i < PDP_SIZE - 1; i++) { 1136 xpq_queue_pte_update(l3_pd + i * sizeof(pd_entry_t), 1137 xpmap_ptom(pmap->pm_pdirpa[i]) | PG_V); 1138 } 1139 splx(s); 1140 tlbflush(); 1141 #else /* PAE */ 1142 lcr3(pmap_pdirpa(pmap, 0)); 1143 #endif /* PAE */ 1144 #endif /* i386 */ 1145 1146 #ifdef __x86_64__ 1147 int i, s; 1148 pd_entry_t *old_pgd, *new_pgd; 1149 paddr_t addr; 1150 struct cpu_info *ci; 1151 1152 /* kernel pmap always in cr3 and should never go in user cr3 */ 1153 if (pmap_pdirpa(pmap, 0) != pmap_pdirpa(pmap_kernel(), 0)) { 1154 ci = curcpu(); 1155 /* 1156 * Map user space address in kernel space and load 1157 * user cr3 1158 */ 1159 s = splvm(); 1160 new_pgd = pmap->pm_pdir; 1161 old_pgd = pmap_kernel()->pm_pdir; 1162 addr = xpmap_ptom(pmap_pdirpa(pmap_kernel(), 0)); 1163 for (i = 0; i < PDIR_SLOT_PTE; 1164 i++, addr += sizeof(pd_entry_t)) { 1165 if ((new_pgd[i] & PG_V) || (old_pgd[i] & PG_V)) 1166 xpq_queue_pte_update(addr, new_pgd[i]); 1167 } 1168 tlbflush(); 1169 xen_set_user_pgd(pmap_pdirpa(pmap, 0)); 1170 ci->ci_xen_current_user_pgd = pmap_pdirpa(pmap, 0); 1171 splx(s); 1172 } 1173 #endif /* __x86_64__ */ 1174 } 1175