1 /* $NetBSD: cpu.c,v 1.34 2009/07/30 13:56:57 cegger Exp $ */ 2 /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp */ 3 4 /*- 5 * Copyright (c) 2000 The NetBSD Foundation, Inc. 6 * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi, 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to The NetBSD Foundation 10 * by RedBack Networks Inc. 11 * 12 * Author: Bill Sommerfeld 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 25 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 26 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 27 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33 * POSSIBILITY OF SUCH DAMAGE. 34 */ 35 36 /* 37 * Copyright (c) 1999 Stefan Grefen 38 * 39 * Redistribution and use in source and binary forms, with or without 40 * modification, are permitted provided that the following conditions 41 * are met: 42 * 1. Redistributions of source code must retain the above copyright 43 * notice, this list of conditions and the following disclaimer. 44 * 2. Redistributions in binary form must reproduce the above copyright 45 * notice, this list of conditions and the following disclaimer in the 46 * documentation and/or other materials provided with the distribution. 47 * 3. All advertising materials mentioning features or use of this software 48 * must display the following acknowledgement: 49 * This product includes software developed by the NetBSD 50 * Foundation, Inc. and its contributors. 51 * 4. Neither the name of The NetBSD Foundation nor the names of its 52 * contributors may be used to endorse or promote products derived 53 * from this software without specific prior written permission. 54 * 55 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 56 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 57 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 58 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 59 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 60 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 61 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 62 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 63 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 64 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 65 * SUCH DAMAGE. 66 */ 67 68 #include <sys/cdefs.h> 69 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.34 2009/07/30 13:56:57 cegger Exp $"); 70 71 #include "opt_ddb.h" 72 #include "opt_multiprocessor.h" 73 #include "opt_mpbios.h" /* for MPDEBUG */ 74 #include "opt_mtrr.h" 75 #include "opt_xen.h" 76 77 #include "lapic.h" 78 #include "ioapic.h" 79 80 #include <sys/param.h> 81 #include <sys/proc.h> 82 #include <sys/user.h> 83 #include <sys/systm.h> 84 #include <sys/device.h> 85 #include <sys/kmem.h> 86 #include <sys/cpu.h> 87 #include <sys/atomic.h> 88 #include <sys/reboot.h> 89 90 #include <uvm/uvm_extern.h> 91 92 #include <machine/cpufunc.h> 93 #include <machine/cpuvar.h> 94 #include <machine/pmap.h> 95 #include <machine/vmparam.h> 96 #include <machine/mpbiosvar.h> 97 #include <machine/pcb.h> 98 #include <machine/specialreg.h> 99 #include <machine/segments.h> 100 #include <machine/gdt.h> 101 #include <machine/mtrr.h> 102 #include <machine/pio.h> 103 104 #include <xen/vcpuvar.h> 105 106 #if NLAPIC > 0 107 #include <machine/apicvar.h> 108 #include <machine/i82489reg.h> 109 #include <machine/i82489var.h> 110 #endif 111 112 #include <dev/ic/mc146818reg.h> 113 #include <dev/isa/isareg.h> 114 115 #define X86_MAXPROCS 32 116 117 int cpu_match(device_t, cfdata_t, void *); 118 void cpu_attach(device_t, device_t, void *); 119 int vcpu_match(device_t, cfdata_t, void *); 120 void vcpu_attach(device_t, device_t, void *); 121 void cpu_attach_common(device_t, device_t, void *); 122 void cpu_offline_md(void); 123 124 struct cpu_softc { 125 device_t sc_dev; /* device tree glue */ 126 struct cpu_info *sc_info; /* pointer to CPU info */ 127 bool sc_wasonline; 128 }; 129 130 int mp_cpu_start(struct cpu_info *, paddr_t); 131 void mp_cpu_start_cleanup(struct cpu_info *); 132 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 133 mp_cpu_start_cleanup }; 134 135 CFATTACH_DECL_NEW(cpu, sizeof(struct cpu_softc), 136 cpu_match, cpu_attach, NULL, NULL); 137 CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc), 138 vcpu_match, vcpu_attach, NULL, NULL); 139 140 /* 141 * Statically-allocated CPU info for the primary CPU (or the only 142 * CPU, on uniprocessors). The CPU info list is initialized to 143 * point at it. 144 */ 145 #ifdef TRAPLOG 146 #include <machine/tlog.h> 147 struct tlog tlog_primary; 148 #endif 149 struct cpu_info cpu_info_primary = { 150 .ci_dev = 0, 151 .ci_self = &cpu_info_primary, 152 .ci_idepth = -1, 153 .ci_curlwp = &lwp0, 154 .ci_curldt = -1, 155 #ifdef TRAPLOG 156 .ci_tlog = &tlog_primary, 157 #endif 158 159 }; 160 struct cpu_info phycpu_info_primary = { 161 .ci_dev = 0, 162 .ci_self = &phycpu_info_primary, 163 }; 164 165 struct cpu_info *cpu_info_list = &cpu_info_primary; 166 167 static void cpu_set_tss_gates(struct cpu_info *ci); 168 169 uint32_t cpus_attached = 0; 170 uint32_t cpus_running = 0; 171 172 bool x86_mp_online; 173 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 174 175 struct cpu_info *phycpu_info[X86_MAXPROCS] = { &cpu_info_primary }; 176 177 #ifdef MULTIPROCESSOR 178 /* 179 * Array of CPU info structures. Must be statically-allocated because 180 * curproc, etc. are used early. 181 */ 182 struct cpu_info *cpu_info[X86_MAXPROCS] = { &cpu_info_primary }; 183 184 void cpu_hatch(void *); 185 static void cpu_boot_secondary(struct cpu_info *ci); 186 static void cpu_start_secondary(struct cpu_info *ci); 187 static void cpu_copy_trampoline(void); 188 189 /* 190 * Runs once per boot once multiprocessor goo has been detected and 191 * the local APIC on the boot processor has been mapped. 192 * 193 * Called from lapic_boot_init() (from mpbios_scan()). 194 */ 195 void 196 cpu_init_first(void) 197 { 198 int cpunum = lapic_cpu_number(); 199 200 if (cpunum != 0) { 201 cpu_info[0] = NULL; 202 cpu_info[cpunum] = &cpu_info_primary; 203 } 204 205 cpu_copy_trampoline(); 206 } 207 #endif 208 209 int 210 cpu_match(device_t parent, cfdata_t match, void *aux) 211 { 212 213 return 1; 214 } 215 216 void 217 cpu_attach(device_t parent, device_t self, void *aux) 218 { 219 struct cpu_softc *sc = device_private(self); 220 struct cpu_attach_args *caa = aux; 221 struct cpu_info *ci; 222 uintptr_t ptr; 223 int cpunum = caa->cpu_number; 224 225 sc->sc_dev = self; 226 227 if (cpus_attached == ~0) { 228 aprint_error(": increase MAXCPUS\n"); 229 return; 230 } 231 232 /* 233 * If we're an Application Processor, allocate a cpu_info 234 * structure, otherwise use the primary's. 235 */ 236 if (caa->cpu_role == CPU_ROLE_AP) { 237 if ((boothowto & RB_MD1) != 0) { 238 aprint_error(": multiprocessor boot disabled\n"); 239 if (!pmf_device_register(self, NULL, NULL)) 240 aprint_error_dev(self, 241 "couldn't establish power handler\n"); 242 return; 243 } 244 aprint_naive(": Application Processor\n"); 245 ptr = (uintptr_t)kmem_zalloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 246 KM_SLEEP); 247 ci = (struct cpu_info *)((ptr + CACHE_LINE_SIZE - 1) & 248 ~(CACHE_LINE_SIZE - 1)); 249 ci->ci_curldt = -1; 250 if (phycpu_info[cpunum] != NULL) 251 panic("cpu at apic id %d already attached?", cpunum); 252 phycpu_info[cpunum] = ci; 253 } else { 254 aprint_naive(": %s Processor\n", 255 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 256 ci = &phycpu_info_primary; 257 if (cpunum != 0) { 258 phycpu_info[0] = NULL; 259 phycpu_info[cpunum] = ci; 260 } 261 } 262 263 ci->ci_self = ci; 264 sc->sc_info = ci; 265 266 ci->ci_dev = self; 267 ci->ci_cpuid = caa->cpu_number; 268 ci->ci_vcpu = NULL; 269 270 printf(": "); 271 switch (caa->cpu_role) { 272 case CPU_ROLE_SP: 273 printf("(uniprocessor)\n"); 274 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY); 275 break; 276 277 case CPU_ROLE_BP: 278 printf("(boot processor)\n"); 279 atomic_or_32(&ci->ci_flags, CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY); 280 break; 281 282 case CPU_ROLE_AP: 283 /* 284 * report on an AP 285 */ 286 printf("(application processor)\n"); 287 break; 288 289 default: 290 panic("unknown processor type??\n"); 291 } 292 293 atomic_or_32(&cpus_attached, ci->ci_cpumask); 294 295 return; 296 } 297 298 int 299 vcpu_match(device_t parent, cfdata_t match, void *aux) 300 { 301 struct vcpu_attach_args *vcaa = aux; 302 303 if (strcmp(vcaa->vcaa_name, match->cf_name) == 0) 304 return 1; 305 return 0; 306 } 307 308 void 309 vcpu_attach(device_t parent, device_t self, void *aux) 310 { 311 struct vcpu_attach_args *vcaa = aux; 312 313 cpu_attach_common(parent, self, &vcaa->vcaa_caa); 314 } 315 316 static void 317 cpu_vm_init(struct cpu_info *ci) 318 { 319 int ncolors = 2, i; 320 321 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 322 struct x86_cache_info *cai; 323 int tcolors; 324 325 cai = &ci->ci_cinfo[i]; 326 327 tcolors = atop(cai->cai_totalsize); 328 switch(cai->cai_associativity) { 329 case 0xff: 330 tcolors = 1; /* fully associative */ 331 break; 332 case 0: 333 case 1: 334 break; 335 default: 336 tcolors /= cai->cai_associativity; 337 } 338 ncolors = max(ncolors, tcolors); 339 } 340 341 /* 342 * Knowing the size of the largest cache on this CPU, re-color 343 * our pages. 344 */ 345 if (ncolors <= uvmexp.ncolors) 346 return; 347 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 348 uvm_page_recolor(ncolors); 349 } 350 351 void 352 cpu_attach_common(device_t parent, device_t self, void *aux) 353 { 354 struct cpu_softc *sc = device_private(self); 355 struct cpu_attach_args *caa = aux; 356 struct cpu_info *ci; 357 uintptr_t ptr; 358 int cpunum = caa->cpu_number; 359 360 sc->sc_dev = self; 361 362 /* 363 * If we're an Application Processor, allocate a cpu_info 364 * structure, otherwise use the primary's. 365 */ 366 if (caa->cpu_role == CPU_ROLE_AP) { 367 if (cpunum >= X86_MAXPROCS) { 368 aprint_error(": apic id %d ignored, " 369 "please increase X86_MAXPROCS\n", cpunum); 370 } 371 372 aprint_naive(": Application Processor\n"); 373 ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 374 KM_SLEEP); 375 ci = (struct cpu_info *)((ptr + CACHE_LINE_SIZE - 1) & 376 ~(CACHE_LINE_SIZE - 1)); 377 memset(ci, 0, sizeof(*ci)); 378 #if defined(MULTIPROCESSOR) 379 if (cpu_info[cpunum] != NULL) 380 panic("cpu at apic id %d already attached?", cpunum); 381 cpu_info[cpunum] = ci; 382 #endif 383 #ifdef TRAPLOG 384 ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP); 385 #endif 386 } else { 387 aprint_naive(": %s Processor\n", 388 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 389 ci = &cpu_info_primary; 390 #if defined(MULTIPROCESSOR) 391 if (cpunum != lapic_cpu_number()) { 392 panic("%s: running CPU is at apic %d" 393 " instead of at expected %d", 394 device_xname(sc->sc_dev), lapic_cpu_number(), cpunum); 395 } 396 #endif 397 } 398 399 ci->ci_self = ci; 400 sc->sc_info = ci; 401 402 ci->ci_dev = self; 403 ci->ci_cpuid = cpunum; 404 405 KASSERT(HYPERVISOR_shared_info != NULL); 406 ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum]; 407 408 ci->ci_func = caa->cpu_func; 409 410 if (caa->cpu_role == CPU_ROLE_AP) { 411 #if defined(MULTIPROCESSOR) 412 int error; 413 414 error = mi_cpu_attach(ci); 415 if (error != 0) { 416 aprint_normal("\n"); 417 aprint_error_dev(sc->sc_dev, "mi_cpu_attach failed with %d\n", 418 error); 419 return; 420 } 421 #endif 422 } else { 423 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 424 } 425 426 ci->ci_cpumask = (1 << cpu_index(ci)); 427 pmap_reference(pmap_kernel()); 428 ci->ci_pmap = pmap_kernel(); 429 ci->ci_tlbstate = TLBSTATE_STALE; 430 431 /* further PCB init done later. */ 432 433 switch (caa->cpu_role) { 434 case CPU_ROLE_SP: 435 atomic_or_32(&ci->ci_flags, 436 CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY); 437 cpu_intr_init(ci); 438 cpu_get_tsc_freq(ci); 439 cpu_identify(ci); 440 cpu_init(ci); 441 cpu_set_tss_gates(ci); 442 pmap_cpu_init_late(ci); 443 x86_cpu_idle_init(); 444 #if 0 445 x86_errata(); 446 #endif 447 break; 448 449 case CPU_ROLE_BP: 450 atomic_or_32(&ci->ci_flags, 451 CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY); 452 cpu_intr_init(ci); 453 cpu_get_tsc_freq(ci); 454 cpu_identify(ci); 455 cpu_init(ci); 456 cpu_set_tss_gates(ci); 457 pmap_cpu_init_late(ci); 458 x86_cpu_idle_init(); 459 #if NLAPIC > 0 460 /* 461 * Enable local apic 462 */ 463 lapic_enable(); 464 lapic_set_lvt(); 465 lapic_calibrate_timer(ci); 466 #endif 467 #if 0 468 x86_errata(); 469 #endif 470 break; 471 472 case CPU_ROLE_AP: 473 /* 474 * report on an AP 475 */ 476 477 #if defined(MULTIPROCESSOR) 478 cpu_intr_init(ci); 479 gdt_alloc_cpu(ci); 480 cpu_set_tss_gates(ci); 481 pmap_cpu_init_early(ci); 482 pmap_cpu_init_late(ci); 483 cpu_start_secondary(ci); 484 if (ci->ci_flags & CPUF_PRESENT) { 485 struct cpu_info *tmp; 486 487 identifycpu(ci); 488 tmp = cpu_info_list; 489 while (tmp->ci_next) 490 tmp = tmp->ci_next; 491 492 tmp->ci_next = ci; 493 } 494 #else 495 aprint_normal_dev(sc->sc_dev, "not started\n"); 496 #endif 497 break; 498 499 default: 500 aprint_normal("\n"); 501 panic("unknown processor type??\n"); 502 } 503 cpu_vm_init(ci); 504 505 atomic_or_32(&cpus_attached, ci->ci_cpumask); 506 507 #if 0 508 if (!pmf_device_register(self, cpu_suspend, cpu_resume)) 509 aprint_error_dev(self, "couldn't establish power handler\n"); 510 #endif 511 512 #if defined(MULTIPROCESSOR) 513 if (mp_verbose) { 514 struct lwp *l = ci->ci_data.cpu_idlelwp; 515 516 aprint_verbose_dev(sc->sc_dev, "idle lwp at %p, idle sp at 0x%p\n", 517 l, 518 #ifdef i386 519 (void *)l->l_addr->u_pcb.pcb_esp 520 #else 521 (void *)l->l_addr->u_pcb.pcb_rsp 522 #endif 523 ); 524 525 } 526 #endif 527 } 528 529 /* 530 * Initialize the processor appropriately. 531 */ 532 533 void 534 cpu_init(struct cpu_info *ci) 535 { 536 537 /* 538 * On a P6 or above, enable global TLB caching if the 539 * hardware supports it. 540 */ 541 if (cpu_feature & CPUID_PGE) 542 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 543 544 #ifdef XXXMTRR 545 /* 546 * On a P6 or above, initialize MTRR's if the hardware supports them. 547 */ 548 if (cpu_feature & CPUID_MTRR) { 549 if ((ci->ci_flags & CPUF_AP) == 0) 550 i686_mtrr_init_first(); 551 mtrr_init_cpu(ci); 552 } 553 #endif 554 /* 555 * If we have FXSAVE/FXRESTOR, use them. 556 */ 557 if (cpu_feature & CPUID_FXSR) { 558 lcr4(rcr4() | CR4_OSFXSR); 559 560 /* 561 * If we have SSE/SSE2, enable XMM exceptions. 562 */ 563 if (cpu_feature & (CPUID_SSE|CPUID_SSE2)) 564 lcr4(rcr4() | CR4_OSXMMEXCPT); 565 } 566 567 atomic_or_32(&cpus_running, ci->ci_cpumask); 568 569 wbinvd(); 570 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 571 } 572 573 574 #ifdef MULTIPROCESSOR 575 void 576 cpu_boot_secondary_processors(void) 577 { 578 struct cpu_info *ci; 579 u_long i; 580 581 for (i = 0; i < X86_MAXPROCS; i++) { 582 ci = cpu_info[i]; 583 if (ci == NULL) 584 continue; 585 if (ci->ci_data.cpu_idlelwp == NULL) 586 continue; 587 if ((ci->ci_flags & CPUF_PRESENT) == 0) 588 continue; 589 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 590 continue; 591 cpu_boot_secondary(ci); 592 } 593 594 x86_mp_online = true; 595 } 596 597 static void 598 cpu_init_idle_lwp(struct cpu_info *ci) 599 { 600 struct lwp *l = ci->ci_data.cpu_idlelwp; 601 struct pcb *pcb = &l->l_addr->u_pcb; 602 603 pcb->pcb_cr0 = rcr0(); 604 } 605 606 void 607 cpu_init_idle_lwps(void) 608 { 609 struct cpu_info *ci; 610 u_long i; 611 612 for (i = 0; i < X86_MAXPROCS; i++) { 613 ci = cpu_info[i]; 614 if (ci == NULL) 615 continue; 616 if (ci->ci_data.cpu_idlelwp == NULL) 617 continue; 618 if ((ci->ci_flags & CPUF_PRESENT) == 0) 619 continue; 620 cpu_init_idle_lwp(ci); 621 } 622 } 623 624 void 625 cpu_start_secondary(struct cpu_info *ci) 626 { 627 int i; 628 struct pmap *kpm = pmap_kernel(); 629 extern uint32_t mp_pdirpa; 630 631 mp_pdirpa = kpm->pm_pdirpa; /* XXX move elsewhere, not per CPU. */ 632 633 atomic_or_32(&ci->ci_flags, CPUF_AP); 634 635 aprint_debug_dev(ci->ci_dev, "starting\n"); 636 637 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 638 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) 639 return; 640 641 /* 642 * wait for it to become ready 643 */ 644 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 645 #ifdef MPDEBUG 646 extern int cpu_trace[3]; 647 static int otrace[3]; 648 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) { 649 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n", 650 cpu_trace[0], cpu_trace[1], cpu_trace[2]); 651 memcpy(otrace, cpu_trace, sizeof(otrace)); 652 } 653 #endif 654 delay(10); 655 } 656 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 657 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 658 #if defined(MPDEBUG) && defined(DDB) 659 printf("dropping into debugger; continue from here to resume boot\n"); 660 Debugger(); 661 #endif 662 } 663 664 CPU_START_CLEANUP(ci); 665 } 666 667 void 668 cpu_boot_secondary(struct cpu_info *ci) 669 { 670 int i; 671 672 atomic_or_32(&ci->ci_flags, CPUF_GO); 673 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 674 delay(10); 675 } 676 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 677 aprint_error_dev(ci->ci_dev, "CPU failed to start\n"); 678 #if defined(MPDEBUG) && defined(DDB) 679 printf("dropping into debugger; continue from here to resume boot\n"); 680 Debugger(); 681 #endif 682 } 683 } 684 685 /* 686 * The CPU ends up here when its ready to run 687 * This is called from code in mptramp.s; at this point, we are running 688 * in the idle pcb/idle stack of the new CPU. When this function returns, 689 * this processor will enter the idle loop and start looking for work. 690 * 691 * XXX should share some of this with init386 in machdep.c 692 */ 693 void 694 cpu_hatch(void *v) 695 { 696 struct cpu_info *ci = (struct cpu_info *)v; 697 int s, i; 698 uint32_t blacklist_features; 699 700 #ifdef __x86_64__ 701 cpu_init_msrs(ci, true); 702 #endif 703 704 cpu_probe(ci); 705 706 /* not on Xen... */ 707 blacklist_features = ~(CPUID_PGE|CPUID_PSE|CPUID_MTRR|CPUID_FXSR|CPUID_NOX); /* XXX add CPUID_SVM */ 708 709 cpu_feature &= blacklist_features; 710 711 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 712 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 713 while ((ci->ci_flags & CPUF_GO) == 0) { 714 /* Don't use delay, boot CPU may be patching the text. */ 715 for (i = 10000; i != 0; i--) 716 x86_pause(); 717 } 718 719 /* Because the text may have been patched in x86_patch(). */ 720 wbinvd(); 721 x86_flush(); 722 723 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 724 725 lcr3(pmap_kernel()->pm_pdirpa); 726 curlwp->l_addr->u_pcb.pcb_cr3 = pmap_kernel()->pm_pdirpa; 727 lcr0(ci->ci_data.cpu_idlelwp->l_addr->u_pcb.pcb_cr0); 728 cpu_init_idt(); 729 gdt_init_cpu(ci); 730 lapic_enable(); 731 lapic_set_lvt(); 732 lapic_initclocks(); 733 734 #ifdef i386 735 npxinit(ci); 736 #else 737 fpuinit(ci); 738 #endif 739 740 lldt(GSEL(GLDT_SEL, SEL_KPL)); 741 ltr(ci->ci_tss_sel); 742 743 cpu_init(ci); 744 cpu_get_tsc_freq(ci); 745 746 s = splhigh(); 747 #ifdef i386 748 lapic_tpr = 0; 749 #else 750 lcr8(0); 751 #endif 752 x86_enable_intr(); 753 splx(s); 754 #if 0 755 x86_errata(); 756 #endif 757 758 aprint_debug_dev(ci->ci_dev, "CPU %ld running\n", 759 (long)ci->ci_cpuid); 760 } 761 762 #if defined(DDB) 763 764 #include <ddb/db_output.h> 765 #include <machine/db_machdep.h> 766 767 /* 768 * Dump CPU information from ddb. 769 */ 770 void 771 cpu_debug_dump(void) 772 { 773 struct cpu_info *ci; 774 CPU_INFO_ITERATOR cii; 775 776 db_printf("addr dev id flags ipis curlwp fpcurlwp\n"); 777 for (CPU_INFO_FOREACH(cii, ci)) { 778 db_printf("%p %s %ld %x %x %10p %10p\n", 779 ci, 780 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 781 (long)ci->ci_cpuid, 782 ci->ci_flags, ci->ci_ipis, 783 ci->ci_curlwp, 784 ci->ci_fpcurlwp); 785 } 786 } 787 #endif 788 789 static void 790 cpu_copy_trampoline(void) 791 { 792 /* 793 * Copy boot code. 794 */ 795 extern u_char cpu_spinup_trampoline[]; 796 extern u_char cpu_spinup_trampoline_end[]; 797 798 vaddr_t mp_trampoline_vaddr; 799 800 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 801 UVM_KMF_VAONLY); 802 803 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 804 VM_PROT_READ | VM_PROT_WRITE); 805 pmap_update(pmap_kernel()); 806 memcpy((void *)mp_trampoline_vaddr, 807 cpu_spinup_trampoline, 808 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 809 810 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 811 pmap_update(pmap_kernel()); 812 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 813 } 814 815 #endif 816 817 #ifdef i386 818 #if 0 819 static void 820 tss_init(struct i386tss *tss, void *stack, void *func) 821 { 822 memset(tss, 0, sizeof *tss); 823 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 824 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 825 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 826 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 827 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 828 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 829 tss->tss_cr3 = pmap_kernel()->pm_pdirpa; 830 tss->tss_esp = (int)((char *)stack + USPACE - 16); 831 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 832 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 833 tss->__tss_eip = (int)func; 834 } 835 #endif 836 837 /* XXX */ 838 #define IDTVEC(name) __CONCAT(X, name) 839 typedef void (vector)(void); 840 extern vector IDTVEC(tss_trap08); 841 #ifdef DDB 842 extern vector Xintrddbipi; 843 extern int ddb_vec; 844 #endif 845 846 static void 847 cpu_set_tss_gates(struct cpu_info *ci) 848 { 849 #if 0 850 struct segment_descriptor sd; 851 852 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 853 UVM_KMF_WIRED); 854 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 855 IDTVEC(tss_trap08)); 856 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 857 SDT_SYS386TSS, SEL_KPL, 0, 0); 858 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 859 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 860 GSEL(GTRAPTSS_SEL, SEL_KPL)); 861 #endif 862 863 #if defined(DDB) && defined(MULTIPROCESSOR) 864 /* 865 * Set up separate handler for the DDB IPI, so that it doesn't 866 * stomp on a possibly corrupted stack. 867 * 868 * XXX overwriting the gate set in db_machine_init. 869 * Should rearrange the code so that it's set only once. 870 */ 871 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 872 UVM_KMF_WIRED); 873 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, 874 Xintrddbipi); 875 876 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 877 SDT_SYS386TSS, SEL_KPL, 0, 0); 878 ci->ci_gdt[GIPITSS_SEL].sd = sd; 879 880 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 881 GSEL(GIPITSS_SEL, SEL_KPL)); 882 #endif 883 } 884 #else 885 static void 886 cpu_set_tss_gates(struct cpu_info *ci) 887 { 888 889 } 890 #endif /* i386 */ 891 892 int 893 mp_cpu_start(struct cpu_info *ci, paddr_t target) 894 { 895 #if 0 896 #if NLAPIC > 0 897 int error; 898 #endif 899 unsigned short dwordptr[2]; 900 901 /* 902 * Bootstrap code must be addressable in real mode 903 * and it must be page aligned. 904 */ 905 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 906 907 /* 908 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 909 */ 910 911 outb(IO_RTC, NVRAM_RESET); 912 outb(IO_RTC+1, NVRAM_RESET_JUMP); 913 914 /* 915 * "and the warm reset vector (DWORD based at 40:67) to point 916 * to the AP startup code ..." 917 */ 918 919 dwordptr[0] = 0; 920 dwordptr[1] = target >> 4; 921 922 pmap_kenter_pa (0, 0, VM_PROT_READ|VM_PROT_WRITE); 923 memcpy ((uint8_t *) 0x467, dwordptr, 4); 924 pmap_kremove (0, PAGE_SIZE); 925 926 #if NLAPIC > 0 927 /* 928 * ... prior to executing the following sequence:" 929 */ 930 931 if (ci->ci_flags & CPUF_AP) { 932 if ((error = x86_ipi_init(ci->ci_cpuid)) != 0) 933 return error; 934 935 delay(10000); 936 937 if (cpu_feature & CPUID_APIC) { 938 error = x86_ipi_init(ci->ci_cpuid); 939 if (error != 0) { 940 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 941 __func__); 942 return error; 943 } 944 945 delay(10000); 946 947 error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid, 948 LAPIC_DLMODE_STARTUP); 949 if (error != 0) { 950 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 951 __func__); 952 return error; 953 } 954 delay(200); 955 956 error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid, 957 LAPIC_DLMODE_STARTUP); 958 if (error != 0) { 959 aprint_error_dev(ci->ci_dev, "%s: IPI not taken ((3)\n", 960 __func__); 961 return error; 962 } 963 delay(200); 964 } 965 } 966 #endif 967 #endif /* 0 */ 968 return 0; 969 } 970 971 void 972 mp_cpu_start_cleanup(struct cpu_info *ci) 973 { 974 #if 0 975 /* 976 * Ensure the NVRAM reset byte contains something vaguely sane. 977 */ 978 979 outb(IO_RTC, NVRAM_RESET); 980 outb(IO_RTC+1, NVRAM_RESET_RST); 981 #endif 982 } 983 984 #ifdef __x86_64__ 985 986 void 987 cpu_init_msrs(struct cpu_info *ci, bool full) 988 { 989 if (full) { 990 HYPERVISOR_set_segment_base (SEGBASE_FS, 0); 991 HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci); 992 HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0); 993 } 994 } 995 #endif /* __x86_64__ */ 996 997 void 998 cpu_offline_md(void) 999 { 1000 int s; 1001 1002 s = splhigh(); 1003 #ifdef __i386__ 1004 npxsave_cpu(true); 1005 #else 1006 fpusave_cpu(true); 1007 #endif 1008 splx(s); 1009 } 1010 1011 #if 0 1012 /* XXX joerg restructure and restart CPUs individually */ 1013 static bool 1014 cpu_suspend(device_t dv PMF_FN_ARGS) 1015 { 1016 struct cpu_softc *sc = device_private(dv); 1017 struct cpu_info *ci = sc->sc_info; 1018 int err; 1019 1020 if (ci->ci_flags & CPUF_PRIMARY) 1021 return true; 1022 if (ci->ci_data.cpu_idlelwp == NULL) 1023 return true; 1024 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1025 return true; 1026 1027 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1028 1029 if (sc->sc_wasonline) { 1030 mutex_enter(&cpu_lock); 1031 err = cpu_setstate(ci, false); 1032 mutex_exit(&cpu_lock); 1033 1034 if (err) 1035 return false; 1036 } 1037 1038 return true; 1039 } 1040 1041 static bool 1042 cpu_resume(device_t dv PMF_FN_ARGS) 1043 { 1044 struct cpu_softc *sc = device_private(dv); 1045 struct cpu_info *ci = sc->sc_info; 1046 int err = 0; 1047 1048 if (ci->ci_flags & CPUF_PRIMARY) 1049 return true; 1050 if (ci->ci_data.cpu_idlelwp == NULL) 1051 return true; 1052 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1053 return true; 1054 1055 if (sc->sc_wasonline) { 1056 mutex_enter(&cpu_lock); 1057 err = cpu_setstate(ci, true); 1058 mutex_exit(&cpu_lock); 1059 } 1060 1061 return err == 0; 1062 } 1063 #endif 1064 1065 void 1066 cpu_get_tsc_freq(struct cpu_info *ci) 1067 { 1068 const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time; 1069 delay(1000000); 1070 uint64_t freq = 1000000000ULL << 32; 1071 freq = freq / (uint64_t)tinfo->tsc_to_system_mul; 1072 if ( tinfo->tsc_shift < 0 ) 1073 freq = freq << -tinfo->tsc_shift; 1074 else 1075 freq = freq >> tinfo->tsc_shift; 1076 ci->ci_data.cpu_cc_freq = freq; 1077 } 1078 1079 void 1080 x86_cpu_idle_xen(void) 1081 { 1082 struct cpu_info *ci = curcpu(); 1083 1084 KASSERT(ci->ci_ilevel == IPL_NONE); 1085 1086 x86_disable_intr(); 1087 if (!__predict_false(ci->ci_want_resched)) { 1088 idle_block(); 1089 } else { 1090 x86_enable_intr(); 1091 } 1092 } 1093