1 /* $NetBSD: cpu.c,v 1.31 2008/12/23 20:06:16 cegger Exp $ */ 2 /* NetBSD: cpu.c,v 1.18 2004/02/20 17:35:01 yamt Exp */ 3 4 /*- 5 * Copyright (c) 2000 The NetBSD Foundation, Inc. 6 * Copyright (c) 2002, 2006, 2007 YAMAMOTO Takashi, 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to The NetBSD Foundation 10 * by RedBack Networks Inc. 11 * 12 * Author: Bill Sommerfeld 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 24 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 25 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 26 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 27 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 28 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 29 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 30 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 31 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 32 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 33 * POSSIBILITY OF SUCH DAMAGE. 34 */ 35 36 /* 37 * Copyright (c) 1999 Stefan Grefen 38 * 39 * Redistribution and use in source and binary forms, with or without 40 * modification, are permitted provided that the following conditions 41 * are met: 42 * 1. Redistributions of source code must retain the above copyright 43 * notice, this list of conditions and the following disclaimer. 44 * 2. Redistributions in binary form must reproduce the above copyright 45 * notice, this list of conditions and the following disclaimer in the 46 * documentation and/or other materials provided with the distribution. 47 * 3. All advertising materials mentioning features or use of this software 48 * must display the following acknowledgement: 49 * This product includes software developed by the NetBSD 50 * Foundation, Inc. and its contributors. 51 * 4. Neither the name of The NetBSD Foundation nor the names of its 52 * contributors may be used to endorse or promote products derived 53 * from this software without specific prior written permission. 54 * 55 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND ANY 56 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 57 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 58 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR AND CONTRIBUTORS BE LIABLE 59 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 60 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 61 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 62 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 63 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 64 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 65 * SUCH DAMAGE. 66 */ 67 68 #include <sys/cdefs.h> 69 __KERNEL_RCSID(0, "$NetBSD: cpu.c,v 1.31 2008/12/23 20:06:16 cegger Exp $"); 70 71 #include "opt_ddb.h" 72 #include "opt_multiprocessor.h" 73 #include "opt_mpbios.h" /* for MPDEBUG */ 74 #include "opt_mtrr.h" 75 #include "opt_xen.h" 76 77 #include "lapic.h" 78 #include "ioapic.h" 79 80 #include <sys/param.h> 81 #include <sys/proc.h> 82 #include <sys/user.h> 83 #include <sys/systm.h> 84 #include <sys/device.h> 85 #include <sys/kmem.h> 86 #include <sys/cpu.h> 87 #include <sys/atomic.h> 88 89 #include <uvm/uvm_extern.h> 90 91 #include <machine/cpufunc.h> 92 #include <machine/cpuvar.h> 93 #include <machine/pmap.h> 94 #include <machine/vmparam.h> 95 #include <machine/mpbiosvar.h> 96 #include <machine/pcb.h> 97 #include <machine/specialreg.h> 98 #include <machine/segments.h> 99 #include <machine/gdt.h> 100 #include <machine/mtrr.h> 101 #include <machine/pio.h> 102 103 #ifdef XEN3 104 #include <xen/vcpuvar.h> 105 #endif 106 107 #if NLAPIC > 0 108 #include <machine/apicvar.h> 109 #include <machine/i82489reg.h> 110 #include <machine/i82489var.h> 111 #endif 112 113 #include <dev/ic/mc146818reg.h> 114 #include <dev/isa/isareg.h> 115 116 #define X86_MAXPROCS 32 117 118 int cpu_match(device_t, cfdata_t, void *); 119 void cpu_attach(device_t, device_t, void *); 120 #ifdef XEN3 121 int vcpu_match(device_t, cfdata_t, void *); 122 void vcpu_attach(device_t, device_t, void *); 123 #endif 124 void cpu_attach_common(device_t, device_t, void *); 125 void cpu_offline_md(void); 126 127 struct cpu_softc { 128 device_t sc_dev; /* device tree glue */ 129 struct cpu_info *sc_info; /* pointer to CPU info */ 130 }; 131 132 int mp_cpu_start(struct cpu_info *, paddr_t); 133 void mp_cpu_start_cleanup(struct cpu_info *); 134 const struct cpu_functions mp_cpu_funcs = { mp_cpu_start, NULL, 135 mp_cpu_start_cleanup }; 136 137 CFATTACH_DECL_NEW(cpu, sizeof(struct cpu_softc), 138 cpu_match, cpu_attach, NULL, NULL); 139 #ifdef XEN3 140 CFATTACH_DECL_NEW(vcpu, sizeof(struct cpu_softc), 141 vcpu_match, vcpu_attach, NULL, NULL); 142 #endif 143 144 /* 145 * Statically-allocated CPU info for the primary CPU (or the only 146 * CPU, on uniprocessors). The CPU info list is initialized to 147 * point at it. 148 */ 149 #ifdef TRAPLOG 150 #include <machine/tlog.h> 151 struct tlog tlog_primary; 152 #endif 153 struct cpu_info cpu_info_primary = { 154 .ci_dev = 0, 155 .ci_self = &cpu_info_primary, 156 .ci_idepth = -1, 157 .ci_curlwp = &lwp0, 158 .ci_curldt = -1, 159 #ifdef TRAPLOG 160 .ci_tlog = &tlog_primary, 161 #endif 162 163 }; 164 struct cpu_info phycpu_info_primary = { 165 .ci_dev = 0, 166 .ci_self = &phycpu_info_primary, 167 }; 168 169 struct cpu_info *cpu_info_list = &cpu_info_primary; 170 171 static void cpu_set_tss_gates(struct cpu_info *ci); 172 173 uint32_t cpus_attached = 0; 174 uint32_t cpus_running = 0; 175 176 bool x86_mp_online; 177 paddr_t mp_trampoline_paddr = MP_TRAMPOLINE; 178 179 struct cpu_info *phycpu_info[X86_MAXPROCS] = { &cpu_info_primary }; 180 181 #ifdef MULTIPROCESSOR 182 /* 183 * Array of CPU info structures. Must be statically-allocated because 184 * curproc, etc. are used early. 185 */ 186 struct cpu_info *cpu_info[X86_MAXPROCS] = { &cpu_info_primary }; 187 188 void cpu_hatch(void *); 189 static void cpu_boot_secondary(struct cpu_info *ci); 190 static void cpu_start_secondary(struct cpu_info *ci); 191 static void cpu_copy_trampoline(void); 192 193 /* 194 * Runs once per boot once multiprocessor goo has been detected and 195 * the local APIC on the boot processor has been mapped. 196 * 197 * Called from lapic_boot_init() (from mpbios_scan()). 198 */ 199 void 200 cpu_init_first(void) 201 { 202 int cpunum = lapic_cpu_number(); 203 204 if (cpunum != 0) { 205 cpu_info[0] = NULL; 206 cpu_info[cpunum] = &cpu_info_primary; 207 } 208 209 cpu_copy_trampoline(); 210 } 211 #endif 212 213 int 214 cpu_match(device_t parent, cfdata_t match, void *aux) 215 { 216 217 return 1; 218 } 219 220 void 221 cpu_attach(device_t parent, device_t self, void *aux) 222 { 223 #ifdef XEN3 224 struct cpu_softc *sc = device_private(self); 225 struct cpu_attach_args *caa = aux; 226 struct cpu_info *ci; 227 int cpunum = caa->cpu_number; 228 229 sc->sc_dev = self; 230 231 /* 232 * If we're an Application Processor, allocate a cpu_info 233 * structure, otherwise use the primary's. 234 */ 235 if (caa->cpu_role == CPU_ROLE_AP) { 236 ci = kmem_zalloc(sizeof(*ci), KM_SLEEP); 237 ci->ci_curldt = -1; 238 if (phycpu_info[cpunum] != NULL) 239 panic("cpu at apic id %d already attached?", cpunum); 240 phycpu_info[cpunum] = ci; 241 } else { 242 ci = &phycpu_info_primary; 243 if (cpunum != 0) { 244 phycpu_info[0] = NULL; 245 phycpu_info[cpunum] = ci; 246 } 247 } 248 249 ci->ci_self = ci; 250 sc->sc_info = ci; 251 252 ci->ci_dev = self; 253 ci->ci_cpuid = caa->cpu_number; 254 ci->ci_vcpu = NULL; 255 256 printf(": "); 257 switch (caa->cpu_role) { 258 case CPU_ROLE_SP: 259 printf("(uniprocessor)\n"); 260 ci->ci_flags |= CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY; 261 break; 262 263 case CPU_ROLE_BP: 264 printf("(boot processor)\n"); 265 ci->ci_flags |= CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY; 266 break; 267 268 case CPU_ROLE_AP: 269 /* 270 * report on an AP 271 */ 272 printf("(application processor)\n"); 273 break; 274 275 default: 276 panic("unknown processor type??\n"); 277 } 278 return; 279 #else 280 cpu_attach_common(parent, self, aux); 281 #endif 282 } 283 284 #ifdef XEN3 285 int 286 vcpu_match(device_t parent, cfdata_t match, void *aux) 287 { 288 struct vcpu_attach_args *vcaa = aux; 289 290 if (strcmp(vcaa->vcaa_name, match->cf_name) == 0) 291 return 1; 292 return 0; 293 } 294 295 void 296 vcpu_attach(device_t parent, device_t self, void *aux) 297 { 298 struct vcpu_attach_args *vcaa = aux; 299 300 cpu_attach_common(parent, self, &vcaa->vcaa_caa); 301 } 302 #endif 303 304 static void 305 cpu_vm_init(struct cpu_info *ci) 306 { 307 int ncolors = 2, i; 308 309 for (i = CAI_ICACHE; i <= CAI_L2CACHE; i++) { 310 struct x86_cache_info *cai; 311 int tcolors; 312 313 cai = &ci->ci_cinfo[i]; 314 315 tcolors = atop(cai->cai_totalsize); 316 switch(cai->cai_associativity) { 317 case 0xff: 318 tcolors = 1; /* fully associative */ 319 break; 320 case 0: 321 case 1: 322 break; 323 default: 324 tcolors /= cai->cai_associativity; 325 } 326 ncolors = max(ncolors, tcolors); 327 } 328 329 /* 330 * Knowing the size of the largest cache on this CPU, re-color 331 * our pages. 332 */ 333 if (ncolors <= uvmexp.ncolors) 334 return; 335 aprint_debug_dev(ci->ci_dev, "%d page colors\n", ncolors); 336 uvm_page_recolor(ncolors); 337 } 338 339 void 340 cpu_attach_common(device_t parent, device_t self, void *aux) 341 { 342 struct cpu_softc *sc = device_private(self); 343 struct cpu_attach_args *caa = aux; 344 struct cpu_info *ci; 345 uintptr_t ptr; 346 int cpunum = caa->cpu_number; 347 348 sc->sc_dev = self; 349 350 /* 351 * If we're an Application Processor, allocate a cpu_info 352 * structure, otherwise use the primary's. 353 */ 354 if (caa->cpu_role == CPU_ROLE_AP) { 355 if (cpunum >= X86_MAXPROCS) { 356 aprint_error(": apic id %d ignored, " 357 "please increase X86_MAXPROCS\n", cpunum); 358 } 359 360 aprint_naive(": Application Processor\n"); 361 ptr = (uintptr_t)kmem_alloc(sizeof(*ci) + CACHE_LINE_SIZE - 1, 362 KM_SLEEP); 363 ci = (struct cpu_info *)((ptr + CACHE_LINE_SIZE - 1) & 364 ~(CACHE_LINE_SIZE - 1)); 365 memset(ci, 0, sizeof(*ci)); 366 #if defined(MULTIPROCESSOR) 367 if (cpu_info[cpunum] != NULL) 368 panic("cpu at apic id %d already attached?", cpunum); 369 cpu_info[cpunum] = ci; 370 #endif 371 #ifdef TRAPLOG 372 ci->ci_tlog_base = kmem_zalloc(sizeof(struct tlog), KM_SLEEP); 373 #endif 374 } else { 375 aprint_naive(": %s Processor\n", 376 caa->cpu_role == CPU_ROLE_SP ? "Single" : "Boot"); 377 ci = &cpu_info_primary; 378 #if defined(MULTIPROCESSOR) 379 if (cpunum != lapic_cpu_number()) { 380 panic("%s: running CPU is at apic %d" 381 " instead of at expected %d", 382 device_xname(sc->sc_dev), lapic_cpu_number(), cpunum); 383 } 384 #endif 385 } 386 387 ci->ci_self = ci; 388 sc->sc_info = ci; 389 390 ci->ci_dev = self; 391 ci->ci_cpuid = cpunum; 392 393 KASSERT(HYPERVISOR_shared_info != NULL); 394 ci->ci_vcpu = &HYPERVISOR_shared_info->vcpu_info[cpunum]; 395 396 ci->ci_func = caa->cpu_func; 397 398 if (caa->cpu_role == CPU_ROLE_AP) { 399 #if defined(MULTIPROCESSOR) 400 int error; 401 402 error = mi_cpu_attach(ci); 403 if (error != 0) { 404 aprint_normal("\n"); 405 aprint_error_dev(sc->sc_dev, "mi_cpu_attach failed with %d\n", 406 error); 407 return; 408 } 409 #endif 410 } else { 411 KASSERT(ci->ci_data.cpu_idlelwp != NULL); 412 } 413 414 ci->ci_cpumask = (1 << cpu_index(ci)); 415 pmap_reference(pmap_kernel()); 416 ci->ci_pmap = pmap_kernel(); 417 ci->ci_tlbstate = TLBSTATE_STALE; 418 419 /* further PCB init done later. */ 420 421 switch (caa->cpu_role) { 422 case CPU_ROLE_SP: 423 atomic_or_32(&ci->ci_flags, 424 CPUF_PRESENT | CPUF_SP | CPUF_PRIMARY); 425 cpu_intr_init(ci); 426 cpu_get_tsc_freq(ci); 427 cpu_identify(ci); 428 cpu_init(ci); 429 cpu_set_tss_gates(ci); 430 pmap_cpu_init_late(ci); 431 x86_cpu_idle_init(); 432 #if 0 433 x86_errata(); 434 #endif 435 break; 436 437 case CPU_ROLE_BP: 438 atomic_or_32(&ci->ci_flags, 439 CPUF_PRESENT | CPUF_BSP | CPUF_PRIMARY); 440 cpu_intr_init(ci); 441 cpu_get_tsc_freq(ci); 442 cpu_identify(ci); 443 cpu_init(ci); 444 cpu_set_tss_gates(ci); 445 pmap_cpu_init_late(ci); 446 x86_cpu_idle_init(); 447 #if NLAPIC > 0 448 /* 449 * Enable local apic 450 */ 451 lapic_enable(); 452 lapic_set_lvt(); 453 lapic_calibrate_timer(ci); 454 #endif 455 #if 0 456 x86_errata(); 457 #endif 458 break; 459 460 case CPU_ROLE_AP: 461 /* 462 * report on an AP 463 */ 464 465 #if defined(MULTIPROCESSOR) 466 cpu_intr_init(ci); 467 gdt_alloc_cpu(ci); 468 cpu_set_tss_gates(ci); 469 pmap_cpu_init_early(ci); 470 pmap_cpu_init_late(ci); 471 cpu_start_secondary(ci); 472 if (ci->ci_flags & CPUF_PRESENT) { 473 struct cpu_info *tmp; 474 475 identifycpu(ci); 476 tmp = cpu_info_list; 477 while (tmp->ci_next) 478 tmp = tmp->ci_next; 479 480 tmp->ci_next = ci; 481 } 482 #else 483 aprint_normal_dev(sc->sc_dev, "not started\n"); 484 #endif 485 break; 486 487 default: 488 aprint_normal("\n"); 489 panic("unknown processor type??\n"); 490 } 491 cpu_vm_init(ci); 492 493 cpus_attached |= (1 << ci->ci_cpuid); 494 495 #if 0 496 if (!pmf_device_register(self, cpu_suspend, cpu_resume)) 497 aprint_error_dev(self, "couldn't establish power handler\n"); 498 #endif 499 500 #if defined(MULTIPROCESSOR) 501 if (mp_verbose) { 502 struct lwp *l = ci->ci_data.cpu_idlelwp; 503 504 aprint_verbose_dev(sc->sc_dev, "idle lwp at %p, idle sp at 0x%p\n", 505 l, 506 #ifdef i386 507 (void *)l->l_addr->u_pcb.pcb_esp 508 #else 509 (void *)l->l_addr->u_pcb.pcb_rsp 510 #endif 511 ); 512 513 } 514 #endif 515 } 516 517 /* 518 * Initialize the processor appropriately. 519 */ 520 521 void 522 cpu_init(struct cpu_info *ci) 523 { 524 525 /* 526 * On a P6 or above, enable global TLB caching if the 527 * hardware supports it. 528 */ 529 if (cpu_feature & CPUID_PGE) 530 lcr4(rcr4() | CR4_PGE); /* enable global TLB caching */ 531 532 #ifdef XXXMTRR 533 /* 534 * On a P6 or above, initialize MTRR's if the hardware supports them. 535 */ 536 if (cpu_feature & CPUID_MTRR) { 537 if ((ci->ci_flags & CPUF_AP) == 0) 538 i686_mtrr_init_first(); 539 mtrr_init_cpu(ci); 540 } 541 #endif 542 /* 543 * If we have FXSAVE/FXRESTOR, use them. 544 */ 545 if (cpu_feature & CPUID_FXSR) { 546 lcr4(rcr4() | CR4_OSFXSR); 547 548 /* 549 * If we have SSE/SSE2, enable XMM exceptions. 550 */ 551 if (cpu_feature & (CPUID_SSE|CPUID_SSE2)) 552 lcr4(rcr4() | CR4_OSXMMEXCPT); 553 } 554 555 #ifdef MULTIPROCESSOR 556 atomic_or_32(&ci->ci_flags, CPUF_RUNNING); 557 atomic_or_32(&cpus_running, ci->ci_cpumask); 558 #endif 559 } 560 561 562 #ifdef MULTIPROCESSOR 563 void 564 cpu_boot_secondary_processors(void) 565 { 566 struct cpu_info *ci; 567 u_long i; 568 569 for (i = 0; i < X86_MAXPROCS; i++) { 570 ci = cpu_info[i]; 571 if (ci == NULL) 572 continue; 573 if (ci->ci_data.cpu_idlelwp == NULL) 574 continue; 575 if ((ci->ci_flags & CPUF_PRESENT) == 0) 576 continue; 577 if (ci->ci_flags & (CPUF_BSP|CPUF_SP|CPUF_PRIMARY)) 578 continue; 579 cpu_boot_secondary(ci); 580 } 581 582 x86_mp_online = true; 583 } 584 585 static void 586 cpu_init_idle_lwp(struct cpu_info *ci) 587 { 588 struct lwp *l = ci->ci_data.cpu_idlelwp; 589 struct pcb *pcb = &l->l_addr->u_pcb; 590 591 pcb->pcb_cr0 = rcr0(); 592 } 593 594 void 595 cpu_init_idle_lwps(void) 596 { 597 struct cpu_info *ci; 598 u_long i; 599 600 for (i = 0; i < X86_MAXPROCS; i++) { 601 ci = cpu_info[i]; 602 if (ci == NULL) 603 continue; 604 if (ci->ci_data.cpu_idlelwp == NULL) 605 continue; 606 if ((ci->ci_flags & CPUF_PRESENT) == 0) 607 continue; 608 cpu_init_idle_lwp(ci); 609 } 610 } 611 612 void 613 cpu_start_secondary(struct cpu_info *ci) 614 { 615 int i; 616 struct pmap *kpm = pmap_kernel(); 617 extern uint32_t mp_pdirpa; 618 619 mp_pdirpa = kpm->pm_pdirpa; /* XXX move elsewhere, not per CPU. */ 620 621 atomic_or_32(&ci->ci_flags, CPUF_AP); 622 623 aprint_debug_dev(ci->ci_dev, "starting\n"); 624 625 ci->ci_curlwp = ci->ci_data.cpu_idlelwp; 626 if (CPU_STARTUP(ci, mp_trampoline_paddr) != 0) 627 return; 628 629 /* 630 * wait for it to become ready 631 */ 632 for (i = 100000; (!(ci->ci_flags & CPUF_PRESENT)) && i > 0; i--) { 633 #ifdef MPDEBUG 634 extern int cpu_trace[3]; 635 static int otrace[3]; 636 if (memcmp(otrace, cpu_trace, sizeof(otrace)) != 0) { 637 aprint_debug_dev(ci->ci_dev, "trace %02x %02x %02x\n", 638 cpu_trace[0], cpu_trace[1], cpu_trace[2]); 639 memcpy(otrace, cpu_trace, sizeof(otrace)); 640 } 641 #endif 642 delay(10); 643 } 644 if ((ci->ci_flags & CPUF_PRESENT) == 0) { 645 aprint_error_dev(ci->ci_dev, "failed to become ready\n"); 646 #if defined(MPDEBUG) && defined(DDB) 647 printf("dropping into debugger; continue from here to resume boot\n"); 648 Debugger(); 649 #endif 650 } 651 652 CPU_START_CLEANUP(ci); 653 } 654 655 void 656 cpu_boot_secondary(struct cpu_info *ci) 657 { 658 int i; 659 660 atomic_or_32(&ci->ci_flags, CPUF_GO); 661 for (i = 100000; (!(ci->ci_flags & CPUF_RUNNING)) && i > 0; i--) { 662 delay(10); 663 } 664 if ((ci->ci_flags & CPUF_RUNNING) == 0) { 665 aprint_error_dev(ci->ci_dev, "CPU failed to start\n"); 666 #if defined(MPDEBUG) && defined(DDB) 667 printf("dropping into debugger; continue from here to resume boot\n"); 668 Debugger(); 669 #endif 670 } 671 } 672 673 /* 674 * The CPU ends up here when its ready to run 675 * This is called from code in mptramp.s; at this point, we are running 676 * in the idle pcb/idle stack of the new CPU. When this function returns, 677 * this processor will enter the idle loop and start looking for work. 678 * 679 * XXX should share some of this with init386 in machdep.c 680 */ 681 void 682 cpu_hatch(void *v) 683 { 684 struct cpu_info *ci = (struct cpu_info *)v; 685 int s, i; 686 uint32_t blacklist_features; 687 688 #ifdef __x86_64__ 689 cpu_init_msrs(ci, true); 690 #endif 691 692 cpu_probe(ci); 693 694 /* not on Xen... */ 695 blacklist_features = ~(CPUID_PGE|CPUID_PSE|CPUID_MTRR|CPUID_FXSR|CPUID_NOX); /* XXX add CPUID_SVM */ 696 697 cpu_feature &= blacklist_features; 698 699 KDASSERT((ci->ci_flags & CPUF_PRESENT) == 0); 700 atomic_or_32(&ci->ci_flags, CPUF_PRESENT); 701 while ((ci->ci_flags & CPUF_GO) == 0) { 702 /* Don't use delay, boot CPU may be patching the text. */ 703 for (i = 10000; i != 0; i--) 704 x86_pause(); 705 } 706 707 /* Because the text may have been patched in x86_patch(). */ 708 wbinvd(); 709 x86_flush(); 710 711 KASSERT((ci->ci_flags & CPUF_RUNNING) == 0); 712 713 lcr3(pmap_kernel()->pm_pdirpa); 714 curlwp->l_addr->u_pcb.pcb_cr3 = pmap_kernel()->pm_pdirpa; 715 lcr0(ci->ci_data.cpu_idlelwp->l_addr->u_pcb.pcb_cr0); 716 cpu_init_idt(); 717 gdt_init_cpu(ci); 718 lapic_enable(); 719 lapic_set_lvt(); 720 lapic_initclocks(); 721 722 #ifdef i386 723 npxinit(ci); 724 #else 725 fpuinit(ci); 726 #endif 727 728 lldt(GSEL(GLDT_SEL, SEL_KPL)); 729 ltr(ci->ci_tss_sel); 730 731 cpu_init(ci); 732 cpu_get_tsc_freq(ci); 733 734 s = splhigh(); 735 #ifdef i386 736 lapic_tpr = 0; 737 #else 738 lcr8(0); 739 #endif 740 x86_enable_intr(); 741 splx(s); 742 #if 0 743 x86_errata(); 744 #endif 745 746 aprint_debug_dev(ci->ci_dev, "CPU %ld running\n", 747 (long)ci->ci_cpuid); 748 } 749 750 #if defined(DDB) 751 752 #include <ddb/db_output.h> 753 #include <machine/db_machdep.h> 754 755 /* 756 * Dump CPU information from ddb. 757 */ 758 void 759 cpu_debug_dump(void) 760 { 761 struct cpu_info *ci; 762 CPU_INFO_ITERATOR cii; 763 764 db_printf("addr dev id flags ipis curlwp fpcurlwp\n"); 765 for (CPU_INFO_FOREACH(cii, ci)) { 766 db_printf("%p %s %ld %x %x %10p %10p\n", 767 ci, 768 ci->ci_dev == NULL ? "BOOT" : device_xname(ci->ci_dev), 769 (long)ci->ci_cpuid, 770 ci->ci_flags, ci->ci_ipis, 771 ci->ci_curlwp, 772 ci->ci_fpcurlwp); 773 } 774 } 775 #endif 776 777 static void 778 cpu_copy_trampoline(void) 779 { 780 /* 781 * Copy boot code. 782 */ 783 extern u_char cpu_spinup_trampoline[]; 784 extern u_char cpu_spinup_trampoline_end[]; 785 786 vaddr_t mp_trampoline_vaddr; 787 788 mp_trampoline_vaddr = uvm_km_alloc(kernel_map, PAGE_SIZE, 0, 789 UVM_KMF_VAONLY); 790 791 pmap_kenter_pa(mp_trampoline_vaddr, mp_trampoline_paddr, 792 VM_PROT_READ | VM_PROT_WRITE); 793 pmap_update(pmap_kernel()); 794 memcpy((void *)mp_trampoline_vaddr, 795 cpu_spinup_trampoline, 796 cpu_spinup_trampoline_end - cpu_spinup_trampoline); 797 798 pmap_kremove(mp_trampoline_vaddr, PAGE_SIZE); 799 pmap_update(pmap_kernel()); 800 uvm_km_free(kernel_map, mp_trampoline_vaddr, PAGE_SIZE, UVM_KMF_VAONLY); 801 } 802 803 #endif 804 805 #ifdef i386 806 #if 0 807 static void 808 tss_init(struct i386tss *tss, void *stack, void *func) 809 { 810 memset(tss, 0, sizeof *tss); 811 tss->tss_esp0 = tss->tss_esp = (int)((char *)stack + USPACE - 16); 812 tss->tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); 813 tss->__tss_cs = GSEL(GCODE_SEL, SEL_KPL); 814 tss->tss_fs = GSEL(GCPU_SEL, SEL_KPL); 815 tss->tss_gs = tss->__tss_es = tss->__tss_ds = 816 tss->__tss_ss = GSEL(GDATA_SEL, SEL_KPL); 817 tss->tss_cr3 = pmap_kernel()->pm_pdirpa; 818 tss->tss_esp = (int)((char *)stack + USPACE - 16); 819 tss->tss_ldt = GSEL(GLDT_SEL, SEL_KPL); 820 tss->__tss_eflags = PSL_MBO | PSL_NT; /* XXX not needed? */ 821 tss->__tss_eip = (int)func; 822 } 823 #endif 824 825 /* XXX */ 826 #define IDTVEC(name) __CONCAT(X, name) 827 typedef void (vector)(void); 828 extern vector IDTVEC(tss_trap08); 829 #ifdef DDB 830 extern vector Xintrddbipi; 831 extern int ddb_vec; 832 #endif 833 834 static void 835 cpu_set_tss_gates(struct cpu_info *ci) 836 { 837 #if 0 838 struct segment_descriptor sd; 839 840 ci->ci_doubleflt_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 841 UVM_KMF_WIRED); 842 tss_init(&ci->ci_doubleflt_tss, ci->ci_doubleflt_stack, 843 IDTVEC(tss_trap08)); 844 setsegment(&sd, &ci->ci_doubleflt_tss, sizeof(struct i386tss) - 1, 845 SDT_SYS386TSS, SEL_KPL, 0, 0); 846 ci->ci_gdt[GTRAPTSS_SEL].sd = sd; 847 setgate(&idt[8], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 848 GSEL(GTRAPTSS_SEL, SEL_KPL)); 849 #endif 850 851 #if defined(DDB) && defined(MULTIPROCESSOR) 852 /* 853 * Set up separate handler for the DDB IPI, so that it doesn't 854 * stomp on a possibly corrupted stack. 855 * 856 * XXX overwriting the gate set in db_machine_init. 857 * Should rearrange the code so that it's set only once. 858 */ 859 ci->ci_ddbipi_stack = (char *)uvm_km_alloc(kernel_map, USPACE, 0, 860 UVM_KMF_WIRED); 861 tss_init(&ci->ci_ddbipi_tss, ci->ci_ddbipi_stack, 862 Xintrddbipi); 863 864 setsegment(&sd, &ci->ci_ddbipi_tss, sizeof(struct i386tss) - 1, 865 SDT_SYS386TSS, SEL_KPL, 0, 0); 866 ci->ci_gdt[GIPITSS_SEL].sd = sd; 867 868 setgate(&idt[ddb_vec], NULL, 0, SDT_SYSTASKGT, SEL_KPL, 869 GSEL(GIPITSS_SEL, SEL_KPL)); 870 #endif 871 } 872 #else 873 static void 874 cpu_set_tss_gates(struct cpu_info *ci) 875 { 876 877 } 878 #endif /* i386 */ 879 880 int 881 mp_cpu_start(struct cpu_info *ci, paddr_t target) 882 { 883 #if 0 884 #if NLAPIC > 0 885 int error; 886 #endif 887 unsigned short dwordptr[2]; 888 889 /* 890 * Bootstrap code must be addressable in real mode 891 * and it must be page aligned. 892 */ 893 KASSERT(target < 0x10000 && target % PAGE_SIZE == 0); 894 895 /* 896 * "The BSP must initialize CMOS shutdown code to 0Ah ..." 897 */ 898 899 outb(IO_RTC, NVRAM_RESET); 900 outb(IO_RTC+1, NVRAM_RESET_JUMP); 901 902 /* 903 * "and the warm reset vector (DWORD based at 40:67) to point 904 * to the AP startup code ..." 905 */ 906 907 dwordptr[0] = 0; 908 dwordptr[1] = target >> 4; 909 910 pmap_kenter_pa (0, 0, VM_PROT_READ|VM_PROT_WRITE); 911 memcpy ((uint8_t *) 0x467, dwordptr, 4); 912 pmap_kremove (0, PAGE_SIZE); 913 914 #if NLAPIC > 0 915 /* 916 * ... prior to executing the following sequence:" 917 */ 918 919 if (ci->ci_flags & CPUF_AP) { 920 if ((error = x86_ipi_init(ci->ci_cpuid)) != 0) 921 return error; 922 923 delay(10000); 924 925 if (cpu_feature & CPUID_APIC) { 926 error = x86_ipi_init(ci->ci_cpuid); 927 if (error != 0) { 928 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (1)\n", 929 __func__); 930 return error; 931 } 932 933 delay(10000); 934 935 error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid, 936 LAPIC_DLMODE_STARTUP); 937 if (error != 0) { 938 aprint_error_dev(ci->ci_dev, "%s: IPI not taken (2)\n", 939 __func__); 940 return error; 941 } 942 delay(200); 943 944 error = x86_ipi(target / PAGE_SIZE, ci->ci_cpuid, 945 LAPIC_DLMODE_STARTUP); 946 if (error != 0) { 947 aprint_error_dev(ci->ci_dev, "%s: IPI not taken ((3)\n", 948 __func__); 949 return error; 950 } 951 delay(200); 952 } 953 } 954 #endif 955 #endif /* 0 */ 956 return 0; 957 } 958 959 void 960 mp_cpu_start_cleanup(struct cpu_info *ci) 961 { 962 #if 0 963 /* 964 * Ensure the NVRAM reset byte contains something vaguely sane. 965 */ 966 967 outb(IO_RTC, NVRAM_RESET); 968 outb(IO_RTC+1, NVRAM_RESET_RST); 969 #endif 970 } 971 972 #ifdef __x86_64__ 973 974 void 975 cpu_init_msrs(struct cpu_info *ci, bool full) 976 { 977 if (full) { 978 HYPERVISOR_set_segment_base (SEGBASE_FS, 0); 979 HYPERVISOR_set_segment_base (SEGBASE_GS_KERNEL, (uint64_t) ci); 980 HYPERVISOR_set_segment_base (SEGBASE_GS_USER, 0); 981 } 982 } 983 #endif /* __x86_64__ */ 984 985 void 986 cpu_offline_md(void) 987 { 988 int s; 989 990 s = splhigh(); 991 #ifdef __i386__ 992 npxsave_cpu(true); 993 #else 994 fpusave_cpu(true); 995 #endif 996 splx(s); 997 } 998 999 #if 0 1000 /* XXX joerg restructure and restart CPUs individually */ 1001 static bool 1002 cpu_suspend(device_t dv PMF_FN_ARGS) 1003 { 1004 struct cpu_softc *sc = device_private(dv); 1005 struct cpu_info *ci = sc->sc_info; 1006 int err; 1007 1008 if (ci->ci_flags & CPUF_PRIMARY) 1009 return true; 1010 if (ci->ci_data.cpu_idlelwp == NULL) 1011 return true; 1012 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1013 return true; 1014 1015 sc->sc_wasonline = !(ci->ci_schedstate.spc_flags & SPCF_OFFLINE); 1016 1017 if (sc->sc_wasonline) { 1018 mutex_enter(&cpu_lock); 1019 err = cpu_setstate(ci, false); 1020 mutex_exit(&cpu_lock); 1021 1022 if (err) 1023 return false; 1024 } 1025 1026 return true; 1027 } 1028 1029 static bool 1030 cpu_resume(device_t dv PMF_FN_ARGS) 1031 { 1032 struct cpu_softc *sc = device_private(dv); 1033 struct cpu_info *ci = sc->sc_info; 1034 int err = 0; 1035 1036 if (ci->ci_flags & CPUF_PRIMARY) 1037 return true; 1038 if (ci->ci_data.cpu_idlelwp == NULL) 1039 return true; 1040 if ((ci->ci_flags & CPUF_PRESENT) == 0) 1041 return true; 1042 1043 if (sc->sc_wasonline) { 1044 mutex_enter(&cpu_lock); 1045 err = cpu_setstate(ci, true); 1046 mutex_exit(&cpu_lock); 1047 } 1048 1049 return err == 0; 1050 } 1051 #endif 1052 1053 void 1054 cpu_get_tsc_freq(struct cpu_info *ci) 1055 { 1056 #ifdef XEN3 1057 const volatile vcpu_time_info_t *tinfo = &ci->ci_vcpu->time; 1058 delay(1000000); 1059 uint64_t freq = 1000000000ULL << 32; 1060 freq = freq / (uint64_t)tinfo->tsc_to_system_mul; 1061 if ( tinfo->tsc_shift < 0 ) 1062 freq = freq << -tinfo->tsc_shift; 1063 else 1064 freq = freq >> tinfo->tsc_shift; 1065 ci->ci_data.cpu_cc_freq = freq; 1066 #else 1067 /* Xen2 */ 1068 /* XXX this needs to read the shared_info of the CPU being probed.. */ 1069 ci->ci_data.cpu_cc_freq = HYPERVISOR_shared_info->cpu_freq; 1070 #endif /* XEN3 */ 1071 } 1072 1073 void 1074 x86_cpu_idle_xen(void) 1075 { 1076 struct cpu_info *ci = curcpu(); 1077 1078 KASSERT(ci->ci_ilevel == IPL_NONE); 1079 1080 x86_disable_intr(); 1081 if (!__predict_false(ci->ci_want_resched)) { 1082 idle_block(); 1083 } else { 1084 x86_enable_intr(); 1085 } 1086 } 1087