1/* $NetBSD: locore.S,v 1.11 2019/03/19 19:15:57 maxv Exp $ */ 2 3/* 4 * Copyright (c) 1998, 2000, 2007, 2008, 2016, 2017 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Charles M. Hannum and by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#define _LOCORE 33 34/* Override user-land alignment before including asm.h */ 35#define ALIGN_DATA .align 8 36#define ALIGN_TEXT .align 16,0x90 37#define _ALIGN_TEXT ALIGN_TEXT 38 39#include <machine/asm.h> 40#include <machine/param.h> 41#include <machine/pte.h> 42#include <machine/psl.h> 43#include <machine/segments.h> 44#include <machine/specialreg.h> 45#include <machine/trap.h> 46 47#define _KERNEL 48#include <machine/bootinfo.h> 49#undef _KERNEL 50 51#include "pdir.h" 52#include "redef.h" 53 54/* 32bit version of PTE_NX */ 55#define PTE_NX32 0x80000000 56 57#define TABLE_L2_ENTRIES (NKL2_KIMG_ENTRIES + 1) 58#define TABLE_L3_ENTRIES NKL3_KIMG_ENTRIES 59 60#define PROC0_PML4_OFF 0 61#define PROC0_STK_OFF (PROC0_PML4_OFF + 1 * PAGE_SIZE) 62#define PROC0_PTP3_OFF (PROC0_STK_OFF + UPAGES * PAGE_SIZE) 63#define PROC0_PTP2_OFF (PROC0_PTP3_OFF + NKL4_KIMG_ENTRIES * PAGE_SIZE) 64#define PROC0_PTP1_OFF (PROC0_PTP2_OFF + TABLE_L3_ENTRIES * PAGE_SIZE) 65#define TABLESIZE \ 66 ((NKL4_KIMG_ENTRIES + TABLE_L3_ENTRIES + TABLE_L2_ENTRIES + 1 + UPAGES) \ 67 * PAGE_SIZE) 68 69/* 70 * fillkpt - Fill in a kernel page table 71 * eax = pte (page frame | control | status) 72 * ebx = page table address 73 * ecx = number of pages to map 74 * 75 * Each entry is 8 (PDE_SIZE) bytes long: we must set the 4 upper bytes to 0. 76 */ 77#define fillkpt \ 78 cmpl $0,%ecx ; /* zero-sized? */ \ 79 je 2f ; \ 801: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \ 81 movl %eax,(%ebx) ; /* store phys addr */ \ 82 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 83 addl $PAGE_SIZE,%eax ; /* next phys page */ \ 84 loop 1b ; \ 852: ; 86 87/* 88 * fillkpt_nox - Same as fillkpt, but sets the NX/XD bit. 89 */ 90#define fillkpt_nox \ 91 cmpl $0,%ecx ; /* zero-sized? */ \ 92 je 2f ; \ 93 pushl %ebp ; \ 94 movl _C_LABEL(nox_flag),%ebp ; \ 951: movl %ebp,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: NX */ \ 96 movl %eax,(%ebx) ; /* store phys addr */ \ 97 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 98 addl $PAGE_SIZE,%eax ; /* next phys page */ \ 99 loop 1b ; \ 100 popl %ebp ; \ 1012: ; 102 103/* 104 * fillkpt_blank - Fill in a kernel page table with blank entries 105 * ebx = page table address 106 * ecx = number of pages to map 107 */ 108#define fillkpt_blank \ 109 cmpl $0,%ecx ; /* zero-sized? */ \ 110 je 2f ; \ 1111: movl $0,(PDE_SIZE-4)(%ebx) ; /* upper 32 bits: 0 */ \ 112 movl $0,(%ebx) ; /* lower 32 bits: 0 */ \ 113 addl $PDE_SIZE,%ebx ; /* next PTE/PDE */ \ 114 loop 1b ; \ 1152: ; 116 117/* 118 * Initialization 119 */ 120 .data 121 122 .globl _C_LABEL(tablesize) 123 .globl _C_LABEL(nox_flag) 124 .globl _C_LABEL(cpuid_level) 125 .globl _C_LABEL(esym) 126 .globl _C_LABEL(eblob) 127 .globl _C_LABEL(atdevbase) 128 .globl _C_LABEL(PDPpaddr) 129 .globl _C_LABEL(boothowto) 130 .globl _C_LABEL(bootinfo) 131 .globl _C_LABEL(biosbasemem) 132 .globl _C_LABEL(biosextmem) 133 .globl _C_LABEL(stkpa) 134 .globl _C_LABEL(stkva) 135 .globl _C_LABEL(kernpa_start) 136 .globl _C_LABEL(kernpa_end) 137 138 .type _C_LABEL(tablesize), @object 139_C_LABEL(tablesize): .long TABLESIZE 140END(tablesize) 141 .type _C_LABEL(nox_flag), @object 142LABEL(nox_flag) .long 0 /* 32bit NOX flag, set if supported */ 143END(nox_flag) 144 .type _C_LABEL(cpuid_level), @object 145LABEL(cpuid_level) .long -1 /* max. level accepted by cpuid instr */ 146END(cpuid_level) 147 .type _C_LABEL(esym), @object 148LABEL(esym) .quad 0 /* ptr to end of syms */ 149END(esym) 150 .type _C_LABEL(eblob), @object 151LABEL(eblob) .quad 0 /* ptr to end of modules */ 152END(eblob) 153 .type _C_LABEL(atdevbase), @object 154LABEL(atdevbase) .quad 0 /* location of start of iomem in virt */ 155END(atdevbase) 156 .type _C_LABEL(PDPpaddr), @object 157LABEL(PDPpaddr) .quad 0 /* paddr of PTD, for libkvm */ 158END(PDPpaddr) 159 .type _C_LABEL(biosbasemem), @object 160LABEL(biosbasemem) .long 0 /* base memory reported by BIOS */ 161END(biosbasemem) 162 .type _C_LABEL(biosextmem), @object 163LABEL(biosextmem) .long 0 /* extended memory reported by BIOS */ 164END(biosextmem) 165 .type _C_LABEL(stkpa), @object 166LABEL(stkpa) .quad 0 167END(stkpa) 168 .type _C_LABEL(stkva), @object 169LABEL(stkva) .quad 0 170END(stkva) 171 .type _C_LABEL(kernpa_start), @object 172LABEL(kernpa_start) .quad 0 173END(kernpa_start) 174 .type _C_LABEL(kernpa_end), @object 175LABEL(kernpa_end) .quad 0 176END(kernpa_end) 177 178 .globl gdt64_lo 179 .globl gdt64_start 180 181#define GDT64_LIMIT gdt64_end-gdt64_start-1 182/* Temporary gdt64, with base address in low memory */ 183 .type _C_LABEL(gdt64_lo), @object 184LABEL(gdt64_lo) 185 .word GDT64_LIMIT 186 .quad gdt64_start 187END(gdt64_lo) 188.align 64 189#undef GDT64_LIMIT 190 191 .type _C_LABEL(gdt64_start), @object 192LABEL(gdt64_start) 193 .quad 0x0000000000000000 /* always empty */ 194 .quad 0x00af9a000000ffff /* kernel CS */ 195 .quad 0x00cf92000000ffff /* kernel DS */ 196END(gdt64_start) 197gdt64_end: 198 199 .type _C_LABEL(farjmp64), @object 200_C_LABEL(farjmp64): 201 .long longmode 202 .word GSEL(GCODE_SEL, SEL_KPL) 203END(farjmp64) 204 205 /* Space for the temporary stack */ 206 .size tmpstk, tmpstk - . 207 .space 512 208tmpstk: 209 210 .text 211 212ENTRY(start) 213 .code32 214 215 /* Warm boot */ 216 movw $0x1234,0x472 217 218 /* 219 * Load parameters from the stack (32 bits): 220 * boothowto, [bootdev], bootinfo, esym, biosextmem, biosbasemem 221 * We are not interested in 'bootdev'. 222 */ 223 224 /* Load 'boothowto' */ 225 movl 4(%esp),%eax 226 movl %eax,_C_LABEL(boothowto) 227 228 /* Load 'bootinfo' */ 229 movl 12(%esp),%eax 230 testl %eax,%eax /* bootinfo = NULL? */ 231 jz .Lbootinfo_finished 232 233 movl (%eax),%ebx /* number of entries */ 234 movl $_C_LABEL(bootinfo),%ebp 235 movl %ebp,%edx 236 addl $BOOTINFO_MAXSIZE,%ebp 237 movl %ebx,(%edx) 238 addl $4,%edx 239 240.Lbootinfo_entryloop: 241 testl %ebx,%ebx /* no remaining entries? */ 242 jz .Lbootinfo_finished 243 244 addl $4,%eax 245 movl (%eax),%ecx /* address of entry */ 246 pushl %edi 247 pushl %esi 248 pushl %eax 249 250 movl (%ecx),%eax /* btinfo_common::len (size of entry) */ 251 movl %edx,%edi 252 addl (%ecx),%edx /* update dest pointer */ 253 cmpl %ebp,%edx /* beyond bootinfo+BOOTINFO_MAXSIZE? */ 254 jg .Lbootinfo_overflow 255 256 movl %ecx,%esi 257 movl %eax,%ecx 258 259 /* If any modules were loaded, record where they end. */ 260 cmpl $BTINFO_MODULELIST,4(%esi) /* btinfo_common::type */ 261 jne 0f 262 pushl 12(%esi) /* btinfo_modulelist::endpa */ 263 popl _C_LABEL(eblob) 2640: 265 266 /* Record the information about the kernel. */ 267 cmpl $BTINFO_PREKERN,4(%esi) /* btinfo_common::type */ 268 jne 0f 269 pushl 8(%esi) /* btinfo_prekern::kernpa_start */ 270 popl _C_LABEL(kernpa_start) 271 pushl 12(%esi) /* btinfo_prekern::kernpa_end */ 272 popl _C_LABEL(kernpa_end) 2730: 274 275 rep 276 movsb /* copy esi -> edi */ 277 popl %eax 278 popl %esi 279 popl %edi 280 subl $1,%ebx /* decrement the # of entries */ 281 jmp .Lbootinfo_entryloop 282 283.Lbootinfo_overflow: 284 /* 285 * Cleanup for overflow case. Pop the registers, and correct the number 286 * of entries. 287 */ 288 popl %eax 289 popl %esi 290 popl %edi 291 movl $_C_LABEL(bootinfo),%ebp 292 movl %ebp,%edx 293 subl %ebx,(%edx) /* correct the number of entries */ 294 295.Lbootinfo_finished: 296 /* Load 'esym' */ 297 movl 16(%esp),%eax 298 movl $_C_LABEL(esym),%ebp 299 movl %eax,(%ebp) 300 301 /* Load 'biosextmem' */ 302 movl 20(%esp),%eax 303 movl $_C_LABEL(biosextmem),%ebp 304 movl %eax,(%ebp) 305 306 /* Load 'biosbasemem' */ 307 movl 24(%esp),%eax 308 movl $_C_LABEL(biosbasemem),%ebp 309 movl %eax,(%ebp) 310 311 /* 312 * Done with the parameters! 313 */ 314 315 /* First, reset the PSL. */ 316 pushl $PSL_MBO 317 popfl 318 319 /* Switch to new stack now. */ 320 movl $_C_LABEL(tmpstk),%esp 321 322 xorl %eax,%eax 323 cpuid 324 movl %eax,_C_LABEL(cpuid_level) 325 326 /* 327 * Retrieve the NX/XD flag. We use the 32bit version of PTE_NX. 328 */ 329 movl $0x80000001,%eax 330 cpuid 331 andl $CPUID_NOX,%edx 332 jz .Lno_NOX 333 movl $PTE_NX32,_C_LABEL(nox_flag) 334.Lno_NOX: 335 336/* 337 * There are four levels of pages in amd64: PML4 -> PDP -> PD -> PT. They will 338 * be referred to as: L4 -> L3 -> L2 -> L1. 339 * 340 * Physical address space: 341 * +---------------+----------+--------------+--------+---------------------+- 342 * | PREKERN IMAGE |**UNUSED**| KERNEL IMAGE | [SYMS] | [PRELOADED MODULES] | 343 * +---------------+----------+--------------+--------+---------------------+- 344 * (1) (2) (3) (4) 345 * ------------------+ 346 * BOOTSTRAP TABLES | 347 * ------------------+ 348 * (5) 349 * 350 * The virtual address space is the same, since it is identity-mapped (va = pa). 351 * However, the KERNEL IMAGE is mapped as read-only: the prekern reads it, but 352 * won't write to it. (Needed when relocating the kernel.) 353 * 354 * PROC0 STK is obviously not linked as a page level. It just happens to be 355 * caught between L4 and L3. 356 * 357 * (PROC0 STK + L4 + L3 + L2 + L1) is later referred to as BOOTSTRAP TABLES. 358 * 359 * Important note: the prekern segments are properly 4k-aligned 360 * (see prekern.ldscript), so there's no need to enforce alignment. 361 */ 362 363 /* Find end of the prekern image; brings us on (1). */ 364 movl $_C_LABEL(__prekern_end),%edi 365 366 /* Find end of the kernel image; brings us on (2). */ 367 movl _C_LABEL(kernpa_end),%eax 368 testl %eax,%eax 369 jz 1f 370 movl %eax,%edi 3711: 372 373 /* Find end of the kernel symbols; brings us on (3). */ 374 movl _C_LABEL(esym),%eax 375 testl %eax,%eax 376 jz 1f 377 movl %eax,%edi 3781: 379 380 /* Find end of the kernel preloaded modules; brings us on (4). */ 381 movl _C_LABEL(eblob),%eax 382 testl %eax,%eax 383 jz 1f 384 movl %eax,%edi 3851: 386 387 /* We are on (3). Align up for BOOTSTRAP TABLES. */ 388 movl %edi,%esi 389 addl $PGOFSET,%esi 390 andl $~PGOFSET,%esi 391 392 /* We are on the BOOTSTRAP TABLES. Save L4's physical address. */ 393 movl $_C_LABEL(PDPpaddr),%ebp 394 movl %esi,(%ebp) 395 movl $0,4(%ebp) 396 397 /* Now, zero out the BOOTSTRAP TABLES (before filling them in). */ 398 movl %esi,%edi 399 xorl %eax,%eax 400 cld 401 movl $TABLESIZE,%ecx 402 shrl $2,%ecx 403 rep 404 stosl /* copy eax -> edi */ 405 406/* 407 * Build the page tables and levels. We go from L1 to L4, and link the levels 408 * together. 409 */ 410 /* 411 * Build L1. 412 */ 413 leal (PROC0_PTP1_OFF)(%esi),%ebx 414 415 /* Skip the area below the prekern text. */ 416 movl $(PREKERNTEXTOFF - PREKERNBASE),%ecx 417 shrl $PGSHIFT,%ecx 418 fillkpt_blank 419 420 /* Map the prekern text RX. */ 421 movl $(PREKERNTEXTOFF - PREKERNBASE),%eax /* start of TEXT */ 422 movl $_C_LABEL(__rodata_start),%ecx 423 subl %eax,%ecx 424 shrl $PGSHIFT,%ecx 425 orl $(PTE_P),%eax 426 fillkpt 427 428 /* Map the prekern rodata R. */ 429 movl $_C_LABEL(__rodata_start),%eax 430 movl $_C_LABEL(__data_start),%ecx 431 subl %eax,%ecx 432 shrl $PGSHIFT,%ecx 433 orl $(PTE_P),%eax 434 fillkpt_nox 435 436 /* Map the prekern data+bss RW. */ 437 movl $_C_LABEL(__data_start),%eax 438 movl $_C_LABEL(__prekern_end),%ecx 439 subl %eax,%ecx 440 shrl $PGSHIFT,%ecx 441 orl $(PTE_P|PTE_W),%eax 442 fillkpt_nox 443 444 /* Map a RO view of the kernel. */ 445 movl $_C_LABEL(__prekern_end),%eax 446 movl %esi,%ecx /* start of BOOTSTRAP TABLES */ 447 subl %eax,%ecx 448 shrl $PGSHIFT,%ecx 449 orl $(PTE_P),%eax 450 fillkpt_nox 451 452 /* Map the BOOTSTRAP TABLES RW. */ 453 movl %esi,%eax /* start of BOOTSTRAP TABLES */ 454 movl $TABLESIZE,%ecx /* length of BOOTSTRAP TABLES */ 455 shrl $PGSHIFT,%ecx 456 orl $(PTE_P|PTE_W),%eax 457 fillkpt_nox 458 459 /* Map the ISA I/O MEM RW. */ 460 movl $IOM_BEGIN,%eax 461 movl $IOM_SIZE,%ecx /* size of ISA I/O MEM */ 462 shrl $PGSHIFT,%ecx 463 orl $(PTE_P|PTE_W/*|PTE_PCD*/),%eax 464 fillkpt_nox 465 466 /* 467 * Build L2. Linked to L1. 468 */ 469 leal (PROC0_PTP2_OFF)(%esi),%ebx 470 leal (PROC0_PTP1_OFF)(%esi),%eax 471 orl $(PTE_P|PTE_W),%eax 472 movl $(NKL2_KIMG_ENTRIES+1),%ecx 473 fillkpt 474 475 /* 476 * Build L3. Linked to L2. 477 */ 478 leal (PROC0_PTP3_OFF)(%esi),%ebx 479 leal (PROC0_PTP2_OFF)(%esi),%eax 480 orl $(PTE_P|PTE_W),%eax 481 movl $NKL3_KIMG_ENTRIES,%ecx 482 fillkpt 483 484 /* 485 * Build L4. Linked to L3. 486 */ 487 leal (PROC0_PML4_OFF)(%esi),%ebx 488 leal (PROC0_PTP3_OFF)(%esi),%eax 489 orl $(PTE_P|PTE_W),%eax 490 movl $NKL4_KIMG_ENTRIES,%ecx 491 fillkpt 492 493 /* Install recursive top level PDE (one entry) */ 494 leal (PROC0_PML4_OFF + PDIR_SLOT_PTE * PDE_SIZE)(%esi),%ebx 495 leal (PROC0_PML4_OFF)(%esi),%eax 496 orl $(PTE_P|PTE_W),%eax 497 movl $1,%ecx 498 fillkpt_nox 499 500 /* 501 * Startup checklist: 502 * 1. Enable PAE (and SSE while here). 503 */ 504 movl %cr4,%eax 505 orl $(CR4_PAE|CR4_OSFXSR|CR4_OSXMMEXCPT),%eax 506 movl %eax,%cr4 507 508 /* 509 * 2. Set Long Mode Enable in EFER. Also enable the syscall extensions, 510 * and NOX if available. 511 */ 512 movl $MSR_EFER,%ecx 513 rdmsr 514 xorl %eax,%eax 515 orl $(EFER_LME|EFER_SCE),%eax 516 movl _C_LABEL(nox_flag),%ebx 517 cmpl $0,%ebx 518 je .Lskip_NOX 519 orl $(EFER_NXE),%eax 520.Lskip_NOX: 521 wrmsr 522 523 /* 524 * 3. Load %cr3 with pointer to PML4. 525 */ 526 movl %esi,%eax 527 movl %eax,%cr3 528 529 /* 530 * 4. Enable paging and the rest of it. 531 */ 532 movl %cr0,%eax 533 orl $(CR0_PE|CR0_PG|CR0_NE|CR0_TS|CR0_MP|CR0_WP|CR0_AM),%eax 534 movl %eax,%cr0 535 jmp compat 536compat: 537 538 /* 539 * 5. Not quite done yet, we're now in a compatibility segment, in 540 * legacy mode. We must jump to a long mode segment. Need to set up 541 * a GDT with a long mode segment in it to do that. 542 */ 543 movl $_C_LABEL(gdt64_lo),%eax 544 lgdt (%eax) 545 movl $_C_LABEL(farjmp64),%eax 546 ljmp *(%eax) 547 548 .code64 549longmode: 550 551 /* 552 * We have arrived. Everything is identity-mapped. 553 */ 554 555 /* Store atdevbase. */ 556 movq $TABLESIZE,%rdx 557 addq %rsi,%rdx 558 movq %rdx,_C_LABEL(atdevbase)(%rip) 559 560 /* Set up bootstrap stack. */ 561 leaq (PROC0_STK_OFF)(%rsi),%rax 562 movq %rax,_C_LABEL(stkpa)(%rip) 563 leaq (USPACE-FRAMESIZE)(%rax),%rsp 564 xorq %rbp,%rbp /* mark end of frames */ 565 566 xorw %ax,%ax 567 movw %ax,%gs 568 movw %ax,%fs 569 570 movw $GSEL(GDATA_SEL, SEL_KPL),%ax 571 movw %ax,%ss 572 573 /* The first physical page available. */ 574 leaq (TABLESIZE)(%rsi),%rdi 575 576 /* 577 * Continue execution in C. 578 */ 579 call _C_LABEL(init_prekern) 580 581 ret 582END(start) 583 584/* -------------------------------------------------------------------------- */ 585 586ENTRY(cpuid) 587 movq %rbx,%r8 588 movq %rdi,%rax 589 movq %rsi,%rcx 590 movq %rdx,%rsi 591 cpuid 592 movl %eax,0(%rsi) 593 movl %ebx,4(%rsi) 594 movl %ecx,8(%rsi) 595 movl %edx,12(%rsi) 596 movq %r8,%rbx 597 ret 598END(cpuid) 599 600ENTRY(lidt) 601 lidt (%rdi) 602 ret 603END(lidt) 604 605ENTRY(rdtsc) 606 xorq %rax,%rax 607 rdtsc 608 shlq $32,%rdx 609 orq %rdx,%rax 610 ret 611END(rdtsc) 612 613ENTRY(rdseed) 614 rdseed %rax 615 jc .Lrdseed_success 616 movq $(-1),%rax 617 ret 618.Lrdseed_success: 619 movq %rax,(%rdi) 620 xorq %rax,%rax 621 ret 622END(rdseed) 623 624ENTRY(rdrand) 625 rdrand %rax 626 jc .Lrdrand_success 627 movq $(-1),%rax 628 ret 629.Lrdrand_success: 630 movq %rax,(%rdi) 631 xorq %rax,%rax 632 ret 633END(rdrand) 634 635ENTRY(jump_kernel) 636 movq _C_LABEL(stkva),%rsp 637 xorq %rbp,%rbp 638 callq exec_kernel 639END(jump_kernel) 640