1/* $NetBSD: armv6_start.S,v 1.39 2024/09/07 06:17:37 andvar Exp $ */ 2 3/*- 4 * Copyright (c) 2012, 2017, 2018 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Matt Thomas of 3am Software Foundry and Nick Hudson. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32#include "opt_arm_debug.h" 33#include "opt_console.h" 34#include "opt_cpuoptions.h" 35#include "opt_cputypes.h" 36#include "opt_fdt.h" 37#include "opt_kasan.h" 38#include "opt_multiprocessor.h" 39 40#include <sys/cdefs.h> 41 42#include <arm/asm.h> 43#include <arm/armreg.h> 44#include "assym.h" 45 46#if defined(CONSADDR) && defined(CONADDR) 47#error Only one of CONSADDR and CONADDR should be defined 48#endif 49 50#if defined(CONSADDR) 51#define START_CONSADDR CONSADDR 52#endif 53#if defined(CONADDR) 54#define START_CONSADDR CONADDR 55#endif 56 57#if defined(VERBOSE_INIT_ARM) 58#define XPUTC(n) mov r0, n; bl uartputc 59#define VPRINTF(string) bl generic_vprint; .asciz string; .align 2 60#define VPRINTX(regno) mov r0, regno; bl generic_printx 61#else 62#define XPUTC(c) /* nothing */ 63#define VPRINTF(string) /* nothing */ 64#define VPRINTX(regno) /* nothing */ 65#endif 66 67#if defined(FDT) 68#define MD_CPU_HATCH arm_fdt_cpu_hatch 69#endif 70 71/* 72 * A generic kernel start routine. 73 * 74 * At this point, this code has been loaded into SDRAM and the MMU should be off 75 * with data caches disabled. 76 * 77 * linux image type should be used in uboot images to ensure this is the case. 78 */ 79 80 // Use callee saved registers 81 R_L1TABLE .req r4 82 R_VA .req r5 83 R_PA .req r6 84 R_NSEC .req r7 85 R_ATTR .req r8 86 R_DEVATTR .req r9 87 88 R_TMP1 .req r8 89 R_TMP2 .req r9 90 R_VTOPDIFF .req r10 91 R_FDTADDR .req r11 92 R_INDEX .req r11 93 94 .text 95 96ENTRY_NP(generic_start) 97 98#if defined(__ARMEB__) 99 /* Make sure sctlr.u = 1 when cpsr.e = 1. */ 100 mrc p15, 0, R_TMP1, c1, c0, 0 101 orr R_TMP1, R_TMP1, #CPU_CONTROL_UNAL_ENABLE 102 mcr p15, 0, R_TMP1, c1, c0, 0 103 104 setend be /* force big endian */ 105#endif 106 107 /* disable IRQs/FIQs. */ 108 cpsid if 109 110 adr R_TMP1, generic_start 111 ldr R_VTOPDIFF, =generic_start 112 sub R_VTOPDIFF, R_VTOPDIFF, R_TMP1 113 114 ldr R_TMP1, =start_stacks_top 115 sub sp, R_TMP1, R_VTOPDIFF 116 117 mov r4, r0 118 mov r5, r1 119 mov r6, r2 120 mov r7, r3 121 122 // We can now call functions 123 124 VPRINTF("\n\rpc : ") 125 VPRINTX(pc) 126 127 VPRINTF("\n\roff : ") 128 VPRINTX(R_VTOPDIFF) 129 130 VPRINTF("\n\rsp : ") 131 VPRINTX(sp) 132 133 ldr R_TMP1, =(L1_S_SIZE - 1) 134 ands R_TMP2, R_VTOPDIFF, R_TMP1 135 bne arm_bad_vtopdiff 136 137#if defined(FDTBASE) 138 /* 139 * ARM boot protocol has FDT address in r2 which is now in r6 140 */ 141 VPRINTF("\n\rfdt : ") 142 mov R_FDTADDR, r6 // Save fdt_addr_r for mapping later 143 144 VPRINTX(r6) 145#endif 146 147#if defined(VERBOSE_INIT_ARM) 148 VPRINTF("\n\rmidr : ") 149 mrc p15, 0, r0, c0, c0, 0 // MIDR 150 VPRINTX(r0) 151 VPRINTF("\n\rrevidr: ") 152 mrc p15, 0, r0, c0, c0, 6 // REVIDR 153 VPRINTX(r0) 154 VPRINTF("\n\rmpidr : ") 155 mrc p15, 0, r0, c0, c0, 5 // MPIDR 156 VPRINTX(r0) 157 VPRINTF("\n\rttb0 : ") 158 mrc p15, 0, r0, c2, c0, 0 // TTBR0 read 159 VPRINTX(r0) 160 VPRINTF("\n\rttb1 : ") 161 mrc p15, 0, r0, c2, c0, 1 // TTBR1 read 162 VPRINTX(r0) 163 VPRINTF("\n\rttcr : ") 164 mrc p15, 0, r0, c2, c0, 2 // TTBCR read 165 VPRINTX(r0) 166 VPRINTF("\n\r") 167#endif 168 169#if defined(_ARM_ARCH_7) 170 b generic_startv7 171#elif defined(_ARM_ARCH_6) 172 b generic_startv6 173#else 174#error Unsupported CPU 175#endif 176arm_bad_vtopdiff: 177 VPRINTF("\n\rpanic: vtop not L1_FRAME aligned (") 178 VPRINTX(R_VTOPDIFF) 179 VPRINTF(")\n\r") 1801: b 1b 181ASEND(generic_start) 182 183/* 184 * Save the u-boot arguments (including FDT address) and the virtual to physical 185 * offset. 186 * 187 * Uses the following callee saved registers: 188 * 189 * r8 (R_TMP1), r9 (R_TMP2) 190 */ 191generic_savevars: 192 mov R_TMP1, lr 193 /* 194 * Store virtual to physical address difference 195 */ 196 ldr R_TMP2, =kern_vtopdiff 197 sub R_TMP2, R_TMP2, R_VTOPDIFF 198 str R_VTOPDIFF, [R_TMP2] 199 200 /* 201 * store uboot arguments to uboot_args[4] 202 */ 203 ldr R_TMP2, =uboot_args 204 sub R_TMP2, R_VTOPDIFF 205 206 VPRINTF("\n\ruboot : ") 207 VPRINTX(R_TMP2) 208 str r4, [R_TMP2, #(4*0)] 209 str r5, [R_TMP2, #(4*1)] 210 str r6, [R_TMP2, #(4*2)] 211 str r7, [R_TMP2, #(4*3)] 212 213#if defined(FDTBASE) 214 /* 215 * ARM boot protocol has FDT address in r2 which is now in r6 216 */ 217 VPRINTF("\n\rfdt : ") 218 ldr R_TMP2, =fdt_addr_r 219 sub R_TMP2, R_VTOPDIFF 220 str r6, [R_TMP2] 221 222 VPRINTX(r6) 223#endif 224 225 RETr(R_TMP1) 226 227 .ltorg 228 229 /* 230 * Allocate some memory after the kernel image for stacks and 231 * bootstrap L1PT 232 */ 233 .section "_init_memory", "aw", %nobits 234 .p2align INIT_ARM_STACK_SHIFT 235 .global start_stacks_bottom 236 .global start_stacks_top 237start_stacks_bottom: 238 .space INIT_ARM_TOTAL_STACK 239start_stacks_top: 240 241 .section "_init_memory", "aw", %nobits 242 .p2align 14 /* 16KiB aligned */ 243 .global ARM_BOOTSTRAP_LxPT 244ARM_BOOTSTRAP_LxPT: 245TEMP_L1_TABLE: 246 .space L1_TABLE_SIZE 247TEMP_L1_TABLE_END: 248 249 .text 250 .align 2 251 252arm_build_translation_table: 253 push {r0, lr} 254 /* 255 * Initialise the l1pt for identity mapping of the kernel with caches 256 * off. This will be updated further with additional mappings to form 257 * the bootstrap table. 258 */ 259 ldr R_L1TABLE, =TEMP_L1_TABLE 260 sub R_L1TABLE, R_VTOPDIFF 261 262 // PA of kernel rounded down to nearest L1_S boundary 263 adr R_PA, generic_start 264 ldr r0, =(L1_S_SIZE - 1) 265 bic R_PA, R_PA, r0 266 267 // attribute to map kernel 268 ldr R_ATTR, =(L1_S_PROTO | L1_S_AP_KRW | L1_S_NOCACHE) 269 bl arm_boot_l1pt_init 270 271 /* 272 * Set up a preliminary mapping in the MMU to allow us to run 273 * at KERNEL_BASE_VIRT (determined from generic_start) with caches off. 274 */ 275 ldr R_L1TABLE, =TEMP_L1_TABLE 276 sub R_L1TABLE, R_VTOPDIFF 277 278 // Calculate kernel size plus 1M to grow into 279 ldr r0, =generic_start // kernel start VA 280 ldr r1, =TEMP_L1_TABLE_END // kernel end VA 281 add r1, #L1_S_SIZE // Make sure we have 1M to grow into 282 283 ldr r2, =(L1_S_SIZE - 1) 284 285 bic r0, r2 // trunc kernel start to 1M boundary 286 add r1, r2 287 bic r1, r2 // round kernel end to 1M boundary 288 289 sub r1, r0 // kernel size plus 1M to grow into 290 mov R_NSEC, r1, lsr #(L1_S_SHIFT) 291 292 ldr R_VA, =generic_start // VA of kernel 293 bic R_VA, r2 // ...rounded down to L1_S boundary 294 adr R_PA, generic_start // PA of kernel 295 bic R_PA, r2 // ...rounded down to L1_S boundary 296 297 // attribute to map kernel 298 ldr R_ATTR, =(L1_S_PROTO | L1_S_AP_KRW | L1_S_NOCACHE) 299 bl arm_boot_l1pt_entry 300 301#if defined(FDTBASE) 302 VPRINTF("DTB") 303 304 /* 305 * Add DTB identity mapping (1MB) from originally r2 (but saved in 306 * R_FDTADDR) 307 */ 308 ldr r0, =(L1_S_SIZE - 1) /* align DTB PA to 1M */ 309 bic R_VA, R_FDTADDR, r0 310 mov R_PA, R_VA 311 mov R_NSEC, #1 /* 1MB mapping */ 312 313 ldr R_L1TABLE, =TEMP_L1_TABLE 314 sub R_L1TABLE, R_VTOPDIFF 315 bl arm_boot_l1pt_entry 316#endif 317 318#if defined(START_CONSADDR) 319 /* If START_CONSADDR exists add its identity mapping (1MB) */ 320 VPRINTF("\n\rCONSADDR") 321 ldr r0, =(L1_S_SIZE - 1) /* align DTB PA to 1M */ 322 ldr R_VA, =START_CONSADDR 323 bic R_VA, R_VA, r0 324 mov R_PA, R_VA 325 mov R_NSEC, #1 326 327 ldr R_ATTR, =(L1_S_PROTO | L1_S_AP_KRW) 328 orr R_ATTR, R_DEVATTR 329 330 ldr R_L1TABLE, =TEMP_L1_TABLE 331 sub R_L1TABLE, R_VTOPDIFF 332 bl arm_boot_l1pt_entry 333#endif 334 335 XPUTC(#'M') 336 337 pop {r0, pc} 338 339 .ltorg 340 341/* 342 * Set up a preliminary mapping in the MMU to allow us to run at 343 * KERNEL_BASE_VIRT (determined from generic_start) 344 * 345 * On Entry 346 * 347 * R_L1TABLE is the PA of the temporary L1PT 348 * R_ATTR is the attribute bits to set for each section mapping 349 * 350 * No R_VA/R_PA/R_NSEC needed here as we use 'generic_start' and 351 * 'TEMP_L1_TABLE_END' to calculate the initial direct VA:PA mapping 352 */ 353 354/* We push r0 to maintain stack alignment only */ 355arm_boot_l1pt_init: 356 push {r0, lr} 357 358 // Start address to clear memory. 359 mov r0, R_L1TABLE 360 361 // Zero the entire table so all virtual addresses are invalid. 362 add r1, r0, #L1_TABLE_SIZE // Ending address 363 mov r2, #0 364 mov r3, #0 3651: stmia r0!, {r2-r3} 366 stmia r0!, {r2-r3} // 16 bytes per loop 367 cmp r0, r1 368 blt 1b 369 370 // Calculate the size of the kernel in L1_S_SIZE sections 371 ldr r0, =generic_start 372 ldr r1, =TEMP_L1_TABLE_END 373 374 ldr r2, =(L1_S_SIZE - 1) 375 376 bic r0, r2 // trunc kernel start to 1M boundary 377 add r1, r2 378 bic r1, r2 // round kernel end to 1M boundary 379 380 sub r1, r0 // kernel size plus 1M to grow into 381 mov R_NSEC, r1, lsr #(L1_S_SHIFT) 382 383 // identity mapping for size of kernel 384 adr R_VA, generic_start 385 bic R_VA, r2 386 mov R_PA, R_VA 387 388 pop {r0, lr} 389 /* Fallthrough */ 390 391// We push r0 to maintain stack alignment only 392arm_boot_l1pt_entry: 393 push {r0, lr} 394 395 VPRINTF("\n\r") 396 VPRINTX(R_L1TABLE) 397 398 VPRINTF(" va:") 399 VPRINTX(R_VA) 400 401 VPRINTF(" pa:") 402 VPRINTX(R_PA) 403 404 VPRINTF(" nsec:") 405 VPRINTX(R_NSEC) 406 407 VPRINTF(" attr:") 408 VPRINTX(R_ATTR) 409 410 VPRINTF("\n\r") 411 lsr R_VA, R_VA, #L1_S_SHIFT 412 orr R_PA, R_ATTR 413 4142: 415 VPRINTX(R_L1TABLE) 416 XPUTC('[') 417 VPRINTX(R_VA) 418 XPUTC(']') 419 XPUTC('=') 420 VPRINTX(R_PA) 421 422 ldr r0, [R_L1TABLE, R_VA, lsl #2] 423 cmp r0, #0 424 cmpne r0, R_PA 425 bne arm_boot_overlap 426 427 str R_PA, [R_L1TABLE, R_VA, lsl #2] 428 add R_VA, R_VA, #1 429 add R_PA, R_PA, #(L1_S_SIZE) 430 431 VPRINTF("\n\r") 432 433 subs R_NSEC, R_NSEC, #1 434 bhi 2b 435 436 pop {r0, pc} 437 438arm_boot_overlap: 439 VPRINTF("\n\rpanic: overlapping mappings\n\r") 4403: 441 b 3b 442 443#if defined(_ARM_ARCH_7) 444generic_startv7: 445 446 .arch armv7a 447 .arch_extension sec 448 .arch_extension virt 449 450 VPRINTF("v7 : ") 451 452 bl armv7_init 453 bl generic_savevars 454 455 mov R_DEVATTR, #L1_S_V6_XN 456 bl arm_build_translation_table 457 458 /* 459 * Turn on the MMU. Return to virtual address space. 460 */ 461 movw r0, #:lower16:TEMP_L1_TABLE 462 movt r0, #:upper16:TEMP_L1_TABLE 463 sub r0, R_VTOPDIFF 464 465 // Return to virtual address after the call to armv7_mmuinit 466 movw lr, #:lower16:generic_vstartv7 467 movt lr, #:upper16:generic_vstartv7 468 b armv7_mmuinit 469 470generic_vstartv7: 471 // Stack to KVA address 472 add sp, sp, R_VTOPDIFF 473 474 VPRINTF("virtual\n\r") 475 476 VPRINTF("prrr : ") 477 mrc p15, 0, r0, c10, c2, 0 // Primary Region Remap Register (PRRR) 478 VPRINTX(r0) 479 VPRINTF("\n\rnmrr : ") 480 mrc p15, 0, r0, c10, c2, 1 // Normal Memory Remap Register (NMRR) 481 VPRINTX(r0) 482 VPRINTF("\n\r") 483 484#if defined(KASAN) 485 ldr r0, =start_stacks_bottom 486 bl _C_LABEL(kasan_early_init) 487 488 VPRINTF("kasan\n\r") 489#endif 490 491 /* r0 = &cpu_info_store[0] */ 492 movw r0, #:lower16:cpu_info_store 493 movt r0, #:upper16:cpu_info_store 494 495 mrc p15, 0, r1, c0, c0, 0 // MIDR get 496 str r1, [r0, #CI_MIDR] 497 mrc p15, 0, r1, c0, c0, 5 // MPIDR get 498 str r1, [r0, #CI_MPIDR] 499 500 bl arm_cpu_topology_set 501 502 VPRINTF("go\n\r") 503 504 /* 505 * Jump to start in locore.S, which in turn will call initarm and main. 506 */ 507 b start 508 509 /* NOTREACHED */ 510 .ltorg 511#elif defined(_ARM_ARCH_6) 512generic_startv6: 513 VPRINTF("v6 : ") 514 515 bl armv6_init 516 bl generic_savevars 517 518#if defined(ARM_MMU_EXTENDED) 519 mov R_DEVATTR, #L1_S_V6_XN 520#else 521 mov R_DEVATTR, #0 522#endif 523 bl arm_build_translation_table 524 525 XPUTC(#'E') 526 /* 527 * Turn on the MMU. Return to new enabled address space. 528 */ 529 ldr r0, =TEMP_L1_TABLE 530 sub r0, R_VTOPDIFF 531 532 ldr lr, =generic_vstartv6 533 b armv6_mmuinit 534 535generic_vstartv6: 536 // Stack to KVA address 537 add sp, sp, R_VTOPDIFF 538 539 VPRINTF("virtual\n\r") 540 541#if defined(KASAN) 542 ldr r0, =start_stacks_bottom 543 bl _C_LABEL(kasan_early_init) 544 545 VPRINTF("kasan\n\r") 546#endif 547 548 VPRINTF("go\n\r") 549 550 /* 551 * Jump to start in locore.S, which in turn will call initarm and main. 552 */ 553 b start 554 555 /* NOTREACHED */ 556 .ltorg 557 558#endif 559 560#if defined(_ARM_ARCH_7) 561 562// 563// SCTLR register initialization values 564// 565#if defined(__ARMEL__) 566#define CPU_CONTROL_EX_BEND_SET 0 567#else 568#define CPU_CONTROL_EX_BEND_SET CPU_CONTROL_EX_BEND 569#endif 570 571#if defined(ARM32_DISABLE_ALIGNMENT_FAULTS) 572#define CPU_CONTROL_AFLT_ENABLE_CLR CPU_CONTROL_AFLT_ENABLE 573#define CPU_CONTROL_AFLT_ENABLE_SET 0 574#else 575#define CPU_CONTROL_AFLT_ENABLE_CLR 0 576#define CPU_CONTROL_AFLT_ENABLE_SET CPU_CONTROL_AFLT_ENABLE 577#endif 578 579#if defined(ARM_MMU_EXTENDED) 580#define CPU_CONTROL_XP_ENABLE_CLR 0 581#define CPU_CONTROL_XP_ENABLE_SET CPU_CONTROL_XP_ENABLE 582#else 583#define CPU_CONTROL_XP_ENABLE_CLR CPU_CONTROL_XP_ENABLE 584#define CPU_CONTROL_XP_ENABLE_SET 0 585#endif 586 587/* SWP is only usable on uni-processor ARMv7 systems. */ 588#ifdef MULTIPROCESSOR 589#define CPU_CONTROL_XP_SWP_ENABLE 0 590#else 591#define CPU_CONTROL_XP_SWP_ENABLE CPU_CONTROL_SWP_ENABLE 592#endif 593 594// bits to set in the Control Register 595// 596#define CPU_CONTROL_SET ( \ 597 CPU_CONTROL_MMU_ENABLE | \ 598 CPU_CONTROL_UNAL_ENABLE | \ 599 CPU_CONTROL_EX_BEND_SET | \ 600 CPU_CONTROL_AFLT_ENABLE_SET | \ 601 CPU_CONTROL_XP_ENABLE_SET | \ 602 0) 603 604// bits to clear in the Control Register 605// 606#define CPU_CONTROL_CLR ( \ 607 CPU_CONTROL_AFLT_ENABLE_CLR | \ 608 CPU_CONTROL_XP_ENABLE_CLR | \ 609 0) 610 611/* 612 * Perform the initialization of the an ARMv7 core required by NetBSD. 613 * 614 * Uses the following callee saved registers: 615 * 616 * r8 (R_TMP1), r9 (R_TMP2) 617 */ 618armv7_init: 619 620 .arch armv7a 621 .arch_extension sec 622 .arch_extension virt 623 624 mov R_TMP1, lr 625 mov R_TMP2, sp 626 627 /* 628 * Leave HYP mode and move into supervisor mode with IRQs/FIQs 629 * disabled. 630 */ 631 mrs r0, cpsr 632 and r0, r0, #(PSR_MODE) /* Mode is in the low 5 bits of CPSR */ 633 teq r0, #(PSR_HYP32_MODE) /* Hyp Mode? */ 634 bne 1f 635 636 XPUTC('h') 637 638 mov sp, #0 639 640 /* Set CNTVOFF to 0 */ 641 mov r1, #0 642 mcrr p15, 4, r1, r1, c14 643 644 /* Ensure that IRQ, and FIQ will be disabled after eret */ 645 mrs r0, cpsr 646 bic r0, r0, #(PSR_MODE) 647 orr r0, r0, #(PSR_SVC32_MODE) 648 orr r0, r0, #(I32_bit | F32_bit) 649 msr spsr_cxsf, r0 650 /* Exit hypervisor mode */ 651 adr lr, 2f 652 msr elr_hyp, lr 653 eret 654 6551: 656 cpsid if, #PSR_SVC32_MODE // SVC32 with no interrupts 657 6582: 659 mov r0, #0 660 msr spsr_sxc, r0 // set SPSR[23:8] to known value 661 662 mov sp, R_TMP2 663 664 XPUTC('A') 665 666 mrc p15, 0, r0, c1, c0, 0 667 tst r0, #CPU_CONTROL_DC_ENABLE 668 blne armv7_dcache_wbinv_all 669 670 // TeX remap 671 672#define ARMV7_SCTLR_CLEAR ( \ 673 CPU_CONTROL_IC_ENABLE | \ 674 CPU_CONTROL_DC_ENABLE | \ 675 CPU_CONTROL_MMU_ENABLE | \ 676 CPU_CONTROL_BPRD_ENABLE | \ 677 CPU_CONTROL_TR_ENABLE | \ 678 0) 679 680#define ARMV7_SCTLR_SET ( \ 681 CPU_CONTROL_UNAL_ENABLE | \ 682 CPU_CONTROL_XP_SWP_ENABLE | \ 683 0) 684 685 mrc p15, 0, r0, c1, c0, 0 686 movw r1, #:lower16:ARMV7_SCTLR_CLEAR 687 movt r1, #:upper16:ARMV7_SCTLR_CLEAR 688 movw r2, #:lower16:ARMV7_SCTLR_SET 689 movt r2, #:upper16:ARMV7_SCTLR_SET 690 691 mov R_TMP2, r0 // save for printing 692 bic r0, r0, r1 // disable icache/dcache/mmu 693 orr r0, r0, r2 // enable unaligned access 694 695 mcr p15, 0, r0, c1, c0, 0 // SCTLR write 696 dsb 697 isb 698 699 bl armv7_dcache_inv_all 700 mcr p15, 0, r0, c7, c5, 0 /* ICIALLU */ 701 dsb 702 isb 703 704#if defined(VERBOSE_INIT_ARM) 705 XPUTC(#'B') 706 707 VPRINTF(" sctlr:") 708 VPRINTX(R_TMP2) 709 VPRINTF("/") 710 mrc p15, 0, r0, c1, c0, 0 711 VPRINTX(r0) 712 VPRINTF(" ") 713 714 XPUTC(#'C') 715#endif 716 717 bx R_TMP1 // return 718 719 .ltorg 720 721/* 722 * Transitions the CPU to using the TTB passed in r0. 723 * 724 * Uses the following callee saved registers: 725 * 726 * Callee saved: 727 * r4, r5 728 */ 729 730armv7_mmuinit: 731 // Because the MMU may already be on do a typical sequence to set 732 // the Translation Table Base(s). 733 mov r4, lr 734 mov r5, r0 // save TTBR 735 736 XPUTC(#'F') 737 dsb // Drain the write buffers. 738 739 XPUTC(#'G') 740 mrc p15, 0, r1, c0, c0, 5 // MPIDR read 741 cmp r1, #0 742 orrlt r5, r5, #TTBR_MPATTR // MP, cachable (Normal WB) 743 orrge r5, r5, #TTBR_UPATTR // Non-MP, cacheable, normal WB 744 745 XPUTC(#'0') 746 mcr p15, 0, r5, c2, c0, 0 // TTBR0 write 747 748#if defined(ARM_MMU_EXTENDED) 749 // When using split TTBRs, we need to set both since the physical 750 // addresses we were/are using might be in either. 751 XPUTC(#'1') 752 mcr p15, 0, r5, c2, c0, 1 // TTBR1 write 753#endif 754 755 XPUTC(#'H') 756#if defined(ARM_MMU_EXTENDED) 757 XPUTC(#'1') 758 mov r1, #TTBCR_S_N_1 // make sure TTBCR_S_N is 1 759#else 760 XPUTC(#'0') 761 mov r1, #0 // make sure TTBCR is 0 762#endif 763 mcr p15, 0, r1, c2, c0, 2 // TTBCR write 764 765 XPUTC(#'J') 766 mov r1, #0 // get KERNEL_PID 767 mcr p15, 0, r1, c13, c0, 1 // CONTEXTIDR write 768 769 isb 770 771 // Set the Domain Access register. Very important! 772 XPUTC(#'K') 773 mov r1, #DOMAIN_DEFAULT 774 mcr p15, 0, r1, c3, c0, 0 // DACR write 775 776#if 0 777 778/* 779 * Set TEX remap registers 780 * - All is set to uncacheable memory 781 */ 782 ldr r0, =0xAAAAA 783 mcr CP15_PRRR(r0) 784 mov r0, #0 785 mcr CP15_NMRR(r0) 786#endif 787 788 XPUTC(#'I') 789 mov r1, #0 790 mcr p15, 0, r1, c8, c7, 0 // TLBIALL (just this core) 791 dsb 792 isb 793 794 // 795 // Enable the MMU, etc. 796 // 797 XPUTC(#'L') 798 XPUTC(#'\n') 799 XPUTC(#'\r') 800 mrc p15, 0, r1, c1, c0, 0 // SCTLR read 801 802 movw r3, #:lower16:CPU_CONTROL_SET 803 movt r3, #:upper16:CPU_CONTROL_SET 804 movw r2, #:lower16:CPU_CONTROL_CLR 805 movt r2, #:upper16:CPU_CONTROL_CLR 806 orr r0, r1, r3 807 bic r0, r0, r2 808 809 mcr p15, 0, r0, c1, c0, 0 /* SCTLR write */ 810 811 dsb 812 isb 813 814 mcr p15, 0, r0, c8, c7, 0 /* TLBIALL - Flush TLB */ 815 mcr p15, 0, r0, c7, c5, 6 /* BPIALL - Branch predictor invalidate all */ 816 dsb 817 isb 818 819 VPRINTF("MMU\n\r") 820 bx r4 // return 821 822 .p2align 2 823 824 .text 825 826ENTRY_NP(cpu_mpstart) 827#ifndef MULTIPROCESSOR 828 // 829 // If not MULTIPROCESSOR, drop CPU into power saving state. 830 // 8313: wfi 832 b 3b 833#else 834#if defined(__ARMEB__) 835 setend be // switch to BE now 836#endif 837 838 /* disable IRQs/FIQs. */ 839 cpsid if 840 841 adr R_TMP2, cpu_mpstart 842 ldr R_VTOPDIFF, =cpu_mpstart 843 sub R_VTOPDIFF, R_VTOPDIFF, R_TMP2 844 845 mrc p15, 0, r4, c0, c0, 5 // MPIDR get 846 and r4, #(MPIDR_AFF2|MPIDR_AFF1|MPIDR_AFF0) 847 848 mov r0, #0 849 ldr r1, =cpu_mpidr 850 sub r1, R_VTOPDIFF 8511: 852 ldr r2, [r1, r0, lsl #2] // r2 = cpu_mpidr[r0] 853 cmp r2, r4 854 beq 2f // found our mpidr 855 856 add r0, #1 857 cmp r0, #MAXCPUS 858 bne 1b 859 860 // Not found our mpidr in the list - use Aff0 for cpuindex 861 and r0, r4, #7 8622: 863 mov R_INDEX, r0 // save cpu_index for later 864 865 ldr R_TMP1, =start_stacks_top 866 sub sp, R_TMP1, R_VTOPDIFF 867 868 mov r5, R_INDEX 869 lsl r5, #INIT_ARM_STACK_SHIFT 870 sub sp, sp, r5 871 872#if defined(VERBOSE_INIT_ARM) 873 VPRINTF("\n\rmidr : ") 874 mrc p15, 0, r0, c0, c0, 0 // MIDR 875 VPRINTX(r0) 876 VPRINTF("\n\rrevidr: ") 877 mrc p15, 0, r0, c0, c0, 6 // REVIDR 878 VPRINTX(r0) 879 VPRINTF("\n\rmpidr : ") 880 mrc p15, 0, r0, c0, c0, 5 // MPIDR 881 VPRINTX(r0) 882#endif 883 VPRINTF("\n\rindex : ") 884 VPRINTX(R_INDEX) 885 VPRINTF("\n\rsp : ") 886 VPRINTX(sp) 887 XPUTC('\n') 888 XPUTC('\r') 889 890 // disables and clears caches 891 bl armv7_init 892 893 movw r0, #:lower16:TEMP_L1_TABLE 894 movt r0, #:upper16:TEMP_L1_TABLE 895 sub r0, R_VTOPDIFF 896 897 movw lr, #:lower16:armv7_mpcontinuation 898 movt lr, #:upper16:armv7_mpcontinuation 899 b armv7_mmuinit 900ASEND(cpu_mpstart) 901 902/* 903 * Now running with real kernel VA via bootstrap tables 904 */ 905armv7_mpcontinuation: 906 // Stack to KVA address 907 add sp, sp, R_VTOPDIFF 908 909 VPRINTF("virtual\n\r") 910 911 // index into cpu_mpidr[] or cpu_number if not found 912 mov r0, R_INDEX 913 bl cpu_init_secondary_processor 914 915 /* 916 * Wait for cpu_boot_secondary_processors 917 */ 918 919 /* r6 = &arm_cpu_mbox[0] */ 920 movw r6, #:lower16:arm_cpu_mbox 921 movt r6, #:upper16:arm_cpu_mbox 922 923 mov r5, #1 // bitmask... 924 lsl r5, R_INDEX // ... for our cpu 925 926 /* wait for the mailbox start bit to become true */ 9271: ldr r2, [r6] // load mbox 928 dmb // make it a load-acquire 929 tst r2, r5 // is our bit set? 930 wfeeq // no, back to waiting 931 beq 1b // no, and try again 932 933 movw r0, #:lower16:cpu_info 934 movt r0, #:upper16:cpu_info // get pointer to cpu_info 935 ldr r5, [r0, R_INDEX, lsl #2] // load our cpu_info 936 ldr r6, [r5, #CI_IDLELWP] // get the idlelwp 937 ldr r7, [r6, #L_PCB] // now get its pcb 938 ldr sp, [r7, #PCB_KSP] // finally, we can load our SP 939#if defined(TPIDRPRW_IS_CURCPU) 940 mcr p15, 0, r5, c13, c0, 4 // squirrel away curcpu() 941#elif defined(TPIDRPRW_IS_CURLWP) 942 mcr p15, 0, r6, c13, c0, 4 // squirrel away curlwp() 943#else 944#error either TPIDRPRW_IS_CURCPU or TPIDRPRW_IS_CURLWP must be defined 945#endif 946 /* 947 * No membar needed because we're not switching from a 948 * previous lwp, and the idle lwp we're switching to can't be 949 * holding locks already; see cpu_switchto. 950 */ 951 str r6, [r5, #CI_CURLWP] // and note we are running on it 952 953 mov r0, r5 // pass cpu_info 954 mov r1, R_INDEX // pass cpu_index 955 movw r2, #:lower16:MD_CPU_HATCH // pass md_cpu_hatch 956 movt r2, #:upper16:MD_CPU_HATCH // pass md_cpu_hatch 957 bl _C_LABEL(cpu_hatch) 958 b _C_LABEL(idle_loop) // never to return 959ASEND(armv7_mpcontinuation) 960#endif // MULTIPROCESSOR 961 962#elif defined(_ARM_ARCH_6) 963 964ENTRY_NP(armv6_init) 965/* 966 * Workaround Erratum 411920 967 * 968 * - value of arg 'reg' Should Be Zero 969 */ 970#define Invalidate_I_cache(reg) \ 971 .p2align 5; \ 972 mcr p15, 0, reg, c7, c5, 0; /* Invalidate Entire I cache */ \ 973 mcr p15, 0, reg, c7, c5, 0; /* Invalidate Entire I cache */ \ 974 mcr p15, 0, reg, c7, c5, 0; /* Invalidate Entire I cache */ \ 975 mcr p15, 0, reg, c7, c5, 0; /* Invalidate Entire I cache */ \ 976 nop; \ 977 nop; \ 978 nop; \ 979 nop; \ 980 nop; \ 981 nop; \ 982 nop; \ 983 nop; \ 984 nop; \ 985 nop; \ 986 nop; 987 988 mov R_TMP1, lr 989 mov r0, #0 /* SBZ */ 990 Invalidate_I_cache(r0) 991 992 mcr p15, 0, r0, c7, c14, 0 /* Clean and Invalidate Entire Data Cache */ 993 994 ldr r2, =(CPU_CONTROL_IC_ENABLE|CPU_CONTROL_DC_ENABLE) 995 /* Disable I+D caches */ 996 mrc p15, 0, r1, c1, c0, 0 /* " " " */ 997 mov R_TMP2, r1 998 bic r1, r1, r2 /* " " " */ 999 mcr p15, 0, r1, c1, c0, 0 /* " " " */ 1000 1001 mcr p15, 0, r0, c7, c10, 4 /* Drain the write buffers. */ 1002 1003#if defined(VERBOSE_INIT_ARM) 1004 XPUTC(#'B') 1005 1006 VPRINTF(" sctlr:") 1007 VPRINTX(R_TMP2) 1008 VPRINTF("/") 1009 mrc p15, 0, r0, c1, c0, 0 1010 VPRINTX(r0) 1011 VPRINTF(" ") 1012 1013 XPUTC(#'C') 1014#endif 1015 1016 bx R_TMP1 1017 1018 .ltorg 1019 1020armv6_mmuinit: 1021 mov r4, lr 1022 mov r5, r0 1023 1024 XPUTC(#'0') 1025 mcr p15, 0, r5, c2, c0, 0 // TTBR0 write 1026 1027#if defined(ARM_MMU_EXTENDED) 1028 // When using split TTBRs, we need to set both since the physical 1029 // addresses we were/are using might be in either. 1030 XPUTC(#'1') 1031 mcr p15, 0, r5, c2, c0, 1 /* TTBR1 write */ 1032#endif 1033 1034 XPUTC(#'H') 1035#if defined(ARM_MMU_EXTENDED) 1036 XPUTC(#'1') 1037 mov r1, #TTBCR_S_N_1 /* make sure TTBCR_S_N is 1 */ 1038#else 1039 XPUTC(#'0') 1040 mov r1, #0 /* make sure TTBCR is 0 */ 1041#endif 1042 mcr p15, 0, r1, c2, c0, 2 /* TTBCR write */ 1043 1044 XPUTC(#'I') 1045 1046 mov r0, #0 1047 mcr p15, 0, r0, c8, c7, 0 /* Invalidate TLBs */ 1048 1049 XPUTC(#'K') 1050 /* Set the Domain Access register. Very important! */ 1051 mov r0, #DOMAIN_DEFAULT 1052 mcr p15, 0, r0, c3, c0, 0 1053 1054 /* 1055 * Enable the MMU, etc. 1056 */ 1057 1058#if defined(VERBOSE_INIT_ARM) 1059 VPRINTF(" sctlr:") 1060 mrc p15, 0, r0, c1, c0, 0 1061 VPRINTX(r0) 1062 VPRINTF("/") 1063#endif 1064 1065 mrc p15, 0, r0, c1, c0, 0 1066 1067 ldr r1, Lcontrol_wax 1068 and r0, r0, r1 1069 ldr r1, Lcontrol_clr 1070 bic r0, r0, r1 1071 ldr r1, Lcontrol_set 1072 orr r0, r0, r1 1073 1074 mov r6, r0 1075 VPRINTX(r6) 1076 VPRINTF(" ") 1077 1078 .align 5 1079 /* turn mmu on! */ 1080 mov r0, r6 1081 mcr p15, 0, r0, c1, c0, 0 1082 1083 /* 1084 * Ensure that the coprocessor has finished turning on the MMU. 1085 */ 1086 mrc p15, 0, r0, c0, c0, 0 /* Read an arbitrary value. */ 1087 mov r0, r0 /* Stall until read completes. */ 1088 1089 nop 1090 nop 1091 nop 1092 nop 1093 nop 1094 1095 mov pc, r4 1096 1097 .ltorg 1098 1099 /* bits to set in the Control Register */ 1100Lcontrol_set: 1101#if defined(ARM_MMU_EXTENDED) 1102#define CPU_CONTROL_EXTRA CPU_CONTROL_XP_ENABLE 1103#else 1104#define CPU_CONTROL_EXTRA CPU_CONTROL_SYST_ENABLE 1105#endif 1106#if defined(__ARMEL__) 1107#define CPU_CONTROL_EX_BEND_SET 0 1108#else 1109#define CPU_CONTROL_EX_BEND_SET CPU_CONTROL_EX_BEND 1110#endif 1111 .word CPU_CONTROL_MMU_ENABLE | \ 1112 CPU_CONTROL_WBUF_ENABLE | /* not defined in 1176 (SBO) */ \ 1113 CPU_CONTROL_32BP_ENABLE | /* SBO */ \ 1114 CPU_CONTROL_32BD_ENABLE | /* SBO */ \ 1115 CPU_CONTROL_LABT_ENABLE | /* SBO */ \ 1116 (1 << 16) | /* SBO - Global enable for data tcm */ \ 1117 (1 << 18) | /* SBO - Global enable for insn tcm */ \ 1118 CPU_CONTROL_UNAL_ENABLE | \ 1119 CPU_CONTROL_EXTRA | \ 1120 CPU_CONTROL_EX_BEND_SET 1121 1122 /* bits to clear in the Control Register */ 1123Lcontrol_clr: 1124 .word 0 1125 1126 /* bits to "write as existing" in the Control Register */ 1127Lcontrol_wax: 1128 .word (3 << 30) | \ 1129 (1 << 29) | \ 1130 (1 << 28) | \ 1131 (3 << 26) | \ 1132 (3 << 19) | \ 1133 (1 << 17) | \ 1134 (1 << 10) 1135#endif 1136 1137ENTRY_NP(generic_vprint) 1138 push {r4, lr} 1139 1140 mov r4, lr 1141 b 2f 11421: 1143 bl uartputc 1144 11452: 1146 ldrb r0, [r4], #1 1147 cmp r0, #0 1148 bne 1b 1149 1150 add lr, r4, #3 1151 bic lr, #3 1152 1153 pop {r4} 1154 add sp, sp, #4 1155 mov pc, lr 1156ASEND(generic_vprint) 1157 1158ENTRY_NP(generic_prints) 1159 push {r4, lr} 1160 1161 mov r4, r0 11621: 1163 ldrb r0, [r4], #1 1164 cmp r0, #0 1165 popeq {r4, pc} 1166 1167 bl uartputc 1168 b 1b 1169ASEND(generic_prints) 1170 1171ENTRY_NP(generic_printx) 1172 push {r4, r5, r6, lr} 1173 mov r5, r0 1174 1175 mov r0, #'0' 1176 bl uartputc 1177 mov r0, #'x' 1178 bl uartputc 1179 1180 // Word size in bits 1181 mov r4, #32 11821: 1183 sub r4, r4, #4 // nibble shift 1184 lsr r3, r5, r4 // extract ... 1185 and r3, r3, #0xf // ... nibble 1186 1187 cmp r3, #9 1188 add r0, r3, #'0' 1189 addgt r0, r3, #'a' - 10 1190 bl uartputc 1191 1192 cmp r4, #0 1193 bne 1b 1194 pop {r4, r5, r6, pc} 1195ASEND(generic_printx) 1196