1 /* $NetBSD: arm32_kvminit.c,v 1.41 2017/12/10 21:38:26 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 2002, 2003, 2005 Genetec Corporation. All rights reserved. 5 * Written by Hiroyuki Bessho for Genetec Corporation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of Genetec Corporation may not be used to endorse or 16 * promote products derived from this software without specific prior 17 * written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY GENETEC CORPORATION ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENETEC CORPORATION 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * Copyright (c) 2001 Wasabi Systems, Inc. 32 * All rights reserved. 33 * 34 * Written by Jason R. Thorpe for Wasabi Systems, Inc. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 3. All advertising materials mentioning features or use of this software 45 * must display the following acknowledgement: 46 * This product includes software developed for the NetBSD Project by 47 * Wasabi Systems, Inc. 48 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 49 * or promote products derived from this software without specific prior 50 * written permission. 51 * 52 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 53 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 54 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 55 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 56 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 57 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 58 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 59 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 60 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 61 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 62 * POSSIBILITY OF SUCH DAMAGE. 63 * 64 * Copyright (c) 1997,1998 Mark Brinicombe. 65 * Copyright (c) 1997,1998 Causality Limited. 66 * All rights reserved. 67 * 68 * Redistribution and use in source and binary forms, with or without 69 * modification, are permitted provided that the following conditions 70 * are met: 71 * 1. Redistributions of source code must retain the above copyright 72 * notice, this list of conditions and the following disclaimer. 73 * 2. Redistributions in binary form must reproduce the above copyright 74 * notice, this list of conditions and the following disclaimer in the 75 * documentation and/or other materials provided with the distribution. 76 * 3. All advertising materials mentioning features or use of this software 77 * must display the following acknowledgement: 78 * This product includes software developed by Mark Brinicombe 79 * for the NetBSD Project. 80 * 4. The name of the company nor the name of the author may be used to 81 * endorse or promote products derived from this software without specific 82 * prior written permission. 83 * 84 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 85 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 86 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 87 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 88 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 89 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 90 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 91 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 92 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 93 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 94 * SUCH DAMAGE. 95 * 96 * Copyright (c) 2007 Microsoft 97 * All rights reserved. 98 * 99 * Redistribution and use in source and binary forms, with or without 100 * modification, are permitted provided that the following conditions 101 * are met: 102 * 1. Redistributions of source code must retain the above copyright 103 * notice, this list of conditions and the following disclaimer. 104 * 2. Redistributions in binary form must reproduce the above copyright 105 * notice, this list of conditions and the following disclaimer in the 106 * documentation and/or other materials provided with the distribution. 107 * 3. All advertising materials mentioning features or use of this software 108 * must display the following acknowledgement: 109 * This product includes software developed by Microsoft 110 * 111 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 112 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 113 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 114 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTERS BE LIABLE FOR ANY DIRECT, 115 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 116 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 117 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 118 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 119 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 120 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 121 * SUCH DAMAGE. 122 */ 123 124 #include "opt_fdt.h" 125 #include "opt_multiprocessor.h" 126 127 #include <sys/cdefs.h> 128 __KERNEL_RCSID(0, "$NetBSD: arm32_kvminit.c,v 1.41 2017/12/10 21:38:26 skrll Exp $"); 129 130 #include <sys/param.h> 131 #include <sys/device.h> 132 #include <sys/kernel.h> 133 #include <sys/reboot.h> 134 #include <sys/bus.h> 135 136 #include <dev/cons.h> 137 138 #include <uvm/uvm_extern.h> 139 140 #include <arm/locore.h> 141 #include <arm/db_machdep.h> 142 #include <arm/undefined.h> 143 #include <arm/bootconfig.h> 144 #include <arm/arm32/machdep.h> 145 146 #if defined(FDT) 147 #include <arch/evbarm/fdt/platform.h> 148 #endif 149 150 #ifdef MULTIPROCESSOR 151 #ifndef __HAVE_CPU_UAREA_ALLOC_IDLELWP 152 #error __HAVE_CPU_UAREA_ALLOC_IDLELWP required to not waste pages for idlestack 153 #endif 154 #endif 155 156 struct bootmem_info bootmem_info; 157 158 extern void *msgbufaddr; 159 paddr_t msgbufphys; 160 paddr_t physical_start; 161 paddr_t physical_end; 162 163 extern char etext[]; 164 extern char __data_start[], _edata[]; 165 extern char __bss_start[], __bss_end__[]; 166 extern char _end[]; 167 168 /* Page tables for mapping kernel VM */ 169 #define KERNEL_L2PT_VMDATA_NUM 8 /* start with 32MB of KVM */ 170 171 /* 172 * Macros to translate between physical and virtual for a subset of the 173 * kernel address space. *Not* for general use. 174 */ 175 #if defined(KERNEL_BASE_VOFFSET) 176 #define KERN_VTOPHYS(bmi, va) \ 177 ((paddr_t)((vaddr_t)(va) - KERNEL_BASE_VOFFSET)) 178 #define KERN_PHYSTOV(bmi, pa) \ 179 ((vaddr_t)((paddr_t)(pa) + KERNEL_BASE_VOFFSET)) 180 #else 181 #define KERN_VTOPHYS(bmi, va) \ 182 ((paddr_t)((vaddr_t)(va) - KERNEL_BASE + (bmi)->bmi_start)) 183 #define KERN_PHYSTOV(bmi, pa) \ 184 ((vaddr_t)((paddr_t)(pa) - (bmi)->bmi_start + KERNEL_BASE)) 185 #endif 186 187 void 188 arm32_bootmem_init(paddr_t memstart, psize_t memsize, vsize_t kernelstart) 189 { 190 struct bootmem_info * const bmi = &bootmem_info; 191 pv_addr_t *pv = bmi->bmi_freeblocks; 192 193 #ifdef VERBOSE_INIT_ARM 194 printf("%s: memstart=%#lx, memsize=%#lx, kernelstart=%#lx\n", 195 __func__, memstart, memsize, kernelstart); 196 #endif 197 198 physical_start = bmi->bmi_start = memstart; 199 physical_end = bmi->bmi_end = memstart + memsize; 200 #ifndef ARM_HAS_LPAE 201 if (physical_end == 0) { 202 physical_end = -PAGE_SIZE; 203 memsize -= PAGE_SIZE; 204 bmi->bmi_end -= PAGE_SIZE; 205 #ifdef VERBOSE_INIT_ARM 206 printf("%s: memsize shrunk by a page to avoid ending at 4GB\n", 207 __func__); 208 #endif 209 } 210 #endif 211 physmem = memsize / PAGE_SIZE; 212 213 /* 214 * Let's record where the kernel lives. 215 */ 216 bmi->bmi_kernelstart = kernelstart; 217 bmi->bmi_kernelend = KERN_VTOPHYS(bmi, round_page((vaddr_t)_end)); 218 219 #if defined(FDT) 220 fdt_add_reserved_memory_range(bmi->bmi_kernelstart, 221 bmi->bmi_kernelend - bmi->bmi_kernelstart); 222 #endif 223 224 #ifdef VERBOSE_INIT_ARM 225 printf("%s: kernelend=%#lx\n", __func__, bmi->bmi_kernelend); 226 #endif 227 228 /* 229 * Now the rest of the free memory must be after the kernel. 230 */ 231 pv->pv_pa = bmi->bmi_kernelend; 232 pv->pv_va = KERN_PHYSTOV(bmi, pv->pv_pa); 233 pv->pv_size = bmi->bmi_end - bmi->bmi_kernelend; 234 bmi->bmi_freepages += pv->pv_size / PAGE_SIZE; 235 #ifdef VERBOSE_INIT_ARM 236 printf("%s: adding %lu free pages: [%#lx..%#lx] (VA %#lx)\n", 237 __func__, pv->pv_size / PAGE_SIZE, pv->pv_pa, 238 pv->pv_pa + pv->pv_size - 1, pv->pv_va); 239 #endif 240 pv++; 241 242 #if !defined(FDT) 243 /* 244 * Add a free block for any memory before the kernel. 245 */ 246 if (bmi->bmi_start < bmi->bmi_kernelstart) { 247 pv->pv_pa = bmi->bmi_start; 248 pv->pv_va = KERN_PHYSTOV(bmi, pv->pv_pa); 249 pv->pv_size = bmi->bmi_kernelstart - pv->pv_pa; 250 bmi->bmi_freepages += pv->pv_size / PAGE_SIZE; 251 #ifdef VERBOSE_INIT_ARM 252 printf("%s: adding %lu free pages: [%#lx..%#lx] (VA %#lx)\n", 253 __func__, pv->pv_size / PAGE_SIZE, pv->pv_pa, 254 pv->pv_pa + pv->pv_size - 1, pv->pv_va); 255 #endif 256 pv++; 257 } 258 #endif 259 260 bmi->bmi_nfreeblocks = pv - bmi->bmi_freeblocks; 261 262 SLIST_INIT(&bmi->bmi_freechunks); 263 SLIST_INIT(&bmi->bmi_chunks); 264 } 265 266 static bool 267 concat_pvaddr(pv_addr_t *acc_pv, pv_addr_t *pv) 268 { 269 if (acc_pv->pv_pa + acc_pv->pv_size == pv->pv_pa 270 && acc_pv->pv_va + acc_pv->pv_size == pv->pv_va 271 && acc_pv->pv_prot == pv->pv_prot 272 && acc_pv->pv_cache == pv->pv_cache) { 273 #ifdef VERBOSE_INIT_ARMX 274 printf("%s: appending pv %p (%#lx..%#lx) to %#lx..%#lx\n", 275 __func__, pv, pv->pv_pa, pv->pv_pa + pv->pv_size + 1, 276 acc_pv->pv_pa, acc_pv->pv_pa + acc_pv->pv_size + 1); 277 #endif 278 acc_pv->pv_size += pv->pv_size; 279 return true; 280 } 281 282 return false; 283 } 284 285 static void 286 add_pages(struct bootmem_info *bmi, pv_addr_t *pv) 287 { 288 pv_addr_t **pvp = &SLIST_FIRST(&bmi->bmi_chunks); 289 while ((*pvp) != NULL && (*pvp)->pv_va <= pv->pv_va) { 290 pv_addr_t * const pv0 = (*pvp); 291 KASSERT(SLIST_NEXT(pv0, pv_list) == NULL || pv0->pv_pa < SLIST_NEXT(pv0, pv_list)->pv_pa); 292 if (concat_pvaddr(pv0, pv)) { 293 #ifdef VERBOSE_INIT_ARM 294 printf("%s: %s pv %p (%#lx..%#lx) to %#lx..%#lx\n", 295 __func__, "appending", pv, 296 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 297 pv0->pv_pa, pv0->pv_pa + pv0->pv_size - pv->pv_size - 1); 298 #endif 299 pv = SLIST_NEXT(pv0, pv_list); 300 if (pv != NULL && concat_pvaddr(pv0, pv)) { 301 #ifdef VERBOSE_INIT_ARM 302 printf("%s: %s pv %p (%#lx..%#lx) to %#lx..%#lx\n", 303 __func__, "merging", pv, 304 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 305 pv0->pv_pa, 306 pv0->pv_pa + pv0->pv_size - pv->pv_size - 1); 307 #endif 308 SLIST_REMOVE_AFTER(pv0, pv_list); 309 SLIST_INSERT_HEAD(&bmi->bmi_freechunks, pv, pv_list); 310 } 311 return; 312 } 313 KASSERT(pv->pv_va != (*pvp)->pv_va); 314 pvp = &SLIST_NEXT(*pvp, pv_list); 315 } 316 KASSERT((*pvp) == NULL || pv->pv_va < (*pvp)->pv_va); 317 pv_addr_t * const new_pv = SLIST_FIRST(&bmi->bmi_freechunks); 318 KASSERT(new_pv != NULL); 319 SLIST_REMOVE_HEAD(&bmi->bmi_freechunks, pv_list); 320 *new_pv = *pv; 321 SLIST_NEXT(new_pv, pv_list) = *pvp; 322 (*pvp) = new_pv; 323 #ifdef VERBOSE_INIT_ARM 324 printf("%s: adding pv %p (pa %#lx, va %#lx, %lu pages) ", 325 __func__, new_pv, new_pv->pv_pa, new_pv->pv_va, 326 new_pv->pv_size / PAGE_SIZE); 327 if (SLIST_NEXT(new_pv, pv_list)) 328 printf("before pa %#lx\n", SLIST_NEXT(new_pv, pv_list)->pv_pa); 329 else 330 printf("at tail\n"); 331 #endif 332 } 333 334 static void 335 valloc_pages(struct bootmem_info *bmi, pv_addr_t *pv, size_t npages, 336 int prot, int cache, bool zero_p) 337 { 338 size_t nbytes = npages * PAGE_SIZE; 339 pv_addr_t *free_pv = bmi->bmi_freeblocks; 340 size_t free_idx = 0; 341 static bool l1pt_found; 342 343 KASSERT(npages > 0); 344 345 /* 346 * If we haven't allocated the kernel L1 page table and we are aligned 347 * at a L1 table boundary, alloc the memory for it. 348 */ 349 if (!l1pt_found 350 && (free_pv->pv_pa & (L1_TABLE_SIZE - 1)) == 0 351 && free_pv->pv_size >= L1_TABLE_SIZE) { 352 l1pt_found = true; 353 valloc_pages(bmi, &kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE, 354 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 355 add_pages(bmi, &kernel_l1pt); 356 } 357 358 while (nbytes > free_pv->pv_size) { 359 free_pv++; 360 free_idx++; 361 if (free_idx == bmi->bmi_nfreeblocks) { 362 panic("%s: could not allocate %zu bytes", 363 __func__, nbytes); 364 } 365 } 366 367 /* 368 * As we allocate the memory, make sure that we don't walk over 369 * our current first level translation table. 370 */ 371 KASSERT((armreg_ttbr_read() & ~(L1_TABLE_SIZE - 1)) != free_pv->pv_pa); 372 373 #if defined(FDT) 374 fdt_add_reserved_memory_range(free_pv->pv_pa, nbytes); 375 #endif 376 pv->pv_pa = free_pv->pv_pa; 377 pv->pv_va = free_pv->pv_va; 378 pv->pv_size = nbytes; 379 pv->pv_prot = prot; 380 pv->pv_cache = cache; 381 382 /* 383 * If PTE_PAGETABLE uses the same cache modes as PTE_CACHE 384 * just use PTE_CACHE. 385 */ 386 if (cache == PTE_PAGETABLE 387 && pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt 388 && pte_l2_l_cache_mode == pte_l2_l_cache_mode_pt 389 && pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt) 390 pv->pv_cache = PTE_CACHE; 391 392 free_pv->pv_pa += nbytes; 393 free_pv->pv_va += nbytes; 394 free_pv->pv_size -= nbytes; 395 if (free_pv->pv_size == 0) { 396 --bmi->bmi_nfreeblocks; 397 for (; free_idx < bmi->bmi_nfreeblocks; free_idx++) { 398 free_pv[0] = free_pv[1]; 399 } 400 } 401 402 bmi->bmi_freepages -= npages; 403 404 if (zero_p) 405 memset((void *)pv->pv_va, 0, nbytes); 406 } 407 408 void 409 arm32_kernel_vm_init(vaddr_t kernel_vm_base, vaddr_t vectors, vaddr_t iovbase, 410 const struct pmap_devmap *devmap, bool mapallmem_p) 411 { 412 struct bootmem_info * const bmi = &bootmem_info; 413 #ifdef MULTIPROCESSOR 414 const size_t cpu_num = arm_cpu_max; 415 #else 416 const size_t cpu_num = 1; 417 #endif 418 #ifdef ARM_HAS_VBAR 419 const bool map_vectors_p = false; 420 #elif defined(CPU_ARMV7) || defined(CPU_ARM11) 421 const bool map_vectors_p = vectors == ARM_VECTORS_HIGH 422 || (armreg_pfr1_read() & ARM_PFR1_SEC_MASK) == 0; 423 #else 424 const bool map_vectors_p = true; 425 #endif 426 427 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 428 KASSERT(mapallmem_p); 429 #ifdef ARM_MMU_EXTENDED 430 /* 431 * The direct map VA space ends at the start of the kernel VM space. 432 */ 433 pmap_directlimit = kernel_vm_base; 434 #else 435 KASSERT(kernel_vm_base - KERNEL_BASE >= physical_end - physical_start); 436 #endif /* ARM_MMU_EXTENDED */ 437 #endif /* __HAVE_MM_MD_DIRECT_MAPPED_PHYS */ 438 439 /* 440 * Calculate the number of L2 pages needed for mapping the 441 * kernel + data + stuff. Assume 2 L2 pages for kernel, 1 for vectors, 442 * and 1 for IO 443 */ 444 size_t kernel_size = bmi->bmi_kernelend; 445 kernel_size -= (bmi->bmi_kernelstart & -L2_S_SEGSIZE); 446 kernel_size += L1_TABLE_SIZE_REAL; 447 kernel_size += PAGE_SIZE * KERNEL_L2PT_VMDATA_NUM; 448 if (map_vectors_p) { 449 kernel_size += PAGE_SIZE; /* L2PT for VECTORS */ 450 } 451 if (iovbase) { 452 kernel_size += PAGE_SIZE; /* L2PT for IO */ 453 } 454 kernel_size += 455 cpu_num * (ABT_STACK_SIZE + FIQ_STACK_SIZE + IRQ_STACK_SIZE 456 + UND_STACK_SIZE + UPAGES) * PAGE_SIZE; 457 kernel_size += round_page(MSGBUFSIZE); 458 kernel_size += 0x10000; /* slop */ 459 if (!mapallmem_p) { 460 kernel_size += PAGE_SIZE 461 * ((kernel_size + L2_S_SEGSIZE - 1) / L2_S_SEGSIZE); 462 } 463 kernel_size = round_page(kernel_size); 464 465 /* 466 * Now we know how many L2 pages it will take. 467 */ 468 const size_t KERNEL_L2PT_KERNEL_NUM = 469 round_page(kernel_size + L2_S_SEGSIZE - 1) / L2_S_SEGSIZE; 470 471 #ifdef VERBOSE_INIT_ARM 472 printf("%s: %zu L2 pages are needed to map %#zx kernel bytes\n", 473 __func__, KERNEL_L2PT_KERNEL_NUM, kernel_size); 474 #endif 475 476 KASSERT(KERNEL_L2PT_KERNEL_NUM + KERNEL_L2PT_VMDATA_NUM < __arraycount(bmi->bmi_l2pts)); 477 pv_addr_t * const kernel_l2pt = bmi->bmi_l2pts; 478 pv_addr_t * const vmdata_l2pt = kernel_l2pt + KERNEL_L2PT_KERNEL_NUM; 479 pv_addr_t msgbuf; 480 pv_addr_t text; 481 pv_addr_t data; 482 pv_addr_t chunks[KERNEL_L2PT_KERNEL_NUM+KERNEL_L2PT_VMDATA_NUM+11]; 483 #if ARM_MMU_XSCALE == 1 484 pv_addr_t minidataclean; 485 #endif 486 487 /* 488 * We need to allocate some fixed page tables to get the kernel going. 489 * 490 * We are going to allocate our bootstrap pages from the beginning of 491 * the free space that we just calculated. We allocate one page 492 * directory and a number of page tables and store the physical 493 * addresses in the bmi_l2pts array in bootmem_info. 494 * 495 * The kernel page directory must be on a 16K boundary. The page 496 * tables must be on 4K boundaries. What we do is allocate the 497 * page directory on the first 16K boundary that we encounter, and 498 * the page tables on 4K boundaries otherwise. Since we allocate 499 * at least 3 L2 page tables, we are guaranteed to encounter at 500 * least one 16K aligned region. 501 */ 502 503 #ifdef VERBOSE_INIT_ARM 504 printf("%s: allocating page tables for", __func__); 505 #endif 506 for (size_t i = 0; i < __arraycount(chunks); i++) { 507 SLIST_INSERT_HEAD(&bmi->bmi_freechunks, &chunks[i], pv_list); 508 } 509 510 kernel_l1pt.pv_pa = 0; 511 kernel_l1pt.pv_va = 0; 512 513 /* 514 * Allocate the L2 pages, but if we get to a page that is aligned for 515 * an L1 page table, we will allocate the pages for it first and then 516 * allocate the L2 page. 517 */ 518 519 if (map_vectors_p) { 520 /* 521 * First allocate L2 page for the vectors. 522 */ 523 #ifdef VERBOSE_INIT_ARM 524 printf(" vector"); 525 #endif 526 valloc_pages(bmi, &bmi->bmi_vector_l2pt, 1, 527 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 528 add_pages(bmi, &bmi->bmi_vector_l2pt); 529 } 530 531 /* 532 * Now allocate L2 pages for the kernel 533 */ 534 #ifdef VERBOSE_INIT_ARM 535 printf(" kernel"); 536 #endif 537 for (size_t idx = 0; idx < KERNEL_L2PT_KERNEL_NUM; ++idx) { 538 valloc_pages(bmi, &kernel_l2pt[idx], 1, 539 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 540 add_pages(bmi, &kernel_l2pt[idx]); 541 } 542 543 /* 544 * Now allocate L2 pages for the initial kernel VA space. 545 */ 546 #ifdef VERBOSE_INIT_ARM 547 printf(" vm"); 548 #endif 549 for (size_t idx = 0; idx < KERNEL_L2PT_VMDATA_NUM; ++idx) { 550 valloc_pages(bmi, &vmdata_l2pt[idx], 1, 551 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 552 add_pages(bmi, &vmdata_l2pt[idx]); 553 } 554 555 /* 556 * If someone wanted a L2 page for I/O, allocate it now. 557 */ 558 if (iovbase) { 559 #ifdef VERBOSE_INIT_ARM 560 printf(" io"); 561 #endif 562 valloc_pages(bmi, &bmi->bmi_io_l2pt, 1, 563 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 564 add_pages(bmi, &bmi->bmi_io_l2pt); 565 } 566 567 #ifdef VERBOSE_INIT_ARM 568 printf("%s: allocating stacks\n", __func__); 569 #endif 570 571 /* Allocate stacks for all modes and CPUs */ 572 valloc_pages(bmi, &abtstack, ABT_STACK_SIZE * cpu_num, 573 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 574 add_pages(bmi, &abtstack); 575 valloc_pages(bmi, &fiqstack, FIQ_STACK_SIZE * cpu_num, 576 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 577 add_pages(bmi, &fiqstack); 578 valloc_pages(bmi, &irqstack, IRQ_STACK_SIZE * cpu_num, 579 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 580 add_pages(bmi, &irqstack); 581 valloc_pages(bmi, &undstack, UND_STACK_SIZE * cpu_num, 582 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 583 add_pages(bmi, &undstack); 584 valloc_pages(bmi, &idlestack, UPAGES * cpu_num, /* SVC32 */ 585 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 586 add_pages(bmi, &idlestack); 587 valloc_pages(bmi, &kernelstack, UPAGES, /* SVC32 */ 588 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 589 add_pages(bmi, &kernelstack); 590 591 /* Allocate the message buffer from the end of memory. */ 592 const size_t msgbuf_pgs = round_page(MSGBUFSIZE) / PAGE_SIZE; 593 valloc_pages(bmi, &msgbuf, msgbuf_pgs, 594 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, false); 595 add_pages(bmi, &msgbuf); 596 msgbufphys = msgbuf.pv_pa; 597 msgbufaddr = (void *)msgbuf.pv_va; 598 599 if (map_vectors_p) { 600 /* 601 * Allocate a page for the system vector page. 602 * This page will just contain the system vectors and can be 603 * shared by all processes. 604 */ 605 valloc_pages(bmi, &systempage, 1, VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE, 606 PTE_CACHE, true); 607 } 608 systempage.pv_va = vectors; 609 610 /* 611 * If the caller needed a few extra pages for some reason, allocate 612 * them now. 613 */ 614 #if ARM_MMU_XSCALE == 1 615 #if (ARM_NMMUS > 1) 616 if (xscale_use_minidata) 617 #endif 618 valloc_pages(bmi, &minidataclean, 1, 619 VM_PROT_READ|VM_PROT_WRITE, 0, true); 620 #endif 621 622 /* 623 * Ok we have allocated physical pages for the primary kernel 624 * page tables and stacks. Let's just confirm that. 625 */ 626 if (kernel_l1pt.pv_va == 0 627 && (!kernel_l1pt.pv_pa || (kernel_l1pt.pv_pa & (L1_TABLE_SIZE - 1)) != 0)) 628 panic("%s: Failed to allocate or align the kernel " 629 "page directory", __func__); 630 631 632 #ifdef VERBOSE_INIT_ARM 633 printf("Creating L1 page table at 0x%08lx\n", kernel_l1pt.pv_pa); 634 #endif 635 636 /* 637 * Now we start construction of the L1 page table 638 * We start by mapping the L2 page tables into the L1. 639 * This means that we can replace L1 mappings later on if necessary 640 */ 641 vaddr_t l1pt_va = kernel_l1pt.pv_va; 642 paddr_t l1pt_pa = kernel_l1pt.pv_pa; 643 644 if (map_vectors_p) { 645 /* Map the L2 pages tables in the L1 page table */ 646 pmap_link_l2pt(l1pt_va, systempage.pv_va & -L2_S_SEGSIZE, 647 &bmi->bmi_vector_l2pt); 648 #ifdef VERBOSE_INIT_ARM 649 printf("%s: adding L2 pt (VA %#lx, PA %#lx) " 650 "for VA %#lx\n (vectors)", 651 __func__, bmi->bmi_vector_l2pt.pv_va, 652 bmi->bmi_vector_l2pt.pv_pa, systempage.pv_va); 653 #endif 654 } 655 656 const vaddr_t kernel_base = 657 KERN_PHYSTOV(bmi, bmi->bmi_kernelstart & -L2_S_SEGSIZE); 658 for (size_t idx = 0; idx < KERNEL_L2PT_KERNEL_NUM; idx++) { 659 pmap_link_l2pt(l1pt_va, kernel_base + idx * L2_S_SEGSIZE, 660 &kernel_l2pt[idx]); 661 #ifdef VERBOSE_INIT_ARM 662 printf("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (kernel)\n", 663 __func__, kernel_l2pt[idx].pv_va, 664 kernel_l2pt[idx].pv_pa, kernel_base + idx * L2_S_SEGSIZE); 665 #endif 666 } 667 668 for (size_t idx = 0; idx < KERNEL_L2PT_VMDATA_NUM; idx++) { 669 pmap_link_l2pt(l1pt_va, kernel_vm_base + idx * L2_S_SEGSIZE, 670 &vmdata_l2pt[idx]); 671 #ifdef VERBOSE_INIT_ARM 672 printf("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (vm)\n", 673 __func__, vmdata_l2pt[idx].pv_va, vmdata_l2pt[idx].pv_pa, 674 kernel_vm_base + idx * L2_S_SEGSIZE); 675 #endif 676 } 677 if (iovbase) { 678 pmap_link_l2pt(l1pt_va, iovbase & -L2_S_SEGSIZE, &bmi->bmi_io_l2pt); 679 #ifdef VERBOSE_INIT_ARM 680 printf("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (io)\n", 681 __func__, bmi->bmi_io_l2pt.pv_va, bmi->bmi_io_l2pt.pv_pa, 682 iovbase & -L2_S_SEGSIZE); 683 #endif 684 } 685 686 /* update the top of the kernel VM */ 687 pmap_curmaxkvaddr = 688 kernel_vm_base + (KERNEL_L2PT_VMDATA_NUM * L2_S_SEGSIZE); 689 690 #ifdef VERBOSE_INIT_ARM 691 printf("Mapping kernel\n"); 692 #endif 693 694 extern char etext[], _end[]; 695 size_t totalsize = bmi->bmi_kernelend - bmi->bmi_kernelstart; 696 size_t textsize = KERN_VTOPHYS(bmi, (uintptr_t)etext) - bmi->bmi_kernelstart; 697 698 textsize = (textsize + PGOFSET) & ~PGOFSET; 699 700 /* start at offset of kernel in RAM */ 701 702 text.pv_pa = bmi->bmi_kernelstart; 703 text.pv_va = KERN_PHYSTOV(bmi, bmi->bmi_kernelstart); 704 text.pv_size = textsize; 705 text.pv_prot = VM_PROT_READ | VM_PROT_EXECUTE; 706 text.pv_cache = PTE_CACHE; 707 708 #ifdef VERBOSE_INIT_ARM 709 printf("%s: adding chunk for kernel text %#lx..%#lx (VA %#lx)\n", 710 __func__, text.pv_pa, text.pv_pa + text.pv_size - 1, text.pv_va); 711 #endif 712 713 add_pages(bmi, &text); 714 715 data.pv_pa = text.pv_pa + textsize; 716 data.pv_va = text.pv_va + textsize; 717 data.pv_size = totalsize - textsize; 718 data.pv_prot = VM_PROT_READ|VM_PROT_WRITE; 719 data.pv_cache = PTE_CACHE; 720 721 #ifdef VERBOSE_INIT_ARM 722 printf("%s: adding chunk for kernel data/bss %#lx..%#lx (VA %#lx)\n", 723 __func__, data.pv_pa, data.pv_pa + data.pv_size - 1, data.pv_va); 724 #endif 725 726 add_pages(bmi, &data); 727 728 #ifdef VERBOSE_INIT_ARM 729 printf("Listing Chunks\n"); 730 731 pv_addr_t *lpv; 732 SLIST_FOREACH(lpv, &bmi->bmi_chunks, pv_list) { 733 printf("%s: pv %p: chunk VA %#lx..%#lx " 734 "(PA %#lx, prot %d, cache %d)\n", 735 __func__, lpv, lpv->pv_va, lpv->pv_va + lpv->pv_size - 1, 736 lpv->pv_pa, lpv->pv_prot, lpv->pv_cache); 737 } 738 printf("\nMapping Chunks\n"); 739 #endif 740 741 pv_addr_t cur_pv; 742 pv_addr_t *pv = SLIST_FIRST(&bmi->bmi_chunks); 743 if (!mapallmem_p || pv->pv_pa == bmi->bmi_start) { 744 cur_pv = *pv; 745 KASSERTMSG(cur_pv.pv_va >= KERNEL_BASE, "%#lx", cur_pv.pv_va); 746 pv = SLIST_NEXT(pv, pv_list); 747 } else { 748 cur_pv.pv_va = KERNEL_BASE; 749 cur_pv.pv_pa = KERN_VTOPHYS(bmi, cur_pv.pv_va); 750 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_pa; 751 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 752 cur_pv.pv_cache = PTE_CACHE; 753 } 754 while (pv != NULL) { 755 if (mapallmem_p) { 756 if (concat_pvaddr(&cur_pv, pv)) { 757 pv = SLIST_NEXT(pv, pv_list); 758 continue; 759 } 760 if (cur_pv.pv_pa + cur_pv.pv_size < pv->pv_pa) { 761 /* 762 * See if we can extend the current pv to emcompass the 763 * hole, and if so do it and retry the concatenation. 764 */ 765 if (cur_pv.pv_prot == (VM_PROT_READ|VM_PROT_WRITE) 766 && cur_pv.pv_cache == PTE_CACHE) { 767 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_va; 768 continue; 769 } 770 771 /* 772 * We couldn't so emit the current chunk and then 773 */ 774 #ifdef VERBOSE_INIT_ARM 775 printf("%s: mapping chunk VA %#lx..%#lx " 776 "(PA %#lx, prot %d, cache %d)\n", 777 __func__, 778 cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 779 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 780 #endif 781 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 782 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 783 784 /* 785 * set the current chunk to the hole and try again. 786 */ 787 cur_pv.pv_pa += cur_pv.pv_size; 788 cur_pv.pv_va += cur_pv.pv_size; 789 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_va; 790 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 791 cur_pv.pv_cache = PTE_CACHE; 792 continue; 793 } 794 } 795 796 /* 797 * The new pv didn't concatenate so emit the current one 798 * and use the new pv as the current pv. 799 */ 800 #ifdef VERBOSE_INIT_ARM 801 printf("%s: mapping chunk VA %#lx..%#lx " 802 "(PA %#lx, prot %d, cache %d)\n", 803 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 804 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 805 #endif 806 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 807 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 808 cur_pv = *pv; 809 pv = SLIST_NEXT(pv, pv_list); 810 } 811 812 /* 813 * If we are mapping all of memory, let's map the rest of memory. 814 */ 815 if (mapallmem_p && cur_pv.pv_pa + cur_pv.pv_size < bmi->bmi_end) { 816 if (cur_pv.pv_prot == (VM_PROT_READ | VM_PROT_WRITE) 817 && cur_pv.pv_cache == PTE_CACHE) { 818 cur_pv.pv_size = bmi->bmi_end - cur_pv.pv_pa; 819 } else { 820 KASSERTMSG(cur_pv.pv_va + cur_pv.pv_size <= kernel_vm_base, 821 "%#lx >= %#lx", cur_pv.pv_va + cur_pv.pv_size, 822 kernel_vm_base); 823 #ifdef VERBOSE_INIT_ARM 824 printf("%s: mapping chunk VA %#lx..%#lx " 825 "(PA %#lx, prot %d, cache %d)\n", 826 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 827 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 828 #endif 829 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 830 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 831 cur_pv.pv_pa += cur_pv.pv_size; 832 cur_pv.pv_va += cur_pv.pv_size; 833 cur_pv.pv_size = bmi->bmi_end - cur_pv.pv_pa; 834 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 835 cur_pv.pv_cache = PTE_CACHE; 836 } 837 } 838 839 // The amount we can direct is limited by the start of the 840 // virtual part of the kernel address space. Don't overrun 841 // into it. 842 if (mapallmem_p && cur_pv.pv_va + cur_pv.pv_size > kernel_vm_base) { 843 cur_pv.pv_size = kernel_vm_base - cur_pv.pv_va; 844 } 845 846 /* 847 * Now we map the final chunk. 848 */ 849 #ifdef VERBOSE_INIT_ARM 850 printf("%s: mapping last chunk VA %#lx..%#lx (PA %#lx, prot %d, cache %d)\n", 851 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 852 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 853 #endif 854 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 855 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 856 857 /* 858 * Now we map the stuff that isn't directly after the kernel 859 */ 860 if (map_vectors_p) { 861 /* Map the vector page. */ 862 pmap_map_entry(l1pt_va, systempage.pv_va, systempage.pv_pa, 863 VM_PROT_READ|VM_PROT_WRITE|VM_PROT_EXECUTE, PTE_CACHE); 864 } 865 866 /* Map the Mini-Data cache clean area. */ 867 #if ARM_MMU_XSCALE == 1 868 #if (ARM_NMMUS > 1) 869 if (xscale_use_minidata) 870 #endif 871 xscale_setup_minidata(l1pt_va, minidataclean.pv_va, 872 minidataclean.pv_pa); 873 #endif 874 875 /* 876 * Map integrated peripherals at same address in first level page 877 * table so that we can continue to use console. 878 */ 879 if (devmap) 880 pmap_devmap_bootstrap(l1pt_va, devmap); 881 882 #ifdef VERBOSE_INIT_ARM 883 /* Tell the user about where all the bits and pieces live. */ 884 printf("%22s Physical Virtual Num\n", " "); 885 printf("%22s Starting Ending Starting Ending Pages\n", " "); 886 887 static const char mem_fmt[] = 888 "%20s: 0x%08lx 0x%08lx 0x%08lx 0x%08lx %u\n"; 889 static const char mem_fmt_nov[] = 890 "%20s: 0x%08lx 0x%08lx %zu\n"; 891 892 printf(mem_fmt, "SDRAM", bmi->bmi_start, bmi->bmi_end - 1, 893 KERN_PHYSTOV(bmi, bmi->bmi_start), KERN_PHYSTOV(bmi, bmi->bmi_end - 1), 894 (int)physmem); 895 printf(mem_fmt, "text section", 896 text.pv_pa, text.pv_pa + text.pv_size - 1, 897 text.pv_va, text.pv_va + text.pv_size - 1, 898 (int)(text.pv_size / PAGE_SIZE)); 899 printf(mem_fmt, "data section", 900 KERN_VTOPHYS(bmi, __data_start), KERN_VTOPHYS(bmi, _edata), 901 (vaddr_t)__data_start, (vaddr_t)_edata, 902 (int)((round_page((vaddr_t)_edata) 903 - trunc_page((vaddr_t)__data_start)) / PAGE_SIZE)); 904 printf(mem_fmt, "bss section", 905 KERN_VTOPHYS(bmi, __bss_start), KERN_VTOPHYS(bmi, __bss_end__), 906 (vaddr_t)__bss_start, (vaddr_t)__bss_end__, 907 (int)((round_page((vaddr_t)__bss_end__) 908 - trunc_page((vaddr_t)__bss_start)) / PAGE_SIZE)); 909 printf(mem_fmt, "L1 page directory", 910 kernel_l1pt.pv_pa, kernel_l1pt.pv_pa + L1_TABLE_SIZE - 1, 911 kernel_l1pt.pv_va, kernel_l1pt.pv_va + L1_TABLE_SIZE - 1, 912 L1_TABLE_SIZE / PAGE_SIZE); 913 printf(mem_fmt, "ABT stack (CPU 0)", 914 abtstack.pv_pa, abtstack.pv_pa + (ABT_STACK_SIZE * PAGE_SIZE) - 1, 915 abtstack.pv_va, abtstack.pv_va + (ABT_STACK_SIZE * PAGE_SIZE) - 1, 916 ABT_STACK_SIZE); 917 printf(mem_fmt, "FIQ stack (CPU 0)", 918 fiqstack.pv_pa, fiqstack.pv_pa + (FIQ_STACK_SIZE * PAGE_SIZE) - 1, 919 fiqstack.pv_va, fiqstack.pv_va + (FIQ_STACK_SIZE * PAGE_SIZE) - 1, 920 FIQ_STACK_SIZE); 921 printf(mem_fmt, "IRQ stack (CPU 0)", 922 irqstack.pv_pa, irqstack.pv_pa + (IRQ_STACK_SIZE * PAGE_SIZE) - 1, 923 irqstack.pv_va, irqstack.pv_va + (IRQ_STACK_SIZE * PAGE_SIZE) - 1, 924 IRQ_STACK_SIZE); 925 printf(mem_fmt, "UND stack (CPU 0)", 926 undstack.pv_pa, undstack.pv_pa + (UND_STACK_SIZE * PAGE_SIZE) - 1, 927 undstack.pv_va, undstack.pv_va + (UND_STACK_SIZE * PAGE_SIZE) - 1, 928 UND_STACK_SIZE); 929 printf(mem_fmt, "IDLE stack (CPU 0)", 930 idlestack.pv_pa, idlestack.pv_pa + (UPAGES * PAGE_SIZE) - 1, 931 idlestack.pv_va, idlestack.pv_va + (UPAGES * PAGE_SIZE) - 1, 932 UPAGES); 933 printf(mem_fmt, "SVC stack", 934 kernelstack.pv_pa, kernelstack.pv_pa + (UPAGES * PAGE_SIZE) - 1, 935 kernelstack.pv_va, kernelstack.pv_va + (UPAGES * PAGE_SIZE) - 1, 936 UPAGES); 937 printf(mem_fmt, "Message Buffer", 938 msgbuf.pv_pa, msgbuf.pv_pa + (msgbuf_pgs * PAGE_SIZE) - 1, 939 msgbuf.pv_va, msgbuf.pv_va + (msgbuf_pgs * PAGE_SIZE) - 1, 940 (int)msgbuf_pgs); 941 if (map_vectors_p) { 942 printf(mem_fmt, "Exception Vectors", 943 systempage.pv_pa, systempage.pv_pa + PAGE_SIZE - 1, 944 systempage.pv_va, systempage.pv_va + PAGE_SIZE - 1, 945 1); 946 } 947 for (size_t i = 0; i < bmi->bmi_nfreeblocks; i++) { 948 pv = &bmi->bmi_freeblocks[i]; 949 950 printf(mem_fmt_nov, "Free Memory", 951 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 952 pv->pv_size / PAGE_SIZE); 953 } 954 #endif 955 /* 956 * Now we have the real page tables in place so we can switch to them. 957 * Once this is done we will be running with the REAL kernel page 958 * tables. 959 */ 960 961 #if defined(VERBOSE_INIT_ARM) 962 printf("TTBR0=%#x", armreg_ttbr_read()); 963 #ifdef _ARM_ARCH_6 964 printf(" TTBR1=%#x TTBCR=%#x CONTEXTIDR=%#x", 965 armreg_ttbr1_read(), armreg_ttbcr_read(), 966 armreg_contextidr_read()); 967 #endif 968 printf("\n"); 969 #endif 970 971 /* Switch tables */ 972 #ifdef VERBOSE_INIT_ARM 973 printf("switching to new L1 page table @%#lx...", l1pt_pa); 974 #endif 975 976 #ifdef ARM_MMU_EXTENDED 977 cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) 978 | (DOMAIN_CLIENT << (PMAP_DOMAIN_USER*2))); 979 #else 980 cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) | DOMAIN_CLIENT); 981 #endif 982 cpu_idcache_wbinv_all(); 983 #ifdef VERBOSE_INIT_ARM 984 printf(" ttb"); 985 #endif 986 #ifdef ARM_MMU_EXTENDED 987 /* 988 * TTBCR should have been initialized by the MD start code. 989 */ 990 KASSERT((armreg_contextidr_read() & 0xff) == 0); 991 KASSERT(armreg_ttbcr_read() == __SHIFTIN(1, TTBCR_S_N)); 992 /* 993 * Disable lookups via TTBR0 until there is an activated pmap. 994 */ 995 armreg_ttbcr_write(armreg_ttbcr_read() | TTBCR_S_PD0); 996 cpu_setttb(l1pt_pa, KERNEL_PID); 997 arm_isb(); 998 #else 999 cpu_setttb(l1pt_pa, true); 1000 cpu_domains(DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)); 1001 #endif 1002 cpu_tlb_flushID(); 1003 1004 #ifdef VERBOSE_INIT_ARM 1005 #ifdef ARM_MMU_EXTENDED 1006 printf(" (TTBCR=%#x TTBR0=%#x TTBR1=%#x)", 1007 armreg_ttbcr_read(), armreg_ttbr_read(), armreg_ttbr1_read()); 1008 #else 1009 printf(" (TTBR0=%#x)", armreg_ttbr_read()); 1010 #endif 1011 #endif 1012 1013 #ifdef MULTIPROCESSOR 1014 /* 1015 * Kick the secondaries to load the TTB. After which they'll go 1016 * back to sleep to wait for the final kick so they will hatch. 1017 */ 1018 #ifdef VERBOSE_INIT_ARM 1019 printf(" hatchlings"); 1020 #endif 1021 cpu_boot_secondary_processors(); 1022 #endif 1023 1024 #ifdef VERBOSE_INIT_ARM 1025 printf(" OK\n"); 1026 #endif 1027 } 1028