1 /* $NetBSD: aarch64_machdep.c,v 1.70 2023/07/16 21:36:40 riastradh Exp $ */
2
3 /*-
4 * Copyright (c) 2014 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Matt Thomas of 3am Software Foundry.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #include <sys/cdefs.h>
33 __KERNEL_RCSID(1, "$NetBSD: aarch64_machdep.c,v 1.70 2023/07/16 21:36:40 riastradh Exp $");
34
35 #include "opt_arm_debug.h"
36 #include "opt_cpuoptions.h"
37 #include "opt_ddb.h"
38 #include "opt_fdt.h"
39 #include "opt_kernhist.h"
40 #include "opt_modular.h"
41
42 #include <sys/param.h>
43 #include <sys/types.h>
44 #include <sys/asan.h>
45 #include <sys/boot_flag.h>
46 #include <sys/bus.h>
47 #include <sys/core.h>
48 #include <sys/conf.h>
49 #include <sys/kauth.h>
50 #include <sys/kcore.h>
51 #include <sys/module.h>
52 #include <sys/msgbuf.h>
53 #include <sys/reboot.h>
54 #include <sys/sysctl.h>
55 #include <sys/xcall.h>
56
57 #include <dev/mm.h>
58
59 #include <uvm/uvm.h>
60
61 #include <machine/bootconfig.h>
62
63 #include <arm/cpufunc.h>
64
65 #include <aarch64/armreg.h>
66 #ifdef DDB
67 #include <aarch64/db_machdep.h>
68 #endif
69 #include <aarch64/frame.h>
70 #include <aarch64/machdep.h>
71 #include <aarch64/pmap.h>
72 #include <aarch64/pte.h>
73 #include <aarch64/vmparam.h>
74 #include <aarch64/kcore.h>
75
76 #include <arm/fdt/arm_fdtvar.h>
77 #include <dev/fdt/fdtvar.h>
78 #include <dev/fdt/fdt_memory.h>
79
80 #ifdef VERBOSE_INIT_ARM
81 #define VPRINTF(...) printf(__VA_ARGS__)
82 #else
83 #define VPRINTF(...) __nothing
84 #endif
85
86 char cpu_model[32];
87 char machine[] = MACHINE;
88 char machine_arch[] = MACHINE_ARCH;
89
90 const pcu_ops_t * const pcu_ops_md_defs[PCU_UNIT_COUNT] = {
91 [PCU_FPU] = &pcu_fpu_ops
92 };
93
94 struct vm_map *phys_map;
95
96 #ifdef MODULAR
97 vaddr_t module_start, module_end;
98 static struct vm_map module_map_store;
99 #endif
100
101 #ifdef KASAN
102 vaddr_t kasan_kernelstart;
103 vaddr_t kasan_kernelsize;
104 #endif
105
106 /* XXX */
107 vaddr_t physical_start;
108 vaddr_t physical_end;
109 /* filled in before cleaning bss. keep in .data */
110 u_long kern_vtopdiff __attribute__((__section__(".data")));
111
112 /* extra physical memory allocated from round_page(_end[]) */
113 long kernend_extra;
114
115 /* dump configuration */
116 int cpu_dump(void);
117 int cpu_dumpsize(void);
118 u_long cpu_dump_mempagecnt(void);
119
120 uint32_t dumpmag = 0x8fca0101; /* magic number for savecore */
121 int dumpsize = 0; /* also for savecore */
122 long dumplo = 0;
123
124 int aarch64_bti_enabled __read_mostly;
125
126 static void
bti_init(void)127 bti_init(void)
128 {
129 #ifdef ARMV85_BTI
130 extern uint64_t pmap_attr_gp;
131 uint64_t reg;
132
133 reg = reg_id_aa64pfr1_el1_read();
134
135 if (reg >= ID_AA64PFR1_EL1_BT_SUPPORTED) {
136 pmap_attr_gp = LX_BLKPAG_GP;
137 aarch64_bti_enabled = 1;
138 }
139 #endif
140 }
141
142 void
cpu_kernel_vm_init(uint64_t memory_start __unused,uint64_t memory_size __unused)143 cpu_kernel_vm_init(uint64_t memory_start __unused, uint64_t memory_size __unused)
144 {
145 extern char __kernel_text[];
146 extern char _end[];
147 extern char __data_start[];
148 extern char __rodata_start[];
149 u_int blk;
150
151 bti_init();
152
153 vaddr_t kernstart = trunc_page((vaddr_t)__kernel_text);
154 vaddr_t kernend = round_page((vaddr_t)_end);
155 paddr_t kernstart_phys = KERN_VTOPHYS(kernstart);
156 paddr_t kernend_phys = KERN_VTOPHYS(kernend);
157 vaddr_t data_start = (vaddr_t)__data_start;
158 vaddr_t rodata_start = (vaddr_t)__rodata_start;
159
160 /* add direct mappings of whole memory */
161 const pt_entry_t dmattr =
162 LX_BLKPAG_ATTR_NORMAL_WB |
163 LX_BLKPAG_AP_RW |
164 LX_BLKPAG_PXN |
165 LX_BLKPAG_UXN;
166 for (blk = 0; blk < bootconfig.dramblocks; blk++) {
167 uint64_t start, end;
168
169 start = trunc_page(bootconfig.dram[blk].address);
170 end = round_page(bootconfig.dram[blk].address +
171 (uint64_t)bootconfig.dram[blk].pages * PAGE_SIZE);
172
173 pmapboot_enter_range(AARCH64_PA_TO_KVA(start), start,
174 end - start, dmattr, printf);
175 }
176
177 /* Disable translation table walks using TTBR0 */
178 uint64_t tcr = reg_tcr_el1_read();
179 reg_tcr_el1_write(tcr | TCR_EPD0);
180 isb();
181
182 aarch64_tlbi_all();
183
184 /*
185 * at this point, whole kernel image is mapped as "rwx".
186 * permission should be changed to:
187 *
188 * text rwx => r-x
189 * rodata rwx => r--
190 * data rwx => rw- (.bss included)
191 *
192 * kernel image has mapped by L2 block. (2Mbyte)
193 */
194 pmapboot_protect(L2_TRUNC_BLOCK(kernstart),
195 L2_TRUNC_BLOCK(data_start), VM_PROT_WRITE);
196 pmapboot_protect(L2_ROUND_BLOCK(rodata_start),
197 L2_ROUND_BLOCK(kernend), VM_PROT_EXECUTE);
198
199 aarch64_tlbi_all();
200
201 VPRINTF("%s: kernel phys start %lx end %lx+%lx\n", __func__,
202 kernstart_phys, kernend_phys, kernend_extra);
203 fdt_memory_remove_range(kernstart_phys,
204 kernend_phys - kernstart_phys + kernend_extra);
205
206 #ifdef KASAN
207 kasan_kernelstart = kernstart;
208 kasan_kernelsize = L2_ROUND_BLOCK(kernend) - kernstart;
209 #endif
210 }
211
212
213
214 /*
215 * Upper region: 0xffff_ffff_ffff_ffff Top of virtual memory
216 *
217 * 0xffff_ffff_ffe0_0000 End of KVA
218 * = VM_MAX_KERNEL_ADDRESS
219 *
220 * 0xffff_c000_4000_0000 Start of KVA
221 *
222 * 0xffff_c000_0???_???? End of kernel
223 * = _end[]
224 * 0xffff_c000_00??_???? Start of kernel
225 * = __kernel_text[]
226 *
227 * 0xffff_c000_0000_0000 Kernel base address
228 * = VM_MIN_KERNEL_ADDRESS
229 *
230 * 0xffff_bfff_ffff_ffff End of direct mapped
231 * 0xffff_0000_0000_0000 Start of direct mapped
232 * = AARCH64_DIRECTMAP_START
233 *
234 * Hole: 0xfffe_ffff_ffff_ffff
235 * 0x0001_0000_0000_0000
236 *
237 * Lower region: 0x0000_ffff_ffff_ffff End of user address space
238 * = VM_MAXUSER_ADDRESS
239 *
240 * 0x0000_0000_0000_0000 Start of user address space
241 */
242 vaddr_t
initarm_common(vaddr_t kvm_base,vsize_t kvm_size,const struct boot_physmem * bp,size_t nbp)243 initarm_common(vaddr_t kvm_base, vsize_t kvm_size,
244 const struct boot_physmem *bp, size_t nbp)
245 {
246 extern char __kernel_text[];
247 extern char _end[];
248 extern char lwp0uspace[];
249
250 struct pcb *pcb;
251 struct trapframe *tf;
252 psize_t memsize_total;
253 vaddr_t kernstart, kernend;
254 vaddr_t kernstart_l2 __unused, kernend_l2; /* L2 table 2MB aligned */
255 vaddr_t kernelvmstart;
256 size_t i;
257
258 cputype = cpu_idnum(); /* for compatible arm */
259
260 kernstart = trunc_page((vaddr_t)__kernel_text);
261 kernend = round_page((vaddr_t)_end);
262
263 kernstart_l2 = L2_TRUNC_BLOCK(kernstart);
264 kernend_l2 = L2_ROUND_BLOCK(kernend);
265
266 kernelvmstart = kernend_l2;
267
268 #ifdef MODULAR
269 /*
270 * The aarch64 compilers (gcc & llvm) use R_AARCH_CALL26/R_AARCH_JUMP26
271 * for function calls (bl)/jumps(b). At this time, neither compiler
272 * supports -mlong-calls therefore the kernel modules should be loaded
273 * within the maximum range of +/-128MB from kernel text.
274 */
275 #define MODULE_RESERVED_MAX (1024 * 1024 * 128)
276 #define MODULE_RESERVED_SIZE (1024 * 1024 * 32) /* good enough? */
277 module_start = kernelvmstart;
278 module_end = kernend_l2 + MODULE_RESERVED_SIZE;
279 if (module_end >= kernstart_l2 + MODULE_RESERVED_MAX)
280 module_end = kernstart_l2 + MODULE_RESERVED_MAX;
281 KASSERT(module_end > kernend_l2);
282 kernelvmstart = module_end;
283 #endif /* MODULAR */
284
285 KASSERT(kernelvmstart < VM_KERNEL_VM_BASE);
286
287 kernelvmstart = VM_KERNEL_VM_BASE;
288
289 paddr_t kernstart_phys __unused = KERN_VTOPHYS(kernstart);
290 paddr_t kernend_phys __unused = KERN_VTOPHYS(kernend);
291
292 physical_start = bootconfig.dram[0].address;
293 physical_end = bootconfig.dram[bootconfig.dramblocks - 1].address +
294 ptoa(bootconfig.dram[bootconfig.dramblocks - 1].pages);
295
296 /*
297 * msgbuf is allocated from the bottom of any one of memory blocks
298 * to avoid corruption due to bootloader or changing kernel layout.
299 */
300 paddr_t msgbufaddr = 0;
301 for (i = 0; i < bootconfig.dramblocks; i++) {
302 /* this block has enough space for msgbuf? */
303 if (bootconfig.dram[i].pages < atop(round_page(MSGBUFSIZE)))
304 continue;
305
306 /* allocate msgbuf from the bottom of this block */
307 bootconfig.dram[i].pages -= atop(round_page(MSGBUFSIZE));
308 msgbufaddr = bootconfig.dram[i].address +
309 ptoa(bootconfig.dram[i].pages);
310 break;
311 }
312 KASSERT(msgbufaddr != 0); /* no space for msgbuf */
313 initmsgbuf((void *)AARCH64_PA_TO_KVA(msgbufaddr), MSGBUFSIZE);
314
315 VPRINTF(
316 "------------------------------------------\n"
317 "kern_vtopdiff = 0x%016lx\n"
318 "physical_start = 0x%016lx\n"
319 "kernel_start_phys = 0x%016lx\n"
320 "kernel_end_phys = 0x%016lx\n"
321 "pagetables_start_phys = 0x%016lx\n"
322 "pagetables_end_phys = 0x%016lx\n"
323 "msgbuf = 0x%016lx\n"
324 "physical_end = 0x%016lx\n"
325 "VM_MIN_KERNEL_ADDRESS = 0x%016lx\n"
326 "kernel_start_l2 = 0x%016lx\n"
327 "kernel_start = 0x%016lx\n"
328 "kernel_end = 0x%016lx\n"
329 "(extra) = 0x%016lx\n"
330 "kernel_end_l2 = 0x%016lx\n"
331 #ifdef MODULAR
332 "module_start = 0x%016lx\n"
333 "module_end = 0x%016lx\n"
334 #endif
335 "(kernel va area) = 0x%016lx\n"
336 "(devmap va area) = 0x%016lx\n"
337 "VM_MAX_KERNEL_ADDRESS = 0x%016lx\n"
338 "------------------------------------------\n",
339 kern_vtopdiff,
340 physical_start,
341 kernstart_phys,
342 kernend_phys,
343 round_page(kernend_phys),
344 round_page(kernend_phys) + kernend_extra,
345 msgbufaddr,
346 physical_end,
347 VM_MIN_KERNEL_ADDRESS,
348 kernstart_l2,
349 kernstart,
350 kernend,
351 kernend_extra,
352 kernend_l2,
353 #ifdef MODULAR
354 module_start,
355 module_end,
356 #endif
357 VM_KERNEL_VM_BASE,
358 VM_KERNEL_IO_BASE,
359 VM_MAX_KERNEL_ADDRESS);
360
361 #ifdef DDB
362 db_machdep_cpu_init();
363 #endif
364
365 uvm_md_init();
366
367 /* register free physical memory blocks */
368 memsize_total = 0;
369
370 KASSERT(bp != NULL || nbp == 0);
371 KASSERT(bp == NULL || nbp != 0);
372
373 KDASSERT(bootconfig.dramblocks <= DRAM_BLOCKS);
374 for (i = 0; i < bootconfig.dramblocks; i++) {
375 paddr_t start, end;
376
377 /* empty is end */
378 if (bootconfig.dram[i].address == 0 &&
379 bootconfig.dram[i].pages == 0)
380 break;
381
382 start = atop(bootconfig.dram[i].address);
383 end = start + bootconfig.dram[i].pages;
384
385 int vm_freelist = VM_FREELIST_DEFAULT;
386
387 VPRINTF("block %2zu start %08lx end %08lx\n", i, ptoa(start),
388 ptoa(end));
389
390 /*
391 * This assumes the bp list is sorted in ascending
392 * order.
393 */
394 paddr_t segend = end;
395 for (size_t j = 0; j < nbp && start < end; j++) {
396 paddr_t bp_start = bp[j].bp_start;
397 paddr_t bp_end = bp_start + bp[j].bp_pages;
398
399 VPRINTF(" bp %2zu start %08lx end %08lx\n",
400 j, ptoa(bp_start), ptoa(bp_end));
401
402 KASSERT(bp_start < bp_end);
403 if (start >= bp_end || segend < bp_start)
404 continue;
405
406 if (start < bp_start)
407 start = bp_start;
408
409 if (start < bp_end) {
410 if (segend > bp_end) {
411 segend = bp_end;
412 }
413 vm_freelist = bp[j].bp_freelist;
414
415 VPRINTF(" start %08lx end %08lx"
416 "... loading in freelist %d\n", ptoa(start),
417 ptoa(segend), vm_freelist);
418
419 uvm_page_physload(start, segend, start, segend,
420 vm_freelist);
421
422 memsize_total += ptoa(segend - start);
423 start = segend;
424 segend = end;
425 }
426 }
427 }
428 physmem = atop(memsize_total);
429
430 /*
431 * kernel image is mapped on L2 table (2MB*n) by locore.S
432 * virtual space start from 2MB aligned kernend
433 */
434 pmap_bootstrap(kernelvmstart, VM_MAX_KERNEL_ADDRESS);
435
436 kasan_init();
437
438 /*
439 * setup lwp0
440 */
441 uvm_lwp_setuarea(&lwp0, (vaddr_t)lwp0uspace);
442 memset(&lwp0.l_md, 0, sizeof(lwp0.l_md));
443 pcb = lwp_getpcb(&lwp0);
444 memset(pcb, 0, sizeof(struct pcb));
445
446 tf = (struct trapframe *)(lwp0uspace + USPACE) - 1;
447 memset(tf, 0, sizeof(struct trapframe));
448 tf->tf_spsr = SPSR_M_EL0T;
449 lwp0.l_md.md_utf = pcb->pcb_tf = tf;
450
451 return (vaddr_t)tf;
452 }
453
454 /*
455 * machine dependent system variables.
456 */
457 static void
set_user_tagged_address(void * arg1,void * arg2)458 set_user_tagged_address(void *arg1, void *arg2)
459 {
460 uint64_t enable = PTRTOUINT64(arg1);
461 uint64_t tcr = reg_tcr_el1_read();
462
463 if (enable)
464 tcr |= TCR_TBI0;
465 else
466 tcr &= ~TCR_TBI0;
467 reg_tcr_el1_write(tcr);
468 }
469
470 static int
sysctl_machdep_tagged_address(SYSCTLFN_ARGS)471 sysctl_machdep_tagged_address(SYSCTLFN_ARGS)
472 {
473 struct sysctlnode node;
474 int error, cur, val;
475 uint64_t tcr;
476
477 tcr = reg_tcr_el1_read();
478 cur = val = (tcr & TCR_TBI0) ? 1 : 0;
479
480 node = *rnode;
481 node.sysctl_data = &val;
482 error = sysctl_lookup(SYSCTLFN_CALL(&node));
483 if (error || newp == NULL)
484 return error;
485 if (val < 0 || val > 1)
486 return EINVAL;
487
488 if (cur != val) {
489 uint64_t where = xc_broadcast(0, set_user_tagged_address,
490 UINT64TOPTR(val), NULL);
491 xc_wait(where);
492 }
493
494 return 0;
495 }
496
497 SYSCTL_SETUP(sysctl_machdep_setup, "sysctl machdep subtree setup")
498 {
499 sysctl_createv(clog, 0, NULL, NULL,
500 CTLFLAG_PERMANENT,
501 CTLTYPE_NODE, "machdep", NULL,
502 NULL, 0, NULL, 0,
503 CTL_MACHDEP, CTL_EOL);
504
505 sysctl_createv(clog, 0, NULL, NULL,
506 CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
507 CTLTYPE_INT, "tagged_address",
508 SYSCTL_DESCR("top byte ignored in the address calculation"),
509 sysctl_machdep_tagged_address, 0, NULL, 0,
510 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
511
512 sysctl_createv(clog, 0, NULL, NULL,
513 CTLFLAG_PERMANENT,
514 CTLTYPE_INT, "pan",
515 SYSCTL_DESCR("Whether Privileged Access Never is enabled"),
516 NULL, 0,
517 &aarch64_pan_enabled, 0,
518 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
519
520 sysctl_createv(clog, 0, NULL, NULL,
521 CTLFLAG_PERMANENT,
522 CTLTYPE_INT, "pac",
523 SYSCTL_DESCR("Whether Pointer Authentication is enabled"),
524 NULL, 0,
525 &aarch64_pac_enabled, 0,
526 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
527
528 sysctl_createv(clog, 0, NULL, NULL,
529 CTLFLAG_PERMANENT,
530 CTLTYPE_INT, "bti",
531 SYSCTL_DESCR("Whether Branch Target Identification is enabled"),
532 NULL, 0,
533 &aarch64_bti_enabled, 0,
534 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
535
536 sysctl_createv(clog, 0, NULL, NULL,
537 CTLFLAG_PERMANENT,
538 CTLTYPE_INT, "hafdbs",
539 SYSCTL_DESCR("Whether Hardware updates to Access flag and Dirty state is enabled"),
540 NULL, 0,
541 &aarch64_hafdbs_enabled, 0,
542 CTL_MACHDEP, CTL_CREATE, CTL_EOL);
543 }
544
545 void
parse_mi_bootargs(char * args)546 parse_mi_bootargs(char *args)
547 {
548 const char *p = args;
549
550 while (*p != '\0') {
551 while (isspace(*p))
552 p++;
553
554 /* parse single dash (`-') options */
555 if (*p == '-') {
556 p++;
557 while (!isspace(*p) && *p != '\0') {
558 BOOT_FLAG(*p, boothowto);
559 p++;
560 }
561 continue;
562 }
563
564 /* skip normal argument */
565 while (!isspace(*p) && *p != '\0')
566 p++;
567 }
568 }
569
570 void
machdep_init(void)571 machdep_init(void)
572 {
573 /* clear cpu reset hook for early boot */
574 cpu_reset_address0 = NULL;
575
576 configure_cpu_traps();
577 }
578
579 #ifdef MODULAR
580 /* Push any modules loaded by the boot loader */
581 void
module_init_md(void)582 module_init_md(void)
583 {
584 #ifdef FDT
585 arm_fdt_module_init();
586 #endif
587 }
588 #endif /* MODULAR */
589
590 static bool
in_dram_p(paddr_t pa,psize_t size)591 in_dram_p(paddr_t pa, psize_t size)
592 {
593 int i;
594
595 for (i = 0; i < bootconfig.dramblocks; i++) {
596 paddr_t s, e;
597 s = bootconfig.dram[i].address;
598 e = bootconfig.dram[i].address + ptoa(bootconfig.dram[i].pages);
599 if ((s <= pa) && ((pa + size) <= e))
600 return true;
601 }
602 return false;
603 }
604
605 bool
mm_md_direct_mapped_phys(paddr_t pa,vaddr_t * vap)606 mm_md_direct_mapped_phys(paddr_t pa, vaddr_t *vap)
607 {
608 if (in_dram_p(pa, 0)) {
609 *vap = AARCH64_PA_TO_KVA(pa);
610 return true;
611 }
612 return false;
613 }
614
615 int
mm_md_physacc(paddr_t pa,vm_prot_t prot)616 mm_md_physacc(paddr_t pa, vm_prot_t prot)
617 {
618 if (in_dram_p(pa, 0))
619 return 0;
620
621 if (pa >= AARCH64_MAX_PA)
622 return EFAULT;
623
624 return kauth_authorize_machdep(kauth_cred_get(),
625 KAUTH_MACHDEP_UNMANAGEDMEM, NULL, NULL, NULL, NULL);
626 }
627
628 #ifdef __HAVE_MM_MD_KERNACC
629 int
mm_md_kernacc(void * ptr,vm_prot_t prot,bool * handled)630 mm_md_kernacc(void *ptr, vm_prot_t prot, bool *handled)
631 {
632 extern char __kernel_text[];
633 extern char _end[];
634 extern char __data_start[];
635 extern char __rodata_start[];
636
637 vaddr_t kernstart = trunc_page((vaddr_t)__kernel_text);
638 vaddr_t kernend = round_page((vaddr_t)_end);
639 paddr_t kernstart_phys = KERN_VTOPHYS(kernstart);
640 vaddr_t data_start = (vaddr_t)__data_start;
641 vaddr_t rodata_start = (vaddr_t)__rodata_start;
642 vsize_t rosize = kernend - rodata_start;
643
644 const vaddr_t v = (vaddr_t)ptr;
645
646 #define IN_RANGE(addr,sta,end) (((sta) <= (addr)) && ((addr) < (end)))
647
648 *handled = false;
649 if (IN_RANGE(v, kernstart, kernend)) {
650 *handled = true;
651 if ((v < data_start) && (prot & VM_PROT_WRITE))
652 return EFAULT;
653 } else if (IN_RANGE(v, AARCH64_DIRECTMAP_START, AARCH64_DIRECTMAP_END)) {
654 /*
655 * if defined PMAP_MAP_POOLPAGE, direct mapped address
656 * will be appeared as kvm(3) address.
657 */
658 paddr_t pa = AARCH64_KVA_TO_PA(v);
659 if (in_dram_p(pa, 0)) {
660 *handled = true;
661 if (IN_RANGE(pa, kernstart_phys,
662 kernstart_phys + rosize) &&
663 (prot & VM_PROT_WRITE))
664 return EFAULT;
665 }
666 }
667 return 0;
668 }
669 #endif
670
671 void
cpu_startup(void)672 cpu_startup(void)
673 {
674 vaddr_t maxaddr, minaddr;
675
676 consinit();
677
678 #ifdef FDT
679 const struct fdt_platform * const plat = fdt_platform_find();
680 if (plat->fp_startup != NULL)
681 plat->fp_startup();
682 #endif
683
684 /*
685 * Allocate a submap for physio.
686 */
687 minaddr = 0;
688 phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr,
689 VM_PHYS_SIZE, 0, FALSE, NULL);
690
691 #ifdef MODULAR
692 uvm_map_setup(&module_map_store, module_start, module_end, 0);
693 module_map_store.pmap = pmap_kernel();
694 module_map = &module_map_store;
695 #endif
696
697 /* Hello! */
698 banner();
699
700 cpu_startup_hook();
701 }
702
__weak_alias(cpu_startup_hook,cpu_startup_default)703 __weak_alias(cpu_startup_hook,cpu_startup_default)
704 void
705 cpu_startup_default(void)
706 {
707 }
708
709 /*
710 * cpu_dump: dump the machine-dependent kernel core dump headers.
711 */
712 int
cpu_dump(void)713 cpu_dump(void)
714 {
715 int (*dump)(dev_t, daddr_t, void *, size_t);
716 char bf[dbtob(1)];
717 kcore_seg_t *segp;
718 cpu_kcore_hdr_t *cpuhdrp;
719 phys_ram_seg_t *memsegp;
720 const struct bdevsw *bdev;
721 int i;
722
723 bdev = bdevsw_lookup(dumpdev);
724 if (bdev == NULL)
725 return (ENXIO);
726 dump = bdev->d_dump;
727
728 memset(bf, 0, sizeof bf);
729 segp = (kcore_seg_t *)bf;
730 cpuhdrp = (cpu_kcore_hdr_t *)&bf[ALIGN(sizeof(*segp))];
731 memsegp = &cpuhdrp->kh_ramsegs[0];
732
733 /*
734 * Generate a segment header.
735 */
736 CORE_SETMAGIC(*segp, KCORE_MAGIC, MID_MACHINE, CORE_CPU);
737 segp->c_size = dbtob(1) - ALIGN(sizeof(*segp));
738
739 /*
740 * Add the machine-dependent header info.
741 */
742 cpuhdrp->kh_tcr1 = reg_tcr_el1_read();
743 cpuhdrp->kh_ttbr1 = reg_ttbr1_el1_read();
744 cpuhdrp->kh_nramsegs = bootconfig.dramblocks;
745
746 /*
747 * Fill in the memory segment descriptors.
748 */
749 for (i = 0; i < bootconfig.dramblocks; i++) {
750 memsegp[i].start = bootconfig.dram[i].address;
751 memsegp[i].size = ptoa(bootconfig.dram[i].pages);
752 }
753
754 return (dump(dumpdev, dumplo, bf, dbtob(1)));
755 }
756
757 void
dumpsys(void)758 dumpsys(void)
759 {
760 const struct bdevsw *bdev;
761 daddr_t blkno;
762 int psize;
763 int error;
764 paddr_t addr = 0, end;
765 int block;
766 psize_t len;
767 vaddr_t dumpspace;
768
769 /* flush everything out of caches */
770 cpu_dcache_wbinv_all();
771
772 if (dumpdev == NODEV)
773 return;
774 if (dumpsize == 0) {
775 cpu_dumpconf();
776 }
777 if (dumplo <= 0 || dumpsize == 0) {
778 printf("\ndump to dev %u,%u not possible\n",
779 major(dumpdev), minor(dumpdev));
780 delay(5000000);
781 return;
782 }
783 printf("\ndumping to dev %u,%u offset %ld\n",
784 major(dumpdev), minor(dumpdev), dumplo);
785
786
787 bdev = bdevsw_lookup(dumpdev);
788 if (bdev == NULL || bdev->d_psize == NULL)
789 return;
790 psize = bdev_size(dumpdev);
791 printf("dump ");
792 if (psize == -1) {
793 printf("area unavailable\n");
794 return;
795 }
796
797 if ((error = cpu_dump()) != 0)
798 goto err;
799
800 blkno = dumplo + cpu_dumpsize();
801 error = 0;
802 len = dumpsize;
803
804 for (block = 0; block < bootconfig.dramblocks && error == 0; ++block) {
805 addr = bootconfig.dram[block].address;
806 end = bootconfig.dram[block].address +
807 ptoa(bootconfig.dram[block].pages);
808 for (; addr < end; addr += PAGE_SIZE) {
809 if (((len * PAGE_SIZE) % (1024*1024)) == 0)
810 printf("%lu ", (len * PAGE_SIZE) / (1024 * 1024));
811
812 if (!mm_md_direct_mapped_phys(addr, &dumpspace)) {
813 error = ENOMEM;
814 goto err;
815 }
816 error = (*bdev->d_dump)(dumpdev,
817 blkno, (void *) dumpspace, PAGE_SIZE);
818
819 if (error)
820 goto err;
821 blkno += btodb(PAGE_SIZE);
822 len--;
823 }
824 }
825 err:
826 switch (error) {
827 case ENXIO:
828 printf("device bad\n");
829 break;
830
831 case EFAULT:
832 printf("device not ready\n");
833 break;
834
835 case EINVAL:
836 printf("area improper\n");
837 break;
838
839 case EIO:
840 printf("i/o error\n");
841 break;
842
843 case EINTR:
844 printf("aborted from console\n");
845 break;
846
847 case ENOMEM:
848 printf("no direct map for %lx\n", addr);
849 break;
850
851 case 0:
852 printf("succeeded\n");
853 break;
854
855 default:
856 printf("error %d\n", error);
857 break;
858 }
859 printf("\n\n");
860 delay(5000000);
861 }
862
863 /*
864 * cpu_dumpsize: calculate size of machine-dependent kernel core dump headers.
865 */
866 int
cpu_dumpsize(void)867 cpu_dumpsize(void)
868 {
869 int size;
870
871 size = ALIGN(sizeof(kcore_seg_t)) + ALIGN(sizeof(cpu_kcore_hdr_t)) +
872 ALIGN(bootconfig.dramblocks * sizeof(phys_ram_seg_t));
873 if (roundup(size, dbtob(1)) != dbtob(1))
874 return -1;
875
876 return (1);
877 }
878
879 /*
880 * cpu_dump_mempagecnt: calculate the size of RAM (in pages) to be dumped.
881 */
882 u_long
cpu_dump_mempagecnt(void)883 cpu_dump_mempagecnt(void)
884 {
885 u_long i, n;
886
887 n = 0;
888 for (i = 0; i < bootconfig.dramblocks; i++) {
889 n += bootconfig.dram[i].pages;
890 }
891
892 return (n);
893 }
894
895 /*
896 * This is called by main to set dumplo and dumpsize.
897 * Dumps always skip the first PAGE_SIZE of disk space
898 * in case there might be a disk label stored there.
899 * If there is extra space, put dump at the end to
900 * reduce the chance that swapping trashes it.
901 */
902
903 void
cpu_dumpconf(void)904 cpu_dumpconf(void)
905 {
906 u_long nblks, dumpblks; /* size of dump area */
907
908 if (dumpdev == NODEV)
909 return;
910 nblks = bdev_size(dumpdev);
911 if (nblks <= ctod(1))
912 return;
913
914 dumpblks = cpu_dumpsize();
915 if (dumpblks < 0)
916 goto bad;
917 dumpblks += ctod(cpu_dump_mempagecnt());
918
919 /* If dump won't fit (incl. room for possible label), punt. */
920 if (dumpblks > (nblks - ctod(1)))
921 goto bad;
922
923 /* Put dump at end of partition */
924 dumplo = nblks - dumpblks;
925
926 /* dumpsize is in page units, and doesn't include headers. */
927 dumpsize = cpu_dump_mempagecnt();
928 return;
929
930 bad:
931 dumpsize = 0;
932 }
933