1 /* $NetBSD: fault.c,v 1.119 2024/02/02 22:00:32 andvar Exp $ */
2
3 /*
4 * Copyright 2003 Wasabi Systems, Inc.
5 * All rights reserved.
6 *
7 * Written by Steve C. Woodford for Wasabi Systems, Inc.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 * must display the following acknowledgement:
19 * This product includes software developed for the NetBSD Project by
20 * Wasabi Systems, Inc.
21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
22 * or promote products derived from this software without specific prior
23 * written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
36 */
37 /*
38 * Copyright (c) 1994-1997 Mark Brinicombe.
39 * Copyright (c) 1994 Brini.
40 * All rights reserved.
41 *
42 * This code is derived from software written for Brini by Mark Brinicombe
43 *
44 * Redistribution and use in source and binary forms, with or without
45 * modification, are permitted provided that the following conditions
46 * are met:
47 * 1. Redistributions of source code must retain the above copyright
48 * notice, this list of conditions and the following disclaimer.
49 * 2. Redistributions in binary form must reproduce the above copyright
50 * notice, this list of conditions and the following disclaimer in the
51 * documentation and/or other materials provided with the distribution.
52 * 3. All advertising materials mentioning features or use of this software
53 * must display the following acknowledgement:
54 * This product includes software developed by Brini.
55 * 4. The name of the company nor the name of the author may be used to
56 * endorse or promote products derived from this software without specific
57 * prior written permission.
58 *
59 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED
60 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
61 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
62 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
63 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
64 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
65 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
66 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
67 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
68 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
69 * SUCH DAMAGE.
70 *
71 * RiscBSD kernel project
72 *
73 * fault.c
74 *
75 * Fault handlers
76 *
77 * Created : 28/11/94
78 */
79
80 #include "opt_ddb.h"
81 #include "opt_kgdb.h"
82 #include "opt_multiprocessor.h"
83
84 #include <sys/types.h>
85 __KERNEL_RCSID(0, "$NetBSD: fault.c,v 1.119 2024/02/02 22:00:32 andvar Exp $");
86
87 #include <sys/param.h>
88
89 #include <sys/cpu.h>
90 #include <sys/intr.h>
91 #include <sys/kauth.h>
92 #include <sys/kernel.h>
93 #include <sys/proc.h>
94 #include <sys/systm.h>
95
96 #include <uvm/uvm_extern.h>
97 #include <uvm/uvm_stat.h>
98 #ifdef UVMHIST
99 #include <uvm/uvm.h>
100 #endif
101
102 #include <arm/locore.h>
103
104 #include <machine/pcb.h>
105 #if defined(DDB) || defined(KGDB)
106 #include <machine/db_machdep.h>
107 #ifdef KGDB
108 #include <sys/kgdb.h>
109 #endif
110 #if !defined(DDB)
111 #define kdb_trap kgdb_trap
112 #endif
113 #endif
114
115 #include <arch/arm/arm/disassem.h>
116 #include <arm/arm32/machdep.h>
117
118 #if defined(DEBUG) && !defined(MULTIPROCESSOR)
119 int last_fault_code; /* For the benefit of pmap_fault_fixup() */
120 #endif
121
122 #if defined(CPU_ARM6) || defined(CPU_ARM7) || defined(CPU_ARM7TDMI)
123 /* These CPUs may need data/prefetch abort fixups */
124 #define CPU_ABORT_FIXUP_REQUIRED
125 #endif
126
127 struct data_abort {
128 int (*func)(trapframe_t *, u_int, u_int, struct lwp *, ksiginfo_t *);
129 const char *desc;
130 };
131
132 static int dab_fatal(trapframe_t *, u_int, u_int, struct lwp *, ksiginfo_t *);
133 static int dab_align(trapframe_t *, u_int, u_int, struct lwp *, ksiginfo_t *);
134 static int dab_buserr(trapframe_t *, u_int, u_int, struct lwp *, ksiginfo_t *);
135
136 static const struct data_abort data_aborts[] = {
137 {dab_fatal, "Vector Exception"},
138 {dab_align, "Alignment Fault 1"},
139 {dab_fatal, "Terminal Exception"},
140 {dab_align, "Alignment Fault 3"},
141 {dab_buserr, "External Linefetch Abort (S)"},
142 {NULL, "Translation Fault (S)"},
143 {dab_buserr, "External Linefetch Abort (P)"},
144 {NULL, "Translation Fault (P)"},
145 {dab_buserr, "External Non-Linefetch Abort (S)"},
146 {NULL, "Domain Fault (S)"},
147 {dab_buserr, "External Non-Linefetch Abort (P)"},
148 {NULL, "Domain Fault (P)"},
149 {dab_buserr, "External Translation Abort (L1)"},
150 {NULL, "Permission Fault (S)"},
151 {dab_buserr, "External Translation Abort (L2)"},
152 {NULL, "Permission Fault (P)"}
153 };
154
155 /* Determine if 'x' is a permission fault */
156 #define IS_PERMISSION_FAULT(x) \
157 (((1 << ((x) & FAULT_TYPE_MASK)) & \
158 ((1 << FAULT_PERM_P) | (1 << FAULT_PERM_S))) != 0)
159
160 #if 0
161 /* maybe one day we'll do emulations */
162 #define TRAPSIGNAL(l,k) (*(l)->l_proc->p_emul->e_trapsignal)((l), (k))
163 #else
164 #define TRAPSIGNAL(l,k) trapsignal((l), (k))
165 #endif
166
167 static inline void
call_trapsignal(struct lwp * l,const struct trapframe * tf,ksiginfo_t * ksi)168 call_trapsignal(struct lwp *l, const struct trapframe *tf, ksiginfo_t *ksi)
169 {
170 if (l->l_proc->p_pid == 1 || cpu_printfataltraps) {
171 printf("%d.%d(%s): trap: signo=%d code=%d addr=%p trap=%#x\n",
172 l->l_proc->p_pid, l->l_lid, l->l_proc->p_comm,
173 ksi->ksi_signo, ksi->ksi_code, ksi->ksi_addr,
174 ksi->ksi_trap);
175 printf("r0=%08x r1=%08x r2=%08x r3=%08x\n",
176 tf->tf_r0, tf->tf_r1, tf->tf_r2, tf->tf_r3);
177 printf("r4=%08x r5=%08x r6=%08x r7=%08x\n",
178 tf->tf_r4, tf->tf_r5, tf->tf_r6, tf->tf_r7);
179 printf("r8=%08x r9=%08x rA=%08x rB=%08x\n",
180 tf->tf_r8, tf->tf_r9, tf->tf_r10, tf->tf_r11);
181 printf("ip=%08x sp=%08x lr=%08x pc=%08x spsr=%08x\n",
182 tf->tf_r12, tf->tf_usr_sp, tf->tf_usr_lr, tf->tf_pc,
183 tf->tf_spsr);
184 }
185
186 TRAPSIGNAL(l, ksi);
187 }
188
189 static inline int
data_abort_fixup(trapframe_t * tf,u_int fsr,u_int far,struct lwp * l)190 data_abort_fixup(trapframe_t *tf, u_int fsr, u_int far, struct lwp *l)
191 {
192 #ifdef CPU_ABORT_FIXUP_REQUIRED
193 int error;
194
195 /* Call the CPU specific data abort fixup routine */
196 error = cpu_dataabt_fixup(tf);
197 if (__predict_true(error != ABORT_FIXUP_FAILED))
198 return error;
199
200 /*
201 * Oops, couldn't fix up the instruction
202 */
203 printf("%s: fixup for %s mode data abort failed.\n", __func__,
204 TRAP_USERMODE(tf) ? "user" : "kernel");
205 #ifdef THUMB_CODE
206 if (tf->tf_spsr & PSR_T_bit) {
207 printf("pc = 0x%08x, opcode 0x%04x, 0x%04x, insn = ",
208 tf->tf_pc, *((uint16 *)(tf->tf_pc & ~1)),
209 *((uint16 *)((tf->tf_pc + 2) & ~1)));
210 }
211 else
212 #endif
213 {
214 printf("pc = 0x%08x, opcode 0x%08x, insn = ", tf->tf_pc,
215 *((u_int *)tf->tf_pc));
216 }
217 disassemble(tf->tf_pc);
218
219 /* Die now if this happened in kernel mode */
220 if (!TRAP_USERMODE(tf))
221 dab_fatal(tf, fsr, far, l, NULL);
222
223 return error;
224 #else
225 return ABORT_FIXUP_OK;
226 #endif /* CPU_ABORT_FIXUP_REQUIRED */
227 }
228
229 void
data_abort_handler(trapframe_t * tf)230 data_abort_handler(trapframe_t *tf)
231 {
232 struct vm_map *map;
233 struct lwp * const l = curlwp;
234 struct cpu_info * const ci = curcpu();
235 u_int far, fsr;
236 vm_prot_t ftype;
237 void *onfault;
238 vaddr_t va;
239 int error;
240 ksiginfo_t ksi;
241
242 UVMHIST_FUNC(__func__);
243 UVMHIST_CALLED(maphist);
244
245 /* Grab FAR/FSR before enabling interrupts */
246 far = cpu_faultaddress();
247 fsr = cpu_faultstatus();
248
249 /* Update vmmeter statistics */
250 ci->ci_data.cpu_ntrap++;
251
252 /* Re-enable interrupts if they were enabled previously */
253 KASSERT(!TRAP_USERMODE(tf) || VALID_PSR(tf->tf_spsr));
254 #ifdef __NO_FIQ
255 if (__predict_true((tf->tf_spsr & I32_bit) != I32_bit))
256 restore_interrupts(tf->tf_spsr & IF32_bits);
257 #else
258 if (__predict_true((tf->tf_spsr & IF32_bits) != IF32_bits))
259 restore_interrupts(tf->tf_spsr & IF32_bits);
260 #endif
261
262 /* Get the current lwp structure */
263
264 UVMHIST_LOG(maphist, " (l=%#jx, far=%#jx, fsr=%#jx",
265 (uintptr_t)l, far, fsr, 0);
266 UVMHIST_LOG(maphist, " tf=%#jx, pc=%#jx)",
267 (uintptr_t)tf, (uintptr_t)tf->tf_pc, 0, 0);
268
269 /* Data abort came from user mode? */
270 bool user = (TRAP_USERMODE(tf) != 0);
271
272 /* Grab the current pcb */
273 struct pcb * const pcb = lwp_getpcb(l);
274
275 curcpu()->ci_abt_evs[fsr & FAULT_TYPE_MASK].ev_count++;
276
277 /* Invoke the appropriate handler, if necessary */
278 if (__predict_false(data_aborts[fsr & FAULT_TYPE_MASK].func != NULL)) {
279 #ifdef DIAGNOSTIC
280 printf("%s: data_aborts fsr=0x%x far=0x%x\n",
281 __func__, fsr, far);
282 #endif
283 if ((data_aborts[fsr & FAULT_TYPE_MASK].func)(tf, fsr, far,
284 l, &ksi))
285 goto do_trapsignal;
286 goto out;
287 }
288
289 /*
290 * At this point, we're dealing with one of the following data aborts:
291 *
292 * FAULT_TRANS_S - Translation -- Section
293 * FAULT_TRANS_P - Translation -- Page
294 * FAULT_DOMAIN_S - Domain -- Section
295 * FAULT_DOMAIN_P - Domain -- Page
296 * FAULT_PERM_S - Permission -- Section
297 * FAULT_PERM_P - Permission -- Page
298 *
299 * These are the main virtual memory-related faults signalled by
300 * the MMU.
301 */
302
303 KASSERTMSG(!user || tf == lwp_trapframe(l), "tf %p vs %p", tf,
304 lwp_trapframe(l));
305
306 /*
307 * Make sure the Program Counter is sane. We could fall foul of
308 * someone executing Thumb code, in which case the PC might not
309 * be word-aligned. This would cause a kernel alignment fault
310 * further down if we have to decode the current instruction.
311 */
312 #ifdef THUMB_CODE
313 /*
314 * XXX: It would be nice to be able to support Thumb in the kernel
315 * at some point.
316 */
317 if (__predict_false(!user && (tf->tf_pc & 3) != 0)) {
318 printf("\n%s: Misaligned Kernel-mode Program Counter\n",
319 __func__);
320 dab_fatal(tf, fsr, far, l, NULL);
321 }
322 #else
323 if (__predict_false((tf->tf_pc & 3) != 0)) {
324 if (user) {
325 /*
326 * Give the user an illegal instruction signal.
327 */
328 /* Deliver a SIGILL to the process */
329 KSI_INIT_TRAP(&ksi);
330 ksi.ksi_signo = SIGILL;
331 ksi.ksi_code = ILL_ILLOPC;
332 ksi.ksi_addr = (uint32_t *)(intptr_t) far;
333 ksi.ksi_trap = fsr;
334 goto do_trapsignal;
335 }
336
337 /*
338 * The kernel never executes Thumb code.
339 */
340 printf("\n%s: Misaligned Kernel-mode Program Counter\n",
341 __func__);
342 dab_fatal(tf, fsr, far, l, NULL);
343 }
344 #endif
345
346 /* See if the CPU state needs to be fixed up */
347 switch (data_abort_fixup(tf, fsr, far, l)) {
348 case ABORT_FIXUP_RETURN:
349 return;
350 case ABORT_FIXUP_FAILED:
351 /* Deliver a SIGILL to the process */
352 KSI_INIT_TRAP(&ksi);
353 ksi.ksi_signo = SIGILL;
354 ksi.ksi_code = ILL_ILLOPC;
355 ksi.ksi_addr = (uint32_t *)(intptr_t) far;
356 ksi.ksi_trap = fsr;
357 goto do_trapsignal;
358 default:
359 break;
360 }
361
362 va = trunc_page((vaddr_t)far);
363
364 /*
365 * It is only a kernel address space fault iff:
366 * 1. user == 0 and
367 * 2. pcb_onfault not set or
368 * 3. pcb_onfault set and not LDRT/LDRBT/STRT/STRBT instruction.
369 */
370 if (!user && (va >= VM_MIN_KERNEL_ADDRESS ||
371 (va < VM_MIN_ADDRESS && vector_page == ARM_VECTORS_LOW)) &&
372 __predict_true((pcb->pcb_onfault == NULL ||
373 (read_insn(tf->tf_pc, false) & 0x05200000) != 0x04200000))) {
374 map = kernel_map;
375
376 /* Was the fault due to the FPE ? */
377 if (__predict_false((tf->tf_spsr & PSR_MODE)==PSR_UND32_MODE)) {
378 KSI_INIT_TRAP(&ksi);
379 ksi.ksi_signo = SIGSEGV;
380 ksi.ksi_code = SEGV_ACCERR;
381 ksi.ksi_addr = (uint32_t *)(intptr_t) far;
382 ksi.ksi_trap = fsr;
383
384 /*
385 * Force exit via userret()
386 * This is necessary as the FPE is an extension to
387 * userland that actually runs in a privileged mode
388 * but uses USR mode permissions for its accesses.
389 */
390 user = true;
391 goto do_trapsignal;
392 }
393 } else {
394 map = &l->l_proc->p_vmspace->vm_map;
395 }
396
397 /*
398 * We need to know whether the page should be mapped as R or R/W.
399 * Before ARMv6, the MMU did not give us the info as to whether the
400 * fault was caused by a read or a write.
401 *
402 * However, we know that a permission fault can only be the result of
403 * a write to a read-only location, so we can deal with those quickly.
404 *
405 * Otherwise we need to disassemble the instruction responsible to
406 * determine if it was a write.
407 */
408 if (CPU_IS_ARMV6_P() || CPU_IS_ARMV7_P()) {
409 ftype = (fsr & FAULT_WRITE) ? VM_PROT_WRITE : VM_PROT_READ;
410 } else if (IS_PERMISSION_FAULT(fsr)) {
411 ftype = VM_PROT_WRITE;
412 } else {
413 #ifdef THUMB_CODE
414 /* Fast track the ARM case. */
415 if (__predict_false(tf->tf_spsr & PSR_T_bit)) {
416 u_int insn = read_thumb_insn(tf->tf_pc, user);
417 u_int insn_f8 = insn & 0xf800;
418 u_int insn_fe = insn & 0xfe00;
419
420 if (insn_f8 == 0x6000 || /* STR(1) */
421 insn_f8 == 0x7000 || /* STRB(1) */
422 insn_f8 == 0x8000 || /* STRH(1) */
423 insn_f8 == 0x9000 || /* STR(3) */
424 insn_f8 == 0xc000 || /* STM */
425 insn_fe == 0x5000 || /* STR(2) */
426 insn_fe == 0x5200 || /* STRH(2) */
427 insn_fe == 0x5400) /* STRB(2) */
428 ftype = VM_PROT_WRITE;
429 else
430 ftype = VM_PROT_READ;
431 }
432 else
433 #endif
434 {
435 u_int insn = read_insn(tf->tf_pc, user);
436
437 if (((insn & 0x0c100000) == 0x04000000) || /* STR[B] */
438 ((insn & 0x0e1000b0) == 0x000000b0) || /* STR[HD]*/
439 ((insn & 0x0a100000) == 0x08000000) || /* STM/CDT*/
440 ((insn & 0x0f9000f0) == 0x01800090)) /* STREX[BDH] */
441 ftype = VM_PROT_WRITE;
442 else if ((insn & 0x0fb00ff0) == 0x01000090)/* SWP */
443 ftype = VM_PROT_READ | VM_PROT_WRITE;
444 else
445 ftype = VM_PROT_READ;
446 }
447 }
448
449 /*
450 * See if the fault is as a result of ref/mod emulation,
451 * or domain mismatch.
452 */
453 #if defined(DEBUG) && !defined(MULTIPROCESSOR)
454 last_fault_code = fsr;
455 #endif
456 if (pmap_fault_fixup(map->pmap, va, ftype, user)) {
457 UVMHIST_LOG(maphist, " <- ref/mod emul", 0, 0, 0, 0);
458 goto out;
459 }
460
461 if (__predict_false(curcpu()->ci_intr_depth > 0)) {
462 if (pcb->pcb_onfault) {
463 tf->tf_r0 = EINVAL;
464 tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault;
465 return;
466 }
467 printf("\nNon-emulated page fault with intr_depth > 0\n");
468 dab_fatal(tf, fsr, far, l, NULL);
469 }
470
471 #ifdef PMAP_FAULTINFO
472 struct pcb_faultinfo * const pfi = &pcb->pcb_faultinfo;
473 struct proc * const p = curproc;
474
475 if (p->p_pid == pfi->pfi_lastpid && va == pfi->pfi_faultaddr) {
476 if (++pfi->pfi_repeats > 4) {
477 tlb_asid_t asid = tlb_get_asid();
478 pt_entry_t *ptep = pfi->pfi_faultptep;
479
480 printf("%s: fault #%u (%x/%s) for %#" PRIxVADDR
481 "(%#x) at pc %#" PRIxREGISTER " curpid=%u/%u "
482 "ptep@%p=%#" PRIxPTE ")\n", __func__,
483 pfi->pfi_repeats, fsr & FAULT_TYPE_MASK,
484 data_aborts[fsr & FAULT_TYPE_MASK].desc, va,
485 far, tf->tf_pc, map->pmap->pm_pai[0].pai_asid,
486 asid, ptep, ptep ? *ptep : 0);
487 cpu_Debugger();
488 }
489 } else {
490 pfi->pfi_lastpid = p->p_pid;
491 pfi->pfi_faultaddr = va;
492 pfi->pfi_repeats = 0;
493 pfi->pfi_faultptep = NULL;
494 pfi->pfi_faulttype = fsr & FAULT_TYPE_MASK;
495 }
496 #endif /* PMAP_FAULTINFO */
497
498 onfault = pcb->pcb_onfault;
499 pcb->pcb_onfault = NULL;
500 error = uvm_fault(map, va, ftype);
501 pcb->pcb_onfault = onfault;
502
503 if (__predict_true(error == 0)) {
504 if (user)
505 uvm_grow(l->l_proc, va); /* Record any stack growth */
506 UVMHIST_LOG(maphist, " <- uvm", 0, 0, 0, 0);
507 goto out;
508 }
509
510 if (user == 0) {
511 if (pcb->pcb_onfault) {
512 tf->tf_r0 = error;
513 tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault;
514 return;
515 }
516
517 printf("\nuvm_fault(%p, %lx, %x) -> %x\n", map, va, ftype,
518 error);
519 dab_fatal(tf, fsr, far, l, NULL);
520 }
521
522 KSI_INIT_TRAP(&ksi);
523
524 switch (error) {
525 case ENOMEM:
526 printf("UVM: pid %d (%s), uid %d killed: "
527 "out of swap\n", l->l_proc->p_pid, l->l_proc->p_comm,
528 l->l_cred ? kauth_cred_geteuid(l->l_cred) : -1);
529 ksi.ksi_signo = SIGKILL;
530 break;
531 case EACCES:
532 ksi.ksi_signo = SIGSEGV;
533 ksi.ksi_code = SEGV_ACCERR;
534 break;
535 case EINVAL:
536 ksi.ksi_signo = SIGBUS;
537 ksi.ksi_code = BUS_ADRERR;
538 break;
539 default:
540 ksi.ksi_signo = SIGSEGV;
541 ksi.ksi_code = SEGV_MAPERR;
542 break;
543 }
544 ksi.ksi_addr = (uint32_t *)(intptr_t) far;
545 ksi.ksi_trap = fsr;
546 UVMHIST_LOG(maphist, " <- error (%jd)", error, 0, 0, 0);
547
548 do_trapsignal:
549 call_trapsignal(l, tf, &ksi);
550 out:
551 /* If returning to user mode, make sure to invoke userret() */
552 if (user)
553 userret(l);
554 }
555
556 /*
557 * dab_fatal() handles the following data aborts:
558 *
559 * FAULT_WRTBUF_0 - Vector Exception
560 * FAULT_WRTBUF_1 - Terminal Exception
561 *
562 * We should never see these on a properly functioning system.
563 *
564 * This function is also called by the other handlers if they
565 * detect a fatal problem.
566 *
567 * Note: If 'l' is NULL, we assume we're dealing with a prefetch abort.
568 */
569 static int
dab_fatal(trapframe_t * tf,u_int fsr,u_int far,struct lwp * l,ksiginfo_t * ksi)570 dab_fatal(trapframe_t *tf, u_int fsr, u_int far, struct lwp *l, ksiginfo_t *ksi)
571 {
572 const char * const mode = TRAP_USERMODE(tf) ? "user" : "kernel";
573
574 if (l != NULL) {
575 printf("Fatal %s mode data abort: '%s'\n", mode,
576 data_aborts[fsr & FAULT_TYPE_MASK].desc);
577 printf("trapframe: %p\nFSR=%08x, FAR=", tf, fsr);
578 if ((fsr & FAULT_IMPRECISE) == 0)
579 printf("%08x, ", far);
580 else
581 printf("Invalid, ");
582 printf("spsr=%08x\n", tf->tf_spsr);
583 } else {
584 printf("Fatal %s mode prefetch abort at 0x%08x\n",
585 mode, tf->tf_pc);
586 printf("trapframe: %p, spsr=%08x\n", tf, tf->tf_spsr);
587 }
588
589 printf("r0 =%08x, r1 =%08x, r2 =%08x, r3 =%08x\n",
590 tf->tf_r0, tf->tf_r1, tf->tf_r2, tf->tf_r3);
591 printf("r4 =%08x, r5 =%08x, r6 =%08x, r7 =%08x\n",
592 tf->tf_r4, tf->tf_r5, tf->tf_r6, tf->tf_r7);
593 printf("r8 =%08x, r9 =%08x, r10=%08x, r11=%08x\n",
594 tf->tf_r8, tf->tf_r9, tf->tf_r10, tf->tf_r11);
595 printf("r12=%08x, ", tf->tf_r12);
596
597 if (TRAP_USERMODE(tf))
598 printf("usp=%08x, ulr=%08x",
599 tf->tf_usr_sp, tf->tf_usr_lr);
600 else
601 printf("ssp=%08x, slr=%08x",
602 tf->tf_svc_sp, tf->tf_svc_lr);
603 printf(", pc =%08x\n\n", tf->tf_pc);
604
605 #if defined(DDB) || defined(KGDB)
606 kdb_trap(T_FAULT, tf);
607 #endif
608 panic("Fatal abort");
609 /*NOTREACHED*/
610 }
611
612 /*
613 * dab_align() handles the following data aborts:
614 *
615 * FAULT_ALIGN_0 - Alignment fault
616 * FAULT_ALIGN_0 - Alignment fault
617 *
618 * These faults are fatal if they happen in kernel mode. Otherwise, we
619 * deliver a bus error to the process.
620 */
621 static int
dab_align(trapframe_t * tf,u_int fsr,u_int far,struct lwp * l,ksiginfo_t * ksi)622 dab_align(trapframe_t *tf, u_int fsr, u_int far, struct lwp *l, ksiginfo_t *ksi)
623 {
624 /* Alignment faults are always fatal if they occur in kernel mode */
625 if (!TRAP_USERMODE(tf))
626 dab_fatal(tf, fsr, far, l, NULL);
627
628 /* pcb_onfault *must* be NULL at this point */
629 KDASSERT(((struct pcb *)lwp_getpcb(l))->pcb_onfault == NULL);
630
631 /* See if the CPU state needs to be fixed up */
632 (void) data_abort_fixup(tf, fsr, far, l);
633
634 /* Deliver a bus error signal to the process */
635 KSI_INIT_TRAP(ksi);
636 ksi->ksi_signo = SIGBUS;
637 ksi->ksi_code = BUS_ADRALN;
638 ksi->ksi_addr = (uint32_t *)(intptr_t)far;
639 ksi->ksi_trap = fsr;
640
641 KASSERTMSG(tf == lwp_trapframe(l), "tf %p vs %p", tf, lwp_trapframe(l));
642
643 return 1;
644 }
645
646 /*
647 * dab_buserr() handles the following data aborts:
648 *
649 * FAULT_BUSERR_0 - External Abort on Linefetch -- Section
650 * FAULT_BUSERR_1 - External Abort on Linefetch -- Page
651 * FAULT_BUSERR_2 - External Abort on Non-linefetch -- Section
652 * FAULT_BUSERR_3 - External Abort on Non-linefetch -- Page
653 * FAULT_BUSTRNL1 - External abort on Translation -- Level 1
654 * FAULT_BUSTRNL2 - External abort on Translation -- Level 2
655 *
656 * If pcb_onfault is set, flag the fault and return to the handler.
657 * If the fault occurred in user mode, give the process a SIGBUS.
658 *
659 * Note: On XScale, FAULT_BUSERR_0, FAULT_BUSERR_1, and FAULT_BUSERR_2
660 * can be flagged as imprecise in the FSR. This causes a real headache
661 * since some of the machine state is lost. In this case, tf->tf_pc
662 * may not actually point to the offending instruction. In fact, if
663 * we've taken a double abort fault, it generally points somewhere near
664 * the top of "data_abort_entry" in exception.S.
665 *
666 * In all other cases, these data aborts are considered fatal.
667 */
668 static int
dab_buserr(trapframe_t * tf,u_int fsr,u_int far,struct lwp * l,ksiginfo_t * ksi)669 dab_buserr(trapframe_t *tf, u_int fsr, u_int far, struct lwp *l,
670 ksiginfo_t *ksi)
671 {
672 struct pcb *pcb = lwp_getpcb(l);
673
674 #ifdef __XSCALE__
675 if ((fsr & FAULT_IMPRECISE) != 0 &&
676 (tf->tf_spsr & PSR_MODE) == PSR_ABT32_MODE) {
677 /*
678 * Oops, an imprecise, double abort fault. We've lost the
679 * r14_abt/spsr_abt values corresponding to the original
680 * abort, and the spsr saved in the trapframe indicates
681 * ABT mode.
682 */
683 tf->tf_spsr &= ~PSR_MODE;
684
685 /*
686 * We use a simple heuristic to determine if the double abort
687 * happened as a result of a kernel or user mode access.
688 * If the current trapframe is at the top of the kernel stack,
689 * the fault _must_ have come from user mode.
690 */
691 if (tf != ((trapframe_t *)pcb->pcb_ksp) - 1) {
692 /*
693 * Kernel mode. We're either about to die a
694 * spectacular death, or pcb_onfault will come
695 * to our rescue. Either way, the current value
696 * of tf->tf_pc is irrelevant.
697 */
698 tf->tf_spsr |= PSR_SVC32_MODE;
699 if (pcb->pcb_onfault == NULL)
700 printf("\nKernel mode double abort!\n");
701 } else {
702 /*
703 * User mode. We've lost the program counter at the
704 * time of the fault (not that it was accurate anyway;
705 * it's not called an imprecise fault for nothing).
706 * About all we can do is copy r14_usr to tf_pc and
707 * hope for the best. The process is about to get a
708 * SIGBUS, so it's probably history anyway.
709 */
710 tf->tf_spsr |= PSR_USR32_MODE;
711 tf->tf_pc = tf->tf_usr_lr;
712 #ifdef THUMB_CODE
713 tf->tf_spsr &= ~PSR_T_bit;
714 if (tf->tf_usr_lr & 1)
715 tf->tf_spsr |= PSR_T_bit;
716 #endif
717 }
718 }
719
720 /* FAR is invalid for imprecise exceptions */
721 if ((fsr & FAULT_IMPRECISE) != 0)
722 far = 0;
723 #endif /* __XSCALE__ */
724
725 if (pcb->pcb_onfault) {
726 KDASSERT(TRAP_USERMODE(tf) == 0);
727 tf->tf_r0 = EFAULT;
728 tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault;
729 return 0;
730 }
731
732 /* See if the CPU state needs to be fixed up */
733 (void) data_abort_fixup(tf, fsr, far, l);
734
735 /*
736 * At this point, if the fault happened in kernel mode, we're toast
737 */
738 if (!TRAP_USERMODE(tf))
739 dab_fatal(tf, fsr, far, l, NULL);
740
741 /* Deliver a bus error signal to the process */
742 KSI_INIT_TRAP(ksi);
743 ksi->ksi_signo = SIGBUS;
744 ksi->ksi_code = BUS_ADRERR;
745 ksi->ksi_addr = (uint32_t *)(intptr_t)far;
746 ksi->ksi_trap = fsr;
747
748 KASSERTMSG(tf == lwp_trapframe(l), "tf %p vs %p", tf, lwp_trapframe(l));
749
750 return 1;
751 }
752
753 static inline int
prefetch_abort_fixup(trapframe_t * tf)754 prefetch_abort_fixup(trapframe_t *tf)
755 {
756 #ifdef CPU_ABORT_FIXUP_REQUIRED
757 int error;
758
759 /* Call the CPU specific prefetch abort fixup routine */
760 error = cpu_prefetchabt_fixup(tf);
761 if (__predict_true(error != ABORT_FIXUP_FAILED))
762 return error;
763
764 /*
765 * Oops, couldn't fix up the instruction
766 */
767 printf("%s: fixup for %s mode prefetch abort failed.\n", __func__,
768 TRAP_USERMODE(tf) ? "user" : "kernel");
769 #ifdef THUMB_CODE
770 if (tf->tf_spsr & PSR_T_bit) {
771 printf("pc = 0x%08x, opcode 0x%04x, 0x%04x, insn = ",
772 tf->tf_pc, *((uint16 *)(tf->tf_pc & ~1)),
773 *((uint16 *)((tf->tf_pc + 2) & ~1)));
774 }
775 else
776 #endif
777 {
778 printf("pc = 0x%08x, opcode 0x%08x, insn = ", tf->tf_pc,
779 *((u_int *)tf->tf_pc));
780 }
781 disassemble(tf->tf_pc);
782
783 /* Die now if this happened in kernel mode */
784 if (!TRAP_USERMODE(tf))
785 dab_fatal(tf, 0, tf->tf_pc, NULL, NULL);
786
787 return error;
788 #else
789 return ABORT_FIXUP_OK;
790 #endif /* CPU_ABORT_FIXUP_REQUIRED */
791 }
792
793 /*
794 * void prefetch_abort_handler(trapframe_t *tf)
795 *
796 * Abort handler called when instruction execution occurs at
797 * a non existent or restricted (access permissions) memory page.
798 * If the address is invalid and we were in SVC mode then panic as
799 * the kernel should never prefetch abort.
800 * If the address is invalid and the page is mapped then the user process
801 * does not have read permission so send it a signal.
802 * Otherwise fault the page in and try again.
803 */
804 void
prefetch_abort_handler(trapframe_t * tf)805 prefetch_abort_handler(trapframe_t *tf)
806 {
807 struct lwp *l;
808 struct pcb *pcb __diagused;
809 struct vm_map *map;
810 vaddr_t fault_pc, va;
811 ksiginfo_t ksi;
812 int error, user;
813
814 UVMHIST_FUNC(__func__);
815 UVMHIST_CALLED(maphist);
816
817 /* Update vmmeter statistics */
818 curcpu()->ci_data.cpu_ntrap++;
819
820 l = curlwp;
821 pcb = lwp_getpcb(l);
822 user = TRAP_USERMODE(tf) != 0;
823
824 /*
825 * Enable IRQ's (disabled by the abort) This always comes
826 * from user mode so we know interrupts were not disabled.
827 * But we check anyway.
828 */
829 KASSERT(!user || VALID_PSR(tf->tf_spsr));
830 #ifdef __NO_FIQ
831 if (__predict_true((tf->tf_spsr & I32_bit) != I32_bit))
832 restore_interrupts(tf->tf_spsr & IF32_bits);
833 #else
834 if (__predict_true((tf->tf_spsr & IF32_bits) != IF32_bits))
835 restore_interrupts(tf->tf_spsr & IF32_bits);
836 #endif
837
838 /* See if the CPU state needs to be fixed up */
839 switch (prefetch_abort_fixup(tf)) {
840 case ABORT_FIXUP_RETURN:
841 KASSERT(!TRAP_USERMODE(tf) || VALID_PSR(tf->tf_spsr));
842 return;
843 case ABORT_FIXUP_FAILED:
844 /* Deliver a SIGILL to the process */
845 KSI_INIT_TRAP(&ksi);
846 ksi.ksi_signo = SIGILL;
847 ksi.ksi_code = ILL_ILLOPC;
848 ksi.ksi_addr = (uint32_t *)(intptr_t) tf->tf_pc;
849 KASSERTMSG(tf == lwp_trapframe(l), "tf %p vs %p", tf,
850 lwp_trapframe(l));
851 goto do_trapsignal;
852 default:
853 break;
854 }
855
856 /* Prefetch aborts cannot happen in kernel mode */
857 if (__predict_false(!user))
858 dab_fatal(tf, 0, tf->tf_pc, NULL, NULL);
859
860 /* Get fault address */
861 fault_pc = tf->tf_pc;
862 KASSERTMSG(tf == lwp_trapframe(l), "tf %p vs %p", tf, lwp_trapframe(l));
863 UVMHIST_LOG(maphist, " (pc=%#jx, l=%#jx, tf=%#jx)",
864 fault_pc, (uintptr_t)l, (uintptr_t)tf, 0);
865
866 #ifdef THUMB_CODE
867 recheck:
868 #endif
869 /* Ok validate the address, can only execute in USER space */
870 if (__predict_false(fault_pc >= VM_MAXUSER_ADDRESS ||
871 (fault_pc < VM_MIN_ADDRESS && vector_page == ARM_VECTORS_LOW))) {
872 KSI_INIT_TRAP(&ksi);
873 ksi.ksi_signo = SIGSEGV;
874 ksi.ksi_code = SEGV_ACCERR;
875 ksi.ksi_addr = (uint32_t *)(intptr_t) fault_pc;
876 ksi.ksi_trap = fault_pc;
877 goto do_trapsignal;
878 }
879
880 map = &l->l_proc->p_vmspace->vm_map;
881 va = trunc_page(fault_pc);
882
883 /*
884 * See if the pmap can handle this fault on its own...
885 */
886 #if defined(DEBUG) && !defined(MULTIPROCESSOR)
887 last_fault_code = -1;
888 #endif
889 if (pmap_fault_fixup(map->pmap, va, VM_PROT_READ|VM_PROT_EXECUTE, 1)) {
890 UVMHIST_LOG (maphist, " <- emulated", 0, 0, 0, 0);
891 goto out;
892 }
893
894 #ifdef DIAGNOSTIC
895 if (__predict_false(curcpu()->ci_intr_depth > 0)) {
896 printf("\nNon-emulated prefetch abort with intr_depth > 0\n");
897 dab_fatal(tf, 0, tf->tf_pc, NULL, NULL);
898 }
899 #endif
900
901 KASSERT(pcb->pcb_onfault == NULL);
902 error = uvm_fault(map, va, VM_PROT_READ|VM_PROT_EXECUTE);
903
904 if (__predict_true(error == 0)) {
905 UVMHIST_LOG (maphist, " <- uvm", 0, 0, 0, 0);
906 goto out;
907 }
908 KSI_INIT_TRAP(&ksi);
909
910 UVMHIST_LOG (maphist, " <- fatal (%jd)", error, 0, 0, 0);
911
912 if (error == ENOMEM) {
913 printf("UVM: pid %d (%s), uid %d killed: "
914 "out of swap\n", l->l_proc->p_pid, l->l_proc->p_comm,
915 l->l_cred ? kauth_cred_geteuid(l->l_cred) : -1);
916 ksi.ksi_signo = SIGKILL;
917 } else
918 ksi.ksi_signo = SIGSEGV;
919
920 ksi.ksi_code = SEGV_MAPERR;
921 ksi.ksi_addr = (uint32_t *)(intptr_t) fault_pc;
922 ksi.ksi_trap = fault_pc;
923
924 do_trapsignal:
925 call_trapsignal(l, tf, &ksi);
926
927 out:
928
929 #ifdef THUMB_CODE
930 #define THUMB_32BIT(hi) (((hi) & 0xe000) == 0xe000 && ((hi) & 0x1800))
931 /* thumb-32 instruction was located on page boundary? */
932 if ((tf->tf_spsr & PSR_T_bit) &&
933 ((fault_pc & PAGE_MASK) == (PAGE_SIZE - THUMB_INSN_SIZE)) &&
934 THUMB_32BIT(*(uint16_t *)tf->tf_pc)) {
935 fault_pc = tf->tf_pc + THUMB_INSN_SIZE;
936 goto recheck;
937 }
938 #endif /* THUMB_CODE */
939
940 KASSERT(!TRAP_USERMODE(tf) || VALID_PSR(tf->tf_spsr));
941 userret(l);
942 }
943
944 /*
945 * Tentatively read an 8, 16, or 32-bit value from 'addr'.
946 * If the read succeeds, the value is written to 'rptr' and zero is returned.
947 * Else, return EFAULT.
948 */
949 int
badaddr_read(void * addr,size_t size,void * rptr)950 badaddr_read(void *addr, size_t size, void *rptr)
951 {
952 extern int badaddr_read_1(const uint8_t *, uint8_t *);
953 extern int badaddr_read_2(const uint16_t *, uint16_t *);
954 extern int badaddr_read_4(const uint32_t *, uint32_t *);
955 union {
956 uint8_t v1;
957 uint16_t v2;
958 uint32_t v4;
959 } u;
960 int rv, s;
961
962 cpu_drain_writebuf();
963
964 s = splhigh();
965
966 /* Read from the test address. */
967 switch (size) {
968 case sizeof(uint8_t):
969 rv = badaddr_read_1(addr, &u.v1);
970 if (rv == 0 && rptr)
971 *(uint8_t *) rptr = u.v1;
972 break;
973
974 case sizeof(uint16_t):
975 rv = badaddr_read_2(addr, &u.v2);
976 if (rv == 0 && rptr)
977 *(uint16_t *) rptr = u.v2;
978 break;
979
980 case sizeof(uint32_t):
981 rv = badaddr_read_4(addr, &u.v4);
982 if (rv == 0 && rptr)
983 *(uint32_t *) rptr = u.v4;
984 break;
985
986 default:
987 panic("%s: invalid size (%zu)", __func__, size);
988 }
989
990 splx(s);
991
992 /* Return EFAULT if the address was invalid, else zero */
993 return rv;
994 }
995