xref: /netbsd-src/sys/arch/arm/arm32/fault.c (revision cd22f25e6f6d1cc1f197fe8c5468a80f51d1c4e1)
1 /*	$NetBSD: fault.c,v 1.67 2008/04/27 18:58:44 matt Exp $	*/
2 
3 /*
4  * Copyright 2003 Wasabi Systems, Inc.
5  * All rights reserved.
6  *
7  * Written by Steve C. Woodford for Wasabi Systems, Inc.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  * 3. All advertising materials mentioning features or use of this software
18  *    must display the following acknowledgement:
19  *      This product includes software developed for the NetBSD Project by
20  *      Wasabi Systems, Inc.
21  * 4. The name of Wasabi Systems, Inc. may not be used to endorse
22  *    or promote products derived from this software without specific prior
23  *    written permission.
24  *
25  * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
29  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35  * POSSIBILITY OF SUCH DAMAGE.
36  */
37 /*
38  * Copyright (c) 1994-1997 Mark Brinicombe.
39  * Copyright (c) 1994 Brini.
40  * All rights reserved.
41  *
42  * This code is derived from software written for Brini by Mark Brinicombe
43  *
44  * Redistribution and use in source and binary forms, with or without
45  * modification, are permitted provided that the following conditions
46  * are met:
47  * 1. Redistributions of source code must retain the above copyright
48  *    notice, this list of conditions and the following disclaimer.
49  * 2. Redistributions in binary form must reproduce the above copyright
50  *    notice, this list of conditions and the following disclaimer in the
51  *    documentation and/or other materials provided with the distribution.
52  * 3. All advertising materials mentioning features or use of this software
53  *    must display the following acknowledgement:
54  *	This product includes software developed by Brini.
55  * 4. The name of the company nor the name of the author may be used to
56  *    endorse or promote products derived from this software without specific
57  *    prior written permission.
58  *
59  * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED
60  * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
61  * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
62  * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
63  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
64  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
65  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
66  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
67  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
68  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
69  * SUCH DAMAGE.
70  *
71  * RiscBSD kernel project
72  *
73  * fault.c
74  *
75  * Fault handlers
76  *
77  * Created      : 28/11/94
78  */
79 
80 #include "opt_ddb.h"
81 #include "opt_kgdb.h"
82 
83 #include <sys/types.h>
84 __KERNEL_RCSID(0, "$NetBSD: fault.c,v 1.67 2008/04/27 18:58:44 matt Exp $");
85 
86 #include <sys/param.h>
87 #include <sys/systm.h>
88 #include <sys/proc.h>
89 #include <sys/user.h>
90 #include <sys/kernel.h>
91 #include <sys/kauth.h>
92 #include <sys/cpu.h>
93 
94 #include <uvm/uvm_extern.h>
95 #include <uvm/uvm_stat.h>
96 #ifdef UVMHIST
97 #include <uvm/uvm.h>
98 #endif
99 
100 #include <arm/cpuconf.h>
101 
102 #include <machine/frame.h>
103 #include <arm/arm32/katelib.h>
104 #include <machine/intr.h>
105 #if defined(DDB) || defined(KGDB)
106 #include <machine/db_machdep.h>
107 #ifdef KGDB
108 #include <sys/kgdb.h>
109 #endif
110 #if !defined(DDB)
111 #define kdb_trap	kgdb_trap
112 #endif
113 #endif
114 
115 #include <arch/arm/arm/disassem.h>
116 #include <arm/arm32/machdep.h>
117 
118 extern char fusubailout[];
119 
120 #ifdef DEBUG
121 int last_fault_code;	/* For the benefit of pmap_fault_fixup() */
122 #endif
123 
124 #if defined(CPU_ARM3) || defined(CPU_ARM6) || \
125     defined(CPU_ARM7) || defined(CPU_ARM7TDMI)
126 /* These CPUs may need data/prefetch abort fixups */
127 #define	CPU_ABORT_FIXUP_REQUIRED
128 #endif
129 
130 struct data_abort {
131 	int (*func)(trapframe_t *, u_int, u_int, struct lwp *, ksiginfo_t *);
132 	const char *desc;
133 };
134 
135 static int dab_fatal(trapframe_t *, u_int, u_int, struct lwp *, ksiginfo_t *);
136 static int dab_align(trapframe_t *, u_int, u_int, struct lwp *, ksiginfo_t *);
137 static int dab_buserr(trapframe_t *, u_int, u_int, struct lwp *, ksiginfo_t *);
138 
139 static const struct data_abort data_aborts[] = {
140 	{dab_fatal,	"Vector Exception"},
141 	{dab_align,	"Alignment Fault 1"},
142 	{dab_fatal,	"Terminal Exception"},
143 	{dab_align,	"Alignment Fault 3"},
144 	{dab_buserr,	"External Linefetch Abort (S)"},
145 	{NULL,		"Translation Fault (S)"},
146 	{dab_buserr,	"External Linefetch Abort (P)"},
147 	{NULL,		"Translation Fault (P)"},
148 	{dab_buserr,	"External Non-Linefetch Abort (S)"},
149 	{NULL,		"Domain Fault (S)"},
150 	{dab_buserr,	"External Non-Linefetch Abort (P)"},
151 	{NULL,		"Domain Fault (P)"},
152 	{dab_buserr,	"External Translation Abort (L1)"},
153 	{NULL,		"Permission Fault (S)"},
154 	{dab_buserr,	"External Translation Abort (L2)"},
155 	{NULL,		"Permission Fault (P)"}
156 };
157 
158 /* Determine if a fault came from user mode */
159 #define	TRAP_USERMODE(tf)	((tf->tf_spsr & PSR_MODE) == PSR_USR32_MODE)
160 
161 /* Determine if 'x' is a permission fault */
162 #define	IS_PERMISSION_FAULT(x)					\
163 	(((1 << ((x) & FAULT_TYPE_MASK)) &			\
164 	  ((1 << FAULT_PERM_P) | (1 << FAULT_PERM_S))) != 0)
165 
166 #if 0
167 /* maybe one day we'll do emulations */
168 #define	TRAPSIGNAL(l,k)	(*(l)->l_proc->p_emul->e_trapsignal)((l), (k))
169 #else
170 #define	TRAPSIGNAL(l,k)	trapsignal((l), (k))
171 #endif
172 
173 static inline void
174 call_trapsignal(struct lwp *l, ksiginfo_t *ksi)
175 {
176 
177 	KERNEL_LOCK(1, l);
178 	TRAPSIGNAL(l, ksi);
179 	KERNEL_UNLOCK_LAST(l);
180 }
181 
182 static inline int
183 data_abort_fixup(trapframe_t *tf, u_int fsr, u_int far, struct lwp *l)
184 {
185 #ifdef CPU_ABORT_FIXUP_REQUIRED
186 	int error;
187 
188 	/* Call the CPU specific data abort fixup routine */
189 	error = cpu_dataabt_fixup(tf);
190 	if (__predict_true(error != ABORT_FIXUP_FAILED))
191 		return (error);
192 
193 	/*
194 	 * Oops, couldn't fix up the instruction
195 	 */
196 	printf("data_abort_fixup: fixup for %s mode data abort failed.\n",
197 	    TRAP_USERMODE(tf) ? "user" : "kernel");
198 #ifdef THUMB_CODE
199 	if (tf->tf_spsr & PSR_T_bit) {
200 		printf("pc = 0x%08x, opcode 0x%04x, 0x%04x, insn = ",
201 		    tf->tf_pc, *((u_int16 *)(tf->tf_pc & ~1)),
202 		    *((u_int16 *)((tf->tf_pc + 2) & ~1)));
203 	}
204 	else
205 #endif
206 	{
207 		printf("pc = 0x%08x, opcode 0x%08x, insn = ", tf->tf_pc,
208 		    *((u_int *)tf->tf_pc));
209 	}
210 	disassemble(tf->tf_pc);
211 
212 	/* Die now if this happened in kernel mode */
213 	if (!TRAP_USERMODE(tf))
214 		dab_fatal(tf, fsr, far, l, NULL);
215 
216 	return (error);
217 #else
218 	return (ABORT_FIXUP_OK);
219 #endif /* CPU_ABORT_FIXUP_REQUIRED */
220 }
221 
222 void
223 data_abort_handler(trapframe_t *tf)
224 {
225 	struct vm_map *map;
226 	struct pcb *pcb;
227 	struct lwp *l;
228 	u_int user, far, fsr;
229 	vm_prot_t ftype;
230 	void *onfault;
231 	vaddr_t va;
232 	int error;
233 	ksiginfo_t ksi;
234 
235 	UVMHIST_FUNC("data_abort_handler");
236 
237 	/* Grab FAR/FSR before enabling interrupts */
238 	far = cpu_faultaddress();
239 	fsr = cpu_faultstatus();
240 
241 	UVMHIST_CALLED(maphist);
242 	/* Update vmmeter statistics */
243 	uvmexp.traps++;
244 
245 	/* Re-enable interrupts if they were enabled previously */
246 	if (__predict_true((tf->tf_spsr & I32_bit) == 0))
247 		enable_interrupts(I32_bit);
248 
249 	/* Get the current lwp structure */
250 	KASSERT(curlwp != NULL);
251 	l = curlwp;
252 
253 	UVMHIST_LOG(maphist, " (pc=0x%x, l=0x%x, far=0x%x, fsr=0x%x)",
254 	    tf->tf_pc, l, far, fsr);
255 
256 	/* Data abort came from user mode? */
257 	if ((user = TRAP_USERMODE(tf)) != 0)
258 		LWP_CACHE_CREDS(l, l->l_proc);
259 
260 	/* Grab the current pcb */
261 	pcb = &l->l_addr->u_pcb;
262 
263 	/* Invoke the appropriate handler, if necessary */
264 	if (__predict_false(data_aborts[fsr & FAULT_TYPE_MASK].func != NULL)) {
265 		if ((data_aborts[fsr & FAULT_TYPE_MASK].func)(tf, fsr, far,
266 		    l, &ksi))
267 			goto do_trapsignal;
268 		goto out;
269 	}
270 
271 	/*
272 	 * At this point, we're dealing with one of the following data aborts:
273 	 *
274 	 *  FAULT_TRANS_S  - Translation -- Section
275 	 *  FAULT_TRANS_P  - Translation -- Page
276 	 *  FAULT_DOMAIN_S - Domain -- Section
277 	 *  FAULT_DOMAIN_P - Domain -- Page
278 	 *  FAULT_PERM_S   - Permission -- Section
279 	 *  FAULT_PERM_P   - Permission -- Page
280 	 *
281 	 * These are the main virtual memory-related faults signalled by
282 	 * the MMU.
283 	 */
284 
285 	/* fusubailout is used by [fs]uswintr to avoid page faulting */
286 	if (__predict_false(pcb->pcb_onfault == fusubailout)) {
287 		tf->tf_r0 = EFAULT;
288 		tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault;
289 		return;
290 	}
291 
292 	if (user)
293 		l->l_addr->u_pcb.pcb_tf = tf;
294 
295 	/*
296 	 * Make sure the Program Counter is sane. We could fall foul of
297 	 * someone executing Thumb code, in which case the PC might not
298 	 * be word-aligned. This would cause a kernel alignment fault
299 	 * further down if we have to decode the current instruction.
300 	 */
301 #ifdef THUMB_CODE
302 	/*
303 	 * XXX: It would be nice to be able to support Thumb in the kernel
304 	 * at some point.
305 	 */
306 	if (__predict_false(!user && (tf->tf_pc & 3) != 0)) {
307 		printf("\ndata_abort_fault: Misaligned Kernel-mode "
308 		    "Program Counter\n");
309 		dab_fatal(tf, fsr, far, l, NULL);
310 	}
311 #else
312 	if (__predict_false((tf->tf_pc & 3) != 0)) {
313 		if (user) {
314 			/*
315 			 * Give the user an illegal instruction signal.
316 			 */
317 			/* Deliver a SIGILL to the process */
318 			KSI_INIT_TRAP(&ksi);
319 			ksi.ksi_signo = SIGILL;
320 			ksi.ksi_code = ILL_ILLOPC;
321 			ksi.ksi_addr = (u_int32_t *)(intptr_t) far;
322 			ksi.ksi_trap = fsr;
323 			goto do_trapsignal;
324 		}
325 
326 		/*
327 		 * The kernel never executes Thumb code.
328 		 */
329 		printf("\ndata_abort_fault: Misaligned Kernel-mode "
330 		    "Program Counter\n");
331 		dab_fatal(tf, fsr, far, l, NULL);
332 	}
333 #endif
334 
335 	/* See if the CPU state needs to be fixed up */
336 	switch (data_abort_fixup(tf, fsr, far, l)) {
337 	case ABORT_FIXUP_RETURN:
338 		return;
339 	case ABORT_FIXUP_FAILED:
340 		/* Deliver a SIGILL to the process */
341 		KSI_INIT_TRAP(&ksi);
342 		ksi.ksi_signo = SIGILL;
343 		ksi.ksi_code = ILL_ILLOPC;
344 		ksi.ksi_addr = (u_int32_t *)(intptr_t) far;
345 		ksi.ksi_trap = fsr;
346 		goto do_trapsignal;
347 	default:
348 		break;
349 	}
350 
351 	va = trunc_page((vaddr_t)far);
352 
353 	/*
354 	 * It is only a kernel address space fault iff:
355 	 *	1. user == 0  and
356 	 *	2. pcb_onfault not set or
357 	 *	3. pcb_onfault set and not LDRT/LDRBT/STRT/STRBT instruction.
358 	 */
359 	if (user == 0 && (va >= VM_MIN_KERNEL_ADDRESS ||
360 	    (va < VM_MIN_ADDRESS && vector_page == ARM_VECTORS_LOW)) &&
361 	    __predict_true((pcb->pcb_onfault == NULL ||
362 	     (ReadWord(tf->tf_pc) & 0x05200000) != 0x04200000))) {
363 		map = kernel_map;
364 
365 		/* Was the fault due to the FPE/IPKDB ? */
366 		if (__predict_false((tf->tf_spsr & PSR_MODE)==PSR_UND32_MODE)) {
367 			KSI_INIT_TRAP(&ksi);
368 			ksi.ksi_signo = SIGSEGV;
369 			ksi.ksi_code = SEGV_ACCERR;
370 			ksi.ksi_addr = (u_int32_t *)(intptr_t) far;
371 			ksi.ksi_trap = fsr;
372 
373 			/*
374 			 * Force exit via userret()
375 			 * This is necessary as the FPE is an extension to
376 			 * userland that actually runs in a priveledged mode
377 			 * but uses USR mode permissions for its accesses.
378 			 */
379 			user = 1;
380 			goto do_trapsignal;
381 		}
382 	} else
383 		map = &l->l_proc->p_vmspace->vm_map;
384 
385 	/*
386 	 * We need to know whether the page should be mapped
387 	 * as R or R/W. The MMU does not give us the info as
388 	 * to whether the fault was caused by a read or a write.
389 	 *
390 	 * However, we know that a permission fault can only be
391 	 * the result of a write to a read-only location, so
392 	 * we can deal with those quickly.
393 	 *
394 	 * Otherwise we need to disassemble the instruction
395 	 * responsible to determine if it was a write.
396 	 */
397 	if (IS_PERMISSION_FAULT(fsr))
398 		ftype = VM_PROT_WRITE;
399 	else {
400 #ifdef THUMB_CODE
401 		/* Fast track the ARM case.  */
402 		if (__predict_false(tf->tf_spsr & PSR_T_bit)) {
403 			u_int insn = fusword((void *)(tf->tf_pc & ~1));
404 			u_int insn_f8 = insn & 0xf800;
405 			u_int insn_fe = insn & 0xfe00;
406 
407 			if (insn_f8 == 0x6000 || /* STR(1) */
408 			    insn_f8 == 0x7000 || /* STRB(1) */
409 			    insn_f8 == 0x8000 || /* STRH(1) */
410 			    insn_f8 == 0x9000 || /* STR(3) */
411 			    insn_f8 == 0xc000 || /* STM */
412 			    insn_fe == 0x5000 || /* STR(2) */
413 			    insn_fe == 0x5200 || /* STRH(2) */
414 			    insn_fe == 0x5400)   /* STRB(2) */
415 				ftype = VM_PROT_WRITE;
416 			else
417 				ftype = VM_PROT_READ;
418 		}
419 		else
420 #endif
421 		{
422 			u_int insn = ReadWord(tf->tf_pc);
423 
424 			if (((insn & 0x0c100000) == 0x04000000) || /* STR[B] */
425 			    ((insn & 0x0e1000b0) == 0x000000b0) || /* STR[HD]*/
426 			    ((insn & 0x0a100000) == 0x08000000))   /* STM/CDT*/
427 				ftype = VM_PROT_WRITE;
428 			else if ((insn & 0x0fb00ff0) == 0x01000090)/* SWP */
429 				ftype = VM_PROT_READ | VM_PROT_WRITE;
430 			else
431 				ftype = VM_PROT_READ;
432 		}
433 	}
434 
435 	/*
436 	 * See if the fault is as a result of ref/mod emulation,
437 	 * or domain mismatch.
438 	 */
439 #ifdef DEBUG
440 	last_fault_code = fsr;
441 #endif
442 	if (pmap_fault_fixup(map->pmap, va, ftype, user)) {
443 		UVMHIST_LOG(maphist, " <- ref/mod emul", 0, 0, 0, 0);
444 		goto out;
445 	}
446 
447 	if (__predict_false(curcpu()->ci_intr_depth > 0)) {
448 		if (pcb->pcb_onfault) {
449 			tf->tf_r0 = EINVAL;
450 			tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault;
451 			return;
452 		}
453 		printf("\nNon-emulated page fault with intr_depth > 0\n");
454 		dab_fatal(tf, fsr, far, l, NULL);
455 	}
456 
457 	onfault = pcb->pcb_onfault;
458 	pcb->pcb_onfault = NULL;
459 	error = uvm_fault(map, va, ftype);
460 	pcb->pcb_onfault = onfault;
461 
462 	if (__predict_true(error == 0)) {
463 		if (user)
464 			uvm_grow(l->l_proc, va); /* Record any stack growth */
465 		UVMHIST_LOG(maphist, " <- uvm", 0, 0, 0, 0);
466 		goto out;
467 	}
468 
469 	if (user == 0) {
470 		if (pcb->pcb_onfault) {
471 			tf->tf_r0 = error;
472 			tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault;
473 			return;
474 		}
475 
476 		printf("\nuvm_fault(%p, %lx, %x) -> %x\n", map, va, ftype,
477 		    error);
478 		dab_fatal(tf, fsr, far, l, NULL);
479 	}
480 
481 	KSI_INIT_TRAP(&ksi);
482 
483 	if (error == ENOMEM) {
484 		printf("UVM: pid %d (%s), uid %d killed: "
485 		    "out of swap\n", l->l_proc->p_pid, l->l_proc->p_comm,
486 		    l->l_cred ? kauth_cred_geteuid(l->l_cred) : -1);
487 		ksi.ksi_signo = SIGKILL;
488 	} else
489 		ksi.ksi_signo = SIGSEGV;
490 
491 	ksi.ksi_code = (error == EACCES) ? SEGV_ACCERR : SEGV_MAPERR;
492 	ksi.ksi_addr = (u_int32_t *)(intptr_t) far;
493 	ksi.ksi_trap = fsr;
494 	UVMHIST_LOG(maphist, " <- erorr (%d)", error, 0, 0, 0);
495 
496 do_trapsignal:
497 	call_trapsignal(l, &ksi);
498 out:
499 	/* If returning to user mode, make sure to invoke userret() */
500 	if (user)
501 		userret(l);
502 }
503 
504 /*
505  * dab_fatal() handles the following data aborts:
506  *
507  *  FAULT_WRTBUF_0 - Vector Exception
508  *  FAULT_WRTBUF_1 - Terminal Exception
509  *
510  * We should never see these on a properly functioning system.
511  *
512  * This function is also called by the other handlers if they
513  * detect a fatal problem.
514  *
515  * Note: If 'l' is NULL, we assume we're dealing with a prefetch abort.
516  */
517 static int
518 dab_fatal(trapframe_t *tf, u_int fsr, u_int far, struct lwp *l, ksiginfo_t *ksi)
519 {
520 	const char *mode;
521 
522 	mode = TRAP_USERMODE(tf) ? "user" : "kernel";
523 
524 	if (l != NULL) {
525 		printf("Fatal %s mode data abort: '%s'\n", mode,
526 		    data_aborts[fsr & FAULT_TYPE_MASK].desc);
527 		printf("trapframe: %p\nFSR=%08x, FAR=", tf, fsr);
528 		if ((fsr & FAULT_IMPRECISE) == 0)
529 			printf("%08x, ", far);
530 		else
531 			printf("Invalid,  ");
532 		printf("spsr=%08x\n", tf->tf_spsr);
533 	} else {
534 		printf("Fatal %s mode prefetch abort at 0x%08x\n",
535 		    mode, tf->tf_pc);
536 		printf("trapframe: %p, spsr=%08x\n", tf, tf->tf_spsr);
537 	}
538 
539 	printf("r0 =%08x, r1 =%08x, r2 =%08x, r3 =%08x\n",
540 	    tf->tf_r0, tf->tf_r1, tf->tf_r2, tf->tf_r3);
541 	printf("r4 =%08x, r5 =%08x, r6 =%08x, r7 =%08x\n",
542 	    tf->tf_r4, tf->tf_r5, tf->tf_r6, tf->tf_r7);
543 	printf("r8 =%08x, r9 =%08x, r10=%08x, r11=%08x\n",
544 	    tf->tf_r8, tf->tf_r9, tf->tf_r10, tf->tf_r11);
545 	printf("r12=%08x, ", tf->tf_r12);
546 
547 	if (TRAP_USERMODE(tf))
548 		printf("usp=%08x, ulr=%08x",
549 		    tf->tf_usr_sp, tf->tf_usr_lr);
550 	else
551 		printf("ssp=%08x, slr=%08x",
552 		    tf->tf_svc_sp, tf->tf_svc_lr);
553 	printf(", pc =%08x\n\n", tf->tf_pc);
554 
555 #if defined(DDB) || defined(KGDB)
556 	kdb_trap(T_FAULT, tf);
557 #endif
558 	panic("Fatal abort");
559 	/*NOTREACHED*/
560 }
561 
562 /*
563  * dab_align() handles the following data aborts:
564  *
565  *  FAULT_ALIGN_0 - Alignment fault
566  *  FAULT_ALIGN_0 - Alignment fault
567  *
568  * These faults are fatal if they happen in kernel mode. Otherwise, we
569  * deliver a bus error to the process.
570  */
571 static int
572 dab_align(trapframe_t *tf, u_int fsr, u_int far, struct lwp *l, ksiginfo_t *ksi)
573 {
574 
575 	/* Alignment faults are always fatal if they occur in kernel mode */
576 	if (!TRAP_USERMODE(tf))
577 		dab_fatal(tf, fsr, far, l, NULL);
578 
579 	/* pcb_onfault *must* be NULL at this point */
580 	KDASSERT(l->l_addr->u_pcb.pcb_onfault == NULL);
581 
582 	/* See if the CPU state needs to be fixed up */
583 	(void) data_abort_fixup(tf, fsr, far, l);
584 
585 	/* Deliver a bus error signal to the process */
586 	KSI_INIT_TRAP(ksi);
587 	ksi->ksi_signo = SIGBUS;
588 	ksi->ksi_code = BUS_ADRALN;
589 	ksi->ksi_addr = (u_int32_t *)(intptr_t)far;
590 	ksi->ksi_trap = fsr;
591 
592 	l->l_addr->u_pcb.pcb_tf = tf;
593 
594 	return (1);
595 }
596 
597 /*
598  * dab_buserr() handles the following data aborts:
599  *
600  *  FAULT_BUSERR_0 - External Abort on Linefetch -- Section
601  *  FAULT_BUSERR_1 - External Abort on Linefetch -- Page
602  *  FAULT_BUSERR_2 - External Abort on Non-linefetch -- Section
603  *  FAULT_BUSERR_3 - External Abort on Non-linefetch -- Page
604  *  FAULT_BUSTRNL1 - External abort on Translation -- Level 1
605  *  FAULT_BUSTRNL2 - External abort on Translation -- Level 2
606  *
607  * If pcb_onfault is set, flag the fault and return to the handler.
608  * If the fault occurred in user mode, give the process a SIGBUS.
609  *
610  * Note: On XScale, FAULT_BUSERR_0, FAULT_BUSERR_1, and FAULT_BUSERR_2
611  * can be flagged as imprecise in the FSR. This causes a real headache
612  * since some of the machine state is lost. In this case, tf->tf_pc
613  * may not actually point to the offending instruction. In fact, if
614  * we've taken a double abort fault, it generally points somewhere near
615  * the top of "data_abort_entry" in exception.S.
616  *
617  * In all other cases, these data aborts are considered fatal.
618  */
619 static int
620 dab_buserr(trapframe_t *tf, u_int fsr, u_int far, struct lwp *l,
621     ksiginfo_t *ksi)
622 {
623 	struct pcb *pcb = &l->l_addr->u_pcb;
624 
625 #ifdef __XSCALE__
626 	if ((fsr & FAULT_IMPRECISE) != 0 &&
627 	    (tf->tf_spsr & PSR_MODE) == PSR_ABT32_MODE) {
628 		/*
629 		 * Oops, an imprecise, double abort fault. We've lost the
630 		 * r14_abt/spsr_abt values corresponding to the original
631 		 * abort, and the spsr saved in the trapframe indicates
632 		 * ABT mode.
633 		 */
634 		tf->tf_spsr &= ~PSR_MODE;
635 
636 		/*
637 		 * We use a simple heuristic to determine if the double abort
638 		 * happened as a result of a kernel or user mode access.
639 		 * If the current trapframe is at the top of the kernel stack,
640 		 * the fault _must_ have come from user mode.
641 		 */
642 		if (tf != ((trapframe_t *)pcb->pcb_un.un_32.pcb32_sp) - 1) {
643 			/*
644 			 * Kernel mode. We're either about to die a
645 			 * spectacular death, or pcb_onfault will come
646 			 * to our rescue. Either way, the current value
647 			 * of tf->tf_pc is irrelevant.
648 			 */
649 			tf->tf_spsr |= PSR_SVC32_MODE;
650 			if (pcb->pcb_onfault == NULL)
651 				printf("\nKernel mode double abort!\n");
652 		} else {
653 			/*
654 			 * User mode. We've lost the program counter at the
655 			 * time of the fault (not that it was accurate anyway;
656 			 * it's not called an imprecise fault for nothing).
657 			 * About all we can do is copy r14_usr to tf_pc and
658 			 * hope for the best. The process is about to get a
659 			 * SIGBUS, so it's probably history anyway.
660 			 */
661 			tf->tf_spsr |= PSR_USR32_MODE;
662 			tf->tf_pc = tf->tf_usr_lr;
663 #ifdef THUMB_CODE
664 			tf->tf_spsr &= ~PSR_T_bit;
665 			if (tf->tf_usr_lr & 1)
666 				tf->tf_spsr |= PSR_T_bit;
667 #endif
668 		}
669 	}
670 
671 	/* FAR is invalid for imprecise exceptions */
672 	if ((fsr & FAULT_IMPRECISE) != 0)
673 		far = 0;
674 #endif /* __XSCALE__ */
675 
676 	if (pcb->pcb_onfault) {
677 		KDASSERT(TRAP_USERMODE(tf) == 0);
678 		tf->tf_r0 = EFAULT;
679 		tf->tf_pc = (register_t)(intptr_t) pcb->pcb_onfault;
680 		return (0);
681 	}
682 
683 	/* See if the CPU state needs to be fixed up */
684 	(void) data_abort_fixup(tf, fsr, far, l);
685 
686 	/*
687 	 * At this point, if the fault happened in kernel mode, we're toast
688 	 */
689 	if (!TRAP_USERMODE(tf))
690 		dab_fatal(tf, fsr, far, l, NULL);
691 
692 	/* Deliver a bus error signal to the process */
693 	KSI_INIT_TRAP(ksi);
694 	ksi->ksi_signo = SIGBUS;
695 	ksi->ksi_code = BUS_ADRERR;
696 	ksi->ksi_addr = (u_int32_t *)(intptr_t)far;
697 	ksi->ksi_trap = fsr;
698 
699 	l->l_addr->u_pcb.pcb_tf = tf;
700 
701 	return (1);
702 }
703 
704 static inline int
705 prefetch_abort_fixup(trapframe_t *tf)
706 {
707 #ifdef CPU_ABORT_FIXUP_REQUIRED
708 	int error;
709 
710 	/* Call the CPU specific prefetch abort fixup routine */
711 	error = cpu_prefetchabt_fixup(tf);
712 	if (__predict_true(error != ABORT_FIXUP_FAILED))
713 		return (error);
714 
715 	/*
716 	 * Oops, couldn't fix up the instruction
717 	 */
718 	printf(
719 	    "prefetch_abort_fixup: fixup for %s mode prefetch abort failed.\n",
720 	    TRAP_USERMODE(tf) ? "user" : "kernel");
721 #ifdef THUMB_CODE
722 	if (tf->tf_spsr & PSR_T_bit) {
723 		printf("pc = 0x%08x, opcode 0x%04x, 0x%04x, insn = ",
724 		    tf->tf_pc, *((u_int16 *)(tf->tf_pc & ~1),
725 		    *((u_int16 *)((tf->tf_pc + 2) & ~1));
726 	}
727 	else
728 #endif
729 	{
730 		printf("pc = 0x%08x, opcode 0x%08x, insn = ", tf->tf_pc,
731 		    *((u_int *)tf->tf_pc));
732 	}
733 	disassemble(tf->tf_pc);
734 
735 	/* Die now if this happened in kernel mode */
736 	if (!TRAP_USERMODE(tf))
737 		dab_fatal(tf, 0, tf->tf_pc, NULL, NULL);
738 
739 	return (error);
740 #else
741 	return (ABORT_FIXUP_OK);
742 #endif /* CPU_ABORT_FIXUP_REQUIRED */
743 }
744 
745 /*
746  * void prefetch_abort_handler(trapframe_t *tf)
747  *
748  * Abort handler called when instruction execution occurs at
749  * a non existent or restricted (access permissions) memory page.
750  * If the address is invalid and we were in SVC mode then panic as
751  * the kernel should never prefetch abort.
752  * If the address is invalid and the page is mapped then the user process
753  * does no have read permission so send it a signal.
754  * Otherwise fault the page in and try again.
755  */
756 void
757 prefetch_abort_handler(trapframe_t *tf)
758 {
759 	struct lwp *l;
760 	struct vm_map *map;
761 	vaddr_t fault_pc, va;
762 	ksiginfo_t ksi;
763 	int error, user;
764 
765 	UVMHIST_FUNC("prefetch_abort_handler"); UVMHIST_CALLED(maphist);
766 
767 	/* Update vmmeter statistics */
768 	uvmexp.traps++;
769 
770 	l = curlwp;
771 
772 	if ((user = TRAP_USERMODE(tf)) != 0)
773 		LWP_CACHE_CREDS(l, l->l_proc);
774 
775 	/*
776 	 * Enable IRQ's (disabled by the abort) This always comes
777 	 * from user mode so we know interrupts were not disabled.
778 	 * But we check anyway.
779 	 */
780 	if (__predict_true((tf->tf_spsr & I32_bit) == 0))
781 		enable_interrupts(I32_bit);
782 
783 	/* See if the CPU state needs to be fixed up */
784 	switch (prefetch_abort_fixup(tf)) {
785 	case ABORT_FIXUP_RETURN:
786 		return;
787 	case ABORT_FIXUP_FAILED:
788 		/* Deliver a SIGILL to the process */
789 		KSI_INIT_TRAP(&ksi);
790 		ksi.ksi_signo = SIGILL;
791 		ksi.ksi_code = ILL_ILLOPC;
792 		ksi.ksi_addr = (u_int32_t *)(intptr_t) tf->tf_pc;
793 		l->l_addr->u_pcb.pcb_tf = tf;
794 		goto do_trapsignal;
795 	default:
796 		break;
797 	}
798 
799 	/* Prefetch aborts cannot happen in kernel mode */
800 	if (__predict_false(!user))
801 		dab_fatal(tf, 0, tf->tf_pc, NULL, NULL);
802 
803 	/* Get fault address */
804 	fault_pc = tf->tf_pc;
805 	l = curlwp;
806 	l->l_addr->u_pcb.pcb_tf = tf;
807 	UVMHIST_LOG(maphist, " (pc=0x%x, l=0x%x, tf=0x%x)", fault_pc, l, tf,
808 	    0);
809 
810 	/* Ok validate the address, can only execute in USER space */
811 	if (__predict_false(fault_pc >= VM_MAXUSER_ADDRESS ||
812 	    (fault_pc < VM_MIN_ADDRESS && vector_page == ARM_VECTORS_LOW))) {
813 		KSI_INIT_TRAP(&ksi);
814 		ksi.ksi_signo = SIGSEGV;
815 		ksi.ksi_code = SEGV_ACCERR;
816 		ksi.ksi_addr = (u_int32_t *)(intptr_t) fault_pc;
817 		ksi.ksi_trap = fault_pc;
818 		goto do_trapsignal;
819 	}
820 
821 	map = &l->l_proc->p_vmspace->vm_map;
822 	va = trunc_page(fault_pc);
823 
824 	/*
825 	 * See if the pmap can handle this fault on its own...
826 	 */
827 #ifdef DEBUG
828 	last_fault_code = -1;
829 #endif
830 	if (pmap_fault_fixup(map->pmap, va, VM_PROT_READ, 1)) {
831 		UVMHIST_LOG (maphist, " <- emulated", 0, 0, 0, 0);
832 		goto out;
833 	}
834 
835 #ifdef DIAGNOSTIC
836 	if (__predict_false(l->l_cpu->ci_intr_depth > 0)) {
837 		printf("\nNon-emulated prefetch abort with intr_depth > 0\n");
838 		dab_fatal(tf, 0, tf->tf_pc, NULL, NULL);
839 	}
840 #endif
841 	error = uvm_fault(map, va, VM_PROT_READ);
842 
843 	if (__predict_true(error == 0)) {
844 		UVMHIST_LOG (maphist, " <- uvm", 0, 0, 0, 0);
845 		goto out;
846 	}
847 	KSI_INIT_TRAP(&ksi);
848 
849 	UVMHIST_LOG (maphist, " <- fatal (%d)", error, 0, 0, 0);
850 	if (error == ENOMEM) {
851 		printf("UVM: pid %d (%s), uid %d killed: "
852 		    "out of swap\n", l->l_proc->p_pid, l->l_proc->p_comm,
853 		    l->l_cred ? kauth_cred_geteuid(l->l_cred) : -1);
854 		ksi.ksi_signo = SIGKILL;
855 	} else
856 		ksi.ksi_signo = SIGSEGV;
857 
858 	ksi.ksi_code = SEGV_MAPERR;
859 	ksi.ksi_addr = (u_int32_t *)(intptr_t) fault_pc;
860 	ksi.ksi_trap = fault_pc;
861 
862 do_trapsignal:
863 	call_trapsignal(l, &ksi);
864 
865 out:
866 	userret(l);
867 }
868 
869 /*
870  * Tentatively read an 8, 16, or 32-bit value from 'addr'.
871  * If the read succeeds, the value is written to 'rptr' and zero is returned.
872  * Else, return EFAULT.
873  */
874 int
875 badaddr_read(void *addr, size_t size, void *rptr)
876 {
877 	extern int badaddr_read_1(const uint8_t *, uint8_t *);
878 	extern int badaddr_read_2(const uint16_t *, uint16_t *);
879 	extern int badaddr_read_4(const uint32_t *, uint32_t *);
880 	union {
881 		uint8_t v1;
882 		uint16_t v2;
883 		uint32_t v4;
884 	} u;
885 	struct pcb *curpcb_save;
886 	int rv, s;
887 
888 	cpu_drain_writebuf();
889 
890 	/*
891 	 * We might be called at interrupt time, so arrange to steal
892 	 * lwp0's PCB temporarily, if required, so that pcb_onfault
893 	 * handling works correctly.
894 	 */
895 	s = splhigh();
896 	if ((curpcb_save = curpcb) == NULL)
897 		curpcb = &lwp0.l_addr->u_pcb;
898 
899 	/* Read from the test address. */
900 	switch (size) {
901 	case sizeof(uint8_t):
902 		rv = badaddr_read_1(addr, &u.v1);
903 		if (rv == 0 && rptr)
904 			*(uint8_t *) rptr = u.v1;
905 		break;
906 
907 	case sizeof(uint16_t):
908 		rv = badaddr_read_2(addr, &u.v2);
909 		if (rv == 0 && rptr)
910 			*(uint16_t *) rptr = u.v2;
911 		break;
912 
913 	case sizeof(uint32_t):
914 		rv = badaddr_read_4(addr, &u.v4);
915 		if (rv == 0 && rptr)
916 			*(uint32_t *) rptr = u.v4;
917 		break;
918 
919 	default:
920 		curpcb = curpcb_save;
921 		panic("badaddr: invalid size (%lu)", (u_long) size);
922 	}
923 
924 	/* Restore curpcb */
925 	curpcb = curpcb_save;
926 	splx(s);
927 
928 	/* Return EFAULT if the address was invalid, else zero */
929 	return (rv);
930 }
931