1 /* $NetBSD: fault.c,v 1.26 2024/02/07 04:20:26 msaitoh Exp $ */
2
3 /*
4 * Copyright (c) 2017 Ryo Shimizu
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
20 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26 * POSSIBILITY OF SUCH DAMAGE.
27 */
28
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(0, "$NetBSD: fault.c,v 1.26 2024/02/07 04:20:26 msaitoh Exp $");
31
32 #include "opt_compat_netbsd32.h"
33 #include "opt_cpuoptions.h"
34 #include "opt_ddb.h"
35 #include "opt_uvmhist.h"
36
37 #include <sys/param.h>
38 #include <sys/kauth.h>
39 #include <sys/proc.h>
40 #include <sys/siginfo.h>
41
42 #include <uvm/uvm.h>
43
44 #include <aarch64/frame.h>
45 #include <aarch64/machdep.h>
46 #include <aarch64/armreg.h>
47 #include <aarch64/db_machdep.h>
48
49 #include <arm/cpufunc.h>
50
51 UVMHIST_DECL(pmaphist);
52
53 const char * const fault_status_code[] = {
54 [ESR_ISS_FSC_ADDRESS_SIZE_FAULT_0] = "Address Size Fault L0",
55 [ESR_ISS_FSC_ADDRESS_SIZE_FAULT_1] = "Address Size Fault L1",
56 [ESR_ISS_FSC_ADDRESS_SIZE_FAULT_2] = "Address Size Fault L2",
57 [ESR_ISS_FSC_ADDRESS_SIZE_FAULT_3] = "Address Size Fault L3",
58 [ESR_ISS_FSC_TRANSLATION_FAULT_0] = "Translation Fault L0",
59 [ESR_ISS_FSC_TRANSLATION_FAULT_1] = "Translation Fault L1",
60 [ESR_ISS_FSC_TRANSLATION_FAULT_2] = "Translation Fault L2",
61 [ESR_ISS_FSC_TRANSLATION_FAULT_3] = "Translation Fault L3",
62 [ESR_ISS_FSC_ACCESS_FAULT_0] = "Access Flag Fault L0",
63 [ESR_ISS_FSC_ACCESS_FAULT_1] = "Access Flag Fault L1",
64 [ESR_ISS_FSC_ACCESS_FAULT_2] = "Access Flag Fault L2",
65 [ESR_ISS_FSC_ACCESS_FAULT_3] = "Access Flag Fault L3",
66 [ESR_ISS_FSC_PERM_FAULT_0] = "Permission Fault L0",
67 [ESR_ISS_FSC_PERM_FAULT_1] = "Permission Fault L1",
68 [ESR_ISS_FSC_PERM_FAULT_2] = "Permission Fault L2",
69 [ESR_ISS_FSC_PERM_FAULT_3] = "Permission Fault L3",
70 [ESR_ISS_FSC_SYNC_EXTERNAL_ABORT] =
71 "Synchronous External Abort",
72 [ESR_ISS_FSC_SYNC_EXTERNAL_ABORT_TTWALK_0] =
73 "Synchronous External Abort on translation table walk L0",
74 [ESR_ISS_FSC_SYNC_EXTERNAL_ABORT_TTWALK_1] =
75 "Synchronous External Abort on translation table walk L1",
76 [ESR_ISS_FSC_SYNC_EXTERNAL_ABORT_TTWALK_2] =
77 "Synchronous External Abort on translation table walk L2",
78 [ESR_ISS_FSC_SYNC_EXTERNAL_ABORT_TTWALK_3] =
79 "Synchronous External Abort on translation table walk L3",
80 [ESR_ISS_FSC_SYNC_PARITY_ERROR] =
81 "Synchronous Parity error",
82 [ESR_ISS_FSC_SYNC_PARITY_ERROR_ON_TTWALK_0] =
83 "Synchronous Parity error on translation table walk L0",
84 [ESR_ISS_FSC_SYNC_PARITY_ERROR_ON_TTWALK_1] =
85 "Synchronous Parity error on translation table walk L1",
86 [ESR_ISS_FSC_SYNC_PARITY_ERROR_ON_TTWALK_2] =
87 "Synchronous Parity error on translation table walk L2",
88 [ESR_ISS_FSC_SYNC_PARITY_ERROR_ON_TTWALK_3] =
89 "Synchronous Parity error on translation table walk L3",
90 [ESR_ISS_FSC_ALIGNMENT_FAULT] = "Alignment Fault",
91 [ESR_ISS_FSC_TLB_CONFLICT_FAULT] = "TLB Conflict Fault",
92 [ESR_ISS_FSC_LOCKDOWN_ABORT] = "Lockdown Abort",
93 [ESR_ISS_FSC_UNSUPPORTED_EXCLUSIVE] = "Unsupported exclusive",
94 [ESR_ISS_FSC_FIRST_LEVEL_DOMAIN_FAULT] =
95 "First Level Domain Fault",
96 [ESR_ISS_FSC_SECOND_LEVEL_DOMAIN_FAULT] =
97 "Second Level Domain Fault"
98 };
99
100 static bool
is_fatal_abort(uint32_t esr)101 is_fatal_abort(uint32_t esr)
102 {
103 uint32_t fsc;
104
105 fsc = __SHIFTOUT(esr, ESR_ISS_DATAABORT_DFSC);
106
107 switch (fsc) {
108 case ESR_ISS_FSC_TRANSLATION_FAULT_0:
109 case ESR_ISS_FSC_TRANSLATION_FAULT_1:
110 case ESR_ISS_FSC_TRANSLATION_FAULT_2:
111 case ESR_ISS_FSC_TRANSLATION_FAULT_3:
112 case ESR_ISS_FSC_ACCESS_FAULT_0:
113 case ESR_ISS_FSC_ACCESS_FAULT_1:
114 case ESR_ISS_FSC_ACCESS_FAULT_2:
115 case ESR_ISS_FSC_ACCESS_FAULT_3:
116 case ESR_ISS_FSC_PERM_FAULT_0:
117 case ESR_ISS_FSC_PERM_FAULT_1:
118 case ESR_ISS_FSC_PERM_FAULT_2:
119 case ESR_ISS_FSC_PERM_FAULT_3:
120 return false;
121 }
122 return true;
123 }
124
125 /* SPSR_M is SPSR_M_EL0T or SPSR_M_USR32 ? */
126 #define IS_SPSR_USER(spsr) \
127 (((spsr) & (SPSR_M & ~SPSR_A32)) == 0)
128
129 void
data_abort_handler(struct trapframe * tf,uint32_t eclass)130 data_abort_handler(struct trapframe *tf, uint32_t eclass)
131 {
132 struct proc *p;
133 struct lwp *l;
134 struct vm_map *map;
135 struct faultbuf *fb;
136 vaddr_t va;
137 uint32_t esr, fsc, rw;
138 vm_prot_t ftype;
139 int error = EFAULT, len;
140 const bool user = IS_SPSR_USER(tf->tf_spsr) ? true : false;
141 bool is_pan_trap = false;
142
143 bool fatalabort;
144 const char *faultstr;
145 static char panicinfo[256];
146
147 UVMHIST_FUNC(__func__);
148 UVMHIST_CALLED(pmaphist);
149
150 __asm __volatile ("clrex");
151
152 l = curlwp;
153
154 esr = tf->tf_esr;
155 rw = __SHIFTOUT(esr, ESR_ISS_DATAABORT_WnR); /* 0 if IFSC */
156
157 fatalabort = is_fatal_abort(esr);
158 if (fatalabort)
159 goto do_fault;
160
161 p = l->l_proc;
162 va = trunc_page((vaddr_t)tf->tf_far);
163
164 /* eliminate address tag if ECR_EL1.TBI[01] is enabled */
165 va = aarch64_untag_address(va);
166
167 if ((VM_MIN_KERNEL_ADDRESS <= va) && (va < VM_MAX_KERNEL_ADDRESS)) {
168 map = kernel_map;
169 UVMHIST_LOG(pmaphist, "use kernel_map %p", map, 0, 0, 0);
170 } else if (VM_MIN_ADDRESS <= va && va <= VM_MAX_ADDRESS) {
171 map = &p->p_vmspace->vm_map;
172 UVMHIST_LOG(pmaphist, "use user vm_map %p (kernel_map=%p)",
173 map, kernel_map, 0, 0);
174 } else
175 goto do_fault;
176
177 if ((eclass == ESR_EC_INSN_ABT_EL0) || (eclass == ESR_EC_INSN_ABT_EL1))
178 ftype = VM_PROT_EXECUTE;
179 else if (__SHIFTOUT(esr, ESR_ISS_DATAABORT_CM))
180 ftype = VM_PROT_READ;
181 else
182 ftype = (rw == 0) ? VM_PROT_READ : VM_PROT_WRITE;
183
184 if (ftype & VM_PROT_EXECUTE) {
185 UVMHIST_LOG(pmaphist, "pagefault %016jx %016jx user=%jd EXEC",
186 tf->tf_far, va, user, 0);
187 } else {
188 UVMHIST_LOG(pmaphist, "pagefault %016lx %016lx user=%jd "
189 "write=%jd", tf->tf_far, va, user, rw);
190 }
191
192 if (__predict_false(!user && (map != kernel_map) &&
193 (tf->tf_spsr & SPSR_PAN))) {
194 /*
195 * We were in kernel mode, faulted on a user address,
196 * and had PAN enabled. This is a fatal fault.
197 */
198 is_pan_trap = true;
199 goto handle_fault;
200 }
201
202 /* reference/modified emulation */
203 #ifdef ARMV81_HAFDBS
204 if (aarch64_hafdbs_enabled == ID_AA64MMFR1_EL1_HAFDBS_NONE ||
205 (aarch64_hafdbs_enabled == ID_AA64MMFR1_EL1_HAFDBS_A &&
206 ftype == VM_PROT_WRITE))
207 #endif
208 {
209 if (pmap_fault_fixup(map->pmap, va, ftype, user)) {
210 UVMHIST_LOG(pmaphist, "fixed: va=%016llx", tf->tf_far, 0, 0, 0);
211 return;
212 }
213 }
214
215 fb = cpu_disable_onfault();
216 error = uvm_fault(map, va, ftype);
217 cpu_enable_onfault(fb);
218 if (__predict_true(error == 0)) {
219 if (user)
220 uvm_grow(p, va);
221
222 UVMHIST_LOG(pmaphist, "uvm_fault success: far=%016lx, va=%016llx",
223 tf->tf_far, va, 0, 0);
224 return;
225 }
226
227 do_fault:
228 /* faultbail path? */
229 kpreempt_disable();
230 const bool intrdepthzero = (curcpu()->ci_intr_depth == 0);
231 kpreempt_enable();
232 if (intrdepthzero) {
233 fb = cpu_disable_onfault();
234 if (fb != NULL) {
235 cpu_jump_onfault(tf, fb, error);
236 return;
237 }
238 }
239
240 handle_fault:
241 fsc = __SHIFTOUT(esr, ESR_ISS_DATAABORT_DFSC); /* also IFSC */
242 if (user) {
243 if (!fatalabort) {
244 switch (error) {
245 case ENOMEM:
246 printf("UVM: pid %d (%s), uid %d killed: "
247 "out of swap\n",
248 l->l_proc->p_pid, l->l_proc->p_comm,
249 l->l_cred ?
250 kauth_cred_geteuid(l->l_cred) : -1);
251 do_trapsignal(l, SIGKILL, 0,
252 (void *)tf->tf_far, esr);
253 break;
254 case EACCES:
255 do_trapsignal(l, SIGSEGV, SEGV_ACCERR,
256 (void *)tf->tf_far, esr);
257 break;
258 case EINVAL:
259 do_trapsignal(l, SIGBUS, BUS_ADRERR,
260 (void *)tf->tf_far, esr);
261 break;
262 default:
263 do_trapsignal(l, SIGSEGV, SEGV_MAPERR,
264 (void *)tf->tf_far, esr);
265 break;
266 }
267 } else {
268 /*
269 * fatal abort in usermode
270 */
271 switch (fsc) {
272 case ESR_ISS_FSC_TLB_CONFLICT_FAULT:
273 case ESR_ISS_FSC_LOCKDOWN_ABORT:
274 case ESR_ISS_FSC_UNSUPPORTED_EXCLUSIVE:
275 case ESR_ISS_FSC_FIRST_LEVEL_DOMAIN_FAULT:
276 case ESR_ISS_FSC_SECOND_LEVEL_DOMAIN_FAULT:
277 default:
278 do_trapsignal(l, SIGSEGV, SEGV_MAPERR,
279 (void *)tf->tf_far, esr);
280 break;
281 case ESR_ISS_FSC_ADDRESS_SIZE_FAULT_0:
282 case ESR_ISS_FSC_ADDRESS_SIZE_FAULT_1:
283 case ESR_ISS_FSC_ADDRESS_SIZE_FAULT_2:
284 case ESR_ISS_FSC_ADDRESS_SIZE_FAULT_3:
285 do_trapsignal(l, SIGBUS, BUS_ADRERR,
286 (void *)tf->tf_far, esr);
287 break;
288 case ESR_ISS_FSC_SYNC_EXTERNAL_ABORT:
289 case ESR_ISS_FSC_SYNC_EXTERNAL_ABORT_TTWALK_0:
290 case ESR_ISS_FSC_SYNC_EXTERNAL_ABORT_TTWALK_1:
291 case ESR_ISS_FSC_SYNC_EXTERNAL_ABORT_TTWALK_2:
292 case ESR_ISS_FSC_SYNC_EXTERNAL_ABORT_TTWALK_3:
293 case ESR_ISS_FSC_SYNC_PARITY_ERROR:
294 case ESR_ISS_FSC_SYNC_PARITY_ERROR_ON_TTWALK_0:
295 case ESR_ISS_FSC_SYNC_PARITY_ERROR_ON_TTWALK_1:
296 case ESR_ISS_FSC_SYNC_PARITY_ERROR_ON_TTWALK_2:
297 case ESR_ISS_FSC_SYNC_PARITY_ERROR_ON_TTWALK_3:
298 do_trapsignal(l, SIGBUS, BUS_OBJERR,
299 (void *)tf->tf_far, esr);
300 break;
301 case ESR_ISS_FSC_ALIGNMENT_FAULT:
302 do_trapsignal(l, SIGBUS, BUS_ADRALN,
303 (void *)tf->tf_far, esr);
304 break;
305 }
306 }
307
308 #undef DEBUG_DUMP_ON_USERFAULT /* DEBUG */
309 #undef DEBUG_DDB_ON_USERFAULT /* DEBUG */
310
311 #if defined(DEBUG_DUMP_ON_USERFAULT) || \
312 (defined(DDB) && defined(DEBUG_DDB_ON_USERFAULT))
313 __nothing;
314 #else
315 return;
316 #endif
317 }
318
319 /*
320 * fatal abort. analyze fault status code to show by panic()
321 */
322 len = snprintf(panicinfo, sizeof(panicinfo), "Trap: %s:",
323 eclass_trapname(eclass));
324
325 if ((fsc >= __arraycount(fault_status_code)) ||
326 ((faultstr = fault_status_code[fsc]) == NULL))
327 len += snprintf(panicinfo + len, sizeof(panicinfo) - len,
328 " unknown fault status 0x%x ", fsc);
329 else
330 len += snprintf(panicinfo + len, sizeof(panicinfo) - len,
331 " %s", faultstr);
332
333 if ((__SHIFTOUT(esr, ESR_EC) == ESR_EC_DATA_ABT_EL1) ||
334 (__SHIFTOUT(esr, ESR_EC) == ESR_EC_DATA_ABT_EL0))
335 len += snprintf(panicinfo + len, sizeof(panicinfo) - len,
336 " with %s access", (rw == 0) ? "read" : "write");
337
338 len += snprintf(panicinfo + len, sizeof(panicinfo) - len,
339 " for %016"PRIxREGISTER, tf->tf_far);
340
341 if (__SHIFTOUT(esr, ESR_ISS_DATAABORT_EA) != 0)
342 len += snprintf(panicinfo + len, sizeof(panicinfo) - len,
343 ", External abort");
344
345 if (__SHIFTOUT(esr, ESR_ISS_DATAABORT_S1PTW) != 0)
346 len += snprintf(panicinfo + len, sizeof(panicinfo) - len,
347 ", State 2 Fault");
348
349 if (is_pan_trap)
350 len += snprintf(panicinfo + len, sizeof(panicinfo) - len,
351 ", PAN Set");
352
353 len += snprintf(panicinfo + len, sizeof(panicinfo) - len,
354 ": pc %016"PRIxREGISTER, tf->tf_pc);
355
356 if (tf->tf_pc == tf->tf_far) { /* XXX? */
357 /* fault address is pc. the causal instruction cannot be read */
358 len += snprintf(panicinfo + len, sizeof(panicinfo) - len,
359 ": opcode unknown");
360 }
361 #ifdef DDB
362 else {
363 /* ...and disassemble the instruction */
364 len += snprintf(panicinfo + len, sizeof(panicinfo) - len,
365 ": %s", strdisasm(tf->tf_pc, tf->tf_spsr));
366 }
367 #endif
368
369 if (user) {
370 #if defined(DEBUG_DDB_ON_USERFAULT) && defined(DDB)
371 printf("%s\n", panicinfo);
372 Debugger();
373 #elif defined(DEBUG_DUMP_ON_USERFAULT)
374 printf("%s\n", panicinfo);
375 dump_trapframe(tf, printf);
376 #endif
377 }
378
379 if (!user)
380 panic("%s\n", panicinfo);
381 }
382