1 /* $NetBSD: trap.c,v 1.199 2024/12/28 13:48:07 martin Exp $ */ 2 3 /* 4 * Copyright (c) 1996-2002 Eduardo Horvath. All rights reserved. 5 * Copyright (c) 1996 6 * The President and Fellows of Harvard College. All rights reserved. 7 * Copyright (c) 1992, 1993 8 * The Regents of the University of California. All rights reserved. 9 * 10 * This software was developed by the Computer Systems Engineering group 11 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and 12 * contributed to Berkeley. 13 * 14 * All advertising materials mentioning features or use of this software 15 * must display the following acknowledgement: 16 * This product includes software developed by the University of 17 * California, Lawrence Berkeley Laboratory. 18 * This product includes software developed by Harvard University. 19 * 20 * Redistribution and use in source and binary forms, with or without 21 * modification, are permitted provided that the following conditions 22 * are met: 23 * 1. Redistributions of source code must retain the above copyright 24 * notice, this list of conditions and the following disclaimer. 25 * 2. Redistributions in binary form must reproduce the above copyright 26 * notice, this list of conditions and the following disclaimer in the 27 * documentation and/or other materials provided with the distribution. 28 * 3. All advertising materials mentioning features or use of this software 29 * must display the following acknowledgement: 30 * This product includes software developed by the University of 31 * California, Berkeley and its contributors. 32 * This product includes software developed by Harvard University. 33 * 4. Neither the name of the University nor the names of its contributors 34 * may be used to endorse or promote products derived from this software 35 * without specific prior written permission. 36 * 37 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 38 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 39 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 40 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 41 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 42 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 43 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 44 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 45 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 46 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 47 * SUCH DAMAGE. 48 * 49 * @(#)trap.c 8.4 (Berkeley) 9/23/93 50 */ 51 52 #include <sys/cdefs.h> 53 __KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.199 2024/12/28 13:48:07 martin Exp $"); 54 55 #include "opt_ddb.h" 56 #include "opt_multiprocessor.h" 57 #include "opt_compat_netbsd32.h" 58 59 #include <sys/param.h> 60 #include <sys/systm.h> 61 #include <sys/pool.h> 62 #include <sys/proc.h> 63 #include <sys/ras.h> 64 #include <sys/kernel.h> 65 #include <sys/resource.h> 66 #include <sys/signal.h> 67 #include <sys/wait.h> 68 #include <sys/syscall.h> 69 #include <sys/syslog.h> 70 #include <sys/kauth.h> 71 72 #include <uvm/uvm_extern.h> 73 74 #include <machine/cpu.h> 75 #include <machine/ctlreg.h> 76 #include <machine/trap.h> 77 #include <machine/instr.h> 78 #include <machine/pcb.h> 79 #include <machine/pmap.h> 80 #include <machine/userret.h> 81 82 #ifdef DDB 83 #include <machine/db_machdep.h> 84 #else 85 #include <machine/frame.h> 86 #endif 87 88 #include <sparc64/sparc64/cache.h> 89 90 #include <sparc/fpu/fpu_extern.h> 91 92 #ifndef offsetof 93 #define offsetof(s, f) ((size_t)&((s *)0)->f) 94 #endif 95 96 #ifdef TRAPSTATS 97 /* trapstats */ 98 int protfix = 0; 99 int udmiss = 0; /* Number of normal/nucleus data/text miss/protection faults */ 100 int udhit = 0; 101 int udprot = 0; 102 int utmiss = 0; 103 int kdmiss = 0; 104 int kdhit = 0; 105 int kdprot = 0; 106 int ktmiss = 0; 107 int iveccnt = 0; /* number if normal/nucleus interrupt/interrupt vector faults */ 108 int uintrcnt = 0; 109 int kiveccnt = 0; 110 int kintrcnt = 0; 111 int intristk = 0; /* interrupts when already on intrstack */ 112 int intrpoll = 0; /* interrupts not using vector lists */ 113 int wfill = 0; 114 int kwfill = 0; 115 int wspill = 0; 116 int wspillskip = 0; 117 int rftucnt = 0; 118 int rftuld = 0; 119 int rftudone = 0; 120 int rftkcnt[5] = { 0, 0, 0, 0, 0 }; 121 #endif 122 123 #ifdef DEBUG 124 #define RW_64 0x1 125 #define RW_ERR 0x2 126 #define RW_FOLLOW 0x4 127 int rwindow_debug = RW_ERR; 128 #define TDB_ADDFLT 0x1 129 #define TDB_TXTFLT 0x2 130 #define TDB_TRAP 0x4 131 #define TDB_SYSCALL 0x8 132 #define TDB_FOLLOW 0x10 133 #define TDB_FRAME 0x20 134 #define TDB_NSAVED 0x40 135 #define TDB_TL 0x80 136 #define TDB_STOPSIG 0x100 137 #define TDB_STOPCALL 0x200 138 #define TDB_STOPCPIO 0x400 139 #define TDB_SYSTOP 0x800 140 int trapdebug = 0/*|TDB_SYSCALL|TDB_STOPSIG|TDB_STOPCPIO|TDB_ADDFLT|TDB_FOLLOW*/; 141 /* #define inline */ 142 #endif 143 144 #ifdef DDB 145 #if 1 146 #define DEBUGGER(t,f) do { kdb_trap(t,f); } while (0) 147 #else 148 #define DEBUGGER(t,f) Debugger() 149 #endif 150 #else 151 #define DEBUGGER(t,f) 152 #define Debugger() 153 #endif 154 155 struct evcnt ecc_corrected = 156 EVCNT_INITIALIZER(EVCNT_TYPE_MISC,0,"ECC","corrected"); 157 EVCNT_ATTACH_STATIC(ecc_corrected); 158 159 /* 160 * Initial FPU state is all registers == all 1s, everything else == all 0s. 161 * This makes every floating point register a signalling NaN, with sign bit 162 * set, no matter how it is interpreted. Appendix N of the Sparc V8 document 163 * seems to imply that we should do this, and it does make sense. 164 */ 165 const struct fpstate64 initfpstate __aligned(SPARC64_BLOCK_SIZE) = { 166 .fs_regs = 167 { ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, 168 ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, 169 ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, 170 ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0, ~0 }, 171 .fs_qsize = 0 172 }; 173 174 /* 175 * There are more than 100 trap types, but most are unused. 176 * 177 * Trap type 0 is taken over as an `Asynchronous System Trap'. 178 * This is left-over Vax emulation crap that should be fixed. 179 * 180 * Traps not supported on the spitfire are marked with `*', 181 * and additions are marked with `+' 182 */ 183 static const char T[] = "*trap"; 184 const char *trap_type[] = { 185 /* non-user vectors */ 186 "ast", /* 0 */ 187 "power on reset", /* 1 */ 188 "watchdog reset", /* 2 */ 189 "externally initiated reset",/*3 */ 190 "software initiated reset",/* 4 */ 191 "RED state exception", /* 5 */ 192 T, T, /* 6..7 */ 193 "instruction access exception", /* 8 */ 194 "*instruction MMU miss",/* 9 */ 195 "instruction access error",/* 0a */ 196 T, T, T, T, T, /* 0b..0f */ 197 "illegal instruction", /* 10 */ 198 "privileged opcode", /* 11 */ 199 "*unimplemented LDD", /* 12 */ 200 "*unimplemented STD", /* 13 */ 201 T, T, T, T, /* 14..17 */ 202 T, T, T, T, T, T, T, T, /* 18..1f */ 203 "fp disabled", /* 20 */ 204 "fp exception ieee 754",/* 21 */ 205 "fp exception other", /* 22 */ 206 "tag overflow", /* 23 */ 207 "clean window", /* 24 */ 208 T, T, T, /* 25..27 -- trap continues */ 209 "division by zero", /* 28 */ 210 "*internal processor error",/* 29 */ 211 T, T, T, T, T, T, /* 2a..2f */ 212 "data access exception",/* 30 */ 213 "*data access MMU miss",/* 31 */ 214 "data access error", /* 32 */ 215 "*data access protection",/* 33 */ 216 "mem address not aligned", /* 34 */ 217 "LDDF mem address not aligned",/* 35 */ 218 "STDF mem address not aligned",/* 36 */ 219 "privileged action", /* 37 */ 220 "LDQF mem address not aligned",/* 38 */ 221 "STQF mem address not aligned",/* 39 */ 222 T, T, T, T, T, T, /* 3a..3f */ 223 "*async data error", /* 40 */ 224 "level 1 int", /* 41 */ 225 "level 2 int", /* 42 */ 226 "level 3 int", /* 43 */ 227 "level 4 int", /* 44 */ 228 "level 5 int", /* 45 */ 229 "level 6 int", /* 46 */ 230 "level 7 int", /* 47 */ 231 "level 8 int", /* 48 */ 232 "level 9 int", /* 49 */ 233 "level 10 int", /* 4a */ 234 "level 11 int", /* 4b */ 235 "level 12 int", /* 4c */ 236 "level 13 int", /* 4d */ 237 "level 14 int", /* 4e */ 238 "level 15 int", /* 4f */ 239 T, T, T, T, T, T, T, T, /* 50..57 */ 240 T, T, T, T, T, T, T, T, /* 58..5f */ 241 "+interrupt vector", /* 60 */ 242 "+PA_watchpoint", /* 61 */ 243 "+VA_watchpoint", /* 62 */ 244 "+corrected ECC error", /* 63 */ 245 "+fast instruction access MMU miss",/* 64 */ 246 T, T, T, /* 65..67 -- trap continues */ 247 "+fast data access MMU miss",/* 68 */ 248 T, T, T, /* 69..6b -- trap continues */ 249 "+fast data access protection",/* 6c */ 250 T, T, T, /* 6d..6f -- trap continues */ 251 "+fast ECC error", /* 70 */ 252 T, T, T, T, T, T, T, /* 71..77 */ 253 T, T, T, T, T, T, T, T, /* 78..7f */ 254 "spill 0 normal", /* 80 */ 255 T, T, T, /* 81..83 -- trap continues */ 256 "spill 1 normal", /* 84 */ 257 T, T, T, /* 85..87 -- trap continues */ 258 "spill 2 normal", /* 88 */ 259 T, T, T, /* 89..8b -- trap continues */ 260 "spill 3 normal", /* 8c */ 261 T, T, T, /* 8d..8f -- trap continues */ 262 "spill 4 normal", /* 90 */ 263 T, T, T, /* 91..93 -- trap continues */ 264 "spill 5 normal", /* 94 */ 265 T, T, T, /* 95..97 -- trap continues */ 266 "spill 6 normal", /* 98 */ 267 T, T, T, /* 99..9b -- trap continues */ 268 "spill 7 normal", /* 9c */ 269 T, T, T, /* 9c..9f -- trap continues */ 270 "spill 0 other", /* a0 */ 271 T, T, T, /* a1..a3 -- trap continues */ 272 "spill 1 other", /* a4 */ 273 T, T, T, /* a5..a7 -- trap continues */ 274 "spill 2 other", /* a8 */ 275 T, T, T, /* a9..ab -- trap continues */ 276 "spill 3 other", /* ac */ 277 T, T, T, /* ad..af -- trap continues */ 278 "spill 4 other", /* b0 */ 279 T, T, T, /* b1..b3 -- trap continues */ 280 "spill 5 other", /* b4 */ 281 T, T, T, /* b5..b7 -- trap continues */ 282 "spill 6 other", /* b8 */ 283 T, T, T, /* b9..bb -- trap continues */ 284 "spill 7 other", /* bc */ 285 T, T, T, /* bc..bf -- trap continues */ 286 "fill 0 normal", /* c0 */ 287 T, T, T, /* c1..c3 -- trap continues */ 288 "fill 1 normal", /* c4 */ 289 T, T, T, /* c5..c7 -- trap continues */ 290 "fill 2 normal", /* c8 */ 291 T, T, T, /* c9..cb -- trap continues */ 292 "fill 3 normal", /* cc */ 293 T, T, T, /* cd..cf -- trap continues */ 294 "fill 4 normal", /* d0 */ 295 T, T, T, /* d1..d3 -- trap continues */ 296 "fill 5 normal", /* d4 */ 297 T, T, T, /* d5..d7 -- trap continues */ 298 "fill 6 normal", /* d8 */ 299 T, T, T, /* d9..db -- trap continues */ 300 "fill 7 normal", /* dc */ 301 T, T, T, /* dc..df -- trap continues */ 302 "fill 0 other", /* e0 */ 303 T, T, T, /* e1..e3 -- trap continues */ 304 "fill 1 other", /* e4 */ 305 T, T, T, /* e5..e7 -- trap continues */ 306 "fill 2 other", /* e8 */ 307 T, T, T, /* e9..eb -- trap continues */ 308 "fill 3 other", /* ec */ 309 T, T, T, /* ed..ef -- trap continues */ 310 "fill 4 other", /* f0 */ 311 T, T, T, /* f1..f3 -- trap continues */ 312 "fill 5 other", /* f4 */ 313 T, T, T, /* f5..f7 -- trap continues */ 314 "fill 6 other", /* f8 */ 315 T, T, T, /* f9..fb -- trap continues */ 316 "fill 7 other", /* fc */ 317 T, T, T, /* fc..ff -- trap continues */ 318 319 /* user (software trap) vectors */ 320 "syscall", /* 100 */ 321 "breakpoint", /* 101 */ 322 "zero divide", /* 102 */ 323 "flush windows", /* 103 */ 324 "clean windows", /* 104 */ 325 "range check", /* 105 */ 326 "fix align", /* 106 */ 327 "integer overflow", /* 107 */ 328 "svr4 syscall", /* 108 */ 329 "4.4 syscall", /* 109 */ 330 "kgdb exec", /* 10a */ 331 T, T, T, T, T, /* 10b..10f */ 332 T, T, T, T, T, T, T, T, /* 11a..117 */ 333 T, T, T, T, T, T, T, T, /* 118..11f */ 334 "svr4 getcc", /* 120 */ 335 "svr4 setcc", /* 121 */ 336 "svr4 getpsr", /* 122 */ 337 "svr4 setpsr", /* 123 */ 338 "svr4 gethrtime", /* 124 */ 339 "svr4 gethrvtime", /* 125 */ 340 T, /* 126 */ 341 "svr4 gethrestime", /* 127 */ 342 T, T, T, T, T, T, T, T, /* 128..12f */ 343 T, T, /* 130..131 */ 344 "get condition codes", /* 132 */ 345 "set condition codes", /* 133 */ 346 T, T, T, T, /* 134..137 */ 347 T, T, T, T, T, T, T, T, /* 138..13f */ 348 T, T, T, T, T, T, T, T, /* 140..147 */ 349 T, T, T, T, T, T, T, T, /* 148..14f */ 350 T, T, T, T, T, T, T, T, /* 150..157 */ 351 T, T, T, T, T, T, T, T, /* 158..15f */ 352 T, T, T, T, /* 160..163 */ 353 "SVID syscall64", /* 164 */ 354 "SPARC Intl syscall64", /* 165 */ 355 "OS vendor spec syscall", /* 166 */ 356 "HW OEM syscall", /* 167 */ 357 "ret from deferred trap", /* 168 */ 358 }; 359 360 #define N_TRAP_TYPES (sizeof trap_type / sizeof *trap_type) 361 362 void trap(struct trapframe64 *, unsigned int, vaddr_t, long); 363 void data_access_fault(struct trapframe64 *, unsigned int, vaddr_t, vaddr_t, 364 vaddr_t, u_long); 365 void data_access_error(struct trapframe64 *, unsigned int, vaddr_t, u_long, 366 vaddr_t, u_long); 367 void text_access_fault(struct trapframe64 *tf, unsigned int type, vaddr_t pc, 368 u_long sfsr); 369 void text_access_error(struct trapframe64 *, unsigned int, vaddr_t, u_long, 370 vaddr_t, u_long); 371 void ecc_corrected_error(unsigned int type, vaddr_t pc); 372 373 #ifdef DEBUG 374 void print_trapframe(struct trapframe64 *); 375 376 void 377 print_trapframe(struct trapframe64 *tf) 378 { 379 380 printf("Trapframe %p:\ttstate: %lx\tpc: %lx\tnpc: %lx\n", 381 tf, (u_long)tf->tf_tstate, (u_long)tf->tf_pc, (u_long)tf->tf_npc); 382 printf("fault: %p\ty: %x\t", 383 (void *)(u_long)tf->tf_fault, (int)tf->tf_y); 384 printf("pil: %d\toldpil: %d\ttt: %x\nGlobals:\n", 385 (int)tf->tf_pil, (int)tf->tf_oldpil, (int)tf->tf_tt); 386 printf("%08x%08x %08x%08x %08x%08x %08x%08x\n", 387 (u_int)(tf->tf_global[0]>>32), (u_int)tf->tf_global[0], 388 (u_int)(tf->tf_global[1]>>32), (u_int)tf->tf_global[1], 389 (u_int)(tf->tf_global[2]>>32), (u_int)tf->tf_global[2], 390 (u_int)(tf->tf_global[3]>>32), (u_int)tf->tf_global[3]); 391 printf("%08x%08x %08x%08x %08x%08x %08x%08x\nOuts:\n", 392 (u_int)(tf->tf_global[4]>>32), (u_int)tf->tf_global[4], 393 (u_int)(tf->tf_global[5]>>32), (u_int)tf->tf_global[5], 394 (u_int)(tf->tf_global[6]>>32), (u_int)tf->tf_global[6], 395 (u_int)(tf->tf_global[7]>>32), (u_int)tf->tf_global[7]); 396 #ifdef DEBUG 397 printf("%08x%08x %08x%08x %08x%08x %08x%08x\n", 398 (u_int)(tf->tf_out[0]>>32), (u_int)tf->tf_out[0], 399 (u_int)(tf->tf_out[1]>>32), (u_int)tf->tf_out[1], 400 (u_int)(tf->tf_out[2]>>32), (u_int)tf->tf_out[2], 401 (u_int)(tf->tf_out[3]>>32), (u_int)tf->tf_out[3]); 402 printf("%08x%08x %08x%08x %08x%08x %08x%08x\nLocals:\n", 403 (u_int)(tf->tf_out[4]>>32), (u_int)tf->tf_out[4], 404 (u_int)(tf->tf_out[5]>>32), (u_int)tf->tf_out[5], 405 (u_int)(tf->tf_out[6]>>32), (u_int)tf->tf_out[6], 406 (u_int)(tf->tf_out[7]>>32), (u_int)tf->tf_out[7]); 407 printf("%08x%08x %08x%08x %08x%08x %08x%08x\n", 408 (u_int)(tf->tf_local[0]>>32), (u_int)tf->tf_local[0], 409 (u_int)(tf->tf_local[1]>>32), (u_int)tf->tf_local[1], 410 (u_int)(tf->tf_local[2]>>32), (u_int)tf->tf_local[2], 411 (u_int)(tf->tf_local[3]>>32), (u_int)tf->tf_local[3]); 412 printf("%08x%08x %08x%08x %08x%08x %08x%08x\nIns:\n", 413 (u_int)(tf->tf_local[4]>>32), (u_int)tf->tf_local[4], 414 (u_int)(tf->tf_local[5]>>32), (u_int)tf->tf_local[5], 415 (u_int)(tf->tf_local[6]>>32), (u_int)tf->tf_local[6], 416 (u_int)(tf->tf_local[7]>>32), (u_int)tf->tf_local[7]); 417 printf("%08x%08x %08x%08x %08x%08x %08x%08x\n", 418 (u_int)(tf->tf_in[0]>>32), (u_int)tf->tf_in[0], 419 (u_int)(tf->tf_in[1]>>32), (u_int)tf->tf_in[1], 420 (u_int)(tf->tf_in[2]>>32), (u_int)tf->tf_in[2], 421 (u_int)(tf->tf_in[3]>>32), (u_int)tf->tf_in[3]); 422 printf("%08x%08x %08x%08x %08x%08x %08x%08x\n", 423 (u_int)(tf->tf_in[4]>>32), (u_int)tf->tf_in[4], 424 (u_int)(tf->tf_in[5]>>32), (u_int)tf->tf_in[5], 425 (u_int)(tf->tf_in[6]>>32), (u_int)tf->tf_in[6], 426 (u_int)(tf->tf_in[7]>>32), (u_int)tf->tf_in[7]); 427 #endif 428 429 } 430 #endif 431 432 /* 433 * Called from locore.s trap handling, for non-MMU-related traps. 434 * (MMU-related traps go through data_access_fault, below.) 435 */ 436 void 437 trap(struct trapframe64 *tf, unsigned int type, vaddr_t pc, long tstate) 438 { 439 struct lwp *l; 440 struct proc *p; 441 struct pcb *pcb; 442 int64_t n; 443 u_quad_t sticks; 444 int pstate = tstate >> TSTATE_PSTATE_SHIFT; 445 ksiginfo_t ksi; 446 int error; 447 int code, sig; 448 #ifdef MULTIPROCESSOR 449 int s; 450 #define disintr() s = intr_disable() 451 #define rstintr() intr_restore(s) 452 #else 453 #define disintr() /* nothing */ 454 #define rstintr() /* nothing */ 455 #endif 456 457 /* This steps the PC over the trap. */ 458 #define ADVANCE (n = tf->tf_npc, tf->tf_pc = n, tf->tf_npc = n + 4) 459 460 #ifdef DEBUG 461 if (tf->tf_pc == tf->tf_npc) { 462 printf("trap: tpc %p == tnpc %p\n", 463 (void *)(u_long)tf->tf_pc, (void *)(u_long)tf->tf_npc); 464 Debugger(); 465 } 466 if ((trapdebug & TDB_NSAVED && curpcb->pcb_nsaved) || 467 trapdebug & (TDB_FOLLOW | TDB_TRAP)) { 468 char sbuf[sizeof(PSTATE_BITS) + 64]; 469 470 printf("trap: type 0x%x: pc=%lx &tf=%p\n", 471 type, pc, tf); 472 snprintb(sbuf, sizeof(sbuf), PSTATE_BITS, pstate); 473 printf(" npc=%lx pstate=%s %s\n", 474 (long)tf->tf_npc, sbuf, 475 type < N_TRAP_TYPES ? trap_type[type] : 476 ((type == T_AST) ? "ast" : 477 ((type == T_RWRET) ? "rwret" : T))); 478 } 479 if ((trapdebug & (TDB_FOLLOW | TDB_TRAP)) || 480 ((trapdebug & TDB_TL) && gettl())) { 481 char sbuf[sizeof(PSTATE_BITS) + 64]; 482 483 extern int trap_trace_dis; 484 trap_trace_dis = 1; 485 printf("trap: type 0x%x: lvl=%d pc=%lx &tf=%p", 486 type, gettl(), pc, tf); 487 snprintb(sbuf, sizeof(sbuf), PSTATE_BITS, pstate); 488 printf(" npc=%lx pstate=%s %s\n", 489 (long)tf->tf_npc, sbuf, 490 type < N_TRAP_TYPES ? trap_type[type] : 491 ((type == T_AST) ? "ast" : 492 ((type == T_RWRET) ? "rwret" : T))); 493 #ifdef DDB 494 kdb_trap(type, tf); 495 #endif 496 } 497 #endif 498 499 curcpu()->ci_data.cpu_ntrap++; 500 501 /* 502 * Generally, kernel traps cause a panic. Any exceptions are 503 * handled early here. 504 */ 505 if (pstate & PSTATE_PRIV) { 506 #ifdef DDB 507 if (type == T_BREAKPOINT) { 508 write_all_windows(); 509 if (kdb_trap(type, tf)) { 510 /* ADVANCE; */ 511 return; 512 } 513 } 514 if (type == T_PA_WATCHPT || type == T_VA_WATCHPT) { 515 if (kdb_trap(type, tf)) { 516 /* DDB must turn off watchpoints or something */ 517 return; 518 } 519 } 520 #endif 521 /* 522 * The kernel needs to use FPU registers for block 523 * load/store. If we trap in privileged code, save 524 * the FPU state if there is any and enable the FPU. 525 * 526 * We rely on the kernel code properly enabling the FPU 527 * in %fprs, otherwise we'll hang here trying to enable 528 * the FPU. 529 */ 530 if (type == T_FPDISABLED) { 531 struct lwp *newfplwp; 532 533 /* New scheme */ 534 if (CLKF_INTR((struct clockframe *)tf) || !curlwp) { 535 newfplwp = &lwp0; 536 } else { 537 newfplwp = curlwp; 538 /* force other cpus to give up this fpstate */ 539 if (newfplwp->l_md.md_fpstate) 540 fpusave_lwp(newfplwp, true); 541 } 542 if (fplwp != newfplwp) { 543 disintr(); 544 if (fplwp != NULL) { 545 /* someone else had it, maybe? */ 546 KASSERT(fplwp->l_md.md_fpstate != NULL); 547 savefpstate(fplwp->l_md.md_fpstate); 548 fplwp = NULL; 549 } 550 rstintr(); 551 /* If we have an allocated fpstate, load it */ 552 if (newfplwp->l_md.md_fpstate != NULL) { 553 fplwp = newfplwp; 554 loadfpstate(fplwp->l_md.md_fpstate); 555 } else 556 fplwp = NULL; 557 } 558 /* Enable the FPU */ 559 tf->tf_tstate |= TSTATE_PEF; 560 return; 561 } else if (type == T_ECCERR) { 562 ecc_corrected_error(type, pc); 563 return; 564 } else if (type == T_FAST_ECC_ERROR) { 565 /* Disable D$, clear error, enable D$, continue. */ 566 membar_Sync(); 567 sp_blast_dcache_disabled(dcache_size, dcache_line_size); 568 return; 569 } 570 goto dopanic; 571 } 572 l = curlwp; 573 p = l->l_proc; 574 sticks = p->p_sticks; 575 pcb = lwp_getpcb(l); 576 l->l_md.md_tf = tf; /* for ptrace/signals */ 577 578 sig = 0; 579 580 switch (type) { 581 582 default: 583 if (type < 0x100) { 584 extern int trap_trace_dis; 585 dopanic: 586 trap_trace_dis = 1; 587 588 { 589 char sb[sizeof(PSTATE_BITS) + 64]; 590 591 printf("trap type 0x%x: cpu %d, pc=%lx", 592 type, cpu_number(), pc); 593 snprintb(sb, sizeof(sb), PSTATE_BITS, pstate); 594 printf(" npc=%lx pstate=%s\n", 595 (long)tf->tf_npc, sb); 596 DEBUGGER(type, tf); 597 panic("%s", type < N_TRAP_TYPES ? trap_type[type] : T); 598 } 599 /* NOTREACHED */ 600 } 601 602 /* the following message is gratuitous */ 603 /* ... but leave it in until we find anything */ 604 printf("%s[%d]: unimplemented software trap 0x%x\n", 605 p->p_comm, p->p_pid, type); 606 607 KSI_INIT_TRAP(&ksi); 608 sig = SIGILL; 609 ksi.ksi_trap = type; 610 ksi.ksi_code = ILL_ILLTRP; 611 ksi.ksi_addr = (void *)pc; 612 break; 613 614 case T_AST: 615 want_ast = 0; 616 if (l->l_pflag & LP_OWEUPC) { 617 l->l_pflag &= ~LP_OWEUPC; 618 ADDUPROF(l); 619 } 620 break; 621 622 case T_INST_EXCEPT: 623 case T_TEXTFAULT: 624 #ifdef DEBUG 625 /* This is not an MMU issue!!!! */ 626 printf("trap: pid=%d.%d comm=%s textfault at %lx!! sending SIGILL due to trap %d: %s\n", 627 l->l_proc->p_pid, l->l_lid, l->l_proc->p_comm, 628 pc, type, type < N_TRAP_TYPES ? trap_type[type] : T); 629 #endif 630 /* FALLTHROUGH */ 631 case T_ILLINST: 632 #if defined(DDB) && defined(DEBUG) 633 if (trapdebug & TDB_STOPSIG) 634 Debugger(); 635 #endif 636 KSI_INIT_TRAP(&ksi); 637 sig = SIGILL; 638 ksi.ksi_trap = type; 639 ksi.ksi_code = ILL_ILLOPC; 640 ksi.ksi_addr = (void *)pc; 641 break; 642 643 case T_PRIVINST: 644 printf("trap: pid=%d.%d comm=%s privinst!! sending SIGILL due to trap %d: %s\n", 645 l->l_proc->p_pid, l->l_lid, l->l_proc->p_comm, 646 type, type < N_TRAP_TYPES ? trap_type[type] : T); 647 #if defined(DDB) && defined(DEBUG) 648 if (trapdebug & TDB_STOPSIG) 649 Debugger(); 650 #endif 651 KSI_INIT_TRAP(&ksi); 652 sig = SIGILL; 653 ksi.ksi_trap = type; 654 ksi.ksi_code = ILL_PRVOPC; 655 ksi.ksi_addr = (void *)pc; 656 break; 657 658 case T_PRIVACT: 659 KSI_INIT_TRAP(&ksi); 660 sig = SIGILL; 661 ksi.ksi_trap = type; 662 ksi.ksi_code = ILL_PRVOPC; 663 ksi.ksi_addr = (void *)pc; 664 break; 665 666 case T_FPDISABLED: { 667 struct fpstate64 *fs = l->l_md.md_fpstate; 668 669 if (fs == NULL) { 670 /* NOTE: fpstate must be 64-byte aligned */ 671 fs = pool_cache_get(fpstate_cache, PR_WAITOK); 672 *fs = initfpstate; 673 l->l_md.md_fpstate = fs; 674 } 675 /* 676 * We may have more FPEs stored up and/or ops queued. 677 * If they exist, handle them and get out. Otherwise, 678 * resolve the FPU state, turn it on, and try again. 679 * 680 * Ultras should never have a FPU queue. 681 */ 682 if (fs->fs_qsize) { 683 printf("trap: Warning fs_qsize is %d\n",fs->fs_qsize); 684 fpu_cleanup(l, fs); 685 break; 686 } 687 if (fplwp != l) { /* we do not have it */ 688 /* but maybe another CPU has it? */ 689 fpusave_lwp(l, true); 690 disintr(); 691 if (fplwp != NULL) { /* someone else had it */ 692 KASSERT(fplwp->l_md.md_fpstate != NULL); 693 savefpstate(fplwp->l_md.md_fpstate); 694 } 695 loadfpstate(fs); 696 fplwp = l; /* now we do have it */ 697 rstintr(); 698 } 699 tf->tf_tstate |= TSTATE_PEF; 700 break; 701 } 702 703 case T_ALIGN: 704 case T_LDDF_ALIGN: 705 case T_STDF_ALIGN: 706 { 707 int64_t dsfsr = 0, dsfar = 0; 708 #ifdef DEBUG 709 int64_t isfsr = 0; 710 #endif 711 if (!CPU_ISSUN4V) { 712 dsfsr = ldxa(SFSR, ASI_DMMU); 713 if (dsfsr & SFSR_FV) 714 dsfar = ldxa(SFAR, ASI_DMMU); 715 } else { 716 paddr_t mmu_fsa_dfa = cpus->ci_mmufsa 717 + offsetof(struct mmufsa, dfa); 718 dsfar = ldxa(mmu_fsa_dfa, ASI_PHYS_CACHED); 719 } 720 #ifdef DEBUG 721 if (!CPU_ISSUN4V) { 722 isfsr = ldxa(SFSR, ASI_IMMU); 723 } else { 724 paddr_t mmu_fsa_ifa = cpus->ci_mmufsa 725 + offsetof(struct mmufsa, ifa); 726 isfsr = ldxa(mmu_fsa_ifa, ASI_PHYS_CACHED); 727 } 728 #endif 729 /* 730 * If we're busy doing copyin/copyout continue 731 */ 732 pcb = lwp_getpcb(l); 733 if (pcb && pcb->pcb_onfault) { 734 tf->tf_pc = (vaddr_t)pcb->pcb_onfault; 735 tf->tf_npc = tf->tf_pc + 4; 736 break; 737 } 738 739 #ifdef DEBUG 740 #define fmt64(x) (u_int)((x)>>32), (u_int)((x)) 741 if (!CPU_ISSUN4V) { 742 printf("Alignment error: pid=%d.%d comm=%s dsfsr=%08x:%08x " 743 "dsfar=%x:%x isfsr=%08x:%08x pc=%lx\n", 744 l->l_proc->p_pid, l->l_lid, l->l_proc->p_comm, fmt64(dsfsr), fmt64(dsfar), 745 fmt64(isfsr), pc); 746 } else { 747 748 printf("Alignment error: pid=%d.%d comm=%s pc=%lx\n", 749 l->l_proc->p_pid, l->l_lid, l->l_proc->p_comm, pc); 750 paddr_t mmufsa_dfa_addr = cpus->ci_mmufsa + offsetof(struct mmufsa, dfa); 751 paddr_t mmufsa_dfc_addr = cpus->ci_mmufsa + offsetof(struct mmufsa, dfc); 752 int64_t dfa = ldxa(mmufsa_dfa_addr, ASI_PHYS_CACHED); 753 printf("dfa = %016lx\n", dfa); 754 int64_t dfc = ldxa(mmufsa_dfc_addr, ASI_PHYS_CACHED); 755 printf("dfc = %016lx\n", dfc); 756 } 757 #endif 758 759 #if defined(DDB) && defined(DEBUG) 760 if (trapdebug & TDB_STOPSIG) { 761 write_all_windows(); 762 kdb_trap(type, tf); 763 } 764 #endif 765 if ((l->l_proc->p_md.md_flags & MDP_FIXALIGN) != 0 && 766 fixalign(l, tf) == 0) { 767 ADVANCE; 768 break; 769 } 770 KSI_INIT_TRAP(&ksi); 771 sig = SIGBUS; 772 ksi.ksi_trap = type; 773 ksi.ksi_code = BUS_ADRALN; 774 ksi.ksi_addr = (void*)(intptr_t)dsfar; 775 } 776 break; 777 778 case T_FP_IEEE_754: 779 case T_FP_OTHER: 780 /* 781 * Clean up after a floating point exception. 782 * fpu_cleanup can (and usually does) modify the 783 * state we save here, so we must `give up' the FPU 784 * chip context. (The software and hardware states 785 * will not match once fpu_cleanup does its job, so 786 * we must not save again later.) 787 */ 788 if (l != fplwp) 789 panic("fpe without being the FP user"); 790 disintr(); 791 KASSERT(l->l_md.md_fpstate != NULL); 792 savefpstate(l->l_md.md_fpstate); 793 fplwp = NULL; 794 rstintr(); 795 /* tf->tf_tstate &= ~TSTATE_PEF */ /* share_fpu will do this */ 796 if (l->l_md.md_fpstate->fs_qsize == 0) { 797 error = copyin((void *)pc, 798 &l->l_md.md_fpstate->fs_queue[0].fq_instr, 799 sizeof(int)); 800 if (error) { 801 sig = SIGBUS; 802 KSI_INIT_TRAP(&ksi); 803 ksi.ksi_trap = type; 804 ksi.ksi_code = BUS_OBJERR; 805 ksi.ksi_addr = (void *)pc; 806 break; 807 } 808 l->l_md.md_fpstate->fs_qsize = 1; 809 code = fpu_cleanup(l, l->l_md.md_fpstate); 810 ADVANCE; 811 } else 812 code = fpu_cleanup(l, l->l_md.md_fpstate); 813 814 if (code != 0) { 815 sig = SIGFPE; 816 KSI_INIT_TRAP(&ksi); 817 ksi.ksi_trap = type; 818 ksi.ksi_code = code; 819 ksi.ksi_addr = (void *)pc; 820 } 821 break; 822 823 case T_TAGOF: 824 KSI_INIT_TRAP(&ksi); 825 sig = SIGEMT; 826 ksi.ksi_trap = type; 827 ksi.ksi_code = SI_NOINFO; 828 ksi.ksi_addr = (void *)pc; 829 break; 830 831 case T_BREAKPOINT: 832 if (p->p_raslist == NULL || 833 (ras_lookup(p, (void *)(intptr_t)tf->tf_pc) == (void *)-1)) { 834 sig = SIGTRAP; 835 KSI_INIT_TRAP(&ksi); 836 ksi.ksi_trap = type; 837 ksi.ksi_code = TRAP_BRKPT; 838 ksi.ksi_addr = (void *)pc; 839 } 840 break; 841 842 case T_IDIV0: 843 case T_DIV0: 844 ADVANCE; 845 sig = SIGFPE; 846 KSI_INIT_TRAP(&ksi); 847 ksi.ksi_trap = type; 848 ksi.ksi_code = FPE_INTDIV; 849 ksi.ksi_addr = (void *)pc; 850 break; 851 852 case T_CLEANWIN: 853 uprintf("T_CLEANWIN\n"); /* XXX Should not get this */ 854 ADVANCE; 855 break; 856 857 case T_FLUSHWIN: 858 /* Software window flush for v8 software */ 859 write_all_windows(); 860 ADVANCE; 861 break; 862 863 case T_RANGECHECK: 864 printf("T_RANGECHECK\n"); /* XXX */ 865 ADVANCE; 866 sig = SIGILL; 867 KSI_INIT_TRAP(&ksi); 868 ksi.ksi_trap = type; 869 ksi.ksi_code = ILL_ILLADR; 870 ksi.ksi_addr = (void *)pc; 871 break; 872 873 case T_FIXALIGN: 874 #ifdef DEBUG_ALIGN 875 uprintf("T_FIXALIGN\n"); 876 #endif 877 /* User wants us to fix alignment faults */ 878 l->l_proc->p_md.md_flags |= MDP_FIXALIGN; 879 ADVANCE; 880 break; 881 882 case T_INTOF: 883 uprintf("T_INTOF\n"); /* XXX */ 884 ADVANCE; 885 sig = SIGFPE; 886 KSI_INIT_TRAP(&ksi); 887 ksi.ksi_trap = type; 888 ksi.ksi_code = FPE_INTOVF; 889 ksi.ksi_addr = (void *)pc; 890 break; 891 case T_ECCERR: 892 ecc_corrected_error(type, pc); 893 break; 894 } 895 if (sig != 0) { 896 ksi.ksi_signo = sig; 897 trapsignal(l, &ksi); 898 } 899 userret(l, pc, sticks); 900 share_fpu(l, tf); 901 #undef ADVANCE 902 #ifdef DEBUG 903 if (trapdebug & (TDB_FOLLOW | TDB_TRAP)) { 904 printf("trap: done\n"); 905 /* if (type != T_BREAKPOINT) Debugger(); */ 906 } 907 #endif 908 } 909 910 /* 911 * Save windows from PCB into user stack, and return 0. This is used on 912 * window overflow pseudo-traps (from locore.s, just before returning to 913 * user mode) and when ptrace or sendsig needs a consistent state. 914 * As a side effect, rwindow_save() always sets pcb_nsaved to 0. 915 * 916 * If the windows cannot be saved, pcb_nsaved is restored and we return -1. 917 * 918 * XXXXXX This cannot work properly. I need to re-examine this register 919 * window thing entirely. 920 */ 921 int 922 rwindow_save(struct lwp *l) 923 { 924 struct pcb *pcb = lwp_getpcb(l); 925 struct rwindow64 *rw = &pcb->pcb_rw[0]; 926 uint64_t rwdest; 927 int i, j; 928 929 i = pcb->pcb_nsaved; 930 #ifdef DEBUG 931 if (rwindow_debug & RW_FOLLOW) 932 printf("rwindow_save(%p): nsaved %d\n", l, i); 933 #endif 934 if (i == 0) 935 return (0); 936 #ifdef DEBUG 937 if (rwindow_debug & RW_FOLLOW) 938 printf("%s[%d]: rwindow: pcb->stack:", l->l_proc->p_comm, l->l_proc->p_pid); 939 #endif 940 941 while (i > 0) { 942 rwdest = rw[i--].rw_in[6]; 943 #ifdef DEBUG 944 if (rwindow_debug & RW_FOLLOW) 945 printf("window %d at %lx\n", i, (long)rwdest); 946 #endif 947 if (rwdest & 1) { 948 #ifdef DEBUG 949 if (rwindow_debug & RW_64) { 950 printf("rwindow_save: 64-bit tf to %p+BIAS " 951 "or %p\n", 952 (void *)(long)rwdest, 953 (void *)(long)(rwdest+BIAS)); 954 Debugger(); 955 } 956 #endif 957 rwdest += BIAS; 958 if (copyout((void *)&rw[i], (void *)(u_long)rwdest, 959 sizeof(*rw))) { 960 #ifdef DEBUG 961 if (rwindow_debug & (RW_ERR | RW_64)) 962 printf("rwindow_save: 64-bit pcb copyout " 963 "to %p failed\n", 964 (void *)(long)rwdest); 965 #endif 966 return (-1); 967 } 968 #ifdef DEBUG 969 if (rwindow_debug & RW_64) { 970 printf("Finished copyout(%p, %p, %lx)\n", 971 (void *)&rw[i], (void *)(long)rwdest, 972 sizeof(*rw)); 973 Debugger(); 974 } 975 #endif 976 } else { 977 struct rwindow32 rwstack; 978 979 /* 32-bit window */ 980 for (j = 0; j < 8; j++) { 981 rwstack.rw_local[j] = (int)rw[i].rw_local[j]; 982 rwstack.rw_in[j] = (int)rw[i].rw_in[j]; 983 } 984 /* Must truncate rwdest */ 985 if (copyout(&rwstack, (void *)(u_long)(u_int)rwdest, 986 sizeof(rwstack))) { 987 #ifdef DEBUG 988 if (rwindow_debug & RW_ERR) 989 printf("rwindow_save: 32-bit pcb " 990 "copyout to %p (%p) failed\n", 991 (void *)(u_long)(u_int)rwdest, 992 (void *)(u_long)rwdest); 993 #endif 994 return (-1); 995 } 996 } 997 } 998 pcb->pcb_nsaved = 0; 999 #ifdef DEBUG 1000 if (rwindow_debug & RW_FOLLOW) { 1001 printf("\n"); 1002 Debugger(); 1003 } 1004 #endif 1005 return (0); 1006 } 1007 1008 /* 1009 * Kill user windows (before exec) by writing back to stack or pcb 1010 * and then erasing any pcb tracks. Otherwise we might try to write 1011 * the registers into the new process after the exec. 1012 */ 1013 void 1014 cpu_vmspace_exec(struct lwp *l, vaddr_t start, vaddr_t end) 1015 { 1016 struct pcb *pcb = lwp_getpcb(l); 1017 1018 write_user_windows(); 1019 pcb->pcb_nsaved = 0; 1020 } 1021 1022 /* 1023 * This routine handles MMU generated faults. About half 1024 * of them could be recoverable through uvm_fault. 1025 */ 1026 void 1027 data_access_fault(struct trapframe64 *tf, unsigned int type, vaddr_t pc, 1028 vaddr_t addr, vaddr_t sfva, u_long sfsr) 1029 { 1030 uint64_t tstate; 1031 struct lwp *l; 1032 struct proc *p; 1033 struct pcb *pcb; 1034 struct vmspace *vm; 1035 vaddr_t va; 1036 int rv; 1037 vm_prot_t access_type; 1038 vaddr_t onfault; 1039 u_quad_t sticks; 1040 ksiginfo_t ksi; 1041 1042 #ifdef DEBUG 1043 if (tf->tf_pc == tf->tf_npc) { 1044 printf("data_access_fault: tpc %lx == tnpc %lx\n", 1045 (long)tf->tf_pc, (long)tf->tf_npc); 1046 Debugger(); 1047 } 1048 write_user_windows(); 1049 if ((curpcb->pcb_nsaved > 8) || 1050 (trapdebug & TDB_NSAVED && curpcb->pcb_nsaved) || 1051 (trapdebug & (TDB_ADDFLT | TDB_FOLLOW))) { 1052 printf("%ld: data_access_fault(%p, %x, %p, %p, %lx, %lx) " 1053 "nsaved=%d\n", 1054 (long)(curproc?curproc->p_pid:-1), tf, type, 1055 (void *)pc, (void *)addr, 1056 sfva, sfsr, (int)curpcb->pcb_nsaved); 1057 #ifdef DDB 1058 if ((trapdebug & TDB_NSAVED && curpcb->pcb_nsaved)) 1059 Debugger(); 1060 #endif 1061 } 1062 if (trapdebug & TDB_FRAME) { 1063 print_trapframe(tf); 1064 } 1065 if ((trapdebug & TDB_TL) && gettl()) { 1066 printf("%ld: data_access_fault(%p, %x, %p, %p, %lx, %lx) " 1067 "nsaved=%d\n", 1068 (long)(curproc?curproc->p_pid:-1), tf, type, 1069 (void*)addr, (void*)pc, 1070 sfva, sfsr, (int)curpcb->pcb_nsaved); 1071 Debugger(); 1072 } 1073 if (trapdebug & TDB_STOPCALL) { 1074 Debugger(); 1075 } 1076 #endif 1077 1078 curcpu()->ci_data.cpu_ntrap++; 1079 l = curlwp; 1080 p = l->l_proc; 1081 pcb = lwp_getpcb(l); 1082 onfault = (vaddr_t)pcb->pcb_onfault; 1083 sticks = p->p_sticks; 1084 tstate = tf->tf_tstate; 1085 1086 #ifdef _LP64 1087 /* deal with invalid VAs early */ 1088 if (__predict_false(addr >= (1UL<<HOLESHIFT) && addr < (1UL<<63))) { 1089 1090 if (tstate & TSTATE_PRIV) { 1091 if (onfault) { 1092 tf->tf_pc = onfault; 1093 tf->tf_npc = onfault + 4; 1094 tf->tf_out[0] = EFAULT; 1095 return; 1096 } 1097 panic("fault type %u for invalid va %lx", type, addr); 1098 } 1099 1100 KSI_INIT_TRAP(&ksi); 1101 ksi.ksi_signo = SIGSEGV; 1102 ksi.ksi_code = SEGV_ACCERR; 1103 ksi.ksi_trap = type; 1104 ksi.ksi_addr = (void *)pc; 1105 trapsignal(l, &ksi); 1106 userret(l, pc, sticks); 1107 share_fpu(l, tf); 1108 return; 1109 } 1110 #endif 1111 1112 /* Find the faulting va to give to uvm_fault */ 1113 va = trunc_page(addr); 1114 1115 /* 1116 * Now munch on protections. 1117 * 1118 * If it was a FAST_DATA_ACCESS_MMU_MISS we have no idea what the 1119 * access was since the SFSR is not set. But we should never get 1120 * here from there. 1121 */ 1122 if (type == T_FDMMU_MISS || (sfsr & SFSR_FV) == 0) { 1123 /* Punt */ 1124 access_type = VM_PROT_READ; 1125 } else { 1126 access_type = (sfsr & SFSR_W) ? VM_PROT_WRITE : VM_PROT_READ; 1127 } 1128 if (tstate & TSTATE_PRIV) { 1129 extern char Lfsbail[]; 1130 1131 rv = EFAULT; 1132 1133 /* 1134 * If this was an access that we shouldn't try to page in, 1135 * resume at the fault handler without any action. 1136 */ 1137 if (onfault == (vaddr_t)Lfsbail) 1138 goto kfault; 1139 1140 /* 1141 * During autoconfiguration, faults are never OK unless 1142 * pcb_onfault is set. Once running normally we must allow 1143 * exec() to cause copy-on-write faults to kernel addresses. 1144 */ 1145 if (cold) 1146 goto kfault; 1147 if (!(addr & TLB_TAG_ACCESS_CTX)) { 1148 /* CTXT == NUCLEUS */ 1149 curpcb->pcb_onfault = NULL; 1150 rv = uvm_fault(kernel_map, va, access_type); 1151 curpcb->pcb_onfault = (void *)onfault; 1152 #ifdef DEBUG 1153 if (trapdebug & (TDB_ADDFLT | TDB_FOLLOW)) 1154 printf("cpu%d: data_access_fault: kernel " 1155 "uvm_fault(%p, %lx, %x) " 1156 "sez %x -- %s\n", cpu_number(), 1157 kernel_map, va, access_type, rv, 1158 rv ? "failure" : "success"); 1159 #endif 1160 if (rv == 0) 1161 return; 1162 goto kfault; 1163 } 1164 } else { 1165 l->l_md.md_tf = tf; 1166 } 1167 1168 vm = p->p_vmspace; 1169 /* alas! must call the horrible vm code */ 1170 pcb->pcb_onfault = NULL; 1171 rv = uvm_fault(&vm->vm_map, va, access_type); 1172 pcb->pcb_onfault = (void *)onfault; 1173 1174 #ifdef DEBUG 1175 if (trapdebug & (TDB_ADDFLT | TDB_FOLLOW)) 1176 printf("cpu%d: data_access_fault: %s uvm_fault(%p, %lx, %x) " 1177 "sez %x -- %s\n", cpu_number(), 1178 &vm->vm_map == kernel_map ? "kernel!!!" : "user", 1179 &vm->vm_map, va, access_type, rv, 1180 rv ? "failure" : "success"); 1181 #endif 1182 1183 /* 1184 * If this was a stack access we keep track of the maximum 1185 * accessed stack size. Also, if uvm_fault gets a protection 1186 * failure it is due to accessing the stack region outside 1187 * the current limit and we need to reflect that as an access 1188 * error. 1189 */ 1190 if ((void *)va >= vm->vm_maxsaddr) { 1191 if (rv == 0) 1192 uvm_grow(p, va); 1193 else if (rv == EACCES) 1194 rv = EFAULT; 1195 } 1196 if (rv != 0) { 1197 1198 /* 1199 * Pagein failed. If doing copyin/out, return to onfault 1200 * address. Any other page fault in kernel, die; if user 1201 * fault, deliver SIGSEGV. 1202 */ 1203 if (tstate & TSTATE_PRIV) { 1204 kfault: 1205 if (!onfault) { 1206 extern int trap_trace_dis; 1207 1208 /* Disable traptrace for printf */ 1209 trap_trace_dis = 1; 1210 (void) splhigh(); 1211 printf("cpu%d: data fault: pc=%lx rpc=%"PRIx64" addr=%lx\n", 1212 cpu_number(), pc, tf->tf_in[7], addr); 1213 DEBUGGER(type, tf); 1214 panic("kernel fault"); 1215 /* NOTREACHED */ 1216 } 1217 #ifdef DEBUG 1218 if (trapdebug & 1219 (TDB_ADDFLT | TDB_FOLLOW | TDB_STOPCPIO)) { 1220 printf("data_access_fault: copyin/out of %p " 1221 "fault -- recover\n", (void *)addr); 1222 DEBUGGER(type, tf); 1223 } 1224 #endif 1225 tf->tf_pc = onfault; 1226 tf->tf_npc = onfault + 4; 1227 tf->tf_out[0] = (rv == EACCES) ? EFAULT : rv; 1228 return; 1229 } 1230 #ifdef DEBUG 1231 if (trapdebug & (TDB_ADDFLT | TDB_STOPSIG)) { 1232 extern int trap_trace_dis; 1233 trap_trace_dis = 1; 1234 printf("cpu%d: data_access_fault at addr %p: " 1235 "sending SIGSEGV\n", cpu_number(), (void *)addr); 1236 printf("%ld: data_access_fault(%p, %x, %p, %p, " 1237 "%lx, %lx) nsaved=%d\n", 1238 (long)(curproc ? curproc->p_pid : -1), tf, type, 1239 (void *)addr, (void *)pc, 1240 sfva, sfsr, (int)curpcb->pcb_nsaved); 1241 Debugger(); 1242 } 1243 #endif 1244 KSI_INIT_TRAP(&ksi); 1245 switch (rv) { 1246 case EINVAL: 1247 ksi.ksi_signo = SIGBUS; 1248 ksi.ksi_code = BUS_ADRERR; 1249 break; 1250 case EACCES: 1251 ksi.ksi_signo = SIGSEGV; 1252 ksi.ksi_code = SEGV_ACCERR; 1253 break; 1254 case ENOMEM: 1255 ksi.ksi_signo = SIGKILL; 1256 printf("UVM: pid %d (%s), uid %d killed: out of swap\n", 1257 p->p_pid, p->p_comm, 1258 l->l_cred ? kauth_cred_geteuid(l->l_cred) : -1); 1259 break; 1260 default: 1261 ksi.ksi_signo = SIGSEGV; 1262 ksi.ksi_code = SEGV_MAPERR; 1263 break; 1264 } 1265 ksi.ksi_trap = type; 1266 ksi.ksi_addr = (void *)sfva; 1267 trapsignal(l, &ksi); 1268 } 1269 if ((tstate & TSTATE_PRIV) == 0) { 1270 userret(l, pc, sticks); 1271 share_fpu(l, tf); 1272 } 1273 #ifdef DEBUG 1274 if (trapdebug & (TDB_ADDFLT | TDB_FOLLOW)) 1275 printf("data_access_fault: done\n"); 1276 if (trapdebug & TDB_FRAME) { 1277 print_trapframe(tf); 1278 } 1279 if (trapdebug & (TDB_ADDFLT | TDB_FOLLOW)) { 1280 extern void *return_from_trap(void); 1281 1282 if ((void *)(u_long)tf->tf_pc == (void *)return_from_trap) { 1283 printf("Returning from stack datafault\n"); 1284 } 1285 } 1286 #endif 1287 } 1288 1289 /* 1290 * This routine handles deferred errors caused by the memory 1291 * or I/O bus subsystems. Most of these are fatal, and even 1292 * if they are not, recovery is painful. Also, the TPC and 1293 * TNPC values are probably not valid if we're not doing a 1294 * special PEEK/POKE code sequence. 1295 */ 1296 void 1297 data_access_error(struct trapframe64 *tf, unsigned int type, vaddr_t afva, 1298 u_long afsr, vaddr_t sfva, u_long sfsr) 1299 { 1300 u_long pc; 1301 uint64_t tstate; 1302 struct lwp *l; 1303 struct pcb *pcb; 1304 vaddr_t onfault; 1305 u_quad_t sticks; 1306 ksiginfo_t ksi; 1307 1308 #ifdef DEBUG 1309 if (tf->tf_pc == tf->tf_npc) { 1310 printf("data_access_error: tpc %lx == tnpc %lx\n", 1311 (long)tf->tf_pc, (long)tf->tf_npc); 1312 Debugger(); 1313 } 1314 write_user_windows(); 1315 if ((trapdebug & TDB_NSAVED && curpcb->pcb_nsaved) || 1316 trapdebug & (TDB_ADDFLT | TDB_FOLLOW)) { 1317 char buf[768]; 1318 1319 snprintb(buf, sizeof buf, SFSR_BITS, sfsr); 1320 printf("%d data_access_error(%lx, %lx, %lx, %p)=%lx @ %p %s\n", 1321 curproc?curproc->p_pid:-1, 1322 (long)type, (long)sfva, (long)afva, tf, 1323 (long)tf->tf_tstate, 1324 (void *)(u_long)tf->tf_pc, buf); 1325 } 1326 if (trapdebug & TDB_FRAME) { 1327 print_trapframe(tf); 1328 } 1329 if ((trapdebug & TDB_TL) && gettl()) { 1330 char buf[768]; 1331 snprintb(buf, sizeof buf, SFSR_BITS, sfsr); 1332 1333 printf("%d tl %d data_access_error(%lx, %lx, %lx, %p)=" 1334 "%lx @ %lx %s\n", 1335 curproc ? curproc->p_pid : -1, gettl(), 1336 (long)type, (long)sfva, (long)afva, tf, 1337 (long)tf->tf_tstate, 1338 (long)tf->tf_pc, buf); 1339 Debugger(); 1340 } 1341 if (trapdebug & TDB_STOPCALL) { 1342 Debugger(); 1343 } 1344 #endif 1345 1346 curcpu()->ci_data.cpu_ntrap++; 1347 pc = tf->tf_pc; 1348 tstate = tf->tf_tstate; 1349 1350 /* 1351 * Catch PCI config space reads. 1352 */ 1353 if (curcpu()->ci_pci_probe) { 1354 #ifdef DIAGNOSTIC 1355 printf("data_access_error in pci_conf_read: pc=%lx addr=%lx\n", 1356 pc, afva); 1357 #endif 1358 curcpu()->ci_pci_fault = true; 1359 /* 1360 * The contents of TNPC are undefined, so make it points to 1361 * the next instruction. 1362 */ 1363 tf->tf_npc = pc + 4; 1364 return; 1365 } 1366 1367 l = curlwp; 1368 pcb = lwp_getpcb(l); 1369 sticks = l->l_proc->p_sticks; 1370 1371 printf("data error type %x sfsr=%lx sfva=%lx afsr=%lx afva=%lx tf=%p\n", 1372 type, sfsr, sfva, afsr, afva, tf); 1373 1374 if (afsr == 0) { 1375 printf("data_access_error: no fault\n"); 1376 goto out; /* No fault. Why were we called? */ 1377 } 1378 1379 if (tstate & TSTATE_PRIV) { 1380 onfault = (vaddr_t)pcb->pcb_onfault; 1381 if (!onfault) { 1382 extern int trap_trace_dis; 1383 char buf[768]; 1384 1385 trap_trace_dis = 1; /* Disable traptrace for printf */ 1386 snprintb(buf, sizeof buf, SFSR_BITS, sfsr); 1387 (void) splhigh(); 1388 printf("data fault: pc=%lx addr=%lx sfsr=%s\n", 1389 (u_long)pc, (long)sfva, buf); 1390 DEBUGGER(type, tf); 1391 panic("kernel fault"); 1392 /* NOTREACHED */ 1393 } 1394 1395 /* 1396 * If this was a privileged error but not a probe, we 1397 * cannot recover, so panic. 1398 */ 1399 if (afsr & ASFR_PRIV) { 1400 char buf[128]; 1401 1402 snprintb(buf, sizeof(buf), AFSR_BITS, afsr); 1403 panic("Privileged Async Fault: AFAR %p AFSR %lx\n%s", 1404 (void *)afva, afsr, buf); 1405 /* NOTREACHED */ 1406 } 1407 #ifdef DEBUG 1408 if (trapdebug & (TDB_ADDFLT | TDB_FOLLOW | TDB_STOPCPIO)) { 1409 printf("data_access_error: kern fault -- " 1410 "skipping instr\n"); 1411 if (trapdebug & TDB_STOPCPIO) { 1412 DEBUGGER(type, tf); 1413 } 1414 } 1415 #endif 1416 tf->tf_pc = onfault; 1417 tf->tf_npc = onfault + 4; 1418 tf->tf_out[0] = EFAULT; 1419 return; 1420 } 1421 #ifdef DEBUG 1422 if (trapdebug & (TDB_ADDFLT | TDB_STOPSIG)) { 1423 extern int trap_trace_dis; 1424 1425 trap_trace_dis = 1; 1426 printf("data_access_error at %p: sending SIGSEGV\n", 1427 (void *)(u_long)afva); 1428 Debugger(); 1429 } 1430 #endif 1431 KSI_INIT_TRAP(&ksi); 1432 ksi.ksi_signo = SIGSEGV; 1433 ksi.ksi_code = SEGV_MAPERR; 1434 ksi.ksi_trap = type; 1435 ksi.ksi_addr = (void *)afva; 1436 trapsignal(l, &ksi); 1437 out: 1438 if ((tstate & TSTATE_PRIV) == 0) { 1439 userret(l, pc, sticks); 1440 share_fpu(l, tf); 1441 } 1442 #ifdef DEBUG 1443 if (trapdebug & (TDB_ADDFLT | TDB_FOLLOW)) 1444 printf("data_access_error: done\n"); 1445 if (trapdebug & TDB_FRAME) { 1446 print_trapframe(tf); 1447 } 1448 #endif 1449 } 1450 1451 /* 1452 * This routine handles MMU generated faults. About half 1453 * of them could be recoverable through uvm_fault. 1454 */ 1455 void 1456 text_access_fault(struct trapframe64 *tf, unsigned int type, vaddr_t pc, 1457 u_long sfsr) 1458 { 1459 uint64_t tstate; 1460 struct lwp *l; 1461 struct proc *p; 1462 struct vmspace *vm; 1463 vaddr_t va; 1464 int rv; 1465 vm_prot_t access_type; 1466 u_quad_t sticks; 1467 ksiginfo_t ksi; 1468 1469 #ifdef DEBUG 1470 if (tf->tf_pc == tf->tf_npc) { 1471 printf("text_access_fault: tpc %p == tnpc %p\n", 1472 (void *)(u_long)tf->tf_pc, (void *)(u_long)tf->tf_npc); 1473 Debugger(); 1474 } 1475 write_user_windows(); 1476 if (((trapdebug & TDB_NSAVED) && curpcb->pcb_nsaved) || 1477 (trapdebug & (TDB_TXTFLT | TDB_FOLLOW))) 1478 printf("%d text_access_fault(%x, %lx, %p)\n", 1479 curproc?curproc->p_pid:-1, type, pc, tf); 1480 if (trapdebug & TDB_FRAME) { 1481 print_trapframe(tf); 1482 } 1483 if ((trapdebug & TDB_TL) && gettl()) { 1484 printf("%d tl %d text_access_fault(%x, %lx, %p)\n", 1485 curproc?curproc->p_pid:-1, gettl(), type, pc, tf); 1486 Debugger(); 1487 } 1488 if (trapdebug & TDB_STOPCALL) { 1489 Debugger(); 1490 } 1491 #endif 1492 1493 curcpu()->ci_data.cpu_ntrap++; 1494 l = curlwp; 1495 p = l->l_proc; 1496 sticks = p->p_sticks; 1497 tstate = tf->tf_tstate; 1498 va = trunc_page(pc); 1499 1500 /* Now munch on protections... */ 1501 1502 access_type = VM_PROT_EXECUTE; 1503 if (tstate & TSTATE_PRIV) { 1504 extern int trap_trace_dis; 1505 trap_trace_dis = 1; /* Disable traptrace for printf */ 1506 (void) splhigh(); 1507 printf("text_access_fault: pc=%lx va=%lx\n", pc, va); 1508 DEBUGGER(type, tf); 1509 panic("kernel fault"); 1510 /* NOTREACHED */ 1511 } else 1512 l->l_md.md_tf = tf; 1513 1514 vm = p->p_vmspace; 1515 /* alas! must call the horrible vm code */ 1516 KASSERT(curpcb->pcb_onfault == NULL); 1517 rv = uvm_fault(&vm->vm_map, va, access_type); 1518 1519 #ifdef DEBUG 1520 if (trapdebug & (TDB_TXTFLT | TDB_FOLLOW)) 1521 printf("text_access_fault: uvm_fault(%p, %lx, %x) sez %x\n", 1522 &vm->vm_map, va, access_type, rv); 1523 #endif 1524 if (rv != 0) { 1525 1526 /* 1527 * Pagein failed, deliver SIGSEGV. 1528 */ 1529 #ifdef DEBUG 1530 if (trapdebug & (TDB_TXTFLT | TDB_STOPSIG)) { 1531 extern int trap_trace_dis; 1532 trap_trace_dis = 1; 1533 printf("text_access_fault at %p: sending SIGSEGV\n", 1534 (void *)(u_long)va); 1535 Debugger(); 1536 } 1537 #endif 1538 KSI_INIT_TRAP(&ksi); 1539 ksi.ksi_signo = SIGSEGV; 1540 ksi.ksi_code = (rv == EACCES ? SEGV_ACCERR : SEGV_MAPERR); 1541 ksi.ksi_trap = type; 1542 ksi.ksi_addr = (void *)pc; 1543 trapsignal(l, &ksi); 1544 } 1545 userret(l, pc, sticks); 1546 share_fpu(l, tf); 1547 #ifdef DEBUG 1548 if (trapdebug & (TDB_TXTFLT | TDB_FOLLOW)) { 1549 printf("text_access_fault: done\n"); 1550 /* kdb_trap(T_BREAKPOINT, tf); */ 1551 } 1552 if (trapdebug & TDB_FRAME) { 1553 print_trapframe(tf); 1554 } 1555 #endif 1556 } 1557 1558 1559 /* 1560 * This routine handles deferred errors caused by the memory 1561 * or I/O bus subsystems. Most of these are fatal, and even 1562 * if they are not, recovery is painful. Also, the TPC and 1563 * TNPC values are probably not valid if we're not doing a 1564 * special PEEK/POKE code sequence. 1565 */ 1566 void 1567 text_access_error(struct trapframe64 *tf, unsigned int type, vaddr_t pc, 1568 u_long sfsr, vaddr_t afva, u_long afsr) 1569 { 1570 int64_t tstate; 1571 struct lwp *l; 1572 struct proc *p; 1573 struct vmspace *vm; 1574 vaddr_t va; 1575 int rv; 1576 vm_prot_t access_type; 1577 u_quad_t sticks; 1578 ksiginfo_t ksi; 1579 char buf[768]; 1580 1581 #ifdef DEBUG 1582 if (tf->tf_pc == tf->tf_npc) { 1583 printf("text_access_error: tpc %p == tnpc %p\n", 1584 (void *)(u_long)tf->tf_pc, (void *)(u_long)tf->tf_npc); 1585 Debugger(); 1586 } 1587 write_user_windows(); 1588 if ((trapdebug & TDB_NSAVED && curpcb->pcb_nsaved) || 1589 trapdebug & (TDB_TXTFLT | TDB_FOLLOW)) { 1590 snprintb(buf, sizeof buf, SFSR_BITS, sfsr); 1591 printf("%ld text_access_error(%lx, %lx, %lx, %p)=%lx @ %lx %s\n", 1592 (long)(curproc?curproc->p_pid:-1), 1593 (long)type, pc, (long)afva, tf, (long)tf->tf_tstate, 1594 (long)tf->tf_pc, buf); 1595 } 1596 if (trapdebug & TDB_FRAME) { 1597 print_trapframe(tf); 1598 } 1599 if ((trapdebug & TDB_TL) && gettl()) { 1600 snprintb(buf, sizeof buf, SFSR_BITS, sfsr); 1601 printf("%d tl %d text_access_error(%lx, %lx, %lx, %p)=%lx @ %lx %s\n", 1602 curproc?curproc->p_pid:-1, gettl(), 1603 (long)type, (long)pc, (long)afva, tf, 1604 (long)tf->tf_tstate, (long)tf->tf_pc, buf); 1605 Debugger(); 1606 } 1607 if (trapdebug & TDB_STOPCALL) { 1608 Debugger(); 1609 } 1610 #endif 1611 curcpu()->ci_data.cpu_ntrap++; 1612 l = curlwp; 1613 p = l->l_proc; 1614 sticks = p->p_sticks; 1615 1616 tstate = tf->tf_tstate; 1617 1618 if ((afsr) != 0) { 1619 extern int trap_trace_dis; 1620 1621 trap_trace_dis++; /* Disable traptrace for printf */ 1622 printf("text_access_error: memory error...\n"); 1623 printf("text memory error type %d sfsr=%lx sfva=%lx afsr=%lx afva=%lx tf=%p\n", 1624 type, sfsr, pc, afsr, afva, tf); 1625 trap_trace_dis--; /* Reenable traptrace for printf */ 1626 1627 if (tstate & TSTATE_PRIV) 1628 panic("text_access_error: kernel memory error"); 1629 1630 /* User fault -- Berr */ 1631 KSI_INIT_TRAP(&ksi); 1632 ksi.ksi_signo = SIGBUS; 1633 ksi.ksi_code = BUS_OBJERR; 1634 ksi.ksi_trap = type; 1635 ksi.ksi_addr = (void *)afva; 1636 trapsignal(l, &ksi); 1637 } 1638 1639 if ((sfsr & SFSR_FV) == 0 || (sfsr & SFSR_FT) == 0) 1640 goto out; /* No fault. Why were we called? */ 1641 1642 va = trunc_page(pc); 1643 1644 /* Now munch on protections... */ 1645 1646 access_type = VM_PROT_EXECUTE; 1647 if (tstate & TSTATE_PRIV) { 1648 extern int trap_trace_dis; 1649 trap_trace_dis = 1; /* Disable traptrace for printf */ 1650 snprintb(buf, sizeof buf, SFSR_BITS, sfsr); 1651 (void) splhigh(); 1652 printf("text error: pc=%lx sfsr=%s\n", pc, buf); 1653 DEBUGGER(type, tf); 1654 panic("kernel fault"); 1655 /* NOTREACHED */ 1656 } else 1657 l->l_md.md_tf = tf; 1658 1659 vm = p->p_vmspace; 1660 /* alas! must call the horrible vm code */ 1661 KASSERT(curpcb->pcb_onfault == NULL); 1662 rv = uvm_fault(&vm->vm_map, va, access_type); 1663 1664 if (rv != 0) { 1665 /* 1666 * Pagein failed, deliver SIGSEGV. 1667 */ 1668 #ifdef DEBUG 1669 if (trapdebug & (TDB_TXTFLT | TDB_STOPSIG)) { 1670 extern int trap_trace_dis; 1671 trap_trace_dis = 1; 1672 printf("text_access_error at %p: sending SIGSEGV\n", 1673 (void *)(u_long)va); 1674 Debugger(); 1675 } 1676 #endif 1677 KSI_INIT_TRAP(&ksi); 1678 ksi.ksi_signo = SIGSEGV; 1679 ksi.ksi_code = (rv == EACCES ? SEGV_ACCERR : SEGV_MAPERR); 1680 ksi.ksi_trap = type; 1681 ksi.ksi_addr = (void *)pc; 1682 trapsignal(l, &ksi); 1683 } 1684 out: 1685 if ((tstate & TSTATE_PRIV) == 0) { 1686 userret(l, pc, sticks); 1687 share_fpu(l, tf); 1688 } 1689 #ifdef DEBUG 1690 if (trapdebug & (TDB_TXTFLT | TDB_FOLLOW)) 1691 printf("text_access_error: done\n"); 1692 if (trapdebug & TDB_FRAME) { 1693 print_trapframe(tf); 1694 } 1695 #endif 1696 } 1697 1698 /* 1699 * Handle an ECC corrected event. 1700 */ 1701 void 1702 ecc_corrected_error(unsigned int type, vaddr_t pc) 1703 { 1704 uint64_t eeer, afar, afsr; 1705 char buf[128]; 1706 int s; 1707 1708 /* Clear the error */ 1709 eeer = ldxa(0, ASI_ERROR_EN_REG); 1710 s = intr_disable(); 1711 stxa(0, ASI_ERROR_EN_REG, 1712 eeer & ~(P_EER_NCEEN | P_EER_CEEN)); 1713 membar_Sync(); 1714 intr_restore(s); 1715 1716 /* Flush the caches in order ensure no corrupt data got installed. */ 1717 blast_dcache(); 1718 blast_icache(); 1719 1720 #if 0 1721 /* Ensure the caches are still turned on (should be). */ 1722 cache_enable(PCPU_GET(impl)); 1723 #endif 1724 1725 /* Grab the current AFSR/AFAR, and clear the error from the AFSR. */ 1726 afar = ldxa(0, ASI_AFAR); 1727 afsr = ldxa(0, ASI_AFSR); 1728 s = intr_disable(); 1729 stxa(0, ASI_AFSR, ldxa(0, ASI_AFSR)); 1730 membar_Sync(); 1731 intr_restore(s); 1732 ecc_corrected.ev_count++; 1733 snprintb(buf, sizeof(buf), AFSR_BITS, afsr); 1734 printf("corrected ECC error: pc %p afsr %"PRIx64" (%s) addr %"PRIx64"\n", (void *)pc, afsr, buf, afar); 1735 1736 /* Turn (non-)correctable error reporting back on. */ 1737 s = intr_disable(); 1738 stxa(0, ASI_ERROR_EN_REG, eeer); 1739 membar_Sync(); 1740 intr_restore(s); 1741 } 1742