1 /*- 2 * Copyright (c) 2015-2018 Ruslan Bukin <br@bsdpad.com> 3 * All rights reserved. 4 * 5 * Portions of this software were developed by SRI International and the 6 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 7 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 8 * 9 * Portions of this software were developed by the University of Cambridge 10 * Computer Laboratory as part of the CTSRD Project, with support from the 11 * UK Higher Education Innovation Fund (HEIF). 12 * 13 * Redistribution and use in source and binary forms, with or without 14 * modification, are permitted provided that the following conditions 15 * are met: 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in the 20 * documentation and/or other materials provided with the distribution. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 35 #include <sys/cdefs.h> 36 #include "opt_ddb.h" 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/bus.h> 41 #include <sys/intr.h> 42 #include <sys/kernel.h> 43 #include <sys/ktr.h> 44 #include <sys/lock.h> 45 #include <sys/mutex.h> 46 #include <sys/proc.h> 47 #include <sys/ptrace.h> 48 #include <sys/syscall.h> 49 #include <sys/sysent.h> 50 #ifdef KDB 51 #include <sys/kdb.h> 52 #endif 53 54 #include <vm/vm.h> 55 #include <vm/pmap.h> 56 #include <vm/vm_kern.h> 57 #include <vm/vm_map.h> 58 #include <vm/vm_param.h> 59 #include <vm/vm_extern.h> 60 61 #include <machine/fpe.h> 62 #include <machine/frame.h> 63 #include <machine/pcb.h> 64 #include <machine/pcpu.h> 65 66 #include <machine/resource.h> 67 68 #ifdef KDTRACE_HOOKS 69 #include <sys/dtrace_bsd.h> 70 #endif 71 72 #ifdef DDB 73 #include <ddb/ddb.h> 74 #include <ddb/db_sym.h> 75 #endif 76 77 int (*dtrace_invop_jump_addr)(struct trapframe *); 78 79 /* Called from exception.S */ 80 void do_trap_supervisor(struct trapframe *); 81 void do_trap_user(struct trapframe *); 82 83 static __inline void 84 call_trapsignal(struct thread *td, int sig, int code, void *addr, int trapno) 85 { 86 ksiginfo_t ksi; 87 88 ksiginfo_init_trap(&ksi); 89 ksi.ksi_signo = sig; 90 ksi.ksi_code = code; 91 ksi.ksi_addr = addr; 92 ksi.ksi_trapno = trapno; 93 trapsignal(td, &ksi); 94 } 95 96 int 97 cpu_fetch_syscall_args(struct thread *td) 98 { 99 struct proc *p; 100 syscallarg_t *ap, *dst_ap; 101 struct syscall_args *sa; 102 103 p = td->td_proc; 104 sa = &td->td_sa; 105 ap = &td->td_frame->tf_a[0]; 106 dst_ap = &sa->args[0]; 107 108 sa->code = td->td_frame->tf_t[0]; 109 sa->original_code = sa->code; 110 111 if (__predict_false(sa->code == SYS_syscall || sa->code == SYS___syscall)) { 112 sa->code = *ap++; 113 } else { 114 *dst_ap++ = *ap++; 115 } 116 117 if (__predict_false(sa->code >= p->p_sysent->sv_size)) 118 sa->callp = &nosys_sysent; 119 else 120 sa->callp = &p->p_sysent->sv_table[sa->code]; 121 122 KASSERT(sa->callp->sy_narg <= nitems(sa->args), 123 ("Syscall %d takes too many arguments", sa->code)); 124 125 memcpy(dst_ap, ap, (NARGREG - 1) * sizeof(*dst_ap)); 126 127 td->td_retval[0] = 0; 128 td->td_retval[1] = 0; 129 130 return (0); 131 } 132 133 #include "../../kern/subr_syscall.c" 134 135 static void 136 print_with_symbol(const char *name, uint64_t value) 137 { 138 #ifdef DDB 139 c_db_sym_t sym; 140 db_expr_t sym_value; 141 db_expr_t offset; 142 const char *sym_name; 143 #endif 144 145 printf("%7s: 0x%016lx", name, value); 146 147 #ifdef DDB 148 if (value >= VM_MIN_KERNEL_ADDRESS) { 149 sym = db_search_symbol(value, DB_STGY_ANY, &offset); 150 if (sym != C_DB_SYM_NULL) { 151 db_symbol_values(sym, &sym_name, &sym_value); 152 if (offset != 0) 153 printf(" (%s + 0x%lx)", sym_name, offset); 154 else 155 printf(" (%s)", sym_name); 156 } 157 } 158 #endif 159 printf("\n"); 160 } 161 162 static void 163 dump_regs(struct trapframe *frame) 164 { 165 char name[6]; 166 int i; 167 168 for (i = 0; i < nitems(frame->tf_t); i++) { 169 snprintf(name, sizeof(name), "t[%d]", i); 170 print_with_symbol(name, frame->tf_t[i]); 171 } 172 173 for (i = 0; i < nitems(frame->tf_s); i++) { 174 snprintf(name, sizeof(name), "s[%d]", i); 175 print_with_symbol(name, frame->tf_s[i]); 176 } 177 178 for (i = 0; i < nitems(frame->tf_a); i++) { 179 snprintf(name, sizeof(name), "a[%d]", i); 180 print_with_symbol(name, frame->tf_a[i]); 181 } 182 183 print_with_symbol("ra", frame->tf_ra); 184 print_with_symbol("sp", frame->tf_sp); 185 print_with_symbol("gp", frame->tf_gp); 186 print_with_symbol("tp", frame->tf_tp); 187 print_with_symbol("sepc", frame->tf_sepc); 188 printf("sstatus: 0x%016lx\n", frame->tf_sstatus); 189 printf("stval : 0x%016lx\n", frame->tf_stval); 190 } 191 192 static void 193 ecall_handler(void) 194 { 195 struct thread *td; 196 197 td = curthread; 198 199 syscallenter(td); 200 syscallret(td); 201 } 202 203 static void 204 page_fault_handler(struct trapframe *frame, int usermode) 205 { 206 struct vm_map *map; 207 uint64_t stval; 208 struct thread *td; 209 struct pcb *pcb; 210 vm_prot_t ftype; 211 vm_offset_t va; 212 struct proc *p; 213 int error, sig, ucode; 214 #ifdef KDB 215 bool handled; 216 #endif 217 218 #ifdef KDB 219 if (kdb_active) { 220 kdb_reenter(); 221 return; 222 } 223 #endif 224 225 td = curthread; 226 p = td->td_proc; 227 pcb = td->td_pcb; 228 stval = frame->tf_stval; 229 230 if (usermode) { 231 if (!VIRT_IS_VALID(stval)) { 232 call_trapsignal(td, SIGSEGV, SEGV_MAPERR, (void *)stval, 233 frame->tf_scause & SCAUSE_CODE); 234 goto done; 235 } 236 map = &p->p_vmspace->vm_map; 237 } else { 238 /* 239 * Enable interrupts for the duration of the page fault. For 240 * user faults this was done already in do_trap_user(). 241 */ 242 if ((frame->tf_sstatus & SSTATUS_SIE) != 0) 243 intr_enable(); 244 245 if (stval >= VM_MIN_KERNEL_ADDRESS) { 246 map = kernel_map; 247 } else { 248 if (pcb->pcb_onfault == 0) 249 goto fatal; 250 map = &p->p_vmspace->vm_map; 251 } 252 } 253 254 va = trunc_page(stval); 255 256 if (frame->tf_scause == SCAUSE_STORE_PAGE_FAULT) { 257 ftype = VM_PROT_WRITE; 258 } else if (frame->tf_scause == SCAUSE_INST_PAGE_FAULT) { 259 ftype = VM_PROT_EXECUTE; 260 } else { 261 ftype = VM_PROT_READ; 262 } 263 264 if (VIRT_IS_VALID(va) && pmap_fault(map->pmap, va, ftype)) 265 goto done; 266 267 if (td->td_critnest != 0 || td->td_intr_nesting_level != 0 || 268 WITNESS_CHECK(WARN_SLEEPOK | WARN_GIANTOK, NULL, 269 "Kernel page fault") != 0) 270 goto fatal; 271 272 error = vm_fault_trap(map, va, ftype, VM_FAULT_NORMAL, &sig, &ucode); 273 if (error != KERN_SUCCESS) { 274 if (usermode) { 275 call_trapsignal(td, sig, ucode, (void *)stval, 276 frame->tf_scause & SCAUSE_CODE); 277 } else { 278 if (pcb->pcb_onfault != 0) { 279 frame->tf_a[0] = error; 280 frame->tf_sepc = pcb->pcb_onfault; 281 return; 282 } 283 goto fatal; 284 } 285 } 286 287 done: 288 if (usermode) 289 userret(td, frame); 290 return; 291 292 fatal: 293 dump_regs(frame); 294 #ifdef KDB 295 if (debugger_on_trap) { 296 kdb_why = KDB_WHY_TRAP; 297 handled = kdb_trap(frame->tf_scause & SCAUSE_CODE, 0, frame); 298 kdb_why = KDB_WHY_UNSET; 299 if (handled) 300 return; 301 } 302 #endif 303 panic("Fatal page fault at %#lx: %#lx", frame->tf_sepc, stval); 304 } 305 306 void 307 do_trap_supervisor(struct trapframe *frame) 308 { 309 uint64_t exception; 310 311 /* Ensure we came from supervisor mode, interrupts disabled */ 312 KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) == 313 SSTATUS_SPP, ("Came from S mode with interrupts enabled")); 314 315 KASSERT((csr_read(sstatus) & (SSTATUS_SUM)) == 0, 316 ("Came from S mode with SUM enabled")); 317 318 exception = frame->tf_scause & SCAUSE_CODE; 319 if ((frame->tf_scause & SCAUSE_INTR) != 0) { 320 /* Interrupt */ 321 intr_irq_handler(frame, INTR_ROOT_IRQ); 322 return; 323 } 324 325 #ifdef KDTRACE_HOOKS 326 if (dtrace_trap_func != NULL && (*dtrace_trap_func)(frame, exception)) 327 return; 328 #endif 329 330 CTR4(KTR_TRAP, "%s: exception=%lu, sepc=%#lx, stval=%#lx", __func__, 331 exception, frame->tf_sepc, frame->tf_stval); 332 333 switch (exception) { 334 case SCAUSE_LOAD_ACCESS_FAULT: 335 case SCAUSE_STORE_ACCESS_FAULT: 336 case SCAUSE_INST_ACCESS_FAULT: 337 dump_regs(frame); 338 panic("Memory access exception at %#lx: %#lx", 339 frame->tf_sepc, frame->tf_stval); 340 break; 341 case SCAUSE_LOAD_MISALIGNED: 342 case SCAUSE_STORE_MISALIGNED: 343 case SCAUSE_INST_MISALIGNED: 344 dump_regs(frame); 345 panic("Misaligned address exception at %#lx: %#lx", 346 frame->tf_sepc, frame->tf_stval); 347 break; 348 case SCAUSE_STORE_PAGE_FAULT: 349 case SCAUSE_LOAD_PAGE_FAULT: 350 case SCAUSE_INST_PAGE_FAULT: 351 page_fault_handler(frame, 0); 352 break; 353 case SCAUSE_BREAKPOINT: 354 #ifdef KDTRACE_HOOKS 355 if (dtrace_invop_jump_addr != NULL && 356 dtrace_invop_jump_addr(frame) == 0) 357 break; 358 #endif 359 #ifdef KDB 360 kdb_trap(exception, 0, frame); 361 #else 362 dump_regs(frame); 363 panic("No debugger in kernel."); 364 #endif 365 break; 366 case SCAUSE_ILLEGAL_INSTRUCTION: 367 dump_regs(frame); 368 panic("Illegal instruction 0x%0*lx at %#lx", 369 (frame->tf_stval & 0x3) != 0x3 ? 4 : 8, 370 frame->tf_stval, frame->tf_sepc); 371 break; 372 default: 373 dump_regs(frame); 374 panic("Unknown kernel exception %#lx trap value %#lx", 375 exception, frame->tf_stval); 376 } 377 } 378 379 void 380 do_trap_user(struct trapframe *frame) 381 { 382 uint64_t exception; 383 struct thread *td; 384 struct pcb *pcb; 385 386 td = curthread; 387 pcb = td->td_pcb; 388 389 KASSERT(td->td_frame == frame, 390 ("%s: td_frame %p != frame %p", __func__, td->td_frame, frame)); 391 392 /* Ensure we came from usermode, interrupts disabled */ 393 KASSERT((csr_read(sstatus) & (SSTATUS_SPP | SSTATUS_SIE)) == 0, 394 ("Came from U mode with interrupts enabled")); 395 396 KASSERT((csr_read(sstatus) & (SSTATUS_SUM)) == 0, 397 ("Came from U mode with SUM enabled")); 398 399 exception = frame->tf_scause & SCAUSE_CODE; 400 if ((frame->tf_scause & SCAUSE_INTR) != 0) { 401 /* Interrupt */ 402 intr_irq_handler(frame, INTR_ROOT_IRQ); 403 return; 404 } 405 intr_enable(); 406 407 CTR4(KTR_TRAP, "%s: exception=%lu, sepc=%#lx, stval=%#lx", __func__, 408 exception, frame->tf_sepc, frame->tf_stval); 409 410 switch (exception) { 411 case SCAUSE_LOAD_ACCESS_FAULT: 412 case SCAUSE_STORE_ACCESS_FAULT: 413 case SCAUSE_INST_ACCESS_FAULT: 414 call_trapsignal(td, SIGBUS, BUS_ADRERR, (void *)frame->tf_sepc, 415 exception); 416 userret(td, frame); 417 break; 418 case SCAUSE_LOAD_MISALIGNED: 419 case SCAUSE_STORE_MISALIGNED: 420 case SCAUSE_INST_MISALIGNED: 421 call_trapsignal(td, SIGBUS, BUS_ADRALN, (void *)frame->tf_sepc, 422 exception); 423 userret(td, frame); 424 break; 425 case SCAUSE_STORE_PAGE_FAULT: 426 case SCAUSE_LOAD_PAGE_FAULT: 427 case SCAUSE_INST_PAGE_FAULT: 428 page_fault_handler(frame, 1); 429 break; 430 case SCAUSE_ECALL_USER: 431 frame->tf_sepc += 4; /* Next instruction */ 432 ecall_handler(); 433 break; 434 case SCAUSE_ILLEGAL_INSTRUCTION: 435 if ((pcb->pcb_fpflags & PCB_FP_STARTED) == 0) { 436 /* 437 * May be a FPE trap. Enable FPE usage 438 * for this thread and try again. 439 */ 440 fpe_state_clear(); 441 frame->tf_sstatus &= ~SSTATUS_FS_MASK; 442 frame->tf_sstatus |= SSTATUS_FS_CLEAN; 443 pcb->pcb_fpflags |= PCB_FP_STARTED; 444 break; 445 } 446 call_trapsignal(td, SIGILL, ILL_ILLTRP, (void *)frame->tf_sepc, 447 exception); 448 userret(td, frame); 449 break; 450 case SCAUSE_BREAKPOINT: 451 call_trapsignal(td, SIGTRAP, TRAP_BRKPT, (void *)frame->tf_sepc, 452 exception); 453 userret(td, frame); 454 break; 455 default: 456 dump_regs(frame); 457 panic("Unknown userland exception %#lx, trap value %#lx", 458 exception, frame->tf_stval); 459 } 460 } 461