1/* $NetBSD: amd64_trap.S,v 1.55 2023/02/27 16:24:28 riastradh Exp $ */ 2 3/* 4 * Copyright (c) 1998, 2007, 2008, 2017 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Charles M. Hannum, by Andrew Doran and by Maxime Villard. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32/* 33 * Copyright (c) 2001 Wasabi Systems, Inc. 34 * All rights reserved. 35 * 36 * Written by Frank van der Linden for Wasabi Systems, Inc. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed for the NetBSD Project by 49 * Wasabi Systems, Inc. 50 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 51 * or promote products derived from this software without specific prior 52 * written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 55 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67#include <machine/asm.h> 68 69#include "opt_xen.h" 70#include "opt_dtrace.h" 71 72#define ALIGN_TEXT .align 16,0x90 73 74#include <machine/frameasm.h> 75#include <machine/segments.h> 76#include <machine/trap.h> 77#include <machine/specialreg.h> 78 79#include "assym.h" 80 81/* 82 * Trap and fault vector routines 83 * 84 * On exit from the kernel to user mode, we always need to check for ASTs. In 85 * addition, we need to do this atomically; otherwise an interrupt may occur 86 * which causes an AST, but it won't get processed until the next kernel entry 87 * (possibly the next clock tick). Thus, we disable interrupt before checking, 88 * and only enable them again on the final `iret' or before calling the AST 89 * handler. 90 */ 91 92#ifdef XENPV 93#define PRE_TRAP CLI(cx); movq (%rsp),%rcx ; movq 8(%rsp),%r11 ; addq $0x10,%rsp 94#else 95#define PRE_TRAP 96#endif 97 98#define TRAPENTRY \ 99 INTRENTRY ; \ 100 jmp .Lalltraps_noentry 101 102#define TRAP_NJ(a) PRE_TRAP ; pushq $(a) 103#define ZTRAP_NJ(a) PRE_TRAP ; pushq $0 ; pushq $(a) 104#define TRAP(a) TRAP_NJ(a) ; TRAPENTRY 105#define ZTRAP(a) ZTRAP_NJ(a) ; TRAPENTRY 106 107 .text 108 109/* 110 * ASM macro, used to leave the IST3 stack and to put ourselves on a non-IST 111 * stack. Only RDX, RCX and RAX are allowed to be used. 112 * 113 * +------------------------------+ 114 * The iret frame we copy is: | rip | cs | rflags | rsp | ss | 115 * +------------------------------+ 116 */ 117.macro IST3_LEAVE is_user 118 .if \is_user 119 movq CPUVAR(CURLWP),%rax 120 movq L_PCB(%rax),%rax 121 movq PCB_RSP0(%rax),%rax 122 .else 123 movq TF_RSP(%rsp),%rax 124 andq $(~0xF),%rax 125 .endif 126 127 subq $(5*8),%rax 128 movq %rax,CPUVAR(SCRATCH) 129 130 /* Copy the iret frame. */ 131 movq TF_SS(%rsp),%rcx 132 movq %rcx,(4*8)(%rax) 133 movq TF_RSP(%rsp),%rcx 134 movq %rcx,(3*8)(%rax) 135 movq TF_RFLAGS(%rsp),%rcx 136 movq %rcx,(2*8)(%rax) 137 movq TF_CS(%rsp),%rcx 138 movq %rcx,(1*8)(%rax) 139 movq TF_RIP(%rsp),%rcx 140 movq %rcx,(0*8)(%rax) 141 142 /* Restore. */ 143 movq TF_RDX(%rsp),%rdx 144 movq TF_RCX(%rsp),%rcx 145 movq TF_RAX(%rsp),%rax 146 147 /* Zero out the stack we used, RDX+RCX+RAX+IRET. */ 148 movq $0,TF_RDX(%rsp) 149 movq $0,TF_RCX(%rsp) 150 movq $0,TF_RAX(%rsp) 151 movq $0,TF_RIP(%rsp) 152 movq $0,TF_CS(%rsp) 153 movq $0,TF_RFLAGS(%rsp) 154 movq $0,TF_RSP(%rsp) 155 movq $0,TF_SS(%rsp) 156 157 movq CPUVAR(SCRATCH),%rsp 158.endm 159 160 TEXT_USER_BEGIN 161 162IDTVEC(trap00) /* #DE - Divide-by-zero error */ 163 ZTRAP(T_DIVIDE) 164IDTVEC_END(trap00) 165 166/* 167 * Handle the SS shadow, CVE-2018-8897. 168 * 169 * We are running on the IST3 stack. If we are under an SS shadow, ignore 170 * the exception and return immediately. Otherwise, copy the iret frame 171 * onto the non-IST stack, and ZTRAP on it as usual. 172 * 173 * IST3 is used temporarily, and is mapped in userland by SVS. It contains 174 * a few secrets, the values of the CPU context. These secrets are zeroed 175 * out when we leave. 176 * 177 * When we ignore an SS shadow, we can't zero out the iret frame. It is 178 * not a problem, because in this particular case, the frame is known not 179 * to contain secrets. 180 */ 181IDTVEC(trap01) /* #DB - Debug */ 182#ifndef XENPV 183 subq $(TF_REGSIZE+16),%rsp 184 185 /* We clobber only RDX, RCX and RAX. */ 186 movq %rdx,TF_RDX(%rsp) 187 movq %rcx,TF_RCX(%rsp) 188 movq %rax,TF_RAX(%rsp) 189 190 testb $SEL_UPL,TF_CS(%rsp) 191 jnz .Luser_dbentry 192 193 movl $MSR_GSBASE,%ecx 194 rdmsr 195 cmpl $VM_SPACE_SEP_HIGH32,%edx 196 jae .Lkern_dbentry 197 198 /* SS shadow, ignore the exception. */ 199 xorq %rax,%rax 200 movq %rax,%dr6 201 202 /* Restore and zero out. */ 203 movq TF_RDX(%rsp),%rdx 204 movq TF_RCX(%rsp),%rcx 205 movq TF_RAX(%rsp),%rax 206 movq $0,TF_RDX(%rsp) 207 movq $0,TF_RCX(%rsp) 208 movq $0,TF_RAX(%rsp) 209 210 addq $(TF_REGSIZE+16),%rsp 211 iretq 212 213.Lkern_dbentry: 214 IST3_LEAVE 0 215 ZTRAP(T_TRCTRAP) 216 217.Luser_dbentry: 218 swapgs 219 SVS_ENTER_ALTSTACK 220 IST3_LEAVE 1 221 ZTRAP_NJ(T_TRCTRAP) 222 subq $TF_REGSIZE,%rsp 223 INTR_SAVE_GPRS 224 cld 225 SMAP_ENABLE 226 IBRS_ENTER 227 KMSAN_ENTER 228 movw %gs,TF_GS(%rsp) 229 movw %fs,TF_FS(%rsp) 230 movw %es,TF_ES(%rsp) 231 movw %ds,TF_DS(%rsp) 232 233 jmp .Lalltraps_noentry 234#else /* !XENPV */ 235 ZTRAP(T_TRCTRAP) 236#endif /* !XENPV */ 237IDTVEC_END(trap01) 238 239/* 240 * Non Maskable Interrupts are a special case: they can be triggered even 241 * with interrupts disabled, and once triggered they block further NMIs 242 * until an 'iret' instruction is executed. 243 * 244 * Therefore we don't enable interrupts, because the CPU could switch to 245 * another LWP, call 'iret' and unintentionally leave the NMI mode. 246 * 247 * We need to be careful about %gs too, because it is possible that we were 248 * running in kernel mode with a userland %gs. 249 */ 250IDTVEC(trap02) /* NMI - Non-maskable interrupt */ 251#if defined(XENPV) 252 ZTRAP(T_NMI) 253#else /* XENPV */ 254 ZTRAP_NJ(T_NMI) 255 subq $TF_REGSIZE,%rsp 256 INTR_SAVE_GPRS 257 258 testb $SEL_UPL,TF_CS(%rsp) 259 jz 1f 260 IBRS_ENTER 2611: 262 263 cld 264 SMAP_ENABLE 265 movw %gs,TF_GS(%rsp) 266 movw %fs,TF_FS(%rsp) 267 movw %es,TF_ES(%rsp) 268 movw %ds,TF_DS(%rsp) 269 270 SVS_ENTER_NMI 271 KMSAN_ENTER 272 273 movl $MSR_GSBASE,%ecx 274 rdmsr 275 cmpl $VM_SPACE_SEP_HIGH32,%edx 276 jae .Lnoswapgs 277 278 swapgs 279 movq %rsp,%rdi 280 incq CPUVAR(NTRAP) 281 call _C_LABEL(nmitrap) 282 swapgs 283 jmp .Lnmileave 284 285.Lnoswapgs: 286 movq %rsp,%rdi 287 incq CPUVAR(NTRAP) 288 call _C_LABEL(nmitrap) 289 290.Lnmileave: 291 testb $SEL_UPL,TF_CS(%rsp) 292 jz 1f 293 MDS_LEAVE 294 IBRS_LEAVE 2951: 296 297 KMSAN_LEAVE 298 SVS_LEAVE_NMI 299 INTR_RESTORE_GPRS 300 addq $TF_REGSIZE+16,%rsp 301 iretq 302#endif /* XENPV */ 303IDTVEC_END(trap02) 304 305IDTVEC(trap03) /* #BP - Breakpoint */ 306#ifndef KDTRACE_HOOKS 307 ZTRAP(T_BPTFLT) 308#else 309 ZTRAP_NJ(T_BPTFLT) 310 INTRENTRY 311 STI(si) 312 /* 313 * DTrace Function Boundary Trace (fbt) probes are triggered 314 * by int3 (0xcc). 315 */ 316 /* Check if there is no DTrace hook registered. */ 317 cmpq $0,dtrace_invop_jump_addr 318 je calltrap 319 320 /* 321 * Set our jump address for the jump back in the event that 322 * the exception wasn't caused by DTrace at all. 323 */ 324 /* XXX: This doesn't look right for SMP - unless it is a 325 * constant - so why set it everytime. (dsl) */ 326 movq $calltrap, dtrace_invop_calltrap_addr(%rip) 327 328 /* Jump to the code hooked in by DTrace. */ 329 movq dtrace_invop_jump_addr, %rax 330 jmpq *dtrace_invop_jump_addr 331#endif 332IDTVEC_END(trap03) 333 334IDTVEC(trap04) /* #OF - Overflow */ 335 ZTRAP(T_OFLOW) 336IDTVEC_END(trap04) 337 338IDTVEC(trap05) /* #BR - BOUND range exceeded */ 339 ZTRAP(T_BOUND) 340IDTVEC_END(trap05) 341 342IDTVEC(trap06) /* #UD - Invalid opcode */ 343 ZTRAP(T_PRIVINFLT) 344IDTVEC_END(trap06) 345 346IDTVEC(trap07) /* #NM - Device not available (x87) */ 347 ZTRAP_NJ(T_DNA) 348 INTRENTRY 349#ifdef DIAGNOSTIC 350 movzbl CPUVAR(ILEVEL),%ebx 351#endif 352 movq %rsp,%rdi 353 call _C_LABEL(fpudna) 354 jmp .Lalltraps_checkusr 355IDTVEC_END(trap07) 356 357/* 358 * Double faults execute on a particular stack, and we must not jump out 359 * of it. So don't enable interrupts. 360 */ 361IDTVEC(trap08) /* #DF - Double fault */ 362#if defined(XENPV) 363 TRAP(T_DOUBLEFLT) 364#else /* XENPV */ 365 TRAP_NJ(T_DOUBLEFLT) 366 subq $TF_REGSIZE,%rsp 367 INTR_SAVE_GPRS 368 369 testb $SEL_UPL,TF_CS(%rsp) 370 jz 1f 371 IBRS_ENTER 372 swapgs 3731: 374 375 SVS_ENTER_ALTSTACK 376 377 cld 378 SMAP_ENABLE 379 movw %gs,TF_GS(%rsp) 380 movw %fs,TF_FS(%rsp) 381 movw %es,TF_ES(%rsp) 382 movw %ds,TF_DS(%rsp) 383 384 movq %rsp,%rdi 385 incq CPUVAR(NTRAP) 386 call _C_LABEL(doubletrap) 387 388 testb $SEL_UPL,TF_CS(%rsp) 389 jz 1f 390 MDS_LEAVE 391 SVS_LEAVE_ALTSTACK 392 IBRS_LEAVE 393 swapgs 3941: 395 396 INTR_RESTORE_GPRS 397 addq $TF_REGSIZE+16,%rsp 398 iretq 399#endif /* XENPV */ 400IDTVEC_END(trap08) 401 402IDTVEC(trap09) /* Coprocessor segment overrun (legacy x87) */ 403 ZTRAP(T_FPOPFLT) 404IDTVEC_END(trap09) 405 406IDTVEC(trap10) /* #TS - Invalid TSS */ 407 TRAP(T_TSSFLT) 408IDTVEC_END(trap10) 409 410#ifdef XENPV 411/* 412 * I don't believe XEN generates in-kernel traps for the 413 * equivalent of iret, if it does this code would be needed 414 * in order to copy the user segment registers into the fault frame. 415 */ 416#define kernuser_reenter alltraps 417#endif /* XENPV */ 418 419IDTVEC(trap11) /* #NP - Segment not present */ 420 TRAP_NJ(T_SEGNPFLT) 421 jmp kernuser_reenter 422IDTVEC_END(trap11) 423 424IDTVEC(trap12) /* #SS - Stack fault */ 425 TRAP_NJ(T_STKFLT) 426 jmp kernuser_reenter 427IDTVEC_END(trap12) 428 429IDTVEC(trap13) /* #GP - General protection */ 430 TRAP_NJ(T_PROTFLT) 431 jmp kernuser_reenter 432IDTVEC_END(trap13) 433 434IDTVEC(trap14) /* #PF - Page fault */ 435 TRAP(T_PAGEFLT) 436IDTVEC_END(trap14) 437 438IDTVEC(trap15) /* XXX ??? */ 439 ZTRAP_NJ(T_ASTFLT) 440 INTRENTRY 441#ifdef DIAGNOSTIC 442 movzbl CPUVAR(ILEVEL),%ebx 443#endif 444 jmp .Lalltraps_checkusr 445IDTVEC_END(trap15) 446 447IDTVEC(trap16) /* #MF - x87 floating-point exception */ 448 ZTRAP_NJ(T_ARITHTRAP) 449.Ldo_fputrap: 450 INTRENTRY 451#ifdef XENPV 452 /* traps are called with interrupts enabled, and we may have been 453 * interrupted just before the CLI in the trap macro. 454 * we have to check if a FPU reload is needed. 455 */ 456 movq CPUVAR(CURLWP),%r14 457 HANDLE_DEFERRED_FPU 458#endif /* XENPV */ 459#ifdef DIAGNOSTIC 460 movzbl CPUVAR(ILEVEL),%ebx 461#endif 462 movq %rsp,%rdi 463 call _C_LABEL(fputrap) 464 jmp .Lalltraps_checkusr 465IDTVEC_END(trap16) 466 467IDTVEC(trap17) /* #AC - Alignment check */ 468 TRAP(T_ALIGNFLT) 469IDTVEC_END(trap17) 470 471IDTVEC(trap18) /* #MC - Machine check */ 472 ZTRAP(T_MCA) 473IDTVEC_END(trap18) 474 475IDTVEC(trap19) /* #XM - SIMD floating-point exception */ 476 ZTRAP_NJ(T_XMM) 477 jmp .Ldo_fputrap 478IDTVEC_END(trap19) 479 480IDTVEC(trap20) /* #VE - Virtualization (Intel) */ 481IDTVEC(trap21) /* #CP - Control protection */ 482IDTVEC(trap22) 483IDTVEC(trap23) 484IDTVEC(trap24) 485IDTVEC(trap25) 486IDTVEC(trap26) 487IDTVEC(trap27) 488IDTVEC(trap28) /* #HV - Hypervisor injection (AMD) */ 489IDTVEC(trap29) /* #VC - VMM communication (AMD) */ 490IDTVEC(trap30) /* #SX - Security (AMD) */ 491IDTVEC(trap31) 492 /* 20 - 31 reserved for future exp */ 493 ZTRAP(T_RESERVED) 494IDTVEC_END(trap20) 495IDTVEC_END(trap21) 496IDTVEC_END(trap22) 497IDTVEC_END(trap23) 498IDTVEC_END(trap24) 499IDTVEC_END(trap25) 500IDTVEC_END(trap26) 501IDTVEC_END(trap27) 502IDTVEC_END(trap28) 503IDTVEC_END(trap29) 504IDTVEC_END(trap30) 505IDTVEC_END(trap31) 506 507IDTVEC(intrspurious) 508 ZTRAP_NJ(T_ASTFLT) 509 INTRENTRY 510#ifdef DIAGNOSTIC 511 movzbl CPUVAR(ILEVEL),%ebx 512#endif 513 jmp .Lalltraps_checkusr 514IDTVEC_END(intrspurious) 515 516#ifndef kernuser_reenter 517/* 518 * We need to worry about traps in kernel mode while the kernel %gs isn't 519 * loaded. When such traps happen, we have CPL=0 and %gs=userland, and we 520 * must perform an additional swapgs to get %gs=kernel. 521 */ 522 523#define TF_SMALL(val, reg) (val - TF_REGSIZE)(reg) 524#define TF_SMALL_REGPUSHED(val, reg) (val - (TF_REGSIZE - 8))(reg) 525 526/* 527 * It is possible that we received a trap in kernel mode, but with the user 528 * context loaded. There are five cases where this can happen: 529 * 530 * o Execution of IRETQ. 531 * o Reload of ES. 532 * o Reload of DS. 533 * o Reload of FS. 534 * o Reload of GS. 535 * 536 * When this happens, the kernel is re-entered in kernel mode, but the 537 * previous context is in kernel mode too. 538 * 539 * We have two iret frames in the stack. In the first one, we also pushed 540 * 'trapno' and 'err'. The 'rsp' field points to the outer iret frame: 541 * 542 * +---------------------------------------------------+ 543 * | trapno | err | rip | cs=ring0 | rflags | rsp | ss | 544 * +-------------------------------------------|-------+ 545 * | 546 * +---------------------------------+ 547 * | 548 * | +------------------------------------+ 549 * +--> | rip | cs=ring3 | rflags | rsp | ss | 550 * +------------------------------------+ 551 * 552 * We perform a three-step procedure: 553 * 554 * o We update RSP to point to the outer frame. This outer frame is in the 555 * same stack as the current frame, and likely just after the current 556 * frame. 557 * 558 * o We push, in this outer frame, the 'err' and 'trapno' fields of the 559 * CURRENT frame. 560 * 561 * o We do a normal INTRENTRY. Now that RSP points to the outer frame, 562 * everything behaves as if we had received a trap from the outer frame, 563 * that is to say, from userland directly. 564 * 565 * Finally, we jump to 'calltrap' and handle the trap smoothly. 566 * 567 * Two notes regarding SVS: 568 * 569 * o With SVS, we will receive the trap while the user page tables are 570 * loaded. That's not a problem, we don't touch anything unmapped here. 571 * 572 * o With SVS, when the user page tables are loaded, the stack is really 573 * small, and can contain only one trapframe structure. Therefore, in 574 * intrfastexit, we must save the GPRs and pop their part of the stack 575 * right away. If we weren't doing that, and the reload of ES faulted for 576 * example, then the CPU would try to push an iret frame on the current 577 * stack (nested), and would double-fault because it touches the redzone 578 * below the stack (see the documentation in x86/x86/svs.c). By popping 579 * the GPR part of the stack, we leave enough stack for the CPU to push 580 * an iret frame, and for us to push one 8-byte register (%rdi) too. 581 */ 582 _ALIGN_TEXT 583LABEL(kernuser_reenter) 584 testb $SEL_UPL,TF_SMALL(TF_CS, %rsp) 585 jz .Lkernelmode 586 587.Lnormal_entry: 588 INTRENTRY 589 sti 590 jmp calltrap 591 592.Lkernelmode: 593 /* We will clobber %rdi */ 594 pushq %rdi 595 596 /* Case 1: fault on iretq? */ 597 leaq do_iret(%rip),%rdi 598 cmpq %rdi,TF_SMALL_REGPUSHED(TF_RIP, %rsp) 599 jne 5f 600 movq TF_SMALL_REGPUSHED(TF_RSP, %rsp),%rdi /* get %rsp */ 601 testb $SEL_UPL,8(%rdi) /* check %cs of outer iret frame */ 602 je .Lnormal_entry /* jump if iret was to kernel */ 603 jmp .Lkernelmode_but_user /* to user - must restore %gs */ 6045: 605 606 /* Case 2: move to %es? */ 607 leaq do_mov_es(%rip),%rdi 608 cmpq %rdi,TF_SMALL_REGPUSHED(TF_RIP, %rsp) 609 je .Lkernelmode_but_user 610 611 /* Case 3: move to %ds? */ 612 leaq do_mov_ds(%rip),%rdi 613 cmpq %rdi,TF_SMALL_REGPUSHED(TF_RIP, %rsp) 614 je .Lkernelmode_but_user 615 616 /* Case 4: move to %fs? */ 617 leaq do_mov_fs(%rip),%rdi 618 cmpq %rdi,TF_SMALL_REGPUSHED(TF_RIP, %rsp) 619 je .Lkernelmode_but_user 620 621 /* Case 5: move to %gs? */ 622 leaq do_mov_gs(%rip),%rdi 623 cmpq %rdi,TF_SMALL_REGPUSHED(TF_RIP, %rsp) 624 je .Lkernelmode_but_user 625 626 /* None of the above cases: normal kernel fault */ 627 popq %rdi 628 jmp .Lnormal_entry 629 630.Lkernelmode_but_user: 631 /* 632 * Here we have %rdi pushed on the stack, hence 8+. 633 */ 634 movq %rsp,%rdi 635 movq TF_SMALL_REGPUSHED(TF_RSP, %rsp),%rsp 636 637 /* Push tf_err and tf_trapno */ 638 pushq 8+8(%rdi) /* 8+8(%rdi) = current TF_ERR */ 639 pushq 8+0(%rdi) /* 8+0(%rdi) = current TF_TRAPNO */ 640 641 /* Restore %rdi */ 642 movq (%rdi),%rdi 643 644 jmp .Lnormal_entry 645END(kernuser_reenter) 646#endif 647 648 TEXT_USER_END 649 650/* 651 * All traps go through here. Call the generic trap handler, and 652 * check for ASTs afterwards. 653 */ 654ENTRY(alltraps) 655 INTRENTRY 656.Lalltraps_noentry: 657 STI(si) 658 659calltrap: 660#ifdef DIAGNOSTIC 661 movzbl CPUVAR(ILEVEL),%ebx 662#endif 663 movq %rsp,%rdi 664 incq CPUVAR(NTRAP) 665 call _C_LABEL(trap) 666 667.Lalltraps_checkusr: 668 testb $SEL_RPL,TF_CS(%rsp) 669 jz 6f 670 671.Lalltraps_checkast: 672 movq CPUVAR(CURLWP),%r14 673 /* Check for ASTs on exit to user mode. */ 674 CLI(si) 675 CHECK_ASTPENDING(%r14) 676 je 3f 677 CLEAR_ASTPENDING(%r14) 678 STI(si) 679 movl $T_ASTFLT,TF_TRAPNO(%rsp) 680 movq %rsp,%rdi 681 incq CPUVAR(NTRAP) 682 KMSAN_INIT_ARG(8) 683 call _C_LABEL(trap) 684 jmp .Lalltraps_checkast /* re-check ASTs */ 6853: CHECK_DEFERRED_SWITCH 686 jnz 9f 687 HANDLE_DEFERRED_FPU 688 6896: 690#ifdef DIAGNOSTIC 691 cmpb CPUVAR(ILEVEL),%bl 692 jne .Lspl_error 693#endif 694 INTRFASTEXIT 695 6969: STI(si) 697 call _C_LABEL(do_pmap_load) 698 jmp .Lalltraps_checkast /* re-check ASTs */ 699 700#ifdef DIAGNOSTIC 701.Lspl_error: 702 STI(si) 703 movabsq $4f,%rdi 704 movzbl CPUVAR(ILEVEL),%esi 705 call _C_LABEL(panic) 7064: .asciz "spl not lowered on trap exit, ilevel=%x" 707#endif 708END(alltraps) 709 710#ifdef KDTRACE_HOOKS 711 .bss 712 .globl dtrace_invop_jump_addr 713 .align 8 714 .type dtrace_invop_jump_addr, @object 715 .size dtrace_invop_jump_addr, 8 716dtrace_invop_jump_addr: 717 .zero 8 718 .globl dtrace_invop_calltrap_addr 719 .align 8 720 .type dtrace_invop_calltrap_addr, @object 721 .size dtrace_invop_calltrap_addr, 8 722dtrace_invop_calltrap_addr: 723 .zero 8 724#endif 725 726 .section .rodata 727 728LABEL(x86_exceptions) 729 .quad _C_LABEL(Xtrap00), _C_LABEL(Xtrap01) 730 .quad _C_LABEL(Xtrap02), _C_LABEL(Xtrap03) 731 .quad _C_LABEL(Xtrap04), _C_LABEL(Xtrap05) 732 .quad _C_LABEL(Xtrap06), _C_LABEL(Xtrap07) 733 .quad _C_LABEL(Xtrap08), _C_LABEL(Xtrap09) 734 .quad _C_LABEL(Xtrap10), _C_LABEL(Xtrap11) 735 .quad _C_LABEL(Xtrap12), _C_LABEL(Xtrap13) 736 .quad _C_LABEL(Xtrap14), _C_LABEL(Xtrap15) 737 .quad _C_LABEL(Xtrap16), _C_LABEL(Xtrap17) 738 .quad _C_LABEL(Xtrap18), _C_LABEL(Xtrap19) 739 .quad _C_LABEL(Xtrap20), _C_LABEL(Xtrap21) 740 .quad _C_LABEL(Xtrap22), _C_LABEL(Xtrap23) 741 .quad _C_LABEL(Xtrap24), _C_LABEL(Xtrap25) 742 .quad _C_LABEL(Xtrap26), _C_LABEL(Xtrap27) 743 .quad _C_LABEL(Xtrap28), _C_LABEL(Xtrap29) 744 .quad _C_LABEL(Xtrap30), _C_LABEL(Xtrap31) 745END(x86_exceptions) 746 747