xref: /dflybsd-src/sys/platform/pc64/x86_64/exception.S (revision add2647ee1fad67c3a55c30eb1190d71a9a0eea0)

/*-
 * Copyright (c) 1989, 1990 William F. Jolitz.
 * Copyright (c) 1990 The Regents of the University of California.
 * Copyright (c) 2007 The FreeBSD Foundation
 * Copyright (c) 2008 The DragonFly Project.
 * Copyright (c) 2008 Jordan Gordeev.
 * All rights reserved.
 *
 * Portions of this software were developed by A. Joseph Koshy under
 * sponsorship from the FreeBSD Foundation and Google, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 */

#if 0 /* JG */
#include "opt_atpic.h"
#endif

#include <machine/asmacros.h>
#include <machine/psl.h>
#include <machine/trap.h>
#include <machine/segments.h>

#include "assym.s"

	.text

	.globl	lwkt_switch_return

/*****************************************************************************/
/* Trap handling                                                             */
/*****************************************************************************/
/*
 * Trap and fault vector routines.
 *
 * All traps are 'interrupt gates', SDT_SYSIGT.  An interrupt gate pushes
 * state on the stack but also disables interrupts.  This is important for
 * us for the use of the swapgs instruction.  We cannot be interrupted
 * until the GS.base value is correct.  For most traps, we automatically
 * then enable interrupts if the interrupted context had them enabled.
 *
 * The cpu will push a certain amount of state onto the kernel stack for
 * the current process.  See x86_64/include/frame.h.
 * This includes the current RFLAGS (status register, which includes
 * the interrupt disable state prior to the trap), the code segment register,
 * and the return instruction pointer are pushed by the cpu.  The cpu
 * will also push an 'error' code for certain traps.  We push a dummy
 * error code for those traps where the cpu doesn't in order to maintain
 * a consistent frame.  We also push a contrived 'trap number'.
 *
 * The cpu does not push the general registers, we must do that, and we
 * must restore them prior to calling 'iret'.  The cpu adjusts the %cs and
 * %ss segment registers, but does not mess with %ds, %es, or %fs.  Thus we
 * must load them with appropriate values for supervisor mode operation.
 */

MCOUNT_LABEL(user)
MCOUNT_LABEL(btrap)

/*
 * Interrupts must be disabled for all traps, otherwise horrible %gs
 * issues will occur.
 */

/* Regular traps; The cpu does not supply tf_err for these. */
#define	TRAP(a)	 \
	PUSH_FRAME_TFRIP ;			\
	movq $0,TF_XFLAGS(%rsp) ;		\
	movq $(a),TF_TRAPNO(%rsp) ;		\
	movq $0,TF_ADDR(%rsp) ;			\
	movq $0,TF_ERR(%rsp) ;			\
	jmp alltraps

/* This group of traps have tf_err already pushed by the cpu */
#define	TRAP_ERR(a)				\
	PUSH_FRAME_TFERR ;			\
	movq $(a),TF_TRAPNO(%rsp) ;		\
	movq $0,TF_ADDR(%rsp) ;			\
	movq $0,TF_XFLAGS(%rsp) ;		\
	jmp alltraps

/*
 * Due to a historical artifact, it is possible for a #DB exception
 * to occur in certain bad places that would normlally be protected by
 * the interrupt gate's interrupt disablement.
 *
 * Due to this possibly occuring in the system call entry code, we also
 * run #DB on an ist2 stack to force the cpu to load a new %rsp, otherwise
 * it might push the cpu exception frame onto the user stack.  To make things
 * easier we just point ist2 at our trampoline area.
 */
IDTVEC(dbg)
#ifdef DIRECT_DISALLOW_SS_CPUBUG
	/*
	 * Directly disallow #DB faults which can occur at critical points
	 * in the code due to a historical artifact of how the cpu operates.
	 * %gs state might not match RPL.  Test the %rip and iretq immediately
	 * (valid %gs and %cr3 state not needed).  If we don't need kernel
	 * reporting we can enable this and its a bit safer from unintended
	 * consequences.
	 *
	 * If this is not enabled the kernel still catches the problem.  It
	 * will report the problem and continue properly.
	 */
	cmpq	$Xbpt,0(%rsp)
	je	200f
	cmpq	$Xfast_syscall,0(%rsp)
	je	200f
#endif

	/*
	 * Ok, regardless of the RPL mask in the trap frame, we took
	 * the trap on a separate stack via ist2.  This means we
	 * must copy it appropriately.
	 *
	 * If coming from userland we can skip directly to the normal
	 * TRAP code because it will handle the fact that we are on an
	 * alternative stack (dbgstack set by ist2), even though it isn't
	 * the trampoline stack).  The frame will be moved to the correct
	 * kernel stack.
	 */
	testb   $SEL_RPL_MASK,TF_CS-TF_RIP(%rsp)
	jnz	210f				/* jnz from userland */

	/*
	 * From kernel - %gs and %cr3 may be inconsistent.  Save original
	 * values and load consistent values, restore after return.
	 *
	 * The trap handler is NOT allowed to block for this case.
	 */
	subq	$TR_RIP, %rsp
	movq	%rax, TR_RAX(%rsp)
	movq	%rcx, TR_RCX(%rsp)
	movq	%rdx, TR_RDX(%rsp)

	cld
	movq	%cr3,%rax			/* save CR3 */
	movq	%rax, TR_PCB_CR3_SAVED(%rsp)
	movl	$MSR_GSBASE,%ecx		/* save %gs */
	rdmsr
	shlq	$32,%rdx
	orq	%rdx,%rax
	movq	%rax, TR_PCB_GS_SAVED(%rsp)
	movq	TR_PCB_GS_KERNEL(%rsp),%rdx	/* retrieve kernel %gs */
	movl	%edx,%eax
	shrq	$32,%rdx
	wrmsr
	movq	PCPU(trampoline)+TR_PCB_CR3,%rax
	movq	%rax,%cr3

	movq	TR_RDX(%rsp), %rdx
	movq	TR_RCX(%rsp), %rcx
	movq	TR_RAX(%rsp), %rax
	addq	$TR_RIP, %rsp

	/*
	 * We are coming from the kernel.
	 *
	 * We are on the IST2 stack and, in fact, we have to *STAY* on this
	 * stack so no longer try to shift our frame to the kernel %rsp
	 * in the trap frame, since this %rsp might actually be a user %rsp
	 * in the mov mem,%ss + syscall DBG trap case.
	 *
	 * Run the normal trap.  Because TF_CS is at a kernel RPL, the
	 * normal code will skip the usual swapgs and KMMU (trampoline)
	 * code.  We've handled the rest.
	 *
	 * NOTE: at this point the trampframe is above the normal stack
	 *	 frame.  The trap code will be ignorant of the special
	 *	 TR_* registers above the cpu hardware frame portion,
	 *	 and the TR_* registers below it will be overwritten.
	 */
	PUSH_FRAME_TFRIP
	movq	$0,TF_XFLAGS(%rsp)
	movq	$T_TRCTRAP,TF_TRAPNO(%rsp)
	movq	$0,TF_ADDR(%rsp)
	movq	$0,TF_ERR(%rsp)

	FAKE_MCOUNT(TF_RIP(%rsp))
	cld
	movq	%rsp, %rdi
	call	trap
	MEXITCOUNT

	/*
	 * Pop the frame (since we're coming from kernel mode, this will
	 * not mess with %cr3 or %gs), then restore %cr3 and %gs for our
	 * iretq.  Not optimal but more readable and this is not a
	 * critical path.
	 */
	POP_FRAME(nop)

	subq	$TR_RIP, %rsp
	movq	%rax, TR_RAX(%rsp)
	movq	%rcx, TR_RCX(%rsp)
	movq	%rdx, TR_RDX(%rsp)

	movl	$MSR_GSBASE,%ecx		/* restore %gs */
	movq	TR_PCB_GS_SAVED(%rsp),%rdx
	movl	%edx,%eax
	shrq	$32,%rdx
	wrmsr

	movq	TR_PCB_CR3_SAVED(%rsp),%rax	/* restore %cr3 */
	movq	%rax,%cr3

	movq	TR_RAX(%rsp),%rax
	movq	TR_RCX(%rsp),%rcx
	movq	TR_RDX(%rsp),%rdx
	addq	$TR_RIP, %rsp

	/*
	 * Direct iretq. No point jumping to doreti because the
	 * exception code that deals with iretq faults can't handle
	 * non-deterministic %gs/%cr3 state.
	 */
#ifdef DIRECT_DISALLOW_SS_CPUBUG
200:
#endif
	iretq

	/*
	 * From userland (normal trap path)
	 */
210:
	TRAP(T_TRCTRAP)
	/* NOT REACHED */

IDTVEC(bpt)
	TRAP(T_BPTFLT)
IDTVEC(div)
	TRAP(T_DIVIDE)
IDTVEC(ofl)
	TRAP(T_OFLOW)
IDTVEC(bnd)
	TRAP(T_BOUND)
IDTVEC(ill)
	TRAP(T_PRIVINFLT)
IDTVEC(dna)
	TRAP(T_DNA)
IDTVEC(fpusegm)
	TRAP(T_FPOPFLT)
IDTVEC(mchk)
	TRAP(T_MCHK)
IDTVEC(fpu)
	TRAP(T_ARITHTRAP)
IDTVEC(xmm)
	TRAP(T_XMMFLT)

IDTVEC(tss)
	TRAP_ERR(T_TSSFLT)
IDTVEC(missing)
	TRAP_ERR(T_SEGNPFLT)
IDTVEC(stk)
	TRAP_ERR(T_STKFLT)
IDTVEC(align)
	TRAP_ERR(T_ALIGNFLT)

	/*
	 * alltraps entry point.  Use swapgs if this is the first time in the
	 * kernel from userland.  Reenable interrupts if they were enabled
	 * before the trap.
	 *
	 * WARNING!  %gs not available until after our swapgs code
	 */
	SUPERALIGN_TEXT
	.globl	alltraps
	.type	alltraps,@function
alltraps:

#if 0
alltraps_pushregs:
	movq	%rdi,TF_RDI(%rsp)
alltraps_pushregs_no_rdi:
	movq	%rsi,TF_RSI(%rsp)
	movq	%rdx,TF_RDX(%rsp)
	movq	%rcx,TF_RCX(%rsp)
	movq	%r8,TF_R8(%rsp)
	movq	%r9,TF_R9(%rsp)
	movq	%rax,TF_RAX(%rsp)
	movq	%rbx,TF_RBX(%rsp)
	movq	%rbp,TF_RBP(%rsp)
	movq	%r10,TF_R10(%rsp)
	movq	%r11,TF_R11(%rsp)
	movq	%r12,TF_R12(%rsp)
	movq	%r13,TF_R13(%rsp)
	movq	%r14,TF_R14(%rsp)
	movq	%r15,TF_R15(%rsp)
#endif
	sti
	FAKE_MCOUNT(TF_RIP(%rsp))
	.globl	calltrap
	.type	calltrap,@function
calltrap:
	cld
	movq	%rsp, %rdi
	call	trap
	MEXITCOUNT
	jmp	doreti			/* Handle any pending ASTs */

IDTVEC(dblfault)
	PUSH_FRAME_TFERR
	movq	$T_DOUBLEFLT,TF_TRAPNO(%rsp)
	movq	$0,TF_ADDR(%rsp)
	movq	$0,TF_XFLAGS(%rsp)

	cld
	movq	%rsp, %rdi
	call	dblfault_handler
2:	hlt
	jmp	2b

	/*
	 * We need to save the contents of %cr2 before PUSH_FRAME* messes
	 * with %cr3.
	 */
IDTVEC(page)
	PUSH_FRAME_TFERR_SAVECR2
	movq	$T_PAGEFLT,TF_TRAPNO(%rsp)
	movq	$0,TF_XFLAGS(%rsp)
	jmp	alltraps

	/*
	 * We have to special-case this one.  If we get a trap in doreti() at
	 * the iretq stage, we'll reenter as a kernel exception with the
	 * wrong gs and isolation state.  We have to act as through we came
	 * in from userland.
	 */
IDTVEC(prot)
	pushq	%r10
	leaq	doreti_iret(%rip),%r10
	cmpq	%r10,TF_RIP-TF_ERR+8(%rsp)		/* +8 due to pushq */
	jne	prot_normal
	testb	$SEL_RPL_MASK,TF_CS-TF_ERR+8(%rsp)      /* +8 due to pushq */
	jnz	prot_normal

	/*
	 * Special fault during iretq
	 */
	popq	%r10
	swapgs
	KMMUENTER_TFERR
	subq	$TF_ERR,%rsp
	PUSH_FRAME_REGS
	movq	$T_PROTFLT,TF_TRAPNO(%rsp)
	movq	$0,TF_ADDR(%rsp)
	movq	$0,TF_XFLAGS(%rsp)
	jmp	alltraps

prot_normal:
	popq	%r10
	PUSH_FRAME_TFERR
	movq	$T_PROTFLT,TF_TRAPNO(%rsp)
	movq	$0,TF_ADDR(%rsp)
	movq	$0,TF_XFLAGS(%rsp)
	jmp	alltraps

/*
 * Fast syscall entry point.  We enter here with just our new %cs/%ss set,
 * and the new privilige level.  We are still running on the old user stack
 * pointer.  We have to juggle a few things around to find our stack etc.
 * swapgs gives us access to our PCPU space only.
 *
 * We use GD_TRAMPOLINE+TR_CR2 to save the user stack pointer temporarily.
 */
IDTVEC(fast_syscall)
	swapgs					/* get kernel %gs */
	movq	%rsp,PCPU(trampoline)+TR_CR2	/* save user %rsp */
	movq	PCPU(common_tss)+TSS_RSP0,%rsp

	/*
	 * NOTE: KMMUENTER_SYSCALL does not actually use the stack but
	 *	 adjust the stack pointer for correctness in case we
	 *	 do in the future.
	 */
	subq	$TR_PCB_RSP,%rsp
	KMMUENTER_SYSCALL
	movq	PCPU(trampoline)+TR_PCB_RSP,%rsp

	/* Now emulate a trapframe. Make the 8 byte alignment odd for call. */
	subq	$TF_SIZE,%rsp
	/* defer TF_RSP till we have a spare register */
	movq	%r11,TF_RFLAGS(%rsp)
	movq	%rcx,TF_RIP(%rsp)	/* %rcx original value is in %r10 */
	movq	PCPU(trampoline)+TR_CR2,%r11	/* %r11 already saved */
	movq	%r11,TF_RSP(%rsp)	/* user stack pointer */
	orl	$RQF_QUICKRET,PCPU(reqflags)
	movq	$KUDSEL,TF_SS(%rsp)
	movq	$KUCSEL,TF_CS(%rsp)
	movq	$2,TF_ERR(%rsp)
	movq	$T_FAST_SYSCALL,TF_TRAPNO(%rsp)	/* for the vkernel */
	movq	$0,TF_XFLAGS(%rsp)	/* note: used in signal frame */
	movq	%rdi,TF_RDI(%rsp)	/* arg 1 */
	movq	%rsi,TF_RSI(%rsp)	/* arg 2 */
	movq	%rdx,TF_RDX(%rsp)	/* arg 3 */
	movq	%r10,TF_RCX(%rsp)	/* arg 4 */
	movq	%r8,TF_R8(%rsp)		/* arg 5 */
	movq	%r9,TF_R9(%rsp)		/* arg 6 */
	movq	%rax,TF_RAX(%rsp)	/* syscall number */
	movq	%rbx,TF_RBX(%rsp)	/* C preserved */
	movq	%rbp,TF_RBP(%rsp)	/* C preserved */
	movq	%r12,TF_R12(%rsp)	/* C preserved */
	movq	%r13,TF_R13(%rsp)	/* C preserved */
	movq	%r14,TF_R14(%rsp)	/* C preserved */
	movq	%r15,TF_R15(%rsp)	/* C preserved */

	xorq	%rax,%rax		/* SECURITY CLEAR REGS */
	movq	%rax,%rbx
	movq	%rax,%rcx
	movq	%rax,%rdx
	movq	%rax,%rsi
	movq	%rax,%rdi
	movq	%rax,%rbp
	movq	%rax,%r8
	movq	%rax,%r9
	movq	%rax,%r10
	movq	%rax,%r11
	movq	%rax,%r12
	movq	%rax,%r13
	movq	%rax,%r14
	movq	%rax,%r15

	sti
	FAKE_MCOUNT(TF_RIP(%rsp))
	movq	%rsp, %rdi
	call	syscall2

	/*
	 * Fast return from system call
	 */
	cli
	testl	$RQF_IPIQ|RQF_TIMER|RQF_INTPEND|RQF_AST_MASK,PCPU(reqflags)
	jnz	1f
	testl	$RQF_QUICKRET,PCPU(reqflags)
	jz	1f
	MEXITCOUNT

	movq	TF_RBX(%rsp),%rbx	/* SECURITY RESTORE */
	movq	TF_RCX(%rsp),%rcx
	movq	TF_RBP(%rsp),%rbp
	movq	TF_R8(%rsp),%r8
	movq	TF_R9(%rsp),%r9
	xorq	%r10,%r10		/* (security - clear scratch) */
	movq	%r10,%r11
	movq	TF_R12(%rsp),%r12
	movq	TF_R13(%rsp),%r13
	movq	TF_R14(%rsp),%r14
	movq	TF_R15(%rsp),%r15

	movq	TF_RDI(%rsp),%rdi	/* NORMAL RESTORE */
	movq	TF_RSI(%rsp),%rsi
	movq	TF_RDX(%rsp),%rdx
	movq	TF_RAX(%rsp),%rax
	movq	TF_RFLAGS(%rsp),%r11
	movq	TF_RIP(%rsp),%rcx
	movq	TF_RSP(%rsp),%rsp
	KMMUEXIT_SYSCALL
	swapgs
	sysretq

	/*
	 * Normal slow / full iret
	 */
1:
	MEXITCOUNT
	jmp	doreti

/*
 * Here for CYA insurance, in case a "syscall" instruction gets
 * issued from 32 bit compatibility mode. MSR_CSTAR has to point
 * to *something* if EFER_SCE is enabled.
 */
IDTVEC(fast_syscall32)
	sysret

/*
 * NMI handling is special.
 *
 * First, an NMI is taken on its own pcpu stack.  RFLAGS.IF, %gs, and %cr3
 * will be inconsistent when interrupt supervisor mode.
 *
 * Second, the processor treats NMIs specially, blocking further NMIs
 * until an 'iretq' instruction is executed.  We therefore need to
 * execute the NMI handler with interrupts disabled to prevent a
 * nested interrupt from executing an 'iretq' instruction and
 * inadvertently taking the processor out of NMI mode.
 */
IDTVEC(nmi)
	/*
	 * We don't need to special-case entry from userland, %gs will
	 * be consistent with expectations.
	 */
	testb   $SEL_RPL_MASK,TF_CS-TF_RIP(%rsp) ; /* from userland? */ \
	jnz	200f

	/*
	 * From kernel - %gs and %cr3 may be inconsistent.  Save original
	 * values and load consistent values, restore on return.
	 *
	 * The trap handler is NOT allowed to block for this case.
	 */
	subq	$TR_RIP, %rsp
	movq	%rax, TR_RAX(%rsp)
	movq	%rcx, TR_RCX(%rsp)
	movq	%rdx, TR_RDX(%rsp)

	cld
	movq	%cr3,%rax			/* save CR3 */
	movq	%rax, TR_PCB_CR3_SAVED(%rsp)
	movl	$MSR_GSBASE,%ecx		/* save %gs */
	rdmsr
	shlq	$32,%rdx
	orq	%rdx,%rax
	movq	%rax, TR_PCB_GS_SAVED(%rsp)
	movq	TR_PCB_GS_KERNEL(%rsp),%rdx	/* retrieve kernel %gs */
	movl	%edx,%eax
	shrq	$32,%rdx
	wrmsr
#if 0
	movq	TR_PCB_CR3(%rsp),%rax		/* retrieve kernel %cr3 */
#endif
	movq	PCPU(trampoline)+TR_PCB_CR3,%rax
	movq	%rax,%cr3

	movq	TR_RDX(%rsp), %rdx
	movq	TR_RCX(%rsp), %rcx
	movq	TR_RAX(%rsp), %rax
	addq	$TR_RIP, %rsp

	/*
	 * Ok, run the normal trap.  Because TF_CS is at a kernel RPL,
	 * the normal code will skip the usual swapgs and KMMU (trampoline)
	 * code.  We've handled the rest.
	 *
	 * NOTE: at this point the trampframe is above the normal stack
	 *	 frame.  The trap code will be ignorant of the special
	 *	 TR_* registers above the cpu hardware frame portion,
	 *	 and the TR_* registers below it will be overwritten.
	 */
	PUSH_FRAME_TFRIP
	movq	$0,TF_XFLAGS(%rsp)
	movq	$T_NMI,TF_TRAPNO(%rsp)
	movq	$0,TF_ADDR(%rsp)
	movq	$0,TF_ERR(%rsp)

	FAKE_MCOUNT(TF_RIP(%rsp))
	cld
	movq	%rsp, %rdi
	call	trap
	MEXITCOUNT

	/*
	 * Pop the frame (since we're coming from kernel mode, this will
	 * not mess with %cr3 or %gs), then restore %cr3 and %gs for our
	 * iretq.  Not optimal but more readable and this is not a
	 * critical path.
	 */
	POP_FRAME(nop)

	subq	$TR_RIP, %rsp
	movq	%rax, TR_RAX(%rsp)
	movq	%rcx, TR_RCX(%rsp)
	movq	%rdx, TR_RDX(%rsp)

	movl	$MSR_GSBASE,%ecx		/* restore %gs */
	movq	TR_PCB_GS_SAVED(%rsp),%rdx
	movl	%edx,%eax
	shrq	$32,%rdx
	wrmsr

	movq	TR_PCB_CR3_SAVED(%rsp),%rax	/* restore %cr3 */
	movq	%rax,%cr3

	movq	TR_RAX(%rsp),%rax
	movq	TR_RCX(%rsp),%rcx
	movq	TR_RDX(%rsp),%rdx
	addq	$TR_RIP, %rsp

	/*
	 * Direct iretq. No point jumping to doreti because the
	 * exception code that deals with iretq faults can't handle
	 * non-deterministic %gs/%cr3 state.
	 */
	iretq

	/*
	 * From userland (normal trap path)
	 */
200:
	PUSH_FRAME_TFRIP
	movq	$0,TF_XFLAGS(%rsp)
	movq	$T_NMI,TF_TRAPNO(%rsp)
	movq	$0,TF_ADDR(%rsp)
	movq	$0,TF_ERR(%rsp)

	FAKE_MCOUNT(TF_RIP(%rsp))
	cld
	movq	%rsp, %rdi
	call	trap
	MEXITCOUNT

	POP_FRAME(jmp doreti_iret)

/*
 * Reserved (unconfigured) traps rsvd00 - rsvdff
 */
.macro reservetrap a b
IDTVEC(rsvd\a\b)
	TRAP(T_RESERVED + 0x\a\b)
.endm

.macro reservegrp a
reservetrap \a 0
reservetrap \a 1
reservetrap \a 2
reservetrap \a 3
reservetrap \a 4
reservetrap \a 5
reservetrap \a 6
reservetrap \a 7
reservetrap \a 8
reservetrap \a 9
reservetrap \a a
reservetrap \a b
reservetrap \a c
reservetrap \a d
reservetrap \a e
reservetrap \a f
.endm

reservegrp 0
reservegrp 1
reservegrp 2
reservegrp 3
reservegrp 4
reservegrp 5
reservegrp 6
reservegrp 7
reservegrp 8
reservegrp 9
reservegrp a
reservegrp b
reservegrp c
reservegrp d
reservegrp e
reservegrp f

/*
 * This function is what cpu_heavy_restore jumps to after a new process
 * is created.  The LWKT subsystem switches while holding a critical
 * section and we maintain that abstraction here (e.g. because
 * cpu_heavy_restore needs it due to PCB_*() manipulation), then get out of
 * it before calling the initial function (typically fork_return()) and/or
 * returning to user mode.
 *
 * The MP lock is not held at any point but the critcount is bumped
 * on entry to prevent interruption of the trampoline at a bad point.
 *
 * This is effectively what td->td_switch() returns to.  It 'returns' the
 * old thread in %rax and since this is not returning to a td->td_switch()
 * call from lwkt_switch() we must handle the cleanup for the old thread
 * by calling lwkt_switch_return().
 *
 * fork_trampoline(%rax:otd, %rbx:func, %r12:arg)
 */
ENTRY(fork_trampoline)
	movq	%rax,%rdi
	call	lwkt_switch_return
	movq	PCPU(curthread),%rax
	decl	TD_CRITCOUNT(%rax)

	/*
	 * cpu_set_fork_handler intercepts this function call to
	 * have this call a non-return function to stay in kernel mode.
	 *
	 * initproc has its own fork handler, start_init(), which DOES
	 * return.
	 *
	 * %rbx - chaining function (typically fork_return)
	 * %r12 -> %rdi (argument)
	 * frame-> %rsi (trap frame)
	 *
	 *   void (func:rbx)(arg:rdi, trapframe:rsi)
	 */
	movq	%rsp, %rsi		/* pass trapframe by reference */
	movq	%r12, %rdi		/* arg1 */
	call	*%rbx			/* function */

	/* cut from syscall */

	sti
	call	splz

	/*
	 * Return via doreti to handle ASTs.
	 *
	 * trapframe is at the top of the stack.
	 */
	MEXITCOUNT
	jmp	doreti

/*
 * To efficiently implement classification of trap and interrupt handlers
 * for profiling, there must be only trap handlers between the labels btrap
 * and bintr, and only interrupt handlers between the labels bintr and
 * eintr.  This is implemented (partly) by including files that contain
 * some of the handlers.  Before including the files, set up a normal asm
 * environment so that the included files doen't need to know that they are
 * included.
 */

	.data
	.p2align 4

	.text
	SUPERALIGN_TEXT
MCOUNT_LABEL(bintr)

#if 0 /* JG */
#include <x86_64/x86_64/apic_vector.S>
#endif

#ifdef DEV_ATPIC
	.data
	.p2align 4
	.text
	SUPERALIGN_TEXT

#include <x86_64/isa/atpic_vector.S>
#endif

	.text
MCOUNT_LABEL(eintr)