Lines Matching +full:fault +full:- +full:inject
1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
165 "HLT triggers a VM-exit");
169 0, "PAUSE triggers a VM-exit");
173 0, "WBINVD triggers a VM-exit");
210 static int pirvec = -1;
420 return "mce-during-entry";
424 return "apic-access";
446 return "apic-write";
493 * "Virtualizing MSR-Based APIC Accesses".
549 if (x == -1) {
560 * It is still sub-optimal because the invvpid will invalidate
694 * - bit 54 indicates support for INS/OUTS decoding
703 /* Check support for primary processor-based VM-execution controls */
710 "primary processor-based controls\n");
714 /* Clear the processor-based ctl bits that are set on demand */
717 /* Check support for secondary processor-based VM-execution controls */
724 "secondary processor-based controls\n");
734 /* Check support for pin-based VM-execution controls */
741 "pin-based controls\n");
745 /* Check support for VM-exit controls */
756 /* Check support for VM-entry controls */
794 * Support a pass-through-based implementation of these via the
795 * "enable RDTSCP" VM-execution control and the "RDTSC exiting"
796 * VM-execution control.
798 * The "enable RDTSCP" VM-execution control applies to both RDPID
800 * Instruction Behavior in VMX Non-root operation"); this is why
801 * only this VM-execution control needs to be enabled in order to
805 * The "RDTSC exiting" VM-execution control applies to both RDTSC
807 * already set up for RDTSC and RDTSCP pass-through by the current
816 * bitmap is currently per-VM rather than per-vCPU while the
818 * per-vCPU basis).
966 * CR0_PE and CR0_PG can be set to zero in VMX non-root operation
1006 KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present",
1008 KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d "
1009 "has invalid type %d", vector, gd->gd_type));
1010 KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d "
1011 "has invalid dpl %d", vector, gd->gd_dpl));
1012 KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor "
1013 "for vector %d has invalid selector %d", vector, gd->gd_selector));
1014 KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid "
1015 "IST %d", vector, gd->gd_ist));
1017 func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset);
1060 vmx->vm = vm;
1062 vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pmltop));
1065 * Clean up EPTP-tagged guest physical and combined mappings
1073 ept_invalidate_mappings(vmx->eptp);
1075 vmx->msr_bitmap = malloc_aligned(PAGE_SIZE, PAGE_SIZE, M_VMX,
1077 msr_bitmap_initialize(vmx->msr_bitmap);
1082 * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are
1083 * always restored from the vmcs host state area on vm-exit.
1093 * The TSC MSR is exposed read-only. Writes are disallowed as
1100 * guest RDTSCP support are enabled (since, as per Table 2-2 in SDM
1103 * exposed read-only so that the VMM can do one fewer MSR read per
1104 * exit than if this register were exposed read-write; the guest
1125 vmx->pmap = pmap;
1142 vcpu->vmx = vmx;
1143 vcpu->vcpu = vcpu1;
1144 vcpu->vcpuid = vcpuid;
1145 vcpu->vmcs = malloc_aligned(sizeof(*vmcs), PAGE_SIZE, M_VMX,
1147 vcpu->apic_page = malloc_aligned(PAGE_SIZE, PAGE_SIZE, M_VMX,
1149 vcpu->pir_desc = malloc_aligned(sizeof(*vcpu->pir_desc), 64, M_VMX,
1152 vmcs = vcpu->vmcs;
1153 vmcs->identifier = vmx_revision();
1167 error += vmwrite(VMCS_HOST_RSP, (u_long)&vcpu->ctx);
1168 error += vmwrite(VMCS_EPTP, vmx->eptp);
1171 if (vcpu_trap_wbinvd(vcpu->vcpu)) {
1178 error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap));
1191 if (vcpu_trace_exceptions(vcpu->vcpu))
1197 vcpu->ctx.guest_dr6 = DBREG_DR6_RESERVED1;
1201 error += vmwrite(VMCS_VIRTUAL_APIC, vtophys(vcpu->apic_page));
1213 error += vmwrite(VMCS_PIR_DESC, vtophys(vcpu->pir_desc));
1218 vcpu->cap.set = 0;
1219 vcpu->cap.set |= cap_rdpid != 0 ? 1 << VM_CAP_RDPID : 0;
1220 vcpu->cap.set |= cap_rdtscp != 0 ? 1 << VM_CAP_RDTSCP : 0;
1221 vcpu->cap.proc_ctls = procbased_ctls;
1222 vcpu->cap.proc_ctls2 = procbased_ctls2;
1223 vcpu->cap.exc_bitmap = exc_bitmap;
1225 vcpu->state.nextrip = ~0;
1226 vcpu->state.lastcpu = NOCPU;
1227 vcpu->state.vpid = vpid;
1231 * to the power-on register value from the Intel Sys Arch.
1232 * CR0 - 0x60000010
1233 * CR4 - 0
1243 vcpu->ctx.pmap = vmx->pmap;
1253 handled = x86_emulate_cpuid(vcpu->vcpu, (uint64_t *)&vmxctx->guest_rax,
1254 (uint64_t *)&vmxctx->guest_rbx, (uint64_t *)&vmxctx->guest_rcx,
1255 (uint64_t *)&vmxctx->guest_rdx);
1292 vmxstate = &vcpu->state;
1293 if (vmxstate->vpid == 0)
1303 vmxstate->lastcpu = NOCPU;
1307 KASSERT(curthread->td_critnest > 0, ("%s: vcpu %d running outside "
1308 "critical section", __func__, vcpu->vcpuid));
1325 if (atomic_load_long(&pmap->pm_eptgen) == vmx->eptgen[curcpu]) {
1328 invvpid_desc.vpid = vmxstate->vpid;
1331 vmm_stat_incr(vcpu->vcpu, VCPU_INVVPID_DONE, 1);
1337 * 'vmx->eptp' for all vpids.
1339 vmm_stat_incr(vcpu->vcpu, VCPU_INVVPID_SAVED, 1);
1348 vmxstate = &vcpu->state;
1349 if (vmxstate->lastcpu == curcpu)
1352 vmxstate->lastcpu = curcpu;
1354 vmm_stat_incr(vcpu->vcpu, VCPU_MIGRATIONS, 1);
1371 if ((vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) {
1372 vcpu->cap.proc_ctls |= PROCBASED_INT_WINDOW_EXITING;
1373 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls);
1382 KASSERT((vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0,
1383 ("intr_window_exiting not set: %#x", vcpu->cap.proc_ctls));
1384 vcpu->cap.proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING;
1385 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls);
1393 if ((vcpu->cap.proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) {
1394 vcpu->cap.proc_ctls |= PROCBASED_NMI_WINDOW_EXITING;
1395 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls);
1404 KASSERT((vcpu->cap.proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0,
1405 ("nmi_window_exiting not set %#x", vcpu->cap.proc_ctls));
1406 vcpu->cap.proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING;
1407 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls);
1416 if ((vcpu->cap.proc_ctls & PROCBASED_TSC_OFFSET) == 0) {
1417 vcpu->cap.proc_ctls |= PROCBASED_TSC_OFFSET;
1418 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vcpu->cap.proc_ctls);
1425 vm_set_tsc_offset(vcpu->vcpu, offset);
1442 "interruptibility-state %#x", gi));
1446 "VM-entry interruption information %#x", info));
1449 * Inject the virtual NMI. The vector must be the NMI IDT entry
1458 vm_nmi_clear(vcpu->vcpu);
1469 if (vcpu->cap.set & (1 << VM_CAP_MASK_HWINTR)) {
1473 if (vcpu->state.nextrip != guestrip) {
1478 vcpu->state.nextrip, guestrip);
1484 if (vm_entry_intinfo(vcpu->vcpu, &entryinfo)) {
1489 KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject "
1496 * VT-x requires #BP and #OF to be injected as software
1509 if (vm_nmi_pending(vcpu->vcpu)) {
1512 * inject it directly here otherwise enable "NMI window
1513 * exiting" to inject it as soon as we can.
1529 VMX_CTR1(vcpu, "Cannot inject NMI "
1530 "due to VM-entry intr info %#x", info);
1533 VMX_CTR1(vcpu, "Cannot inject NMI due to "
1534 "Guest Interruptibility-state %#x", gi);
1541 extint_pending = vm_extint_pending(vcpu->vcpu);
1549 * If interrupt-window exiting is already in effect then don't bother
1553 if ((vcpu->cap.proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) {
1560 /* Ask the local apic for a vector to inject */
1567 * - maskable interrupt vectors [16,255] can be delivered
1573 /* Ask the legacy pic for a vector to inject */
1574 vatpic_pending_intr(vcpu->vmx->vm, &vector);
1579 * - maskable interrupt vectors [0,255] can be delivered
1589 VMX_CTR2(vcpu, "Cannot inject vector %d due to "
1596 VMX_CTR2(vcpu, "Cannot inject vector %d due to "
1597 "Guest Interruptibility-state %#x", vector, gi);
1605 * - A vectoring VM-entry was aborted due to astpending
1606 * - A VM-exit happened during event injection.
1607 * - An exception was injected above.
1608 * - An NMI was injected above or after "NMI window exiting"
1610 VMX_CTR2(vcpu, "Cannot inject vector %d due to "
1611 "VM-entry intr info %#x", vector, info);
1615 /* Inject the interrupt */
1624 vm_extint_clear(vcpu->vcpu);
1625 vatpic_intr_accepted(vcpu->vmx->vm, vector);
1631 * we can inject that one too.
1633 * Also, interrupt window exiting allows us to inject any
1647 * Set the Interrupt Window Exiting execution control so we can inject
1655 * tracks virtual-NMI blocking in the Guest Interruptibility-state field of
1656 * the VMCS. An IRET instruction in VMX non-root operation will remove any
1657 * virtual-NMI blocking.
1659 * This unblocking occurs even if the IRET causes a fault. In this case the
1660 * hypervisor needs to restore virtual-NMI blocking before resuming the guest.
1667 VMX_CTR0(vcpu, "Restore Virtual-NMI blocking");
1678 VMX_CTR0(vcpu, "Clear Virtual-NMI blocking");
1702 vmxctx = &vcpu->ctx;
1706 * Note that the processor raises a GP# fault on its own if
1708 * emulate that fault here.
1712 if (vmxctx->guest_rcx != 0) {
1713 vm_inject_gp(vcpu->vcpu);
1718 if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) {
1719 vm_inject_ud(vcpu->vcpu);
1723 xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff);
1724 if ((xcrval & ~limits->xcr0_allowed) != 0) {
1725 vm_inject_gp(vcpu->vcpu);
1730 vm_inject_gp(vcpu->vcpu);
1737 vm_inject_gp(vcpu->vcpu);
1748 vm_inject_gp(vcpu->vcpu);
1758 vm_inject_gp(vcpu->vcpu);
1776 vmxctx = &vcpu->ctx;
1780 return (vmxctx->guest_rax);
1782 return (vmxctx->guest_rcx);
1784 return (vmxctx->guest_rdx);
1786 return (vmxctx->guest_rbx);
1790 return (vmxctx->guest_rbp);
1792 return (vmxctx->guest_rsi);
1794 return (vmxctx->guest_rdi);
1796 return (vmxctx->guest_r8);
1798 return (vmxctx->guest_r9);
1800 return (vmxctx->guest_r10);
1802 return (vmxctx->guest_r11);
1804 return (vmxctx->guest_r12);
1806 return (vmxctx->guest_r13);
1808 return (vmxctx->guest_r14);
1810 return (vmxctx->guest_r15);
1821 vmxctx = &vcpu->ctx;
1825 vmxctx->guest_rax = regval;
1828 vmxctx->guest_rcx = regval;
1831 vmxctx->guest_rdx = regval;
1834 vmxctx->guest_rbx = regval;
1840 vmxctx->guest_rbp = regval;
1843 vmxctx->guest_rsi = regval;
1846 vmxctx->guest_rdi = regval;
1849 vmxctx->guest_r8 = regval;
1852 vmxctx->guest_r9 = regval;
1855 vmxctx->guest_r10 = regval;
1858 vmxctx->guest_r11 = regval;
1861 vmxctx->guest_r12 = regval;
1864 vmxctx->guest_r13 = regval;
1867 vmxctx->guest_r14 = regval;
1870 vmxctx->guest_r15 = regval;
1899 * the "IA-32e mode guest" bit in VM-entry control must be
1948 vlapic = vm_lapic(vcpu->vcpu);
2062 vis->seg_name = VM_REG_GUEST_ES;
2065 vis->seg_name = vm_segment_name(s);
2068 error = vmx_getdesc(vcpu, vis->seg_name, &vis->seg_desc);
2075 paging->cr3 = vmcs_guest_cr3();
2076 paging->cpl = vmx_cpl();
2077 paging->cpu_mode = vmx_cpu_mode();
2078 paging->paging_mode = vmx_paging_mode();
2087 paging = &vmexit->u.inst_emul.paging;
2089 vmexit->exitcode = VM_EXITCODE_INST_EMUL;
2090 vmexit->inst_length = 0;
2091 vmexit->u.inst_emul.gpa = gpa;
2092 vmexit->u.inst_emul.gla = gla;
2094 switch (paging->cpu_mode) {
2096 vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
2097 vmexit->u.inst_emul.cs_d = 0;
2101 vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE);
2103 vmexit->u.inst_emul.cs_d = SEG_DESC_DEF32(csar);
2106 vmexit->u.inst_emul.cs_base = 0;
2107 vmexit->u.inst_emul.cs_d = 0;
2110 vie_init(&vmexit->u.inst_emul.vie, NULL, 0);
2133 /* EPT fault on an instruction fetch doesn't make sense here */
2137 /* EPT fault must be a read fault or a write fault */
2145 * guest-physical address that is a translation of a guest-linear
2161 proc_ctls2 = vcpu->cap.proc_ctls2;
2170 proc_ctls2 = vcpu->cap.proc_ctls2;
2187 * In general there should not be any APIC write VM-exits
2188 * unless APIC-access virtualization is enabled.
2190 * However self-IPI virtualization can legitimately trigger
2191 * an APIC-write VM-exit so treat it specially.
2195 apic_regs = (uint32_t *)(vlapic->apic_page);
2262 qual = vmexit->u.vmx.exit_qualification;
2309 * Regardless of whether the APIC-access is allowed this handler
2311 * - if the access is allowed then it is handled by emulating the
2312 * instruction that caused the VM-exit (outside the critical section)
2313 * - if the access is not allowed then it will be converted to an
2345 error = lapic_wrmsr(vcpu->vcpu, num, val, retu);
2361 error = lapic_rdmsr(vcpu->vcpu, num, &result, retu);
2367 vmxctx = &vcpu->ctx;
2399 vmxctx = &vcpu->ctx;
2401 vcpuid = vcpu->vcpuid;
2404 qual = vmexit->u.vmx.exit_qualification;
2405 reason = vmexit->u.vmx.exit_reason;
2406 vmexit->exitcode = VM_EXITCODE_BOGUS;
2408 vmm_stat_incr(vcpu->vcpu, VMEXIT_COUNT, 1);
2412 * VM-entry failures during or after loading guest state.
2414 * These VM-exits are uncommon but must be handled specially
2415 * as most VM-exit fields are not populated as usual.
2418 VMX_CTR0(vcpu, "Handling MCE during VM-entry");
2425 * be handled specially by re-injecting the event if the IDT
2439 error = vm_exit_intinfo(vcpu->vcpu, exitintinfo);
2444 * If 'virtual NMIs' are being used and the VM-exit
2446 * VM-entry, then clear "blocking by NMI" in the
2447 * Guest Interruptibility-State so the NMI can be
2448 * reinjected on the subsequent VM-entry.
2463 * Update VM-entry instruction length if the event being
2469 vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2475 ts = &vmexit->u.task_switch;
2476 ts->tsssel = qual & 0xffff;
2477 ts->reason = vmx_task_switch_reason(qual);
2478 ts->ext = 0;
2479 ts->errcode_valid = 0;
2480 vmx_paging_info(&ts->paging);
2494 if (ts->reason == TSR_IDT_GATE) {
2503 ts->ext = 1;
2504 vmexit->inst_length = 0;
2506 ts->errcode_valid = 1;
2507 ts->errcode = vmcs_idt_vectoring_err();
2511 vmexit->exitcode = VM_EXITCODE_TASK_SWITCH;
2514 "%s errcode 0x%016lx", ts->reason, ts->tsssel,
2515 ts->ext ? "external" : "internal",
2516 ((uint64_t)ts->errcode << 32) | ts->errcode_valid);
2519 vmm_stat_incr(vcpu->vcpu, VMEXIT_CR_ACCESS, 1);
2534 vmm_stat_incr(vcpu->vcpu, VMEXIT_RDMSR, 1);
2536 ecx = vmxctx->guest_rcx;
2541 vmexit->exitcode = VM_EXITCODE_RDMSR;
2542 vmexit->u.msr.code = ecx;
2547 KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2552 vmm_stat_incr(vcpu->vcpu, VMEXIT_WRMSR, 1);
2554 eax = vmxctx->guest_rax;
2555 ecx = vmxctx->guest_rcx;
2556 edx = vmxctx->guest_rdx;
2564 vmexit->exitcode = VM_EXITCODE_WRMSR;
2565 vmexit->u.msr.code = ecx;
2566 vmexit->u.msr.wval = (uint64_t)edx << 32 | eax;
2571 KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS,
2576 vmm_stat_incr(vcpu->vcpu, VMEXIT_HLT, 1);
2578 vmexit->exitcode = VM_EXITCODE_HLT;
2579 vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2581 vmexit->u.hlt.intr_status =
2584 vmexit->u.hlt.intr_status = 0;
2587 vmm_stat_incr(vcpu->vcpu, VMEXIT_MTRAP, 1);
2589 vmexit->exitcode = VM_EXITCODE_MTRAP;
2590 vmexit->inst_length = 0;
2593 vmm_stat_incr(vcpu->vcpu, VMEXIT_PAUSE, 1);
2595 vmexit->exitcode = VM_EXITCODE_PAUSE;
2598 vmm_stat_incr(vcpu->vcpu, VMEXIT_INTR_WINDOW, 1);
2609 * host interrupt handler in the VM's softc. We will inject
2629 * VM-exit but not increment the instruction pointer.
2631 vmm_stat_incr(vcpu->vcpu, VMEXIT_EXTINT, 1);
2636 if (vm_nmi_pending(vcpu->vcpu))
2639 vmm_stat_incr(vcpu->vcpu, VMEXIT_NMI_WINDOW, 1);
2642 vmm_stat_incr(vcpu->vcpu, VMEXIT_INOUT, 1);
2643 vmexit->exitcode = VM_EXITCODE_INOUT;
2644 vmexit->u.inout.bytes = (qual & 0x7) + 1;
2645 vmexit->u.inout.in = in = (qual & 0x8) ? 1 : 0;
2646 vmexit->u.inout.string = (qual & 0x10) ? 1 : 0;
2647 vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0;
2648 vmexit->u.inout.port = (uint16_t)(qual >> 16);
2649 vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax);
2650 if (vmexit->u.inout.string) {
2652 vmexit->exitcode = VM_EXITCODE_INOUT_STR;
2653 vis = &vmexit->u.inout_str;
2654 vmx_paging_info(&vis->paging);
2655 vis->rflags = vmcs_read(VMCS_GUEST_RFLAGS);
2656 vis->cr0 = vmcs_read(VMCS_GUEST_CR0);
2657 vis->index = inout_str_index(vcpu, in);
2658 vis->count = inout_str_count(vcpu, vis->inout.rep);
2659 vis->addrsize = inout_str_addrsize(inst_info);
2665 vmm_stat_incr(vcpu->vcpu, VMEXIT_CPUID, 1);
2670 vmm_stat_incr(vcpu->vcpu, VMEXIT_EXCEPTION, 1);
2679 * If Virtual NMIs control is 1 and the VM-exit is due to a
2680 * fault encountered during the execution of IRET then we must
2681 * restore the state of "virtual-NMI blocking" before resuming
2713 (vcpu->cap.set & (1 << VM_CAP_BPT_EXIT))) {
2714 vmexit->exitcode = VM_EXITCODE_BPT;
2715 vmexit->u.bpt.inst_length = vmexit->inst_length;
2716 vmexit->inst_length = 0;
2727 * Software exceptions exhibit trap-like behavior. This in
2728 * turn requires populating the VM-entry instruction length
2733 vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length);
2745 error = vm_inject_exception(vcpu->vcpu, intr_vec,
2754 * memory then this must be a nested page fault otherwise
2758 if (vm_mem_allocated(vcpu->vcpu, gpa) ||
2760 vmexit->exitcode = VM_EXITCODE_PAGING;
2761 vmexit->inst_length = 0;
2762 vmexit->u.paging.gpa = gpa;
2763 vmexit->u.paging.fault_type = ept_fault_type(qual);
2764 vmm_stat_incr(vcpu->vcpu, VMEXIT_NESTED_FAULT, 1);
2769 vmm_stat_incr(vcpu->vcpu, VMEXIT_INST_EMUL, 1);
2774 * If Virtual NMIs control is 1 and the VM-exit is due to an
2775 * EPT fault during the execution of IRET then we must restore
2776 * the state of "virtual-NMI blocking" before resuming.
2786 vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI;
2787 vmexit->u.ioapic_eoi.vector = qual & 0xFF;
2789 vmexit->inst_length = 0; /* trap-like */
2797 * APIC-write VM exit is trap-like so the %rip is already
2800 vmexit->inst_length = 0;
2801 vlapic = vm_lapic(vcpu->vcpu);
2812 vmexit->exitcode = VM_EXITCODE_MONITOR;
2816 vmexit->exitcode = VM_EXITCODE_MWAIT;
2819 vlapic = vm_lapic(vcpu->vcpu);
2821 vmexit->inst_length = 0;
2835 vmexit->exitcode = VM_EXITCODE_VMINSN;
2845 vmm_stat_incr(vcpu->vcpu, VMEXIT_UNKNOWN, 1);
2860 vmexit->rip += vmexit->inst_length;
2861 vmexit->inst_length = 0;
2862 vmcs_write(VMCS_GUEST_RIP, vmexit->rip);
2864 if (vmexit->exitcode == VM_EXITCODE_BOGUS) {
2869 vmexit->exitcode = VM_EXITCODE_VMX;
2870 vmexit->u.vmx.status = VM_SUCCESS;
2871 vmexit->u.vmx.inst_type = 0;
2872 vmexit->u.vmx.inst_error = 0;
2890 KASSERT(vmxctx->inst_fail_status != VM_SUCCESS,
2892 vmxctx->inst_fail_status));
2894 vmexit->inst_length = 0;
2895 vmexit->exitcode = VM_EXITCODE_VMX;
2896 vmexit->u.vmx.status = vmxctx->inst_fail_status;
2897 vmexit->u.vmx.inst_error = vmcs_instruction_error();
2898 vmexit->u.vmx.exit_reason = ~0;
2899 vmexit->u.vmx.exit_qualification = ~0;
2904 vmexit->u.vmx.inst_type = rc;
2912 * If the NMI-exiting VM execution control is set to '1' then an NMI in
2913 * non-root operation causes a VM-exit. NMI blocking is in effect so it is
2926 if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION)
2947 vmxctx->host_dr7 = rdr7();
2948 vmxctx->host_debugctl = rdmsr(MSR_DEBUGCTLMSR);
2965 vmxctx->host_tf = rflags & PSL_T;
2969 vmxctx->host_dr0 = rdr0();
2970 vmxctx->host_dr1 = rdr1();
2971 vmxctx->host_dr2 = rdr2();
2972 vmxctx->host_dr3 = rdr3();
2973 vmxctx->host_dr6 = rdr6();
2976 load_dr0(vmxctx->guest_dr0);
2977 load_dr1(vmxctx->guest_dr1);
2978 load_dr2(vmxctx->guest_dr2);
2979 load_dr3(vmxctx->guest_dr3);
2980 load_dr6(vmxctx->guest_dr6);
2988 vmxctx->guest_dr0 = rdr0();
2989 vmxctx->guest_dr1 = rdr1();
2990 vmxctx->guest_dr2 = rdr2();
2991 vmxctx->guest_dr3 = rdr3();
2992 vmxctx->guest_dr6 = rdr6();
2998 load_dr0(vmxctx->host_dr0);
2999 load_dr1(vmxctx->host_dr1);
3000 load_dr2(vmxctx->host_dr2);
3001 load_dr3(vmxctx->host_dr3);
3002 load_dr6(vmxctx->host_dr6);
3003 wrmsr(MSR_DEBUGCTLMSR, vmxctx->host_debugctl);
3004 load_dr7(vmxctx->host_dr7);
3005 write_rflags(read_rflags() | vmxctx->host_tf);
3016 CPU_SET_ATOMIC(cpu, &pmap->pm_active);
3017 smr_enter(pmap->pm_eptsmr);
3018 eptgen = atomic_load_long(&pmap->pm_eptgen);
3019 if (eptgen != vmx->eptgen[cpu]) {
3020 vmx->eptgen[cpu] = eptgen;
3022 (struct invept_desc){ .eptp = vmx->eptp, ._res = 0 });
3029 smr_exit(pmap->pm_eptsmr);
3030 CPU_CLR_ATOMIC(curcpu, &pmap->pm_active);
3048 vmx = vcpu->vmx;
3049 vmcs = vcpu->vmcs;
3050 vmxctx = &vcpu->ctx;
3051 vlapic = vm_lapic(vcpu->vcpu);
3052 vmexit = vm_exitinfo(vcpu->vcpu);
3055 KASSERT(vmxctx->pmap == pmap,
3056 ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap));
3103 * triple fault.
3107 vm_exit_suspended(vcpu->vcpu, rip);
3111 if (vcpu_rendezvous_pending(vcpu->vcpu, evinfo)) {
3113 vm_exit_rendezvous(vcpu->vcpu, rip);
3119 vm_exit_reqidle(vcpu->vcpu, rip);
3123 if (vcpu_should_yield(vcpu->vcpu)) {
3125 vm_exit_astpending(vcpu->vcpu, rip);
3131 if (vcpu_debugged(vcpu->vcpu)) {
3133 vm_exit_debug(vcpu->vcpu, rip);
3142 if ((vcpu->cap.proc_ctls & PROCBASED_USE_TPR_SHADOW) != 0) {
3184 * EPTP-tagged TLB entries if required.
3200 vmexit->rip = rip = vmcs_guest_rip();
3201 vmexit->inst_length = vmexit_instruction_length();
3202 vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason();
3203 vmexit->u.vmx.exit_qualification = vmcs_exit_qualification();
3206 vcpu->state.nextrip = rip;
3218 rip = vmexit->rip;
3225 if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) ||
3226 (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) {
3228 handled, vmexit->exitcode);
3232 vmexit->exitcode);
3245 vpid_free(vcpu->state.vpid);
3246 free(vcpu->pir_desc, M_VMX);
3247 free(vcpu->apic_page, M_VMX);
3248 free(vcpu->vmcs, M_VMX);
3258 vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
3260 free(vmx->msr_bitmap, M_VMX);
3272 return (&vmxctx->guest_rax);
3274 return (&vmxctx->guest_rbx);
3276 return (&vmxctx->guest_rcx);
3278 return (&vmxctx->guest_rdx);
3280 return (&vmxctx->guest_rsi);
3282 return (&vmxctx->guest_rdi);
3284 return (&vmxctx->guest_rbp);
3286 return (&vmxctx->guest_r8);
3288 return (&vmxctx->guest_r9);
3290 return (&vmxctx->guest_r10);
3292 return (&vmxctx->guest_r11);
3294 return (&vmxctx->guest_r12);
3296 return (&vmxctx->guest_r13);
3298 return (&vmxctx->guest_r14);
3300 return (&vmxctx->guest_r15);
3302 return (&vmxctx->guest_cr2);
3304 return (&vmxctx->guest_dr0);
3306 return (&vmxctx->guest_dr1);
3308 return (&vmxctx->guest_dr2);
3310 return (&vmxctx->guest_dr3);
3312 return (&vmxctx->guest_dr6);
3349 error = vmcs_getreg(vcpu->vmcs, running,
3370 vmcs = vcpu->vmcs;
3388 shreg = -1;
3409 struct vmx *vmx = vcpu->vmx;
3411 running = vcpu_is_running(vcpu->vcpu, &hostcpu);
3413 panic("vmx_getreg: %s%d is running", vm_name(vmx->vm),
3414 vcpu->vcpuid);
3420 *retval = vcpu->guest_msrs[IDX_MSR_KGSBASE];
3423 *retval = vlapic_get_cr8(vm_lapic(vcpu->vcpu));
3427 if (vmxctx_getreg(&vcpu->ctx, reg, retval) == 0)
3430 return (vmcs_getreg(vcpu->vmcs, running, reg, retval));
3440 struct vmx *vmx = vcpu->vmx;
3442 running = vcpu_is_running(vcpu->vcpu, &hostcpu);
3444 panic("vmx_setreg: %s%d is running", vm_name(vmx->vm),
3445 vcpu->vcpuid);
3450 if (vmxctx_setreg(&vcpu->ctx, reg, val) == 0)
3457 error = vmcs_setreg(vcpu->vmcs, running, reg, val);
3461 * If the "load EFER" VM-entry control is 1 then the
3462 * value of EFER.LMA must be identical to "IA-32e mode guest"
3463 * bit in the VM-entry control.
3467 vmcs_getreg(vcpu->vmcs, running,
3473 vmcs_setreg(vcpu->vmcs, running,
3482 error = vmcs_setreg(vcpu->vmcs, running,
3494 pmap = vcpu->ctx.pmap;
3507 struct vmx *vmx = vcpu->vmx;
3509 running = vcpu_is_running(vcpu->vcpu, &hostcpu);
3511 panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm),
3512 vcpu->vcpuid);
3514 return (vmcs_getdesc(vcpu->vmcs, running, reg, desc));
3522 struct vmx *vmx = vcpu->vmx;
3524 running = vcpu_is_running(vcpu->vcpu, &hostcpu);
3526 panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm),
3527 vcpu->vcpuid);
3529 return (vmcs_setdesc(vcpu->vmcs, running, reg, desc));
3541 vcap = vcpu->cap.set;
3590 struct vmcs *vmcs = vcpu->vmcs;
3606 pptr = &vcpu->cap.proc_ctls;
3615 pptr = &vcpu->cap.proc_ctls;
3624 pptr = &vcpu->cap.proc_ctls;
3644 pptr = &vcpu->cap.proc_ctls2;
3653 pptr = &vcpu->cap.proc_ctls2;
3663 if (vcpu->cap.exc_bitmap != 0xffffffff) {
3664 pptr = &vcpu->cap.exc_bitmap;
3673 vlapic = vm_lapic(vcpu->vcpu);
3674 vlapic->ipi_exit = val;
3707 vcpu->cap.set |= (1 << type);
3709 vcpu->cap.set &= ~(1 << type);
3738 VLAPIC_CTR2(vlapic, msg " assert %s-triggered vector %d", \
3740 VLAPIC_CTR1(vlapic, msg " pir0 0x%016lx", pir_desc->pir[0]); \
3741 VLAPIC_CTR1(vlapic, msg " pir1 0x%016lx", pir_desc->pir[1]); \
3742 VLAPIC_CTR1(vlapic, msg " pir2 0x%016lx", pir_desc->pir[2]); \
3743 VLAPIC_CTR1(vlapic, msg " pir3 0x%016lx", pir_desc->pir[3]); \
3748 * vlapic->ops handlers that utilize the APICv hardware assist described in
3760 pir_desc = vlapic_vtx->pir_desc;
3769 atomic_set_long(&pir_desc->pir[idx], mask);
3773 * transition from 0->1.
3778 * the 0->1 'pending' transition with a notification, but the vCPU
3780 * need to then be notified if a high-priority interrupt arrived which
3785 * to-be-injected interrupt exceed the priorities already present, the
3787 * cleared whenever the 'pending' bit makes another 0->1 transition.
3789 if (atomic_cmpset_long(&pir_desc->pending, 0, 1) != 0) {
3791 vlapic_vtx->pending_prio = 0;
3793 const u_int old_prio = vlapic_vtx->pending_prio;
3797 atomic_set_int(&vlapic_vtx->pending_prio, prio_bit);
3825 pir_desc = vlapic_vtx->pir_desc;
3826 lapic = vlapic->apic_page;
3835 vmexit = vm_exitinfo(vlapic->vcpu);
3836 KASSERT(vmexit->exitcode == VM_EXITCODE_HLT,
3838 rvi = vmexit->u.hlt.intr_status & APIC_TPR_INT;
3839 ppr = lapic->ppr & APIC_TPR_INT;
3843 pending = atomic_load_acq_long(&pir_desc->pending);
3857 VLAPIC_CTR1(vlapic, "HLT with non-zero PPR %d", lapic->ppr);
3860 for (i = 3; i >= 0; i--) {
3861 pirval = pir_desc->pir[i];
3863 vpr = (i * 64 + flsl(pirval) - 1) & APIC_TPR_INT;
3869 * If the highest-priority pending interrupt falls short of the
3871 * have any stale bits which would preclude a higher-priority interrupt
3876 const u_int old = vlapic_vtx->pending_prio;
3879 vlapic_vtx->pending_prio = prio_bit;
3901 KASSERT(!vcpu_is_running(vlapic->vcpu, NULL),
3905 vmcs = vlapic_vtx->vcpu->vmcs;
3927 vcpu = vlapic_vtx->vcpu;
3928 vmcs = vcpu->vmcs;
3930 proc_ctls = vcpu->cap.proc_ctls;
3934 vcpu->cap.proc_ctls = proc_ctls;
3952 vcpu = vlapic_vtx->vcpu;
3953 vmx = vcpu->vmx;
3954 vmcs = vcpu->vmcs;
3956 proc_ctls2 = vcpu->cap.proc_ctls2;
3962 vcpu->cap.proc_ctls2 = proc_ctls2;
3968 if (vlapic->vcpuid == 0) {
3973 error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE);
4005 int rvi, pirbase = -1;
4009 pir_desc = vlapic_vtx->pir_desc;
4010 if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) {
4017 pirbase = -1;
4018 lapic = vlapic->apic_page;
4020 val = atomic_readandclear_long(&pir_desc->pir[0]);
4022 lapic->irr0 |= val;
4023 lapic->irr1 |= val >> 32;
4028 val = atomic_readandclear_long(&pir_desc->pir[1]);
4030 lapic->irr2 |= val;
4031 lapic->irr3 |= val >> 32;
4036 val = atomic_readandclear_long(&pir_desc->pir[2]);
4038 lapic->irr4 |= val;
4039 lapic->irr5 |= val >> 32;
4044 val = atomic_readandclear_long(&pir_desc->pir[3]);
4046 lapic->irr6 |= val;
4047 lapic->irr7 |= val >> 32;
4056 * interrupts on VM-entry.
4060 * CPU-Y is sending a posted interrupt to CPU-X, which
4062 * CPU-X will eventually exit and the state seen in s/w is
4065 * CPU-X CPU-Y
4076 rvi = pirbase + flsl(pirval) - 1;
4097 vmx = vcpu->vmx;
4100 vlapic->vm = vmx->vm;
4101 vlapic->vcpu = vcpu->vcpu;
4102 vlapic->vcpuid = vcpu->vcpuid;
4103 vlapic->apic_page = (struct LAPIC *)vcpu->apic_page;
4106 vlapic_vtx->pir_desc = vcpu->pir_desc;
4107 vlapic_vtx->vcpu = vcpu;
4110 vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_ts;
4114 vlapic->ops.set_intr_ready = vmx_set_intr_ready;
4115 vlapic->ops.pending_intr = vmx_pending_intr;
4116 vlapic->ops.intr_accepted = vmx_intr_accepted;
4117 vlapic->ops.set_tmr = vmx_set_tmr;
4118 vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_vid;
4122 vlapic->ops.post_intr = vmx_post_intr;
4149 vmx = vcpu->vmx;
4150 vmcs = vcpu->vmcs;
4152 run = vcpu_is_running(vcpu->vcpu, &hostcpu);
4154 printf("%s: %s%d is running", __func__, vm_name(vmx->vm),
4155 vcpu->vcpuid);
4214 SNAPSHOT_BUF_OR_LEAVE(vcpu->guest_msrs,
4215 sizeof(vcpu->guest_msrs), meta, err, done);
4217 SNAPSHOT_BUF_OR_LEAVE(vcpu->pir_desc,
4218 sizeof(*vcpu->pir_desc), meta, err, done);
4220 SNAPSHOT_BUF_OR_LEAVE(&vcpu->mtrr,
4221 sizeof(vcpu->mtrr), meta, err, done);
4223 vmxctx = &vcpu->ctx;
4224 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdi, meta, err, done);
4225 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rsi, meta, err, done);
4226 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdx, meta, err, done);
4227 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rcx, meta, err, done);
4228 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r8, meta, err, done);
4229 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r9, meta, err, done);
4230 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rax, meta, err, done);
4231 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rbx, meta, err, done);
4232 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rbp, meta, err, done);
4233 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r10, meta, err, done);
4234 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r11, meta, err, done);
4235 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r12, meta, err, done);
4236 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r13, meta, err, done);
4237 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r14, meta, err, done);
4238 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r15, meta, err, done);
4239 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_cr2, meta, err, done);
4240 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr0, meta, err, done);
4241 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr1, meta, err, done);
4242 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr2, meta, err, done);
4243 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr3, meta, err, done);
4244 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr6, meta, err, done);
4258 vmx = vcpu->vmx;
4259 vmcs = vcpu->vmcs;
4261 running = vcpu_is_running(vcpu->vcpu, &hostcpu);
4263 printf("%s: %s%d is running", __func__, vm_name(vmx->vm),
4264 vcpu->vcpuid);