1/* $OpenBSD: vmm_support.S,v 1.28 2024/07/09 11:15:58 deraadt Exp $ */ 2/* 3 * Copyright (c) 2014 Mike Larkin <mlarkin@openbsd.org> 4 * 5 * Permission to use, copy, modify, and distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 */ 17 18#include "assym.h" 19#include <machine/param.h> 20#include <machine/asm.h> 21#include <machine/codepatch.h> 22#include <machine/psl.h> 23#include <machine/specialreg.h> 24 25/* 26 * XXX duplicated in vmmvar.h due to song-and-dance with sys/rwlock.h inclusion 27 * here 28 */ 29#define VMX_FAIL_LAUNCH_UNKNOWN 1 30#define VMX_FAIL_LAUNCH_INVALID_VMCS 2 31#define VMX_FAIL_LAUNCH_VALID_VMCS 3 32 33 .global vmxon 34 .global vmxoff 35 .global vmclear 36 .global vmptrld 37 .global vmptrst 38 .global vmwrite 39 .global vmread 40 .global invvpid 41 .global invept 42 .global vmx_enter_guest 43 .global vmm_dispatch_intr 44 .global svm_enter_guest 45 46 .text 47 .code64 48 .align 16,0xcc 49vmm_dispatch_intr: 50 movq %rsp, %r11 /* r11 = temporary register */ 51 andq $0xFFFFFFFFFFFFFFF0, %rsp 52 movw %ss, %ax 53 pushq %rax 54 pushq %r11 55 pushfq 56 movw %cs, %ax 57 pushq %rax 58 cli 59 callq *%rdi 60 movq $0,-8(%rsp) 61 ret 62 lfence 63 64ENTRY(vmxon) 65 RETGUARD_SETUP(vmxon, r11) 66 xorq %rax, %rax 67 vmxon (%rdi) 68 setna %al 69 RETGUARD_CHECK(vmxon, r11) 70 ret 71 lfence 72END(vmxon) 73 74ENTRY(vmxoff) 75 RETGUARD_SETUP(vmxoff, r11) 76 xorq %rax, %rax 77 vmxoff 78 setna %al 79 RETGUARD_CHECK(vmxoff, r11) 80 ret 81 lfence 82END(vmxoff) 83 84ENTRY(vmclear) 85 RETGUARD_SETUP(vmclear, r11) 86 xorq %rax, %rax 87 vmclear (%rdi) 88 setna %al 89 RETGUARD_CHECK(vmclear, r11) 90 ret 91 lfence 92END(vmclear) 93 94ENTRY(vmptrld) 95 RETGUARD_SETUP(vmptrld, r11) 96 xorq %rax, %rax 97 vmptrld (%rdi) 98 setna %al 99 RETGUARD_CHECK(vmptrld, r11) 100 ret 101 lfence 102END(vmptrld) 103 104ENTRY(vmptrst) 105 RETGUARD_SETUP(vmptrst, r11) 106 xorq %rax, %rax 107 vmptrst (%rdi) 108 setna %al 109 RETGUARD_CHECK(vmptrst, r11) 110 ret 111 lfence 112 113ENTRY(vmwrite) 114 RETGUARD_SETUP(vmwrite, r11) 115 xorq %rax, %rax 116 vmwrite %rsi, %rdi 117 setna %al 118 RETGUARD_CHECK(vmwrite, r11) 119 ret 120 lfence 121END(vmwrite) 122 123ENTRY(vmread) 124 RETGUARD_SETUP(vmread, r11) 125 xorq %rax, %rax 126 vmread %rdi, (%rsi) 127 setna %al 128 RETGUARD_CHECK(vmread, r11) 129 ret 130 lfence 131END(vmread) 132 133/* 134 * Intel SDM Vol 3C, 31.2 defines different "vmfail" types, but there's no 135 * need to distinguish between CF=1 and ZF=1 for invvpid or invept. 136 */ 137ENTRY(invvpid) 138 RETGUARD_SETUP(invvpid, r11) 139 invvpid (%rsi), %rdi 140 jbe invvpid_fail 141 xorq %rax, %rax 142 jmp invvpid_ret 143invvpid_fail: 144 movq $1, %rax 145invvpid_ret: 146 RETGUARD_CHECK(invvpid, r11) 147 ret 148 lfence 149END(invvpid) 150 151ENTRY(invept) 152 RETGUARD_SETUP(invept, r11) 153 invept (%rsi), %rdi 154 jbe invept_fail 155 xorq %rax, %rax 156 jmp invept_ret 157invept_fail: 158 movq $1, %rax 159invept_ret: 160 RETGUARD_CHECK(invept, r11) 161 ret 162 lfence 163END(invept) 164 165ENTRY(vmx_enter_guest) 166 RETGUARD_SETUP(vmx_enter_guest, r11) 167 movq %rdx, %r8 /* resume flag */ 168 movq %rcx, %r9 /* L1DF MSR support */ 169 testq %r8, %r8 170 jnz skip_init 171 172 /* 173 * XXX make vmx_exit_handler a global and put this in the per-vcpu 174 * init code 175 */ 176 movq $VMCS_HOST_IA32_RIP, %rdi 177 movq $vmx_exit_handler_asm, %rax 178 vmwrite %rax, %rdi /* Host RIP */ 179 180skip_init: 181 RETGUARD_PUSH(r11) 182 183 /* Preserve callee-preserved registers as per AMD64 ABI */ 184 pushq %r15 185 pushq %r14 186 pushq %r13 187 pushq %r12 188 pushq %rbp 189 pushq %rbx 190 pushq %rsi /* Guest Regs Pointer */ 191 192 /* 193 * XXX this MDS mitigation and the L1TF mitigation are believed 194 * XXX to overlap in some cases, but Intel hasn't provided the 195 * XXX information yet to make the correct choices. 196 */ 197 CODEPATCH_START 198 xorl %eax,%eax 199 xorl %ebx,%ebx 200 xorl %ecx,%ecx 201 xorl %edx,%edx 202 xorl %esi,%esi 203 xorl %edi,%edi 204 xorl %ebp,%ebp 205 /* 206 * r8 is a boolean flagging launch or resume 207 * r9 is 0-2 about the CPU 208 */ 209 xorl %r10d,%r10d 210 xorl %r11d,%r11d 211 xorl %r12d,%r12d 212 xorl %r13d,%r13d 213 xorl %r14d,%r14d 214 xorl %r15d,%r15d 215 subq $8, %rsp 216 movw %ds, (%rsp) 217 verw (%rsp) 218 addq $8, %rsp 219 CODEPATCH_END(CPTAG_MDS_VMM) 220 movq (%rsp),%rsi /* reload now that it's mucked with */ 221 222 movq $VMCS_HOST_IA32_RSP, %rdi 223 movq %rsp, %rax 224 vmwrite %rax, %rdi /* Host RSP */ 225 226 /* 227 * Intel L1TF vulnerability fix 228 * 229 * Certain Intel CPUs are broken and allow guest VMs to bypass 230 * EPT entirely as their address harvesting logic treats guest 231 * PTEs as host physical addresses. Flush L1 Dcache to prevent 232 * information leakage by command MSR or manually reading a 233 * bunch of junk in order to fill sizeof(L1 Dcache)*2. 234 * 235 * %r9 (inherited from parameter 4 in %rcx earlier) 236 * determines the flushing requirements 237 * 0 - use manual "junk read" flush 238 * 1 - use MSR command 239 * 2 (VMX_SKIP_L1D_FLUSH) - no flush required on this CPU 240 */ 241 cmpq $VMX_SKIP_L1D_FLUSH, %r9 242 je done_flush 243 244 testq %r9, %r9 245 jz no_l1df_msr 246 247 /* CPU has command MSR */ 248 movq $MSR_FLUSH_CMD, %rcx 249 xorq %rdx, %rdx 250 movq $FLUSH_CMD_L1D_FLUSH, %rax 251 wrmsr 252 jmp done_flush 253 254no_l1df_msr: 255 xorq %r9, %r9 256l1df_tlb_loop: 257 /* XXX get the right L1 size from cpuid */ 258 cmpq $VMX_L1D_FLUSH_SIZE, %r9 259 je l1df_tlb_done 260 movb l1tf_flush_region(%r9), %al 261 addq $PAGE_SIZE, %r9 262 jmp l1df_tlb_loop 263 264l1df_tlb_done: 265 /* 266 * Serialize: ensure previous TLB loads don't pull PTDs 267 * or other PA-containing data into the L1D. 268 */ 269 xorq %rax, %rax 270 cpuid 271 272 xorq %r9, %r9 273l1df_load_cache: 274 movb l1tf_flush_region(%r9), %al 275 /* XXX get the right cacheline size from cpuid */ 276 addq $0x40, %r9 277 cmpq $VMX_L1D_FLUSH_SIZE, %r9 278 jne l1df_load_cache 279 lfence 280 281done_flush: 282 testq %r8, %r8 283 jnz do_resume 284 285 /* Restore guest registers */ 286 movq 0xa0(%rsi), %rax 287 movq %rax, %dr0 288 movq 0xa8(%rsi), %rax 289 movq %rax, %dr1 290 movq 0xb0(%rsi), %rax 291 movq %rax, %dr2 292 movq 0xb8(%rsi), %rax 293 movq %rax, %dr3 294 movq 0xc0(%rsi), %rax 295 movq %rax, %dr6 296 movq 0x78(%rsi), %rax 297 movq %rax, %cr2 298 movq 0x70(%rsi), %r15 299 movq 0x68(%rsi), %r14 300 movq 0x60(%rsi), %r13 301 movq 0x58(%rsi), %r12 302 movq 0x50(%rsi), %r11 303 movq 0x48(%rsi), %r10 304 movq 0x40(%rsi), %r9 305 movq %rsi, %r8 306 /* XXX get the right cacheline size from cpuid */ 307 addq $0x40, %r8 308 clflush (%r8) 309 movq 0x38(%rsi), %r8 310 movq 0x30(%rsi), %rbp 311 movq 0x28(%rsi), %rdi 312 movq 0x20(%rsi), %rdx 313 movq 0x18(%rsi), %rcx 314 movq 0x10(%rsi), %rbx 315 movq 0x08(%rsi), %rax 316 clflush (%rsi) 317 movq 0x00(%rsi), %rsi 318 319 vmlaunch 320 jmp fail_launch_or_resume 321do_resume: 322 /* Restore guest registers */ 323 movq 0xa0(%rsi), %rax 324 movq %rax, %dr0 325 movq 0xa8(%rsi), %rax 326 movq %rax, %dr1 327 movq 0xb0(%rsi), %rax 328 movq %rax, %dr2 329 movq 0xb8(%rsi), %rax 330 movq %rax, %dr3 331 movq 0xc0(%rsi), %rax 332 movq %rax, %dr6 333 movq 0x78(%rsi), %rax 334 movq %rax, %cr2 335 movq 0x70(%rsi), %r15 336 movq 0x68(%rsi), %r14 337 movq 0x60(%rsi), %r13 338 movq 0x58(%rsi), %r12 339 movq 0x50(%rsi), %r11 340 movq 0x48(%rsi), %r10 341 movq 0x40(%rsi), %r9 342 movq %rsi, %r8 343 /* XXX get the right cacheline size from cpuid */ 344 addq $0x40, %r8 345 clflush (%r8) 346 movq 0x38(%rsi), %r8 347 movq 0x30(%rsi), %rbp 348 movq 0x28(%rsi), %rdi 349 movq 0x20(%rsi), %rdx 350 movq 0x18(%rsi), %rcx 351 movq 0x10(%rsi), %rbx 352 movq 0x08(%rsi), %rax 353 clflush (%rsi) 354 movq 0x00(%rsi), %rsi 355 356 vmresume 357fail_launch_or_resume: 358 RET_STACK_REFILL_WITH_RCX 359 360 /* Failed launch/resume (fell through) */ 361 jc fail_launch_invalid_vmcs /* Invalid VMCS */ 362 jz fail_launch_valid_vmcs /* Valid VMCS, failed launch/resume */ 363 364 /* Unknown failure mode (not documented as per Intel SDM) */ 365fail_launch_unknown: 366 movq $VMX_FAIL_LAUNCH_UNKNOWN, %rdi 367 popq %rsi 368 jmp restore_host 369 370fail_launch_invalid_vmcs: 371 movq $VMX_FAIL_LAUNCH_INVALID_VMCS, %rdi 372 popq %rsi 373 jmp restore_host 374 375fail_launch_valid_vmcs: 376 movq $VMCS_INSTRUCTION_ERROR, %rdi 377 popq %rsi 378 vmread %rdi, %rax 379 /* XXX check failure of vmread */ 380 movl %eax, 0x80(%rsi) 381 movq $VMX_FAIL_LAUNCH_VALID_VMCS, %rdi 382 jmp restore_host 383 384vmx_exit_handler_asm: 385 /* Preserve guest registers not saved in VMCS */ 386 pushq %rsi 387 pushq %rdi 388 movq 0x10(%rsp), %rdi 389 movq 0x8(%rsp), %rsi 390 movq %rsi, (%rdi) 391 popq %rdi 392 popq %rsi /* discard */ 393 394 popq %rsi 395 movq %rax, 0x8(%rsi) 396 movq %rbx, 0x10(%rsi) 397 movq %rcx, 0x18(%rsi) 398 movq %rdx, 0x20(%rsi) 399 movq %rdi, 0x28(%rsi) 400 movq %rbp, 0x30(%rsi) 401 movq %r8, 0x38(%rsi) 402 movq %r9, 0x40(%rsi) 403 movq %r10, 0x48(%rsi) 404 movq %r11, 0x50(%rsi) 405 movq %r12, 0x58(%rsi) 406 movq %r13, 0x60(%rsi) 407 movq %r14, 0x68(%rsi) 408 movq %r15, 0x70(%rsi) 409 movq %cr2, %rax 410 movq %rax, 0x78(%rsi) 411 movq %dr0, %rax 412 movq %rax, 0xa0(%rsi) 413 movq %dr1, %rax 414 movq %rax, 0xa8(%rsi) 415 movq %dr2, %rax 416 movq %rax, 0xb0(%rsi) 417 movq %dr3, %rax 418 movq %rax, 0xb8(%rsi) 419 movq %dr6, %rax 420 movq %rax, 0xc0(%rsi) 421 422 /* %rdi = 0 means we took an exit */ 423 xorq %rdi, %rdi 424 425 RET_STACK_REFILL_WITH_RCX 426 427restore_host: 428 popq %rbx 429 popq %rbp 430 popq %r12 431 popq %r13 432 popq %r14 433 popq %r15 434 435 RETGUARD_POP(r11) 436 437 movq %rdi, %rax 438 RETGUARD_CHECK(vmx_enter_guest, r11) 439 ret 440 lfence 441END(vmx_enter_guest) 442 443ENTRY(svm_enter_guest) 444 RETGUARD_SETUP(svm_enter_guest, r11) 445 clgi 446 movq %rdi, %r8 447 pushfq 448 449 pushq %rdx /* gdt pointer */ 450 451 /* 452 * Save (possibly) lazy-switched selectors 453 */ 454 strw %ax 455 pushw %ax 456 movw %es, %ax 457 pushw %ax 458 movw %ds, %ax 459 pushw %ax 460 movw %ss, %ax 461 pushw %ax 462 463 movq $MSR_FSBASE, %rcx 464 rdmsr 465 pushq %rax 466 pushq %rdx 467 pushw %fs 468 movq $MSR_GSBASE, %rcx 469 rdmsr 470 pushq %rax 471 pushq %rdx 472 pushw %gs 473 movq $MSR_KERNELGSBASE, %rcx 474 rdmsr 475 pushq %rax 476 pushq %rdx 477 478 /* 479 * Save various MSRs 480 */ 481 movq $MSR_STAR, %rcx 482 rdmsr 483 pushq %rax 484 pushq %rdx 485 486 movq $MSR_LSTAR, %rcx 487 rdmsr 488 pushq %rax 489 pushq %rdx 490 491 movq $MSR_SFMASK, %rcx 492 rdmsr 493 pushq %rax 494 pushq %rdx 495 496 RETGUARD_PUSH(r11) 497 498 /* Preserve callee-preserved registers as per AMD64 ABI */ 499 pushq %r15 500 pushq %r14 501 pushq %r13 502 pushq %r12 503 pushq %rbp 504 pushq %rbx 505 pushq %rsi /* Guest Regs Pointer */ 506 507 /* Restore guest registers */ 508 movq %r8, %rax /* rax = vmcb pa */ 509 movq 0xa0(%rsi), %r8 510 movq %r8, %dr0 511 movq 0xa8(%rsi), %r8 512 movq %r8, %dr1 513 movq 0xb0(%rsi), %r8 514 movq %r8, %dr2 515 movq 0xb8(%rsi), %r8 516 movq %r8, %dr3 517 /* %dr6 is saved in the VMCB */ 518 movq 0x78(%rsi), %r8 519 movq %r8, %cr2 520 movq 0x70(%rsi), %r15 521 movq 0x68(%rsi), %r14 522 movq 0x60(%rsi), %r13 523 movq 0x58(%rsi), %r12 524 movq 0x50(%rsi), %r11 525 movq 0x48(%rsi), %r10 526 movq 0x40(%rsi), %r9 527 movq 0x38(%rsi), %r8 528 movq 0x30(%rsi), %rbp 529 movq 0x28(%rsi), %rdi 530 movq 0x20(%rsi), %rdx 531 movq 0x18(%rsi), %rcx 532 movq 0x10(%rsi), %rbx 533 /* %rax at 0x08(%rsi) is not needed in SVM */ 534 movq 0x00(%rsi), %rsi 535 536 vmload %rax 537 vmrun %rax 538 vmsave %rax 539 540 /* Preserve guest registers not saved in VMCB */ 541 pushq %rsi 542 pushq %rdi 543 movq 0x10(%rsp), %rdi 544 movq 0x8(%rsp), %rsi 545 movq %rsi, (%rdi) 546 popq %rdi 547 popq %rsi /* discard */ 548 549 popq %rsi 550 /* %rax at 0x08(%rsi) is not needed in SVM */ 551 movq %rbx, 0x10(%rsi) 552 movq %rcx, 0x18(%rsi) 553 movq %rdx, 0x20(%rsi) 554 movq %rdi, 0x28(%rsi) 555 movq %rbp, 0x30(%rsi) 556 movq %r8, 0x38(%rsi) 557 movq %r9, 0x40(%rsi) 558 movq %r10, 0x48(%rsi) 559 movq %r11, 0x50(%rsi) 560 movq %r12, 0x58(%rsi) 561 movq %r13, 0x60(%rsi) 562 movq %r14, 0x68(%rsi) 563 movq %r15, 0x70(%rsi) 564 movq %cr2, %rax 565 movq %rax, 0x78(%rsi) 566 movq %dr0, %rax 567 movq %rax, 0xa0(%rsi) 568 movq %dr1, %rax 569 movq %rax, 0xa8(%rsi) 570 movq %dr2, %rax 571 movq %rax, 0xb0(%rsi) 572 movq %dr3, %rax 573 movq %rax, 0xb8(%rsi) 574 /* %dr6 is saved in the VMCB */ 575 576 /* %rdi = 0 means we took an exit */ 577 xorq %rdi, %rdi 578 579restore_host_svm: 580 popq %rbx 581 popq %rbp 582 popq %r12 583 popq %r13 584 popq %r14 585 popq %r15 586 587 RETGUARD_POP(r11) 588 589 /* 590 * Restore saved MSRs 591 */ 592 popq %rdx 593 popq %rax 594 movq $MSR_SFMASK, %rcx 595 wrmsr 596 597 /* make sure guest doesn't bleed into host */ 598 xorl %edx, %edx 599 xorl %eax, %eax 600 movq $MSR_CSTAR, %rcx 601 wrmsr 602 603 popq %rdx 604 popq %rax 605 movq $MSR_LSTAR, %rcx 606 wrmsr 607 608 popq %rdx 609 popq %rax 610 movq $MSR_STAR, %rcx 611 wrmsr 612 613 /* 614 * popw %gs will reset gsbase to 0, so preserve it 615 * first. This is to accommodate possibly lazy-switched 616 * selectors from above 617 */ 618 cli 619 popq %rdx 620 popq %rax 621 movq $MSR_KERNELGSBASE, %rcx 622 wrmsr 623 624 popw %gs 625 popq %rdx 626 popq %rax 627 movq $MSR_GSBASE, %rcx 628 wrmsr 629 630 popw %fs 631 popq %rdx 632 popq %rax 633 movq $MSR_FSBASE, %rcx 634 wrmsr 635 636 popw %ax 637 movw %ax, %ss 638 popw %ax 639 movw %ax, %ds 640 popw %ax 641 movw %ax, %es 642 643 xorq %rax, %rax 644 lldtw %ax /* Host LDT is always 0 */ 645 646 popw %ax /* ax = saved TR */ 647 648 popq %rdx 649 addq $0x2, %rdx 650 movq (%rdx), %rdx 651 652 /* rdx = GDTR base addr */ 653 andb $0xF9, 5(%rdx, %rax) 654 655 ltrw %ax 656 657 popfq 658 659 movq %rdi, %rax 660 661 RETGUARD_CHECK(svm_enter_guest, r11) 662 ret 663 lfence 664END(svm_enter_guest) 665