1 /* $NetBSD: frameasm.h,v 1.48 2020/04/25 15:26:16 bouyer Exp $ */ 2 3 #ifndef _AMD64_MACHINE_FRAMEASM_H 4 #define _AMD64_MACHINE_FRAMEASM_H 5 6 #ifdef _KERNEL_OPT 7 #include "opt_xen.h" 8 #include "opt_svs.h" 9 #include "opt_kcov.h" 10 #include "opt_kmsan.h" 11 #endif 12 13 /* 14 * Macros to define pushing/popping frames for interrupts, traps 15 * and system calls. Currently all the same; will diverge later. 16 */ 17 18 #ifdef XENPV 19 #define HYPERVISOR_iret hypercall_page + (__HYPERVISOR_iret * 32) 20 /* Xen do not need swapgs, done by hypervisor */ 21 #define swapgs 22 #define iretq pushq $0 ; jmp HYPERVISOR_iret 23 #define XEN_ONLY2(x,y) x,y 24 #define NOT_XEN(x) 25 26 #define CLI(temp_reg) \ 27 movq CPUVAR(VCPU),%r ## temp_reg ; \ 28 movb $1,EVTCHN_UPCALL_MASK(%r ## temp_reg); 29 30 #define STI(temp_reg) \ 31 movq CPUVAR(VCPU),%r ## temp_reg ; \ 32 movb $0,EVTCHN_UPCALL_MASK(%r ## temp_reg); 33 34 #define PUSHF(temp_reg) \ 35 movq CPUVAR(VCPU),%r ## temp_reg ; \ 36 movzbl EVTCHN_UPCALL_MASK(%r ## temp_reg), %e ## temp_reg; \ 37 pushq %r ## temp_reg 38 39 #define POPF \ 40 popq %rdi; \ 41 call _C_LABEL(xen_write_psl) 42 43 44 #else /* XENPV */ 45 #define XEN_ONLY2(x,y) 46 #define NOT_XEN(x) x 47 #define CLI(temp_reg) cli 48 #define STI(temp_reg) sti 49 #define PUSHF(temp_reg) pushf 50 #define POPL popl 51 #endif /* XEN */ 52 53 #define HP_NAME_CLAC 1 54 #define HP_NAME_STAC 2 55 #define HP_NAME_NOLOCK 3 56 #define HP_NAME_RETFENCE 4 57 #define HP_NAME_SVS_ENTER 5 58 #define HP_NAME_SVS_LEAVE 6 59 #define HP_NAME_SVS_ENTER_ALT 7 60 #define HP_NAME_SVS_LEAVE_ALT 8 61 #define HP_NAME_IBRS_ENTER 9 62 #define HP_NAME_IBRS_LEAVE 10 63 #define HP_NAME_SVS_ENTER_NMI 11 64 #define HP_NAME_SVS_LEAVE_NMI 12 65 #define HP_NAME_MDS_LEAVE 13 66 67 #define HOTPATCH(name, size) \ 68 123: ; \ 69 .pushsection .rodata.hotpatch, "a" ; \ 70 .byte name ; \ 71 .byte size ; \ 72 .quad 123b ; \ 73 .popsection 74 75 #define SMAP_ENABLE \ 76 HOTPATCH(HP_NAME_CLAC, 3) ; \ 77 .byte 0x0F, 0x1F, 0x00 ; \ 78 79 #define SMAP_DISABLE \ 80 HOTPATCH(HP_NAME_STAC, 3) ; \ 81 .byte 0x0F, 0x1F, 0x00 ; \ 82 83 /* 84 * IBRS 85 */ 86 87 #define IBRS_ENTER_BYTES 12 88 #define IBRS_ENTER \ 89 HOTPATCH(HP_NAME_IBRS_ENTER, IBRS_ENTER_BYTES) ; \ 90 NOIBRS_ENTER 91 #define NOIBRS_ENTER \ 92 .byte 0xEB, (IBRS_ENTER_BYTES-2) /* jmp */ ; \ 93 .fill (IBRS_ENTER_BYTES-2),1,0xCC 94 95 #define IBRS_LEAVE_BYTES 12 96 #define IBRS_LEAVE \ 97 HOTPATCH(HP_NAME_IBRS_LEAVE, IBRS_LEAVE_BYTES) ; \ 98 NOIBRS_LEAVE 99 #define NOIBRS_LEAVE \ 100 .byte 0xEB, (IBRS_LEAVE_BYTES-2) /* jmp */ ; \ 101 .fill (IBRS_LEAVE_BYTES-2),1,0xCC 102 103 /* 104 * MDS 105 */ 106 107 #define MDS_LEAVE_BYTES 10 108 #define MDS_LEAVE \ 109 HOTPATCH(HP_NAME_MDS_LEAVE, MDS_LEAVE_BYTES) ; \ 110 NOMDS_LEAVE 111 #define NOMDS_LEAVE \ 112 .byte 0xEB, (MDS_LEAVE_BYTES-2) /* jmp */ ; \ 113 .fill (MDS_LEAVE_BYTES-2),1,0xCC 114 115 #define SWAPGS NOT_XEN(swapgs) 116 117 /* 118 * These are used on interrupt or trap entry or exit. 119 */ 120 #define INTR_SAVE_GPRS \ 121 movq %rdi,TF_RDI(%rsp) ; \ 122 movq %rsi,TF_RSI(%rsp) ; \ 123 movq %rdx,TF_RDX(%rsp) ; \ 124 movq %rcx,TF_RCX(%rsp) ; \ 125 movq %r8,TF_R8(%rsp) ; \ 126 movq %r9,TF_R9(%rsp) ; \ 127 movq %r10,TF_R10(%rsp) ; \ 128 movq %r11,TF_R11(%rsp) ; \ 129 movq %r12,TF_R12(%rsp) ; \ 130 movq %r13,TF_R13(%rsp) ; \ 131 movq %r14,TF_R14(%rsp) ; \ 132 movq %r15,TF_R15(%rsp) ; \ 133 movq %rbp,TF_RBP(%rsp) ; \ 134 movq %rbx,TF_RBX(%rsp) ; \ 135 movq %rax,TF_RAX(%rsp) 136 137 #define INTR_RESTORE_GPRS \ 138 movq TF_RDI(%rsp),%rdi ; \ 139 movq TF_RSI(%rsp),%rsi ; \ 140 movq TF_RDX(%rsp),%rdx ; \ 141 movq TF_RCX(%rsp),%rcx ; \ 142 movq TF_R8(%rsp),%r8 ; \ 143 movq TF_R9(%rsp),%r9 ; \ 144 movq TF_R10(%rsp),%r10 ; \ 145 movq TF_R11(%rsp),%r11 ; \ 146 movq TF_R12(%rsp),%r12 ; \ 147 movq TF_R13(%rsp),%r13 ; \ 148 movq TF_R14(%rsp),%r14 ; \ 149 movq TF_R15(%rsp),%r15 ; \ 150 movq TF_RBP(%rsp),%rbp ; \ 151 movq TF_RBX(%rsp),%rbx ; \ 152 movq TF_RAX(%rsp),%rax 153 154 #define TEXT_USER_BEGIN .pushsection .text.user, "ax" 155 #define TEXT_USER_END .popsection 156 157 #ifdef SVS 158 159 /* XXX: put this somewhere else */ 160 #define SVS_UTLS 0xffffff0000000000 /* PMAP_PCPU_BASE */ 161 #define UTLS_KPDIRPA 0 162 #define UTLS_SCRATCH 8 163 #define UTLS_RSP0 16 164 165 #define SVS_ENTER_BYTES 22 166 #define NOSVS_ENTER \ 167 .byte 0xEB, (SVS_ENTER_BYTES-2) /* jmp */ ; \ 168 .fill (SVS_ENTER_BYTES-2),1,0xCC 169 #define SVS_ENTER \ 170 HOTPATCH(HP_NAME_SVS_ENTER, SVS_ENTER_BYTES) ; \ 171 NOSVS_ENTER 172 173 #define SVS_LEAVE_BYTES 21 174 #define NOSVS_LEAVE \ 175 .byte 0xEB, (SVS_LEAVE_BYTES-2) /* jmp */ ; \ 176 .fill (SVS_LEAVE_BYTES-2),1,0xCC 177 #define SVS_LEAVE \ 178 HOTPATCH(HP_NAME_SVS_LEAVE, SVS_LEAVE_BYTES) ; \ 179 NOSVS_LEAVE 180 181 #define SVS_ENTER_ALT_BYTES 23 182 #define NOSVS_ENTER_ALTSTACK \ 183 .byte 0xEB, (SVS_ENTER_ALT_BYTES-2) /* jmp */ ; \ 184 .fill (SVS_ENTER_ALT_BYTES-2),1,0xCC 185 #define SVS_ENTER_ALTSTACK \ 186 HOTPATCH(HP_NAME_SVS_ENTER_ALT, SVS_ENTER_ALT_BYTES) ; \ 187 NOSVS_ENTER_ALTSTACK 188 189 #define SVS_LEAVE_ALT_BYTES 22 190 #define NOSVS_LEAVE_ALTSTACK \ 191 .byte 0xEB, (SVS_LEAVE_ALT_BYTES-2) /* jmp */ ; \ 192 .fill (SVS_LEAVE_ALT_BYTES-2),1,0xCC 193 #define SVS_LEAVE_ALTSTACK \ 194 HOTPATCH(HP_NAME_SVS_LEAVE_ALT, SVS_LEAVE_ALT_BYTES) ; \ 195 NOSVS_LEAVE_ALTSTACK 196 197 #define SVS_ENTER_NMI_BYTES 22 198 #define NOSVS_ENTER_NMI \ 199 .byte 0xEB, (SVS_ENTER_NMI_BYTES-2) /* jmp */ ; \ 200 .fill (SVS_ENTER_NMI_BYTES-2),1,0xCC 201 #define SVS_ENTER_NMI \ 202 HOTPATCH(HP_NAME_SVS_ENTER_NMI, SVS_ENTER_NMI_BYTES) ; \ 203 NOSVS_ENTER_NMI 204 205 #define SVS_LEAVE_NMI_BYTES 11 206 #define NOSVS_LEAVE_NMI \ 207 .byte 0xEB, (SVS_LEAVE_NMI_BYTES-2) /* jmp */ ; \ 208 .fill (SVS_LEAVE_NMI_BYTES-2),1,0xCC 209 #define SVS_LEAVE_NMI \ 210 HOTPATCH(HP_NAME_SVS_LEAVE_NMI, SVS_LEAVE_NMI_BYTES) ; \ 211 NOSVS_LEAVE_NMI 212 213 #else 214 #define SVS_ENTER /* nothing */ 215 #define SVS_ENTER_NMI /* nothing */ 216 #define SVS_LEAVE /* nothing */ 217 #define SVS_LEAVE_NMI /* nothing */ 218 #define SVS_ENTER_ALTSTACK /* nothing */ 219 #define SVS_LEAVE_ALTSTACK /* nothing */ 220 #endif 221 222 #ifdef KMSAN 223 #define KMSAN_ENTER \ 224 movq %rsp,%rdi ; \ 225 movq $TF_REGSIZE+16+40,%rsi ; \ 226 xorq %rdx,%rdx ; \ 227 callq kmsan_mark ; \ 228 callq kmsan_intr_enter 229 #define KMSAN_LEAVE \ 230 pushq %rbp ; \ 231 movq %rsp,%rbp ; \ 232 callq kmsan_intr_leave ; \ 233 popq %rbp 234 #define KMSAN_INIT_ARG(sz) \ 235 pushq %rax ; \ 236 pushq %rcx ; \ 237 pushq %rdx ; \ 238 pushq %rsi ; \ 239 pushq %rdi ; \ 240 pushq %r8 ; \ 241 pushq %r9 ; \ 242 pushq %r10 ; \ 243 pushq %r11 ; \ 244 movq $sz,%rdi ; \ 245 callq _C_LABEL(kmsan_init_arg); \ 246 popq %r11 ; \ 247 popq %r10 ; \ 248 popq %r9 ; \ 249 popq %r8 ; \ 250 popq %rdi ; \ 251 popq %rsi ; \ 252 popq %rdx ; \ 253 popq %rcx ; \ 254 popq %rax 255 #define KMSAN_INIT_RET(sz) \ 256 pushq %rax ; \ 257 pushq %rcx ; \ 258 pushq %rdx ; \ 259 pushq %rsi ; \ 260 pushq %rdi ; \ 261 pushq %r8 ; \ 262 pushq %r9 ; \ 263 pushq %r10 ; \ 264 pushq %r11 ; \ 265 movq $sz,%rdi ; \ 266 callq _C_LABEL(kmsan_init_ret); \ 267 popq %r11 ; \ 268 popq %r10 ; \ 269 popq %r9 ; \ 270 popq %r8 ; \ 271 popq %rdi ; \ 272 popq %rsi ; \ 273 popq %rdx ; \ 274 popq %rcx ; \ 275 popq %rax 276 #else 277 #define KMSAN_ENTER /* nothing */ 278 #define KMSAN_LEAVE /* nothing */ 279 #define KMSAN_INIT_ARG(sz) /* nothing */ 280 #define KMSAN_INIT_RET(sz) /* nothing */ 281 #endif 282 283 #ifdef KCOV 284 #define KCOV_DISABLE \ 285 incl CPUVAR(IDEPTH) 286 #define KCOV_ENABLE \ 287 decl CPUVAR(IDEPTH) 288 #else 289 #define KCOV_DISABLE /* nothing */ 290 #define KCOV_ENABLE /* nothing */ 291 #endif 292 293 #define INTRENTRY \ 294 subq $TF_REGSIZE,%rsp ; \ 295 INTR_SAVE_GPRS ; \ 296 cld ; \ 297 SMAP_ENABLE ; \ 298 testb $SEL_UPL,TF_CS(%rsp) ; \ 299 je 98f ; \ 300 SWAPGS ; \ 301 IBRS_ENTER ; \ 302 SVS_ENTER ; \ 303 movw %gs,TF_GS(%rsp) ; \ 304 movw %fs,TF_FS(%rsp) ; \ 305 movw %es,TF_ES(%rsp) ; \ 306 movw %ds,TF_DS(%rsp) ; \ 307 98: KMSAN_ENTER 308 309 #define INTRFASTEXIT \ 310 jmp intrfastexit 311 312 #define INTR_RECURSE_HWFRAME \ 313 movq %rsp,%r10 ; \ 314 movl %ss,%r11d ; \ 315 pushq %r11 ; \ 316 pushq %r10 ; \ 317 pushfq ; \ 318 pushq $GSEL(GCODE_SEL,SEL_KPL); \ 319 /* XEN: We must fixup CS, as even kernel mode runs at CPL 3 */ \ 320 XEN_ONLY2(andb $0xfc,(%rsp);) \ 321 pushq %r13 ; 322 323 #define INTR_RECURSE_ENTRY \ 324 subq $TF_REGSIZE,%rsp ; \ 325 INTR_SAVE_GPRS ; \ 326 cld ; \ 327 KMSAN_ENTER 328 329 #define CHECK_DEFERRED_SWITCH \ 330 cmpl $0, CPUVAR(WANT_PMAPLOAD) 331 332 #define CHECK_ASTPENDING(reg) cmpl $0, L_MD_ASTPENDING(reg) 333 #define CLEAR_ASTPENDING(reg) movl $0, L_MD_ASTPENDING(reg) 334 335 /* 336 * If the FPU state is not in the CPU, restore it. Executed with interrupts 337 * disabled. 338 * 339 * %r14 is curlwp, must not be modified 340 * %rbx must not be modified 341 */ 342 #define HANDLE_DEFERRED_FPU \ 343 testl $MDL_FPU_IN_CPU,L_MD_FLAGS(%r14) ; \ 344 jnz 1f ; \ 345 call _C_LABEL(fpu_handle_deferred) ; \ 346 orl $MDL_FPU_IN_CPU,L_MD_FLAGS(%r14) ; \ 347 1: 348 349 #endif /* _AMD64_MACHINE_FRAMEASM_H */ 350