1 /* $NetBSD: frameasm.h,v 1.54 2022/04/09 12:07:00 riastradh Exp $ */ 2 3 #ifndef _AMD64_MACHINE_FRAMEASM_H 4 #define _AMD64_MACHINE_FRAMEASM_H 5 6 #ifdef _KERNEL_OPT 7 #include "opt_xen.h" 8 #include "opt_svs.h" 9 #include "opt_kcov.h" 10 #include "opt_kmsan.h" 11 #endif 12 13 /* 14 * Macros to define pushing/popping frames for interrupts, traps 15 * and system calls. Currently all the same; will diverge later. 16 */ 17 18 #ifdef XENPV 19 #define HYPERVISOR_iret hypercall_page + (__HYPERVISOR_iret * 32) 20 /* Xen do not need swapgs, done by hypervisor */ 21 #define swapgs 22 #define iretq pushq $0 ; jmp HYPERVISOR_iret 23 #define XEN_ONLY2(x,y) x,y 24 #define NOT_XEN(x) 25 26 #define CLI(temp_reg) \ 27 movq CPUVAR(VCPU),%r ## temp_reg ; \ 28 movb $1,EVTCHN_UPCALL_MASK(%r ## temp_reg); 29 30 #define STI(temp_reg) \ 31 movq CPUVAR(VCPU),%r ## temp_reg ; \ 32 movb $0,EVTCHN_UPCALL_MASK(%r ## temp_reg); 33 34 #define PUSHF(temp_reg) \ 35 movq CPUVAR(VCPU),%r ## temp_reg ; \ 36 movzbl EVTCHN_UPCALL_MASK(%r ## temp_reg), %e ## temp_reg; \ 37 pushq %r ## temp_reg 38 39 #define POPF \ 40 popq %rdi; \ 41 call _C_LABEL(xen_write_psl) 42 43 44 #else /* XENPV */ 45 #define XEN_ONLY2(x,y) 46 #define NOT_XEN(x) x 47 #define CLI(temp_reg) cli 48 #define STI(temp_reg) sti 49 #define PUSHF(temp_reg) pushf 50 #define POPL popl 51 #endif /* XENPV */ 52 53 #define HP_NAME_CLAC 1 54 #define HP_NAME_STAC 2 55 #define HP_NAME_NOLOCK 3 56 #define HP_NAME_RETFENCE 4 57 #define HP_NAME_SVS_ENTER 5 58 #define HP_NAME_SVS_LEAVE 6 59 #define HP_NAME_SVS_ENTER_ALT 7 60 #define HP_NAME_SVS_LEAVE_ALT 8 61 #define HP_NAME_IBRS_ENTER 9 62 #define HP_NAME_IBRS_LEAVE 10 63 #define HP_NAME_SVS_ENTER_NMI 11 64 #define HP_NAME_SVS_LEAVE_NMI 12 65 #define HP_NAME_MDS_LEAVE 13 66 #define HP_NAME_SSE2_MFENCE 14 67 68 #define HOTPATCH(name, size) \ 69 123: ; \ 70 .pushsection .rodata.hotpatch, "a" ; \ 71 .byte name ; \ 72 .byte size ; \ 73 .quad 123b ; \ 74 .popsection 75 76 #define SMAP_ENABLE \ 77 HOTPATCH(HP_NAME_CLAC, 3) ; \ 78 .byte 0x0F, 0x1F, 0x00 ; \ 79 80 #define SMAP_DISABLE \ 81 HOTPATCH(HP_NAME_STAC, 3) ; \ 82 .byte 0x0F, 0x1F, 0x00 ; \ 83 84 /* 85 * IBRS 86 */ 87 88 #define IBRS_ENTER_BYTES 12 89 #define IBRS_ENTER \ 90 HOTPATCH(HP_NAME_IBRS_ENTER, IBRS_ENTER_BYTES) ; \ 91 NOIBRS_ENTER 92 #define NOIBRS_ENTER \ 93 .byte 0xEB, (IBRS_ENTER_BYTES-2) /* jmp */ ; \ 94 .fill (IBRS_ENTER_BYTES-2),1,0xCC 95 96 #define IBRS_LEAVE_BYTES 12 97 #define IBRS_LEAVE \ 98 HOTPATCH(HP_NAME_IBRS_LEAVE, IBRS_LEAVE_BYTES) ; \ 99 NOIBRS_LEAVE 100 #define NOIBRS_LEAVE \ 101 .byte 0xEB, (IBRS_LEAVE_BYTES-2) /* jmp */ ; \ 102 .fill (IBRS_LEAVE_BYTES-2),1,0xCC 103 104 /* 105 * MDS 106 */ 107 108 #define MDS_LEAVE_BYTES 10 109 #define MDS_LEAVE \ 110 HOTPATCH(HP_NAME_MDS_LEAVE, MDS_LEAVE_BYTES) ; \ 111 NOMDS_LEAVE 112 #define NOMDS_LEAVE \ 113 .byte 0xEB, (MDS_LEAVE_BYTES-2) /* jmp */ ; \ 114 .fill (MDS_LEAVE_BYTES-2),1,0xCC 115 116 #define SWAPGS NOT_XEN(swapgs) 117 118 /* 119 * These are used on interrupt or trap entry or exit. 120 */ 121 #define INTR_SAVE_GPRS \ 122 movq %rdi,TF_RDI(%rsp) ; \ 123 movq %rsi,TF_RSI(%rsp) ; \ 124 movq %rdx,TF_RDX(%rsp) ; \ 125 movq %rcx,TF_RCX(%rsp) ; \ 126 movq %r8,TF_R8(%rsp) ; \ 127 movq %r9,TF_R9(%rsp) ; \ 128 movq %r10,TF_R10(%rsp) ; \ 129 movq %r11,TF_R11(%rsp) ; \ 130 movq %r12,TF_R12(%rsp) ; \ 131 movq %r13,TF_R13(%rsp) ; \ 132 movq %r14,TF_R14(%rsp) ; \ 133 movq %r15,TF_R15(%rsp) ; \ 134 movq %rbp,TF_RBP(%rsp) ; \ 135 movq %rbx,TF_RBX(%rsp) ; \ 136 movq %rax,TF_RAX(%rsp) 137 138 #define INTR_RESTORE_GPRS \ 139 movq TF_RDI(%rsp),%rdi ; \ 140 movq TF_RSI(%rsp),%rsi ; \ 141 movq TF_RDX(%rsp),%rdx ; \ 142 movq TF_RCX(%rsp),%rcx ; \ 143 movq TF_R8(%rsp),%r8 ; \ 144 movq TF_R9(%rsp),%r9 ; \ 145 movq TF_R10(%rsp),%r10 ; \ 146 movq TF_R11(%rsp),%r11 ; \ 147 movq TF_R12(%rsp),%r12 ; \ 148 movq TF_R13(%rsp),%r13 ; \ 149 movq TF_R14(%rsp),%r14 ; \ 150 movq TF_R15(%rsp),%r15 ; \ 151 movq TF_RBP(%rsp),%rbp ; \ 152 movq TF_RBX(%rsp),%rbx ; \ 153 movq TF_RAX(%rsp),%rax 154 155 #define TEXT_USER_BEGIN .pushsection .text.user, "ax" 156 #define TEXT_USER_END .popsection 157 158 #ifdef SVS 159 160 /* XXX: put this somewhere else */ 161 #define SVS_UTLS 0xffffff0000000000 /* PMAP_PCPU_BASE */ 162 #define UTLS_KPDIRPA 0 163 #define UTLS_SCRATCH 8 164 #define UTLS_RSP0 16 165 166 #define SVS_ENTER_BYTES 22 167 #define NOSVS_ENTER \ 168 .byte 0xEB, (SVS_ENTER_BYTES-2) /* jmp */ ; \ 169 .fill (SVS_ENTER_BYTES-2),1,0xCC 170 #define SVS_ENTER \ 171 HOTPATCH(HP_NAME_SVS_ENTER, SVS_ENTER_BYTES) ; \ 172 NOSVS_ENTER 173 174 #define SVS_LEAVE_BYTES 21 175 #define NOSVS_LEAVE \ 176 .byte 0xEB, (SVS_LEAVE_BYTES-2) /* jmp */ ; \ 177 .fill (SVS_LEAVE_BYTES-2),1,0xCC 178 #define SVS_LEAVE \ 179 HOTPATCH(HP_NAME_SVS_LEAVE, SVS_LEAVE_BYTES) ; \ 180 NOSVS_LEAVE 181 182 #define SVS_ENTER_ALT_BYTES 23 183 #define NOSVS_ENTER_ALTSTACK \ 184 .byte 0xEB, (SVS_ENTER_ALT_BYTES-2) /* jmp */ ; \ 185 .fill (SVS_ENTER_ALT_BYTES-2),1,0xCC 186 #define SVS_ENTER_ALTSTACK \ 187 HOTPATCH(HP_NAME_SVS_ENTER_ALT, SVS_ENTER_ALT_BYTES) ; \ 188 NOSVS_ENTER_ALTSTACK 189 190 #define SVS_LEAVE_ALT_BYTES 22 191 #define NOSVS_LEAVE_ALTSTACK \ 192 .byte 0xEB, (SVS_LEAVE_ALT_BYTES-2) /* jmp */ ; \ 193 .fill (SVS_LEAVE_ALT_BYTES-2),1,0xCC 194 #define SVS_LEAVE_ALTSTACK \ 195 HOTPATCH(HP_NAME_SVS_LEAVE_ALT, SVS_LEAVE_ALT_BYTES) ; \ 196 NOSVS_LEAVE_ALTSTACK 197 198 #define SVS_ENTER_NMI_BYTES 22 199 #define NOSVS_ENTER_NMI \ 200 .byte 0xEB, (SVS_ENTER_NMI_BYTES-2) /* jmp */ ; \ 201 .fill (SVS_ENTER_NMI_BYTES-2),1,0xCC 202 #define SVS_ENTER_NMI \ 203 HOTPATCH(HP_NAME_SVS_ENTER_NMI, SVS_ENTER_NMI_BYTES) ; \ 204 NOSVS_ENTER_NMI 205 206 #define SVS_LEAVE_NMI_BYTES 11 207 #define NOSVS_LEAVE_NMI \ 208 .byte 0xEB, (SVS_LEAVE_NMI_BYTES-2) /* jmp */ ; \ 209 .fill (SVS_LEAVE_NMI_BYTES-2),1,0xCC 210 #define SVS_LEAVE_NMI \ 211 HOTPATCH(HP_NAME_SVS_LEAVE_NMI, SVS_LEAVE_NMI_BYTES) ; \ 212 NOSVS_LEAVE_NMI 213 214 #else 215 #define SVS_ENTER /* nothing */ 216 #define SVS_ENTER_NMI /* nothing */ 217 #define SVS_LEAVE /* nothing */ 218 #define SVS_LEAVE_NMI /* nothing */ 219 #define SVS_ENTER_ALTSTACK /* nothing */ 220 #define SVS_LEAVE_ALTSTACK /* nothing */ 221 #endif 222 223 #ifdef KMSAN 224 /* XXX this belongs somewhere else. */ 225 #define KMSAN_ENTER \ 226 movq %rsp,%rdi ; \ 227 movq $TF_REGSIZE+16+40,%rsi ; \ 228 xorq %rdx,%rdx ; \ 229 callq kmsan_mark ; \ 230 callq kmsan_intr_enter 231 #define KMSAN_LEAVE \ 232 pushq %rbp ; \ 233 movq %rsp,%rbp ; \ 234 callq kmsan_intr_leave ; \ 235 popq %rbp 236 #define KMSAN_INIT_ARG(sz) \ 237 pushq %rax ; \ 238 pushq %rcx ; \ 239 pushq %rdx ; \ 240 pushq %rsi ; \ 241 pushq %rdi ; \ 242 pushq %r8 ; \ 243 pushq %r9 ; \ 244 pushq %r10 ; \ 245 pushq %r11 ; \ 246 movq $sz,%rdi ; \ 247 callq _C_LABEL(kmsan_init_arg); \ 248 popq %r11 ; \ 249 popq %r10 ; \ 250 popq %r9 ; \ 251 popq %r8 ; \ 252 popq %rdi ; \ 253 popq %rsi ; \ 254 popq %rdx ; \ 255 popq %rcx ; \ 256 popq %rax 257 #define KMSAN_INIT_RET(sz) \ 258 pushq %rax ; \ 259 pushq %rcx ; \ 260 pushq %rdx ; \ 261 pushq %rsi ; \ 262 pushq %rdi ; \ 263 pushq %r8 ; \ 264 pushq %r9 ; \ 265 pushq %r10 ; \ 266 pushq %r11 ; \ 267 movq $sz,%rdi ; \ 268 callq _C_LABEL(kmsan_init_ret); \ 269 popq %r11 ; \ 270 popq %r10 ; \ 271 popq %r9 ; \ 272 popq %r8 ; \ 273 popq %rdi ; \ 274 popq %rsi ; \ 275 popq %rdx ; \ 276 popq %rcx ; \ 277 popq %rax 278 #else 279 #define KMSAN_ENTER /* nothing */ 280 #define KMSAN_LEAVE /* nothing */ 281 #define KMSAN_INIT_ARG(sz) /* nothing */ 282 #define KMSAN_INIT_RET(sz) /* nothing */ 283 #endif 284 285 #ifdef KCOV 286 #define KCOV_DISABLE \ 287 incl CPUVAR(IDEPTH) 288 #define KCOV_ENABLE \ 289 decl CPUVAR(IDEPTH) 290 #else 291 #define KCOV_DISABLE /* nothing */ 292 #define KCOV_ENABLE /* nothing */ 293 #endif 294 295 #define INTRENTRY \ 296 subq $TF_REGSIZE,%rsp ; \ 297 INTR_SAVE_GPRS ; \ 298 cld ; \ 299 SMAP_ENABLE ; \ 300 testb $SEL_UPL,TF_CS(%rsp) ; \ 301 je 98f ; \ 302 SWAPGS ; \ 303 IBRS_ENTER ; \ 304 SVS_ENTER ; \ 305 movw %gs,TF_GS(%rsp) ; \ 306 movw %fs,TF_FS(%rsp) ; \ 307 movw %es,TF_ES(%rsp) ; \ 308 movw %ds,TF_DS(%rsp) ; \ 309 98: KMSAN_ENTER 310 311 #define INTRFASTEXIT \ 312 jmp intrfastexit 313 314 #define INTR_RECURSE_HWFRAME \ 315 movq %rsp,%r10 ; \ 316 movl %ss,%r11d ; \ 317 pushq %r11 ; \ 318 pushq %r10 ; \ 319 pushfq ; \ 320 pushq $GSEL(GCODE_SEL,SEL_KPL); \ 321 /* XEN: We must fixup CS, as even kernel mode runs at CPL 3 */ \ 322 XEN_ONLY2(andb $0xfc,(%rsp);) \ 323 pushq %r13 ; 324 325 #define INTR_RECURSE_ENTRY \ 326 subq $TF_REGSIZE,%rsp ; \ 327 INTR_SAVE_GPRS ; \ 328 cld ; \ 329 KMSAN_ENTER 330 331 #define CHECK_DEFERRED_SWITCH \ 332 cmpl $0, CPUVAR(WANT_PMAPLOAD) 333 334 #define CHECK_ASTPENDING(reg) cmpl $0, L_MD_ASTPENDING(reg) 335 #define CLEAR_ASTPENDING(reg) movl $0, L_MD_ASTPENDING(reg) 336 337 /* 338 * If the FPU state is not in the CPU, restore it. Executed with interrupts 339 * disabled. 340 * 341 * %r14 is curlwp, must not be modified 342 * %rbx must not be modified 343 */ 344 #define HANDLE_DEFERRED_FPU \ 345 testl $MDL_FPU_IN_CPU,L_MD_FLAGS(%r14) ; \ 346 jnz 1f ; \ 347 call _C_LABEL(fpu_handle_deferred) ; \ 348 orl $MDL_FPU_IN_CPU,L_MD_FLAGS(%r14) ; \ 349 1: 350 351 #endif /* _AMD64_MACHINE_FRAMEASM_H */ 352