1 /* $OpenBSD: cpufunc.h,v 1.43 2024/11/08 12:08:22 bluhm Exp $ */ 2 /* $NetBSD: cpufunc.h,v 1.3 2003/05/08 10:27:43 fvdl Exp $ */ 3 4 /*- 5 * Copyright (c) 1998 The NetBSD Foundation, Inc. 6 * All rights reserved. 7 * 8 * This code is derived from software contributed to The NetBSD Foundation 9 * by Charles M. Hannum. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #ifndef _MACHINE_CPUFUNC_H_ 34 #define _MACHINE_CPUFUNC_H_ 35 36 /* 37 * Functions to provide access to i386-specific instructions. 38 */ 39 40 #include <sys/types.h> 41 42 #include <machine/specialreg.h> 43 44 #if defined(_KERNEL) && !defined (_STANDALONE) 45 46 static __inline void 47 invlpg(u_int64_t addr) 48 { 49 __asm volatile("invlpg (%0)" : : "r" (addr) : "memory"); 50 } 51 52 static __inline void 53 sidt(void *p) 54 { 55 __asm volatile("sidt (%0)" : : "r" (p) : "memory"); 56 } 57 58 static __inline void 59 lidt(void *p) 60 { 61 __asm volatile("lidt (%0)" : : "r" (p) : "memory"); 62 } 63 64 static __inline void 65 sgdt(void *p) 66 { 67 __asm volatile("sgdt (%0)" : : "r" (p) : "memory"); 68 } 69 70 static __inline void 71 bare_lgdt(struct region_descriptor *p) 72 { 73 __asm volatile("lgdt (%0)" : : "r" (p) : "memory"); 74 } 75 76 static __inline void 77 sldt(u_short *sel) 78 { 79 __asm volatile("sldt (%0)" : : "r" (sel) : "memory"); 80 } 81 82 static __inline void 83 lldt(u_short sel) 84 { 85 __asm volatile("lldt %0" : : "r" (sel)); 86 } 87 88 static __inline void 89 ltr(u_short sel) 90 { 91 __asm volatile("ltr %0" : : "r" (sel)); 92 } 93 94 static __inline void 95 lcr8(u_int val) 96 { 97 u_int64_t val64 = val; 98 __asm volatile("movq %0,%%cr8" : : "r" (val64)); 99 } 100 101 /* 102 * Upper 32 bits are reserved anyway, so just keep this 32bits. 103 */ 104 static __inline void 105 lcr0(u_int val) 106 { 107 u_int64_t val64 = val; 108 __asm volatile("movq %0,%%cr0" : : "r" (val64)); 109 } 110 111 static __inline u_int 112 rcr0(void) 113 { 114 u_int64_t val64; 115 u_int val; 116 __asm volatile("movq %%cr0,%0" : "=r" (val64)); 117 val = val64; 118 return val; 119 } 120 121 static __inline u_int64_t 122 rcr2(void) 123 { 124 u_int64_t val; 125 __asm volatile("movq %%cr2,%0" : "=r" (val)); 126 return val; 127 } 128 129 static __inline void 130 lcr3(u_int64_t val) 131 { 132 __asm volatile("movq %0,%%cr3" : : "r" (val)); 133 } 134 135 static __inline u_int64_t 136 rcr3(void) 137 { 138 u_int64_t val; 139 __asm volatile("movq %%cr3,%0" : "=r" (val)); 140 return val; 141 } 142 143 /* 144 * Same as for cr0. Don't touch upper 32 bits. 145 */ 146 static __inline void 147 lcr4(u_int val) 148 { 149 u_int64_t val64 = val; 150 151 __asm volatile("movq %0,%%cr4" : : "r" (val64)); 152 } 153 154 static __inline u_int 155 rcr4(void) 156 { 157 u_int64_t val64; 158 __asm volatile("movq %%cr4,%0" : "=r" (val64)); 159 return (u_int) val64; 160 } 161 162 static __inline void 163 tlbflush(void) 164 { 165 u_int64_t val; 166 __asm volatile("movq %%cr3,%0" : "=r" (val)); 167 __asm volatile("movq %0,%%cr3" : : "r" (val)); 168 } 169 170 static inline void 171 invpcid(uint64_t type, paddr_t pcid, paddr_t addr) 172 { 173 uint64_t desc[2] = { pcid, addr }; 174 asm volatile("invpcid %0,%1" : : "m"(desc[0]), "r"(type)); 175 } 176 #define INVPCID_ADDR 0 177 #define INVPCID_PCID 1 178 #define INVPCID_ALL 2 179 #define INVPCID_NON_GLOBAL 3 180 181 #ifdef notyet 182 void setidt(int idx, /*XXX*/caddr_t func, int typ, int dpl); 183 #endif 184 185 186 /* XXXX ought to be in psl.h with spl() functions */ 187 188 static __inline u_long 189 read_rflags(void) 190 { 191 u_long ef; 192 193 __asm volatile("pushfq; popq %0" : "=r" (ef)); 194 return (ef); 195 } 196 197 static __inline void 198 write_rflags(u_long ef) 199 { 200 __asm volatile("pushq %0; popfq" : : "r" (ef)); 201 } 202 203 static __inline void 204 intr_enable(void) 205 { 206 __asm volatile("sti"); 207 } 208 209 static __inline u_long 210 intr_disable(void) 211 { 212 u_long ef; 213 214 ef = read_rflags(); 215 __asm volatile("cli"); 216 return (ef); 217 } 218 219 static __inline void 220 intr_restore(u_long ef) 221 { 222 write_rflags(ef); 223 } 224 225 static __inline u_int64_t 226 rdmsr(u_int msr) 227 { 228 uint32_t hi, lo; 229 __asm volatile("rdmsr" : "=d" (hi), "=a" (lo) : "c" (msr)); 230 return (((uint64_t)hi << 32) | (uint64_t) lo); 231 } 232 233 static __inline int 234 rdpkru(u_int ecx) 235 { 236 uint32_t edx, pkru; 237 asm volatile("rdpkru " : "=a" (pkru), "=d" (edx) : "c" (ecx)); 238 return pkru; 239 } 240 241 static __inline void 242 wrpkru(u_int ecx, uint32_t pkru) 243 { 244 uint32_t edx = 0; 245 asm volatile("wrpkru" : : "a" (pkru), "c" (ecx), "d" (edx)); 246 } 247 248 static __inline void 249 wrmsr(u_int msr, u_int64_t newval) 250 { 251 __asm volatile("wrmsr" : 252 : "a" (newval & 0xffffffff), "d" (newval >> 32), "c" (msr)); 253 } 254 255 /* 256 * Some of the undocumented AMD64 MSRs need a 'passcode' to access. 257 * 258 * See LinuxBIOSv2: src/cpu/amd/model_fxx/model_fxx_init.c 259 */ 260 261 #define OPTERON_MSR_PASSCODE 0x9c5a203a 262 263 static __inline u_int64_t 264 rdmsr_locked(u_int msr, u_int code) 265 { 266 uint32_t hi, lo; 267 __asm volatile("rdmsr" 268 : "=d" (hi), "=a" (lo) 269 : "c" (msr), "D" (code)); 270 return (((uint64_t)hi << 32) | (uint64_t) lo); 271 } 272 273 static __inline void 274 wrmsr_locked(u_int msr, u_int code, u_int64_t newval) 275 { 276 __asm volatile("wrmsr" : 277 : "a" (newval & 0xffffffff), "d" (newval >> 32), "c" (msr), "D" (code)); 278 } 279 280 static __inline void 281 wbinvd(void) 282 { 283 __asm volatile("wbinvd" : : : "memory"); 284 } 285 286 #ifdef MULTIPROCESSOR 287 int wbinvd_on_all_cpus(void); 288 void wbinvd_on_all_cpus_acked(void); 289 #else 290 static inline int 291 wbinvd_on_all_cpus(void) 292 { 293 wbinvd(); 294 return 0; 295 } 296 297 static inline int 298 wbinvd_on_all_cpus_acked(void) 299 { 300 wbinvd(); 301 return 0; 302 } 303 #endif /* MULTIPROCESSOR */ 304 305 static __inline void 306 clflush(u_int64_t addr) 307 { 308 __asm volatile("clflush %0" : "+m" (*(volatile char *)addr)); 309 } 310 311 static __inline void 312 mfence(void) 313 { 314 __asm volatile("mfence" : : : "memory"); 315 } 316 317 static __inline u_int64_t 318 rdtsc(void) 319 { 320 uint32_t hi, lo; 321 322 __asm volatile("rdtsc" : "=d" (hi), "=a" (lo)); 323 return (((uint64_t)hi << 32) | (uint64_t) lo); 324 } 325 326 static __inline u_int64_t 327 rdtscp(void) 328 { 329 uint32_t hi, lo; 330 331 __asm volatile("rdtscp" : "=d" (hi), "=a" (lo) : : "ecx"); 332 return (((uint64_t)hi << 32) | (uint64_t) lo); 333 } 334 335 static __inline u_int64_t 336 rdtsc_lfence(void) 337 { 338 uint32_t hi, lo; 339 340 __asm volatile("lfence; rdtsc" : "=d" (hi), "=a" (lo)); 341 return (((uint64_t)hi << 32) | (uint64_t) lo); 342 } 343 344 static __inline u_int64_t 345 rdpmc(u_int pmc) 346 { 347 uint32_t hi, lo; 348 349 __asm volatile("rdpmc" : "=d" (hi), "=a" (lo) : "c" (pmc)); 350 return (((uint64_t)hi << 32) | (uint64_t) lo); 351 } 352 353 static __inline void 354 monitor(const volatile void *addr, u_long extensions, u_int hints) 355 { 356 357 __asm volatile("monitor" 358 : : "a" (addr), "c" (extensions), "d" (hints)); 359 } 360 361 static __inline void 362 mwait(u_long extensions, u_int hints) 363 { 364 365 __asm volatile( 366 " mwait ;" 367 " mov $8,%%rcx ;" 368 " .align 16,0x90 ;" 369 "3: call 5f ;" 370 "4: pause ;" 371 " lfence ;" 372 " call 4b ;" 373 " .align 16,0xcc ;" 374 "5: call 7f ;" 375 "6: pause ;" 376 " lfence ;" 377 " call 6b ;" 378 " .align 16,0xcc ;" 379 "7: loop 3b ;" 380 " add $(16*8),%%rsp" 381 : "+c" (extensions) : "a" (hints)); 382 } 383 384 static __inline void 385 xsetbv(uint32_t reg, uint64_t mask) 386 { 387 uint32_t lo, hi; 388 389 lo = mask; 390 hi = mask >> 32; 391 __asm volatile("xsetbv" :: "c" (reg), "a" (lo), "d" (hi) : "memory"); 392 } 393 394 static __inline uint64_t 395 xgetbv(uint32_t reg) 396 { 397 uint32_t lo, hi; 398 399 __asm volatile("xgetbv" : "=a" (lo), "=d" (hi) : "c" (reg)); 400 401 return (((uint64_t)hi << 32) | (uint64_t)lo); 402 } 403 404 static __inline void 405 stgi(void) 406 { 407 __asm volatile("stgi"); 408 } 409 410 static __inline void 411 clgi(void) 412 { 413 __asm volatile("clgi"); 414 } 415 416 /* Break into DDB. */ 417 static __inline void 418 breakpoint(void) 419 { 420 __asm volatile("int $3"); 421 } 422 423 void amd64_errata(struct cpu_info *); 424 void cpu_ucode_setup(void); 425 void cpu_ucode_apply(struct cpu_info *); 426 427 struct cpu_info_full; 428 void cpu_enter_pages(struct cpu_info_full *); 429 430 int rdmsr_safe(u_int msr, uint64_t *); 431 432 #endif /* _KERNEL */ 433 434 #endif /* !_MACHINE_CPUFUNC_H_ */ 435