1 /* $OpenBSD: cpu.c,v 1.135 2025/01/25 12:29:35 kettenis Exp $ */ 2 3 /* 4 * Copyright (c) 2016 Dale Rahn <drahn@dalerahn.com> 5 * Copyright (c) 2017 Mark Kettenis <kettenis@openbsd.org> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include "kstat.h" 21 22 #include <sys/param.h> 23 #include <sys/systm.h> 24 #include <sys/proc.h> 25 #include <sys/malloc.h> 26 #include <sys/device.h> 27 #include <sys/sysctl.h> 28 #include <sys/task.h> 29 #include <sys/user.h> 30 #include <sys/kstat.h> 31 32 #include <uvm/uvm_extern.h> 33 34 #include <machine/fdt.h> 35 #include <machine/elf.h> 36 37 #include <dev/ofw/openfirm.h> 38 #include <dev/ofw/ofw_clock.h> 39 #include <dev/ofw/ofw_regulator.h> 40 #include <dev/ofw/ofw_thermal.h> 41 #include <dev/ofw/fdt.h> 42 43 #include <machine/cpufunc.h> 44 45 #include "psci.h" 46 #if NPSCI > 0 47 #include <dev/fdt/pscivar.h> 48 #endif 49 50 /* CPU Identification */ 51 #define CPU_IMPL_ARM 0x41 52 #define CPU_IMPL_CAVIUM 0x43 53 #define CPU_IMPL_AMCC 0x50 54 #define CPU_IMPL_QCOM 0x51 55 #define CPU_IMPL_APPLE 0x61 56 #define CPU_IMPL_AMPERE 0xc0 57 58 /* ARM */ 59 #define CPU_PART_CORTEX_A34 0xd02 60 #define CPU_PART_CORTEX_A53 0xd03 61 #define CPU_PART_CORTEX_A35 0xd04 62 #define CPU_PART_CORTEX_A55 0xd05 63 #define CPU_PART_CORTEX_A65 0xd06 64 #define CPU_PART_CORTEX_A57 0xd07 65 #define CPU_PART_CORTEX_A72 0xd08 66 #define CPU_PART_CORTEX_A73 0xd09 67 #define CPU_PART_CORTEX_A75 0xd0a 68 #define CPU_PART_CORTEX_A76 0xd0b 69 #define CPU_PART_NEOVERSE_N1 0xd0c 70 #define CPU_PART_CORTEX_A77 0xd0d 71 #define CPU_PART_CORTEX_A76AE 0xd0e 72 #define CPU_PART_NEOVERSE_V1 0xd40 73 #define CPU_PART_CORTEX_A78 0xd41 74 #define CPU_PART_CORTEX_A78AE 0xd42 75 #define CPU_PART_CORTEX_A65AE 0xd43 76 #define CPU_PART_CORTEX_X1 0xd44 77 #define CPU_PART_CORTEX_A510 0xd46 78 #define CPU_PART_CORTEX_A710 0xd47 79 #define CPU_PART_CORTEX_X2 0xd48 80 #define CPU_PART_NEOVERSE_N2 0xd49 81 #define CPU_PART_NEOVERSE_E1 0xd4a 82 #define CPU_PART_CORTEX_A78C 0xd4b 83 #define CPU_PART_CORTEX_X1C 0xd4c 84 #define CPU_PART_CORTEX_A715 0xd4d 85 #define CPU_PART_CORTEX_X3 0xd4e 86 #define CPU_PART_NEOVERSE_V2 0xd4f 87 #define CPU_PART_CORTEX_A520 0xd80 88 #define CPU_PART_CORTEX_A720 0xd81 89 #define CPU_PART_CORTEX_X4 0xd82 90 #define CPU_PART_NEOVERSE_V3AE 0xd83 91 #define CPU_PART_NEOVERSE_V3 0xd84 92 #define CPU_PART_CORTEX_X925 0xd85 93 #define CPU_PART_CORTEX_A725 0xd87 94 #define CPU_PART_CORTEX_A520AE 0xd88 95 #define CPU_PART_CORTEX_A720AE 0xd89 96 #define CPU_PART_NEOVERSE_N3 0xd8e 97 98 /* Cavium */ 99 #define CPU_PART_THUNDERX_T88 0x0a1 100 #define CPU_PART_THUNDERX_T81 0x0a2 101 #define CPU_PART_THUNDERX_T83 0x0a3 102 #define CPU_PART_THUNDERX2_T99 0x0af 103 104 /* Applied Micro */ 105 #define CPU_PART_X_GENE 0x000 106 107 /* Qualcomm */ 108 #define CPU_PART_ORYON 0x001 109 #define CPU_PART_KRYO400_GOLD 0x804 110 #define CPU_PART_KRYO400_SILVER 0x805 111 112 /* Apple */ 113 #define CPU_PART_ICESTORM 0x022 114 #define CPU_PART_FIRESTORM 0x023 115 #define CPU_PART_ICESTORM_PRO 0x024 116 #define CPU_PART_FIRESTORM_PRO 0x025 117 #define CPU_PART_ICESTORM_MAX 0x028 118 #define CPU_PART_FIRESTORM_MAX 0x029 119 #define CPU_PART_BLIZZARD 0x032 120 #define CPU_PART_AVALANCHE 0x033 121 #define CPU_PART_BLIZZARD_PRO 0x034 122 #define CPU_PART_AVALANCHE_PRO 0x035 123 #define CPU_PART_BLIZZARD_MAX 0x038 124 #define CPU_PART_AVALANCHE_MAX 0x039 125 126 /* Ampere */ 127 #define CPU_PART_AMPERE1 0xac3 128 129 #define CPU_IMPL(midr) (((midr) >> 24) & 0xff) 130 #define CPU_PART(midr) (((midr) >> 4) & 0xfff) 131 #define CPU_VAR(midr) (((midr) >> 20) & 0xf) 132 #define CPU_REV(midr) (((midr) >> 0) & 0xf) 133 134 struct cpu_cores { 135 int id; 136 char *name; 137 }; 138 139 struct cpu_cores cpu_cores_none[] = { 140 { 0, NULL }, 141 }; 142 143 struct cpu_cores cpu_cores_arm[] = { 144 { CPU_PART_CORTEX_A34, "Cortex-A34" }, 145 { CPU_PART_CORTEX_A35, "Cortex-A35" }, 146 { CPU_PART_CORTEX_A53, "Cortex-A53" }, 147 { CPU_PART_CORTEX_A55, "Cortex-A55" }, 148 { CPU_PART_CORTEX_A57, "Cortex-A57" }, 149 { CPU_PART_CORTEX_A65, "Cortex-A65" }, 150 { CPU_PART_CORTEX_A65AE, "Cortex-A65AE" }, 151 { CPU_PART_CORTEX_A72, "Cortex-A72" }, 152 { CPU_PART_CORTEX_A73, "Cortex-A73" }, 153 { CPU_PART_CORTEX_A75, "Cortex-A75" }, 154 { CPU_PART_CORTEX_A76, "Cortex-A76" }, 155 { CPU_PART_CORTEX_A76AE, "Cortex-A76AE" }, 156 { CPU_PART_CORTEX_A77, "Cortex-A77" }, 157 { CPU_PART_CORTEX_A78, "Cortex-A78" }, 158 { CPU_PART_CORTEX_A78AE, "Cortex-A78AE" }, 159 { CPU_PART_CORTEX_A78C, "Cortex-A78C" }, 160 { CPU_PART_CORTEX_A510, "Cortex-A510" }, 161 { CPU_PART_CORTEX_A520, "Cortex-A520" }, 162 { CPU_PART_CORTEX_A520AE, "Cortex-A520AE" }, 163 { CPU_PART_CORTEX_A710, "Cortex-A710" }, 164 { CPU_PART_CORTEX_A715, "Cortex-A715" }, 165 { CPU_PART_CORTEX_A720, "Cortex-A720" }, 166 { CPU_PART_CORTEX_A720AE, "Cortex-A720AE" }, 167 { CPU_PART_CORTEX_A725, "Cortex-A725" }, 168 { CPU_PART_CORTEX_X1, "Cortex-X1" }, 169 { CPU_PART_CORTEX_X1C, "Cortex-X1C" }, 170 { CPU_PART_CORTEX_X2, "Cortex-X2" }, 171 { CPU_PART_CORTEX_X3, "Cortex-X3" }, 172 { CPU_PART_CORTEX_X4, "Cortex-X4" }, 173 { CPU_PART_CORTEX_X925, "Cortex-X925" }, 174 { CPU_PART_NEOVERSE_E1, "Neoverse E1" }, 175 { CPU_PART_NEOVERSE_N1, "Neoverse N1" }, 176 { CPU_PART_NEOVERSE_N2, "Neoverse N2" }, 177 { CPU_PART_NEOVERSE_N3, "Neoverse N3" }, 178 { CPU_PART_NEOVERSE_V1, "Neoverse V1" }, 179 { CPU_PART_NEOVERSE_V2, "Neoverse V2" }, 180 { CPU_PART_NEOVERSE_V3, "Neoverse V3" }, 181 { CPU_PART_NEOVERSE_V3AE, "Neoverse V3AE" }, 182 { 0, NULL }, 183 }; 184 185 struct cpu_cores cpu_cores_cavium[] = { 186 { CPU_PART_THUNDERX_T88, "ThunderX T88" }, 187 { CPU_PART_THUNDERX_T81, "ThunderX T81" }, 188 { CPU_PART_THUNDERX_T83, "ThunderX T83" }, 189 { CPU_PART_THUNDERX2_T99, "ThunderX2 T99" }, 190 { 0, NULL }, 191 }; 192 193 struct cpu_cores cpu_cores_amcc[] = { 194 { CPU_PART_X_GENE, "X-Gene" }, 195 { 0, NULL }, 196 }; 197 198 struct cpu_cores cpu_cores_qcom[] = { 199 { CPU_PART_KRYO400_GOLD, "Kryo 400 Gold" }, 200 { CPU_PART_KRYO400_SILVER, "Kryo 400 Silver" }, 201 { CPU_PART_ORYON, "Oryon" }, 202 { 0, NULL }, 203 }; 204 205 struct cpu_cores cpu_cores_apple[] = { 206 { CPU_PART_ICESTORM, "Icestorm" }, 207 { CPU_PART_FIRESTORM, "Firestorm" }, 208 { CPU_PART_ICESTORM_PRO, "Icestorm Pro" }, 209 { CPU_PART_FIRESTORM_PRO, "Firestorm Pro" }, 210 { CPU_PART_ICESTORM_MAX, "Icestorm Max" }, 211 { CPU_PART_FIRESTORM_MAX, "Firestorm Max" }, 212 { CPU_PART_BLIZZARD, "Blizzard" }, 213 { CPU_PART_AVALANCHE, "Avalanche" }, 214 { CPU_PART_BLIZZARD_PRO, "Blizzard Pro" }, 215 { CPU_PART_AVALANCHE_PRO, "Avalanche Pro" }, 216 { CPU_PART_BLIZZARD_MAX, "Blizzard Max" }, 217 { CPU_PART_AVALANCHE_MAX, "Avalanche Max" }, 218 { 0, NULL }, 219 }; 220 221 struct cpu_cores cpu_cores_ampere[] = { 222 { CPU_PART_AMPERE1, "AmpereOne" }, 223 { 0, NULL }, 224 }; 225 226 /* arm cores makers */ 227 const struct implementers { 228 int id; 229 char *name; 230 struct cpu_cores *corelist; 231 } cpu_implementers[] = { 232 { CPU_IMPL_ARM, "ARM", cpu_cores_arm }, 233 { CPU_IMPL_CAVIUM, "Cavium", cpu_cores_cavium }, 234 { CPU_IMPL_AMCC, "Applied Micro", cpu_cores_amcc }, 235 { CPU_IMPL_QCOM, "Qualcomm", cpu_cores_qcom }, 236 { CPU_IMPL_APPLE, "Apple", cpu_cores_apple }, 237 { CPU_IMPL_AMPERE, "Ampere", cpu_cores_ampere }, 238 { 0, NULL }, 239 }; 240 241 char cpu_model[64]; 242 int cpu_node; 243 244 uint64_t cpu_id_aa64isar0; 245 uint64_t cpu_id_aa64isar1; 246 uint64_t cpu_id_aa64isar2; 247 uint64_t cpu_id_aa64mmfr0; 248 uint64_t cpu_id_aa64mmfr1; 249 uint64_t cpu_id_aa64mmfr2; 250 uint64_t cpu_id_aa64pfr0; 251 uint64_t cpu_id_aa64pfr1; 252 253 int arm64_has_lse; 254 int arm64_has_rng; 255 #ifdef CRYPTO 256 int arm64_has_aes; 257 #endif 258 259 extern char trampoline_vectors_none[]; 260 extern char trampoline_vectors_loop_8[]; 261 extern char trampoline_vectors_loop_11[]; 262 extern char trampoline_vectors_loop_24[]; 263 extern char trampoline_vectors_loop_32[]; 264 #if NPSCI > 0 265 extern char trampoline_vectors_psci_hvc[]; 266 extern char trampoline_vectors_psci_smc[]; 267 #endif 268 extern char trampoline_vectors_clrbhb[]; 269 270 struct cpu_info *cpu_info_list = &cpu_info_primary; 271 272 int cpu_match(struct device *, void *, void *); 273 void cpu_attach(struct device *, struct device *, void *); 274 275 const struct cfattach cpu_ca = { 276 sizeof(struct device), cpu_match, cpu_attach 277 }; 278 279 struct cfdriver cpu_cd = { 280 NULL, "cpu", DV_DULL 281 }; 282 283 struct timeout cpu_rng_to; 284 void cpu_rng(void *); 285 286 void cpu_opp_init(struct cpu_info *, uint32_t); 287 void cpu_psci_init(struct cpu_info *); 288 void cpu_psci_idle_cycle(void); 289 290 void cpu_flush_bp_noop(void); 291 void cpu_flush_bp_psci(void); 292 void cpu_serror_apple(void); 293 294 #if NKSTAT > 0 295 void cpu_kstat_attach(struct cpu_info *ci); 296 void cpu_opp_kstat_attach(struct cpu_info *ci); 297 #endif 298 299 void 300 cpu_rng(void *arg) 301 { 302 struct timeout *to = arg; 303 uint64_t rndr; 304 int ret; 305 306 ret = __builtin_arm_rndrrs(&rndr); 307 if (ret) 308 ret = __builtin_arm_rndr(&rndr); 309 if (ret == 0) { 310 enqueue_randomness(rndr & 0xffffffff); 311 enqueue_randomness(rndr >> 32); 312 } 313 314 if (to) 315 timeout_add_msec(to, 1000); 316 } 317 318 /* 319 * Enable mitigation for Spectre-V2 branch target injection 320 * vulnerabilities (CVE-2017-5715). 321 */ 322 void 323 cpu_mitigate_spectre_v2(struct cpu_info *ci) 324 { 325 uint64_t id; 326 327 /* 328 * By default we let the firmware decide what mitigation is 329 * necessary. 330 */ 331 ci->ci_flush_bp = cpu_flush_bp_psci; 332 333 /* Some specific CPUs are known not to be vulnerable. */ 334 switch (CPU_IMPL(ci->ci_midr)) { 335 case CPU_IMPL_ARM: 336 switch (CPU_PART(ci->ci_midr)) { 337 case CPU_PART_CORTEX_A35: 338 case CPU_PART_CORTEX_A53: 339 case CPU_PART_CORTEX_A55: 340 /* Not vulnerable. */ 341 ci->ci_flush_bp = cpu_flush_bp_noop; 342 break; 343 } 344 break; 345 case CPU_IMPL_QCOM: 346 switch (CPU_PART(ci->ci_midr)) { 347 case CPU_PART_KRYO400_SILVER: 348 /* Not vulnerable. */ 349 ci->ci_flush_bp = cpu_flush_bp_noop; 350 break; 351 } 352 } 353 354 /* 355 * The architecture has been updated to explicitly tell us if 356 * we're not vulnerable to Spectre-V2. 357 */ 358 id = READ_SPECIALREG(id_aa64pfr0_el1); 359 if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_IMPL) 360 ci->ci_flush_bp = cpu_flush_bp_noop; 361 } 362 363 /* 364 * Enable mitigation for Spectre-BHB branch history injection 365 * vulnerabilities (CVE-2022-23960). 366 */ 367 void 368 cpu_mitigate_spectre_bhb(struct cpu_info *ci) 369 { 370 uint64_t id; 371 372 /* 373 * If we know the CPU, we can add a branchy loop that cleans 374 * the BHB. 375 */ 376 switch (CPU_IMPL(ci->ci_midr)) { 377 case CPU_IMPL_ARM: 378 switch (CPU_PART(ci->ci_midr)) { 379 case CPU_PART_CORTEX_A57: 380 case CPU_PART_CORTEX_A72: 381 ci->ci_trampoline_vectors = 382 (vaddr_t)trampoline_vectors_loop_8; 383 break; 384 case CPU_PART_CORTEX_A76: 385 case CPU_PART_CORTEX_A76AE: 386 case CPU_PART_CORTEX_A77: 387 case CPU_PART_NEOVERSE_N1: 388 ci->ci_trampoline_vectors = 389 (vaddr_t)trampoline_vectors_loop_24; 390 break; 391 case CPU_PART_CORTEX_A78: 392 case CPU_PART_CORTEX_A78AE: 393 case CPU_PART_CORTEX_A78C: 394 case CPU_PART_CORTEX_X1: 395 case CPU_PART_CORTEX_X2: 396 case CPU_PART_CORTEX_A710: 397 case CPU_PART_NEOVERSE_N2: 398 case CPU_PART_NEOVERSE_V1: 399 ci->ci_trampoline_vectors = 400 (vaddr_t)trampoline_vectors_loop_32; 401 break; 402 } 403 break; 404 case CPU_IMPL_AMPERE: 405 switch (CPU_PART(ci->ci_midr)) { 406 case CPU_PART_AMPERE1: 407 ci->ci_trampoline_vectors = 408 (vaddr_t)trampoline_vectors_loop_11; 409 break; 410 } 411 break; 412 } 413 414 /* 415 * If we're not using a loop, let firmware decide. This also 416 * covers the original Spectre-V2 in addition to Spectre-BHB. 417 */ 418 #if NPSCI > 0 419 if (ci->ci_trampoline_vectors == (vaddr_t)trampoline_vectors_none && 420 smccc_needs_arch_workaround_3()) { 421 ci->ci_flush_bp = cpu_flush_bp_noop; 422 if (psci_method() == PSCI_METHOD_HVC) 423 ci->ci_trampoline_vectors = 424 (vaddr_t)trampoline_vectors_psci_hvc; 425 if (psci_method() == PSCI_METHOD_SMC) 426 ci->ci_trampoline_vectors = 427 (vaddr_t)trampoline_vectors_psci_smc; 428 } 429 #endif 430 431 /* Prefer CLRBHB to mitigate Spectre-BHB. */ 432 id = READ_SPECIALREG(id_aa64isar2_el1); 433 if (ID_AA64ISAR2_CLRBHB(id) >= ID_AA64ISAR2_CLRBHB_IMPL) 434 ci->ci_trampoline_vectors = (vaddr_t)trampoline_vectors_clrbhb; 435 436 /* ECBHB tells us Spectre-BHB is mitigated. */ 437 id = READ_SPECIALREG(id_aa64mmfr1_el1); 438 if (ID_AA64MMFR1_ECBHB(id) >= ID_AA64MMFR1_ECBHB_IMPL) 439 ci->ci_trampoline_vectors = (vaddr_t)trampoline_vectors_none; 440 441 /* 442 * The architecture has been updated to explicitly tell us if 443 * we're not vulnerable to Spectre-BHB. 444 */ 445 id = READ_SPECIALREG(id_aa64pfr0_el1); 446 if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_HCXT) 447 ci->ci_trampoline_vectors = (vaddr_t)trampoline_vectors_none; 448 } 449 450 /* 451 * Enable mitigation for Spectre-V4 speculative store bypass 452 * vulnerabilities (CVE-2018-3639). 453 */ 454 void 455 cpu_mitigate_spectre_v4(struct cpu_info *ci) 456 { 457 uint64_t id; 458 459 switch (CPU_IMPL(ci->ci_midr)) { 460 case CPU_IMPL_ARM: 461 switch (CPU_PART(ci->ci_midr)) { 462 case CPU_PART_CORTEX_A35: 463 case CPU_PART_CORTEX_A53: 464 case CPU_PART_CORTEX_A55: 465 /* Not vulnerable. */ 466 return; 467 } 468 break; 469 case CPU_IMPL_QCOM: 470 switch (CPU_PART(ci->ci_midr)) { 471 case CPU_PART_KRYO400_SILVER: 472 /* Not vulnerable. */ 473 return; 474 } 475 break; 476 } 477 478 /* SSBS tells us Spectre-V4 is mitigated. */ 479 id = READ_SPECIALREG(id_aa64pfr1_el1); 480 if (ID_AA64PFR1_SSBS(id) >= ID_AA64PFR1_SSBS_PSTATE) 481 return; 482 483 /* Enable firmware workaround if required. */ 484 smccc_enable_arch_workaround_2(); 485 } 486 487 void 488 cpu_identify(struct cpu_info *ci) 489 { 490 static uint64_t prev_id_aa64isar0; 491 static uint64_t prev_id_aa64isar1; 492 static uint64_t prev_id_aa64isar2; 493 static uint64_t prev_id_aa64mmfr0; 494 static uint64_t prev_id_aa64mmfr1; 495 static uint64_t prev_id_aa64mmfr2; 496 static uint64_t prev_id_aa64pfr0; 497 static uint64_t prev_id_aa64pfr1; 498 uint64_t midr, impl, part; 499 uint64_t clidr, ccsidr, id; 500 uint32_t ctr, sets, ways, line; 501 const char *impl_name = NULL; 502 const char *part_name = NULL; 503 const char *il1p_name = NULL; 504 const char *sep; 505 struct cpu_cores *coreselecter = cpu_cores_none; 506 int ccidx; 507 int i; 508 509 midr = READ_SPECIALREG(midr_el1); 510 impl = CPU_IMPL(midr); 511 part = CPU_PART(midr); 512 ci->ci_midr = midr; 513 514 for (i = 0; cpu_implementers[i].name; i++) { 515 if (impl == cpu_implementers[i].id) { 516 impl_name = cpu_implementers[i].name; 517 coreselecter = cpu_implementers[i].corelist; 518 break; 519 } 520 } 521 522 for (i = 0; coreselecter[i].name; i++) { 523 if (part == coreselecter[i].id) { 524 part_name = coreselecter[i].name; 525 break; 526 } 527 } 528 529 if (impl_name && part_name) { 530 printf(" %s %s r%llup%llu", impl_name, part_name, CPU_VAR(midr), 531 CPU_REV(midr)); 532 533 if (CPU_IS_PRIMARY(ci)) 534 snprintf(cpu_model, sizeof(cpu_model), 535 "%s %s r%llup%llu", impl_name, part_name, 536 CPU_VAR(midr), CPU_REV(midr)); 537 } else { 538 printf(" Unknown, MIDR 0x%llx", midr); 539 540 if (CPU_IS_PRIMARY(ci)) 541 snprintf(cpu_model, sizeof(cpu_model), "Unknown"); 542 } 543 544 /* Print cache information. */ 545 546 ctr = READ_SPECIALREG(ctr_el0); 547 switch (ctr & CTR_IL1P_MASK) { 548 case CTR_IL1P_AIVIVT: 549 il1p_name = "AIVIVT "; 550 break; 551 case CTR_IL1P_VIPT: 552 il1p_name = "VIPT "; 553 break; 554 case CTR_IL1P_PIPT: 555 il1p_name = "PIPT "; 556 break; 557 } 558 559 id = READ_SPECIALREG(id_aa64mmfr2_el1); 560 clidr = READ_SPECIALREG(clidr_el1); 561 if (ID_AA64MMFR2_CCIDX(id) > ID_AA64MMFR2_CCIDX_IMPL) { 562 /* Reserved value. Don't print cache information. */ 563 clidr = 0; 564 } else if (ID_AA64MMFR2_CCIDX(id) == ID_AA64MMFR2_CCIDX_IMPL) { 565 /* CCSIDR_EL1 uses the new 64-bit format. */ 566 ccidx = 1; 567 } else { 568 /* CCSIDR_EL1 uses the old 32-bit format. */ 569 ccidx = 0; 570 } 571 for (i = 0; i < 7; i++) { 572 if ((clidr & CLIDR_CTYPE_MASK) == 0) 573 break; 574 printf("\n%s:", ci->ci_dev->dv_xname); 575 sep = ""; 576 if (clidr & CLIDR_CTYPE_INSN) { 577 WRITE_SPECIALREG(csselr_el1, 578 i << CSSELR_LEVEL_SHIFT | CSSELR_IND); 579 __asm volatile("isb"); 580 ccsidr = READ_SPECIALREG(ccsidr_el1); 581 if (ccidx) { 582 sets = CCSIDR_CCIDX_SETS(ccsidr); 583 ways = CCSIDR_CCIDX_WAYS(ccsidr); 584 line = CCSIDR_CCIDX_LINE_SIZE(ccsidr); 585 } else { 586 sets = CCSIDR_SETS(ccsidr); 587 ways = CCSIDR_WAYS(ccsidr); 588 line = CCSIDR_LINE_SIZE(ccsidr); 589 } 590 printf("%s %dKB %db/line %d-way L%d %sI-cache", sep, 591 (sets * ways * line) / 1024, line, ways, (i + 1), 592 il1p_name); 593 il1p_name = ""; 594 sep = ","; 595 } 596 if (clidr & CLIDR_CTYPE_DATA) { 597 WRITE_SPECIALREG(csselr_el1, i << CSSELR_LEVEL_SHIFT); 598 __asm volatile("isb"); 599 ccsidr = READ_SPECIALREG(ccsidr_el1); 600 if (ccidx) { 601 sets = CCSIDR_CCIDX_SETS(ccsidr); 602 ways = CCSIDR_CCIDX_WAYS(ccsidr); 603 line = CCSIDR_CCIDX_LINE_SIZE(ccsidr); 604 } else { 605 sets = CCSIDR_SETS(ccsidr); 606 ways = CCSIDR_WAYS(ccsidr); 607 line = CCSIDR_LINE_SIZE(ccsidr); 608 } 609 printf("%s %dKB %db/line %d-way L%d D-cache", sep, 610 (sets * ways * line) / 1024, line, ways, (i + 1)); 611 sep = ","; 612 } 613 if (clidr & CLIDR_CTYPE_UNIFIED) { 614 WRITE_SPECIALREG(csselr_el1, i << CSSELR_LEVEL_SHIFT); 615 __asm volatile("isb"); 616 ccsidr = READ_SPECIALREG(ccsidr_el1); 617 if (ccidx) { 618 sets = CCSIDR_CCIDX_SETS(ccsidr); 619 ways = CCSIDR_CCIDX_WAYS(ccsidr); 620 line = CCSIDR_CCIDX_LINE_SIZE(ccsidr); 621 } else { 622 sets = CCSIDR_SETS(ccsidr); 623 ways = CCSIDR_WAYS(ccsidr); 624 line = CCSIDR_LINE_SIZE(ccsidr); 625 } 626 printf("%s %dKB %db/line %d-way L%d cache", sep, 627 (sets * ways * line) / 1024, line, ways, (i + 1)); 628 } 629 clidr >>= 3; 630 } 631 632 cpu_mitigate_spectre_v2(ci); 633 cpu_mitigate_spectre_bhb(ci); 634 cpu_mitigate_spectre_v4(ci); 635 636 /* 637 * Apple CPUs provide detailed information for SError. 638 */ 639 if (impl == CPU_IMPL_APPLE) 640 ci->ci_serror = cpu_serror_apple; 641 642 /* 643 * Skip printing CPU features if they are identical to the 644 * previous CPU. 645 */ 646 if (READ_SPECIALREG(id_aa64isar0_el1) == prev_id_aa64isar0 && 647 READ_SPECIALREG(id_aa64isar1_el1) == prev_id_aa64isar1 && 648 READ_SPECIALREG(id_aa64isar2_el1) == prev_id_aa64isar2 && 649 READ_SPECIALREG(id_aa64mmfr0_el1) == prev_id_aa64mmfr0 && 650 READ_SPECIALREG(id_aa64mmfr1_el1) == prev_id_aa64mmfr1 && 651 READ_SPECIALREG(id_aa64mmfr2_el1) == prev_id_aa64mmfr2 && 652 READ_SPECIALREG(id_aa64pfr0_el1) == prev_id_aa64pfr0 && 653 READ_SPECIALREG(id_aa64pfr1_el1) == prev_id_aa64pfr1) 654 return; 655 656 /* 657 * Print CPU features encoded in the ID registers. 658 */ 659 660 if (READ_SPECIALREG(id_aa64isar0_el1) != cpu_id_aa64isar0) { 661 printf("\n%s: mismatched ID_AA64ISAR0_EL1", 662 ci->ci_dev->dv_xname); 663 } 664 if (READ_SPECIALREG(id_aa64isar1_el1) != cpu_id_aa64isar1) { 665 printf("\n%s: mismatched ID_AA64ISAR1_EL1", 666 ci->ci_dev->dv_xname); 667 } 668 if (READ_SPECIALREG(id_aa64isar2_el1) != cpu_id_aa64isar2) { 669 printf("\n%s: mismatched ID_AA64ISAR2_EL1", 670 ci->ci_dev->dv_xname); 671 } 672 if (READ_SPECIALREG(id_aa64mmfr0_el1) != cpu_id_aa64mmfr0) { 673 printf("\n%s: mismatched ID_AA64MMFR0_EL1", 674 ci->ci_dev->dv_xname); 675 } 676 if (READ_SPECIALREG(id_aa64mmfr1_el1) != cpu_id_aa64mmfr1) { 677 printf("\n%s: mismatched ID_AA64MMFR1_EL1", 678 ci->ci_dev->dv_xname); 679 } 680 if (READ_SPECIALREG(id_aa64mmfr2_el1) != cpu_id_aa64mmfr2) { 681 printf("\n%s: mismatched ID_AA64MMFR2_EL1", 682 ci->ci_dev->dv_xname); 683 } 684 id = READ_SPECIALREG(id_aa64pfr0_el1); 685 /* Allow CSV2/CVS3 to be different. */ 686 id &= ~ID_AA64PFR0_CSV2_MASK; 687 id &= ~ID_AA64PFR0_CSV3_MASK; 688 /* Ignore 32-bit support in all exception levels. */ 689 id &= ~ID_AA64PFR0_EL0_MASK; 690 id &= ~ID_AA64PFR0_EL1_MASK; 691 id &= ~ID_AA64PFR0_EL2_MASK; 692 id &= ~ID_AA64PFR0_EL3_MASK; 693 if (id != cpu_id_aa64pfr0) { 694 printf("\n%s: mismatched ID_AA64PFR0_EL1", 695 ci->ci_dev->dv_xname); 696 } 697 if (READ_SPECIALREG(id_aa64pfr1_el1) != cpu_id_aa64pfr1) { 698 printf("\n%s: mismatched ID_AA64PFR1_EL1", 699 ci->ci_dev->dv_xname); 700 } 701 702 printf("\n%s: ", ci->ci_dev->dv_xname); 703 704 /* 705 * ID_AA64ISAR0 706 */ 707 id = READ_SPECIALREG(id_aa64isar0_el1); 708 sep = ""; 709 710 if (ID_AA64ISAR0_RNDR(id) >= ID_AA64ISAR0_RNDR_IMPL) { 711 printf("%sRNDR", sep); 712 sep = ","; 713 arm64_has_rng = 1; 714 } 715 716 if (ID_AA64ISAR0_TLB(id) >= ID_AA64ISAR0_TLB_IOS) { 717 printf("%sTLBIOS", sep); 718 sep = ","; 719 } 720 if (ID_AA64ISAR0_TLB(id) >= ID_AA64ISAR0_TLB_IRANGE) 721 printf("+IRANGE"); 722 723 if (ID_AA64ISAR0_TS(id) >= ID_AA64ISAR0_TS_BASE) { 724 printf("%sTS", sep); 725 sep = ","; 726 } 727 if (ID_AA64ISAR0_TS(id) >= ID_AA64ISAR0_TS_AXFLAG) 728 printf("+AXFLAG"); 729 730 if (ID_AA64ISAR0_FHM(id) >= ID_AA64ISAR0_FHM_IMPL) { 731 printf("%sFHM", sep); 732 sep = ","; 733 } 734 735 if (ID_AA64ISAR0_DP(id) >= ID_AA64ISAR0_DP_IMPL) { 736 printf("%sDP", sep); 737 sep = ","; 738 } 739 740 if (ID_AA64ISAR0_SM4(id) >= ID_AA64ISAR0_SM4_IMPL) { 741 printf("%sSM4", sep); 742 sep = ","; 743 } 744 745 if (ID_AA64ISAR0_SM3(id) >= ID_AA64ISAR0_SM3_IMPL) { 746 printf("%sSM3", sep); 747 sep = ","; 748 } 749 750 if (ID_AA64ISAR0_SHA3(id) >= ID_AA64ISAR0_SHA3_IMPL) { 751 printf("%sSHA3", sep); 752 sep = ","; 753 } 754 755 if (ID_AA64ISAR0_RDM(id) >= ID_AA64ISAR0_RDM_IMPL) { 756 printf("%sRDM", sep); 757 sep = ","; 758 } 759 760 if (ID_AA64ISAR0_ATOMIC(id) >= ID_AA64ISAR0_ATOMIC_IMPL) { 761 printf("%sAtomic", sep); 762 sep = ","; 763 arm64_has_lse = 1; 764 } 765 766 if (ID_AA64ISAR0_CRC32(id) >= ID_AA64ISAR0_CRC32_BASE) { 767 printf("%sCRC32", sep); 768 sep = ","; 769 } 770 771 if (ID_AA64ISAR0_SHA2(id) >= ID_AA64ISAR0_SHA2_BASE) { 772 printf("%sSHA2", sep); 773 sep = ","; 774 } 775 if (ID_AA64ISAR0_SHA2(id) >= ID_AA64ISAR0_SHA2_512) 776 printf("+SHA512"); 777 778 if (ID_AA64ISAR0_SHA1(id) >= ID_AA64ISAR0_SHA1_BASE) { 779 printf("%sSHA1", sep); 780 sep = ","; 781 } 782 783 if (ID_AA64ISAR0_AES(id) >= ID_AA64ISAR0_AES_BASE) { 784 printf("%sAES", sep); 785 sep = ","; 786 #ifdef CRYPTO 787 arm64_has_aes = 1; 788 #endif 789 } 790 if (ID_AA64ISAR0_AES(id) >= ID_AA64ISAR0_AES_PMULL) 791 printf("+PMULL"); 792 793 /* 794 * ID_AA64ISAR1 795 */ 796 id = READ_SPECIALREG(id_aa64isar1_el1); 797 798 if (ID_AA64ISAR1_LS64(id) >= ID_AA64ISAR1_LS64_BASE) { 799 printf("%sLS64", sep); 800 sep = ","; 801 } 802 if (ID_AA64ISAR1_LS64(id) >= ID_AA64ISAR1_LS64_V) 803 printf("+V"); 804 if (ID_AA64ISAR1_LS64(id) >= ID_AA64ISAR1_LS64_ACCDATA) 805 printf("+ACCDATA"); 806 807 if (ID_AA64ISAR1_XS(id) >= ID_AA64ISAR1_XS_IMPL) { 808 printf("%sXS", sep); 809 sep = ","; 810 } 811 812 if (ID_AA64ISAR1_I8MM(id) >= ID_AA64ISAR1_I8MM_IMPL) { 813 printf("%sI8MM", sep); 814 sep = ","; 815 } 816 817 if (ID_AA64ISAR1_DGH(id) >= ID_AA64ISAR1_DGH_IMPL) { 818 printf("%sDGH", sep); 819 sep = ","; 820 } 821 822 if (ID_AA64ISAR1_BF16(id) >= ID_AA64ISAR1_BF16_BASE) { 823 printf("%sBF16", sep); 824 sep = ","; 825 } 826 if (ID_AA64ISAR1_BF16(id) >= ID_AA64ISAR1_BF16_EBF) 827 printf("+EBF"); 828 829 if (ID_AA64ISAR1_SPECRES(id) >= ID_AA64ISAR1_SPECRES_IMPL) { 830 printf("%sSPECRES", sep); 831 sep = ","; 832 } 833 834 if (ID_AA64ISAR1_SB(id) >= ID_AA64ISAR1_SB_IMPL) { 835 printf("%sSB", sep); 836 sep = ","; 837 } 838 839 if (ID_AA64ISAR1_FRINTTS(id) >= ID_AA64ISAR1_FRINTTS_IMPL) { 840 printf("%sFRINTTS", sep); 841 sep = ","; 842 } 843 844 if (ID_AA64ISAR1_GPI(id) >= ID_AA64ISAR1_GPI_IMPL) { 845 printf("%sGPI", sep); 846 sep = ","; 847 } 848 849 if (ID_AA64ISAR1_GPA(id) >= ID_AA64ISAR1_GPA_IMPL) { 850 printf("%sGPA", sep); 851 sep = ","; 852 } 853 854 if (ID_AA64ISAR1_LRCPC(id) >= ID_AA64ISAR1_LRCPC_BASE) { 855 printf("%sLRCPC", sep); 856 sep = ","; 857 } 858 if (ID_AA64ISAR1_LRCPC(id) >= ID_AA64ISAR1_LRCPC_LDAPUR) 859 printf("+LDAPUR"); 860 861 if (ID_AA64ISAR1_FCMA(id) >= ID_AA64ISAR1_FCMA_IMPL) { 862 printf("%sFCMA", sep); 863 sep = ","; 864 } 865 866 if (ID_AA64ISAR1_JSCVT(id) >= ID_AA64ISAR1_JSCVT_IMPL) { 867 printf("%sJSCVT", sep); 868 sep = ","; 869 } 870 871 if (ID_AA64ISAR1_API(id) >= ID_AA64ISAR1_API_PAC) { 872 printf("%sAPI", sep); 873 sep = ","; 874 } 875 if (ID_AA64ISAR1_API(id) == ID_AA64ISAR1_API_EPAC) 876 printf("+EPAC"); 877 else if (ID_AA64ISAR1_API(id) >= ID_AA64ISAR1_API_EPAC2) 878 printf("+EPAC2"); 879 if (ID_AA64ISAR1_API(id) >= ID_AA64ISAR1_API_FPAC) 880 printf("+FPAC"); 881 if (ID_AA64ISAR1_API(id) >= ID_AA64ISAR1_API_FPAC_COMBINED) 882 printf("+COMBINED"); 883 884 if (ID_AA64ISAR1_APA(id) >= ID_AA64ISAR1_APA_PAC) { 885 printf("%sAPA", sep); 886 sep = ","; 887 } 888 if (ID_AA64ISAR1_APA(id) == ID_AA64ISAR1_APA_EPAC) 889 printf("+EPAC"); 890 else if (ID_AA64ISAR1_APA(id) >= ID_AA64ISAR1_APA_EPAC2) 891 printf("+EPAC2"); 892 if (ID_AA64ISAR1_APA(id) >= ID_AA64ISAR1_APA_FPAC) 893 printf("+FPAC"); 894 if (ID_AA64ISAR1_APA(id) >= ID_AA64ISAR1_APA_FPAC_COMBINED) 895 printf("+COMBINED"); 896 897 if (ID_AA64ISAR1_DPB(id) >= ID_AA64ISAR1_DPB_IMPL) { 898 printf("%sDPB", sep); 899 sep = ","; 900 } 901 if (ID_AA64ISAR1_DPB(id) >= ID_AA64ISAR1_DPB_DCCVADP) 902 printf("+DCCVADP"); 903 904 /* 905 * ID_AA64ISAR2 906 */ 907 id = READ_SPECIALREG(id_aa64isar2_el1); 908 909 if (ID_AA64ISAR2_CSSC(id) >= ID_AA64ISAR2_CSSC_IMPL) { 910 printf("%sCSSC", sep); 911 sep = ","; 912 } 913 914 if (ID_AA64ISAR2_RPRFM(id) >= ID_AA64ISAR2_RPRFM_IMPL) { 915 printf("%sRPRFM", sep); 916 sep = ","; 917 } 918 919 if (ID_AA64ISAR2_CLRBHB(id) >= ID_AA64ISAR2_CLRBHB_IMPL) { 920 printf("%sCLRBHB", sep); 921 sep = ","; 922 } 923 924 if (ID_AA64ISAR2_BC(id) >= ID_AA64ISAR2_BC_IMPL) { 925 printf("%sBC", sep); 926 sep = ","; 927 } 928 929 if (ID_AA64ISAR2_MOPS(id) >= ID_AA64ISAR2_MOPS_IMPL) { 930 printf("%sMOPS", sep); 931 sep = ","; 932 } 933 934 if (ID_AA64ISAR2_GPA3(id) >= ID_AA64ISAR2_GPA3_IMPL) { 935 printf("%sGPA3", sep); 936 sep = ","; 937 } 938 939 if (ID_AA64ISAR2_APA3(id) >= ID_AA64ISAR2_APA3_PAC) { 940 printf("%sAPA3", sep); 941 sep = ","; 942 } 943 if (ID_AA64ISAR2_APA3(id) == ID_AA64ISAR2_APA3_EPAC) 944 printf("+EPAC"); 945 else if (ID_AA64ISAR2_APA3(id) >= ID_AA64ISAR2_APA3_EPAC2) 946 printf("+EPAC2"); 947 if (ID_AA64ISAR2_APA3(id) >= ID_AA64ISAR2_APA3_FPAC) 948 printf("+FPAC"); 949 if (ID_AA64ISAR2_APA3(id) >= ID_AA64ISAR2_APA3_FPAC_COMBINED) 950 printf("+COMBINED"); 951 952 if (ID_AA64ISAR2_RPRES(id) >= ID_AA64ISAR2_RPRES_IMPL) { 953 printf("%sRPRES", sep); 954 sep = ","; 955 } 956 957 if (ID_AA64ISAR2_WFXT(id) >= ID_AA64ISAR2_WFXT_IMPL) { 958 printf("%sWFXT", sep); 959 sep = ","; 960 } 961 962 /* 963 * ID_AA64MMFR0 964 * 965 * We only print ASIDBits for now. 966 */ 967 id = READ_SPECIALREG(id_aa64mmfr0_el1); 968 969 if (ID_AA64MMFR0_ECV(id) >= ID_AA64MMFR0_ECV_IMPL) { 970 printf("%sECV", sep); 971 sep = ","; 972 } 973 if (ID_AA64MMFR0_ECV(id) >= ID_AA64MMFR0_ECV_CNTHCTL) 974 printf("+CNTHCTL"); 975 976 if (ID_AA64MMFR0_ASID_BITS(id) == ID_AA64MMFR0_ASID_BITS_16) { 977 printf("%sASID16", sep); 978 sep = ","; 979 } 980 981 /* 982 * ID_AA64MMFR1 983 * 984 * We omit printing most virtualization related fields for now. 985 */ 986 id = READ_SPECIALREG(id_aa64mmfr1_el1); 987 988 if (ID_AA64MMFR1_AFP(id) >= ID_AA64MMFR1_AFP_IMPL) { 989 printf("%sAFP", sep); 990 sep = ","; 991 } 992 993 if (ID_AA64MMFR1_SPECSEI(id) >= ID_AA64MMFR1_SPECSEI_IMPL) { 994 printf("%sSpecSEI", sep); 995 sep = ","; 996 } 997 998 if (ID_AA64MMFR1_PAN(id) >= ID_AA64MMFR1_PAN_IMPL) { 999 printf("%sPAN", sep); 1000 sep = ","; 1001 } 1002 if (ID_AA64MMFR1_PAN(id) >= ID_AA64MMFR1_PAN_ATS1E1) 1003 printf("+ATS1E1"); 1004 if (ID_AA64MMFR1_PAN(id) >= ID_AA64MMFR1_PAN_EPAN) 1005 printf("+EPAN"); 1006 1007 if (ID_AA64MMFR1_LO(id) >= ID_AA64MMFR1_LO_IMPL) { 1008 printf("%sLO", sep); 1009 sep = ","; 1010 } 1011 1012 if (ID_AA64MMFR1_HPDS(id) >= ID_AA64MMFR1_HPDS_IMPL) { 1013 printf("%sHPDS", sep); 1014 sep = ","; 1015 } 1016 1017 if (ID_AA64MMFR1_VH(id) >= ID_AA64MMFR1_VH_IMPL) { 1018 printf("%sVH", sep); 1019 sep = ","; 1020 } 1021 1022 if (ID_AA64MMFR1_HAFDBS(id) >= ID_AA64MMFR1_HAFDBS_AF) { 1023 printf("%sHAF", sep); 1024 sep = ","; 1025 } 1026 if (ID_AA64MMFR1_HAFDBS(id) >= ID_AA64MMFR1_HAFDBS_AF_DBS) 1027 printf("DBS"); 1028 1029 if (ID_AA64MMFR1_ECBHB(id) >= ID_AA64MMFR1_ECBHB_IMPL) { 1030 printf("%sECBHB", sep); 1031 sep = ","; 1032 } 1033 1034 /* 1035 * ID_AA64MMFR2 1036 */ 1037 id = READ_SPECIALREG(id_aa64mmfr2_el1); 1038 1039 if (ID_AA64MMFR2_IDS(id) >= ID_AA64MMFR2_IDS_IMPL) { 1040 printf("%sIDS", sep); 1041 sep = ","; 1042 } 1043 1044 if (ID_AA64MMFR2_AT(id) >= ID_AA64MMFR2_AT_IMPL) { 1045 printf("%sAT", sep); 1046 sep = ","; 1047 } 1048 1049 /* 1050 * ID_AA64PFR0 1051 */ 1052 id = READ_SPECIALREG(id_aa64pfr0_el1); 1053 1054 if (ID_AA64PFR0_CSV3(id) >= ID_AA64PFR0_CSV3_IMPL) { 1055 printf("%sCSV3", sep); 1056 sep = ","; 1057 } 1058 1059 if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_IMPL) { 1060 printf("%sCSV2", sep); 1061 sep = ","; 1062 } 1063 if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_SCXT) 1064 printf("+SCXT"); 1065 if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_HCXT) 1066 printf("+HCXT"); 1067 1068 if (ID_AA64PFR0_DIT(id) >= ID_AA64PFR0_DIT_IMPL) { 1069 printf("%sDIT", sep); 1070 sep = ","; 1071 } 1072 1073 if (ID_AA64PFR0_ADV_SIMD(id) != ID_AA64PFR0_ADV_SIMD_NONE && 1074 ID_AA64PFR0_ADV_SIMD(id) >= ID_AA64PFR0_ADV_SIMD_HP) { 1075 printf("%sAdvSIMD+HP", sep); 1076 sep = ","; 1077 } 1078 1079 if (ID_AA64PFR0_FP(id) != ID_AA64PFR0_FP_NONE && 1080 ID_AA64PFR0_FP(id) >= ID_AA64PFR0_FP_HP) { 1081 printf("%sFP+HP", sep); 1082 sep = ","; 1083 } 1084 1085 /* 1086 * ID_AA64PFR1 1087 */ 1088 id = READ_SPECIALREG(id_aa64pfr1_el1); 1089 1090 if (ID_AA64PFR1_BT(id) >= ID_AA64PFR1_BT_IMPL) { 1091 printf("%sBT", sep); 1092 sep = ","; 1093 } 1094 1095 if (ID_AA64PFR1_SSBS(id) >= ID_AA64PFR1_SSBS_PSTATE) { 1096 printf("%sSSBS", sep); 1097 sep = ","; 1098 } 1099 if (ID_AA64PFR1_SSBS(id) >= ID_AA64PFR1_SSBS_PSTATE_MSR) 1100 printf("+MSR"); 1101 1102 if (ID_AA64PFR1_MTE(id) >= ID_AA64PFR1_MTE_IMPL) { 1103 printf("%sMTE", sep); 1104 sep = ","; 1105 } 1106 1107 prev_id_aa64isar0 = READ_SPECIALREG(id_aa64isar0_el1); 1108 prev_id_aa64isar1 = READ_SPECIALREG(id_aa64isar1_el1); 1109 prev_id_aa64isar2 = READ_SPECIALREG(id_aa64isar2_el1); 1110 prev_id_aa64mmfr0 = READ_SPECIALREG(id_aa64mmfr0_el1); 1111 prev_id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); 1112 prev_id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1); 1113 prev_id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1); 1114 prev_id_aa64pfr1 = READ_SPECIALREG(id_aa64pfr1_el1); 1115 1116 #ifdef CPU_DEBUG 1117 id = READ_SPECIALREG(id_aa64afr0_el1); 1118 printf("\nID_AA64AFR0_EL1: 0x%016llx", id); 1119 id = READ_SPECIALREG(id_aa64afr1_el1); 1120 printf("\nID_AA64AFR1_EL1: 0x%016llx", id); 1121 id = READ_SPECIALREG(id_aa64dfr0_el1); 1122 printf("\nID_AA64DFR0_EL1: 0x%016llx", id); 1123 id = READ_SPECIALREG(id_aa64dfr1_el1); 1124 printf("\nID_AA64DFR1_EL1: 0x%016llx", id); 1125 id = READ_SPECIALREG(id_aa64isar0_el1); 1126 printf("\nID_AA64ISAR0_EL1: 0x%016llx", id); 1127 id = READ_SPECIALREG(id_aa64isar1_el1); 1128 printf("\nID_AA64ISAR1_EL1: 0x%016llx", id); 1129 id = READ_SPECIALREG(id_aa64isar2_el1); 1130 printf("\nID_AA64ISAR2_EL1: 0x%016llx", id); 1131 id = READ_SPECIALREG(id_aa64mmfr0_el1); 1132 printf("\nID_AA64MMFR0_EL1: 0x%016llx", id); 1133 id = READ_SPECIALREG(id_aa64mmfr1_el1); 1134 printf("\nID_AA64MMFR1_EL1: 0x%016llx", id); 1135 id = READ_SPECIALREG(id_aa64mmfr2_el1); 1136 printf("\nID_AA64MMFR2_EL1: 0x%016llx", id); 1137 id = READ_SPECIALREG(id_aa64pfr0_el1); 1138 printf("\nID_AA64PFR0_EL1: 0x%016llx", id); 1139 id = READ_SPECIALREG(id_aa64pfr1_el1); 1140 printf("\nID_AA64PFR1_EL1: 0x%016llx", id); 1141 #endif 1142 } 1143 1144 void 1145 cpu_identify_cleanup(void) 1146 { 1147 uint64_t id_aa64mmfr2; 1148 uint64_t value; 1149 1150 /* ID_AA64ISAR0_EL1 */ 1151 value = cpu_id_aa64isar0 & ID_AA64ISAR0_MASK; 1152 value &= ~ID_AA64ISAR0_TLB_MASK; 1153 cpu_id_aa64isar0 = value; 1154 1155 /* ID_AA64ISAR1_EL1 */ 1156 value = cpu_id_aa64isar1 & ID_AA64ISAR1_MASK; 1157 value &= ~ID_AA64ISAR1_SPECRES_MASK; 1158 cpu_id_aa64isar1 = value; 1159 1160 /* ID_AA64ISAR2_EL1 */ 1161 value = cpu_id_aa64isar2 & ID_AA64ISAR2_MASK; 1162 value &= ~ID_AA64ISAR2_CLRBHB_MASK; 1163 cpu_id_aa64isar2 = value; 1164 1165 /* ID_AA64MMFR0_EL1 */ 1166 value = 0; 1167 value |= cpu_id_aa64mmfr0 & ID_AA64MMFR0_ECV_MASK; 1168 cpu_id_aa64mmfr0 = value; 1169 1170 /* ID_AA64MMFR1_EL1 */ 1171 value = 0; 1172 value |= cpu_id_aa64mmfr1 & ID_AA64MMFR1_AFP_MASK; 1173 cpu_id_aa64mmfr1 = value; 1174 1175 /* ID_AA64MMFR2_EL1 */ 1176 value = 0; 1177 value |= cpu_id_aa64mmfr2 & ID_AA64MMFR2_AT_MASK; 1178 cpu_id_aa64mmfr2 = value; 1179 1180 /* ID_AA64PFR0_EL1 */ 1181 value = 0; 1182 value |= cpu_id_aa64pfr0 & ID_AA64PFR0_FP_MASK; 1183 value |= cpu_id_aa64pfr0 & ID_AA64PFR0_ADV_SIMD_MASK; 1184 value |= cpu_id_aa64pfr0 & ID_AA64PFR0_DIT_MASK; 1185 cpu_id_aa64pfr0 = value; 1186 1187 /* ID_AA64PFR1_EL1 */ 1188 value = 0; 1189 value |= cpu_id_aa64pfr1 & ID_AA64PFR1_BT_MASK; 1190 value |= cpu_id_aa64pfr1 & ID_AA64PFR1_SSBS_MASK; 1191 cpu_id_aa64pfr1 = value; 1192 1193 /* HWCAP */ 1194 hwcap |= HWCAP_FP; /* OpenBSD assumes Floating-point support */ 1195 hwcap |= HWCAP_ASIMD; /* OpenBSD assumes Advanced SIMD support */ 1196 /* HWCAP_EVTSTRM: OpenBSD kernel doesn't configure event stream */ 1197 if (ID_AA64ISAR0_AES(cpu_id_aa64isar0) >= ID_AA64ISAR0_AES_BASE) 1198 hwcap |= HWCAP_AES; 1199 if (ID_AA64ISAR0_AES(cpu_id_aa64isar0) >= ID_AA64ISAR0_AES_PMULL) 1200 hwcap |= HWCAP_PMULL; 1201 if (ID_AA64ISAR0_SHA1(cpu_id_aa64isar0) >= ID_AA64ISAR0_SHA1_BASE) 1202 hwcap |= HWCAP_SHA1; 1203 if (ID_AA64ISAR0_SHA2(cpu_id_aa64isar0) >= ID_AA64ISAR0_SHA2_BASE) 1204 hwcap |= HWCAP_SHA2; 1205 if (ID_AA64ISAR0_CRC32(cpu_id_aa64isar0) >= ID_AA64ISAR0_CRC32_BASE) 1206 hwcap |= HWCAP_CRC32; 1207 if (ID_AA64ISAR0_ATOMIC(cpu_id_aa64isar0) >= ID_AA64ISAR0_ATOMIC_IMPL) 1208 hwcap |= HWCAP_ATOMICS; 1209 if (ID_AA64PFR0_FP(cpu_id_aa64pfr0) != ID_AA64PFR0_FP_NONE && 1210 ID_AA64PFR0_FP(cpu_id_aa64pfr0) >= ID_AA64PFR0_FP_HP) 1211 hwcap |= HWCAP_FPHP; 1212 if (ID_AA64PFR0_ADV_SIMD(cpu_id_aa64pfr0) != ID_AA64PFR0_ADV_SIMD_NONE && 1213 ID_AA64PFR0_ADV_SIMD(cpu_id_aa64pfr0) >= ID_AA64PFR0_ADV_SIMD_HP) 1214 hwcap |= HWCAP_ASIMDHP; 1215 id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1); 1216 if (ID_AA64MMFR2_IDS(id_aa64mmfr2) >= ID_AA64MMFR2_IDS_IMPL) 1217 hwcap |= HWCAP_CPUID; 1218 if (ID_AA64ISAR0_RDM(cpu_id_aa64isar0) >= ID_AA64ISAR0_RDM_IMPL) 1219 hwcap |= HWCAP_ASIMDRDM; 1220 if (ID_AA64ISAR1_JSCVT(cpu_id_aa64isar1) >= ID_AA64ISAR1_JSCVT_IMPL) 1221 hwcap |= HWCAP_JSCVT; 1222 if (ID_AA64ISAR1_FCMA(cpu_id_aa64isar1) >= ID_AA64ISAR1_FCMA_IMPL) 1223 hwcap |= HWCAP_FCMA; 1224 if (ID_AA64ISAR1_LRCPC(cpu_id_aa64isar1) >= ID_AA64ISAR1_LRCPC_BASE) 1225 hwcap |= HWCAP_LRCPC; 1226 if (ID_AA64ISAR1_DPB(cpu_id_aa64isar1) >= ID_AA64ISAR1_DPB_IMPL) 1227 hwcap |= HWCAP_DCPOP; 1228 if (ID_AA64ISAR0_SHA3(cpu_id_aa64isar0) >= ID_AA64ISAR0_SHA3_IMPL) 1229 hwcap |= HWCAP_SHA3; 1230 if (ID_AA64ISAR0_SM3(cpu_id_aa64isar0) >= ID_AA64ISAR0_SM3_IMPL) 1231 hwcap |= HWCAP_SM3; 1232 if (ID_AA64ISAR0_SM4(cpu_id_aa64isar0) >= ID_AA64ISAR0_SM4_IMPL) 1233 hwcap |= HWCAP_SM4; 1234 if (ID_AA64ISAR0_DP(cpu_id_aa64isar0) >= ID_AA64ISAR0_DP_IMPL) 1235 hwcap |= HWCAP_ASIMDDP; 1236 if (ID_AA64ISAR0_SHA2(cpu_id_aa64isar0) >= ID_AA64ISAR0_SHA2_512) 1237 hwcap |= HWCAP_SHA512; 1238 /* HWCAP_SVE: OpenBSD kernel doesn't provide SVE support */ 1239 if (ID_AA64ISAR0_FHM(cpu_id_aa64isar0) >= ID_AA64ISAR0_FHM_IMPL) 1240 hwcap |= HWCAP_ASIMDFHM; 1241 if (ID_AA64PFR0_DIT(cpu_id_aa64pfr0) >= ID_AA64PFR0_DIT_IMPL) 1242 hwcap |= HWCAP_DIT; 1243 if (ID_AA64MMFR2_AT(cpu_id_aa64mmfr2) >= ID_AA64MMFR2_AT_IMPL) 1244 hwcap |= HWCAP_USCAT; 1245 if (ID_AA64ISAR1_LRCPC(cpu_id_aa64isar1) >= ID_AA64ISAR1_LRCPC_LDAPUR) 1246 hwcap |= HWCAP_ILRCPC; 1247 if (ID_AA64ISAR0_TS(cpu_id_aa64isar0) >= ID_AA64ISAR0_TS_BASE) 1248 hwcap |= HWCAP_FLAGM; 1249 if (ID_AA64PFR1_SSBS(cpu_id_aa64pfr1) >= ID_AA64PFR1_SSBS_PSTATE_MSR) 1250 hwcap |= HWCAP_SSBS; 1251 if (ID_AA64ISAR1_SB(cpu_id_aa64isar1) >= ID_AA64ISAR1_SB_IMPL) 1252 hwcap |= HWCAP_SB; 1253 if (ID_AA64ISAR1_APA(cpu_id_aa64isar1) >= ID_AA64ISAR1_APA_PAC || 1254 ID_AA64ISAR1_API(cpu_id_aa64isar1) >= ID_AA64ISAR1_API_PAC) 1255 hwcap |= HWCAP_PACA; 1256 if (ID_AA64ISAR1_GPA(cpu_id_aa64isar1) >= ID_AA64ISAR1_GPA_IMPL || 1257 ID_AA64ISAR1_GPI(cpu_id_aa64isar1) >= ID_AA64ISAR1_GPI_IMPL) 1258 hwcap |= HWCAP_PACG; 1259 1260 /* HWCAP2 */ 1261 if (ID_AA64ISAR1_DPB(cpu_id_aa64isar1) >= ID_AA64ISAR1_DPB_DCCVADP) 1262 hwcap2 |= HWCAP2_DCPODP; 1263 /* HWCAP2_SVE2: OpenBSD kernel doesn't provide SVE support */ 1264 /* HWCAP2_SVEAES: OpenBSD kernel doesn't provide SVE support */ 1265 /* HWCAP2_SVEPMULL: OpenBSD kernel doesn't provide SVE support */ 1266 /* HWCAP2_SVEBITPERM: OpenBSD kernel doesn't provide SVE support */ 1267 /* HWCAP2_SVESHA3: OpenBSD kernel doesn't provide SVE support */ 1268 /* HWCAP2_SVESM4: OpenBSD kernel doesn't provide SVE support */ 1269 if (ID_AA64ISAR0_TS(cpu_id_aa64isar0) >= ID_AA64ISAR0_TS_AXFLAG) 1270 hwcap2 |= HWCAP2_FLAGM2; 1271 if (ID_AA64ISAR1_FRINTTS(cpu_id_aa64isar1) >= ID_AA64ISAR1_FRINTTS_IMPL) 1272 hwcap2 |= HWCAP2_FRINT; 1273 /* HWCAP2_SVEI8MM: OpenBSD kernel doesn't provide SVE support */ 1274 /* HWCAP2_SVEF32MM: OpenBSD kernel doesn't provide SVE support */ 1275 /* HWCAP2_SVEF64MM: OpenBSD kernel doesn't provide SVE support */ 1276 /* HWCAP2_SVEBF16: OpenBSD kernel doesn't provide SVE support */ 1277 if (ID_AA64ISAR1_I8MM(cpu_id_aa64isar1) >= ID_AA64ISAR1_I8MM_IMPL) 1278 hwcap2 |= HWCAP2_I8MM; 1279 if (ID_AA64ISAR1_BF16(cpu_id_aa64isar1) >= ID_AA64ISAR1_BF16_BASE) 1280 hwcap2 |= HWCAP2_BF16; 1281 if (ID_AA64ISAR1_DGH(cpu_id_aa64isar1) >= ID_AA64ISAR1_DGH_IMPL) 1282 hwcap2 |= HWCAP2_DGH; 1283 if (ID_AA64ISAR0_RNDR(cpu_id_aa64isar0) >= ID_AA64ISAR0_RNDR_IMPL) 1284 hwcap2 |= HWCAP2_RNG; 1285 if (ID_AA64PFR1_BT(cpu_id_aa64pfr1) >= ID_AA64PFR1_BT_IMPL) 1286 hwcap2 |= HWCAP2_BTI; 1287 /* HWCAP2_MTE: OpenBSD kernel doesn't provide MTE support */ 1288 if (ID_AA64MMFR0_ECV(cpu_id_aa64mmfr0) >= ID_AA64MMFR0_ECV_IMPL) 1289 hwcap2 |= HWCAP2_ECV; 1290 if (ID_AA64MMFR1_AFP(cpu_id_aa64mmfr1) >= ID_AA64MMFR1_AFP_IMPL) 1291 hwcap2 |= HWCAP2_AFP; 1292 if (ID_AA64ISAR2_RPRES(cpu_id_aa64isar2) >= ID_AA64ISAR2_RPRES_IMPL) 1293 hwcap2 |= HWCAP2_RPRES; 1294 /* HWCAP2_MTE3: OpenBSD kernel doesn't provide MTE support */ 1295 /* HWCAP2_SME: OpenBSD kernel doesn't provide SME support */ 1296 /* HWCAP2_SME_I16I64: OpenBSD kernel doesn't provide SME support */ 1297 /* HWCAP2_SME_F64F64: OpenBSD kernel doesn't provide SME support */ 1298 /* HWCAP2_SME_I8I32: OpenBSD kernel doesn't provide SME support */ 1299 /* HWCAP2_SME_F16F32: OpenBSD kernel doesn't provide SME support */ 1300 /* HWCAP2_SME_B16F32: OpenBSD kernel doesn't provide SME support */ 1301 /* HWCAP2_SME_F32F32: OpenBSD kernel doesn't provide SME support */ 1302 /* HWCAP2_SME_FA64: OpenBSD kernel doesn't provide SME support */ 1303 if (ID_AA64ISAR2_WFXT(cpu_id_aa64isar2) >= ID_AA64ISAR2_WFXT_IMPL) 1304 hwcap2 |= HWCAP2_WFXT; 1305 if (ID_AA64ISAR1_BF16(cpu_id_aa64isar1) >= ID_AA64ISAR1_BF16_EBF) 1306 hwcap2 |= HWCAP2_EBF16; 1307 /* HWCAP2_SVE_EBF16: OpenBSD kernel doesn't provide SVE support */ 1308 if (ID_AA64ISAR2_CSSC(cpu_id_aa64isar2) >= ID_AA64ISAR2_CSSC_IMPL) 1309 hwcap2 |= HWCAP2_CSSC; 1310 if (ID_AA64ISAR2_RPRFM(cpu_id_aa64isar2) >= ID_AA64ISAR2_RPRFM_IMPL) 1311 hwcap2 |= HWCAP2_RPRFM; 1312 /* HWCAP2_SVE2P1: OpenBSD kernel doesn't provide SVE support */ 1313 /* HWCAP2_SME2: OpenBSD kernel doesn't provide SME support */ 1314 /* HWCAP2_SME2P1: OpenBSD kernel doesn't provide SME support */ 1315 /* HWCAP2_SME_I16I32: OpenBSD kernel doesn't provide SME support */ 1316 /* HWCAP2_SME_BI32I32: OpenBSD kernel doesn't provide SME support */ 1317 /* HWCAP2_SME_B16B16: OpenBSD kernel doesn't provide SME support */ 1318 /* HWCAP2_SME_F16F16: OpenBSD kernel doesn't provide SME support */ 1319 if (ID_AA64ISAR2_MOPS(cpu_id_aa64isar2) >= ID_AA64ISAR2_MOPS_IMPL) 1320 hwcap2 |= HWCAP2_MOPS; 1321 if (ID_AA64ISAR2_BC(cpu_id_aa64isar2) >= ID_AA64ISAR2_BC_IMPL) 1322 hwcap2 |= HWCAP2_HBC; 1323 } 1324 1325 void cpu_init(void); 1326 int cpu_start_secondary(struct cpu_info *ci, int, uint64_t); 1327 int cpu_clockspeed(int *); 1328 1329 int 1330 cpu_match(struct device *parent, void *cfdata, void *aux) 1331 { 1332 struct fdt_attach_args *faa = aux; 1333 uint64_t mpidr = READ_SPECIALREG(mpidr_el1); 1334 char buf[32]; 1335 1336 if (OF_getprop(faa->fa_node, "device_type", buf, sizeof(buf)) <= 0 || 1337 strcmp(buf, "cpu") != 0) 1338 return 0; 1339 1340 if (ncpus < MAXCPUS || faa->fa_reg[0].addr == (mpidr & MPIDR_AFF)) 1341 return 1; 1342 1343 return 0; 1344 } 1345 1346 void 1347 cpu_attach(struct device *parent, struct device *dev, void *aux) 1348 { 1349 struct fdt_attach_args *faa = aux; 1350 struct cpu_info *ci; 1351 void *kstack; 1352 #ifdef MULTIPROCESSOR 1353 uint64_t mpidr = READ_SPECIALREG(mpidr_el1); 1354 #endif 1355 uint32_t opp; 1356 1357 KASSERT(faa->fa_nreg > 0); 1358 1359 #ifdef MULTIPROCESSOR 1360 if (faa->fa_reg[0].addr == (mpidr & MPIDR_AFF)) { 1361 ci = &cpu_info_primary; 1362 ci->ci_flags |= CPUF_RUNNING | CPUF_PRESENT | CPUF_PRIMARY; 1363 } else { 1364 ci = malloc(sizeof(*ci), M_DEVBUF, M_WAITOK | M_ZERO); 1365 cpu_info[dev->dv_unit] = ci; 1366 ci->ci_next = cpu_info_list->ci_next; 1367 cpu_info_list->ci_next = ci; 1368 ci->ci_flags |= CPUF_AP; 1369 ncpus++; 1370 } 1371 #else 1372 ci = &cpu_info_primary; 1373 #endif 1374 1375 ci->ci_dev = dev; 1376 ci->ci_cpuid = dev->dv_unit; 1377 ci->ci_mpidr = faa->fa_reg[0].addr; 1378 ci->ci_node = faa->fa_node; 1379 ci->ci_self = ci; 1380 1381 printf(" mpidr %llx:", ci->ci_mpidr); 1382 1383 kstack = km_alloc(USPACE, &kv_any, &kp_zero, &kd_waitok); 1384 ci->ci_el1_stkend = (vaddr_t)kstack + USPACE - 16; 1385 ci->ci_trampoline_vectors = (vaddr_t)trampoline_vectors_none; 1386 1387 #ifdef MULTIPROCESSOR 1388 if (ci->ci_flags & CPUF_AP) { 1389 char buf[32]; 1390 uint64_t spinup_data = 0; 1391 int spinup_method = 0; 1392 int timeout = 10000; 1393 int len; 1394 1395 len = OF_getprop(ci->ci_node, "enable-method", 1396 buf, sizeof(buf)); 1397 if (strcmp(buf, "psci") == 0) { 1398 spinup_method = 1; 1399 } else if (strcmp(buf, "spin-table") == 0) { 1400 spinup_method = 2; 1401 spinup_data = OF_getpropint64(ci->ci_node, 1402 "cpu-release-addr", 0); 1403 } 1404 1405 clockqueue_init(&ci->ci_queue); 1406 sched_init_cpu(ci); 1407 if (cpu_start_secondary(ci, spinup_method, spinup_data)) { 1408 atomic_setbits_int(&ci->ci_flags, CPUF_IDENTIFY); 1409 __asm volatile("dsb sy; sev" ::: "memory"); 1410 1411 while ((ci->ci_flags & CPUF_IDENTIFIED) == 0 && 1412 --timeout) 1413 delay(1000); 1414 if (timeout == 0) { 1415 printf(" failed to identify"); 1416 ci->ci_flags = 0; 1417 } 1418 } else { 1419 printf(" failed to spin up"); 1420 ci->ci_flags = 0; 1421 } 1422 } else { 1423 #endif 1424 cpu_id_aa64isar0 = READ_SPECIALREG(id_aa64isar0_el1); 1425 cpu_id_aa64isar1 = READ_SPECIALREG(id_aa64isar1_el1); 1426 cpu_id_aa64isar2 = READ_SPECIALREG(id_aa64isar2_el1); 1427 cpu_id_aa64mmfr0 = READ_SPECIALREG(id_aa64mmfr0_el1); 1428 cpu_id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); 1429 cpu_id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1); 1430 cpu_id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1); 1431 cpu_id_aa64pfr1 = READ_SPECIALREG(id_aa64pfr1_el1); 1432 1433 /* 1434 * The CSV2/CSV3 "features" are handled on a 1435 * per-processor basis. So it is fine if these fields 1436 * differ between CPU cores. Mask off these fields to 1437 * prevent exporting these to userland. 1438 */ 1439 cpu_id_aa64pfr0 &= ~ID_AA64PFR0_CSV2_MASK; 1440 cpu_id_aa64pfr0 &= ~ID_AA64PFR0_CSV3_MASK; 1441 1442 /* 1443 * We only support 64-bit mode, so we don't care about 1444 * differences in support for 32-bit mode between 1445 * cores. Mask off these fields as well. 1446 */ 1447 cpu_id_aa64pfr0 &= ~ID_AA64PFR0_EL0_MASK; 1448 cpu_id_aa64pfr0 &= ~ID_AA64PFR0_EL1_MASK; 1449 cpu_id_aa64pfr0 &= ~ID_AA64PFR0_EL2_MASK; 1450 cpu_id_aa64pfr0 &= ~ID_AA64PFR0_EL3_MASK; 1451 1452 /* 1453 * Lenovo X13s ships with broken EL2 firmware that 1454 * hangs the machine if we enable PAuth. 1455 */ 1456 if (hw_vendor && hw_prod && strcmp(hw_vendor, "LENOVO") == 0) { 1457 if (strncmp(hw_prod, "21BX", 4) == 0 || 1458 strncmp(hw_prod, "21BY", 4) == 0) { 1459 cpu_id_aa64isar1 &= ~ID_AA64ISAR1_APA_MASK; 1460 cpu_id_aa64isar1 &= ~ID_AA64ISAR1_GPA_MASK; 1461 } 1462 } 1463 1464 cpu_identify(ci); 1465 1466 if (OF_getproplen(ci->ci_node, "clocks") > 0) { 1467 cpu_node = ci->ci_node; 1468 cpu_cpuspeed = cpu_clockspeed; 1469 } 1470 1471 cpu_init(); 1472 1473 if (arm64_has_rng) { 1474 timeout_set(&cpu_rng_to, cpu_rng, &cpu_rng_to); 1475 cpu_rng(&cpu_rng_to); 1476 } 1477 #ifdef MULTIPROCESSOR 1478 } 1479 #endif 1480 1481 #if NKSTAT > 0 1482 cpu_kstat_attach(ci); 1483 #endif 1484 1485 opp = OF_getpropint(ci->ci_node, "operating-points-v2", 0); 1486 if (opp) 1487 cpu_opp_init(ci, opp); 1488 1489 cpu_psci_init(ci); 1490 1491 printf("\n"); 1492 } 1493 1494 void 1495 cpu_init(void) 1496 { 1497 uint64_t id_aa64mmfr1, sctlr; 1498 uint64_t id_aa64pfr0; 1499 uint64_t tcr; 1500 1501 WRITE_SPECIALREG(ttbr0_el1, pmap_kernel()->pm_pt0pa); 1502 __asm volatile("isb"); 1503 tcr = READ_SPECIALREG(tcr_el1); 1504 tcr &= ~TCR_T0SZ(0x3f); 1505 tcr |= TCR_T0SZ(64 - USER_SPACE_BITS); 1506 tcr |= TCR_A1; 1507 WRITE_SPECIALREG(tcr_el1, tcr); 1508 cpu_tlb_flush(); 1509 1510 /* Enable PAN. */ 1511 id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); 1512 if (ID_AA64MMFR1_PAN(id_aa64mmfr1) >= ID_AA64MMFR1_PAN_IMPL) { 1513 sctlr = READ_SPECIALREG(sctlr_el1); 1514 sctlr &= ~SCTLR_SPAN; 1515 if (ID_AA64MMFR1_PAN(id_aa64mmfr1) >= ID_AA64MMFR1_PAN_EPAN) 1516 sctlr |= SCTLR_EPAN; 1517 WRITE_SPECIALREG(sctlr_el1, sctlr); 1518 } 1519 1520 /* Enable DIT. */ 1521 id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1); 1522 if (ID_AA64PFR0_DIT(id_aa64pfr0) >= ID_AA64PFR0_DIT_IMPL) 1523 __asm volatile (".arch armv8.4-a; msr dit, #1"); 1524 1525 /* Enable PAuth. */ 1526 if (ID_AA64ISAR1_APA(cpu_id_aa64isar1) >= ID_AA64ISAR1_APA_PAC || 1527 ID_AA64ISAR1_API(cpu_id_aa64isar1) >= ID_AA64ISAR1_API_PAC) { 1528 sctlr = READ_SPECIALREG(sctlr_el1); 1529 sctlr |= SCTLR_EnIA | SCTLR_EnDA; 1530 sctlr |= SCTLR_EnIB | SCTLR_EnDB; 1531 WRITE_SPECIALREG(sctlr_el1, sctlr); 1532 } 1533 1534 /* Enable strict BTI compatibility for PACIASP and PACIBSP. */ 1535 if (ID_AA64PFR1_BT(cpu_id_aa64pfr1) >= ID_AA64PFR1_BT_IMPL) { 1536 sctlr = READ_SPECIALREG(sctlr_el1); 1537 sctlr |= SCTLR_BT0 | SCTLR_BT1; 1538 WRITE_SPECIALREG(sctlr_el1, sctlr); 1539 } 1540 1541 /* Initialize debug registers. */ 1542 WRITE_SPECIALREG(mdscr_el1, DBG_MDSCR_TDCC); 1543 WRITE_SPECIALREG(oslar_el1, 0); 1544 } 1545 1546 void 1547 cpu_flush_bp_noop(void) 1548 { 1549 } 1550 1551 void 1552 cpu_flush_bp_psci(void) 1553 { 1554 #if NPSCI > 0 1555 psci_flush_bp(); 1556 #endif 1557 } 1558 1559 void 1560 cpu_serror_apple(void) 1561 { 1562 __asm volatile("dsb sy; isb" ::: "memory"); 1563 printf("l2c_err_sts 0x%llx\n", READ_SPECIALREG(s3_3_c15_c8_0)); 1564 printf("l2c_err_adr 0x%llx\n", READ_SPECIALREG(s3_3_c15_c9_0)); 1565 printf("l2c_err_inf 0x%llx\n", READ_SPECIALREG(s3_3_c15_c10_0)); 1566 } 1567 1568 int 1569 cpu_clockspeed(int *freq) 1570 { 1571 *freq = clock_get_frequency(cpu_node, NULL) / 1000000; 1572 return 0; 1573 } 1574 1575 #ifdef MULTIPROCESSOR 1576 1577 void cpu_boot_secondary(struct cpu_info *ci); 1578 void cpu_hatch_secondary(void); 1579 void cpu_hatch_secondary_spin(void); 1580 1581 void cpu_suspend_cycle(void); 1582 1583 void 1584 cpu_boot_secondary_processors(void) 1585 { 1586 struct cpu_info *ci; 1587 CPU_INFO_ITERATOR cii; 1588 1589 CPU_INFO_FOREACH(cii, ci) { 1590 if ((ci->ci_flags & CPUF_AP) == 0) 1591 continue; 1592 if (ci->ci_flags & CPUF_PRIMARY) 1593 continue; 1594 1595 ci->ci_randseed = (arc4random() & 0x7fffffff) + 1; 1596 cpu_boot_secondary(ci); 1597 } 1598 } 1599 1600 void 1601 cpu_start_spin_table(struct cpu_info *ci, uint64_t start, uint64_t data) 1602 { 1603 extern paddr_t cpu_hatch_ci; 1604 1605 pmap_extract(pmap_kernel(), (vaddr_t)ci, &cpu_hatch_ci); 1606 cpu_dcache_wb_range((vaddr_t)&cpu_hatch_ci, sizeof(paddr_t)); 1607 1608 /* this reuses the zero page for the core */ 1609 vaddr_t start_pg = zero_page + (PAGE_SIZE * ci->ci_cpuid); 1610 paddr_t pa = trunc_page(data); 1611 uint64_t offset = data - pa; 1612 uint64_t *startvec = (uint64_t *)(start_pg + offset); 1613 1614 pmap_kenter_cache(start_pg, pa, PROT_READ|PROT_WRITE, PMAP_CACHE_CI); 1615 1616 *startvec = start; 1617 __asm volatile("dsb sy; sev" ::: "memory"); 1618 1619 pmap_kremove(start_pg, PAGE_SIZE); 1620 } 1621 1622 int 1623 cpu_start_secondary(struct cpu_info *ci, int method, uint64_t data) 1624 { 1625 vaddr_t start_va; 1626 paddr_t ci_pa, start_pa; 1627 uint64_t ttbr1; 1628 int32_t status; 1629 1630 __asm("mrs %x0, ttbr1_el1": "=r"(ttbr1)); 1631 ci->ci_ttbr1 = ttbr1; 1632 cpu_dcache_wb_range((vaddr_t)ci, sizeof(*ci)); 1633 1634 switch (method) { 1635 #if NPSCI > 0 1636 case 1: 1637 /* psci */ 1638 start_va = (vaddr_t)cpu_hatch_secondary; 1639 pmap_extract(pmap_kernel(), start_va, &start_pa); 1640 pmap_extract(pmap_kernel(), (vaddr_t)ci, &ci_pa); 1641 status = psci_cpu_on(ci->ci_mpidr, start_pa, ci_pa); 1642 return (status == PSCI_SUCCESS); 1643 #endif 1644 case 2: 1645 /* spin-table */ 1646 start_va = (vaddr_t)cpu_hatch_secondary_spin; 1647 pmap_extract(pmap_kernel(), start_va, &start_pa); 1648 cpu_start_spin_table(ci, start_pa, data); 1649 return 1; 1650 } 1651 1652 return 0; 1653 } 1654 1655 void 1656 cpu_boot_secondary(struct cpu_info *ci) 1657 { 1658 atomic_setbits_int(&ci->ci_flags, CPUF_GO); 1659 __asm volatile("dsb sy; sev" ::: "memory"); 1660 1661 /* 1662 * Send an interrupt as well to make sure the CPU wakes up 1663 * regardless of whether it is in a WFE or a WFI loop. 1664 */ 1665 arm_send_ipi(ci, ARM_IPI_NOP); 1666 1667 while ((ci->ci_flags & CPUF_RUNNING) == 0) 1668 __asm volatile("wfe"); 1669 } 1670 1671 void 1672 cpu_init_secondary(struct cpu_info *ci) 1673 { 1674 struct proc *p; 1675 struct pcb *pcb; 1676 struct trapframe *tf; 1677 struct switchframe *sf; 1678 int s; 1679 1680 ci->ci_flags |= CPUF_PRESENT; 1681 __asm volatile("dsb sy" ::: "memory"); 1682 1683 if ((ci->ci_flags & CPUF_IDENTIFIED) == 0) { 1684 while ((ci->ci_flags & CPUF_IDENTIFY) == 0) 1685 __asm volatile("wfe"); 1686 1687 cpu_identify(ci); 1688 atomic_setbits_int(&ci->ci_flags, CPUF_IDENTIFIED); 1689 __asm volatile("dsb sy" ::: "memory"); 1690 } 1691 1692 while ((ci->ci_flags & CPUF_GO) == 0) 1693 __asm volatile("wfe"); 1694 1695 cpu_init(); 1696 1697 /* 1698 * Start from a clean slate regardless of whether this is the 1699 * initial power up or a wakeup of a suspended CPU. 1700 */ 1701 1702 ci->ci_curproc = NULL; 1703 ci->ci_curpcb = NULL; 1704 ci->ci_curpm = NULL; 1705 ci->ci_cpl = IPL_NONE; 1706 ci->ci_ipending = 0; 1707 ci->ci_idepth = 0; 1708 1709 #ifdef DIAGNOSTIC 1710 ci->ci_mutex_level = 0; 1711 #endif 1712 1713 /* 1714 * Re-create the switchframe for this CPUs idle process. 1715 */ 1716 1717 p = ci->ci_schedstate.spc_idleproc; 1718 pcb = &p->p_addr->u_pcb; 1719 1720 tf = (struct trapframe *)((u_long)p->p_addr 1721 + USPACE 1722 - sizeof(struct trapframe) 1723 - 0x10); 1724 1725 tf = (struct trapframe *)STACKALIGN(tf); 1726 pcb->pcb_tf = tf; 1727 1728 sf = (struct switchframe *)tf - 1; 1729 sf->sf_x19 = (uint64_t)sched_idle; 1730 sf->sf_x20 = (uint64_t)ci; 1731 sf->sf_lr = (uint64_t)proc_trampoline; 1732 pcb->pcb_sp = (uint64_t)sf; 1733 1734 s = splhigh(); 1735 arm_intr_cpu_enable(); 1736 cpu_startclock(); 1737 1738 atomic_setbits_int(&ci->ci_flags, CPUF_RUNNING); 1739 __asm volatile("dsb sy; sev" ::: "memory"); 1740 1741 spllower(IPL_NONE); 1742 1743 sched_toidle(); 1744 } 1745 1746 void 1747 cpu_halt(void) 1748 { 1749 struct cpu_info *ci = curcpu(); 1750 vaddr_t start_va; 1751 paddr_t ci_pa, start_pa; 1752 int count = 0; 1753 u_long psw; 1754 int32_t status; 1755 1756 KERNEL_ASSERT_UNLOCKED(); 1757 SCHED_ASSERT_UNLOCKED(); 1758 1759 start_va = (vaddr_t)cpu_hatch_secondary; 1760 pmap_extract(pmap_kernel(), start_va, &start_pa); 1761 pmap_extract(pmap_kernel(), (vaddr_t)ci, &ci_pa); 1762 1763 psw = intr_disable(); 1764 1765 atomic_clearbits_int(&ci->ci_flags, 1766 CPUF_RUNNING | CPUF_PRESENT | CPUF_GO); 1767 1768 #if NPSCI > 0 1769 if (psci_can_suspend()) 1770 psci_cpu_off(); 1771 #endif 1772 1773 /* 1774 * If we failed to turn ourselves off using PSCI, declare that 1775 * we're still present and spin in a low power state until 1776 * we're told to wake up again by the primary CPU. 1777 */ 1778 1779 atomic_setbits_int(&ci->ci_flags, CPUF_PRESENT); 1780 1781 /* Mask clock interrupts. */ 1782 WRITE_SPECIALREG(cntv_ctl_el0, 1783 READ_SPECIALREG(cntv_ctl_el0) | CNTV_CTL_IMASK); 1784 1785 while ((ci->ci_flags & CPUF_GO) == 0) { 1786 #if NPSCI > 0 1787 if (ci->ci_psci_suspend_param) { 1788 status = psci_cpu_suspend(ci->ci_psci_suspend_param, 1789 start_pa, ci_pa); 1790 if (status != PSCI_SUCCESS) 1791 ci->ci_psci_suspend_param = 0; 1792 } else 1793 #endif 1794 cpu_suspend_cycle(); 1795 count++; 1796 } 1797 1798 atomic_setbits_int(&ci->ci_flags, CPUF_RUNNING); 1799 __asm volatile("dsb sy; sev" ::: "memory"); 1800 1801 intr_restore(psw); 1802 1803 /* Unmask clock interrupts. */ 1804 WRITE_SPECIALREG(cntv_ctl_el0, 1805 READ_SPECIALREG(cntv_ctl_el0) & ~CNTV_CTL_IMASK); 1806 } 1807 1808 void 1809 cpu_kick(struct cpu_info *ci) 1810 { 1811 /* force cpu to enter kernel */ 1812 if (ci != curcpu()) 1813 arm_send_ipi(ci, ARM_IPI_NOP); 1814 } 1815 1816 void 1817 cpu_unidle(struct cpu_info *ci) 1818 { 1819 /* 1820 * This could send IPI or SEV depending on if the other 1821 * processor is sleeping (WFI or WFE), in userland, or if the 1822 * cpu is in other possible wait states? 1823 */ 1824 if (ci != curcpu()) 1825 arm_send_ipi(ci, ARM_IPI_NOP); 1826 } 1827 1828 #endif 1829 1830 int cpu_suspended; 1831 1832 #ifdef SUSPEND 1833 1834 void cpu_hatch_primary(void); 1835 1836 void (*cpu_suspend_cycle_fcn)(void) = cpu_wfi; 1837 label_t cpu_suspend_jmpbuf; 1838 1839 void 1840 cpu_suspend_cycle(void) 1841 { 1842 cpu_suspend_cycle_fcn(); 1843 } 1844 1845 void 1846 cpu_init_primary(void) 1847 { 1848 cpu_init(); 1849 1850 cpu_startclock(); 1851 1852 longjmp(&cpu_suspend_jmpbuf); 1853 } 1854 1855 int 1856 cpu_suspend_primary(void) 1857 { 1858 struct cpu_info *ci = curcpu(); 1859 vaddr_t start_va; 1860 paddr_t ci_pa, start_pa; 1861 uint64_t ttbr1; 1862 int32_t status; 1863 int count = 0; 1864 1865 __asm("mrs %x0, ttbr1_el1": "=r"(ttbr1)); 1866 ci->ci_ttbr1 = ttbr1; 1867 cpu_dcache_wb_range((vaddr_t)ci, sizeof(*ci)); 1868 1869 start_va = (vaddr_t)cpu_hatch_primary; 1870 pmap_extract(pmap_kernel(), start_va, &start_pa); 1871 pmap_extract(pmap_kernel(), (vaddr_t)ci, &ci_pa); 1872 1873 #if NPSCI > 0 1874 if (psci_can_suspend()) { 1875 if (setjmp(&cpu_suspend_jmpbuf)) { 1876 /* XXX wait for debug output on Allwinner A64 */ 1877 delay(200000); 1878 return 0; 1879 } 1880 1881 psci_system_suspend(start_pa, ci_pa); 1882 1883 return EOPNOTSUPP; 1884 } 1885 #endif 1886 1887 if (setjmp(&cpu_suspend_jmpbuf)) 1888 goto resume; 1889 1890 /* 1891 * If PSCI doesn't support SYSTEM_SUSPEND, spin in a low power 1892 * state waiting for an interrupt that wakes us up again. 1893 */ 1894 1895 /* Mask clock interrupts. */ 1896 WRITE_SPECIALREG(cntv_ctl_el0, 1897 READ_SPECIALREG(cntv_ctl_el0) | CNTV_CTL_IMASK); 1898 1899 /* 1900 * All non-wakeup interrupts should be masked at this point; 1901 * re-enable interrupts such that wakeup interrupts actually 1902 * wake us up. Set a flag such that drivers can tell we're 1903 * suspended and change their behaviour accordingly. They can 1904 * wake us up by clearing the flag. 1905 */ 1906 cpu_suspended = 1; 1907 arm_intr_func.setipl(IPL_NONE); 1908 intr_enable(); 1909 1910 while (cpu_suspended) { 1911 #if NPSCI > 0 1912 if (ci->ci_psci_suspend_param) { 1913 status = psci_cpu_suspend(ci->ci_psci_suspend_param, 1914 start_pa, ci_pa); 1915 if (status != PSCI_SUCCESS) 1916 ci->ci_psci_suspend_param = 0; 1917 } else 1918 #endif 1919 cpu_suspend_cycle(); 1920 count++; 1921 } 1922 1923 resume: 1924 intr_disable(); 1925 arm_intr_func.setipl(IPL_HIGH); 1926 1927 /* Unmask clock interrupts. */ 1928 WRITE_SPECIALREG(cntv_ctl_el0, 1929 READ_SPECIALREG(cntv_ctl_el0) & ~CNTV_CTL_IMASK); 1930 1931 return 0; 1932 } 1933 1934 #ifdef MULTIPROCESSOR 1935 1936 void 1937 cpu_resume_secondary(struct cpu_info *ci) 1938 { 1939 int timeout = 10000; 1940 1941 if (ci->ci_flags & CPUF_PRESENT) 1942 return; 1943 1944 cpu_start_secondary(ci, 1, 0); 1945 while ((ci->ci_flags & CPUF_PRESENT) == 0 && --timeout) 1946 delay(1000); 1947 if (timeout == 0) { 1948 printf("%s: failed to spin up\n", 1949 ci->ci_dev->dv_xname); 1950 ci->ci_flags = 0; 1951 } 1952 } 1953 1954 #endif 1955 1956 #endif 1957 1958 /* 1959 * Dynamic voltage and frequency scaling implementation. 1960 */ 1961 1962 extern int perflevel; 1963 1964 struct opp { 1965 uint64_t opp_hz; 1966 uint32_t opp_microvolt; 1967 }; 1968 1969 struct opp_table { 1970 LIST_ENTRY(opp_table) ot_list; 1971 uint32_t ot_phandle; 1972 1973 struct opp *ot_opp; 1974 u_int ot_nopp; 1975 uint64_t ot_opp_hz_min; 1976 uint64_t ot_opp_hz_max; 1977 1978 struct cpu_info *ot_master; 1979 }; 1980 1981 LIST_HEAD(, opp_table) opp_tables = LIST_HEAD_INITIALIZER(opp_tables); 1982 struct task cpu_opp_task; 1983 1984 void cpu_opp_mountroot(struct device *); 1985 void cpu_opp_dotask(void *); 1986 void cpu_opp_setperf(int); 1987 1988 uint32_t cpu_opp_get_cooling_level(void *, uint32_t *); 1989 void cpu_opp_set_cooling_level(void *, uint32_t *, uint32_t); 1990 1991 void 1992 cpu_opp_init(struct cpu_info *ci, uint32_t phandle) 1993 { 1994 struct opp_table *ot; 1995 struct cooling_device *cd; 1996 int count, node, child; 1997 uint32_t opp_hz, opp_microvolt; 1998 uint32_t values[3]; 1999 int i, j, len; 2000 2001 LIST_FOREACH(ot, &opp_tables, ot_list) { 2002 if (ot->ot_phandle == phandle) { 2003 ci->ci_opp_table = ot; 2004 return; 2005 } 2006 } 2007 2008 node = OF_getnodebyphandle(phandle); 2009 if (node == 0) 2010 return; 2011 2012 if (!OF_is_compatible(node, "operating-points-v2")) 2013 return; 2014 2015 count = 0; 2016 for (child = OF_child(node); child != 0; child = OF_peer(child)) { 2017 if (OF_getproplen(child, "turbo-mode") == 0) 2018 continue; 2019 count++; 2020 } 2021 if (count == 0) 2022 return; 2023 2024 ot = malloc(sizeof(struct opp_table), M_DEVBUF, M_ZERO | M_WAITOK); 2025 ot->ot_phandle = phandle; 2026 ot->ot_opp = mallocarray(count, sizeof(struct opp), 2027 M_DEVBUF, M_ZERO | M_WAITOK); 2028 ot->ot_nopp = count; 2029 2030 count = 0; 2031 for (child = OF_child(node); child != 0; child = OF_peer(child)) { 2032 if (OF_getproplen(child, "turbo-mode") == 0) 2033 continue; 2034 opp_hz = OF_getpropint64(child, "opp-hz", 0); 2035 len = OF_getpropintarray(child, "opp-microvolt", 2036 values, sizeof(values)); 2037 opp_microvolt = 0; 2038 if (len == sizeof(uint32_t) || len == 3 * sizeof(uint32_t)) 2039 opp_microvolt = values[0]; 2040 2041 /* Insert into the array, keeping things sorted. */ 2042 for (i = 0; i < count; i++) { 2043 if (opp_hz < ot->ot_opp[i].opp_hz) 2044 break; 2045 } 2046 for (j = count; j > i; j--) 2047 ot->ot_opp[j] = ot->ot_opp[j - 1]; 2048 ot->ot_opp[i].opp_hz = opp_hz; 2049 ot->ot_opp[i].opp_microvolt = opp_microvolt; 2050 count++; 2051 } 2052 2053 ot->ot_opp_hz_min = ot->ot_opp[0].opp_hz; 2054 ot->ot_opp_hz_max = ot->ot_opp[count - 1].opp_hz; 2055 2056 if (OF_getproplen(node, "opp-shared") == 0) 2057 ot->ot_master = ci; 2058 2059 LIST_INSERT_HEAD(&opp_tables, ot, ot_list); 2060 2061 ci->ci_opp_table = ot; 2062 ci->ci_opp_max = ot->ot_nopp - 1; 2063 ci->ci_cpu_supply = OF_getpropint(ci->ci_node, "cpu-supply", 0); 2064 2065 cd = malloc(sizeof(struct cooling_device), M_DEVBUF, M_ZERO | M_WAITOK); 2066 cd->cd_node = ci->ci_node; 2067 cd->cd_cookie = ci; 2068 cd->cd_get_level = cpu_opp_get_cooling_level; 2069 cd->cd_set_level = cpu_opp_set_cooling_level; 2070 cooling_device_register(cd); 2071 2072 /* 2073 * Do additional checks at mountroot when all the clocks and 2074 * regulators are available. 2075 */ 2076 config_mountroot(ci->ci_dev, cpu_opp_mountroot); 2077 } 2078 2079 void 2080 cpu_opp_mountroot(struct device *self) 2081 { 2082 struct cpu_info *ci; 2083 CPU_INFO_ITERATOR cii; 2084 int count = 0; 2085 int level = 0; 2086 2087 if (cpu_setperf) 2088 return; 2089 2090 CPU_INFO_FOREACH(cii, ci) { 2091 struct opp_table *ot = ci->ci_opp_table; 2092 uint64_t curr_hz; 2093 uint32_t curr_microvolt; 2094 int error; 2095 2096 if (ot == NULL) 2097 continue; 2098 2099 #if NKSTAT > 0 2100 cpu_opp_kstat_attach(ci); 2101 #endif 2102 2103 /* Skip if this table is shared and we're not the master. */ 2104 if (ot->ot_master && ot->ot_master != ci) 2105 continue; 2106 2107 /* PWM regulators may need to be explicitly enabled. */ 2108 regulator_enable(ci->ci_cpu_supply); 2109 2110 curr_hz = clock_get_frequency(ci->ci_node, NULL); 2111 curr_microvolt = regulator_get_voltage(ci->ci_cpu_supply); 2112 2113 /* Disable if clock isn't implemented. */ 2114 error = ENODEV; 2115 if (curr_hz != 0) 2116 error = clock_set_frequency(ci->ci_node, NULL, curr_hz); 2117 if (error) { 2118 ci->ci_opp_table = NULL; 2119 printf("%s: clock not implemented\n", 2120 ci->ci_dev->dv_xname); 2121 continue; 2122 } 2123 2124 /* Disable if regulator isn't implemented. */ 2125 error = ci->ci_cpu_supply ? ENODEV : 0; 2126 if (ci->ci_cpu_supply && curr_microvolt != 0) 2127 error = regulator_set_voltage(ci->ci_cpu_supply, 2128 curr_microvolt); 2129 if (error) { 2130 ci->ci_opp_table = NULL; 2131 printf("%s: regulator not implemented\n", 2132 ci->ci_dev->dv_xname); 2133 continue; 2134 } 2135 2136 /* 2137 * Initialize performance level based on the current 2138 * speed of the first CPU that supports DVFS. 2139 */ 2140 if (level == 0) { 2141 uint64_t min, max; 2142 uint64_t level_hz; 2143 2144 min = ot->ot_opp_hz_min; 2145 max = ot->ot_opp_hz_max; 2146 level_hz = clock_get_frequency(ci->ci_node, NULL); 2147 if (level_hz < min) 2148 level_hz = min; 2149 if (level_hz > max) 2150 level_hz = max; 2151 level = howmany(100 * (level_hz - min), (max - min)); 2152 } 2153 2154 count++; 2155 } 2156 2157 if (count > 0) { 2158 task_set(&cpu_opp_task, cpu_opp_dotask, NULL); 2159 cpu_setperf = cpu_opp_setperf; 2160 2161 perflevel = (level > 0) ? level : 0; 2162 cpu_setperf(perflevel); 2163 } 2164 } 2165 2166 void 2167 cpu_opp_dotask(void *arg) 2168 { 2169 struct cpu_info *ci; 2170 CPU_INFO_ITERATOR cii; 2171 2172 CPU_INFO_FOREACH(cii, ci) { 2173 struct opp_table *ot = ci->ci_opp_table; 2174 uint64_t curr_hz, opp_hz; 2175 uint32_t curr_microvolt, opp_microvolt; 2176 int opp_idx; 2177 int error = 0; 2178 2179 if (ot == NULL) 2180 continue; 2181 2182 /* Skip if this table is shared and we're not the master. */ 2183 if (ot->ot_master && ot->ot_master != ci) 2184 continue; 2185 2186 opp_idx = MIN(ci->ci_opp_idx, ci->ci_opp_max); 2187 opp_hz = ot->ot_opp[opp_idx].opp_hz; 2188 opp_microvolt = ot->ot_opp[opp_idx].opp_microvolt; 2189 2190 curr_hz = clock_get_frequency(ci->ci_node, NULL); 2191 curr_microvolt = regulator_get_voltage(ci->ci_cpu_supply); 2192 2193 if (error == 0 && opp_hz < curr_hz) 2194 error = clock_set_frequency(ci->ci_node, NULL, opp_hz); 2195 if (error == 0 && ci->ci_cpu_supply && 2196 opp_microvolt != 0 && opp_microvolt != curr_microvolt) { 2197 error = regulator_set_voltage(ci->ci_cpu_supply, 2198 opp_microvolt); 2199 } 2200 if (error == 0 && opp_hz > curr_hz) 2201 error = clock_set_frequency(ci->ci_node, NULL, opp_hz); 2202 2203 if (error) 2204 printf("%s: DVFS failed\n", ci->ci_dev->dv_xname); 2205 } 2206 } 2207 2208 void 2209 cpu_opp_setperf(int level) 2210 { 2211 struct cpu_info *ci; 2212 CPU_INFO_ITERATOR cii; 2213 2214 CPU_INFO_FOREACH(cii, ci) { 2215 struct opp_table *ot = ci->ci_opp_table; 2216 uint64_t min, max; 2217 uint64_t level_hz, opp_hz; 2218 int opp_idx = -1; 2219 int i; 2220 2221 if (ot == NULL) 2222 continue; 2223 2224 /* Skip if this table is shared and we're not the master. */ 2225 if (ot->ot_master && ot->ot_master != ci) 2226 continue; 2227 2228 min = ot->ot_opp_hz_min; 2229 max = ot->ot_opp_hz_max; 2230 level_hz = min + (level * (max - min)) / 100; 2231 opp_hz = min; 2232 for (i = 0; i < ot->ot_nopp; i++) { 2233 if (ot->ot_opp[i].opp_hz <= level_hz && 2234 ot->ot_opp[i].opp_hz >= opp_hz) 2235 opp_hz = ot->ot_opp[i].opp_hz; 2236 } 2237 2238 /* Find index of selected operating point. */ 2239 for (i = 0; i < ot->ot_nopp; i++) { 2240 if (ot->ot_opp[i].opp_hz == opp_hz) { 2241 opp_idx = i; 2242 break; 2243 } 2244 } 2245 KASSERT(opp_idx >= 0); 2246 2247 ci->ci_opp_idx = opp_idx; 2248 } 2249 2250 /* 2251 * Update the hardware from a task since setting the 2252 * regulators might need process context. 2253 */ 2254 task_add(systq, &cpu_opp_task); 2255 } 2256 2257 uint32_t 2258 cpu_opp_get_cooling_level(void *cookie, uint32_t *cells) 2259 { 2260 struct cpu_info *ci = cookie; 2261 struct opp_table *ot = ci->ci_opp_table; 2262 2263 return ot->ot_nopp - ci->ci_opp_max - 1; 2264 } 2265 2266 void 2267 cpu_opp_set_cooling_level(void *cookie, uint32_t *cells, uint32_t level) 2268 { 2269 struct cpu_info *ci = cookie; 2270 struct opp_table *ot = ci->ci_opp_table; 2271 int opp_max; 2272 2273 if (level > (ot->ot_nopp - 1)) 2274 level = ot->ot_nopp - 1; 2275 2276 opp_max = (ot->ot_nopp - level - 1); 2277 if (ci->ci_opp_max != opp_max) { 2278 ci->ci_opp_max = opp_max; 2279 task_add(systq, &cpu_opp_task); 2280 } 2281 } 2282 2283 2284 void 2285 cpu_psci_init(struct cpu_info *ci) 2286 { 2287 uint32_t *domains; 2288 uint32_t *domain; 2289 uint32_t *states; 2290 uint32_t ncells; 2291 uint32_t cluster; 2292 int idx, len, node; 2293 2294 /* 2295 * Find the shallowest (for now) idle state for this CPU. 2296 * This should be the first one that is listed. We'll use it 2297 * in the idle loop. 2298 */ 2299 2300 len = OF_getproplen(ci->ci_node, "cpu-idle-states"); 2301 if (len < (int)sizeof(uint32_t)) 2302 return; 2303 2304 states = malloc(len, M_TEMP, M_WAITOK); 2305 OF_getpropintarray(ci->ci_node, "cpu-idle-states", states, len); 2306 node = OF_getnodebyphandle(states[0]); 2307 free(states, M_TEMP, len); 2308 if (node) { 2309 uint32_t entry, exit, residency, param; 2310 int32_t features; 2311 2312 param = OF_getpropint(node, "arm,psci-suspend-param", 0); 2313 entry = OF_getpropint(node, "entry-latency-us", 0); 2314 exit = OF_getpropint(node, "exit-latency-us", 0); 2315 residency = OF_getpropint(node, "min-residency-us", 0); 2316 ci->ci_psci_idle_latency += entry + exit + 2 * residency; 2317 2318 /* Skip states that stop the local timer. */ 2319 if (OF_getpropbool(node, "local-timer-stop")) 2320 ci->ci_psci_idle_param = 0; 2321 2322 /* Skip powerdown states. */ 2323 features = psci_features(CPU_SUSPEND); 2324 if (features == PSCI_NOT_SUPPORTED || 2325 (features & PSCI_FEATURE_POWER_STATE_EXT) == 0) { 2326 if (param & PSCI_POWER_STATE_POWERDOWN) 2327 param = 0; 2328 } else { 2329 if (param & PSCI_POWER_STATE_EXT_POWERDOWN) 2330 param = 0; 2331 } 2332 2333 if (param) { 2334 ci->ci_psci_idle_param = param; 2335 cpu_idle_cycle_fcn = cpu_psci_idle_cycle; 2336 } 2337 } 2338 2339 /* 2340 * Hunt for the deepest idle state for this CPU. This is 2341 * fairly complicated as it requires traversing quite a few 2342 * nodes in the device tree. The first step is to look up the 2343 * "psci" power domain for this CPU. 2344 */ 2345 2346 idx = OF_getindex(ci->ci_node, "psci", "power-domain-names"); 2347 if (idx < 0) 2348 return; 2349 2350 len = OF_getproplen(ci->ci_node, "power-domains"); 2351 if (len <= 0) 2352 return; 2353 2354 domains = malloc(len, M_TEMP, M_WAITOK); 2355 OF_getpropintarray(ci->ci_node, "power-domains", domains, len); 2356 2357 domain = domains; 2358 while (domain && domain < domains + (len / sizeof(uint32_t))) { 2359 if (idx == 0) 2360 break; 2361 2362 node = OF_getnodebyphandle(domain[0]); 2363 if (node == 0) 2364 break; 2365 2366 ncells = OF_getpropint(node, "#power-domain-cells", 0); 2367 domain = domain + ncells + 1; 2368 idx--; 2369 } 2370 2371 node = idx == 0 ? OF_getnodebyphandle(domain[0]) : 0; 2372 free(domains, M_TEMP, len); 2373 if (node == 0) 2374 return; 2375 2376 /* 2377 * We found the "psci" power domain. If this power domain has 2378 * a parent power domain, stash its phandle away for later. 2379 */ 2380 2381 cluster = OF_getpropint(node, "power-domains", 0); 2382 2383 /* 2384 * Get the deepest idle state for the CPU; this should be the 2385 * last one that is listed. 2386 */ 2387 2388 len = OF_getproplen(node, "domain-idle-states"); 2389 if (len < (int)sizeof(uint32_t)) 2390 return; 2391 2392 states = malloc(len, M_TEMP, M_WAITOK); 2393 OF_getpropintarray(node, "domain-idle-states", states, len); 2394 2395 node = OF_getnodebyphandle(states[len / sizeof(uint32_t) - 1]); 2396 free(states, M_TEMP, len); 2397 if (node == 0) 2398 return; 2399 2400 ci->ci_psci_suspend_param = 2401 OF_getpropint(node, "arm,psci-suspend-param", 0); 2402 2403 /* 2404 * Qualcomm Snapdragon always seem to operate in OS Initiated 2405 * mode. This means that the last CPU to suspend can pick the 2406 * idle state that powers off the entire cluster. In our case 2407 * that will always be the primary CPU. 2408 */ 2409 2410 #ifdef MULTIPROCESSOR 2411 if (ci->ci_flags & CPUF_AP) 2412 return; 2413 #endif 2414 2415 node = OF_getnodebyphandle(cluster); 2416 if (node == 0) 2417 return; 2418 2419 /* 2420 * Get the deepest idle state for the cluster; this should be 2421 * the last one that is listed. 2422 */ 2423 2424 states = malloc(len, M_TEMP, M_WAITOK); 2425 OF_getpropintarray(node, "domain-idle-states", states, len); 2426 2427 node = OF_getnodebyphandle(states[len / sizeof(uint32_t) - 1]); 2428 free(states, M_TEMP, len); 2429 if (node == 0) 2430 return; 2431 2432 ci->ci_psci_suspend_param = 2433 OF_getpropint(node, "arm,psci-suspend-param", 0); 2434 } 2435 2436 void 2437 cpu_psci_idle_cycle(void) 2438 { 2439 struct cpu_info *ci = curcpu(); 2440 struct timeval start, stop; 2441 u_long itime; 2442 2443 microuptime(&start); 2444 2445 if (ci->ci_prev_sleep > ci->ci_psci_idle_latency) 2446 psci_cpu_suspend(ci->ci_psci_idle_param, 0, 0); 2447 else 2448 cpu_wfi(); 2449 2450 microuptime(&stop); 2451 timersub(&stop, &start, &stop); 2452 itime = stop.tv_sec * 1000000 + stop.tv_usec; 2453 2454 ci->ci_last_itime = itime; 2455 itime >>= 1; 2456 ci->ci_prev_sleep = (ci->ci_prev_sleep + (ci->ci_prev_sleep >> 1) 2457 + itime) >> 1; 2458 } 2459 2460 #if NKSTAT > 0 2461 2462 struct cpu_kstats { 2463 struct kstat_kv ck_impl; 2464 struct kstat_kv ck_part; 2465 struct kstat_kv ck_rev; 2466 }; 2467 2468 void 2469 cpu_kstat_attach(struct cpu_info *ci) 2470 { 2471 struct kstat *ks; 2472 struct cpu_kstats *ck; 2473 uint64_t impl, part; 2474 const char *impl_name = NULL, *part_name = NULL; 2475 const struct cpu_cores *coreselecter = cpu_cores_none; 2476 int i; 2477 2478 ks = kstat_create(ci->ci_dev->dv_xname, 0, "mach", 0, KSTAT_T_KV, 0); 2479 if (ks == NULL) { 2480 printf("%s: unable to create cpu kstats\n", 2481 ci->ci_dev->dv_xname); 2482 return; 2483 } 2484 2485 ck = malloc(sizeof(*ck), M_DEVBUF, M_WAITOK); 2486 2487 impl = CPU_IMPL(ci->ci_midr); 2488 part = CPU_PART(ci->ci_midr); 2489 2490 for (i = 0; cpu_implementers[i].name; i++) { 2491 if (impl == cpu_implementers[i].id) { 2492 impl_name = cpu_implementers[i].name; 2493 coreselecter = cpu_implementers[i].corelist; 2494 break; 2495 } 2496 } 2497 2498 if (impl_name) { 2499 kstat_kv_init(&ck->ck_impl, "impl", KSTAT_KV_T_ISTR); 2500 strlcpy(kstat_kv_istr(&ck->ck_impl), impl_name, 2501 sizeof(kstat_kv_istr(&ck->ck_impl))); 2502 } else 2503 kstat_kv_init(&ck->ck_impl, "impl", KSTAT_KV_T_NULL); 2504 2505 for (i = 0; coreselecter[i].name; i++) { 2506 if (part == coreselecter[i].id) { 2507 part_name = coreselecter[i].name; 2508 break; 2509 } 2510 } 2511 2512 if (part_name) { 2513 kstat_kv_init(&ck->ck_part, "part", KSTAT_KV_T_ISTR); 2514 strlcpy(kstat_kv_istr(&ck->ck_part), part_name, 2515 sizeof(kstat_kv_istr(&ck->ck_part))); 2516 } else 2517 kstat_kv_init(&ck->ck_part, "part", KSTAT_KV_T_NULL); 2518 2519 kstat_kv_init(&ck->ck_rev, "rev", KSTAT_KV_T_ISTR); 2520 snprintf(kstat_kv_istr(&ck->ck_rev), sizeof(kstat_kv_istr(&ck->ck_rev)), 2521 "r%llup%llu", CPU_VAR(ci->ci_midr), CPU_REV(ci->ci_midr)); 2522 2523 ks->ks_softc = ci; 2524 ks->ks_data = ck; 2525 ks->ks_datalen = sizeof(*ck); 2526 ks->ks_read = kstat_read_nop; 2527 2528 kstat_install(ks); 2529 2530 /* XXX should we have a ci->ci_kstat = ks? */ 2531 } 2532 2533 struct cpu_opp_kstats { 2534 struct kstat_kv coppk_freq; 2535 struct kstat_kv coppk_supply_v; 2536 }; 2537 2538 int 2539 cpu_opp_kstat_read(struct kstat *ks) 2540 { 2541 struct cpu_info *ci = ks->ks_softc; 2542 struct cpu_opp_kstats *coppk = ks->ks_data; 2543 2544 struct opp_table *ot = ci->ci_opp_table; 2545 struct cpu_info *oci; 2546 struct timespec now, diff; 2547 2548 /* rate limit */ 2549 getnanouptime(&now); 2550 timespecsub(&now, &ks->ks_updated, &diff); 2551 if (diff.tv_sec < 1) 2552 return (0); 2553 2554 if (ot == NULL) 2555 return (0); 2556 2557 oci = ot->ot_master; 2558 if (oci == NULL) 2559 oci = ci; 2560 2561 kstat_kv_freq(&coppk->coppk_freq) = 2562 clock_get_frequency(oci->ci_node, NULL); 2563 2564 if (oci->ci_cpu_supply) { 2565 kstat_kv_volts(&coppk->coppk_supply_v) = 2566 regulator_get_voltage(oci->ci_cpu_supply); 2567 } 2568 2569 ks->ks_updated = now; 2570 2571 return (0); 2572 } 2573 2574 void 2575 cpu_opp_kstat_attach(struct cpu_info *ci) 2576 { 2577 struct kstat *ks; 2578 struct cpu_opp_kstats *coppk; 2579 struct opp_table *ot = ci->ci_opp_table; 2580 struct cpu_info *oci = ot->ot_master; 2581 2582 if (oci == NULL) 2583 oci = ci; 2584 2585 ks = kstat_create(ci->ci_dev->dv_xname, 0, "dt-opp", 0, 2586 KSTAT_T_KV, 0); 2587 if (ks == NULL) { 2588 printf("%s: unable to create cpu dt-opp kstats\n", 2589 ci->ci_dev->dv_xname); 2590 return; 2591 } 2592 2593 coppk = malloc(sizeof(*coppk), M_DEVBUF, M_WAITOK); 2594 2595 kstat_kv_init(&coppk->coppk_freq, "freq", KSTAT_KV_T_FREQ); 2596 kstat_kv_init(&coppk->coppk_supply_v, "supply", 2597 oci->ci_cpu_supply ? KSTAT_KV_T_VOLTS_DC : KSTAT_KV_T_NULL); 2598 2599 ks->ks_softc = oci; 2600 ks->ks_data = coppk; 2601 ks->ks_datalen = sizeof(*coppk); 2602 ks->ks_read = cpu_opp_kstat_read; 2603 2604 kstat_install(ks); 2605 2606 /* XXX should we have a ci->ci_opp_kstat = ks? */ 2607 } 2608 2609 #endif /* NKSTAT > 0 */ 2610