1 /* $NetBSD: cpu.c,v 1.59 2021/03/09 16:44:27 ryo Exp $ */ 2 3 /* 4 * Copyright (c) 2017 Ryo Shimizu <ryo@nerv.org> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 20 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(1, "$NetBSD: cpu.c,v 1.59 2021/03/09 16:44:27 ryo Exp $"); 31 32 #include "locators.h" 33 #include "opt_arm_debug.h" 34 #include "opt_ddb.h" 35 #include "opt_fdt.h" 36 #include "opt_multiprocessor.h" 37 38 #include <sys/param.h> 39 #include <sys/atomic.h> 40 #include <sys/cpu.h> 41 #include <sys/device.h> 42 #include <sys/kmem.h> 43 #include <sys/reboot.h> 44 #include <sys/rndsource.h> 45 #include <sys/sysctl.h> 46 #include <sys/systm.h> 47 48 #include <crypto/aes/aes_impl.h> 49 #include <crypto/aes/arch/arm/aes_armv8.h> 50 #include <crypto/aes/arch/arm/aes_neon.h> 51 #include <crypto/chacha/chacha_impl.h> 52 #include <crypto/chacha/arch/arm/chacha_neon.h> 53 54 #include <aarch64/armreg.h> 55 #include <aarch64/cpu.h> 56 #include <aarch64/cpu_counter.h> 57 #ifdef DDB 58 #include <aarch64/db_machdep.h> 59 #endif 60 #include <aarch64/machdep.h> 61 62 #include <arm/cpufunc.h> 63 #include <arm/cpu_topology.h> 64 #ifdef FDT 65 #include <arm/fdt/arm_fdtvar.h> 66 #endif 67 68 #ifdef VERBOSE_INIT_ARM 69 #define VPRINTF(...) printf(__VA_ARGS__) 70 #else 71 #define VPRINTF(...) __nothing 72 #endif 73 74 void cpu_attach(device_t, cpuid_t); 75 static void identify_aarch64_model(uint32_t, char *, size_t); 76 static void cpu_identify(device_t self, struct cpu_info *); 77 static void cpu_identify1(device_t self, struct cpu_info *); 78 static void cpu_identify2(device_t self, struct cpu_info *); 79 static void cpu_init_counter(struct cpu_info *); 80 static void cpu_setup_id(struct cpu_info *); 81 static void cpu_setup_sysctl(device_t, struct cpu_info *); 82 static void cpu_setup_rng(device_t, struct cpu_info *); 83 static void cpu_setup_aes(device_t, struct cpu_info *); 84 static void cpu_setup_chacha(device_t, struct cpu_info *); 85 86 #ifdef MULTIPROCESSOR 87 #define NCPUINFO MAXCPUS 88 #else 89 #define NCPUINFO 1 90 #endif /* MULTIPROCESSOR */ 91 92 /* 93 * Our exported cpu_info structs; these will be first used by the 94 * secondary cpus as part of cpu_mpstart and the hatching process. 95 */ 96 struct cpu_info cpu_info_store[NCPUINFO] = { 97 [0] = { 98 .ci_cpl = IPL_HIGH, 99 .ci_curlwp = &lwp0 100 } 101 }; 102 103 void 104 cpu_attach(device_t dv, cpuid_t id) 105 { 106 struct cpu_info *ci; 107 const int unit = device_unit(dv); 108 109 if (unit == 0) { 110 ci = curcpu(); 111 ci->ci_cpuid = id; 112 cpu_setup_id(ci); 113 } else { 114 #ifdef MULTIPROCESSOR 115 if ((boothowto & RB_MD1) != 0) { 116 aprint_naive("\n"); 117 aprint_normal(": multiprocessor boot disabled\n"); 118 return; 119 } 120 121 KASSERT(unit < MAXCPUS); 122 ci = &cpu_info_store[unit]; 123 124 ci->ci_cpl = IPL_HIGH; 125 ci->ci_cpuid = id; 126 /* ci_id is stored by own cpus when hatching */ 127 128 cpu_info[ncpu] = ci; 129 if (cpu_hatched_p(unit) == 0) { 130 ci->ci_dev = dv; 131 dv->dv_private = ci; 132 ci->ci_index = -1; 133 134 aprint_naive(": disabled\n"); 135 aprint_normal(": disabled (unresponsive)\n"); 136 return; 137 } 138 #else /* MULTIPROCESSOR */ 139 aprint_naive(": disabled\n"); 140 aprint_normal(": disabled (uniprocessor kernel)\n"); 141 return; 142 #endif /* MULTIPROCESSOR */ 143 } 144 145 ci->ci_dev = dv; 146 dv->dv_private = ci; 147 148 ci->ci_kfpu_spl = -1; 149 150 arm_cpu_do_topology(ci); 151 cpu_identify(ci->ci_dev, ci); 152 153 #ifdef MULTIPROCESSOR 154 if (unit != 0) { 155 mi_cpu_attach(ci); 156 return; 157 } 158 #endif /* MULTIPROCESSOR */ 159 160 set_cpufuncs(); 161 fpu_attach(ci); 162 163 cpu_identify1(dv, ci); 164 165 /* aarch64_getcacheinfo(0) was called by locore.S */ 166 aarch64_printcacheinfo(dv); 167 cpu_identify2(dv, ci); 168 169 cpu_init_counter(ci); 170 171 cpu_setup_sysctl(dv, ci); 172 cpu_setup_rng(dv, ci); 173 cpu_setup_aes(dv, ci); 174 cpu_setup_chacha(dv, ci); 175 } 176 177 struct cpuidtab { 178 uint32_t cpu_partnum; 179 const char *cpu_name; 180 const char *cpu_vendor; 181 const char *cpu_architecture; 182 }; 183 184 #define CPU_PARTMASK (CPU_ID_IMPLEMENTOR_MASK | CPU_ID_PARTNO_MASK) 185 186 const struct cpuidtab cpuids[] = { 187 { CPU_ID_CORTEXA35R0 & CPU_PARTMASK, "Cortex-A35", "Arm", "v8-A" }, 188 { CPU_ID_CORTEXA53R0 & CPU_PARTMASK, "Cortex-A53", "Arm", "v8-A" }, 189 { CPU_ID_CORTEXA57R0 & CPU_PARTMASK, "Cortex-A57", "Arm", "v8-A" }, 190 { CPU_ID_CORTEXA55R1 & CPU_PARTMASK, "Cortex-A55", "Arm", "v8.2-A+" }, 191 { CPU_ID_CORTEXA65R0 & CPU_PARTMASK, "Cortex-A65", "Arm", "v8.2-A+" }, 192 { CPU_ID_CORTEXA72R0 & CPU_PARTMASK, "Cortex-A72", "Arm", "v8-A" }, 193 { CPU_ID_CORTEXA73R0 & CPU_PARTMASK, "Cortex-A73", "Arm", "v8-A" }, 194 { CPU_ID_CORTEXA75R2 & CPU_PARTMASK, "Cortex-A75", "Arm", "v8.2-A+" }, 195 { CPU_ID_CORTEXA76R3 & CPU_PARTMASK, "Cortex-A76", "Arm", "v8.2-A+" }, 196 { CPU_ID_CORTEXA76AER1 & CPU_PARTMASK, "Cortex-A76AE", "Arm", "v8.2-A+" }, 197 { CPU_ID_CORTEXA77R0 & CPU_PARTMASK, "Cortex-A77", "Arm", "v8.2-A+" }, 198 { CPU_ID_NVIDIADENVER2 & CPU_PARTMASK, "Denver2", "NVIDIA", "v8-A" }, 199 { CPU_ID_EMAG8180 & CPU_PARTMASK, "eMAG", "Ampere", "v8-A" }, 200 { CPU_ID_NEOVERSEE1R1 & CPU_PARTMASK, "Neoverse E1", "Arm", "v8.2-A+" }, 201 { CPU_ID_NEOVERSEN1R3 & CPU_PARTMASK, "Neoverse N1", "Arm", "v8.2-A+" }, 202 { CPU_ID_THUNDERXRX, "ThunderX", "Cavium", "v8-A" }, 203 { CPU_ID_THUNDERX81XXRX, "ThunderX CN81XX", "Cavium", "v8-A" }, 204 { CPU_ID_THUNDERX83XXRX, "ThunderX CN83XX", "Cavium", "v8-A" }, 205 { CPU_ID_THUNDERX2RX, "ThunderX2", "Marvell", "v8.1-A" }, 206 }; 207 208 static void 209 identify_aarch64_model(uint32_t cpuid, char *buf, size_t len) 210 { 211 int i; 212 uint32_t cpupart, variant, revision; 213 214 cpupart = cpuid & CPU_PARTMASK; 215 variant = __SHIFTOUT(cpuid, CPU_ID_VARIANT_MASK); 216 revision = __SHIFTOUT(cpuid, CPU_ID_REVISION_MASK); 217 218 for (i = 0; i < __arraycount(cpuids); i++) { 219 if (cpupart == cpuids[i].cpu_partnum) { 220 snprintf(buf, len, "%s %s r%dp%d (%s)", 221 cpuids[i].cpu_vendor, cpuids[i].cpu_name, 222 variant, revision, 223 cpuids[i].cpu_architecture); 224 return; 225 } 226 } 227 228 snprintf(buf, len, "unknown CPU (ID = 0x%08x)", cpuid); 229 } 230 231 static void 232 cpu_identify(device_t self, struct cpu_info *ci) 233 { 234 char model[128]; 235 const char *m; 236 237 identify_aarch64_model(ci->ci_id.ac_midr, model, sizeof(model)); 238 if (ci->ci_index == 0) { 239 m = cpu_getmodel(); 240 if (m == NULL || *m == 0) 241 cpu_setmodel("%s", model); 242 } 243 244 aprint_naive("\n"); 245 aprint_normal(": %s, id 0x%lx\n", model, ci->ci_cpuid); 246 } 247 248 static void 249 cpu_identify1(device_t self, struct cpu_info *ci) 250 { 251 uint64_t ctr, clidr, sctlr; /* for cache */ 252 253 /* SCTLR - System Control Register */ 254 sctlr = reg_sctlr_el1_read(); 255 if (sctlr & SCTLR_I) 256 aprint_verbose_dev(self, "IC enabled"); 257 else 258 aprint_verbose_dev(self, "IC disabled"); 259 260 if (sctlr & SCTLR_C) 261 aprint_verbose(", DC enabled"); 262 else 263 aprint_verbose(", DC disabled"); 264 265 if (sctlr & SCTLR_A) 266 aprint_verbose(", Alignment check enabled\n"); 267 else { 268 switch (sctlr & (SCTLR_SA | SCTLR_SA0)) { 269 case SCTLR_SA | SCTLR_SA0: 270 aprint_verbose( 271 ", EL0/EL1 stack Alignment check enabled\n"); 272 break; 273 case SCTLR_SA: 274 aprint_verbose(", EL1 stack Alignment check enabled\n"); 275 break; 276 case SCTLR_SA0: 277 aprint_verbose(", EL0 stack Alignment check enabled\n"); 278 break; 279 case 0: 280 aprint_verbose(", Alignment check disabled\n"); 281 break; 282 } 283 } 284 285 /* 286 * CTR - Cache Type Register 287 */ 288 ctr = reg_ctr_el0_read(); 289 clidr = reg_clidr_el1_read(); 290 aprint_verbose_dev(self, "Cache Writeback Granule %" PRIu64 "B," 291 " Exclusives Reservation Granule %" PRIu64 "B\n", 292 __SHIFTOUT(ctr, CTR_EL0_CWG_LINE) * 4, 293 __SHIFTOUT(ctr, CTR_EL0_ERG_LINE) * 4); 294 295 aprint_verbose_dev(self, "Dcache line %ld, Icache line %ld" 296 ", DIC=%lu, IDC=%lu, LoUU=%lu, LoC=%lu, LoUIS=%lu\n", 297 sizeof(int) << __SHIFTOUT(ctr, CTR_EL0_DMIN_LINE), 298 sizeof(int) << __SHIFTOUT(ctr, CTR_EL0_IMIN_LINE), 299 __SHIFTOUT(ctr, CTR_EL0_DIC), 300 __SHIFTOUT(ctr, CTR_EL0_IDC), 301 __SHIFTOUT(clidr, CLIDR_LOUU), 302 __SHIFTOUT(clidr, CLIDR_LOC), 303 __SHIFTOUT(clidr, CLIDR_LOUIS)); 304 } 305 306 307 /* 308 * identify vfp, etc. 309 */ 310 static void 311 cpu_identify2(device_t self, struct cpu_info *ci) 312 { 313 struct aarch64_sysctl_cpu_id *id = &ci->ci_id; 314 uint64_t dfr0; 315 316 if (!CPU_IS_PRIMARY(ci)) { 317 cpu_setup_id(ci); 318 cpu_setup_sysctl(self, ci); 319 } 320 321 dfr0 = reg_id_aa64dfr0_el1_read(); 322 323 aprint_debug_dev(self, "midr=0x%" PRIx32 " mpidr=0x%" PRIx32 "\n", 324 (uint32_t)ci->ci_id.ac_midr, (uint32_t)ci->ci_id.ac_mpidr); 325 aprint_verbose_dev(self, "revID=0x%" PRIx64, id->ac_revidr); 326 327 /* ID_AA64DFR0_EL1 */ 328 switch (__SHIFTOUT(dfr0, ID_AA64DFR0_EL1_PMUVER)) { 329 case ID_AA64DFR0_EL1_PMUVER_V3: 330 aprint_verbose(", PMCv3"); 331 break; 332 case ID_AA64DFR0_EL1_PMUVER_NOV3: 333 aprint_verbose(", PMC"); 334 break; 335 } 336 337 /* ID_AA64MMFR0_EL1 */ 338 switch (__SHIFTOUT(id->ac_aa64mmfr0, ID_AA64MMFR0_EL1_TGRAN4)) { 339 case ID_AA64MMFR0_EL1_TGRAN4_4KB: 340 aprint_verbose(", 4k table"); 341 break; 342 } 343 switch (__SHIFTOUT(id->ac_aa64mmfr0, ID_AA64MMFR0_EL1_TGRAN16)) { 344 case ID_AA64MMFR0_EL1_TGRAN16_16KB: 345 aprint_verbose(", 16k table"); 346 break; 347 } 348 switch (__SHIFTOUT(id->ac_aa64mmfr0, ID_AA64MMFR0_EL1_TGRAN64)) { 349 case ID_AA64MMFR0_EL1_TGRAN64_64KB: 350 aprint_verbose(", 64k table"); 351 break; 352 } 353 354 switch (__SHIFTOUT(id->ac_aa64mmfr0, ID_AA64MMFR0_EL1_ASIDBITS)) { 355 case ID_AA64MMFR0_EL1_ASIDBITS_8BIT: 356 aprint_verbose(", 8bit ASID"); 357 break; 358 case ID_AA64MMFR0_EL1_ASIDBITS_16BIT: 359 aprint_verbose(", 16bit ASID"); 360 break; 361 } 362 aprint_verbose("\n"); 363 364 365 366 aprint_verbose_dev(self, "auxID=0x%" PRIx64, ci->ci_id.ac_aa64isar0); 367 368 /* PFR0 */ 369 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_CSV3)) { 370 case ID_AA64PFR0_EL1_CSV3_IMPL: 371 aprint_verbose(", CSV3"); 372 break; 373 } 374 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_CSV2)) { 375 case ID_AA64PFR0_EL1_CSV2_IMPL: 376 aprint_verbose(", CSV2"); 377 break; 378 } 379 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_GIC)) { 380 case ID_AA64PFR0_EL1_GIC_CPUIF_EN: 381 aprint_verbose(", GICv3"); 382 break; 383 } 384 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_FP)) { 385 case ID_AA64PFR0_EL1_FP_NONE: 386 break; 387 default: 388 aprint_verbose(", FP"); 389 break; 390 } 391 392 /* ISAR0 */ 393 switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_CRC32)) { 394 case ID_AA64ISAR0_EL1_CRC32_CRC32X: 395 aprint_verbose(", CRC32"); 396 break; 397 } 398 switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_SHA1)) { 399 case ID_AA64ISAR0_EL1_SHA1_SHA1CPMHSU: 400 aprint_verbose(", SHA1"); 401 break; 402 } 403 switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_SHA2)) { 404 case ID_AA64ISAR0_EL1_SHA2_SHA256HSU: 405 aprint_verbose(", SHA256"); 406 break; 407 } 408 switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_AES)) { 409 case ID_AA64ISAR0_EL1_AES_AES: 410 aprint_verbose(", AES"); 411 break; 412 case ID_AA64ISAR0_EL1_AES_PMUL: 413 aprint_verbose(", AES+PMULL"); 414 break; 415 } 416 switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_RNDR)) { 417 case ID_AA64ISAR0_EL1_RNDR_RNDRRS: 418 aprint_verbose(", RNDRRS"); 419 break; 420 } 421 422 /* PFR0:DIT -- data-independent timing support */ 423 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_DIT)) { 424 case ID_AA64PFR0_EL1_DIT_IMPL: 425 aprint_verbose(", DIT"); 426 break; 427 } 428 429 /* PFR0:AdvSIMD */ 430 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_ADVSIMD)) { 431 case ID_AA64PFR0_EL1_ADV_SIMD_NONE: 432 break; 433 default: 434 aprint_verbose(", NEON"); 435 break; 436 } 437 438 /* MVFR0/MVFR1 */ 439 switch (__SHIFTOUT(id->ac_mvfr0, MVFR0_FPROUND)) { 440 case MVFR0_FPROUND_ALL: 441 aprint_verbose(", rounding"); 442 break; 443 } 444 switch (__SHIFTOUT(id->ac_mvfr0, MVFR0_FPTRAP)) { 445 case MVFR0_FPTRAP_TRAP: 446 aprint_verbose(", exceptions"); 447 break; 448 } 449 switch (__SHIFTOUT(id->ac_mvfr1, MVFR1_FPDNAN)) { 450 case MVFR1_FPDNAN_NAN: 451 aprint_verbose(", NaN propagation"); 452 break; 453 } 454 switch (__SHIFTOUT(id->ac_mvfr1, MVFR1_FPFTZ)) { 455 case MVFR1_FPFTZ_DENORMAL: 456 aprint_verbose(", denormals"); 457 break; 458 } 459 switch (__SHIFTOUT(id->ac_mvfr0, MVFR0_SIMDREG)) { 460 case MVFR0_SIMDREG_16x64: 461 aprint_verbose(", 16x64bitRegs"); 462 break; 463 case MVFR0_SIMDREG_32x64: 464 aprint_verbose(", 32x64bitRegs"); 465 break; 466 } 467 switch (__SHIFTOUT(id->ac_mvfr1, MVFR1_SIMDFMAC)) { 468 case MVFR1_SIMDFMAC_FMAC: 469 aprint_verbose(", Fused Multiply-Add"); 470 break; 471 } 472 473 aprint_verbose("\n"); 474 } 475 476 /* 477 * Enable the performance counter, then estimate frequency for 478 * the current PE and store the result in cpu_cc_freq. 479 */ 480 static void 481 cpu_init_counter(struct cpu_info *ci) 482 { 483 const uint64_t dfr0 = reg_id_aa64dfr0_el1_read(); 484 const u_int pmuver = __SHIFTOUT(dfr0, ID_AA64DFR0_EL1_PMUVER); 485 if (pmuver == ID_AA64DFR0_EL1_PMUVER_NONE) { 486 /* Performance Monitors Extension not implemented. */ 487 return; 488 } 489 490 reg_pmcr_el0_write(PMCR_E | PMCR_C); 491 reg_pmcntenset_el0_write(PMCNTEN_C); 492 493 const uint32_t prev = cpu_counter32(); 494 delay(100000); 495 ci->ci_data.cpu_cc_freq = (cpu_counter32() - prev) * 10; 496 } 497 498 /* 499 * Fill in this CPUs id data. Must be called from hatched cpus. 500 */ 501 static void 502 cpu_setup_id(struct cpu_info *ci) 503 { 504 struct aarch64_sysctl_cpu_id *id = &ci->ci_id; 505 506 memset(id, 0, sizeof *id); 507 508 id->ac_midr = reg_midr_el1_read(); 509 id->ac_revidr = reg_revidr_el1_read(); 510 id->ac_mpidr = reg_mpidr_el1_read(); 511 512 id->ac_aa64dfr0 = reg_id_aa64dfr0_el1_read(); 513 id->ac_aa64dfr1 = reg_id_aa64dfr1_el1_read(); 514 515 id->ac_aa64isar0 = reg_id_aa64isar0_el1_read(); 516 id->ac_aa64isar1 = reg_id_aa64isar1_el1_read(); 517 518 id->ac_aa64mmfr0 = reg_id_aa64mmfr0_el1_read(); 519 id->ac_aa64mmfr1 = reg_id_aa64mmfr1_el1_read(); 520 id->ac_aa64mmfr2 = reg_id_aa64mmfr2_el1_read(); 521 522 id->ac_mvfr0 = reg_mvfr0_el1_read(); 523 id->ac_mvfr1 = reg_mvfr1_el1_read(); 524 id->ac_mvfr2 = reg_mvfr2_el1_read(); 525 526 id->ac_clidr = reg_clidr_el1_read(); 527 id->ac_ctr = reg_ctr_el0_read(); 528 529 /* Only in ARMv8.2. */ 530 id->ac_aa64zfr0 = 0 /* reg_id_aa64zfr0_el1_read() */; 531 532 id->ac_aa64pfr0 = reg_id_aa64pfr0_el1_read(); 533 id->ac_aa64pfr1 = reg_id_aa64pfr1_el1_read(); 534 } 535 536 /* 537 * setup the per-cpu sysctl tree. 538 */ 539 static void 540 cpu_setup_sysctl(device_t dv, struct cpu_info *ci) 541 { 542 const struct sysctlnode *cpunode = NULL; 543 544 sysctl_createv(NULL, 0, NULL, &cpunode, 545 CTLFLAG_PERMANENT, 546 CTLTYPE_NODE, device_xname(dv), NULL, 547 NULL, 0, NULL, 0, 548 CTL_MACHDEP, 549 CTL_CREATE, CTL_EOL); 550 551 if (cpunode == NULL) 552 return; 553 554 sysctl_createv(NULL, 0, &cpunode, NULL, 555 CTLFLAG_PERMANENT, 556 CTLTYPE_STRUCT, "cpu_id", NULL, 557 NULL, 0, &ci->ci_id, sizeof(ci->ci_id), 558 CTL_CREATE, CTL_EOL); 559 } 560 561 static struct krndsource rndrrs_source; 562 563 static void 564 rndrrs_get(size_t nbytes, void *cookie) 565 { 566 /* Entropy bits per data byte, wild-arse guess. */ 567 const unsigned bpb = 4; 568 size_t nbits = nbytes*NBBY; 569 uint64_t x; 570 int error; 571 572 while (nbits) { 573 /* 574 * x := random 64-bit sample 575 * error := Z bit, set to 1 if sample is bad 576 * 577 * XXX This should be done by marking the function 578 * __attribute__((target("arch=armv8.5-a+rng"))) and 579 * using `mrs %0, rndrrs', but: 580 * 581 * (a) the version of gcc we use doesn't support that, 582 * and 583 * (b) clang doesn't seem to like `rndrrs' itself. 584 * 585 * So we use the numeric encoding for now. 586 */ 587 __asm __volatile("" 588 "mrs %0, s3_3_c2_c4_1\n" 589 "cset %w1, eq" 590 : "=r"(x), "=r"(error)); 591 if (error) 592 break; 593 rnd_add_data_sync(&rndrrs_source, &x, sizeof(x), 594 bpb*sizeof(x)); 595 nbits -= MIN(nbits, bpb*sizeof(x)); 596 } 597 598 explicit_memset(&x, 0, sizeof x); 599 } 600 601 /* 602 * setup the RNDRRS entropy source 603 */ 604 static void 605 cpu_setup_rng(device_t dv, struct cpu_info *ci) 606 { 607 struct aarch64_sysctl_cpu_id *id = &ci->ci_id; 608 609 /* Probably shared between cores. */ 610 if (!CPU_IS_PRIMARY(ci)) 611 return; 612 613 /* Verify that it is supported. */ 614 switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_RNDR)) { 615 case ID_AA64ISAR0_EL1_RNDR_RNDRRS: 616 break; 617 default: 618 return; 619 } 620 621 /* Attach it. */ 622 rndsource_setcb(&rndrrs_source, rndrrs_get, NULL); 623 rnd_attach_source(&rndrrs_source, "rndrrs", RND_TYPE_RNG, 624 RND_FLAG_DEFAULT|RND_FLAG_HASCB); 625 } 626 627 /* 628 * setup the AES implementation 629 */ 630 static void 631 cpu_setup_aes(device_t dv, struct cpu_info *ci) 632 { 633 struct aarch64_sysctl_cpu_id *id = &ci->ci_id; 634 635 /* Check for ARMv8.0-AES support. */ 636 switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_AES)) { 637 case ID_AA64ISAR0_EL1_AES_AES: 638 case ID_AA64ISAR0_EL1_AES_PMUL: 639 aes_md_init(&aes_armv8_impl); 640 return; 641 default: 642 break; 643 } 644 645 /* Failing that, check for SIMD support. */ 646 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_ADVSIMD)) { 647 case ID_AA64PFR0_EL1_ADV_SIMD_IMPL: 648 aes_md_init(&aes_neon_impl); 649 return; 650 default: 651 break; 652 } 653 } 654 655 /* 656 * setup the ChaCha implementation 657 */ 658 static void 659 cpu_setup_chacha(device_t dv, struct cpu_info *ci) 660 { 661 struct aarch64_sysctl_cpu_id *id = &ci->ci_id; 662 663 /* Check for SIMD support. */ 664 switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_ADVSIMD)) { 665 case ID_AA64PFR0_EL1_ADV_SIMD_IMPL: 666 chacha_md_init(&chacha_neon_impl); 667 return; 668 default: 669 break; 670 } 671 } 672 673 #ifdef MULTIPROCESSOR 674 void 675 cpu_hatch(struct cpu_info *ci) 676 { 677 KASSERT(curcpu() == ci); 678 679 mutex_enter(&cpu_hatch_lock); 680 681 set_cpufuncs(); 682 fpu_attach(ci); 683 684 cpu_identify1(ci->ci_dev, ci); 685 aarch64_getcacheinfo(device_unit(ci->ci_dev)); 686 aarch64_printcacheinfo(ci->ci_dev); 687 cpu_identify2(ci->ci_dev, ci); 688 #ifdef DDB 689 db_machdep_init(); 690 #endif 691 mutex_exit(&cpu_hatch_lock); 692 693 cpu_init_counter(ci); 694 695 intr_cpu_init(ci); 696 697 #ifdef FDT 698 arm_fdt_cpu_hatch(ci); 699 #endif 700 #ifdef MD_CPU_HATCH 701 MD_CPU_HATCH(ci); /* for non-fdt arch? */ 702 #endif 703 704 /* 705 * clear my bit of arm_cpu_mbox to tell cpu_boot_secondary_processors(). 706 * there are cpu0,1,2,3, and if cpu2 is unresponsive, 707 * ci_index are each cpu0=0, cpu1=1, cpu2=undef, cpu3=2. 708 * therefore we have to use device_unit instead of ci_index for mbox. 709 */ 710 711 cpu_clr_mbox(device_unit(ci->ci_dev)); 712 } 713 #endif /* MULTIPROCESSOR */ 714