xref: /netbsd-src/sys/arch/aarch64/aarch64/cpu.c (revision 5dd13e896668a4f217808e435c02749be10ec09c)
1 /* $NetBSD: cpu.c,v 1.82 2024/12/09 21:52:53 jmcneill Exp $ */
2 
3 /*
4  * Copyright (c) 2017 Ryo Shimizu
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18  * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
19  * DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
20  * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
21  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
22  * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
24  * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
25  * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
26  * POSSIBILITY OF SUCH DAMAGE.
27  */
28 
29 #include <sys/cdefs.h>
30 __KERNEL_RCSID(1, "$NetBSD: cpu.c,v 1.82 2024/12/09 21:52:53 jmcneill Exp $");
31 
32 #include "locators.h"
33 #include "opt_arm_debug.h"
34 #include "opt_ddb.h"
35 #include "opt_fdt.h"
36 #include "opt_multiprocessor.h"
37 
38 #include <sys/param.h>
39 #include <sys/atomic.h>
40 #include <sys/cpu.h>
41 #include <sys/device.h>
42 #include <sys/kmem.h>
43 #include <sys/reboot.h>
44 #include <sys/rndsource.h>
45 #include <sys/sdt.h>
46 #include <sys/sysctl.h>
47 #include <sys/systm.h>
48 
49 #include <crypto/aes/aes_impl.h>
50 #include <crypto/aes/arch/arm/aes_armv8.h>
51 #include <crypto/aes/arch/arm/aes_neon.h>
52 #include <crypto/chacha/chacha_impl.h>
53 #include <crypto/chacha/arch/arm/chacha_neon.h>
54 
55 #include <aarch64/armreg.h>
56 #include <aarch64/cpu.h>
57 #include <aarch64/cpu_counter.h>
58 #ifdef DDB
59 #include <aarch64/db_machdep.h>
60 #endif
61 #include <aarch64/machdep.h>
62 
63 #include <arm/cpufunc.h>
64 #include <arm/cpuvar.h>
65 #include <arm/cpu_topology.h>
66 #ifdef FDT
67 #include <arm/fdt/arm_fdtvar.h>
68 #endif
69 
70 #ifdef VERBOSE_INIT_ARM
71 #define VPRINTF(...)	printf(__VA_ARGS__)
72 #else
73 #define VPRINTF(...)	__nothing
74 #endif
75 
76 void cpu_attach(device_t, cpuid_t);
77 void cpu_setup_id(struct cpu_info *);
78 
79 static void identify_aarch64_model(uint32_t, char *, size_t);
80 static void cpu_identify(device_t self, struct cpu_info *);
81 static void cpu_identify1(device_t self, struct cpu_info *);
82 static void cpu_identify2(device_t self, struct cpu_info *);
83 static void cpu_init_counter(struct cpu_info *);
84 static void cpu_setup_sysctl(device_t, struct cpu_info *);
85 static void cpu_setup_rng(device_t, struct cpu_info *);
86 static void cpu_setup_aes(device_t, struct cpu_info *);
87 static void cpu_setup_chacha(device_t, struct cpu_info *);
88 
89 #ifdef MULTIPROCESSOR
90 #define NCPUINFO	MAXCPUS
91 #else
92 #define NCPUINFO	1
93 #endif /* MULTIPROCESSOR */
94 
95 /*
96  * Our exported cpu_info structs; these will be first used by the
97  * secondary cpus as part of cpu_mpstart and the hatching process.
98  */
99 struct cpu_info cpu_info_store[NCPUINFO] = {
100 	[0] = {
101 		.ci_cpl = IPL_HIGH,
102 		.ci_curlwp = &lwp0
103 	}
104 };
105 
106 void
107 cpu_attach(device_t dv, cpuid_t id)
108 {
109 	struct cpu_info *ci;
110 	const int unit = device_unit(dv);
111 
112 	if (unit == 0) {
113 		ci = curcpu();
114 		ci->ci_cpuid = id;
115 	} else {
116 #ifdef MULTIPROCESSOR
117 		if ((boothowto & RB_MD1) != 0) {
118 			aprint_naive("\n");
119 			aprint_normal(": multiprocessor boot disabled\n");
120 			return;
121 		}
122 
123 		KASSERT(unit < MAXCPUS);
124 		ci = &cpu_info_store[unit];
125 
126 		ci->ci_cpl = IPL_HIGH;
127 		ci->ci_cpuid = id;
128 		/* ci_id is stored by own cpus when hatching */
129 
130 		cpu_info[ncpu] = ci;
131 		if (cpu_hatched_p(unit) == 0) {
132 			ci->ci_dev = dv;
133 			device_set_private(dv, ci);
134 			ci->ci_index = -1;
135 
136 			aprint_naive(": disabled\n");
137 			aprint_normal(": disabled (unresponsive)\n");
138 			return;
139 		}
140 #else /* MULTIPROCESSOR */
141 		aprint_naive(": disabled\n");
142 		aprint_normal(": disabled (uniprocessor kernel)\n");
143 		return;
144 #endif /* MULTIPROCESSOR */
145 	}
146 
147 	ci->ci_dev = dv;
148 	device_set_private(dv, ci);
149 
150 	ci->ci_kfpu_spl = -1;
151 
152 	arm_cpu_do_topology(ci);	// XXXNH move this after mi_cpu_attach
153 	cpu_identify(dv, ci);
154 
155 	cpu_setup_sysctl(dv, ci);
156 
157 #ifdef MULTIPROCESSOR
158 	if (unit != 0) {
159 		mi_cpu_attach(ci);
160 		pmap_tlb_info_attach(&pmap_tlb0_info, ci);
161 		aarch64_parsecacheinfo(ci);
162 	}
163 #endif /* MULTIPROCESSOR */
164 
165 	fpu_attach(ci);
166 
167 	cpu_identify1(dv, ci);
168 	aarch64_printcacheinfo(dv, ci);
169 	cpu_identify2(dv, ci);
170 
171 	cpu_setup_rng(dv, ci);
172 
173 	if (unit != 0) {
174 	    return;
175 	}
176 
177 #ifdef DDB
178 	db_machdep_init(ci);
179 #endif
180 
181 	cpu_init_counter(ci);
182 
183 	/* These currently only check the BP. */
184 	cpu_setup_aes(dv, ci);
185 	cpu_setup_chacha(dv, ci);
186 
187 	cpu_rescan(dv, NULL, NULL);
188 }
189 
190 int
191 cpu_rescan(device_t dv, const char *ifattr, const int *locators)
192 {
193 	struct cpu_info *ci = device_private(dv);
194 
195 	if (ifattr_match(ifattr, "cpufeaturebus")) {
196 		struct cpufeature_attach_args cfaa = {
197 			.ci = ci,
198 		};
199 		config_found(dv, &cfaa, NULL, CFARGS(.iattr = "cpufeaturebus"));
200 	}
201 
202 	return 0;
203 }
204 
205 void
206 cpu_childdetached(device_t dv, device_t child)
207 {
208 	/* Nada */
209 }
210 
211 struct cpuidtab {
212 	uint32_t cpu_partnum;
213 	const char *cpu_name;
214 	const char *cpu_vendor;
215 	const char *cpu_architecture;
216 };
217 
218 #define CPU_PARTMASK	(CPU_ID_IMPLEMENTOR_MASK | CPU_ID_PARTNO_MASK)
219 
220 const struct cpuidtab cpuids[] = {
221 	{ CPU_ID_CORTEXA35R0 & CPU_PARTMASK, "Cortex-A35", "Arm", "v8-A" },
222 	{ CPU_ID_CORTEXA53R0 & CPU_PARTMASK, "Cortex-A53", "Arm", "v8-A" },
223 	{ CPU_ID_CORTEXA57R0 & CPU_PARTMASK, "Cortex-A57", "Arm", "v8-A" },
224 	{ CPU_ID_CORTEXA55R1 & CPU_PARTMASK, "Cortex-A55", "Arm", "v8.2-A+" },
225 	{ CPU_ID_CORTEXA65R0 & CPU_PARTMASK, "Cortex-A65", "Arm", "v8.2-A+" },
226 	{ CPU_ID_CORTEXA72R0 & CPU_PARTMASK, "Cortex-A72", "Arm", "v8-A" },
227 	{ CPU_ID_CORTEXA73R0 & CPU_PARTMASK, "Cortex-A73", "Arm", "v8-A" },
228 	{ CPU_ID_CORTEXA75R2 & CPU_PARTMASK, "Cortex-A75", "Arm", "v8.2-A+" },
229 	{ CPU_ID_CORTEXA76R3 & CPU_PARTMASK, "Cortex-A76", "Arm", "v8.2-A+" },
230 	{ CPU_ID_CORTEXA76AER1 & CPU_PARTMASK, "Cortex-A76AE", "Arm", "v8.2-A+" },
231 	{ CPU_ID_CORTEXA77R0 & CPU_PARTMASK, "Cortex-A77", "Arm", "v8.2-A+" },
232 	{ CPU_ID_CORTEXA710R2 & CPU_PARTMASK, "Cortex-A710", "Arm", "v9.0-A" },
233 	{ CPU_ID_NVIDIADENVER2 & CPU_PARTMASK, "Denver2", "NVIDIA", "v8-A" },
234 	{ CPU_ID_EMAG8180 & CPU_PARTMASK, "eMAG", "Ampere", "v8-A" },
235 	{ CPU_ID_NEOVERSEE1R1 & CPU_PARTMASK, "Neoverse E1", "Arm", "v8.2-A+" },
236 	{ CPU_ID_NEOVERSEN1R3 & CPU_PARTMASK, "Neoverse N1", "Arm", "v8.2-A+" },
237 	{ CPU_ID_NEOVERSEV1R1 & CPU_PARTMASK, "Neoverse V1", "Arm", "v8.4-A+" },
238 	{ CPU_ID_NEOVERSEN2R0 & CPU_PARTMASK, "Neoverse N2", "Arm", "v9.0-A" },
239 	{ CPU_ID_THUNDERXRX, "ThunderX", "Cavium", "v8-A" },
240 	{ CPU_ID_THUNDERX81XXRX, "ThunderX CN81XX", "Cavium", "v8-A" },
241 	{ CPU_ID_THUNDERX83XXRX, "ThunderX CN83XX", "Cavium", "v8-A" },
242 	{ CPU_ID_THUNDERX2RX, "ThunderX2", "Marvell", "v8.1-A" },
243 	{ CPU_ID_APPLE_M1_ICESTORM & CPU_PARTMASK, "M1 Icestorm", "Apple", "Apple Silicon" },
244 	{ CPU_ID_APPLE_M1_FIRESTORM & CPU_PARTMASK, "M1 Firestorm", "Apple", "Apple Silicon" },
245 	{ CPU_ID_AMPERE1 & CPU_PARTMASK, "Ampere-1", "Ampere", "v8.6-A+" },
246 	{ CPU_ID_AMPERE1A & CPU_PARTMASK, "Ampere-1A", "Ampere", "v8.6-A+" },
247 	{ CPU_ID_A64FX & CPU_PARTMASK, "A64FX", "Fujitsu", "v8.2-A+" },
248 	{ CPU_ID_ORYON & CPU_PARTMASK, "Oryon", "Qualcomm", "v8.7-A+" },
249 };
250 
251 static void
252 identify_aarch64_model(uint32_t cpuid, char *buf, size_t len)
253 {
254 	int i;
255 	uint32_t cpupart, variant, revision;
256 
257 	cpupart = cpuid & CPU_PARTMASK;
258 	variant = __SHIFTOUT(cpuid, CPU_ID_VARIANT_MASK);
259 	revision = __SHIFTOUT(cpuid, CPU_ID_REVISION_MASK);
260 
261 	for (i = 0; i < __arraycount(cpuids); i++) {
262 		if (cpupart == cpuids[i].cpu_partnum) {
263 			snprintf(buf, len, "%s %s r%dp%d (%s)",
264 			    cpuids[i].cpu_vendor, cpuids[i].cpu_name,
265 			    variant, revision,
266 			    cpuids[i].cpu_architecture);
267 			return;
268 		}
269 	}
270 
271 	snprintf(buf, len, "unknown CPU (ID = 0x%08x)", cpuid);
272 }
273 
274 static void
275 cpu_identify(device_t self, struct cpu_info *ci)
276 {
277 	char model[128];
278 	const char *m;
279 
280 	identify_aarch64_model(ci->ci_id.ac_midr, model, sizeof(model));
281 
282 	aprint_naive("\n");
283 	aprint_normal(": %s, id 0x%lx\n", model, ci->ci_cpuid);
284 	aprint_normal_dev(ci->ci_dev, "package %u, core %u, smt %u, numa %u\n",
285 	    ci->ci_package_id, ci->ci_core_id, ci->ci_smt_id, ci->ci_numa_id);
286 
287 	if (ci->ci_index == 0) {
288 		m = cpu_getmodel();
289 		if (m == NULL || *m == 0)
290 			cpu_setmodel("%s", model);
291 
292 		if (CPU_ID_ERRATA_CAVIUM_THUNDERX_1_1_P(ci->ci_id.ac_midr))
293 			aprint_normal("WARNING: ThunderX Pass 1.1 detected.\n"
294 			    "This has known hardware bugs that may cause the "
295 			    "incorrect operation of atomic operations.\n");
296 	}
297 }
298 
299 static void
300 cpu_identify1(device_t self, struct cpu_info *ci)
301 {
302 	struct aarch64_sysctl_cpu_id *id = &ci->ci_id;
303 	uint64_t sctlr = ci->ci_sctlr_el1;
304 
305 	if (sctlr & SCTLR_I)
306 		aprint_verbose_dev(self, "IC enabled");
307 	else
308 		aprint_verbose_dev(self, "IC disabled");
309 
310 	if (sctlr & SCTLR_C)
311 		aprint_verbose(", DC enabled");
312 	else
313 		aprint_verbose(", DC disabled");
314 
315 	if (sctlr & SCTLR_A)
316 		aprint_verbose(", Alignment check enabled\n");
317 	else {
318 		switch (sctlr & (SCTLR_SA | SCTLR_SA0)) {
319 		case SCTLR_SA | SCTLR_SA0:
320 			aprint_verbose(
321 			    ", EL0/EL1 stack Alignment check enabled\n");
322 			break;
323 		case SCTLR_SA:
324 			aprint_verbose(", EL1 stack Alignment check enabled\n");
325 			break;
326 		case SCTLR_SA0:
327 			aprint_verbose(", EL0 stack Alignment check enabled\n");
328 			break;
329 		case 0:
330 			aprint_verbose(", Alignment check disabled\n");
331 			break;
332 		}
333 	}
334 
335 	/*
336 	 * CTR - Cache Type Register
337 	 */
338 	const uint64_t ctr = id->ac_ctr;
339 	const uint64_t clidr = id->ac_clidr;
340 	aprint_verbose_dev(self, "Cache Writeback Granule %" PRIu64 "B,"
341 	    " Exclusives Reservation Granule %" PRIu64 "B\n",
342 	    __SHIFTOUT(ctr, CTR_EL0_CWG_LINE) * 4,
343 	    __SHIFTOUT(ctr, CTR_EL0_ERG_LINE) * 4);
344 
345 	aprint_verbose_dev(self, "Dcache line %ld, Icache line %ld"
346 	    ", DIC=%lu, IDC=%lu, LoUU=%lu, LoC=%lu, LoUIS=%lu\n",
347 	    sizeof(int) << __SHIFTOUT(ctr, CTR_EL0_DMIN_LINE),
348 	    sizeof(int) << __SHIFTOUT(ctr, CTR_EL0_IMIN_LINE),
349 	    __SHIFTOUT(ctr, CTR_EL0_DIC),
350 	    __SHIFTOUT(ctr, CTR_EL0_IDC),
351 	    __SHIFTOUT(clidr, CLIDR_LOUU),
352 	    __SHIFTOUT(clidr, CLIDR_LOC),
353 	    __SHIFTOUT(clidr, CLIDR_LOUIS));
354 }
355 
356 
357 /*
358  * identify vfp, etc.
359  */
360 static void
361 cpu_identify2(device_t self, struct cpu_info *ci)
362 {
363 	struct aarch64_sysctl_cpu_id * const id = &ci->ci_id;
364 
365 	aprint_debug_dev(self, "midr=0x%" PRIx64 " mpidr=0x%" PRIx64 "\n",
366 	    id->ac_midr, id->ac_mpidr);
367 	aprint_verbose_dev(self, "revID=0x%" PRIx64, id->ac_revidr);
368 
369 	/* ID_AA64DFR0_EL1 */
370 	switch (__SHIFTOUT(id->ac_aa64dfr0, ID_AA64DFR0_EL1_PMUVER)) {
371 	case ID_AA64DFR0_EL1_PMUVER_V3:
372 		aprint_verbose(", PMCv3");
373 		break;
374 	case ID_AA64DFR0_EL1_PMUVER_NOV3:
375 		aprint_verbose(", PMC");
376 		break;
377 	}
378 
379 	/* ID_AA64MMFR0_EL1 */
380 	switch (__SHIFTOUT(id->ac_aa64mmfr0, ID_AA64MMFR0_EL1_TGRAN4)) {
381 	case ID_AA64MMFR0_EL1_TGRAN4_4KB:
382 		aprint_verbose(", 4k table");
383 		break;
384 	}
385 	switch (__SHIFTOUT(id->ac_aa64mmfr0, ID_AA64MMFR0_EL1_TGRAN16)) {
386 	case ID_AA64MMFR0_EL1_TGRAN16_16KB:
387 		aprint_verbose(", 16k table");
388 		break;
389 	}
390 	switch (__SHIFTOUT(id->ac_aa64mmfr0, ID_AA64MMFR0_EL1_TGRAN64)) {
391 	case ID_AA64MMFR0_EL1_TGRAN64_64KB:
392 		aprint_verbose(", 64k table");
393 		break;
394 	}
395 
396 	switch (__SHIFTOUT(id->ac_aa64mmfr0, ID_AA64MMFR0_EL1_ASIDBITS)) {
397 	case ID_AA64MMFR0_EL1_ASIDBITS_8BIT:
398 		aprint_verbose(", 8bit ASID");
399 		break;
400 	case ID_AA64MMFR0_EL1_ASIDBITS_16BIT:
401 		aprint_verbose(", 16bit ASID");
402 		break;
403 	}
404 	aprint_verbose("\n");
405 
406 	aprint_verbose_dev(self, "auxID=0x%" PRIx64, ci->ci_id.ac_aa64isar0);
407 
408 	/* PFR0 */
409 	switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_CSV3)) {
410 	case ID_AA64PFR0_EL1_CSV3_IMPL:
411 		aprint_verbose(", CSV3");
412 		break;
413 	}
414 	switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_CSV2)) {
415 	case ID_AA64PFR0_EL1_CSV2_IMPL:
416 		aprint_verbose(", CSV2");
417 		break;
418 	}
419 	switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_GIC)) {
420 	case ID_AA64PFR0_EL1_GIC_CPUIF_EN:
421 		aprint_verbose(", GICv3");
422 		break;
423 	}
424 	switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_FP)) {
425 	case ID_AA64PFR0_EL1_FP_NONE:
426 		break;
427 	default:
428 		aprint_verbose(", FP");
429 		break;
430 	}
431 
432 	/* ISAR0 */
433 	switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_CRC32)) {
434 	case ID_AA64ISAR0_EL1_CRC32_CRC32X:
435 		aprint_verbose(", CRC32");
436 		break;
437 	}
438 	switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_SHA1)) {
439 	case ID_AA64ISAR0_EL1_SHA1_SHA1CPMHSU:
440 		aprint_verbose(", SHA1");
441 		break;
442 	}
443 	switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_SHA2)) {
444 	case ID_AA64ISAR0_EL1_SHA2_SHA256HSU:
445 		aprint_verbose(", SHA256");
446 		break;
447 	}
448 	switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_AES)) {
449 	case ID_AA64ISAR0_EL1_AES_AES:
450 		aprint_verbose(", AES");
451 		break;
452 	case ID_AA64ISAR0_EL1_AES_PMUL:
453 		aprint_verbose(", AES+PMULL");
454 		break;
455 	}
456 	switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_RNDR)) {
457 	case ID_AA64ISAR0_EL1_RNDR_RNDRRS:
458 		aprint_verbose(", RNDRRS");
459 		break;
460 	}
461 
462 	/* PFR0:DIT -- data-independent timing support */
463 	switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_DIT)) {
464 	case ID_AA64PFR0_EL1_DIT_IMPL:
465 		aprint_verbose(", DIT");
466 		break;
467 	}
468 
469 	/* PFR0:AdvSIMD */
470 	switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_ADVSIMD)) {
471 	case ID_AA64PFR0_EL1_ADV_SIMD_NONE:
472 		break;
473 	default:
474 		aprint_verbose(", NEON");
475 		break;
476 	}
477 
478 	/* MVFR0/MVFR1 */
479 	switch (__SHIFTOUT(id->ac_mvfr0, MVFR0_FPROUND)) {
480 	case MVFR0_FPROUND_ALL:
481 		aprint_verbose(", rounding");
482 		break;
483 	}
484 	switch (__SHIFTOUT(id->ac_mvfr0, MVFR0_FPTRAP)) {
485 	case MVFR0_FPTRAP_TRAP:
486 		aprint_verbose(", exceptions");
487 		break;
488 	}
489 	switch (__SHIFTOUT(id->ac_mvfr1, MVFR1_FPDNAN)) {
490 	case MVFR1_FPDNAN_NAN:
491 		aprint_verbose(", NaN propagation");
492 		break;
493 	}
494 	switch (__SHIFTOUT(id->ac_mvfr1, MVFR1_FPFTZ)) {
495 	case MVFR1_FPFTZ_DENORMAL:
496 		aprint_verbose(", denormals");
497 		break;
498 	}
499 	switch (__SHIFTOUT(id->ac_mvfr0, MVFR0_SIMDREG)) {
500 	case MVFR0_SIMDREG_16x64:
501 		aprint_verbose(", 16x64bitRegs");
502 		break;
503 	case MVFR0_SIMDREG_32x64:
504 		aprint_verbose(", 32x64bitRegs");
505 		break;
506 	}
507 	switch (__SHIFTOUT(id->ac_mvfr1, MVFR1_SIMDFMAC)) {
508 	case MVFR1_SIMDFMAC_FMAC:
509 		aprint_verbose(", Fused Multiply-Add");
510 		break;
511 	}
512 
513 	aprint_verbose("\n");
514 }
515 
516 /*
517  * Enable the performance counter, then estimate frequency for
518  * the current PE and store the result in cpu_cc_freq.
519  */
520 static void
521 cpu_init_counter(struct cpu_info *ci)
522 {
523 	const uint64_t dfr0 = reg_id_aa64dfr0_el1_read();
524 	const u_int pmuver = __SHIFTOUT(dfr0, ID_AA64DFR0_EL1_PMUVER);
525 	if (pmuver == ID_AA64DFR0_EL1_PMUVER_NONE) {
526 		/* Performance Monitors Extension not implemented. */
527 		return;
528 	}
529 	if (pmuver == ID_AA64DFR0_EL1_PMUVER_IMPL) {
530 		/* Non-standard Performance Monitors are not supported. */
531 		return;
532 	}
533 
534 	reg_pmcr_el0_write(PMCR_E | PMCR_C | PMCR_LC);
535 	reg_pmintenclr_el1_write(PMINTEN_C | PMINTEN_P);
536 	reg_pmcntenset_el0_write(PMCNTEN_C);
537 
538 	const uint32_t prev = cpu_counter32();
539 	delay(100000);
540 	ci->ci_data.cpu_cc_freq = (cpu_counter32() - prev) * 10;
541 }
542 
543 /*
544  * Fill in this CPUs id data.  Must be called on all cpus.
545  */
546 void __noasan
547 cpu_setup_id(struct cpu_info *ci)
548 {
549 	struct aarch64_sysctl_cpu_id *id = &ci->ci_id;
550 
551 	/* SCTLR - System Control Register */
552 	ci->ci_sctlr_el1 = reg_sctlr_el1_read();
553 
554 	memset(id, 0, sizeof *id);
555 
556 	id->ac_midr      = reg_midr_el1_read();
557 	id->ac_revidr    = reg_revidr_el1_read();
558 	id->ac_mpidr     = reg_mpidr_el1_read();
559 
560 	id->ac_aa64dfr0  = reg_id_aa64dfr0_el1_read();
561 	id->ac_aa64dfr1  = reg_id_aa64dfr1_el1_read();
562 
563 	id->ac_aa64isar0 = reg_id_aa64isar0_el1_read();
564 	id->ac_aa64isar1 = reg_id_aa64isar1_el1_read();
565 
566 	id->ac_aa64mmfr0 = reg_id_aa64mmfr0_el1_read();
567 	id->ac_aa64mmfr1 = reg_id_aa64mmfr1_el1_read();
568 	id->ac_aa64mmfr2 = reg_id_aa64mmfr2_el1_read();
569 
570 	id->ac_mvfr0     = reg_mvfr0_el1_read();
571 	id->ac_mvfr1     = reg_mvfr1_el1_read();
572 	id->ac_mvfr2     = reg_mvfr2_el1_read();
573 
574 	id->ac_clidr     = reg_clidr_el1_read();
575 	id->ac_ctr       = reg_ctr_el0_read();
576 
577 	/* Only in ARMv8.2. */
578 	id->ac_aa64zfr0  = 0 /* reg_id_aa64zfr0_el1_read() */;
579 
580 	id->ac_aa64pfr0  = reg_id_aa64pfr0_el1_read();
581 	id->ac_aa64pfr1  = reg_id_aa64pfr1_el1_read();
582 }
583 
584 /*
585  * setup the per-cpu sysctl tree.
586  */
587 static void
588 cpu_setup_sysctl(device_t dv, struct cpu_info *ci)
589 {
590 	const struct sysctlnode *cpunode = NULL;
591 
592 	sysctl_createv(NULL, 0, NULL, &cpunode,
593 		       CTLFLAG_PERMANENT,
594 		       CTLTYPE_NODE, device_xname(dv), NULL,
595 		       NULL, 0, NULL, 0,
596 		       CTL_MACHDEP,
597 		       CTL_CREATE, CTL_EOL);
598 
599 	if (cpunode == NULL)
600 		return;
601 
602 	sysctl_createv(NULL, 0, &cpunode, NULL,
603 		       CTLFLAG_PERMANENT,
604 		       CTLTYPE_STRUCT, "cpu_id", NULL,
605 		       NULL, 0, &ci->ci_id, sizeof(ci->ci_id),
606 		       CTL_CREATE, CTL_EOL);
607 }
608 
609 static struct krndsource rndrrs_source;
610 
611 static void
612 rndrrs_get(size_t nbytes, void *cookie)
613 {
614 	/* Entropy bits per data byte, wild-arse guess.  */
615 	const unsigned bpb = 4;
616 	size_t nbits = nbytes*NBBY;
617 	uint64_t x;
618 	int error, bound;
619 
620 	bound = curlwp_bind();	/* bind to CPU for rndrrs_fail evcnt */
621 	while (nbits) {
622 		/*
623 		 * x := random 64-bit sample
624 		 * error := Z bit, set to 1 if sample is bad
625 		 *
626 		 * XXX This should be done by marking the function
627 		 * __attribute__((target("arch=armv8.5-a+rng"))) and
628 		 * using `mrs %0, rndrrs', but:
629 		 *
630 		 * (a) the version of gcc we use doesn't support that,
631 		 * and
632 		 * (b) clang doesn't seem to like `rndrrs' itself.
633 		 *
634 		 * So we use the numeric encoding for now.
635 		 */
636 		__asm __volatile(""
637 		    "mrs	%0, s3_3_c2_c4_1\n"
638 		    "cset	%w1, eq"
639 		    : "=r"(x), "=r"(error));
640 		if (error) {
641 			DTRACE_PROBE(rndrrs_fail);
642 			curcpu()->ci_rndrrs_fail.ev_count++;
643 			break;
644 		}
645 		rnd_add_data_sync(&rndrrs_source, &x, sizeof(x),
646 		    bpb*sizeof(x));
647 		nbits -= MIN(nbits, bpb*sizeof(x));
648 	}
649 	curlwp_bindx(bound);
650 
651 	explicit_memset(&x, 0, sizeof x);
652 }
653 
654 /*
655  * setup the RNDRRS entropy source
656  */
657 static void
658 cpu_setup_rng(device_t dv, struct cpu_info *ci)
659 {
660 	struct aarch64_sysctl_cpu_id *id = &ci->ci_id;
661 
662 	/* Verify that it is supported.  */
663 	switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_RNDR)) {
664 	case ID_AA64ISAR0_EL1_RNDR_RNDRRS:
665 		break;
666 	default:
667 		return;
668 	}
669 
670 	/* Attach event counter for RNDRRS failure.  */
671 	evcnt_attach_dynamic(&ci->ci_rndrrs_fail, EVCNT_TYPE_MISC, NULL,
672 	    ci->ci_cpuname, "rndrrs fail");
673 
674 	/*
675 	 * On the primary CPU, attach random source -- this only
676 	 * happens once globally.
677 	 */
678 	if (!CPU_IS_PRIMARY(ci))
679 		return;
680 	rndsource_setcb(&rndrrs_source, rndrrs_get, NULL);
681 	rnd_attach_source(&rndrrs_source, "rndrrs", RND_TYPE_RNG,
682 	    RND_FLAG_DEFAULT|RND_FLAG_HASCB);
683 }
684 
685 /*
686  * setup the AES implementation
687  */
688 static void
689 cpu_setup_aes(device_t dv, struct cpu_info *ci)
690 {
691 	struct aarch64_sysctl_cpu_id *id = &ci->ci_id;
692 
693 	/* Check for ARMv8.0-AES support.  */
694 	switch (__SHIFTOUT(id->ac_aa64isar0, ID_AA64ISAR0_EL1_AES)) {
695 	case ID_AA64ISAR0_EL1_AES_AES:
696 	case ID_AA64ISAR0_EL1_AES_PMUL:
697 		aes_md_init(&aes_armv8_impl);
698 		return;
699 	default:
700 		break;
701 	}
702 
703 	/* Failing that, check for SIMD support.  */
704 	switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_ADVSIMD)) {
705 	case ID_AA64PFR0_EL1_ADV_SIMD_IMPL:
706 		aes_md_init(&aes_neon_impl);
707 		return;
708 	default:
709 		break;
710 	}
711 }
712 
713 /*
714  * setup the ChaCha implementation
715  */
716 static void
717 cpu_setup_chacha(device_t dv, struct cpu_info *ci)
718 {
719 	struct aarch64_sysctl_cpu_id *id = &ci->ci_id;
720 
721 	/* Check for SIMD support.  */
722 	switch (__SHIFTOUT(id->ac_aa64pfr0, ID_AA64PFR0_EL1_ADVSIMD)) {
723 	case ID_AA64PFR0_EL1_ADV_SIMD_IMPL:
724 		chacha_md_init(&chacha_neon_impl);
725 		return;
726 	default:
727 		break;
728 	}
729 }
730 
731 #ifdef MULTIPROCESSOR
732 /*
733  * Initialise a secondary processor.
734  *
735  * printf isn't available as kmutex(9) relies on curcpu which isn't setup yet.
736  *
737  */
738 void __noasan
739 cpu_init_secondary_processor(int cpuindex)
740 {
741 	struct cpu_info * ci = &cpu_info_store[cpuindex];
742 	struct aarch64_sysctl_cpu_id *id = &ci->ci_id;
743 
744 	aarch64_setcpufuncs(ci);
745 
746 	/* Sets ci->ci_{sctlr,midr,mpidr}, etc */
747 	cpu_setup_id(ci);
748 
749 	arm_cpu_topology_set(ci, id->ac_mpidr);
750 	aarch64_getcacheinfo(ci);
751 
752 	cpu_set_hatched(cpuindex);
753 
754 	/*
755 	 * return to assembly to wait for cpu_boot_secondary_processors
756 	 */
757 }
758 
759 
760 /*
761  * When we are called, the MMU and caches are on and we are running on the stack
762  * of the idlelwp for this cpu.
763  */
764 void
765 cpu_hatch(struct cpu_info *ci)
766 {
767 	KASSERT(curcpu() == ci);
768 	KASSERT((reg_tcr_el1_read() & TCR_EPD0) != 0);
769 
770 #ifdef DDB
771 	db_machdep_cpu_init();
772 #endif
773 
774 	cpu_init_counter(ci);
775 
776 	intr_cpu_init(ci);
777 
778 #ifdef FDT
779 	arm_fdt_cpu_hatch(ci);
780 #endif
781 
782 	/*
783 	 * clear my bit of arm_cpu_mbox to tell cpu_boot_secondary_processors().
784 	 * there are cpu0,1,2,3, and if cpu2 is unresponsive,
785 	 * ci_index are each cpu0=0, cpu1=1, cpu2=undef, cpu3=2.
786 	 * therefore we have to use device_unit instead of ci_index for mbox.
787 	 */
788 
789 	cpu_clr_mbox(device_unit(ci->ci_dev));
790 }
791 #endif /* MULTIPROCESSOR */
792