xref: /openbsd-src/sys/arch/arm64/arm64/cpu.c (revision 6f6231dc4f0a07fe7b8a4824090617757514f823)
1 /*	$OpenBSD: cpu.c,v 1.135 2025/01/25 12:29:35 kettenis Exp $	*/
2 
3 /*
4  * Copyright (c) 2016 Dale Rahn <drahn@dalerahn.com>
5  * Copyright (c) 2017 Mark Kettenis <kettenis@openbsd.org>
6  *
7  * Permission to use, copy, modify, and distribute this software for any
8  * purpose with or without fee is hereby granted, provided that the above
9  * copyright notice and this permission notice appear in all copies.
10  *
11  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
12  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
13  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
14  * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
15  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
16  * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
17  * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
18  */
19 
20 #include "kstat.h"
21 
22 #include <sys/param.h>
23 #include <sys/systm.h>
24 #include <sys/proc.h>
25 #include <sys/malloc.h>
26 #include <sys/device.h>
27 #include <sys/sysctl.h>
28 #include <sys/task.h>
29 #include <sys/user.h>
30 #include <sys/kstat.h>
31 
32 #include <uvm/uvm_extern.h>
33 
34 #include <machine/fdt.h>
35 #include <machine/elf.h>
36 
37 #include <dev/ofw/openfirm.h>
38 #include <dev/ofw/ofw_clock.h>
39 #include <dev/ofw/ofw_regulator.h>
40 #include <dev/ofw/ofw_thermal.h>
41 #include <dev/ofw/fdt.h>
42 
43 #include <machine/cpufunc.h>
44 
45 #include "psci.h"
46 #if NPSCI > 0
47 #include <dev/fdt/pscivar.h>
48 #endif
49 
50 /* CPU Identification */
51 #define CPU_IMPL_ARM		0x41
52 #define CPU_IMPL_CAVIUM		0x43
53 #define CPU_IMPL_AMCC		0x50
54 #define CPU_IMPL_QCOM		0x51
55 #define CPU_IMPL_APPLE		0x61
56 #define CPU_IMPL_AMPERE		0xc0
57 
58 /* ARM */
59 #define CPU_PART_CORTEX_A34	0xd02
60 #define CPU_PART_CORTEX_A53	0xd03
61 #define CPU_PART_CORTEX_A35	0xd04
62 #define CPU_PART_CORTEX_A55	0xd05
63 #define CPU_PART_CORTEX_A65	0xd06
64 #define CPU_PART_CORTEX_A57	0xd07
65 #define CPU_PART_CORTEX_A72	0xd08
66 #define CPU_PART_CORTEX_A73	0xd09
67 #define CPU_PART_CORTEX_A75	0xd0a
68 #define CPU_PART_CORTEX_A76	0xd0b
69 #define CPU_PART_NEOVERSE_N1	0xd0c
70 #define CPU_PART_CORTEX_A77	0xd0d
71 #define CPU_PART_CORTEX_A76AE	0xd0e
72 #define CPU_PART_NEOVERSE_V1	0xd40
73 #define CPU_PART_CORTEX_A78	0xd41
74 #define CPU_PART_CORTEX_A78AE	0xd42
75 #define CPU_PART_CORTEX_A65AE	0xd43
76 #define CPU_PART_CORTEX_X1	0xd44
77 #define CPU_PART_CORTEX_A510	0xd46
78 #define CPU_PART_CORTEX_A710	0xd47
79 #define CPU_PART_CORTEX_X2	0xd48
80 #define CPU_PART_NEOVERSE_N2	0xd49
81 #define CPU_PART_NEOVERSE_E1	0xd4a
82 #define CPU_PART_CORTEX_A78C	0xd4b
83 #define CPU_PART_CORTEX_X1C	0xd4c
84 #define CPU_PART_CORTEX_A715	0xd4d
85 #define CPU_PART_CORTEX_X3	0xd4e
86 #define CPU_PART_NEOVERSE_V2	0xd4f
87 #define CPU_PART_CORTEX_A520	0xd80
88 #define CPU_PART_CORTEX_A720	0xd81
89 #define CPU_PART_CORTEX_X4	0xd82
90 #define CPU_PART_NEOVERSE_V3AE	0xd83
91 #define CPU_PART_NEOVERSE_V3	0xd84
92 #define CPU_PART_CORTEX_X925	0xd85
93 #define CPU_PART_CORTEX_A725	0xd87
94 #define CPU_PART_CORTEX_A520AE	0xd88
95 #define CPU_PART_CORTEX_A720AE	0xd89
96 #define CPU_PART_NEOVERSE_N3	0xd8e
97 
98 /* Cavium */
99 #define CPU_PART_THUNDERX_T88	0x0a1
100 #define CPU_PART_THUNDERX_T81	0x0a2
101 #define CPU_PART_THUNDERX_T83	0x0a3
102 #define CPU_PART_THUNDERX2_T99	0x0af
103 
104 /* Applied Micro */
105 #define CPU_PART_X_GENE		0x000
106 
107 /* Qualcomm */
108 #define CPU_PART_ORYON		0x001
109 #define CPU_PART_KRYO400_GOLD	0x804
110 #define CPU_PART_KRYO400_SILVER	0x805
111 
112 /* Apple */
113 #define CPU_PART_ICESTORM	0x022
114 #define CPU_PART_FIRESTORM	0x023
115 #define CPU_PART_ICESTORM_PRO	0x024
116 #define CPU_PART_FIRESTORM_PRO	0x025
117 #define CPU_PART_ICESTORM_MAX	0x028
118 #define CPU_PART_FIRESTORM_MAX	0x029
119 #define CPU_PART_BLIZZARD	0x032
120 #define CPU_PART_AVALANCHE	0x033
121 #define CPU_PART_BLIZZARD_PRO	0x034
122 #define CPU_PART_AVALANCHE_PRO	0x035
123 #define CPU_PART_BLIZZARD_MAX	0x038
124 #define CPU_PART_AVALANCHE_MAX	0x039
125 
126 /* Ampere */
127 #define CPU_PART_AMPERE1	0xac3
128 
129 #define CPU_IMPL(midr)  (((midr) >> 24) & 0xff)
130 #define CPU_PART(midr)  (((midr) >> 4) & 0xfff)
131 #define CPU_VAR(midr)   (((midr) >> 20) & 0xf)
132 #define CPU_REV(midr)   (((midr) >> 0) & 0xf)
133 
134 struct cpu_cores {
135 	int	id;
136 	char	*name;
137 };
138 
139 struct cpu_cores cpu_cores_none[] = {
140 	{ 0, NULL },
141 };
142 
143 struct cpu_cores cpu_cores_arm[] = {
144 	{ CPU_PART_CORTEX_A34, "Cortex-A34" },
145 	{ CPU_PART_CORTEX_A35, "Cortex-A35" },
146 	{ CPU_PART_CORTEX_A53, "Cortex-A53" },
147 	{ CPU_PART_CORTEX_A55, "Cortex-A55" },
148 	{ CPU_PART_CORTEX_A57, "Cortex-A57" },
149 	{ CPU_PART_CORTEX_A65, "Cortex-A65" },
150 	{ CPU_PART_CORTEX_A65AE, "Cortex-A65AE" },
151 	{ CPU_PART_CORTEX_A72, "Cortex-A72" },
152 	{ CPU_PART_CORTEX_A73, "Cortex-A73" },
153 	{ CPU_PART_CORTEX_A75, "Cortex-A75" },
154 	{ CPU_PART_CORTEX_A76, "Cortex-A76" },
155 	{ CPU_PART_CORTEX_A76AE, "Cortex-A76AE" },
156 	{ CPU_PART_CORTEX_A77, "Cortex-A77" },
157 	{ CPU_PART_CORTEX_A78, "Cortex-A78" },
158 	{ CPU_PART_CORTEX_A78AE, "Cortex-A78AE" },
159 	{ CPU_PART_CORTEX_A78C, "Cortex-A78C" },
160 	{ CPU_PART_CORTEX_A510, "Cortex-A510" },
161 	{ CPU_PART_CORTEX_A520, "Cortex-A520" },
162 	{ CPU_PART_CORTEX_A520AE, "Cortex-A520AE" },
163 	{ CPU_PART_CORTEX_A710, "Cortex-A710" },
164 	{ CPU_PART_CORTEX_A715, "Cortex-A715" },
165 	{ CPU_PART_CORTEX_A720, "Cortex-A720" },
166 	{ CPU_PART_CORTEX_A720AE, "Cortex-A720AE" },
167 	{ CPU_PART_CORTEX_A725, "Cortex-A725" },
168 	{ CPU_PART_CORTEX_X1, "Cortex-X1" },
169 	{ CPU_PART_CORTEX_X1C, "Cortex-X1C" },
170 	{ CPU_PART_CORTEX_X2, "Cortex-X2" },
171 	{ CPU_PART_CORTEX_X3, "Cortex-X3" },
172 	{ CPU_PART_CORTEX_X4, "Cortex-X4" },
173 	{ CPU_PART_CORTEX_X925, "Cortex-X925" },
174 	{ CPU_PART_NEOVERSE_E1, "Neoverse E1" },
175 	{ CPU_PART_NEOVERSE_N1, "Neoverse N1" },
176 	{ CPU_PART_NEOVERSE_N2, "Neoverse N2" },
177 	{ CPU_PART_NEOVERSE_N3, "Neoverse N3" },
178 	{ CPU_PART_NEOVERSE_V1, "Neoverse V1" },
179 	{ CPU_PART_NEOVERSE_V2, "Neoverse V2" },
180 	{ CPU_PART_NEOVERSE_V3, "Neoverse V3" },
181 	{ CPU_PART_NEOVERSE_V3AE, "Neoverse V3AE" },
182 	{ 0, NULL },
183 };
184 
185 struct cpu_cores cpu_cores_cavium[] = {
186 	{ CPU_PART_THUNDERX_T88, "ThunderX T88" },
187 	{ CPU_PART_THUNDERX_T81, "ThunderX T81" },
188 	{ CPU_PART_THUNDERX_T83, "ThunderX T83" },
189 	{ CPU_PART_THUNDERX2_T99, "ThunderX2 T99" },
190 	{ 0, NULL },
191 };
192 
193 struct cpu_cores cpu_cores_amcc[] = {
194 	{ CPU_PART_X_GENE, "X-Gene" },
195 	{ 0, NULL },
196 };
197 
198 struct cpu_cores cpu_cores_qcom[] = {
199 	{ CPU_PART_KRYO400_GOLD, "Kryo 400 Gold" },
200 	{ CPU_PART_KRYO400_SILVER, "Kryo 400 Silver" },
201 	{ CPU_PART_ORYON, "Oryon" },
202 	{ 0, NULL },
203 };
204 
205 struct cpu_cores cpu_cores_apple[] = {
206 	{ CPU_PART_ICESTORM, "Icestorm" },
207 	{ CPU_PART_FIRESTORM, "Firestorm" },
208 	{ CPU_PART_ICESTORM_PRO, "Icestorm Pro" },
209 	{ CPU_PART_FIRESTORM_PRO, "Firestorm Pro" },
210 	{ CPU_PART_ICESTORM_MAX, "Icestorm Max" },
211 	{ CPU_PART_FIRESTORM_MAX, "Firestorm Max" },
212 	{ CPU_PART_BLIZZARD, "Blizzard" },
213 	{ CPU_PART_AVALANCHE, "Avalanche" },
214 	{ CPU_PART_BLIZZARD_PRO, "Blizzard Pro" },
215 	{ CPU_PART_AVALANCHE_PRO, "Avalanche Pro" },
216 	{ CPU_PART_BLIZZARD_MAX, "Blizzard Max" },
217 	{ CPU_PART_AVALANCHE_MAX, "Avalanche Max" },
218 	{ 0, NULL },
219 };
220 
221 struct cpu_cores cpu_cores_ampere[] = {
222 	{ CPU_PART_AMPERE1, "AmpereOne" },
223 	{ 0, NULL },
224 };
225 
226 /* arm cores makers */
227 const struct implementers {
228 	int			id;
229 	char			*name;
230 	struct cpu_cores	*corelist;
231 } cpu_implementers[] = {
232 	{ CPU_IMPL_ARM,	"ARM", cpu_cores_arm },
233 	{ CPU_IMPL_CAVIUM, "Cavium", cpu_cores_cavium },
234 	{ CPU_IMPL_AMCC, "Applied Micro", cpu_cores_amcc },
235 	{ CPU_IMPL_QCOM, "Qualcomm", cpu_cores_qcom },
236 	{ CPU_IMPL_APPLE, "Apple", cpu_cores_apple },
237 	{ CPU_IMPL_AMPERE, "Ampere", cpu_cores_ampere },
238 	{ 0, NULL },
239 };
240 
241 char cpu_model[64];
242 int cpu_node;
243 
244 uint64_t cpu_id_aa64isar0;
245 uint64_t cpu_id_aa64isar1;
246 uint64_t cpu_id_aa64isar2;
247 uint64_t cpu_id_aa64mmfr0;
248 uint64_t cpu_id_aa64mmfr1;
249 uint64_t cpu_id_aa64mmfr2;
250 uint64_t cpu_id_aa64pfr0;
251 uint64_t cpu_id_aa64pfr1;
252 
253 int arm64_has_lse;
254 int arm64_has_rng;
255 #ifdef CRYPTO
256 int arm64_has_aes;
257 #endif
258 
259 extern char trampoline_vectors_none[];
260 extern char trampoline_vectors_loop_8[];
261 extern char trampoline_vectors_loop_11[];
262 extern char trampoline_vectors_loop_24[];
263 extern char trampoline_vectors_loop_32[];
264 #if NPSCI > 0
265 extern char trampoline_vectors_psci_hvc[];
266 extern char trampoline_vectors_psci_smc[];
267 #endif
268 extern char trampoline_vectors_clrbhb[];
269 
270 struct cpu_info *cpu_info_list = &cpu_info_primary;
271 
272 int	cpu_match(struct device *, void *, void *);
273 void	cpu_attach(struct device *, struct device *, void *);
274 
275 const struct cfattach cpu_ca = {
276 	sizeof(struct device), cpu_match, cpu_attach
277 };
278 
279 struct cfdriver cpu_cd = {
280 	NULL, "cpu", DV_DULL
281 };
282 
283 struct timeout cpu_rng_to;
284 void	cpu_rng(void *);
285 
286 void	cpu_opp_init(struct cpu_info *, uint32_t);
287 void	cpu_psci_init(struct cpu_info *);
288 void	cpu_psci_idle_cycle(void);
289 
290 void	cpu_flush_bp_noop(void);
291 void	cpu_flush_bp_psci(void);
292 void	cpu_serror_apple(void);
293 
294 #if NKSTAT > 0
295 void	cpu_kstat_attach(struct cpu_info *ci);
296 void	cpu_opp_kstat_attach(struct cpu_info *ci);
297 #endif
298 
299 void
300 cpu_rng(void *arg)
301 {
302 	struct timeout *to = arg;
303 	uint64_t rndr;
304 	int ret;
305 
306 	ret = __builtin_arm_rndrrs(&rndr);
307 	if (ret)
308 		ret = __builtin_arm_rndr(&rndr);
309 	if (ret == 0) {
310 		enqueue_randomness(rndr & 0xffffffff);
311 		enqueue_randomness(rndr >> 32);
312 	}
313 
314 	if (to)
315 		timeout_add_msec(to, 1000);
316 }
317 
318 /*
319  * Enable mitigation for Spectre-V2 branch target injection
320  * vulnerabilities (CVE-2017-5715).
321  */
322 void
323 cpu_mitigate_spectre_v2(struct cpu_info *ci)
324 {
325 	uint64_t id;
326 
327 	/*
328 	 * By default we let the firmware decide what mitigation is
329 	 * necessary.
330 	 */
331 	ci->ci_flush_bp = cpu_flush_bp_psci;
332 
333 	/* Some specific CPUs are known not to be vulnerable. */
334 	switch (CPU_IMPL(ci->ci_midr)) {
335 	case CPU_IMPL_ARM:
336 		switch (CPU_PART(ci->ci_midr)) {
337 		case CPU_PART_CORTEX_A35:
338 		case CPU_PART_CORTEX_A53:
339 		case CPU_PART_CORTEX_A55:
340 			/* Not vulnerable. */
341 			ci->ci_flush_bp = cpu_flush_bp_noop;
342 			break;
343 		}
344 		break;
345 	case CPU_IMPL_QCOM:
346 		switch (CPU_PART(ci->ci_midr)) {
347 		case CPU_PART_KRYO400_SILVER:
348 			/* Not vulnerable. */
349 			ci->ci_flush_bp = cpu_flush_bp_noop;
350 			break;
351 		}
352 	}
353 
354 	/*
355 	 * The architecture has been updated to explicitly tell us if
356 	 * we're not vulnerable to Spectre-V2.
357 	 */
358 	id = READ_SPECIALREG(id_aa64pfr0_el1);
359 	if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_IMPL)
360 		ci->ci_flush_bp = cpu_flush_bp_noop;
361 }
362 
363 /*
364  * Enable mitigation for Spectre-BHB branch history injection
365  * vulnerabilities (CVE-2022-23960).
366 */
367 void
368 cpu_mitigate_spectre_bhb(struct cpu_info *ci)
369 {
370 	uint64_t id;
371 
372 	/*
373 	 * If we know the CPU, we can add a branchy loop that cleans
374 	 * the BHB.
375 	 */
376 	switch (CPU_IMPL(ci->ci_midr)) {
377 	case CPU_IMPL_ARM:
378 		switch (CPU_PART(ci->ci_midr)) {
379 		case CPU_PART_CORTEX_A57:
380 		case CPU_PART_CORTEX_A72:
381 			ci->ci_trampoline_vectors =
382 			    (vaddr_t)trampoline_vectors_loop_8;
383 			break;
384 		case CPU_PART_CORTEX_A76:
385 		case CPU_PART_CORTEX_A76AE:
386 		case CPU_PART_CORTEX_A77:
387 		case CPU_PART_NEOVERSE_N1:
388 			ci->ci_trampoline_vectors =
389 			    (vaddr_t)trampoline_vectors_loop_24;
390 			break;
391 		case CPU_PART_CORTEX_A78:
392 		case CPU_PART_CORTEX_A78AE:
393 		case CPU_PART_CORTEX_A78C:
394 		case CPU_PART_CORTEX_X1:
395 		case CPU_PART_CORTEX_X2:
396 		case CPU_PART_CORTEX_A710:
397 		case CPU_PART_NEOVERSE_N2:
398 		case CPU_PART_NEOVERSE_V1:
399 			ci->ci_trampoline_vectors =
400 			    (vaddr_t)trampoline_vectors_loop_32;
401 			break;
402 		}
403 		break;
404 	case CPU_IMPL_AMPERE:
405 		switch (CPU_PART(ci->ci_midr)) {
406 		case CPU_PART_AMPERE1:
407 			ci->ci_trampoline_vectors =
408 			    (vaddr_t)trampoline_vectors_loop_11;
409 			break;
410 		}
411 		break;
412 	}
413 
414 	/*
415 	 * If we're not using a loop, let firmware decide.  This also
416 	 * covers the original Spectre-V2 in addition to Spectre-BHB.
417 	 */
418 #if NPSCI > 0
419 	if (ci->ci_trampoline_vectors == (vaddr_t)trampoline_vectors_none &&
420 	    smccc_needs_arch_workaround_3()) {
421 		ci->ci_flush_bp = cpu_flush_bp_noop;
422 		if (psci_method() == PSCI_METHOD_HVC)
423 			ci->ci_trampoline_vectors =
424 			    (vaddr_t)trampoline_vectors_psci_hvc;
425 		if (psci_method() == PSCI_METHOD_SMC)
426 			ci->ci_trampoline_vectors =
427 			    (vaddr_t)trampoline_vectors_psci_smc;
428 	}
429 #endif
430 
431 	/* Prefer CLRBHB to mitigate Spectre-BHB. */
432 	id = READ_SPECIALREG(id_aa64isar2_el1);
433 	if (ID_AA64ISAR2_CLRBHB(id) >= ID_AA64ISAR2_CLRBHB_IMPL)
434 		ci->ci_trampoline_vectors = (vaddr_t)trampoline_vectors_clrbhb;
435 
436 	/* ECBHB tells us Spectre-BHB is mitigated. */
437 	id = READ_SPECIALREG(id_aa64mmfr1_el1);
438 	if (ID_AA64MMFR1_ECBHB(id) >= ID_AA64MMFR1_ECBHB_IMPL)
439 		ci->ci_trampoline_vectors = (vaddr_t)trampoline_vectors_none;
440 
441 	/*
442 	 * The architecture has been updated to explicitly tell us if
443 	 * we're not vulnerable to Spectre-BHB.
444 	 */
445 	id = READ_SPECIALREG(id_aa64pfr0_el1);
446 	if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_HCXT)
447 		ci->ci_trampoline_vectors = (vaddr_t)trampoline_vectors_none;
448 }
449 
450 /*
451  * Enable mitigation for Spectre-V4 speculative store bypass
452  * vulnerabilities (CVE-2018-3639).
453  */
454 void
455 cpu_mitigate_spectre_v4(struct cpu_info *ci)
456 {
457 	uint64_t id;
458 
459 	switch (CPU_IMPL(ci->ci_midr)) {
460 	case CPU_IMPL_ARM:
461 		switch (CPU_PART(ci->ci_midr)) {
462 		case CPU_PART_CORTEX_A35:
463 		case CPU_PART_CORTEX_A53:
464 		case CPU_PART_CORTEX_A55:
465 			/* Not vulnerable. */
466 			return;
467 		}
468 		break;
469 	case CPU_IMPL_QCOM:
470 		switch (CPU_PART(ci->ci_midr)) {
471 		case CPU_PART_KRYO400_SILVER:
472 			/* Not vulnerable. */
473 			return;
474 		}
475 		break;
476 	}
477 
478 	/* SSBS tells us Spectre-V4 is mitigated. */
479 	id = READ_SPECIALREG(id_aa64pfr1_el1);
480 	if (ID_AA64PFR1_SSBS(id) >= ID_AA64PFR1_SSBS_PSTATE)
481 		return;
482 
483 	/* Enable firmware workaround if required. */
484 	smccc_enable_arch_workaround_2();
485 }
486 
487 void
488 cpu_identify(struct cpu_info *ci)
489 {
490 	static uint64_t prev_id_aa64isar0;
491 	static uint64_t prev_id_aa64isar1;
492 	static uint64_t prev_id_aa64isar2;
493 	static uint64_t prev_id_aa64mmfr0;
494 	static uint64_t prev_id_aa64mmfr1;
495 	static uint64_t prev_id_aa64mmfr2;
496 	static uint64_t prev_id_aa64pfr0;
497 	static uint64_t prev_id_aa64pfr1;
498 	uint64_t midr, impl, part;
499 	uint64_t clidr, ccsidr, id;
500 	uint32_t ctr, sets, ways, line;
501 	const char *impl_name = NULL;
502 	const char *part_name = NULL;
503 	const char *il1p_name = NULL;
504 	const char *sep;
505 	struct cpu_cores *coreselecter = cpu_cores_none;
506 	int ccidx;
507 	int i;
508 
509 	midr = READ_SPECIALREG(midr_el1);
510 	impl = CPU_IMPL(midr);
511 	part = CPU_PART(midr);
512 	ci->ci_midr = midr;
513 
514 	for (i = 0; cpu_implementers[i].name; i++) {
515 		if (impl == cpu_implementers[i].id) {
516 			impl_name = cpu_implementers[i].name;
517 			coreselecter = cpu_implementers[i].corelist;
518 			break;
519 		}
520 	}
521 
522 	for (i = 0; coreselecter[i].name; i++) {
523 		if (part == coreselecter[i].id) {
524 			part_name = coreselecter[i].name;
525 			break;
526 		}
527 	}
528 
529 	if (impl_name && part_name) {
530 		printf(" %s %s r%llup%llu", impl_name, part_name, CPU_VAR(midr),
531 		    CPU_REV(midr));
532 
533 		if (CPU_IS_PRIMARY(ci))
534 			snprintf(cpu_model, sizeof(cpu_model),
535 			    "%s %s r%llup%llu", impl_name, part_name,
536 			    CPU_VAR(midr), CPU_REV(midr));
537 	} else {
538 		printf(" Unknown, MIDR 0x%llx", midr);
539 
540 		if (CPU_IS_PRIMARY(ci))
541 			snprintf(cpu_model, sizeof(cpu_model), "Unknown");
542 	}
543 
544 	/* Print cache information. */
545 
546 	ctr = READ_SPECIALREG(ctr_el0);
547 	switch (ctr & CTR_IL1P_MASK) {
548 	case CTR_IL1P_AIVIVT:
549 		il1p_name = "AIVIVT ";
550 		break;
551 	case CTR_IL1P_VIPT:
552 		il1p_name = "VIPT ";
553 		break;
554 	case CTR_IL1P_PIPT:
555 		il1p_name = "PIPT ";
556 		break;
557 	}
558 
559 	id = READ_SPECIALREG(id_aa64mmfr2_el1);
560 	clidr = READ_SPECIALREG(clidr_el1);
561 	if (ID_AA64MMFR2_CCIDX(id) > ID_AA64MMFR2_CCIDX_IMPL) {
562 		/* Reserved value.  Don't print cache information. */
563 		clidr = 0;
564 	} else if (ID_AA64MMFR2_CCIDX(id) == ID_AA64MMFR2_CCIDX_IMPL) {
565 		/* CCSIDR_EL1 uses the new 64-bit format. */
566 		ccidx = 1;
567 	} else {
568 		/* CCSIDR_EL1 uses the old 32-bit format. */
569 		ccidx = 0;
570 	}
571 	for (i = 0; i < 7; i++) {
572 		if ((clidr & CLIDR_CTYPE_MASK) == 0)
573 			break;
574 		printf("\n%s:", ci->ci_dev->dv_xname);
575 		sep = "";
576 		if (clidr & CLIDR_CTYPE_INSN) {
577 			WRITE_SPECIALREG(csselr_el1,
578 			    i << CSSELR_LEVEL_SHIFT | CSSELR_IND);
579 			__asm volatile("isb");
580 			ccsidr = READ_SPECIALREG(ccsidr_el1);
581 			if (ccidx) {
582 				sets = CCSIDR_CCIDX_SETS(ccsidr);
583 				ways = CCSIDR_CCIDX_WAYS(ccsidr);
584 				line = CCSIDR_CCIDX_LINE_SIZE(ccsidr);
585 			} else {
586 				sets = CCSIDR_SETS(ccsidr);
587 				ways = CCSIDR_WAYS(ccsidr);
588 				line = CCSIDR_LINE_SIZE(ccsidr);
589 			}
590 			printf("%s %dKB %db/line %d-way L%d %sI-cache", sep,
591 			    (sets * ways * line) / 1024, line, ways, (i + 1),
592 			    il1p_name);
593 			il1p_name = "";
594 			sep = ",";
595 		}
596 		if (clidr & CLIDR_CTYPE_DATA) {
597 			WRITE_SPECIALREG(csselr_el1, i << CSSELR_LEVEL_SHIFT);
598 			__asm volatile("isb");
599 			ccsidr = READ_SPECIALREG(ccsidr_el1);
600 			if (ccidx) {
601 				sets = CCSIDR_CCIDX_SETS(ccsidr);
602 				ways = CCSIDR_CCIDX_WAYS(ccsidr);
603 				line = CCSIDR_CCIDX_LINE_SIZE(ccsidr);
604 			} else {
605 				sets = CCSIDR_SETS(ccsidr);
606 				ways = CCSIDR_WAYS(ccsidr);
607 				line = CCSIDR_LINE_SIZE(ccsidr);
608 			}
609 			printf("%s %dKB %db/line %d-way L%d D-cache", sep,
610 			    (sets * ways * line) / 1024, line, ways, (i + 1));
611 			sep = ",";
612 		}
613 		if (clidr & CLIDR_CTYPE_UNIFIED) {
614 			WRITE_SPECIALREG(csselr_el1, i << CSSELR_LEVEL_SHIFT);
615 			__asm volatile("isb");
616 			ccsidr = READ_SPECIALREG(ccsidr_el1);
617 			if (ccidx) {
618 				sets = CCSIDR_CCIDX_SETS(ccsidr);
619 				ways = CCSIDR_CCIDX_WAYS(ccsidr);
620 				line = CCSIDR_CCIDX_LINE_SIZE(ccsidr);
621 			} else {
622 				sets = CCSIDR_SETS(ccsidr);
623 				ways = CCSIDR_WAYS(ccsidr);
624 				line = CCSIDR_LINE_SIZE(ccsidr);
625 			}
626 			printf("%s %dKB %db/line %d-way L%d cache", sep,
627 			    (sets * ways * line) / 1024, line, ways, (i + 1));
628 		}
629 		clidr >>= 3;
630 	}
631 
632 	cpu_mitigate_spectre_v2(ci);
633 	cpu_mitigate_spectre_bhb(ci);
634 	cpu_mitigate_spectre_v4(ci);
635 
636 	/*
637 	 * Apple CPUs provide detailed information for SError.
638 	 */
639 	if (impl == CPU_IMPL_APPLE)
640 		ci->ci_serror = cpu_serror_apple;
641 
642 	/*
643 	 * Skip printing CPU features if they are identical to the
644 	 * previous CPU.
645 	 */
646 	if (READ_SPECIALREG(id_aa64isar0_el1) == prev_id_aa64isar0 &&
647 	    READ_SPECIALREG(id_aa64isar1_el1) == prev_id_aa64isar1 &&
648 	    READ_SPECIALREG(id_aa64isar2_el1) == prev_id_aa64isar2 &&
649 	    READ_SPECIALREG(id_aa64mmfr0_el1) == prev_id_aa64mmfr0 &&
650 	    READ_SPECIALREG(id_aa64mmfr1_el1) == prev_id_aa64mmfr1 &&
651 	    READ_SPECIALREG(id_aa64mmfr2_el1) == prev_id_aa64mmfr2 &&
652 	    READ_SPECIALREG(id_aa64pfr0_el1) == prev_id_aa64pfr0 &&
653 	    READ_SPECIALREG(id_aa64pfr1_el1) == prev_id_aa64pfr1)
654 		return;
655 
656 	/*
657 	 * Print CPU features encoded in the ID registers.
658 	 */
659 
660 	if (READ_SPECIALREG(id_aa64isar0_el1) != cpu_id_aa64isar0) {
661 		printf("\n%s: mismatched ID_AA64ISAR0_EL1",
662 		    ci->ci_dev->dv_xname);
663 	}
664 	if (READ_SPECIALREG(id_aa64isar1_el1) != cpu_id_aa64isar1) {
665 		printf("\n%s: mismatched ID_AA64ISAR1_EL1",
666 		    ci->ci_dev->dv_xname);
667 	}
668 	if (READ_SPECIALREG(id_aa64isar2_el1) != cpu_id_aa64isar2) {
669 		printf("\n%s: mismatched ID_AA64ISAR2_EL1",
670 		    ci->ci_dev->dv_xname);
671 	}
672 	if (READ_SPECIALREG(id_aa64mmfr0_el1) != cpu_id_aa64mmfr0) {
673 		printf("\n%s: mismatched ID_AA64MMFR0_EL1",
674 		    ci->ci_dev->dv_xname);
675 	}
676 	if (READ_SPECIALREG(id_aa64mmfr1_el1) != cpu_id_aa64mmfr1) {
677 		printf("\n%s: mismatched ID_AA64MMFR1_EL1",
678 		    ci->ci_dev->dv_xname);
679 	}
680 	if (READ_SPECIALREG(id_aa64mmfr2_el1) != cpu_id_aa64mmfr2) {
681 		printf("\n%s: mismatched ID_AA64MMFR2_EL1",
682 		    ci->ci_dev->dv_xname);
683 	}
684 	id = READ_SPECIALREG(id_aa64pfr0_el1);
685 	/* Allow CSV2/CVS3 to be different. */
686 	id &= ~ID_AA64PFR0_CSV2_MASK;
687 	id &= ~ID_AA64PFR0_CSV3_MASK;
688 	/* Ignore 32-bit support in all exception levels. */
689 	id &= ~ID_AA64PFR0_EL0_MASK;
690 	id &= ~ID_AA64PFR0_EL1_MASK;
691 	id &= ~ID_AA64PFR0_EL2_MASK;
692 	id &= ~ID_AA64PFR0_EL3_MASK;
693 	if (id != cpu_id_aa64pfr0) {
694 		printf("\n%s: mismatched ID_AA64PFR0_EL1",
695 		    ci->ci_dev->dv_xname);
696 	}
697 	if (READ_SPECIALREG(id_aa64pfr1_el1) != cpu_id_aa64pfr1) {
698 		printf("\n%s: mismatched ID_AA64PFR1_EL1",
699 		    ci->ci_dev->dv_xname);
700 	}
701 
702 	printf("\n%s: ", ci->ci_dev->dv_xname);
703 
704 	/*
705 	 * ID_AA64ISAR0
706 	 */
707 	id = READ_SPECIALREG(id_aa64isar0_el1);
708 	sep = "";
709 
710 	if (ID_AA64ISAR0_RNDR(id) >= ID_AA64ISAR0_RNDR_IMPL) {
711 		printf("%sRNDR", sep);
712 		sep = ",";
713 		arm64_has_rng = 1;
714 	}
715 
716 	if (ID_AA64ISAR0_TLB(id) >= ID_AA64ISAR0_TLB_IOS) {
717 		printf("%sTLBIOS", sep);
718 		sep = ",";
719 	}
720 	if (ID_AA64ISAR0_TLB(id) >= ID_AA64ISAR0_TLB_IRANGE)
721 		printf("+IRANGE");
722 
723 	if (ID_AA64ISAR0_TS(id) >= ID_AA64ISAR0_TS_BASE) {
724 		printf("%sTS", sep);
725 		sep = ",";
726 	}
727 	if (ID_AA64ISAR0_TS(id) >= ID_AA64ISAR0_TS_AXFLAG)
728 		printf("+AXFLAG");
729 
730 	if (ID_AA64ISAR0_FHM(id) >= ID_AA64ISAR0_FHM_IMPL) {
731 		printf("%sFHM", sep);
732 		sep = ",";
733 	}
734 
735 	if (ID_AA64ISAR0_DP(id) >= ID_AA64ISAR0_DP_IMPL) {
736 		printf("%sDP", sep);
737 		sep = ",";
738 	}
739 
740 	if (ID_AA64ISAR0_SM4(id) >= ID_AA64ISAR0_SM4_IMPL) {
741 		printf("%sSM4", sep);
742 		sep = ",";
743 	}
744 
745 	if (ID_AA64ISAR0_SM3(id) >= ID_AA64ISAR0_SM3_IMPL) {
746 		printf("%sSM3", sep);
747 		sep = ",";
748 	}
749 
750 	if (ID_AA64ISAR0_SHA3(id) >= ID_AA64ISAR0_SHA3_IMPL) {
751 		printf("%sSHA3", sep);
752 		sep = ",";
753 	}
754 
755 	if (ID_AA64ISAR0_RDM(id) >= ID_AA64ISAR0_RDM_IMPL) {
756 		printf("%sRDM", sep);
757 		sep = ",";
758 	}
759 
760 	if (ID_AA64ISAR0_ATOMIC(id) >= ID_AA64ISAR0_ATOMIC_IMPL) {
761 		printf("%sAtomic", sep);
762 		sep = ",";
763 		arm64_has_lse = 1;
764 	}
765 
766 	if (ID_AA64ISAR0_CRC32(id) >= ID_AA64ISAR0_CRC32_BASE) {
767 		printf("%sCRC32", sep);
768 		sep = ",";
769 	}
770 
771 	if (ID_AA64ISAR0_SHA2(id) >= ID_AA64ISAR0_SHA2_BASE) {
772 		printf("%sSHA2", sep);
773 		sep = ",";
774 	}
775 	if (ID_AA64ISAR0_SHA2(id) >= ID_AA64ISAR0_SHA2_512)
776 		printf("+SHA512");
777 
778 	if (ID_AA64ISAR0_SHA1(id) >= ID_AA64ISAR0_SHA1_BASE) {
779 		printf("%sSHA1", sep);
780 		sep = ",";
781 	}
782 
783 	if (ID_AA64ISAR0_AES(id) >= ID_AA64ISAR0_AES_BASE) {
784 		printf("%sAES", sep);
785 		sep = ",";
786 #ifdef CRYPTO
787 		arm64_has_aes = 1;
788 #endif
789 	}
790 	if (ID_AA64ISAR0_AES(id) >= ID_AA64ISAR0_AES_PMULL)
791 		printf("+PMULL");
792 
793 	/*
794 	 * ID_AA64ISAR1
795 	 */
796 	id = READ_SPECIALREG(id_aa64isar1_el1);
797 
798 	if (ID_AA64ISAR1_LS64(id) >= ID_AA64ISAR1_LS64_BASE) {
799 		printf("%sLS64", sep);
800 		sep = ",";
801 	}
802 	if (ID_AA64ISAR1_LS64(id) >= ID_AA64ISAR1_LS64_V)
803 		printf("+V");
804 	if (ID_AA64ISAR1_LS64(id) >= ID_AA64ISAR1_LS64_ACCDATA)
805 		printf("+ACCDATA");
806 
807 	if (ID_AA64ISAR1_XS(id) >= ID_AA64ISAR1_XS_IMPL) {
808 		printf("%sXS", sep);
809 		sep = ",";
810 	}
811 
812 	if (ID_AA64ISAR1_I8MM(id) >= ID_AA64ISAR1_I8MM_IMPL) {
813 		printf("%sI8MM", sep);
814 		sep = ",";
815 	}
816 
817 	if (ID_AA64ISAR1_DGH(id) >= ID_AA64ISAR1_DGH_IMPL) {
818 		printf("%sDGH", sep);
819 		sep = ",";
820 	}
821 
822 	if (ID_AA64ISAR1_BF16(id) >= ID_AA64ISAR1_BF16_BASE) {
823 		printf("%sBF16", sep);
824 		sep = ",";
825 	}
826 	if (ID_AA64ISAR1_BF16(id) >= ID_AA64ISAR1_BF16_EBF)
827 		printf("+EBF");
828 
829 	if (ID_AA64ISAR1_SPECRES(id) >= ID_AA64ISAR1_SPECRES_IMPL) {
830 		printf("%sSPECRES", sep);
831 		sep = ",";
832 	}
833 
834 	if (ID_AA64ISAR1_SB(id) >= ID_AA64ISAR1_SB_IMPL) {
835 		printf("%sSB", sep);
836 		sep = ",";
837 	}
838 
839 	if (ID_AA64ISAR1_FRINTTS(id) >= ID_AA64ISAR1_FRINTTS_IMPL) {
840 		printf("%sFRINTTS", sep);
841 		sep = ",";
842 	}
843 
844 	if (ID_AA64ISAR1_GPI(id) >= ID_AA64ISAR1_GPI_IMPL) {
845 		printf("%sGPI", sep);
846 		sep = ",";
847 	}
848 
849 	if (ID_AA64ISAR1_GPA(id) >= ID_AA64ISAR1_GPA_IMPL) {
850 		printf("%sGPA", sep);
851 		sep = ",";
852 	}
853 
854 	if (ID_AA64ISAR1_LRCPC(id) >= ID_AA64ISAR1_LRCPC_BASE) {
855 		printf("%sLRCPC", sep);
856 		sep = ",";
857 	}
858 	if (ID_AA64ISAR1_LRCPC(id) >= ID_AA64ISAR1_LRCPC_LDAPUR)
859 		printf("+LDAPUR");
860 
861 	if (ID_AA64ISAR1_FCMA(id) >= ID_AA64ISAR1_FCMA_IMPL) {
862 		printf("%sFCMA", sep);
863 		sep = ",";
864 	}
865 
866 	if (ID_AA64ISAR1_JSCVT(id) >= ID_AA64ISAR1_JSCVT_IMPL) {
867 		printf("%sJSCVT", sep);
868 		sep = ",";
869 	}
870 
871 	if (ID_AA64ISAR1_API(id) >= ID_AA64ISAR1_API_PAC) {
872 		printf("%sAPI", sep);
873 		sep = ",";
874 	}
875 	if (ID_AA64ISAR1_API(id) == ID_AA64ISAR1_API_EPAC)
876 		printf("+EPAC");
877 	else if (ID_AA64ISAR1_API(id) >= ID_AA64ISAR1_API_EPAC2)
878 		printf("+EPAC2");
879 	if (ID_AA64ISAR1_API(id) >= ID_AA64ISAR1_API_FPAC)
880 		printf("+FPAC");
881 	if (ID_AA64ISAR1_API(id) >= ID_AA64ISAR1_API_FPAC_COMBINED)
882 		printf("+COMBINED");
883 
884 	if (ID_AA64ISAR1_APA(id) >= ID_AA64ISAR1_APA_PAC) {
885 		printf("%sAPA", sep);
886 		sep = ",";
887 	}
888 	if (ID_AA64ISAR1_APA(id) == ID_AA64ISAR1_APA_EPAC)
889 		printf("+EPAC");
890 	else if (ID_AA64ISAR1_APA(id) >= ID_AA64ISAR1_APA_EPAC2)
891 		printf("+EPAC2");
892 	if (ID_AA64ISAR1_APA(id) >= ID_AA64ISAR1_APA_FPAC)
893 		printf("+FPAC");
894 	if (ID_AA64ISAR1_APA(id) >= ID_AA64ISAR1_APA_FPAC_COMBINED)
895 		printf("+COMBINED");
896 
897 	if (ID_AA64ISAR1_DPB(id) >= ID_AA64ISAR1_DPB_IMPL) {
898 		printf("%sDPB", sep);
899 		sep = ",";
900 	}
901 	if (ID_AA64ISAR1_DPB(id) >= ID_AA64ISAR1_DPB_DCCVADP)
902 		printf("+DCCVADP");
903 
904 	/*
905 	 * ID_AA64ISAR2
906 	 */
907 	id = READ_SPECIALREG(id_aa64isar2_el1);
908 
909 	if (ID_AA64ISAR2_CSSC(id) >= ID_AA64ISAR2_CSSC_IMPL) {
910 		printf("%sCSSC", sep);
911 		sep = ",";
912 	}
913 
914 	if (ID_AA64ISAR2_RPRFM(id) >= ID_AA64ISAR2_RPRFM_IMPL) {
915 		printf("%sRPRFM", sep);
916 		sep = ",";
917 	}
918 
919 	if (ID_AA64ISAR2_CLRBHB(id) >= ID_AA64ISAR2_CLRBHB_IMPL) {
920 		printf("%sCLRBHB", sep);
921 		sep = ",";
922 	}
923 
924 	if (ID_AA64ISAR2_BC(id) >= ID_AA64ISAR2_BC_IMPL) {
925 		printf("%sBC", sep);
926 		sep = ",";
927 	}
928 
929 	if (ID_AA64ISAR2_MOPS(id) >= ID_AA64ISAR2_MOPS_IMPL) {
930 		printf("%sMOPS", sep);
931 		sep = ",";
932 	}
933 
934 	if (ID_AA64ISAR2_GPA3(id) >= ID_AA64ISAR2_GPA3_IMPL) {
935 		printf("%sGPA3", sep);
936 		sep = ",";
937 	}
938 
939 	if (ID_AA64ISAR2_APA3(id) >= ID_AA64ISAR2_APA3_PAC) {
940 		printf("%sAPA3", sep);
941 		sep = ",";
942 	}
943 	if (ID_AA64ISAR2_APA3(id) == ID_AA64ISAR2_APA3_EPAC)
944 		printf("+EPAC");
945 	else if (ID_AA64ISAR2_APA3(id) >= ID_AA64ISAR2_APA3_EPAC2)
946 		printf("+EPAC2");
947 	if (ID_AA64ISAR2_APA3(id) >= ID_AA64ISAR2_APA3_FPAC)
948 		printf("+FPAC");
949 	if (ID_AA64ISAR2_APA3(id) >= ID_AA64ISAR2_APA3_FPAC_COMBINED)
950 		printf("+COMBINED");
951 
952 	if (ID_AA64ISAR2_RPRES(id) >= ID_AA64ISAR2_RPRES_IMPL) {
953 		printf("%sRPRES", sep);
954 		sep = ",";
955 	}
956 
957 	if (ID_AA64ISAR2_WFXT(id) >= ID_AA64ISAR2_WFXT_IMPL) {
958 		printf("%sWFXT", sep);
959 		sep = ",";
960 	}
961 
962 	/*
963 	 * ID_AA64MMFR0
964 	 *
965 	 * We only print ASIDBits for now.
966 	 */
967 	id = READ_SPECIALREG(id_aa64mmfr0_el1);
968 
969 	if (ID_AA64MMFR0_ECV(id) >= ID_AA64MMFR0_ECV_IMPL) {
970 		printf("%sECV", sep);
971 		sep = ",";
972 	}
973 	if (ID_AA64MMFR0_ECV(id) >= ID_AA64MMFR0_ECV_CNTHCTL)
974 		printf("+CNTHCTL");
975 
976 	if (ID_AA64MMFR0_ASID_BITS(id) == ID_AA64MMFR0_ASID_BITS_16) {
977 		printf("%sASID16", sep);
978 		sep = ",";
979 	}
980 
981 	/*
982 	 * ID_AA64MMFR1
983 	 *
984 	 * We omit printing most virtualization related fields for now.
985 	 */
986 	id = READ_SPECIALREG(id_aa64mmfr1_el1);
987 
988 	if (ID_AA64MMFR1_AFP(id) >= ID_AA64MMFR1_AFP_IMPL) {
989 		printf("%sAFP", sep);
990 		sep = ",";
991 	}
992 
993 	if (ID_AA64MMFR1_SPECSEI(id) >= ID_AA64MMFR1_SPECSEI_IMPL) {
994 		printf("%sSpecSEI", sep);
995 		sep = ",";
996 	}
997 
998 	if (ID_AA64MMFR1_PAN(id) >= ID_AA64MMFR1_PAN_IMPL) {
999 		printf("%sPAN", sep);
1000 		sep = ",";
1001 	}
1002 	if (ID_AA64MMFR1_PAN(id) >= ID_AA64MMFR1_PAN_ATS1E1)
1003 		printf("+ATS1E1");
1004 	if (ID_AA64MMFR1_PAN(id) >= ID_AA64MMFR1_PAN_EPAN)
1005 		printf("+EPAN");
1006 
1007 	if (ID_AA64MMFR1_LO(id) >= ID_AA64MMFR1_LO_IMPL) {
1008 		printf("%sLO", sep);
1009 		sep = ",";
1010 	}
1011 
1012 	if (ID_AA64MMFR1_HPDS(id) >= ID_AA64MMFR1_HPDS_IMPL) {
1013 		printf("%sHPDS", sep);
1014 		sep = ",";
1015 	}
1016 
1017 	if (ID_AA64MMFR1_VH(id) >= ID_AA64MMFR1_VH_IMPL) {
1018 		printf("%sVH", sep);
1019 		sep = ",";
1020 	}
1021 
1022 	if (ID_AA64MMFR1_HAFDBS(id) >= ID_AA64MMFR1_HAFDBS_AF) {
1023 		printf("%sHAF", sep);
1024 		sep = ",";
1025 	}
1026 	if (ID_AA64MMFR1_HAFDBS(id) >= ID_AA64MMFR1_HAFDBS_AF_DBS)
1027 		printf("DBS");
1028 
1029 	if (ID_AA64MMFR1_ECBHB(id) >= ID_AA64MMFR1_ECBHB_IMPL) {
1030 		printf("%sECBHB", sep);
1031 		sep = ",";
1032 	}
1033 
1034 	/*
1035 	 * ID_AA64MMFR2
1036 	 */
1037 	id = READ_SPECIALREG(id_aa64mmfr2_el1);
1038 
1039 	if (ID_AA64MMFR2_IDS(id) >= ID_AA64MMFR2_IDS_IMPL) {
1040 		printf("%sIDS", sep);
1041 		sep = ",";
1042 	}
1043 
1044 	if (ID_AA64MMFR2_AT(id) >= ID_AA64MMFR2_AT_IMPL) {
1045 		printf("%sAT", sep);
1046 		sep = ",";
1047 	}
1048 
1049 	/*
1050 	 * ID_AA64PFR0
1051 	 */
1052 	id = READ_SPECIALREG(id_aa64pfr0_el1);
1053 
1054 	if (ID_AA64PFR0_CSV3(id) >= ID_AA64PFR0_CSV3_IMPL) {
1055 		printf("%sCSV3", sep);
1056 		sep = ",";
1057 	}
1058 
1059 	if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_IMPL) {
1060 		printf("%sCSV2", sep);
1061 		sep = ",";
1062 	}
1063 	if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_SCXT)
1064 		printf("+SCXT");
1065 	if (ID_AA64PFR0_CSV2(id) >= ID_AA64PFR0_CSV2_HCXT)
1066 		printf("+HCXT");
1067 
1068 	if (ID_AA64PFR0_DIT(id) >= ID_AA64PFR0_DIT_IMPL) {
1069 		printf("%sDIT", sep);
1070 		sep = ",";
1071 	}
1072 
1073 	if (ID_AA64PFR0_ADV_SIMD(id) != ID_AA64PFR0_ADV_SIMD_NONE &&
1074 	    ID_AA64PFR0_ADV_SIMD(id) >= ID_AA64PFR0_ADV_SIMD_HP) {
1075 		printf("%sAdvSIMD+HP", sep);
1076 		sep = ",";
1077 	}
1078 
1079 	if (ID_AA64PFR0_FP(id) != ID_AA64PFR0_FP_NONE &&
1080 	    ID_AA64PFR0_FP(id) >= ID_AA64PFR0_FP_HP) {
1081 		printf("%sFP+HP", sep);
1082 		sep = ",";
1083 	}
1084 
1085 	/*
1086 	 * ID_AA64PFR1
1087 	 */
1088 	id = READ_SPECIALREG(id_aa64pfr1_el1);
1089 
1090 	if (ID_AA64PFR1_BT(id) >= ID_AA64PFR1_BT_IMPL) {
1091 		printf("%sBT", sep);
1092 		sep = ",";
1093 	}
1094 
1095 	if (ID_AA64PFR1_SSBS(id) >= ID_AA64PFR1_SSBS_PSTATE) {
1096 		printf("%sSSBS", sep);
1097 		sep = ",";
1098 	}
1099 	if (ID_AA64PFR1_SSBS(id) >= ID_AA64PFR1_SSBS_PSTATE_MSR)
1100 		printf("+MSR");
1101 
1102 	if (ID_AA64PFR1_MTE(id) >= ID_AA64PFR1_MTE_IMPL) {
1103 		printf("%sMTE", sep);
1104 		sep = ",";
1105 	}
1106 
1107 	prev_id_aa64isar0 = READ_SPECIALREG(id_aa64isar0_el1);
1108 	prev_id_aa64isar1 = READ_SPECIALREG(id_aa64isar1_el1);
1109 	prev_id_aa64isar2 = READ_SPECIALREG(id_aa64isar2_el1);
1110 	prev_id_aa64mmfr0 = READ_SPECIALREG(id_aa64mmfr0_el1);
1111 	prev_id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
1112 	prev_id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1);
1113 	prev_id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1);
1114 	prev_id_aa64pfr1 = READ_SPECIALREG(id_aa64pfr1_el1);
1115 
1116 #ifdef CPU_DEBUG
1117 	id = READ_SPECIALREG(id_aa64afr0_el1);
1118 	printf("\nID_AA64AFR0_EL1: 0x%016llx", id);
1119 	id = READ_SPECIALREG(id_aa64afr1_el1);
1120 	printf("\nID_AA64AFR1_EL1: 0x%016llx", id);
1121 	id = READ_SPECIALREG(id_aa64dfr0_el1);
1122 	printf("\nID_AA64DFR0_EL1: 0x%016llx", id);
1123 	id = READ_SPECIALREG(id_aa64dfr1_el1);
1124 	printf("\nID_AA64DFR1_EL1: 0x%016llx", id);
1125 	id = READ_SPECIALREG(id_aa64isar0_el1);
1126 	printf("\nID_AA64ISAR0_EL1: 0x%016llx", id);
1127 	id = READ_SPECIALREG(id_aa64isar1_el1);
1128 	printf("\nID_AA64ISAR1_EL1: 0x%016llx", id);
1129 	id = READ_SPECIALREG(id_aa64isar2_el1);
1130 	printf("\nID_AA64ISAR2_EL1: 0x%016llx", id);
1131 	id = READ_SPECIALREG(id_aa64mmfr0_el1);
1132 	printf("\nID_AA64MMFR0_EL1: 0x%016llx", id);
1133 	id = READ_SPECIALREG(id_aa64mmfr1_el1);
1134 	printf("\nID_AA64MMFR1_EL1: 0x%016llx", id);
1135 	id = READ_SPECIALREG(id_aa64mmfr2_el1);
1136 	printf("\nID_AA64MMFR2_EL1: 0x%016llx", id);
1137 	id = READ_SPECIALREG(id_aa64pfr0_el1);
1138 	printf("\nID_AA64PFR0_EL1: 0x%016llx", id);
1139 	id = READ_SPECIALREG(id_aa64pfr1_el1);
1140 	printf("\nID_AA64PFR1_EL1: 0x%016llx", id);
1141 #endif
1142 }
1143 
1144 void
1145 cpu_identify_cleanup(void)
1146 {
1147 	uint64_t id_aa64mmfr2;
1148 	uint64_t value;
1149 
1150 	/* ID_AA64ISAR0_EL1 */
1151 	value = cpu_id_aa64isar0 & ID_AA64ISAR0_MASK;
1152 	value &= ~ID_AA64ISAR0_TLB_MASK;
1153 	cpu_id_aa64isar0 = value;
1154 
1155 	/* ID_AA64ISAR1_EL1 */
1156 	value = cpu_id_aa64isar1 & ID_AA64ISAR1_MASK;
1157 	value &= ~ID_AA64ISAR1_SPECRES_MASK;
1158 	cpu_id_aa64isar1 = value;
1159 
1160 	/* ID_AA64ISAR2_EL1 */
1161 	value = cpu_id_aa64isar2 & ID_AA64ISAR2_MASK;
1162 	value &= ~ID_AA64ISAR2_CLRBHB_MASK;
1163 	cpu_id_aa64isar2 = value;
1164 
1165 	/* ID_AA64MMFR0_EL1 */
1166 	value = 0;
1167 	value |= cpu_id_aa64mmfr0 & ID_AA64MMFR0_ECV_MASK;
1168 	cpu_id_aa64mmfr0 = value;
1169 
1170 	/* ID_AA64MMFR1_EL1 */
1171 	value = 0;
1172 	value |= cpu_id_aa64mmfr1 & ID_AA64MMFR1_AFP_MASK;
1173 	cpu_id_aa64mmfr1 = value;
1174 
1175 	/* ID_AA64MMFR2_EL1 */
1176 	value = 0;
1177 	value |= cpu_id_aa64mmfr2 & ID_AA64MMFR2_AT_MASK;
1178 	cpu_id_aa64mmfr2 = value;
1179 
1180 	/* ID_AA64PFR0_EL1 */
1181 	value = 0;
1182 	value |= cpu_id_aa64pfr0 & ID_AA64PFR0_FP_MASK;
1183 	value |= cpu_id_aa64pfr0 & ID_AA64PFR0_ADV_SIMD_MASK;
1184 	value |= cpu_id_aa64pfr0 & ID_AA64PFR0_DIT_MASK;
1185 	cpu_id_aa64pfr0 = value;
1186 
1187 	/* ID_AA64PFR1_EL1 */
1188 	value = 0;
1189 	value |= cpu_id_aa64pfr1 & ID_AA64PFR1_BT_MASK;
1190 	value |= cpu_id_aa64pfr1 & ID_AA64PFR1_SSBS_MASK;
1191 	cpu_id_aa64pfr1 = value;
1192 
1193 	/* HWCAP */
1194 	hwcap |= HWCAP_FP;	/* OpenBSD assumes Floating-point support */
1195 	hwcap |= HWCAP_ASIMD;	/* OpenBSD assumes Advanced SIMD support */
1196 	/* HWCAP_EVTSTRM: OpenBSD kernel doesn't configure event stream */
1197 	if (ID_AA64ISAR0_AES(cpu_id_aa64isar0) >= ID_AA64ISAR0_AES_BASE)
1198 		hwcap |= HWCAP_AES;
1199 	if (ID_AA64ISAR0_AES(cpu_id_aa64isar0) >= ID_AA64ISAR0_AES_PMULL)
1200 		hwcap |= HWCAP_PMULL;
1201 	if (ID_AA64ISAR0_SHA1(cpu_id_aa64isar0) >= ID_AA64ISAR0_SHA1_BASE)
1202 		hwcap |= HWCAP_SHA1;
1203 	if (ID_AA64ISAR0_SHA2(cpu_id_aa64isar0) >= ID_AA64ISAR0_SHA2_BASE)
1204 		hwcap |= HWCAP_SHA2;
1205 	if (ID_AA64ISAR0_CRC32(cpu_id_aa64isar0) >= ID_AA64ISAR0_CRC32_BASE)
1206 		hwcap |= HWCAP_CRC32;
1207 	if (ID_AA64ISAR0_ATOMIC(cpu_id_aa64isar0) >= ID_AA64ISAR0_ATOMIC_IMPL)
1208 		hwcap |= HWCAP_ATOMICS;
1209 	if (ID_AA64PFR0_FP(cpu_id_aa64pfr0) != ID_AA64PFR0_FP_NONE &&
1210 	    ID_AA64PFR0_FP(cpu_id_aa64pfr0) >= ID_AA64PFR0_FP_HP)
1211 		hwcap |= HWCAP_FPHP;
1212 	if (ID_AA64PFR0_ADV_SIMD(cpu_id_aa64pfr0) != ID_AA64PFR0_ADV_SIMD_NONE &&
1213 	    ID_AA64PFR0_ADV_SIMD(cpu_id_aa64pfr0) >= ID_AA64PFR0_ADV_SIMD_HP)
1214 		hwcap |= HWCAP_ASIMDHP;
1215 	id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1);
1216 	if (ID_AA64MMFR2_IDS(id_aa64mmfr2) >= ID_AA64MMFR2_IDS_IMPL)
1217 		hwcap |= HWCAP_CPUID;
1218 	if (ID_AA64ISAR0_RDM(cpu_id_aa64isar0) >= ID_AA64ISAR0_RDM_IMPL)
1219 		hwcap |= HWCAP_ASIMDRDM;
1220 	if (ID_AA64ISAR1_JSCVT(cpu_id_aa64isar1) >= ID_AA64ISAR1_JSCVT_IMPL)
1221 		hwcap |= HWCAP_JSCVT;
1222 	if (ID_AA64ISAR1_FCMA(cpu_id_aa64isar1) >= ID_AA64ISAR1_FCMA_IMPL)
1223 		hwcap |= HWCAP_FCMA;
1224 	if (ID_AA64ISAR1_LRCPC(cpu_id_aa64isar1) >= ID_AA64ISAR1_LRCPC_BASE)
1225 		hwcap |= HWCAP_LRCPC;
1226 	if (ID_AA64ISAR1_DPB(cpu_id_aa64isar1) >= ID_AA64ISAR1_DPB_IMPL)
1227 		hwcap |= HWCAP_DCPOP;
1228 	if (ID_AA64ISAR0_SHA3(cpu_id_aa64isar0) >= ID_AA64ISAR0_SHA3_IMPL)
1229 		hwcap |= HWCAP_SHA3;
1230 	if (ID_AA64ISAR0_SM3(cpu_id_aa64isar0) >= ID_AA64ISAR0_SM3_IMPL)
1231 		hwcap |= HWCAP_SM3;
1232 	if (ID_AA64ISAR0_SM4(cpu_id_aa64isar0) >= ID_AA64ISAR0_SM4_IMPL)
1233 		hwcap |= HWCAP_SM4;
1234 	if (ID_AA64ISAR0_DP(cpu_id_aa64isar0) >= ID_AA64ISAR0_DP_IMPL)
1235 		hwcap |= HWCAP_ASIMDDP;
1236 	if (ID_AA64ISAR0_SHA2(cpu_id_aa64isar0) >= ID_AA64ISAR0_SHA2_512)
1237 		hwcap |= HWCAP_SHA512;
1238 	/* HWCAP_SVE: OpenBSD kernel doesn't provide SVE support */
1239 	if (ID_AA64ISAR0_FHM(cpu_id_aa64isar0) >= ID_AA64ISAR0_FHM_IMPL)
1240 		hwcap |= HWCAP_ASIMDFHM;
1241 	if (ID_AA64PFR0_DIT(cpu_id_aa64pfr0) >= ID_AA64PFR0_DIT_IMPL)
1242 		hwcap |= HWCAP_DIT;
1243 	if (ID_AA64MMFR2_AT(cpu_id_aa64mmfr2) >= ID_AA64MMFR2_AT_IMPL)
1244 		hwcap |= HWCAP_USCAT;
1245 	if (ID_AA64ISAR1_LRCPC(cpu_id_aa64isar1) >= ID_AA64ISAR1_LRCPC_LDAPUR)
1246 		hwcap |= HWCAP_ILRCPC;
1247 	if (ID_AA64ISAR0_TS(cpu_id_aa64isar0) >= ID_AA64ISAR0_TS_BASE)
1248 		hwcap |= HWCAP_FLAGM;
1249 	if (ID_AA64PFR1_SSBS(cpu_id_aa64pfr1) >= ID_AA64PFR1_SSBS_PSTATE_MSR)
1250 		hwcap |= HWCAP_SSBS;
1251 	if (ID_AA64ISAR1_SB(cpu_id_aa64isar1) >= ID_AA64ISAR1_SB_IMPL)
1252 		hwcap |= HWCAP_SB;
1253 	if (ID_AA64ISAR1_APA(cpu_id_aa64isar1) >= ID_AA64ISAR1_APA_PAC ||
1254 	    ID_AA64ISAR1_API(cpu_id_aa64isar1) >= ID_AA64ISAR1_API_PAC)
1255 		hwcap |= HWCAP_PACA;
1256 	if (ID_AA64ISAR1_GPA(cpu_id_aa64isar1) >= ID_AA64ISAR1_GPA_IMPL ||
1257 	    ID_AA64ISAR1_GPI(cpu_id_aa64isar1) >= ID_AA64ISAR1_GPI_IMPL)
1258 		hwcap |= HWCAP_PACG;
1259 
1260 	/* HWCAP2 */
1261 	if (ID_AA64ISAR1_DPB(cpu_id_aa64isar1) >= ID_AA64ISAR1_DPB_DCCVADP)
1262 		hwcap2 |= HWCAP2_DCPODP;
1263 	/* HWCAP2_SVE2: OpenBSD kernel doesn't provide SVE support */
1264 	/* HWCAP2_SVEAES: OpenBSD kernel doesn't provide SVE support */
1265 	/* HWCAP2_SVEPMULL: OpenBSD kernel doesn't provide SVE support */
1266 	/* HWCAP2_SVEBITPERM: OpenBSD kernel doesn't provide SVE support */
1267 	/* HWCAP2_SVESHA3: OpenBSD kernel doesn't provide SVE support */
1268 	/* HWCAP2_SVESM4: OpenBSD kernel doesn't provide SVE support */
1269 	if (ID_AA64ISAR0_TS(cpu_id_aa64isar0) >= ID_AA64ISAR0_TS_AXFLAG)
1270 		hwcap2 |= HWCAP2_FLAGM2;
1271 	if (ID_AA64ISAR1_FRINTTS(cpu_id_aa64isar1) >= ID_AA64ISAR1_FRINTTS_IMPL)
1272 		hwcap2 |= HWCAP2_FRINT;
1273 	/* HWCAP2_SVEI8MM: OpenBSD kernel doesn't provide SVE support */
1274 	/* HWCAP2_SVEF32MM: OpenBSD kernel doesn't provide SVE support */
1275 	/* HWCAP2_SVEF64MM: OpenBSD kernel doesn't provide SVE support */
1276 	/* HWCAP2_SVEBF16: OpenBSD kernel doesn't provide SVE support */
1277 	if (ID_AA64ISAR1_I8MM(cpu_id_aa64isar1) >= ID_AA64ISAR1_I8MM_IMPL)
1278 		hwcap2 |= HWCAP2_I8MM;
1279 	if (ID_AA64ISAR1_BF16(cpu_id_aa64isar1) >= ID_AA64ISAR1_BF16_BASE)
1280 		hwcap2 |= HWCAP2_BF16;
1281 	if (ID_AA64ISAR1_DGH(cpu_id_aa64isar1) >= ID_AA64ISAR1_DGH_IMPL)
1282 		hwcap2 |= HWCAP2_DGH;
1283 	if (ID_AA64ISAR0_RNDR(cpu_id_aa64isar0) >= ID_AA64ISAR0_RNDR_IMPL)
1284 		hwcap2 |= HWCAP2_RNG;
1285 	if (ID_AA64PFR1_BT(cpu_id_aa64pfr1) >= ID_AA64PFR1_BT_IMPL)
1286 		hwcap2 |= HWCAP2_BTI;
1287 	/* HWCAP2_MTE: OpenBSD kernel doesn't provide MTE support */
1288 	if (ID_AA64MMFR0_ECV(cpu_id_aa64mmfr0) >= ID_AA64MMFR0_ECV_IMPL)
1289 		hwcap2 |= HWCAP2_ECV;
1290 	if (ID_AA64MMFR1_AFP(cpu_id_aa64mmfr1) >= ID_AA64MMFR1_AFP_IMPL)
1291 		hwcap2 |= HWCAP2_AFP;
1292 	if (ID_AA64ISAR2_RPRES(cpu_id_aa64isar2) >= ID_AA64ISAR2_RPRES_IMPL)
1293 		hwcap2 |= HWCAP2_RPRES;
1294 	/* HWCAP2_MTE3: OpenBSD kernel doesn't provide MTE support */
1295 	/* HWCAP2_SME: OpenBSD kernel doesn't provide SME support */
1296 	/* HWCAP2_SME_I16I64: OpenBSD kernel doesn't provide SME support */
1297 	/* HWCAP2_SME_F64F64: OpenBSD kernel doesn't provide SME support */
1298 	/* HWCAP2_SME_I8I32: OpenBSD kernel doesn't provide SME support */
1299 	/* HWCAP2_SME_F16F32: OpenBSD kernel doesn't provide SME support */
1300 	/* HWCAP2_SME_B16F32: OpenBSD kernel doesn't provide SME support */
1301 	/* HWCAP2_SME_F32F32: OpenBSD kernel doesn't provide SME support */
1302 	/* HWCAP2_SME_FA64: OpenBSD kernel doesn't provide SME support */
1303 	if (ID_AA64ISAR2_WFXT(cpu_id_aa64isar2) >= ID_AA64ISAR2_WFXT_IMPL)
1304 		hwcap2 |= HWCAP2_WFXT;
1305 	if (ID_AA64ISAR1_BF16(cpu_id_aa64isar1) >= ID_AA64ISAR1_BF16_EBF)
1306 		hwcap2 |= HWCAP2_EBF16;
1307 	/* HWCAP2_SVE_EBF16: OpenBSD kernel doesn't provide SVE support */
1308 	if (ID_AA64ISAR2_CSSC(cpu_id_aa64isar2) >= ID_AA64ISAR2_CSSC_IMPL)
1309 		hwcap2 |= HWCAP2_CSSC;
1310 	if (ID_AA64ISAR2_RPRFM(cpu_id_aa64isar2) >= ID_AA64ISAR2_RPRFM_IMPL)
1311 		hwcap2 |= HWCAP2_RPRFM;
1312 	/* HWCAP2_SVE2P1: OpenBSD kernel doesn't provide SVE support */
1313 	/* HWCAP2_SME2: OpenBSD kernel doesn't provide SME support */
1314 	/* HWCAP2_SME2P1: OpenBSD kernel doesn't provide SME support */
1315 	/* HWCAP2_SME_I16I32: OpenBSD kernel doesn't provide SME support */
1316 	/* HWCAP2_SME_BI32I32: OpenBSD kernel doesn't provide SME support */
1317 	/* HWCAP2_SME_B16B16: OpenBSD kernel doesn't provide SME support */
1318 	/* HWCAP2_SME_F16F16: OpenBSD kernel doesn't provide SME support */
1319 	if (ID_AA64ISAR2_MOPS(cpu_id_aa64isar2) >= ID_AA64ISAR2_MOPS_IMPL)
1320 		hwcap2 |= HWCAP2_MOPS;
1321 	if (ID_AA64ISAR2_BC(cpu_id_aa64isar2) >= ID_AA64ISAR2_BC_IMPL)
1322 		hwcap2 |= HWCAP2_HBC;
1323 }
1324 
1325 void	cpu_init(void);
1326 int	cpu_start_secondary(struct cpu_info *ci, int, uint64_t);
1327 int	cpu_clockspeed(int *);
1328 
1329 int
1330 cpu_match(struct device *parent, void *cfdata, void *aux)
1331 {
1332 	struct fdt_attach_args *faa = aux;
1333 	uint64_t mpidr = READ_SPECIALREG(mpidr_el1);
1334 	char buf[32];
1335 
1336 	if (OF_getprop(faa->fa_node, "device_type", buf, sizeof(buf)) <= 0 ||
1337 	    strcmp(buf, "cpu") != 0)
1338 		return 0;
1339 
1340 	if (ncpus < MAXCPUS || faa->fa_reg[0].addr == (mpidr & MPIDR_AFF))
1341 		return 1;
1342 
1343 	return 0;
1344 }
1345 
1346 void
1347 cpu_attach(struct device *parent, struct device *dev, void *aux)
1348 {
1349 	struct fdt_attach_args *faa = aux;
1350 	struct cpu_info *ci;
1351 	void *kstack;
1352 #ifdef MULTIPROCESSOR
1353 	uint64_t mpidr = READ_SPECIALREG(mpidr_el1);
1354 #endif
1355 	uint32_t opp;
1356 
1357 	KASSERT(faa->fa_nreg > 0);
1358 
1359 #ifdef MULTIPROCESSOR
1360 	if (faa->fa_reg[0].addr == (mpidr & MPIDR_AFF)) {
1361 		ci = &cpu_info_primary;
1362 		ci->ci_flags |= CPUF_RUNNING | CPUF_PRESENT | CPUF_PRIMARY;
1363 	} else {
1364 		ci = malloc(sizeof(*ci), M_DEVBUF, M_WAITOK | M_ZERO);
1365 		cpu_info[dev->dv_unit] = ci;
1366 		ci->ci_next = cpu_info_list->ci_next;
1367 		cpu_info_list->ci_next = ci;
1368 		ci->ci_flags |= CPUF_AP;
1369 		ncpus++;
1370 	}
1371 #else
1372 	ci = &cpu_info_primary;
1373 #endif
1374 
1375 	ci->ci_dev = dev;
1376 	ci->ci_cpuid = dev->dv_unit;
1377 	ci->ci_mpidr = faa->fa_reg[0].addr;
1378 	ci->ci_node = faa->fa_node;
1379 	ci->ci_self = ci;
1380 
1381 	printf(" mpidr %llx:", ci->ci_mpidr);
1382 
1383 	kstack = km_alloc(USPACE, &kv_any, &kp_zero, &kd_waitok);
1384 	ci->ci_el1_stkend = (vaddr_t)kstack + USPACE - 16;
1385 	ci->ci_trampoline_vectors = (vaddr_t)trampoline_vectors_none;
1386 
1387 #ifdef MULTIPROCESSOR
1388 	if (ci->ci_flags & CPUF_AP) {
1389 		char buf[32];
1390 		uint64_t spinup_data = 0;
1391 		int spinup_method = 0;
1392 		int timeout = 10000;
1393 		int len;
1394 
1395 		len = OF_getprop(ci->ci_node, "enable-method",
1396 		    buf, sizeof(buf));
1397 		if (strcmp(buf, "psci") == 0) {
1398 			spinup_method = 1;
1399 		} else if (strcmp(buf, "spin-table") == 0) {
1400 			spinup_method = 2;
1401 			spinup_data = OF_getpropint64(ci->ci_node,
1402 			    "cpu-release-addr", 0);
1403 		}
1404 
1405 		clockqueue_init(&ci->ci_queue);
1406 		sched_init_cpu(ci);
1407 		if (cpu_start_secondary(ci, spinup_method, spinup_data)) {
1408 			atomic_setbits_int(&ci->ci_flags, CPUF_IDENTIFY);
1409 			__asm volatile("dsb sy; sev" ::: "memory");
1410 
1411 			while ((ci->ci_flags & CPUF_IDENTIFIED) == 0 &&
1412 			    --timeout)
1413 				delay(1000);
1414 			if (timeout == 0) {
1415 				printf(" failed to identify");
1416 				ci->ci_flags = 0;
1417 			}
1418 		} else {
1419 			printf(" failed to spin up");
1420 			ci->ci_flags = 0;
1421 		}
1422 	} else {
1423 #endif
1424 		cpu_id_aa64isar0 = READ_SPECIALREG(id_aa64isar0_el1);
1425 		cpu_id_aa64isar1 = READ_SPECIALREG(id_aa64isar1_el1);
1426 		cpu_id_aa64isar2 = READ_SPECIALREG(id_aa64isar2_el1);
1427 		cpu_id_aa64mmfr0 = READ_SPECIALREG(id_aa64mmfr0_el1);
1428 		cpu_id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
1429 		cpu_id_aa64mmfr2 = READ_SPECIALREG(id_aa64mmfr2_el1);
1430 		cpu_id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1);
1431 		cpu_id_aa64pfr1 = READ_SPECIALREG(id_aa64pfr1_el1);
1432 
1433 		/*
1434 		 * The CSV2/CSV3 "features" are handled on a
1435 		 * per-processor basis.  So it is fine if these fields
1436 		 * differ between CPU cores.  Mask off these fields to
1437 		 * prevent exporting these to userland.
1438 		 */
1439 		cpu_id_aa64pfr0 &= ~ID_AA64PFR0_CSV2_MASK;
1440 		cpu_id_aa64pfr0 &= ~ID_AA64PFR0_CSV3_MASK;
1441 
1442 		/*
1443 		 * We only support 64-bit mode, so we don't care about
1444 		 * differences in support for 32-bit mode between
1445 		 * cores.  Mask off these fields as well.
1446 		 */
1447 		cpu_id_aa64pfr0 &= ~ID_AA64PFR0_EL0_MASK;
1448 		cpu_id_aa64pfr0 &= ~ID_AA64PFR0_EL1_MASK;
1449 		cpu_id_aa64pfr0 &= ~ID_AA64PFR0_EL2_MASK;
1450 		cpu_id_aa64pfr0 &= ~ID_AA64PFR0_EL3_MASK;
1451 
1452 		/*
1453 		 * Lenovo X13s ships with broken EL2 firmware that
1454 		 * hangs the machine if we enable PAuth.
1455 		 */
1456 		if (hw_vendor && hw_prod && strcmp(hw_vendor, "LENOVO") == 0) {
1457 			if (strncmp(hw_prod, "21BX", 4) == 0 ||
1458 			    strncmp(hw_prod, "21BY", 4) == 0) {
1459 				cpu_id_aa64isar1 &= ~ID_AA64ISAR1_APA_MASK;
1460 				cpu_id_aa64isar1 &= ~ID_AA64ISAR1_GPA_MASK;
1461 			}
1462 		}
1463 
1464 		cpu_identify(ci);
1465 
1466 		if (OF_getproplen(ci->ci_node, "clocks") > 0) {
1467 			cpu_node = ci->ci_node;
1468 			cpu_cpuspeed = cpu_clockspeed;
1469 		}
1470 
1471 		cpu_init();
1472 
1473 		if (arm64_has_rng) {
1474 			timeout_set(&cpu_rng_to, cpu_rng, &cpu_rng_to);
1475 			cpu_rng(&cpu_rng_to);
1476 		}
1477 #ifdef MULTIPROCESSOR
1478 	}
1479 #endif
1480 
1481 #if NKSTAT > 0
1482 	cpu_kstat_attach(ci);
1483 #endif
1484 
1485 	opp = OF_getpropint(ci->ci_node, "operating-points-v2", 0);
1486 	if (opp)
1487 		cpu_opp_init(ci, opp);
1488 
1489 	cpu_psci_init(ci);
1490 
1491 	printf("\n");
1492 }
1493 
1494 void
1495 cpu_init(void)
1496 {
1497 	uint64_t id_aa64mmfr1, sctlr;
1498 	uint64_t id_aa64pfr0;
1499 	uint64_t tcr;
1500 
1501 	WRITE_SPECIALREG(ttbr0_el1, pmap_kernel()->pm_pt0pa);
1502 	__asm volatile("isb");
1503 	tcr = READ_SPECIALREG(tcr_el1);
1504 	tcr &= ~TCR_T0SZ(0x3f);
1505 	tcr |= TCR_T0SZ(64 - USER_SPACE_BITS);
1506 	tcr |= TCR_A1;
1507 	WRITE_SPECIALREG(tcr_el1, tcr);
1508 	cpu_tlb_flush();
1509 
1510 	/* Enable PAN. */
1511 	id_aa64mmfr1 = READ_SPECIALREG(id_aa64mmfr1_el1);
1512 	if (ID_AA64MMFR1_PAN(id_aa64mmfr1) >= ID_AA64MMFR1_PAN_IMPL) {
1513 		sctlr = READ_SPECIALREG(sctlr_el1);
1514 		sctlr &= ~SCTLR_SPAN;
1515 		if (ID_AA64MMFR1_PAN(id_aa64mmfr1) >= ID_AA64MMFR1_PAN_EPAN)
1516 			sctlr |= SCTLR_EPAN;
1517 		WRITE_SPECIALREG(sctlr_el1, sctlr);
1518 	}
1519 
1520 	/* Enable DIT. */
1521 	id_aa64pfr0 = READ_SPECIALREG(id_aa64pfr0_el1);
1522 	if (ID_AA64PFR0_DIT(id_aa64pfr0) >= ID_AA64PFR0_DIT_IMPL)
1523 		__asm volatile (".arch armv8.4-a; msr dit, #1");
1524 
1525 	/* Enable PAuth. */
1526 	if (ID_AA64ISAR1_APA(cpu_id_aa64isar1) >= ID_AA64ISAR1_APA_PAC ||
1527 	    ID_AA64ISAR1_API(cpu_id_aa64isar1) >= ID_AA64ISAR1_API_PAC) {
1528 		sctlr = READ_SPECIALREG(sctlr_el1);
1529 		sctlr |= SCTLR_EnIA | SCTLR_EnDA;
1530 		sctlr |= SCTLR_EnIB | SCTLR_EnDB;
1531 		WRITE_SPECIALREG(sctlr_el1, sctlr);
1532 	}
1533 
1534 	/* Enable strict BTI compatibility for PACIASP and PACIBSP. */
1535 	if (ID_AA64PFR1_BT(cpu_id_aa64pfr1) >= ID_AA64PFR1_BT_IMPL) {
1536 		sctlr = READ_SPECIALREG(sctlr_el1);
1537 		sctlr |= SCTLR_BT0 | SCTLR_BT1;
1538 		WRITE_SPECIALREG(sctlr_el1, sctlr);
1539 	}
1540 
1541 	/* Initialize debug registers. */
1542 	WRITE_SPECIALREG(mdscr_el1, DBG_MDSCR_TDCC);
1543 	WRITE_SPECIALREG(oslar_el1, 0);
1544 }
1545 
1546 void
1547 cpu_flush_bp_noop(void)
1548 {
1549 }
1550 
1551 void
1552 cpu_flush_bp_psci(void)
1553 {
1554 #if NPSCI > 0
1555 	psci_flush_bp();
1556 #endif
1557 }
1558 
1559 void
1560 cpu_serror_apple(void)
1561 {
1562 	__asm volatile("dsb sy; isb" ::: "memory");
1563 	printf("l2c_err_sts 0x%llx\n", READ_SPECIALREG(s3_3_c15_c8_0));
1564 	printf("l2c_err_adr 0x%llx\n", READ_SPECIALREG(s3_3_c15_c9_0));
1565 	printf("l2c_err_inf 0x%llx\n", READ_SPECIALREG(s3_3_c15_c10_0));
1566 }
1567 
1568 int
1569 cpu_clockspeed(int *freq)
1570 {
1571 	*freq = clock_get_frequency(cpu_node, NULL) / 1000000;
1572 	return 0;
1573 }
1574 
1575 #ifdef MULTIPROCESSOR
1576 
1577 void cpu_boot_secondary(struct cpu_info *ci);
1578 void cpu_hatch_secondary(void);
1579 void cpu_hatch_secondary_spin(void);
1580 
1581 void cpu_suspend_cycle(void);
1582 
1583 void
1584 cpu_boot_secondary_processors(void)
1585 {
1586 	struct cpu_info *ci;
1587 	CPU_INFO_ITERATOR cii;
1588 
1589 	CPU_INFO_FOREACH(cii, ci) {
1590 		if ((ci->ci_flags & CPUF_AP) == 0)
1591 			continue;
1592 		if (ci->ci_flags & CPUF_PRIMARY)
1593 			continue;
1594 
1595 		ci->ci_randseed = (arc4random() & 0x7fffffff) + 1;
1596 		cpu_boot_secondary(ci);
1597 	}
1598 }
1599 
1600 void
1601 cpu_start_spin_table(struct cpu_info *ci, uint64_t start, uint64_t data)
1602 {
1603 	extern paddr_t cpu_hatch_ci;
1604 
1605 	pmap_extract(pmap_kernel(), (vaddr_t)ci, &cpu_hatch_ci);
1606 	cpu_dcache_wb_range((vaddr_t)&cpu_hatch_ci, sizeof(paddr_t));
1607 
1608 	/* this reuses the zero page for the core */
1609 	vaddr_t start_pg = zero_page + (PAGE_SIZE * ci->ci_cpuid);
1610 	paddr_t pa = trunc_page(data);
1611 	uint64_t offset = data - pa;
1612 	uint64_t *startvec = (uint64_t *)(start_pg + offset);
1613 
1614 	pmap_kenter_cache(start_pg, pa, PROT_READ|PROT_WRITE, PMAP_CACHE_CI);
1615 
1616 	*startvec = start;
1617 	__asm volatile("dsb sy; sev" ::: "memory");
1618 
1619 	pmap_kremove(start_pg, PAGE_SIZE);
1620 }
1621 
1622 int
1623 cpu_start_secondary(struct cpu_info *ci, int method, uint64_t data)
1624 {
1625 	vaddr_t start_va;
1626 	paddr_t ci_pa, start_pa;
1627 	uint64_t ttbr1;
1628 	int32_t status;
1629 
1630 	__asm("mrs %x0, ttbr1_el1": "=r"(ttbr1));
1631 	ci->ci_ttbr1 = ttbr1;
1632 	cpu_dcache_wb_range((vaddr_t)ci, sizeof(*ci));
1633 
1634 	switch (method) {
1635 #if NPSCI > 0
1636 	case 1:
1637 		/* psci */
1638 		start_va = (vaddr_t)cpu_hatch_secondary;
1639 		pmap_extract(pmap_kernel(), start_va, &start_pa);
1640 		pmap_extract(pmap_kernel(), (vaddr_t)ci, &ci_pa);
1641 		status = psci_cpu_on(ci->ci_mpidr, start_pa, ci_pa);
1642 		return (status == PSCI_SUCCESS);
1643 #endif
1644 	case 2:
1645 		/* spin-table */
1646 		start_va = (vaddr_t)cpu_hatch_secondary_spin;
1647 		pmap_extract(pmap_kernel(), start_va, &start_pa);
1648 		cpu_start_spin_table(ci, start_pa, data);
1649 		return 1;
1650 	}
1651 
1652 	return 0;
1653 }
1654 
1655 void
1656 cpu_boot_secondary(struct cpu_info *ci)
1657 {
1658 	atomic_setbits_int(&ci->ci_flags, CPUF_GO);
1659 	__asm volatile("dsb sy; sev" ::: "memory");
1660 
1661 	/*
1662 	 * Send an interrupt as well to make sure the CPU wakes up
1663 	 * regardless of whether it is in a WFE or a WFI loop.
1664 	 */
1665 	arm_send_ipi(ci, ARM_IPI_NOP);
1666 
1667 	while ((ci->ci_flags & CPUF_RUNNING) == 0)
1668 		__asm volatile("wfe");
1669 }
1670 
1671 void
1672 cpu_init_secondary(struct cpu_info *ci)
1673 {
1674 	struct proc *p;
1675 	struct pcb *pcb;
1676 	struct trapframe *tf;
1677 	struct switchframe *sf;
1678 	int s;
1679 
1680 	ci->ci_flags |= CPUF_PRESENT;
1681 	__asm volatile("dsb sy" ::: "memory");
1682 
1683 	if ((ci->ci_flags & CPUF_IDENTIFIED) == 0) {
1684 		while ((ci->ci_flags & CPUF_IDENTIFY) == 0)
1685 			__asm volatile("wfe");
1686 
1687 		cpu_identify(ci);
1688 		atomic_setbits_int(&ci->ci_flags, CPUF_IDENTIFIED);
1689 		__asm volatile("dsb sy" ::: "memory");
1690 	}
1691 
1692 	while ((ci->ci_flags & CPUF_GO) == 0)
1693 		__asm volatile("wfe");
1694 
1695 	cpu_init();
1696 
1697 	/*
1698 	 * Start from a clean slate regardless of whether this is the
1699 	 * initial power up or a wakeup of a suspended CPU.
1700 	 */
1701 
1702 	ci->ci_curproc = NULL;
1703 	ci->ci_curpcb = NULL;
1704 	ci->ci_curpm = NULL;
1705 	ci->ci_cpl = IPL_NONE;
1706 	ci->ci_ipending = 0;
1707 	ci->ci_idepth = 0;
1708 
1709 #ifdef DIAGNOSTIC
1710 	ci->ci_mutex_level = 0;
1711 #endif
1712 
1713 	/*
1714 	 * Re-create the switchframe for this CPUs idle process.
1715 	 */
1716 
1717 	p = ci->ci_schedstate.spc_idleproc;
1718 	pcb = &p->p_addr->u_pcb;
1719 
1720 	tf = (struct trapframe *)((u_long)p->p_addr
1721 	    + USPACE
1722 	    - sizeof(struct trapframe)
1723 	    - 0x10);
1724 
1725 	tf = (struct trapframe *)STACKALIGN(tf);
1726 	pcb->pcb_tf = tf;
1727 
1728 	sf = (struct switchframe *)tf - 1;
1729 	sf->sf_x19 = (uint64_t)sched_idle;
1730 	sf->sf_x20 = (uint64_t)ci;
1731 	sf->sf_lr = (uint64_t)proc_trampoline;
1732 	pcb->pcb_sp = (uint64_t)sf;
1733 
1734 	s = splhigh();
1735 	arm_intr_cpu_enable();
1736 	cpu_startclock();
1737 
1738 	atomic_setbits_int(&ci->ci_flags, CPUF_RUNNING);
1739 	__asm volatile("dsb sy; sev" ::: "memory");
1740 
1741 	spllower(IPL_NONE);
1742 
1743 	sched_toidle();
1744 }
1745 
1746 void
1747 cpu_halt(void)
1748 {
1749 	struct cpu_info *ci = curcpu();
1750 	vaddr_t start_va;
1751 	paddr_t ci_pa, start_pa;
1752 	int count = 0;
1753 	u_long psw;
1754 	int32_t status;
1755 
1756 	KERNEL_ASSERT_UNLOCKED();
1757 	SCHED_ASSERT_UNLOCKED();
1758 
1759 	start_va = (vaddr_t)cpu_hatch_secondary;
1760 	pmap_extract(pmap_kernel(), start_va, &start_pa);
1761 	pmap_extract(pmap_kernel(), (vaddr_t)ci, &ci_pa);
1762 
1763 	psw = intr_disable();
1764 
1765 	atomic_clearbits_int(&ci->ci_flags,
1766 	    CPUF_RUNNING | CPUF_PRESENT | CPUF_GO);
1767 
1768 #if NPSCI > 0
1769 	if (psci_can_suspend())
1770 		psci_cpu_off();
1771 #endif
1772 
1773 	/*
1774 	 * If we failed to turn ourselves off using PSCI, declare that
1775 	 * we're still present and spin in a low power state until
1776 	 * we're told to wake up again by the primary CPU.
1777 	 */
1778 
1779 	atomic_setbits_int(&ci->ci_flags, CPUF_PRESENT);
1780 
1781 	/* Mask clock interrupts. */
1782 	WRITE_SPECIALREG(cntv_ctl_el0,
1783 	    READ_SPECIALREG(cntv_ctl_el0) | CNTV_CTL_IMASK);
1784 
1785 	while ((ci->ci_flags & CPUF_GO) == 0) {
1786 #if NPSCI > 0
1787 		if (ci->ci_psci_suspend_param) {
1788 			status = psci_cpu_suspend(ci->ci_psci_suspend_param,
1789 			    start_pa, ci_pa);
1790 			if (status != PSCI_SUCCESS)
1791 				ci->ci_psci_suspend_param = 0;
1792 		} else
1793 #endif
1794 			cpu_suspend_cycle();
1795 		count++;
1796 	}
1797 
1798 	atomic_setbits_int(&ci->ci_flags, CPUF_RUNNING);
1799 	__asm volatile("dsb sy; sev" ::: "memory");
1800 
1801 	intr_restore(psw);
1802 
1803 	/* Unmask clock interrupts. */
1804 	WRITE_SPECIALREG(cntv_ctl_el0,
1805 	    READ_SPECIALREG(cntv_ctl_el0) & ~CNTV_CTL_IMASK);
1806 }
1807 
1808 void
1809 cpu_kick(struct cpu_info *ci)
1810 {
1811 	/* force cpu to enter kernel */
1812 	if (ci != curcpu())
1813 		arm_send_ipi(ci, ARM_IPI_NOP);
1814 }
1815 
1816 void
1817 cpu_unidle(struct cpu_info *ci)
1818 {
1819 	/*
1820 	 * This could send IPI or SEV depending on if the other
1821 	 * processor is sleeping (WFI or WFE), in userland, or if the
1822 	 * cpu is in other possible wait states?
1823 	 */
1824 	if (ci != curcpu())
1825 		arm_send_ipi(ci, ARM_IPI_NOP);
1826 }
1827 
1828 #endif
1829 
1830 int cpu_suspended;
1831 
1832 #ifdef SUSPEND
1833 
1834 void cpu_hatch_primary(void);
1835 
1836 void (*cpu_suspend_cycle_fcn)(void) = cpu_wfi;
1837 label_t cpu_suspend_jmpbuf;
1838 
1839 void
1840 cpu_suspend_cycle(void)
1841 {
1842 	cpu_suspend_cycle_fcn();
1843 }
1844 
1845 void
1846 cpu_init_primary(void)
1847 {
1848 	cpu_init();
1849 
1850 	cpu_startclock();
1851 
1852 	longjmp(&cpu_suspend_jmpbuf);
1853 }
1854 
1855 int
1856 cpu_suspend_primary(void)
1857 {
1858 	struct cpu_info *ci = curcpu();
1859 	vaddr_t start_va;
1860 	paddr_t ci_pa, start_pa;
1861 	uint64_t ttbr1;
1862 	int32_t status;
1863 	int count = 0;
1864 
1865 	__asm("mrs %x0, ttbr1_el1": "=r"(ttbr1));
1866 	ci->ci_ttbr1 = ttbr1;
1867 	cpu_dcache_wb_range((vaddr_t)ci, sizeof(*ci));
1868 
1869 	start_va = (vaddr_t)cpu_hatch_primary;
1870 	pmap_extract(pmap_kernel(), start_va, &start_pa);
1871 	pmap_extract(pmap_kernel(), (vaddr_t)ci, &ci_pa);
1872 
1873 #if NPSCI > 0
1874 	if (psci_can_suspend()) {
1875 		if (setjmp(&cpu_suspend_jmpbuf)) {
1876 			/* XXX wait for debug output on Allwinner A64 */
1877 			delay(200000);
1878 			return 0;
1879 		}
1880 
1881 		psci_system_suspend(start_pa, ci_pa);
1882 
1883 		return EOPNOTSUPP;
1884 	}
1885 #endif
1886 
1887 	if (setjmp(&cpu_suspend_jmpbuf))
1888 		goto resume;
1889 
1890 	/*
1891 	 * If PSCI doesn't support SYSTEM_SUSPEND, spin in a low power
1892 	 * state waiting for an interrupt that wakes us up again.
1893 	 */
1894 
1895 	/* Mask clock interrupts. */
1896 	WRITE_SPECIALREG(cntv_ctl_el0,
1897 	    READ_SPECIALREG(cntv_ctl_el0) | CNTV_CTL_IMASK);
1898 
1899 	/*
1900 	 * All non-wakeup interrupts should be masked at this point;
1901 	 * re-enable interrupts such that wakeup interrupts actually
1902 	 * wake us up.  Set a flag such that drivers can tell we're
1903 	 * suspended and change their behaviour accordingly.  They can
1904 	 * wake us up by clearing the flag.
1905 	 */
1906 	cpu_suspended = 1;
1907 	arm_intr_func.setipl(IPL_NONE);
1908 	intr_enable();
1909 
1910 	while (cpu_suspended) {
1911 #if NPSCI > 0
1912 		if (ci->ci_psci_suspend_param) {
1913 			status = psci_cpu_suspend(ci->ci_psci_suspend_param,
1914 			    start_pa, ci_pa);
1915 			if (status != PSCI_SUCCESS)
1916 				ci->ci_psci_suspend_param = 0;
1917 		} else
1918 #endif
1919 			cpu_suspend_cycle();
1920 		count++;
1921 	}
1922 
1923 resume:
1924 	intr_disable();
1925 	arm_intr_func.setipl(IPL_HIGH);
1926 
1927 	/* Unmask clock interrupts. */
1928 	WRITE_SPECIALREG(cntv_ctl_el0,
1929 	    READ_SPECIALREG(cntv_ctl_el0) & ~CNTV_CTL_IMASK);
1930 
1931 	return 0;
1932 }
1933 
1934 #ifdef MULTIPROCESSOR
1935 
1936 void
1937 cpu_resume_secondary(struct cpu_info *ci)
1938 {
1939 	int timeout = 10000;
1940 
1941 	if (ci->ci_flags & CPUF_PRESENT)
1942 		return;
1943 
1944 	cpu_start_secondary(ci, 1, 0);
1945 	while ((ci->ci_flags & CPUF_PRESENT) == 0 && --timeout)
1946 		delay(1000);
1947 	if (timeout == 0) {
1948 		printf("%s: failed to spin up\n",
1949 		    ci->ci_dev->dv_xname);
1950 		ci->ci_flags = 0;
1951 	}
1952 }
1953 
1954 #endif
1955 
1956 #endif
1957 
1958 /*
1959  * Dynamic voltage and frequency scaling implementation.
1960  */
1961 
1962 extern int perflevel;
1963 
1964 struct opp {
1965 	uint64_t opp_hz;
1966 	uint32_t opp_microvolt;
1967 };
1968 
1969 struct opp_table {
1970 	LIST_ENTRY(opp_table) ot_list;
1971 	uint32_t ot_phandle;
1972 
1973 	struct opp *ot_opp;
1974 	u_int ot_nopp;
1975 	uint64_t ot_opp_hz_min;
1976 	uint64_t ot_opp_hz_max;
1977 
1978 	struct cpu_info *ot_master;
1979 };
1980 
1981 LIST_HEAD(, opp_table) opp_tables = LIST_HEAD_INITIALIZER(opp_tables);
1982 struct task cpu_opp_task;
1983 
1984 void	cpu_opp_mountroot(struct device *);
1985 void	cpu_opp_dotask(void *);
1986 void	cpu_opp_setperf(int);
1987 
1988 uint32_t cpu_opp_get_cooling_level(void *, uint32_t *);
1989 void	cpu_opp_set_cooling_level(void *, uint32_t *, uint32_t);
1990 
1991 void
1992 cpu_opp_init(struct cpu_info *ci, uint32_t phandle)
1993 {
1994 	struct opp_table *ot;
1995 	struct cooling_device *cd;
1996 	int count, node, child;
1997 	uint32_t opp_hz, opp_microvolt;
1998 	uint32_t values[3];
1999 	int i, j, len;
2000 
2001 	LIST_FOREACH(ot, &opp_tables, ot_list) {
2002 		if (ot->ot_phandle == phandle) {
2003 			ci->ci_opp_table = ot;
2004 			return;
2005 		}
2006 	}
2007 
2008 	node = OF_getnodebyphandle(phandle);
2009 	if (node == 0)
2010 		return;
2011 
2012 	if (!OF_is_compatible(node, "operating-points-v2"))
2013 		return;
2014 
2015 	count = 0;
2016 	for (child = OF_child(node); child != 0; child = OF_peer(child)) {
2017 		if (OF_getproplen(child, "turbo-mode") == 0)
2018 			continue;
2019 		count++;
2020 	}
2021 	if (count == 0)
2022 		return;
2023 
2024 	ot = malloc(sizeof(struct opp_table), M_DEVBUF, M_ZERO | M_WAITOK);
2025 	ot->ot_phandle = phandle;
2026 	ot->ot_opp = mallocarray(count, sizeof(struct opp),
2027 	    M_DEVBUF, M_ZERO | M_WAITOK);
2028 	ot->ot_nopp = count;
2029 
2030 	count = 0;
2031 	for (child = OF_child(node); child != 0; child = OF_peer(child)) {
2032 		if (OF_getproplen(child, "turbo-mode") == 0)
2033 			continue;
2034 		opp_hz = OF_getpropint64(child, "opp-hz", 0);
2035 		len = OF_getpropintarray(child, "opp-microvolt",
2036 		    values, sizeof(values));
2037 		opp_microvolt = 0;
2038 		if (len == sizeof(uint32_t) || len == 3 * sizeof(uint32_t))
2039 			opp_microvolt = values[0];
2040 
2041 		/* Insert into the array, keeping things sorted. */
2042 		for (i = 0; i < count; i++) {
2043 			if (opp_hz < ot->ot_opp[i].opp_hz)
2044 				break;
2045 		}
2046 		for (j = count; j > i; j--)
2047 			ot->ot_opp[j] = ot->ot_opp[j - 1];
2048 		ot->ot_opp[i].opp_hz = opp_hz;
2049 		ot->ot_opp[i].opp_microvolt = opp_microvolt;
2050 		count++;
2051 	}
2052 
2053 	ot->ot_opp_hz_min = ot->ot_opp[0].opp_hz;
2054 	ot->ot_opp_hz_max = ot->ot_opp[count - 1].opp_hz;
2055 
2056 	if (OF_getproplen(node, "opp-shared") == 0)
2057 		ot->ot_master = ci;
2058 
2059 	LIST_INSERT_HEAD(&opp_tables, ot, ot_list);
2060 
2061 	ci->ci_opp_table = ot;
2062 	ci->ci_opp_max = ot->ot_nopp - 1;
2063 	ci->ci_cpu_supply = OF_getpropint(ci->ci_node, "cpu-supply", 0);
2064 
2065 	cd = malloc(sizeof(struct cooling_device), M_DEVBUF, M_ZERO | M_WAITOK);
2066 	cd->cd_node = ci->ci_node;
2067 	cd->cd_cookie = ci;
2068 	cd->cd_get_level = cpu_opp_get_cooling_level;
2069 	cd->cd_set_level = cpu_opp_set_cooling_level;
2070 	cooling_device_register(cd);
2071 
2072 	/*
2073 	 * Do additional checks at mountroot when all the clocks and
2074 	 * regulators are available.
2075 	 */
2076 	config_mountroot(ci->ci_dev, cpu_opp_mountroot);
2077 }
2078 
2079 void
2080 cpu_opp_mountroot(struct device *self)
2081 {
2082 	struct cpu_info *ci;
2083 	CPU_INFO_ITERATOR cii;
2084 	int count = 0;
2085 	int level = 0;
2086 
2087 	if (cpu_setperf)
2088 		return;
2089 
2090 	CPU_INFO_FOREACH(cii, ci) {
2091 		struct opp_table *ot = ci->ci_opp_table;
2092 		uint64_t curr_hz;
2093 		uint32_t curr_microvolt;
2094 		int error;
2095 
2096 		if (ot == NULL)
2097 			continue;
2098 
2099 #if NKSTAT > 0
2100 		cpu_opp_kstat_attach(ci);
2101 #endif
2102 
2103 		/* Skip if this table is shared and we're not the master. */
2104 		if (ot->ot_master && ot->ot_master != ci)
2105 			continue;
2106 
2107 		/* PWM regulators may need to be explicitly enabled. */
2108 		regulator_enable(ci->ci_cpu_supply);
2109 
2110 		curr_hz = clock_get_frequency(ci->ci_node, NULL);
2111 		curr_microvolt = regulator_get_voltage(ci->ci_cpu_supply);
2112 
2113 		/* Disable if clock isn't implemented. */
2114 		error = ENODEV;
2115 		if (curr_hz != 0)
2116 			error = clock_set_frequency(ci->ci_node, NULL, curr_hz);
2117 		if (error) {
2118 			ci->ci_opp_table = NULL;
2119 			printf("%s: clock not implemented\n",
2120 			       ci->ci_dev->dv_xname);
2121 			continue;
2122 		}
2123 
2124 		/* Disable if regulator isn't implemented. */
2125 		error = ci->ci_cpu_supply ? ENODEV : 0;
2126 		if (ci->ci_cpu_supply && curr_microvolt != 0)
2127 			error = regulator_set_voltage(ci->ci_cpu_supply,
2128 			    curr_microvolt);
2129 		if (error) {
2130 			ci->ci_opp_table = NULL;
2131 			printf("%s: regulator not implemented\n",
2132 			    ci->ci_dev->dv_xname);
2133 			continue;
2134 		}
2135 
2136 		/*
2137 		 * Initialize performance level based on the current
2138 		 * speed of the first CPU that supports DVFS.
2139 		 */
2140 		if (level == 0) {
2141 			uint64_t min, max;
2142 			uint64_t level_hz;
2143 
2144 			min = ot->ot_opp_hz_min;
2145 			max = ot->ot_opp_hz_max;
2146 			level_hz = clock_get_frequency(ci->ci_node, NULL);
2147 			if (level_hz < min)
2148 				level_hz = min;
2149 			if (level_hz > max)
2150 				level_hz = max;
2151 			level = howmany(100 * (level_hz - min), (max - min));
2152 		}
2153 
2154 		count++;
2155 	}
2156 
2157 	if (count > 0) {
2158 		task_set(&cpu_opp_task, cpu_opp_dotask, NULL);
2159 		cpu_setperf = cpu_opp_setperf;
2160 
2161 		perflevel = (level > 0) ? level : 0;
2162 		cpu_setperf(perflevel);
2163 	}
2164 }
2165 
2166 void
2167 cpu_opp_dotask(void *arg)
2168 {
2169 	struct cpu_info *ci;
2170 	CPU_INFO_ITERATOR cii;
2171 
2172 	CPU_INFO_FOREACH(cii, ci) {
2173 		struct opp_table *ot = ci->ci_opp_table;
2174 		uint64_t curr_hz, opp_hz;
2175 		uint32_t curr_microvolt, opp_microvolt;
2176 		int opp_idx;
2177 		int error = 0;
2178 
2179 		if (ot == NULL)
2180 			continue;
2181 
2182 		/* Skip if this table is shared and we're not the master. */
2183 		if (ot->ot_master && ot->ot_master != ci)
2184 			continue;
2185 
2186 		opp_idx = MIN(ci->ci_opp_idx, ci->ci_opp_max);
2187 		opp_hz = ot->ot_opp[opp_idx].opp_hz;
2188 		opp_microvolt = ot->ot_opp[opp_idx].opp_microvolt;
2189 
2190 		curr_hz = clock_get_frequency(ci->ci_node, NULL);
2191 		curr_microvolt = regulator_get_voltage(ci->ci_cpu_supply);
2192 
2193 		if (error == 0 && opp_hz < curr_hz)
2194 			error = clock_set_frequency(ci->ci_node, NULL, opp_hz);
2195 		if (error == 0 && ci->ci_cpu_supply &&
2196 		    opp_microvolt != 0 && opp_microvolt != curr_microvolt) {
2197 			error = regulator_set_voltage(ci->ci_cpu_supply,
2198 			    opp_microvolt);
2199 		}
2200 		if (error == 0 && opp_hz > curr_hz)
2201 			error = clock_set_frequency(ci->ci_node, NULL, opp_hz);
2202 
2203 		if (error)
2204 			printf("%s: DVFS failed\n", ci->ci_dev->dv_xname);
2205 	}
2206 }
2207 
2208 void
2209 cpu_opp_setperf(int level)
2210 {
2211 	struct cpu_info *ci;
2212 	CPU_INFO_ITERATOR cii;
2213 
2214 	CPU_INFO_FOREACH(cii, ci) {
2215 		struct opp_table *ot = ci->ci_opp_table;
2216 		uint64_t min, max;
2217 		uint64_t level_hz, opp_hz;
2218 		int opp_idx = -1;
2219 		int i;
2220 
2221 		if (ot == NULL)
2222 			continue;
2223 
2224 		/* Skip if this table is shared and we're not the master. */
2225 		if (ot->ot_master && ot->ot_master != ci)
2226 			continue;
2227 
2228 		min = ot->ot_opp_hz_min;
2229 		max = ot->ot_opp_hz_max;
2230 		level_hz = min + (level * (max - min)) / 100;
2231 		opp_hz = min;
2232 		for (i = 0; i < ot->ot_nopp; i++) {
2233 			if (ot->ot_opp[i].opp_hz <= level_hz &&
2234 			    ot->ot_opp[i].opp_hz >= opp_hz)
2235 				opp_hz = ot->ot_opp[i].opp_hz;
2236 		}
2237 
2238 		/* Find index of selected operating point. */
2239 		for (i = 0; i < ot->ot_nopp; i++) {
2240 			if (ot->ot_opp[i].opp_hz == opp_hz) {
2241 				opp_idx = i;
2242 				break;
2243 			}
2244 		}
2245 		KASSERT(opp_idx >= 0);
2246 
2247 		ci->ci_opp_idx = opp_idx;
2248 	}
2249 
2250 	/*
2251 	 * Update the hardware from a task since setting the
2252 	 * regulators might need process context.
2253 	 */
2254 	task_add(systq, &cpu_opp_task);
2255 }
2256 
2257 uint32_t
2258 cpu_opp_get_cooling_level(void *cookie, uint32_t *cells)
2259 {
2260 	struct cpu_info *ci = cookie;
2261 	struct opp_table *ot = ci->ci_opp_table;
2262 
2263 	return ot->ot_nopp - ci->ci_opp_max - 1;
2264 }
2265 
2266 void
2267 cpu_opp_set_cooling_level(void *cookie, uint32_t *cells, uint32_t level)
2268 {
2269 	struct cpu_info *ci = cookie;
2270 	struct opp_table *ot = ci->ci_opp_table;
2271 	int opp_max;
2272 
2273 	if (level > (ot->ot_nopp - 1))
2274 		level = ot->ot_nopp - 1;
2275 
2276 	opp_max = (ot->ot_nopp - level - 1);
2277 	if (ci->ci_opp_max != opp_max) {
2278 		ci->ci_opp_max = opp_max;
2279 		task_add(systq, &cpu_opp_task);
2280 	}
2281 }
2282 
2283 
2284 void
2285 cpu_psci_init(struct cpu_info *ci)
2286 {
2287 	uint32_t *domains;
2288 	uint32_t *domain;
2289 	uint32_t *states;
2290 	uint32_t ncells;
2291 	uint32_t cluster;
2292 	int idx, len, node;
2293 
2294 	/*
2295 	 * Find the shallowest (for now) idle state for this CPU.
2296 	 * This should be the first one that is listed.  We'll use it
2297 	 * in the idle loop.
2298 	 */
2299 
2300 	len = OF_getproplen(ci->ci_node, "cpu-idle-states");
2301 	if (len < (int)sizeof(uint32_t))
2302 		return;
2303 
2304 	states = malloc(len, M_TEMP, M_WAITOK);
2305 	OF_getpropintarray(ci->ci_node, "cpu-idle-states", states, len);
2306 	node = OF_getnodebyphandle(states[0]);
2307 	free(states, M_TEMP, len);
2308 	if (node) {
2309 		uint32_t entry, exit, residency, param;
2310 		int32_t features;
2311 
2312 		param = OF_getpropint(node, "arm,psci-suspend-param", 0);
2313 		entry = OF_getpropint(node, "entry-latency-us", 0);
2314 		exit = OF_getpropint(node, "exit-latency-us", 0);
2315 		residency = OF_getpropint(node, "min-residency-us", 0);
2316 		ci->ci_psci_idle_latency += entry + exit + 2 * residency;
2317 
2318 		/* Skip states that stop the local timer. */
2319 		if (OF_getpropbool(node, "local-timer-stop"))
2320 			ci->ci_psci_idle_param = 0;
2321 
2322 		/* Skip powerdown states. */
2323 		features = psci_features(CPU_SUSPEND);
2324 		if (features == PSCI_NOT_SUPPORTED ||
2325 		    (features & PSCI_FEATURE_POWER_STATE_EXT) == 0) {
2326 			if (param & PSCI_POWER_STATE_POWERDOWN)
2327 				param = 0;
2328 		} else {
2329 			if (param & PSCI_POWER_STATE_EXT_POWERDOWN)
2330 				param = 0;
2331 		}
2332 
2333 		if (param) {
2334 			ci->ci_psci_idle_param = param;
2335 			cpu_idle_cycle_fcn = cpu_psci_idle_cycle;
2336 		}
2337 	}
2338 
2339 	/*
2340 	 * Hunt for the deepest idle state for this CPU.  This is
2341 	 * fairly complicated as it requires traversing quite a few
2342 	 * nodes in the device tree.  The first step is to look up the
2343 	 * "psci" power domain for this CPU.
2344 	 */
2345 
2346 	idx = OF_getindex(ci->ci_node, "psci", "power-domain-names");
2347 	if (idx < 0)
2348 		return;
2349 
2350 	len = OF_getproplen(ci->ci_node, "power-domains");
2351 	if (len <= 0)
2352 		return;
2353 
2354 	domains = malloc(len, M_TEMP, M_WAITOK);
2355 	OF_getpropintarray(ci->ci_node, "power-domains", domains, len);
2356 
2357 	domain = domains;
2358 	while (domain && domain < domains + (len / sizeof(uint32_t))) {
2359 		if (idx == 0)
2360 			break;
2361 
2362 		node = OF_getnodebyphandle(domain[0]);
2363 		if (node == 0)
2364 			break;
2365 
2366 		ncells = OF_getpropint(node, "#power-domain-cells", 0);
2367 		domain = domain + ncells + 1;
2368 		idx--;
2369 	}
2370 
2371 	node = idx == 0 ? OF_getnodebyphandle(domain[0]) : 0;
2372 	free(domains, M_TEMP, len);
2373 	if (node == 0)
2374 		return;
2375 
2376 	/*
2377 	 * We found the "psci" power domain.  If this power domain has
2378 	 * a parent power domain, stash its phandle away for later.
2379 	 */
2380 
2381 	cluster = OF_getpropint(node, "power-domains", 0);
2382 
2383 	/*
2384 	 * Get the deepest idle state for the CPU; this should be the
2385 	 * last one that is listed.
2386 	 */
2387 
2388 	len = OF_getproplen(node, "domain-idle-states");
2389 	if (len < (int)sizeof(uint32_t))
2390 		return;
2391 
2392 	states = malloc(len, M_TEMP, M_WAITOK);
2393 	OF_getpropintarray(node, "domain-idle-states", states, len);
2394 
2395 	node = OF_getnodebyphandle(states[len / sizeof(uint32_t) - 1]);
2396 	free(states, M_TEMP, len);
2397 	if (node == 0)
2398 		return;
2399 
2400 	ci->ci_psci_suspend_param =
2401 		OF_getpropint(node, "arm,psci-suspend-param", 0);
2402 
2403 	/*
2404 	 * Qualcomm Snapdragon always seem to operate in OS Initiated
2405 	 * mode.  This means that the last CPU to suspend can pick the
2406 	 * idle state that powers off the entire cluster.  In our case
2407 	 * that will always be the primary CPU.
2408 	 */
2409 
2410 #ifdef MULTIPROCESSOR
2411 	if (ci->ci_flags & CPUF_AP)
2412 		return;
2413 #endif
2414 
2415 	node = OF_getnodebyphandle(cluster);
2416 	if (node == 0)
2417 		return;
2418 
2419 	/*
2420 	 * Get the deepest idle state for the cluster; this should be
2421 	 * the last one that is listed.
2422 	 */
2423 
2424 	states = malloc(len, M_TEMP, M_WAITOK);
2425 	OF_getpropintarray(node, "domain-idle-states", states, len);
2426 
2427 	node = OF_getnodebyphandle(states[len / sizeof(uint32_t) - 1]);
2428 	free(states, M_TEMP, len);
2429 	if (node == 0)
2430 		return;
2431 
2432 	ci->ci_psci_suspend_param =
2433 		OF_getpropint(node, "arm,psci-suspend-param", 0);
2434 }
2435 
2436 void
2437 cpu_psci_idle_cycle(void)
2438 {
2439 	struct cpu_info *ci = curcpu();
2440 	struct timeval start, stop;
2441 	u_long itime;
2442 
2443 	microuptime(&start);
2444 
2445 	if (ci->ci_prev_sleep > ci->ci_psci_idle_latency)
2446 		psci_cpu_suspend(ci->ci_psci_idle_param, 0, 0);
2447 	else
2448 		cpu_wfi();
2449 
2450 	microuptime(&stop);
2451 	timersub(&stop, &start, &stop);
2452 	itime = stop.tv_sec * 1000000 + stop.tv_usec;
2453 
2454 	ci->ci_last_itime = itime;
2455 	itime >>= 1;
2456 	ci->ci_prev_sleep = (ci->ci_prev_sleep + (ci->ci_prev_sleep >> 1)
2457 	    + itime) >> 1;
2458 }
2459 
2460 #if NKSTAT > 0
2461 
2462 struct cpu_kstats {
2463 	struct kstat_kv		ck_impl;
2464 	struct kstat_kv		ck_part;
2465 	struct kstat_kv		ck_rev;
2466 };
2467 
2468 void
2469 cpu_kstat_attach(struct cpu_info *ci)
2470 {
2471 	struct kstat *ks;
2472 	struct cpu_kstats *ck;
2473 	uint64_t impl, part;
2474 	const char *impl_name = NULL, *part_name = NULL;
2475 	const struct cpu_cores *coreselecter = cpu_cores_none;
2476 	int i;
2477 
2478 	ks = kstat_create(ci->ci_dev->dv_xname, 0, "mach", 0, KSTAT_T_KV, 0);
2479 	if (ks == NULL) {
2480 		printf("%s: unable to create cpu kstats\n",
2481 		    ci->ci_dev->dv_xname);
2482 		return;
2483 	}
2484 
2485 	ck = malloc(sizeof(*ck), M_DEVBUF, M_WAITOK);
2486 
2487 	impl = CPU_IMPL(ci->ci_midr);
2488 	part = CPU_PART(ci->ci_midr);
2489 
2490 	for (i = 0; cpu_implementers[i].name; i++) {
2491 		if (impl == cpu_implementers[i].id) {
2492 			impl_name = cpu_implementers[i].name;
2493 			coreselecter = cpu_implementers[i].corelist;
2494 			break;
2495 		}
2496 	}
2497 
2498 	if (impl_name) {
2499 		kstat_kv_init(&ck->ck_impl, "impl", KSTAT_KV_T_ISTR);
2500 		strlcpy(kstat_kv_istr(&ck->ck_impl), impl_name,
2501 		    sizeof(kstat_kv_istr(&ck->ck_impl)));
2502 	} else
2503 		kstat_kv_init(&ck->ck_impl, "impl", KSTAT_KV_T_NULL);
2504 
2505 	for (i = 0; coreselecter[i].name; i++) {
2506 		if (part == coreselecter[i].id) {
2507 			part_name = coreselecter[i].name;
2508 			break;
2509 		}
2510 	}
2511 
2512 	if (part_name) {
2513 		kstat_kv_init(&ck->ck_part, "part", KSTAT_KV_T_ISTR);
2514 		strlcpy(kstat_kv_istr(&ck->ck_part), part_name,
2515 		    sizeof(kstat_kv_istr(&ck->ck_part)));
2516 	} else
2517 		kstat_kv_init(&ck->ck_part, "part", KSTAT_KV_T_NULL);
2518 
2519 	kstat_kv_init(&ck->ck_rev, "rev", KSTAT_KV_T_ISTR);
2520 	snprintf(kstat_kv_istr(&ck->ck_rev), sizeof(kstat_kv_istr(&ck->ck_rev)),
2521 	    "r%llup%llu", CPU_VAR(ci->ci_midr), CPU_REV(ci->ci_midr));
2522 
2523 	ks->ks_softc = ci;
2524 	ks->ks_data = ck;
2525 	ks->ks_datalen = sizeof(*ck);
2526 	ks->ks_read = kstat_read_nop;
2527 
2528 	kstat_install(ks);
2529 
2530 	/* XXX should we have a ci->ci_kstat = ks? */
2531 }
2532 
2533 struct cpu_opp_kstats {
2534 	struct kstat_kv		coppk_freq;
2535 	struct kstat_kv		coppk_supply_v;
2536 };
2537 
2538 int
2539 cpu_opp_kstat_read(struct kstat *ks)
2540 {
2541 	struct cpu_info *ci = ks->ks_softc;
2542 	struct cpu_opp_kstats *coppk = ks->ks_data;
2543 
2544 	struct opp_table *ot = ci->ci_opp_table;
2545 	struct cpu_info *oci;
2546 	struct timespec now, diff;
2547 
2548 	/* rate limit */
2549 	getnanouptime(&now);
2550 	timespecsub(&now, &ks->ks_updated, &diff);
2551 	if (diff.tv_sec < 1)
2552 		return (0);
2553 
2554 	if (ot == NULL)
2555 		return (0);
2556 
2557 	oci = ot->ot_master;
2558 	if (oci == NULL)
2559 		oci = ci;
2560 
2561 	kstat_kv_freq(&coppk->coppk_freq) =
2562 	    clock_get_frequency(oci->ci_node, NULL);
2563 
2564 	if (oci->ci_cpu_supply) {
2565 		kstat_kv_volts(&coppk->coppk_supply_v) =
2566 		    regulator_get_voltage(oci->ci_cpu_supply);
2567 	}
2568 
2569 	ks->ks_updated = now;
2570 
2571 	return (0);
2572 }
2573 
2574 void
2575 cpu_opp_kstat_attach(struct cpu_info *ci)
2576 {
2577 	struct kstat *ks;
2578 	struct cpu_opp_kstats *coppk;
2579 	struct opp_table *ot = ci->ci_opp_table;
2580 	struct cpu_info *oci = ot->ot_master;
2581 
2582 	if (oci == NULL)
2583 		oci = ci;
2584 
2585 	ks = kstat_create(ci->ci_dev->dv_xname, 0, "dt-opp", 0,
2586 	    KSTAT_T_KV, 0);
2587 	if (ks == NULL) {
2588 		printf("%s: unable to create cpu dt-opp kstats\n",
2589 		    ci->ci_dev->dv_xname);
2590 		return;
2591 	}
2592 
2593 	coppk = malloc(sizeof(*coppk), M_DEVBUF, M_WAITOK);
2594 
2595 	kstat_kv_init(&coppk->coppk_freq, "freq", KSTAT_KV_T_FREQ);
2596 	kstat_kv_init(&coppk->coppk_supply_v, "supply",
2597 	    oci->ci_cpu_supply ? KSTAT_KV_T_VOLTS_DC : KSTAT_KV_T_NULL);
2598 
2599 	ks->ks_softc = oci;
2600 	ks->ks_data = coppk;
2601 	ks->ks_datalen = sizeof(*coppk);
2602 	ks->ks_read = cpu_opp_kstat_read;
2603 
2604 	kstat_install(ks);
2605 
2606 	/* XXX should we have a ci->ci_opp_kstat = ks? */
2607 }
2608 
2609 #endif /* NKSTAT > 0 */
2610