1 /* $OpenBSD: cpu.c,v 1.60 2024/06/11 15:44:55 kettenis Exp $ */
2 /* $NetBSD: cpu.c,v 1.56 2004/04/14 04:01:49 bsh Exp $ */
3
4
5 /*
6 * Copyright (c) 1995 Mark Brinicombe.
7 * Copyright (c) 1995 Brini.
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. All advertising materials mentioning features or use of this software
19 * must display the following acknowledgement:
20 * This product includes software developed by Brini.
21 * 4. The name of the company nor the name of the author may be used to
22 * endorse or promote products derived from this software without specific
23 * prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY BRINI ``AS IS'' AND ANY EXPRESS OR IMPLIED
26 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
27 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
28 * IN NO EVENT SHALL BRINI OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
29 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
30 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
31 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * SUCH DAMAGE.
36 *
37 * RiscBSD kernel project
38 *
39 * cpu.c
40 *
41 * Probing and configuration for the master CPU
42 *
43 * Created : 10/10/95
44 */
45
46 #include <sys/param.h>
47 #include <sys/systm.h>
48 #include <sys/malloc.h>
49 #include <sys/device.h>
50 #include <sys/sched.h>
51 #include <sys/sysctl.h>
52 #include <sys/task.h>
53
54 #include <machine/cpu.h>
55 #include <machine/fdt.h>
56
57 #include <arm/cpufunc.h>
58 #include <arm/vfp.h>
59
60 #include <dev/ofw/openfirm.h>
61 #include <dev/ofw/ofw_clock.h>
62 #include <dev/ofw/ofw_regulator.h>
63 #include <dev/ofw/ofw_thermal.h>
64 #include <dev/ofw/fdt.h>
65
66 #include "psci.h"
67 #if NPSCI > 0
68 #include <dev/fdt/pscivar.h>
69 #endif
70
71 /* CPU Identification */
72 #define CPU_IMPL_ARM 0x41
73 #define CPU_IMPL_AMCC 0x50
74
75 #define CPU_PART_CORTEX_A5 0xc05
76 #define CPU_PART_CORTEX_A7 0xc07
77 #define CPU_PART_CORTEX_A8 0xc08
78 #define CPU_PART_CORTEX_A9 0xc09
79 #define CPU_PART_CORTEX_A12 0xc0d
80 #define CPU_PART_CORTEX_A15 0xc0f
81 #define CPU_PART_CORTEX_A17 0xc0e
82 #define CPU_PART_CORTEX_A32 0xd01
83 #define CPU_PART_CORTEX_A53 0xd03
84 #define CPU_PART_CORTEX_A35 0xd04
85 #define CPU_PART_CORTEX_A55 0xd05
86 #define CPU_PART_CORTEX_A57 0xd07
87 #define CPU_PART_CORTEX_A72 0xd08
88 #define CPU_PART_CORTEX_A73 0xd09
89 #define CPU_PART_CORTEX_A75 0xd0a
90
91 #define CPU_PART_X_GENE 0x000
92
93 #define CPU_IMPL(midr) (((midr) >> 24) & 0xff)
94 #define CPU_PART(midr) (((midr) >> 4) & 0xfff)
95 #define CPU_VAR(midr) (((midr) >> 20) & 0xf)
96 #define CPU_REV(midr) (((midr) >> 0) & 0xf)
97
98 struct cpu_cores {
99 int id;
100 char *name;
101 };
102
103 struct cpu_cores cpu_cores_none[] = {
104 { 0, NULL },
105 };
106
107 struct cpu_cores cpu_cores_arm[] = {
108 { CPU_PART_CORTEX_A5, "Cortex-A5" },
109 { CPU_PART_CORTEX_A7, "Cortex-A7" },
110 { CPU_PART_CORTEX_A8, "Cortex-A8" },
111 { CPU_PART_CORTEX_A9, "Cortex-A9" },
112 { CPU_PART_CORTEX_A12, "Cortex-A12" },
113 { CPU_PART_CORTEX_A15, "Cortex-A15" },
114 { CPU_PART_CORTEX_A17, "Cortex-A17" },
115 { CPU_PART_CORTEX_A32, "Cortex-A32" },
116 { CPU_PART_CORTEX_A35, "Cortex-A35" },
117 { CPU_PART_CORTEX_A53, "Cortex-A53" },
118 { CPU_PART_CORTEX_A55, "Cortex-A55" },
119 { CPU_PART_CORTEX_A57, "Cortex-A57" },
120 { CPU_PART_CORTEX_A72, "Cortex-A72" },
121 { CPU_PART_CORTEX_A73, "Cortex-A73" },
122 { CPU_PART_CORTEX_A75, "Cortex-A75" },
123 { 0, NULL },
124 };
125
126 struct cpu_cores cpu_cores_amcc[] = {
127 { CPU_PART_X_GENE, "X-Gene" },
128 { 0, NULL },
129 };
130
131 /* arm cores makers */
132 const struct implementers {
133 int id;
134 char *name;
135 struct cpu_cores *corelist;
136 } cpu_implementers[] = {
137 { CPU_IMPL_ARM, "ARM", cpu_cores_arm },
138 { CPU_IMPL_AMCC, "Applied Micro", cpu_cores_amcc },
139 { 0, NULL },
140 };
141
142 char cpu_model[64];
143 int cpu_node;
144
145 int cpu_match(struct device *, void *, void *);
146 void cpu_attach(struct device *, struct device *, void *);
147
148 const struct cfattach cpu_ca = {
149 sizeof(struct device), cpu_match, cpu_attach
150 };
151
152 struct cfdriver cpu_cd = {
153 NULL, "cpu", DV_DULL
154 };
155
156 void cpu_opp_init_legacy(struct cpu_info *);
157 void cpu_opp_init(struct cpu_info *, uint32_t);
158
159 void cpu_flush_bp_noop(void);
160
161 void
cpu_identify(struct cpu_info * ci)162 cpu_identify(struct cpu_info *ci)
163 {
164 uint32_t midr, impl, part;
165 uint32_t clidr;
166 uint32_t ctr, ccsidr, sets, ways, line;
167 const char *impl_name = NULL;
168 const char *part_name = NULL;
169 const char *il1p_name = NULL;
170 const char *sep;
171 struct cpu_cores *coreselecter = cpu_cores_none;
172 int i;
173
174 __asm volatile("mrc p15, 0, %0, c0, c0, 0" : "=r"(midr));
175 impl = CPU_IMPL(midr);
176 part = CPU_PART(midr);
177
178 for (i = 0; cpu_implementers[i].name; i++) {
179 if (impl == cpu_implementers[i].id) {
180 impl_name = cpu_implementers[i].name;
181 coreselecter = cpu_implementers[i].corelist;
182 break;
183 }
184 }
185
186 for (i = 0; coreselecter[i].name; i++) {
187 if (part == coreselecter[i].id) {
188 part_name = coreselecter[i].name;
189 break;
190 }
191 }
192
193 if (impl_name && part_name) {
194 printf(" %s %s r%up%u", impl_name, part_name, CPU_VAR(midr),
195 CPU_REV(midr));
196
197 if (CPU_IS_PRIMARY(ci))
198 snprintf(cpu_model, sizeof(cpu_model),
199 "%s %s r%up%u", impl_name, part_name,
200 CPU_VAR(midr), CPU_REV(midr));
201 } else {
202 printf(" Unknown, MIDR 0x%x", midr);
203
204 if (CPU_IS_PRIMARY(ci))
205 snprintf(cpu_model, sizeof(cpu_model), "Unknown");
206 }
207
208 /* Print cache information. */
209
210 __asm volatile("mrc p15, 0, %0, c0, c0, 1" : "=r"(ctr));
211 switch (ctr & CTR_IL1P_MASK) {
212 case CTR_IL1P_AIVIVT:
213 il1p_name = "AIVIVT ";
214 break;
215 case CTR_IL1P_VIPT:
216 il1p_name = "VIPT ";
217 break;
218 case CTR_IL1P_PIPT:
219 il1p_name = "PIPT ";
220 break;
221 }
222
223 __asm volatile("mrc p15, 1, %0, c0, c0, 1" : "=r"(clidr));
224 for (i = 0; i < 7; i++) {
225 if ((clidr & CLIDR_CTYPE_MASK) == 0)
226 break;
227 printf("\n%s:", ci->ci_dev->dv_xname);
228 sep = "";
229 if (clidr & CLIDR_CTYPE_INSN) {
230 __asm volatile("mcr p15, 2, %0, c0, c0, 0" ::
231 "r"(i << CSSELR_LEVEL_SHIFT | CSSELR_IND));
232 __asm volatile("mrc p15, 1, %0, c0, c0, 0" : "=r"(ccsidr));
233 sets = CCSIDR_SETS(ccsidr);
234 ways = CCSIDR_WAYS(ccsidr);
235 line = CCSIDR_LINE_SIZE(ccsidr);
236 printf("%s %dKB %db/line %d-way L%d %sI-cache", sep,
237 (sets * ways * line) / 1024, line, ways, (i + 1),
238 il1p_name);
239 il1p_name = "";
240 sep = ",";
241 }
242 if (clidr & CLIDR_CTYPE_DATA) {
243 __asm volatile("mcr p15, 2, %0, c0, c0, 0" ::
244 "r"(i << CSSELR_LEVEL_SHIFT));
245 __asm volatile("mrc p15, 1, %0, c0, c0, 0" : "=r"(ccsidr));
246 sets = CCSIDR_SETS(ccsidr);
247 ways = CCSIDR_WAYS(ccsidr);
248 line = CCSIDR_LINE_SIZE(ccsidr);
249 printf("%s %dKB %db/line %d-way L%d D-cache", sep,
250 (sets * ways * line) / 1024, line, ways, (i + 1));
251 sep = ",";
252 }
253 if (clidr & CLIDR_CTYPE_UNIFIED) {
254 __asm volatile("mcr p15, 2, %0, c0, c0, 0" ::
255 "r"(i << CSSELR_LEVEL_SHIFT));
256 __asm volatile("mrc p15, 1, %0, c0, c0, 0" : "=r"(ccsidr));
257 sets = CCSIDR_SETS(ccsidr);
258 ways = CCSIDR_WAYS(ccsidr);
259 line = CCSIDR_LINE_SIZE(ccsidr);
260 printf("%s %dKB %db/line %d-way L%d cache", sep,
261 (sets * ways * line) / 1024, line, ways, (i + 1));
262 }
263 clidr >>= 3;
264 }
265
266 /*
267 * Some ARM processors are vulnerable to branch target
268 * injection attacks (CVE-2017-5715).
269 */
270 switch (impl) {
271 case CPU_IMPL_ARM:
272 switch (part) {
273 case CPU_PART_CORTEX_A5:
274 case CPU_PART_CORTEX_A7:
275 case CPU_PART_CORTEX_A32:
276 case CPU_PART_CORTEX_A35:
277 case CPU_PART_CORTEX_A53:
278 case CPU_PART_CORTEX_A55:
279 /* Not vulnerable. */
280 ci->ci_flush_bp = cpu_flush_bp_noop;
281 break;
282 case CPU_PART_CORTEX_A8:
283 case CPU_PART_CORTEX_A9:
284 case CPU_PART_CORTEX_A12:
285 case CPU_PART_CORTEX_A17:
286 case CPU_PART_CORTEX_A73:
287 case CPU_PART_CORTEX_A75:
288 default:
289 /* Vulnerable; flush BP cache. */
290 ci->ci_flush_bp = armv7_flush_bp;
291 break;
292 case CPU_PART_CORTEX_A15:
293 case CPU_PART_CORTEX_A72:
294 /*
295 * Vulnerable; BPIALL is "not effective" so
296 * must use ICIALLU and hope the firmware set
297 * the magic bit in the ACTLR that actually
298 * forces a BTB flush.
299 */
300 ci->ci_flush_bp = cortex_a15_flush_bp;
301 break;
302 case CPU_PART_CORTEX_A57:
303 /*
304 * Vulnerable; must disable and enable the MMU
305 * which can be done by a PSCI call on
306 * firmware with the appropriate fixes. Punt
307 * for now.
308 */
309 ci->ci_flush_bp = cpu_flush_bp_noop;
310 }
311 break;
312 default:
313 /* Not much we can do for an unknown processor. */
314 ci->ci_flush_bp = cpu_flush_bp_noop;
315 break;
316 }
317 }
318
319 int cpu_hatch_secondary(struct cpu_info *ci, int, uint64_t);
320 int cpu_clockspeed(int *);
321
322 int
cpu_match(struct device * parent,void * cfdata,void * aux)323 cpu_match(struct device *parent, void *cfdata, void *aux)
324 {
325 struct fdt_attach_args *faa = aux;
326 uint32_t mpidr;
327 char buf[32];
328
329 __asm volatile("mrc p15, 0, %0, c0, c0, 5" : "=r"(mpidr));
330
331 if (OF_getprop(faa->fa_node, "device_type", buf, sizeof(buf)) <= 0 ||
332 strcmp(buf, "cpu") != 0)
333 return 0;
334
335 if (ncpus < MAXCPUS || faa->fa_reg[0].addr == (mpidr & MPIDR_AFF))
336 return 1;
337
338 return 0;
339 }
340
341 void
cpu_attach(struct device * parent,struct device * dev,void * aux)342 cpu_attach(struct device *parent, struct device *dev, void *aux)
343 {
344 struct fdt_attach_args *faa = aux;
345 struct cpu_info *ci;
346 uint32_t mpidr;
347 uint32_t opp;
348
349 __asm volatile("mrc p15, 0, %0, c0, c0, 5" : "=r"(mpidr));
350 KASSERT(faa->fa_nreg > 0);
351
352 #ifdef MULTIPROCESSOR
353 if (faa->fa_reg[0].addr == (mpidr & MPIDR_AFF)) {
354 ci = &cpu_info_primary;
355 ci->ci_flags |= CPUF_RUNNING | CPUF_PRESENT | CPUF_PRIMARY;
356 } else {
357 ci = malloc(sizeof(*ci), M_DEVBUF, M_WAITOK | M_ZERO);
358 cpu_info[dev->dv_unit] = ci;
359 ci->ci_next = cpu_info_list->ci_next;
360 cpu_info_list->ci_next = ci;
361 ci->ci_flags |= CPUF_AP;
362 ncpus++;
363 }
364 #else
365 ci = &cpu_info_primary;
366 #endif
367
368 ci->ci_dev = dev;
369 ci->ci_cpuid = dev->dv_unit;
370 ci->ci_mpidr = faa->fa_reg[0].addr;
371 ci->ci_node = faa->fa_node;
372 ci->ci_self = ci;
373
374 printf(" mpidr %llx:", ci->ci_mpidr);
375
376 #ifdef MULTIPROCESSOR
377 if (ci->ci_flags & CPUF_AP) {
378 char buf[32];
379 uint64_t spinup_data = 0;
380 int spinup_method = 0;
381 int timeout = 10000;
382 int len;
383
384 len = OF_getprop(ci->ci_node, "enable-method",
385 buf, sizeof(buf));
386 if (strcmp(buf, "psci") == 0) {
387 spinup_method = 1;
388 } else if (strcmp(buf, "spin-table") == 0) {
389 spinup_method = 2;
390 spinup_data = OF_getpropint64(ci->ci_node,
391 "cpu-release-addr", 0);
392 }
393
394 clockqueue_init(&ci->ci_queue);
395 sched_init_cpu(ci);
396 if (cpu_hatch_secondary(ci, spinup_method, spinup_data)) {
397 atomic_setbits_int(&ci->ci_flags, CPUF_IDENTIFY);
398 __asm volatile("dsb sy; sev");
399
400 while ((ci->ci_flags & CPUF_IDENTIFIED) == 0 &&
401 --timeout)
402 delay(1000);
403 if (timeout == 0) {
404 printf(" failed to identify");
405 ci->ci_flags = 0;
406 }
407 } else {
408 printf(" failed to spin up");
409 ci->ci_flags = 0;
410 }
411 } else {
412 #endif
413 cpu_identify(ci);
414
415 vfp_init();
416
417 if (OF_getproplen(ci->ci_node, "clocks") > 0) {
418 cpu_node = ci->ci_node;
419 cpu_cpuspeed = cpu_clockspeed;
420 }
421 #ifdef MULTIPROCESSOR
422 }
423 #endif
424
425 opp = OF_getpropint(ci->ci_node, "operating-points-v2", 0);
426 if (opp)
427 cpu_opp_init(ci, opp);
428 else
429 cpu_opp_init_legacy(ci);
430
431 printf("\n");
432 }
433
434 void
cpu_flush_bp_noop(void)435 cpu_flush_bp_noop(void)
436 {
437 }
438
439 int
cpu_clockspeed(int * freq)440 cpu_clockspeed(int *freq)
441 {
442 *freq = clock_get_frequency(cpu_node, NULL) / 1000000;
443 return 0;
444 }
445
446 #ifdef MULTIPROCESSOR
447
448 void cpu_boot_secondary(struct cpu_info *ci);
449 void cpu_hatch(void);
450
451 void
cpu_boot_secondary_processors(void)452 cpu_boot_secondary_processors(void)
453 {
454 struct cpu_info *ci;
455 CPU_INFO_ITERATOR cii;
456
457 CPU_INFO_FOREACH(cii, ci) {
458 if ((ci->ci_flags & CPUF_AP) == 0)
459 continue;
460 if (ci->ci_flags & CPUF_PRIMARY)
461 continue;
462
463 ci->ci_randseed = (arc4random() & 0x7fffffff) + 1;
464 cpu_boot_secondary(ci);
465 }
466 }
467
468 int
cpu_hatch_secondary(struct cpu_info * ci,int method,uint64_t data)469 cpu_hatch_secondary(struct cpu_info *ci, int method, uint64_t data)
470 {
471 extern paddr_t cpu_hatch_ci;
472 paddr_t startaddr;
473 void *kstack;
474 uint32_t ttbr0;
475 int rc = 0;
476
477 kstack = km_alloc(USPACE, &kv_any, &kp_zero, &kd_waitok);
478 ci->ci_pl1_stkend = (vaddr_t)kstack + USPACE - 16;
479
480 kstack = km_alloc(PAGE_SIZE, &kv_any, &kp_zero, &kd_waitok);
481 ci->ci_irq_stkend = (vaddr_t)kstack + PAGE_SIZE;
482 kstack = km_alloc(PAGE_SIZE, &kv_any, &kp_zero, &kd_waitok);
483 ci->ci_abt_stkend = (vaddr_t)kstack + PAGE_SIZE;
484 kstack = km_alloc(PAGE_SIZE, &kv_any, &kp_zero, &kd_waitok);
485 ci->ci_und_stkend = (vaddr_t)kstack + PAGE_SIZE;
486
487 pmap_extract(pmap_kernel(), (vaddr_t)ci, &cpu_hatch_ci);
488
489 __asm volatile("mrc p15, 0, %0, c2, c0, 0" : "=r"(ttbr0));
490 ci->ci_ttbr0 = ttbr0;
491
492 cpu_dcache_wb_range((vaddr_t)&cpu_hatch_ci, sizeof(paddr_t));
493 cpu_dcache_wb_range((vaddr_t)ci, sizeof(*ci));
494
495 pmap_extract(pmap_kernel(), (vaddr_t)cpu_hatch, &startaddr);
496
497 switch (method) {
498 case 1:
499 /* psci */
500 #if NPSCI > 0
501 rc = (psci_cpu_on(ci->ci_mpidr, startaddr, 0) == PSCI_SUCCESS);
502 #endif
503 break;
504 #ifdef notyet
505 case 2:
506 /* spin-table */
507 cpu_hatch_spin_table(ci, startaddr, data);
508 rc = 1;
509 break;
510 #endif
511 default:
512 /* no method to spin up CPU */
513 ci->ci_flags = 0; /* mark cpu as not AP */
514 }
515
516 return rc;
517 }
518
519 void
cpu_boot_secondary(struct cpu_info * ci)520 cpu_boot_secondary(struct cpu_info *ci)
521 {
522 atomic_setbits_int(&ci->ci_flags, CPUF_GO);
523 __asm volatile("dsb sy; sev");
524
525 while ((ci->ci_flags & CPUF_RUNNING) == 0)
526 __asm volatile("wfe");
527 }
528
529 void
cpu_start_secondary(struct cpu_info * ci)530 cpu_start_secondary(struct cpu_info *ci)
531 {
532 int s;
533
534 cpu_setup();
535
536 set_stackptr(PSR_IRQ32_MODE, ci->ci_irq_stkend);
537 set_stackptr(PSR_ABT32_MODE, ci->ci_abt_stkend);
538 set_stackptr(PSR_UND32_MODE, ci->ci_und_stkend);
539
540 ci->ci_flags |= CPUF_PRESENT;
541 __asm volatile("dsb sy");
542
543 while ((ci->ci_flags & CPUF_IDENTIFY) == 0)
544 __asm volatile("wfe");
545
546 cpu_identify(ci);
547 atomic_setbits_int(&ci->ci_flags, CPUF_IDENTIFIED);
548 __asm volatile("dsb sy");
549
550 while ((ci->ci_flags & CPUF_GO) == 0)
551 __asm volatile("wfe");
552
553 s = splhigh();
554 arm_intr_cpu_enable();
555 cpu_startclock();
556
557 atomic_setbits_int(&ci->ci_flags, CPUF_RUNNING);
558 __asm volatile("dsb sy; sev");
559
560 spllower(IPL_NONE);
561
562 sched_toidle();
563 }
564
565 void
cpu_kick(struct cpu_info * ci)566 cpu_kick(struct cpu_info *ci)
567 {
568 /* force cpu to enter kernel */
569 if (ci != curcpu())
570 arm_send_ipi(ci, ARM_IPI_NOP);
571 }
572
573 void
cpu_unidle(struct cpu_info * ci)574 cpu_unidle(struct cpu_info *ci)
575 {
576 /*
577 * This could send IPI or SEV depending on if the other
578 * processor is sleeping (WFI or WFE), in userland, or if the
579 * cpu is in other possible wait states?
580 */
581 if (ci != curcpu())
582 arm_send_ipi(ci, ARM_IPI_NOP);
583 }
584
585 #endif /* MULTIPROCESSOR */
586
587 /*
588 * Dynamic voltage and frequency scaling implementation.
589 */
590
591 extern int perflevel;
592
593 struct opp {
594 uint64_t opp_hz;
595 uint32_t opp_microvolt;
596 };
597
598 struct opp_table {
599 LIST_ENTRY(opp_table) ot_list;
600 uint32_t ot_phandle;
601
602 struct opp *ot_opp;
603 u_int ot_nopp;
604 uint64_t ot_opp_hz_min;
605 uint64_t ot_opp_hz_max;
606
607 struct cpu_info *ot_master;
608 };
609
610 LIST_HEAD(, opp_table) opp_tables = LIST_HEAD_INITIALIZER(opp_tables);
611 struct task cpu_opp_task;
612
613 void cpu_opp_mountroot(struct device *);
614 void cpu_opp_dotask(void *);
615 void cpu_opp_setperf(int);
616
617 uint32_t cpu_opp_get_cooling_level(void *, uint32_t *);
618 void cpu_opp_set_cooling_level(void *, uint32_t *, uint32_t);
619
620 void
cpu_opp_init_legacy(struct cpu_info * ci)621 cpu_opp_init_legacy(struct cpu_info *ci)
622 {
623 struct opp_table *ot;
624 struct cooling_device *cd;
625 uint32_t opp_hz, opp_microvolt;
626 uint32_t *opps, supply;
627 int i, j, len, count;
628
629 supply = OF_getpropint(ci->ci_node, "cpu-supply", 0);
630 if (supply == 0)
631 return;
632
633 len = OF_getproplen(ci->ci_node, "operating-points");
634 if (len <= 0)
635 return;
636
637 opps = malloc(len, M_TEMP, M_WAITOK);
638 OF_getpropintarray(ci->ci_node, "operating-points", opps, len);
639
640 count = len / (2 * sizeof(uint32_t));
641
642 ot = malloc(sizeof(struct opp_table), M_DEVBUF, M_ZERO | M_WAITOK);
643 ot->ot_opp = mallocarray(count, sizeof(struct opp),
644 M_DEVBUF, M_ZERO | M_WAITOK);
645 ot->ot_nopp = count;
646
647 count = 0;
648 while (count < len / (2 * sizeof(uint32_t))) {
649 opp_hz = opps[2 * count] * 1000;
650 opp_microvolt = opps[2 * count + 1];
651
652 /* Insert into the array, keeping things sorted. */
653 for (i = 0; i < count; i++) {
654 if (opp_hz < ot->ot_opp[i].opp_hz)
655 break;
656 }
657 for (j = count; j > i; j--)
658 ot->ot_opp[j] = ot->ot_opp[j - 1];
659 ot->ot_opp[i].opp_hz = opp_hz;
660 ot->ot_opp[i].opp_microvolt = opp_microvolt;
661 count++;
662 }
663
664 ot->ot_opp_hz_min = ot->ot_opp[0].opp_hz;
665 ot->ot_opp_hz_max = ot->ot_opp[count - 1].opp_hz;
666
667 free(opps, M_TEMP, len);
668
669 ci->ci_opp_table = ot;
670 ci->ci_opp_max = ot->ot_nopp - 1;
671 ci->ci_cpu_supply = supply;
672
673 cd = malloc(sizeof(struct cooling_device), M_DEVBUF, M_ZERO | M_WAITOK);
674 cd->cd_node = ci->ci_node;
675 cd->cd_cookie = ci;
676 cd->cd_get_level = cpu_opp_get_cooling_level;
677 cd->cd_set_level = cpu_opp_set_cooling_level;
678 cooling_device_register(cd);
679
680 /*
681 * Do additional checks at mountroot when all the clocks and
682 * regulators are available.
683 */
684 config_mountroot(ci->ci_dev, cpu_opp_mountroot);
685 }
686
687 void
cpu_opp_init(struct cpu_info * ci,uint32_t phandle)688 cpu_opp_init(struct cpu_info *ci, uint32_t phandle)
689 {
690 struct opp_table *ot;
691 struct cooling_device *cd;
692 int count, node, child;
693 uint32_t opp_hz, opp_microvolt;
694 uint32_t values[3];
695 int i, j, len;
696
697 LIST_FOREACH(ot, &opp_tables, ot_list) {
698 if (ot->ot_phandle == phandle) {
699 ci->ci_opp_table = ot;
700 return;
701 }
702 }
703
704 node = OF_getnodebyphandle(phandle);
705 if (node == 0)
706 return;
707
708 if (!OF_is_compatible(node, "operating-points-v2"))
709 return;
710
711 count = 0;
712 for (child = OF_child(node); child != 0; child = OF_peer(child)) {
713 if (OF_getproplen(child, "turbo-mode") == 0)
714 continue;
715 count++;
716 }
717 if (count == 0)
718 return;
719
720 ot = malloc(sizeof(struct opp_table), M_DEVBUF, M_ZERO | M_WAITOK);
721 ot->ot_phandle = phandle;
722 ot->ot_opp = mallocarray(count, sizeof(struct opp),
723 M_DEVBUF, M_ZERO | M_WAITOK);
724 ot->ot_nopp = count;
725
726 count = 0;
727 for (child = OF_child(node); child != 0; child = OF_peer(child)) {
728 if (OF_getproplen(child, "turbo-mode") == 0)
729 continue;
730 opp_hz = OF_getpropint64(child, "opp-hz", 0);
731 len = OF_getpropintarray(child, "opp-microvolt",
732 values, sizeof(values));
733 opp_microvolt = 0;
734 if (len == sizeof(uint32_t) || len == 3 * sizeof(uint32_t))
735 opp_microvolt = values[0];
736
737 /* Insert into the array, keeping things sorted. */
738 for (i = 0; i < count; i++) {
739 if (opp_hz < ot->ot_opp[i].opp_hz)
740 break;
741 }
742 for (j = count; j > i; j--)
743 ot->ot_opp[j] = ot->ot_opp[j - 1];
744 ot->ot_opp[i].opp_hz = opp_hz;
745 ot->ot_opp[i].opp_microvolt = opp_microvolt;
746 count++;
747 }
748
749 ot->ot_opp_hz_min = ot->ot_opp[0].opp_hz;
750 ot->ot_opp_hz_max = ot->ot_opp[count - 1].opp_hz;
751
752 if (OF_getproplen(node, "opp-shared") == 0)
753 ot->ot_master = ci;
754
755 LIST_INSERT_HEAD(&opp_tables, ot, ot_list);
756
757 ci->ci_opp_table = ot;
758 ci->ci_opp_max = ot->ot_nopp - 1;
759 ci->ci_cpu_supply = OF_getpropint(ci->ci_node, "cpu-supply", 0);
760
761 cd = malloc(sizeof(struct cooling_device), M_DEVBUF, M_ZERO | M_WAITOK);
762 cd->cd_node = ci->ci_node;
763 cd->cd_cookie = ci;
764 cd->cd_get_level = cpu_opp_get_cooling_level;
765 cd->cd_set_level = cpu_opp_set_cooling_level;
766 cooling_device_register(cd);
767
768 /*
769 * Do additional checks at mountroot when all the clocks and
770 * regulators are available.
771 */
772 config_mountroot(ci->ci_dev, cpu_opp_mountroot);
773 }
774
775 void
cpu_opp_mountroot(struct device * self)776 cpu_opp_mountroot(struct device *self)
777 {
778 struct cpu_info *ci;
779 CPU_INFO_ITERATOR cii;
780 int count = 0;
781 int level = 0;
782
783 if (cpu_setperf)
784 return;
785
786 CPU_INFO_FOREACH(cii, ci) {
787 struct opp_table *ot = ci->ci_opp_table;
788 uint64_t curr_hz;
789 uint32_t curr_microvolt;
790 int error;
791
792 if (ot == NULL)
793 continue;
794
795 /* Skip if this table is shared and we're not the master. */
796 if (ot->ot_master && ot->ot_master != ci)
797 continue;
798
799 /* PWM regulators may need to be explicitly enabled. */
800 regulator_enable(ci->ci_cpu_supply);
801
802 curr_hz = clock_get_frequency(ci->ci_node, NULL);
803 curr_microvolt = regulator_get_voltage(ci->ci_cpu_supply);
804
805 /* Disable if clock isn't implemented. */
806 error = ENODEV;
807 if (curr_hz != 0)
808 error = clock_set_frequency(ci->ci_node, NULL, curr_hz);
809 if (error) {
810 ci->ci_opp_table = NULL;
811 printf("%s: clock not implemented\n",
812 ci->ci_dev->dv_xname);
813 continue;
814 }
815
816 /* Disable if regulator isn't implemented. */
817 error = ci->ci_cpu_supply ? ENODEV : 0;
818 if (ci->ci_cpu_supply && curr_microvolt != 0)
819 error = regulator_set_voltage(ci->ci_cpu_supply,
820 curr_microvolt);
821 if (error) {
822 ci->ci_opp_table = NULL;
823 printf("%s: regulator not implemented\n",
824 ci->ci_dev->dv_xname);
825 continue;
826 }
827
828 /*
829 * Initialize performance level based on the current
830 * speed of the first CPU that supports DVFS.
831 */
832 if (level == 0) {
833 uint64_t min, max;
834 uint64_t level_hz;
835
836 min = ot->ot_opp_hz_min;
837 max = ot->ot_opp_hz_max;
838 level_hz = clock_get_frequency(ci->ci_node, NULL);
839 if (level_hz < min)
840 level_hz = min;
841 if (level_hz > max)
842 level_hz = max;
843 level = howmany(100 * (level_hz - min), (max - min));
844 }
845
846 count++;
847 }
848
849 if (count > 0) {
850 task_set(&cpu_opp_task, cpu_opp_dotask, NULL);
851 cpu_setperf = cpu_opp_setperf;
852
853 perflevel = (level > 0) ? level : 0;
854 cpu_setperf(perflevel);
855 }
856 }
857
858 void
cpu_opp_dotask(void * arg)859 cpu_opp_dotask(void *arg)
860 {
861 struct cpu_info *ci;
862 CPU_INFO_ITERATOR cii;
863
864 CPU_INFO_FOREACH(cii, ci) {
865 struct opp_table *ot = ci->ci_opp_table;
866 uint64_t curr_hz, opp_hz;
867 uint32_t curr_microvolt, opp_microvolt;
868 int opp_idx;
869 int error = 0;
870
871 if (ot == NULL)
872 continue;
873
874 /* Skip if this table is shared and we're not the master. */
875 if (ot->ot_master && ot->ot_master != ci)
876 continue;
877
878 opp_idx = MIN(ci->ci_opp_idx, ci->ci_opp_max);
879 opp_hz = ot->ot_opp[opp_idx].opp_hz;
880 opp_microvolt = ot->ot_opp[opp_idx].opp_microvolt;
881
882 curr_hz = clock_get_frequency(ci->ci_node, NULL);
883 curr_microvolt = regulator_get_voltage(ci->ci_cpu_supply);
884
885 if (error == 0 && opp_hz < curr_hz)
886 error = clock_set_frequency(ci->ci_node, NULL, opp_hz);
887 if (error == 0 && ci->ci_cpu_supply &&
888 opp_microvolt != 0 && opp_microvolt != curr_microvolt) {
889 error = regulator_set_voltage(ci->ci_cpu_supply,
890 opp_microvolt);
891 }
892 if (error == 0 && opp_hz > curr_hz)
893 error = clock_set_frequency(ci->ci_node, NULL, opp_hz);
894
895 if (error)
896 printf("%s: DVFS failed\n", ci->ci_dev->dv_xname);
897 }
898 }
899
900 void
cpu_opp_setperf(int level)901 cpu_opp_setperf(int level)
902 {
903 struct cpu_info *ci;
904 CPU_INFO_ITERATOR cii;
905
906 CPU_INFO_FOREACH(cii, ci) {
907 struct opp_table *ot = ci->ci_opp_table;
908 uint64_t min, max;
909 uint64_t level_hz, opp_hz;
910 int opp_idx = -1;
911 int i;
912
913 if (ot == NULL)
914 continue;
915
916 /* Skip if this table is shared and we're not the master. */
917 if (ot->ot_master && ot->ot_master != ci)
918 continue;
919
920 min = ot->ot_opp_hz_min;
921 max = ot->ot_opp_hz_max;
922 level_hz = min + (level * (max - min)) / 100;
923 opp_hz = min;
924 for (i = 0; i < ot->ot_nopp; i++) {
925 if (ot->ot_opp[i].opp_hz <= level_hz &&
926 ot->ot_opp[i].opp_hz >= opp_hz)
927 opp_hz = ot->ot_opp[i].opp_hz;
928 }
929
930 /* Find index of selected operating point. */
931 for (i = 0; i < ot->ot_nopp; i++) {
932 if (ot->ot_opp[i].opp_hz == opp_hz) {
933 opp_idx = i;
934 break;
935 }
936 }
937 KASSERT(opp_idx >= 0);
938
939 ci->ci_opp_idx = opp_idx;
940 }
941
942 /*
943 * Update the hardware from a task since setting the
944 * regulators might need process context.
945 */
946 task_add(systq, &cpu_opp_task);
947 }
948
949 uint32_t
cpu_opp_get_cooling_level(void * cookie,uint32_t * cells)950 cpu_opp_get_cooling_level(void *cookie, uint32_t *cells)
951 {
952 struct cpu_info *ci = cookie;
953 struct opp_table *ot = ci->ci_opp_table;
954
955 return ot->ot_nopp - ci->ci_opp_max - 1;
956 }
957
958 void
cpu_opp_set_cooling_level(void * cookie,uint32_t * cells,uint32_t level)959 cpu_opp_set_cooling_level(void *cookie, uint32_t *cells, uint32_t level)
960 {
961 struct cpu_info *ci = cookie;
962 struct opp_table *ot = ci->ci_opp_table;
963 int opp_max;
964
965 if (level > (ot->ot_nopp - 1))
966 level = ot->ot_nopp - 1;
967
968 opp_max = (ot->ot_nopp - level - 1);
969 if (ci->ci_opp_max != opp_max) {
970 ci->ci_opp_max = opp_max;
971 task_add(systq, &cpu_opp_task);
972 }
973 }
974