1 /* $NetBSD: gicv3.c,v 1.54 2022/06/26 11:14:36 jmcneill Exp $ */
2
3 /*-
4 * Copyright (c) 2018 Jared McNeill <jmcneill@invisible.ca>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include "opt_multiprocessor.h"
30
31 #define _INTR_PRIVATE
32
33 #include <sys/cdefs.h>
34 __KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.54 2022/06/26 11:14:36 jmcneill Exp $");
35
36 #include <sys/param.h>
37 #include <sys/kernel.h>
38 #include <sys/bus.h>
39 #include <sys/device.h>
40 #include <sys/intr.h>
41 #include <sys/systm.h>
42 #include <sys/cpu.h>
43 #include <sys/vmem.h>
44 #include <sys/kmem.h>
45 #include <sys/atomic.h>
46
47 #include <machine/cpufunc.h>
48
49 #include <arm/locore.h>
50 #include <arm/armreg.h>
51
52 #include <arm/cortex/gicv3.h>
53 #include <arm/cortex/gic_reg.h>
54
55 #define PICTOSOFTC(pic) \
56 container_of(pic, struct gicv3_softc, sc_pic)
57 #define LPITOSOFTC(lpi) \
58 container_of(lpi, struct gicv3_softc, sc_lpi)
59
60 #define IPL_TO_PRIORITY(sc, ipl) (((0xff - (ipl)) << (sc)->sc_priority_shift) & 0xff)
61 #define IPL_TO_PMR(sc, ipl) (((0xff - (ipl)) << (sc)->sc_pmr_shift) & 0xff)
62 #define IPL_TO_HWPL(ipl) ((ipl) >= IPL_VM ? (ipl) : IPL_NONE)
63
64 #define GIC_SUPPORTS_1OFN(sc) (((sc)->sc_gicd_typer & GICD_TYPER_No1N) == 0)
65
66 #define GIC_PRIO_SHIFT_NS 4
67 #define GIC_PRIO_SHIFT_S 3
68
69 /*
70 * Set to true if you want to use 1 of N interrupt distribution for SPIs
71 * when available. Disabled by default because it causes issues with the
72 * USB stack.
73 */
74 bool gicv3_use_1ofn = false;
75
76 static struct gicv3_softc *gicv3_softc;
77
78 static inline uint32_t
gicd_read_4(struct gicv3_softc * sc,bus_size_t reg)79 gicd_read_4(struct gicv3_softc *sc, bus_size_t reg)
80 {
81 return bus_space_read_4(sc->sc_bst, sc->sc_bsh_d, reg);
82 }
83
84 static inline void
gicd_write_4(struct gicv3_softc * sc,bus_size_t reg,uint32_t val)85 gicd_write_4(struct gicv3_softc *sc, bus_size_t reg, uint32_t val)
86 {
87 bus_space_write_4(sc->sc_bst, sc->sc_bsh_d, reg, val);
88 }
89
90 #ifdef MULTIPROCESSOR
91 static inline uint64_t
gicd_read_8(struct gicv3_softc * sc,bus_size_t reg)92 gicd_read_8(struct gicv3_softc *sc, bus_size_t reg)
93 {
94 return bus_space_read_8(sc->sc_bst, sc->sc_bsh_d, reg);
95 }
96 #endif
97
98 static inline void
gicd_write_8(struct gicv3_softc * sc,bus_size_t reg,uint64_t val)99 gicd_write_8(struct gicv3_softc *sc, bus_size_t reg, uint64_t val)
100 {
101 bus_space_write_8(sc->sc_bst, sc->sc_bsh_d, reg, val);
102 }
103
104 static inline uint32_t
gicr_read_4(struct gicv3_softc * sc,u_int index,bus_size_t reg)105 gicr_read_4(struct gicv3_softc *sc, u_int index, bus_size_t reg)
106 {
107 KASSERT(index < sc->sc_bsh_r_count);
108 return bus_space_read_4(sc->sc_bst, sc->sc_bsh_r[index], reg);
109 }
110
111 static inline void
gicr_write_4(struct gicv3_softc * sc,u_int index,bus_size_t reg,uint32_t val)112 gicr_write_4(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint32_t val)
113 {
114 KASSERT(index < sc->sc_bsh_r_count);
115 bus_space_write_4(sc->sc_bst, sc->sc_bsh_r[index], reg, val);
116 }
117
118 static inline uint64_t
gicr_read_8(struct gicv3_softc * sc,u_int index,bus_size_t reg)119 gicr_read_8(struct gicv3_softc *sc, u_int index, bus_size_t reg)
120 {
121 KASSERT(index < sc->sc_bsh_r_count);
122 return bus_space_read_8(sc->sc_bst, sc->sc_bsh_r[index], reg);
123 }
124
125 static inline void
gicr_write_8(struct gicv3_softc * sc,u_int index,bus_size_t reg,uint64_t val)126 gicr_write_8(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint64_t val)
127 {
128 KASSERT(index < sc->sc_bsh_r_count);
129 bus_space_write_8(sc->sc_bst, sc->sc_bsh_r[index], reg, val);
130 }
131
132 static void
gicv3_unblock_irqs(struct pic_softc * pic,size_t irqbase,uint32_t mask)133 gicv3_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
134 {
135 struct gicv3_softc * const sc = PICTOSOFTC(pic);
136 struct cpu_info * const ci = curcpu();
137 const u_int group = irqbase / 32;
138
139 if (group == 0) {
140 atomic_or_32(&sc->sc_enabled_sgippi, mask);
141 gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, mask);
142 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
143 ;
144 } else {
145 gicd_write_4(sc, GICD_ISENABLERn(group), mask);
146 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
147 ;
148 }
149 }
150
151 static void
gicv3_block_irqs(struct pic_softc * pic,size_t irqbase,uint32_t mask)152 gicv3_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
153 {
154 struct gicv3_softc * const sc = PICTOSOFTC(pic);
155 struct cpu_info * const ci = curcpu();
156 const u_int group = irqbase / 32;
157
158 if (group == 0) {
159 atomic_and_32(&sc->sc_enabled_sgippi, ~mask);
160 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, mask);
161 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
162 ;
163 } else {
164 gicd_write_4(sc, GICD_ICENABLERn(group), mask);
165 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
166 ;
167 }
168 }
169
170 static void
gicv3_establish_irq(struct pic_softc * pic,struct intrsource * is)171 gicv3_establish_irq(struct pic_softc *pic, struct intrsource *is)
172 {
173 struct gicv3_softc * const sc = PICTOSOFTC(pic);
174 const u_int group = is->is_irq / 32;
175 uint32_t ipriority, icfg;
176 uint64_t irouter;
177 u_int n;
178
179 const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl);
180 const u_int ipriority_shift = (is->is_irq & 0x3) * 8;
181 const u_int icfg_shift = (is->is_irq & 0xf) * 2;
182
183 if (group == 0) {
184 /* SGIs and PPIs are per-CPU and always MP-safe */
185 is->is_mpsafe = true;
186 is->is_percpu = true;
187
188 /* Update interrupt configuration and priority on all redistributors */
189 for (n = 0; n < sc->sc_bsh_r_count; n++) {
190 icfg = gicr_read_4(sc, n, GICR_ICFGRn(is->is_irq / 16));
191 if (is->is_type == IST_LEVEL)
192 icfg &= ~(0x2 << icfg_shift);
193 if (is->is_type == IST_EDGE)
194 icfg |= (0x2 << icfg_shift);
195 gicr_write_4(sc, n, GICR_ICFGRn(is->is_irq / 16), icfg);
196
197 ipriority = gicr_read_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4));
198 ipriority &= ~(0xffU << ipriority_shift);
199 ipriority |= (ipriority_val << ipriority_shift);
200 gicr_write_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4), ipriority);
201 }
202 } else {
203 /*
204 * If 1 of N SPI routing is supported, route MP-safe interrupts to all
205 * participating PEs. Otherwise, just route to the primary PE.
206 */
207 if (is->is_mpsafe && GIC_SUPPORTS_1OFN(sc) && gicv3_use_1ofn) {
208 irouter = GICD_IROUTER_Interrupt_Routing_mode;
209 } else {
210 irouter = sc->sc_irouter[0];
211 }
212 gicd_write_8(sc, GICD_IROUTER(is->is_irq), irouter);
213
214 /* Update interrupt configuration */
215 icfg = gicd_read_4(sc, GICD_ICFGRn(is->is_irq / 16));
216 if (is->is_type == IST_LEVEL)
217 icfg &= ~(0x2 << icfg_shift);
218 if (is->is_type == IST_EDGE)
219 icfg |= (0x2 << icfg_shift);
220 gicd_write_4(sc, GICD_ICFGRn(is->is_irq / 16), icfg);
221
222 /* Update interrupt priority */
223 ipriority = gicd_read_4(sc, GICD_IPRIORITYRn(is->is_irq / 4));
224 ipriority &= ~(0xffU << ipriority_shift);
225 ipriority |= (ipriority_val << ipriority_shift);
226 gicd_write_4(sc, GICD_IPRIORITYRn(is->is_irq / 4), ipriority);
227 }
228 }
229
230 static void
gicv3_set_priority(struct pic_softc * pic,int ipl)231 gicv3_set_priority(struct pic_softc *pic, int ipl)
232 {
233 struct gicv3_softc * const sc = PICTOSOFTC(pic);
234 struct cpu_info * const ci = curcpu();
235 const int hwpl = IPL_TO_HWPL(ipl);
236
237 while (hwpl < ci->ci_hwpl) {
238 /* Lowering priority mask */
239 ci->ci_hwpl = hwpl;
240 __insn_barrier();
241 icc_pmr_write(IPL_TO_PMR(sc, hwpl));
242 }
243 __insn_barrier();
244 ci->ci_cpl = ipl;
245 }
246
247 static void
gicv3_dist_enable(struct gicv3_softc * sc)248 gicv3_dist_enable(struct gicv3_softc *sc)
249 {
250 uint32_t gicd_ctrl;
251 u_int n;
252
253 /* Disable the distributor */
254 gicd_ctrl = gicd_read_4(sc, GICD_CTRL);
255 gicd_ctrl &= ~(GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS);
256 gicd_write_4(sc, GICD_CTRL, gicd_ctrl);
257
258 /* Wait for register write to complete */
259 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
260 ;
261
262 /* Clear all INTID enable bits */
263 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32)
264 gicd_write_4(sc, GICD_ICENABLERn(n / 32), ~0);
265
266 /* Set default priorities to lowest */
267 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 4)
268 gicd_write_4(sc, GICD_IPRIORITYRn(n / 4), ~0);
269
270 /* Set all interrupts to G1NS */
271 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) {
272 gicd_write_4(sc, GICD_IGROUPRn(n / 32), ~0);
273 gicd_write_4(sc, GICD_IGRPMODRn(n / 32), 0);
274 }
275
276 /* Set all interrupts level-sensitive by default */
277 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 16)
278 gicd_write_4(sc, GICD_ICFGRn(n / 16), 0);
279
280 /* Wait for register writes to complete */
281 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
282 ;
283
284 /* Enable Affinity routing and G1NS interrupts */
285 gicd_ctrl = GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS;
286 gicd_write_4(sc, GICD_CTRL, gicd_ctrl);
287 }
288
289 static void
gicv3_redist_enable(struct gicv3_softc * sc,struct cpu_info * ci)290 gicv3_redist_enable(struct gicv3_softc *sc, struct cpu_info *ci)
291 {
292 uint32_t icfg;
293 u_int n, o;
294
295 /* Clear INTID enable bits */
296 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, ~0);
297
298 /* Wait for register write to complete */
299 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
300 ;
301
302 /* Set default priorities */
303 for (n = 0; n < 32; n += 4) {
304 uint32_t priority = 0;
305 size_t byte_shift = 0;
306 for (o = 0; o < 4; o++, byte_shift += 8) {
307 struct intrsource * const is = sc->sc_pic.pic_sources[n + o];
308 if (is == NULL)
309 priority |= (0xffU << byte_shift);
310 else {
311 const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl);
312 priority |= ipriority_val << byte_shift;
313 }
314 }
315 gicr_write_4(sc, ci->ci_gic_redist, GICR_IPRIORITYRn(n / 4), priority);
316 }
317
318 /* Set all interrupts to G1NS */
319 gicr_write_4(sc, ci->ci_gic_redist, GICR_IGROUPR0, ~0);
320 gicr_write_4(sc, ci->ci_gic_redist, GICR_IGRPMODR0, 0);
321
322 /* Restore PPI configs */
323 for (n = 0, icfg = 0; n < 16; n++) {
324 struct intrsource * const is = sc->sc_pic.pic_sources[16 + n];
325 if (is != NULL && is->is_type == IST_EDGE)
326 icfg |= (0x2 << (n * 2));
327 }
328 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICFGRn(1), icfg);
329
330 /* Restore current enable bits */
331 gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, sc->sc_enabled_sgippi);
332
333 /* Wait for register write to complete */
334 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
335 ;
336 }
337
338 static uint64_t
gicv3_cpu_identity(void)339 gicv3_cpu_identity(void)
340 {
341 u_int aff3, aff2, aff1, aff0;
342
343 const register_t mpidr = cpu_mpidr_aff_read();
344 aff0 = __SHIFTOUT(mpidr, MPIDR_AFF0);
345 aff1 = __SHIFTOUT(mpidr, MPIDR_AFF1);
346 aff2 = __SHIFTOUT(mpidr, MPIDR_AFF2);
347 aff3 = __SHIFTOUT(mpidr, MPIDR_AFF3);
348
349 return __SHIFTIN(aff0, GICR_TYPER_Affinity_Value_Aff0) |
350 __SHIFTIN(aff1, GICR_TYPER_Affinity_Value_Aff1) |
351 __SHIFTIN(aff2, GICR_TYPER_Affinity_Value_Aff2) |
352 __SHIFTIN(aff3, GICR_TYPER_Affinity_Value_Aff3);
353 }
354
355 static u_int
gicv3_find_redist(struct gicv3_softc * sc)356 gicv3_find_redist(struct gicv3_softc *sc)
357 {
358 uint64_t gicr_typer;
359 u_int n;
360
361 const uint64_t cpu_identity = gicv3_cpu_identity();
362
363 for (n = 0; n < sc->sc_bsh_r_count; n++) {
364 gicr_typer = gicr_read_8(sc, n, GICR_TYPER);
365 if ((gicr_typer & GICR_TYPER_Affinity_Value) == cpu_identity)
366 return n;
367 }
368
369 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0);
370 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1);
371 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2);
372 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3);
373
374 panic("%s: could not find GICv3 redistributor for cpu %d.%d.%d.%d",
375 cpu_name(curcpu()), aff3, aff2, aff1, aff0);
376 }
377
378 static uint64_t
gicv3_sgir(struct gicv3_softc * sc)379 gicv3_sgir(struct gicv3_softc *sc)
380 {
381 const uint64_t cpu_identity = gicv3_cpu_identity();
382
383 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0);
384 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1);
385 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2);
386 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3);
387
388 return __SHIFTIN(__BIT(aff0), ICC_SGIR_EL1_TargetList) |
389 __SHIFTIN(aff1, ICC_SGIR_EL1_Aff1) |
390 __SHIFTIN(aff2, ICC_SGIR_EL1_Aff2) |
391 __SHIFTIN(aff3, ICC_SGIR_EL1_Aff3);
392 }
393
394 static void
gicv3_cpu_init(struct pic_softc * pic,struct cpu_info * ci)395 gicv3_cpu_init(struct pic_softc *pic, struct cpu_info *ci)
396 {
397 struct gicv3_softc * const sc = PICTOSOFTC(pic);
398 uint32_t icc_sre, icc_ctlr, gicr_waker;
399
400 evcnt_attach_dynamic(&ci->ci_intr_preempt, EVCNT_TYPE_MISC, NULL,
401 ci->ci_cpuname, "intr preempt");
402
403 ci->ci_gic_redist = gicv3_find_redist(sc);
404 ci->ci_gic_sgir = gicv3_sgir(sc);
405
406 /* Enable System register access and disable IRQ/FIQ bypass */
407 icc_sre = ICC_SRE_EL1_SRE | ICC_SRE_EL1_DFB | ICC_SRE_EL1_DIB;
408 icc_sre_write(icc_sre);
409
410 /* Mark the connected PE as being awake */
411 gicr_waker = gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER);
412 gicr_waker &= ~GICR_WAKER_ProcessorSleep;
413 gicr_write_4(sc, ci->ci_gic_redist, GICR_WAKER, gicr_waker);
414 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER) & GICR_WAKER_ChildrenAsleep)
415 ;
416
417 /* Set initial priority mask */
418 ci->ci_hwpl = IPL_HIGH;
419 icc_pmr_write(IPL_TO_PMR(sc, IPL_HIGH));
420
421 /* Set the binary point field to the minimum value */
422 icc_bpr1_write(0);
423
424 /* Enable group 1 interrupt signaling */
425 icc_igrpen1_write(ICC_IGRPEN_EL1_Enable);
426
427 /* Set EOI mode */
428 icc_ctlr = icc_ctlr_read();
429 icc_ctlr &= ~ICC_CTLR_EL1_EOImode;
430 icc_ctlr_write(icc_ctlr);
431
432 /* Enable redistributor */
433 gicv3_redist_enable(sc, ci);
434
435 /* Allow IRQ exceptions */
436 ENABLE_INTERRUPT();
437 }
438
439 #ifdef MULTIPROCESSOR
440 static void
gicv3_ipi_send(struct pic_softc * pic,const kcpuset_t * kcp,u_long ipi)441 gicv3_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi)
442 {
443 struct cpu_info *ci;
444 uint64_t sgir;
445
446 sgir = __SHIFTIN(ipi, ICC_SGIR_EL1_INTID);
447 if (kcp == NULL) {
448 /* Interrupts routed to all PEs, excluding "self" */
449 if (ncpu == 1)
450 return;
451 sgir |= ICC_SGIR_EL1_IRM;
452 } else {
453 /* Interrupt to exactly one PE */
454 ci = cpu_lookup(kcpuset_ffs(kcp) - 1);
455 if (ci == curcpu())
456 return;
457 sgir |= ci->ci_gic_sgir;
458 }
459 icc_sgi1r_write(sgir);
460 isb();
461 }
462
463 static void
gicv3_get_affinity(struct pic_softc * pic,size_t irq,kcpuset_t * affinity)464 gicv3_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity)
465 {
466 struct gicv3_softc * const sc = PICTOSOFTC(pic);
467 const size_t group = irq / 32;
468 int n;
469
470 kcpuset_zero(affinity);
471 if (group == 0) {
472 /* All CPUs are targets for group 0 (SGI/PPI) */
473 for (n = 0; n < ncpu; n++) {
474 kcpuset_set(affinity, n);
475 }
476 } else {
477 /* Find distributor targets (SPI) */
478 const uint64_t irouter = gicd_read_8(sc, GICD_IROUTER(irq));
479 for (n = 0; n < ncpu; n++) {
480 if (irouter == GICD_IROUTER_Interrupt_Routing_mode ||
481 irouter == sc->sc_irouter[n])
482 kcpuset_set(affinity, n);
483 }
484 }
485 }
486
487 static int
gicv3_set_affinity(struct pic_softc * pic,size_t irq,const kcpuset_t * affinity)488 gicv3_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity)
489 {
490 struct gicv3_softc * const sc = PICTOSOFTC(pic);
491 const size_t group = irq / 32;
492 uint64_t irouter;
493
494 if (group == 0)
495 return EINVAL;
496
497 const int set = kcpuset_countset(affinity);
498 if (set == 1) {
499 irouter = sc->sc_irouter[kcpuset_ffs(affinity) - 1];
500 } else if (set == ncpu && GIC_SUPPORTS_1OFN(sc) && gicv3_use_1ofn) {
501 irouter = GICD_IROUTER_Interrupt_Routing_mode;
502 } else {
503 return EINVAL;
504 }
505
506 gicd_write_8(sc, GICD_IROUTER(irq), irouter);
507
508 return 0;
509 }
510 #endif
511
512 static const struct pic_ops gicv3_picops = {
513 .pic_unblock_irqs = gicv3_unblock_irqs,
514 .pic_block_irqs = gicv3_block_irqs,
515 .pic_establish_irq = gicv3_establish_irq,
516 .pic_set_priority = gicv3_set_priority,
517 #ifdef MULTIPROCESSOR
518 .pic_cpu_init = gicv3_cpu_init,
519 .pic_ipi_send = gicv3_ipi_send,
520 .pic_get_affinity = gicv3_get_affinity,
521 .pic_set_affinity = gicv3_set_affinity,
522 #endif
523 };
524
525 static void
gicv3_dcache_wb_range(vaddr_t va,vsize_t len)526 gicv3_dcache_wb_range(vaddr_t va, vsize_t len)
527 {
528 cpu_dcache_wb_range(va, len);
529 dsb(sy);
530 }
531
532 static void
gicv3_lpi_unblock_irqs(struct pic_softc * pic,size_t irqbase,uint32_t mask)533 gicv3_lpi_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
534 {
535 struct gicv3_softc * const sc = LPITOSOFTC(pic);
536 int bit;
537
538 while ((bit = ffs(mask)) != 0) {
539 sc->sc_lpiconf.base[irqbase + bit - 1] |= GIC_LPICONF_Enable;
540 if (sc->sc_lpiconf_flush)
541 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1);
542 mask &= ~__BIT(bit - 1);
543 }
544
545 if (!sc->sc_lpiconf_flush)
546 dsb(ishst);
547 }
548
549 static void
gicv3_lpi_block_irqs(struct pic_softc * pic,size_t irqbase,uint32_t mask)550 gicv3_lpi_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
551 {
552 struct gicv3_softc * const sc = LPITOSOFTC(pic);
553 int bit;
554
555 while ((bit = ffs(mask)) != 0) {
556 sc->sc_lpiconf.base[irqbase + bit - 1] &= ~GIC_LPICONF_Enable;
557 if (sc->sc_lpiconf_flush)
558 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1);
559 mask &= ~__BIT(bit - 1);
560 }
561
562 if (!sc->sc_lpiconf_flush)
563 dsb(ishst);
564 }
565
566 static void
gicv3_lpi_establish_irq(struct pic_softc * pic,struct intrsource * is)567 gicv3_lpi_establish_irq(struct pic_softc *pic, struct intrsource *is)
568 {
569 struct gicv3_softc * const sc = LPITOSOFTC(pic);
570
571 sc->sc_lpiconf.base[is->is_irq] = IPL_TO_PRIORITY(sc, is->is_ipl) | GIC_LPICONF_Res1;
572
573 if (sc->sc_lpiconf_flush)
574 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[is->is_irq], 1);
575 else
576 dsb(ishst);
577 }
578
579 static void
gicv3_lpi_cpu_init(struct pic_softc * pic,struct cpu_info * ci)580 gicv3_lpi_cpu_init(struct pic_softc *pic, struct cpu_info *ci)
581 {
582 struct gicv3_softc * const sc = LPITOSOFTC(pic);
583 struct gicv3_lpi_callback *cb;
584 uint64_t propbase, pendbase;
585 uint32_t ctlr;
586
587 /* If physical LPIs are not supported on this redistributor, just return. */
588 const uint64_t typer = gicr_read_8(sc, ci->ci_gic_redist, GICR_TYPER);
589 if ((typer & GICR_TYPER_PLPIS) == 0)
590 return;
591
592 /* Interrupt target address for this CPU, used by ITS when GITS_TYPER.PTA == 0 */
593 sc->sc_processor_id[cpu_index(ci)] = __SHIFTOUT(typer, GICR_TYPER_Processor_Number);
594
595 /* Disable LPIs before making changes */
596 ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR);
597 ctlr &= ~GICR_CTLR_Enable_LPIs;
598 gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr);
599 dsb(sy);
600
601 /* Setup the LPI configuration table */
602 propbase = sc->sc_lpiconf.segs[0].ds_addr |
603 __SHIFTIN(ffs(pic->pic_maxsources) - 1, GICR_PROPBASER_IDbits) |
604 __SHIFTIN(GICR_Shareability_IS, GICR_PROPBASER_Shareability) |
605 __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PROPBASER_InnerCache);
606 gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase);
607 propbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PROPBASER);
608 if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) != GICR_Shareability_IS) {
609 if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) == GICR_Shareability_NS) {
610 propbase &= ~GICR_PROPBASER_Shareability;
611 propbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PROPBASER_Shareability);
612 propbase &= ~GICR_PROPBASER_InnerCache;
613 propbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PROPBASER_InnerCache);
614 gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase);
615 }
616 sc->sc_lpiconf_flush = true;
617 }
618
619 /* Setup the LPI pending table */
620 pendbase = sc->sc_lpipend[cpu_index(ci)].segs[0].ds_addr |
621 __SHIFTIN(GICR_Shareability_IS, GICR_PENDBASER_Shareability) |
622 __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PENDBASER_InnerCache);
623 gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase);
624 pendbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PENDBASER);
625 if (__SHIFTOUT(pendbase, GICR_PENDBASER_Shareability) == GICR_Shareability_NS) {
626 pendbase &= ~GICR_PENDBASER_Shareability;
627 pendbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PENDBASER_Shareability);
628 pendbase &= ~GICR_PENDBASER_InnerCache;
629 pendbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PENDBASER_InnerCache);
630 gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase);
631 }
632
633 /* Enable LPIs */
634 ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR);
635 ctlr |= GICR_CTLR_Enable_LPIs;
636 gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr);
637 dsb(sy);
638
639 /* Setup ITS if present */
640 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list)
641 cb->cpu_init(cb->priv, ci);
642 }
643
644 #ifdef MULTIPROCESSOR
645 static void
gicv3_lpi_get_affinity(struct pic_softc * pic,size_t irq,kcpuset_t * affinity)646 gicv3_lpi_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity)
647 {
648 struct gicv3_softc * const sc = LPITOSOFTC(pic);
649 struct gicv3_lpi_callback *cb;
650
651 kcpuset_zero(affinity);
652 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list)
653 cb->get_affinity(cb->priv, irq, affinity);
654 }
655
656 static int
gicv3_lpi_set_affinity(struct pic_softc * pic,size_t irq,const kcpuset_t * affinity)657 gicv3_lpi_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity)
658 {
659 struct gicv3_softc * const sc = LPITOSOFTC(pic);
660 struct gicv3_lpi_callback *cb;
661 int error = EINVAL;
662
663 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) {
664 error = cb->set_affinity(cb->priv, irq, affinity);
665 if (error != EPASSTHROUGH)
666 return error;
667 }
668
669 return EINVAL;
670 }
671 #endif
672
673 static const struct pic_ops gicv3_lpiops = {
674 .pic_unblock_irqs = gicv3_lpi_unblock_irqs,
675 .pic_block_irqs = gicv3_lpi_block_irqs,
676 .pic_establish_irq = gicv3_lpi_establish_irq,
677 #ifdef MULTIPROCESSOR
678 .pic_cpu_init = gicv3_lpi_cpu_init,
679 .pic_get_affinity = gicv3_lpi_get_affinity,
680 .pic_set_affinity = gicv3_lpi_set_affinity,
681 #endif
682 };
683
684 void
gicv3_dma_alloc(struct gicv3_softc * sc,struct gicv3_dma * dma,bus_size_t len,bus_size_t align)685 gicv3_dma_alloc(struct gicv3_softc *sc, struct gicv3_dma *dma, bus_size_t len, bus_size_t align)
686 {
687 int nsegs, error;
688
689 dma->len = len;
690 error = bus_dmamem_alloc(sc->sc_dmat, dma->len, align, 0, dma->segs, 1, &nsegs, BUS_DMA_WAITOK);
691 if (error)
692 panic("bus_dmamem_alloc failed: %d", error);
693 error = bus_dmamem_map(sc->sc_dmat, dma->segs, nsegs, len, (void **)&dma->base, BUS_DMA_WAITOK);
694 if (error)
695 panic("bus_dmamem_map failed: %d", error);
696 error = bus_dmamap_create(sc->sc_dmat, len, 1, len, 0, BUS_DMA_WAITOK, &dma->map);
697 if (error)
698 panic("bus_dmamap_create failed: %d", error);
699 error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->base, dma->len, NULL, BUS_DMA_WAITOK);
700 if (error)
701 panic("bus_dmamap_load failed: %d", error);
702
703 memset(dma->base, 0, dma->len);
704 bus_dmamap_sync(sc->sc_dmat, dma->map, 0, dma->len, BUS_DMASYNC_PREWRITE);
705 }
706
707 static void
gicv3_lpi_init(struct gicv3_softc * sc)708 gicv3_lpi_init(struct gicv3_softc *sc)
709 {
710 /*
711 * Allocate LPI configuration table
712 */
713 gicv3_dma_alloc(sc, &sc->sc_lpiconf, sc->sc_lpi.pic_maxsources, 0x1000);
714 KASSERT((sc->sc_lpiconf.segs[0].ds_addr & ~GICR_PROPBASER_Physical_Address) == 0);
715
716 /*
717 * Allocate LPI pending tables
718 */
719 const bus_size_t lpipend_sz = (8192 + sc->sc_lpi.pic_maxsources) / NBBY;
720 for (int cpuindex = 0; cpuindex < ncpu; cpuindex++) {
721 gicv3_dma_alloc(sc, &sc->sc_lpipend[cpuindex], lpipend_sz, 0x10000);
722 KASSERT((sc->sc_lpipend[cpuindex].segs[0].ds_addr & ~GICR_PENDBASER_Physical_Address) == 0);
723 }
724 }
725
726 void
gicv3_irq_handler(void * frame)727 gicv3_irq_handler(void *frame)
728 {
729 struct cpu_info * const ci = curcpu();
730 struct gicv3_softc * const sc = gicv3_softc;
731 struct pic_softc *pic;
732 const int oldipl = ci->ci_cpl;
733 const int oldhwpl = IPL_TO_HWPL(oldipl);
734
735 ci->ci_data.cpu_nintr++;
736
737 if (ci->ci_hwpl != oldhwpl) {
738 ci->ci_hwpl = oldhwpl;
739 icc_pmr_write(IPL_TO_PMR(sc, oldhwpl));
740 if (oldhwpl == IPL_HIGH) {
741 return;
742 }
743 }
744
745 for (;;) {
746 const uint32_t iar = icc_iar1_read();
747 dsb(sy);
748 const uint32_t irq = __SHIFTOUT(iar, ICC_IAR_INTID);
749 if (irq == ICC_IAR_INTID_SPURIOUS)
750 break;
751
752 pic = irq >= GIC_LPI_BASE ? &sc->sc_lpi : &sc->sc_pic;
753 if (irq - pic->pic_irqbase >= pic->pic_maxsources)
754 continue;
755
756 struct intrsource * const is = pic->pic_sources[irq - pic->pic_irqbase];
757 KASSERT(is != NULL);
758
759 const bool early_eoi = irq < GIC_LPI_BASE && is->is_type == IST_EDGE;
760
761 const int ipl = is->is_ipl;
762 if (__predict_false(ipl < ci->ci_cpl)) {
763 pic_do_pending_ints(I32_bit, ipl, frame);
764 } else if (ci->ci_cpl != ipl) {
765 KASSERT(ipl >= IPL_VM);
766 icc_pmr_write(IPL_TO_PMR(sc, ipl));
767 ci->ci_hwpl = ci->ci_cpl = ipl;
768 }
769
770 if (early_eoi) {
771 icc_eoi1r_write(iar);
772 isb();
773 }
774
775 const int64_t nintr = ci->ci_data.cpu_nintr;
776
777 ENABLE_INTERRUPT();
778 pic_dispatch(is, frame);
779 DISABLE_INTERRUPT();
780
781 if (nintr != ci->ci_data.cpu_nintr)
782 ci->ci_intr_preempt.ev_count++;
783
784 if (!early_eoi) {
785 icc_eoi1r_write(iar);
786 isb();
787 }
788 }
789
790 pic_do_pending_ints(I32_bit, oldipl, frame);
791 }
792
793 static bool
gicv3_cpuif_is_nonsecure(struct gicv3_softc * sc)794 gicv3_cpuif_is_nonsecure(struct gicv3_softc *sc)
795 {
796 /*
797 * Write 0 to bit7 and see if it sticks. This is only possible if
798 * we have a non-secure view of the PMR register.
799 */
800 const uint32_t opmr = icc_pmr_read();
801 icc_pmr_write(0);
802 const uint32_t npmr = icc_pmr_read();
803 icc_pmr_write(opmr);
804
805 return (npmr & GICC_PMR_NONSECURE) == 0;
806 }
807
808 static bool
gicv3_dist_is_nonsecure(struct gicv3_softc * sc)809 gicv3_dist_is_nonsecure(struct gicv3_softc *sc)
810 {
811 const uint32_t gicd_ctrl = gicd_read_4(sc, GICD_CTRL);
812
813 /*
814 * If security is enabled, we have a non-secure view of the IPRIORITYRn
815 * registers and LPI configuration priority fields.
816 */
817 return (gicd_ctrl & GICD_CTRL_DS) == 0;
818 }
819
820 /*
821 * Rockchip RK3399 provides a different view of int priority registers
822 * depending on which firmware is in use. This is hard to detect in
823 * a way that could possibly break other boards, so only do this
824 * detection if we know we are on a RK3399 SoC.
825 */
826 static void
gicv3_quirk_rockchip_rk3399(struct gicv3_softc * sc)827 gicv3_quirk_rockchip_rk3399(struct gicv3_softc *sc)
828 {
829 /* Detect the number of supported PMR bits */
830 icc_pmr_write(0xff);
831 const uint8_t pmrbits = icc_pmr_read();
832
833 /* Detect the number of supported IPRIORITYRn bits */
834 const uint32_t oiprio = gicd_read_4(sc, GICD_IPRIORITYRn(8));
835 gicd_write_4(sc, GICD_IPRIORITYRn(8), oiprio | 0xff);
836 const uint8_t pribits = gicd_read_4(sc, GICD_IPRIORITYRn(8)) & 0xff;
837 gicd_write_4(sc, GICD_IPRIORITYRn(8), oiprio);
838
839 /*
840 * If we see fewer PMR bits than IPRIORITYRn bits here, it means
841 * we have a secure view of IPRIORITYRn (this is not supposed to
842 * happen!).
843 */
844 if (pmrbits < pribits) {
845 aprint_verbose_dev(sc->sc_dev,
846 "buggy RK3399 firmware detected; applying workaround\n");
847 sc->sc_priority_shift = GIC_PRIO_SHIFT_S;
848 }
849 }
850
851 int
gicv3_init(struct gicv3_softc * sc)852 gicv3_init(struct gicv3_softc *sc)
853 {
854 CPU_INFO_ITERATOR cii;
855 struct cpu_info *ci;
856
857 KASSERT(CPU_IS_PRIMARY(curcpu()));
858
859 LIST_INIT(&sc->sc_lpi_callbacks);
860
861 /* Store route to CPU for SPIs */
862 sc->sc_irouter = kmem_zalloc(sizeof(*sc->sc_irouter) * ncpu, KM_SLEEP);
863 for (CPU_INFO_FOREACH(cii, ci)) {
864 KASSERT(cpu_index(ci) < ncpu);
865 sc->sc_irouter[cpu_index(ci)] = ci->ci_cpuid;
866 }
867
868 sc->sc_gicd_typer = gicd_read_4(sc, GICD_TYPER);
869
870 /*
871 * We don't always have a consistent view of priorities between the
872 * CPU interface (ICC_PMR_EL1) and the GICD/GICR registers. Detect
873 * if we are making secure or non-secure accesses to each, and adjust
874 * the values that we write to each accordingly.
875 */
876 const bool dist_ns = gicv3_dist_is_nonsecure(sc);
877 sc->sc_priority_shift = dist_ns ? GIC_PRIO_SHIFT_NS : GIC_PRIO_SHIFT_S;
878 const bool cpuif_ns = gicv3_cpuif_is_nonsecure(sc);
879 sc->sc_pmr_shift = cpuif_ns ? GIC_PRIO_SHIFT_NS : GIC_PRIO_SHIFT_S;
880
881 if ((sc->sc_quirks & GICV3_QUIRK_RK3399) != 0)
882 gicv3_quirk_rockchip_rk3399(sc);
883
884 aprint_verbose_dev(sc->sc_dev,
885 "iidr 0x%08x, cpuif %ssecure, dist %ssecure, "
886 "priority shift %d, pmr shift %d, quirks %#x\n",
887 gicd_read_4(sc, GICD_IIDR),
888 cpuif_ns ? "non-" : "",
889 dist_ns ? "non-" : "",
890 sc->sc_priority_shift,
891 sc->sc_pmr_shift,
892 sc->sc_quirks);
893
894 sc->sc_pic.pic_ops = &gicv3_picops;
895 sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(sc->sc_gicd_typer);
896 snprintf(sc->sc_pic.pic_name, sizeof(sc->sc_pic.pic_name), "gicv3");
897 #ifdef MULTIPROCESSOR
898 sc->sc_pic.pic_cpus = kcpuset_running;
899 #endif
900 pic_add(&sc->sc_pic, 0);
901
902 if ((sc->sc_gicd_typer & GICD_TYPER_LPIS) != 0) {
903 sc->sc_lpipend = kmem_zalloc(sizeof(*sc->sc_lpipend) * ncpu, KM_SLEEP);
904 sc->sc_processor_id = kmem_zalloc(sizeof(*sc->sc_processor_id) * ncpu, KM_SLEEP);
905
906 sc->sc_lpi.pic_ops = &gicv3_lpiops;
907 sc->sc_lpi.pic_maxsources = 8192; /* Min. required by GICv3 spec */
908 snprintf(sc->sc_lpi.pic_name, sizeof(sc->sc_lpi.pic_name), "gicv3-lpi");
909 pic_add(&sc->sc_lpi, GIC_LPI_BASE);
910
911 sc->sc_lpi_pool = vmem_create("gicv3-lpi", 0, sc->sc_lpi.pic_maxsources,
912 1, NULL, NULL, NULL, 0, VM_SLEEP, IPL_HIGH);
913 if (sc->sc_lpi_pool == NULL)
914 panic("failed to create gicv3 lpi pool\n");
915
916 gicv3_lpi_init(sc);
917 }
918
919 KASSERT(gicv3_softc == NULL);
920 gicv3_softc = sc;
921
922 for (int i = 0; i < sc->sc_bsh_r_count; i++) {
923 const uint64_t gicr_typer = gicr_read_8(sc, i, GICR_TYPER);
924 const u_int aff0 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff0);
925 const u_int aff1 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff1);
926 const u_int aff2 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff2);
927 const u_int aff3 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff3);
928
929 aprint_debug_dev(sc->sc_dev, "redist %d: cpu %d.%d.%d.%d\n",
930 i, aff3, aff2, aff1, aff0);
931 }
932
933 gicv3_dist_enable(sc);
934
935 gicv3_cpu_init(&sc->sc_pic, curcpu());
936 if ((sc->sc_gicd_typer & GICD_TYPER_LPIS) != 0)
937 gicv3_lpi_cpu_init(&sc->sc_lpi, curcpu());
938
939 #ifdef MULTIPROCESSOR
940 intr_establish_xname(IPI_AST, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast");
941 intr_establish_xname(IPI_XCALL, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall");
942 intr_establish_xname(IPI_GENERIC, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic");
943 intr_establish_xname(IPI_NOP, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop");
944 intr_establish_xname(IPI_SHOOTDOWN, IPL_SCHED, IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown");
945 #ifdef DDB
946 intr_establish_xname(IPI_DDB, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb");
947 #endif
948 #ifdef __HAVE_PREEMPTION
949 intr_establish_xname(IPI_KPREEMPT, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt");
950 #endif
951 #endif
952
953 return 0;
954 }
955