1 /* $NetBSD: apple_intc.c,v 1.9 2022/06/28 10:42:22 jmcneill Exp $ */
2
3 /*-
4 * Copyright (c) 2021 Jared McNeill <jmcneill@invisible.ca>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 #include "opt_ddb.h"
30 #include "opt_multiprocessor.h"
31
32 #define _INTR_PRIVATE
33
34 #include <sys/cdefs.h>
35 __KERNEL_RCSID(0, "$NetBSD: apple_intc.c,v 1.9 2022/06/28 10:42:22 jmcneill Exp $");
36
37 #include <sys/param.h>
38 #include <sys/bus.h>
39 #include <sys/device.h>
40 #include <sys/intr.h>
41 #include <sys/kernel.h>
42 #include <sys/lwp.h>
43 #include <sys/systm.h>
44 #include <sys/cpu.h>
45 #include <sys/kmem.h>
46 #include <sys/atomic.h>
47
48 #include <dev/fdt/fdtvar.h>
49
50 #include <dev/pci/pcireg.h>
51 #include <dev/pci/pcivar.h>
52
53 #include <arm/cpu.h>
54 #include <arm/cpufunc.h>
55 #include <arm/armreg.h>
56 #include <arm/locore.h>
57 #include <arm/pic/picvar.h>
58 #include <arm/fdt/arm_fdtvar.h>
59
60 /*
61 * AIC registers
62 */
63 #define AIC_INFO 0x0004
64 #define AIC_INFO_NIRQ __BITS(15,0)
65 #define AIC_WHOAMI 0x2000
66 #define AIC_EVENT 0x2004
67 #define AIC_EVENT_TYPE __BITS(31,16)
68 #define AIC_EVENT_TYPE_NONE 0
69 #define AIC_EVENT_TYPE_IRQ 1
70 #define AIC_EVENT_TYPE_IPI 4
71 #define AIC_EVENT_DATA __BITS(15,0)
72 #define AIC_EVENT_IPI_OTHER 1
73 #define AIC_IPI_SEND 0x2008
74 #define AIC_IPI_ACK 0x200c
75 #define AIC_IPI_MASK_CLR 0x2028
76 #define AIC_IPI_OTHER __BIT(0)
77 #define AIC_AFFINITY(irqno) (0x3000 + (irqno) * 4)
78 #define AIC_SW_SET(irqno) (0x4000 + (irqno) / 32 * 4)
79 #define AIC_SW_CLR(irqno) (0x4080 + (irqno) / 32 * 4)
80 #define AIC_MASK_SET(irqno) (0x4100 + (irqno) / 32 * 4)
81 #define AIC_MASK_CLR(irqno) (0x4180 + (irqno) / 32 * 4)
82 #define AIC_MASK_BIT(irqno) __BIT((irqno) & 0x1f)
83
84 static const struct device_compatible_entry compat_data[] = {
85 { .compat = "apple,aic" },
86 DEVICE_COMPAT_EOL
87 };
88
89 struct apple_intc_softc;
90
91 struct apple_intc_percpu {
92 struct apple_intc_softc *pc_sc;
93 u_int pc_cpuid;
94 u_int pc_ipimask;
95
96 struct pic_softc pc_pic;
97 };
98
99 #define LOCALPIC_SOURCE_TIMER 0
100 #define LOCALPIC_SOURCE_IPI 1
101
102 struct apple_intc_softc {
103 device_t sc_dev; /* device handle */
104 bus_space_tag_t sc_bst; /* mmio tag */
105 bus_space_handle_t sc_bsh; /* mmio handle */
106 u_int sc_nirq; /* number of supported IRQs */
107 u_int *sc_cpuid; /* map of cpu index to AIC CPU ID */
108 struct apple_intc_percpu *sc_pc; /* per-CPU data for timer and IPIs */
109
110 struct pic_softc sc_pic;
111 };
112
113 static struct apple_intc_softc *intc_softc;
114
115 #define PICTOSOFTC(pic) container_of(pic, struct apple_intc_softc, sc_pic)
116 #define PICTOPERCPU(pic) container_of(pic, struct apple_intc_percpu, pc_pic)
117
118 #define AIC_READ(sc, reg) \
119 bus_space_read_4((sc)->sc_bst, (sc)->sc_bsh, (reg))
120 #define AIC_WRITE(sc, reg, val) \
121 bus_space_write_4((sc)->sc_bst, (sc)->sc_bsh, (reg), (val))
122
123 static void
apple_intc_unblock_irqs(struct pic_softc * pic,size_t irqbase,uint32_t mask)124 apple_intc_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
125 {
126 struct apple_intc_softc * const sc = PICTOSOFTC(pic);
127
128 AIC_WRITE(sc, AIC_SW_SET(irqbase), mask);
129 AIC_WRITE(sc, AIC_MASK_CLR(irqbase), mask);
130 }
131
132 static void
apple_intc_block_irqs(struct pic_softc * pic,size_t irqbase,uint32_t mask)133 apple_intc_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
134 {
135 }
136
137 static void
apple_intc_establish_irq(struct pic_softc * pic,struct intrsource * is)138 apple_intc_establish_irq(struct pic_softc *pic, struct intrsource *is)
139 {
140 struct apple_intc_softc * const sc = PICTOSOFTC(pic);
141
142 KASSERT(is->is_type == IST_LEVEL);
143
144 /* Route to primary PE by default */
145 AIC_WRITE(sc, AIC_AFFINITY(is->is_irq), __BIT(0));
146 AIC_WRITE(sc, AIC_MASK_CLR(is->is_irq),
147 AIC_MASK_BIT(is->is_irq));
148 }
149
150 static void
apple_intc_set_priority(struct pic_softc * pic,int ipl)151 apple_intc_set_priority(struct pic_softc *pic, int ipl)
152 {
153 curcpu()->ci_cpl = ipl;
154 }
155
156 static void
apple_intc_cpu_init(struct pic_softc * pic,struct cpu_info * ci)157 apple_intc_cpu_init(struct pic_softc *pic, struct cpu_info *ci)
158 {
159 struct apple_intc_softc * const sc = PICTOSOFTC(pic);
160 const u_int cpuno = cpu_index(ci);
161
162 sc->sc_cpuid[cpuno] = AIC_READ(sc, AIC_WHOAMI);
163 }
164
165 static const struct pic_ops apple_intc_picops = {
166 .pic_unblock_irqs = apple_intc_unblock_irqs,
167 .pic_block_irqs = apple_intc_block_irqs,
168 .pic_establish_irq = apple_intc_establish_irq,
169 .pic_set_priority = apple_intc_set_priority,
170 #ifdef MULTIPROCESSOR
171 .pic_cpu_init = apple_intc_cpu_init,
172 #endif
173 };
174
175 static void
apple_intc_local_unblock_irqs(struct pic_softc * pic,size_t irqbase,uint32_t mask)176 apple_intc_local_unblock_irqs(struct pic_softc *pic, size_t irqbase,
177 uint32_t mask)
178 {
179 KASSERT(irqbase == 0);
180
181 if ((mask & __BIT(LOCALPIC_SOURCE_TIMER)) != 0) {
182 gtmr_cntv_ctl_write(gtmr_cntv_ctl_read() & ~CNTCTL_IMASK);
183 isb();
184 }
185 }
186
187 static void
apple_intc_local_block_irqs(struct pic_softc * pic,size_t irqbase,uint32_t mask)188 apple_intc_local_block_irqs(struct pic_softc *pic, size_t irqbase,
189 uint32_t mask)
190 {
191 KASSERT(irqbase == 0);
192
193 if ((mask & __BIT(LOCALPIC_SOURCE_TIMER)) != 0) {
194 gtmr_cntv_ctl_write(gtmr_cntv_ctl_read() | CNTCTL_IMASK);
195 isb();
196 }
197 }
198
199 static void
apple_intc_local_establish_irq(struct pic_softc * pic,struct intrsource * is)200 apple_intc_local_establish_irq(struct pic_softc *pic, struct intrsource *is)
201 {
202 }
203
204 #ifdef MULTIPROCESSOR
205 static void
apple_intc_local_ipi_send(struct pic_softc * pic,const kcpuset_t * kcp,u_long ipi)206 apple_intc_local_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi)
207 {
208 struct apple_intc_percpu * const pc = PICTOPERCPU(pic);
209 struct apple_intc_softc * const sc = pc->pc_sc;
210 const u_int target = sc->sc_cpuid[pc->pc_cpuid];
211
212 atomic_or_32(&pc->pc_ipimask, __BIT(ipi));
213 AIC_WRITE(sc, AIC_IPI_SEND, __BIT(target));
214 }
215 #endif /* MULTIPROCESSOR */
216
217 static const struct pic_ops apple_intc_localpicops = {
218 .pic_unblock_irqs = apple_intc_local_unblock_irqs,
219 .pic_block_irqs = apple_intc_local_block_irqs,
220 .pic_establish_irq = apple_intc_local_establish_irq,
221 #ifdef MULTIPROCESSOR
222 .pic_ipi_send = apple_intc_local_ipi_send,
223 #endif
224 };
225
226 static void *
apple_intc_fdt_establish(device_t dev,u_int * specifier,int ipl,int flags,int (* func)(void *),void * arg,const char * xname)227 apple_intc_fdt_establish(device_t dev, u_int *specifier, int ipl, int flags,
228 int (*func)(void *), void *arg, const char *xname)
229 {
230 struct apple_intc_softc * const sc = device_private(dev);
231
232 /* 1st cell is the interrupt type (0=IRQ, 1=FIQ) */
233 const u_int type = be32toh(specifier[0]);
234 /* 2nd cell is the interrupt number */
235 const u_int intno = be32toh(specifier[1]);
236 /* 3rd cell is the interrupt flags */
237
238 const u_int mpsafe = (flags & FDT_INTR_MPSAFE) ? IST_MPSAFE : 0;
239
240 if (type == 0)
241 return intr_establish_xname(intno, ipl, IST_LEVEL | mpsafe,
242 func, arg, xname);
243
244 /* interate over CPUs for LOCALPIC_SOURCE_TIMER */
245 CPU_INFO_ITERATOR cii;
246 struct cpu_info *ci;
247 void *ih = NULL;
248 for (CPU_INFO_FOREACH(cii, ci)) {
249 const cpuid_t cpuno = cpu_index(ci);
250 struct apple_intc_percpu * const pc = &sc->sc_pc[cpuno];
251 struct pic_softc * const pic = &pc->pc_pic;
252 const int irq = pic->pic_irqbase + LOCALPIC_SOURCE_TIMER;
253
254 void *ihn = intr_establish_xname(irq, ipl, IST_LEVEL | mpsafe,
255 func, arg, xname);
256 if (cpuno == 0)
257 ih = ihn;
258 }
259 return ih;
260 }
261
262 static void
apple_intc_fdt_disestablish(device_t dev,void * ih)263 apple_intc_fdt_disestablish(device_t dev, void *ih)
264 {
265 intr_disestablish(ih);
266 }
267
268 static bool
apple_intc_fdt_intrstr(device_t dev,u_int * specifier,char * buf,size_t buflen)269 apple_intc_fdt_intrstr(device_t dev, u_int *specifier, char *buf, size_t buflen)
270 {
271 if (!specifier)
272 return false;
273
274 /* 1st cell is the interrupt type (0=IRQ, 1=FIQ) */
275 const u_int type = be32toh(specifier[0]);
276 /* 2nd cell is the interrupt number */
277 const u_int intno = be32toh(specifier[1]);
278
279 snprintf(buf, buflen, "%s %u", type == 0 ? "irq" : "fiq", intno);
280
281 return true;
282 }
283
284 static const struct fdtbus_interrupt_controller_func apple_intc_fdt_funcs = {
285 .establish = apple_intc_fdt_establish,
286 .disestablish = apple_intc_fdt_disestablish,
287 .intrstr = apple_intc_fdt_intrstr,
288 };
289
290 static void
apple_intc_mark_pending(struct pic_softc * pic,u_int intno)291 apple_intc_mark_pending(struct pic_softc *pic, u_int intno)
292 {
293 const int base = intno & ~0x1f;
294 const uint32_t pending = __BIT(intno & 0x1f);
295 pic_mark_pending_sources(pic, base, pending);
296 }
297
298 static void
apple_intc_irq_handler(void * frame)299 apple_intc_irq_handler(void *frame)
300 {
301 struct cpu_info * const ci = curcpu();
302 struct apple_intc_softc * const sc = intc_softc;
303 struct pic_softc *pic;
304 struct intrsource *is;
305 const int oldipl = ci->ci_cpl;
306 uint16_t evtype, evdata;
307 bus_size_t clr_reg;
308 uint32_t clr_val;
309
310 ci->ci_data.cpu_nintr++;
311
312 for (;;) {
313 const uint32_t ev = AIC_READ(sc, AIC_EVENT);
314 evtype = __SHIFTOUT(ev, AIC_EVENT_TYPE);
315 evdata = __SHIFTOUT(ev, AIC_EVENT_DATA);
316
317 dsb(sy);
318 isb();
319
320 if (evtype == AIC_EVENT_TYPE_IRQ) {
321 KASSERT(evdata < sc->sc_nirq);
322 pic = &sc->sc_pic;
323 is = pic->pic_sources[evdata];
324 KASSERT(is != NULL);
325
326 AIC_WRITE(sc, AIC_SW_CLR(evdata),
327 __BIT(evdata & 0x1f));
328
329 clr_reg = AIC_MASK_CLR(evdata);
330 clr_val = AIC_MASK_BIT(evdata);
331 } else if (evtype == AIC_EVENT_TYPE_IPI) {
332 KASSERT(evdata == AIC_EVENT_IPI_OTHER);
333 pic = &sc->sc_pc[cpu_index(ci)].pc_pic;
334 is = pic->pic_sources[LOCALPIC_SOURCE_IPI];
335 KASSERT(is != NULL);
336
337 AIC_WRITE(sc, AIC_IPI_ACK, AIC_IPI_OTHER);
338
339 clr_reg = 0;
340 clr_val = 0;
341 } else {
342 break;
343 }
344
345 if (ci->ci_cpl >= is->is_ipl) {
346 apple_intc_mark_pending(pic, is->is_irq);
347 } else {
348 pic_set_priority(ci, is->is_ipl);
349 ENABLE_INTERRUPT();
350 pic_dispatch(is, frame);
351 DISABLE_INTERRUPT();
352
353 if (clr_val != 0) {
354 AIC_WRITE(sc, clr_reg, clr_val);
355 }
356 }
357 }
358
359 if (oldipl != IPL_HIGH) {
360 pic_do_pending_ints(DAIF_I|DAIF_F, oldipl, frame);
361 }
362 }
363
364 static void
apple_intc_fiq_handler(void * frame)365 apple_intc_fiq_handler(void *frame)
366 {
367 struct cpu_info * const ci = curcpu();
368 struct apple_intc_softc * const sc = intc_softc;
369 struct pic_softc * const pic = &sc->sc_pc[cpu_index(ci)].pc_pic;
370 const int oldipl = ci->ci_cpl;
371
372 ci->ci_data.cpu_nintr++;
373
374 struct intrsource * const is = pic->pic_sources[LOCALPIC_SOURCE_TIMER];
375
376 dsb(sy);
377 isb();
378
379 if (oldipl >= is->is_ipl) {
380 apple_intc_mark_pending(pic, LOCALPIC_SOURCE_TIMER);
381 } else {
382 pic_set_priority(ci, is->is_ipl);
383 pic_dispatch(is, frame);
384 }
385
386 if (oldipl != IPL_HIGH) {
387 pic_do_pending_ints(DAIF_I|DAIF_F, oldipl, frame);
388 }
389 }
390
391 #ifdef MULTIPROCESSOR
392 static int
apple_intc_ipi_handler(void * priv)393 apple_intc_ipi_handler(void *priv)
394 {
395 struct apple_intc_percpu * const pc = priv;
396 struct apple_intc_softc * const sc = pc->pc_sc;
397 uint32_t ipimask, bit;
398
399 AIC_WRITE(sc, AIC_IPI_MASK_CLR, AIC_IPI_OTHER);
400 ipimask = atomic_swap_32(&pc->pc_ipimask, 0);
401
402 while ((bit = ffs(ipimask)) > 0) {
403 const u_int ipi = bit - 1;
404
405 switch (ipi) {
406 case IPI_AST:
407 pic_ipi_ast(priv);
408 break;
409 case IPI_NOP:
410 pic_ipi_nop(priv);
411 break;
412 #ifdef __HAVE_PREEMPTION
413 case IPI_KPREEMPT:
414 pic_ipi_kpreempt(priv);
415 break;
416 #endif
417 case IPI_XCALL:
418 pic_ipi_xcall(priv);
419 break;
420 case IPI_GENERIC:
421 pic_ipi_generic(priv);
422 break;
423 case IPI_SHOOTDOWN:
424 pic_ipi_shootdown(priv);
425 break;
426 #ifdef DDB
427 case IPI_DDB:
428 pic_ipi_ddb(priv);
429 break;
430 #endif
431 }
432 ipimask &= ~__BIT(ipi);
433 }
434
435 return 1;
436 }
437 #endif /* MULTIPROCESSOR */
438
439 static int
apple_intc_match(device_t parent,cfdata_t cf,void * aux)440 apple_intc_match(device_t parent, cfdata_t cf, void *aux)
441 {
442 struct fdt_attach_args * const faa = aux;
443
444 return of_compatible_match(faa->faa_phandle, compat_data);
445 }
446
447 static void
apple_intc_attach(device_t parent,device_t self,void * aux)448 apple_intc_attach(device_t parent, device_t self, void *aux)
449 {
450 struct apple_intc_softc * const sc = device_private(self);
451 struct fdt_attach_args * const faa = aux;
452 const int phandle = faa->faa_phandle;
453 bus_addr_t addr;
454 bus_size_t size;
455 int error;
456
457 if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) {
458 aprint_error(": couldn't get registers\n");
459 return;
460 }
461
462 sc->sc_dev = self;
463 sc->sc_bst = faa->faa_bst;
464 if (bus_space_map(sc->sc_bst, addr, size, 0, &sc->sc_bsh) != 0) {
465 aprint_error(": couldn't map registers\n");
466 return;
467 }
468
469 sc->sc_nirq = AIC_READ(sc, AIC_INFO) & AIC_INFO_NIRQ;
470
471 aprint_naive("\n");
472 aprint_normal(": Apple AIC (%u IRQs, 1 FIQ)\n", sc->sc_nirq);
473 KASSERT(sc->sc_nirq % 32 == 0);
474
475 sc->sc_pic.pic_ops = &apple_intc_picops;
476 sc->sc_pic.pic_maxsources = sc->sc_nirq;
477 snprintf(sc->sc_pic.pic_name, sizeof(sc->sc_pic.pic_name), "AIC");
478 pic_add(&sc->sc_pic, 0);
479
480 error = fdtbus_register_interrupt_controller(self, phandle,
481 &apple_intc_fdt_funcs);
482 if (error) {
483 aprint_error_dev(self, "couldn't register with fdtbus: %d\n",
484 error);
485 return;
486 }
487
488 KASSERT(intc_softc == NULL);
489 intc_softc = sc;
490 arm_fdt_irq_set_handler(apple_intc_irq_handler);
491 arm_fdt_fiq_set_handler(apple_intc_fiq_handler);
492
493 KASSERT(ncpu != 0);
494 sc->sc_cpuid = kmem_zalloc(sizeof(*sc->sc_cpuid) * ncpu, KM_SLEEP);
495 sc->sc_pc = kmem_zalloc(sizeof(*sc->sc_pc) * ncpu, KM_SLEEP);
496
497 CPU_INFO_ITERATOR cii;
498 struct cpu_info *ci;
499 for (CPU_INFO_FOREACH(cii, ci)) {
500 const cpuid_t cpuno = cpu_index(ci);
501 struct apple_intc_percpu * const pc = &sc->sc_pc[cpuno];
502 struct pic_softc * const pic = &pc->pc_pic;
503
504 pc->pc_sc = sc;
505 pc->pc_cpuid = cpuno;
506
507 #ifdef MULTIPROCESSOR
508 pic->pic_cpus = ci->ci_kcpuset;
509 #endif
510 pic->pic_ops = &apple_intc_localpicops;
511 pic->pic_maxsources = 2;
512 snprintf(pic->pic_name, sizeof(pic->pic_name), "AIC/%lu", cpuno);
513
514 pic_add(pic, PIC_IRQBASE_ALLOC);
515
516 #ifdef MULTIPROCESSOR
517 intr_establish_xname(pic->pic_irqbase + LOCALPIC_SOURCE_IPI,
518 IPL_HIGH, IST_LEVEL | IST_MPSAFE, apple_intc_ipi_handler,
519 pc, "ipi");
520 #endif
521 }
522
523 apple_intc_cpu_init(&sc->sc_pic, curcpu());
524 }
525
526 CFATTACH_DECL_NEW(apple_intc, sizeof(struct apple_intc_softc),
527 apple_intc_match, apple_intc_attach, NULL, NULL);
528