xref: /netbsd-src/sys/arch/arm/cortex/gicv3.c (revision a87250117ad8929d62ddf579163d64e532dfc524)
1 /* $NetBSD: gicv3.c,v 1.54 2022/06/26 11:14:36 jmcneill Exp $ */
2 
3 /*-
4  * Copyright (c) 2018 Jared McNeill <jmcneill@invisible.ca>
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
21  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
22  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
23  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
24  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  */
28 
29 #include "opt_multiprocessor.h"
30 
31 #define	_INTR_PRIVATE
32 
33 #include <sys/cdefs.h>
34 __KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.54 2022/06/26 11:14:36 jmcneill Exp $");
35 
36 #include <sys/param.h>
37 #include <sys/kernel.h>
38 #include <sys/bus.h>
39 #include <sys/device.h>
40 #include <sys/intr.h>
41 #include <sys/systm.h>
42 #include <sys/cpu.h>
43 #include <sys/vmem.h>
44 #include <sys/kmem.h>
45 #include <sys/atomic.h>
46 
47 #include <machine/cpufunc.h>
48 
49 #include <arm/locore.h>
50 #include <arm/armreg.h>
51 
52 #include <arm/cortex/gicv3.h>
53 #include <arm/cortex/gic_reg.h>
54 
55 #define	PICTOSOFTC(pic)	\
56 	container_of(pic, struct gicv3_softc, sc_pic)
57 #define	LPITOSOFTC(lpi) \
58 	container_of(lpi, struct gicv3_softc, sc_lpi)
59 
60 #define	IPL_TO_PRIORITY(sc, ipl)	(((0xff - (ipl)) << (sc)->sc_priority_shift) & 0xff)
61 #define	IPL_TO_PMR(sc, ipl)		(((0xff - (ipl)) << (sc)->sc_pmr_shift) & 0xff)
62 #define	IPL_TO_HWPL(ipl)		((ipl) >= IPL_VM ? (ipl) : IPL_NONE)
63 
64 #define	GIC_SUPPORTS_1OFN(sc)		(((sc)->sc_gicd_typer & GICD_TYPER_No1N) == 0)
65 
66 #define	GIC_PRIO_SHIFT_NS		4
67 #define	GIC_PRIO_SHIFT_S		3
68 
69 /*
70  * Set to true if you want to use 1 of N interrupt distribution for SPIs
71  * when available. Disabled by default because it causes issues with the
72  * USB stack.
73  */
74 bool gicv3_use_1ofn = false;
75 
76 static struct gicv3_softc *gicv3_softc;
77 
78 static inline uint32_t
gicd_read_4(struct gicv3_softc * sc,bus_size_t reg)79 gicd_read_4(struct gicv3_softc *sc, bus_size_t reg)
80 {
81 	return bus_space_read_4(sc->sc_bst, sc->sc_bsh_d, reg);
82 }
83 
84 static inline void
gicd_write_4(struct gicv3_softc * sc,bus_size_t reg,uint32_t val)85 gicd_write_4(struct gicv3_softc *sc, bus_size_t reg, uint32_t val)
86 {
87 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_d, reg, val);
88 }
89 
90 #ifdef MULTIPROCESSOR
91 static inline uint64_t
gicd_read_8(struct gicv3_softc * sc,bus_size_t reg)92 gicd_read_8(struct gicv3_softc *sc, bus_size_t reg)
93 {
94 	return bus_space_read_8(sc->sc_bst, sc->sc_bsh_d, reg);
95 }
96 #endif
97 
98 static inline void
gicd_write_8(struct gicv3_softc * sc,bus_size_t reg,uint64_t val)99 gicd_write_8(struct gicv3_softc *sc, bus_size_t reg, uint64_t val)
100 {
101 	bus_space_write_8(sc->sc_bst, sc->sc_bsh_d, reg, val);
102 }
103 
104 static inline uint32_t
gicr_read_4(struct gicv3_softc * sc,u_int index,bus_size_t reg)105 gicr_read_4(struct gicv3_softc *sc, u_int index, bus_size_t reg)
106 {
107 	KASSERT(index < sc->sc_bsh_r_count);
108 	return bus_space_read_4(sc->sc_bst, sc->sc_bsh_r[index], reg);
109 }
110 
111 static inline void
gicr_write_4(struct gicv3_softc * sc,u_int index,bus_size_t reg,uint32_t val)112 gicr_write_4(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint32_t val)
113 {
114 	KASSERT(index < sc->sc_bsh_r_count);
115 	bus_space_write_4(sc->sc_bst, sc->sc_bsh_r[index], reg, val);
116 }
117 
118 static inline uint64_t
gicr_read_8(struct gicv3_softc * sc,u_int index,bus_size_t reg)119 gicr_read_8(struct gicv3_softc *sc, u_int index, bus_size_t reg)
120 {
121 	KASSERT(index < sc->sc_bsh_r_count);
122 	return bus_space_read_8(sc->sc_bst, sc->sc_bsh_r[index], reg);
123 }
124 
125 static inline void
gicr_write_8(struct gicv3_softc * sc,u_int index,bus_size_t reg,uint64_t val)126 gicr_write_8(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint64_t val)
127 {
128 	KASSERT(index < sc->sc_bsh_r_count);
129 	bus_space_write_8(sc->sc_bst, sc->sc_bsh_r[index], reg, val);
130 }
131 
132 static void
gicv3_unblock_irqs(struct pic_softc * pic,size_t irqbase,uint32_t mask)133 gicv3_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
134 {
135 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
136 	struct cpu_info * const ci = curcpu();
137 	const u_int group = irqbase / 32;
138 
139 	if (group == 0) {
140 		atomic_or_32(&sc->sc_enabled_sgippi, mask);
141 		gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, mask);
142 		while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
143 			;
144 	} else {
145 		gicd_write_4(sc, GICD_ISENABLERn(group), mask);
146 		while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
147 			;
148 	}
149 }
150 
151 static void
gicv3_block_irqs(struct pic_softc * pic,size_t irqbase,uint32_t mask)152 gicv3_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
153 {
154 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
155 	struct cpu_info * const ci = curcpu();
156 	const u_int group = irqbase / 32;
157 
158 	if (group == 0) {
159 		atomic_and_32(&sc->sc_enabled_sgippi, ~mask);
160 		gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, mask);
161 		while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
162 			;
163 	} else {
164 		gicd_write_4(sc, GICD_ICENABLERn(group), mask);
165 		while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
166 			;
167 	}
168 }
169 
170 static void
gicv3_establish_irq(struct pic_softc * pic,struct intrsource * is)171 gicv3_establish_irq(struct pic_softc *pic, struct intrsource *is)
172 {
173 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
174 	const u_int group = is->is_irq / 32;
175 	uint32_t ipriority, icfg;
176 	uint64_t irouter;
177 	u_int n;
178 
179 	const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl);
180 	const u_int ipriority_shift = (is->is_irq & 0x3) * 8;
181 	const u_int icfg_shift = (is->is_irq & 0xf) * 2;
182 
183 	if (group == 0) {
184 		/* SGIs and PPIs are per-CPU and always MP-safe */
185 		is->is_mpsafe = true;
186 		is->is_percpu = true;
187 
188 		/* Update interrupt configuration and priority on all redistributors */
189 		for (n = 0; n < sc->sc_bsh_r_count; n++) {
190 			icfg = gicr_read_4(sc, n, GICR_ICFGRn(is->is_irq / 16));
191 			if (is->is_type == IST_LEVEL)
192 				icfg &= ~(0x2 << icfg_shift);
193 			if (is->is_type == IST_EDGE)
194 				icfg |= (0x2 << icfg_shift);
195 			gicr_write_4(sc, n, GICR_ICFGRn(is->is_irq / 16), icfg);
196 
197 			ipriority = gicr_read_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4));
198 			ipriority &= ~(0xffU << ipriority_shift);
199 			ipriority |= (ipriority_val << ipriority_shift);
200 			gicr_write_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4), ipriority);
201 		}
202 	} else {
203 		/*
204 		 * If 1 of N SPI routing is supported, route MP-safe interrupts to all
205 		 * participating PEs. Otherwise, just route to the primary PE.
206 		 */
207 		if (is->is_mpsafe && GIC_SUPPORTS_1OFN(sc) && gicv3_use_1ofn) {
208 			irouter = GICD_IROUTER_Interrupt_Routing_mode;
209 		} else {
210 			irouter = sc->sc_irouter[0];
211 		}
212 		gicd_write_8(sc, GICD_IROUTER(is->is_irq), irouter);
213 
214 		/* Update interrupt configuration */
215 		icfg = gicd_read_4(sc, GICD_ICFGRn(is->is_irq / 16));
216 		if (is->is_type == IST_LEVEL)
217 			icfg &= ~(0x2 << icfg_shift);
218 		if (is->is_type == IST_EDGE)
219 			icfg |= (0x2 << icfg_shift);
220 		gicd_write_4(sc, GICD_ICFGRn(is->is_irq / 16), icfg);
221 
222 		/* Update interrupt priority */
223 		ipriority = gicd_read_4(sc, GICD_IPRIORITYRn(is->is_irq / 4));
224 		ipriority &= ~(0xffU << ipriority_shift);
225 		ipriority |= (ipriority_val << ipriority_shift);
226 		gicd_write_4(sc, GICD_IPRIORITYRn(is->is_irq / 4), ipriority);
227 	}
228 }
229 
230 static void
gicv3_set_priority(struct pic_softc * pic,int ipl)231 gicv3_set_priority(struct pic_softc *pic, int ipl)
232 {
233 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
234 	struct cpu_info * const ci = curcpu();
235 	const int hwpl = IPL_TO_HWPL(ipl);
236 
237 	while (hwpl < ci->ci_hwpl) {
238 		/* Lowering priority mask */
239 		ci->ci_hwpl = hwpl;
240 		__insn_barrier();
241 		icc_pmr_write(IPL_TO_PMR(sc, hwpl));
242 	}
243 	__insn_barrier();
244 	ci->ci_cpl = ipl;
245 }
246 
247 static void
gicv3_dist_enable(struct gicv3_softc * sc)248 gicv3_dist_enable(struct gicv3_softc *sc)
249 {
250 	uint32_t gicd_ctrl;
251 	u_int n;
252 
253 	/* Disable the distributor */
254 	gicd_ctrl = gicd_read_4(sc, GICD_CTRL);
255 	gicd_ctrl &= ~(GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS);
256 	gicd_write_4(sc, GICD_CTRL, gicd_ctrl);
257 
258 	/* Wait for register write to complete */
259 	while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
260 		;
261 
262 	/* Clear all INTID enable bits */
263 	for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32)
264 		gicd_write_4(sc, GICD_ICENABLERn(n / 32), ~0);
265 
266 	/* Set default priorities to lowest */
267 	for (n = 32; n < sc->sc_pic.pic_maxsources; n += 4)
268 		gicd_write_4(sc, GICD_IPRIORITYRn(n / 4), ~0);
269 
270 	/* Set all interrupts to G1NS */
271 	for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) {
272 		gicd_write_4(sc, GICD_IGROUPRn(n / 32), ~0);
273 		gicd_write_4(sc, GICD_IGRPMODRn(n / 32), 0);
274 	}
275 
276 	/* Set all interrupts level-sensitive by default */
277 	for (n = 32; n < sc->sc_pic.pic_maxsources; n += 16)
278 		gicd_write_4(sc, GICD_ICFGRn(n / 16), 0);
279 
280 	/* Wait for register writes to complete */
281 	while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP)
282 		;
283 
284 	/* Enable Affinity routing and G1NS interrupts */
285 	gicd_ctrl = GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS;
286 	gicd_write_4(sc, GICD_CTRL, gicd_ctrl);
287 }
288 
289 static void
gicv3_redist_enable(struct gicv3_softc * sc,struct cpu_info * ci)290 gicv3_redist_enable(struct gicv3_softc *sc, struct cpu_info *ci)
291 {
292 	uint32_t icfg;
293 	u_int n, o;
294 
295 	/* Clear INTID enable bits */
296 	gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, ~0);
297 
298 	/* Wait for register write to complete */
299 	while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
300 		;
301 
302 	/* Set default priorities */
303 	for (n = 0; n < 32; n += 4) {
304 		uint32_t priority = 0;
305 		size_t byte_shift = 0;
306 		for (o = 0; o < 4; o++, byte_shift += 8) {
307 			struct intrsource * const is = sc->sc_pic.pic_sources[n + o];
308 			if (is == NULL)
309 				priority |= (0xffU << byte_shift);
310 			else {
311 				const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl);
312 				priority |= ipriority_val << byte_shift;
313 			}
314 		}
315 		gicr_write_4(sc, ci->ci_gic_redist, GICR_IPRIORITYRn(n / 4), priority);
316 	}
317 
318 	/* Set all interrupts to G1NS */
319 	gicr_write_4(sc, ci->ci_gic_redist, GICR_IGROUPR0, ~0);
320 	gicr_write_4(sc, ci->ci_gic_redist, GICR_IGRPMODR0, 0);
321 
322 	/* Restore PPI configs */
323 	for (n = 0, icfg = 0; n < 16; n++) {
324 		struct intrsource * const is = sc->sc_pic.pic_sources[16 + n];
325 		if (is != NULL && is->is_type == IST_EDGE)
326 			icfg |= (0x2 << (n * 2));
327 	}
328 	gicr_write_4(sc, ci->ci_gic_redist, GICR_ICFGRn(1), icfg);
329 
330 	/* Restore current enable bits */
331 	gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, sc->sc_enabled_sgippi);
332 
333 	/* Wait for register write to complete */
334 	while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP)
335 		;
336 }
337 
338 static uint64_t
gicv3_cpu_identity(void)339 gicv3_cpu_identity(void)
340 {
341 	u_int aff3, aff2, aff1, aff0;
342 
343 	const register_t mpidr = cpu_mpidr_aff_read();
344 	aff0 = __SHIFTOUT(mpidr, MPIDR_AFF0);
345 	aff1 = __SHIFTOUT(mpidr, MPIDR_AFF1);
346 	aff2 = __SHIFTOUT(mpidr, MPIDR_AFF2);
347 	aff3 = __SHIFTOUT(mpidr, MPIDR_AFF3);
348 
349 	return __SHIFTIN(aff0, GICR_TYPER_Affinity_Value_Aff0) |
350 	       __SHIFTIN(aff1, GICR_TYPER_Affinity_Value_Aff1) |
351 	       __SHIFTIN(aff2, GICR_TYPER_Affinity_Value_Aff2) |
352 	       __SHIFTIN(aff3, GICR_TYPER_Affinity_Value_Aff3);
353 }
354 
355 static u_int
gicv3_find_redist(struct gicv3_softc * sc)356 gicv3_find_redist(struct gicv3_softc *sc)
357 {
358 	uint64_t gicr_typer;
359 	u_int n;
360 
361 	const uint64_t cpu_identity = gicv3_cpu_identity();
362 
363 	for (n = 0; n < sc->sc_bsh_r_count; n++) {
364 		gicr_typer = gicr_read_8(sc, n, GICR_TYPER);
365 		if ((gicr_typer & GICR_TYPER_Affinity_Value) == cpu_identity)
366 			return n;
367 	}
368 
369 	const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0);
370 	const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1);
371 	const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2);
372 	const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3);
373 
374 	panic("%s: could not find GICv3 redistributor for cpu %d.%d.%d.%d",
375 	    cpu_name(curcpu()), aff3, aff2, aff1, aff0);
376 }
377 
378 static uint64_t
gicv3_sgir(struct gicv3_softc * sc)379 gicv3_sgir(struct gicv3_softc *sc)
380 {
381 	const uint64_t cpu_identity = gicv3_cpu_identity();
382 
383 	const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0);
384 	const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1);
385 	const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2);
386 	const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3);
387 
388 	return __SHIFTIN(__BIT(aff0), ICC_SGIR_EL1_TargetList) |
389 	       __SHIFTIN(aff1, ICC_SGIR_EL1_Aff1) |
390 	       __SHIFTIN(aff2, ICC_SGIR_EL1_Aff2) |
391 	       __SHIFTIN(aff3, ICC_SGIR_EL1_Aff3);
392 }
393 
394 static void
gicv3_cpu_init(struct pic_softc * pic,struct cpu_info * ci)395 gicv3_cpu_init(struct pic_softc *pic, struct cpu_info *ci)
396 {
397 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
398 	uint32_t icc_sre, icc_ctlr, gicr_waker;
399 
400 	evcnt_attach_dynamic(&ci->ci_intr_preempt, EVCNT_TYPE_MISC, NULL,
401 	    ci->ci_cpuname, "intr preempt");
402 
403 	ci->ci_gic_redist = gicv3_find_redist(sc);
404 	ci->ci_gic_sgir = gicv3_sgir(sc);
405 
406 	/* Enable System register access and disable IRQ/FIQ bypass */
407 	icc_sre = ICC_SRE_EL1_SRE | ICC_SRE_EL1_DFB | ICC_SRE_EL1_DIB;
408 	icc_sre_write(icc_sre);
409 
410 	/* Mark the connected PE as being awake */
411 	gicr_waker = gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER);
412 	gicr_waker &= ~GICR_WAKER_ProcessorSleep;
413 	gicr_write_4(sc, ci->ci_gic_redist, GICR_WAKER, gicr_waker);
414 	while (gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER) & GICR_WAKER_ChildrenAsleep)
415 		;
416 
417 	/* Set initial priority mask */
418 	ci->ci_hwpl = IPL_HIGH;
419 	icc_pmr_write(IPL_TO_PMR(sc, IPL_HIGH));
420 
421 	/* Set the binary point field to the minimum value */
422 	icc_bpr1_write(0);
423 
424 	/* Enable group 1 interrupt signaling */
425 	icc_igrpen1_write(ICC_IGRPEN_EL1_Enable);
426 
427 	/* Set EOI mode */
428 	icc_ctlr = icc_ctlr_read();
429 	icc_ctlr &= ~ICC_CTLR_EL1_EOImode;
430 	icc_ctlr_write(icc_ctlr);
431 
432 	/* Enable redistributor */
433 	gicv3_redist_enable(sc, ci);
434 
435 	/* Allow IRQ exceptions */
436 	ENABLE_INTERRUPT();
437 }
438 
439 #ifdef MULTIPROCESSOR
440 static void
gicv3_ipi_send(struct pic_softc * pic,const kcpuset_t * kcp,u_long ipi)441 gicv3_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi)
442 {
443 	struct cpu_info *ci;
444 	uint64_t sgir;
445 
446 	sgir = __SHIFTIN(ipi, ICC_SGIR_EL1_INTID);
447 	if (kcp == NULL) {
448 		/* Interrupts routed to all PEs, excluding "self" */
449 		if (ncpu == 1)
450 			return;
451 		sgir |= ICC_SGIR_EL1_IRM;
452 	} else {
453 		/* Interrupt to exactly one PE */
454 		ci = cpu_lookup(kcpuset_ffs(kcp) - 1);
455 		if (ci == curcpu())
456 			return;
457 		sgir |= ci->ci_gic_sgir;
458 	}
459 	icc_sgi1r_write(sgir);
460 	isb();
461 }
462 
463 static void
gicv3_get_affinity(struct pic_softc * pic,size_t irq,kcpuset_t * affinity)464 gicv3_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity)
465 {
466 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
467 	const size_t group = irq / 32;
468 	int n;
469 
470 	kcpuset_zero(affinity);
471 	if (group == 0) {
472 		/* All CPUs are targets for group 0 (SGI/PPI) */
473 		for (n = 0; n < ncpu; n++) {
474 			kcpuset_set(affinity, n);
475 		}
476 	} else {
477 		/* Find distributor targets (SPI) */
478 		const uint64_t irouter = gicd_read_8(sc, GICD_IROUTER(irq));
479 		for (n = 0; n < ncpu; n++) {
480 			if (irouter == GICD_IROUTER_Interrupt_Routing_mode ||
481 			    irouter == sc->sc_irouter[n])
482 				kcpuset_set(affinity, n);
483 		}
484 	}
485 }
486 
487 static int
gicv3_set_affinity(struct pic_softc * pic,size_t irq,const kcpuset_t * affinity)488 gicv3_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity)
489 {
490 	struct gicv3_softc * const sc = PICTOSOFTC(pic);
491 	const size_t group = irq / 32;
492 	uint64_t irouter;
493 
494 	if (group == 0)
495 		return EINVAL;
496 
497 	const int set = kcpuset_countset(affinity);
498 	if (set == 1) {
499 		irouter = sc->sc_irouter[kcpuset_ffs(affinity) - 1];
500 	} else if (set == ncpu && GIC_SUPPORTS_1OFN(sc) && gicv3_use_1ofn) {
501 		irouter = GICD_IROUTER_Interrupt_Routing_mode;
502 	} else {
503 		return EINVAL;
504 	}
505 
506 	gicd_write_8(sc, GICD_IROUTER(irq), irouter);
507 
508 	return 0;
509 }
510 #endif
511 
512 static const struct pic_ops gicv3_picops = {
513 	.pic_unblock_irqs = gicv3_unblock_irqs,
514 	.pic_block_irqs = gicv3_block_irqs,
515 	.pic_establish_irq = gicv3_establish_irq,
516 	.pic_set_priority = gicv3_set_priority,
517 #ifdef MULTIPROCESSOR
518 	.pic_cpu_init = gicv3_cpu_init,
519 	.pic_ipi_send = gicv3_ipi_send,
520 	.pic_get_affinity = gicv3_get_affinity,
521 	.pic_set_affinity = gicv3_set_affinity,
522 #endif
523 };
524 
525 static void
gicv3_dcache_wb_range(vaddr_t va,vsize_t len)526 gicv3_dcache_wb_range(vaddr_t va, vsize_t len)
527 {
528 	cpu_dcache_wb_range(va, len);
529 	dsb(sy);
530 }
531 
532 static void
gicv3_lpi_unblock_irqs(struct pic_softc * pic,size_t irqbase,uint32_t mask)533 gicv3_lpi_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
534 {
535 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
536 	int bit;
537 
538 	while ((bit = ffs(mask)) != 0) {
539 		sc->sc_lpiconf.base[irqbase + bit - 1] |= GIC_LPICONF_Enable;
540 		if (sc->sc_lpiconf_flush)
541 			gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1);
542 		mask &= ~__BIT(bit - 1);
543 	}
544 
545 	if (!sc->sc_lpiconf_flush)
546 		dsb(ishst);
547 }
548 
549 static void
gicv3_lpi_block_irqs(struct pic_softc * pic,size_t irqbase,uint32_t mask)550 gicv3_lpi_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask)
551 {
552 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
553 	int bit;
554 
555 	while ((bit = ffs(mask)) != 0) {
556 		sc->sc_lpiconf.base[irqbase + bit - 1] &= ~GIC_LPICONF_Enable;
557 		if (sc->sc_lpiconf_flush)
558 			gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1);
559 		mask &= ~__BIT(bit - 1);
560 	}
561 
562 	if (!sc->sc_lpiconf_flush)
563 		dsb(ishst);
564 }
565 
566 static void
gicv3_lpi_establish_irq(struct pic_softc * pic,struct intrsource * is)567 gicv3_lpi_establish_irq(struct pic_softc *pic, struct intrsource *is)
568 {
569 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
570 
571 	sc->sc_lpiconf.base[is->is_irq] = IPL_TO_PRIORITY(sc, is->is_ipl) | GIC_LPICONF_Res1;
572 
573 	if (sc->sc_lpiconf_flush)
574 		gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[is->is_irq], 1);
575 	else
576 		dsb(ishst);
577 }
578 
579 static void
gicv3_lpi_cpu_init(struct pic_softc * pic,struct cpu_info * ci)580 gicv3_lpi_cpu_init(struct pic_softc *pic, struct cpu_info *ci)
581 {
582 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
583 	struct gicv3_lpi_callback *cb;
584 	uint64_t propbase, pendbase;
585 	uint32_t ctlr;
586 
587 	/* If physical LPIs are not supported on this redistributor, just return. */
588 	const uint64_t typer = gicr_read_8(sc, ci->ci_gic_redist, GICR_TYPER);
589 	if ((typer & GICR_TYPER_PLPIS) == 0)
590 		return;
591 
592 	/* Interrupt target address for this CPU, used by ITS when GITS_TYPER.PTA == 0 */
593 	sc->sc_processor_id[cpu_index(ci)] = __SHIFTOUT(typer, GICR_TYPER_Processor_Number);
594 
595 	/* Disable LPIs before making changes */
596 	ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR);
597 	ctlr &= ~GICR_CTLR_Enable_LPIs;
598 	gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr);
599 	dsb(sy);
600 
601 	/* Setup the LPI configuration table */
602 	propbase = sc->sc_lpiconf.segs[0].ds_addr |
603 	    __SHIFTIN(ffs(pic->pic_maxsources) - 1, GICR_PROPBASER_IDbits) |
604 	    __SHIFTIN(GICR_Shareability_IS, GICR_PROPBASER_Shareability) |
605 	    __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PROPBASER_InnerCache);
606 	gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase);
607 	propbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PROPBASER);
608 	if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) != GICR_Shareability_IS) {
609 		if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) == GICR_Shareability_NS) {
610 			propbase &= ~GICR_PROPBASER_Shareability;
611 			propbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PROPBASER_Shareability);
612 			propbase &= ~GICR_PROPBASER_InnerCache;
613 			propbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PROPBASER_InnerCache);
614 			gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase);
615 		}
616 		sc->sc_lpiconf_flush = true;
617 	}
618 
619 	/* Setup the LPI pending table */
620 	pendbase = sc->sc_lpipend[cpu_index(ci)].segs[0].ds_addr |
621 	    __SHIFTIN(GICR_Shareability_IS, GICR_PENDBASER_Shareability) |
622 	    __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PENDBASER_InnerCache);
623 	gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase);
624 	pendbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PENDBASER);
625 	if (__SHIFTOUT(pendbase, GICR_PENDBASER_Shareability) == GICR_Shareability_NS) {
626 		pendbase &= ~GICR_PENDBASER_Shareability;
627 		pendbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PENDBASER_Shareability);
628 		pendbase &= ~GICR_PENDBASER_InnerCache;
629 		pendbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PENDBASER_InnerCache);
630 		gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase);
631 	}
632 
633 	/* Enable LPIs */
634 	ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR);
635 	ctlr |= GICR_CTLR_Enable_LPIs;
636 	gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr);
637 	dsb(sy);
638 
639 	/* Setup ITS if present */
640 	LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list)
641 		cb->cpu_init(cb->priv, ci);
642 }
643 
644 #ifdef MULTIPROCESSOR
645 static void
gicv3_lpi_get_affinity(struct pic_softc * pic,size_t irq,kcpuset_t * affinity)646 gicv3_lpi_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity)
647 {
648 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
649 	struct gicv3_lpi_callback *cb;
650 
651 	kcpuset_zero(affinity);
652 	LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list)
653 		cb->get_affinity(cb->priv, irq, affinity);
654 }
655 
656 static int
gicv3_lpi_set_affinity(struct pic_softc * pic,size_t irq,const kcpuset_t * affinity)657 gicv3_lpi_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity)
658 {
659 	struct gicv3_softc * const sc = LPITOSOFTC(pic);
660 	struct gicv3_lpi_callback *cb;
661 	int error = EINVAL;
662 
663 	LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) {
664 		error = cb->set_affinity(cb->priv, irq, affinity);
665 		if (error != EPASSTHROUGH)
666 			return error;
667 	}
668 
669 	return EINVAL;
670 }
671 #endif
672 
673 static const struct pic_ops gicv3_lpiops = {
674 	.pic_unblock_irqs = gicv3_lpi_unblock_irqs,
675 	.pic_block_irqs = gicv3_lpi_block_irqs,
676 	.pic_establish_irq = gicv3_lpi_establish_irq,
677 #ifdef MULTIPROCESSOR
678 	.pic_cpu_init = gicv3_lpi_cpu_init,
679 	.pic_get_affinity = gicv3_lpi_get_affinity,
680 	.pic_set_affinity = gicv3_lpi_set_affinity,
681 #endif
682 };
683 
684 void
gicv3_dma_alloc(struct gicv3_softc * sc,struct gicv3_dma * dma,bus_size_t len,bus_size_t align)685 gicv3_dma_alloc(struct gicv3_softc *sc, struct gicv3_dma *dma, bus_size_t len, bus_size_t align)
686 {
687 	int nsegs, error;
688 
689 	dma->len = len;
690 	error = bus_dmamem_alloc(sc->sc_dmat, dma->len, align, 0, dma->segs, 1, &nsegs, BUS_DMA_WAITOK);
691 	if (error)
692 		panic("bus_dmamem_alloc failed: %d", error);
693 	error = bus_dmamem_map(sc->sc_dmat, dma->segs, nsegs, len, (void **)&dma->base, BUS_DMA_WAITOK);
694 	if (error)
695 		panic("bus_dmamem_map failed: %d", error);
696 	error = bus_dmamap_create(sc->sc_dmat, len, 1, len, 0, BUS_DMA_WAITOK, &dma->map);
697 	if (error)
698 		panic("bus_dmamap_create failed: %d", error);
699 	error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->base, dma->len, NULL, BUS_DMA_WAITOK);
700 	if (error)
701 		panic("bus_dmamap_load failed: %d", error);
702 
703 	memset(dma->base, 0, dma->len);
704 	bus_dmamap_sync(sc->sc_dmat, dma->map, 0, dma->len, BUS_DMASYNC_PREWRITE);
705 }
706 
707 static void
gicv3_lpi_init(struct gicv3_softc * sc)708 gicv3_lpi_init(struct gicv3_softc *sc)
709 {
710 	/*
711 	 * Allocate LPI configuration table
712 	 */
713 	gicv3_dma_alloc(sc, &sc->sc_lpiconf, sc->sc_lpi.pic_maxsources, 0x1000);
714 	KASSERT((sc->sc_lpiconf.segs[0].ds_addr & ~GICR_PROPBASER_Physical_Address) == 0);
715 
716 	/*
717 	 * Allocate LPI pending tables
718 	 */
719 	const bus_size_t lpipend_sz = (8192 + sc->sc_lpi.pic_maxsources) / NBBY;
720 	for (int cpuindex = 0; cpuindex < ncpu; cpuindex++) {
721 		gicv3_dma_alloc(sc, &sc->sc_lpipend[cpuindex], lpipend_sz, 0x10000);
722 		KASSERT((sc->sc_lpipend[cpuindex].segs[0].ds_addr & ~GICR_PENDBASER_Physical_Address) == 0);
723 	}
724 }
725 
726 void
gicv3_irq_handler(void * frame)727 gicv3_irq_handler(void *frame)
728 {
729 	struct cpu_info * const ci = curcpu();
730 	struct gicv3_softc * const sc = gicv3_softc;
731 	struct pic_softc *pic;
732 	const int oldipl = ci->ci_cpl;
733 	const int oldhwpl = IPL_TO_HWPL(oldipl);
734 
735 	ci->ci_data.cpu_nintr++;
736 
737 	if (ci->ci_hwpl != oldhwpl) {
738 		ci->ci_hwpl = oldhwpl;
739 		icc_pmr_write(IPL_TO_PMR(sc, oldhwpl));
740 		if (oldhwpl == IPL_HIGH) {
741 			return;
742 		}
743 	}
744 
745 	for (;;) {
746 		const uint32_t iar = icc_iar1_read();
747 		dsb(sy);
748 		const uint32_t irq = __SHIFTOUT(iar, ICC_IAR_INTID);
749 		if (irq == ICC_IAR_INTID_SPURIOUS)
750 			break;
751 
752 		pic = irq >= GIC_LPI_BASE ? &sc->sc_lpi : &sc->sc_pic;
753 		if (irq - pic->pic_irqbase >= pic->pic_maxsources)
754 			continue;
755 
756 		struct intrsource * const is = pic->pic_sources[irq - pic->pic_irqbase];
757 		KASSERT(is != NULL);
758 
759 		const bool early_eoi = irq < GIC_LPI_BASE && is->is_type == IST_EDGE;
760 
761 		const int ipl = is->is_ipl;
762 		if (__predict_false(ipl < ci->ci_cpl)) {
763 			pic_do_pending_ints(I32_bit, ipl, frame);
764 		} else if (ci->ci_cpl != ipl) {
765 			KASSERT(ipl >= IPL_VM);
766 			icc_pmr_write(IPL_TO_PMR(sc, ipl));
767 			ci->ci_hwpl = ci->ci_cpl = ipl;
768 		}
769 
770 		if (early_eoi) {
771 			icc_eoi1r_write(iar);
772 			isb();
773 		}
774 
775 		const int64_t nintr = ci->ci_data.cpu_nintr;
776 
777 		ENABLE_INTERRUPT();
778 		pic_dispatch(is, frame);
779 		DISABLE_INTERRUPT();
780 
781 		if (nintr != ci->ci_data.cpu_nintr)
782 			ci->ci_intr_preempt.ev_count++;
783 
784 		if (!early_eoi) {
785 			icc_eoi1r_write(iar);
786 			isb();
787 		}
788 	}
789 
790 	pic_do_pending_ints(I32_bit, oldipl, frame);
791 }
792 
793 static bool
gicv3_cpuif_is_nonsecure(struct gicv3_softc * sc)794 gicv3_cpuif_is_nonsecure(struct gicv3_softc *sc)
795 {
796 	/*
797 	 * Write 0 to bit7 and see if it sticks. This is only possible if
798 	 * we have a non-secure view of the PMR register.
799 	 */
800 	const uint32_t opmr = icc_pmr_read();
801 	icc_pmr_write(0);
802 	const uint32_t npmr = icc_pmr_read();
803 	icc_pmr_write(opmr);
804 
805 	return (npmr & GICC_PMR_NONSECURE) == 0;
806 }
807 
808 static bool
gicv3_dist_is_nonsecure(struct gicv3_softc * sc)809 gicv3_dist_is_nonsecure(struct gicv3_softc *sc)
810 {
811 	const uint32_t gicd_ctrl = gicd_read_4(sc, GICD_CTRL);
812 
813 	/*
814 	 * If security is enabled, we have a non-secure view of the IPRIORITYRn
815 	 * registers and LPI configuration priority fields.
816 	 */
817 	return (gicd_ctrl & GICD_CTRL_DS) == 0;
818 }
819 
820 /*
821  * Rockchip RK3399 provides a different view of int priority registers
822  * depending on which firmware is in use. This is hard to detect in
823  * a way that could possibly break other boards, so only do this
824  * detection if we know we are on a RK3399 SoC.
825  */
826 static void
gicv3_quirk_rockchip_rk3399(struct gicv3_softc * sc)827 gicv3_quirk_rockchip_rk3399(struct gicv3_softc *sc)
828 {
829 	/* Detect the number of supported PMR bits */
830 	icc_pmr_write(0xff);
831 	const uint8_t pmrbits = icc_pmr_read();
832 
833 	/* Detect the number of supported IPRIORITYRn bits */
834 	const uint32_t oiprio = gicd_read_4(sc, GICD_IPRIORITYRn(8));
835 	gicd_write_4(sc, GICD_IPRIORITYRn(8), oiprio | 0xff);
836 	const uint8_t pribits = gicd_read_4(sc, GICD_IPRIORITYRn(8)) & 0xff;
837 	gicd_write_4(sc, GICD_IPRIORITYRn(8), oiprio);
838 
839 	/*
840 	 * If we see fewer PMR bits than IPRIORITYRn bits here, it means
841 	 * we have a secure view of IPRIORITYRn (this is not supposed to
842 	 * happen!).
843 	 */
844 	if (pmrbits < pribits) {
845 		aprint_verbose_dev(sc->sc_dev,
846 		    "buggy RK3399 firmware detected; applying workaround\n");
847 		sc->sc_priority_shift = GIC_PRIO_SHIFT_S;
848 	}
849 }
850 
851 int
gicv3_init(struct gicv3_softc * sc)852 gicv3_init(struct gicv3_softc *sc)
853 {
854 	CPU_INFO_ITERATOR cii;
855 	struct cpu_info *ci;
856 
857 	KASSERT(CPU_IS_PRIMARY(curcpu()));
858 
859 	LIST_INIT(&sc->sc_lpi_callbacks);
860 
861 	/* Store route to CPU for SPIs */
862 	sc->sc_irouter = kmem_zalloc(sizeof(*sc->sc_irouter) * ncpu, KM_SLEEP);
863 	for (CPU_INFO_FOREACH(cii, ci)) {
864 		KASSERT(cpu_index(ci) < ncpu);
865 		sc->sc_irouter[cpu_index(ci)] = ci->ci_cpuid;
866 	}
867 
868 	sc->sc_gicd_typer = gicd_read_4(sc, GICD_TYPER);
869 
870 	/*
871 	 * We don't always have a consistent view of priorities between the
872 	 * CPU interface (ICC_PMR_EL1) and the GICD/GICR registers. Detect
873 	 * if we are making secure or non-secure accesses to each, and adjust
874 	 * the values that we write to each accordingly.
875 	 */
876 	const bool dist_ns = gicv3_dist_is_nonsecure(sc);
877 	sc->sc_priority_shift = dist_ns ? GIC_PRIO_SHIFT_NS : GIC_PRIO_SHIFT_S;
878 	const bool cpuif_ns = gicv3_cpuif_is_nonsecure(sc);
879 	sc->sc_pmr_shift = cpuif_ns ? GIC_PRIO_SHIFT_NS : GIC_PRIO_SHIFT_S;
880 
881 	if ((sc->sc_quirks & GICV3_QUIRK_RK3399) != 0)
882 		gicv3_quirk_rockchip_rk3399(sc);
883 
884 	aprint_verbose_dev(sc->sc_dev,
885 	    "iidr 0x%08x, cpuif %ssecure, dist %ssecure, "
886 	    "priority shift %d, pmr shift %d, quirks %#x\n",
887 	    gicd_read_4(sc, GICD_IIDR),
888 	    cpuif_ns ? "non-" : "",
889 	    dist_ns ? "non-" : "",
890 	    sc->sc_priority_shift,
891 	    sc->sc_pmr_shift,
892 	    sc->sc_quirks);
893 
894 	sc->sc_pic.pic_ops = &gicv3_picops;
895 	sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(sc->sc_gicd_typer);
896 	snprintf(sc->sc_pic.pic_name, sizeof(sc->sc_pic.pic_name), "gicv3");
897 #ifdef MULTIPROCESSOR
898 	sc->sc_pic.pic_cpus = kcpuset_running;
899 #endif
900 	pic_add(&sc->sc_pic, 0);
901 
902 	if ((sc->sc_gicd_typer & GICD_TYPER_LPIS) != 0) {
903 		sc->sc_lpipend = kmem_zalloc(sizeof(*sc->sc_lpipend) * ncpu, KM_SLEEP);
904 		sc->sc_processor_id = kmem_zalloc(sizeof(*sc->sc_processor_id) * ncpu, KM_SLEEP);
905 
906 		sc->sc_lpi.pic_ops = &gicv3_lpiops;
907 		sc->sc_lpi.pic_maxsources = 8192;	/* Min. required by GICv3 spec */
908 		snprintf(sc->sc_lpi.pic_name, sizeof(sc->sc_lpi.pic_name), "gicv3-lpi");
909 		pic_add(&sc->sc_lpi, GIC_LPI_BASE);
910 
911 		sc->sc_lpi_pool = vmem_create("gicv3-lpi", 0, sc->sc_lpi.pic_maxsources,
912 		    1, NULL, NULL, NULL, 0, VM_SLEEP, IPL_HIGH);
913 		if (sc->sc_lpi_pool == NULL)
914 			panic("failed to create gicv3 lpi pool\n");
915 
916 		gicv3_lpi_init(sc);
917 	}
918 
919 	KASSERT(gicv3_softc == NULL);
920 	gicv3_softc = sc;
921 
922 	for (int i = 0; i < sc->sc_bsh_r_count; i++) {
923 		const uint64_t gicr_typer = gicr_read_8(sc, i, GICR_TYPER);
924 		const u_int aff0 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff0);
925 		const u_int aff1 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff1);
926 		const u_int aff2 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff2);
927 		const u_int aff3 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff3);
928 
929 		aprint_debug_dev(sc->sc_dev, "redist %d: cpu %d.%d.%d.%d\n",
930 		    i, aff3, aff2, aff1, aff0);
931 	}
932 
933 	gicv3_dist_enable(sc);
934 
935 	gicv3_cpu_init(&sc->sc_pic, curcpu());
936 	if ((sc->sc_gicd_typer & GICD_TYPER_LPIS) != 0)
937 		gicv3_lpi_cpu_init(&sc->sc_lpi, curcpu());
938 
939 #ifdef MULTIPROCESSOR
940 	intr_establish_xname(IPI_AST, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast");
941 	intr_establish_xname(IPI_XCALL, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall");
942 	intr_establish_xname(IPI_GENERIC, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic");
943 	intr_establish_xname(IPI_NOP, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop");
944 	intr_establish_xname(IPI_SHOOTDOWN, IPL_SCHED, IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown");
945 #ifdef DDB
946 	intr_establish_xname(IPI_DDB, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb");
947 #endif
948 #ifdef __HAVE_PREEMPTION
949 	intr_establish_xname(IPI_KPREEMPT, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt");
950 #endif
951 #endif
952 
953 	return 0;
954 }
955