1 /* $NetBSD: gicv3.c,v 1.44 2021/03/28 11:13:24 jmcneill Exp $ */ 2 3 /*- 4 * Copyright (c) 2018 Jared McNeill <jmcneill@invisible.ca> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include "opt_multiprocessor.h" 30 31 #define _INTR_PRIVATE 32 33 #include <sys/cdefs.h> 34 __KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.44 2021/03/28 11:13:24 jmcneill Exp $"); 35 36 #include <sys/param.h> 37 #include <sys/kernel.h> 38 #include <sys/bus.h> 39 #include <sys/device.h> 40 #include <sys/intr.h> 41 #include <sys/systm.h> 42 #include <sys/cpu.h> 43 #include <sys/vmem.h> 44 #include <sys/kmem.h> 45 #include <sys/atomic.h> 46 47 #include <machine/cpufunc.h> 48 49 #include <arm/locore.h> 50 #include <arm/armreg.h> 51 52 #include <arm/cortex/gicv3.h> 53 #include <arm/cortex/gic_reg.h> 54 55 #define PICTOSOFTC(pic) \ 56 ((void *)((uintptr_t)(pic) - offsetof(struct gicv3_softc, sc_pic))) 57 #define LPITOSOFTC(lpi) \ 58 ((void *)((uintptr_t)(lpi) - offsetof(struct gicv3_softc, sc_lpi))) 59 60 #define IPL_TO_PRIORITY(sc, ipl) (((0xff - (ipl)) << (sc)->sc_priority_shift) & 0xff) 61 #define IPL_TO_PMR(sc, ipl) (((0xff - (ipl)) << (sc)->sc_pmr_shift) & 0xff) 62 63 #define GIC_SUPPORTS_1OFN(sc) (((sc)->sc_gicd_typer & GICD_TYPER_No1N) == 0) 64 65 #define GIC_PRIO_SHIFT_NS 4 66 #define GIC_PRIO_SHIFT_S 3 67 68 /* 69 * Set to true if you want to use 1 of N interrupt distribution for SPIs 70 * when available. Disabled by default because it causes issues with the 71 * USB stack. 72 */ 73 bool gicv3_use_1ofn = false; 74 75 static struct gicv3_softc *gicv3_softc; 76 77 static inline uint32_t 78 gicd_read_4(struct gicv3_softc *sc, bus_size_t reg) 79 { 80 return bus_space_read_4(sc->sc_bst, sc->sc_bsh_d, reg); 81 } 82 83 static inline void 84 gicd_write_4(struct gicv3_softc *sc, bus_size_t reg, uint32_t val) 85 { 86 bus_space_write_4(sc->sc_bst, sc->sc_bsh_d, reg, val); 87 } 88 89 #ifdef MULTIPROCESSOR 90 static inline uint64_t 91 gicd_read_8(struct gicv3_softc *sc, bus_size_t reg) 92 { 93 return bus_space_read_8(sc->sc_bst, sc->sc_bsh_d, reg); 94 } 95 #endif 96 97 static inline void 98 gicd_write_8(struct gicv3_softc *sc, bus_size_t reg, uint64_t val) 99 { 100 bus_space_write_8(sc->sc_bst, sc->sc_bsh_d, reg, val); 101 } 102 103 static inline uint32_t 104 gicr_read_4(struct gicv3_softc *sc, u_int index, bus_size_t reg) 105 { 106 KASSERT(index < sc->sc_bsh_r_count); 107 return bus_space_read_4(sc->sc_bst, sc->sc_bsh_r[index], reg); 108 } 109 110 static inline void 111 gicr_write_4(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint32_t val) 112 { 113 KASSERT(index < sc->sc_bsh_r_count); 114 bus_space_write_4(sc->sc_bst, sc->sc_bsh_r[index], reg, val); 115 } 116 117 static inline uint64_t 118 gicr_read_8(struct gicv3_softc *sc, u_int index, bus_size_t reg) 119 { 120 KASSERT(index < sc->sc_bsh_r_count); 121 return bus_space_read_8(sc->sc_bst, sc->sc_bsh_r[index], reg); 122 } 123 124 static inline void 125 gicr_write_8(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint64_t val) 126 { 127 KASSERT(index < sc->sc_bsh_r_count); 128 bus_space_write_8(sc->sc_bst, sc->sc_bsh_r[index], reg, val); 129 } 130 131 static void 132 gicv3_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 133 { 134 struct gicv3_softc * const sc = PICTOSOFTC(pic); 135 struct cpu_info * const ci = curcpu(); 136 const u_int group = irqbase / 32; 137 138 if (group == 0) { 139 atomic_or_32(&sc->sc_enabled_sgippi, mask); 140 gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, mask); 141 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 142 ; 143 } else { 144 gicd_write_4(sc, GICD_ISENABLERn(group), mask); 145 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 146 ; 147 } 148 } 149 150 static void 151 gicv3_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 152 { 153 struct gicv3_softc * const sc = PICTOSOFTC(pic); 154 struct cpu_info * const ci = curcpu(); 155 const u_int group = irqbase / 32; 156 157 if (group == 0) { 158 atomic_and_32(&sc->sc_enabled_sgippi, ~mask); 159 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, mask); 160 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 161 ; 162 } else { 163 gicd_write_4(sc, GICD_ICENABLERn(group), mask); 164 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 165 ; 166 } 167 } 168 169 static void 170 gicv3_establish_irq(struct pic_softc *pic, struct intrsource *is) 171 { 172 struct gicv3_softc * const sc = PICTOSOFTC(pic); 173 const u_int group = is->is_irq / 32; 174 uint32_t ipriority, icfg; 175 uint64_t irouter; 176 u_int n; 177 178 const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl); 179 const u_int ipriority_shift = (is->is_irq & 0x3) * 8; 180 const u_int icfg_shift = (is->is_irq & 0xf) * 2; 181 182 if (group == 0) { 183 /* SGIs and PPIs are always MP-safe */ 184 is->is_mpsafe = true; 185 186 /* Update interrupt configuration and priority on all redistributors */ 187 for (n = 0; n < sc->sc_bsh_r_count; n++) { 188 icfg = gicr_read_4(sc, n, GICR_ICFGRn(is->is_irq / 16)); 189 if (is->is_type == IST_LEVEL) 190 icfg &= ~(0x2 << icfg_shift); 191 if (is->is_type == IST_EDGE) 192 icfg |= (0x2 << icfg_shift); 193 gicr_write_4(sc, n, GICR_ICFGRn(is->is_irq / 16), icfg); 194 195 ipriority = gicr_read_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4)); 196 ipriority &= ~(0xffU << ipriority_shift); 197 ipriority |= (ipriority_val << ipriority_shift); 198 gicr_write_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4), ipriority); 199 } 200 } else { 201 /* 202 * If 1 of N SPI routing is supported, route MP-safe interrupts to all 203 * participating PEs. Otherwise, just route to the primary PE. 204 */ 205 if (is->is_mpsafe && GIC_SUPPORTS_1OFN(sc) && gicv3_use_1ofn) { 206 irouter = GICD_IROUTER_Interrupt_Routing_mode; 207 } else { 208 irouter = sc->sc_irouter[0]; 209 } 210 gicd_write_8(sc, GICD_IROUTER(is->is_irq), irouter); 211 212 /* Update interrupt configuration */ 213 icfg = gicd_read_4(sc, GICD_ICFGRn(is->is_irq / 16)); 214 if (is->is_type == IST_LEVEL) 215 icfg &= ~(0x2 << icfg_shift); 216 if (is->is_type == IST_EDGE) 217 icfg |= (0x2 << icfg_shift); 218 gicd_write_4(sc, GICD_ICFGRn(is->is_irq / 16), icfg); 219 220 /* Update interrupt priority */ 221 ipriority = gicd_read_4(sc, GICD_IPRIORITYRn(is->is_irq / 4)); 222 ipriority &= ~(0xffU << ipriority_shift); 223 ipriority |= (ipriority_val << ipriority_shift); 224 gicd_write_4(sc, GICD_IPRIORITYRn(is->is_irq / 4), ipriority); 225 } 226 } 227 228 static void 229 gicv3_set_priority(struct pic_softc *pic, int ipl) 230 { 231 struct gicv3_softc * const sc = PICTOSOFTC(pic); 232 struct cpu_info * const ci = curcpu(); 233 const uint8_t newpmr = IPL_TO_PMR(sc, ipl); 234 235 if (newpmr > ci->ci_hwpl) { 236 /* Lowering priority mask */ 237 ci->ci_hwpl = newpmr; 238 icc_pmr_write(newpmr); 239 } 240 } 241 242 static void 243 gicv3_dist_enable(struct gicv3_softc *sc) 244 { 245 uint32_t gicd_ctrl; 246 u_int n; 247 248 /* Disable the distributor */ 249 gicd_ctrl = gicd_read_4(sc, GICD_CTRL); 250 gicd_ctrl &= ~(GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS); 251 gicd_write_4(sc, GICD_CTRL, gicd_ctrl); 252 253 /* Wait for register write to complete */ 254 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 255 ; 256 257 /* Clear all INTID enable bits */ 258 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) 259 gicd_write_4(sc, GICD_ICENABLERn(n / 32), ~0); 260 261 /* Set default priorities to lowest */ 262 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 4) 263 gicd_write_4(sc, GICD_IPRIORITYRn(n / 4), ~0); 264 265 /* Set all interrupts to G1NS */ 266 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) { 267 gicd_write_4(sc, GICD_IGROUPRn(n / 32), ~0); 268 gicd_write_4(sc, GICD_IGRPMODRn(n / 32), 0); 269 } 270 271 /* Set all interrupts level-sensitive by default */ 272 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 16) 273 gicd_write_4(sc, GICD_ICFGRn(n / 16), 0); 274 275 /* Wait for register writes to complete */ 276 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 277 ; 278 279 /* Enable Affinity routing and G1NS interrupts */ 280 gicd_ctrl = GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS; 281 gicd_write_4(sc, GICD_CTRL, gicd_ctrl); 282 } 283 284 static void 285 gicv3_redist_enable(struct gicv3_softc *sc, struct cpu_info *ci) 286 { 287 uint32_t icfg; 288 u_int n, o; 289 290 /* Clear INTID enable bits */ 291 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, ~0); 292 293 /* Wait for register write to complete */ 294 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 295 ; 296 297 /* Set default priorities */ 298 for (n = 0; n < 32; n += 4) { 299 uint32_t priority = 0; 300 size_t byte_shift = 0; 301 for (o = 0; o < 4; o++, byte_shift += 8) { 302 struct intrsource * const is = sc->sc_pic.pic_sources[n + o]; 303 if (is == NULL) 304 priority |= (0xffU << byte_shift); 305 else { 306 const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl); 307 priority |= ipriority_val << byte_shift; 308 } 309 } 310 gicr_write_4(sc, ci->ci_gic_redist, GICR_IPRIORITYRn(n / 4), priority); 311 } 312 313 /* Set all interrupts to G1NS */ 314 gicr_write_4(sc, ci->ci_gic_redist, GICR_IGROUPR0, ~0); 315 gicr_write_4(sc, ci->ci_gic_redist, GICR_IGRPMODR0, 0); 316 317 /* Restore PPI configs */ 318 for (n = 0, icfg = 0; n < 16; n++) { 319 struct intrsource * const is = sc->sc_pic.pic_sources[16 + n]; 320 if (is != NULL && is->is_type == IST_EDGE) 321 icfg |= (0x2 << (n * 2)); 322 } 323 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICFGRn(1), icfg); 324 325 /* Restore current enable bits */ 326 gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, sc->sc_enabled_sgippi); 327 328 /* Wait for register write to complete */ 329 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 330 ; 331 } 332 333 static uint64_t 334 gicv3_cpu_identity(void) 335 { 336 u_int aff3, aff2, aff1, aff0; 337 338 const register_t mpidr = cpu_mpidr_aff_read(); 339 aff0 = __SHIFTOUT(mpidr, MPIDR_AFF0); 340 aff1 = __SHIFTOUT(mpidr, MPIDR_AFF1); 341 aff2 = __SHIFTOUT(mpidr, MPIDR_AFF2); 342 aff3 = __SHIFTOUT(mpidr, MPIDR_AFF3); 343 344 return __SHIFTIN(aff0, GICR_TYPER_Affinity_Value_Aff0) | 345 __SHIFTIN(aff1, GICR_TYPER_Affinity_Value_Aff1) | 346 __SHIFTIN(aff2, GICR_TYPER_Affinity_Value_Aff2) | 347 __SHIFTIN(aff3, GICR_TYPER_Affinity_Value_Aff3); 348 } 349 350 static u_int 351 gicv3_find_redist(struct gicv3_softc *sc) 352 { 353 uint64_t gicr_typer; 354 u_int n; 355 356 const uint64_t cpu_identity = gicv3_cpu_identity(); 357 358 for (n = 0; n < sc->sc_bsh_r_count; n++) { 359 gicr_typer = gicr_read_8(sc, n, GICR_TYPER); 360 if ((gicr_typer & GICR_TYPER_Affinity_Value) == cpu_identity) 361 return n; 362 } 363 364 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 365 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 366 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 367 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 368 369 panic("%s: could not find GICv3 redistributor for cpu %d.%d.%d.%d", 370 cpu_name(curcpu()), aff3, aff2, aff1, aff0); 371 } 372 373 static uint64_t 374 gicv3_sgir(struct gicv3_softc *sc) 375 { 376 const uint64_t cpu_identity = gicv3_cpu_identity(); 377 378 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 379 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 380 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 381 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 382 383 return __SHIFTIN(__BIT(aff0), ICC_SGIR_EL1_TargetList) | 384 __SHIFTIN(aff1, ICC_SGIR_EL1_Aff1) | 385 __SHIFTIN(aff2, ICC_SGIR_EL1_Aff2) | 386 __SHIFTIN(aff3, ICC_SGIR_EL1_Aff3); 387 } 388 389 static void 390 gicv3_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 391 { 392 struct gicv3_softc * const sc = PICTOSOFTC(pic); 393 uint32_t icc_sre, icc_ctlr, gicr_waker; 394 395 evcnt_attach_dynamic(&ci->ci_intr_preempt, EVCNT_TYPE_MISC, NULL, 396 ci->ci_cpuname, "intr preempt"); 397 398 ci->ci_gic_redist = gicv3_find_redist(sc); 399 ci->ci_gic_sgir = gicv3_sgir(sc); 400 401 /* Store route to CPU for SPIs */ 402 const uint64_t cpu_identity = gicv3_cpu_identity(); 403 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 404 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 405 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 406 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 407 sc->sc_irouter[cpu_index(ci)] = 408 __SHIFTIN(aff0, GICD_IROUTER_Aff0) | 409 __SHIFTIN(aff1, GICD_IROUTER_Aff1) | 410 __SHIFTIN(aff2, GICD_IROUTER_Aff2) | 411 __SHIFTIN(aff3, GICD_IROUTER_Aff3); 412 413 /* Enable System register access and disable IRQ/FIQ bypass */ 414 icc_sre = ICC_SRE_EL1_SRE | ICC_SRE_EL1_DFB | ICC_SRE_EL1_DIB; 415 icc_sre_write(icc_sre); 416 417 /* Mark the connected PE as being awake */ 418 gicr_waker = gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER); 419 gicr_waker &= ~GICR_WAKER_ProcessorSleep; 420 gicr_write_4(sc, ci->ci_gic_redist, GICR_WAKER, gicr_waker); 421 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER) & GICR_WAKER_ChildrenAsleep) 422 ; 423 424 /* Set initial priority mask */ 425 ci->ci_hwpl = IPL_TO_PMR(sc, IPL_HIGH); 426 icc_pmr_write(ci->ci_hwpl); 427 428 /* Set the binary point field to the minimum value */ 429 icc_bpr1_write(0); 430 431 /* Enable group 1 interrupt signaling */ 432 icc_igrpen1_write(ICC_IGRPEN_EL1_Enable); 433 434 /* Set EOI mode */ 435 icc_ctlr = icc_ctlr_read(); 436 icc_ctlr &= ~ICC_CTLR_EL1_EOImode; 437 icc_ctlr_write(icc_ctlr); 438 439 /* Enable redistributor */ 440 gicv3_redist_enable(sc, ci); 441 442 /* Allow IRQ exceptions */ 443 ENABLE_INTERRUPT(); 444 } 445 446 #ifdef MULTIPROCESSOR 447 static void 448 gicv3_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi) 449 { 450 struct cpu_info *ci; 451 uint64_t sgir; 452 453 sgir = __SHIFTIN(ipi, ICC_SGIR_EL1_INTID); 454 if (kcp == NULL) { 455 /* Interrupts routed to all PEs, excluding "self" */ 456 if (ncpu == 1) 457 return; 458 sgir |= ICC_SGIR_EL1_IRM; 459 } else { 460 /* Interrupt to exactly one PE */ 461 ci = cpu_lookup(kcpuset_ffs(kcp) - 1); 462 if (ci == curcpu()) 463 return; 464 sgir |= ci->ci_gic_sgir; 465 } 466 icc_sgi1r_write(sgir); 467 isb(); 468 } 469 470 static void 471 gicv3_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 472 { 473 struct gicv3_softc * const sc = PICTOSOFTC(pic); 474 const size_t group = irq / 32; 475 int n; 476 477 kcpuset_zero(affinity); 478 if (group == 0) { 479 /* All CPUs are targets for group 0 (SGI/PPI) */ 480 for (n = 0; n < ncpu; n++) { 481 if (sc->sc_irouter[n] != UINT64_MAX) 482 kcpuset_set(affinity, n); 483 } 484 } else { 485 /* Find distributor targets (SPI) */ 486 const uint64_t irouter = gicd_read_8(sc, GICD_IROUTER(irq)); 487 for (n = 0; n < ncpu; n++) { 488 if (irouter == GICD_IROUTER_Interrupt_Routing_mode || 489 irouter == sc->sc_irouter[n]) 490 kcpuset_set(affinity, n); 491 } 492 } 493 } 494 495 static int 496 gicv3_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity) 497 { 498 struct gicv3_softc * const sc = PICTOSOFTC(pic); 499 const size_t group = irq / 32; 500 uint64_t irouter; 501 502 if (group == 0) 503 return EINVAL; 504 505 const int set = kcpuset_countset(affinity); 506 if (set == 1) { 507 irouter = sc->sc_irouter[kcpuset_ffs(affinity) - 1]; 508 } else if (set == ncpu && GIC_SUPPORTS_1OFN(sc) && gicv3_use_1ofn) { 509 irouter = GICD_IROUTER_Interrupt_Routing_mode; 510 } else { 511 return EINVAL; 512 } 513 514 gicd_write_8(sc, GICD_IROUTER(irq), irouter); 515 516 return 0; 517 } 518 #endif 519 520 static const struct pic_ops gicv3_picops = { 521 .pic_unblock_irqs = gicv3_unblock_irqs, 522 .pic_block_irqs = gicv3_block_irqs, 523 .pic_establish_irq = gicv3_establish_irq, 524 .pic_set_priority = gicv3_set_priority, 525 #ifdef MULTIPROCESSOR 526 .pic_cpu_init = gicv3_cpu_init, 527 .pic_ipi_send = gicv3_ipi_send, 528 .pic_get_affinity = gicv3_get_affinity, 529 .pic_set_affinity = gicv3_set_affinity, 530 #endif 531 }; 532 533 static void 534 gicv3_dcache_wb_range(vaddr_t va, vsize_t len) 535 { 536 cpu_dcache_wb_range(va, len); 537 dsb(sy); 538 } 539 540 static void 541 gicv3_lpi_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 542 { 543 struct gicv3_softc * const sc = LPITOSOFTC(pic); 544 int bit; 545 546 while ((bit = ffs(mask)) != 0) { 547 sc->sc_lpiconf.base[irqbase + bit - 1] |= GIC_LPICONF_Enable; 548 if (sc->sc_lpiconf_flush) 549 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1); 550 mask &= ~__BIT(bit - 1); 551 } 552 553 if (!sc->sc_lpiconf_flush) 554 dsb(ishst); 555 } 556 557 static void 558 gicv3_lpi_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 559 { 560 struct gicv3_softc * const sc = LPITOSOFTC(pic); 561 int bit; 562 563 while ((bit = ffs(mask)) != 0) { 564 sc->sc_lpiconf.base[irqbase + bit - 1] &= ~GIC_LPICONF_Enable; 565 if (sc->sc_lpiconf_flush) 566 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1); 567 mask &= ~__BIT(bit - 1); 568 } 569 570 if (!sc->sc_lpiconf_flush) 571 dsb(ishst); 572 } 573 574 static void 575 gicv3_lpi_establish_irq(struct pic_softc *pic, struct intrsource *is) 576 { 577 struct gicv3_softc * const sc = LPITOSOFTC(pic); 578 579 sc->sc_lpiconf.base[is->is_irq] = IPL_TO_PRIORITY(sc, is->is_ipl) | GIC_LPICONF_Res1; 580 581 if (sc->sc_lpiconf_flush) 582 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[is->is_irq], 1); 583 else 584 dsb(ishst); 585 } 586 587 static void 588 gicv3_lpi_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 589 { 590 struct gicv3_softc * const sc = LPITOSOFTC(pic); 591 struct gicv3_lpi_callback *cb; 592 uint64_t propbase, pendbase; 593 uint32_t ctlr; 594 595 /* If physical LPIs are not supported on this redistributor, just return. */ 596 const uint64_t typer = gicr_read_8(sc, ci->ci_gic_redist, GICR_TYPER); 597 if ((typer & GICR_TYPER_PLPIS) == 0) 598 return; 599 600 /* Interrupt target address for this CPU, used by ITS when GITS_TYPER.PTA == 0 */ 601 sc->sc_processor_id[cpu_index(ci)] = __SHIFTOUT(typer, GICR_TYPER_Processor_Number); 602 603 /* Disable LPIs before making changes */ 604 ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); 605 ctlr &= ~GICR_CTLR_Enable_LPIs; 606 gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); 607 dsb(sy); 608 609 /* Setup the LPI configuration table */ 610 propbase = sc->sc_lpiconf.segs[0].ds_addr | 611 __SHIFTIN(ffs(pic->pic_maxsources) - 1, GICR_PROPBASER_IDbits) | 612 __SHIFTIN(GICR_Shareability_IS, GICR_PROPBASER_Shareability) | 613 __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PROPBASER_InnerCache); 614 gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase); 615 propbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PROPBASER); 616 if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) != GICR_Shareability_IS) { 617 if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) == GICR_Shareability_NS) { 618 propbase &= ~GICR_PROPBASER_Shareability; 619 propbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PROPBASER_Shareability); 620 propbase &= ~GICR_PROPBASER_InnerCache; 621 propbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PROPBASER_InnerCache); 622 gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase); 623 } 624 sc->sc_lpiconf_flush = true; 625 } 626 627 /* Setup the LPI pending table */ 628 pendbase = sc->sc_lpipend[cpu_index(ci)].segs[0].ds_addr | 629 __SHIFTIN(GICR_Shareability_IS, GICR_PENDBASER_Shareability) | 630 __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PENDBASER_InnerCache); 631 gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase); 632 pendbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PENDBASER); 633 if (__SHIFTOUT(pendbase, GICR_PENDBASER_Shareability) == GICR_Shareability_NS) { 634 pendbase &= ~GICR_PENDBASER_Shareability; 635 pendbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PENDBASER_Shareability); 636 pendbase &= ~GICR_PENDBASER_InnerCache; 637 pendbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PENDBASER_InnerCache); 638 gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase); 639 } 640 641 /* Enable LPIs */ 642 ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); 643 ctlr |= GICR_CTLR_Enable_LPIs; 644 gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); 645 dsb(sy); 646 647 /* Setup ITS if present */ 648 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) 649 cb->cpu_init(cb->priv, ci); 650 } 651 652 #ifdef MULTIPROCESSOR 653 static void 654 gicv3_lpi_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 655 { 656 struct gicv3_softc * const sc = LPITOSOFTC(pic); 657 struct gicv3_lpi_callback *cb; 658 659 kcpuset_zero(affinity); 660 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) 661 cb->get_affinity(cb->priv, irq, affinity); 662 } 663 664 static int 665 gicv3_lpi_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity) 666 { 667 struct gicv3_softc * const sc = LPITOSOFTC(pic); 668 struct gicv3_lpi_callback *cb; 669 int error = EINVAL; 670 671 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) { 672 error = cb->set_affinity(cb->priv, irq, affinity); 673 if (error != EPASSTHROUGH) 674 return error; 675 } 676 677 return EINVAL; 678 } 679 #endif 680 681 static const struct pic_ops gicv3_lpiops = { 682 .pic_unblock_irqs = gicv3_lpi_unblock_irqs, 683 .pic_block_irqs = gicv3_lpi_block_irqs, 684 .pic_establish_irq = gicv3_lpi_establish_irq, 685 #ifdef MULTIPROCESSOR 686 .pic_cpu_init = gicv3_lpi_cpu_init, 687 .pic_get_affinity = gicv3_lpi_get_affinity, 688 .pic_set_affinity = gicv3_lpi_set_affinity, 689 #endif 690 }; 691 692 void 693 gicv3_dma_alloc(struct gicv3_softc *sc, struct gicv3_dma *dma, bus_size_t len, bus_size_t align) 694 { 695 int nsegs, error; 696 697 dma->len = len; 698 error = bus_dmamem_alloc(sc->sc_dmat, dma->len, align, 0, dma->segs, 1, &nsegs, BUS_DMA_WAITOK); 699 if (error) 700 panic("bus_dmamem_alloc failed: %d", error); 701 error = bus_dmamem_map(sc->sc_dmat, dma->segs, nsegs, len, (void **)&dma->base, BUS_DMA_WAITOK); 702 if (error) 703 panic("bus_dmamem_map failed: %d", error); 704 error = bus_dmamap_create(sc->sc_dmat, len, 1, len, 0, BUS_DMA_WAITOK, &dma->map); 705 if (error) 706 panic("bus_dmamap_create failed: %d", error); 707 error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->base, dma->len, NULL, BUS_DMA_WAITOK); 708 if (error) 709 panic("bus_dmamap_load failed: %d", error); 710 711 memset(dma->base, 0, dma->len); 712 bus_dmamap_sync(sc->sc_dmat, dma->map, 0, dma->len, BUS_DMASYNC_PREWRITE); 713 } 714 715 static void 716 gicv3_lpi_init(struct gicv3_softc *sc) 717 { 718 /* 719 * Allocate LPI configuration table 720 */ 721 gicv3_dma_alloc(sc, &sc->sc_lpiconf, sc->sc_lpi.pic_maxsources, 0x1000); 722 KASSERT((sc->sc_lpiconf.segs[0].ds_addr & ~GICR_PROPBASER_Physical_Address) == 0); 723 724 /* 725 * Allocate LPI pending tables 726 */ 727 const bus_size_t lpipend_sz = (8192 + sc->sc_lpi.pic_maxsources) / NBBY; 728 for (int cpuindex = 0; cpuindex < ncpu; cpuindex++) { 729 gicv3_dma_alloc(sc, &sc->sc_lpipend[cpuindex], lpipend_sz, 0x10000); 730 KASSERT((sc->sc_lpipend[cpuindex].segs[0].ds_addr & ~GICR_PENDBASER_Physical_Address) == 0); 731 } 732 } 733 734 void 735 gicv3_irq_handler(void *frame) 736 { 737 struct cpu_info * const ci = curcpu(); 738 struct gicv3_softc * const sc = gicv3_softc; 739 struct pic_softc *pic; 740 const int oldipl = ci->ci_cpl; 741 const uint8_t pmr = IPL_TO_PMR(sc, oldipl); 742 743 ci->ci_data.cpu_nintr++; 744 745 if (ci->ci_hwpl != pmr) { 746 ci->ci_hwpl = pmr; 747 icc_pmr_write(pmr); 748 if (oldipl == IPL_HIGH) { 749 return; 750 } 751 } 752 753 for (;;) { 754 const uint32_t iar = icc_iar1_read(); 755 dsb(sy); 756 const uint32_t irq = __SHIFTOUT(iar, ICC_IAR_INTID); 757 if (irq == ICC_IAR_INTID_SPURIOUS) 758 break; 759 760 pic = irq >= GIC_LPI_BASE ? &sc->sc_lpi : &sc->sc_pic; 761 if (irq - pic->pic_irqbase >= pic->pic_maxsources) 762 continue; 763 764 struct intrsource * const is = pic->pic_sources[irq - pic->pic_irqbase]; 765 KASSERT(is != NULL); 766 767 const bool early_eoi = irq < GIC_LPI_BASE && is->is_type == IST_EDGE; 768 769 const int ipl = is->is_ipl; 770 if (__predict_false(ipl < ci->ci_cpl)) { 771 pic_do_pending_ints(I32_bit, ipl, frame); 772 } else if (ci->ci_cpl != ipl) { 773 icc_pmr_write(IPL_TO_PMR(sc, ipl)); 774 ci->ci_hwpl = ci->ci_cpl = ipl; 775 } 776 777 if (early_eoi) { 778 icc_eoi1r_write(iar); 779 isb(); 780 } 781 782 const int64_t nintr = ci->ci_data.cpu_nintr; 783 784 ENABLE_INTERRUPT(); 785 pic_dispatch(is, frame); 786 DISABLE_INTERRUPT(); 787 788 if (nintr != ci->ci_data.cpu_nintr) 789 ci->ci_intr_preempt.ev_count++; 790 791 if (!early_eoi) { 792 icc_eoi1r_write(iar); 793 isb(); 794 } 795 } 796 797 pic_do_pending_ints(I32_bit, oldipl, frame); 798 } 799 800 static bool 801 gicv3_cpuif_is_nonsecure(struct gicv3_softc *sc) 802 { 803 /* 804 * Write 0 to bit7 and see if it sticks. This is only possible if 805 * we have a non-secure view of the PMR register. 806 */ 807 const uint32_t opmr = icc_pmr_read(); 808 icc_pmr_write(0); 809 const uint32_t npmr = icc_pmr_read(); 810 icc_pmr_write(opmr); 811 812 return (npmr & GICC_PMR_NONSECURE) == 0; 813 } 814 815 static bool 816 gicv3_dist_is_nonsecure(struct gicv3_softc *sc) 817 { 818 const uint32_t gicd_ctrl = gicd_read_4(sc, GICD_CTRL); 819 820 /* 821 * If security is enabled, we have a non-secure view of the IPRIORITYRn 822 * registers and LPI configuration priority fields. 823 */ 824 return (gicd_ctrl & GICD_CTRL_DS) == 0; 825 } 826 827 /* 828 * Rockchip RK3399 provides a different view of int priority registers 829 * depending on which firmware is in use. This is hard to detect in 830 * a way that could possibly break other boards, so only do this 831 * detection if we know we are on a RK3399 SoC. 832 */ 833 static void 834 gicv3_quirk_rockchip_rk3399(struct gicv3_softc *sc) 835 { 836 /* Detect the number of supported PMR bits */ 837 icc_pmr_write(0xff); 838 const uint8_t pmrbits = icc_pmr_read(); 839 840 /* Detect the number of supported IPRIORITYRn bits */ 841 const uint32_t oiprio = gicd_read_4(sc, GICD_IPRIORITYRn(8)); 842 gicd_write_4(sc, GICD_IPRIORITYRn(8), oiprio | 0xff); 843 const uint8_t pribits = gicd_read_4(sc, GICD_IPRIORITYRn(8)) & 0xff; 844 gicd_write_4(sc, GICD_IPRIORITYRn(8), oiprio); 845 846 /* 847 * If we see fewer PMR bits than IPRIORITYRn bits here, it means 848 * we have a secure view of IPRIORITYRn (this is not supposed to 849 * happen!). 850 */ 851 if (pmrbits < pribits) { 852 aprint_verbose_dev(sc->sc_dev, 853 "buggy RK3399 firmware detected; applying workaround\n"); 854 sc->sc_priority_shift = GIC_PRIO_SHIFT_S; 855 } 856 } 857 858 int 859 gicv3_init(struct gicv3_softc *sc) 860 { 861 int n; 862 863 KASSERT(CPU_IS_PRIMARY(curcpu())); 864 865 LIST_INIT(&sc->sc_lpi_callbacks); 866 867 sc->sc_irouter = kmem_zalloc(sizeof(*sc->sc_irouter) * ncpu, KM_SLEEP); 868 for (n = 0; n < ncpu; n++) 869 sc->sc_irouter[n] = UINT64_MAX; 870 871 sc->sc_gicd_typer = gicd_read_4(sc, GICD_TYPER); 872 873 /* 874 * We don't always have a consistent view of priorities between the 875 * CPU interface (ICC_PMR_EL1) and the GICD/GICR registers. Detect 876 * if we are making secure or non-secure accesses to each, and adjust 877 * the values that we write to each accordingly. 878 */ 879 const bool dist_ns = gicv3_dist_is_nonsecure(sc); 880 sc->sc_priority_shift = dist_ns ? GIC_PRIO_SHIFT_NS : GIC_PRIO_SHIFT_S; 881 const bool cpuif_ns = gicv3_cpuif_is_nonsecure(sc); 882 sc->sc_pmr_shift = cpuif_ns ? GIC_PRIO_SHIFT_NS : GIC_PRIO_SHIFT_S; 883 884 if ((sc->sc_quirks & GICV3_QUIRK_RK3399) != 0) 885 gicv3_quirk_rockchip_rk3399(sc); 886 887 aprint_verbose_dev(sc->sc_dev, 888 "iidr 0x%08x, cpuif %ssecure, dist %ssecure, " 889 "priority shift %d, pmr shift %d, quirks %#x\n", 890 gicd_read_4(sc, GICD_IIDR), 891 cpuif_ns ? "non-" : "", 892 dist_ns ? "non-" : "", 893 sc->sc_priority_shift, 894 sc->sc_pmr_shift, 895 sc->sc_quirks); 896 897 sc->sc_pic.pic_ops = &gicv3_picops; 898 sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(sc->sc_gicd_typer); 899 snprintf(sc->sc_pic.pic_name, sizeof(sc->sc_pic.pic_name), "gicv3"); 900 #ifdef MULTIPROCESSOR 901 sc->sc_pic.pic_cpus = kcpuset_running; 902 #endif 903 pic_add(&sc->sc_pic, 0); 904 905 if ((sc->sc_gicd_typer & GICD_TYPER_LPIS) != 0) { 906 sc->sc_lpipend = kmem_zalloc(sizeof(*sc->sc_lpipend) * ncpu, KM_SLEEP); 907 sc->sc_processor_id = kmem_zalloc(sizeof(*sc->sc_processor_id) * ncpu, KM_SLEEP); 908 909 sc->sc_lpi.pic_ops = &gicv3_lpiops; 910 sc->sc_lpi.pic_maxsources = 8192; /* Min. required by GICv3 spec */ 911 snprintf(sc->sc_lpi.pic_name, sizeof(sc->sc_lpi.pic_name), "gicv3-lpi"); 912 pic_add(&sc->sc_lpi, GIC_LPI_BASE); 913 914 sc->sc_lpi_pool = vmem_create("gicv3-lpi", 0, sc->sc_lpi.pic_maxsources, 915 1, NULL, NULL, NULL, 0, VM_SLEEP, IPL_HIGH); 916 if (sc->sc_lpi_pool == NULL) 917 panic("failed to create gicv3 lpi pool\n"); 918 919 gicv3_lpi_init(sc); 920 } 921 922 KASSERT(gicv3_softc == NULL); 923 gicv3_softc = sc; 924 925 for (int i = 0; i < sc->sc_bsh_r_count; i++) { 926 const uint64_t gicr_typer = gicr_read_8(sc, i, GICR_TYPER); 927 const u_int aff0 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff0); 928 const u_int aff1 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff1); 929 const u_int aff2 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff2); 930 const u_int aff3 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff3); 931 932 aprint_debug_dev(sc->sc_dev, "redist %d: cpu %d.%d.%d.%d\n", 933 i, aff3, aff2, aff1, aff0); 934 } 935 936 gicv3_dist_enable(sc); 937 938 gicv3_cpu_init(&sc->sc_pic, curcpu()); 939 if ((sc->sc_gicd_typer & GICD_TYPER_LPIS) != 0) 940 gicv3_lpi_cpu_init(&sc->sc_lpi, curcpu()); 941 942 #ifdef MULTIPROCESSOR 943 intr_establish_xname(IPI_AST, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast"); 944 intr_establish_xname(IPI_XCALL, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall"); 945 intr_establish_xname(IPI_GENERIC, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic"); 946 intr_establish_xname(IPI_NOP, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop"); 947 intr_establish_xname(IPI_SHOOTDOWN, IPL_SCHED, IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown"); 948 #ifdef DDB 949 intr_establish_xname(IPI_DDB, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb"); 950 #endif 951 #ifdef __HAVE_PREEMPTION 952 intr_establish_xname(IPI_KPREEMPT, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt"); 953 #endif 954 #endif 955 956 return 0; 957 } 958