1 /* $NetBSD: gicv3.c,v 1.49 2021/10/02 20:52:09 skrll Exp $ */ 2 3 /*- 4 * Copyright (c) 2018 Jared McNeill <jmcneill@invisible.ca> 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 #include "opt_multiprocessor.h" 30 #include "opt_gic.h" 31 32 #define _INTR_PRIVATE 33 34 #include <sys/cdefs.h> 35 __KERNEL_RCSID(0, "$NetBSD: gicv3.c,v 1.49 2021/10/02 20:52:09 skrll Exp $"); 36 37 #include <sys/param.h> 38 #include <sys/kernel.h> 39 #include <sys/bus.h> 40 #include <sys/device.h> 41 #include <sys/intr.h> 42 #include <sys/systm.h> 43 #include <sys/cpu.h> 44 #include <sys/vmem.h> 45 #include <sys/kmem.h> 46 #include <sys/atomic.h> 47 48 #include <machine/cpufunc.h> 49 50 #include <arm/locore.h> 51 #include <arm/armreg.h> 52 53 #include <arm/cortex/gicv3.h> 54 #include <arm/cortex/gic_reg.h> 55 56 #ifdef GIC_SPLFUNCS 57 #include <arm/cortex/gic_splfuncs.h> 58 #endif 59 60 #define PICTOSOFTC(pic) \ 61 ((void *)((uintptr_t)(pic) - offsetof(struct gicv3_softc, sc_pic))) 62 #define LPITOSOFTC(lpi) \ 63 ((void *)((uintptr_t)(lpi) - offsetof(struct gicv3_softc, sc_lpi))) 64 65 #define IPL_TO_PRIORITY(sc, ipl) (((0xff - (ipl)) << (sc)->sc_priority_shift) & 0xff) 66 #define IPL_TO_PMR(sc, ipl) (((0xff - (ipl)) << (sc)->sc_pmr_shift) & 0xff) 67 68 #define GIC_SUPPORTS_1OFN(sc) (((sc)->sc_gicd_typer & GICD_TYPER_No1N) == 0) 69 70 #define GIC_PRIO_SHIFT_NS 4 71 #define GIC_PRIO_SHIFT_S 3 72 73 /* 74 * Set to true if you want to use 1 of N interrupt distribution for SPIs 75 * when available. Disabled by default because it causes issues with the 76 * USB stack. 77 */ 78 bool gicv3_use_1ofn = false; 79 80 static struct gicv3_softc *gicv3_softc; 81 82 static inline uint32_t 83 gicd_read_4(struct gicv3_softc *sc, bus_size_t reg) 84 { 85 return bus_space_read_4(sc->sc_bst, sc->sc_bsh_d, reg); 86 } 87 88 static inline void 89 gicd_write_4(struct gicv3_softc *sc, bus_size_t reg, uint32_t val) 90 { 91 bus_space_write_4(sc->sc_bst, sc->sc_bsh_d, reg, val); 92 } 93 94 #ifdef MULTIPROCESSOR 95 static inline uint64_t 96 gicd_read_8(struct gicv3_softc *sc, bus_size_t reg) 97 { 98 return bus_space_read_8(sc->sc_bst, sc->sc_bsh_d, reg); 99 } 100 #endif 101 102 static inline void 103 gicd_write_8(struct gicv3_softc *sc, bus_size_t reg, uint64_t val) 104 { 105 bus_space_write_8(sc->sc_bst, sc->sc_bsh_d, reg, val); 106 } 107 108 static inline uint32_t 109 gicr_read_4(struct gicv3_softc *sc, u_int index, bus_size_t reg) 110 { 111 KASSERT(index < sc->sc_bsh_r_count); 112 return bus_space_read_4(sc->sc_bst, sc->sc_bsh_r[index], reg); 113 } 114 115 static inline void 116 gicr_write_4(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint32_t val) 117 { 118 KASSERT(index < sc->sc_bsh_r_count); 119 bus_space_write_4(sc->sc_bst, sc->sc_bsh_r[index], reg, val); 120 } 121 122 static inline uint64_t 123 gicr_read_8(struct gicv3_softc *sc, u_int index, bus_size_t reg) 124 { 125 KASSERT(index < sc->sc_bsh_r_count); 126 return bus_space_read_8(sc->sc_bst, sc->sc_bsh_r[index], reg); 127 } 128 129 static inline void 130 gicr_write_8(struct gicv3_softc *sc, u_int index, bus_size_t reg, uint64_t val) 131 { 132 KASSERT(index < sc->sc_bsh_r_count); 133 bus_space_write_8(sc->sc_bst, sc->sc_bsh_r[index], reg, val); 134 } 135 136 static void 137 gicv3_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 138 { 139 struct gicv3_softc * const sc = PICTOSOFTC(pic); 140 struct cpu_info * const ci = curcpu(); 141 const u_int group = irqbase / 32; 142 143 if (group == 0) { 144 atomic_or_32(&sc->sc_enabled_sgippi, mask); 145 gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, mask); 146 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 147 ; 148 } else { 149 gicd_write_4(sc, GICD_ISENABLERn(group), mask); 150 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 151 ; 152 } 153 } 154 155 static void 156 gicv3_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 157 { 158 struct gicv3_softc * const sc = PICTOSOFTC(pic); 159 struct cpu_info * const ci = curcpu(); 160 const u_int group = irqbase / 32; 161 162 if (group == 0) { 163 atomic_and_32(&sc->sc_enabled_sgippi, ~mask); 164 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, mask); 165 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 166 ; 167 } else { 168 gicd_write_4(sc, GICD_ICENABLERn(group), mask); 169 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 170 ; 171 } 172 } 173 174 static void 175 gicv3_establish_irq(struct pic_softc *pic, struct intrsource *is) 176 { 177 struct gicv3_softc * const sc = PICTOSOFTC(pic); 178 const u_int group = is->is_irq / 32; 179 uint32_t ipriority, icfg; 180 uint64_t irouter; 181 u_int n; 182 183 const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl); 184 const u_int ipriority_shift = (is->is_irq & 0x3) * 8; 185 const u_int icfg_shift = (is->is_irq & 0xf) * 2; 186 187 if (group == 0) { 188 /* SGIs and PPIs are per-CPU and always MP-safe */ 189 is->is_mpsafe = true; 190 is->is_percpu = true; 191 192 /* Update interrupt configuration and priority on all redistributors */ 193 for (n = 0; n < sc->sc_bsh_r_count; n++) { 194 icfg = gicr_read_4(sc, n, GICR_ICFGRn(is->is_irq / 16)); 195 if (is->is_type == IST_LEVEL) 196 icfg &= ~(0x2 << icfg_shift); 197 if (is->is_type == IST_EDGE) 198 icfg |= (0x2 << icfg_shift); 199 gicr_write_4(sc, n, GICR_ICFGRn(is->is_irq / 16), icfg); 200 201 ipriority = gicr_read_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4)); 202 ipriority &= ~(0xffU << ipriority_shift); 203 ipriority |= (ipriority_val << ipriority_shift); 204 gicr_write_4(sc, n, GICR_IPRIORITYRn(is->is_irq / 4), ipriority); 205 } 206 } else { 207 /* 208 * If 1 of N SPI routing is supported, route MP-safe interrupts to all 209 * participating PEs. Otherwise, just route to the primary PE. 210 */ 211 if (is->is_mpsafe && GIC_SUPPORTS_1OFN(sc) && gicv3_use_1ofn) { 212 irouter = GICD_IROUTER_Interrupt_Routing_mode; 213 } else { 214 irouter = sc->sc_irouter[0]; 215 } 216 gicd_write_8(sc, GICD_IROUTER(is->is_irq), irouter); 217 218 /* Update interrupt configuration */ 219 icfg = gicd_read_4(sc, GICD_ICFGRn(is->is_irq / 16)); 220 if (is->is_type == IST_LEVEL) 221 icfg &= ~(0x2 << icfg_shift); 222 if (is->is_type == IST_EDGE) 223 icfg |= (0x2 << icfg_shift); 224 gicd_write_4(sc, GICD_ICFGRn(is->is_irq / 16), icfg); 225 226 /* Update interrupt priority */ 227 ipriority = gicd_read_4(sc, GICD_IPRIORITYRn(is->is_irq / 4)); 228 ipriority &= ~(0xffU << ipriority_shift); 229 ipriority |= (ipriority_val << ipriority_shift); 230 gicd_write_4(sc, GICD_IPRIORITYRn(is->is_irq / 4), ipriority); 231 } 232 } 233 234 static void 235 gicv3_set_priority(struct pic_softc *pic, int ipl) 236 { 237 struct gicv3_softc * const sc = PICTOSOFTC(pic); 238 struct cpu_info * const ci = curcpu(); 239 240 if (ipl < ci->ci_hwpl) { 241 /* Lowering priority mask */ 242 ci->ci_hwpl = ipl; 243 icc_pmr_write(IPL_TO_PMR(sc, ipl)); 244 } 245 } 246 247 static void 248 gicv3_dist_enable(struct gicv3_softc *sc) 249 { 250 uint32_t gicd_ctrl; 251 u_int n; 252 253 /* Disable the distributor */ 254 gicd_ctrl = gicd_read_4(sc, GICD_CTRL); 255 gicd_ctrl &= ~(GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS); 256 gicd_write_4(sc, GICD_CTRL, gicd_ctrl); 257 258 /* Wait for register write to complete */ 259 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 260 ; 261 262 /* Clear all INTID enable bits */ 263 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) 264 gicd_write_4(sc, GICD_ICENABLERn(n / 32), ~0); 265 266 /* Set default priorities to lowest */ 267 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 4) 268 gicd_write_4(sc, GICD_IPRIORITYRn(n / 4), ~0); 269 270 /* Set all interrupts to G1NS */ 271 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 32) { 272 gicd_write_4(sc, GICD_IGROUPRn(n / 32), ~0); 273 gicd_write_4(sc, GICD_IGRPMODRn(n / 32), 0); 274 } 275 276 /* Set all interrupts level-sensitive by default */ 277 for (n = 32; n < sc->sc_pic.pic_maxsources; n += 16) 278 gicd_write_4(sc, GICD_ICFGRn(n / 16), 0); 279 280 /* Wait for register writes to complete */ 281 while (gicd_read_4(sc, GICD_CTRL) & GICD_CTRL_RWP) 282 ; 283 284 /* Enable Affinity routing and G1NS interrupts */ 285 gicd_ctrl = GICD_CTRL_EnableGrp1A | GICD_CTRL_ARE_NS; 286 gicd_write_4(sc, GICD_CTRL, gicd_ctrl); 287 } 288 289 static void 290 gicv3_redist_enable(struct gicv3_softc *sc, struct cpu_info *ci) 291 { 292 uint32_t icfg; 293 u_int n, o; 294 295 /* Clear INTID enable bits */ 296 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICENABLER0, ~0); 297 298 /* Wait for register write to complete */ 299 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 300 ; 301 302 /* Set default priorities */ 303 for (n = 0; n < 32; n += 4) { 304 uint32_t priority = 0; 305 size_t byte_shift = 0; 306 for (o = 0; o < 4; o++, byte_shift += 8) { 307 struct intrsource * const is = sc->sc_pic.pic_sources[n + o]; 308 if (is == NULL) 309 priority |= (0xffU << byte_shift); 310 else { 311 const u_int ipriority_val = IPL_TO_PRIORITY(sc, is->is_ipl); 312 priority |= ipriority_val << byte_shift; 313 } 314 } 315 gicr_write_4(sc, ci->ci_gic_redist, GICR_IPRIORITYRn(n / 4), priority); 316 } 317 318 /* Set all interrupts to G1NS */ 319 gicr_write_4(sc, ci->ci_gic_redist, GICR_IGROUPR0, ~0); 320 gicr_write_4(sc, ci->ci_gic_redist, GICR_IGRPMODR0, 0); 321 322 /* Restore PPI configs */ 323 for (n = 0, icfg = 0; n < 16; n++) { 324 struct intrsource * const is = sc->sc_pic.pic_sources[16 + n]; 325 if (is != NULL && is->is_type == IST_EDGE) 326 icfg |= (0x2 << (n * 2)); 327 } 328 gicr_write_4(sc, ci->ci_gic_redist, GICR_ICFGRn(1), icfg); 329 330 /* Restore current enable bits */ 331 gicr_write_4(sc, ci->ci_gic_redist, GICR_ISENABLER0, sc->sc_enabled_sgippi); 332 333 /* Wait for register write to complete */ 334 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR) & GICR_CTLR_RWP) 335 ; 336 } 337 338 static uint64_t 339 gicv3_cpu_identity(void) 340 { 341 u_int aff3, aff2, aff1, aff0; 342 343 const register_t mpidr = cpu_mpidr_aff_read(); 344 aff0 = __SHIFTOUT(mpidr, MPIDR_AFF0); 345 aff1 = __SHIFTOUT(mpidr, MPIDR_AFF1); 346 aff2 = __SHIFTOUT(mpidr, MPIDR_AFF2); 347 aff3 = __SHIFTOUT(mpidr, MPIDR_AFF3); 348 349 return __SHIFTIN(aff0, GICR_TYPER_Affinity_Value_Aff0) | 350 __SHIFTIN(aff1, GICR_TYPER_Affinity_Value_Aff1) | 351 __SHIFTIN(aff2, GICR_TYPER_Affinity_Value_Aff2) | 352 __SHIFTIN(aff3, GICR_TYPER_Affinity_Value_Aff3); 353 } 354 355 static u_int 356 gicv3_find_redist(struct gicv3_softc *sc) 357 { 358 uint64_t gicr_typer; 359 u_int n; 360 361 const uint64_t cpu_identity = gicv3_cpu_identity(); 362 363 for (n = 0; n < sc->sc_bsh_r_count; n++) { 364 gicr_typer = gicr_read_8(sc, n, GICR_TYPER); 365 if ((gicr_typer & GICR_TYPER_Affinity_Value) == cpu_identity) 366 return n; 367 } 368 369 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 370 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 371 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 372 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 373 374 panic("%s: could not find GICv3 redistributor for cpu %d.%d.%d.%d", 375 cpu_name(curcpu()), aff3, aff2, aff1, aff0); 376 } 377 378 static uint64_t 379 gicv3_sgir(struct gicv3_softc *sc) 380 { 381 const uint64_t cpu_identity = gicv3_cpu_identity(); 382 383 const u_int aff0 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff0); 384 const u_int aff1 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff1); 385 const u_int aff2 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff2); 386 const u_int aff3 = __SHIFTOUT(cpu_identity, GICR_TYPER_Affinity_Value_Aff3); 387 388 return __SHIFTIN(__BIT(aff0), ICC_SGIR_EL1_TargetList) | 389 __SHIFTIN(aff1, ICC_SGIR_EL1_Aff1) | 390 __SHIFTIN(aff2, ICC_SGIR_EL1_Aff2) | 391 __SHIFTIN(aff3, ICC_SGIR_EL1_Aff3); 392 } 393 394 static void 395 gicv3_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 396 { 397 struct gicv3_softc * const sc = PICTOSOFTC(pic); 398 uint32_t icc_sre, icc_ctlr, gicr_waker; 399 400 evcnt_attach_dynamic(&ci->ci_intr_preempt, EVCNT_TYPE_MISC, NULL, 401 ci->ci_cpuname, "intr preempt"); 402 403 ci->ci_gic_redist = gicv3_find_redist(sc); 404 ci->ci_gic_sgir = gicv3_sgir(sc); 405 406 /* Enable System register access and disable IRQ/FIQ bypass */ 407 icc_sre = ICC_SRE_EL1_SRE | ICC_SRE_EL1_DFB | ICC_SRE_EL1_DIB; 408 icc_sre_write(icc_sre); 409 410 /* Mark the connected PE as being awake */ 411 gicr_waker = gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER); 412 gicr_waker &= ~GICR_WAKER_ProcessorSleep; 413 gicr_write_4(sc, ci->ci_gic_redist, GICR_WAKER, gicr_waker); 414 while (gicr_read_4(sc, ci->ci_gic_redist, GICR_WAKER) & GICR_WAKER_ChildrenAsleep) 415 ; 416 417 /* Set initial priority mask */ 418 ci->ci_hwpl = IPL_HIGH; 419 icc_pmr_write(IPL_TO_PMR(sc, IPL_HIGH)); 420 421 /* Set the binary point field to the minimum value */ 422 icc_bpr1_write(0); 423 424 /* Enable group 1 interrupt signaling */ 425 icc_igrpen1_write(ICC_IGRPEN_EL1_Enable); 426 427 /* Set EOI mode */ 428 icc_ctlr = icc_ctlr_read(); 429 icc_ctlr &= ~ICC_CTLR_EL1_EOImode; 430 icc_ctlr_write(icc_ctlr); 431 432 /* Enable redistributor */ 433 gicv3_redist_enable(sc, ci); 434 435 /* Allow IRQ exceptions */ 436 ENABLE_INTERRUPT(); 437 } 438 439 #ifdef MULTIPROCESSOR 440 static void 441 gicv3_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi) 442 { 443 struct cpu_info *ci; 444 uint64_t sgir; 445 446 sgir = __SHIFTIN(ipi, ICC_SGIR_EL1_INTID); 447 if (kcp == NULL) { 448 /* Interrupts routed to all PEs, excluding "self" */ 449 if (ncpu == 1) 450 return; 451 sgir |= ICC_SGIR_EL1_IRM; 452 } else { 453 /* Interrupt to exactly one PE */ 454 ci = cpu_lookup(kcpuset_ffs(kcp) - 1); 455 if (ci == curcpu()) 456 return; 457 sgir |= ci->ci_gic_sgir; 458 } 459 icc_sgi1r_write(sgir); 460 isb(); 461 } 462 463 static void 464 gicv3_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 465 { 466 struct gicv3_softc * const sc = PICTOSOFTC(pic); 467 const size_t group = irq / 32; 468 int n; 469 470 kcpuset_zero(affinity); 471 if (group == 0) { 472 /* All CPUs are targets for group 0 (SGI/PPI) */ 473 for (n = 0; n < ncpu; n++) { 474 kcpuset_set(affinity, n); 475 } 476 } else { 477 /* Find distributor targets (SPI) */ 478 const uint64_t irouter = gicd_read_8(sc, GICD_IROUTER(irq)); 479 for (n = 0; n < ncpu; n++) { 480 if (irouter == GICD_IROUTER_Interrupt_Routing_mode || 481 irouter == sc->sc_irouter[n]) 482 kcpuset_set(affinity, n); 483 } 484 } 485 } 486 487 static int 488 gicv3_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity) 489 { 490 struct gicv3_softc * const sc = PICTOSOFTC(pic); 491 const size_t group = irq / 32; 492 uint64_t irouter; 493 494 if (group == 0) 495 return EINVAL; 496 497 const int set = kcpuset_countset(affinity); 498 if (set == 1) { 499 irouter = sc->sc_irouter[kcpuset_ffs(affinity) - 1]; 500 } else if (set == ncpu && GIC_SUPPORTS_1OFN(sc) && gicv3_use_1ofn) { 501 irouter = GICD_IROUTER_Interrupt_Routing_mode; 502 } else { 503 return EINVAL; 504 } 505 506 gicd_write_8(sc, GICD_IROUTER(irq), irouter); 507 508 return 0; 509 } 510 #endif 511 512 static const struct pic_ops gicv3_picops = { 513 .pic_unblock_irqs = gicv3_unblock_irqs, 514 .pic_block_irqs = gicv3_block_irqs, 515 .pic_establish_irq = gicv3_establish_irq, 516 .pic_set_priority = gicv3_set_priority, 517 #ifdef MULTIPROCESSOR 518 .pic_cpu_init = gicv3_cpu_init, 519 .pic_ipi_send = gicv3_ipi_send, 520 .pic_get_affinity = gicv3_get_affinity, 521 .pic_set_affinity = gicv3_set_affinity, 522 #endif 523 }; 524 525 static void 526 gicv3_dcache_wb_range(vaddr_t va, vsize_t len) 527 { 528 cpu_dcache_wb_range(va, len); 529 dsb(sy); 530 } 531 532 static void 533 gicv3_lpi_unblock_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 534 { 535 struct gicv3_softc * const sc = LPITOSOFTC(pic); 536 int bit; 537 538 while ((bit = ffs(mask)) != 0) { 539 sc->sc_lpiconf.base[irqbase + bit - 1] |= GIC_LPICONF_Enable; 540 if (sc->sc_lpiconf_flush) 541 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1); 542 mask &= ~__BIT(bit - 1); 543 } 544 545 if (!sc->sc_lpiconf_flush) 546 dsb(ishst); 547 } 548 549 static void 550 gicv3_lpi_block_irqs(struct pic_softc *pic, size_t irqbase, uint32_t mask) 551 { 552 struct gicv3_softc * const sc = LPITOSOFTC(pic); 553 int bit; 554 555 while ((bit = ffs(mask)) != 0) { 556 sc->sc_lpiconf.base[irqbase + bit - 1] &= ~GIC_LPICONF_Enable; 557 if (sc->sc_lpiconf_flush) 558 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[irqbase + bit - 1], 1); 559 mask &= ~__BIT(bit - 1); 560 } 561 562 if (!sc->sc_lpiconf_flush) 563 dsb(ishst); 564 } 565 566 static void 567 gicv3_lpi_establish_irq(struct pic_softc *pic, struct intrsource *is) 568 { 569 struct gicv3_softc * const sc = LPITOSOFTC(pic); 570 571 sc->sc_lpiconf.base[is->is_irq] = IPL_TO_PRIORITY(sc, is->is_ipl) | GIC_LPICONF_Res1; 572 573 if (sc->sc_lpiconf_flush) 574 gicv3_dcache_wb_range((vaddr_t)&sc->sc_lpiconf.base[is->is_irq], 1); 575 else 576 dsb(ishst); 577 } 578 579 static void 580 gicv3_lpi_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 581 { 582 struct gicv3_softc * const sc = LPITOSOFTC(pic); 583 struct gicv3_lpi_callback *cb; 584 uint64_t propbase, pendbase; 585 uint32_t ctlr; 586 587 /* If physical LPIs are not supported on this redistributor, just return. */ 588 const uint64_t typer = gicr_read_8(sc, ci->ci_gic_redist, GICR_TYPER); 589 if ((typer & GICR_TYPER_PLPIS) == 0) 590 return; 591 592 /* Interrupt target address for this CPU, used by ITS when GITS_TYPER.PTA == 0 */ 593 sc->sc_processor_id[cpu_index(ci)] = __SHIFTOUT(typer, GICR_TYPER_Processor_Number); 594 595 /* Disable LPIs before making changes */ 596 ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); 597 ctlr &= ~GICR_CTLR_Enable_LPIs; 598 gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); 599 dsb(sy); 600 601 /* Setup the LPI configuration table */ 602 propbase = sc->sc_lpiconf.segs[0].ds_addr | 603 __SHIFTIN(ffs(pic->pic_maxsources) - 1, GICR_PROPBASER_IDbits) | 604 __SHIFTIN(GICR_Shareability_IS, GICR_PROPBASER_Shareability) | 605 __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PROPBASER_InnerCache); 606 gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase); 607 propbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PROPBASER); 608 if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) != GICR_Shareability_IS) { 609 if (__SHIFTOUT(propbase, GICR_PROPBASER_Shareability) == GICR_Shareability_NS) { 610 propbase &= ~GICR_PROPBASER_Shareability; 611 propbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PROPBASER_Shareability); 612 propbase &= ~GICR_PROPBASER_InnerCache; 613 propbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PROPBASER_InnerCache); 614 gicr_write_8(sc, ci->ci_gic_redist, GICR_PROPBASER, propbase); 615 } 616 sc->sc_lpiconf_flush = true; 617 } 618 619 /* Setup the LPI pending table */ 620 pendbase = sc->sc_lpipend[cpu_index(ci)].segs[0].ds_addr | 621 __SHIFTIN(GICR_Shareability_IS, GICR_PENDBASER_Shareability) | 622 __SHIFTIN(GICR_Cache_NORMAL_RA_WA_WB, GICR_PENDBASER_InnerCache); 623 gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase); 624 pendbase = gicr_read_8(sc, ci->ci_gic_redist, GICR_PENDBASER); 625 if (__SHIFTOUT(pendbase, GICR_PENDBASER_Shareability) == GICR_Shareability_NS) { 626 pendbase &= ~GICR_PENDBASER_Shareability; 627 pendbase |= __SHIFTIN(GICR_Shareability_NS, GICR_PENDBASER_Shareability); 628 pendbase &= ~GICR_PENDBASER_InnerCache; 629 pendbase |= __SHIFTIN(GICR_Cache_NORMAL_NC, GICR_PENDBASER_InnerCache); 630 gicr_write_8(sc, ci->ci_gic_redist, GICR_PENDBASER, pendbase); 631 } 632 633 /* Enable LPIs */ 634 ctlr = gicr_read_4(sc, ci->ci_gic_redist, GICR_CTLR); 635 ctlr |= GICR_CTLR_Enable_LPIs; 636 gicr_write_4(sc, ci->ci_gic_redist, GICR_CTLR, ctlr); 637 dsb(sy); 638 639 /* Setup ITS if present */ 640 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) 641 cb->cpu_init(cb->priv, ci); 642 } 643 644 #ifdef MULTIPROCESSOR 645 static void 646 gicv3_lpi_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 647 { 648 struct gicv3_softc * const sc = LPITOSOFTC(pic); 649 struct gicv3_lpi_callback *cb; 650 651 kcpuset_zero(affinity); 652 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) 653 cb->get_affinity(cb->priv, irq, affinity); 654 } 655 656 static int 657 gicv3_lpi_set_affinity(struct pic_softc *pic, size_t irq, const kcpuset_t *affinity) 658 { 659 struct gicv3_softc * const sc = LPITOSOFTC(pic); 660 struct gicv3_lpi_callback *cb; 661 int error = EINVAL; 662 663 LIST_FOREACH(cb, &sc->sc_lpi_callbacks, list) { 664 error = cb->set_affinity(cb->priv, irq, affinity); 665 if (error != EPASSTHROUGH) 666 return error; 667 } 668 669 return EINVAL; 670 } 671 #endif 672 673 static const struct pic_ops gicv3_lpiops = { 674 .pic_unblock_irqs = gicv3_lpi_unblock_irqs, 675 .pic_block_irqs = gicv3_lpi_block_irqs, 676 .pic_establish_irq = gicv3_lpi_establish_irq, 677 #ifdef MULTIPROCESSOR 678 .pic_cpu_init = gicv3_lpi_cpu_init, 679 .pic_get_affinity = gicv3_lpi_get_affinity, 680 .pic_set_affinity = gicv3_lpi_set_affinity, 681 #endif 682 }; 683 684 void 685 gicv3_dma_alloc(struct gicv3_softc *sc, struct gicv3_dma *dma, bus_size_t len, bus_size_t align) 686 { 687 int nsegs, error; 688 689 dma->len = len; 690 error = bus_dmamem_alloc(sc->sc_dmat, dma->len, align, 0, dma->segs, 1, &nsegs, BUS_DMA_WAITOK); 691 if (error) 692 panic("bus_dmamem_alloc failed: %d", error); 693 error = bus_dmamem_map(sc->sc_dmat, dma->segs, nsegs, len, (void **)&dma->base, BUS_DMA_WAITOK); 694 if (error) 695 panic("bus_dmamem_map failed: %d", error); 696 error = bus_dmamap_create(sc->sc_dmat, len, 1, len, 0, BUS_DMA_WAITOK, &dma->map); 697 if (error) 698 panic("bus_dmamap_create failed: %d", error); 699 error = bus_dmamap_load(sc->sc_dmat, dma->map, dma->base, dma->len, NULL, BUS_DMA_WAITOK); 700 if (error) 701 panic("bus_dmamap_load failed: %d", error); 702 703 memset(dma->base, 0, dma->len); 704 bus_dmamap_sync(sc->sc_dmat, dma->map, 0, dma->len, BUS_DMASYNC_PREWRITE); 705 } 706 707 static void 708 gicv3_lpi_init(struct gicv3_softc *sc) 709 { 710 /* 711 * Allocate LPI configuration table 712 */ 713 gicv3_dma_alloc(sc, &sc->sc_lpiconf, sc->sc_lpi.pic_maxsources, 0x1000); 714 KASSERT((sc->sc_lpiconf.segs[0].ds_addr & ~GICR_PROPBASER_Physical_Address) == 0); 715 716 /* 717 * Allocate LPI pending tables 718 */ 719 const bus_size_t lpipend_sz = (8192 + sc->sc_lpi.pic_maxsources) / NBBY; 720 for (int cpuindex = 0; cpuindex < ncpu; cpuindex++) { 721 gicv3_dma_alloc(sc, &sc->sc_lpipend[cpuindex], lpipend_sz, 0x10000); 722 KASSERT((sc->sc_lpipend[cpuindex].segs[0].ds_addr & ~GICR_PENDBASER_Physical_Address) == 0); 723 } 724 } 725 726 void 727 gicv3_irq_handler(void *frame) 728 { 729 struct cpu_info * const ci = curcpu(); 730 struct gicv3_softc * const sc = gicv3_softc; 731 struct pic_softc *pic; 732 const int oldipl = ci->ci_cpl; 733 734 ci->ci_data.cpu_nintr++; 735 736 if (ci->ci_hwpl != oldipl) { 737 ci->ci_hwpl = oldipl; 738 icc_pmr_write(IPL_TO_PMR(sc, oldipl)); 739 if (oldipl == IPL_HIGH) { 740 return; 741 } 742 } 743 744 for (;;) { 745 const uint32_t iar = icc_iar1_read(); 746 dsb(sy); 747 const uint32_t irq = __SHIFTOUT(iar, ICC_IAR_INTID); 748 if (irq == ICC_IAR_INTID_SPURIOUS) 749 break; 750 751 pic = irq >= GIC_LPI_BASE ? &sc->sc_lpi : &sc->sc_pic; 752 if (irq - pic->pic_irqbase >= pic->pic_maxsources) 753 continue; 754 755 struct intrsource * const is = pic->pic_sources[irq - pic->pic_irqbase]; 756 KASSERT(is != NULL); 757 758 const bool early_eoi = irq < GIC_LPI_BASE && is->is_type == IST_EDGE; 759 760 const int ipl = is->is_ipl; 761 if (__predict_false(ipl < ci->ci_cpl)) { 762 pic_do_pending_ints(I32_bit, ipl, frame); 763 } else if (ci->ci_cpl != ipl) { 764 icc_pmr_write(IPL_TO_PMR(sc, ipl)); 765 ci->ci_hwpl = ci->ci_cpl = ipl; 766 } 767 768 if (early_eoi) { 769 icc_eoi1r_write(iar); 770 isb(); 771 } 772 773 const int64_t nintr = ci->ci_data.cpu_nintr; 774 775 ENABLE_INTERRUPT(); 776 pic_dispatch(is, frame); 777 DISABLE_INTERRUPT(); 778 779 if (nintr != ci->ci_data.cpu_nintr) 780 ci->ci_intr_preempt.ev_count++; 781 782 if (!early_eoi) { 783 icc_eoi1r_write(iar); 784 isb(); 785 } 786 } 787 788 pic_do_pending_ints(I32_bit, oldipl, frame); 789 } 790 791 static bool 792 gicv3_cpuif_is_nonsecure(struct gicv3_softc *sc) 793 { 794 /* 795 * Write 0 to bit7 and see if it sticks. This is only possible if 796 * we have a non-secure view of the PMR register. 797 */ 798 const uint32_t opmr = icc_pmr_read(); 799 icc_pmr_write(0); 800 const uint32_t npmr = icc_pmr_read(); 801 icc_pmr_write(opmr); 802 803 return (npmr & GICC_PMR_NONSECURE) == 0; 804 } 805 806 static bool 807 gicv3_dist_is_nonsecure(struct gicv3_softc *sc) 808 { 809 const uint32_t gicd_ctrl = gicd_read_4(sc, GICD_CTRL); 810 811 /* 812 * If security is enabled, we have a non-secure view of the IPRIORITYRn 813 * registers and LPI configuration priority fields. 814 */ 815 return (gicd_ctrl & GICD_CTRL_DS) == 0; 816 } 817 818 /* 819 * Rockchip RK3399 provides a different view of int priority registers 820 * depending on which firmware is in use. This is hard to detect in 821 * a way that could possibly break other boards, so only do this 822 * detection if we know we are on a RK3399 SoC. 823 */ 824 static void 825 gicv3_quirk_rockchip_rk3399(struct gicv3_softc *sc) 826 { 827 /* Detect the number of supported PMR bits */ 828 icc_pmr_write(0xff); 829 const uint8_t pmrbits = icc_pmr_read(); 830 831 /* Detect the number of supported IPRIORITYRn bits */ 832 const uint32_t oiprio = gicd_read_4(sc, GICD_IPRIORITYRn(8)); 833 gicd_write_4(sc, GICD_IPRIORITYRn(8), oiprio | 0xff); 834 const uint8_t pribits = gicd_read_4(sc, GICD_IPRIORITYRn(8)) & 0xff; 835 gicd_write_4(sc, GICD_IPRIORITYRn(8), oiprio); 836 837 /* 838 * If we see fewer PMR bits than IPRIORITYRn bits here, it means 839 * we have a secure view of IPRIORITYRn (this is not supposed to 840 * happen!). 841 */ 842 if (pmrbits < pribits) { 843 aprint_verbose_dev(sc->sc_dev, 844 "buggy RK3399 firmware detected; applying workaround\n"); 845 sc->sc_priority_shift = GIC_PRIO_SHIFT_S; 846 } 847 } 848 849 int 850 gicv3_init(struct gicv3_softc *sc) 851 { 852 CPU_INFO_ITERATOR cii; 853 struct cpu_info *ci; 854 855 KASSERT(CPU_IS_PRIMARY(curcpu())); 856 857 LIST_INIT(&sc->sc_lpi_callbacks); 858 859 /* Store route to CPU for SPIs */ 860 sc->sc_irouter = kmem_zalloc(sizeof(*sc->sc_irouter) * ncpu, KM_SLEEP); 861 for (CPU_INFO_FOREACH(cii, ci)) { 862 KASSERT(cpu_index(ci) < ncpu); 863 sc->sc_irouter[cpu_index(ci)] = ci->ci_cpuid; 864 } 865 866 sc->sc_gicd_typer = gicd_read_4(sc, GICD_TYPER); 867 868 /* 869 * We don't always have a consistent view of priorities between the 870 * CPU interface (ICC_PMR_EL1) and the GICD/GICR registers. Detect 871 * if we are making secure or non-secure accesses to each, and adjust 872 * the values that we write to each accordingly. 873 */ 874 const bool dist_ns = gicv3_dist_is_nonsecure(sc); 875 sc->sc_priority_shift = dist_ns ? GIC_PRIO_SHIFT_NS : GIC_PRIO_SHIFT_S; 876 const bool cpuif_ns = gicv3_cpuif_is_nonsecure(sc); 877 sc->sc_pmr_shift = cpuif_ns ? GIC_PRIO_SHIFT_NS : GIC_PRIO_SHIFT_S; 878 879 if ((sc->sc_quirks & GICV3_QUIRK_RK3399) != 0) 880 gicv3_quirk_rockchip_rk3399(sc); 881 882 aprint_verbose_dev(sc->sc_dev, 883 "iidr 0x%08x, cpuif %ssecure, dist %ssecure, " 884 "priority shift %d, pmr shift %d, quirks %#x\n", 885 gicd_read_4(sc, GICD_IIDR), 886 cpuif_ns ? "non-" : "", 887 dist_ns ? "non-" : "", 888 sc->sc_priority_shift, 889 sc->sc_pmr_shift, 890 sc->sc_quirks); 891 892 sc->sc_pic.pic_ops = &gicv3_picops; 893 sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(sc->sc_gicd_typer); 894 snprintf(sc->sc_pic.pic_name, sizeof(sc->sc_pic.pic_name), "gicv3"); 895 #ifdef MULTIPROCESSOR 896 sc->sc_pic.pic_cpus = kcpuset_running; 897 #endif 898 pic_add(&sc->sc_pic, 0); 899 900 if ((sc->sc_gicd_typer & GICD_TYPER_LPIS) != 0) { 901 sc->sc_lpipend = kmem_zalloc(sizeof(*sc->sc_lpipend) * ncpu, KM_SLEEP); 902 sc->sc_processor_id = kmem_zalloc(sizeof(*sc->sc_processor_id) * ncpu, KM_SLEEP); 903 904 sc->sc_lpi.pic_ops = &gicv3_lpiops; 905 sc->sc_lpi.pic_maxsources = 8192; /* Min. required by GICv3 spec */ 906 snprintf(sc->sc_lpi.pic_name, sizeof(sc->sc_lpi.pic_name), "gicv3-lpi"); 907 pic_add(&sc->sc_lpi, GIC_LPI_BASE); 908 909 sc->sc_lpi_pool = vmem_create("gicv3-lpi", 0, sc->sc_lpi.pic_maxsources, 910 1, NULL, NULL, NULL, 0, VM_SLEEP, IPL_HIGH); 911 if (sc->sc_lpi_pool == NULL) 912 panic("failed to create gicv3 lpi pool\n"); 913 914 gicv3_lpi_init(sc); 915 } 916 917 KASSERT(gicv3_softc == NULL); 918 gicv3_softc = sc; 919 920 for (int i = 0; i < sc->sc_bsh_r_count; i++) { 921 const uint64_t gicr_typer = gicr_read_8(sc, i, GICR_TYPER); 922 const u_int aff0 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff0); 923 const u_int aff1 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff1); 924 const u_int aff2 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff2); 925 const u_int aff3 = __SHIFTOUT(gicr_typer, GICR_TYPER_Affinity_Value_Aff3); 926 927 aprint_debug_dev(sc->sc_dev, "redist %d: cpu %d.%d.%d.%d\n", 928 i, aff3, aff2, aff1, aff0); 929 } 930 931 gicv3_dist_enable(sc); 932 933 gicv3_cpu_init(&sc->sc_pic, curcpu()); 934 if ((sc->sc_gicd_typer & GICD_TYPER_LPIS) != 0) 935 gicv3_lpi_cpu_init(&sc->sc_lpi, curcpu()); 936 937 #ifdef MULTIPROCESSOR 938 intr_establish_xname(IPI_AST, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast"); 939 intr_establish_xname(IPI_XCALL, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall"); 940 intr_establish_xname(IPI_GENERIC, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic"); 941 intr_establish_xname(IPI_NOP, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop"); 942 intr_establish_xname(IPI_SHOOTDOWN, IPL_SCHED, IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown"); 943 #ifdef DDB 944 intr_establish_xname(IPI_DDB, IPL_HIGH, IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb"); 945 #endif 946 #ifdef __HAVE_PREEMPTION 947 intr_establish_xname(IPI_KPREEMPT, IPL_VM, IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt"); 948 #endif 949 #endif 950 951 #ifdef GIC_SPLFUNCS 952 gic_spl_init(); 953 #endif 954 955 return 0; 956 } 957