1 /* $OpenBSD: agintc.c,v 1.62 2025/01/24 20:17:28 kettenis Exp $ */ 2 /* 3 * Copyright (c) 2007, 2009, 2011, 2017 Dale Rahn <drahn@dalerahn.com> 4 * Copyright (c) 2018 Mark Kettenis <kettenis@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 /* 20 * This is a device driver for the GICv3/GICv4 IP from ARM as specified 21 * in IHI0069C, an example of this hardware is the GIC 500. 22 */ 23 24 #include <sys/param.h> 25 #include <sys/systm.h> 26 #include <sys/queue.h> 27 #include <sys/malloc.h> 28 #include <sys/device.h> 29 #include <sys/evcount.h> 30 31 #include <machine/bus.h> 32 #include <machine/cpufunc.h> 33 #include <machine/fdt.h> 34 35 #include <dev/ofw/fdt.h> 36 #include <dev/ofw/openfirm.h> 37 38 #include <machine/simplebusvar.h> 39 40 #define ICC_PMR s3_0_c4_c6_0 41 #define ICC_IAR0 s3_0_c12_c8_0 42 #define ICC_EOIR0 s3_0_c12_c8_1 43 #define ICC_HPPIR0 s3_0_c12_c8_2 44 #define ICC_BPR0 s3_0_c12_c8_3 45 46 #define ICC_DIR s3_0_c12_c11_1 47 #define ICC_RPR s3_0_c12_c11_3 48 #define ICC_SGI1R s3_0_c12_c11_5 49 #define ICC_SGI0R s3_0_c12_c11_7 50 51 #define ICC_IAR1 s3_0_c12_c12_0 52 #define ICC_EOIR1 s3_0_c12_c12_1 53 #define ICC_HPPIR1 s3_0_c12_c12_2 54 #define ICC_BPR1 s3_0_c12_c12_3 55 #define ICC_CTLR s3_0_c12_c12_4 56 #define ICC_SRE_EL1 s3_0_c12_c12_5 57 #define ICC_SRE_EL1_EN 0x7 58 #define ICC_IGRPEN0 s3_0_c12_c12_6 59 #define ICC_IGRPEN1 s3_0_c12_c12_7 60 61 #define _STR(x) #x 62 #define STR(x) _STR(x) 63 64 /* distributor registers */ 65 #define GICD_CTLR 0x0000 66 /* non-secure */ 67 #define GICD_CTLR_RWP (1U << 31) 68 #define GICD_CTLR_EnableGrp1 (1 << 0) 69 #define GICD_CTLR_EnableGrp1A (1 << 1) 70 #define GICD_CTLR_ARE_NS (1 << 4) 71 #define GICD_CTLR_DS (1 << 6) 72 #define GICD_TYPER 0x0004 73 #define GICD_TYPER_MBIS (1 << 16) 74 #define GICD_TYPER_LPIS (1 << 17) 75 #define GICD_TYPER_ITLINE_M 0x1f 76 #define GICD_IIDR 0x0008 77 #define GICD_SETSPI_NSR 0x0040 78 #define GICD_CLRSPI_NSR 0x0048 79 #define GICD_IGROUPR(i) (0x0080 + (IRQ_TO_REG32(i) * 4)) 80 #define GICD_ISENABLER(i) (0x0100 + (IRQ_TO_REG32(i) * 4)) 81 #define GICD_ICENABLER(i) (0x0180 + (IRQ_TO_REG32(i) * 4)) 82 #define GICD_ISPENDR(i) (0x0200 + (IRQ_TO_REG32(i) * 4)) 83 #define GICD_ICPENDR(i) (0x0280 + (IRQ_TO_REG32(i) * 4)) 84 #define GICD_ISACTIVER(i) (0x0300 + (IRQ_TO_REG32(i) * 4)) 85 #define GICD_ICACTIVER(i) (0x0380 + (IRQ_TO_REG32(i) * 4)) 86 #define GICD_IPRIORITYR(i) (0x0400 + (i)) 87 #define GICD_ICFGR(i) (0x0c00 + (IRQ_TO_REG16(i) * 4)) 88 #define GICD_ICFGR_TRIG_LEVEL(i) (0x0 << (IRQ_TO_REG16BIT(i) * 2)) 89 #define GICD_ICFGR_TRIG_EDGE(i) (0x2 << (IRQ_TO_REG16BIT(i) * 2)) 90 #define GICD_ICFGR_TRIG_MASK(i) (0x2 << (IRQ_TO_REG16BIT(i) * 2)) 91 #define GICD_IGRPMODR(i) (0x0d00 + (IRQ_TO_REG32(i) * 4)) 92 #define GICD_NSACR(i) (0x0e00 + (IRQ_TO_REG16(i) * 4)) 93 #define GICD_IROUTER(i) (0x6000 + ((i) * 8)) 94 95 /* redistributor registers */ 96 #define GICR_CTLR 0x00000 97 #define GICR_CTLR_RWP ((1U << 31) | (1 << 3)) 98 #define GICR_CTLR_ENABLE_LPIS (1 << 0) 99 #define GICR_IIDR 0x00004 100 #define GICR_TYPER 0x00008 101 #define GICR_TYPER_LAST (1 << 4) 102 #define GICR_TYPER_VLPIS (1 << 1) 103 #define GICR_WAKER 0x00014 104 #define GICR_WAKER_X31 (1U << 31) 105 #define GICR_WAKER_CHILDRENASLEEP (1 << 2) 106 #define GICR_WAKER_PROCESSORSLEEP (1 << 1) 107 #define GICR_WAKER_X0 (1 << 0) 108 #define GICR_PROPBASER 0x00070 109 #define GICR_PROPBASER_ISH (1ULL << 10) 110 #define GICR_PROPBASER_IC_NORM_NC (1ULL << 7) 111 #define GICR_PENDBASER 0x00078 112 #define GICR_PENDBASER_PTZ (1ULL << 62) 113 #define GICR_PENDBASER_ISH (1ULL << 10) 114 #define GICR_PENDBASER_IC_NORM_NC (1ULL << 7) 115 #define GICR_IGROUPR0 0x10080 116 #define GICR_ISENABLE0 0x10100 117 #define GICR_ICENABLE0 0x10180 118 #define GICR_ISPENDR0 0x10200 119 #define GICR_ICPENDR0 0x10280 120 #define GICR_ISACTIVE0 0x10300 121 #define GICR_ICACTIVE0 0x10380 122 #define GICR_IPRIORITYR(i) (0x10400 + (i)) 123 #define GICR_ICFGR0 0x10c00 124 #define GICR_ICFGR1 0x10c04 125 #define GICR_IGRPMODR0 0x10d00 126 127 #define GICR_PROP_SIZE (64 * 1024) 128 #define GICR_PROP_GROUP1 (1 << 1) 129 #define GICR_PROP_ENABLE (1 << 0) 130 #define GICR_PEND_SIZE (64 * 1024) 131 132 #define PPI_BASE 16 133 #define SPI_BASE 32 134 #define LPI_BASE 8192 135 136 #define IRQ_TO_REG32(i) (((i) >> 5) & 0x1f) 137 #define IRQ_TO_REG32BIT(i) ((i) & 0x1f) 138 139 #define IRQ_TO_REG16(i) (((i) >> 4) & 0x3f) 140 #define IRQ_TO_REG16BIT(i) ((i) & 0xf) 141 142 #define IRQ_ENABLE 1 143 #define IRQ_DISABLE 0 144 145 struct agintc_mbi_range { 146 int mr_base; 147 int mr_span; 148 void **mr_mbi; 149 }; 150 151 struct agintc_lpi_info { 152 struct agintc_msi_softc *li_msic; 153 struct cpu_info *li_ci; 154 uint32_t li_deviceid; 155 uint32_t li_eventid; 156 struct intrhand *li_ih; 157 }; 158 159 struct agintc_softc { 160 struct simplebus_softc sc_sbus; 161 struct intrq *sc_handler; 162 struct agintc_lpi_info **sc_lpi; 163 bus_space_tag_t sc_iot; 164 bus_space_handle_t sc_d_ioh; 165 bus_space_handle_t *sc_r_ioh; 166 bus_space_handle_t *sc_rbase_ioh; 167 bus_dma_tag_t sc_dmat; 168 uint16_t *sc_processor; 169 int sc_cpuremap[MAXCPUS]; 170 int sc_nintr; 171 int sc_nlpi; 172 bus_addr_t sc_mbi_addr; 173 int sc_mbi_nranges; 174 struct agintc_mbi_range *sc_mbi_ranges; 175 int sc_prio_shift; 176 int sc_pmr_shift; 177 int sc_rk3399_quirk; 178 struct evcount sc_spur; 179 int sc_ncells; 180 int sc_num_redist; 181 int sc_num_redist_regions; 182 struct agintc_dmamem *sc_prop; 183 struct agintc_dmamem *sc_pend; 184 struct interrupt_controller sc_ic; 185 int sc_ipi_num[3]; /* id for each ipi */ 186 int sc_ipi_reason[MAXCPUS]; /* cause of ipi */ 187 void *sc_ipi_irq[3]; /* irqhandle for each ipi */ 188 }; 189 struct agintc_softc *agintc_sc; 190 191 struct intrhand { 192 TAILQ_ENTRY(intrhand) ih_list; /* link on intrq list */ 193 int (*ih_func)(void *); /* handler */ 194 void *ih_arg; /* arg for handler */ 195 int ih_ipl; /* IPL_* */ 196 int ih_flags; 197 int ih_irq; /* IRQ number */ 198 struct evcount ih_count; 199 char *ih_name; 200 struct cpu_info *ih_ci; /* CPU the IRQ runs on */ 201 }; 202 203 struct intrq { 204 TAILQ_HEAD(, intrhand) iq_list; /* handler list */ 205 struct cpu_info *iq_ci; /* CPU the IRQ runs on */ 206 int iq_irq_max; /* IRQ to mask while handling */ 207 int iq_irq_min; /* lowest IRQ when shared */ 208 int iq_ist; /* share type */ 209 int iq_route; 210 }; 211 212 struct agintc_dmamem { 213 bus_dmamap_t adm_map; 214 bus_dma_segment_t adm_seg; 215 size_t adm_size; 216 caddr_t adm_kva; 217 }; 218 219 #define AGINTC_DMA_MAP(_adm) ((_adm)->adm_map) 220 #define AGINTC_DMA_LEN(_adm) ((_adm)->adm_size) 221 #define AGINTC_DMA_DVA(_adm) ((_adm)->adm_map->dm_segs[0].ds_addr) 222 #define AGINTC_DMA_KVA(_adm) ((void *)(_adm)->adm_kva) 223 224 struct agintc_dmamem *agintc_dmamem_alloc(bus_dma_tag_t, bus_size_t, 225 bus_size_t); 226 void agintc_dmamem_free(bus_dma_tag_t, struct agintc_dmamem *); 227 228 int agintc_match(struct device *, void *, void *); 229 void agintc_attach(struct device *, struct device *, void *); 230 void agintc_mbiinit(struct agintc_softc *, int, bus_addr_t); 231 void agintc_cpuinit(void); 232 int agintc_spllower(int); 233 void agintc_splx(int); 234 int agintc_splraise(int); 235 void agintc_setipl(int); 236 void agintc_enable_wakeup(void); 237 void agintc_disable_wakeup(void); 238 void agintc_calc_mask(void); 239 void agintc_calc_irq(struct agintc_softc *sc, int irq); 240 void *agintc_intr_establish(int, int, int, struct cpu_info *, 241 int (*)(void *), void *, char *); 242 void *agintc_intr_establish_fdt(void *cookie, int *cell, int level, 243 struct cpu_info *, int (*func)(void *), void *arg, char *name); 244 void *agintc_intr_establish_mbi(void *, uint64_t *, uint64_t *, 245 int , struct cpu_info *, int (*)(void *), void *, char *); 246 void agintc_intr_disestablish(void *); 247 void agintc_intr_set_wakeup(void *); 248 void agintc_irq_handler(void *); 249 uint32_t agintc_iack(void); 250 void agintc_eoi(uint32_t); 251 void agintc_set_priority(struct agintc_softc *sc, int, int); 252 void agintc_intr_enable(struct agintc_softc *, int); 253 void agintc_intr_disable(struct agintc_softc *, int); 254 void agintc_intr_config(struct agintc_softc *, int, int); 255 void agintc_route(struct agintc_softc *, int, int, 256 struct cpu_info *); 257 void agintc_route_irq(void *, int, struct cpu_info *); 258 void agintc_intr_barrier(void *); 259 void agintc_r_wait_rwp(struct agintc_softc *sc); 260 261 int agintc_ipi_ddb(void *v); 262 int agintc_ipi_halt(void *v); 263 int agintc_ipi_nop(void *v); 264 int agintc_ipi_combined(void *); 265 void agintc_send_ipi(struct cpu_info *, int); 266 267 void agintc_msi_discard(struct agintc_lpi_info *); 268 void agintc_msi_inv(struct agintc_lpi_info *); 269 270 const struct cfattach agintc_ca = { 271 sizeof (struct agintc_softc), agintc_match, agintc_attach 272 }; 273 274 struct cfdriver agintc_cd = { 275 NULL, "agintc", DV_DULL 276 }; 277 278 static char *agintc_compatibles[] = { 279 "arm,gic-v3", 280 "arm,gic-v4", 281 NULL 282 }; 283 284 int 285 agintc_match(struct device *parent, void *cfdata, void *aux) 286 { 287 struct fdt_attach_args *faa = aux; 288 int i; 289 290 for (i = 0; agintc_compatibles[i]; i++) 291 if (OF_is_compatible(faa->fa_node, agintc_compatibles[i])) 292 return (1); 293 294 return (0); 295 } 296 297 static void 298 __isb(void) 299 { 300 __asm volatile("isb"); 301 } 302 303 void 304 agintc_attach(struct device *parent, struct device *self, void *aux) 305 { 306 struct agintc_softc *sc = (struct agintc_softc *)self; 307 struct fdt_attach_args *faa = aux; 308 struct cpu_info *ci; 309 CPU_INFO_ITERATOR cii; 310 u_long psw; 311 uint32_t typer; 312 uint32_t nsacr, oldnsacr; 313 uint32_t pmr, oldpmr; 314 uint32_t ctrl, bits; 315 uint32_t affinity; 316 uint64_t redist_stride; 317 int i, nbits, nintr; 318 int idx, offset, nredist; 319 #ifdef MULTIPROCESSOR 320 int nipi, ipiirq[3]; 321 #endif 322 323 psw = intr_disable(); 324 arm_init_smask(); 325 326 sc->sc_iot = faa->fa_iot; 327 sc->sc_dmat = faa->fa_dmat; 328 329 sc->sc_num_redist_regions = 330 OF_getpropint(faa->fa_node, "#redistributor-regions", 1); 331 332 if (faa->fa_nreg < sc->sc_num_redist_regions + 1) 333 panic("%s: missing registers", __func__); 334 335 if (bus_space_map(sc->sc_iot, faa->fa_reg[0].addr, 336 faa->fa_reg[0].size, 0, &sc->sc_d_ioh)) 337 panic("%s: GICD bus_space_map failed", __func__); 338 339 sc->sc_rbase_ioh = mallocarray(sc->sc_num_redist_regions, 340 sizeof(*sc->sc_rbase_ioh), M_DEVBUF, M_WAITOK); 341 for (idx = 0; idx < sc->sc_num_redist_regions; idx++) { 342 if (bus_space_map(sc->sc_iot, faa->fa_reg[1 + idx].addr, 343 faa->fa_reg[1 + idx].size, 0, &sc->sc_rbase_ioh[idx])) 344 panic("%s: GICR bus_space_map failed", __func__); 345 } 346 347 typer = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_TYPER); 348 349 if (typer & GICD_TYPER_LPIS) { 350 /* Allocate redistributor tables */ 351 sc->sc_prop = agintc_dmamem_alloc(sc->sc_dmat, 352 GICR_PROP_SIZE, GICR_PROP_SIZE); 353 if (sc->sc_prop == NULL) { 354 printf(": can't alloc LPI config table\n"); 355 goto unmap; 356 } 357 sc->sc_pend = agintc_dmamem_alloc(sc->sc_dmat, 358 GICR_PEND_SIZE, GICR_PEND_SIZE); 359 if (sc->sc_pend == NULL) { 360 printf(": can't alloc LPI pending table\n"); 361 goto unmap; 362 } 363 364 /* Minimum number of LPIs supported by any implementation. */ 365 sc->sc_nlpi = 8192; 366 } 367 368 if (typer & GICD_TYPER_MBIS) 369 agintc_mbiinit(sc, faa->fa_node, faa->fa_reg[0].addr); 370 371 /* 372 * We are guaranteed to have at least 16 priority levels, so 373 * in principle we just want to use the top 4 bits of the 374 * (non-secure) priority field. 375 */ 376 sc->sc_prio_shift = sc->sc_pmr_shift = 4; 377 378 /* 379 * If the system supports two security states and SCR_EL3.FIQ 380 * is zero, the non-secure shifted view applies. We detect 381 * this by checking whether the number of writable bits 382 * matches the number of implemented priority bits. If that 383 * is the case we will need to adjust the priorities that we 384 * write into ICC_PMR_EL1 accordingly. 385 * 386 * On Ampere eMAG it appears as if there are five writable 387 * bits when we write 0xff. But for higher priorities 388 * (smaller values) only the top 4 bits stick. So we use 0xbf 389 * instead to determine the number of writable bits. 390 */ 391 ctrl = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR); 392 if ((ctrl & GICD_CTLR_DS) == 0) { 393 __asm volatile("mrs %x0, "STR(ICC_CTLR_EL1) : "=r"(ctrl)); 394 nbits = ICC_CTLR_EL1_PRIBITS(ctrl) + 1; 395 __asm volatile("mrs %x0, "STR(ICC_PMR) : "=r"(oldpmr)); 396 __asm volatile("msr "STR(ICC_PMR)", %x0" :: "r"(0xbf)); 397 __asm volatile("mrs %x0, "STR(ICC_PMR) : "=r"(pmr)); 398 __asm volatile("msr "STR(ICC_PMR)", %x0" :: "r"(oldpmr)); 399 if (nbits == 8 - (ffs(pmr) - 1)) 400 sc->sc_pmr_shift--; 401 } 402 403 /* 404 * The Rockchip RK3399 is busted. Its GIC-500 treats all 405 * access to its memory mapped registers as "secure". As a 406 * result, several registers don't behave as expected. For 407 * example, the GICD_IPRIORITYRn and GICR_IPRIORITYRn 408 * registers expose the full priority range available to 409 * secure interrupts. We need to be aware of this and write 410 * an adjusted priority value into these registers. We also 411 * need to be careful not to touch any bits that shouldn't be 412 * writable in non-secure mode. 413 * 414 * We check whether we have secure mode access to these 415 * registers by attempting to write to the GICD_NSACR register 416 * and check whether its contents actually change. In that 417 * case we need to adjust the priorities we write into 418 * GICD_IPRIORITYRn and GICRIPRIORITYRn accordingly. 419 */ 420 oldnsacr = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_NSACR(32)); 421 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, GICD_NSACR(32), 422 oldnsacr ^ 0xffffffff); 423 nsacr = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_NSACR(32)); 424 if (nsacr != oldnsacr) { 425 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, GICD_NSACR(32), 426 oldnsacr); 427 sc->sc_rk3399_quirk = 1; 428 sc->sc_prio_shift--; 429 printf(" sec"); 430 } 431 432 printf(" shift %d:%d", sc->sc_prio_shift, sc->sc_pmr_shift); 433 434 evcount_attach(&sc->sc_spur, "irq1023/spur", NULL); 435 436 __asm volatile("msr "STR(ICC_SRE_EL1)", %x0" : : "r" (ICC_SRE_EL1_EN)); 437 __isb(); 438 439 nintr = 32 * (typer & GICD_TYPER_ITLINE_M); 440 nintr += 32; /* ICD_ICTR + 1, irq 0-31 is SGI, 32+ is PPI */ 441 sc->sc_nintr = nintr; 442 443 agintc_sc = sc; /* save this for global access */ 444 445 /* find the redistributors. */ 446 idx = 0; 447 offset = 0; 448 redist_stride = OF_getpropint64(faa->fa_node, "redistributor-stride", 0); 449 for (nredist = 0; idx < sc->sc_num_redist_regions; nredist++) { 450 uint64_t typer; 451 int32_t sz; 452 453 typer = bus_space_read_8(sc->sc_iot, sc->sc_rbase_ioh[idx], 454 offset + GICR_TYPER); 455 456 if (redist_stride == 0) { 457 sz = (64 * 1024 * 2); 458 if (typer & GICR_TYPER_VLPIS) 459 sz += (64 * 1024 * 2); 460 } else 461 sz = redist_stride; 462 463 #ifdef DEBUG_AGINTC 464 printf("probing redistributor %d %x\n", nredist, offset); 465 #endif 466 467 offset += sz; 468 if (offset >= faa->fa_reg[1 + idx].size || 469 typer & GICR_TYPER_LAST) { 470 offset = 0; 471 idx++; 472 } 473 } 474 475 sc->sc_num_redist = nredist; 476 printf(" nirq %d nredist %d", nintr, sc->sc_num_redist); 477 478 sc->sc_r_ioh = mallocarray(sc->sc_num_redist, 479 sizeof(*sc->sc_r_ioh), M_DEVBUF, M_WAITOK); 480 sc->sc_processor = mallocarray(sc->sc_num_redist, 481 sizeof(*sc->sc_processor), M_DEVBUF, M_WAITOK); 482 483 /* submap and configure the redistributors. */ 484 idx = 0; 485 offset = 0; 486 for (nredist = 0; nredist < sc->sc_num_redist; nredist++) { 487 uint64_t typer; 488 int32_t sz; 489 490 typer = bus_space_read_8(sc->sc_iot, sc->sc_rbase_ioh[idx], 491 offset + GICR_TYPER); 492 493 if (redist_stride == 0) { 494 sz = (64 * 1024 * 2); 495 if (typer & GICR_TYPER_VLPIS) 496 sz += (64 * 1024 * 2); 497 } else 498 sz = redist_stride; 499 500 affinity = bus_space_read_8(sc->sc_iot, 501 sc->sc_rbase_ioh[idx], offset + GICR_TYPER) >> 32; 502 CPU_INFO_FOREACH(cii, ci) { 503 if (affinity == (((ci->ci_mpidr >> 8) & 0xff000000) | 504 (ci->ci_mpidr & 0x00ffffff))) 505 break; 506 } 507 if (ci != NULL) 508 sc->sc_cpuremap[ci->ci_cpuid] = nredist; 509 510 sc->sc_processor[nredist] = bus_space_read_8(sc->sc_iot, 511 sc->sc_rbase_ioh[idx], offset + GICR_TYPER) >> 8; 512 513 bus_space_subregion(sc->sc_iot, sc->sc_rbase_ioh[idx], 514 offset, sz, &sc->sc_r_ioh[nredist]); 515 516 if (sc->sc_nlpi > 0) { 517 bus_space_write_8(sc->sc_iot, sc->sc_rbase_ioh[idx], 518 offset + GICR_PROPBASER, 519 AGINTC_DMA_DVA(sc->sc_prop) | 520 GICR_PROPBASER_ISH | GICR_PROPBASER_IC_NORM_NC | 521 fls(LPI_BASE + sc->sc_nlpi - 1) - 1); 522 bus_space_write_8(sc->sc_iot, sc->sc_rbase_ioh[idx], 523 offset + GICR_PENDBASER, 524 AGINTC_DMA_DVA(sc->sc_pend) | 525 GICR_PENDBASER_ISH | GICR_PENDBASER_IC_NORM_NC | 526 GICR_PENDBASER_PTZ); 527 bus_space_write_4(sc->sc_iot, sc->sc_rbase_ioh[idx], 528 offset + GICR_CTLR, GICR_CTLR_ENABLE_LPIS); 529 } 530 531 offset += sz; 532 if (offset >= faa->fa_reg[1 + idx].size || 533 typer & GICR_TYPER_LAST) { 534 offset = 0; 535 idx++; 536 } 537 } 538 539 /* Disable all interrupts, clear all pending */ 540 for (i = 1; i < nintr / 32; i++) { 541 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 542 GICD_ICACTIVER(i * 32), ~0); 543 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 544 GICD_ICENABLER(i * 32), ~0); 545 } 546 547 for (i = 4; i < nintr; i += 4) { 548 /* lowest priority ?? */ 549 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 550 GICD_IPRIORITYR(i), 0xffffffff); 551 } 552 553 /* Set all interrupts to G1NS */ 554 for (i = 1; i < nintr / 32; i++) { 555 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 556 GICD_IGROUPR(i * 32), ~0); 557 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 558 GICD_IGRPMODR(i * 32), 0); 559 } 560 561 for (i = 2; i < nintr / 16; i++) { 562 /* irq 32 - N */ 563 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 564 GICD_ICFGR(i * 16), 0); 565 } 566 567 agintc_cpuinit(); 568 569 sc->sc_handler = mallocarray(nintr, 570 sizeof(*sc->sc_handler), M_DEVBUF, M_ZERO | M_WAITOK); 571 for (i = 0; i < nintr; i++) 572 TAILQ_INIT(&sc->sc_handler[i].iq_list); 573 sc->sc_lpi = mallocarray(sc->sc_nlpi, 574 sizeof(*sc->sc_lpi), M_DEVBUF, M_ZERO | M_WAITOK); 575 576 /* set priority to IPL_HIGH until configure lowers to desired IPL */ 577 agintc_setipl(IPL_HIGH); 578 579 /* initialize all interrupts as disabled */ 580 agintc_calc_mask(); 581 582 /* insert self as interrupt handler */ 583 arm_set_intr_handler(agintc_splraise, agintc_spllower, agintc_splx, 584 agintc_setipl, agintc_irq_handler, NULL, 585 agintc_enable_wakeup, agintc_disable_wakeup); 586 587 /* enable interrupts */ 588 ctrl = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR); 589 bits = GICD_CTLR_ARE_NS | GICD_CTLR_EnableGrp1A | GICD_CTLR_EnableGrp1; 590 if (sc->sc_rk3399_quirk) { 591 bits &= ~GICD_CTLR_EnableGrp1A; 592 bits <<= 1; 593 } 594 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR, ctrl | bits); 595 596 __asm volatile("msr "STR(ICC_PMR)", %x0" :: "r"(0xff)); 597 __asm volatile("msr "STR(ICC_BPR1)", %x0" :: "r"(0)); 598 __asm volatile("msr "STR(ICC_IGRPEN1)", %x0" :: "r"(1)); 599 600 #ifdef MULTIPROCESSOR 601 /* setup IPI interrupts */ 602 603 /* 604 * Ideally we want three IPI interrupts, one for NOP, one for 605 * DDB and one for HALT. However we can survive if only one 606 * is available; it is possible that most are not available to 607 * the non-secure OS. 608 */ 609 nipi = 0; 610 for (i = 0; i < 16; i++) { 611 int hwcpu = sc->sc_cpuremap[cpu_number()]; 612 int reg, oldreg; 613 614 oldreg = bus_space_read_1(sc->sc_iot, sc->sc_r_ioh[hwcpu], 615 GICR_IPRIORITYR(i)); 616 bus_space_write_1(sc->sc_iot, sc->sc_r_ioh[hwcpu], 617 GICR_IPRIORITYR(i), oldreg ^ 0x20); 618 619 /* if this interrupt is not usable, pri will be unmodified */ 620 reg = bus_space_read_1(sc->sc_iot, sc->sc_r_ioh[hwcpu], 621 GICR_IPRIORITYR(i)); 622 if (reg == oldreg) 623 continue; 624 625 /* return to original value, will be set when used */ 626 bus_space_write_1(sc->sc_iot, sc->sc_r_ioh[hwcpu], 627 GICR_IPRIORITYR(i), oldreg); 628 629 if (nipi == 0) 630 printf(" ipi: %d", i); 631 else 632 printf(", %d", i); 633 ipiirq[nipi++] = i; 634 if (nipi == 3) 635 break; 636 } 637 638 if (nipi == 0) 639 panic("no irq available for IPI"); 640 641 switch (nipi) { 642 case 1: 643 sc->sc_ipi_irq[0] = agintc_intr_establish(ipiirq[0], 644 IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL, 645 agintc_ipi_combined, sc, "ipi"); 646 sc->sc_ipi_num[ARM_IPI_NOP] = ipiirq[0]; 647 sc->sc_ipi_num[ARM_IPI_DDB] = ipiirq[0]; 648 sc->sc_ipi_num[ARM_IPI_HALT] = ipiirq[0]; 649 break; 650 case 2: 651 sc->sc_ipi_irq[0] = agintc_intr_establish(ipiirq[0], 652 IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL, 653 agintc_ipi_nop, sc, "ipinop"); 654 sc->sc_ipi_num[ARM_IPI_NOP] = ipiirq[0]; 655 sc->sc_ipi_irq[1] = agintc_intr_establish(ipiirq[1], 656 IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL, 657 agintc_ipi_combined, sc, "ipi"); 658 sc->sc_ipi_num[ARM_IPI_DDB] = ipiirq[1]; 659 sc->sc_ipi_num[ARM_IPI_HALT] = ipiirq[1]; 660 break; 661 case 3: 662 sc->sc_ipi_irq[0] = agintc_intr_establish(ipiirq[0], 663 IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL, 664 agintc_ipi_nop, sc, "ipinop"); 665 sc->sc_ipi_num[ARM_IPI_NOP] = ipiirq[0]; 666 sc->sc_ipi_irq[1] = agintc_intr_establish(ipiirq[1], 667 IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL, 668 agintc_ipi_ddb, sc, "ipiddb"); 669 sc->sc_ipi_num[ARM_IPI_DDB] = ipiirq[1]; 670 sc->sc_ipi_irq[2] = agintc_intr_establish(ipiirq[2], 671 IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL, 672 agintc_ipi_halt, sc, "ipihalt"); 673 sc->sc_ipi_num[ARM_IPI_HALT] = ipiirq[2]; 674 break; 675 default: 676 panic("nipi unexpected number %d", nipi); 677 } 678 679 intr_send_ipi_func = agintc_send_ipi; 680 #endif 681 682 sc->sc_ic.ic_node = faa->fa_node; 683 sc->sc_ic.ic_cookie = self; 684 sc->sc_ic.ic_establish = agintc_intr_establish_fdt; 685 sc->sc_ic.ic_disestablish = agintc_intr_disestablish; 686 sc->sc_ic.ic_route = agintc_route_irq; 687 sc->sc_ic.ic_cpu_enable = agintc_cpuinit; 688 sc->sc_ic.ic_barrier = agintc_intr_barrier; 689 if (sc->sc_mbi_nranges > 0) 690 sc->sc_ic.ic_establish_msi = agintc_intr_establish_mbi; 691 sc->sc_ic.ic_set_wakeup = agintc_intr_set_wakeup; 692 arm_intr_register_fdt(&sc->sc_ic); 693 694 intr_restore(psw); 695 696 /* Attach ITS. */ 697 simplebus_attach(parent, &sc->sc_sbus.sc_dev, faa); 698 699 return; 700 701 unmap: 702 if (sc->sc_r_ioh) { 703 free(sc->sc_r_ioh, M_DEVBUF, 704 sc->sc_num_redist * sizeof(*sc->sc_r_ioh)); 705 } 706 if (sc->sc_processor) { 707 free(sc->sc_processor, M_DEVBUF, 708 sc->sc_num_redist * sizeof(*sc->sc_processor)); 709 } 710 711 if (sc->sc_pend) 712 agintc_dmamem_free(sc->sc_dmat, sc->sc_pend); 713 if (sc->sc_prop) 714 agintc_dmamem_free(sc->sc_dmat, sc->sc_prop); 715 716 for (idx = 0; idx < sc->sc_num_redist_regions; idx++) { 717 bus_space_unmap(sc->sc_iot, sc->sc_rbase_ioh[idx], 718 faa->fa_reg[1 + idx].size); 719 } 720 free(sc->sc_rbase_ioh, M_DEVBUF, 721 sc->sc_num_redist_regions * sizeof(*sc->sc_rbase_ioh)); 722 723 bus_space_unmap(sc->sc_iot, sc->sc_d_ioh, faa->fa_reg[0].size); 724 } 725 726 void 727 agintc_mbiinit(struct agintc_softc *sc, int node, bus_addr_t addr) 728 { 729 uint32_t *ranges; 730 int i, len; 731 732 if (OF_getproplen(node, "msi-controller") != 0) 733 return; 734 735 len = OF_getproplen(node, "mbi-ranges"); 736 if (len <= 0 || len % 2 * sizeof(uint32_t) != 0) 737 return; 738 739 ranges = malloc(len, M_TEMP, M_WAITOK); 740 OF_getpropintarray(node, "mbi-ranges", ranges, len); 741 742 sc->sc_mbi_nranges = len / (2 * sizeof(uint32_t)); 743 sc->sc_mbi_ranges = mallocarray(sc->sc_mbi_nranges, 744 sizeof(struct agintc_mbi_range), M_DEVBUF, M_WAITOK); 745 746 for (i = 0; i < sc->sc_mbi_nranges; i++) { 747 sc->sc_mbi_ranges[i].mr_base = ranges[2 * i + 0]; 748 sc->sc_mbi_ranges[i].mr_span = ranges[2 * i + 1]; 749 sc->sc_mbi_ranges[i].mr_mbi = 750 mallocarray(sc->sc_mbi_ranges[i].mr_span, 751 sizeof(void *), M_DEVBUF, M_WAITOK | M_ZERO); 752 } 753 754 free(ranges, M_TEMP, len); 755 756 addr = OF_getpropint64(node, "mbi-alias", addr); 757 sc->sc_mbi_addr = addr + GICD_SETSPI_NSR; 758 759 printf(" mbi"); 760 } 761 762 /* Initialize redistributors on each core. */ 763 void 764 agintc_cpuinit(void) 765 { 766 struct agintc_softc *sc = agintc_sc; 767 uint32_t waker; 768 int timeout = 100000; 769 int hwcpu; 770 int i; 771 772 hwcpu = sc->sc_cpuremap[cpu_number()]; 773 waker = bus_space_read_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 774 GICR_WAKER); 775 waker &= ~(GICR_WAKER_PROCESSORSLEEP); 776 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], GICR_WAKER, 777 waker); 778 779 do { 780 waker = bus_space_read_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 781 GICR_WAKER); 782 } while (--timeout && (waker & GICR_WAKER_CHILDRENASLEEP)); 783 if (timeout == 0) 784 printf("%s: waker timed out\n", __func__); 785 786 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 787 GICR_ICENABLE0, ~0); 788 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 789 GICR_ICPENDR0, ~0); 790 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 791 GICR_ICACTIVE0, ~0); 792 for (i = 0; i < 32; i += 4) { 793 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 794 GICR_IPRIORITYR(i), ~0); 795 } 796 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 797 GICR_IGROUPR0, ~0); 798 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 799 GICR_IGRPMODR0, 0); 800 801 if (sc->sc_ipi_irq[0] != NULL) 802 agintc_route_irq(sc->sc_ipi_irq[0], IRQ_ENABLE, curcpu()); 803 if (sc->sc_ipi_irq[1] != NULL) 804 agintc_route_irq(sc->sc_ipi_irq[1], IRQ_ENABLE, curcpu()); 805 if (sc->sc_ipi_irq[2] != NULL) 806 agintc_route_irq(sc->sc_ipi_irq[2], IRQ_ENABLE, curcpu()); 807 808 __asm volatile("msr "STR(ICC_PMR)", %x0" :: "r"(0xff)); 809 __asm volatile("msr "STR(ICC_BPR1)", %x0" :: "r"(0)); 810 __asm volatile("msr "STR(ICC_IGRPEN1)", %x0" :: "r"(1)); 811 intr_enable(); 812 } 813 814 void 815 agintc_set_priority(struct agintc_softc *sc, int irq, int ipl) 816 { 817 struct cpu_info *ci = curcpu(); 818 int hwcpu = sc->sc_cpuremap[ci->ci_cpuid]; 819 uint32_t prival; 820 821 prival = ((0xff - ipl) << sc->sc_prio_shift) & 0xff; 822 823 if (irq >= SPI_BASE) { 824 bus_space_write_1(sc->sc_iot, sc->sc_d_ioh, 825 GICD_IPRIORITYR(irq), prival); 826 } else { 827 /* only sets local redistributor */ 828 bus_space_write_1(sc->sc_iot, sc->sc_r_ioh[hwcpu], 829 GICR_IPRIORITYR(irq), prival); 830 } 831 } 832 833 void 834 agintc_setipl(int ipl) 835 { 836 struct agintc_softc *sc = agintc_sc; 837 struct cpu_info *ci = curcpu(); 838 u_long psw; 839 uint32_t prival; 840 841 /* disable here is only to keep hardware in sync with ci->ci_cpl */ 842 psw = intr_disable(); 843 ci->ci_cpl = ipl; 844 845 prival = ((0xff - ipl) << sc->sc_pmr_shift) & 0xff; 846 __asm volatile("msr "STR(ICC_PMR)", %x0" : : "r" (prival)); 847 __isb(); 848 849 intr_restore(psw); 850 } 851 852 void 853 agintc_enable_wakeup(void) 854 { 855 struct agintc_softc *sc = agintc_sc; 856 struct intrhand *ih; 857 uint8_t *prop; 858 int irq, wakeup; 859 860 for (irq = 0; irq < sc->sc_nintr; irq++) { 861 /* No handler? Disabled already. */ 862 if (TAILQ_EMPTY(&sc->sc_handler[irq].iq_list)) 863 continue; 864 /* Unless we're WAKEUP, disable. */ 865 wakeup = 0; 866 TAILQ_FOREACH(ih, &sc->sc_handler[irq].iq_list, ih_list) { 867 if (ih->ih_flags & IPL_WAKEUP) { 868 wakeup = 1; 869 break; 870 } 871 } 872 if (!wakeup) 873 agintc_intr_disable(sc, irq); 874 } 875 876 for (irq = 0; irq < sc->sc_nlpi; irq++) { 877 if (sc->sc_lpi[irq] == NULL) 878 continue; 879 ih = sc->sc_lpi[irq]->li_ih; 880 KASSERT(ih != NULL); 881 if (ih->ih_flags & IPL_WAKEUP) 882 continue; 883 prop = AGINTC_DMA_KVA(sc->sc_prop); 884 prop[irq] &= ~GICR_PROP_ENABLE; 885 /* Make globally visible. */ 886 cpu_dcache_wb_range((vaddr_t)&prop[irq], 887 sizeof(*prop)); 888 __asm volatile("dsb sy"); 889 /* Invalidate cache */ 890 agintc_msi_inv(sc->sc_lpi[irq]); 891 } 892 } 893 894 void 895 agintc_disable_wakeup(void) 896 { 897 struct agintc_softc *sc = agintc_sc; 898 struct intrhand *ih; 899 uint8_t *prop; 900 int irq, wakeup; 901 902 for (irq = 0; irq < sc->sc_nintr; irq++) { 903 /* No handler? Keep disabled. */ 904 if (TAILQ_EMPTY(&sc->sc_handler[irq].iq_list)) 905 continue; 906 /* WAKEUPs are already enabled. */ 907 wakeup = 0; 908 TAILQ_FOREACH(ih, &sc->sc_handler[irq].iq_list, ih_list) { 909 if (ih->ih_flags & IPL_WAKEUP) { 910 wakeup = 1; 911 break; 912 } 913 } 914 if (!wakeup) 915 agintc_intr_enable(sc, irq); 916 } 917 918 for (irq = 0; irq < sc->sc_nlpi; irq++) { 919 if (sc->sc_lpi[irq] == NULL) 920 continue; 921 ih = sc->sc_lpi[irq]->li_ih; 922 KASSERT(ih != NULL); 923 if (ih->ih_flags & IPL_WAKEUP) 924 continue; 925 prop = AGINTC_DMA_KVA(sc->sc_prop); 926 prop[irq] |= GICR_PROP_ENABLE; 927 /* Make globally visible. */ 928 cpu_dcache_wb_range((vaddr_t)&prop[irq], 929 sizeof(*prop)); 930 __asm volatile("dsb sy"); 931 /* Invalidate cache */ 932 agintc_msi_inv(sc->sc_lpi[irq]); 933 } 934 } 935 936 void 937 agintc_intr_enable(struct agintc_softc *sc, int irq) 938 { 939 struct cpu_info *ci = curcpu(); 940 int hwcpu = sc->sc_cpuremap[ci->ci_cpuid]; 941 int bit = 1 << IRQ_TO_REG32BIT(irq); 942 943 if (irq >= 32) { 944 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 945 GICD_ISENABLER(irq), bit); 946 } else { 947 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 948 GICR_ISENABLE0, bit); 949 } 950 } 951 952 void 953 agintc_intr_disable(struct agintc_softc *sc, int irq) 954 { 955 struct cpu_info *ci = curcpu(); 956 int hwcpu = sc->sc_cpuremap[ci->ci_cpuid]; 957 958 if (irq >= 32) { 959 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 960 GICD_ICENABLER(irq), 1 << IRQ_TO_REG32BIT(irq)); 961 } else { 962 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 963 GICR_ICENABLE0, 1 << IRQ_TO_REG32BIT(irq)); 964 } 965 } 966 967 void 968 agintc_intr_config(struct agintc_softc *sc, int irq, int type) 969 { 970 uint32_t reg; 971 972 /* Don't dare to change SGIs or PPIs (yet) */ 973 if (irq < 32) 974 return; 975 976 reg = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_ICFGR(irq)); 977 reg &= ~GICD_ICFGR_TRIG_MASK(irq); 978 if (type == IST_EDGE_RISING) 979 reg |= GICD_ICFGR_TRIG_EDGE(irq); 980 else 981 reg |= GICD_ICFGR_TRIG_LEVEL(irq); 982 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, GICD_ICFGR(irq), reg); 983 } 984 985 void 986 agintc_calc_mask(void) 987 { 988 struct agintc_softc *sc = agintc_sc; 989 int irq; 990 991 for (irq = 0; irq < sc->sc_nintr; irq++) 992 agintc_calc_irq(sc, irq); 993 } 994 995 void 996 agintc_calc_irq(struct agintc_softc *sc, int irq) 997 { 998 struct cpu_info *ci = sc->sc_handler[irq].iq_ci; 999 struct intrhand *ih; 1000 int max = IPL_NONE; 1001 int min = IPL_HIGH; 1002 1003 TAILQ_FOREACH(ih, &sc->sc_handler[irq].iq_list, ih_list) { 1004 if (ih->ih_ipl > max) 1005 max = ih->ih_ipl; 1006 1007 if (ih->ih_ipl < min) 1008 min = ih->ih_ipl; 1009 } 1010 1011 if (max == IPL_NONE) 1012 min = IPL_NONE; 1013 1014 if (sc->sc_handler[irq].iq_irq_max == max && 1015 sc->sc_handler[irq].iq_irq_min == min) 1016 return; 1017 1018 sc->sc_handler[irq].iq_irq_max = max; 1019 sc->sc_handler[irq].iq_irq_min = min; 1020 1021 #ifdef DEBUG_AGINTC 1022 if (min != IPL_NONE) 1023 printf("irq %d to block at %d %d \n", irq, max, min ); 1024 #endif 1025 /* Enable interrupts at lower levels, clear -> enable */ 1026 /* Set interrupt priority/enable */ 1027 if (min != IPL_NONE) { 1028 agintc_set_priority(sc, irq, min); 1029 agintc_route(sc, irq, IRQ_ENABLE, ci); 1030 agintc_intr_enable(sc, irq); 1031 } else { 1032 agintc_intr_disable(sc, irq); 1033 agintc_route(sc, irq, IRQ_DISABLE, ci); 1034 } 1035 } 1036 1037 void 1038 agintc_splx(int new) 1039 { 1040 struct cpu_info *ci = curcpu(); 1041 1042 if (ci->ci_ipending & arm_smask[new]) 1043 arm_do_pending_intr(new); 1044 1045 agintc_setipl(new); 1046 } 1047 1048 int 1049 agintc_spllower(int new) 1050 { 1051 struct cpu_info *ci = curcpu(); 1052 int old = ci->ci_cpl; 1053 1054 agintc_splx(new); 1055 return (old); 1056 } 1057 1058 int 1059 agintc_splraise(int new) 1060 { 1061 struct cpu_info *ci = curcpu(); 1062 int old = ci->ci_cpl; 1063 1064 /* 1065 * setipl must always be called because there is a race window 1066 * where the variable is updated before the mask is set 1067 * an interrupt occurs in that window without the mask always 1068 * being set, the hardware might not get updated on the next 1069 * splraise completely messing up spl protection. 1070 */ 1071 if (old > new) 1072 new = old; 1073 1074 agintc_setipl(new); 1075 return (old); 1076 } 1077 1078 uint32_t 1079 agintc_iack(void) 1080 { 1081 int irq; 1082 1083 __asm volatile("mrs %x0, "STR(ICC_IAR1) : "=r" (irq)); 1084 __asm volatile("dsb sy"); 1085 return irq; 1086 } 1087 1088 void 1089 agintc_route_irq(void *v, int enable, struct cpu_info *ci) 1090 { 1091 struct agintc_softc *sc = agintc_sc; 1092 struct intrhand *ih = v; 1093 1094 if (enable) { 1095 agintc_set_priority(sc, ih->ih_irq, 1096 sc->sc_handler[ih->ih_irq].iq_irq_min); 1097 agintc_route(sc, ih->ih_irq, IRQ_ENABLE, ci); 1098 agintc_intr_enable(sc, ih->ih_irq); 1099 } 1100 } 1101 1102 void 1103 agintc_route(struct agintc_softc *sc, int irq, int enable, struct cpu_info *ci) 1104 { 1105 /* XXX does not yet support 'participating node' */ 1106 if (irq >= 32) { 1107 #ifdef DEBUG_AGINTC 1108 printf("router %x irq %d val %016llx\n", GICD_IROUTER(irq), 1109 irq, ci->ci_mpidr & MPIDR_AFF); 1110 #endif 1111 bus_space_write_8(sc->sc_iot, sc->sc_d_ioh, 1112 GICD_IROUTER(irq), ci->ci_mpidr & MPIDR_AFF); 1113 } 1114 } 1115 1116 void 1117 agintc_intr_barrier(void *cookie) 1118 { 1119 struct intrhand *ih = cookie; 1120 1121 sched_barrier(ih->ih_ci); 1122 } 1123 1124 void 1125 agintc_run_handler(struct intrhand *ih, void *frame, int s) 1126 { 1127 void *arg; 1128 int handled; 1129 1130 #ifdef MULTIPROCESSOR 1131 int need_lock; 1132 1133 if (ih->ih_flags & IPL_MPSAFE) 1134 need_lock = 0; 1135 else 1136 need_lock = s < IPL_SCHED; 1137 1138 if (need_lock) 1139 KERNEL_LOCK(); 1140 #endif 1141 1142 if (ih->ih_arg) 1143 arg = ih->ih_arg; 1144 else 1145 arg = frame; 1146 1147 handled = ih->ih_func(arg); 1148 if (handled) 1149 ih->ih_count.ec_count++; 1150 1151 #ifdef MULTIPROCESSOR 1152 if (need_lock) 1153 KERNEL_UNLOCK(); 1154 #endif 1155 } 1156 1157 void 1158 agintc_irq_handler(void *frame) 1159 { 1160 struct agintc_softc *sc = agintc_sc; 1161 struct intrhand *ih; 1162 int irq, pri, s; 1163 1164 irq = agintc_iack(); 1165 1166 #ifdef DEBUG_AGINTC 1167 if (irq != 30) 1168 printf("irq %d fired\n", irq); 1169 else { 1170 static int cnt = 0; 1171 if ((cnt++ % 100) == 0) { 1172 printf("irq %d fired * _100\n", irq); 1173 #ifdef DDB 1174 db_enter(); 1175 #endif 1176 } 1177 } 1178 #endif 1179 1180 if (irq == 1023) { 1181 sc->sc_spur.ec_count++; 1182 return; 1183 } 1184 1185 if ((irq >= sc->sc_nintr && irq < LPI_BASE) || 1186 irq >= LPI_BASE + sc->sc_nlpi) { 1187 return; 1188 } 1189 1190 if (irq >= LPI_BASE) { 1191 if (sc->sc_lpi[irq - LPI_BASE] == NULL) 1192 return; 1193 ih = sc->sc_lpi[irq - LPI_BASE]->li_ih; 1194 KASSERT(ih != NULL); 1195 1196 s = agintc_splraise(ih->ih_ipl); 1197 intr_enable(); 1198 agintc_run_handler(ih, frame, s); 1199 intr_disable(); 1200 agintc_eoi(irq); 1201 1202 agintc_splx(s); 1203 return; 1204 } 1205 1206 pri = sc->sc_handler[irq].iq_irq_max; 1207 s = agintc_splraise(pri); 1208 intr_enable(); 1209 TAILQ_FOREACH(ih, &sc->sc_handler[irq].iq_list, ih_list) { 1210 agintc_run_handler(ih, frame, s); 1211 } 1212 intr_disable(); 1213 agintc_eoi(irq); 1214 1215 agintc_splx(s); 1216 } 1217 1218 void * 1219 agintc_intr_establish_fdt(void *cookie, int *cell, int level, 1220 struct cpu_info *ci, int (*func)(void *), void *arg, char *name) 1221 { 1222 struct agintc_softc *sc = agintc_sc; 1223 int irq; 1224 int type; 1225 1226 /* 2nd cell contains the interrupt number */ 1227 irq = cell[1]; 1228 1229 /* 1st cell contains type: 0 SPI (32-X), 1 PPI (16-31) */ 1230 if (cell[0] == 0) 1231 irq += SPI_BASE; 1232 else if (cell[0] == 1) 1233 irq += PPI_BASE; 1234 else 1235 panic("%s: bogus interrupt type", sc->sc_sbus.sc_dev.dv_xname); 1236 1237 /* SPIs are only active-high level or low-to-high edge */ 1238 if (cell[2] & 0x3) 1239 type = IST_EDGE_RISING; 1240 else 1241 type = IST_LEVEL_HIGH; 1242 1243 return agintc_intr_establish(irq, type, level, ci, func, arg, name); 1244 } 1245 1246 void * 1247 agintc_intr_establish(int irqno, int type, int level, struct cpu_info *ci, 1248 int (*func)(void *), void *arg, char *name) 1249 { 1250 struct agintc_softc *sc = agintc_sc; 1251 struct intrhand *ih; 1252 u_long psw; 1253 1254 if (irqno < 0 || (irqno >= sc->sc_nintr && irqno < LPI_BASE) || 1255 irqno >= LPI_BASE + sc->sc_nlpi) 1256 panic("agintc_intr_establish: bogus irqnumber %d: %s", 1257 irqno, name); 1258 1259 if (ci == NULL) 1260 ci = &cpu_info_primary; 1261 1262 ih = malloc(sizeof *ih, M_DEVBUF, M_WAITOK); 1263 ih->ih_func = func; 1264 ih->ih_arg = arg; 1265 ih->ih_ipl = level & IPL_IRQMASK; 1266 ih->ih_flags = level & IPL_FLAGMASK; 1267 ih->ih_irq = irqno; 1268 ih->ih_name = name; 1269 ih->ih_ci = ci; 1270 1271 psw = intr_disable(); 1272 1273 if (irqno < LPI_BASE) { 1274 if (!TAILQ_EMPTY(&sc->sc_handler[irqno].iq_list) && 1275 sc->sc_handler[irqno].iq_ci != ci) { 1276 intr_restore(psw); 1277 free(ih, M_DEVBUF, sizeof *ih); 1278 return NULL; 1279 } 1280 TAILQ_INSERT_TAIL(&sc->sc_handler[irqno].iq_list, ih, ih_list); 1281 sc->sc_handler[irqno].iq_ci = ci; 1282 } 1283 1284 if (name != NULL) 1285 evcount_attach(&ih->ih_count, name, &ih->ih_irq); 1286 1287 #ifdef DEBUG_AGINTC 1288 printf("%s: irq %d level %d [%s]\n", __func__, irqno, level, name); 1289 #endif 1290 1291 if (irqno < LPI_BASE) { 1292 agintc_intr_config(sc, irqno, type); 1293 agintc_calc_irq(sc, irqno); 1294 } else { 1295 uint8_t *prop = AGINTC_DMA_KVA(sc->sc_prop); 1296 1297 prop[irqno - LPI_BASE] = (((0xff - ih->ih_ipl) << 4) & 0xff) | 1298 GICR_PROP_GROUP1 | GICR_PROP_ENABLE; 1299 1300 /* Make globally visible. */ 1301 cpu_dcache_wb_range((vaddr_t)&prop[irqno - LPI_BASE], 1302 sizeof(*prop)); 1303 __asm volatile("dsb sy"); 1304 } 1305 1306 intr_restore(psw); 1307 return (ih); 1308 } 1309 1310 void 1311 agintc_intr_disestablish(void *cookie) 1312 { 1313 struct agintc_softc *sc = agintc_sc; 1314 struct intrhand *ih = cookie; 1315 int irqno = ih->ih_irq; 1316 u_long psw; 1317 struct agintc_mbi_range *mr; 1318 int i; 1319 1320 psw = intr_disable(); 1321 1322 if (irqno < LPI_BASE) { 1323 TAILQ_REMOVE(&sc->sc_handler[irqno].iq_list, ih, ih_list); 1324 agintc_calc_irq(sc, irqno); 1325 1326 /* In case this is an MBI, free it */ 1327 for (i = 0; i < sc->sc_mbi_nranges; i++) { 1328 mr = &sc->sc_mbi_ranges[i]; 1329 if (irqno < mr->mr_base) 1330 continue; 1331 if (irqno >= mr->mr_base + mr->mr_span) 1332 break; 1333 if (mr->mr_mbi[irqno - mr->mr_base] != NULL) 1334 mr->mr_mbi[irqno - mr->mr_base] = NULL; 1335 } 1336 } else { 1337 uint8_t *prop = AGINTC_DMA_KVA(sc->sc_prop); 1338 1339 prop[irqno - LPI_BASE] = 0; 1340 1341 /* Make globally visible. */ 1342 cpu_dcache_wb_range((vaddr_t)&prop[irqno - LPI_BASE], 1343 sizeof(*prop)); 1344 __asm volatile("dsb sy"); 1345 } 1346 1347 if (ih->ih_name != NULL) 1348 evcount_detach(&ih->ih_count); 1349 1350 intr_restore(psw); 1351 1352 free(ih, M_DEVBUF, 0); 1353 } 1354 1355 void 1356 agintc_intr_set_wakeup(void *cookie) 1357 { 1358 struct intrhand *ih = cookie; 1359 1360 ih->ih_flags |= IPL_WAKEUP; 1361 } 1362 1363 void * 1364 agintc_intr_establish_mbi(void *self, uint64_t *addr, uint64_t *data, 1365 int level, struct cpu_info *ci, int (*func)(void *), void *arg, char *name) 1366 { 1367 struct agintc_softc *sc = agintc_sc; 1368 struct agintc_mbi_range *mr; 1369 void *cookie; 1370 int i, j, hwcpu; 1371 1372 if (ci == NULL) 1373 ci = &cpu_info_primary; 1374 hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid]; 1375 1376 for (i = 0; i < sc->sc_mbi_nranges; i++) { 1377 mr = &sc->sc_mbi_ranges[i]; 1378 for (j = 0; j < mr->mr_span; j++) { 1379 if (mr->mr_mbi[j] != NULL) 1380 continue; 1381 1382 cookie = agintc_intr_establish(mr->mr_base + j, 1383 IST_EDGE_RISING, level, ci, func, arg, name); 1384 if (cookie == NULL) 1385 return NULL; 1386 1387 *addr = sc->sc_mbi_addr; 1388 *data = mr->mr_base + j; 1389 1390 mr->mr_mbi[j] = cookie; 1391 return cookie; 1392 } 1393 } 1394 1395 return NULL; 1396 } 1397 1398 void 1399 agintc_eoi(uint32_t eoi) 1400 { 1401 __asm volatile("msr "STR(ICC_EOIR1)", %x0" :: "r" (eoi)); 1402 __isb(); 1403 } 1404 1405 void 1406 agintc_d_wait_rwp(struct agintc_softc *sc) 1407 { 1408 int count = 100000; 1409 uint32_t v; 1410 1411 do { 1412 v = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR); 1413 } while (--count && (v & GICD_CTLR_RWP)); 1414 1415 if (count == 0) 1416 panic("%s: RWP timed out 0x08%x", __func__, v); 1417 } 1418 1419 void 1420 agintc_r_wait_rwp(struct agintc_softc *sc) 1421 { 1422 struct cpu_info *ci = curcpu(); 1423 int hwcpu = sc->sc_cpuremap[ci->ci_cpuid]; 1424 int count = 100000; 1425 uint32_t v; 1426 1427 do { 1428 v = bus_space_read_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 1429 GICR_CTLR); 1430 } while (--count && (v & GICR_CTLR_RWP)); 1431 1432 if (count == 0) 1433 panic("%s: RWP timed out 0x08%x", __func__, v); 1434 } 1435 1436 #ifdef MULTIPROCESSOR 1437 int 1438 agintc_ipi_ddb(void *v) 1439 { 1440 /* XXX */ 1441 #ifdef DDB 1442 db_enter(); 1443 #endif 1444 return 1; 1445 } 1446 1447 int 1448 agintc_ipi_halt(void *v) 1449 { 1450 struct agintc_softc *sc = v; 1451 int old = curcpu()->ci_cpl; 1452 1453 intr_disable(); 1454 agintc_eoi(sc->sc_ipi_num[ARM_IPI_HALT]); 1455 agintc_setipl(IPL_NONE); 1456 1457 cpu_halt(); 1458 1459 agintc_setipl(old); 1460 intr_enable(); 1461 return 1; 1462 } 1463 1464 int 1465 agintc_ipi_nop(void *v) 1466 { 1467 /* Nothing to do here, just enough to wake up from WFI */ 1468 return 1; 1469 } 1470 1471 int 1472 agintc_ipi_combined(void *v) 1473 { 1474 struct agintc_softc *sc = v; 1475 1476 if (sc->sc_ipi_reason[cpu_number()] == ARM_IPI_DDB) { 1477 sc->sc_ipi_reason[cpu_number()] = ARM_IPI_NOP; 1478 return agintc_ipi_ddb(v); 1479 } else if (sc->sc_ipi_reason[cpu_number()] == ARM_IPI_HALT) { 1480 sc->sc_ipi_reason[cpu_number()] = ARM_IPI_NOP; 1481 return agintc_ipi_halt(v); 1482 } else { 1483 return agintc_ipi_nop(v); 1484 } 1485 } 1486 1487 void 1488 agintc_send_ipi(struct cpu_info *ci, int id) 1489 { 1490 struct agintc_softc *sc = agintc_sc; 1491 uint64_t sendmask; 1492 1493 if (ci == curcpu() && id == ARM_IPI_NOP) 1494 return; 1495 1496 /* never overwrite IPI_DDB or IPI_HALT with IPI_NOP */ 1497 if (id == ARM_IPI_DDB || id == ARM_IPI_HALT) 1498 sc->sc_ipi_reason[ci->ci_cpuid] = id; 1499 1500 /* will only send 1 cpu */ 1501 sendmask = (ci->ci_mpidr & MPIDR_AFF3) << 16; 1502 sendmask |= (ci->ci_mpidr & MPIDR_AFF2) << 16; 1503 sendmask |= (ci->ci_mpidr & MPIDR_AFF1) << 8; 1504 sendmask |= 1 << (ci->ci_mpidr & 0x0f); 1505 sendmask |= (sc->sc_ipi_num[id] << 24); 1506 1507 __asm volatile ("msr " STR(ICC_SGI1R)", %x0" ::"r"(sendmask)); 1508 } 1509 #endif 1510 1511 /* 1512 * GICv3 ITS controller for MSI interrupts. 1513 */ 1514 #define GITS_CTLR 0x0000 1515 #define GITS_CTLR_ENABLED (1UL << 0) 1516 #define GITS_TYPER 0x0008 1517 #define GITS_TYPER_CIL (1ULL << 36) 1518 #define GITS_TYPER_CIDBITS(x) (((x) >> 32) & 0xf) 1519 #define GITS_TYPER_HCC(x) (((x) >> 24) & 0xff) 1520 #define GITS_TYPER_PTA (1ULL << 19) 1521 #define GITS_TYPER_DEVBITS(x) (((x) >> 13) & 0x1f) 1522 #define GITS_TYPER_ITE_SZ(x) (((x) >> 4) & 0xf) 1523 #define GITS_TYPER_PHYS (1ULL << 0) 1524 #define GITS_CBASER 0x0080 1525 #define GITS_CBASER_VALID (1ULL << 63) 1526 #define GITS_CBASER_IC_NORM_NC (1ULL << 59) 1527 #define GITS_CBASER_MASK 0x1ffffffffff000ULL 1528 #define GITS_CWRITER 0x0088 1529 #define GITS_CREADR 0x0090 1530 #define GITS_BASER(i) (0x0100 + ((i) * 8)) 1531 #define GITS_BASER_VALID (1ULL << 63) 1532 #define GITS_BASER_INDIRECT (1ULL << 62) 1533 #define GITS_BASER_IC_NORM_NC (1ULL << 59) 1534 #define GITS_BASER_TYPE_MASK (7ULL << 56) 1535 #define GITS_BASER_TYPE_DEVICE (1ULL << 56) 1536 #define GITS_BASER_TYPE_COLL (4ULL << 56) 1537 #define GITS_BASER_TTE_SZ(x) (((x) >> 48) & 0x1f) 1538 #define GITS_BASER_PGSZ_MASK (3ULL << 8) 1539 #define GITS_BASER_PGSZ_4K (0ULL << 8) 1540 #define GITS_BASER_PGSZ_16K (1ULL << 8) 1541 #define GITS_BASER_PGSZ_64K (2ULL << 8) 1542 #define GITS_BASER_SZ_MASK (0xffULL) 1543 #define GITS_BASER_PA_MASK 0x7ffffffff000ULL 1544 #define GITS_TRANSLATER 0x10040 1545 1546 #define GITS_NUM_BASER 8 1547 1548 struct gits_cmd { 1549 uint8_t cmd; 1550 uint32_t deviceid; 1551 uint32_t eventid; 1552 uint32_t intid; 1553 uint64_t dw2; 1554 uint64_t dw3; 1555 }; 1556 1557 #define GITS_CMD_VALID (1ULL << 63) 1558 1559 /* ITS commands */ 1560 #define SYNC 0x05 1561 #define MAPD 0x08 1562 #define MAPC 0x09 1563 #define MAPTI 0x0a 1564 #define INV 0x0c 1565 #define INVALL 0x0d 1566 #define DISCARD 0x0f 1567 1568 #define GITS_CMDQ_SIZE (64 * 1024) 1569 #define GITS_CMDQ_NENTRIES (GITS_CMDQ_SIZE / sizeof(struct gits_cmd)) 1570 1571 struct agintc_msi_device { 1572 LIST_ENTRY(agintc_msi_device) md_list; 1573 1574 uint32_t md_deviceid; 1575 uint32_t md_events; 1576 struct agintc_dmamem *md_itt; 1577 }; 1578 1579 int agintc_msi_match(struct device *, void *, void *); 1580 void agintc_msi_attach(struct device *, struct device *, void *); 1581 void *agintc_intr_establish_msi(void *, uint64_t *, uint64_t *, 1582 int , struct cpu_info *, int (*)(void *), void *, char *); 1583 void agintc_intr_disestablish_msi(void *); 1584 void agintc_intr_barrier_msi(void *); 1585 1586 struct agintc_msi_softc { 1587 struct device sc_dev; 1588 bus_space_tag_t sc_iot; 1589 bus_space_handle_t sc_ioh; 1590 bus_dma_tag_t sc_dmat; 1591 1592 bus_addr_t sc_msi_addr; 1593 int sc_msi_delta; 1594 1595 struct agintc_dmamem *sc_cmdq; 1596 uint16_t sc_cmdidx; 1597 1598 int sc_devbits; 1599 uint32_t sc_deviceid_max; 1600 struct agintc_dmamem *sc_dtt; 1601 size_t sc_dtt_pgsz; 1602 uint8_t sc_dte_sz; 1603 int sc_dtt_indirect; 1604 int sc_cidbits; 1605 struct agintc_dmamem *sc_ctt; 1606 size_t sc_ctt_pgsz; 1607 uint8_t sc_cte_sz; 1608 uint8_t sc_ite_sz; 1609 1610 LIST_HEAD(, agintc_msi_device) sc_msi_devices; 1611 1612 struct interrupt_controller sc_ic; 1613 }; 1614 1615 const struct cfattach agintcmsi_ca = { 1616 sizeof (struct agintc_msi_softc), agintc_msi_match, agintc_msi_attach 1617 }; 1618 1619 struct cfdriver agintcmsi_cd = { 1620 NULL, "agintcmsi", DV_DULL 1621 }; 1622 1623 void agintc_msi_send_cmd(struct agintc_msi_softc *, struct gits_cmd *); 1624 void agintc_msi_wait_cmd(struct agintc_msi_softc *); 1625 1626 #define CPU_IMPL(midr) (((midr) >> 24) & 0xff) 1627 #define CPU_PART(midr) (((midr) >> 4) & 0xfff) 1628 1629 #define CPU_IMPL_QCOM 0x51 1630 #define CPU_PART_ORYON 0x001 1631 1632 int 1633 agintc_msi_match(struct device *parent, void *cfdata, void *aux) 1634 { 1635 struct fdt_attach_args *faa = aux; 1636 1637 /* 1638 * XXX For some reason MSIs don't work on Qualcomm X1E SoCs in 1639 * ACPI mode. So skip attaching the ITS in that case. MSIs 1640 * work fine when booting with a DTB. 1641 */ 1642 if (OF_is_compatible(OF_peer(0), "openbsd,acpi") && 1643 CPU_IMPL(curcpu()->ci_midr) == CPU_IMPL_QCOM && 1644 CPU_PART(curcpu()->ci_midr) == CPU_PART_ORYON) 1645 return 0; 1646 1647 return OF_is_compatible(faa->fa_node, "arm,gic-v3-its"); 1648 } 1649 1650 void 1651 agintc_msi_attach(struct device *parent, struct device *self, void *aux) 1652 { 1653 struct agintc_msi_softc *sc = (struct agintc_msi_softc *)self; 1654 struct fdt_attach_args *faa = aux; 1655 struct gits_cmd cmd; 1656 uint32_t pre_its[2]; 1657 uint64_t typer; 1658 int i, hwcpu; 1659 1660 if (faa->fa_nreg < 1) { 1661 printf(": no registers\n"); 1662 return; 1663 } 1664 1665 sc->sc_iot = faa->fa_iot; 1666 if (bus_space_map(sc->sc_iot, faa->fa_reg[0].addr, 1667 faa->fa_reg[0].size, 0, &sc->sc_ioh)) { 1668 printf(": can't map registers\n"); 1669 return; 1670 } 1671 sc->sc_dmat = faa->fa_dmat; 1672 1673 sc->sc_msi_addr = faa->fa_reg[0].addr + GITS_TRANSLATER; 1674 if (OF_getpropintarray(faa->fa_node, "socionext,synquacer-pre-its", 1675 pre_its, sizeof(pre_its)) == sizeof(pre_its)) { 1676 sc->sc_msi_addr = pre_its[0]; 1677 sc->sc_msi_delta = 4; 1678 } 1679 1680 typer = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_TYPER); 1681 if ((typer & GITS_TYPER_PHYS) == 0 || typer & GITS_TYPER_PTA) { 1682 printf(": unsupported type 0x%016llx\n", typer); 1683 goto unmap; 1684 } 1685 sc->sc_ite_sz = GITS_TYPER_ITE_SZ(typer) + 1; 1686 sc->sc_devbits = GITS_TYPER_DEVBITS(typer) + 1; 1687 if (typer & GITS_TYPER_CIL) 1688 sc->sc_cidbits = GITS_TYPER_CIDBITS(typer) + 1; 1689 else 1690 sc->sc_cidbits = 16; 1691 1692 /* Set up command queue. */ 1693 sc->sc_cmdq = agintc_dmamem_alloc(sc->sc_dmat, 1694 GITS_CMDQ_SIZE, GITS_CMDQ_SIZE); 1695 if (sc->sc_cmdq == NULL) { 1696 printf(": can't alloc command queue\n"); 1697 goto unmap; 1698 } 1699 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_CBASER, 1700 AGINTC_DMA_DVA(sc->sc_cmdq) | GITS_CBASER_IC_NORM_NC | 1701 (GITS_CMDQ_SIZE / PAGE_SIZE) - 1 | GITS_CBASER_VALID); 1702 1703 /* Set up device translation table. */ 1704 for (i = 0; i < GITS_NUM_BASER; i++) { 1705 uint64_t baser; 1706 paddr_t dtt_pa; 1707 size_t size; 1708 1709 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1710 if ((baser & GITS_BASER_TYPE_MASK) != GITS_BASER_TYPE_DEVICE) 1711 continue; 1712 1713 /* Determine the maximum supported page size. */ 1714 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1715 (baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_64K); 1716 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1717 if ((baser & GITS_BASER_PGSZ_MASK) == GITS_BASER_PGSZ_64K) 1718 goto dfound; 1719 1720 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1721 (baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_16K); 1722 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1723 if ((baser & GITS_BASER_PGSZ_MASK) == GITS_BASER_PGSZ_16K) 1724 goto dfound; 1725 1726 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1727 (baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_4K); 1728 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1729 1730 dfound: 1731 switch (baser & GITS_BASER_PGSZ_MASK) { 1732 case GITS_BASER_PGSZ_4K: 1733 sc->sc_dtt_pgsz = PAGE_SIZE; 1734 break; 1735 case GITS_BASER_PGSZ_16K: 1736 sc->sc_dtt_pgsz = 4 * PAGE_SIZE; 1737 break; 1738 case GITS_BASER_PGSZ_64K: 1739 sc->sc_dtt_pgsz = 16 * PAGE_SIZE; 1740 break; 1741 } 1742 1743 /* Calculate table size. */ 1744 sc->sc_dte_sz = GITS_BASER_TTE_SZ(baser) + 1; 1745 size = (1ULL << sc->sc_devbits) * sc->sc_dte_sz; 1746 size = roundup(size, sc->sc_dtt_pgsz); 1747 1748 /* Might make sense to go indirect */ 1749 if (size > 2 * sc->sc_dtt_pgsz) { 1750 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1751 baser | GITS_BASER_INDIRECT); 1752 if (bus_space_read_8(sc->sc_iot, sc->sc_ioh, 1753 GITS_BASER(i)) & GITS_BASER_INDIRECT) 1754 sc->sc_dtt_indirect = 1; 1755 } 1756 if (sc->sc_dtt_indirect) { 1757 size = (1ULL << sc->sc_devbits); 1758 size /= (sc->sc_dtt_pgsz / sc->sc_dte_sz); 1759 size *= sizeof(uint64_t); 1760 size = roundup(size, sc->sc_dtt_pgsz); 1761 } 1762 1763 /* Clamp down to maximum configurable num pages */ 1764 if (size / sc->sc_dtt_pgsz > GITS_BASER_SZ_MASK + 1) 1765 size = (GITS_BASER_SZ_MASK + 1) * sc->sc_dtt_pgsz; 1766 1767 /* Calculate max deviceid based off configured size */ 1768 sc->sc_deviceid_max = (size / sc->sc_dte_sz) - 1; 1769 if (sc->sc_dtt_indirect) 1770 sc->sc_deviceid_max = ((size / sizeof(uint64_t)) * 1771 (sc->sc_dtt_pgsz / sc->sc_dte_sz)) - 1; 1772 1773 /* Allocate table. */ 1774 sc->sc_dtt = agintc_dmamem_alloc(sc->sc_dmat, 1775 size, sc->sc_dtt_pgsz); 1776 if (sc->sc_dtt == NULL) { 1777 printf(": can't alloc translation table\n"); 1778 goto unmap; 1779 } 1780 1781 /* Configure table. */ 1782 dtt_pa = AGINTC_DMA_DVA(sc->sc_dtt); 1783 KASSERT((dtt_pa & GITS_BASER_PA_MASK) == dtt_pa); 1784 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1785 GITS_BASER_IC_NORM_NC | baser & GITS_BASER_PGSZ_MASK | 1786 dtt_pa | (size / sc->sc_dtt_pgsz) - 1 | 1787 (sc->sc_dtt_indirect ? GITS_BASER_INDIRECT : 0) | 1788 GITS_BASER_VALID); 1789 } 1790 1791 /* Set up collection translation table. */ 1792 for (i = 0; i < GITS_NUM_BASER; i++) { 1793 uint64_t baser; 1794 paddr_t ctt_pa; 1795 size_t size; 1796 1797 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1798 if ((baser & GITS_BASER_TYPE_MASK) != GITS_BASER_TYPE_COLL) 1799 continue; 1800 1801 /* Determine the maximum supported page size. */ 1802 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1803 (baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_64K); 1804 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1805 if ((baser & GITS_BASER_PGSZ_MASK) == GITS_BASER_PGSZ_64K) 1806 goto cfound; 1807 1808 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1809 (baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_16K); 1810 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1811 if ((baser & GITS_BASER_PGSZ_MASK) == GITS_BASER_PGSZ_16K) 1812 goto cfound; 1813 1814 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1815 (baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_4K); 1816 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1817 1818 cfound: 1819 switch (baser & GITS_BASER_PGSZ_MASK) { 1820 case GITS_BASER_PGSZ_4K: 1821 sc->sc_ctt_pgsz = PAGE_SIZE; 1822 break; 1823 case GITS_BASER_PGSZ_16K: 1824 sc->sc_ctt_pgsz = 4 * PAGE_SIZE; 1825 break; 1826 case GITS_BASER_PGSZ_64K: 1827 sc->sc_ctt_pgsz = 16 * PAGE_SIZE; 1828 break; 1829 } 1830 1831 /* Calculate table size. */ 1832 sc->sc_cte_sz = GITS_BASER_TTE_SZ(baser) + 1; 1833 size = (1ULL << sc->sc_cidbits) * sc->sc_cte_sz; 1834 size = roundup(size, sc->sc_ctt_pgsz); 1835 1836 /* Allocate table. */ 1837 sc->sc_ctt = agintc_dmamem_alloc(sc->sc_dmat, 1838 size, sc->sc_ctt_pgsz); 1839 if (sc->sc_ctt == NULL) { 1840 printf(": can't alloc translation table\n"); 1841 goto unmap; 1842 } 1843 1844 /* Configure table. */ 1845 ctt_pa = AGINTC_DMA_DVA(sc->sc_ctt); 1846 KASSERT((ctt_pa & GITS_BASER_PA_MASK) == ctt_pa); 1847 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1848 GITS_BASER_IC_NORM_NC | baser & GITS_BASER_PGSZ_MASK | 1849 ctt_pa | (size / sc->sc_ctt_pgsz) - 1 | GITS_BASER_VALID); 1850 } 1851 1852 /* Enable ITS. */ 1853 bus_space_write_4(sc->sc_iot, sc->sc_ioh, GITS_CTLR, 1854 GITS_CTLR_ENABLED); 1855 1856 LIST_INIT(&sc->sc_msi_devices); 1857 1858 /* Create one collection per core. */ 1859 KASSERT(ncpus <= agintc_sc->sc_num_redist); 1860 for (i = 0; i < ncpus; i++) { 1861 hwcpu = agintc_sc->sc_cpuremap[i]; 1862 memset(&cmd, 0, sizeof(cmd)); 1863 cmd.cmd = MAPC; 1864 cmd.dw2 = GITS_CMD_VALID | 1865 (agintc_sc->sc_processor[hwcpu] << 16) | i; 1866 agintc_msi_send_cmd(sc, &cmd); 1867 agintc_msi_wait_cmd(sc); 1868 } 1869 1870 printf("\n"); 1871 1872 sc->sc_ic.ic_node = faa->fa_node; 1873 sc->sc_ic.ic_cookie = sc; 1874 sc->sc_ic.ic_establish_msi = agintc_intr_establish_msi; 1875 sc->sc_ic.ic_disestablish = agintc_intr_disestablish_msi; 1876 sc->sc_ic.ic_barrier = agintc_intr_barrier_msi; 1877 sc->sc_ic.ic_gic_its_id = OF_getpropint(faa->fa_node, 1878 "openbsd,gic-its-id", 0); 1879 arm_intr_register_fdt(&sc->sc_ic); 1880 return; 1881 1882 unmap: 1883 if (sc->sc_dtt) 1884 agintc_dmamem_free(sc->sc_dmat, sc->sc_dtt); 1885 if (sc->sc_cmdq) 1886 agintc_dmamem_free(sc->sc_dmat, sc->sc_cmdq); 1887 1888 bus_space_unmap(sc->sc_iot, sc->sc_ioh, faa->fa_reg[0].size); 1889 } 1890 1891 void 1892 agintc_msi_send_cmd(struct agintc_msi_softc *sc, struct gits_cmd *cmd) 1893 { 1894 struct gits_cmd *queue = AGINTC_DMA_KVA(sc->sc_cmdq); 1895 1896 memcpy(&queue[sc->sc_cmdidx], cmd, sizeof(*cmd)); 1897 1898 /* Make globally visible. */ 1899 cpu_dcache_wb_range((vaddr_t)&queue[sc->sc_cmdidx], sizeof(*cmd)); 1900 __asm volatile("dsb sy"); 1901 1902 sc->sc_cmdidx++; 1903 sc->sc_cmdidx %= GITS_CMDQ_NENTRIES; 1904 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_CWRITER, 1905 sc->sc_cmdidx * sizeof(*cmd)); 1906 } 1907 1908 void 1909 agintc_msi_wait_cmd(struct agintc_msi_softc *sc) 1910 { 1911 uint64_t creadr; 1912 int timo; 1913 1914 for (timo = 1000; timo > 0; timo--) { 1915 creadr = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_CREADR); 1916 if (creadr == sc->sc_cmdidx * sizeof(struct gits_cmd)) 1917 break; 1918 delay(1); 1919 } 1920 if (timo == 0) 1921 printf("%s: command queue timeout\n", sc->sc_dev.dv_xname); 1922 } 1923 1924 int 1925 agintc_msi_create_device_table(struct agintc_msi_softc *sc, uint32_t deviceid) 1926 { 1927 uint64_t *table = AGINTC_DMA_KVA(sc->sc_dtt); 1928 uint32_t idx = deviceid / (sc->sc_dtt_pgsz / sc->sc_dte_sz); 1929 struct agintc_dmamem *dtt; 1930 paddr_t dtt_pa; 1931 1932 /* Out of bounds */ 1933 if (deviceid > sc->sc_deviceid_max) 1934 return ENXIO; 1935 1936 /* No need to adjust */ 1937 if (!sc->sc_dtt_indirect) 1938 return 0; 1939 1940 /* Table already allocated */ 1941 if (table[idx]) 1942 return 0; 1943 1944 /* FIXME: leaks */ 1945 dtt = agintc_dmamem_alloc(sc->sc_dmat, 1946 sc->sc_dtt_pgsz, sc->sc_dtt_pgsz); 1947 if (dtt == NULL) 1948 return ENOMEM; 1949 1950 dtt_pa = AGINTC_DMA_DVA(dtt); 1951 KASSERT((dtt_pa & GITS_BASER_PA_MASK) == dtt_pa); 1952 table[idx] = dtt_pa | GITS_BASER_VALID; 1953 cpu_dcache_wb_range((vaddr_t)&table[idx], sizeof(table[idx])); 1954 __asm volatile("dsb sy"); 1955 return 0; 1956 } 1957 1958 struct agintc_msi_device * 1959 agintc_msi_create_device(struct agintc_msi_softc *sc, uint32_t deviceid) 1960 { 1961 struct agintc_msi_device *md; 1962 struct gits_cmd cmd; 1963 1964 if (deviceid > sc->sc_deviceid_max) 1965 return NULL; 1966 1967 if (agintc_msi_create_device_table(sc, deviceid) != 0) 1968 return NULL; 1969 1970 md = malloc(sizeof(*md), M_DEVBUF, M_ZERO | M_WAITOK); 1971 md->md_deviceid = deviceid; 1972 md->md_itt = agintc_dmamem_alloc(sc->sc_dmat, 1973 32 * sc->sc_ite_sz, PAGE_SIZE); 1974 LIST_INSERT_HEAD(&sc->sc_msi_devices, md, md_list); 1975 1976 memset(&cmd, 0, sizeof(cmd)); 1977 cmd.cmd = MAPD; 1978 cmd.deviceid = deviceid; 1979 cmd.eventid = 4; /* size */ 1980 cmd.dw2 = AGINTC_DMA_DVA(md->md_itt) | GITS_CMD_VALID; 1981 agintc_msi_send_cmd(sc, &cmd); 1982 agintc_msi_wait_cmd(sc); 1983 1984 return md; 1985 } 1986 1987 struct agintc_msi_device * 1988 agintc_msi_find_device(struct agintc_msi_softc *sc, uint32_t deviceid) 1989 { 1990 struct agintc_msi_device *md; 1991 1992 LIST_FOREACH(md, &sc->sc_msi_devices, md_list) { 1993 if (md->md_deviceid == deviceid) 1994 return md; 1995 } 1996 1997 return agintc_msi_create_device(sc, deviceid); 1998 } 1999 2000 void 2001 agintc_msi_discard(struct agintc_lpi_info *li) 2002 { 2003 struct agintc_msi_softc *sc; 2004 struct cpu_info *ci; 2005 struct gits_cmd cmd; 2006 int hwcpu; 2007 2008 sc = li->li_msic; 2009 ci = li->li_ci; 2010 hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid]; 2011 2012 memset(&cmd, 0, sizeof(cmd)); 2013 cmd.cmd = DISCARD; 2014 cmd.deviceid = li->li_deviceid; 2015 cmd.eventid = li->li_eventid; 2016 agintc_msi_send_cmd(sc, &cmd); 2017 2018 memset(&cmd, 0, sizeof(cmd)); 2019 cmd.cmd = SYNC; 2020 cmd.dw2 = agintc_sc->sc_processor[hwcpu] << 16; 2021 agintc_msi_send_cmd(sc, &cmd); 2022 agintc_msi_wait_cmd(sc); 2023 } 2024 2025 void 2026 agintc_msi_inv(struct agintc_lpi_info *li) 2027 { 2028 struct agintc_msi_softc *sc; 2029 struct cpu_info *ci; 2030 struct gits_cmd cmd; 2031 int hwcpu; 2032 2033 sc = li->li_msic; 2034 ci = li->li_ci; 2035 hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid]; 2036 2037 memset(&cmd, 0, sizeof(cmd)); 2038 cmd.cmd = INV; 2039 cmd.deviceid = li->li_deviceid; 2040 cmd.eventid = li->li_eventid; 2041 agintc_msi_send_cmd(sc, &cmd); 2042 2043 memset(&cmd, 0, sizeof(cmd)); 2044 cmd.cmd = SYNC; 2045 cmd.dw2 = agintc_sc->sc_processor[hwcpu] << 16; 2046 agintc_msi_send_cmd(sc, &cmd); 2047 agintc_msi_wait_cmd(sc); 2048 } 2049 2050 void * 2051 agintc_intr_establish_msi(void *self, uint64_t *addr, uint64_t *data, 2052 int level, struct cpu_info *ci, int (*func)(void *), void *arg, char *name) 2053 { 2054 struct agintc_msi_softc *sc = (struct agintc_msi_softc *)self; 2055 struct agintc_msi_device *md; 2056 struct gits_cmd cmd; 2057 uint32_t deviceid = *data; 2058 uint32_t eventid; 2059 int i, hwcpu; 2060 2061 if (ci == NULL) 2062 ci = &cpu_info_primary; 2063 hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid]; 2064 2065 md = agintc_msi_find_device(sc, deviceid); 2066 if (md == NULL) 2067 return NULL; 2068 2069 eventid = *addr; 2070 if (eventid > 0 && (md->md_events & (1U << eventid))) 2071 return NULL; 2072 for (; eventid < 32; eventid++) { 2073 if ((md->md_events & (1U << eventid)) == 0) { 2074 md->md_events |= (1U << eventid); 2075 break; 2076 } 2077 } 2078 if (eventid >= 32) 2079 return NULL; 2080 2081 for (i = 0; i < agintc_sc->sc_nlpi; i++) { 2082 if (agintc_sc->sc_lpi[i] != NULL) 2083 continue; 2084 2085 agintc_sc->sc_lpi[i] = malloc(sizeof(struct agintc_lpi_info), 2086 M_DEVBUF, M_WAITOK | M_ZERO); 2087 agintc_sc->sc_lpi[i]->li_msic = sc; 2088 agintc_sc->sc_lpi[i]->li_ci = ci; 2089 agintc_sc->sc_lpi[i]->li_deviceid = deviceid; 2090 agintc_sc->sc_lpi[i]->li_eventid = eventid; 2091 agintc_sc->sc_lpi[i]->li_ih = 2092 agintc_intr_establish(LPI_BASE + i, 2093 IST_EDGE_RISING, level, ci, func, arg, name); 2094 if (agintc_sc->sc_lpi[i]->li_ih == NULL) { 2095 free(agintc_sc->sc_lpi[i], M_DEVBUF, 2096 sizeof(struct agintc_lpi_info)); 2097 agintc_sc->sc_lpi[i] = NULL; 2098 return NULL; 2099 } 2100 2101 memset(&cmd, 0, sizeof(cmd)); 2102 cmd.cmd = MAPTI; 2103 cmd.deviceid = deviceid; 2104 cmd.eventid = eventid; 2105 cmd.intid = LPI_BASE + i; 2106 cmd.dw2 = ci->ci_cpuid; 2107 agintc_msi_send_cmd(sc, &cmd); 2108 2109 memset(&cmd, 0, sizeof(cmd)); 2110 cmd.cmd = SYNC; 2111 cmd.dw2 = agintc_sc->sc_processor[hwcpu] << 16; 2112 agintc_msi_send_cmd(sc, &cmd); 2113 agintc_msi_wait_cmd(sc); 2114 2115 *addr = sc->sc_msi_addr + deviceid * sc->sc_msi_delta; 2116 *data = eventid; 2117 return &agintc_sc->sc_lpi[i]; 2118 } 2119 2120 return NULL; 2121 } 2122 2123 void 2124 agintc_intr_disestablish_msi(void *cookie) 2125 { 2126 struct agintc_lpi_info *li = *(void **)cookie; 2127 2128 agintc_intr_disestablish(li->li_ih); 2129 agintc_msi_discard(li); 2130 agintc_msi_inv(li); 2131 2132 free(li, M_DEVBUF, sizeof(*li)); 2133 *(void **)cookie = NULL; 2134 } 2135 2136 void 2137 agintc_intr_barrier_msi(void *cookie) 2138 { 2139 struct agintc_lpi_info *li = *(void **)cookie; 2140 2141 agintc_intr_barrier(li->li_ih); 2142 } 2143 2144 struct agintc_dmamem * 2145 agintc_dmamem_alloc(bus_dma_tag_t dmat, bus_size_t size, bus_size_t align) 2146 { 2147 struct agintc_dmamem *adm; 2148 int nsegs; 2149 2150 adm = malloc(sizeof(*adm), M_DEVBUF, M_WAITOK | M_ZERO); 2151 adm->adm_size = size; 2152 2153 if (bus_dmamap_create(dmat, size, 1, size, 0, 2154 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0) 2155 goto admfree; 2156 2157 if (bus_dmamem_alloc(dmat, size, align, 0, &adm->adm_seg, 1, 2158 &nsegs, BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0) 2159 goto destroy; 2160 2161 if (bus_dmamem_map(dmat, &adm->adm_seg, nsegs, size, 2162 &adm->adm_kva, BUS_DMA_WAITOK | BUS_DMA_NOCACHE) != 0) 2163 goto free; 2164 2165 if (bus_dmamap_load_raw(dmat, adm->adm_map, &adm->adm_seg, 2166 nsegs, size, BUS_DMA_WAITOK) != 0) 2167 goto unmap; 2168 2169 /* Make globally visible. */ 2170 cpu_dcache_wb_range((vaddr_t)adm->adm_kva, size); 2171 __asm volatile("dsb sy"); 2172 return adm; 2173 2174 unmap: 2175 bus_dmamem_unmap(dmat, adm->adm_kva, size); 2176 free: 2177 bus_dmamem_free(dmat, &adm->adm_seg, 1); 2178 destroy: 2179 bus_dmamap_destroy(dmat, adm->adm_map); 2180 admfree: 2181 free(adm, M_DEVBUF, sizeof(*adm)); 2182 2183 return NULL; 2184 } 2185 2186 void 2187 agintc_dmamem_free(bus_dma_tag_t dmat, struct agintc_dmamem *adm) 2188 { 2189 bus_dmamem_unmap(dmat, adm->adm_kva, adm->adm_size); 2190 bus_dmamem_free(dmat, &adm->adm_seg, 1); 2191 bus_dmamap_destroy(dmat, adm->adm_map); 2192 free(adm, M_DEVBUF, sizeof(*adm)); 2193 } 2194