1 /* $OpenBSD: agintc.c,v 1.27 2020/09/05 14:47:21 deraadt Exp $ */ 2 /* 3 * Copyright (c) 2007, 2009, 2011, 2017 Dale Rahn <drahn@dalerahn.com> 4 * Copyright (c) 2018 Mark Kettenis <kettenis@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 /* 20 * This is a device driver for the GICv3/GICv4 IP from ARM as specified 21 * in IHI0069C, an example of this hardware is the GIC 500. 22 */ 23 24 #include <sys/param.h> 25 #include <sys/systm.h> 26 #include <sys/queue.h> 27 #include <sys/malloc.h> 28 #include <sys/device.h> 29 #include <sys/evcount.h> 30 31 #include <machine/bus.h> 32 #include <machine/cpufunc.h> 33 #include <machine/fdt.h> 34 35 #include <dev/ofw/fdt.h> 36 #include <dev/ofw/openfirm.h> 37 38 #include <arm64/dev/simplebusvar.h> 39 40 #define ICC_PMR s3_0_c4_c6_0 41 #define ICC_IAR0 s3_0_c12_c8_0 42 #define ICC_EOIR0 s3_0_c12_c8_1 43 #define ICC_HPPIR0 s3_0_c12_c8_2 44 #define ICC_BPR0 s3_0_c12_c8_3 45 46 #define ICC_DIR s3_0_c12_c11_1 47 #define ICC_RPR s3_0_c12_c11_3 48 #define ICC_SGI1R s3_0_c12_c11_5 49 #define ICC_SGI0R s3_0_c12_c11_7 50 51 #define ICC_IAR1 s3_0_c12_c12_0 52 #define ICC_EOIR1 s3_0_c12_c12_1 53 #define ICC_HPPIR1 s3_0_c12_c12_2 54 #define ICC_BPR1 s3_0_c12_c12_3 55 #define ICC_CTLR s3_0_c12_c12_4 56 #define ICC_SRE_EL1 s3_0_c12_c12_5 57 #define ICC_SRE_EL1_EN 0x7 58 #define ICC_IGRPEN0 s3_0_c12_c12_6 59 #define ICC_IGRPEN1 s3_0_c12_c12_7 60 61 #define _STR(x) #x 62 #define STR(x) _STR(x) 63 64 /* distributor registers */ 65 #define GICD_CTLR 0x0000 66 /* non-secure */ 67 #define GICD_CTLR_RWP (1U << 31) 68 #define GICD_CTLR_EnableGrp1 (1 << 0) 69 #define GICD_CTLR_EnableGrp1A (1 << 1) 70 #define GICD_CTLR_ARE_NS (1 << 4) 71 #define GICD_CTLR_DS (1 << 6) 72 #define GICD_TYPER 0x0004 73 #define GICD_TYPER_LPIS (1 << 16) 74 #define GICD_TYPER_ITLINE_M 0x1f 75 #define GICD_IIDR 0x0008 76 #define GICD_ISENABLER(i) (0x0100 + (IRQ_TO_REG32(i) * 4)) 77 #define GICD_ICENABLER(i) (0x0180 + (IRQ_TO_REG32(i) * 4)) 78 #define GICD_ISPENDR(i) (0x0200 + (IRQ_TO_REG32(i) * 4)) 79 #define GICD_ICPENDR(i) (0x0280 + (IRQ_TO_REG32(i) * 4)) 80 #define GICD_ISACTIVER(i) (0x0300 + (IRQ_TO_REG32(i) * 4)) 81 #define GICD_ICACTIVER(i) (0x0380 + (IRQ_TO_REG32(i) * 4)) 82 #define GICD_IPRIORITYR(i) (0x0400 + (i)) 83 #define GICD_ICFGR(i) (0x0c00 + (IRQ_TO_REG16(i) * 4)) 84 #define GICD_NSACR(i) (0x0e00 + (IRQ_TO_REG16(i) * 4)) 85 #define GICD_IROUTER(i) (0x6000 + ((i) * 8)) 86 87 /* redistributor registers */ 88 #define GICR_CTLR 0x00000 89 #define GICR_CTLR_RWP ((1U << 31) | (1 << 3)) 90 #define GICR_CTLR_ENABLE_LPIS (1 << 0) 91 #define GICR_IIDR 0x00004 92 #define GICR_TYPER 0x00008 93 #define GICR_TYPER_LAST (1 << 4) 94 #define GICR_TYPER_VLPIS (1 << 1) 95 #define GICR_WAKER 0x00014 96 #define GICR_WAKER_X31 (1U << 31) 97 #define GICR_WAKER_CHILDRENASLEEP (1 << 2) 98 #define GICR_WAKER_PROCESSORSLEEP (1 << 1) 99 #define GICR_WAKER_X0 (1 << 0) 100 #define GICR_PROPBASER 0x00070 101 #define GICR_PROPBASER_ISH (1ULL << 10) 102 #define GICR_PROPBASER_IC_NORM_NC (1ULL << 7) 103 #define GICR_PENDBASER 0x00078 104 #define GICR_PENDBASER_PTZ (1ULL << 62) 105 #define GICR_PENDBASER_ISH (1ULL << 10) 106 #define GICR_PENDBASER_IC_NORM_NC (1ULL << 7) 107 #define GICR_IGROUP0 0x10080 108 #define GICR_ISENABLE0 0x10100 109 #define GICR_ICENABLE0 0x10180 110 #define GICR_ISPENDR0 0x10200 111 #define GICR_ICPENDR0 0x10280 112 #define GICR_ISACTIVE0 0x10300 113 #define GICR_ICACTIVE0 0x10380 114 #define GICR_IPRIORITYR(i) (0x10400 + (i)) 115 #define GICR_ICFGR0 0x10c00 116 #define GICR_ICFGR1 0x10c04 117 118 #define GICR_PROP_SIZE (64 * 1024) 119 #define GICR_PROP_GROUP1 (1 << 1) 120 #define GICR_PROP_ENABLE (1 << 0) 121 #define GICR_PEND_SIZE (64 * 1024) 122 123 #define PPI_BASE 16 124 #define SPI_BASE 32 125 #define LPI_BASE 8192 126 127 #define IRQ_TO_REG32(i) (((i) >> 5) & 0x7) 128 #define IRQ_TO_REG32BIT(i) ((i) & 0x1f) 129 130 #define IRQ_TO_REG16(i) (((i) >> 4) & 0xf) 131 #define IRQ_TO_REG16BIT(i) ((i) & 0xf) 132 133 #define IRQ_ENABLE 1 134 #define IRQ_DISABLE 0 135 136 struct agintc_softc { 137 struct simplebus_softc sc_sbus; 138 struct intrq *sc_handler; 139 struct intrhand **sc_lpi_handler; 140 bus_space_tag_t sc_iot; 141 bus_space_handle_t sc_d_ioh; 142 bus_space_handle_t *sc_r_ioh; 143 bus_space_handle_t sc_redist_base; 144 bus_dma_tag_t sc_dmat; 145 uint16_t *sc_processor; 146 int sc_cpuremap[MAXCPUS]; 147 int sc_nintr; 148 int sc_nlpi; 149 int sc_prio_shift; 150 int sc_pmr_shift; 151 int sc_rk3399_quirk; 152 struct evcount sc_spur; 153 int sc_ncells; 154 int sc_num_redist; 155 struct agintc_dmamem *sc_prop; 156 struct agintc_dmamem *sc_pend; 157 struct interrupt_controller sc_ic; 158 int sc_ipi_num[2]; /* id for NOP and DDB ipi */ 159 int sc_ipi_reason[MAXCPUS]; /* NOP or DDB caused */ 160 void *sc_ipi_irq[2]; /* irqhandle for each ipi */ 161 }; 162 struct agintc_softc *agintc_sc; 163 164 struct intrhand { 165 TAILQ_ENTRY(intrhand) ih_list; /* link on intrq list */ 166 int (*ih_func)(void *); /* handler */ 167 void *ih_arg; /* arg for handler */ 168 int ih_ipl; /* IPL_* */ 169 int ih_flags; 170 int ih_irq; /* IRQ number */ 171 struct evcount ih_count; 172 char *ih_name; 173 struct cpu_info *ih_ci; /* CPU the IRQ runs on */ 174 }; 175 176 struct intrq { 177 TAILQ_HEAD(, intrhand) iq_list; /* handler list */ 178 struct cpu_info *iq_ci; /* CPU the IRQ runs on */ 179 int iq_irq_max; /* IRQ to mask while handling */ 180 int iq_irq_min; /* lowest IRQ when shared */ 181 int iq_ist; /* share type */ 182 int iq_route; 183 }; 184 185 struct agintc_dmamem { 186 bus_dmamap_t adm_map; 187 bus_dma_segment_t adm_seg; 188 size_t adm_size; 189 caddr_t adm_kva; 190 }; 191 192 #define AGINTC_DMA_MAP(_adm) ((_adm)->adm_map) 193 #define AGINTC_DMA_LEN(_adm) ((_adm)->adm_size) 194 #define AGINTC_DMA_DVA(_adm) ((_adm)->adm_map->dm_segs[0].ds_addr) 195 #define AGINTC_DMA_KVA(_adm) ((void *)(_adm)->adm_kva) 196 197 struct agintc_dmamem *agintc_dmamem_alloc(bus_dma_tag_t, bus_size_t, 198 bus_size_t); 199 void agintc_dmamem_free(bus_dma_tag_t, struct agintc_dmamem *); 200 201 int agintc_match(struct device *, void *, void *); 202 void agintc_attach(struct device *, struct device *, void *); 203 void agintc_cpuinit(void); 204 int agintc_spllower(int); 205 void agintc_splx(int); 206 int agintc_splraise(int); 207 void agintc_setipl(int); 208 void agintc_calc_mask(void); 209 void agintc_calc_irq(struct agintc_softc *sc, int irq); 210 void *agintc_intr_establish(int, int, struct cpu_info *ci, 211 int (*)(void *), void *, char *); 212 void *agintc_intr_establish_fdt(void *cookie, int *cell, int level, 213 struct cpu_info *, int (*func)(void *), void *arg, char *name); 214 void agintc_intr_disestablish(void *); 215 void agintc_irq_handler(void *); 216 uint32_t agintc_iack(void); 217 void agintc_eoi(uint32_t); 218 void agintc_set_priority(struct agintc_softc *sc, int, int); 219 void agintc_intr_enable(struct agintc_softc *, int); 220 void agintc_intr_disable(struct agintc_softc *, int); 221 void agintc_route(struct agintc_softc *, int, int, 222 struct cpu_info *); 223 void agintc_route_irq(void *, int, struct cpu_info *); 224 void agintc_intr_barrier(void *); 225 void agintc_wait_rwp(struct agintc_softc *sc); 226 void agintc_r_wait_rwp(struct agintc_softc *sc); 227 uint32_t agintc_r_ictlr(void); 228 229 int agintc_ipi_ddb(void *v); 230 int agintc_ipi_nop(void *v); 231 int agintc_ipi_combined(void *); 232 void agintc_send_ipi(struct cpu_info *, int); 233 234 struct cfattach agintc_ca = { 235 sizeof (struct agintc_softc), agintc_match, agintc_attach 236 }; 237 238 struct cfdriver agintc_cd = { 239 NULL, "agintc", DV_DULL 240 }; 241 242 static char *agintc_compatibles[] = { 243 "arm,gic-v3", 244 "arm,gic-v4", 245 NULL 246 }; 247 248 int 249 agintc_match(struct device *parent, void *cfdata, void *aux) 250 { 251 struct fdt_attach_args *faa = aux; 252 int i; 253 254 for (i = 0; agintc_compatibles[i]; i++) 255 if (OF_is_compatible(faa->fa_node, agintc_compatibles[i])) 256 return (1); 257 258 return (0); 259 } 260 261 static void 262 __isb(void) 263 { 264 __asm volatile("isb"); 265 } 266 267 void 268 agintc_attach(struct device *parent, struct device *self, void *aux) 269 { 270 struct agintc_softc *sc = (struct agintc_softc *)self; 271 struct fdt_attach_args *faa = aux; 272 struct cpu_info *ci; 273 CPU_INFO_ITERATOR cii; 274 uint32_t typer; 275 uint32_t nsacr, oldnsacr; 276 uint32_t pmr, oldpmr; 277 uint32_t ctrl, bits; 278 uint32_t affinity; 279 int i, j, nbits, nintr; 280 int psw; 281 int offset, nredist; 282 #ifdef MULTIPROCESSOR 283 int nipi, ipiirq[2]; 284 #endif 285 286 psw = disable_interrupts(); 287 arm_init_smask(); 288 289 sc->sc_iot = faa->fa_iot; 290 sc->sc_dmat = faa->fa_dmat; 291 292 /* First row: distributor */ 293 if (bus_space_map(sc->sc_iot, faa->fa_reg[0].addr, 294 faa->fa_reg[0].size, 0, &sc->sc_d_ioh)) 295 panic("%s: ICD bus_space_map failed!", __func__); 296 297 /* Second row: redistributor */ 298 if (bus_space_map(sc->sc_iot, faa->fa_reg[1].addr, 299 faa->fa_reg[1].size, 0, &sc->sc_redist_base)) 300 panic("%s: ICP bus_space_map failed!", __func__); 301 302 typer = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_TYPER); 303 304 if (typer & GICD_TYPER_LPIS) { 305 /* Allocate redistributor tables */ 306 sc->sc_prop = agintc_dmamem_alloc(sc->sc_dmat, 307 GICR_PROP_SIZE, GICR_PROP_SIZE); 308 if (sc->sc_prop == NULL) { 309 printf(": can't alloc LPI config table\n"); 310 goto unmap; 311 } 312 sc->sc_pend = agintc_dmamem_alloc(sc->sc_dmat, 313 GICR_PEND_SIZE, GICR_PEND_SIZE); 314 if (sc->sc_prop == NULL) { 315 printf(": can't alloc LPI pending table\n"); 316 goto unmap; 317 } 318 319 /* Minimum number of LPIs supported by any implementation. */ 320 sc->sc_nlpi = 8192; 321 } 322 323 /* 324 * We are guaranteed to have at least 16 priority levels, so 325 * in principle we just want to use the top 4 bits of the 326 * (non-secure) priority field. 327 */ 328 sc->sc_prio_shift = sc->sc_pmr_shift = 4; 329 330 /* 331 * If the system supports two security states and SCR_EL3.FIQ 332 * is zero, the non-secure shifted view applies. We detect 333 * this by checking whether the number of writable bits 334 * matches the number of implemented priority bits. If that 335 * is the case we will need to adjust the priorities that we 336 * write into ICC_PMR_EL1 accordingly. 337 * 338 * On Ampere eMAG it appears as if there are five writable 339 * bits when we write 0xff. But for higher priorities 340 * (smaller values) only the top 4 bits stick. So we use 0xbf 341 * instead to determine the number of writable bits. 342 */ 343 ctrl = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR); 344 if ((ctrl & GICD_CTLR_DS) == 0) { 345 __asm volatile("mrs %x0, "STR(ICC_CTLR_EL1) : "=r"(ctrl)); 346 nbits = ICC_CTLR_EL1_PRIBITS(ctrl) + 1; 347 __asm volatile("mrs %x0, "STR(ICC_PMR) : "=r"(oldpmr)); 348 __asm volatile("msr "STR(ICC_PMR)", %x0" :: "r"(0xbf)); 349 __asm volatile("mrs %x0, "STR(ICC_PMR) : "=r"(pmr)); 350 __asm volatile("msr "STR(ICC_PMR)", %x0" :: "r"(oldpmr)); 351 if (nbits == 8 - (ffs(pmr) - 1)) 352 sc->sc_pmr_shift--; 353 } 354 355 /* 356 * The Rockchip RK3399 is busted. Its GIC-500 treats all 357 * access to its memory mapped registers as "secure". As a 358 * result, several registers don't behave as expected. For 359 * example, the GICD_IPRIORITYRn and GICR_IPRIORITYRn 360 * registers expose the full priority range available to 361 * secure interrupts. We need to be aware of this and write 362 * an adjusted priority value into these registers. We also 363 * need to be careful not to touch any bits that shouldn't be 364 * writable in non-secure mode. 365 * 366 * We check whether we have secure mode access to these 367 * registers by attempting to write to the GICD_NSACR register 368 * and check whether its contents actually change. In that 369 * case we need to adjust the priorities we write into 370 * GICD_IPRIORITYRn and GICRIPRIORITYRn accordingly. 371 */ 372 oldnsacr = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_NSACR(32)); 373 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, GICD_NSACR(32), 374 oldnsacr ^ 0xffffffff); 375 nsacr = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_NSACR(32)); 376 if (nsacr != oldnsacr) { 377 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, GICD_NSACR(32), 378 oldnsacr); 379 sc->sc_rk3399_quirk = 1; 380 sc->sc_prio_shift--; 381 printf(" sec"); 382 } 383 384 printf(" shift %d:%d", sc->sc_prio_shift, sc->sc_pmr_shift); 385 386 evcount_attach(&sc->sc_spur, "irq1023/spur", NULL); 387 388 __asm volatile("msr "STR(ICC_SRE_EL1)", %x0" : : "r" (ICC_SRE_EL1_EN)); 389 __isb(); 390 391 nintr = 32 * (typer & GICD_TYPER_ITLINE_M); 392 nintr += 32; /* ICD_ICTR + 1, irq 0-31 is SGI, 32+ is PPI */ 393 sc->sc_nintr = nintr; 394 395 agintc_sc = sc; /* save this for global access */ 396 397 /* find the redistributors. */ 398 offset = 0; 399 for (nredist = 0; ; nredist++) { 400 int32_t sz = (64 * 1024 * 2); 401 uint64_t typer; 402 403 typer = bus_space_read_8(sc->sc_iot, sc->sc_redist_base, 404 offset + GICR_TYPER); 405 406 if (typer & GICR_TYPER_VLPIS) 407 sz += (64 * 1024 * 2); 408 409 #ifdef DEBUG_AGINTC 410 printf("probing redistributor %d %x\n", nredist, offset); 411 #endif 412 413 offset += sz; 414 415 if (typer & GICR_TYPER_LAST) { 416 sc->sc_num_redist = nredist + 1; 417 break; 418 } 419 } 420 421 printf(" nirq %d nredist %d", nintr, sc->sc_num_redist); 422 423 sc->sc_r_ioh = mallocarray(sc->sc_num_redist, 424 sizeof(*sc->sc_r_ioh), M_DEVBUF, M_WAITOK); 425 sc->sc_processor = mallocarray(sc->sc_num_redist, 426 sizeof(*sc->sc_processor), M_DEVBUF, M_WAITOK); 427 428 /* submap and configure the redistributors. */ 429 offset = 0; 430 for (nredist = 0; nredist < sc->sc_num_redist; nredist++) { 431 int32_t sz = (64 * 1024 * 2); 432 uint64_t typer; 433 434 typer = bus_space_read_8(sc->sc_iot, sc->sc_redist_base, 435 offset + GICR_TYPER); 436 437 if (typer & GICR_TYPER_VLPIS) 438 sz += (64 * 1024 * 2); 439 440 affinity = bus_space_read_8(sc->sc_iot, 441 sc->sc_redist_base, offset + GICR_TYPER) >> 32; 442 CPU_INFO_FOREACH(cii, ci) { 443 if (affinity == (((ci->ci_mpidr >> 8) & 0xff000000) | 444 (ci->ci_mpidr & 0x00ffffff))) 445 break; 446 } 447 if (ci != NULL) 448 sc->sc_cpuremap[ci->ci_cpuid] = nredist; 449 #ifdef MULTIPROCESSOR 450 else 451 panic("%s: no CPU found for affinity %08x", 452 sc->sc_sbus.sc_dev.dv_xname, affinity); 453 #endif 454 455 sc->sc_processor[nredist] = bus_space_read_8(sc->sc_iot, 456 sc->sc_redist_base, offset + GICR_TYPER) >> 8; 457 458 bus_space_subregion(sc->sc_iot, sc->sc_redist_base, 459 offset, sz, &sc->sc_r_ioh[nredist]); 460 461 if (sc->sc_nlpi > 0) { 462 bus_space_write_8(sc->sc_iot, sc->sc_redist_base, 463 offset + GICR_PROPBASER, 464 AGINTC_DMA_DVA(sc->sc_prop) | 465 GICR_PROPBASER_ISH | GICR_PROPBASER_IC_NORM_NC | 466 fls(LPI_BASE + sc->sc_nlpi - 1) - 1); 467 bus_space_write_8(sc->sc_iot, sc->sc_redist_base, 468 offset + GICR_PENDBASER, 469 AGINTC_DMA_DVA(sc->sc_pend) | 470 GICR_PENDBASER_ISH | GICR_PENDBASER_IC_NORM_NC | 471 GICR_PENDBASER_PTZ); 472 bus_space_write_4(sc->sc_iot, sc->sc_redist_base, 473 offset + GICR_CTLR, GICR_CTLR_ENABLE_LPIS); 474 } 475 476 offset += sz; 477 } 478 479 /* Disable all interrupts, clear all pending */ 480 for (i = 1; i < nintr / 32; i++) { 481 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 482 GICD_ICENABLER(i * 32), ~0); 483 for (j = 0; j < 32; j++) { 484 __asm volatile("msr "STR(ICC_DIR)", %x0" : : 485 "r" ((i * 32) + j)); 486 __isb(); 487 } 488 } 489 490 for (i = 4; i < nintr; i += 4) { 491 /* lowest priority ?? */ 492 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 493 GICD_IPRIORITYR(i), 0xffffffff); 494 } 495 496 for (i = 2; i < nintr / 16; i++) { 497 /* irq 32 - N */ 498 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 499 GICD_ICFGR(i * 16), 0); 500 } 501 502 agintc_cpuinit(); 503 504 sc->sc_handler = mallocarray(nintr, 505 sizeof(*sc->sc_handler), M_DEVBUF, M_ZERO | M_WAITOK); 506 for (i = 0; i < nintr; i++) 507 TAILQ_INIT(&sc->sc_handler[i].iq_list); 508 sc->sc_lpi_handler = mallocarray(sc->sc_nlpi, 509 sizeof(*sc->sc_lpi_handler), M_DEVBUF, M_ZERO | M_WAITOK); 510 511 /* set priority to IPL_HIGH until configure lowers to desired IPL */ 512 agintc_setipl(IPL_HIGH); 513 514 /* initialize all interrupts as disabled */ 515 agintc_calc_mask(); 516 517 /* insert self as interrupt handler */ 518 arm_set_intr_handler(agintc_splraise, agintc_spllower, agintc_splx, 519 agintc_setipl, agintc_irq_handler); 520 521 /* enable interrupts */ 522 ctrl = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR); 523 bits = GICD_CTLR_ARE_NS | GICD_CTLR_EnableGrp1A | GICD_CTLR_EnableGrp1; 524 if (sc->sc_rk3399_quirk) { 525 bits &= ~GICD_CTLR_EnableGrp1A; 526 bits <<= 1; 527 } 528 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR, ctrl | bits); 529 530 __asm volatile("msr "STR(ICC_PMR)", %x0" :: "r"(0xff)); 531 __asm volatile("msr "STR(ICC_BPR1)", %x0" :: "r"(0)); 532 __asm volatile("msr "STR(ICC_IGRPEN1)", %x0" :: "r"(1)); 533 534 #ifdef MULTIPROCESSOR 535 /* setup IPI interrupts */ 536 537 /* 538 * Ideally we want two IPI interrupts, one for NOP and one for 539 * DDB, however we can survive if only one is available it is 540 * possible that most are not available to the non-secure OS. 541 */ 542 nipi = 0; 543 for (i = 0; i < 16; i++) { 544 int hwcpu = sc->sc_cpuremap[cpu_number()]; 545 int reg, oldreg; 546 547 oldreg = bus_space_read_1(sc->sc_iot, sc->sc_r_ioh[hwcpu], 548 GICR_IPRIORITYR(i)); 549 bus_space_write_1(sc->sc_iot, sc->sc_r_ioh[hwcpu], 550 GICR_IPRIORITYR(i), oldreg ^ 0x20); 551 552 /* if this interrupt is not usable, pri will be unmodified */ 553 reg = bus_space_read_1(sc->sc_iot, sc->sc_r_ioh[hwcpu], 554 GICR_IPRIORITYR(i)); 555 if (reg == oldreg) 556 continue; 557 558 /* return to original value, will be set when used */ 559 bus_space_write_1(sc->sc_iot, sc->sc_r_ioh[hwcpu], 560 GICR_IPRIORITYR(i), oldreg); 561 562 if (nipi == 0) 563 printf(" ipi: %d", i); 564 else 565 printf(", %d", i); 566 ipiirq[nipi++] = i; 567 if (nipi == 2) 568 break; 569 } 570 571 if (nipi == 0) 572 panic("no irq available for IPI"); 573 574 switch (nipi) { 575 case 1: 576 sc->sc_ipi_irq[0] = agintc_intr_establish(ipiirq[0], 577 IPL_IPI|IPL_MPSAFE, NULL, agintc_ipi_combined, sc, "ipi"); 578 sc->sc_ipi_num[ARM_IPI_NOP] = ipiirq[0]; 579 sc->sc_ipi_num[ARM_IPI_DDB] = ipiirq[0]; 580 break; 581 case 2: 582 sc->sc_ipi_irq[0] = agintc_intr_establish(ipiirq[0], 583 IPL_IPI|IPL_MPSAFE, NULL, agintc_ipi_nop, sc, "ipinop"); 584 sc->sc_ipi_num[ARM_IPI_NOP] = ipiirq[0]; 585 sc->sc_ipi_irq[1] = agintc_intr_establish(ipiirq[1], 586 IPL_IPI|IPL_MPSAFE, NULL, agintc_ipi_ddb, sc, "ipiddb"); 587 sc->sc_ipi_num[ARM_IPI_DDB] = ipiirq[1]; 588 break; 589 default: 590 panic("nipi unexpected number %d", nipi); 591 } 592 593 intr_send_ipi_func = agintc_send_ipi; 594 #endif 595 596 sc->sc_ic.ic_node = faa->fa_node; 597 sc->sc_ic.ic_cookie = self; 598 sc->sc_ic.ic_establish = agintc_intr_establish_fdt; 599 sc->sc_ic.ic_disestablish = agintc_intr_disestablish; 600 sc->sc_ic.ic_route = agintc_route_irq; 601 sc->sc_ic.ic_cpu_enable = agintc_cpuinit; 602 sc->sc_ic.ic_barrier = agintc_intr_barrier; 603 arm_intr_register_fdt(&sc->sc_ic); 604 605 restore_interrupts(psw); 606 607 /* Attach ITS. */ 608 simplebus_attach(parent, &sc->sc_sbus.sc_dev, faa); 609 610 return; 611 612 unmap: 613 if (sc->sc_r_ioh) { 614 free(sc->sc_r_ioh, M_DEVBUF, 615 sc->sc_num_redist * sizeof(*sc->sc_r_ioh)); 616 } 617 if (sc->sc_processor) { 618 free(sc->sc_processor, M_DEVBUF, 619 sc->sc_num_redist * sizeof(*sc->sc_processor)); 620 } 621 622 if (sc->sc_pend) 623 agintc_dmamem_free(sc->sc_dmat, sc->sc_pend); 624 if (sc->sc_prop) 625 agintc_dmamem_free(sc->sc_dmat, sc->sc_prop); 626 627 bus_space_unmap(sc->sc_iot, sc->sc_redist_base, faa->fa_reg[1].size); 628 bus_space_unmap(sc->sc_iot, sc->sc_d_ioh, faa->fa_reg[0].size); 629 } 630 631 /* Initialize redistributors on each core. */ 632 void 633 agintc_cpuinit(void) 634 { 635 struct agintc_softc *sc = agintc_sc; 636 uint32_t waker; 637 int timeout = 100000; 638 int hwcpu; 639 int i; 640 641 hwcpu = sc->sc_cpuremap[cpu_number()]; 642 waker = bus_space_read_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 643 GICR_WAKER); 644 waker &= ~(GICR_WAKER_PROCESSORSLEEP); 645 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], GICR_WAKER, 646 waker); 647 648 do { 649 waker = bus_space_read_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 650 GICR_WAKER); 651 } while (--timeout && (waker & GICR_WAKER_CHILDRENASLEEP)); 652 if (timeout == 0) 653 printf("%s: waker timed out\n", __func__); 654 655 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 656 GICR_ICENABLE0, ~0); 657 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 658 GICR_ICPENDR0, ~0); 659 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 660 GICR_ICACTIVE0, ~0); 661 for (i = 0; i < 32; i += 4) { 662 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 663 GICR_IPRIORITYR(i), ~0); 664 } 665 666 if (sc->sc_ipi_irq[0] != NULL) 667 agintc_route_irq(sc->sc_ipi_irq[0], IRQ_ENABLE, curcpu()); 668 if (sc->sc_ipi_irq[1] != NULL) 669 agintc_route_irq(sc->sc_ipi_irq[1], IRQ_ENABLE, curcpu()); 670 671 __asm volatile("msr "STR(ICC_PMR)", %x0" :: "r"(0xff)); 672 __asm volatile("msr "STR(ICC_BPR1)", %x0" :: "r"(0)); 673 __asm volatile("msr "STR(ICC_IGRPEN1)", %x0" :: "r"(1)); 674 enable_interrupts(); 675 } 676 677 void 678 agintc_set_priority(struct agintc_softc *sc, int irq, int ipl) 679 { 680 struct cpu_info *ci = curcpu(); 681 int hwcpu = sc->sc_cpuremap[ci->ci_cpuid]; 682 uint32_t prival; 683 684 prival = ((0xff - ipl) << sc->sc_prio_shift) & 0xff; 685 686 if (irq >= SPI_BASE) { 687 bus_space_write_1(sc->sc_iot, sc->sc_d_ioh, 688 GICD_IPRIORITYR(irq), prival); 689 } else { 690 /* only sets local redistributor */ 691 bus_space_write_1(sc->sc_iot, sc->sc_r_ioh[hwcpu], 692 GICR_IPRIORITYR(irq), prival); 693 } 694 } 695 696 void 697 agintc_setipl(int ipl) 698 { 699 struct agintc_softc *sc = agintc_sc; 700 struct cpu_info *ci = curcpu(); 701 int psw; 702 uint32_t prival; 703 704 /* disable here is only to keep hardware in sync with ci->ci_cpl */ 705 psw = disable_interrupts(); 706 ci->ci_cpl = ipl; 707 708 prival = ((0xff - ipl) << sc->sc_pmr_shift) & 0xff; 709 __asm volatile("msr "STR(ICC_PMR)", %x0" : : "r" (prival)); 710 __isb(); 711 712 restore_interrupts(psw); 713 } 714 715 void 716 agintc_intr_enable(struct agintc_softc *sc, int irq) 717 { 718 struct cpu_info *ci = curcpu(); 719 int hwcpu = sc->sc_cpuremap[ci->ci_cpuid]; 720 int bit = 1 << IRQ_TO_REG32BIT(irq); 721 722 if (irq >= 32) { 723 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 724 GICD_ISENABLER(irq), bit); 725 } else { 726 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 727 GICR_ISENABLE0, bit); 728 } 729 } 730 731 void 732 agintc_intr_disable(struct agintc_softc *sc, int irq) 733 { 734 struct cpu_info *ci = curcpu(); 735 int hwcpu = sc->sc_cpuremap[ci->ci_cpuid]; 736 737 if (irq >= 32) { 738 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 739 GICD_ICENABLER(irq), 1 << IRQ_TO_REG32BIT(irq)); 740 } else { 741 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 742 GICR_ICENABLE0, 1 << IRQ_TO_REG32BIT(irq)); 743 } 744 } 745 746 void 747 agintc_calc_mask(void) 748 { 749 struct agintc_softc *sc = agintc_sc; 750 int irq; 751 752 for (irq = 0; irq < sc->sc_nintr; irq++) 753 agintc_calc_irq(sc, irq); 754 } 755 756 void 757 agintc_calc_irq(struct agintc_softc *sc, int irq) 758 { 759 struct cpu_info *ci = sc->sc_handler[irq].iq_ci; 760 struct intrhand *ih; 761 int max = IPL_NONE; 762 int min = IPL_HIGH; 763 764 TAILQ_FOREACH(ih, &sc->sc_handler[irq].iq_list, ih_list) { 765 if (ih->ih_ipl > max) 766 max = ih->ih_ipl; 767 768 if (ih->ih_ipl < min) 769 min = ih->ih_ipl; 770 } 771 772 if (max == IPL_NONE) 773 min = IPL_NONE; 774 775 if (sc->sc_handler[irq].iq_irq_max == max && 776 sc->sc_handler[irq].iq_irq_min == min) 777 return; 778 779 sc->sc_handler[irq].iq_irq_max = max; 780 sc->sc_handler[irq].iq_irq_min = min; 781 782 #ifdef DEBUG_AGINTC 783 if (min != IPL_NONE) 784 printf("irq %d to block at %d %d \n", irq, max, min ); 785 #endif 786 /* Enable interrupts at lower levels, clear -> enable */ 787 /* Set interrupt priority/enable */ 788 if (min != IPL_NONE) { 789 agintc_set_priority(sc, irq, min); 790 agintc_route(sc, irq, IRQ_ENABLE, ci); 791 agintc_intr_enable(sc, irq); 792 } else { 793 agintc_intr_disable(sc, irq); 794 agintc_route(sc, irq, IRQ_DISABLE, ci); 795 } 796 } 797 798 void 799 agintc_splx(int new) 800 { 801 struct cpu_info *ci = curcpu(); 802 803 if (ci->ci_ipending & arm_smask[new]) 804 arm_do_pending_intr(new); 805 806 agintc_setipl(new); 807 } 808 809 int 810 agintc_spllower(int new) 811 { 812 struct cpu_info *ci = curcpu(); 813 int old = ci->ci_cpl; 814 815 agintc_splx(new); 816 return (old); 817 } 818 819 int 820 agintc_splraise(int new) 821 { 822 struct cpu_info *ci = curcpu(); 823 int old = ci->ci_cpl; 824 825 /* 826 * setipl must always be called because there is a race window 827 * where the variable is updated before the mask is set 828 * an interrupt occurs in that window without the mask always 829 * being set, the hardware might not get updated on the next 830 * splraise completely messing up spl protection. 831 */ 832 if (old > new) 833 new = old; 834 835 agintc_setipl(new); 836 return (old); 837 } 838 839 uint32_t 840 agintc_iack(void) 841 { 842 int irq; 843 844 __asm volatile("mrs %x0, "STR(ICC_IAR1) : "=r" (irq)); 845 __asm volatile("dsb sy"); 846 return irq; 847 } 848 849 void 850 agintc_route_irq(void *v, int enable, struct cpu_info *ci) 851 { 852 struct agintc_softc *sc = agintc_sc; 853 struct intrhand *ih = v; 854 855 if (enable) { 856 agintc_set_priority(sc, ih->ih_irq, 857 sc->sc_handler[ih->ih_irq].iq_irq_min); 858 agintc_route(sc, ih->ih_irq, IRQ_ENABLE, ci); 859 agintc_intr_enable(sc, ih->ih_irq); 860 } 861 } 862 863 void 864 agintc_route(struct agintc_softc *sc, int irq, int enable, struct cpu_info *ci) 865 { 866 /* XXX does not yet support 'participating node' */ 867 if (irq >= 32) { 868 #ifdef DEBUG_AGINTC 869 printf("router %x irq %d val %016llx\n", GICD_IROUTER(irq), 870 irq, ci->ci_mpidr & MPIDR_AFF); 871 #endif 872 bus_space_write_8(sc->sc_iot, sc->sc_d_ioh, 873 GICD_IROUTER(irq), ci->ci_mpidr & MPIDR_AFF); 874 } 875 } 876 877 void 878 agintc_intr_barrier(void *cookie) 879 { 880 struct intrhand *ih = cookie; 881 882 sched_barrier(ih->ih_ci); 883 } 884 885 void 886 agintc_run_handler(struct intrhand *ih, void *frame, int s) 887 { 888 void *arg; 889 int handled; 890 891 #ifdef MULTIPROCESSOR 892 int need_lock; 893 894 if (ih->ih_flags & IPL_MPSAFE) 895 need_lock = 0; 896 else 897 need_lock = s < IPL_SCHED; 898 899 if (need_lock) 900 KERNEL_LOCK(); 901 #endif 902 903 if (ih->ih_arg != 0) 904 arg = ih->ih_arg; 905 else 906 arg = frame; 907 908 enable_interrupts(); 909 handled = ih->ih_func(arg); 910 disable_interrupts(); 911 if (handled) 912 ih->ih_count.ec_count++; 913 914 #ifdef MULTIPROCESSOR 915 if (need_lock) 916 KERNEL_UNLOCK(); 917 #endif 918 } 919 920 void 921 agintc_irq_handler(void *frame) 922 { 923 struct agintc_softc *sc = agintc_sc; 924 struct intrhand *ih; 925 int irq, pri, s; 926 927 irq = agintc_iack(); 928 929 #ifdef DEBUG_AGINTC 930 if (irq != 30) 931 printf("irq %d fired\n", irq); 932 else { 933 static int cnt = 0; 934 if ((cnt++ % 100) == 0) { 935 printf("irq %d fired * _100\n", irq); 936 #ifdef DDB 937 db_enter(); 938 #endif 939 } 940 } 941 #endif 942 943 if (irq == 1023) { 944 sc->sc_spur.ec_count++; 945 return; 946 } 947 948 if ((irq >= sc->sc_nintr && irq < LPI_BASE) || 949 irq >= LPI_BASE + sc->sc_nlpi) { 950 return; 951 } 952 953 if (irq >= LPI_BASE) { 954 ih = sc->sc_lpi_handler[irq - LPI_BASE]; 955 if (ih == NULL) 956 return; 957 958 s = agintc_splraise(ih->ih_ipl); 959 agintc_run_handler(ih, frame, s); 960 agintc_eoi(irq); 961 962 agintc_splx(s); 963 return; 964 } 965 966 pri = sc->sc_handler[irq].iq_irq_max; 967 s = agintc_splraise(pri); 968 TAILQ_FOREACH(ih, &sc->sc_handler[irq].iq_list, ih_list) { 969 agintc_run_handler(ih, frame, s); 970 } 971 agintc_eoi(irq); 972 973 agintc_splx(s); 974 } 975 976 void * 977 agintc_intr_establish_fdt(void *cookie, int *cell, int level, 978 struct cpu_info *ci, int (*func)(void *), void *arg, char *name) 979 { 980 struct agintc_softc *sc = agintc_sc; 981 int irq; 982 983 /* 2nd cell contains the interrupt number */ 984 irq = cell[1]; 985 986 /* 1st cell contains type: 0 SPI (32-X), 1 PPI (16-31) */ 987 if (cell[0] == 0) 988 irq += SPI_BASE; 989 else if (cell[0] == 1) 990 irq += PPI_BASE; 991 else 992 panic("%s: bogus interrupt type", sc->sc_sbus.sc_dev.dv_xname); 993 994 return agintc_intr_establish(irq, level, ci, func, arg, name); 995 } 996 997 void * 998 agintc_intr_establish(int irqno, int level, struct cpu_info *ci, 999 int (*func)(void *), void *arg, char *name) 1000 { 1001 struct agintc_softc *sc = agintc_sc; 1002 struct intrhand *ih; 1003 int psw; 1004 1005 if (irqno < 0 || (irqno >= sc->sc_nintr && irqno < LPI_BASE) || 1006 irqno >= LPI_BASE + sc->sc_nlpi) 1007 panic("agintc_intr_establish: bogus irqnumber %d: %s", 1008 irqno, name); 1009 1010 if (ci == NULL) 1011 ci = &cpu_info_primary; 1012 1013 ih = malloc(sizeof *ih, M_DEVBUF, M_WAITOK); 1014 ih->ih_func = func; 1015 ih->ih_arg = arg; 1016 ih->ih_ipl = level & IPL_IRQMASK; 1017 ih->ih_flags = level & IPL_FLAGMASK; 1018 ih->ih_irq = irqno; 1019 ih->ih_name = name; 1020 ih->ih_ci = ci; 1021 1022 psw = disable_interrupts(); 1023 1024 if (irqno < LPI_BASE) { 1025 if (!TAILQ_EMPTY(&sc->sc_handler[irqno].iq_list) && 1026 sc->sc_handler[irqno].iq_ci != ci) { 1027 free(ih, M_DEVBUF, sizeof *ih); 1028 restore_interrupts(psw); 1029 return NULL; 1030 } 1031 TAILQ_INSERT_TAIL(&sc->sc_handler[irqno].iq_list, ih, ih_list); 1032 sc->sc_handler[irqno].iq_ci = ci; 1033 } else 1034 sc->sc_lpi_handler[irqno - LPI_BASE] = ih; 1035 1036 if (name != NULL) 1037 evcount_attach(&ih->ih_count, name, &ih->ih_irq); 1038 1039 #ifdef DEBUG_AGINTC 1040 printf("%s: irq %d level %d [%s]\n", __func__, irqno, level, name); 1041 #endif 1042 1043 if (irqno < LPI_BASE) { 1044 agintc_calc_irq(sc, irqno); 1045 } else { 1046 uint8_t *prop = AGINTC_DMA_KVA(sc->sc_prop); 1047 1048 prop[irqno - LPI_BASE] = (((0xff - ih->ih_ipl) << 4) & 0xff) | 1049 GICR_PROP_GROUP1 | GICR_PROP_ENABLE; 1050 1051 /* Make globally visible. */ 1052 cpu_dcache_wb_range((vaddr_t)prop, 1); 1053 __asm volatile("dsb sy"); 1054 } 1055 1056 restore_interrupts(psw); 1057 return (ih); 1058 } 1059 1060 void 1061 agintc_intr_disestablish(void *cookie) 1062 { 1063 struct agintc_softc *sc = agintc_sc; 1064 struct intrhand *ih = cookie; 1065 int irqno = ih->ih_irq; 1066 int psw; 1067 1068 psw = disable_interrupts(); 1069 1070 TAILQ_REMOVE(&sc->sc_handler[irqno].iq_list, ih, ih_list); 1071 if (ih->ih_name != NULL) 1072 evcount_detach(&ih->ih_count); 1073 1074 agintc_calc_irq(sc, irqno); 1075 1076 restore_interrupts(psw); 1077 1078 free(ih, M_DEVBUF, 0); 1079 } 1080 1081 void 1082 agintc_eoi(uint32_t eoi) 1083 { 1084 __asm volatile("msr "STR(ICC_EOIR1)", %x0" :: "r" (eoi)); 1085 __isb(); 1086 } 1087 1088 void 1089 agintc_d_wait_rwp(struct agintc_softc *sc) 1090 { 1091 int count = 100000; 1092 uint32_t v; 1093 1094 do { 1095 v = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR); 1096 } while (--count && (v & GICD_CTLR_RWP)); 1097 1098 if (count == 0) 1099 panic("%s: RWP timed out 0x08%x", __func__, v); 1100 } 1101 1102 void 1103 agintc_r_wait_rwp(struct agintc_softc *sc) 1104 { 1105 struct cpu_info *ci = curcpu(); 1106 int hwcpu = sc->sc_cpuremap[ci->ci_cpuid]; 1107 int count = 100000; 1108 uint32_t v; 1109 1110 do { 1111 v = bus_space_read_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 1112 GICR_CTLR); 1113 } while (--count && (v & GICR_CTLR_RWP)); 1114 1115 if (count == 0) 1116 panic("%s: RWP timed out 0x08%x", __func__, v); 1117 } 1118 1119 #ifdef MULTIPROCESSOR 1120 int 1121 agintc_ipi_ddb(void *v) 1122 { 1123 /* XXX */ 1124 #ifdef DDB 1125 db_enter(); 1126 #endif 1127 return 1; 1128 } 1129 1130 int 1131 agintc_ipi_nop(void *v) 1132 { 1133 /* Nothing to do here, just enough to wake up from WFI */ 1134 return 1; 1135 } 1136 1137 int 1138 agintc_ipi_combined(void *v) 1139 { 1140 struct agintc_softc *sc = v; 1141 1142 if (sc->sc_ipi_reason[cpu_number()] == ARM_IPI_DDB) { 1143 sc->sc_ipi_reason[cpu_number()] = ARM_IPI_NOP; 1144 return agintc_ipi_ddb(v); 1145 } else { 1146 return agintc_ipi_nop(v); 1147 } 1148 } 1149 1150 void 1151 agintc_send_ipi(struct cpu_info *ci, int id) 1152 { 1153 struct agintc_softc *sc = agintc_sc; 1154 uint64_t sendmask; 1155 1156 if (ci == curcpu() && id == ARM_IPI_NOP) 1157 return; 1158 1159 /* never overwrite IPI_DDB with IPI_NOP */ 1160 if (id == ARM_IPI_DDB) 1161 sc->sc_ipi_reason[ci->ci_cpuid] = id; 1162 1163 /* will only send 1 cpu */ 1164 sendmask = (ci->ci_mpidr & MPIDR_AFF3) << 16; 1165 sendmask |= (ci->ci_mpidr & MPIDR_AFF2) << 16; 1166 sendmask |= (ci->ci_mpidr & MPIDR_AFF1) << 8; 1167 sendmask |= 1 << (ci->ci_mpidr & 0x0f); 1168 sendmask |= (sc->sc_ipi_num[id] << 24); 1169 1170 __asm volatile ("msr " STR(ICC_SGI1R)", %x0" ::"r"(sendmask)); 1171 } 1172 #endif 1173 1174 /* 1175 * GICv3 ITS controller for MSI interrupts. 1176 */ 1177 #define GITS_CTLR 0x0000 1178 #define GITS_CTLR_ENABLED (1UL << 0) 1179 #define GITS_TYPER 0x0008 1180 #define GITS_TYPER_CIL (1ULL << 36) 1181 #define GITS_TYPER_HCC(x) (((x) >> 24) & 0xff) 1182 #define GITS_TYPER_PTA (1ULL << 19) 1183 #define GITS_TYPER_DEVBITS(x) (((x) >> 13) & 0x1f) 1184 #define GITS_TYPER_ITE_SZ(x) (((x) >> 4) & 0xf) 1185 #define GITS_TYPER_PHYS (1ULL << 0) 1186 #define GITS_CBASER 0x0080 1187 #define GITS_CBASER_VALID (1ULL << 63) 1188 #define GITS_CBASER_IC_NORM_NC (1ULL << 59) 1189 #define GITS_CBASER_MASK 0x1ffffffffff000ULL 1190 #define GITS_CWRITER 0x0088 1191 #define GITS_CREADR 0x0090 1192 #define GITS_BASER(i) (0x0100 + ((i) * 8)) 1193 #define GITS_BASER_VALID (1ULL << 63) 1194 #define GITS_BASER_INDIRECT (1ULL << 62) 1195 #define GITS_BASER_IC_NORM_NC (1ULL << 59) 1196 #define GITS_BASER_TYPE_MASK (7ULL << 56) 1197 #define GITS_BASER_TYPE_DEVICE (1ULL << 56) 1198 #define GITS_BASER_DTE_SZ(x) (((x) >> 48) & 0x1f) 1199 #define GITS_BASER_PGSZ_MASK (3ULL << 8) 1200 #define GITS_BASER_PGSZ_4K (0ULL << 8) 1201 #define GITS_BASER_PGSZ_16K (1ULL << 8) 1202 #define GITS_BASER_PGSZ_64K (2ULL << 8) 1203 #define GITS_BASER_PA_MASK 0x7ffffffff000ULL 1204 #define GITS_TRANSLATER 0x10040 1205 1206 #define GITS_NUM_BASER 8 1207 1208 struct gits_cmd { 1209 uint8_t cmd; 1210 uint32_t deviceid; 1211 uint32_t eventid; 1212 uint32_t intid; 1213 uint64_t dw2; 1214 uint64_t dw3; 1215 }; 1216 1217 #define GITS_CMD_VALID (1ULL << 63) 1218 1219 /* ITS commands */ 1220 #define SYNC 0x05 1221 #define MAPD 0x08 1222 #define MAPC 0x09 1223 #define MAPTI 0x0a 1224 1225 #define GITS_CMDQ_SIZE (64 * 1024) 1226 #define GITS_CMDQ_NENTRIES (GITS_CMDQ_SIZE / sizeof(struct gits_cmd)) 1227 1228 struct agintc_msi_device { 1229 LIST_ENTRY(agintc_msi_device) md_list; 1230 1231 uint32_t md_deviceid; 1232 uint32_t md_eventid; 1233 struct agintc_dmamem *md_itt; 1234 }; 1235 1236 int agintc_msi_match(struct device *, void *, void *); 1237 void agintc_msi_attach(struct device *, struct device *, void *); 1238 void *agintc_intr_establish_msi(void *, uint64_t *, uint64_t *, 1239 int , struct cpu_info *, int (*)(void *), void *, char *); 1240 void agintc_intr_disestablish_msi(void *); 1241 void agintc_intr_barrier_msi(void *); 1242 1243 struct agintc_msi_softc { 1244 struct device sc_dev; 1245 bus_space_tag_t sc_iot; 1246 bus_space_handle_t sc_ioh; 1247 bus_dma_tag_t sc_dmat; 1248 1249 bus_addr_t sc_msi_addr; 1250 int sc_msi_delta; 1251 1252 int sc_nlpi; 1253 void **sc_lpi; 1254 1255 struct agintc_dmamem *sc_cmdq; 1256 uint16_t sc_cmdidx; 1257 1258 int sc_devbits; 1259 struct agintc_dmamem *sc_dtt; 1260 size_t sc_dtt_pgsz; 1261 uint8_t sc_dte_sz; 1262 uint8_t sc_ite_sz; 1263 1264 LIST_HEAD(, agintc_msi_device) sc_msi_devices; 1265 1266 struct interrupt_controller sc_ic; 1267 }; 1268 1269 struct cfattach agintcmsi_ca = { 1270 sizeof (struct agintc_msi_softc), agintc_msi_match, agintc_msi_attach 1271 }; 1272 1273 struct cfdriver agintcmsi_cd = { 1274 NULL, "agintcmsi", DV_DULL 1275 }; 1276 1277 void agintc_msi_send_cmd(struct agintc_msi_softc *, struct gits_cmd *); 1278 void agintc_msi_wait_cmd(struct agintc_msi_softc *); 1279 1280 int 1281 agintc_msi_match(struct device *parent, void *cfdata, void *aux) 1282 { 1283 struct fdt_attach_args *faa = aux; 1284 1285 return OF_is_compatible(faa->fa_node, "arm,gic-v3-its"); 1286 } 1287 1288 void 1289 agintc_msi_attach(struct device *parent, struct device *self, void *aux) 1290 { 1291 struct agintc_msi_softc *sc = (struct agintc_msi_softc *)self; 1292 struct fdt_attach_args *faa = aux; 1293 struct gits_cmd cmd; 1294 uint32_t pre_its[2]; 1295 uint64_t typer; 1296 int i, hwcpu; 1297 1298 if (faa->fa_nreg < 1) { 1299 printf(": no registers\n"); 1300 return; 1301 } 1302 1303 sc->sc_iot = faa->fa_iot; 1304 if (bus_space_map(sc->sc_iot, faa->fa_reg[0].addr, 1305 faa->fa_reg[0].size, 0, &sc->sc_ioh)) { 1306 printf(": can't map registers\n"); 1307 return; 1308 } 1309 sc->sc_dmat = faa->fa_dmat; 1310 1311 sc->sc_msi_addr = faa->fa_reg[0].addr + GITS_TRANSLATER; 1312 if (OF_getpropintarray(faa->fa_node, "socionext,synquacer-pre-its", 1313 pre_its, sizeof(pre_its)) == sizeof(pre_its)) { 1314 sc->sc_msi_addr = pre_its[0]; 1315 sc->sc_msi_delta = 4; 1316 } 1317 1318 typer = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_TYPER); 1319 if ((typer & GITS_TYPER_PHYS) == 0 || typer & GITS_TYPER_PTA || 1320 GITS_TYPER_HCC(typer) < ncpus || typer & GITS_TYPER_CIL) { 1321 printf(": unsupported type 0x%016llx\n", typer); 1322 goto unmap; 1323 } 1324 sc->sc_ite_sz = GITS_TYPER_ITE_SZ(typer) + 1; 1325 sc->sc_devbits = GITS_TYPER_DEVBITS(typer) + 1; 1326 1327 sc->sc_nlpi = agintc_sc->sc_nlpi; 1328 sc->sc_lpi = mallocarray(sc->sc_nlpi, sizeof(void *), M_DEVBUF, 1329 M_WAITOK|M_ZERO); 1330 1331 /* Set up command queue. */ 1332 sc->sc_cmdq = agintc_dmamem_alloc(sc->sc_dmat, 1333 GITS_CMDQ_SIZE, GITS_CMDQ_SIZE); 1334 if (sc->sc_cmdq == NULL) { 1335 printf(": can't alloc command queue\n"); 1336 goto unmap; 1337 } 1338 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_CBASER, 1339 AGINTC_DMA_DVA(sc->sc_cmdq) | GITS_CBASER_IC_NORM_NC | 1340 (GITS_CMDQ_SIZE / PAGE_SIZE) - 1 | GITS_CBASER_VALID); 1341 1342 /* Set up device translation table. */ 1343 for (i = 0; i < GITS_NUM_BASER; i++) { 1344 uint64_t baser; 1345 paddr_t dtt_pa; 1346 size_t size; 1347 1348 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1349 if ((baser & GITS_BASER_TYPE_MASK) != GITS_BASER_TYPE_DEVICE) 1350 continue; 1351 1352 /* Determine the maximum supported page size. */ 1353 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1354 (baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_64K); 1355 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1356 if ((baser & GITS_BASER_PGSZ_MASK) == GITS_BASER_PGSZ_64K) 1357 goto found; 1358 1359 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1360 (baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_16K); 1361 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1362 if ((baser & GITS_BASER_PGSZ_MASK) == GITS_BASER_PGSZ_16K) 1363 goto found; 1364 1365 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1366 (baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_4K); 1367 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1368 1369 found: 1370 switch (baser & GITS_BASER_PGSZ_MASK) { 1371 case GITS_BASER_PGSZ_4K: 1372 sc->sc_dtt_pgsz = PAGE_SIZE; 1373 break; 1374 case GITS_BASER_PGSZ_16K: 1375 sc->sc_dtt_pgsz = 4 * PAGE_SIZE; 1376 break; 1377 case GITS_BASER_PGSZ_64K: 1378 sc->sc_dtt_pgsz = 16 * PAGE_SIZE; 1379 break; 1380 } 1381 1382 /* Calculate table size. */ 1383 sc->sc_dte_sz = GITS_BASER_DTE_SZ(baser) + 1; 1384 size = (1ULL << sc->sc_devbits) * sc->sc_dte_sz; 1385 size = roundup(size, sc->sc_dtt_pgsz); 1386 1387 /* Allocate table. */ 1388 sc->sc_dtt = agintc_dmamem_alloc(sc->sc_dmat, 1389 size, sc->sc_dtt_pgsz); 1390 if (sc->sc_dtt == NULL) { 1391 printf(": can't alloc translation table\n"); 1392 goto unmap; 1393 } 1394 1395 /* Configure table. */ 1396 dtt_pa = AGINTC_DMA_DVA(sc->sc_dtt); 1397 KASSERT((dtt_pa & GITS_BASER_PA_MASK) == dtt_pa); 1398 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1399 GITS_BASER_IC_NORM_NC | baser & GITS_BASER_PGSZ_MASK | 1400 dtt_pa | (size / sc->sc_dtt_pgsz) - 1 | GITS_BASER_VALID); 1401 } 1402 1403 /* Enable ITS. */ 1404 bus_space_write_4(sc->sc_iot, sc->sc_ioh, GITS_CTLR, 1405 GITS_CTLR_ENABLED); 1406 1407 LIST_INIT(&sc->sc_msi_devices); 1408 1409 /* Create one collection per core. */ 1410 KASSERT(ncpus <= agintc_sc->sc_num_redist); 1411 for (i = 0; i < ncpus; i++) { 1412 hwcpu = agintc_sc->sc_cpuremap[i]; 1413 memset(&cmd, 0, sizeof(cmd)); 1414 cmd.cmd = MAPC; 1415 cmd.dw2 = GITS_CMD_VALID | 1416 (agintc_sc->sc_processor[hwcpu] << 16) | i; 1417 agintc_msi_send_cmd(sc, &cmd); 1418 agintc_msi_wait_cmd(sc); 1419 } 1420 1421 printf("\n"); 1422 1423 sc->sc_ic.ic_node = faa->fa_node; 1424 sc->sc_ic.ic_cookie = sc; 1425 sc->sc_ic.ic_establish_msi = agintc_intr_establish_msi; 1426 sc->sc_ic.ic_disestablish = agintc_intr_disestablish_msi; 1427 sc->sc_ic.ic_barrier = agintc_intr_barrier_msi; 1428 arm_intr_register_fdt(&sc->sc_ic); 1429 return; 1430 1431 unmap: 1432 if (sc->sc_dtt) 1433 agintc_dmamem_free(sc->sc_dmat, sc->sc_dtt); 1434 if (sc->sc_cmdq) 1435 agintc_dmamem_free(sc->sc_dmat, sc->sc_cmdq); 1436 1437 if (sc->sc_lpi) 1438 free(sc->sc_lpi, M_DEVBUF, sc->sc_nlpi * sizeof(void *)); 1439 1440 bus_space_unmap(sc->sc_iot, sc->sc_ioh, faa->fa_reg[0].size); 1441 } 1442 1443 void 1444 agintc_msi_send_cmd(struct agintc_msi_softc *sc, struct gits_cmd *cmd) 1445 { 1446 struct gits_cmd *queue = AGINTC_DMA_KVA(sc->sc_cmdq); 1447 1448 memcpy(&queue[sc->sc_cmdidx], cmd, sizeof(*cmd)); 1449 1450 /* Make globally visible. */ 1451 cpu_dcache_wb_range((vaddr_t)&queue[sc->sc_cmdidx], sizeof(*cmd)); 1452 __asm volatile("dsb sy"); 1453 1454 sc->sc_cmdidx++; 1455 sc->sc_cmdidx %= GITS_CMDQ_NENTRIES; 1456 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_CWRITER, 1457 sc->sc_cmdidx * sizeof(*cmd)); 1458 } 1459 1460 void 1461 agintc_msi_wait_cmd(struct agintc_msi_softc *sc) 1462 { 1463 uint64_t creadr; 1464 int timo; 1465 1466 for (timo = 1000; timo > 0; timo--) { 1467 creadr = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_CREADR); 1468 if (creadr == sc->sc_cmdidx * sizeof(struct gits_cmd)) 1469 break; 1470 delay(1); 1471 } 1472 if (timo == 0) 1473 printf("%s: command queue timeout\n", sc->sc_dev.dv_xname); 1474 } 1475 1476 struct agintc_msi_device * 1477 agintc_msi_create_device(struct agintc_msi_softc *sc, uint32_t deviceid) 1478 { 1479 struct agintc_msi_device *md; 1480 struct gits_cmd cmd; 1481 1482 md = malloc(sizeof(*md), M_DEVBUF, M_ZERO | M_WAITOK); 1483 md->md_deviceid = deviceid; 1484 md->md_itt = agintc_dmamem_alloc(sc->sc_dmat, 1485 32 * sc->sc_ite_sz, PAGE_SIZE); 1486 LIST_INSERT_HEAD(&sc->sc_msi_devices, md, md_list); 1487 1488 memset(&cmd, 0, sizeof(cmd)); 1489 cmd.cmd = MAPD; 1490 cmd.deviceid = deviceid; 1491 cmd.eventid = 4; /* size */ 1492 cmd.dw2 = AGINTC_DMA_DVA(md->md_itt) | GITS_CMD_VALID; 1493 agintc_msi_send_cmd(sc, &cmd); 1494 agintc_msi_wait_cmd(sc); 1495 1496 return md; 1497 } 1498 1499 struct agintc_msi_device * 1500 agintc_msi_find_device(struct agintc_msi_softc *sc, uint32_t deviceid) 1501 { 1502 struct agintc_msi_device *md; 1503 1504 LIST_FOREACH(md, &sc->sc_msi_devices, md_list) { 1505 if (md->md_deviceid == deviceid) 1506 return md; 1507 } 1508 1509 return agintc_msi_create_device(sc, deviceid); 1510 } 1511 1512 void * 1513 agintc_intr_establish_msi(void *self, uint64_t *addr, uint64_t *data, 1514 int level, struct cpu_info *ci, int (*func)(void *), void *arg, char *name) 1515 { 1516 struct agintc_msi_softc *sc = (struct agintc_msi_softc *)self; 1517 struct agintc_msi_device *md; 1518 struct gits_cmd cmd; 1519 uint32_t deviceid = *data; 1520 uint32_t eventid; 1521 void *cookie; 1522 int i, hwcpu; 1523 1524 if (ci == NULL) 1525 ci = &cpu_info_primary; 1526 hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid]; 1527 1528 md = agintc_msi_find_device(sc, deviceid); 1529 if (md == NULL) 1530 return NULL; 1531 1532 eventid = md->md_eventid++; 1533 if (eventid >= 32) 1534 return NULL; 1535 1536 for (i = 0; i < sc->sc_nlpi; i++) { 1537 if (sc->sc_lpi[i] != NULL) 1538 continue; 1539 1540 cookie = agintc_intr_establish(LPI_BASE + i, 1541 level, ci, func, arg, name); 1542 if (cookie == NULL) 1543 return NULL; 1544 1545 memset(&cmd, 0, sizeof(cmd)); 1546 cmd.cmd = MAPTI; 1547 cmd.deviceid = deviceid; 1548 cmd.eventid = eventid; 1549 cmd.intid = LPI_BASE + i; 1550 cmd.dw2 = ci->ci_cpuid; 1551 agintc_msi_send_cmd(sc, &cmd); 1552 1553 memset(&cmd, 0, sizeof(cmd)); 1554 cmd.cmd = SYNC; 1555 cmd.dw2 = agintc_sc->sc_processor[hwcpu] << 16; 1556 agintc_msi_send_cmd(sc, &cmd); 1557 agintc_msi_wait_cmd(sc); 1558 1559 *addr = sc->sc_msi_addr + deviceid * sc->sc_msi_delta; 1560 *data = eventid; 1561 sc->sc_lpi[i] = cookie; 1562 return &sc->sc_lpi[i]; 1563 } 1564 1565 return NULL; 1566 } 1567 1568 void 1569 agintc_intr_disestablish_msi(void *cookie) 1570 { 1571 agintc_intr_disestablish(*(void **)cookie); 1572 *(void **)cookie = NULL; 1573 } 1574 1575 void 1576 agintc_intr_barrier_msi(void *cookie) 1577 { 1578 agintc_intr_barrier(*(void **)cookie); 1579 } 1580 1581 struct agintc_dmamem * 1582 agintc_dmamem_alloc(bus_dma_tag_t dmat, bus_size_t size, bus_size_t align) 1583 { 1584 struct agintc_dmamem *adm; 1585 int nsegs; 1586 1587 adm = malloc(sizeof(*adm), M_DEVBUF, M_WAITOK | M_ZERO); 1588 adm->adm_size = size; 1589 1590 if (bus_dmamap_create(dmat, size, 1, size, 0, 1591 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0) 1592 goto admfree; 1593 1594 if (bus_dmamem_alloc(dmat, size, align, 0, &adm->adm_seg, 1, 1595 &nsegs, BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0) 1596 goto destroy; 1597 1598 if (bus_dmamem_map(dmat, &adm->adm_seg, nsegs, size, 1599 &adm->adm_kva, BUS_DMA_WAITOK | BUS_DMA_NOCACHE) != 0) 1600 goto free; 1601 1602 if (bus_dmamap_load_raw(dmat, adm->adm_map, &adm->adm_seg, 1603 nsegs, size, BUS_DMA_WAITOK) != 0) 1604 goto unmap; 1605 1606 /* Make globally visible. */ 1607 cpu_dcache_wb_range((vaddr_t)adm->adm_kva, size); 1608 __asm volatile("dsb sy"); 1609 return adm; 1610 1611 unmap: 1612 bus_dmamem_unmap(dmat, adm->adm_kva, size); 1613 free: 1614 bus_dmamem_free(dmat, &adm->adm_seg, 1); 1615 destroy: 1616 bus_dmamap_destroy(dmat, adm->adm_map); 1617 admfree: 1618 free(adm, M_DEVBUF, sizeof(*adm)); 1619 1620 return NULL; 1621 } 1622 1623 void 1624 agintc_dmamem_free(bus_dma_tag_t dmat, struct agintc_dmamem *adm) 1625 { 1626 bus_dmamem_unmap(dmat, adm->adm_kva, adm->adm_size); 1627 bus_dmamem_free(dmat, &adm->adm_seg, 1); 1628 bus_dmamap_destroy(dmat, adm->adm_map); 1629 free(adm, M_DEVBUF, sizeof(*adm)); 1630 } 1631