1 /* $NetBSD: gic.c,v 1.38 2018/11/16 23:25:09 jmcneill Exp $ */ 2 /*- 3 * Copyright (c) 2012 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Matt Thomas of 3am Software Foundry. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 22 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 * POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 #include "opt_ddb.h" 32 #include "opt_multiprocessor.h" 33 34 #define _INTR_PRIVATE 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: gic.c,v 1.38 2018/11/16 23:25:09 jmcneill Exp $"); 38 39 #include <sys/param.h> 40 #include <sys/bus.h> 41 #include <sys/cpu.h> 42 #include <sys/device.h> 43 #include <sys/evcnt.h> 44 #include <sys/intr.h> 45 #include <sys/proc.h> 46 #include <sys/atomic.h> 47 48 #include <arm/armreg.h> 49 #include <arm/atomic.h> 50 #include <arm/cpufunc.h> 51 #include <arm/locore.h> 52 53 #include <arm/cortex/gic_reg.h> 54 #include <arm/cortex/mpcore_var.h> 55 56 void armgic_irq_handler(void *); 57 58 #define ARMGIC_SGI_IPIBASE 0 59 60 /* 61 * SGIs 8-16 are reserved for use by ARM Trusted Firmware. 62 */ 63 __CTASSERT(ARMGIC_SGI_IPIBASE + NIPI <= 8); 64 65 static int armgic_match(device_t, cfdata_t, void *); 66 static void armgic_attach(device_t, device_t, void *); 67 68 static void armgic_set_priority(struct pic_softc *, int); 69 static void armgic_unblock_irqs(struct pic_softc *, size_t, uint32_t); 70 static void armgic_block_irqs(struct pic_softc *, size_t, uint32_t); 71 static void armgic_establish_irq(struct pic_softc *, struct intrsource *); 72 #if 0 73 static void armgic_source_name(struct pic_softc *, int, char *, size_t); 74 #endif 75 76 #ifdef MULTIPROCESSOR 77 static void armgic_cpu_init(struct pic_softc *, struct cpu_info *); 78 static void armgic_ipi_send(struct pic_softc *, const kcpuset_t *, u_long); 79 static void armgic_get_affinity(struct pic_softc *, size_t, kcpuset_t *); 80 static int armgic_set_affinity(struct pic_softc *, size_t, const kcpuset_t *); 81 #endif 82 83 static const struct pic_ops armgic_picops = { 84 .pic_unblock_irqs = armgic_unblock_irqs, 85 .pic_block_irqs = armgic_block_irqs, 86 .pic_establish_irq = armgic_establish_irq, 87 #if 0 88 .pic_source_name = armgic_source_name, 89 #endif 90 .pic_set_priority = armgic_set_priority, 91 #ifdef MULTIPROCESSOR 92 .pic_cpu_init = armgic_cpu_init, 93 .pic_ipi_send = armgic_ipi_send, 94 .pic_get_affinity = armgic_get_affinity, 95 .pic_set_affinity = armgic_set_affinity, 96 #endif 97 }; 98 99 #define PICTOSOFTC(pic) ((struct armgic_softc *)(pic)) 100 101 static struct armgic_softc { 102 struct pic_softc sc_pic; 103 device_t sc_dev; 104 bus_space_tag_t sc_memt; 105 bus_space_handle_t sc_gicch; 106 bus_space_handle_t sc_gicdh; 107 size_t sc_gic_lines; 108 uint32_t sc_gic_type; 109 uint32_t sc_gic_valid_lines[1024/32]; 110 uint32_t sc_enabled_local; 111 #ifdef MULTIPROCESSOR 112 uint32_t sc_target[MAXCPUS]; 113 uint32_t sc_mptargets; 114 #endif 115 uint32_t sc_bptargets; 116 } armgic_softc = { 117 .sc_pic = { 118 .pic_ops = &armgic_picops, 119 .pic_name = "armgic", 120 }, 121 }; 122 123 static struct intrsource armgic_dummy_source; 124 125 __CTASSERT(NIPL == 8); 126 127 /* 128 * GIC register are always in little-endian. It is assumed the bus_space 129 * will do any endian conversion required. 130 */ 131 static inline uint32_t 132 gicc_read(struct armgic_softc *sc, bus_size_t o) 133 { 134 return bus_space_read_4(sc->sc_memt, sc->sc_gicch, o); 135 } 136 137 static inline void 138 gicc_write(struct armgic_softc *sc, bus_size_t o, uint32_t v) 139 { 140 bus_space_write_4(sc->sc_memt, sc->sc_gicch, o, v); 141 } 142 143 static inline uint32_t 144 gicd_read(struct armgic_softc *sc, bus_size_t o) 145 { 146 return bus_space_read_4(sc->sc_memt, sc->sc_gicdh, o); 147 } 148 149 static inline void 150 gicd_write(struct armgic_softc *sc, bus_size_t o, uint32_t v) 151 { 152 bus_space_write_4(sc->sc_memt, sc->sc_gicdh, o, v); 153 } 154 155 static uint32_t 156 gicd_find_targets(struct armgic_softc *sc) 157 { 158 uint32_t targets = 0; 159 160 /* 161 * GICD_ITARGETSR0 through 7 are read-only, and each field returns 162 * a value that corresponds only to the processor reading the 163 * register. Use this to determine the current processor's 164 * CPU interface number. 165 */ 166 for (int i = 0; i < 8; i++) { 167 targets = gicd_read(sc, GICD_ITARGETSRn(i)); 168 if (targets != 0) 169 break; 170 } 171 targets |= (targets >> 16); 172 targets |= (targets >> 8); 173 targets &= 0xff; 174 175 return targets ? targets : 1; 176 } 177 178 /* 179 * In the GIC prioritization scheme, lower numbers have higher priority. 180 * Only write priorities that could be non-secure. 181 */ 182 static inline uint32_t 183 armgic_ipl_to_priority(int ipl) 184 { 185 return GICC_PMR_NONSECURE 186 | ((IPL_HIGH - ipl) * GICC_PMR_NS_PRIORITIES / NIPL); 187 } 188 189 #if 0 190 static inline int 191 armgic_priority_to_ipl(uint32_t priority) 192 { 193 return IPL_HIGH 194 - (priority & ~GICC_PMR_NONSECURE) * NIPL / GICC_PMR_NS_PRIORITIES; 195 } 196 #endif 197 198 static void 199 armgic_unblock_irqs(struct pic_softc *pic, size_t irq_base, uint32_t irq_mask) 200 { 201 struct armgic_softc * const sc = PICTOSOFTC(pic); 202 const size_t group = irq_base / 32; 203 204 if (group == 0) 205 sc->sc_enabled_local |= irq_mask; 206 207 gicd_write(sc, GICD_ISENABLERn(group), irq_mask); 208 } 209 210 static void 211 armgic_block_irqs(struct pic_softc *pic, size_t irq_base, uint32_t irq_mask) 212 { 213 struct armgic_softc * const sc = PICTOSOFTC(pic); 214 const size_t group = irq_base / 32; 215 216 if (group == 0) 217 sc->sc_enabled_local &= ~irq_mask; 218 219 gicd_write(sc, GICD_ICENABLERn(group), irq_mask); 220 } 221 222 static void 223 armgic_set_priority(struct pic_softc *pic, int ipl) 224 { 225 struct armgic_softc * const sc = PICTOSOFTC(pic); 226 227 const uint32_t priority = armgic_ipl_to_priority(ipl); 228 gicc_write(sc, GICC_PMR, priority); 229 } 230 231 #ifdef MULTIPROCESSOR 232 static void 233 armgic_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 234 { 235 struct armgic_softc * const sc = PICTOSOFTC(pic); 236 const size_t group = irq / 32; 237 int n; 238 239 kcpuset_zero(affinity); 240 if (group == 0) { 241 /* All CPUs are targets for group 0 (SGI/PPI) */ 242 for (n = 0; n < MAXCPUS; n++) { 243 if (sc->sc_target[n] != 0) 244 kcpuset_set(affinity, n); 245 } 246 } else { 247 /* Find distributor targets (SPI) */ 248 const u_int byte_shift = 8 * (irq & 3); 249 const bus_size_t targets_reg = GICD_ITARGETSRn(irq / 4); 250 const uint32_t targets = gicd_read(sc, targets_reg); 251 const uint32_t targets_val = (targets >> byte_shift) & 0xff; 252 253 for (n = 0; n < MAXCPUS; n++) { 254 if (sc->sc_target[n] & targets_val) 255 kcpuset_set(affinity, n); 256 } 257 } 258 } 259 260 static int 261 armgic_set_affinity(struct pic_softc *pic, size_t irq, 262 const kcpuset_t *affinity) 263 { 264 struct armgic_softc * const sc = PICTOSOFTC(pic); 265 const size_t group = irq / 32; 266 if (group == 0) 267 return EINVAL; 268 269 const u_int byte_shift = 8 * (irq & 3); 270 const bus_size_t targets_reg = GICD_ITARGETSRn(irq / 4); 271 uint32_t targets_val = 0; 272 int n; 273 274 for (n = 0; n < MAXCPUS; n++) { 275 if (kcpuset_isset(affinity, n)) 276 targets_val |= sc->sc_target[n]; 277 } 278 279 uint32_t targets = gicd_read(sc, targets_reg); 280 targets &= ~(0xff << byte_shift); 281 targets |= (targets_val << byte_shift); 282 gicd_write(sc, targets_reg, targets); 283 284 return 0; 285 } 286 #endif 287 288 #ifdef __HAVE_PIC_FAST_SOFTINTS 289 void 290 softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep_p) 291 { 292 lwp_t **lp = &l->l_cpu->ci_softlwps[level]; 293 KASSERT(*lp == NULL || *lp == l); 294 *lp = l; 295 /* 296 * Really easy. Just tell it to trigger the local CPU. 297 */ 298 *machdep_p = GICD_SGIR_TargetListFilter_Me 299 | __SHIFTIN(level, GICD_SGIR_SGIINTID); 300 } 301 302 void 303 softint_trigger(uintptr_t machdep) 304 { 305 306 gicd_write(&armgic_softc, GICD_SGIR, machdep); 307 } 308 #endif 309 310 void 311 armgic_irq_handler(void *tf) 312 { 313 struct cpu_info * const ci = curcpu(); 314 struct armgic_softc * const sc = &armgic_softc; 315 const int old_ipl = ci->ci_cpl; 316 #ifdef DIAGNOSTIC 317 const int old_mtx_count = ci->ci_mtx_count; 318 const int old_l_biglocks = ci->ci_curlwp->l_biglocks; 319 #endif 320 #ifdef DEBUG 321 size_t n = 0; 322 #endif 323 324 ci->ci_data.cpu_nintr++; 325 326 for (;;) { 327 uint32_t iar = gicc_read(sc, GICC_IAR); 328 uint32_t irq = __SHIFTOUT(iar, GICC_IAR_IRQ); 329 330 if (irq == GICC_IAR_IRQ_SPURIOUS || 331 irq == GICC_IAR_IRQ_SSPURIOUS) { 332 iar = gicc_read(sc, GICC_IAR); 333 irq = __SHIFTOUT(iar, GICC_IAR_IRQ); 334 if (irq == GICC_IAR_IRQ_SPURIOUS) 335 break; 336 if (irq == GICC_IAR_IRQ_SSPURIOUS) { 337 break; 338 } 339 } 340 341 KASSERTMSG(old_ipl != IPL_HIGH, "old_ipl %d pmr %#x hppir %#x", 342 old_ipl, gicc_read(sc, GICC_PMR), gicc_read(sc, GICC_HPPIR)); 343 344 //const uint32_t cpuid = __SHIFTOUT(iar, GICC_IAR_CPUID_MASK); 345 struct intrsource * const is = sc->sc_pic.pic_sources[irq]; 346 KASSERT(is != &armgic_dummy_source); 347 348 /* 349 * GIC has asserted IPL for us so we can just update ci_cpl. 350 * 351 * But it's not that simple. We may have already bumped ci_cpl 352 * due to a high priority interrupt and now we are about to 353 * dispatch one lower than the previous. It's possible for 354 * that previous interrupt to have deferred some interrupts 355 * so we need deal with those when lowering to the current 356 * interrupt's ipl. 357 * 358 * However, if are just raising ipl, we can just update ci_cpl. 359 */ 360 const int ipl = is->is_ipl; 361 if (__predict_false(ipl < ci->ci_cpl)) { 362 pic_do_pending_ints(I32_bit, ipl, tf); 363 KASSERT(ci->ci_cpl == ipl); 364 } else { 365 KASSERTMSG(ipl > ci->ci_cpl, "ipl %d cpl %d hw-ipl %#x", 366 ipl, ci->ci_cpl, 367 gicc_read(sc, GICC_PMR)); 368 gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(ipl)); 369 ci->ci_cpl = ipl; 370 } 371 cpsie(I32_bit); 372 pic_dispatch(is, tf); 373 cpsid(I32_bit); 374 gicc_write(sc, GICC_EOIR, iar); 375 #ifdef DEBUG 376 n++; 377 KDASSERTMSG(n < 5, "%s: processed too many (%zu)", 378 ci->ci_data.cpu_name, n); 379 #endif 380 } 381 382 /* 383 * Now handle any pending ints. 384 */ 385 pic_do_pending_ints(I32_bit, old_ipl, tf); 386 KASSERTMSG(ci->ci_cpl == old_ipl, "ci_cpl %d old_ipl %d", ci->ci_cpl, old_ipl); 387 KASSERT(old_mtx_count == ci->ci_mtx_count); 388 KASSERT(old_l_biglocks == ci->ci_curlwp->l_biglocks); 389 } 390 391 void 392 armgic_establish_irq(struct pic_softc *pic, struct intrsource *is) 393 { 394 struct armgic_softc * const sc = PICTOSOFTC(pic); 395 const size_t group = is->is_irq / 32; 396 const u_int irq = is->is_irq & 31; 397 const u_int byte_shift = 8 * (irq & 3); 398 const u_int twopair_shift = 2 * (irq & 15); 399 400 KASSERTMSG(sc->sc_gic_valid_lines[group] & __BIT(irq), 401 "irq %u: not valid (group[%zu]=0x%08x [0x%08x])", 402 is->is_irq, group, sc->sc_gic_valid_lines[group], 403 (uint32_t)__BIT(irq)); 404 405 KASSERTMSG(is->is_type == IST_LEVEL || is->is_type == IST_EDGE, 406 "irq %u: type %u unsupported", is->is_irq, is->is_type); 407 408 const bus_size_t targets_reg = GICD_ITARGETSRn(is->is_irq / 4); 409 const bus_size_t cfg_reg = GICD_ICFGRn(is->is_irq / 16); 410 uint32_t targets = gicd_read(sc, targets_reg); 411 uint32_t cfg = gicd_read(sc, cfg_reg); 412 413 if (group > 0) { 414 /* 415 * There are 4 irqs per TARGETS register. For now bind 416 * to the primary cpu. 417 */ 418 targets &= ~(0xff << byte_shift); 419 #if 0 420 #ifdef MULTIPROCESSOR 421 if (is->is_mpsafe) { 422 targets |= sc->sc_mptargets << byte_shift; 423 } else 424 #endif 425 #endif 426 targets |= sc->sc_bptargets << byte_shift; 427 gicd_write(sc, targets_reg, targets); 428 429 /* 430 * There are 16 irqs per CFG register. 10=EDGE 00=LEVEL 431 */ 432 uint32_t new_cfg = cfg; 433 uint32_t old_cfg = (cfg >> twopair_shift) & 3; 434 if (is->is_type == IST_LEVEL && (old_cfg & 2) != 0) { 435 new_cfg &= ~(3 << twopair_shift); 436 } else if (is->is_type == IST_EDGE && (old_cfg & 2) == 0) { 437 new_cfg |= 2 << twopair_shift; 438 } 439 if (new_cfg != cfg) { 440 gicd_write(sc, cfg_reg, new_cfg); 441 } 442 #ifdef MULTIPROCESSOR 443 } else { 444 /* 445 * All group 0 interrupts are per processor and MPSAFE by 446 * default. 447 */ 448 is->is_mpsafe = true; 449 #endif 450 } 451 452 /* 453 * There are 4 irqs per PRIORITY register. Map the IPL 454 * to GIC priority. 455 */ 456 const bus_size_t priority_reg = GICD_IPRIORITYRn(is->is_irq / 4); 457 uint32_t priority = gicd_read(sc, priority_reg); 458 priority &= ~(0xff << byte_shift); 459 priority |= armgic_ipl_to_priority(is->is_ipl) << byte_shift; 460 gicd_write(sc, priority_reg, priority); 461 } 462 463 #ifdef MULTIPROCESSOR 464 static void 465 armgic_cpu_init_priorities(struct armgic_softc *sc) 466 { 467 /* Set lowest priority, i.e. disable interrupts */ 468 for (size_t i = 0; i < sc->sc_pic.pic_maxsources; i += 4) { 469 const bus_size_t priority_reg = GICD_IPRIORITYRn(i / 4); 470 gicd_write(sc, priority_reg, ~0); 471 } 472 } 473 474 static void 475 armgic_cpu_update_priorities(struct armgic_softc *sc) 476 { 477 uint32_t enabled = sc->sc_enabled_local; 478 for (size_t i = 0; i < sc->sc_pic.pic_maxsources; i += 4, enabled >>= 4) { 479 const bus_size_t priority_reg = GICD_IPRIORITYRn(i / 4); 480 uint32_t priority = gicd_read(sc, priority_reg); 481 uint32_t byte_mask = 0xff; 482 size_t byte_shift = 0; 483 for (size_t j = 0; j < 4; j++, byte_mask <<= 8, byte_shift += 8) { 484 struct intrsource * const is = sc->sc_pic.pic_sources[i+j]; 485 priority |= byte_mask; 486 if (is == NULL || is == &armgic_dummy_source) 487 continue; 488 priority &= ~byte_mask; 489 priority |= armgic_ipl_to_priority(is->is_ipl) << byte_shift; 490 } 491 gicd_write(sc, priority_reg, priority); 492 } 493 } 494 495 static void 496 armgic_cpu_init_targets(struct armgic_softc *sc) 497 { 498 /* 499 * Update the mpsafe targets 500 */ 501 for (size_t irq = 32; irq < sc->sc_pic.pic_maxsources; irq++) { 502 struct intrsource * const is = sc->sc_pic.pic_sources[irq]; 503 const bus_size_t targets_reg = GICD_ITARGETSRn(irq / 4); 504 if (is != NULL && is->is_mpsafe) { 505 const u_int byte_shift = 8 * (irq & 3); 506 uint32_t targets = gicd_read(sc, targets_reg); 507 targets |= sc->sc_mptargets << byte_shift; 508 gicd_write(sc, targets_reg, targets); 509 } 510 } 511 } 512 513 void 514 armgic_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 515 { 516 struct armgic_softc * const sc = PICTOSOFTC(pic); 517 sc->sc_target[cpu_index(ci)] = gicd_find_targets(sc); 518 atomic_or_32(&sc->sc_mptargets, sc->sc_target[cpu_index(ci)]); 519 KASSERTMSG(ci->ci_cpl == IPL_HIGH, "ipl %d not IPL_HIGH", ci->ci_cpl); 520 armgic_cpu_init_priorities(sc); 521 if (!CPU_IS_PRIMARY(ci)) { 522 if (popcount(sc->sc_mptargets) != 1) { 523 armgic_cpu_init_targets(sc); 524 } 525 if (sc->sc_enabled_local) { 526 armgic_cpu_update_priorities(sc); 527 gicd_write(sc, GICD_ISENABLERn(0), 528 sc->sc_enabled_local); 529 } 530 } 531 gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(ci->ci_cpl)); // set PMR 532 gicc_write(sc, GICC_CTRL, GICC_CTRL_V1_Enable); // enable interrupt 533 cpsie(I32_bit); // allow IRQ exceptions 534 } 535 536 void 537 armgic_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi) 538 { 539 struct armgic_softc * const sc = PICTOSOFTC(pic); 540 541 #if 0 542 if (ipi == IPI_NOP) { 543 __asm __volatile("sev"); 544 return; 545 } 546 #endif 547 548 uint32_t sgir = __SHIFTIN(ARMGIC_SGI_IPIBASE + ipi, GICD_SGIR_SGIINTID); 549 if (kcp != NULL) { 550 uint32_t targets_val = 0; 551 for (int n = 0; n < MAXCPUS; n++) { 552 if (kcpuset_isset(kcp, n)) 553 targets_val |= sc->sc_target[n]; 554 } 555 sgir |= __SHIFTIN(targets_val, GICD_SGIR_TargetList); 556 sgir |= GICD_SGIR_TargetListFilter_List; 557 } else { 558 if (ncpu == 1) 559 return; 560 sgir |= GICD_SGIR_TargetListFilter_NotMe; 561 } 562 563 gicd_write(sc, GICD_SGIR, sgir); 564 } 565 #endif 566 567 int 568 armgic_match(device_t parent, cfdata_t cf, void *aux) 569 { 570 struct mpcore_attach_args * const mpcaa = aux; 571 572 if (strcmp(cf->cf_name, mpcaa->mpcaa_name) != 0) 573 return 0; 574 if (!CPU_ID_CORTEX_P(cputype) || CPU_ID_CORTEX_A8_P(cputype)) 575 return 0; 576 577 return 1; 578 } 579 580 void 581 armgic_attach(device_t parent, device_t self, void *aux) 582 { 583 struct armgic_softc * const sc = &armgic_softc; 584 struct mpcore_attach_args * const mpcaa = aux; 585 586 sc->sc_dev = self; 587 self->dv_private = sc; 588 589 sc->sc_memt = mpcaa->mpcaa_memt; /* provided for us */ 590 bus_space_subregion(sc->sc_memt, mpcaa->mpcaa_memh, mpcaa->mpcaa_off1, 591 4096, &sc->sc_gicdh); 592 bus_space_subregion(sc->sc_memt, mpcaa->mpcaa_memh, mpcaa->mpcaa_off2, 593 4096, &sc->sc_gicch); 594 595 sc->sc_gic_type = gicd_read(sc, GICD_TYPER); 596 sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(sc->sc_gic_type); 597 598 gicc_write(sc, GICC_CTRL, 0); /* disable all interrupts */ 599 gicd_write(sc, GICD_CTRL, 0); /* disable all interrupts */ 600 601 gicc_write(sc, GICC_PMR, 0xff); 602 uint32_t pmr = gicc_read(sc, GICC_PMR); 603 u_int priorities = 1 << popcount32(pmr); 604 605 const uint32_t iidr = gicc_read(sc, GICC_IIDR); 606 const int iidr_prod = __SHIFTOUT(iidr, GICC_IIDR_ProductID); 607 const int iidr_arch = __SHIFTOUT(iidr, GICC_IIDR_ArchVersion); 608 const int iidr_rev = __SHIFTOUT(iidr, GICC_IIDR_Revision); 609 const int iidr_imp = __SHIFTOUT(iidr, GICC_IIDR_Implementer); 610 611 /* 612 * Find the boot processor's CPU interface number. 613 */ 614 sc->sc_bptargets = gicd_find_targets(sc); 615 616 /* 617 * Let's find out how many real sources we have. 618 */ 619 for (size_t i = 0, group = 0; 620 i < sc->sc_pic.pic_maxsources; 621 i += 32, group++) { 622 /* 623 * To figure what sources are real, one enables all interrupts 624 * and then reads back the enable mask so which ones really 625 * got enabled. 626 */ 627 gicd_write(sc, GICD_ISENABLERn(group), 0xffffffff); 628 uint32_t valid = gicd_read(sc, GICD_ISENABLERn(group)); 629 630 /* 631 * Now disable (clear enable) them again. 632 */ 633 gicd_write(sc, GICD_ICENABLERn(group), valid); 634 635 /* 636 * Count how many are valid. 637 */ 638 sc->sc_gic_lines += popcount32(valid); 639 sc->sc_gic_valid_lines[group] = valid; 640 } 641 642 aprint_normal(": Generic Interrupt Controller, " 643 "%zu sources (%zu valid)\n", 644 sc->sc_pic.pic_maxsources, sc->sc_gic_lines); 645 aprint_debug_dev(sc->sc_dev, "Architecture version %d" 646 " (0x%x:%d rev %d)\n", iidr_arch, iidr_imp, iidr_prod, 647 iidr_rev); 648 649 #ifdef MULTIPROCESSOR 650 sc->sc_pic.pic_cpus = kcpuset_running; 651 #endif 652 pic_add(&sc->sc_pic, 0); 653 654 /* 655 * Force the GICD to IPL_HIGH and then enable interrupts. 656 */ 657 struct cpu_info * const ci = curcpu(); 658 KASSERTMSG(ci->ci_cpl == IPL_HIGH, "ipl %d not IPL_HIGH", ci->ci_cpl); 659 armgic_set_priority(&sc->sc_pic, ci->ci_cpl); // set PMR 660 gicd_write(sc, GICD_CTRL, GICD_CTRL_Enable); // enable Distributer 661 gicc_write(sc, GICC_CTRL, GICC_CTRL_V1_Enable); // enable CPU interrupts 662 cpsie(I32_bit); // allow interrupt exceptions 663 664 /* 665 * For each line that isn't valid, we set the intrsource for it to 666 * point at a dummy source so that pic_intr_establish will fail for it. 667 */ 668 for (size_t i = 0, group = 0; 669 i < sc->sc_pic.pic_maxsources; 670 i += 32, group++) { 671 uint32_t invalid = ~sc->sc_gic_valid_lines[group]; 672 for (size_t j = 0; invalid && j < 32; j++, invalid >>= 1) { 673 if (invalid & 1) { 674 sc->sc_pic.pic_sources[i + j] = 675 &armgic_dummy_source; 676 } 677 } 678 } 679 #ifdef __HAVE_PIC_FAST_SOFTINTS 680 intr_establish_xname(SOFTINT_BIO, IPL_SOFTBIO, IST_MPSAFE | IST_EDGE, 681 pic_handle_softint, (void *)SOFTINT_BIO, "softint bio"); 682 intr_establish_xname(SOFTINT_CLOCK, IPL_SOFTCLOCK, IST_MPSAFE | IST_EDGE, 683 pic_handle_softint, (void *)SOFTINT_CLOCK, "softint clock"); 684 intr_establish_xname(SOFTINT_NET, IPL_SOFTNET, IST_MPSAFE | IST_EDGE, 685 pic_handle_softint, (void *)SOFTINT_NET, "softint net"); 686 intr_establish_xname(SOFTINT_SERIAL, IPL_SOFTSERIAL, IST_MPSAFE | IST_EDGE, 687 pic_handle_softint, (void *)SOFTINT_SERIAL, "softint serial"); 688 #endif 689 #ifdef MULTIPROCESSOR 690 armgic_cpu_init(&sc->sc_pic, curcpu()); 691 692 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_AST, IPL_VM, 693 IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast"); 694 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_XCALL, IPL_HIGH, 695 IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall"); 696 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_GENERIC, IPL_HIGH, 697 IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic"); 698 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_NOP, IPL_VM, 699 IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop"); 700 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_SHOOTDOWN, IPL_SCHED, 701 IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown"); 702 #ifdef DDB 703 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_DDB, IPL_HIGH, 704 IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb"); 705 #endif 706 #ifdef __HAVE_PREEMPTION 707 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_KPREEMPT, IPL_VM, 708 IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt"); 709 #endif 710 #endif 711 712 const u_int ppis = popcount32(sc->sc_gic_valid_lines[0] >> 16); 713 const u_int sgis = popcount32(sc->sc_gic_valid_lines[0] & 0xffff); 714 aprint_normal_dev(sc->sc_dev, "%u Priorities, %zu SPIs, %u PPIs, " 715 "%u SGIs\n", priorities, sc->sc_gic_lines - ppis - sgis, ppis, 716 sgis); 717 } 718 719 CFATTACH_DECL_NEW(armgic, 0, 720 armgic_match, armgic_attach, NULL, NULL); 721