1 /* $NetBSD: gic.c,v 1.51 2021/10/21 04:47:57 skrll Exp $ */ 2 /*- 3 * Copyright (c) 2012 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Matt Thomas of 3am Software Foundry. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 22 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 * POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 #include "opt_ddb.h" 32 #include "opt_multiprocessor.h" 33 #include "opt_gic.h" 34 35 #define _INTR_PRIVATE 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: gic.c,v 1.51 2021/10/21 04:47:57 skrll Exp $"); 39 40 #include <sys/param.h> 41 #include <sys/bus.h> 42 #include <sys/cpu.h> 43 #include <sys/device.h> 44 #include <sys/evcnt.h> 45 #include <sys/intr.h> 46 #include <sys/proc.h> 47 #include <sys/atomic.h> 48 49 #include <arm/armreg.h> 50 #include <arm/cpufunc.h> 51 #include <arm/locore.h> 52 53 #include <arm/cortex/gic_reg.h> 54 #include <arm/cortex/mpcore_var.h> 55 56 #ifdef GIC_SPLFUNCS 57 #include <arm/cortex/gic_splfuncs.h> 58 #endif 59 60 void armgic_irq_handler(void *); 61 62 #define ARMGIC_SGI_IPIBASE 0 63 64 /* 65 * SGIs 8-16 are reserved for use by ARM Trusted Firmware. 66 */ 67 __CTASSERT(ARMGIC_SGI_IPIBASE + NIPI <= 8); 68 69 static int armgic_match(device_t, cfdata_t, void *); 70 static void armgic_attach(device_t, device_t, void *); 71 72 static void armgic_set_priority(struct pic_softc *, int); 73 static void armgic_unblock_irqs(struct pic_softc *, size_t, uint32_t); 74 static void armgic_block_irqs(struct pic_softc *, size_t, uint32_t); 75 static void armgic_establish_irq(struct pic_softc *, struct intrsource *); 76 #if 0 77 static void armgic_source_name(struct pic_softc *, int, char *, size_t); 78 #endif 79 80 #ifdef MULTIPROCESSOR 81 static void armgic_cpu_init(struct pic_softc *, struct cpu_info *); 82 static void armgic_ipi_send(struct pic_softc *, const kcpuset_t *, u_long); 83 static void armgic_get_affinity(struct pic_softc *, size_t, kcpuset_t *); 84 static int armgic_set_affinity(struct pic_softc *, size_t, const kcpuset_t *); 85 #endif 86 87 static const struct pic_ops armgic_picops = { 88 .pic_unblock_irqs = armgic_unblock_irqs, 89 .pic_block_irqs = armgic_block_irqs, 90 .pic_establish_irq = armgic_establish_irq, 91 #if 0 92 .pic_source_name = armgic_source_name, 93 #endif 94 .pic_set_priority = armgic_set_priority, 95 #ifdef MULTIPROCESSOR 96 .pic_cpu_init = armgic_cpu_init, 97 .pic_ipi_send = armgic_ipi_send, 98 .pic_get_affinity = armgic_get_affinity, 99 .pic_set_affinity = armgic_set_affinity, 100 #endif 101 }; 102 103 #define PICTOSOFTC(pic) ((struct armgic_softc *)(pic)) 104 105 static struct armgic_softc { 106 struct pic_softc sc_pic; 107 device_t sc_dev; 108 bus_space_tag_t sc_memt; 109 bus_space_handle_t sc_gicch; 110 bus_space_handle_t sc_gicdh; 111 size_t sc_gic_lines; 112 uint32_t sc_gic_type; 113 uint32_t sc_gic_valid_lines[1024/32]; 114 uint32_t sc_enabled_local; 115 #ifdef MULTIPROCESSOR 116 uint32_t sc_target[MAXCPUS]; 117 uint32_t sc_mptargets; 118 #endif 119 uint32_t sc_bptargets; 120 } armgic_softc = { 121 .sc_pic = { 122 .pic_ops = &armgic_picops, 123 .pic_name = "armgic", 124 }, 125 }; 126 127 static struct intrsource armgic_dummy_source; 128 129 __CTASSERT(NIPL == 8); 130 131 /* 132 * GIC register are always in little-endian. It is assumed the bus_space 133 * will do any endian conversion required. 134 */ 135 static inline uint32_t 136 gicc_read(struct armgic_softc *sc, bus_size_t o) 137 { 138 return bus_space_read_4(sc->sc_memt, sc->sc_gicch, o); 139 } 140 141 static inline void 142 gicc_write(struct armgic_softc *sc, bus_size_t o, uint32_t v) 143 { 144 bus_space_write_4(sc->sc_memt, sc->sc_gicch, o, v); 145 } 146 147 static inline uint32_t 148 gicd_read(struct armgic_softc *sc, bus_size_t o) 149 { 150 return bus_space_read_4(sc->sc_memt, sc->sc_gicdh, o); 151 } 152 153 static inline void 154 gicd_write(struct armgic_softc *sc, bus_size_t o, uint32_t v) 155 { 156 bus_space_write_4(sc->sc_memt, sc->sc_gicdh, o, v); 157 } 158 159 static uint32_t 160 gicd_find_targets(struct armgic_softc *sc) 161 { 162 uint32_t targets = 0; 163 164 /* 165 * GICD_ITARGETSR0 through 7 are read-only, and each field returns 166 * a value that corresponds only to the processor reading the 167 * register. Use this to determine the current processor's 168 * CPU interface number. 169 */ 170 for (int i = 0; i < 8; i++) { 171 targets = gicd_read(sc, GICD_ITARGETSRn(i)); 172 if (targets != 0) 173 break; 174 } 175 targets |= (targets >> 16); 176 targets |= (targets >> 8); 177 targets &= 0xff; 178 179 return targets ? targets : 1; 180 } 181 182 /* 183 * In the GIC prioritization scheme, lower numbers have higher priority. 184 * Only write priorities that could be non-secure. 185 */ 186 static inline uint32_t 187 armgic_ipl_to_priority(int ipl) 188 { 189 return GICC_PMR_NONSECURE 190 | ((IPL_HIGH - ipl) * GICC_PMR_NS_PRIORITIES / NIPL); 191 } 192 193 #if 0 194 static inline int 195 armgic_priority_to_ipl(uint32_t priority) 196 { 197 return IPL_HIGH 198 - (priority & ~GICC_PMR_NONSECURE) * NIPL / GICC_PMR_NS_PRIORITIES; 199 } 200 #endif 201 202 static void 203 armgic_unblock_irqs(struct pic_softc *pic, size_t irq_base, uint32_t irq_mask) 204 { 205 struct armgic_softc * const sc = PICTOSOFTC(pic); 206 const size_t group = irq_base / 32; 207 208 if (group == 0) 209 sc->sc_enabled_local |= irq_mask; 210 211 gicd_write(sc, GICD_ISENABLERn(group), irq_mask); 212 } 213 214 static void 215 armgic_block_irqs(struct pic_softc *pic, size_t irq_base, uint32_t irq_mask) 216 { 217 struct armgic_softc * const sc = PICTOSOFTC(pic); 218 const size_t group = irq_base / 32; 219 220 if (group == 0) 221 sc->sc_enabled_local &= ~irq_mask; 222 223 gicd_write(sc, GICD_ICENABLERn(group), irq_mask); 224 } 225 226 static void 227 armgic_set_priority(struct pic_softc *pic, int ipl) 228 { 229 struct armgic_softc * const sc = PICTOSOFTC(pic); 230 struct cpu_info * const ci = curcpu(); 231 232 if (ipl < ci->ci_hwpl) { 233 /* Lowering priority mask */ 234 ci->ci_hwpl = ipl; 235 gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(ipl)); 236 } 237 } 238 239 #ifdef MULTIPROCESSOR 240 static void 241 armgic_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 242 { 243 struct armgic_softc * const sc = PICTOSOFTC(pic); 244 const size_t group = irq / 32; 245 int n; 246 247 kcpuset_zero(affinity); 248 if (group == 0) { 249 /* All CPUs are targets for group 0 (SGI/PPI) */ 250 for (n = 0; n < MAXCPUS; n++) { 251 if (sc->sc_target[n] != 0) 252 kcpuset_set(affinity, n); 253 } 254 } else { 255 /* Find distributor targets (SPI) */ 256 const u_int byte_shift = 8 * (irq & 3); 257 const bus_size_t targets_reg = GICD_ITARGETSRn(irq / 4); 258 const uint32_t targets = gicd_read(sc, targets_reg); 259 const uint32_t targets_val = (targets >> byte_shift) & 0xff; 260 261 for (n = 0; n < MAXCPUS; n++) { 262 if (sc->sc_target[n] & targets_val) 263 kcpuset_set(affinity, n); 264 } 265 } 266 } 267 268 static int 269 armgic_set_affinity(struct pic_softc *pic, size_t irq, 270 const kcpuset_t *affinity) 271 { 272 struct armgic_softc * const sc = PICTOSOFTC(pic); 273 const size_t group = irq / 32; 274 if (group == 0) 275 return EINVAL; 276 277 const u_int byte_shift = 8 * (irq & 3); 278 const bus_size_t targets_reg = GICD_ITARGETSRn(irq / 4); 279 uint32_t targets_val = 0; 280 int n; 281 282 for (n = 0; n < MAXCPUS; n++) { 283 if (kcpuset_isset(affinity, n)) 284 targets_val |= sc->sc_target[n]; 285 } 286 287 uint32_t targets = gicd_read(sc, targets_reg); 288 targets &= ~(0xff << byte_shift); 289 targets |= (targets_val << byte_shift); 290 gicd_write(sc, targets_reg, targets); 291 292 return 0; 293 } 294 #endif 295 296 #ifdef __HAVE_PIC_FAST_SOFTINTS 297 void 298 softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep_p) 299 { 300 lwp_t **lp = &l->l_cpu->ci_softlwps[level]; 301 KASSERT(*lp == NULL || *lp == l); 302 *lp = l; 303 /* 304 * Really easy. Just tell it to trigger the local CPU. 305 */ 306 *machdep_p = GICD_SGIR_TargetListFilter_Me 307 | __SHIFTIN(level, GICD_SGIR_SGIINTID); 308 } 309 310 void 311 softint_trigger(uintptr_t machdep) 312 { 313 314 gicd_write(&armgic_softc, GICD_SGIR, machdep); 315 } 316 #endif 317 318 void 319 armgic_irq_handler(void *tf) 320 { 321 struct cpu_info * const ci = curcpu(); 322 struct armgic_softc * const sc = &armgic_softc; 323 const int old_ipl = ci->ci_cpl; 324 #ifdef DIAGNOSTIC 325 const int old_mtx_count = ci->ci_mtx_count; 326 const int old_l_biglocks = ci->ci_curlwp->l_biglocks; 327 #endif 328 #ifdef DEBUG 329 size_t n = 0; 330 #endif 331 332 ci->ci_data.cpu_nintr++; 333 334 /* 335 * Raise ci_hwpl (and PMR) to ci_cpl and IAR will tell us if the 336 * interrupt that got us here can have its handler run or not. 337 */ 338 if (ci->ci_hwpl <= old_ipl) { 339 ci->ci_hwpl = old_ipl; 340 gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(old_ipl)); 341 /* 342 * we'll get no interrupts when PMR is IPL_HIGH, so bail 343 * early. 344 */ 345 if (old_ipl == IPL_HIGH) { 346 return; 347 } 348 } 349 350 for (;;) { 351 uint32_t iar = gicc_read(sc, GICC_IAR); 352 uint32_t irq = __SHIFTOUT(iar, GICC_IAR_IRQ); 353 354 if (irq == GICC_IAR_IRQ_SPURIOUS || 355 irq == GICC_IAR_IRQ_SSPURIOUS) { 356 iar = gicc_read(sc, GICC_IAR); 357 irq = __SHIFTOUT(iar, GICC_IAR_IRQ); 358 if (irq == GICC_IAR_IRQ_SPURIOUS) 359 break; 360 if (irq == GICC_IAR_IRQ_SSPURIOUS) { 361 break; 362 } 363 } 364 365 KASSERTMSG(old_ipl != IPL_HIGH, "old_ipl %d pmr %#x hppir %#x", 366 old_ipl, gicc_read(sc, GICC_PMR), gicc_read(sc, GICC_HPPIR)); 367 368 //const uint32_t cpuid = __SHIFTOUT(iar, GICC_IAR_CPUID_MASK); 369 struct intrsource * const is = sc->sc_pic.pic_sources[irq]; 370 KASSERT(is != &armgic_dummy_source); 371 372 /* 373 * GIC has asserted IPL for us so we can just update ci_cpl. 374 * 375 * But it's not that simple. We may have already bumped ci_cpl 376 * due to a high priority interrupt and now we are about to 377 * dispatch one lower than the previous. It's possible for 378 * that previous interrupt to have deferred some interrupts 379 * so we need deal with those when lowering to the current 380 * interrupt's ipl. 381 * 382 * However, if are just raising ipl, we can just update ci_cpl. 383 */ 384 385 /* Surely we can KASSERT(ipl < ci->ci_cpl); */ 386 const int ipl = is->is_ipl; 387 if (__predict_false(ipl < ci->ci_cpl)) { 388 pic_do_pending_ints(I32_bit, ipl, tf); 389 KASSERT(ci->ci_cpl == ipl); 390 } else if (ci->ci_cpl != ipl) { 391 KASSERTMSG(ipl > ci->ci_cpl, "ipl %d cpl %d hw-ipl %#x", 392 ipl, ci->ci_cpl, 393 gicc_read(sc, GICC_PMR)); 394 gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(ipl)); 395 ci->ci_hwpl = ci->ci_cpl = ipl; 396 } 397 ENABLE_INTERRUPT(); 398 pic_dispatch(is, tf); 399 DISABLE_INTERRUPT(); 400 gicc_write(sc, GICC_EOIR, iar); 401 #ifdef DEBUG 402 n++; 403 KDASSERTMSG(n < 5, "%s: processed too many (%zu)", 404 ci->ci_data.cpu_name, n); 405 #endif 406 } 407 408 /* 409 * Now handle any pending ints. 410 */ 411 pic_do_pending_ints(I32_bit, old_ipl, tf); 412 KASSERTMSG(ci->ci_cpl == old_ipl, "ci_cpl %d old_ipl %d", ci->ci_cpl, old_ipl); 413 KASSERT(old_mtx_count == ci->ci_mtx_count); 414 KASSERT(old_l_biglocks == ci->ci_curlwp->l_biglocks); 415 } 416 417 void 418 armgic_establish_irq(struct pic_softc *pic, struct intrsource *is) 419 { 420 struct armgic_softc * const sc = PICTOSOFTC(pic); 421 const size_t group = is->is_irq / 32; 422 const u_int irq = is->is_irq & 31; 423 const u_int byte_shift = 8 * (irq & 3); 424 const u_int twopair_shift = 2 * (irq & 15); 425 426 KASSERTMSG(sc->sc_gic_valid_lines[group] & __BIT(irq), 427 "irq %u: not valid (group[%zu]=0x%08x [0x%08x])", 428 is->is_irq, group, sc->sc_gic_valid_lines[group], 429 (uint32_t)__BIT(irq)); 430 431 KASSERTMSG(is->is_type == IST_LEVEL || is->is_type == IST_EDGE, 432 "irq %u: type %u unsupported", is->is_irq, is->is_type); 433 434 const bus_size_t targets_reg = GICD_ITARGETSRn(is->is_irq / 4); 435 const bus_size_t cfg_reg = GICD_ICFGRn(is->is_irq / 16); 436 uint32_t targets = gicd_read(sc, targets_reg); 437 uint32_t cfg = gicd_read(sc, cfg_reg); 438 439 if (group > 0) { 440 /* 441 * There are 4 irqs per TARGETS register. For now bind 442 * to the primary cpu. 443 */ 444 targets &= ~(0xffU << byte_shift); 445 #if 0 446 #ifdef MULTIPROCESSOR 447 if (is->is_mpsafe) { 448 targets |= sc->sc_mptargets << byte_shift; 449 } else 450 #endif 451 #endif 452 targets |= sc->sc_bptargets << byte_shift; 453 gicd_write(sc, targets_reg, targets); 454 455 /* 456 * There are 16 irqs per CFG register. 10=EDGE 00=LEVEL 457 */ 458 uint32_t new_cfg = cfg; 459 uint32_t old_cfg = (cfg >> twopair_shift) & __BITS(1, 0); 460 if (is->is_type == IST_LEVEL && (old_cfg & __BIT(1)) != 0) { 461 new_cfg &= ~(__BITS(1, 0) << twopair_shift); 462 } else if (is->is_type == IST_EDGE && (old_cfg & 2) == 0) { 463 new_cfg |= __BIT(1) << twopair_shift; 464 } 465 if (new_cfg != cfg) { 466 gicd_write(sc, cfg_reg, new_cfg); 467 } 468 #ifdef MULTIPROCESSOR 469 } else { 470 /* 471 * All group 0 interrupts are per processor and MPSAFE by 472 * default. 473 */ 474 is->is_mpsafe = true; 475 is->is_percpu = true; 476 #endif 477 } 478 479 /* 480 * There are 4 irqs per PRIORITY register. Map the IPL 481 * to GIC priority. 482 */ 483 const bus_size_t priority_reg = GICD_IPRIORITYRn(is->is_irq / 4); 484 uint32_t priority = gicd_read(sc, priority_reg); 485 priority &= ~(0xffU << byte_shift); 486 priority |= armgic_ipl_to_priority(is->is_ipl) << byte_shift; 487 gicd_write(sc, priority_reg, priority); 488 } 489 490 #ifdef MULTIPROCESSOR 491 static void 492 armgic_cpu_init_priorities(struct armgic_softc *sc) 493 { 494 /* Set lowest priority, i.e. disable interrupts */ 495 for (size_t i = 0; i < sc->sc_pic.pic_maxsources; i += 4) { 496 const bus_size_t priority_reg = GICD_IPRIORITYRn(i / 4); 497 gicd_write(sc, priority_reg, ~0); 498 } 499 } 500 501 static void 502 armgic_cpu_update_priorities(struct armgic_softc *sc) 503 { 504 uint32_t enabled = sc->sc_enabled_local; 505 for (size_t i = 0; i < sc->sc_pic.pic_maxsources; i += 4, enabled >>= 4) { 506 const bus_size_t priority_reg = GICD_IPRIORITYRn(i / 4); 507 uint32_t priority = gicd_read(sc, priority_reg); 508 uint32_t byte_mask = 0xff; 509 size_t byte_shift = 0; 510 for (size_t j = 0; j < 4; j++, byte_mask <<= 8, byte_shift += 8) { 511 struct intrsource * const is = sc->sc_pic.pic_sources[i+j]; 512 priority |= byte_mask; 513 if (is == NULL || is == &armgic_dummy_source) 514 continue; 515 priority &= ~byte_mask; 516 priority |= armgic_ipl_to_priority(is->is_ipl) << byte_shift; 517 } 518 gicd_write(sc, priority_reg, priority); 519 } 520 } 521 522 static void 523 armgic_cpu_init_targets(struct armgic_softc *sc) 524 { 525 /* 526 * Update the mpsafe targets 527 */ 528 for (size_t irq = 32; irq < sc->sc_pic.pic_maxsources; irq++) { 529 struct intrsource * const is = sc->sc_pic.pic_sources[irq]; 530 const bus_size_t targets_reg = GICD_ITARGETSRn(irq / 4); 531 if (is != NULL && is->is_mpsafe) { 532 const u_int byte_shift = 8 * (irq & 3); 533 uint32_t targets = gicd_read(sc, targets_reg); 534 #if 0 535 targets |= sc->sc_mptargets << byte_shift; 536 #else 537 targets |= sc->sc_bptargets << byte_shift; 538 #endif 539 gicd_write(sc, targets_reg, targets); 540 } 541 } 542 } 543 544 void 545 armgic_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 546 { 547 struct armgic_softc * const sc = PICTOSOFTC(pic); 548 sc->sc_target[cpu_index(ci)] = gicd_find_targets(sc); 549 atomic_or_32(&sc->sc_mptargets, sc->sc_target[cpu_index(ci)]); 550 KASSERTMSG(ci->ci_cpl == IPL_HIGH, "ipl %d not IPL_HIGH", ci->ci_cpl); 551 armgic_cpu_init_priorities(sc); 552 if (!CPU_IS_PRIMARY(ci)) { 553 if (popcount(sc->sc_mptargets) != 1) { 554 armgic_cpu_init_targets(sc); 555 } 556 if (sc->sc_enabled_local) { 557 armgic_cpu_update_priorities(sc); 558 gicd_write(sc, GICD_ISENABLERn(0), 559 sc->sc_enabled_local); 560 } 561 } 562 ci->ci_hwpl = ci->ci_cpl; 563 gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(ci->ci_cpl)); // set PMR 564 gicc_write(sc, GICC_CTRL, GICC_CTRL_V1_Enable); // enable interrupt 565 ENABLE_INTERRUPT(); // allow IRQ exceptions 566 } 567 568 void 569 armgic_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi) 570 { 571 struct armgic_softc * const sc = PICTOSOFTC(pic); 572 573 #if 0 574 if (ipi == IPI_NOP) { 575 sev(); 576 return; 577 } 578 #endif 579 580 uint32_t sgir = __SHIFTIN(ARMGIC_SGI_IPIBASE + ipi, GICD_SGIR_SGIINTID); 581 if (kcp != NULL) { 582 uint32_t targets_val = 0; 583 for (int n = 0; n < MAXCPUS; n++) { 584 if (kcpuset_isset(kcp, n)) 585 targets_val |= sc->sc_target[n]; 586 } 587 sgir |= __SHIFTIN(targets_val, GICD_SGIR_TargetList); 588 sgir |= GICD_SGIR_TargetListFilter_List; 589 } else { 590 if (ncpu == 1) 591 return; 592 sgir |= GICD_SGIR_TargetListFilter_NotMe; 593 } 594 595 gicd_write(sc, GICD_SGIR, sgir); 596 } 597 #endif 598 599 int 600 armgic_match(device_t parent, cfdata_t cf, void *aux) 601 { 602 struct mpcore_attach_args * const mpcaa = aux; 603 604 if (strcmp(cf->cf_name, mpcaa->mpcaa_name) != 0) 605 return 0; 606 607 return 1; 608 } 609 610 void 611 armgic_attach(device_t parent, device_t self, void *aux) 612 { 613 struct armgic_softc * const sc = &armgic_softc; 614 struct mpcore_attach_args * const mpcaa = aux; 615 616 sc->sc_dev = self; 617 self->dv_private = sc; 618 619 sc->sc_memt = mpcaa->mpcaa_memt; /* provided for us */ 620 bus_space_subregion(sc->sc_memt, mpcaa->mpcaa_memh, mpcaa->mpcaa_off1, 621 4096, &sc->sc_gicdh); 622 bus_space_subregion(sc->sc_memt, mpcaa->mpcaa_memh, mpcaa->mpcaa_off2, 623 4096, &sc->sc_gicch); 624 625 sc->sc_gic_type = gicd_read(sc, GICD_TYPER); 626 sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(sc->sc_gic_type); 627 628 gicc_write(sc, GICC_CTRL, 0); /* disable all interrupts */ 629 gicd_write(sc, GICD_CTRL, 0); /* disable all interrupts */ 630 631 gicc_write(sc, GICC_PMR, 0xff); 632 uint32_t pmr = gicc_read(sc, GICC_PMR); 633 u_int priorities = 1 << popcount32(pmr); 634 635 const uint32_t iidr = gicc_read(sc, GICC_IIDR); 636 const int iidr_prod = __SHIFTOUT(iidr, GICC_IIDR_ProductID); 637 const int iidr_arch = __SHIFTOUT(iidr, GICC_IIDR_ArchVersion); 638 const int iidr_rev = __SHIFTOUT(iidr, GICC_IIDR_Revision); 639 const int iidr_imp = __SHIFTOUT(iidr, GICC_IIDR_Implementer); 640 641 /* 642 * Find the boot processor's CPU interface number. 643 */ 644 sc->sc_bptargets = gicd_find_targets(sc); 645 646 /* 647 * Let's find out how many real sources we have. 648 */ 649 for (size_t i = 0, group = 0; 650 i < sc->sc_pic.pic_maxsources; 651 i += 32, group++) { 652 /* 653 * To figure what sources are real, one enables all interrupts 654 * and then reads back the enable mask so which ones really 655 * got enabled. 656 */ 657 gicd_write(sc, GICD_ISENABLERn(group), 0xffffffff); 658 uint32_t valid = gicd_read(sc, GICD_ISENABLERn(group)); 659 660 /* 661 * Now disable (clear enable) them again. 662 */ 663 gicd_write(sc, GICD_ICENABLERn(group), valid); 664 665 /* 666 * Count how many are valid. 667 */ 668 sc->sc_gic_lines += popcount32(valid); 669 sc->sc_gic_valid_lines[group] = valid; 670 } 671 672 aprint_normal(": Generic Interrupt Controller, " 673 "%zu sources (%zu valid)\n", 674 sc->sc_pic.pic_maxsources, sc->sc_gic_lines); 675 aprint_debug_dev(sc->sc_dev, "Architecture version %d" 676 " (0x%x:%d rev %d)\n", iidr_arch, iidr_imp, iidr_prod, 677 iidr_rev); 678 679 #ifdef MULTIPROCESSOR 680 sc->sc_pic.pic_cpus = kcpuset_running; 681 #endif 682 pic_add(&sc->sc_pic, 0); 683 684 /* 685 * Force the GICD to IPL_HIGH and then enable interrupts. 686 */ 687 struct cpu_info * const ci = curcpu(); 688 KASSERTMSG(ci->ci_cpl == IPL_HIGH, "ipl %d not IPL_HIGH", ci->ci_cpl); 689 armgic_set_priority(&sc->sc_pic, ci->ci_cpl); // set PMR 690 gicd_write(sc, GICD_CTRL, GICD_CTRL_Enable); // enable Distributer 691 gicc_write(sc, GICC_CTRL, GICC_CTRL_V1_Enable); // enable CPU interrupts 692 ENABLE_INTERRUPT(); // allow interrupt exceptions 693 694 /* 695 * For each line that isn't valid, we set the intrsource for it to 696 * point at a dummy source so that pic_intr_establish will fail for it. 697 */ 698 for (size_t i = 0, group = 0; 699 i < sc->sc_pic.pic_maxsources; 700 i += 32, group++) { 701 uint32_t invalid = ~sc->sc_gic_valid_lines[group]; 702 for (size_t j = 0; invalid && j < 32; j++, invalid >>= 1) { 703 if (invalid & 1) { 704 sc->sc_pic.pic_sources[i + j] = 705 &armgic_dummy_source; 706 } 707 } 708 } 709 #ifdef __HAVE_PIC_FAST_SOFTINTS 710 intr_establish_xname(SOFTINT_BIO, IPL_SOFTBIO, IST_MPSAFE | IST_EDGE, 711 pic_handle_softint, (void *)SOFTINT_BIO, "softint bio"); 712 intr_establish_xname(SOFTINT_CLOCK, IPL_SOFTCLOCK, IST_MPSAFE | IST_EDGE, 713 pic_handle_softint, (void *)SOFTINT_CLOCK, "softint clock"); 714 intr_establish_xname(SOFTINT_NET, IPL_SOFTNET, IST_MPSAFE | IST_EDGE, 715 pic_handle_softint, (void *)SOFTINT_NET, "softint net"); 716 intr_establish_xname(SOFTINT_SERIAL, IPL_SOFTSERIAL, IST_MPSAFE | IST_EDGE, 717 pic_handle_softint, (void *)SOFTINT_SERIAL, "softint serial"); 718 #endif 719 #ifdef MULTIPROCESSOR 720 armgic_cpu_init(&sc->sc_pic, curcpu()); 721 722 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_AST, IPL_VM, 723 IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast"); 724 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_XCALL, IPL_HIGH, 725 IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall"); 726 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_GENERIC, IPL_HIGH, 727 IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic"); 728 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_NOP, IPL_VM, 729 IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop"); 730 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_SHOOTDOWN, IPL_SCHED, 731 IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown"); 732 #ifdef DDB 733 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_DDB, IPL_HIGH, 734 IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb"); 735 #endif 736 #ifdef __HAVE_PREEMPTION 737 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_KPREEMPT, IPL_VM, 738 IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt"); 739 #endif 740 #endif 741 742 const u_int ppis = popcount32(sc->sc_gic_valid_lines[0] >> 16); 743 const u_int sgis = popcount32(sc->sc_gic_valid_lines[0] & 0xffff); 744 aprint_normal_dev(sc->sc_dev, "%u Priorities, %zu SPIs, %u PPIs, " 745 "%u SGIs\n", priorities, sc->sc_gic_lines - ppis - sgis, ppis, 746 sgis); 747 748 #ifdef GIC_SPLFUNCS 749 gic_spl_init(); 750 #endif 751 } 752 753 CFATTACH_DECL_NEW(armgic, 0, 754 armgic_match, armgic_attach, NULL, NULL); 755