1 /* $NetBSD: gic.c,v 1.47 2021/03/28 09:11:38 skrll Exp $ */ 2 /*- 3 * Copyright (c) 2012 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Matt Thomas of 3am Software Foundry. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 19 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 20 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 22 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 28 * POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 #include "opt_ddb.h" 32 #include "opt_multiprocessor.h" 33 34 #define _INTR_PRIVATE 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: gic.c,v 1.47 2021/03/28 09:11:38 skrll Exp $"); 38 39 #include <sys/param.h> 40 #include <sys/bus.h> 41 #include <sys/cpu.h> 42 #include <sys/device.h> 43 #include <sys/evcnt.h> 44 #include <sys/intr.h> 45 #include <sys/proc.h> 46 #include <sys/atomic.h> 47 48 #include <arm/armreg.h> 49 #include <arm/cpufunc.h> 50 #include <arm/locore.h> 51 52 #include <arm/cortex/gic_reg.h> 53 #include <arm/cortex/mpcore_var.h> 54 55 void armgic_irq_handler(void *); 56 57 #define ARMGIC_SGI_IPIBASE 0 58 59 /* 60 * SGIs 8-16 are reserved for use by ARM Trusted Firmware. 61 */ 62 __CTASSERT(ARMGIC_SGI_IPIBASE + NIPI <= 8); 63 64 static int armgic_match(device_t, cfdata_t, void *); 65 static void armgic_attach(device_t, device_t, void *); 66 67 static void armgic_set_priority(struct pic_softc *, int); 68 static void armgic_unblock_irqs(struct pic_softc *, size_t, uint32_t); 69 static void armgic_block_irqs(struct pic_softc *, size_t, uint32_t); 70 static void armgic_establish_irq(struct pic_softc *, struct intrsource *); 71 #if 0 72 static void armgic_source_name(struct pic_softc *, int, char *, size_t); 73 #endif 74 75 #ifdef MULTIPROCESSOR 76 static void armgic_cpu_init(struct pic_softc *, struct cpu_info *); 77 static void armgic_ipi_send(struct pic_softc *, const kcpuset_t *, u_long); 78 static void armgic_get_affinity(struct pic_softc *, size_t, kcpuset_t *); 79 static int armgic_set_affinity(struct pic_softc *, size_t, const kcpuset_t *); 80 #endif 81 82 static const struct pic_ops armgic_picops = { 83 .pic_unblock_irqs = armgic_unblock_irqs, 84 .pic_block_irqs = armgic_block_irqs, 85 .pic_establish_irq = armgic_establish_irq, 86 #if 0 87 .pic_source_name = armgic_source_name, 88 #endif 89 .pic_set_priority = armgic_set_priority, 90 #ifdef MULTIPROCESSOR 91 .pic_cpu_init = armgic_cpu_init, 92 .pic_ipi_send = armgic_ipi_send, 93 .pic_get_affinity = armgic_get_affinity, 94 .pic_set_affinity = armgic_set_affinity, 95 #endif 96 }; 97 98 #define PICTOSOFTC(pic) ((struct armgic_softc *)(pic)) 99 100 static struct armgic_softc { 101 struct pic_softc sc_pic; 102 device_t sc_dev; 103 bus_space_tag_t sc_memt; 104 bus_space_handle_t sc_gicch; 105 bus_space_handle_t sc_gicdh; 106 size_t sc_gic_lines; 107 uint32_t sc_gic_type; 108 uint32_t sc_gic_valid_lines[1024/32]; 109 uint32_t sc_enabled_local; 110 #ifdef MULTIPROCESSOR 111 uint32_t sc_target[MAXCPUS]; 112 uint32_t sc_mptargets; 113 #endif 114 uint32_t sc_bptargets; 115 } armgic_softc = { 116 .sc_pic = { 117 .pic_ops = &armgic_picops, 118 .pic_name = "armgic", 119 }, 120 }; 121 122 static struct intrsource armgic_dummy_source; 123 124 __CTASSERT(NIPL == 8); 125 126 /* 127 * GIC register are always in little-endian. It is assumed the bus_space 128 * will do any endian conversion required. 129 */ 130 static inline uint32_t 131 gicc_read(struct armgic_softc *sc, bus_size_t o) 132 { 133 return bus_space_read_4(sc->sc_memt, sc->sc_gicch, o); 134 } 135 136 static inline void 137 gicc_write(struct armgic_softc *sc, bus_size_t o, uint32_t v) 138 { 139 bus_space_write_4(sc->sc_memt, sc->sc_gicch, o, v); 140 } 141 142 static inline uint32_t 143 gicd_read(struct armgic_softc *sc, bus_size_t o) 144 { 145 return bus_space_read_4(sc->sc_memt, sc->sc_gicdh, o); 146 } 147 148 static inline void 149 gicd_write(struct armgic_softc *sc, bus_size_t o, uint32_t v) 150 { 151 bus_space_write_4(sc->sc_memt, sc->sc_gicdh, o, v); 152 } 153 154 static uint32_t 155 gicd_find_targets(struct armgic_softc *sc) 156 { 157 uint32_t targets = 0; 158 159 /* 160 * GICD_ITARGETSR0 through 7 are read-only, and each field returns 161 * a value that corresponds only to the processor reading the 162 * register. Use this to determine the current processor's 163 * CPU interface number. 164 */ 165 for (int i = 0; i < 8; i++) { 166 targets = gicd_read(sc, GICD_ITARGETSRn(i)); 167 if (targets != 0) 168 break; 169 } 170 targets |= (targets >> 16); 171 targets |= (targets >> 8); 172 targets &= 0xff; 173 174 return targets ? targets : 1; 175 } 176 177 /* 178 * In the GIC prioritization scheme, lower numbers have higher priority. 179 * Only write priorities that could be non-secure. 180 */ 181 static inline uint32_t 182 armgic_ipl_to_priority(int ipl) 183 { 184 return GICC_PMR_NONSECURE 185 | ((IPL_HIGH - ipl) * GICC_PMR_NS_PRIORITIES / NIPL); 186 } 187 188 #if 0 189 static inline int 190 armgic_priority_to_ipl(uint32_t priority) 191 { 192 return IPL_HIGH 193 - (priority & ~GICC_PMR_NONSECURE) * NIPL / GICC_PMR_NS_PRIORITIES; 194 } 195 #endif 196 197 static void 198 armgic_unblock_irqs(struct pic_softc *pic, size_t irq_base, uint32_t irq_mask) 199 { 200 struct armgic_softc * const sc = PICTOSOFTC(pic); 201 const size_t group = irq_base / 32; 202 203 if (group == 0) 204 sc->sc_enabled_local |= irq_mask; 205 206 gicd_write(sc, GICD_ISENABLERn(group), irq_mask); 207 } 208 209 static void 210 armgic_block_irqs(struct pic_softc *pic, size_t irq_base, uint32_t irq_mask) 211 { 212 struct armgic_softc * const sc = PICTOSOFTC(pic); 213 const size_t group = irq_base / 32; 214 215 if (group == 0) 216 sc->sc_enabled_local &= ~irq_mask; 217 218 gicd_write(sc, GICD_ICENABLERn(group), irq_mask); 219 } 220 221 static void 222 armgic_set_priority(struct pic_softc *pic, int ipl) 223 { 224 struct armgic_softc * const sc = PICTOSOFTC(pic); 225 struct cpu_info * const ci = curcpu(); 226 227 const uint32_t priority = armgic_ipl_to_priority(ipl); 228 if (priority > ci->ci_hwpl) { 229 /* Lowering priority mask */ 230 ci->ci_hwpl = priority; 231 gicc_write(sc, GICC_PMR, priority); 232 } 233 } 234 235 #ifdef MULTIPROCESSOR 236 static void 237 armgic_get_affinity(struct pic_softc *pic, size_t irq, kcpuset_t *affinity) 238 { 239 struct armgic_softc * const sc = PICTOSOFTC(pic); 240 const size_t group = irq / 32; 241 int n; 242 243 kcpuset_zero(affinity); 244 if (group == 0) { 245 /* All CPUs are targets for group 0 (SGI/PPI) */ 246 for (n = 0; n < MAXCPUS; n++) { 247 if (sc->sc_target[n] != 0) 248 kcpuset_set(affinity, n); 249 } 250 } else { 251 /* Find distributor targets (SPI) */ 252 const u_int byte_shift = 8 * (irq & 3); 253 const bus_size_t targets_reg = GICD_ITARGETSRn(irq / 4); 254 const uint32_t targets = gicd_read(sc, targets_reg); 255 const uint32_t targets_val = (targets >> byte_shift) & 0xff; 256 257 for (n = 0; n < MAXCPUS; n++) { 258 if (sc->sc_target[n] & targets_val) 259 kcpuset_set(affinity, n); 260 } 261 } 262 } 263 264 static int 265 armgic_set_affinity(struct pic_softc *pic, size_t irq, 266 const kcpuset_t *affinity) 267 { 268 struct armgic_softc * const sc = PICTOSOFTC(pic); 269 const size_t group = irq / 32; 270 if (group == 0) 271 return EINVAL; 272 273 const u_int byte_shift = 8 * (irq & 3); 274 const bus_size_t targets_reg = GICD_ITARGETSRn(irq / 4); 275 uint32_t targets_val = 0; 276 int n; 277 278 for (n = 0; n < MAXCPUS; n++) { 279 if (kcpuset_isset(affinity, n)) 280 targets_val |= sc->sc_target[n]; 281 } 282 283 uint32_t targets = gicd_read(sc, targets_reg); 284 targets &= ~(0xff << byte_shift); 285 targets |= (targets_val << byte_shift); 286 gicd_write(sc, targets_reg, targets); 287 288 return 0; 289 } 290 #endif 291 292 #ifdef __HAVE_PIC_FAST_SOFTINTS 293 void 294 softint_init_md(lwp_t *l, u_int level, uintptr_t *machdep_p) 295 { 296 lwp_t **lp = &l->l_cpu->ci_softlwps[level]; 297 KASSERT(*lp == NULL || *lp == l); 298 *lp = l; 299 /* 300 * Really easy. Just tell it to trigger the local CPU. 301 */ 302 *machdep_p = GICD_SGIR_TargetListFilter_Me 303 | __SHIFTIN(level, GICD_SGIR_SGIINTID); 304 } 305 306 void 307 softint_trigger(uintptr_t machdep) 308 { 309 310 gicd_write(&armgic_softc, GICD_SGIR, machdep); 311 } 312 #endif 313 314 void 315 armgic_irq_handler(void *tf) 316 { 317 struct cpu_info * const ci = curcpu(); 318 struct armgic_softc * const sc = &armgic_softc; 319 const int old_ipl = ci->ci_cpl; 320 #ifdef DIAGNOSTIC 321 const int old_mtx_count = ci->ci_mtx_count; 322 const int old_l_biglocks = ci->ci_curlwp->l_biglocks; 323 #endif 324 #ifdef DEBUG 325 size_t n = 0; 326 #endif 327 328 ci->ci_data.cpu_nintr++; 329 330 const uint32_t priority = armgic_ipl_to_priority(old_ipl); 331 if (ci->ci_hwpl != priority) { 332 ci->ci_hwpl = priority; 333 gicc_write(sc, GICC_PMR, priority); 334 if (old_ipl == IPL_HIGH) { 335 return; 336 } 337 } 338 339 for (;;) { 340 uint32_t iar = gicc_read(sc, GICC_IAR); 341 uint32_t irq = __SHIFTOUT(iar, GICC_IAR_IRQ); 342 343 if (irq == GICC_IAR_IRQ_SPURIOUS || 344 irq == GICC_IAR_IRQ_SSPURIOUS) { 345 iar = gicc_read(sc, GICC_IAR); 346 irq = __SHIFTOUT(iar, GICC_IAR_IRQ); 347 if (irq == GICC_IAR_IRQ_SPURIOUS) 348 break; 349 if (irq == GICC_IAR_IRQ_SSPURIOUS) { 350 break; 351 } 352 } 353 354 KASSERTMSG(old_ipl != IPL_HIGH, "old_ipl %d pmr %#x hppir %#x", 355 old_ipl, gicc_read(sc, GICC_PMR), gicc_read(sc, GICC_HPPIR)); 356 357 //const uint32_t cpuid = __SHIFTOUT(iar, GICC_IAR_CPUID_MASK); 358 struct intrsource * const is = sc->sc_pic.pic_sources[irq]; 359 KASSERT(is != &armgic_dummy_source); 360 361 /* 362 * GIC has asserted IPL for us so we can just update ci_cpl. 363 * 364 * But it's not that simple. We may have already bumped ci_cpl 365 * due to a high priority interrupt and now we are about to 366 * dispatch one lower than the previous. It's possible for 367 * that previous interrupt to have deferred some interrupts 368 * so we need deal with those when lowering to the current 369 * interrupt's ipl. 370 * 371 * However, if are just raising ipl, we can just update ci_cpl. 372 */ 373 const int ipl = is->is_ipl; 374 if (__predict_false(ipl < ci->ci_cpl)) { 375 pic_do_pending_ints(I32_bit, ipl, tf); 376 KASSERT(ci->ci_cpl == ipl); 377 } else { 378 KASSERTMSG(ipl > ci->ci_cpl, "ipl %d cpl %d hw-ipl %#x", 379 ipl, ci->ci_cpl, 380 gicc_read(sc, GICC_PMR)); 381 gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(ipl)); 382 ci->ci_hwpl = ci->ci_cpl = ipl; 383 } 384 ENABLE_INTERRUPT(); 385 pic_dispatch(is, tf); 386 DISABLE_INTERRUPT(); 387 gicc_write(sc, GICC_EOIR, iar); 388 #ifdef DEBUG 389 n++; 390 KDASSERTMSG(n < 5, "%s: processed too many (%zu)", 391 ci->ci_data.cpu_name, n); 392 #endif 393 } 394 395 /* 396 * Now handle any pending ints. 397 */ 398 pic_do_pending_ints(I32_bit, old_ipl, tf); 399 KASSERTMSG(ci->ci_cpl == old_ipl, "ci_cpl %d old_ipl %d", ci->ci_cpl, old_ipl); 400 KASSERT(old_mtx_count == ci->ci_mtx_count); 401 KASSERT(old_l_biglocks == ci->ci_curlwp->l_biglocks); 402 } 403 404 void 405 armgic_establish_irq(struct pic_softc *pic, struct intrsource *is) 406 { 407 struct armgic_softc * const sc = PICTOSOFTC(pic); 408 const size_t group = is->is_irq / 32; 409 const u_int irq = is->is_irq & 31; 410 const u_int byte_shift = 8 * (irq & 3); 411 const u_int twopair_shift = 2 * (irq & 15); 412 413 KASSERTMSG(sc->sc_gic_valid_lines[group] & __BIT(irq), 414 "irq %u: not valid (group[%zu]=0x%08x [0x%08x])", 415 is->is_irq, group, sc->sc_gic_valid_lines[group], 416 (uint32_t)__BIT(irq)); 417 418 KASSERTMSG(is->is_type == IST_LEVEL || is->is_type == IST_EDGE, 419 "irq %u: type %u unsupported", is->is_irq, is->is_type); 420 421 const bus_size_t targets_reg = GICD_ITARGETSRn(is->is_irq / 4); 422 const bus_size_t cfg_reg = GICD_ICFGRn(is->is_irq / 16); 423 uint32_t targets = gicd_read(sc, targets_reg); 424 uint32_t cfg = gicd_read(sc, cfg_reg); 425 426 if (group > 0) { 427 /* 428 * There are 4 irqs per TARGETS register. For now bind 429 * to the primary cpu. 430 */ 431 targets &= ~(0xffU << byte_shift); 432 #if 0 433 #ifdef MULTIPROCESSOR 434 if (is->is_mpsafe) { 435 targets |= sc->sc_mptargets << byte_shift; 436 } else 437 #endif 438 #endif 439 targets |= sc->sc_bptargets << byte_shift; 440 gicd_write(sc, targets_reg, targets); 441 442 /* 443 * There are 16 irqs per CFG register. 10=EDGE 00=LEVEL 444 */ 445 uint32_t new_cfg = cfg; 446 uint32_t old_cfg = (cfg >> twopair_shift) & __BITS(1, 0); 447 if (is->is_type == IST_LEVEL && (old_cfg & __BIT(1)) != 0) { 448 new_cfg &= ~(__BITS(1, 0) << twopair_shift); 449 } else if (is->is_type == IST_EDGE && (old_cfg & 2) == 0) { 450 new_cfg |= __BIT(1) << twopair_shift; 451 } 452 if (new_cfg != cfg) { 453 gicd_write(sc, cfg_reg, new_cfg); 454 } 455 #ifdef MULTIPROCESSOR 456 } else { 457 /* 458 * All group 0 interrupts are per processor and MPSAFE by 459 * default. 460 */ 461 is->is_mpsafe = true; 462 #endif 463 } 464 465 /* 466 * There are 4 irqs per PRIORITY register. Map the IPL 467 * to GIC priority. 468 */ 469 const bus_size_t priority_reg = GICD_IPRIORITYRn(is->is_irq / 4); 470 uint32_t priority = gicd_read(sc, priority_reg); 471 priority &= ~(0xffU << byte_shift); 472 priority |= armgic_ipl_to_priority(is->is_ipl) << byte_shift; 473 gicd_write(sc, priority_reg, priority); 474 } 475 476 #ifdef MULTIPROCESSOR 477 static void 478 armgic_cpu_init_priorities(struct armgic_softc *sc) 479 { 480 /* Set lowest priority, i.e. disable interrupts */ 481 for (size_t i = 0; i < sc->sc_pic.pic_maxsources; i += 4) { 482 const bus_size_t priority_reg = GICD_IPRIORITYRn(i / 4); 483 gicd_write(sc, priority_reg, ~0); 484 } 485 } 486 487 static void 488 armgic_cpu_update_priorities(struct armgic_softc *sc) 489 { 490 uint32_t enabled = sc->sc_enabled_local; 491 for (size_t i = 0; i < sc->sc_pic.pic_maxsources; i += 4, enabled >>= 4) { 492 const bus_size_t priority_reg = GICD_IPRIORITYRn(i / 4); 493 uint32_t priority = gicd_read(sc, priority_reg); 494 uint32_t byte_mask = 0xff; 495 size_t byte_shift = 0; 496 for (size_t j = 0; j < 4; j++, byte_mask <<= 8, byte_shift += 8) { 497 struct intrsource * const is = sc->sc_pic.pic_sources[i+j]; 498 priority |= byte_mask; 499 if (is == NULL || is == &armgic_dummy_source) 500 continue; 501 priority &= ~byte_mask; 502 priority |= armgic_ipl_to_priority(is->is_ipl) << byte_shift; 503 } 504 gicd_write(sc, priority_reg, priority); 505 } 506 } 507 508 static void 509 armgic_cpu_init_targets(struct armgic_softc *sc) 510 { 511 /* 512 * Update the mpsafe targets 513 */ 514 for (size_t irq = 32; irq < sc->sc_pic.pic_maxsources; irq++) { 515 struct intrsource * const is = sc->sc_pic.pic_sources[irq]; 516 const bus_size_t targets_reg = GICD_ITARGETSRn(irq / 4); 517 if (is != NULL && is->is_mpsafe) { 518 const u_int byte_shift = 8 * (irq & 3); 519 uint32_t targets = gicd_read(sc, targets_reg); 520 #if 0 521 targets |= sc->sc_mptargets << byte_shift; 522 #else 523 targets |= sc->sc_bptargets << byte_shift; 524 #endif 525 gicd_write(sc, targets_reg, targets); 526 } 527 } 528 } 529 530 void 531 armgic_cpu_init(struct pic_softc *pic, struct cpu_info *ci) 532 { 533 struct armgic_softc * const sc = PICTOSOFTC(pic); 534 sc->sc_target[cpu_index(ci)] = gicd_find_targets(sc); 535 atomic_or_32(&sc->sc_mptargets, sc->sc_target[cpu_index(ci)]); 536 KASSERTMSG(ci->ci_cpl == IPL_HIGH, "ipl %d not IPL_HIGH", ci->ci_cpl); 537 armgic_cpu_init_priorities(sc); 538 if (!CPU_IS_PRIMARY(ci)) { 539 if (popcount(sc->sc_mptargets) != 1) { 540 armgic_cpu_init_targets(sc); 541 } 542 if (sc->sc_enabled_local) { 543 armgic_cpu_update_priorities(sc); 544 gicd_write(sc, GICD_ISENABLERn(0), 545 sc->sc_enabled_local); 546 } 547 } 548 ci->ci_hwpl = armgic_ipl_to_priority(ci->ci_cpl); 549 gicc_write(sc, GICC_PMR, armgic_ipl_to_priority(ci->ci_cpl)); // set PMR 550 gicc_write(sc, GICC_CTRL, GICC_CTRL_V1_Enable); // enable interrupt 551 ENABLE_INTERRUPT(); // allow IRQ exceptions 552 } 553 554 void 555 armgic_ipi_send(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi) 556 { 557 struct armgic_softc * const sc = PICTOSOFTC(pic); 558 559 #if 0 560 if (ipi == IPI_NOP) { 561 sev(); 562 return; 563 } 564 #endif 565 566 uint32_t sgir = __SHIFTIN(ARMGIC_SGI_IPIBASE + ipi, GICD_SGIR_SGIINTID); 567 if (kcp != NULL) { 568 uint32_t targets_val = 0; 569 for (int n = 0; n < MAXCPUS; n++) { 570 if (kcpuset_isset(kcp, n)) 571 targets_val |= sc->sc_target[n]; 572 } 573 sgir |= __SHIFTIN(targets_val, GICD_SGIR_TargetList); 574 sgir |= GICD_SGIR_TargetListFilter_List; 575 } else { 576 if (ncpu == 1) 577 return; 578 sgir |= GICD_SGIR_TargetListFilter_NotMe; 579 } 580 581 gicd_write(sc, GICD_SGIR, sgir); 582 } 583 #endif 584 585 int 586 armgic_match(device_t parent, cfdata_t cf, void *aux) 587 { 588 struct mpcore_attach_args * const mpcaa = aux; 589 590 if (strcmp(cf->cf_name, mpcaa->mpcaa_name) != 0) 591 return 0; 592 593 return 1; 594 } 595 596 void 597 armgic_attach(device_t parent, device_t self, void *aux) 598 { 599 struct armgic_softc * const sc = &armgic_softc; 600 struct mpcore_attach_args * const mpcaa = aux; 601 602 sc->sc_dev = self; 603 self->dv_private = sc; 604 605 sc->sc_memt = mpcaa->mpcaa_memt; /* provided for us */ 606 bus_space_subregion(sc->sc_memt, mpcaa->mpcaa_memh, mpcaa->mpcaa_off1, 607 4096, &sc->sc_gicdh); 608 bus_space_subregion(sc->sc_memt, mpcaa->mpcaa_memh, mpcaa->mpcaa_off2, 609 4096, &sc->sc_gicch); 610 611 sc->sc_gic_type = gicd_read(sc, GICD_TYPER); 612 sc->sc_pic.pic_maxsources = GICD_TYPER_LINES(sc->sc_gic_type); 613 614 gicc_write(sc, GICC_CTRL, 0); /* disable all interrupts */ 615 gicd_write(sc, GICD_CTRL, 0); /* disable all interrupts */ 616 617 gicc_write(sc, GICC_PMR, 0xff); 618 uint32_t pmr = gicc_read(sc, GICC_PMR); 619 u_int priorities = 1 << popcount32(pmr); 620 621 const uint32_t iidr = gicc_read(sc, GICC_IIDR); 622 const int iidr_prod = __SHIFTOUT(iidr, GICC_IIDR_ProductID); 623 const int iidr_arch = __SHIFTOUT(iidr, GICC_IIDR_ArchVersion); 624 const int iidr_rev = __SHIFTOUT(iidr, GICC_IIDR_Revision); 625 const int iidr_imp = __SHIFTOUT(iidr, GICC_IIDR_Implementer); 626 627 /* 628 * Find the boot processor's CPU interface number. 629 */ 630 sc->sc_bptargets = gicd_find_targets(sc); 631 632 /* 633 * Let's find out how many real sources we have. 634 */ 635 for (size_t i = 0, group = 0; 636 i < sc->sc_pic.pic_maxsources; 637 i += 32, group++) { 638 /* 639 * To figure what sources are real, one enables all interrupts 640 * and then reads back the enable mask so which ones really 641 * got enabled. 642 */ 643 gicd_write(sc, GICD_ISENABLERn(group), 0xffffffff); 644 uint32_t valid = gicd_read(sc, GICD_ISENABLERn(group)); 645 646 /* 647 * Now disable (clear enable) them again. 648 */ 649 gicd_write(sc, GICD_ICENABLERn(group), valid); 650 651 /* 652 * Count how many are valid. 653 */ 654 sc->sc_gic_lines += popcount32(valid); 655 sc->sc_gic_valid_lines[group] = valid; 656 } 657 658 aprint_normal(": Generic Interrupt Controller, " 659 "%zu sources (%zu valid)\n", 660 sc->sc_pic.pic_maxsources, sc->sc_gic_lines); 661 aprint_debug_dev(sc->sc_dev, "Architecture version %d" 662 " (0x%x:%d rev %d)\n", iidr_arch, iidr_imp, iidr_prod, 663 iidr_rev); 664 665 #ifdef MULTIPROCESSOR 666 sc->sc_pic.pic_cpus = kcpuset_running; 667 #endif 668 pic_add(&sc->sc_pic, 0); 669 670 /* 671 * Force the GICD to IPL_HIGH and then enable interrupts. 672 */ 673 struct cpu_info * const ci = curcpu(); 674 KASSERTMSG(ci->ci_cpl == IPL_HIGH, "ipl %d not IPL_HIGH", ci->ci_cpl); 675 armgic_set_priority(&sc->sc_pic, ci->ci_cpl); // set PMR 676 gicd_write(sc, GICD_CTRL, GICD_CTRL_Enable); // enable Distributer 677 gicc_write(sc, GICC_CTRL, GICC_CTRL_V1_Enable); // enable CPU interrupts 678 ENABLE_INTERRUPT(); // allow interrupt exceptions 679 680 /* 681 * For each line that isn't valid, we set the intrsource for it to 682 * point at a dummy source so that pic_intr_establish will fail for it. 683 */ 684 for (size_t i = 0, group = 0; 685 i < sc->sc_pic.pic_maxsources; 686 i += 32, group++) { 687 uint32_t invalid = ~sc->sc_gic_valid_lines[group]; 688 for (size_t j = 0; invalid && j < 32; j++, invalid >>= 1) { 689 if (invalid & 1) { 690 sc->sc_pic.pic_sources[i + j] = 691 &armgic_dummy_source; 692 } 693 } 694 } 695 #ifdef __HAVE_PIC_FAST_SOFTINTS 696 intr_establish_xname(SOFTINT_BIO, IPL_SOFTBIO, IST_MPSAFE | IST_EDGE, 697 pic_handle_softint, (void *)SOFTINT_BIO, "softint bio"); 698 intr_establish_xname(SOFTINT_CLOCK, IPL_SOFTCLOCK, IST_MPSAFE | IST_EDGE, 699 pic_handle_softint, (void *)SOFTINT_CLOCK, "softint clock"); 700 intr_establish_xname(SOFTINT_NET, IPL_SOFTNET, IST_MPSAFE | IST_EDGE, 701 pic_handle_softint, (void *)SOFTINT_NET, "softint net"); 702 intr_establish_xname(SOFTINT_SERIAL, IPL_SOFTSERIAL, IST_MPSAFE | IST_EDGE, 703 pic_handle_softint, (void *)SOFTINT_SERIAL, "softint serial"); 704 #endif 705 #ifdef MULTIPROCESSOR 706 armgic_cpu_init(&sc->sc_pic, curcpu()); 707 708 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_AST, IPL_VM, 709 IST_MPSAFE | IST_EDGE, pic_ipi_ast, (void *)-1, "IPI ast"); 710 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_XCALL, IPL_HIGH, 711 IST_MPSAFE | IST_EDGE, pic_ipi_xcall, (void *)-1, "IPI xcall"); 712 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_GENERIC, IPL_HIGH, 713 IST_MPSAFE | IST_EDGE, pic_ipi_generic, (void *)-1, "IPI generic"); 714 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_NOP, IPL_VM, 715 IST_MPSAFE | IST_EDGE, pic_ipi_nop, (void *)-1, "IPI nop"); 716 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_SHOOTDOWN, IPL_SCHED, 717 IST_MPSAFE | IST_EDGE, pic_ipi_shootdown, (void *)-1, "IPI shootdown"); 718 #ifdef DDB 719 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_DDB, IPL_HIGH, 720 IST_MPSAFE | IST_EDGE, pic_ipi_ddb, NULL, "IPI ddb"); 721 #endif 722 #ifdef __HAVE_PREEMPTION 723 intr_establish_xname(ARMGIC_SGI_IPIBASE + IPI_KPREEMPT, IPL_VM, 724 IST_MPSAFE | IST_EDGE, pic_ipi_kpreempt, (void *)-1, "IPI kpreempt"); 725 #endif 726 #endif 727 728 const u_int ppis = popcount32(sc->sc_gic_valid_lines[0] >> 16); 729 const u_int sgis = popcount32(sc->sc_gic_valid_lines[0] & 0xffff); 730 aprint_normal_dev(sc->sc_dev, "%u Priorities, %zu SPIs, %u PPIs, " 731 "%u SGIs\n", priorities, sc->sc_gic_lines - ppis - sgis, ppis, 732 sgis); 733 } 734 735 CFATTACH_DECL_NEW(armgic, 0, 736 armgic_match, armgic_attach, NULL, NULL); 737