1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1991 The Regents of the University of California. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to Berkeley by 8 * William Jolitz. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 /*- 35 * Copyright (c) 2002 Benno Rice. 36 * All rights reserved. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 47 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 48 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 49 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 50 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 51 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 52 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 53 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 54 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 55 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 56 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 57 * SUCH DAMAGE. 58 * form: src/sys/i386/isa/intr_machdep.c,v 1.57 2001/07/20 59 */ 60 61 #include "opt_isa.h" 62 63 #include <sys/param.h> 64 #include <sys/systm.h> 65 #include <sys/kernel.h> 66 #include <sys/queue.h> 67 #include <sys/bus.h> 68 #include <sys/cpuset.h> 69 #include <sys/interrupt.h> 70 #include <sys/ktr.h> 71 #include <sys/lock.h> 72 #include <sys/malloc.h> 73 #include <sys/mutex.h> 74 #include <sys/pcpu.h> 75 #include <sys/smp.h> 76 #include <sys/syslog.h> 77 #include <sys/vmmeter.h> 78 #include <sys/proc.h> 79 80 #include <machine/frame.h> 81 #include <machine/intr_machdep.h> 82 #include <machine/md_var.h> 83 #include <machine/smp.h> 84 #include <machine/trap.h> 85 86 #include "pic_if.h" 87 88 static MALLOC_DEFINE(M_INTR, "intr", "interrupt handler data"); 89 90 struct powerpc_intr { 91 struct intr_event *event; 92 long *cntp; 93 void *priv; /* PIC-private data */ 94 device_t pic; 95 u_int irq; 96 u_int intline; 97 u_int vector; 98 u_int cntindex; 99 int fwcode; 100 int ipi; 101 int pi_domain; 102 enum intr_trigger trig; 103 enum intr_polarity pol; 104 cpuset_t pi_cpuset; 105 }; 106 107 struct pic { 108 device_t dev; 109 uint32_t node; 110 u_int irqs; 111 u_int ipis; 112 int base; 113 }; 114 115 static u_int intrcnt_index = 0; 116 static struct mtx intr_table_lock; 117 static struct powerpc_intr **powerpc_intrs; 118 static struct pic piclist[MAX_PICS]; 119 static u_int nvectors; /* Allocated vectors */ 120 static u_int npics; /* PICs registered */ 121 #ifdef DEV_ISA 122 static u_int nirqs = 16; /* Allocated IRQS (ISA pre-allocated). */ 123 #else 124 static u_int nirqs = 0; /* Allocated IRQs. */ 125 #endif 126 static u_int stray_count; 127 128 #define INTRNAME_LEN (MAXCOMLEN + 1) 129 u_long *intrcnt; 130 char *intrnames; 131 size_t sintrcnt = sizeof(intrcnt); 132 size_t sintrnames = sizeof(intrnames); 133 int nintrcnt; 134 135 /* 136 * Just to start 137 */ 138 #ifdef __powerpc64__ 139 u_int num_io_irqs = 768; 140 #else 141 u_int num_io_irqs = 256; 142 #endif 143 144 device_t root_pic; 145 146 #ifdef SMP 147 static void *ipi_cookie; 148 #endif 149 150 static int powerpc_setup_intr_int(const char *name, u_int irq, driver_filter_t 151 filter, driver_intr_t handler, void *arg, enum intr_type flags, void 152 **cookiep, int domain, bool ipi); 153 154 static void 155 intrcnt_setname(const char *name, int index) 156 { 157 158 snprintf(intrnames + INTRNAME_LEN * index, INTRNAME_LEN, "%-*s", 159 INTRNAME_LEN - 1, name); 160 } 161 162 static void 163 intr_init(void *dummy __unused) 164 { 165 166 mtx_init(&intr_table_lock, "intr sources lock", NULL, MTX_DEF); 167 } 168 SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL); 169 170 static void 171 intr_init_sources(void *arg __unused) 172 { 173 174 powerpc_intrs = mallocarray(num_io_irqs, sizeof(*powerpc_intrs), 175 M_INTR, M_WAITOK | M_ZERO); 176 nintrcnt = 1 + num_io_irqs * 2 + mp_ncpus * 2; 177 #ifdef COUNT_IPIS 178 if (mp_ncpus > 1) 179 nintrcnt += 8 * mp_ncpus; 180 #endif 181 intrcnt = mallocarray(nintrcnt, sizeof(u_long), M_INTR, M_WAITOK | 182 M_ZERO); 183 intrnames = mallocarray(nintrcnt, INTRNAME_LEN, M_INTR, M_WAITOK | 184 M_ZERO); 185 sintrcnt = nintrcnt * sizeof(u_long); 186 sintrnames = nintrcnt * INTRNAME_LEN; 187 188 intrcnt_setname("???", 0); 189 intrcnt_index = 1; 190 } 191 /* 192 * This needs to happen before SI_SUB_CPU 193 */ 194 SYSINIT(intr_init_sources, SI_SUB_KLD, SI_ORDER_ANY, intr_init_sources, NULL); 195 196 #ifdef SMP 197 static void 198 smp_intr_init(void *dummy __unused) 199 { 200 struct powerpc_intr *i; 201 int vector; 202 203 for (vector = 0; vector < nvectors; vector++) { 204 i = powerpc_intrs[vector]; 205 if (i != NULL && i->event != NULL && i->pic == root_pic) 206 PIC_BIND(i->pic, i->intline, i->pi_cpuset, &i->priv); 207 } 208 } 209 SYSINIT(smp_intr_init, SI_SUB_SMP, SI_ORDER_ANY, smp_intr_init, NULL); 210 #endif 211 212 void 213 intrcnt_add(const char *name, u_long **countp) 214 { 215 int idx; 216 217 idx = atomic_fetchadd_int(&intrcnt_index, 1); 218 KASSERT(idx < nintrcnt, ("intrcnt_add: Interrupt counter index %d/%d" 219 "reached nintrcnt : %d", intrcnt_index, idx, nintrcnt)); 220 *countp = &intrcnt[idx]; 221 intrcnt_setname(name, idx); 222 } 223 224 static struct powerpc_intr * 225 intr_lookup(u_int irq) 226 { 227 char intrname[16]; 228 struct powerpc_intr *i, *iscan; 229 int vector; 230 231 mtx_lock(&intr_table_lock); 232 for (vector = 0; vector < nvectors; vector++) { 233 i = powerpc_intrs[vector]; 234 if (i != NULL && i->irq == irq) { 235 mtx_unlock(&intr_table_lock); 236 return (i); 237 } 238 } 239 240 i = malloc(sizeof(*i), M_INTR, M_NOWAIT); 241 if (i == NULL) { 242 mtx_unlock(&intr_table_lock); 243 return (NULL); 244 } 245 246 i->event = NULL; 247 i->cntp = NULL; 248 i->priv = NULL; 249 i->trig = INTR_TRIGGER_CONFORM; 250 i->pol = INTR_POLARITY_CONFORM; 251 i->irq = irq; 252 i->pic = NULL; 253 i->vector = -1; 254 i->fwcode = 0; 255 i->ipi = 0; 256 257 #ifdef SMP 258 i->pi_cpuset = all_cpus; 259 #else 260 CPU_SETOF(0, &i->pi_cpuset); 261 #endif 262 263 for (vector = 0; vector < num_io_irqs && vector <= nvectors; 264 vector++) { 265 iscan = powerpc_intrs[vector]; 266 if (iscan != NULL && iscan->irq == irq) 267 break; 268 if (iscan == NULL && i->vector == -1) 269 i->vector = vector; 270 iscan = NULL; 271 } 272 273 if (iscan == NULL && i->vector != -1) { 274 powerpc_intrs[i->vector] = i; 275 i->cntindex = atomic_fetchadd_int(&intrcnt_index, 1); 276 i->cntp = &intrcnt[i->cntindex]; 277 sprintf(intrname, "irq%u:", i->irq); 278 intrcnt_setname(intrname, i->cntindex); 279 nvectors++; 280 } 281 mtx_unlock(&intr_table_lock); 282 283 if (iscan != NULL || i->vector == -1) { 284 free(i, M_INTR); 285 i = iscan; 286 } 287 288 return (i); 289 } 290 291 static int 292 powerpc_map_irq(struct powerpc_intr *i) 293 { 294 struct pic *p; 295 u_int cnt; 296 int idx; 297 298 for (idx = 0; idx < npics; idx++) { 299 p = &piclist[idx]; 300 cnt = p->irqs + p->ipis; 301 if (i->irq >= p->base && i->irq < p->base + cnt) 302 break; 303 } 304 if (idx == npics) 305 return (EINVAL); 306 307 i->intline = i->irq - p->base; 308 i->pic = p->dev; 309 310 /* Try a best guess if that failed */ 311 if (i->pic == NULL) 312 i->pic = root_pic; 313 314 return (0); 315 } 316 317 static void 318 powerpc_intr_eoi(void *arg) 319 { 320 struct powerpc_intr *i = arg; 321 322 PIC_EOI(i->pic, i->intline, i->priv); 323 } 324 325 static void 326 powerpc_intr_pre_ithread(void *arg) 327 { 328 struct powerpc_intr *i = arg; 329 330 PIC_MASK(i->pic, i->intline, i->priv); 331 PIC_EOI(i->pic, i->intline, i->priv); 332 } 333 334 static void 335 powerpc_intr_post_ithread(void *arg) 336 { 337 struct powerpc_intr *i = arg; 338 339 PIC_UNMASK(i->pic, i->intline, i->priv); 340 } 341 342 static int 343 powerpc_assign_intr_cpu(void *arg, int cpu) 344 { 345 #ifdef SMP 346 struct powerpc_intr *i = arg; 347 348 if (cpu == NOCPU) 349 i->pi_cpuset = all_cpus; 350 else 351 CPU_SETOF(cpu, &i->pi_cpuset); 352 353 if (!cold && i->pic != NULL && i->pic == root_pic) 354 PIC_BIND(i->pic, i->intline, i->pi_cpuset, &i->priv); 355 356 return (0); 357 #else 358 return (EOPNOTSUPP); 359 #endif 360 } 361 362 u_int 363 powerpc_register_pic(device_t dev, uint32_t node, u_int irqs, u_int ipis, 364 u_int atpic) 365 { 366 struct pic *p; 367 u_int irq; 368 int idx; 369 370 mtx_lock(&intr_table_lock); 371 372 /* XXX see powerpc_get_irq(). */ 373 for (idx = 0; idx < npics; idx++) { 374 p = &piclist[idx]; 375 if (p->node != node) 376 continue; 377 if (node != 0 || p->dev == dev) 378 break; 379 } 380 p = &piclist[idx]; 381 382 p->dev = dev; 383 p->node = node; 384 p->irqs = irqs; 385 p->ipis = ipis; 386 if (idx == npics) { 387 #ifdef DEV_ISA 388 p->base = (atpic) ? 0 : nirqs; 389 #else 390 p->base = nirqs; 391 #endif 392 irq = p->base + irqs + ipis; 393 nirqs = MAX(nirqs, irq); 394 npics++; 395 } 396 397 KASSERT(npics < MAX_PICS, 398 ("Number of PICs exceeds maximum (%d)", MAX_PICS)); 399 400 mtx_unlock(&intr_table_lock); 401 402 return (p->base); 403 } 404 405 u_int 406 powerpc_get_irq(uint32_t node, u_int pin) 407 { 408 int idx; 409 410 if (node == 0) 411 return (pin); 412 413 mtx_lock(&intr_table_lock); 414 for (idx = 0; idx < npics; idx++) { 415 if (piclist[idx].node == node) { 416 mtx_unlock(&intr_table_lock); 417 return (piclist[idx].base + pin); 418 } 419 } 420 421 /* 422 * XXX we should never encounter an unregistered PIC, but that 423 * can only be done when we properly support bus enumeration 424 * using multiple passes. Until then, fake an entry and give it 425 * some adhoc maximum number of IRQs and IPIs. 426 */ 427 piclist[idx].dev = NULL; 428 piclist[idx].node = node; 429 piclist[idx].irqs = 124; 430 piclist[idx].ipis = 4; 431 piclist[idx].base = nirqs; 432 nirqs += (1 << 25); 433 npics++; 434 435 KASSERT(npics < MAX_PICS, 436 ("Number of PICs exceeds maximum (%d)", MAX_PICS)); 437 438 mtx_unlock(&intr_table_lock); 439 440 return (piclist[idx].base + pin); 441 } 442 443 int 444 powerpc_enable_intr(void) 445 { 446 struct powerpc_intr *i; 447 int error, vector; 448 #ifdef SMP 449 int n; 450 #endif 451 452 if (npics == 0) 453 panic("no PIC detected\n"); 454 455 if (root_pic == NULL) 456 root_pic = piclist[0].dev; 457 458 KASSERT(root_pic != NULL, ("no root PIC!")); 459 460 #ifdef SMP 461 /* Install an IPI handler. */ 462 if (mp_ncpus > 1) { 463 for (n = 0; n < npics; n++) { 464 if (piclist[n].dev != root_pic) 465 continue; 466 467 KASSERT(piclist[n].ipis != 0, 468 ("%s: SMP root PIC does not supply any IPIs", 469 __func__)); 470 error = powerpc_setup_intr_int("IPI", 471 MAP_IRQ(piclist[n].node, piclist[n].irqs), 472 powerpc_ipi_handler, NULL, NULL, 473 INTR_TYPE_MISC | INTR_EXCL, &ipi_cookie, 474 0 /* domain XXX */, true); 475 if (error) { 476 printf("unable to setup IPI handler\n"); 477 return (error); 478 } 479 } 480 } 481 #endif 482 483 for (vector = 0; vector < nvectors; vector++) { 484 i = powerpc_intrs[vector]; 485 if (i == NULL) 486 continue; 487 488 error = powerpc_map_irq(i); 489 if (error) 490 continue; 491 492 if (i->trig == INTR_TRIGGER_INVALID) 493 PIC_TRANSLATE_CODE(i->pic, i->intline, i->fwcode, 494 &i->trig, &i->pol); 495 if (i->trig != INTR_TRIGGER_CONFORM || 496 i->pol != INTR_POLARITY_CONFORM) 497 PIC_CONFIG(i->pic, i->intline, i->trig, i->pol); 498 499 if (i->event != NULL) 500 PIC_ENABLE(i->pic, i->intline, vector, &i->priv); 501 } 502 503 return (0); 504 } 505 506 int 507 powerpc_setup_intr(const char *name, u_int irq, driver_filter_t filter, 508 driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep, 509 int domain) 510 { 511 512 return (powerpc_setup_intr_int(name, irq, filter, handler, arg, flags, 513 cookiep, domain, false)); 514 } 515 516 517 static int 518 powerpc_setup_intr_int(const char *name, u_int irq, driver_filter_t filter, 519 driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep, 520 int domain, bool ipi) 521 { 522 struct powerpc_intr *i; 523 int error, enable = 0; 524 525 i = intr_lookup(irq); 526 if (i == NULL) 527 return (ENOMEM); 528 529 if (i->event == NULL) { 530 error = intr_event_create(&i->event, (void *)i, 0, irq, 531 powerpc_intr_pre_ithread, powerpc_intr_post_ithread, 532 (ipi ? NULL : powerpc_intr_eoi), powerpc_assign_intr_cpu, 533 "irq%u:", irq); 534 if (error) 535 return (error); 536 537 enable = 1; 538 } 539 i->ipi = ipi; 540 541 error = intr_event_add_handler(i->event, name, filter, handler, arg, 542 intr_priority(flags), flags, cookiep); 543 if (error) 544 return (error); 545 i->pi_domain = domain; 546 if (strcmp(name, "IPI") != 0) { 547 CPU_ZERO(&i->pi_cpuset); 548 CPU_COPY(&cpuset_domain[domain], &i->pi_cpuset); 549 } 550 mtx_lock(&intr_table_lock); 551 intrcnt_setname(i->event->ie_fullname, i->cntindex); 552 mtx_unlock(&intr_table_lock); 553 554 if (!cold) { 555 error = powerpc_map_irq(i); 556 557 if (!error) { 558 if (i->trig == INTR_TRIGGER_INVALID) 559 PIC_TRANSLATE_CODE(i->pic, i->intline, 560 i->fwcode, &i->trig, &i->pol); 561 562 if (i->trig != INTR_TRIGGER_CONFORM || 563 i->pol != INTR_POLARITY_CONFORM) 564 PIC_CONFIG(i->pic, i->intline, i->trig, i->pol); 565 566 if (i->pic == root_pic) 567 PIC_BIND(i->pic, i->intline, i->pi_cpuset, &i->priv); 568 569 if (enable) 570 PIC_ENABLE(i->pic, i->intline, i->vector, 571 &i->priv); 572 } 573 } 574 return (error); 575 } 576 577 int 578 powerpc_teardown_intr(void *cookie) 579 { 580 581 return (intr_event_remove_handler(cookie)); 582 } 583 584 #ifdef SMP 585 int 586 powerpc_bind_intr(u_int irq, u_char cpu) 587 { 588 struct powerpc_intr *i; 589 590 i = intr_lookup(irq); 591 if (i == NULL) 592 return (ENOMEM); 593 594 return (intr_event_bind(i->event, cpu)); 595 } 596 #endif 597 598 int 599 powerpc_fw_config_intr(int irq, int sense_code) 600 { 601 struct powerpc_intr *i; 602 603 i = intr_lookup(irq); 604 if (i == NULL) 605 return (ENOMEM); 606 607 i->trig = INTR_TRIGGER_INVALID; 608 i->pol = INTR_POLARITY_CONFORM; 609 i->fwcode = sense_code; 610 611 if (!cold && i->pic != NULL) { 612 PIC_TRANSLATE_CODE(i->pic, i->intline, i->fwcode, &i->trig, 613 &i->pol); 614 PIC_CONFIG(i->pic, i->intline, i->trig, i->pol); 615 } 616 617 return (0); 618 } 619 620 int 621 powerpc_config_intr(int irq, enum intr_trigger trig, enum intr_polarity pol) 622 { 623 struct powerpc_intr *i; 624 625 i = intr_lookup(irq); 626 if (i == NULL) 627 return (ENOMEM); 628 629 i->trig = trig; 630 i->pol = pol; 631 632 if (!cold && i->pic != NULL) 633 PIC_CONFIG(i->pic, i->intline, trig, pol); 634 635 return (0); 636 } 637 638 void 639 powerpc_dispatch_intr(u_int vector, struct trapframe *tf) 640 { 641 struct powerpc_intr *i; 642 struct intr_event *ie; 643 644 i = powerpc_intrs[vector]; 645 if (i == NULL) 646 goto stray; 647 648 (*i->cntp)++; 649 650 ie = i->event; 651 KASSERT(ie != NULL, ("%s: interrupt without an event", __func__)); 652 653 /* 654 * IPIs are magical and need to be EOI'ed before filtering. 655 * This prevents races in IPI handling. 656 */ 657 if (i->ipi) 658 PIC_EOI(i->pic, i->intline, i->priv); 659 660 if (intr_event_handle(ie, tf) != 0) { 661 goto stray; 662 } 663 return; 664 665 stray: 666 stray_count++; 667 if (stray_count <= INTR_STRAY_LOG_MAX) { 668 printf("stray irq %d\n", i ? i->irq : -1); 669 if (stray_count >= INTR_STRAY_LOG_MAX) { 670 printf("got %d stray interrupts, not logging anymore\n", 671 INTR_STRAY_LOG_MAX); 672 } 673 } 674 if (i != NULL) 675 PIC_MASK(i->pic, i->intline, i->priv); 676 } 677 678 void 679 powerpc_intr_mask(u_int irq) 680 { 681 struct powerpc_intr *i; 682 683 i = intr_lookup(irq); 684 if (i == NULL || i->pic == NULL) 685 return; 686 687 PIC_MASK(i->pic, i->intline, i->priv); 688 } 689 690 void 691 powerpc_intr_unmask(u_int irq) 692 { 693 struct powerpc_intr *i; 694 695 i = intr_lookup(irq); 696 if (i == NULL || i->pic == NULL) 697 return; 698 699 PIC_UNMASK(i->pic, i->intline, i->priv); 700 } 701