1 /* $OpenBSD: plic.c,v 1.6 2021/05/13 19:26:25 kettenis Exp $ */ 2 3 /* 4 * Copyright (c) 2020, Mars Li <mengshi.li.mars@gmail.com> 5 * Copyright (c) 2020, Brian Bamsch <bbamsch@google.com> 6 * 7 * Permission to use, copy, modify, and distribute this software for any 8 * purpose with or without fee is hereby granted, provided that the above 9 * copyright notice and this permission notice appear in all copies. 10 * 11 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 12 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 13 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 14 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 15 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 16 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 17 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 18 */ 19 20 #include <sys/param.h> 21 #include <sys/systm.h> 22 #include <sys/queue.h> 23 #include <sys/malloc.h> 24 #include <sys/device.h> 25 #include <sys/evcount.h> 26 27 #include <machine/bus.h> 28 #include <machine/fdt.h> 29 #include <machine/cpu.h> 30 #include "riscv64/dev/riscv_cpu_intc.h" 31 32 #include <dev/ofw/openfirm.h> 33 #include <dev/ofw/fdt.h> 34 35 /* 36 * This driver implements a version of the RISC-V PLIC with the actual layout 37 * specified in chapter 8 of the SiFive U5 Coreplex Series Manual: 38 * 39 * https://static.dev.sifive.com/U54-MC-RVCoreIP.pdf 40 * 41 * The largest number supported by devices marked as 'sifive,plic-1.0.0', is 42 * 1024, of which device 0 is defined as non-existent by the RISC-V Privileged 43 * Spec. 44 */ 45 46 #define PLIC_MAX_IRQS 1024 47 48 #define PLIC_PRIORITY_BASE 0x000000U 49 50 #define PLIC_ENABLE_BASE 0x002000U 51 #define PLIC_ENABLE_STRIDE 0x80U 52 #define IRQ_ENABLE 1 53 #define IRQ_DISABLE 0 54 55 #define PLIC_CONTEXT_BASE 0x200000U 56 #define PLIC_CONTEXT_STRIDE 0x1000U 57 #define PLIC_CONTEXT_THRESHOLD 0x0U 58 #define PLIC_CONTEXT_CLAIM 0x4U 59 60 #define PLIC_PRIORITY(n) (PLIC_PRIORITY_BASE + (n) * sizeof(uint32_t)) 61 #define PLIC_ENABLE(sc, n, h) \ 62 (sc->sc_contexts[h].enable_offset + ((n) / 32) * sizeof(uint32_t)) 63 #define PLIC_THRESHOLD(sc, h) \ 64 (sc->sc_contexts[h].context_offset + PLIC_CONTEXT_THRESHOLD) 65 #define PLIC_CLAIM(sc, h) \ 66 (sc->sc_contexts[h].context_offset + PLIC_CONTEXT_CLAIM) 67 68 69 struct plic_intrhand { 70 TAILQ_ENTRY(plic_intrhand) ih_list; /* link on intrq list */ 71 int (*ih_func)(void *); /* handler */ 72 void *ih_arg; /* arg for handler */ 73 int ih_ipl; /* IPL_* */ 74 int ih_flags; 75 int ih_irq; /* IRQ number */ 76 struct evcount ih_count; 77 char *ih_name; 78 }; 79 80 /* 81 * One interrupt source could have multiple handler attached, 82 * each handler could have different priority level, 83 * we track the max and min priority level. 84 */ 85 struct plic_irqsrc { 86 TAILQ_HEAD(, plic_intrhand) is_list; /* handler list */ 87 int is_irq_max; /* IRQ to mask while handling */ 88 int is_irq_min; /* lowest IRQ when shared */ 89 }; 90 91 struct plic_context { 92 bus_size_t enable_offset; 93 bus_size_t context_offset; 94 }; 95 96 struct plic_softc { 97 struct device sc_dev; 98 int sc_node; 99 bus_space_tag_t sc_iot; 100 bus_space_handle_t sc_ioh; 101 struct plic_irqsrc *sc_isrcs; 102 struct plic_context sc_contexts[MAXCPUS]; 103 int sc_ndev; 104 struct interrupt_controller sc_intc; 105 }; 106 struct plic_softc *plic = NULL; 107 108 int plic_match(struct device *, void *, void *); 109 void plic_attach(struct device *, struct device *, void *); 110 int plic_irq_handler(void *); 111 int plic_irq_dispatch(uint32_t, void *); 112 void *plic_intr_establish(int, int, int (*)(void *), 113 void *, char *); 114 void *plic_intr_establish_fdt(void *, int *, int, int (*)(void *), 115 void *, char *); 116 void plic_intr_disestablish(void *); 117 void plic_intr_route(void *, int, struct cpu_info *); 118 119 void plic_splx(int); 120 int plic_spllower(int); 121 int plic_splraise(int); 122 void plic_setipl(int); 123 void plic_calc_mask(void); 124 125 /* helper function */ 126 int plic_get_cpuid(int); 127 void plic_set_priority(int, uint32_t); 128 void plic_set_threshold(int, uint32_t); 129 void plic_intr_route_grid(int, int, int); 130 void plic_intr_enable_with_pri(int, uint32_t, int); 131 void plic_intr_disable(int, int); 132 133 134 struct cfattach plic_ca = { 135 sizeof(struct plic_softc), plic_match, plic_attach, 136 }; 137 138 struct cfdriver plic_cd = { 139 NULL, "plic", DV_DULL 140 }; 141 142 int plic_attached = 0; 143 144 int 145 plic_match(struct device *parent, void *cfdata, void *aux) 146 { 147 struct fdt_attach_args *faa = aux; 148 149 if (plic_attached) 150 return 0; // Only expect one instance of PLIC 151 152 return (OF_is_compatible(faa->fa_node, "riscv,plic0") || 153 OF_is_compatible(faa->fa_node, "sifive,plic-1.0.0")); 154 } 155 156 void 157 plic_attach(struct device *parent, struct device *dev, void *aux) 158 { 159 struct plic_softc *sc; 160 struct fdt_attach_args *faa; 161 uint32_t *cells; 162 uint32_t irq; 163 uint32_t cpu; 164 int node; 165 int len; 166 int ncell; 167 int context; 168 int i; 169 struct cpu_info *ci; 170 CPU_INFO_ITERATOR cii; 171 172 if (plic_attached) 173 return; 174 175 plic = sc = (struct plic_softc *)dev; 176 faa = (struct fdt_attach_args *)aux; 177 178 if (faa->fa_nreg < 1) 179 return; 180 181 sc->sc_node = node = faa->fa_node; 182 sc->sc_iot = faa->fa_iot; 183 184 /* determine number of devices sending intr to this ic */ 185 sc->sc_ndev = OF_getpropint(faa->fa_node, "riscv,ndev", -1); 186 if (sc->sc_ndev < 0) { 187 printf(": unable to resolve number of devices\n"); 188 return; 189 } 190 191 if (sc->sc_ndev >= PLIC_MAX_IRQS) { 192 printf(": invalid ndev (%d)\n", sc->sc_ndev); 193 return; 194 } 195 196 /* map interrupt controller to va space */ 197 if (bus_space_map(sc->sc_iot, faa->fa_reg[0].addr, 198 faa->fa_reg[0].size, 0, &sc->sc_ioh)) 199 panic("%s: bus_space_map failed!", __func__); 200 201 sc->sc_isrcs = mallocarray(PLIC_MAX_IRQS, sizeof(struct plic_irqsrc), 202 M_DEVBUF, M_ZERO | M_NOWAIT); 203 204 for (irq = 1; irq <= sc->sc_ndev; irq++) { 205 TAILQ_INIT(&sc->sc_isrcs[irq].is_list); 206 plic_set_priority(irq, 0);// Mask interrupt 207 } 208 209 /* 210 * Calculate the per-cpu enable and context register offsets. 211 * 212 * This is tricky for a few reasons. The PLIC divides the interrupt 213 * enable, threshold, and claim bits by "context" 214 * 215 * The tricky part is that the PLIC spec imposes no restrictions on how 216 * these contexts are laid out. So for example, there is no guarantee 217 * that each CPU will have both a machine mode and supervisor context, 218 * or that different PLIC implementations will organize the context 219 * registers in the same way. On top of this, we must handle the fact 220 * that cpuid != hartid, as they may have been renumbered during boot. 221 * We perform the following steps: 222 * 223 * 1. Examine the PLIC's "interrupts-extended" property and skip any 224 * entries that are not for supervisor external interrupts. 225 * 226 * 2. Walk up the device tree to find the corresponding CPU, using node 227 * property to identify the cpuid. 228 * 229 * 3. Calculate the register offsets based on the context number. 230 */ 231 len = OF_getproplen(node, "interrupts-extended"); 232 if (len <= 0) { 233 printf(": could not find interrupts-extended\n"); 234 return; 235 } 236 237 cells = malloc(len, M_TEMP, M_WAITOK); 238 ncell = len / sizeof(*cells); 239 if (OF_getpropintarray(node, "interrupts-extended", cells, len) < 0) { 240 printf(": failed to read interrupts-extended\n"); 241 free(cells, M_TEMP, len); 242 return; 243 } 244 245 for (i = 0, context = 0; i < ncell; i += 2, context++) { 246 /* Skip M-mode external interrupts */ 247 if (cells[i + 1] != IRQ_EXTERNAL_SUPERVISOR) 248 continue; 249 250 /* Get the corresponding cpuid. */ 251 cpu = plic_get_cpuid(OF_getnodebyphandle(cells[i])); 252 if (cpu < 0) { 253 printf(": invalid hart!\n"); 254 free(cells, M_TEMP, len); 255 return; 256 } 257 258 /* 259 * Set the enable and context register offsets for the CPU. 260 * 261 * We assume S-mode handler always comes later than M-mode 262 * handler, but this might be a little fragile. 263 * 264 * XXX 265 * sifive spec doesn't list hart0 S-mode enable/contexts 266 * in its memory map, but QEMU emulates hart0 S-mode 267 * enable/contexts? Otherwise the following offset calculation 268 * would point to hart1 M-mode enable/contexts. 269 */ 270 sc->sc_contexts[cpu].enable_offset = PLIC_ENABLE_BASE + 271 context * PLIC_ENABLE_STRIDE; 272 sc->sc_contexts[cpu].context_offset = PLIC_CONTEXT_BASE + 273 context * PLIC_CONTEXT_STRIDE; 274 } 275 276 free(cells, M_TEMP, len); 277 278 /* Set CPU interrupt priority thresholds to minimum */ 279 CPU_INFO_FOREACH(cii, ci) { 280 plic_set_threshold(ci->ci_cpuid, 0); 281 } 282 283 plic_setipl(IPL_HIGH); /* XXX ??? */ 284 plic_calc_mask(); 285 286 /* 287 * insert self into the external interrupt handler entry in 288 * global interrupt handler vector 289 */ 290 riscv_intc_intr_establish(IRQ_EXTERNAL_SUPERVISOR, 0, 291 plic_irq_handler, NULL, "plic0"); 292 293 /* 294 * From now on, spl update must be enforeced to plic, so 295 * spl* routine should be updated. 296 */ 297 riscv_set_intr_func(plic_splraise, plic_spllower, 298 plic_splx, plic_setipl); 299 300 plic_attached = 1; 301 302 /* enable external interrupt */ 303 csr_set(sie, SIE_SEIE); 304 305 sc->sc_intc.ic_node = faa->fa_node; 306 sc->sc_intc.ic_cookie = sc; 307 sc->sc_intc.ic_establish = plic_intr_establish_fdt; 308 sc->sc_intc.ic_disestablish = plic_intr_disestablish; 309 sc->sc_intc.ic_route = plic_intr_route; 310 // sc->sc_intc.ic_cpu_enable = XXX Per-CPU Initialization? 311 312 riscv_intr_register_fdt(&sc->sc_intc); 313 314 printf("\n"); 315 } 316 317 int 318 plic_irq_handler(void *frame) 319 { 320 struct plic_softc* sc; 321 uint32_t pending; 322 uint32_t cpu; 323 int handled = 0; 324 325 sc = plic; 326 cpu = cpu_number(); 327 328 pending = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 329 PLIC_CLAIM(sc, cpu)); 330 331 if (pending >= sc->sc_ndev) { 332 printf("plic0: pending %x\n", pending); 333 return 0; 334 } 335 336 if (pending) { 337 handled = plic_irq_dispatch(pending, frame); 338 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 339 PLIC_CLAIM(sc, cpu), pending); 340 341 //#define DEBUG_INTC 342 #ifdef DEBUG_INTC 343 if (handled == 0) { 344 printf("plic handled == 0 on pending %d\n", pending); 345 } 346 #endif /* DEBUG_INTC */ 347 } 348 349 return handled; 350 } 351 352 int 353 plic_irq_dispatch(uint32_t irq, void *frame) 354 { 355 int pri, s; 356 int handled = 0; 357 struct plic_softc* sc; 358 struct plic_intrhand *ih; 359 void *arg; 360 361 #ifdef DEBUG_INTC 362 printf("plic irq %d fired\n", irq); 363 #endif 364 365 sc = plic; 366 pri = sc->sc_isrcs[irq].is_irq_max; 367 s = plic_splraise(pri); 368 TAILQ_FOREACH(ih, &sc->sc_isrcs[irq].is_list, ih_list) { 369 #ifdef MULTIPROCESSOR 370 int need_lock; 371 372 if (ih->ih_flags & IPL_MPSAFE) 373 need_lock = 0; 374 else 375 need_lock = s < IPL_SCHED; 376 377 if (need_lock) 378 KERNEL_LOCK(); 379 #endif 380 381 if (ih->ih_arg != 0) 382 arg = ih->ih_arg; 383 else 384 arg = frame; 385 386 intr_enable(); 387 handled = ih->ih_func(arg); 388 intr_disable(); 389 if (handled) 390 ih->ih_count.ec_count++; 391 392 #ifdef MULTIPROCESSOR 393 if (need_lock) 394 KERNEL_UNLOCK(); 395 #endif 396 } 397 398 plic_splx(s); 399 return handled; 400 } 401 402 void * 403 plic_intr_establish(int irqno, int level, int (*func)(void *), 404 void *arg, char *name) 405 { 406 struct plic_softc *sc = plic; 407 struct plic_intrhand *ih; 408 u_long sie; 409 410 if (irqno < 0 || irqno >= PLIC_MAX_IRQS) 411 panic("plic_intr_establish: bogus irqnumber %d: %s", 412 irqno, name); 413 414 ih = malloc(sizeof *ih, M_DEVBUF, M_WAITOK); 415 ih->ih_func = func; 416 ih->ih_arg = arg; 417 ih->ih_ipl = level & IPL_IRQMASK; 418 ih->ih_flags = level & IPL_FLAGMASK; 419 ih->ih_irq = irqno; 420 ih->ih_name = name; 421 422 sie = intr_disable(); 423 424 TAILQ_INSERT_TAIL(&sc->sc_isrcs[irqno].is_list, ih, ih_list); 425 426 if (name != NULL) 427 evcount_attach(&ih->ih_count, name, &ih->ih_irq); 428 429 #ifdef DEBUG_INTC 430 printf("%s irq %d level %d [%s]\n", __func__, irqno, level, 431 name); 432 #endif 433 434 plic_calc_mask(); 435 436 intr_restore(sie); 437 return (ih); 438 } 439 440 void * 441 plic_intr_establish_fdt(void *cookie, int *cell, int level, 442 int (*func)(void *), void *arg, char *name) 443 { 444 return plic_intr_establish(cell[0], level, func, arg, name); 445 } 446 447 void 448 plic_intr_disestablish(void *cookie) 449 { 450 struct plic_softc *sc = plic; 451 struct plic_intrhand *ih = cookie; 452 int irqno = ih->ih_irq; 453 u_long sie; 454 455 sie = intr_disable(); 456 457 TAILQ_REMOVE(&sc->sc_isrcs[irqno].is_list, ih, ih_list); 458 if (ih->ih_name != NULL) 459 evcount_detach(&ih->ih_count); 460 461 intr_restore(sie); 462 463 free(ih, M_DEVBUF, 0); 464 } 465 466 void 467 plic_intr_route(void *cookie, int enable, struct cpu_info *ci) 468 { 469 struct plic_softc *sc = plic; 470 struct plic_intrhand *ih = cookie; 471 472 int irq = ih->ih_irq; 473 int cpu = ci->ci_cpuid; 474 uint32_t min_pri = sc->sc_isrcs[irq].is_irq_min; 475 476 if (enable == IRQ_ENABLE) { 477 plic_intr_enable_with_pri(irq, min_pri, cpu); 478 } else { 479 plic_intr_route_grid(irq, IRQ_DISABLE, cpu); 480 } 481 } 482 483 void 484 plic_splx(int new) 485 { 486 /* XXX 487 * how to do pending external interrupt ? 488 * After set the new threshold, if there is any pending 489 * external interrupts whose priority is now greater than the 490 * threshold, they will get passed through plic to cpu, 491 * trigger a new claim/complete cycle. 492 * So there is no need to handle pending external intr here. 493 * 494 */ 495 struct cpu_info *ci = curcpu(); 496 497 /* Pending software intr is handled here */ 498 if (ci->ci_ipending & riscv_smask[new]) 499 riscv_do_pending_intr(new); 500 501 plic_setipl(new); 502 } 503 504 int 505 plic_spllower(int new) 506 { 507 struct cpu_info *ci = curcpu(); 508 int old = ci->ci_cpl; 509 plic_splx(new); 510 return (old); 511 } 512 513 int 514 plic_splraise(int new) 515 { 516 struct cpu_info *ci = curcpu(); 517 int old; 518 old = ci->ci_cpl; 519 520 /* 521 * setipl must always be called because there is a race window 522 * where the variable is updated before the mask is set 523 * an interrupt occurs in that window without the mask always 524 * being set, the hardware might not get updated on the next 525 * splraise completely messing up spl protection. 526 */ 527 if (old > new) 528 new = old; 529 530 plic_setipl(new); 531 532 return (old); 533 } 534 535 void 536 plic_setipl(int new) 537 { 538 struct cpu_info *ci = curcpu(); 539 u_long sie; 540 541 /* disable here is only to keep hardware in sync with ci->ci_cpl */ 542 sie = intr_disable(); 543 ci->ci_cpl = new; 544 545 /* higher values are higher priority */ 546 plic_set_threshold(ci->ci_cpuid, new); 547 548 intr_restore(sie); 549 } 550 551 /* 552 * update the max/min priority for an interrupt src, 553 * and enforce the updated priority to plic. 554 * this should be called whenever a new handler is attached. 555 */ 556 void 557 plic_calc_mask(void) 558 { 559 struct cpu_info *ci = curcpu(); 560 struct plic_softc *sc = plic; 561 struct plic_intrhand *ih; 562 int irq; 563 564 /* PLIC irq 0 is reserved, thus we start from 1 */ 565 for (irq = 1; irq <= sc->sc_ndev; irq++) { 566 int max = IPL_NONE; 567 int min = IPL_HIGH; 568 TAILQ_FOREACH(ih, &sc->sc_isrcs[irq].is_list, ih_list) { 569 if (ih->ih_ipl > max) 570 max = ih->ih_ipl; 571 572 if (ih->ih_ipl < min) 573 min = ih->ih_ipl; 574 } 575 576 if (max == IPL_NONE) 577 min = IPL_NONE; 578 579 if (sc->sc_isrcs[irq].is_irq_max == max && 580 sc->sc_isrcs[irq].is_irq_min == min) 581 continue; 582 583 sc->sc_isrcs[irq].is_irq_max = max; 584 sc->sc_isrcs[irq].is_irq_min = min; 585 586 /* Enable interrupts at lower levels, clear -> enable */ 587 /* Set interrupt priority/enable */ 588 if (min != IPL_NONE) { 589 plic_intr_enable_with_pri(irq, min, ci->ci_cpuid); 590 } else { 591 plic_intr_disable(irq, ci->ci_cpuid); 592 } 593 } 594 595 plic_setipl(ci->ci_cpl); 596 } 597 598 /***************** helper functions *****************/ 599 600 /* 601 * OpenBSD saves cpu node info in ci struct, so we can search 602 * cpuid by node matching 603 */ 604 int 605 plic_get_cpuid(int intc) 606 { 607 uint32_t hart; 608 int parent_node; 609 struct cpu_info *ci; 610 CPU_INFO_ITERATOR cii; 611 612 /* Check the interrupt controller layout. */ 613 if (OF_getpropintarray(intc, "#interrupt-cells", &hart, 614 sizeof(hart)) < 0) { 615 printf(": could not find #interrupt-cells for phandle %u\n", intc); 616 return (-1); 617 } 618 619 /* 620 * The parent of the interrupt-controller is the CPU we are 621 * interested in, so search for its OF node index. 622 */ 623 parent_node = OF_parent(intc); 624 CPU_INFO_FOREACH(cii, ci) { 625 if (ci->ci_node == parent_node) 626 return ci->ci_cpuid; 627 } 628 return -1; 629 } 630 631 /* update priority for intr src 'irq' */ 632 void 633 plic_set_priority(int irq, uint32_t pri) 634 { 635 struct plic_softc *sc = plic; 636 uint32_t prival; 637 638 /* 639 * sifive plic only has 0 - 7 priority levels, yet OpenBSD defines 640 * 0 - 12 priority levels(level 1 - 4 are for SOFT*, level 12 641 * is for IPI. They should NEVER be passed to plic. 642 * So we calculate plic priority in the following way: 643 */ 644 if (pri <= 4 || pri >= 12)//invalid input 645 prival = 0;//effectively disable this intr source 646 else 647 prival = pri - 4; 648 649 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 650 PLIC_PRIORITY(irq), prival); 651 } 652 653 /* update threshold for 'cpu' */ 654 void 655 plic_set_threshold(int cpu, uint32_t threshold) 656 { 657 struct plic_softc *sc = plic; 658 uint32_t prival; 659 660 if (threshold < 4) // enable everything (as far as plic is concerned) 661 prival = 0; 662 else if (threshold >= 12) // invalid priority level ? 663 prival = IPL_HIGH - 4; // XXX Device-specific high threshold 664 else // everything else 665 prival = threshold - 4; // XXX Device-specific threshold offset 666 667 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 668 PLIC_THRESHOLD(sc, cpu), prival); 669 } 670 671 /* 672 * turns on/off the route from intr source 'irq' 673 * to context 'ci' based on 'enable' 674 */ 675 void 676 plic_intr_route_grid(int irq, int enable, int cpu) 677 { 678 struct plic_softc *sc = plic; 679 uint32_t val, mask; 680 681 if (irq == 0) 682 return; 683 684 KASSERT(cpu < MAXCPUS); 685 686 mask = (1 << (irq % 32)); 687 val = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 688 PLIC_ENABLE(sc, irq, cpu)); 689 if (enable == IRQ_ENABLE) 690 val |= mask; 691 else 692 val &= ~mask; 693 694 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 695 PLIC_ENABLE(sc, irq, cpu), val); 696 } 697 698 /* 699 * Enable intr src 'irq' to cpu 'cpu' by setting: 700 * - priority 701 * - threshold 702 * - enable bit 703 */ 704 void 705 plic_intr_enable_with_pri(int irq, uint32_t min_pri, int cpu) 706 { 707 plic_set_priority(irq, min_pri); 708 plic_set_threshold(cpu, min_pri-1); 709 plic_intr_route_grid(irq, IRQ_ENABLE, cpu); 710 } 711 712 void 713 plic_intr_disable(int irq, int cpu) 714 { 715 plic_set_priority(irq, 0); 716 plic_set_threshold(cpu, IPL_HIGH); 717 plic_intr_route_grid(irq, IRQ_DISABLE, cpu); 718 } 719 /***************** end of helper functions *****************/ 720