1 /* $OpenBSD: intr.c,v 1.12 2024/08/06 09:07:15 kettenis Exp $ */ 2 3 /* 4 * Copyright (c) 2011 Dale Rahn <drahn@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/param.h> 20 #include <sys/systm.h> 21 #include <sys/atomic.h> 22 #include <sys/malloc.h> 23 24 #include <machine/cpu.h> 25 #include <machine/intr.h> 26 #include <machine/sbi.h> 27 28 #include <dev/ofw/openfirm.h> 29 30 int riscv_intr_get_parent(int); 31 uint32_t riscv_intr_map_msi(int, uint64_t *); 32 33 void *riscv_intr_prereg_establish_fdt(void *, int *, int, struct cpu_info *, 34 int (*)(void *), void *, char *); 35 void riscv_intr_prereg_disestablish_fdt(void *); 36 void riscv_intr_prereg_barrier_fdt(void *); 37 38 int riscv_dflt_splraise(int); 39 int riscv_dflt_spllower(int); 40 void riscv_dflt_splx(int); 41 void riscv_dflt_setipl(int); 42 43 void riscv_dflt_intr(void *); 44 void riscv_cpu_intr(void *); 45 46 #define SI_TO_IRQBIT(x) (1 << (x)) 47 uint32_t riscv_smask[NIPL]; 48 49 struct riscv_intr_func riscv_intr_func = { 50 riscv_dflt_splraise, 51 riscv_dflt_spllower, 52 riscv_dflt_splx, 53 riscv_dflt_setipl 54 }; 55 56 void 57 riscv_dflt_intr(void *frame) 58 { 59 panic("%s", __func__); 60 } 61 62 void (*riscv_intr_dispatch)(void *) = riscv_dflt_intr; 63 64 void 65 riscv_cpu_intr(void *frame) 66 { 67 struct cpu_info *ci = curcpu(); 68 69 ci->ci_idepth++; 70 (*riscv_intr_dispatch)(frame); 71 ci->ci_idepth--; 72 } 73 74 /* 75 * Find the interrupt parent by walking up the tree. 76 */ 77 int 78 riscv_intr_get_parent(int node) 79 { 80 uint32_t phandle; 81 82 while (node) { 83 phandle = OF_getpropint(node, "interrupt-parent", 0); 84 if (phandle) 85 return OF_getnodebyphandle(phandle); 86 node = OF_parent(node); 87 if (OF_getpropbool(node, "interrupt-controller")) 88 return node; 89 } 90 91 return 0; 92 } 93 94 uint32_t 95 riscv_intr_map_msi(int node, uint64_t *data) 96 { 97 uint64_t msi_base; 98 uint32_t phandle = 0; 99 uint32_t *cell; 100 uint32_t *map; 101 uint32_t mask, rid_base, rid; 102 int i, len, length, mcells, ncells; 103 104 len = OF_getproplen(node, "msi-map"); 105 if (len <= 0) { 106 while (node && !phandle) { 107 phandle = OF_getpropint(node, "msi-parent", 0); 108 node = OF_parent(node); 109 } 110 111 return phandle; 112 } 113 114 map = malloc(len, M_TEMP, M_WAITOK); 115 OF_getpropintarray(node, "msi-map", map, len); 116 117 mask = OF_getpropint(node, "msi-map-mask", 0xffff); 118 rid = *data & mask; 119 120 cell = map; 121 ncells = len / sizeof(uint32_t); 122 while (ncells > 1) { 123 node = OF_getnodebyphandle(cell[1]); 124 if (node == 0) 125 goto out; 126 127 /* 128 * Some device trees (e.g. those for the Rockchip 129 * RK3399 boards) are missing a #msi-cells property. 130 * Assume the msi-specifier uses a single cell in that 131 * case. 132 */ 133 mcells = OF_getpropint(node, "#msi-cells", 1); 134 if (ncells < mcells + 3) 135 goto out; 136 137 rid_base = cell[0]; 138 length = cell[2 + mcells]; 139 msi_base = cell[2]; 140 for (i = 1; i < mcells; i++) { 141 msi_base <<= 32; 142 msi_base |= cell[2 + i]; 143 } 144 if (rid >= rid_base && rid < rid_base + length) { 145 *data = msi_base + (rid - rid_base); 146 phandle = cell[1]; 147 break; 148 } 149 150 cell += (3 + mcells); 151 ncells -= (3 + mcells); 152 } 153 154 out: 155 free(map, M_TEMP, len); 156 return phandle; 157 } 158 159 /* 160 * Interrupt pre-registration. 161 * 162 * To allow device drivers to establish interrupt handlers before all 163 * relevant interrupt controllers have been attached, we support 164 * pre-registration of interrupt handlers. For each node in the 165 * device tree that has an "interrupt-controller" property, we 166 * register a dummy interrupt controller that simply stashes away all 167 * relevant details of the interrupt handler being established. 168 * Later, when the real interrupt controller registers itself, we 169 * establish those interrupt handlers based on that information. 170 */ 171 172 #define MAX_INTERRUPT_CELLS 4 173 174 struct intr_prereg { 175 LIST_ENTRY(intr_prereg) ip_list; 176 uint32_t ip_phandle; 177 uint32_t ip_cell[MAX_INTERRUPT_CELLS]; 178 179 int ip_level; 180 struct cpu_info *ip_ci; 181 int (*ip_func)(void *); 182 void *ip_arg; 183 char *ip_name; 184 185 struct interrupt_controller *ip_ic; 186 void *ip_ih; 187 }; 188 189 LIST_HEAD(, intr_prereg) prereg_interrupts = 190 LIST_HEAD_INITIALIZER(prereg_interrupts); 191 192 void * 193 riscv_intr_prereg_establish_fdt(void *cookie, int *cell, int level, 194 struct cpu_info *ci, int (*func)(void *), void *arg, char *name) 195 { 196 struct interrupt_controller *ic = cookie; 197 struct intr_prereg *ip; 198 int i; 199 200 ip = malloc(sizeof(struct intr_prereg), M_DEVBUF, M_ZERO | M_WAITOK); 201 ip->ip_phandle = ic->ic_phandle; 202 for (i = 0; i < ic->ic_cells; i++) 203 ip->ip_cell[i] = cell[i]; 204 ip->ip_level = level; 205 ip->ip_ci = ci; 206 ip->ip_func = func; 207 ip->ip_arg = arg; 208 ip->ip_name = name; 209 LIST_INSERT_HEAD(&prereg_interrupts, ip, ip_list); 210 211 return ip; 212 } 213 214 void 215 riscv_intr_prereg_disestablish_fdt(void *cookie) 216 { 217 struct intr_prereg *ip = cookie; 218 struct interrupt_controller *ic = ip->ip_ic; 219 220 if (ip->ip_ic != NULL && ip->ip_ih != NULL) 221 ic->ic_disestablish(ip->ip_ih); 222 223 if (ip->ip_ic != NULL) 224 LIST_REMOVE(ip, ip_list); 225 226 free(ip, M_DEVBUF, sizeof(*ip)); 227 } 228 229 void 230 riscv_intr_prereg_barrier_fdt(void *cookie) 231 { 232 struct intr_prereg *ip = cookie; 233 struct interrupt_controller *ic = ip->ip_ic; 234 235 if (ip->ip_ic != NULL && ip->ip_ih != NULL) 236 ic->ic_barrier(ip->ip_ih); 237 } 238 239 void 240 riscv_intr_init_fdt_recurse(int node) 241 { 242 struct interrupt_controller *ic; 243 244 if (OF_getproplen(node, "interrupt-controller") >= 0) { 245 ic = malloc(sizeof(struct interrupt_controller), 246 M_DEVBUF, M_ZERO | M_WAITOK); 247 ic->ic_node = node; 248 ic->ic_cookie = ic; 249 ic->ic_establish = riscv_intr_prereg_establish_fdt; 250 ic->ic_disestablish = riscv_intr_prereg_disestablish_fdt; 251 ic->ic_barrier = riscv_intr_prereg_barrier_fdt; 252 riscv_intr_register_fdt(ic); 253 } 254 255 for (node = OF_child(node); node; node = OF_peer(node)) 256 riscv_intr_init_fdt_recurse(node); 257 } 258 259 void 260 riscv_intr_init_fdt(void) 261 { 262 int node = OF_peer(0); 263 264 if (node) 265 riscv_intr_init_fdt_recurse(node); 266 } 267 268 LIST_HEAD(, interrupt_controller) interrupt_controllers = 269 LIST_HEAD_INITIALIZER(interrupt_controllers); 270 271 void 272 riscv_intr_register_fdt(struct interrupt_controller *ic) 273 { 274 struct intr_prereg *ip, *tip; 275 276 ic->ic_cells = OF_getpropint(ic->ic_node, "#interrupt-cells", 0); 277 ic->ic_phandle = OF_getpropint(ic->ic_node, "phandle", 0); 278 KASSERT(ic->ic_cells <= MAX_INTERRUPT_CELLS); 279 280 LIST_INSERT_HEAD(&interrupt_controllers, ic, ic_list); 281 282 /* Establish pre-registered interrupt handlers. */ 283 LIST_FOREACH_SAFE(ip, &prereg_interrupts, ip_list, tip) { 284 if (ip->ip_phandle != ic->ic_phandle) 285 continue; 286 287 ip->ip_ic = ic; 288 if (ic->ic_establish)/* riscv_cpu_intc sets this to NULL */ 289 { 290 ip->ip_ih = ic->ic_establish(ic->ic_cookie, ip->ip_cell, 291 ip->ip_level, ip->ip_ci, ip->ip_func, ip->ip_arg, ip->ip_name); 292 if (ip->ip_ih == NULL) 293 printf("can't establish interrupt %s\n", ip->ip_name); 294 } 295 296 LIST_REMOVE(ip, ip_list); 297 } 298 } 299 300 void * 301 riscv_intr_establish_fdt(int node, int level, int (*func)(void *), 302 void *cookie, char *name) 303 { 304 return riscv_intr_establish_fdt_idx(node, 0, level, func, cookie, name); 305 } 306 307 void * 308 riscv_intr_establish_fdt_cpu(int node, int level, struct cpu_info *ci, 309 int (*func)(void *), void *cookie, char *name) 310 { 311 return riscv_intr_establish_fdt_idx_cpu(node, 0, level, ci, func, 312 cookie, name); 313 } 314 315 void * 316 riscv_intr_establish_fdt_idx(int node, int idx, int level, int (*func)(void *), 317 void *cookie, char *name) 318 { 319 return riscv_intr_establish_fdt_idx_cpu(node, idx, level, NULL, func, 320 cookie, name); 321 } 322 323 void * 324 riscv_intr_establish_fdt_idx_cpu(int node, int idx, int level, 325 struct cpu_info *ci, int (*func)(void *), void *cookie, char *name) 326 { 327 struct interrupt_controller *ic; 328 int i, len, ncells, parent; 329 int extended = 1; 330 uint32_t *cell, *cells, phandle; 331 struct machine_intr_handle *ih; 332 void *val = NULL; 333 334 len = OF_getproplen(node, "interrupts-extended"); 335 if (len <= 0) { 336 len = OF_getproplen(node, "interrupts"); 337 extended = 0; 338 } 339 if (len <= 0 || (len % sizeof(uint32_t) != 0)) 340 return NULL; 341 342 /* Old style. */ 343 if (!extended) { 344 parent = riscv_intr_get_parent(node); 345 LIST_FOREACH(ic, &interrupt_controllers, ic_list) { 346 if (ic->ic_node == parent) 347 break; 348 } 349 350 if (ic == NULL) 351 return NULL; 352 } 353 354 cell = cells = malloc(len, M_TEMP, M_WAITOK); 355 if (extended) 356 OF_getpropintarray(node, "interrupts-extended", cells, len); 357 else 358 OF_getpropintarray(node, "interrupts", cells, len); 359 ncells = len / sizeof(uint32_t); 360 361 for (i = 0; i <= idx && ncells > 0; i++) { 362 if (extended) { 363 phandle = cell[0]; 364 365 /* Handle "empty" phandle reference. */ 366 if (phandle == 0) { 367 cell++; 368 ncells--; 369 continue; 370 } 371 372 LIST_FOREACH(ic, &interrupt_controllers, ic_list) { 373 if (ic->ic_phandle == phandle) 374 break; 375 } 376 377 if (ic == NULL) 378 break; 379 380 cell++; 381 ncells--; 382 } 383 384 if (i == idx && ncells >= ic->ic_cells && ic->ic_establish) { 385 val = ic->ic_establish(ic->ic_cookie, cell, level, 386 ci, func, cookie, name); 387 break; 388 } 389 390 cell += ic->ic_cells; 391 ncells -= ic->ic_cells; 392 } 393 394 free(cells, M_TEMP, len); 395 396 if (val == NULL) 397 return NULL; 398 399 ih = malloc(sizeof(*ih), M_DEVBUF, M_WAITOK); 400 ih->ih_ic = ic; 401 ih->ih_ih = val; 402 403 return ih; 404 } 405 406 void * 407 riscv_intr_establish_fdt_imap_cpu(int node, int *reg, int nreg, int level, 408 struct cpu_info *ci, int (*func)(void *), void *cookie, char *name) 409 { 410 struct interrupt_controller *ic; 411 struct machine_intr_handle *ih; 412 uint32_t *cell; 413 uint32_t map_mask[4], *map; 414 int len, acells, ncells; 415 void *val = NULL; 416 417 if (nreg != sizeof(map_mask)) 418 return NULL; 419 420 if (OF_getpropintarray(node, "interrupt-map-mask", map_mask, 421 sizeof(map_mask)) != sizeof(map_mask)) 422 return NULL; 423 424 len = OF_getproplen(node, "interrupt-map"); 425 if (len <= 0) 426 return NULL; 427 428 map = malloc(len, M_DEVBUF, M_WAITOK); 429 OF_getpropintarray(node, "interrupt-map", map, len); 430 431 cell = map; 432 ncells = len / sizeof(uint32_t); 433 while (ncells > 5) { 434 LIST_FOREACH(ic, &interrupt_controllers, ic_list) { 435 if (ic->ic_phandle == cell[4]) 436 break; 437 } 438 439 if (ic == NULL) 440 break; 441 442 acells = OF_getpropint(ic->ic_node, "#address-cells", 0); 443 if (ncells >= (5 + acells + ic->ic_cells) && 444 (reg[0] & map_mask[0]) == cell[0] && 445 (reg[1] & map_mask[1]) == cell[1] && 446 (reg[2] & map_mask[2]) == cell[2] && 447 (reg[3] & map_mask[3]) == cell[3] && 448 ic->ic_establish) { 449 val = ic->ic_establish(ic->ic_cookie, &cell[5 + acells], 450 level, ci, func, cookie, name); 451 break; 452 } 453 454 cell += (5 + acells + ic->ic_cells); 455 ncells -= (5 + acells + ic->ic_cells); 456 } 457 458 if (val == NULL) { 459 free(map, M_DEVBUF, len); 460 return NULL; 461 } 462 463 ih = malloc(sizeof(*ih), M_DEVBUF, M_WAITOK); 464 ih->ih_ic = ic; 465 ih->ih_ih = val; 466 467 free(map, M_DEVBUF, len); 468 return ih; 469 } 470 471 void * 472 riscv_intr_establish_fdt_msi_cpu(int node, uint64_t *addr, uint64_t *data, 473 int level, struct cpu_info *ci, int (*func)(void *), void *cookie, 474 char *name) 475 { 476 struct interrupt_controller *ic; 477 struct machine_intr_handle *ih; 478 uint32_t phandle; 479 void *val = NULL; 480 481 phandle = riscv_intr_map_msi(node, data); 482 LIST_FOREACH(ic, &interrupt_controllers, ic_list) { 483 if (ic->ic_phandle == phandle) 484 break; 485 } 486 487 if (ic == NULL || ic->ic_establish_msi == NULL) 488 return NULL; 489 490 val = ic->ic_establish_msi(ic->ic_cookie, addr, data, 491 level, ci, func, cookie, name); 492 493 ih = malloc(sizeof(*ih), M_DEVBUF, M_WAITOK); 494 ih->ih_ic = ic; 495 ih->ih_ih = val; 496 497 return ih; 498 } 499 500 void 501 riscv_intr_disestablish_fdt(void *cookie) 502 { 503 struct machine_intr_handle *ih = cookie; 504 struct interrupt_controller *ic = ih->ih_ic; 505 506 ic->ic_disestablish(ih->ih_ih); 507 free(ih, M_DEVBUF, sizeof(*ih)); 508 } 509 510 void 511 riscv_intr_enable(void *cookie) 512 { 513 struct machine_intr_handle *ih = cookie; 514 struct interrupt_controller *ic = ih->ih_ic; 515 516 KASSERT(ic->ic_enable != NULL); 517 ic->ic_enable(ih->ih_ih); 518 } 519 520 void 521 riscv_intr_disable(void *cookie) 522 { 523 struct machine_intr_handle *ih = cookie; 524 struct interrupt_controller *ic = ih->ih_ic; 525 526 KASSERT(ic->ic_disable != NULL); 527 ic->ic_disable(ih->ih_ih); 528 } 529 530 void 531 riscv_intr_route(void *cookie, int enable, struct cpu_info *ci) 532 { 533 struct machine_intr_handle *ih = cookie; 534 struct interrupt_controller *ic = ih->ih_ic; 535 536 if (ic->ic_route) 537 ic->ic_route(ih->ih_ih, enable, ci); 538 } 539 540 void 541 riscv_intr_cpu_enable(void) 542 { 543 struct interrupt_controller *ic; 544 545 LIST_FOREACH(ic, &interrupt_controllers, ic_list) 546 if (ic->ic_cpu_enable) 547 ic->ic_cpu_enable(); 548 } 549 550 int 551 riscv_dflt_splraise(int newcpl) 552 { 553 struct cpu_info *ci = curcpu(); 554 int oldcpl; 555 556 oldcpl = ci->ci_cpl; 557 558 if (newcpl < oldcpl) 559 newcpl = oldcpl; 560 561 ci->ci_cpl = newcpl; 562 563 return oldcpl; 564 } 565 566 int 567 riscv_dflt_spllower(int newcpl) 568 { 569 struct cpu_info *ci = curcpu(); 570 int oldcpl; 571 572 oldcpl = ci->ci_cpl; 573 574 splx(newcpl); 575 576 return oldcpl; 577 } 578 579 void 580 riscv_dflt_splx(int newcpl) 581 { 582 struct cpu_info *ci = curcpu(); 583 584 if (ci->ci_ipending & riscv_smask[newcpl]) 585 riscv_do_pending_intr(newcpl); 586 ci->ci_cpl = newcpl; 587 } 588 589 void 590 riscv_dflt_setipl(int newcpl) 591 { 592 struct cpu_info *ci = curcpu(); 593 594 ci->ci_cpl = newcpl; 595 } 596 597 void 598 riscv_do_pending_intr(int pcpl) 599 { 600 struct cpu_info *ci = curcpu(); 601 u_long sie; 602 603 sie = intr_disable(); 604 605 #define DO_SOFTINT(si, ipl) \ 606 if ((ci->ci_ipending & riscv_smask[pcpl]) & \ 607 SI_TO_IRQBIT(si)) { \ 608 ci->ci_ipending &= ~SI_TO_IRQBIT(si); \ 609 riscv_intr_func.setipl(ipl); \ 610 intr_restore(sie); \ 611 softintr_dispatch(si); \ 612 sie = intr_disable(); \ 613 } 614 615 do { 616 DO_SOFTINT(SIR_TTY, IPL_SOFTTTY); 617 DO_SOFTINT(SIR_NET, IPL_SOFTNET); 618 DO_SOFTINT(SIR_CLOCK, IPL_SOFTCLOCK); 619 DO_SOFTINT(SIR_SOFT, IPL_SOFT); 620 } while (ci->ci_ipending & riscv_smask[pcpl]); 621 622 /* Don't use splx... we are here already! */ 623 riscv_intr_func.setipl(pcpl); 624 intr_restore(sie); 625 } 626 627 void 628 riscv_set_intr_func(int (*raise)(int), int (*lower)(int), void (*x)(int), 629 void (*setipl)(int)) 630 { 631 riscv_intr_func.raise = raise; 632 riscv_intr_func.lower = lower; 633 riscv_intr_func.x = x; 634 riscv_intr_func.setipl = setipl; 635 } 636 637 void 638 riscv_set_intr_handler(void (*intr_handle)(void *)) 639 { 640 riscv_intr_dispatch = intr_handle; 641 } 642 643 void 644 riscv_init_smask(void) 645 { 646 static int inited = 0; 647 int i; 648 649 if (inited) 650 return; 651 inited = 1; 652 653 for (i = IPL_NONE; i <= IPL_HIGH; i++) { 654 riscv_smask[i] = 0; 655 if (i < IPL_SOFT) 656 riscv_smask[i] |= SI_TO_IRQBIT(SIR_SOFT); 657 if (i < IPL_SOFTCLOCK) 658 riscv_smask[i] |= SI_TO_IRQBIT(SIR_CLOCK); 659 if (i < IPL_SOFTNET) 660 riscv_smask[i] |= SI_TO_IRQBIT(SIR_NET); 661 if (i < IPL_SOFTTTY) 662 riscv_smask[i] |= SI_TO_IRQBIT(SIR_TTY); 663 } 664 } 665 666 /* provide functions for asm */ 667 #undef splraise 668 #undef spllower 669 #undef splx 670 671 int 672 splraise(int ipl) 673 { 674 return riscv_intr_func.raise(ipl); 675 } 676 677 int _spllower(int ipl); /* XXX - called from asm? */ 678 int 679 _spllower(int ipl) 680 { 681 return riscv_intr_func.lower(ipl); 682 } 683 int 684 spllower(int ipl) 685 { 686 return riscv_intr_func.lower(ipl); 687 } 688 689 void 690 splx(int ipl) 691 { 692 riscv_intr_func.x(ipl); 693 } 694 695 696 #ifdef DIAGNOSTIC 697 void 698 riscv_splassert_check(int wantipl, const char *func) 699 { 700 int oldipl = curcpu()->ci_cpl; 701 702 if (oldipl < wantipl) { 703 splassert_fail(wantipl, oldipl, func); 704 /* 705 * If the splassert_ctl is set to not panic, raise the ipl 706 * in a feeble attempt to reduce damage. 707 */ 708 riscv_intr_func.setipl(wantipl); 709 } 710 711 if (wantipl == IPL_NONE && curcpu()->ci_idepth != 0) { 712 splassert_fail(-1, curcpu()->ci_idepth, func); 713 } 714 } 715 #endif 716 717 void 718 intr_barrier(void *cookie) 719 { 720 struct machine_intr_handle *ih = cookie; 721 struct interrupt_controller *ic = ih->ih_ic; 722 723 ic->ic_barrier(ih->ih_ih); 724 } 725 726 /* 727 * IPI implementation 728 */ 729 730 #ifdef MULTIPROCESSOR 731 732 void 733 intr_send_ipi(struct cpu_info *ci, int reason) 734 { 735 unsigned long hart_mask; 736 737 if (ci == curcpu() && reason == IPI_NOP) 738 return; 739 740 if (reason != IPI_NOP) 741 atomic_setbits_int(&ci->ci_ipi_reason, reason); 742 743 hart_mask = (1UL << ci->ci_hartid); 744 sbi_send_ipi(&hart_mask); 745 } 746 747 int 748 ipi_intr(void *frame) 749 { 750 struct cpu_info *ci = curcpu(); 751 int pending; 752 753 csr_clear(sip, SIP_SSIP); 754 pending = atomic_swap_uint(&ci->ci_ipi_reason, IPI_NOP); 755 756 if (pending & IPI_DDB) 757 db_enter(); 758 759 return 1; 760 } 761 762 #endif 763