1 /* $NetBSD: e500_intr.c,v 1.16 2011/09/27 01:02:35 jym Exp $ */ 2 /*- 3 * Copyright (c) 2010, 2011 The NetBSD Foundation, Inc. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The NetBSD Foundation 7 * by Raytheon BBN Technologies Corp and Defense Advanced Research Projects 8 * Agency and which was developed by Matt Thomas of 3am Software Foundry. 9 * 10 * This material is based upon work supported by the Defense Advanced Research 11 * Projects Agency and Space and Naval Warfare Systems Center, Pacific, under 12 * Contract No. N66001-09-C-2073. 13 * Approved for Public Release, Distribution Unlimited 14 * 15 * Redistribution and use in source and binary forms, with or without 16 * modification, are permitted provided that the following conditions 17 * are met: 18 * 1. Redistributions of source code must retain the above copyright 19 * notice, this list of conditions and the following disclaimer. 20 * 2. Redistributions in binary form must reproduce the above copyright 21 * notice, this list of conditions and the following disclaimer in the 22 * documentation and/or other materials provided with the distribution. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 25 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 26 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 28 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 29 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 30 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 31 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 32 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 33 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 34 * POSSIBILITY OF SUCH DAMAGE. 35 */ 36 37 #include "opt_mpc85xx.h" 38 39 #define __INTR_PRIVATE 40 41 #include <sys/cdefs.h> 42 __KERNEL_RCSID(0, "$NetBSD: e500_intr.c,v 1.16 2011/09/27 01:02:35 jym Exp $"); 43 44 #include <sys/param.h> 45 #include <sys/proc.h> 46 #include <sys/intr.h> 47 #include <sys/cpu.h> 48 #include <sys/kmem.h> 49 #include <sys/atomic.h> 50 #include <sys/bus.h> 51 #include <sys/xcall.h> 52 #include <sys/bitops.h> 53 54 #include <uvm/uvm_extern.h> 55 56 #ifdef __HAVE_FAST_SOFTINTS 57 #include <powerpc/softint.h> 58 #endif 59 60 #include <powerpc/spr.h> 61 #include <powerpc/booke/spr.h> 62 63 #include <powerpc/booke/cpuvar.h> 64 #include <powerpc/booke/e500reg.h> 65 #include <powerpc/booke/e500var.h> 66 #include <powerpc/booke/openpicreg.h> 67 68 #define IPL2CTPR(ipl) ((ipl) + 15 - IPL_HIGH) 69 #define CTPR2IPL(ctpr) ((ctpr) - (15 - IPL_HIGH)) 70 71 #define IST_PERCPU_P(ist) ((ist) >= IST_TIMER) 72 73 struct e500_intr_irq_info { 74 bus_addr_t irq_vpr; 75 bus_addr_t irq_dr; 76 u_int irq_vector; 77 }; 78 79 struct intr_source { 80 int (*is_func)(void *); 81 void *is_arg; 82 int8_t is_ipl; 83 uint8_t is_ist; 84 uint8_t is_irq; 85 bus_size_t is_vpr; 86 bus_size_t is_dr; 87 }; 88 89 #define INTR_SOURCE_INITIALIZER \ 90 { .is_func = e500_intr_spurious, .is_arg = NULL, \ 91 .is_irq = -1, .is_ipl = IPL_NONE, .is_ist = IST_NONE, } 92 93 struct e500_intr_name { 94 uint8_t in_irq; 95 const char in_name[15]; 96 }; 97 98 static const struct e500_intr_name e500_onchip_intr_names[] = { 99 { ISOURCE_L2, "l2" }, 100 { ISOURCE_ECM, "ecm" }, 101 { ISOURCE_DDR, "ddr" }, 102 { ISOURCE_LBC, "lbc" }, 103 { ISOURCE_DMA_CHAN1, "dma-chan1" }, 104 { ISOURCE_DMA_CHAN2, "dma-chan2" }, 105 { ISOURCE_DMA_CHAN3, "dma-chan3" }, 106 { ISOURCE_DMA_CHAN4, "dma-chan4" }, 107 { ISOURCE_PCI1, "pci1" }, 108 { ISOURCE_PCIEX2, "pcie2" }, 109 { ISOURCE_PCIEX , "pcie1" }, 110 { ISOURCE_PCIEX3, "pcie3" }, 111 { ISOURCE_USB1, "usb1" }, 112 { ISOURCE_ETSEC1_TX, "etsec1-tx" }, 113 { ISOURCE_ETSEC1_RX, "etsec1-rx" }, 114 { ISOURCE_ETSEC3_TX, "etsec3-tx" }, 115 { ISOURCE_ETSEC3_RX, "etsec3-rx" }, 116 { ISOURCE_ETSEC3_ERR, "etsec3-err" }, 117 { ISOURCE_ETSEC1_ERR, "etsec1-err" }, 118 { ISOURCE_ETSEC2_TX, "etsec2-tx" }, 119 { ISOURCE_ETSEC2_RX, "etsec2-rx" }, 120 { ISOURCE_ETSEC4_TX, "etsec4-tx" }, 121 { ISOURCE_ETSEC4_RX, "etsec4-rx" }, 122 { ISOURCE_ETSEC4_ERR, "etsec4-err" }, 123 { ISOURCE_ETSEC2_ERR, "etsec2-err" }, 124 { ISOURCE_DUART, "duart" }, 125 { ISOURCE_I2C, "i2c" }, 126 { ISOURCE_PERFMON, "perfmon" }, 127 { ISOURCE_SECURITY1, "sec1" }, 128 { ISOURCE_GPIO, "gpio" }, 129 { ISOURCE_SRIO_EWPU, "srio-ewpu" }, 130 { ISOURCE_SRIO_ODBELL, "srio-odbell" }, 131 { ISOURCE_SRIO_IDBELL, "srio-idbell" }, 132 { ISOURCE_SRIO_OMU1, "srio-omu1" }, 133 { ISOURCE_SRIO_IMU1, "srio-imu1" }, 134 { ISOURCE_SRIO_OMU2, "srio-omu2" }, 135 { ISOURCE_SRIO_IMU2, "srio-imu2" }, 136 { ISOURCE_SECURITY2, "sec2" }, 137 { ISOURCE_SPI, "spi" }, 138 { ISOURCE_ETSEC1_PTP, "etsec1-ptp" }, 139 { ISOURCE_ETSEC2_PTP, "etsec2-ptp" }, 140 { ISOURCE_ETSEC3_PTP, "etsec3-ptp" }, 141 { ISOURCE_ETSEC4_PTP, "etsec4-ptp" }, 142 { ISOURCE_ESDHC, "esdhc" }, 143 { 0, "" }, 144 }; 145 146 const struct e500_intr_name default_external_intr_names[] = { 147 { 0, "" }, 148 }; 149 150 static const struct e500_intr_name e500_msigroup_intr_names[] = { 151 { 0, "msigroup0" }, 152 { 1, "msigroup1" }, 153 { 2, "msigroup2" }, 154 { 3, "msigroup3" }, 155 { 4, "msigroup4" }, 156 { 5, "msigroup5" }, 157 { 6, "msigroup6" }, 158 { 7, "msigroup7" }, 159 { 0, "" }, 160 }; 161 162 static const struct e500_intr_name e500_timer_intr_names[] = { 163 { 0, "timer0" }, 164 { 1, "timer1" }, 165 { 2, "timer2" }, 166 { 3, "timer3" }, 167 { 0, "" }, 168 }; 169 170 static const struct e500_intr_name e500_ipi_intr_names[] = { 171 { 0, "ipi0" }, 172 { 1, "ipi1" }, 173 { 2, "ipi2" }, 174 { 3, "ipi3" }, 175 { 0, "" }, 176 }; 177 178 static const struct e500_intr_name e500_mi_intr_names[] = { 179 { 0, "mi0" }, 180 { 1, "mi1" }, 181 { 2, "mi2" }, 182 { 3, "mi3" }, 183 { 0, "" }, 184 }; 185 186 struct e500_intr_info { 187 u_int ii_external_sources; 188 uint32_t ii_onchip_bitmap[2]; 189 u_int ii_onchip_sources; 190 u_int ii_msigroup_sources; 191 u_int ii_ipi_sources; /* per-cpu */ 192 u_int ii_timer_sources; /* per-cpu */ 193 u_int ii_mi_sources; /* per-cpu */ 194 u_int ii_percpu_sources; 195 const struct e500_intr_name *ii_external_intr_names; 196 const struct e500_intr_name *ii_onchip_intr_names; 197 u_int8_t ii_ist_vectors[IST_MAX+1]; 198 }; 199 200 static kmutex_t e500_intr_lock __cacheline_aligned; 201 static struct e500_intr_info e500_intr_info; 202 203 #define INTR_INFO_DECL(lc_chip, UC_CHIP) \ 204 static const struct e500_intr_info lc_chip##_intr_info = { \ 205 .ii_external_sources = UC_CHIP ## _EXTERNALSOURCES, \ 206 .ii_onchip_bitmap = UC_CHIP ## _ONCHIPBITMAP, \ 207 .ii_onchip_sources = UC_CHIP ## _ONCHIPSOURCES, \ 208 .ii_msigroup_sources = UC_CHIP ## _MSIGROUPSOURCES, \ 209 .ii_timer_sources = UC_CHIP ## _TIMERSOURCES, \ 210 .ii_ipi_sources = UC_CHIP ## _IPISOURCES, \ 211 .ii_mi_sources = UC_CHIP ## _MISOURCES, \ 212 .ii_percpu_sources = UC_CHIP ## _TIMERSOURCES \ 213 + UC_CHIP ## _IPISOURCES + UC_CHIP ## _MISOURCES, \ 214 .ii_external_intr_names = lc_chip ## _external_intr_names, \ 215 .ii_onchip_intr_names = lc_chip ## _onchip_intr_names, \ 216 .ii_ist_vectors = { \ 217 [IST_NONE] = ~0, \ 218 [IST_EDGE] = 0, \ 219 [IST_LEVEL_LOW] = 0, \ 220 [IST_LEVEL_HIGH] = 0, \ 221 [IST_PULSE] = 0, \ 222 [IST_ONCHIP] = UC_CHIP ## _EXTERNALSOURCES, \ 223 [IST_MSIGROUP] = UC_CHIP ## _EXTERNALSOURCES \ 224 + UC_CHIP ## _ONCHIPSOURCES, \ 225 [IST_TIMER] = UC_CHIP ## _EXTERNALSOURCES \ 226 + UC_CHIP ## _ONCHIPSOURCES \ 227 + UC_CHIP ## _MSIGROUPSOURCES, \ 228 [IST_IPI] = UC_CHIP ## _EXTERNALSOURCES \ 229 + UC_CHIP ## _ONCHIPSOURCES \ 230 + UC_CHIP ## _MSIGROUPSOURCES \ 231 + UC_CHIP ## _TIMERSOURCES, \ 232 [IST_MI] = UC_CHIP ## _EXTERNALSOURCES \ 233 + UC_CHIP ## _ONCHIPSOURCES \ 234 + UC_CHIP ## _MSIGROUPSOURCES \ 235 + UC_CHIP ## _TIMERSOURCES \ 236 + UC_CHIP ## _IPISOURCES, \ 237 [IST_MAX] = UC_CHIP ## _EXTERNALSOURCES \ 238 + UC_CHIP ## _ONCHIPSOURCES \ 239 + UC_CHIP ## _MSIGROUPSOURCES \ 240 + UC_CHIP ## _TIMERSOURCES \ 241 + UC_CHIP ## _IPISOURCES \ 242 + UC_CHIP ## _MISOURCES, \ 243 }, \ 244 } 245 246 #ifdef MPC8536 247 #define mpc8536_external_intr_names default_external_intr_names 248 const struct e500_intr_name mpc8536_onchip_intr_names[] = { 249 { ISOURCE_SATA2, "sata2" }, 250 { ISOURCE_USB2, "usb2" }, 251 { ISOURCE_USB3, "usb3" }, 252 { ISOURCE_SATA1, "sata1" }, 253 { 0, "" }, 254 }; 255 256 INTR_INFO_DECL(mpc8536, MPC8536); 257 #endif 258 259 #ifdef MPC8544 260 #define mpc8544_external_intr_names default_external_intr_names 261 const struct e500_intr_name mpc8544_onchip_intr_names[] = { 262 { 0, "" }, 263 }; 264 265 INTR_INFO_DECL(mpc8544, MPC8544); 266 #endif 267 #ifdef MPC8548 268 #define mpc8548_external_intr_names default_external_intr_names 269 const struct e500_intr_name mpc8548_onchip_intr_names[] = { 270 { ISOURCE_PCI1, "pci1" }, 271 { ISOURCE_PCI2, "pci2" }, 272 { 0, "" }, 273 }; 274 275 INTR_INFO_DECL(mpc8548, MPC8548); 276 #endif 277 #ifdef MPC8555 278 #define mpc8555_external_intr_names default_external_intr_names 279 const struct e500_intr_name mpc8555_onchip_intr_names[] = { 280 { ISOURCE_PCI2, "pci2" }, 281 { ISOURCE_CPM, "CPM" }, 282 { 0, "" }, 283 }; 284 285 INTR_INFO_DECL(mpc8555, MPC8555); 286 #endif 287 #ifdef MPC8568 288 #define mpc8568_external_intr_names default_external_intr_names 289 const struct e500_intr_name mpc8568_onchip_intr_names[] = { 290 { ISOURCE_QEB_LOW, "QEB low" }, 291 { ISOURCE_QEB_PORT, "QEB port" }, 292 { ISOURCE_QEB_IECC, "QEB iram ecc" }, 293 { ISOURCE_QEB_MUECC, "QEB ram ecc" }, 294 { ISOURCE_TLU1, "tlu1" }, 295 { ISOURCE_QEB_HIGH, "QEB high" }, 296 { 0, "" }, 297 }; 298 299 INTR_INFO_DECL(mpc8568, MPC8568); 300 #endif 301 #ifdef MPC8572 302 #define mpc8572_external_intr_names default_external_intr_names 303 const struct e500_intr_name mpc8572_onchip_intr_names[] = { 304 { ISOURCE_PCIEX3_MPC8572, "pcie3" }, 305 { ISOURCE_FEC, "fec" }, 306 { ISOURCE_PME_GENERAL, "pme" }, 307 { ISOURCE_TLU1, "tlu1" }, 308 { ISOURCE_TLU2, "tlu2" }, 309 { ISOURCE_PME_CHAN1, "pme-chan1" }, 310 { ISOURCE_PME_CHAN2, "pme-chan2" }, 311 { ISOURCE_PME_CHAN3, "pme-chan3" }, 312 { ISOURCE_PME_CHAN4, "pme-chan4" }, 313 { ISOURCE_DMA2_CHAN1, "dma2-chan1" }, 314 { ISOURCE_DMA2_CHAN2, "dma2-chan2" }, 315 { ISOURCE_DMA2_CHAN3, "dma2-chan3" }, 316 { ISOURCE_DMA2_CHAN4, "dma2-chan4" }, 317 { 0, "" }, 318 }; 319 320 INTR_INFO_DECL(mpc8572, MPC8572); 321 #endif 322 #ifdef P2020 323 #define p20x0_external_intr_names default_external_intr_names 324 const struct e500_intr_name p20x0_onchip_intr_names[] = { 325 { ISOURCE_PCIEX3_MPC8572, "pcie3" }, 326 { ISOURCE_DMA2_CHAN1, "dma2-chan1" }, 327 { ISOURCE_DMA2_CHAN2, "dma2-chan2" }, 328 { ISOURCE_DMA2_CHAN3, "dma2-chan3" }, 329 { ISOURCE_DMA2_CHAN4, "dma2-chan4" }, 330 { 0, "" }, 331 }; 332 333 INTR_INFO_DECL(p20x0, P20x0); 334 #endif 335 336 static const char ist_names[][12] = { 337 [IST_NONE] = "none", 338 [IST_EDGE] = "edge", 339 [IST_LEVEL_LOW] = "level-", 340 [IST_LEVEL_HIGH] = "level+", 341 [IST_PULSE] = "pulse", 342 [IST_MSI] = "msi", 343 [IST_ONCHIP] = "onchip", 344 [IST_MSIGROUP] = "msigroup", 345 [IST_TIMER] = "timer", 346 [IST_IPI] = "ipi", 347 [IST_MI] = "msgint", 348 }; 349 350 static struct intr_source *e500_intr_sources; 351 static const struct intr_source *e500_intr_last_source; 352 353 static void *e500_intr_establish(int, int, int, int (*)(void *), void *); 354 static void e500_intr_disestablish(void *); 355 static void e500_intr_cpu_attach(struct cpu_info *ci); 356 static void e500_intr_cpu_hatch(struct cpu_info *ci); 357 static void e500_intr_cpu_send_ipi(cpuid_t, uintptr_t); 358 static void e500_intr_init(void); 359 static const char *e500_intr_string(int, int); 360 static const char *e500_intr_typename(int); 361 static void e500_critintr(struct trapframe *tf); 362 static void e500_decrintr(struct trapframe *tf); 363 static void e500_extintr(struct trapframe *tf); 364 static void e500_fitintr(struct trapframe *tf); 365 static void e500_wdogintr(struct trapframe *tf); 366 static void e500_spl0(void); 367 static int e500_splraise(int); 368 static void e500_splx(int); 369 370 const struct intrsw e500_intrsw = { 371 .intrsw_establish = e500_intr_establish, 372 .intrsw_disestablish = e500_intr_disestablish, 373 .intrsw_init = e500_intr_init, 374 .intrsw_cpu_attach = e500_intr_cpu_attach, 375 .intrsw_cpu_hatch = e500_intr_cpu_hatch, 376 .intrsw_cpu_send_ipi = e500_intr_cpu_send_ipi, 377 .intrsw_string = e500_intr_string, 378 .intrsw_typename = e500_intr_typename, 379 380 .intrsw_critintr = e500_critintr, 381 .intrsw_decrintr = e500_decrintr, 382 .intrsw_extintr = e500_extintr, 383 .intrsw_fitintr = e500_fitintr, 384 .intrsw_wdogintr = e500_wdogintr, 385 386 .intrsw_splraise = e500_splraise, 387 .intrsw_splx = e500_splx, 388 .intrsw_spl0 = e500_spl0, 389 390 #ifdef __HAVE_FAST_SOFTINTS 391 .intrsw_softint_init_md = powerpc_softint_init_md, 392 .intrsw_softint_trigger = powerpc_softint_trigger, 393 #endif 394 }; 395 396 static inline uint32_t 397 openpic_read(struct cpu_softc *cpu, bus_size_t offset) 398 { 399 400 return bus_space_read_4(cpu->cpu_bst, cpu->cpu_bsh, 401 OPENPIC_BASE + offset); 402 } 403 404 static inline void 405 openpic_write(struct cpu_softc *cpu, bus_size_t offset, uint32_t val) 406 { 407 408 return bus_space_write_4(cpu->cpu_bst, cpu->cpu_bsh, 409 OPENPIC_BASE + offset, val); 410 } 411 412 static const char * 413 e500_intr_external_name_lookup(int irq) 414 { 415 prop_array_t extirqs = board_info_get_object("external-irqs"); 416 prop_string_t irqname = prop_array_get(extirqs, irq); 417 KASSERT(irqname != NULL); 418 KASSERT(prop_object_type(irqname) == PROP_TYPE_STRING); 419 420 return prop_string_cstring_nocopy(irqname); 421 } 422 423 static const char * 424 e500_intr_name_lookup(const struct e500_intr_name *names, int irq) 425 { 426 for (; names->in_name[0] != '\0'; names++) { 427 if (names->in_irq == irq) 428 return names->in_name; 429 } 430 431 return NULL; 432 } 433 434 static const char * 435 e500_intr_onchip_name_lookup(int irq) 436 { 437 const char *name; 438 439 name = e500_intr_name_lookup(e500_intr_info.ii_onchip_intr_names, irq); 440 if (name == NULL) 441 name = e500_intr_name_lookup(e500_onchip_intr_names, irq); 442 443 return name; 444 } 445 446 static inline void 447 e500_splset(struct cpu_info *ci, int ipl) 448 { 449 struct cpu_softc * const cpu = ci->ci_softc; 450 451 //KASSERT(!cpu_intr_p() || ipl >= IPL_VM); 452 KASSERT((curlwp->l_pflag & LP_INTR) == 0 || ipl != IPL_NONE); 453 #if 0 454 u_int ctpr = ipl; 455 KASSERT(openpic_read(cpu, OPENPIC_CTPR) == ci->ci_cpl); 456 #elif 0 457 u_int old_ctpr = (ci->ci_cpl >= IPL_VM ? 15 : ci->ci_cpl); 458 u_int ctpr = (ipl >= IPL_VM ? 15 : ipl); 459 KASSERT(openpic_read(cpu, OPENPIC_CTPR) == old_ctpr); 460 #else 461 const u_int ctpr = IPL2CTPR(ipl); 462 KASSERT(openpic_read(cpu, OPENPIC_CTPR) == IPL2CTPR(ci->ci_cpl)); 463 #endif 464 openpic_write(cpu, OPENPIC_CTPR, ctpr); 465 KASSERT(openpic_read(cpu, OPENPIC_CTPR) == ctpr); 466 ci->ci_cpl = ipl; 467 } 468 469 static void 470 e500_spl0(void) 471 { 472 wrtee(0); 473 474 struct cpu_info * const ci = curcpu(); 475 476 #ifdef __HAVE_FAST_SOFTINTS 477 if (__predict_false(ci->ci_data.cpu_softints != 0)) { 478 e500_splset(ci, IPL_HIGH); 479 powerpc_softint(ci, IPL_NONE, 480 (vaddr_t)__builtin_return_address(0)); 481 } 482 #endif /* __HAVE_FAST_SOFTINTS */ 483 e500_splset(ci, IPL_NONE); 484 485 wrtee(PSL_EE); 486 } 487 488 static void 489 e500_splx(int ipl) 490 { 491 struct cpu_info * const ci = curcpu(); 492 const int old_ipl = ci->ci_cpl; 493 494 KASSERT(mfmsr() & PSL_CE); 495 496 if (ipl == old_ipl) 497 return; 498 499 if (__predict_false(ipl > old_ipl)) { 500 printf("%s: %p: cpl=%u: ignoring splx(%u) to raise ipl\n", 501 __func__, __builtin_return_address(0), old_ipl, ipl); 502 if (old_ipl == IPL_NONE) 503 Debugger(); 504 } 505 506 // const 507 register_t msr = wrtee(0); 508 #ifdef __HAVE_FAST_SOFTINTS 509 const u_int softints = (ci->ci_data.cpu_softints << ipl) & IPL_SOFTMASK; 510 if (__predict_false(softints != 0)) { 511 e500_splset(ci, IPL_HIGH); 512 powerpc_softint(ci, ipl, 513 (vaddr_t)__builtin_return_address(0)); 514 } 515 #endif /* __HAVE_FAST_SOFTINTS */ 516 e500_splset(ci, ipl); 517 #if 1 518 if (ipl < IPL_VM && old_ipl >= IPL_VM) 519 msr = PSL_EE; 520 #endif 521 wrtee(msr); 522 } 523 524 static int 525 e500_splraise(int ipl) 526 { 527 struct cpu_info * const ci = curcpu(); 528 const int old_ipl = ci->ci_cpl; 529 530 KASSERT(mfmsr() & PSL_CE); 531 532 if (old_ipl < ipl) { 533 //const 534 register_t msr = wrtee(0); 535 e500_splset(ci, ipl); 536 #if 1 537 if (old_ipl < IPL_VM && ipl >= IPL_VM) 538 msr = 0; 539 #endif 540 wrtee(msr); 541 } else if (ipl == IPL_NONE) { 542 panic("%s: %p: cpl=%u: attempt to splraise(IPL_NONE)", 543 __func__, __builtin_return_address(0), old_ipl); 544 #if 0 545 } else if (old_ipl > ipl) { 546 printf("%s: %p: cpl=%u: ignoring splraise(%u) to lower ipl\n", 547 __func__, __builtin_return_address(0), old_ipl, ipl); 548 #endif 549 } 550 551 return old_ipl; 552 } 553 554 static int 555 e500_intr_spurious(void *arg) 556 { 557 return 0; 558 } 559 560 static bool 561 e500_intr_irq_info_get(struct cpu_info *ci, u_int irq, int ipl, int ist, 562 struct e500_intr_irq_info *ii) 563 { 564 const struct e500_intr_info * const info = &e500_intr_info; 565 bool ok; 566 567 #if DEBUG > 2 568 printf("%s(%p,irq=%u,ipl=%u,ist=%u,%p)\n", __func__, ci, irq, ipl, ist, ii); 569 #endif 570 571 if (ipl < IPL_VM || ipl > IPL_HIGH) { 572 #if DEBUG > 2 573 printf("%s:%d ipl=%u\n", __func__, __LINE__, ipl); 574 #endif 575 return false; 576 } 577 578 if (ist <= IST_NONE || ist >= IST_MAX) { 579 #if DEBUG > 2 580 printf("%s:%d ist=%u\n", __func__, __LINE__, ist); 581 #endif 582 return false; 583 } 584 585 ii->irq_vector = irq + info->ii_ist_vectors[ist]; 586 if (IST_PERCPU_P(ist) && ist != IST_IPI) 587 ii->irq_vector += ci->ci_cpuid * info->ii_percpu_sources; 588 589 switch (ist) { 590 default: 591 ii->irq_vpr = OPENPIC_EIVPR(irq); 592 ii->irq_dr = OPENPIC_EIDR(irq); 593 ok = irq < info->ii_external_sources 594 && (ist == IST_EDGE 595 || ist == IST_LEVEL_LOW 596 || ist == IST_LEVEL_HIGH); 597 break; 598 case IST_PULSE: 599 ok = false; 600 break; 601 case IST_ONCHIP: 602 ii->irq_vpr = OPENPIC_IIVPR(irq); 603 ii->irq_dr = OPENPIC_IIDR(irq); 604 ok = irq < 32 * __arraycount(info->ii_onchip_bitmap); 605 #if DEBUG > 2 606 printf("%s: irq=%u: ok=%u\n", __func__, irq, ok); 607 #endif 608 ok = ok && (info->ii_onchip_bitmap[irq/32] & (1 << (irq & 31))); 609 #if DEBUG > 2 610 printf("%s: %08x%08x -> %08x%08x: ok=%u\n", __func__, 611 irq < 32 ? 0 : (1 << irq), irq < 32 ? (1 << irq) : 0, 612 info->ii_onchip_bitmap[1], info->ii_onchip_bitmap[0], 613 ok); 614 #endif 615 break; 616 case IST_MSIGROUP: 617 ii->irq_vpr = OPENPIC_MSIVPR(irq); 618 ii->irq_dr = OPENPIC_MSIDR(irq); 619 ok = irq < info->ii_msigroup_sources 620 && ipl == IPL_VM; 621 break; 622 case IST_TIMER: 623 ii->irq_vpr = OPENPIC_GTVPR(ci->ci_cpuid, irq); 624 ii->irq_dr = OPENPIC_GTDR(ci->ci_cpuid, irq); 625 ok = irq < info->ii_timer_sources; 626 #if DEBUG > 2 627 printf("%s: IST_TIMER irq=%u: ok=%u\n", __func__, irq, ok); 628 #endif 629 break; 630 case IST_IPI: 631 ii->irq_vpr = OPENPIC_IPIVPR(irq); 632 ii->irq_dr = OPENPIC_IPIDR(irq); 633 ok = irq < info->ii_ipi_sources; 634 break; 635 case IST_MI: 636 ii->irq_vpr = OPENPIC_MIVPR(irq); 637 ii->irq_dr = OPENPIC_MIDR(irq); 638 ok = irq < info->ii_mi_sources; 639 break; 640 } 641 642 return ok; 643 } 644 645 static const char * 646 e500_intr_string(int irq, int ist) 647 { 648 struct cpu_info * const ci = curcpu(); 649 struct cpu_softc * const cpu = ci->ci_softc; 650 struct e500_intr_irq_info ii; 651 652 if (!e500_intr_irq_info_get(ci, irq, IPL_VM, ist, &ii)) 653 return NULL; 654 655 return cpu->cpu_evcnt_intrs[ii.irq_vector].ev_name; 656 } 657 658 __CTASSERT(__arraycount(ist_names) == IST_MAX); 659 660 static const char * 661 e500_intr_typename(int ist) 662 { 663 if (IST_NONE <= ist && ist < IST_MAX) 664 return ist_names[ist]; 665 666 return NULL; 667 } 668 669 static void * 670 e500_intr_cpu_establish(struct cpu_info *ci, int irq, int ipl, int ist, 671 int (*handler)(void *), void *arg) 672 { 673 struct cpu_softc * const cpu = ci->ci_softc; 674 struct e500_intr_irq_info ii; 675 676 KASSERT(ipl >= IPL_VM && ipl <= IPL_HIGH); 677 KASSERT(ist > IST_NONE && ist < IST_MAX && ist != IST_MSI); 678 679 if (!e500_intr_irq_info_get(ci, irq, ipl, ist, &ii)) { 680 printf("%s: e500_intr_irq_info_get(%p,%u,%u,%u,%p) failed\n", 681 __func__, ci, irq, ipl, ist, &ii); 682 return NULL; 683 } 684 685 struct intr_source * const is = &e500_intr_sources[ii.irq_vector]; 686 mutex_enter(&e500_intr_lock); 687 if (is->is_ipl != IPL_NONE) 688 return NULL; 689 690 is->is_func = handler; 691 is->is_arg = arg; 692 is->is_ipl = ipl; 693 is->is_ist = ist; 694 is->is_irq = irq; 695 is->is_vpr = ii.irq_vpr; 696 is->is_dr = ii.irq_dr; 697 698 uint32_t vpr = VPR_PRIORITY_MAKE(IPL2CTPR(ipl)) 699 | VPR_VECTOR_MAKE(((ii.irq_vector + 1) << 4) | ipl) 700 | (ist == IST_LEVEL_LOW 701 ? VPR_LEVEL_LOW 702 : (ist == IST_LEVEL_HIGH 703 ? VPR_LEVEL_HIGH 704 : (ist == IST_ONCHIP 705 ? VPR_P_HIGH 706 : 0))); 707 708 /* 709 * All interrupts go to the primary except per-cpu interrupts which get 710 * routed to the appropriate cpu. 711 */ 712 uint32_t dr = openpic_read(cpu, ii.irq_dr); 713 714 dr |= 1 << (IST_PERCPU_P(ist) ? ci->ci_cpuid : 0); 715 716 /* 717 * Update the vector/priority and destination registers keeping the 718 * interrupt masked. 719 */ 720 const register_t msr = wrtee(0); /* disable interrupts */ 721 openpic_write(cpu, ii.irq_vpr, vpr | VPR_MSK); 722 openpic_write(cpu, ii.irq_dr, dr); 723 724 /* 725 * Now unmask the interrupt. 726 */ 727 openpic_write(cpu, ii.irq_vpr, vpr); 728 729 wrtee(msr); /* re-enable interrupts */ 730 731 mutex_exit(&e500_intr_lock); 732 733 return is; 734 } 735 736 static void * 737 e500_intr_establish(int irq, int ipl, int ist, 738 int (*handler)(void *), void *arg) 739 { 740 return e500_intr_cpu_establish(curcpu(), irq, ipl, ist, handler, arg); 741 } 742 743 static void 744 e500_intr_disestablish(void *vis) 745 { 746 struct cpu_softc * const cpu = curcpu()->ci_softc; 747 struct intr_source * const is = vis; 748 struct e500_intr_irq_info ii; 749 750 KASSERT(e500_intr_sources <= is); 751 KASSERT(is < e500_intr_last_source); 752 KASSERT(!cpu_intr_p()); 753 754 bool ok = e500_intr_irq_info_get(curcpu(), is->is_irq, is->is_ipl, 755 is->is_ist, &ii); 756 (void)ok; /* appease gcc */ 757 KASSERT(ok); 758 KASSERT(is - e500_intr_sources == ii.irq_vector); 759 760 mutex_enter(&e500_intr_lock); 761 /* 762 * Mask the source using the mask (MSK) bit in the vector/priority reg. 763 */ 764 uint32_t vpr = openpic_read(cpu, ii.irq_vpr); 765 openpic_write(cpu, ii.irq_vpr, VPR_MSK | vpr); 766 767 /* 768 * Wait for the Activity (A) bit for the source to be cleared. 769 */ 770 while (openpic_read(cpu, ii.irq_vpr) & VPR_A) 771 ; 772 773 /* 774 * Now the source can be modified. 775 */ 776 openpic_write(cpu, ii.irq_dr, 0); /* stop delivery */ 777 openpic_write(cpu, ii.irq_vpr, VPR_MSK); /* mask/reset it */ 778 779 *is = (struct intr_source)INTR_SOURCE_INITIALIZER; 780 781 mutex_exit(&e500_intr_lock); 782 } 783 784 static void 785 e500_critintr(struct trapframe *tf) 786 { 787 panic("%s: srr0/srr1=%#lx/%#lx", __func__, tf->tf_srr0, tf->tf_srr1); 788 } 789 790 static void 791 e500_decrintr(struct trapframe *tf) 792 { 793 panic("%s: srr0/srr1=%#lx/%#lx", __func__, tf->tf_srr0, tf->tf_srr1); 794 } 795 796 static void 797 e500_fitintr(struct trapframe *tf) 798 { 799 panic("%s: srr0/srr1=%#lx/%#lx", __func__, tf->tf_srr0, tf->tf_srr1); 800 } 801 802 static void 803 e500_wdogintr(struct trapframe *tf) 804 { 805 mtspr(SPR_TSR, TSR_ENW|TSR_WIS); 806 panic("%s: tf=%p tb=%"PRId64" srr0/srr1=%#lx/%#lx", __func__, tf, 807 mftb(), tf->tf_srr0, tf->tf_srr1); 808 } 809 810 static void 811 e500_extintr(struct trapframe *tf) 812 { 813 struct cpu_info * const ci = curcpu(); 814 struct cpu_softc * const cpu = ci->ci_softc; 815 const int old_ipl = ci->ci_cpl; 816 817 KASSERT(mfmsr() & PSL_CE); 818 819 #if 0 820 // printf("%s(%p): idepth=%d enter\n", __func__, tf, ci->ci_idepth); 821 if ((register_t)tf >= (register_t)curlwp->l_addr + USPACE 822 || (register_t)tf < (register_t)curlwp->l_addr + NBPG) { 823 printf("%s(entry): pid %d.%d (%s): srr0/srr1=%#lx/%#lx: invalid tf addr %p\n", 824 __func__, curlwp->l_proc->p_pid, curlwp->l_lid, 825 curlwp->l_proc->p_comm, tf->tf_srr0, tf->tf_srr1, tf); 826 } 827 #endif 828 829 830 ci->ci_data.cpu_nintr++; 831 tf->tf_cf.cf_idepth = ci->ci_idepth++; 832 cpu->cpu_pcpls[ci->ci_idepth] = old_ipl; 833 #if 1 834 if (mfmsr() & PSL_EE) 835 panic("%s(%p): MSR[EE] is on (%#lx)!", __func__, tf, mfmsr()); 836 if (old_ipl == IPL_HIGH 837 || IPL2CTPR(old_ipl) != openpic_read(cpu, OPENPIC_CTPR)) 838 panic("%s(%p): old_ipl(%u) == IPL_HIGH(%u) " 839 "|| old_ipl + %u != OPENPIC_CTPR (%u)", 840 __func__, tf, old_ipl, IPL_HIGH, 841 15 - IPL_HIGH, openpic_read(cpu, OPENPIC_CTPR)); 842 #else 843 if (old_ipl >= IPL_VM) 844 panic("%s(%p): old_ipl(%u) >= IPL_VM(%u) CTPR=%u", 845 __func__, tf, old_ipl, IPL_VM, openpic_read(cpu, OPENPIC_CTPR)); 846 #endif 847 848 for (;;) { 849 /* 850 * Find out the pending interrupt. 851 */ 852 if (mfmsr() & PSL_EE) 853 panic("%s(%p): MSR[EE] turned on (%#lx)!", __func__, tf, mfmsr()); 854 if (IPL2CTPR(old_ipl) != openpic_read(cpu, OPENPIC_CTPR)) 855 panic("%s(%p): %d: old_ipl(%u) + %u != OPENPIC_CTPR (%u)", 856 __func__, tf, __LINE__, old_ipl, 857 15 - IPL_HIGH, openpic_read(cpu, OPENPIC_CTPR)); 858 const uint32_t iack = openpic_read(cpu, OPENPIC_IACK); 859 #ifdef DIAGNOSTIC 860 const int ipl = iack & 0xf; 861 #endif 862 const int irq = (iack >> 4) - 1; 863 #if 0 864 printf("%s: iack=%d ipl=%d irq=%d <%s>\n", 865 __func__, iack, ipl, irq, 866 (iack != IRQ_SPURIOUS ? 867 cpu->cpu_evcnt_intrs[irq].ev_name : "spurious")); 868 #endif 869 if (IPL2CTPR(old_ipl) != openpic_read(cpu, OPENPIC_CTPR)) 870 panic("%s(%p): %d: old_ipl(%u) + %u != OPENPIC_CTPR (%u)", 871 __func__, tf, __LINE__, old_ipl, 872 15 - IPL_HIGH, openpic_read(cpu, OPENPIC_CTPR)); 873 if (iack == IRQ_SPURIOUS) 874 break; 875 876 struct intr_source * const is = &e500_intr_sources[irq]; 877 if (__predict_true(is < e500_intr_last_source)) { 878 /* 879 * Timer interrupts get their argument overriden with 880 * the pointer to the trapframe. 881 */ 882 KASSERT(is->is_ipl == ipl); 883 void *arg = (is->is_ist == IST_TIMER ? tf : is->is_arg); 884 if (is->is_ipl <= old_ipl) 885 panic("%s(%p): %s (%u): is->is_ipl (%u) <= old_ipl (%u)\n", 886 __func__, tf, 887 cpu->cpu_evcnt_intrs[irq].ev_name, irq, 888 is->is_ipl, old_ipl); 889 KASSERT(is->is_ipl > old_ipl); 890 e500_splset(ci, is->is_ipl); /* change IPL */ 891 if (__predict_false(is->is_func == NULL)) { 892 aprint_error_dev(ci->ci_dev, 893 "interrupt from unestablished irq %d\n", 894 irq); 895 } else { 896 int (*func)(void *) = is->is_func; 897 wrtee(PSL_EE); 898 int rv = (*func)(arg); 899 wrtee(0); 900 #if DEBUG > 2 901 printf("%s: %s handler %p(%p) returned %d\n", 902 __func__, 903 cpu->cpu_evcnt_intrs[irq].ev_name, 904 func, arg, rv); 905 #endif 906 if (rv == 0) 907 cpu->cpu_evcnt_spurious_intr.ev_count++; 908 } 909 e500_splset(ci, old_ipl); /* restore IPL */ 910 cpu->cpu_evcnt_intrs[irq].ev_count++; 911 } else { 912 aprint_error_dev(ci->ci_dev, 913 "interrupt from illegal irq %d\n", irq); 914 cpu->cpu_evcnt_spurious_intr.ev_count++; 915 } 916 /* 917 * If this is a nested interrupt, simply ack it and exit 918 * because the loop we interrupted will complete looking 919 * for interrupts. 920 */ 921 if (mfmsr() & PSL_EE) 922 panic("%s(%p): MSR[EE] left on (%#lx)!", __func__, tf, mfmsr()); 923 if (IPL2CTPR(old_ipl) != openpic_read(cpu, OPENPIC_CTPR)) 924 panic("%s(%p): %d: old_ipl(%u) + %u != OPENPIC_CTPR (%u)", 925 __func__, tf, __LINE__, old_ipl, 926 15 - IPL_HIGH, openpic_read(cpu, OPENPIC_CTPR)); 927 928 openpic_write(cpu, OPENPIC_EOI, 0); 929 if (IPL2CTPR(old_ipl) != openpic_read(cpu, OPENPIC_CTPR)) 930 panic("%s(%p): %d: old_ipl(%u) + %u != OPENPIC_CTPR (%u)", 931 __func__, tf, __LINE__, old_ipl, 932 15 - IPL_HIGH, openpic_read(cpu, OPENPIC_CTPR)); 933 if (ci->ci_idepth > 0) 934 break; 935 } 936 937 ci->ci_idepth--; 938 939 #ifdef __HAVE_FAST_SOFTINTS 940 /* 941 * Before exiting, deal with any softints that need to be dealt with. 942 */ 943 const u_int softints = (ci->ci_data.cpu_softints << old_ipl) & IPL_SOFTMASK; 944 if (__predict_false(softints != 0)) { 945 KASSERT(old_ipl < IPL_VM); 946 e500_splset(ci, IPL_HIGH); /* pop to high */ 947 powerpc_softint(ci, old_ipl, /* deal with them */ 948 tf->tf_srr0); 949 e500_splset(ci, old_ipl); /* and drop back */ 950 } 951 #endif /* __HAVE_FAST_SOFTINTS */ 952 #if 1 953 KASSERT(ci->ci_cpl == old_ipl); 954 #else 955 e500_splset(ci, old_ipl); /* and drop back */ 956 #endif 957 958 /* 959 * If we interrupted while power-saving and we need to exit idle, 960 * we need to clear PSL_POW so we won't go back into power-saving. 961 */ 962 if (__predict_false(tf->tf_srr1 & PSL_POW) && ci->ci_want_resched) 963 tf->tf_srr1 &= ~PSL_POW; 964 965 // printf("%s(%p): idepth=%d exit\n", __func__, tf, ci->ci_idepth); 966 } 967 968 static void 969 e500_intr_init(void) 970 { 971 struct cpu_info * const ci = curcpu(); 972 struct cpu_softc * const cpu = ci->ci_softc; 973 const uint32_t frr = openpic_read(cpu, OPENPIC_FRR); 974 const u_int nirq = FRR_NIRQ_GET(frr) + 1; 975 // const u_int ncpu = FRR_NCPU_GET(frr) + 1; 976 struct intr_source *is; 977 struct e500_intr_info * const ii = &e500_intr_info; 978 979 const uint16_t svr = (mfspr(SPR_SVR) & ~0x80000) >> 16; 980 switch (svr) { 981 #ifdef MPC8536 982 case SVR_MPC8536v1 >> 16: 983 *ii = mpc8536_intr_info; 984 break; 985 #endif 986 #ifdef MPC8544 987 case SVR_MPC8544v1 >> 16: 988 *ii = mpc8544_intr_info; 989 break; 990 #endif 991 #ifdef MPC8548 992 case SVR_MPC8543v1 >> 16: 993 case SVR_MPC8548v1 >> 16: 994 *ii = mpc8548_intr_info; 995 break; 996 #endif 997 #ifdef MPC8555 998 case SVR_MPC8541v1 >> 16: 999 case SVR_MPC8555v1 >> 16: 1000 *ii = mpc8555_intr_info; 1001 break; 1002 #endif 1003 #ifdef MPC8568 1004 case SVR_MPC8568v1 >> 16: 1005 *ii = mpc8568_intr_info; 1006 break; 1007 #endif 1008 #ifdef MPC8572 1009 case SVR_MPC8572v1 >> 16: 1010 *ii = mpc8572_intr_info; 1011 break; 1012 #endif 1013 #ifdef P2020 1014 case SVR_P2010v2 >> 16: 1015 case SVR_P2020v2 >> 16: 1016 *ii = p20x0_intr_info; 1017 break; 1018 #endif 1019 default: 1020 panic("%s: don't know how to deal with SVR %#lx", 1021 __func__, mfspr(SPR_SVR)); 1022 } 1023 1024 /* 1025 * We need to be in mixed mode. 1026 */ 1027 openpic_write(cpu, OPENPIC_GCR, GCR_M); 1028 1029 /* 1030 * Make we and the openpic both agree about the current SPL level. 1031 */ 1032 e500_splset(ci, ci->ci_cpl); 1033 1034 /* 1035 * Allow the required number of interrupt sources. 1036 */ 1037 is = kmem_zalloc(nirq * sizeof(*is), KM_SLEEP); 1038 KASSERT(is); 1039 e500_intr_sources = is; 1040 e500_intr_last_source = is + nirq; 1041 1042 /* 1043 * Initialize all the external interrupts as active low. 1044 */ 1045 for (u_int irq = 0; irq < e500_intr_info.ii_external_sources; irq++) { 1046 openpic_write(cpu, OPENPIC_EIVPR(irq), 1047 VPR_VECTOR_MAKE(irq) | VPR_LEVEL_LOW); 1048 } 1049 } 1050 1051 static void 1052 e500_idlespin(void) 1053 { 1054 KASSERTMSG(curcpu()->ci_cpl == IPL_NONE, 1055 "%s: cpu%u: ci_cpl (%d) != 0", __func__, cpu_number(), 1056 curcpu()->ci_cpl); 1057 KASSERTMSG(CTPR2IPL(openpic_read(curcpu()->ci_softc, OPENPIC_CTPR)) == IPL_NONE, 1058 "%s: cpu%u: CTPR (%d) != IPL_NONE", __func__, cpu_number(), 1059 CTPR2IPL(openpic_read(curcpu()->ci_softc, OPENPIC_CTPR))); 1060 KASSERT(mfmsr() & PSL_EE); 1061 1062 if (powersave > 0) 1063 mtmsr(mfmsr() | PSL_POW); 1064 } 1065 1066 static void 1067 e500_intr_cpu_attach(struct cpu_info *ci) 1068 { 1069 struct cpu_softc * const cpu = ci->ci_softc; 1070 const char * const xname = device_xname(ci->ci_dev); 1071 1072 const u_int32_t frr = openpic_read(cpu, OPENPIC_FRR); 1073 const u_int nirq = FRR_NIRQ_GET(frr) + 1; 1074 // const u_int ncpu = FRR_NCPU_GET(frr) + 1; 1075 1076 const struct e500_intr_info * const info = &e500_intr_info; 1077 1078 cpu->cpu_clock_gtbcr = OPENPIC_GTBCR(ci->ci_cpuid, E500_CLOCK_TIMER); 1079 1080 cpu->cpu_evcnt_intrs = 1081 kmem_zalloc(nirq * sizeof(cpu->cpu_evcnt_intrs[0]), KM_SLEEP); 1082 KASSERT(cpu->cpu_evcnt_intrs); 1083 1084 struct evcnt *evcnt = cpu->cpu_evcnt_intrs; 1085 for (size_t j = 0; j < info->ii_external_sources; j++, evcnt++) { 1086 const char *name = e500_intr_external_name_lookup(j); 1087 evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR, NULL, xname, name); 1088 } 1089 KASSERT(evcnt == cpu->cpu_evcnt_intrs + info->ii_ist_vectors[IST_ONCHIP]); 1090 for (size_t j = 0; j < info->ii_onchip_sources; j++, evcnt++) { 1091 if (info->ii_onchip_bitmap[j / 32] & __BIT(j & 31)) { 1092 const char *name = e500_intr_onchip_name_lookup(j); 1093 if (name != NULL) { 1094 evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR, 1095 NULL, xname, name); 1096 #ifdef DIAGNOSTIC 1097 } else { 1098 printf("%s: missing evcnt for onchip irq %zu\n", 1099 __func__, j); 1100 #endif 1101 } 1102 } 1103 } 1104 1105 KASSERT(evcnt == cpu->cpu_evcnt_intrs + info->ii_ist_vectors[IST_MSIGROUP]); 1106 for (size_t j = 0; j < info->ii_msigroup_sources; j++, evcnt++) { 1107 evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR, 1108 NULL, xname, e500_msigroup_intr_names[j].in_name); 1109 } 1110 1111 KASSERT(evcnt == cpu->cpu_evcnt_intrs + info->ii_ist_vectors[IST_TIMER]); 1112 evcnt += ci->ci_cpuid * info->ii_percpu_sources; 1113 for (size_t j = 0; j < info->ii_timer_sources; j++, evcnt++) { 1114 evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR, 1115 NULL, xname, e500_timer_intr_names[j].in_name); 1116 } 1117 1118 for (size_t j = 0; j < info->ii_ipi_sources; j++, evcnt++) { 1119 evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR, 1120 NULL, xname, e500_ipi_intr_names[j].in_name); 1121 } 1122 1123 for (size_t j = 0; j < info->ii_mi_sources; j++, evcnt++) { 1124 evcnt_attach_dynamic(evcnt, EVCNT_TYPE_INTR, 1125 NULL, xname, e500_mi_intr_names[j].in_name); 1126 } 1127 1128 ci->ci_idlespin = e500_idlespin; 1129 } 1130 1131 static void 1132 e500_intr_cpu_send_ipi(cpuid_t target, uint32_t ipimsg) 1133 { 1134 struct cpu_info * const ci = curcpu(); 1135 struct cpu_softc * const cpu = ci->ci_softc; 1136 uint32_t dstmask; 1137 1138 if (target >= CPU_MAXNUM) { 1139 CPU_INFO_ITERATOR cii; 1140 struct cpu_info *dst_ci; 1141 1142 KASSERT(target == IPI_DST_NOTME || target == IPI_DST_ALL); 1143 1144 dstmask = 0; 1145 for (CPU_INFO_FOREACH(cii, dst_ci)) { 1146 if (target == IPI_DST_ALL || ci != dst_ci) { 1147 dstmask |= 1 << cpu_index(ci); 1148 if (ipimsg) 1149 atomic_or_32(&dst_ci->ci_pending_ipis, 1150 ipimsg); 1151 } 1152 } 1153 } else { 1154 struct cpu_info * const dst_ci = cpu_lookup(target); 1155 KASSERT(dst_ci != NULL); 1156 KASSERTMSG(target == cpu_index(dst_ci), 1157 "%s: target (%lu) != cpu_index(cpu%u)", 1158 __func__, target, cpu_index(dst_ci)); 1159 dstmask = (1 << target); 1160 if (ipimsg) 1161 atomic_or_32(&dst_ci->ci_pending_ipis, ipimsg); 1162 } 1163 1164 openpic_write(cpu, OPENPIC_IPIDR(0), dstmask); 1165 } 1166 1167 typedef void (*ipifunc_t)(void); 1168 1169 #ifdef __HAVE_PREEEMPTION 1170 static void 1171 e500_ipi_kpreempt(void) 1172 { 1173 poowerpc_softint_trigger(1 << IPL_NONE); 1174 } 1175 #endif 1176 1177 static const ipifunc_t e500_ipifuncs[] = { 1178 [ilog2(IPI_XCALL)] = xc_ipi_handler, 1179 [ilog2(IPI_HALT)] = e500_ipi_halt, 1180 #ifdef __HAVE_PREEMPTION 1181 [ilog2(IPI_KPREEMPT)] = e500_ipi_kpreempt, 1182 #endif 1183 [ilog2(IPI_TLB1SYNC)] = e500_tlb1_sync, 1184 }; 1185 1186 static int 1187 e500_ipi_intr(void *v) 1188 { 1189 struct cpu_info * const ci = curcpu(); 1190 1191 ci->ci_ev_ipi.ev_count++; 1192 1193 uint32_t pending_ipis = atomic_swap_32(&ci->ci_pending_ipis, 0); 1194 for (u_int ipi = 31; pending_ipis != 0; ipi--, pending_ipis <<= 1) { 1195 const u_int bits = __builtin_clz(pending_ipis); 1196 ipi -= bits; 1197 pending_ipis <<= bits; 1198 KASSERT(e500_ipifuncs[ipi] != NULL); 1199 (*e500_ipifuncs[ipi])(); 1200 } 1201 1202 return 1; 1203 } 1204 1205 static void 1206 e500_intr_cpu_hatch(struct cpu_info *ci) 1207 { 1208 /* 1209 * Establish clock interrupt for this CPU. 1210 */ 1211 if (e500_intr_cpu_establish(ci, E500_CLOCK_TIMER, IPL_CLOCK, IST_TIMER, 1212 e500_clock_intr, NULL) == NULL) 1213 panic("%s: failed to establish clock interrupt!", __func__); 1214 1215 /* 1216 * Establish the IPI interrupts for this CPU. 1217 */ 1218 if (e500_intr_cpu_establish(ci, 0, IPL_VM, IST_IPI, e500_ipi_intr, 1219 NULL) == NULL) 1220 panic("%s: failed to establish ipi interrupt!", __func__); 1221 1222 /* 1223 * Enable watchdog interrupts. 1224 */ 1225 uint32_t tcr = mfspr(SPR_TCR); 1226 tcr |= TCR_WIE; 1227 mtspr(SPR_TCR, tcr); 1228 } 1229