1 /* 2 * Copyright (c) 1996, by Steve Passe 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. The name of the developer may NOT be used to endorse or promote products 11 * derived from this software without specific prior written permission. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD: src/sys/i386/i386/mpapic.c,v 1.37.2.7 2003/01/25 02:31:47 peter Exp $ 26 */ 27 28 #include <sys/param.h> 29 #include <sys/systm.h> 30 #include <sys/kernel.h> 31 #include <sys/bus.h> 32 #include <sys/machintr.h> 33 #include <sys/thread2.h> 34 35 #include <machine/pmap.h> 36 #include <machine_base/icu/icu_var.h> 37 #include <machine_base/apic/lapic.h> 38 #include <machine_base/apic/ioapic.h> 39 #include <machine_base/apic/ioapic_abi.h> 40 #include <machine_base/apic/apicvar.h> 41 42 #define IOAPIC_COUNT_MAX 16 43 #define IOAPIC_ID_MASK (IOAPIC_COUNT_MAX - 1) 44 45 struct ioapic_info { 46 int io_idx; 47 int io_apic_id; 48 void *io_addr; 49 int io_npin; 50 int io_gsi_base; 51 52 TAILQ_ENTRY(ioapic_info) io_link; 53 }; 54 TAILQ_HEAD(ioapic_info_list, ioapic_info); 55 56 struct ioapic_intsrc { 57 int int_gsi; 58 enum intr_trigger int_trig; 59 enum intr_polarity int_pola; 60 }; 61 62 struct ioapic_conf { 63 struct ioapic_info_list ioc_list; 64 struct ioapic_intsrc ioc_intsrc[16]; /* XXX magic number */ 65 }; 66 67 static int ioapic_config(void); 68 static void ioapic_setup(const struct ioapic_info *); 69 static int ioapic_alloc_apic_id(int); 70 static void ioapic_set_apic_id(const struct ioapic_info *); 71 static void ioapic_gsi_setup(int); 72 static const struct ioapic_info * 73 ioapic_gsi_search(int); 74 static void ioapic_pin_prog(void *, int, int, 75 enum intr_trigger, enum intr_polarity, uint32_t); 76 77 static struct ioapic_conf ioapic_conf; 78 79 static TAILQ_HEAD(, ioapic_enumerator) ioapic_enumerators = 80 TAILQ_HEAD_INITIALIZER(ioapic_enumerators); 81 82 int ioapic_enable = 1; /* I/O APIC is enabled by default */ 83 84 static int 85 ioapic_config(void) 86 { 87 struct ioapic_enumerator *e; 88 struct ioapic_info *info; 89 int start_apic_id = 0; 90 int error, i, probe; 91 register_t ef = 0; 92 93 TAILQ_INIT(&ioapic_conf.ioc_list); 94 /* XXX magic number */ 95 for (i = 0; i < 16; ++i) 96 ioapic_conf.ioc_intsrc[i].int_gsi = -1; 97 98 probe = 1; 99 TUNABLE_INT_FETCH("hw.ioapic_probe", &probe); 100 if (!probe) { 101 kprintf("IOAPIC: warning I/O APIC will not be probed\n"); 102 return ENXIO; 103 } 104 105 TAILQ_FOREACH(e, &ioapic_enumerators, ioapic_link) { 106 error = e->ioapic_probe(e); 107 if (!error) 108 break; 109 } 110 if (e == NULL) { 111 kprintf("IOAPIC: can't find I/O APIC\n"); 112 return ENXIO; 113 } 114 115 crit_enter(); 116 117 ef = read_rflags(); 118 cpu_disable_intr(); 119 120 /* 121 * Switch to I/O APIC MachIntrABI and reconfigure 122 * the default IDT entries. 123 */ 124 MachIntrABI = MachIntrABI_IOAPIC; 125 MachIntrABI.setdefault(); 126 127 e->ioapic_enumerate(e); 128 129 /* 130 * Setup index 131 */ 132 i = 0; 133 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) 134 info->io_idx = i++; 135 136 if (i > IOAPIC_COUNT_MAX) /* XXX magic number */ 137 panic("ioapic_config: more than 16 I/O APIC\n"); 138 139 /* 140 * Setup APIC ID 141 */ 142 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 143 int apic_id; 144 145 apic_id = ioapic_alloc_apic_id(start_apic_id); 146 if (apic_id == NAPICID) { 147 kprintf("IOAPIC: can't alloc APIC ID for " 148 "%dth I/O APIC\n", info->io_idx); 149 break; 150 } 151 info->io_apic_id = apic_id; 152 153 start_apic_id = apic_id + 1; 154 } 155 if (info != NULL) { 156 /* 157 * xAPIC allows I/O APIC's APIC ID to be same 158 * as the LAPIC's APIC ID 159 */ 160 kprintf("IOAPIC: use xAPIC model to alloc APIC ID " 161 "for I/O APIC\n"); 162 163 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) 164 info->io_apic_id = info->io_idx; 165 } 166 167 /* 168 * Warning about any GSI holes 169 */ 170 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 171 const struct ioapic_info *prev_info; 172 173 prev_info = TAILQ_PREV(info, ioapic_info_list, io_link); 174 if (prev_info != NULL) { 175 if (info->io_gsi_base != 176 prev_info->io_gsi_base + prev_info->io_npin) { 177 kprintf("IOAPIC: warning gsi hole " 178 "[%d, %d]\n", 179 prev_info->io_gsi_base + 180 prev_info->io_npin, 181 info->io_gsi_base - 1); 182 } 183 } 184 } 185 186 if (bootverbose) { 187 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 188 kprintf("IOAPIC: idx %d, apic id %d, " 189 "gsi base %d, npin %d\n", 190 info->io_idx, 191 info->io_apic_id, 192 info->io_gsi_base, 193 info->io_npin); 194 } 195 } 196 197 /* 198 * Setup all I/O APIC 199 */ 200 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) 201 ioapic_setup(info); 202 ioapic_abi_fixup_irqmap(); 203 204 write_rflags(ef); 205 206 MachIntrABI.cleanup(); 207 208 crit_exit(); 209 210 return 0; 211 } 212 213 void 214 ioapic_enumerator_register(struct ioapic_enumerator *ne) 215 { 216 struct ioapic_enumerator *e; 217 218 TAILQ_FOREACH(e, &ioapic_enumerators, ioapic_link) { 219 if (e->ioapic_prio < ne->ioapic_prio) { 220 TAILQ_INSERT_BEFORE(e, ne, ioapic_link); 221 return; 222 } 223 } 224 TAILQ_INSERT_TAIL(&ioapic_enumerators, ne, ioapic_link); 225 } 226 227 void 228 ioapic_add(void *addr, int gsi_base, int npin) 229 { 230 struct ioapic_info *info, *ninfo; 231 int gsi_end; 232 233 gsi_end = gsi_base + npin - 1; 234 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 235 if ((gsi_base >= info->io_gsi_base && 236 gsi_base < info->io_gsi_base + info->io_npin) || 237 (gsi_end >= info->io_gsi_base && 238 gsi_end < info->io_gsi_base + info->io_npin)) { 239 panic("ioapic_add: overlapped gsi, base %d npin %d, " 240 "hit base %d, npin %d\n", gsi_base, npin, 241 info->io_gsi_base, info->io_npin); 242 } 243 if (info->io_addr == addr) 244 panic("ioapic_add: duplicated addr %p\n", addr); 245 } 246 247 ninfo = kmalloc(sizeof(*ninfo), M_DEVBUF, M_WAITOK | M_ZERO); 248 ninfo->io_addr = addr; 249 ninfo->io_npin = npin; 250 ninfo->io_gsi_base = gsi_base; 251 ninfo->io_apic_id = -1; 252 253 /* 254 * Create IOAPIC list in ascending order of GSI base 255 */ 256 TAILQ_FOREACH_REVERSE(info, &ioapic_conf.ioc_list, 257 ioapic_info_list, io_link) { 258 if (ninfo->io_gsi_base > info->io_gsi_base) { 259 TAILQ_INSERT_AFTER(&ioapic_conf.ioc_list, 260 info, ninfo, io_link); 261 break; 262 } 263 } 264 if (info == NULL) 265 TAILQ_INSERT_HEAD(&ioapic_conf.ioc_list, ninfo, io_link); 266 } 267 268 void 269 ioapic_intsrc(int irq, int gsi, enum intr_trigger trig, enum intr_polarity pola) 270 { 271 struct ioapic_intsrc *int_src; 272 273 KKASSERT(irq < 16); 274 int_src = &ioapic_conf.ioc_intsrc[irq]; 275 276 if (gsi == 0) { 277 /* Don't allow mixed mode */ 278 kprintf("IOAPIC: warning intsrc irq %d -> gsi 0\n", irq); 279 return; 280 } 281 282 if (int_src->int_gsi != -1) { 283 if (int_src->int_gsi != gsi) { 284 kprintf("IOAPIC: warning intsrc irq %d, gsi " 285 "%d -> %d\n", irq, int_src->int_gsi, gsi); 286 } 287 if (int_src->int_trig != trig) { 288 kprintf("IOAPIC: warning intsrc irq %d, trig " 289 "%s -> %s\n", irq, 290 intr_str_trigger(int_src->int_trig), 291 intr_str_trigger(trig)); 292 } 293 if (int_src->int_pola != pola) { 294 kprintf("IOAPIC: warning intsrc irq %d, pola " 295 "%s -> %s\n", irq, 296 intr_str_polarity(int_src->int_pola), 297 intr_str_polarity(pola)); 298 } 299 } 300 int_src->int_gsi = gsi; 301 int_src->int_trig = trig; 302 int_src->int_pola = pola; 303 } 304 305 static void 306 ioapic_set_apic_id(const struct ioapic_info *info) 307 { 308 uint32_t id; 309 int apic_id; 310 311 id = ioapic_read(info->io_addr, IOAPIC_ID); 312 313 id &= ~APIC_ID_MASK; 314 id |= (info->io_apic_id << 24); 315 316 ioapic_write(info->io_addr, IOAPIC_ID, id); 317 318 /* 319 * Re-read && test 320 */ 321 id = ioapic_read(info->io_addr, IOAPIC_ID); 322 apic_id = (id & APIC_ID_MASK) >> 24; 323 324 /* 325 * I/O APIC ID is a 4bits field 326 */ 327 if ((apic_id & IOAPIC_ID_MASK) != 328 (info->io_apic_id & IOAPIC_ID_MASK)) { 329 panic("ioapic_set_apic_id: can't set apic id to %d, " 330 "currently set to %d\n", info->io_apic_id, apic_id); 331 } 332 } 333 334 static void 335 ioapic_gsi_setup(int gsi) 336 { 337 enum intr_trigger trig; 338 enum intr_polarity pola; 339 int irq; 340 341 if (gsi == 0) { 342 /* ExtINT */ 343 imen_lock(); 344 ioapic_extpin_setup(ioapic_gsi_ioaddr(gsi), 345 ioapic_gsi_pin(gsi), 0); 346 imen_unlock(); 347 return; 348 } 349 350 for (irq = 0; irq < 16; ++irq) { 351 const struct ioapic_intsrc *int_src = 352 &ioapic_conf.ioc_intsrc[irq]; 353 354 if (gsi == int_src->int_gsi) { 355 trig = int_src->int_trig; 356 pola = int_src->int_pola; 357 break; 358 } 359 } 360 361 if (irq == 16) { 362 /* 363 * No explicit IRQ to GSI mapping; 364 * use the default 1:1 mapping 365 */ 366 irq = gsi; 367 if (irq < 16) { 368 if (ioapic_conf.ioc_intsrc[irq].int_gsi >= 0) { 369 /* 370 * This IRQ is mapped to different GSI, 371 * don't do the default configuration. 372 * The configuration of the target GSI 373 * will finally setup this IRQ. 374 * 375 * This GSI is not used, disable it. 376 */ 377 ioapic_pin_setup(ioapic_gsi_ioaddr(gsi), 378 ioapic_gsi_pin(gsi), 0, 379 INTR_TRIGGER_EDGE, INTR_POLARITY_HIGH); 380 return; 381 } 382 trig = INTR_TRIGGER_EDGE; 383 pola = INTR_POLARITY_HIGH; 384 } else { 385 trig = INTR_TRIGGER_LEVEL; 386 pola = INTR_POLARITY_LOW; 387 } 388 } 389 390 ioapic_abi_set_irqmap(irq, gsi, trig, pola); 391 } 392 393 void * 394 ioapic_gsi_ioaddr(int gsi) 395 { 396 const struct ioapic_info *info; 397 398 info = ioapic_gsi_search(gsi); 399 return info->io_addr; 400 } 401 402 int 403 ioapic_gsi_pin(int gsi) 404 { 405 const struct ioapic_info *info; 406 407 info = ioapic_gsi_search(gsi); 408 return gsi - info->io_gsi_base; 409 } 410 411 static const struct ioapic_info * 412 ioapic_gsi_search(int gsi) 413 { 414 const struct ioapic_info *info; 415 416 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 417 if (gsi >= info->io_gsi_base && 418 gsi < info->io_gsi_base + info->io_npin) 419 return info; 420 } 421 panic("ioapic_gsi_search: no I/O APIC\n"); 422 } 423 424 int 425 ioapic_gsi(int idx, int pin) 426 { 427 const struct ioapic_info *info; 428 429 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 430 if (info->io_idx == idx) 431 break; 432 } 433 if (info == NULL) 434 return -1; 435 if (pin >= info->io_npin) 436 return -1; 437 return info->io_gsi_base + pin; 438 } 439 440 void 441 ioapic_extpin_setup(void *addr, int pin, int vec) 442 { 443 ioapic_pin_prog(addr, pin, vec, 444 INTR_TRIGGER_CONFORM, INTR_POLARITY_CONFORM, IOART_DELEXINT); 445 } 446 447 int 448 ioapic_extpin_gsi(void) 449 { 450 return 0; 451 } 452 453 void 454 ioapic_pin_setup(void *addr, int pin, int vec, 455 enum intr_trigger trig, enum intr_polarity pola) 456 { 457 /* 458 * Always clear an I/O APIC pin before [re]programming it. This is 459 * particularly important if the pin is set up for a level interrupt 460 * as the IOART_REM_IRR bit might be set. When we reprogram the 461 * vector any EOI from pending ints on this pin could be lost and 462 * IRR might never get reset. 463 * 464 * To fix this problem, clear the vector and make sure it is 465 * programmed as an edge interrupt. This should theoretically 466 * clear IRR so we can later, safely program it as a level 467 * interrupt. 468 */ 469 ioapic_pin_prog(addr, pin, vec, INTR_TRIGGER_EDGE, INTR_POLARITY_HIGH, 470 IOART_DELFIXED); 471 ioapic_pin_prog(addr, pin, vec, trig, pola, IOART_DELFIXED); 472 } 473 474 static void 475 ioapic_pin_prog(void *addr, int pin, int vec, 476 enum intr_trigger trig, enum intr_polarity pola, uint32_t del_mode) 477 { 478 uint32_t flags, target; 479 int select; 480 481 KKASSERT(del_mode == IOART_DELEXINT || del_mode == IOART_DELFIXED); 482 483 select = IOAPIC_REDTBL0 + (2 * pin); 484 485 flags = ioapic_read(addr, select) & IOART_RESV; 486 flags |= IOART_INTMSET | IOART_DESTPHY; 487 #ifdef foo 488 flags |= del_mode; 489 #else 490 /* 491 * We only support limited I/O APIC mixed mode, 492 * so even for ExtINT, we still use "fixed" 493 * delivery mode. 494 */ 495 flags |= IOART_DELFIXED; 496 #endif 497 498 if (del_mode == IOART_DELEXINT) { 499 KKASSERT(trig == INTR_TRIGGER_CONFORM && 500 pola == INTR_POLARITY_CONFORM); 501 flags |= IOART_TRGREDG | IOART_INTAHI; 502 } else { 503 switch (trig) { 504 case INTR_TRIGGER_EDGE: 505 flags |= IOART_TRGREDG; 506 break; 507 508 case INTR_TRIGGER_LEVEL: 509 flags |= IOART_TRGRLVL; 510 break; 511 512 case INTR_TRIGGER_CONFORM: 513 panic("ioapic_pin_prog: trig conform is not " 514 "supported\n"); 515 } 516 switch (pola) { 517 case INTR_POLARITY_HIGH: 518 flags |= IOART_INTAHI; 519 break; 520 521 case INTR_POLARITY_LOW: 522 flags |= IOART_INTALO; 523 break; 524 525 case INTR_POLARITY_CONFORM: 526 panic("ioapic_pin_prog: pola conform is not " 527 "supported\n"); 528 } 529 } 530 531 target = ioapic_read(addr, select + 1) & IOART_HI_DEST_RESV; 532 target |= (CPUID_TO_APICID(0) << IOART_HI_DEST_SHIFT) & 533 IOART_HI_DEST_MASK; 534 535 ioapic_write(addr, select, flags | vec); 536 ioapic_write(addr, select + 1, target); 537 } 538 539 static void 540 ioapic_setup(const struct ioapic_info *info) 541 { 542 int i; 543 544 ioapic_set_apic_id(info); 545 546 for (i = 0; i < info->io_npin; ++i) 547 ioapic_gsi_setup(info->io_gsi_base + i); 548 } 549 550 static int 551 ioapic_alloc_apic_id(int start) 552 { 553 for (;;) { 554 const struct ioapic_info *info; 555 int apic_id, apic_id16; 556 557 apic_id = lapic_unused_apic_id(start); 558 if (apic_id == NAPICID) { 559 kprintf("IOAPIC: can't find unused APIC ID\n"); 560 return apic_id; 561 } 562 apic_id16 = apic_id & IOAPIC_ID_MASK; 563 564 /* 565 * Check against other I/O APIC's APIC ID's lower 4bits. 566 * 567 * The new APIC ID will have to be different from others 568 * in the lower 4bits, no matter whether xAPIC is used 569 * or not. 570 */ 571 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 572 if (info->io_apic_id == -1) { 573 info = NULL; 574 break; 575 } 576 if ((info->io_apic_id & IOAPIC_ID_MASK) == apic_id16) 577 break; 578 } 579 if (info == NULL) 580 return apic_id; 581 582 kprintf("IOAPIC: APIC ID %d has same lower 4bits as " 583 "%dth I/O APIC, keep searching...\n", 584 apic_id, info->io_idx); 585 586 start = apic_id + 1; 587 } 588 panic("ioapic_unused_apic_id: never reached\n"); 589 } 590 591 /* 592 * Map a physical memory address representing I/O into KVA. The I/O 593 * block is assumed not to cross a page boundary. 594 */ 595 void * 596 ioapic_map(vm_paddr_t pa) 597 { 598 KKASSERT(pa < 0x100000000LL); 599 600 return pmap_mapdev_uncacheable(pa, PAGE_SIZE); 601 } 602 603 static void 604 ioapic_sysinit(void *dummy __unused) 605 { 606 int error; 607 608 if (!ioapic_enable) 609 return; 610 611 KASSERT(lapic_enable, ("I/O APIC is enabled, but LAPIC is disabled\n")); 612 error = ioapic_config(); 613 if (error) { 614 ioapic_enable = 0; 615 icu_reinit_noioapic(); 616 lapic_fixup_noioapic(); 617 } 618 } 619 SYSINIT(ioapic, SI_BOOT2_IOAPIC, SI_ORDER_FIRST, ioapic_sysinit, NULL) 620