1 /* $NetBSD: pci_msi_machdep.c,v 1.15 2021/03/14 08:10:23 skrll Exp $ */ 2 3 /* 4 * Copyright (c) 2015 Internet Initiative Japan Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * TODO 31 * 32 * - PBA (Pending Bit Array) support 33 * - HyperTransport mapping support 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: pci_msi_machdep.c,v 1.15 2021/03/14 08:10:23 skrll Exp $"); 38 39 #include "opt_intrdebug.h" 40 #include "ioapic.h" 41 42 #include <sys/types.h> 43 #include <sys/param.h> 44 #include <sys/time.h> 45 #include <sys/systm.h> 46 #include <sys/cpu.h> 47 #include <sys/errno.h> 48 #include <sys/device.h> 49 #include <sys/intr.h> 50 #include <sys/kmem.h> 51 52 #include <dev/pci/pcivar.h> 53 54 #include <machine/i82093var.h> 55 #include <machine/pic.h> 56 57 #include <x86/pci/msipic.h> 58 #include <x86/pci/pci_msi_machdep.h> 59 60 #ifdef INTRDEBUG 61 #define MSIDEBUG 62 #endif 63 64 #ifdef MSIDEBUG 65 #define DPRINTF(msg) printf msg 66 #else 67 #define DPRINTF(msg) 68 #endif 69 70 static pci_intr_handle_t 71 pci_msi_calculate_handle(struct pic *msi_pic, int vector) 72 { 73 pci_intr_handle_t pih; 74 75 KASSERT(msipic_is_msi_pic(msi_pic)); 76 77 pih = __SHIFTIN((uint64_t)msipic_get_devid(msi_pic), MSI_INT_DEV_MASK) 78 | __SHIFTIN((uint64_t)vector, MSI_INT_VEC_MASK) 79 | APIC_INT_VIA_MSI; 80 if (msi_pic->pic_type == PIC_MSI) 81 MSI_INT_MAKE_MSI(pih); 82 else if (msi_pic->pic_type == PIC_MSIX) 83 MSI_INT_MAKE_MSIX(pih); 84 else 85 panic("%s: Unexpected pic_type: %d\n", __func__, 86 msi_pic->pic_type); 87 88 return pih; 89 } 90 91 #ifdef __HAVE_PCI_MSI_MSIX 92 static pci_intr_handle_t * 93 pci_msi_alloc_vectors(struct pic *msi_pic, uint *table_indexes, int *count) 94 { 95 struct intrsource *isp; 96 pci_intr_handle_t *vectors, pih; 97 int i; 98 const char *intrstr; 99 char intrstr_buf[INTRIDBUF]; 100 101 vectors = kmem_zalloc(sizeof(vectors[0]) * (*count), KM_SLEEP); 102 mutex_enter(&cpu_lock); 103 for (i = 0; i < *count; i++) { 104 u_int table_index; 105 106 if (table_indexes == NULL) 107 table_index = i; 108 else 109 table_index = table_indexes[i]; 110 111 pih = pci_msi_calculate_handle(msi_pic, table_index); 112 113 intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf, 114 sizeof(intrstr_buf)); 115 isp = intr_allocate_io_intrsource(intrstr); 116 if (isp == NULL) { 117 mutex_exit(&cpu_lock); 118 DPRINTF(("can't allocate io_intersource\n")); 119 kmem_free(vectors, sizeof(vectors[0]) * (*count)); 120 return NULL; 121 } 122 123 vectors[i] = pih; 124 } 125 mutex_exit(&cpu_lock); 126 127 return vectors; 128 } 129 #endif /* __HAVE_PCI_MSI_MSIX */ 130 131 static void 132 pci_msi_free_vectors(struct pic *msi_pic, pci_intr_handle_t *pihs, int count) 133 { 134 pci_intr_handle_t pih; 135 int i; 136 const char *intrstr; 137 char intrstr_buf[INTRIDBUF]; 138 139 mutex_enter(&cpu_lock); 140 for (i = 0; i < count; i++) { 141 pih = pci_msi_calculate_handle(msi_pic, i); 142 intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf, 143 sizeof(intrstr_buf)); 144 intr_free_io_intrsource(intrstr); 145 } 146 mutex_exit(&cpu_lock); 147 148 kmem_free(pihs, sizeof(pihs[0]) * count); 149 } 150 151 #ifdef __HAVE_PCI_MSI_MSIX 152 static int 153 pci_msi_alloc_common(pci_intr_handle_t **ihps, int *count, 154 const struct pci_attach_args *pa, bool exact) 155 { 156 struct pic *msi_pic; 157 pci_intr_handle_t *vectors; 158 int error, i; 159 160 #if NIOAPIC > 0 161 if (nioapics == 0) { 162 DPRINTF(("no IOAPIC.\n")); 163 return ENODEV; 164 } 165 #endif 166 167 if ((pa->pa_flags & PCI_FLAGS_MSI_OKAY) == 0) { 168 DPRINTF(("PCI host bridge does not support MSI.\n")); 169 return ENODEV; 170 } 171 172 msi_pic = msipic_construct_msi_pic(pa); 173 if (msi_pic == NULL) { 174 DPRINTF(("cannot allocate MSI pic.\n")); 175 return EINVAL; 176 } 177 178 #ifdef XENPV 179 if (xen_pci_msi_probe(msi_pic, *count)) { 180 DPRINTF(("xen_pci_msi_probe() failed\n")); 181 msipic_destruct_msi_pic(msi_pic); 182 return EINVAL; 183 } 184 #endif 185 186 vectors = NULL; 187 while (*count > 0) { 188 vectors = pci_msi_alloc_vectors(msi_pic, NULL, count); 189 if (vectors != NULL) 190 break; 191 192 if (exact) { 193 DPRINTF(("cannot allocate MSI vectors.\n")); 194 msipic_destruct_msi_pic(msi_pic); 195 return ENOMEM; 196 } else { 197 (*count) >>= 1; /* must be power of 2. */ 198 continue; 199 } 200 } 201 if (vectors == NULL) { 202 DPRINTF(("cannot allocate MSI vectors.\n")); 203 msipic_destruct_msi_pic(msi_pic); 204 return ENOMEM; 205 } 206 207 for (i = 0; i < *count; i++) { 208 MSI_INT_MAKE_MSI(vectors[i]); 209 } 210 211 error = msipic_set_msi_vectors(msi_pic, NULL, *count); 212 if (error) { 213 pci_msi_free_vectors(msi_pic, vectors, *count); 214 msipic_destruct_msi_pic(msi_pic); 215 return error; 216 } 217 218 *ihps = vectors; 219 return 0; 220 } 221 #endif /* __HAVE_PCI_MSI_MSIX */ 222 223 static void * 224 pci_msi_common_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, 225 int level, int (*func)(void *), void *arg, struct pic *pic, 226 const char *xname) 227 { 228 int irq, pin; 229 bool mpsafe; 230 231 KASSERT(INT_VIA_MSI(ih)); 232 233 irq = -1; 234 pin = MSI_INT_VEC(ih); 235 mpsafe = ((ih & MPSAFE_MASK) != 0); 236 237 return intr_establish_xname(irq, pic, pin, IST_EDGE, level, func, arg, 238 mpsafe, xname); 239 } 240 241 static void 242 pci_msi_common_disestablish(pci_chipset_tag_t pc, void *cookie) 243 { 244 245 intr_disestablish(cookie); 246 } 247 248 #ifdef __HAVE_PCI_MSI_MSIX 249 static int 250 pci_msix_alloc_common(pci_intr_handle_t **ihps, u_int *table_indexes, 251 int *count, const struct pci_attach_args *pa, bool exact) 252 { 253 struct pic *msix_pic; 254 pci_intr_handle_t *vectors; 255 int error, i; 256 257 #if NIOAPIC > 0 258 if (nioapics == 0) { 259 DPRINTF(("no IOAPIC.\n")); 260 return ENODEV; 261 } 262 #endif 263 264 if ((pa->pa_flags & PCI_FLAGS_MSIX_OKAY) == 0) { 265 DPRINTF(("PCI host bridge does not support MSI-X.\n")); 266 return ENODEV; 267 } 268 269 msix_pic = msipic_construct_msix_pic(pa); 270 if (msix_pic == NULL) 271 return EINVAL; 272 273 #ifdef XENPV 274 if (xen_pci_msi_probe(msix_pic, *count)) { 275 DPRINTF(("xen_pci_msi_probe() failed\n")); 276 msipic_destruct_msix_pic(msix_pic); 277 return EINVAL; 278 } 279 #endif 280 281 vectors = NULL; 282 while (*count > 0) { 283 vectors = pci_msi_alloc_vectors(msix_pic, table_indexes, count); 284 if (vectors != NULL) 285 break; 286 287 if (exact) { 288 DPRINTF(("cannot allocate MSI-X vectors.\n")); 289 msipic_destruct_msix_pic(msix_pic); 290 return ENOMEM; 291 } else { 292 (*count)--; 293 continue; 294 } 295 } 296 if (vectors == NULL) { 297 DPRINTF(("cannot allocate MSI-X vectors.\n")); 298 msipic_destruct_msix_pic(msix_pic); 299 return ENOMEM; 300 } 301 302 for (i = 0; i < *count; i++) { 303 MSI_INT_MAKE_MSIX(vectors[i]); 304 } 305 306 error = msipic_set_msi_vectors(msix_pic, vectors, *count); 307 if (error) { 308 pci_msi_free_vectors(msix_pic, vectors, *count); 309 msipic_destruct_msix_pic(msix_pic); 310 return error; 311 } 312 313 *ihps = vectors; 314 return 0; 315 } 316 317 static int 318 x86_pci_msi_alloc(pci_intr_handle_t **ihps, int *count, 319 const struct pci_attach_args *pa) 320 { 321 322 return pci_msi_alloc_common(ihps, count, pa, false); 323 } 324 325 static int 326 x86_pci_msi_alloc_exact(pci_intr_handle_t **ihps, int count, 327 const struct pci_attach_args *pa) 328 { 329 330 return pci_msi_alloc_common(ihps, &count, pa, true); 331 } 332 #endif /* __HAVE_PCI_MSI_MSIX */ 333 334 static void 335 x86_pci_msi_release_internal(pci_intr_handle_t *pihs, int count) 336 { 337 struct pic *pic; 338 339 pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0])); 340 if (pic == NULL) 341 return; 342 343 pci_msi_free_vectors(pic, pihs, count); 344 msipic_destruct_msi_pic(pic); 345 } 346 347 #ifdef __HAVE_PCI_MSI_MSIX 348 static int 349 x86_pci_msix_alloc(pci_intr_handle_t **ihps, int *count, 350 const struct pci_attach_args *pa) 351 { 352 353 return pci_msix_alloc_common(ihps, NULL, count, pa, false); 354 } 355 356 static int 357 x86_pci_msix_alloc_exact(pci_intr_handle_t **ihps, int count, 358 const struct pci_attach_args *pa) 359 { 360 361 return pci_msix_alloc_common(ihps, NULL, &count, pa, true); 362 } 363 364 static int 365 x86_pci_msix_alloc_map(pci_intr_handle_t **ihps, u_int *table_indexes, 366 int count, const struct pci_attach_args *pa) 367 { 368 369 return pci_msix_alloc_common(ihps, table_indexes, &count, pa, true); 370 } 371 #endif /* __HAVE_PCI_MSI_MSIX */ 372 373 static void 374 x86_pci_msix_release_internal(pci_intr_handle_t *pihs, int count) 375 { 376 struct pic *pic; 377 378 pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0])); 379 if (pic == NULL) 380 return; 381 382 pci_msi_free_vectors(pic, pihs, count); 383 msipic_destruct_msix_pic(pic); 384 } 385 386 /*****************************************************************************/ 387 /* 388 * extern for MD code. 389 */ 390 391 /* 392 * Return intrid for a MSI/MSI-X device. 393 * "buf" must be allocated by caller. 394 */ 395 const char * 396 x86_pci_msi_string(pci_chipset_tag_t pc, pci_intr_handle_t ih, char *buf, 397 size_t len) 398 { 399 int dev, vec; 400 401 KASSERT(INT_VIA_MSI(ih)); 402 403 dev = MSI_INT_DEV(ih); 404 vec = MSI_INT_VEC(ih); 405 if (MSI_INT_IS_MSIX(ih)) 406 snprintf(buf, len, "msix%d vec %d", dev, vec); 407 else 408 snprintf(buf, len, "msi%d vec %d", dev, vec); 409 410 return buf; 411 } 412 413 /* 414 * Release MSI handles. 415 */ 416 void 417 x86_pci_msi_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count) 418 { 419 420 if (count < 1) 421 return; 422 423 return x86_pci_msi_release_internal(pihs, count); 424 } 425 426 /* 427 * Establish a MSI handle. 428 * If multiple MSI handle is requied to establish, device driver must call 429 * this function for each handle. 430 */ 431 void * 432 x86_pci_msi_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, 433 int level, int (*func)(void *), void *arg, const char *xname) 434 { 435 struct pic *pic; 436 437 pic = msipic_find_msi_pic(MSI_INT_DEV(ih)); 438 if (pic == NULL) { 439 DPRINTF(("pci_intr_handler has no msi_pic\n")); 440 return NULL; 441 } 442 443 return pci_msi_common_establish(pc, ih, level, func, arg, pic, xname); 444 } 445 446 /* 447 * Disestablish a MSI handle. 448 * If multiple MSI handle is requied to disestablish, device driver must call 449 * this function for each handle. 450 */ 451 void 452 x86_pci_msi_disestablish(pci_chipset_tag_t pc, void *cookie) 453 { 454 455 pci_msi_common_disestablish(pc, cookie); 456 } 457 458 /* 459 * Release MSI-X handles. 460 */ 461 void 462 x86_pci_msix_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count) 463 { 464 465 if (count < 1) 466 return; 467 468 return x86_pci_msix_release_internal(pihs, count); 469 } 470 471 /* 472 * Establish a MSI-X handle. 473 * If multiple MSI-X handle is requied to establish, device driver must call 474 * this function for each handle. 475 */ 476 void * 477 x86_pci_msix_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, 478 int level, int (*func)(void *), void *arg, const char *xname) 479 { 480 struct pic *pic; 481 482 pic = msipic_find_msi_pic(MSI_INT_DEV(ih)); 483 if (pic == NULL) { 484 DPRINTF(("pci_intr_handler has no msi_pic\n")); 485 return NULL; 486 } 487 488 return pci_msi_common_establish(pc, ih, level, func, arg, pic, xname); 489 } 490 491 /* 492 * Disestablish a MSI-X handle. 493 * If multiple MSI-X handle is requied to disestablish, device driver must call 494 * this function for each handle. 495 */ 496 void 497 x86_pci_msix_disestablish(pci_chipset_tag_t pc, void *cookie) 498 { 499 500 pci_msi_common_disestablish(pc, cookie); 501 } 502 503 #ifdef __HAVE_PCI_MSI_MSIX 504 /*****************************************************************************/ 505 /* 506 * extern for MI code. 507 */ 508 509 /* 510 * This function is used by device drivers like pci_intr_map(). 511 * 512 * "ihps" is the array of vector numbers which MSI used instead of IRQ number. 513 * "count" must be power of 2. 514 * "count" can decrease if struct intrsource cannot be allocated. 515 * if count == 0, return non-zero value. 516 */ 517 int 518 pci_msi_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 519 int *count) 520 { 521 int hw_max; 522 523 /* MSI vector count must be power of 2. */ 524 KASSERT(*count > 0); 525 KASSERT(((*count - 1) & *count) == 0); 526 527 hw_max = pci_msi_count(pa->pa_pc, pa->pa_tag); 528 if (hw_max == 0) 529 return ENODEV; 530 531 if (*count > hw_max) { 532 DPRINTF(("cut off MSI count to %d\n", hw_max)); 533 *count = hw_max; /* cut off hw_max */ 534 } 535 536 return x86_pci_msi_alloc(ihps, count, pa); 537 } 538 539 /* 540 * This function is used by device drivers like pci_intr_map(). 541 * 542 * "ihps" is the array of vector numbers which MSI used instead of IRQ number. 543 * "count" must be power of 2. 544 * "count" can not decrease. 545 * If "count" struct intrsources cannot be allocated, return non-zero value. 546 */ 547 int 548 pci_msi_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 549 int count) 550 { 551 int hw_max; 552 553 /* MSI vector count must be power of 2. */ 554 KASSERT(count > 0); 555 KASSERT(((count - 1) & count) == 0); 556 557 hw_max = pci_msi_count(pa->pa_pc, pa->pa_tag); 558 if (hw_max == 0) 559 return ENODEV; 560 561 if (count > hw_max) { 562 DPRINTF(("over hardware max MSI count %d\n", hw_max)); 563 return EINVAL; 564 } 565 566 return x86_pci_msi_alloc_exact(ihps, count, pa); 567 } 568 569 /* 570 * This function is used by device drivers like pci_intr_map(). 571 * 572 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number. 573 * "count" can decrease if enough struct intrsources cannot be allocated. 574 * if count == 0, return non-zero value. 575 */ 576 int 577 pci_msix_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 578 int *count) 579 { 580 int hw_max; 581 582 KASSERT(*count > 0); 583 584 hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag); 585 if (hw_max == 0) 586 return ENODEV; 587 588 if (*count > hw_max) { 589 DPRINTF(("cut off MSI-X count to %d\n", hw_max)); 590 *count = hw_max; /* cut off hw_max */ 591 } 592 593 return x86_pci_msix_alloc(ihps, count, pa); 594 } 595 596 /* 597 * This function is used by device drivers like pci_intr_map(). 598 * 599 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number. 600 * "count" can not decrease. 601 * If "count" struct intrsource cannot be allocated, return non-zero value. 602 */ 603 int 604 pci_msix_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 605 int count) 606 { 607 int hw_max; 608 609 KASSERT(count > 0); 610 611 hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag); 612 if (hw_max == 0) 613 return ENODEV; 614 615 if (count > hw_max) { 616 DPRINTF(("over hardware max MSI-X count %d\n", hw_max)); 617 return EINVAL; 618 } 619 620 return x86_pci_msix_alloc_exact(ihps, count, pa); 621 } 622 623 /* 624 * This function is used by device drivers like pci_intr_map(). 625 * Futhermore, this function can map each handle to a MSI-X table index. 626 * 627 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number. 628 * "count" can not decrease. 629 * "map" size must be equal to "count". 630 * If "count" struct intrsource cannot be allocated, return non-zero value. 631 * e.g. 632 * If "map" = { 1, 4, 0 }, 633 * 1st handle is bound to MSI-X index 1 634 * 2nd handle is bound to MSI-X index 4 635 * 3rd handle is bound to MSI-X index 0 636 */ 637 int 638 pci_msix_alloc_map(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 639 u_int *table_indexes, int count) 640 { 641 int hw_max, i, j; 642 643 KASSERT(count > 0); 644 645 hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag); 646 if (hw_max == 0) 647 return ENODEV; 648 649 if (count > hw_max) { 650 DPRINTF(("over hardware max MSI-X count %d\n", hw_max)); 651 return EINVAL; 652 } 653 654 /* check not to duplicate table_index */ 655 for (i = 0; i < count; i++) { 656 u_int basing = table_indexes[i]; 657 658 KASSERT(table_indexes[i] < PCI_MSIX_MAX_VECTORS); 659 if (basing >= hw_max) { 660 DPRINTF(("table index is over hardware max MSI-X index %d\n", 661 hw_max - 1)); 662 return EINVAL; 663 } 664 665 for (j = i + 1; j < count; j++) { 666 if (basing == table_indexes[j]) { 667 DPRINTF(("MSI-X table index duplicated\n")); 668 return EINVAL; 669 } 670 } 671 } 672 673 return x86_pci_msix_alloc_map(ihps, table_indexes, count, pa); 674 } 675 #endif /* __HAVE_PCI_MSI_MSIX */ 676