1 /* $NetBSD: pci_msi_machdep.c,v 1.13 2017/07/28 14:26:50 maxv Exp $ */ 2 3 /* 4 * Copyright (c) 2015 Internet Initiative Japan Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * TODO 31 * 32 * - PBA (Pending Bit Array) support 33 * - HyperTransport mapping support 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: pci_msi_machdep.c,v 1.13 2017/07/28 14:26:50 maxv Exp $"); 38 39 #include "opt_intrdebug.h" 40 #include "ioapic.h" 41 42 #include <sys/types.h> 43 #include <sys/param.h> 44 #include <sys/time.h> 45 #include <sys/systm.h> 46 #include <sys/cpu.h> 47 #include <sys/errno.h> 48 #include <sys/device.h> 49 #include <sys/intr.h> 50 #include <sys/kmem.h> 51 52 #include <dev/pci/pcivar.h> 53 54 #include <machine/i82093var.h> 55 #include <machine/pic.h> 56 57 #include <x86/pci/msipic.h> 58 #include <x86/pci/pci_msi_machdep.h> 59 60 #ifdef INTRDEBUG 61 #define MSIDEBUG 62 #endif 63 64 #ifdef MSIDEBUG 65 #define DPRINTF(msg) printf msg 66 #else 67 #define DPRINTF(msg) 68 #endif 69 70 static pci_intr_handle_t 71 pci_msi_calculate_handle(struct pic *msi_pic, int vector) 72 { 73 pci_intr_handle_t pih; 74 75 KASSERT(msipic_is_msi_pic(msi_pic)); 76 77 pih = __SHIFTIN((uint64_t)msipic_get_devid(msi_pic), MSI_INT_DEV_MASK) 78 | __SHIFTIN((uint64_t)vector, MSI_INT_VEC_MASK) 79 | APIC_INT_VIA_MSI; 80 if (msi_pic->pic_type == PIC_MSI) 81 MSI_INT_MAKE_MSI(pih); 82 else if (msi_pic->pic_type == PIC_MSIX) 83 MSI_INT_MAKE_MSIX(pih); 84 else 85 panic("%s: Unexpected pic_type: %d\n", __func__, 86 msi_pic->pic_type); 87 88 return pih; 89 } 90 91 #ifdef __HAVE_PCI_MSI_MSIX 92 static pci_intr_handle_t * 93 pci_msi_alloc_vectors(struct pic *msi_pic, uint *table_indexes, int *count) 94 { 95 struct intrsource *isp; 96 pci_intr_handle_t *vectors, pih; 97 int i; 98 const char *intrstr; 99 char intrstr_buf[INTRIDBUF]; 100 101 vectors = kmem_zalloc(sizeof(vectors[0]) * (*count), KM_SLEEP); 102 mutex_enter(&cpu_lock); 103 for (i = 0; i < *count; i++) { 104 u_int table_index; 105 106 if (table_indexes == NULL) 107 table_index = i; 108 else 109 table_index = table_indexes[i]; 110 111 pih = pci_msi_calculate_handle(msi_pic, table_index); 112 113 intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf, 114 sizeof(intrstr_buf)); 115 isp = intr_allocate_io_intrsource(intrstr); 116 if (isp == NULL) { 117 mutex_exit(&cpu_lock); 118 DPRINTF(("can't allocate io_intersource\n")); 119 kmem_free(vectors, sizeof(vectors[0]) * (*count)); 120 return NULL; 121 } 122 123 vectors[i] = pih; 124 } 125 mutex_exit(&cpu_lock); 126 127 return vectors; 128 } 129 #endif /* __HAVE_PCI_MSI_MSIX */ 130 131 static void 132 pci_msi_free_vectors(struct pic *msi_pic, pci_intr_handle_t *pihs, int count) 133 { 134 pci_intr_handle_t pih; 135 int i; 136 const char *intrstr; 137 char intrstr_buf[INTRIDBUF]; 138 139 mutex_enter(&cpu_lock); 140 for (i = 0; i < count; i++) { 141 pih = pci_msi_calculate_handle(msi_pic, i); 142 intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf, 143 sizeof(intrstr_buf)); 144 intr_free_io_intrsource(intrstr); 145 } 146 mutex_exit(&cpu_lock); 147 148 kmem_free(pihs, sizeof(pihs[0]) * count); 149 } 150 151 #ifdef __HAVE_PCI_MSI_MSIX 152 static int 153 pci_msi_alloc_common(pci_intr_handle_t **ihps, int *count, 154 const struct pci_attach_args *pa, bool exact) 155 { 156 struct pic *msi_pic; 157 pci_intr_handle_t *vectors; 158 int error, i; 159 160 #if NIOAPIC > 0 161 if (nioapics == 0) { 162 DPRINTF(("no IOAPIC.\n")); 163 return ENODEV; 164 } 165 #endif 166 167 if ((pa->pa_flags & PCI_FLAGS_MSI_OKAY) == 0) { 168 DPRINTF(("PCI host bridge does not support MSI.\n")); 169 return ENODEV; 170 } 171 172 msi_pic = msipic_construct_msi_pic(pa); 173 if (msi_pic == NULL) { 174 DPRINTF(("cannot allocate MSI pic.\n")); 175 return EINVAL; 176 } 177 178 vectors = NULL; 179 while (*count > 0) { 180 vectors = pci_msi_alloc_vectors(msi_pic, NULL, count); 181 if (vectors != NULL) 182 break; 183 184 if (exact) { 185 DPRINTF(("cannot allocate MSI vectors.\n")); 186 msipic_destruct_msi_pic(msi_pic); 187 return ENOMEM; 188 } else { 189 (*count) >>= 1; /* must be power of 2. */ 190 continue; 191 } 192 } 193 if (vectors == NULL) { 194 DPRINTF(("cannot allocate MSI vectors.\n")); 195 msipic_destruct_msi_pic(msi_pic); 196 return ENOMEM; 197 } 198 199 for (i = 0; i < *count; i++) { 200 MSI_INT_MAKE_MSI(vectors[i]); 201 } 202 203 error = msipic_set_msi_vectors(msi_pic, NULL, *count); 204 if (error) { 205 pci_msi_free_vectors(msi_pic, vectors, *count); 206 msipic_destruct_msi_pic(msi_pic); 207 return error; 208 } 209 210 *ihps = vectors; 211 return 0; 212 } 213 #endif /* __HAVE_PCI_MSI_MSIX */ 214 215 static void * 216 pci_msi_common_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, 217 int level, int (*func)(void *), void *arg, struct pic *pic, 218 const char *xname) 219 { 220 int irq, pin; 221 bool mpsafe; 222 223 KASSERT(INT_VIA_MSI(ih)); 224 225 irq = -1; 226 pin = MSI_INT_VEC(ih); 227 mpsafe = ((ih & MPSAFE_MASK) != 0); 228 229 return intr_establish_xname(irq, pic, pin, IST_EDGE, level, func, arg, 230 mpsafe, xname); 231 } 232 233 static void 234 pci_msi_common_disestablish(pci_chipset_tag_t pc, void *cookie) 235 { 236 237 intr_disestablish(cookie); 238 } 239 240 #ifdef __HAVE_PCI_MSI_MSIX 241 static int 242 pci_msix_alloc_common(pci_intr_handle_t **ihps, u_int *table_indexes, 243 int *count, const struct pci_attach_args *pa, bool exact) 244 { 245 struct pic *msix_pic; 246 pci_intr_handle_t *vectors; 247 int error, i; 248 249 #if NIOAPIC > 0 250 if (nioapics == 0) { 251 DPRINTF(("no IOAPIC.\n")); 252 return ENODEV; 253 } 254 #endif 255 256 if ((pa->pa_flags & PCI_FLAGS_MSIX_OKAY) == 0) { 257 DPRINTF(("PCI host bridge does not support MSI-X.\n")); 258 return ENODEV; 259 } 260 261 msix_pic = msipic_construct_msix_pic(pa); 262 if (msix_pic == NULL) 263 return EINVAL; 264 265 vectors = NULL; 266 while (*count > 0) { 267 vectors = pci_msi_alloc_vectors(msix_pic, table_indexes, count); 268 if (vectors != NULL) 269 break; 270 271 if (exact) { 272 DPRINTF(("cannot allocate MSI-X vectors.\n")); 273 msipic_destruct_msix_pic(msix_pic); 274 return ENOMEM; 275 } else { 276 (*count)--; 277 continue; 278 } 279 } 280 if (vectors == NULL) { 281 DPRINTF(("cannot allocate MSI-X vectors.\n")); 282 msipic_destruct_msix_pic(msix_pic); 283 return ENOMEM; 284 } 285 286 for (i = 0; i < *count; i++) { 287 MSI_INT_MAKE_MSIX(vectors[i]); 288 } 289 290 error = msipic_set_msi_vectors(msix_pic, vectors, *count); 291 if (error) { 292 pci_msi_free_vectors(msix_pic, vectors, *count); 293 msipic_destruct_msix_pic(msix_pic); 294 return error; 295 } 296 297 *ihps = vectors; 298 return 0; 299 } 300 301 static int 302 x86_pci_msi_alloc(pci_intr_handle_t **ihps, int *count, 303 const struct pci_attach_args *pa) 304 { 305 306 return pci_msi_alloc_common(ihps, count, pa, false); 307 } 308 309 static int 310 x86_pci_msi_alloc_exact(pci_intr_handle_t **ihps, int count, 311 const struct pci_attach_args *pa) 312 { 313 314 return pci_msi_alloc_common(ihps, &count, pa, true); 315 } 316 #endif /* __HAVE_PCI_MSI_MSIX */ 317 318 static void 319 x86_pci_msi_release_internal(pci_intr_handle_t *pihs, int count) 320 { 321 struct pic *pic; 322 323 pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0])); 324 if (pic == NULL) 325 return; 326 327 pci_msi_free_vectors(pic, pihs, count); 328 msipic_destruct_msi_pic(pic); 329 } 330 331 #ifdef __HAVE_PCI_MSI_MSIX 332 static int 333 x86_pci_msix_alloc(pci_intr_handle_t **ihps, int *count, 334 const struct pci_attach_args *pa) 335 { 336 337 return pci_msix_alloc_common(ihps, NULL, count, pa, false); 338 } 339 340 static int 341 x86_pci_msix_alloc_exact(pci_intr_handle_t **ihps, int count, 342 const struct pci_attach_args *pa) 343 { 344 345 return pci_msix_alloc_common(ihps, NULL, &count, pa, true); 346 } 347 348 static int 349 x86_pci_msix_alloc_map(pci_intr_handle_t **ihps, u_int *table_indexes, 350 int count, const struct pci_attach_args *pa) 351 { 352 353 return pci_msix_alloc_common(ihps, table_indexes, &count, pa, true); 354 } 355 #endif /* __HAVE_PCI_MSI_MSIX */ 356 357 static void 358 x86_pci_msix_release_internal(pci_intr_handle_t *pihs, int count) 359 { 360 struct pic *pic; 361 362 pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0])); 363 if (pic == NULL) 364 return; 365 366 pci_msi_free_vectors(pic, pihs, count); 367 msipic_destruct_msix_pic(pic); 368 } 369 370 /*****************************************************************************/ 371 /* 372 * extern for MD code. 373 */ 374 375 /* 376 * Return intrid for a MSI/MSI-X device. 377 * "buf" must be allocated by caller. 378 */ 379 const char * 380 x86_pci_msi_string(pci_chipset_tag_t pc, pci_intr_handle_t ih, char *buf, 381 size_t len) 382 { 383 int dev, vec; 384 385 KASSERT(INT_VIA_MSI(ih)); 386 387 dev = MSI_INT_DEV(ih); 388 vec = MSI_INT_VEC(ih); 389 if (MSI_INT_IS_MSIX(ih)) 390 snprintf(buf, len, "msix%d vec %d", dev, vec); 391 else 392 snprintf(buf, len, "msi%d vec %d", dev, vec); 393 394 return buf; 395 } 396 397 /* 398 * Release MSI handles. 399 */ 400 void 401 x86_pci_msi_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count) 402 { 403 404 if (count < 1) 405 return; 406 407 return x86_pci_msi_release_internal(pihs, count); 408 } 409 410 /* 411 * Establish a MSI handle. 412 * If multiple MSI handle is requied to establish, device driver must call 413 * this function for each handle. 414 */ 415 void * 416 x86_pci_msi_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, 417 int level, int (*func)(void *), void *arg, const char *xname) 418 { 419 struct pic *pic; 420 421 pic = msipic_find_msi_pic(MSI_INT_DEV(ih)); 422 if (pic == NULL) { 423 DPRINTF(("pci_intr_handler has no msi_pic\n")); 424 return NULL; 425 } 426 427 return pci_msi_common_establish(pc, ih, level, func, arg, pic, xname); 428 } 429 430 /* 431 * Disestablish a MSI handle. 432 * If multiple MSI handle is requied to disestablish, device driver must call 433 * this function for each handle. 434 */ 435 void 436 x86_pci_msi_disestablish(pci_chipset_tag_t pc, void *cookie) 437 { 438 439 pci_msi_common_disestablish(pc, cookie); 440 } 441 442 /* 443 * Release MSI-X handles. 444 */ 445 void 446 x86_pci_msix_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count) 447 { 448 449 if (count < 1) 450 return; 451 452 return x86_pci_msix_release_internal(pihs, count); 453 } 454 455 /* 456 * Establish a MSI-X handle. 457 * If multiple MSI-X handle is requied to establish, device driver must call 458 * this function for each handle. 459 */ 460 void * 461 x86_pci_msix_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, 462 int level, int (*func)(void *), void *arg, const char *xname) 463 { 464 struct pic *pic; 465 466 pic = msipic_find_msi_pic(MSI_INT_DEV(ih)); 467 if (pic == NULL) { 468 DPRINTF(("pci_intr_handler has no msi_pic\n")); 469 return NULL; 470 } 471 472 return pci_msi_common_establish(pc, ih, level, func, arg, pic, xname); 473 } 474 475 /* 476 * Disestablish a MSI-X handle. 477 * If multiple MSI-X handle is requied to disestablish, device driver must call 478 * this function for each handle. 479 */ 480 void 481 x86_pci_msix_disestablish(pci_chipset_tag_t pc, void *cookie) 482 { 483 484 pci_msi_common_disestablish(pc, cookie); 485 } 486 487 #ifdef __HAVE_PCI_MSI_MSIX 488 /*****************************************************************************/ 489 /* 490 * extern for MI code. 491 */ 492 493 /* 494 * This function is used by device drivers like pci_intr_map(). 495 * 496 * "ihps" is the array of vector numbers which MSI used instead of IRQ number. 497 * "count" must be power of 2. 498 * "count" can decrease if struct intrsource cannot be allocated. 499 * if count == 0, return non-zero value. 500 */ 501 int 502 pci_msi_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 503 int *count) 504 { 505 int hw_max; 506 507 /* MSI vector count must be power of 2. */ 508 KASSERT(*count > 0); 509 KASSERT(((*count - 1) & *count) == 0); 510 511 hw_max = pci_msi_count(pa->pa_pc, pa->pa_tag); 512 if (hw_max == 0) 513 return ENODEV; 514 515 if (*count > hw_max) { 516 DPRINTF(("cut off MSI count to %d\n", hw_max)); 517 *count = hw_max; /* cut off hw_max */ 518 } 519 520 return x86_pci_msi_alloc(ihps, count, pa); 521 } 522 523 /* 524 * This function is used by device drivers like pci_intr_map(). 525 * 526 * "ihps" is the array of vector numbers which MSI used instead of IRQ number. 527 * "count" must be power of 2. 528 * "count" can not decrease. 529 * If "count" struct intrsources cannot be allocated, return non-zero value. 530 */ 531 int 532 pci_msi_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 533 int count) 534 { 535 int hw_max; 536 537 /* MSI vector count must be power of 2. */ 538 KASSERT(count > 0); 539 KASSERT(((count - 1) & count) == 0); 540 541 hw_max = pci_msi_count(pa->pa_pc, pa->pa_tag); 542 if (hw_max == 0) 543 return ENODEV; 544 545 if (count > hw_max) { 546 DPRINTF(("over hardware max MSI count %d\n", hw_max)); 547 return EINVAL; 548 } 549 550 return x86_pci_msi_alloc_exact(ihps, count, pa); 551 } 552 553 /* 554 * This function is used by device drivers like pci_intr_map(). 555 * 556 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number. 557 * "count" can decrease if enough struct intrsources cannot be allocated. 558 * if count == 0, return non-zero value. 559 */ 560 int 561 pci_msix_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 562 int *count) 563 { 564 int hw_max; 565 566 KASSERT(*count > 0); 567 568 hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag); 569 if (hw_max == 0) 570 return ENODEV; 571 572 if (*count > hw_max) { 573 DPRINTF(("cut off MSI-X count to %d\n", hw_max)); 574 *count = hw_max; /* cut off hw_max */ 575 } 576 577 return x86_pci_msix_alloc(ihps, count, pa); 578 } 579 580 /* 581 * This function is used by device drivers like pci_intr_map(). 582 * 583 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number. 584 * "count" can not decrease. 585 * If "count" struct intrsource cannot be allocated, return non-zero value. 586 */ 587 int 588 pci_msix_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 589 int count) 590 { 591 int hw_max; 592 593 KASSERT(count > 0); 594 595 hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag); 596 if (hw_max == 0) 597 return ENODEV; 598 599 if (count > hw_max) { 600 DPRINTF(("over hardware max MSI-X count %d\n", hw_max)); 601 return EINVAL; 602 } 603 604 return x86_pci_msix_alloc_exact(ihps, count, pa); 605 } 606 607 /* 608 * This function is used by device drivers like pci_intr_map(). 609 * Futhermore, this function can map each handle to a MSI-X table index. 610 * 611 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number. 612 * "count" can not decrease. 613 * "map" size must be equal to "count". 614 * If "count" struct intrsource cannot be allocated, return non-zero value. 615 * e.g. 616 * If "map" = { 1, 4, 0 }, 617 * 1st handle is bound to MSI-X index 1 618 * 2nd handle is bound to MSI-X index 4 619 * 3rd handle is bound to MSI-X index 0 620 */ 621 int 622 pci_msix_alloc_map(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 623 u_int *table_indexes, int count) 624 { 625 int hw_max, i, j; 626 627 KASSERT(count > 0); 628 629 hw_max = pci_msix_count(pa->pa_pc, pa->pa_tag); 630 if (hw_max == 0) 631 return ENODEV; 632 633 if (count > hw_max) { 634 DPRINTF(("over hardware max MSI-X count %d\n", hw_max)); 635 return EINVAL; 636 } 637 638 /* check not to duplicate table_index */ 639 for (i = 0; i < count; i++) { 640 u_int basing = table_indexes[i]; 641 642 KASSERT(table_indexes[i] < PCI_MSIX_MAX_VECTORS); 643 if (basing >= hw_max) { 644 DPRINTF(("table index is over hardware max MSI-X index %d\n", 645 hw_max - 1)); 646 return EINVAL; 647 } 648 649 for (j = i + 1; j < count; j++) { 650 if (basing == table_indexes[j]) { 651 DPRINTF(("MSI-X table index duplicated\n")); 652 return EINVAL; 653 } 654 } 655 } 656 657 return x86_pci_msix_alloc_map(ihps, table_indexes, count, pa); 658 } 659 #endif /* __HAVE_PCI_MSI_MSIX */ 660