1 /* $NetBSD: pci_msi_machdep.c,v 1.6 2015/06/22 03:57:01 msaitoh Exp $ */ 2 3 /* 4 * Copyright (c) 2015 Internet Initiative Japan Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 /* 30 * TODO 31 * 32 * - PBA (Pending Bit Array) support 33 * - HyperTransport mapping support 34 */ 35 36 #include <sys/cdefs.h> 37 __KERNEL_RCSID(0, "$NetBSD: pci_msi_machdep.c,v 1.6 2015/06/22 03:57:01 msaitoh Exp $"); 38 39 #include <sys/types.h> 40 #include <sys/param.h> 41 #include <sys/time.h> 42 #include <sys/systm.h> 43 #include <sys/cpu.h> 44 #include <sys/errno.h> 45 #include <sys/device.h> 46 #include <sys/intr.h> 47 #include <sys/kmem.h> 48 #include <sys/malloc.h> 49 50 #include <dev/pci/pcivar.h> 51 52 #include <machine/i82093var.h> 53 #include <machine/pic.h> 54 55 #include <x86/pci/msipic.h> 56 #include <x86/pci/pci_msi_machdep.h> 57 58 #ifdef INTRDEBUG 59 #define MSIDEBUG 60 #endif 61 62 #ifdef MSIDEBUG 63 #define DPRINTF(msg) printf msg 64 #else 65 #define DPRINTF(msg) 66 #endif 67 68 static pci_intr_handle_t 69 pci_msi_calculate_handle(struct pic *msi_pic, int vector) 70 { 71 pci_intr_handle_t pih; 72 73 KASSERT(msipic_is_msi_pic(msi_pic)); 74 75 pih = __SHIFTIN((uint64_t)msipic_get_devid(msi_pic), MSI_INT_DEV_MASK) 76 | __SHIFTIN((uint64_t)vector, MSI_INT_VEC_MASK) 77 | APIC_INT_VIA_MSI; 78 if (msi_pic->pic_type == PIC_MSI) 79 MSI_INT_MAKE_MSI(pih); 80 else if (msi_pic->pic_type == PIC_MSIX) 81 MSI_INT_MAKE_MSIX(pih); 82 else 83 panic("%s: Unexpected pic_type: %d\n", __func__, 84 msi_pic->pic_type); 85 86 return pih; 87 } 88 89 static pci_intr_handle_t * 90 pci_msi_alloc_vectors(struct pic *msi_pic, uint *table_indexes, int *count) 91 { 92 struct intrsource *isp; 93 pci_intr_handle_t *vectors, pih; 94 int i; 95 const char *intrstr; 96 char intrstr_buf[INTRIDBUF]; 97 98 vectors = kmem_zalloc(sizeof(vectors[0]) * (*count), KM_SLEEP); 99 if (vectors == NULL) { 100 DPRINTF(("cannot allocate vectors\n")); 101 return NULL; 102 } 103 104 mutex_enter(&cpu_lock); 105 for (i = 0; i < *count; i++) { 106 u_int table_index; 107 108 if (table_indexes == NULL) 109 table_index = i; 110 else 111 table_index = table_indexes[i]; 112 113 pih = pci_msi_calculate_handle(msi_pic, table_index); 114 115 intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf, 116 sizeof(intrstr_buf)); 117 isp = intr_allocate_io_intrsource(intrstr); 118 if (isp == NULL) { 119 mutex_exit(&cpu_lock); 120 DPRINTF(("can't allocate io_intersource\n")); 121 kmem_free(vectors, sizeof(vectors[0]) * (*count)); 122 return NULL; 123 } 124 125 vectors[i] = pih; 126 } 127 mutex_exit(&cpu_lock); 128 129 return vectors; 130 } 131 132 static void 133 pci_msi_free_vectors(struct pic *msi_pic, pci_intr_handle_t *pihs, int count) 134 { 135 pci_intr_handle_t pih; 136 int i; 137 const char *intrstr; 138 char intrstr_buf[INTRIDBUF]; 139 140 mutex_enter(&cpu_lock); 141 for (i = 0; i < count; i++) { 142 pih = pci_msi_calculate_handle(msi_pic, i); 143 intrstr = x86_pci_msi_string(NULL, pih, intrstr_buf, 144 sizeof(intrstr_buf)); 145 intr_free_io_intrsource(intrstr); 146 } 147 mutex_exit(&cpu_lock); 148 149 kmem_free(pihs, sizeof(pihs[0]) * count); 150 } 151 152 static int 153 pci_msi_alloc_common(pci_intr_handle_t **ihps, int *count, 154 const struct pci_attach_args *pa, bool exact) 155 { 156 struct pic *msi_pic; 157 pci_intr_handle_t *vectors; 158 int error, i; 159 160 if ((pa->pa_flags & PCI_FLAGS_MSI_OKAY) == 0) { 161 DPRINTF(("PCI host bridge does not support MSI.\n")); 162 return ENODEV; 163 } 164 165 msi_pic = msipic_construct_msi_pic(pa); 166 if (msi_pic == NULL) { 167 DPRINTF(("cannot allocate MSI pic.\n")); 168 return EINVAL; 169 } 170 171 vectors = NULL; 172 while (*count > 0) { 173 vectors = pci_msi_alloc_vectors(msi_pic, NULL, count); 174 if (vectors != NULL) 175 break; 176 177 if (exact) { 178 DPRINTF(("cannot allocate MSI vectors.\n")); 179 msipic_destruct_msi_pic(msi_pic); 180 return ENOMEM; 181 } else { 182 (*count) >>= 1; /* must be power of 2. */ 183 continue; 184 } 185 } 186 if (vectors == NULL) { 187 DPRINTF(("cannot allocate MSI vectors.\n")); 188 msipic_destruct_msi_pic(msi_pic); 189 return ENOMEM; 190 } 191 192 for (i = 0; i < *count; i++) { 193 MSI_INT_MAKE_MSI(vectors[i]); 194 } 195 196 error = msipic_set_msi_vectors(msi_pic, NULL, *count); 197 if (error) { 198 pci_msi_free_vectors(msi_pic, vectors, *count); 199 msipic_destruct_msi_pic(msi_pic); 200 return error; 201 } 202 203 *ihps = vectors; 204 return 0; 205 } 206 207 static void * 208 pci_msi_common_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, 209 int level, int (*func)(void *), void *arg, struct pic *pic) 210 { 211 int irq, pin; 212 bool mpsafe; 213 214 KASSERT(INT_VIA_MSI(ih)); 215 216 irq = -1; 217 pin = MSI_INT_VEC(ih); 218 mpsafe = ((ih & MPSAFE_MASK) != 0); 219 220 return intr_establish(irq, pic, pin, IST_EDGE, level, func, arg, 221 mpsafe); 222 } 223 224 static void 225 pci_msi_common_disestablish(pci_chipset_tag_t pc, void *cookie) 226 { 227 228 intr_disestablish(cookie); 229 } 230 231 static int 232 pci_msix_alloc_common(pci_intr_handle_t **ihps, u_int *table_indexes, 233 int *count, const struct pci_attach_args *pa, bool exact) 234 { 235 struct pic *msix_pic; 236 pci_intr_handle_t *vectors; 237 int error, i; 238 239 if ((pa->pa_flags & PCI_FLAGS_MSIX_OKAY) == 0) { 240 DPRINTF(("PCI host bridge does not support MSI-X.\n")); 241 return ENODEV; 242 } 243 244 msix_pic = msipic_construct_msix_pic(pa); 245 if (msix_pic == NULL) 246 return EINVAL; 247 248 vectors = NULL; 249 while (*count > 0) { 250 vectors = pci_msi_alloc_vectors(msix_pic, table_indexes, count); 251 if (vectors != NULL) 252 break; 253 254 if (exact) { 255 DPRINTF(("cannot allocate MSI-X vectors.\n")); 256 msipic_destruct_msix_pic(msix_pic); 257 return ENOMEM; 258 } else { 259 (*count)--; 260 continue; 261 } 262 } 263 if (vectors == NULL) { 264 DPRINTF(("cannot allocate MSI-X vectors.\n")); 265 msipic_destruct_msix_pic(msix_pic); 266 return ENOMEM; 267 } 268 269 for (i = 0; i < *count; i++) { 270 MSI_INT_MAKE_MSIX(vectors[i]); 271 } 272 273 error = msipic_set_msi_vectors(msix_pic, vectors, *count); 274 if (error) { 275 pci_msi_free_vectors(msix_pic, vectors, *count); 276 msipic_destruct_msix_pic(msix_pic); 277 return error; 278 } 279 280 *ihps = vectors; 281 return 0; 282 } 283 284 static int 285 x86_pci_msi_alloc(pci_intr_handle_t **ihps, int *count, 286 const struct pci_attach_args *pa) 287 { 288 289 return pci_msi_alloc_common(ihps, count, pa, false); 290 } 291 292 static int 293 x86_pci_msi_alloc_exact(pci_intr_handle_t **ihps, int count, 294 const struct pci_attach_args *pa) 295 { 296 297 return pci_msi_alloc_common(ihps, &count, pa, true); 298 } 299 300 static void 301 x86_pci_msi_release_internal(pci_intr_handle_t *pihs, int count) 302 { 303 struct pic *pic; 304 305 pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0])); 306 if (pic == NULL) 307 return; 308 309 pci_msi_free_vectors(pic, pihs, count); 310 msipic_destruct_msi_pic(pic); 311 } 312 313 static int 314 x86_pci_msix_alloc(pci_intr_handle_t **ihps, int *count, 315 const struct pci_attach_args *pa) 316 { 317 318 return pci_msix_alloc_common(ihps, NULL, count, pa, false); 319 } 320 321 static int 322 x86_pci_msix_alloc_exact(pci_intr_handle_t **ihps, int count, 323 const struct pci_attach_args *pa) 324 { 325 326 return pci_msix_alloc_common(ihps, NULL, &count, pa, true); 327 } 328 329 static int 330 x86_pci_msix_alloc_map(pci_intr_handle_t **ihps, u_int *table_indexes, 331 int count, const struct pci_attach_args *pa) 332 { 333 334 return pci_msix_alloc_common(ihps, table_indexes, &count, pa, true); 335 } 336 337 static void 338 x86_pci_msix_release_internal(pci_intr_handle_t *pihs, int count) 339 { 340 struct pic *pic; 341 342 pic = msipic_find_msi_pic(MSI_INT_DEV(pihs[0])); 343 if (pic == NULL) 344 return; 345 346 pci_msi_free_vectors(pic, pihs, count); 347 msipic_destruct_msix_pic(pic); 348 } 349 350 /*****************************************************************************/ 351 /* 352 * extern for MD code. 353 */ 354 355 /* 356 * Return intrid for a MSI/MSI-X device. 357 * "buf" must be allocated by caller. 358 */ 359 const char * 360 x86_pci_msi_string(pci_chipset_tag_t pc, pci_intr_handle_t ih, char *buf, 361 size_t len) 362 { 363 int dev, vec; 364 365 KASSERT(INT_VIA_MSI(ih)); 366 367 dev = MSI_INT_DEV(ih); 368 vec = MSI_INT_VEC(ih); 369 if (MSI_INT_IS_MSIX(ih)) 370 snprintf(buf, len, "msix%d vec %d", dev, vec); 371 else 372 snprintf(buf, len, "msi%d vec %d", dev, vec); 373 374 return buf; 375 } 376 377 /* 378 * Release MSI handles. 379 */ 380 void 381 x86_pci_msi_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count) 382 { 383 384 if (count < 1) 385 return; 386 387 return x86_pci_msi_release_internal(pihs, count); 388 } 389 390 /* 391 * Establish a MSI handle. 392 * If multiple MSI handle is requied to establish, device driver must call 393 * this function for each handle. 394 */ 395 void * 396 x86_pci_msi_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, 397 int level, int (*func)(void *), void *arg) 398 { 399 struct pic *pic; 400 401 pic = msipic_find_msi_pic(MSI_INT_DEV(ih)); 402 if (pic == NULL) { 403 DPRINTF(("pci_intr_handler has no msi_pic\n")); 404 return NULL; 405 } 406 407 return pci_msi_common_establish(pc, ih, level, func, arg, pic); 408 } 409 410 /* 411 * Disestablish a MSI handle. 412 * If multiple MSI handle is requied to disestablish, device driver must call 413 * this function for each handle. 414 */ 415 void 416 x86_pci_msi_disestablish(pci_chipset_tag_t pc, void *cookie) 417 { 418 419 pci_msi_common_disestablish(pc, cookie); 420 } 421 422 /* 423 * Release MSI-X handles. 424 */ 425 void 426 x86_pci_msix_release(pci_chipset_tag_t pc, pci_intr_handle_t *pihs, int count) 427 { 428 429 if (count < 1) 430 return; 431 432 return x86_pci_msix_release_internal(pihs, count); 433 } 434 435 /* 436 * Establish a MSI-X handle. 437 * If multiple MSI-X handle is requied to establish, device driver must call 438 * this function for each handle. 439 */ 440 void * 441 x86_pci_msix_establish(pci_chipset_tag_t pc, pci_intr_handle_t ih, 442 int level, int (*func)(void *), void *arg) 443 { 444 struct pic *pic; 445 446 pic = msipic_find_msi_pic(MSI_INT_DEV(ih)); 447 if (pic == NULL) { 448 DPRINTF(("pci_intr_handler has no msi_pic\n")); 449 return NULL; 450 } 451 452 return pci_msi_common_establish(pc, ih, level, func, arg, pic); 453 } 454 455 /* 456 * Disestablish a MSI-X handle. 457 * If multiple MSI-X handle is requied to disestablish, device driver must call 458 * this function for each handle. 459 */ 460 void 461 x86_pci_msix_disestablish(pci_chipset_tag_t pc, void *cookie) 462 { 463 464 pci_msi_common_disestablish(pc, cookie); 465 } 466 467 /*****************************************************************************/ 468 /* 469 * extern for MI code. 470 */ 471 472 /* 473 * return number of the devices's MSI vectors 474 * return 0 if the device does not support MSI 475 */ 476 int 477 pci_msi_count(const struct pci_attach_args *pa) 478 { 479 pci_chipset_tag_t pc; 480 pcitag_t tag; 481 pcireg_t reg; 482 uint32_t mmc; 483 int count, offset; 484 485 pc = pa->pa_pc; 486 tag = pa->pa_tag; 487 if (pci_get_capability(pc, tag, PCI_CAP_MSI, &offset, NULL) == 0) 488 return 0; 489 490 reg = pci_conf_read(pc, tag, offset + PCI_MSI_CTL); 491 mmc = PCI_MSI_CTL_MMC(reg); 492 count = 1 << mmc; 493 if (count > PCI_MSI_MAX_VECTORS) { 494 aprint_error("detect an illegal device! The device use reserved MMC values.\n"); 495 return 0; 496 } 497 498 return count; 499 } 500 501 /* 502 * This function is used by device drivers like pci_intr_map(). 503 * 504 * "ihps" is the array of vector numbers which MSI used instead of IRQ number. 505 * "count" must be power of 2. 506 * "count" can decrease if struct intrsource cannot be allocated. 507 * if count == 0, return non-zero value. 508 */ 509 int 510 pci_msi_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 511 int *count) 512 { 513 int hw_max; 514 515 /* MSI vector count must be power of 2. */ 516 KASSERT(*count > 0); 517 KASSERT(((*count - 1) & *count) == 0); 518 519 hw_max = pci_msi_count(pa); 520 if (hw_max == 0) 521 return ENODEV; 522 523 if (*count > hw_max) { 524 DPRINTF(("cut off MSI count to %d\n", hw_max)); 525 *count = hw_max; /* cut off hw_max */ 526 } 527 528 return x86_pci_msi_alloc(ihps, count, pa); 529 } 530 531 /* 532 * This function is used by device drivers like pci_intr_map(). 533 * 534 * "ihps" is the array of vector numbers which MSI used instead of IRQ number. 535 * "count" must be power of 2. 536 * "count" can not decrease. 537 * If "count" struct intrsources cannot be allocated, return non-zero value. 538 */ 539 int 540 pci_msi_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 541 int count) 542 { 543 int hw_max; 544 545 /* MSI vector count must be power of 2. */ 546 KASSERT(count > 0); 547 KASSERT(((count - 1) & count) == 0); 548 549 hw_max = pci_msi_count(pa); 550 if (hw_max == 0) 551 return ENODEV; 552 553 if (count > hw_max) { 554 DPRINTF(("over hardware max MSI count %d\n", hw_max)); 555 return EINVAL; 556 } 557 558 return x86_pci_msi_alloc_exact(ihps, count, pa); 559 } 560 561 /* 562 * return number of the devices's MSI-X vectors 563 * return 0 if the device does not support MSI-X 564 */ 565 int 566 pci_msix_count(const struct pci_attach_args *pa) 567 { 568 pci_chipset_tag_t pc; 569 pcitag_t tag; 570 pcireg_t reg; 571 int offset; 572 573 pc = pa->pa_pc; 574 tag = pa->pa_tag; 575 if (pci_get_capability(pc, tag, PCI_CAP_MSIX, &offset, NULL) == 0) 576 return 0; 577 578 reg = pci_conf_read(pc, tag, offset + PCI_MSIX_CTL); 579 580 return PCI_MSIX_CTL_TBLSIZE(reg); 581 } 582 583 /* 584 * This function is used by device drivers like pci_intr_map(). 585 * 586 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number. 587 * "count" can decrease if enough struct intrsources cannot be allocated. 588 * if count == 0, return non-zero value. 589 */ 590 int 591 pci_msix_alloc(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 592 int *count) 593 { 594 int hw_max; 595 596 KASSERT(*count > 0); 597 598 hw_max = pci_msix_count(pa); 599 if (hw_max == 0) 600 return ENODEV; 601 602 if (*count > hw_max) { 603 DPRINTF(("cut off MSI-X count to %d\n", hw_max)); 604 *count = hw_max; /* cut off hw_max */ 605 } 606 607 return x86_pci_msix_alloc(ihps, count, pa); 608 } 609 610 /* 611 * This function is used by device drivers like pci_intr_map(). 612 * 613 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number. 614 * "count" can not decrease. 615 * If "count" struct intrsource cannot be allocated, return non-zero value. 616 */ 617 int 618 pci_msix_alloc_exact(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 619 int count) 620 { 621 int hw_max; 622 623 KASSERT(count > 0); 624 625 hw_max = pci_msix_count(pa); 626 if (hw_max == 0) 627 return ENODEV; 628 629 if (count > hw_max) { 630 DPRINTF(("over hardware max MSI-X count %d\n", hw_max)); 631 return EINVAL; 632 } 633 634 return x86_pci_msix_alloc_exact(ihps, count, pa); 635 } 636 637 /* 638 * This function is used by device drivers like pci_intr_map(). 639 * Futhermore, this function can map each handle to a MSI-X table index. 640 * 641 * "ihps" is the array of vector numbers which MSI-X used instead of IRQ number. 642 * "count" can not decrease. 643 * "map" size must be equal to "count". 644 * If "count" struct intrsource cannot be allocated, return non-zero value. 645 * e.g. 646 * If "map" = { 1, 4, 0 }, 647 * 1st handle is bound to MSI-X index 1 648 * 2nd handle is bound to MSI-X index 4 649 * 3rd handle is bound to MSI-X index 0 650 */ 651 int 652 pci_msix_alloc_map(const struct pci_attach_args *pa, pci_intr_handle_t **ihps, 653 u_int *table_indexes, int count) 654 { 655 int hw_max, i, j; 656 657 KASSERT(count > 0); 658 659 hw_max = pci_msix_count(pa); 660 if (hw_max == 0) 661 return ENODEV; 662 663 if (count > hw_max) { 664 DPRINTF(("over hardware max MSI-X count %d\n", hw_max)); 665 return EINVAL; 666 } 667 668 /* check not to duplicate table_index */ 669 for (i = 0; i < count; i++) { 670 u_int basing = table_indexes[i]; 671 672 KASSERT(table_indexes[i] < PCI_MSIX_MAX_VECTORS); 673 if (basing >= hw_max) { 674 DPRINTF(("table index is over hardware max MSI-X index %d\n", 675 hw_max - 1)); 676 return EINVAL; 677 } 678 679 for (j = i + 1; j < count; j++) { 680 if (basing == table_indexes[j]) { 681 DPRINTF(("MSI-X table index duplicated\n")); 682 return EINVAL; 683 } 684 } 685 } 686 687 return x86_pci_msix_alloc_map(ihps, table_indexes, count, pa); 688 } 689