1 /* $NetBSD: msipic.c,v 1.8 2015/11/17 17:51:42 msaitoh Exp $ */ 2 3 /* 4 * Copyright (c) 2015 Internet Initiative Japan Inc. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 17 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 18 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 19 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 20 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 21 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 22 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 23 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 24 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 25 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29 #include <sys/cdefs.h> 30 __KERNEL_RCSID(0, "$NetBSD: msipic.c,v 1.8 2015/11/17 17:51:42 msaitoh Exp $"); 31 32 #include "opt_intrdebug.h" 33 34 #include <sys/types.h> 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/errno.h> 38 #include <sys/kmem.h> 39 #include <sys/malloc.h> 40 #include <sys/mutex.h> 41 42 #include <dev/pci/pcivar.h> 43 44 #include <machine/i82489reg.h> 45 #include <machine/i82093reg.h> 46 #include <machine/i82093var.h> 47 #include <machine/pic.h> 48 #include <machine/lock.h> 49 50 #include <x86/pci/msipic.h> 51 52 #ifdef INTRDEBUG 53 #define MSIPICDEBUG 54 #endif 55 56 #ifdef MSIPICDEBUG 57 #define DPRINTF(msg) printf msg 58 #else 59 #define DPRINTF(msg) 60 #endif 61 62 #define BUS_SPACE_WRITE_FLUSH(pc, tag) (void)bus_space_read_4(pc, tag, 0) 63 64 #define MSIPICNAMEBUF 16 65 66 /* 67 * A Pseudo pic for single MSI/MSI-X device. 68 * The pic and MSI/MSI-X device are distinbuished by "devid". The "devid" 69 * is managed by below "dev_seqs". 70 */ 71 struct msipic { 72 int mp_bus; 73 int mp_dev; 74 int mp_fun; 75 76 int mp_devid; /* The device id for the MSI/MSI-X device. */ 77 int mp_veccnt; /* The number of MSI/MSI-X vectors. */ 78 79 char mp_pic_name[MSIPICNAMEBUF]; /* The MSI/MSI-X device's name. */ 80 81 struct pci_attach_args mp_pa; 82 bus_space_tag_t mp_bstag; 83 bus_space_handle_t mp_bshandle; 84 bus_size_t mp_bssize; 85 struct pic *mp_pic; 86 87 LIST_ENTRY(msipic) mp_list; 88 }; 89 90 static kmutex_t msipic_list_lock; 91 92 static LIST_HEAD(, msipic) msipic_list = 93 LIST_HEAD_INITIALIZER(msipic_list); 94 95 /* 96 * This struct managements "devid" to use the same "devid" for the device 97 * re-attached. If the device's bus number and device numer and function 98 * number are equal, it is assumed re-attached. 99 */ 100 struct dev_last_used_seq { 101 bool ds_using; 102 int ds_bus; 103 int ds_dev; 104 int ds_fun; 105 }; 106 /* The number of MSI/MSI-X devices supported by system. */ 107 #define NUM_MSI_DEVS 256 108 /* Record devids to use the same devid when the device is re-attached. */ 109 static struct dev_last_used_seq dev_seqs[NUM_MSI_DEVS]; 110 111 static int msipic_allocate_common_msi_devid(const struct pci_attach_args *); 112 static void msipic_release_common_msi_devid(int); 113 114 static struct pic *msipic_find_msi_pic_locked(int); 115 static struct pic *msipic_construct_common_msi_pic(const struct pci_attach_args *, 116 struct pic *); 117 static void msipic_destruct_common_msi_pic(struct pic *); 118 119 static void msi_set_msictl_enablebit(struct pic *, int, int); 120 static void msi_hwmask(struct pic *, int); 121 static void msi_hwunmask(struct pic *, int); 122 static void msi_addroute(struct pic *, struct cpu_info *, int, int, int); 123 static void msi_delroute(struct pic *, struct cpu_info *, int, int, int); 124 125 static void msix_set_vecctl_mask(struct pic *, int, int); 126 static void msix_hwmask(struct pic *, int); 127 static void msix_hwunmask(struct pic *, int); 128 static void msix_addroute(struct pic *, struct cpu_info *, int, int, int); 129 static void msix_delroute(struct pic *, struct cpu_info *, int, int, int); 130 131 /* 132 * Return new "devid" for the device attached first. 133 * Return the same "devid" for the device re-attached after dettached once. 134 * Return -1 if the number of attached MSI/MSI-X devices is over NUM_MSI_DEVS. 135 */ 136 static int 137 msipic_allocate_common_msi_devid(const struct pci_attach_args *pa) 138 { 139 pci_chipset_tag_t pc; 140 pcitag_t tag; 141 int bus, dev, fun, i; 142 143 KASSERT(mutex_owned(&msipic_list_lock)); 144 145 pc = pa->pa_pc; 146 tag = pa->pa_tag; 147 pci_decompose_tag(pc, tag, &bus, &dev, &fun); 148 149 /* if the device was once attached, use same devid */ 150 for (i = 0; i < NUM_MSI_DEVS; i++) { 151 /* skip host bridge */ 152 if (dev_seqs[i].ds_bus == 0 153 && dev_seqs[i].ds_dev == 0 154 && dev_seqs[i].ds_fun == 0) 155 break; 156 157 if (dev_seqs[i].ds_bus == bus 158 && dev_seqs[i].ds_dev == dev 159 && dev_seqs[i].ds_fun == fun) { 160 dev_seqs[i].ds_using = true; 161 return i; 162 } 163 } 164 165 for (i = 0; i < NUM_MSI_DEVS; i++) { 166 if (dev_seqs[i].ds_using == 0) { 167 dev_seqs[i].ds_using = true; 168 dev_seqs[i].ds_bus = bus; 169 dev_seqs[i].ds_dev = dev; 170 dev_seqs[i].ds_fun = fun; 171 return i; 172 } 173 } 174 175 DPRINTF(("too many MSI devices.\n")); 176 return -1; 177 } 178 179 /* 180 * Set the "devid" unused, but keep reserving the "devid" to reuse when 181 * the device is re-attached. 182 */ 183 static void 184 msipic_release_common_msi_devid(int devid) 185 { 186 187 KASSERT(mutex_owned(&msipic_list_lock)); 188 189 if (devid < 0 || NUM_MSI_DEVS <= devid) { 190 DPRINTF(("%s: invalid devid.\n", __func__)); 191 return; 192 } 193 194 dev_seqs[devid].ds_using = false; 195 /* Keep ds_* to reuse the same devid for the same device. */ 196 } 197 198 static struct pic * 199 msipic_find_msi_pic_locked(int devid) 200 { 201 struct msipic *mpp; 202 203 KASSERT(mutex_owned(&msipic_list_lock)); 204 205 LIST_FOREACH(mpp, &msipic_list, mp_list) { 206 if(mpp->mp_devid == devid) 207 return mpp->mp_pic; 208 } 209 return NULL; 210 } 211 212 /* 213 * Return the msi_pic whose device is already registered. 214 * If the device is not registered yet, return NULL. 215 */ 216 struct pic * 217 msipic_find_msi_pic(int devid) 218 { 219 struct pic *msipic; 220 221 mutex_enter(&msipic_list_lock); 222 msipic = msipic_find_msi_pic_locked(devid); 223 mutex_exit(&msipic_list_lock); 224 225 return msipic; 226 } 227 228 /* 229 * A common construct process of MSI and MSI-X. 230 */ 231 static struct pic * 232 msipic_construct_common_msi_pic(const struct pci_attach_args *pa, 233 struct pic *pic_tmpl) 234 { 235 struct pic *pic; 236 struct msipic *msipic; 237 int devid; 238 239 pic = kmem_alloc(sizeof(*pic), KM_SLEEP); 240 if (pic == NULL) 241 return NULL; 242 243 msipic = kmem_zalloc(sizeof(*msipic), KM_SLEEP); 244 if (msipic == NULL) { 245 kmem_free(pic, sizeof(*pic)); 246 return NULL; 247 } 248 249 mutex_enter(&msipic_list_lock); 250 251 devid = msipic_allocate_common_msi_devid(pa); 252 if (devid == -1) { 253 mutex_exit(&msipic_list_lock); 254 kmem_free(pic, sizeof(*pic)); 255 kmem_free(msipic, sizeof(*msipic)); 256 return NULL; 257 } 258 259 memcpy(pic, pic_tmpl, sizeof(*pic)); 260 pic->pic_msipic = msipic; 261 msipic->mp_pic = pic; 262 pci_decompose_tag(pa->pa_pc, pa->pa_tag, 263 &msipic->mp_bus, &msipic->mp_dev, &msipic->mp_fun); 264 memcpy(&msipic->mp_pa, pa, sizeof(msipic->mp_pa)); 265 msipic->mp_devid = devid; 266 /* 267 * pci_msi{,x}_alloc() must be called only once in the device driver. 268 */ 269 KASSERT(msipic_find_msi_pic_locked(msipic->mp_devid) == NULL); 270 271 LIST_INSERT_HEAD(&msipic_list, msipic, mp_list); 272 273 mutex_exit(&msipic_list_lock); 274 275 return pic; 276 } 277 278 static void 279 msipic_destruct_common_msi_pic(struct pic *msi_pic) 280 { 281 struct msipic *msipic; 282 283 if (msi_pic == NULL) 284 return; 285 286 msipic = msi_pic->pic_msipic; 287 mutex_enter(&msipic_list_lock); 288 LIST_REMOVE(msipic, mp_list); 289 msipic_release_common_msi_devid(msipic->mp_devid); 290 mutex_exit(&msipic_list_lock); 291 292 kmem_free(msipic, sizeof(*msipic)); 293 kmem_free(msi_pic, sizeof(*msi_pic)); 294 } 295 296 /* 297 * The pic is MSI/MSI-X pic or not. 298 */ 299 bool 300 msipic_is_msi_pic(struct pic *pic) 301 { 302 303 return (pic->pic_msipic != NULL); 304 } 305 306 /* 307 * Return the MSI/MSI-X devid which is unique for each devices. 308 */ 309 int 310 msipic_get_devid(struct pic *pic) 311 { 312 313 KASSERT(msipic_is_msi_pic(pic)); 314 315 return pic->pic_msipic->mp_devid; 316 } 317 318 #define MSI_MSICTL_ENABLE 1 319 #define MSI_MSICTL_DISABLE 0 320 static void 321 msi_set_msictl_enablebit(struct pic *pic, int msi_vec, int flag) 322 { 323 pci_chipset_tag_t pc; 324 struct pci_attach_args *pa; 325 pcitag_t tag; 326 pcireg_t ctl; 327 int off, err __diagused; 328 329 pc = NULL; 330 pa = &pic->pic_msipic->mp_pa; 331 tag = pa->pa_tag; 332 err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL); 333 KASSERT(err != 0); 334 335 /* 336 * MSI can establish only one vector at once. 337 * So, use whole device mask bit instead of a vector mask bit. 338 */ 339 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 340 if (flag == MSI_MSICTL_ENABLE) 341 ctl |= PCI_MSI_CTL_MSI_ENABLE; 342 else 343 ctl &= ~PCI_MSI_CTL_MSI_ENABLE; 344 345 pci_conf_write(pc, tag, off, ctl); 346 } 347 348 static void 349 msi_hwmask(struct pic *pic, int msi_vec) 350 { 351 352 msi_set_msictl_enablebit(pic, msi_vec, MSI_MSICTL_DISABLE); 353 } 354 355 /* 356 * Do not use pic->hwunmask() immediately after pic->delroute(). 357 * It is required to use pic->addroute() before pic->hwunmask(). 358 */ 359 static void 360 msi_hwunmask(struct pic *pic, int msi_vec) 361 { 362 363 msi_set_msictl_enablebit(pic, msi_vec, MSI_MSICTL_ENABLE); 364 } 365 366 static void 367 msi_addroute(struct pic *pic, struct cpu_info *ci, 368 int unused, int idt_vec, int type) 369 { 370 pci_chipset_tag_t pc; 371 struct pci_attach_args *pa; 372 pcitag_t tag; 373 pcireg_t addr, data, ctl; 374 int off, err __diagused; 375 376 pc = NULL; 377 pa = &pic->pic_msipic->mp_pa; 378 tag = pa->pa_tag; 379 err = pci_get_capability(pc, tag, PCI_CAP_MSI, &off, NULL); 380 KASSERT(err != 0); 381 382 /* 383 * See Intel 64 and IA-32 Architectures Software Developer's Manual 384 * Volume 3 10.11 Message Signalled Interrupts. 385 */ 386 /* 387 * "cpuid" for MSI address is local APIC ID. In NetBSD, the ID is 388 * the same as ci->ci_cpuid. 389 */ 390 addr = LAPIC_MSIADDR_BASE | __SHIFTIN(ci->ci_cpuid, 391 LAPIC_MSIADDR_DSTID_MASK); 392 /* If trigger mode is edge, it don't care level for trigger mode. */ 393 data = __SHIFTIN(idt_vec, LAPIC_MSIDATA_VECTOR_MASK) 394 | LAPIC_MSIDATA_TRGMODE_EDGE | LAPIC_MSIDATA_DM_FIXED; 395 396 ctl = pci_conf_read(pc, tag, off + PCI_MSI_CTL); 397 if (ctl & PCI_MSI_CTL_64BIT_ADDR) { 398 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_LO, addr); 399 pci_conf_write(pc, tag, off + PCI_MSI_MADDR64_HI, 0); 400 pci_conf_write(pc, tag, off + PCI_MSI_MDATA64, data); 401 } else { 402 pci_conf_write(pc, tag, off + PCI_MSI_MADDR, addr); 403 pci_conf_write(pc, tag, off + PCI_MSI_MDATA, data); 404 } 405 ctl |= PCI_MSI_CTL_MSI_ENABLE; 406 pci_conf_write(pc, tag, off + PCI_MSI_CTL, ctl); 407 } 408 409 /* 410 * Do not use pic->hwunmask() immediately after pic->delroute(). 411 * It is required to use pic->addroute() before pic->hwunmask(). 412 */ 413 static void 414 msi_delroute(struct pic *pic, struct cpu_info *ci, 415 int msi_vec, int idt_vec, int type) 416 { 417 418 msi_hwmask(pic, msi_vec); 419 } 420 421 /* 422 * Template for MSI pic. 423 * .pic_msipic is set later in construct_msi_pic(). 424 */ 425 static struct pic msi_pic_tmpl = { 426 .pic_type = PIC_MSI, 427 .pic_vecbase = 0, 428 .pic_apicid = 0, 429 .pic_lock = __SIMPLELOCK_UNLOCKED, /* not used for msi_pic */ 430 .pic_hwmask = msi_hwmask, 431 .pic_hwunmask = msi_hwunmask, 432 .pic_addroute = msi_addroute, 433 .pic_delroute = msi_delroute, 434 .pic_edge_stubs = ioapic_edge_stubs, 435 .pic_ioapic = NULL, 436 }; 437 438 /* 439 * Create pseudo pic for a MSI device. 440 */ 441 struct pic * 442 msipic_construct_msi_pic(const struct pci_attach_args *pa) 443 { 444 struct pic *msi_pic; 445 char pic_name_buf[MSIPICNAMEBUF]; 446 447 msi_pic = msipic_construct_common_msi_pic(pa, &msi_pic_tmpl); 448 if (msi_pic == NULL) { 449 DPRINTF(("cannot allocate MSI pic.\n")); 450 return NULL; 451 } 452 453 memset(pic_name_buf, 0, MSIPICNAMEBUF); 454 snprintf(pic_name_buf, MSIPICNAMEBUF, "msi%d", 455 msi_pic->pic_msipic->mp_devid); 456 strncpy(msi_pic->pic_msipic->mp_pic_name, pic_name_buf, 457 MSIPICNAMEBUF - 1); 458 msi_pic->pic_name = msi_pic->pic_msipic->mp_pic_name; 459 460 return msi_pic; 461 } 462 463 /* 464 * Delete pseudo pic for a MSI device. 465 */ 466 void 467 msipic_destruct_msi_pic(struct pic *msi_pic) 468 { 469 470 msipic_destruct_common_msi_pic(msi_pic); 471 } 472 473 #define MSIX_VECCTL_HWMASK 1 474 #define MSIX_VECCTL_HWUNMASK 0 475 static void 476 msix_set_vecctl_mask(struct pic *pic, int msix_vec, int flag) 477 { 478 bus_space_tag_t bstag; 479 bus_space_handle_t bshandle; 480 uint64_t entry_base; 481 uint32_t vecctl; 482 483 if (msix_vec < 0) { 484 DPRINTF(("%s: invalid MSI-X table index, devid=%d vecid=%d", 485 __func__, msipic_get_devid(pic), msix_vec)); 486 return; 487 } 488 489 entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec; 490 491 bstag = pic->pic_msipic->mp_bstag; 492 bshandle = pic->pic_msipic->mp_bshandle; 493 vecctl = bus_space_read_4(bstag, bshandle, 494 entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL); 495 if (flag == MSIX_VECCTL_HWMASK) 496 vecctl |= PCI_MSIX_VECTCTL_MASK; 497 else 498 vecctl &= ~PCI_MSIX_VECTCTL_MASK; 499 500 bus_space_write_4(bstag, bshandle, 501 entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, vecctl); 502 BUS_SPACE_WRITE_FLUSH(bstag, bshandle); 503 } 504 505 static void 506 msix_hwmask(struct pic *pic, int msix_vec) 507 { 508 509 msix_set_vecctl_mask(pic, msix_vec, MSIX_VECCTL_HWMASK); 510 } 511 512 /* 513 * Do not use pic->hwunmask() immediately after pic->delroute(). 514 * It is required to use pic->addroute() before pic->hwunmask(). 515 */ 516 static void 517 msix_hwunmask(struct pic *pic, int msix_vec) 518 { 519 520 msix_set_vecctl_mask(pic, msix_vec, MSIX_VECCTL_HWUNMASK); 521 } 522 523 static void 524 msix_addroute(struct pic *pic, struct cpu_info *ci, 525 int msix_vec, int idt_vec, int type) 526 { 527 pci_chipset_tag_t pc; 528 struct pci_attach_args *pa; 529 pcitag_t tag; 530 bus_space_tag_t bstag; 531 bus_space_handle_t bshandle; 532 uint64_t entry_base; 533 pcireg_t addr, data, ctl; 534 int off, err __diagused; 535 536 if (msix_vec < 0) { 537 DPRINTF(("%s: invalid MSI-X table index, devid=%d vecid=%d", 538 __func__, msipic_get_devid(pic), msix_vec)); 539 return; 540 } 541 542 pa = &pic->pic_msipic->mp_pa; 543 pc = pa->pa_pc; 544 tag = pa->pa_tag; 545 err = pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL); 546 KASSERT(err != 0); 547 548 entry_base = PCI_MSIX_TABLE_ENTRY_SIZE * msix_vec; 549 550 /* 551 * See Intel 64 and IA-32 Architectures Software Developer's Manual 552 * Volume 3 10.11 Message Signalled Interrupts. 553 */ 554 /* 555 * "cpuid" for MSI-X address is local APIC ID. In NetBSD, the ID is 556 * the same as ci->ci_cpuid. 557 */ 558 addr = LAPIC_MSIADDR_BASE | __SHIFTIN(ci->ci_cpuid, 559 LAPIC_MSIADDR_DSTID_MASK); 560 /* If trigger mode is edge, it don't care level for trigger mode. */ 561 data = __SHIFTIN(idt_vec, LAPIC_MSIDATA_VECTOR_MASK) 562 | LAPIC_MSIDATA_TRGMODE_EDGE | LAPIC_MSIDATA_DM_FIXED; 563 564 bstag = pic->pic_msipic->mp_bstag; 565 bshandle = pic->pic_msipic->mp_bshandle; 566 bus_space_write_4(bstag, bshandle, 567 entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_LO, addr); 568 bus_space_write_4(bstag, bshandle, 569 entry_base + PCI_MSIX_TABLE_ENTRY_ADDR_HI, 0); 570 bus_space_write_4(bstag, bshandle, 571 entry_base + PCI_MSIX_TABLE_ENTRY_DATA, data); 572 bus_space_write_4(bstag, bshandle, 573 entry_base + PCI_MSIX_TABLE_ENTRY_VECTCTL, 0); 574 BUS_SPACE_WRITE_FLUSH(bstag, bshandle); 575 576 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 577 ctl |= PCI_MSIX_CTL_ENABLE; 578 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 579 } 580 581 /* 582 * Do not use pic->hwunmask() immediately after pic->delroute(). 583 * It is required to use pic->addroute() before pic->hwunmask(). 584 */ 585 static void 586 msix_delroute(struct pic *pic, struct cpu_info *ci, 587 int msix_vec, int vec, int type) 588 { 589 590 msix_hwmask(pic, msix_vec); 591 } 592 593 /* 594 * Template for MSI-X pic. 595 * .pic_msipic is set later in construct_msix_pic(). 596 */ 597 static struct pic msix_pic_tmpl = { 598 .pic_type = PIC_MSIX, 599 .pic_vecbase = 0, 600 .pic_apicid = 0, 601 .pic_lock = __SIMPLELOCK_UNLOCKED, /* not used for msix_pic */ 602 .pic_hwmask = msix_hwmask, 603 .pic_hwunmask = msix_hwunmask, 604 .pic_addroute = msix_addroute, 605 .pic_delroute = msix_delroute, 606 .pic_edge_stubs = ioapic_edge_stubs, 607 }; 608 609 struct pic * 610 msipic_construct_msix_pic(const struct pci_attach_args *pa) 611 { 612 struct pic *msix_pic; 613 pci_chipset_tag_t pc; 614 pcitag_t tag; 615 pcireg_t tbl; 616 bus_space_tag_t bstag; 617 bus_space_handle_t bshandle; 618 bus_size_t bssize; 619 size_t table_size; 620 uint32_t table_offset; 621 u_int memtype; 622 bus_addr_t memaddr; 623 int flags; 624 int bir, bar, err, off, table_nentry; 625 char pic_name_buf[MSIPICNAMEBUF]; 626 627 table_nentry = pci_msix_count(pa->pa_pc, pa->pa_tag); 628 if (table_nentry == 0) { 629 DPRINTF(("MSI-X table entry is 0.\n")); 630 return NULL; 631 } 632 633 pc = pa->pa_pc; 634 tag = pa->pa_tag; 635 if (pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL) == 0) { 636 DPRINTF(("%s: no msix capability", __func__)); 637 return NULL; 638 } 639 640 msix_pic = msipic_construct_common_msi_pic(pa, &msix_pic_tmpl); 641 if (msix_pic == NULL) { 642 DPRINTF(("cannot allocate MSI-X pic.\n")); 643 return NULL; 644 } 645 646 memset(pic_name_buf, 0, MSIPICNAMEBUF); 647 snprintf(pic_name_buf, MSIPICNAMEBUF, "msix%d", 648 msix_pic->pic_msipic->mp_devid); 649 strncpy(msix_pic->pic_msipic->mp_pic_name, pic_name_buf, 650 MSIPICNAMEBUF - 1); 651 msix_pic->pic_name = msix_pic->pic_msipic->mp_pic_name; 652 653 tbl = pci_conf_read(pc, tag, off + PCI_MSIX_TBLOFFSET); 654 table_offset = tbl & PCI_MSIX_TBLOFFSET_MASK; 655 bir = tbl & PCI_MSIX_PBABIR_MASK; 656 switch(bir) { 657 case 0: 658 bar = PCI_BAR0; 659 break; 660 case 1: 661 bar = PCI_BAR1; 662 break; 663 case 2: 664 bar = PCI_BAR2; 665 break; 666 case 3: 667 bar = PCI_BAR3; 668 break; 669 case 4: 670 bar = PCI_BAR4; 671 break; 672 case 5: 673 bar = PCI_BAR5; 674 break; 675 default: 676 aprint_error("detect an illegal device! The device use reserved BIR values.\n"); 677 msipic_destruct_common_msi_pic(msix_pic); 678 return NULL; 679 } 680 memtype = pci_mapreg_type(pc, tag, bar); 681 /* 682 * PCI_MSIX_TABLE_ENTRY_SIZE consists below 683 * - Vector Control (32bit) 684 * - Message Data (32bit) 685 * - Message Upper Address (32bit) 686 * - Message Lower Address (32bit) 687 */ 688 table_size = table_nentry * PCI_MSIX_TABLE_ENTRY_SIZE; 689 #if 0 690 err = pci_mapreg_submap(pa, bar, memtype, BUS_SPACE_MAP_LINEAR, 691 roundup(table_size, PAGE_SIZE), table_offset, 692 &bstag, &bshandle, NULL, &bssize); 693 #else 694 /* 695 * Workaround for PCI prefetchable bit. Some chips (e.g. Intel 82599) 696 * report SERR and MSI-X doesn't work. This problem might not be the 697 * driver's bug but our PCI common part or VMs' bug. Until we find a 698 * real reason, we ignore the prefetchable bit. 699 */ 700 if (pci_mapreg_info(pa->pa_pc, pa->pa_tag, bar, memtype, 701 &memaddr, NULL, &flags) != 0) { 702 DPRINTF(("cannot get a map info.\n")); 703 msipic_destruct_common_msi_pic(msix_pic); 704 return NULL; 705 } 706 if ((flags & BUS_SPACE_MAP_PREFETCHABLE) != 0) { 707 DPRINTF(( "clear prefetchable bit\n")); 708 flags &= ~BUS_SPACE_MAP_PREFETCHABLE; 709 } 710 bssize = roundup(table_size, PAGE_SIZE); 711 err = bus_space_map(pa->pa_memt, memaddr + table_offset, bssize, flags, 712 &bshandle); 713 bstag = pa->pa_memt; 714 #endif 715 if (err) { 716 DPRINTF(("cannot map msix table.\n")); 717 msipic_destruct_common_msi_pic(msix_pic); 718 return NULL; 719 } 720 msix_pic->pic_msipic->mp_bstag = bstag; 721 msix_pic->pic_msipic->mp_bshandle = bshandle; 722 msix_pic->pic_msipic->mp_bssize = bssize; 723 724 return msix_pic; 725 } 726 727 /* 728 * Delete pseudo pic for a MSI-X device. 729 */ 730 void 731 msipic_destruct_msix_pic(struct pic *msix_pic) 732 { 733 struct msipic *msipic; 734 735 KASSERT(msipic_is_msi_pic(msix_pic)); 736 KASSERT(msix_pic->pic_type == PIC_MSIX); 737 738 msipic = msix_pic->pic_msipic; 739 bus_space_unmap(msipic->mp_bstag, msipic->mp_bshandle, 740 msipic->mp_bssize); 741 742 msipic_destruct_common_msi_pic(msix_pic); 743 } 744 745 /* 746 * Set the number of MSI vectors for pseudo MSI pic. 747 */ 748 int 749 msipic_set_msi_vectors(struct pic *msi_pic, pci_intr_handle_t *pihs, 750 int count) 751 { 752 753 KASSERT(msipic_is_msi_pic(msi_pic)); 754 755 msi_pic->pic_msipic->mp_veccnt = count; 756 return 0; 757 } 758 759 /* 760 * Initialize the system to use MSI/MSI-X. 761 */ 762 void 763 msipic_init(void) 764 { 765 766 mutex_init(&msipic_list_lock, MUTEX_DEFAULT, IPL_NONE); 767 } 768