1 /* $NetBSD: virtio_pci.c,v 1.7 2019/01/27 02:08:42 pgoyette Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: virtio_pci.c,v 1.7 2019/01/27 02:08:42 pgoyette Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kmem.h> 34 #include <sys/module.h> 35 #include <sys/interrupt.h> 36 37 #include <sys/device.h> 38 39 #include <dev/pci/pcidevs.h> 40 #include <dev/pci/pcireg.h> 41 #include <dev/pci/pcivar.h> 42 43 #define VIRTIO_PRIVATE 44 45 #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */ 46 #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */ 47 48 static int virtio_pci_match(device_t, cfdata_t, void *); 49 static void virtio_pci_attach(device_t, device_t, void *); 50 static int virtio_pci_rescan(device_t, const char *, const int *); 51 static int virtio_pci_detach(device_t, int); 52 53 struct virtio_pci_softc { 54 struct virtio_softc sc_sc; 55 bus_space_tag_t sc_iot; 56 bus_space_handle_t sc_ioh; 57 bus_size_t sc_iosize; 58 struct pci_attach_args sc_pa; 59 pci_intr_handle_t *sc_ihp; 60 void **sc_ihs; 61 int sc_ihs_num; 62 int sc_config_offset; 63 }; 64 65 static void virtio_pci_kick(struct virtio_softc *, uint16_t); 66 static uint8_t virtio_pci_read_device_config_1(struct virtio_softc *, int); 67 static uint16_t virtio_pci_read_device_config_2(struct virtio_softc *, int); 68 static uint32_t virtio_pci_read_device_config_4(struct virtio_softc *, int); 69 static uint64_t virtio_pci_read_device_config_8(struct virtio_softc *, int); 70 static void virtio_pci_write_device_config_1(struct virtio_softc *, int, uint8_t); 71 static void virtio_pci_write_device_config_2(struct virtio_softc *, int, uint16_t); 72 static void virtio_pci_write_device_config_4(struct virtio_softc *, int, uint32_t); 73 static void virtio_pci_write_device_config_8(struct virtio_softc *, int, uint64_t); 74 static uint16_t virtio_pci_read_queue_size(struct virtio_softc *, uint16_t); 75 static void virtio_pci_setup_queue(struct virtio_softc *, uint16_t, uint32_t); 76 static void virtio_pci_set_status(struct virtio_softc *, int); 77 static uint32_t virtio_pci_negotiate_features(struct virtio_softc *, uint32_t); 78 static int virtio_pci_setup_interrupts(struct virtio_softc *); 79 static void virtio_pci_free_interrupts(struct virtio_softc *); 80 81 static int virtio_pci_intr(void *arg); 82 static int virtio_pci_msix_queue_intr(void *); 83 static int virtio_pci_msix_vq_intr(void *); 84 static int virtio_pci_msix_config_intr(void *); 85 static int virtio_pci_setup_msix_vectors(struct virtio_softc *); 86 static int virtio_pci_setup_msix_interrupts(struct virtio_softc *, 87 struct pci_attach_args *); 88 static int virtio_pci_setup_intx_interrupt(struct virtio_softc *, 89 struct pci_attach_args *); 90 91 #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX 0 92 #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX 1 93 94 /* we use the legacy virtio spec, so the PCI registers are host native 95 * byte order, not PCI (i.e. LE) byte order */ 96 #if BYTE_ORDER == BIG_ENDIAN 97 #define REG_HI_OFF 0 98 #define REG_LO_OFF 4 99 #ifndef __BUS_SPACE_HAS_STREAM_METHODS 100 #define bus_space_read_stream_1 bus_space_read_1 101 #define bus_space_write_stream_1 bus_space_write_1 102 static inline uint16_t 103 bus_space_read_stream_2(bus_space_tag_t t, bus_space_handle_t h, 104 bus_size_t o) 105 { 106 return le16toh(bus_space_read_2(t, h, o)); 107 } 108 static inline void 109 bus_space_write_stream_2(bus_space_tag_t t, bus_space_handle_t h, 110 bus_size_t o, uint16_t v) 111 { 112 bus_space_write_2(t, h, o, htole16(v)); 113 } 114 static inline uint32_t 115 bus_space_read_stream_4(bus_space_tag_t t, bus_space_handle_t h, 116 bus_size_t o) 117 { 118 return le32toh(bus_space_read_4(t, h, o)); 119 } 120 static inline void 121 bus_space_write_stream_4(bus_space_tag_t t, bus_space_handle_t h, 122 bus_size_t o, uint32_t v) 123 { 124 bus_space_write_4(t, h, o, htole32(v)); 125 } 126 #endif 127 #else 128 #define REG_HI_OFF 4 129 #define REG_LO_OFF 0 130 #ifndef __BUS_SPACE_HAS_STREAM_METHODS 131 #define bus_space_read_stream_1 bus_space_read_1 132 #define bus_space_read_stream_2 bus_space_read_2 133 #define bus_space_read_stream_4 bus_space_read_4 134 #define bus_space_write_stream_1 bus_space_write_1 135 #define bus_space_write_stream_2 bus_space_write_2 136 #define bus_space_write_stream_4 bus_space_write_4 137 #endif 138 #endif 139 140 141 static const char *virtio_device_name[] = { 142 "Unknown (0)", /* 0 */ 143 "Network", /* 1 */ 144 "Block", /* 2 */ 145 "Console", /* 3 */ 146 "Entropy", /* 4 */ 147 "Memory Balloon", /* 5 */ 148 "I/O Memory", /* 6 */ 149 "Remote Processor Messaging", /* 7 */ 150 "SCSI", /* 8 */ 151 "9P Transport", /* 9 */ 152 "mac80211 wlan", /* 10 */ 153 }; 154 #define NDEVNAMES __arraycount(virtio_device_name) 155 156 CFATTACH_DECL3_NEW(virtio_pci, sizeof(struct virtio_pci_softc), 157 virtio_pci_match, virtio_pci_attach, virtio_pci_detach, NULL, 158 virtio_pci_rescan, NULL, DVF_DETACH_SHUTDOWN); 159 160 static const struct virtio_ops virtio_pci_ops = { 161 .kick = virtio_pci_kick, 162 .read_dev_cfg_1 = virtio_pci_read_device_config_1, 163 .read_dev_cfg_2 = virtio_pci_read_device_config_2, 164 .read_dev_cfg_4 = virtio_pci_read_device_config_4, 165 .read_dev_cfg_8 = virtio_pci_read_device_config_8, 166 .write_dev_cfg_1 = virtio_pci_write_device_config_1, 167 .write_dev_cfg_2 = virtio_pci_write_device_config_2, 168 .write_dev_cfg_4 = virtio_pci_write_device_config_4, 169 .write_dev_cfg_8 = virtio_pci_write_device_config_8, 170 .read_queue_size = virtio_pci_read_queue_size, 171 .setup_queue = virtio_pci_setup_queue, 172 .set_status = virtio_pci_set_status, 173 .neg_features = virtio_pci_negotiate_features, 174 .setup_interrupts = virtio_pci_setup_interrupts, 175 .free_interrupts = virtio_pci_free_interrupts, 176 }; 177 178 static int 179 virtio_pci_match(device_t parent, cfdata_t match, void *aux) 180 { 181 struct pci_attach_args *pa; 182 183 pa = (struct pci_attach_args *)aux; 184 switch (PCI_VENDOR(pa->pa_id)) { 185 case PCI_VENDOR_QUMRANET: 186 if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <= 187 PCI_PRODUCT(pa->pa_id)) && 188 (PCI_PRODUCT(pa->pa_id) <= 189 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) 190 return 1; 191 break; 192 } 193 194 return 0; 195 } 196 197 static void 198 virtio_pci_attach(device_t parent, device_t self, void *aux) 199 { 200 struct virtio_pci_softc * const psc = device_private(self); 201 struct virtio_softc * const sc = &psc->sc_sc; 202 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 203 pci_chipset_tag_t pc = pa->pa_pc; 204 pcitag_t tag = pa->pa_tag; 205 int revision; 206 pcireg_t id; 207 pcireg_t csr; 208 209 revision = PCI_REVISION(pa->pa_class); 210 if (revision != 0) { 211 aprint_normal(": unknown revision 0x%02x; giving up\n", 212 revision); 213 return; 214 } 215 aprint_normal("\n"); 216 aprint_naive("\n"); 217 218 /* subsystem ID shows what I am */ 219 id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 220 aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n", 221 (PCI_SUBSYS_ID(id) < NDEVNAMES? 222 virtio_device_name[PCI_SUBSYS_ID(id)] : "Unknown"), 223 revision); 224 225 csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); 226 csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_IO_ENABLE; 227 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr); 228 229 sc->sc_dev = self; 230 sc->sc_ops = &virtio_pci_ops; 231 psc->sc_pa = *pa; 232 psc->sc_iot = pa->pa_iot; 233 if (pci_dma64_available(pa)) 234 sc->sc_dmat = pa->pa_dmat64; 235 else 236 sc->sc_dmat = pa->pa_dmat; 237 psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 238 239 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, 240 &psc->sc_iot, &psc->sc_ioh, NULL, &psc->sc_iosize)) { 241 aprint_error_dev(self, "can't map i/o space\n"); 242 return; 243 } 244 245 virtio_device_reset(sc); 246 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 247 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 248 249 sc->sc_childdevid = PCI_SUBSYS_ID(id); 250 sc->sc_child = NULL; 251 virtio_pci_rescan(self, "virtio", 0); 252 return; 253 } 254 255 /* ARGSUSED */ 256 static int 257 virtio_pci_rescan(device_t self, const char *attr, const int *scan_flags) 258 { 259 struct virtio_pci_softc * const psc = device_private(self); 260 struct virtio_softc * const sc = &psc->sc_sc; 261 struct virtio_attach_args va; 262 263 if (sc->sc_child) /* Child already attached? */ 264 return 0; 265 266 memset(&va, 0, sizeof(va)); 267 va.sc_childdevid = sc->sc_childdevid; 268 269 config_found_ia(self, attr, &va, NULL); 270 271 if (sc->sc_child == NULL) { 272 aprint_error_dev(self, 273 "no matching child driver; not configured\n"); 274 return 0; 275 } 276 277 if (sc->sc_child == VIRTIO_CHILD_FAILED) { 278 aprint_error_dev(self, 279 "virtio configuration failed\n"); 280 return 0; 281 } 282 283 /* 284 * Make sure child drivers initialize interrupts via call 285 * to virtio_child_attach_finish(). 286 */ 287 KASSERT(psc->sc_ihs_num != 0); 288 289 return 0; 290 } 291 292 293 static int 294 virtio_pci_detach(device_t self, int flags) 295 { 296 struct virtio_pci_softc * const psc = device_private(self); 297 struct virtio_softc * const sc = &psc->sc_sc; 298 int r; 299 300 if (sc->sc_child != NULL) { 301 r = config_detach(sc->sc_child, flags); 302 if (r) 303 return r; 304 } 305 306 /* Check that child detached properly */ 307 KASSERT(sc->sc_child == NULL); 308 KASSERT(sc->sc_vqs == NULL); 309 KASSERT(psc->sc_ihs_num == 0); 310 311 if (psc->sc_iosize) 312 bus_space_unmap(psc->sc_iot, psc->sc_ioh, psc->sc_iosize); 313 psc->sc_iosize = 0; 314 315 return 0; 316 } 317 318 static void 319 virtio_pci_kick(struct virtio_softc *sc, uint16_t idx) 320 { 321 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 322 323 bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, 324 VIRTIO_CONFIG_QUEUE_NOTIFY, idx); 325 } 326 327 static uint8_t 328 virtio_pci_read_device_config_1(struct virtio_softc *sc, int index) 329 { 330 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 331 return bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh, 332 psc->sc_config_offset + index); 333 } 334 335 static uint16_t 336 virtio_pci_read_device_config_2(struct virtio_softc *sc, int index) 337 { 338 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 339 return bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, 340 psc->sc_config_offset + index); 341 } 342 343 static uint32_t 344 virtio_pci_read_device_config_4(struct virtio_softc *sc, int index) 345 { 346 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 347 return bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh, 348 psc->sc_config_offset + index); 349 } 350 351 static uint64_t 352 virtio_pci_read_device_config_8(struct virtio_softc *sc, int index) 353 { 354 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 355 uint64_t r; 356 357 r = bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh, 358 psc->sc_config_offset + index + REG_HI_OFF); 359 r <<= 32; 360 r |= bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh, 361 psc->sc_config_offset + index + REG_LO_OFF); 362 363 return r; 364 } 365 366 static void 367 virtio_pci_write_device_config_1(struct virtio_softc *sc, int index, 368 uint8_t value) 369 { 370 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 371 372 bus_space_write_stream_1(psc->sc_iot, psc->sc_ioh, 373 psc->sc_config_offset + index, value); 374 } 375 376 static void 377 virtio_pci_write_device_config_2(struct virtio_softc *sc, int index, 378 uint16_t value) 379 { 380 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 381 382 bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, 383 psc->sc_config_offset + index, value); 384 } 385 386 static void 387 virtio_pci_write_device_config_4(struct virtio_softc *sc, int index, 388 uint32_t value) 389 { 390 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 391 392 bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh, 393 psc->sc_config_offset + index, value); 394 } 395 396 static void 397 virtio_pci_write_device_config_8(struct virtio_softc *sc, int index, 398 uint64_t value) 399 { 400 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 401 402 bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh, 403 psc->sc_config_offset + index + REG_LO_OFF, 404 value & 0xffffffff); 405 bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh, 406 psc->sc_config_offset + index + REG_HI_OFF, 407 value >> 32); 408 } 409 410 static uint16_t 411 virtio_pci_read_queue_size(struct virtio_softc *sc, uint16_t idx) 412 { 413 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 414 415 bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, 416 VIRTIO_CONFIG_QUEUE_SELECT, idx); 417 return bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, 418 VIRTIO_CONFIG_QUEUE_SIZE); 419 } 420 421 static void 422 virtio_pci_setup_queue(struct virtio_softc *sc, uint16_t idx, uint32_t addr) 423 { 424 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 425 426 bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, 427 VIRTIO_CONFIG_QUEUE_SELECT, idx); 428 bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh, 429 VIRTIO_CONFIG_QUEUE_ADDRESS, addr); 430 431 if (psc->sc_ihs_num > 1) { 432 int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 433 if (sc->sc_child_mq) 434 vec += idx; 435 bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, 436 VIRTIO_CONFIG_MSI_QUEUE_VECTOR, vec); 437 } 438 } 439 440 static void 441 virtio_pci_set_status(struct virtio_softc *sc, int status) 442 { 443 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 444 int old = 0; 445 446 if (status != 0) { 447 old = bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh, 448 VIRTIO_CONFIG_DEVICE_STATUS); 449 } 450 bus_space_write_stream_1(psc->sc_iot, psc->sc_ioh, 451 VIRTIO_CONFIG_DEVICE_STATUS, status|old); 452 } 453 454 static uint32_t 455 virtio_pci_negotiate_features(struct virtio_softc *sc, uint32_t guest_features) 456 { 457 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 458 uint32_t r; 459 460 r = bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh, 461 VIRTIO_CONFIG_DEVICE_FEATURES); 462 r &= guest_features; 463 bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh, 464 VIRTIO_CONFIG_GUEST_FEATURES, r); 465 466 return r; 467 } 468 469 470 static int 471 virtio_pci_setup_msix_vectors(struct virtio_softc *sc) 472 { 473 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 474 int offset, vector, ret, qid; 475 476 offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR; 477 vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 478 479 bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, vector); 480 ret = bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, offset); 481 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 482 vector, ret); 483 if (ret != vector) 484 return -1; 485 486 for (qid = 0; qid < sc->sc_nvqs; qid++) { 487 offset = VIRTIO_CONFIG_QUEUE_SELECT; 488 bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, qid); 489 490 offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR; 491 vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 492 493 if (sc->sc_child_mq) 494 vector += qid; 495 496 bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, vector); 497 ret = bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, offset); 498 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 499 vector, ret); 500 if (ret != vector) 501 return -1; 502 } 503 504 return 0; 505 } 506 507 static int 508 virtio_pci_setup_msix_interrupts(struct virtio_softc *sc, 509 struct pci_attach_args *pa) 510 { 511 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 512 device_t self = sc->sc_dev; 513 pci_chipset_tag_t pc = pa->pa_pc; 514 char intrbuf[PCI_INTRSTR_LEN]; 515 char intr_xname[INTRDEVNAMEBUF]; 516 char const *intrstr; 517 int idx, qid, n; 518 519 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 520 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 521 pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 522 523 snprintf(intr_xname, sizeof(intr_xname), "%s config", 524 device_xname(sc->sc_dev)); 525 526 psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx], 527 sc->sc_ipl, virtio_pci_msix_config_intr, sc, intr_xname); 528 if (psc->sc_ihs[idx] == NULL) { 529 aprint_error_dev(self, "couldn't establish MSI-X for config\n"); 530 goto error; 531 } 532 533 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 534 if (sc->sc_child_mq) { 535 for (qid = 0; qid < sc->sc_nvqs; qid++) { 536 n = idx + qid; 537 538 snprintf(intr_xname, sizeof(intr_xname), "%s vq#%d", 539 device_xname(sc->sc_dev), qid); 540 541 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) { 542 pci_intr_setattr(pc, &psc->sc_ihp[n], 543 PCI_INTR_MPSAFE, true); 544 } 545 546 psc->sc_ihs[n] = pci_intr_establish_xname(pc, psc->sc_ihp[n], 547 sc->sc_ipl, virtio_pci_msix_vq_intr, &sc->sc_vqs[qid], 548 intr_xname); 549 if (psc->sc_ihs[n] == NULL) { 550 aprint_error_dev(self, "couldn't establish MSI-X for a vq\n"); 551 goto error; 552 } 553 } 554 } else { 555 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 556 pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 557 558 snprintf(intr_xname, sizeof(intr_xname), "%s queues", 559 device_xname(sc->sc_dev)); 560 psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx], 561 sc->sc_ipl, virtio_pci_msix_queue_intr, sc, intr_xname); 562 if (psc->sc_ihs[idx] == NULL) { 563 aprint_error_dev(self, "couldn't establish MSI-X for queues\n"); 564 goto error; 565 } 566 } 567 568 if (virtio_pci_setup_msix_vectors(sc) != 0) { 569 aprint_error_dev(self, "couldn't setup MSI-X vectors\n"); 570 goto error; 571 } 572 573 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 574 intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 575 aprint_normal_dev(self, "config interrupting at %s\n", intrstr); 576 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 577 if (sc->sc_child_mq) { 578 kcpuset_t *affinity; 579 int affinity_to, r; 580 581 kcpuset_create(&affinity, false); 582 583 for (qid = 0; qid < sc->sc_nvqs; qid++) { 584 n = idx + qid; 585 affinity_to = (qid / 2) % ncpu; 586 587 intrstr = pci_intr_string(pc, psc->sc_ihp[n], 588 intrbuf, sizeof(intrbuf)); 589 590 kcpuset_zero(affinity); 591 kcpuset_set(affinity, affinity_to); 592 r = interrupt_distribute(psc->sc_ihs[n], affinity, NULL); 593 if (r == 0) { 594 aprint_normal_dev(self, 595 "for vq #%d interrupting at %s affinity to %u\n", 596 qid, intrstr, affinity_to); 597 } else { 598 aprint_normal_dev(self, 599 "for vq #%d interrupting at %s\n", 600 qid, intrstr); 601 } 602 } 603 604 kcpuset_destroy(affinity); 605 } else { 606 intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 607 aprint_normal_dev(self, "queues interrupting at %s\n", intrstr); 608 } 609 610 return 0; 611 612 error: 613 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 614 if (psc->sc_ihs[idx] != NULL) 615 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]); 616 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 617 if (sc->sc_child_mq) { 618 for (qid = 0; qid < sc->sc_nvqs; qid++) { 619 n = idx + qid; 620 if (psc->sc_ihs[n] == NULL) 621 continue; 622 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[n]); 623 } 624 625 } else { 626 if (psc->sc_ihs[idx] != NULL) 627 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]); 628 } 629 630 return -1; 631 } 632 633 static int 634 virtio_pci_setup_intx_interrupt(struct virtio_softc *sc, 635 struct pci_attach_args *pa) 636 { 637 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 638 device_t self = sc->sc_dev; 639 pci_chipset_tag_t pc = pa->pa_pc; 640 char intrbuf[PCI_INTRSTR_LEN]; 641 char const *intrstr; 642 643 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 644 pci_intr_setattr(pc, &psc->sc_ihp[0], PCI_INTR_MPSAFE, true); 645 646 psc->sc_ihs[0] = pci_intr_establish_xname(pc, psc->sc_ihp[0], 647 sc->sc_ipl, virtio_pci_intr, sc, device_xname(sc->sc_dev)); 648 if (psc->sc_ihs[0] == NULL) { 649 aprint_error_dev(self, "couldn't establish INTx\n"); 650 return -1; 651 } 652 653 intrstr = pci_intr_string(pc, psc->sc_ihp[0], intrbuf, sizeof(intrbuf)); 654 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 655 656 return 0; 657 } 658 659 static int 660 virtio_pci_setup_interrupts(struct virtio_softc *sc) 661 { 662 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 663 device_t self = sc->sc_dev; 664 pci_chipset_tag_t pc = psc->sc_pa.pa_pc; 665 int error; 666 int nmsix; 667 int counts[PCI_INTR_TYPE_SIZE]; 668 pci_intr_type_t max_type; 669 670 nmsix = pci_msix_count(psc->sc_pa.pa_pc, psc->sc_pa.pa_tag); 671 aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix); 672 673 /* We need at least two: one for config and the other for queues */ 674 if ((sc->sc_flags & VIRTIO_F_PCI_INTR_MSIX) == 0 || nmsix < 2) { 675 /* Try INTx only */ 676 max_type = PCI_INTR_TYPE_INTX; 677 counts[PCI_INTR_TYPE_INTX] = 1; 678 } else { 679 /* Try MSI-X first and INTx second */ 680 if (sc->sc_child_mq && 681 sc->sc_nvqs > (nmsix - VIRTIO_MSIX_QUEUE_VECTOR_INDEX)) { 682 nmsix = 2; 683 sc->sc_child_mq = false; 684 } 685 686 max_type = PCI_INTR_TYPE_MSIX; 687 counts[PCI_INTR_TYPE_MSIX] = nmsix; 688 counts[PCI_INTR_TYPE_MSI] = 0; 689 counts[PCI_INTR_TYPE_INTX] = 1; 690 } 691 692 retry: 693 error = pci_intr_alloc(&psc->sc_pa, &psc->sc_ihp, counts, max_type); 694 if (error != 0) { 695 aprint_error_dev(self, "couldn't map interrupt\n"); 696 return -1; 697 } 698 699 if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) { 700 psc->sc_ihs = kmem_alloc(sizeof(*psc->sc_ihs) * nmsix, 701 KM_SLEEP); 702 703 error = virtio_pci_setup_msix_interrupts(sc, &psc->sc_pa); 704 if (error != 0) { 705 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * nmsix); 706 pci_intr_release(pc, psc->sc_ihp, nmsix); 707 708 /* Retry INTx */ 709 max_type = PCI_INTR_TYPE_INTX; 710 counts[PCI_INTR_TYPE_INTX] = 1; 711 goto retry; 712 } 713 714 psc->sc_ihs_num = nmsix; 715 psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI; 716 } else if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) { 717 psc->sc_ihs = kmem_alloc(sizeof(*psc->sc_ihs) * 1, 718 KM_SLEEP); 719 720 error = virtio_pci_setup_intx_interrupt(sc, &psc->sc_pa); 721 if (error != 0) { 722 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * 1); 723 pci_intr_release(pc, psc->sc_ihp, 1); 724 return -1; 725 } 726 727 psc->sc_ihs_num = 1; 728 psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 729 } 730 731 return 0; 732 } 733 734 static void 735 virtio_pci_free_interrupts(struct virtio_softc *sc) 736 { 737 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 738 739 for (int i = 0; i < psc->sc_ihs_num; i++) { 740 if (psc->sc_ihs[i] == NULL) 741 continue; 742 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[i]); 743 psc->sc_ihs[i] = NULL; 744 } 745 746 if (psc->sc_ihs_num > 0) 747 pci_intr_release(psc->sc_pa.pa_pc, psc->sc_ihp, psc->sc_ihs_num); 748 749 if (psc->sc_ihs != NULL) { 750 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * psc->sc_ihs_num); 751 psc->sc_ihs = NULL; 752 } 753 psc->sc_ihs_num = 0; 754 } 755 756 /* 757 * Interrupt handler. 758 */ 759 static int 760 virtio_pci_intr(void *arg) 761 { 762 struct virtio_softc *sc = arg; 763 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 764 int isr, r = 0; 765 766 /* check and ack the interrupt */ 767 isr = bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh, 768 VIRTIO_CONFIG_ISR_STATUS); 769 if (isr == 0) 770 return 0; 771 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 772 (sc->sc_config_change != NULL)) 773 r = (sc->sc_config_change)(sc); 774 if (sc->sc_intrhand != NULL) { 775 if (sc->sc_soft_ih != NULL) 776 softint_schedule(sc->sc_soft_ih); 777 else 778 r |= (sc->sc_intrhand)(sc); 779 } 780 781 return r; 782 } 783 784 static int 785 virtio_pci_msix_queue_intr(void *arg) 786 { 787 struct virtio_softc *sc = arg; 788 int r = 0; 789 790 if (sc->sc_intrhand != NULL) { 791 if (sc->sc_soft_ih != NULL) 792 softint_schedule(sc->sc_soft_ih); 793 else 794 r |= (sc->sc_intrhand)(sc); 795 } 796 797 return r; 798 } 799 800 static int 801 virtio_pci_msix_vq_intr(void *arg) 802 { 803 struct virtqueue *vq = arg; 804 int r = 0; 805 806 if (vq->vq_intrhand != NULL) { 807 if (vq->vq_soft_ih) 808 softint_schedule(vq->vq_soft_ih); 809 else 810 r |= vq->vq_intrhand(vq); 811 } 812 813 return r; 814 } 815 816 static int 817 virtio_pci_msix_config_intr(void *arg) 818 { 819 struct virtio_softc *sc = arg; 820 int r = 0; 821 822 if (sc->sc_config_change != NULL) 823 r = (sc->sc_config_change)(sc); 824 return r; 825 } 826 827 MODULE(MODULE_CLASS_DRIVER, virtio_pci, "pci,virtio"); 828 829 #ifdef _MODULE 830 #include "ioconf.c" 831 #endif 832 833 static int 834 virtio_pci_modcmd(modcmd_t cmd, void *opaque) 835 { 836 int error = 0; 837 838 #ifdef _MODULE 839 switch (cmd) { 840 case MODULE_CMD_INIT: 841 error = config_init_component(cfdriver_ioconf_virtio_pci, 842 cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci); 843 break; 844 case MODULE_CMD_FINI: 845 error = config_fini_component(cfdriver_ioconf_virtio_pci, 846 cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci); 847 break; 848 default: 849 error = ENOTTY; 850 break; 851 } 852 #endif 853 854 return error; 855 } 856