1 /* $NetBSD: virtio_pci.c,v 1.13 2020/09/17 17:33:50 jakllsch Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: virtio_pci.c,v 1.13 2020/09/17 17:33:50 jakllsch Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kmem.h> 34 #include <sys/module.h> 35 #include <sys/interrupt.h> 36 37 #include <sys/device.h> 38 39 #include <dev/pci/pcidevs.h> 40 #include <dev/pci/pcireg.h> 41 #include <dev/pci/pcivar.h> 42 43 #define VIRTIO_PRIVATE 44 45 #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */ 46 #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */ 47 48 static int virtio_pci_match(device_t, cfdata_t, void *); 49 static void virtio_pci_attach(device_t, device_t, void *); 50 static int virtio_pci_rescan(device_t, const char *, const int *); 51 static int virtio_pci_detach(device_t, int); 52 53 struct virtio_pci_softc { 54 struct virtio_softc sc_sc; 55 bus_space_tag_t sc_iot; 56 bus_space_handle_t sc_ioh; 57 bus_size_t sc_iosize; 58 struct pci_attach_args sc_pa; 59 pci_intr_handle_t *sc_ihp; 60 void **sc_ihs; 61 int sc_ihs_num; 62 int sc_config_offset; 63 }; 64 65 static void virtio_pci_kick(struct virtio_softc *, uint16_t); 66 static uint8_t virtio_pci_read_device_config_1(struct virtio_softc *, int); 67 static uint16_t virtio_pci_read_device_config_2(struct virtio_softc *, int); 68 static uint32_t virtio_pci_read_device_config_4(struct virtio_softc *, int); 69 static uint64_t virtio_pci_read_device_config_8(struct virtio_softc *, int); 70 static void virtio_pci_write_device_config_1(struct virtio_softc *, int, uint8_t); 71 static void virtio_pci_write_device_config_2(struct virtio_softc *, int, uint16_t); 72 static void virtio_pci_write_device_config_4(struct virtio_softc *, int, uint32_t); 73 static void virtio_pci_write_device_config_8(struct virtio_softc *, int, uint64_t); 74 static uint16_t virtio_pci_read_queue_size(struct virtio_softc *, uint16_t); 75 static void virtio_pci_setup_queue(struct virtio_softc *, uint16_t, uint32_t); 76 static void virtio_pci_set_status(struct virtio_softc *, int); 77 static uint32_t virtio_pci_negotiate_features(struct virtio_softc *, uint32_t); 78 static int virtio_pci_setup_interrupts(struct virtio_softc *); 79 static void virtio_pci_free_interrupts(struct virtio_softc *); 80 81 static int virtio_pci_intr(void *arg); 82 static int virtio_pci_msix_queue_intr(void *); 83 static int virtio_pci_msix_config_intr(void *); 84 static int virtio_pci_setup_msix_vectors(struct virtio_softc *); 85 static int virtio_pci_setup_msix_interrupts(struct virtio_softc *, 86 struct pci_attach_args *); 87 static int virtio_pci_setup_intx_interrupt(struct virtio_softc *, 88 struct pci_attach_args *); 89 90 #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX 0 91 #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX 1 92 93 /* we use the legacy virtio spec, so the PCI registers are host native 94 * byte order, not PCI (i.e. LE) byte order */ 95 #if BYTE_ORDER == BIG_ENDIAN 96 #define REG_HI_OFF 0 97 #define REG_LO_OFF 4 98 #ifndef __BUS_SPACE_HAS_STREAM_METHODS 99 #define bus_space_read_stream_1 bus_space_read_1 100 #define bus_space_write_stream_1 bus_space_write_1 101 static inline uint16_t 102 bus_space_read_stream_2(bus_space_tag_t t, bus_space_handle_t h, 103 bus_size_t o) 104 { 105 return le16toh(bus_space_read_2(t, h, o)); 106 } 107 static inline void 108 bus_space_write_stream_2(bus_space_tag_t t, bus_space_handle_t h, 109 bus_size_t o, uint16_t v) 110 { 111 bus_space_write_2(t, h, o, htole16(v)); 112 } 113 static inline uint32_t 114 bus_space_read_stream_4(bus_space_tag_t t, bus_space_handle_t h, 115 bus_size_t o) 116 { 117 return le32toh(bus_space_read_4(t, h, o)); 118 } 119 static inline void 120 bus_space_write_stream_4(bus_space_tag_t t, bus_space_handle_t h, 121 bus_size_t o, uint32_t v) 122 { 123 bus_space_write_4(t, h, o, htole32(v)); 124 } 125 #endif 126 #else 127 #define REG_HI_OFF 4 128 #define REG_LO_OFF 0 129 #ifndef __BUS_SPACE_HAS_STREAM_METHODS 130 #define bus_space_read_stream_1 bus_space_read_1 131 #define bus_space_read_stream_2 bus_space_read_2 132 #define bus_space_read_stream_4 bus_space_read_4 133 #define bus_space_write_stream_1 bus_space_write_1 134 #define bus_space_write_stream_2 bus_space_write_2 135 #define bus_space_write_stream_4 bus_space_write_4 136 #endif 137 #endif 138 139 140 static const char *virtio_device_name[] = { 141 "Unknown (0)", /* 0 */ 142 "Network", /* 1 */ 143 "Block", /* 2 */ 144 "Console", /* 3 */ 145 "Entropy", /* 4 */ 146 "Memory Balloon", /* 5 */ 147 "I/O Memory", /* 6 */ 148 "Remote Processor Messaging", /* 7 */ 149 "SCSI", /* 8 */ 150 "9P Transport", /* 9 */ 151 "mac80211 wlan", /* 10 */ 152 }; 153 #define NDEVNAMES __arraycount(virtio_device_name) 154 155 CFATTACH_DECL3_NEW(virtio_pci, sizeof(struct virtio_pci_softc), 156 virtio_pci_match, virtio_pci_attach, virtio_pci_detach, NULL, 157 virtio_pci_rescan, NULL, DVF_DETACH_SHUTDOWN); 158 159 static const struct virtio_ops virtio_pci_ops = { 160 .kick = virtio_pci_kick, 161 .read_dev_cfg_1 = virtio_pci_read_device_config_1, 162 .read_dev_cfg_2 = virtio_pci_read_device_config_2, 163 .read_dev_cfg_4 = virtio_pci_read_device_config_4, 164 .read_dev_cfg_8 = virtio_pci_read_device_config_8, 165 .write_dev_cfg_1 = virtio_pci_write_device_config_1, 166 .write_dev_cfg_2 = virtio_pci_write_device_config_2, 167 .write_dev_cfg_4 = virtio_pci_write_device_config_4, 168 .write_dev_cfg_8 = virtio_pci_write_device_config_8, 169 .read_queue_size = virtio_pci_read_queue_size, 170 .setup_queue = virtio_pci_setup_queue, 171 .set_status = virtio_pci_set_status, 172 .neg_features = virtio_pci_negotiate_features, 173 .setup_interrupts = virtio_pci_setup_interrupts, 174 .free_interrupts = virtio_pci_free_interrupts, 175 }; 176 177 static int 178 virtio_pci_match(device_t parent, cfdata_t match, void *aux) 179 { 180 struct pci_attach_args *pa; 181 182 pa = (struct pci_attach_args *)aux; 183 switch (PCI_VENDOR(pa->pa_id)) { 184 case PCI_VENDOR_QUMRANET: 185 if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <= 186 PCI_PRODUCT(pa->pa_id)) && 187 (PCI_PRODUCT(pa->pa_id) <= 188 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) 189 return 1; 190 break; 191 } 192 193 return 0; 194 } 195 196 static void 197 virtio_pci_attach(device_t parent, device_t self, void *aux) 198 { 199 struct virtio_pci_softc * const psc = device_private(self); 200 struct virtio_softc * const sc = &psc->sc_sc; 201 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 202 pci_chipset_tag_t pc = pa->pa_pc; 203 pcitag_t tag = pa->pa_tag; 204 int revision; 205 pcireg_t id; 206 pcireg_t csr; 207 208 revision = PCI_REVISION(pa->pa_class); 209 if (revision != 0) { 210 aprint_normal(": unknown revision 0x%02x; giving up\n", 211 revision); 212 return; 213 } 214 aprint_normal("\n"); 215 aprint_naive("\n"); 216 217 /* subsystem ID shows what I am */ 218 id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 219 aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n", 220 (PCI_SUBSYS_ID(id) < NDEVNAMES? 221 virtio_device_name[PCI_SUBSYS_ID(id)] : "Unknown"), 222 revision); 223 224 csr = pci_conf_read(pc, tag, PCI_COMMAND_STATUS_REG); 225 csr |= PCI_COMMAND_MASTER_ENABLE | PCI_COMMAND_IO_ENABLE; 226 pci_conf_write(pc, tag, PCI_COMMAND_STATUS_REG, csr); 227 228 sc->sc_dev = self; 229 sc->sc_ops = &virtio_pci_ops; 230 psc->sc_pa = *pa; 231 psc->sc_iot = pa->pa_iot; 232 if (pci_dma64_available(pa)) 233 sc->sc_dmat = pa->pa_dmat64; 234 else 235 sc->sc_dmat = pa->pa_dmat; 236 psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 237 238 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, 239 &psc->sc_iot, &psc->sc_ioh, NULL, &psc->sc_iosize)) { 240 aprint_error_dev(self, "can't map i/o space\n"); 241 return; 242 } 243 244 virtio_device_reset(sc); 245 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 246 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 247 248 sc->sc_childdevid = PCI_SUBSYS_ID(id); 249 sc->sc_child = NULL; 250 virtio_pci_rescan(self, "virtio", 0); 251 return; 252 } 253 254 /* ARGSUSED */ 255 static int 256 virtio_pci_rescan(device_t self, const char *attr, const int *scan_flags) 257 { 258 struct virtio_pci_softc * const psc = device_private(self); 259 struct virtio_softc * const sc = &psc->sc_sc; 260 struct virtio_attach_args va; 261 262 if (sc->sc_child) /* Child already attached? */ 263 return 0; 264 265 memset(&va, 0, sizeof(va)); 266 va.sc_childdevid = sc->sc_childdevid; 267 268 config_found_ia(self, attr, &va, NULL); 269 270 if (sc->sc_child == NULL) { 271 aprint_error_dev(self, 272 "no matching child driver; not configured\n"); 273 return 0; 274 } 275 276 if (sc->sc_child == VIRTIO_CHILD_FAILED) { 277 aprint_error_dev(self, 278 "virtio configuration failed\n"); 279 return 0; 280 } 281 282 /* 283 * Make sure child drivers initialize interrupts via call 284 * to virtio_child_attach_finish(). 285 */ 286 KASSERT(psc->sc_ihs_num != 0); 287 288 return 0; 289 } 290 291 292 static int 293 virtio_pci_detach(device_t self, int flags) 294 { 295 struct virtio_pci_softc * const psc = device_private(self); 296 struct virtio_softc * const sc = &psc->sc_sc; 297 int r; 298 299 if (sc->sc_child != NULL) { 300 r = config_detach(sc->sc_child, flags); 301 if (r) 302 return r; 303 } 304 305 /* Check that child detached properly */ 306 KASSERT(sc->sc_child == NULL); 307 KASSERT(sc->sc_vqs == NULL); 308 KASSERT(psc->sc_ihs_num == 0); 309 310 if (psc->sc_iosize) 311 bus_space_unmap(psc->sc_iot, psc->sc_ioh, psc->sc_iosize); 312 psc->sc_iosize = 0; 313 314 return 0; 315 } 316 317 static void 318 virtio_pci_kick(struct virtio_softc *sc, uint16_t idx) 319 { 320 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 321 322 bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, 323 VIRTIO_CONFIG_QUEUE_NOTIFY, idx); 324 } 325 326 static uint8_t 327 virtio_pci_read_device_config_1(struct virtio_softc *sc, int index) 328 { 329 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 330 return bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh, 331 psc->sc_config_offset + index); 332 } 333 334 static uint16_t 335 virtio_pci_read_device_config_2(struct virtio_softc *sc, int index) 336 { 337 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 338 return bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, 339 psc->sc_config_offset + index); 340 } 341 342 static uint32_t 343 virtio_pci_read_device_config_4(struct virtio_softc *sc, int index) 344 { 345 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 346 return bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh, 347 psc->sc_config_offset + index); 348 } 349 350 static uint64_t 351 virtio_pci_read_device_config_8(struct virtio_softc *sc, int index) 352 { 353 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 354 uint64_t r; 355 356 r = bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh, 357 psc->sc_config_offset + index + REG_HI_OFF); 358 r <<= 32; 359 r |= bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh, 360 psc->sc_config_offset + index + REG_LO_OFF); 361 362 return r; 363 } 364 365 static void 366 virtio_pci_write_device_config_1(struct virtio_softc *sc, int index, 367 uint8_t value) 368 { 369 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 370 371 bus_space_write_stream_1(psc->sc_iot, psc->sc_ioh, 372 psc->sc_config_offset + index, value); 373 } 374 375 static void 376 virtio_pci_write_device_config_2(struct virtio_softc *sc, int index, 377 uint16_t value) 378 { 379 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 380 381 bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, 382 psc->sc_config_offset + index, value); 383 } 384 385 static void 386 virtio_pci_write_device_config_4(struct virtio_softc *sc, int index, 387 uint32_t value) 388 { 389 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 390 391 bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh, 392 psc->sc_config_offset + index, value); 393 } 394 395 static void 396 virtio_pci_write_device_config_8(struct virtio_softc *sc, int index, 397 uint64_t value) 398 { 399 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 400 401 bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh, 402 psc->sc_config_offset + index + REG_LO_OFF, 403 value & 0xffffffff); 404 bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh, 405 psc->sc_config_offset + index + REG_HI_OFF, 406 value >> 32); 407 } 408 409 static uint16_t 410 virtio_pci_read_queue_size(struct virtio_softc *sc, uint16_t idx) 411 { 412 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 413 414 bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, 415 VIRTIO_CONFIG_QUEUE_SELECT, idx); 416 return bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, 417 VIRTIO_CONFIG_QUEUE_SIZE); 418 } 419 420 static void 421 virtio_pci_setup_queue(struct virtio_softc *sc, uint16_t idx, uint32_t addr) 422 { 423 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 424 425 bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, 426 VIRTIO_CONFIG_QUEUE_SELECT, idx); 427 bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh, 428 VIRTIO_CONFIG_QUEUE_ADDRESS, addr); 429 430 if (psc->sc_ihs_num > 1) { 431 int vec = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 432 if (sc->sc_child_mq) 433 vec += idx; 434 bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, 435 VIRTIO_CONFIG_MSI_QUEUE_VECTOR, vec); 436 } 437 } 438 439 static void 440 virtio_pci_set_status(struct virtio_softc *sc, int status) 441 { 442 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 443 int old = 0; 444 445 if (status != 0) { 446 old = bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh, 447 VIRTIO_CONFIG_DEVICE_STATUS); 448 } 449 bus_space_write_stream_1(psc->sc_iot, psc->sc_ioh, 450 VIRTIO_CONFIG_DEVICE_STATUS, status|old); 451 } 452 453 static uint32_t 454 virtio_pci_negotiate_features(struct virtio_softc *sc, uint32_t guest_features) 455 { 456 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 457 uint32_t r; 458 459 r = bus_space_read_stream_4(psc->sc_iot, psc->sc_ioh, 460 VIRTIO_CONFIG_DEVICE_FEATURES); 461 r &= guest_features; 462 bus_space_write_stream_4(psc->sc_iot, psc->sc_ioh, 463 VIRTIO_CONFIG_GUEST_FEATURES, r); 464 465 return r; 466 } 467 468 469 static int 470 virtio_pci_setup_msix_vectors(struct virtio_softc *sc) 471 { 472 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 473 int offset, vector, ret, qid; 474 475 offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR; 476 vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 477 478 bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, vector); 479 ret = bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, offset); 480 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 481 vector, ret); 482 if (ret != vector) 483 return -1; 484 485 for (qid = 0; qid < sc->sc_nvqs; qid++) { 486 offset = VIRTIO_CONFIG_QUEUE_SELECT; 487 bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, qid); 488 489 offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR; 490 vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 491 492 if (sc->sc_child_mq) 493 vector += qid; 494 495 bus_space_write_stream_2(psc->sc_iot, psc->sc_ioh, offset, vector); 496 ret = bus_space_read_stream_2(psc->sc_iot, psc->sc_ioh, offset); 497 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 498 vector, ret); 499 if (ret != vector) 500 return -1; 501 } 502 503 return 0; 504 } 505 506 static int 507 virtio_pci_setup_msix_interrupts(struct virtio_softc *sc, 508 struct pci_attach_args *pa) 509 { 510 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 511 device_t self = sc->sc_dev; 512 pci_chipset_tag_t pc = pa->pa_pc; 513 struct virtqueue *vq; 514 char intrbuf[PCI_INTRSTR_LEN]; 515 char intr_xname[INTRDEVNAMEBUF]; 516 char const *intrstr; 517 int idx, qid, n; 518 519 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 520 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 521 pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 522 523 snprintf(intr_xname, sizeof(intr_xname), "%s config", 524 device_xname(sc->sc_dev)); 525 526 psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx], 527 sc->sc_ipl, virtio_pci_msix_config_intr, sc, intr_xname); 528 if (psc->sc_ihs[idx] == NULL) { 529 aprint_error_dev(self, "couldn't establish MSI-X for config\n"); 530 goto error; 531 } 532 533 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 534 if (sc->sc_child_mq) { 535 for (qid = 0; qid < sc->sc_nvqs; qid++) { 536 n = idx + qid; 537 vq = &sc->sc_vqs[qid]; 538 539 snprintf(intr_xname, sizeof(intr_xname), "%s vq#%d", 540 device_xname(sc->sc_dev), qid); 541 542 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) { 543 pci_intr_setattr(pc, &psc->sc_ihp[n], 544 PCI_INTR_MPSAFE, true); 545 } 546 547 psc->sc_ihs[n] = pci_intr_establish_xname(pc, psc->sc_ihp[n], 548 sc->sc_ipl, vq->vq_intrhand, vq->vq_intrhand_arg, intr_xname); 549 if (psc->sc_ihs[n] == NULL) { 550 aprint_error_dev(self, "couldn't establish MSI-X for a vq\n"); 551 goto error; 552 } 553 } 554 } else { 555 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 556 pci_intr_setattr(pc, &psc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 557 558 snprintf(intr_xname, sizeof(intr_xname), "%s queues", 559 device_xname(sc->sc_dev)); 560 psc->sc_ihs[idx] = pci_intr_establish_xname(pc, psc->sc_ihp[idx], 561 sc->sc_ipl, virtio_pci_msix_queue_intr, sc, intr_xname); 562 if (psc->sc_ihs[idx] == NULL) { 563 aprint_error_dev(self, "couldn't establish MSI-X for queues\n"); 564 goto error; 565 } 566 } 567 568 if (virtio_pci_setup_msix_vectors(sc) != 0) { 569 aprint_error_dev(self, "couldn't setup MSI-X vectors\n"); 570 goto error; 571 } 572 573 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 574 intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 575 aprint_normal_dev(self, "config interrupting at %s\n", intrstr); 576 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 577 if (sc->sc_child_mq) { 578 kcpuset_t *affinity; 579 int affinity_to, r; 580 581 kcpuset_create(&affinity, false); 582 583 for (qid = 0; qid < sc->sc_nvqs; qid++) { 584 n = idx + qid; 585 affinity_to = (qid / 2) % ncpu; 586 587 intrstr = pci_intr_string(pc, psc->sc_ihp[n], 588 intrbuf, sizeof(intrbuf)); 589 590 kcpuset_zero(affinity); 591 kcpuset_set(affinity, affinity_to); 592 r = interrupt_distribute(psc->sc_ihs[n], affinity, NULL); 593 if (r == 0) { 594 aprint_normal_dev(self, 595 "for vq #%d interrupting at %s affinity to %u\n", 596 qid, intrstr, affinity_to); 597 } else { 598 aprint_normal_dev(self, 599 "for vq #%d interrupting at %s\n", 600 qid, intrstr); 601 } 602 } 603 604 kcpuset_destroy(affinity); 605 } else { 606 intrstr = pci_intr_string(pc, psc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 607 aprint_normal_dev(self, "queues interrupting at %s\n", intrstr); 608 } 609 610 return 0; 611 612 error: 613 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 614 if (psc->sc_ihs[idx] != NULL) 615 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]); 616 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 617 if (sc->sc_child_mq) { 618 for (qid = 0; qid < sc->sc_nvqs; qid++) { 619 n = idx + qid; 620 if (psc->sc_ihs[n] == NULL) 621 continue; 622 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[n]); 623 } 624 625 } else { 626 if (psc->sc_ihs[idx] != NULL) 627 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[idx]); 628 } 629 630 return -1; 631 } 632 633 static int 634 virtio_pci_setup_intx_interrupt(struct virtio_softc *sc, 635 struct pci_attach_args *pa) 636 { 637 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 638 device_t self = sc->sc_dev; 639 pci_chipset_tag_t pc = pa->pa_pc; 640 char intrbuf[PCI_INTRSTR_LEN]; 641 char const *intrstr; 642 643 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 644 pci_intr_setattr(pc, &psc->sc_ihp[0], PCI_INTR_MPSAFE, true); 645 646 psc->sc_ihs[0] = pci_intr_establish_xname(pc, psc->sc_ihp[0], 647 sc->sc_ipl, virtio_pci_intr, sc, device_xname(sc->sc_dev)); 648 if (psc->sc_ihs[0] == NULL) { 649 aprint_error_dev(self, "couldn't establish INTx\n"); 650 return -1; 651 } 652 653 intrstr = pci_intr_string(pc, psc->sc_ihp[0], intrbuf, sizeof(intrbuf)); 654 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 655 656 return 0; 657 } 658 659 static int 660 virtio_pci_setup_interrupts(struct virtio_softc *sc) 661 { 662 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 663 device_t self = sc->sc_dev; 664 pci_chipset_tag_t pc = psc->sc_pa.pa_pc; 665 pcitag_t tag = psc->sc_pa.pa_tag; 666 int error; 667 int nmsix; 668 int off; 669 int counts[PCI_INTR_TYPE_SIZE]; 670 pci_intr_type_t max_type; 671 pcireg_t ctl; 672 673 nmsix = pci_msix_count(psc->sc_pa.pa_pc, psc->sc_pa.pa_tag); 674 aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix); 675 676 /* We need at least two: one for config and the other for queues */ 677 if ((sc->sc_flags & VIRTIO_F_PCI_INTR_MSIX) == 0 || nmsix < 2) { 678 /* Try INTx only */ 679 max_type = PCI_INTR_TYPE_INTX; 680 counts[PCI_INTR_TYPE_INTX] = 1; 681 } else { 682 /* Try MSI-X first and INTx second */ 683 if (sc->sc_nvqs + VIRTIO_MSIX_QUEUE_VECTOR_INDEX <= nmsix) { 684 nmsix = sc->sc_nvqs + VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 685 } else { 686 sc->sc_child_mq = false; 687 } 688 689 if (sc->sc_child_mq == false) { 690 nmsix = 2; 691 } 692 693 max_type = PCI_INTR_TYPE_MSIX; 694 counts[PCI_INTR_TYPE_MSIX] = nmsix; 695 counts[PCI_INTR_TYPE_MSI] = 0; 696 counts[PCI_INTR_TYPE_INTX] = 1; 697 } 698 699 retry: 700 error = pci_intr_alloc(&psc->sc_pa, &psc->sc_ihp, counts, max_type); 701 if (error != 0) { 702 aprint_error_dev(self, "couldn't map interrupt\n"); 703 return -1; 704 } 705 706 if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) { 707 psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * nmsix, 708 KM_SLEEP); 709 710 error = virtio_pci_setup_msix_interrupts(sc, &psc->sc_pa); 711 if (error != 0) { 712 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * nmsix); 713 pci_intr_release(pc, psc->sc_ihp, nmsix); 714 715 /* Retry INTx */ 716 max_type = PCI_INTR_TYPE_INTX; 717 counts[PCI_INTR_TYPE_INTX] = 1; 718 goto retry; 719 } 720 721 psc->sc_ihs_num = nmsix; 722 psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI; 723 } else if (pci_intr_type(pc, psc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) { 724 psc->sc_ihs = kmem_zalloc(sizeof(*psc->sc_ihs) * 1, 725 KM_SLEEP); 726 727 error = virtio_pci_setup_intx_interrupt(sc, &psc->sc_pa); 728 if (error != 0) { 729 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * 1); 730 pci_intr_release(pc, psc->sc_ihp, 1); 731 return -1; 732 } 733 734 psc->sc_ihs_num = 1; 735 psc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 736 737 error = pci_get_capability(pc, tag, PCI_CAP_MSIX, &off, NULL); 738 if (error != 0) { 739 ctl = pci_conf_read(pc, tag, off + PCI_MSIX_CTL); 740 ctl &= ~PCI_MSIX_CTL_ENABLE; 741 pci_conf_write(pc, tag, off + PCI_MSIX_CTL, ctl); 742 } 743 } 744 745 return 0; 746 } 747 748 static void 749 virtio_pci_free_interrupts(struct virtio_softc *sc) 750 { 751 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 752 753 for (int i = 0; i < psc->sc_ihs_num; i++) { 754 if (psc->sc_ihs[i] == NULL) 755 continue; 756 pci_intr_disestablish(psc->sc_pa.pa_pc, psc->sc_ihs[i]); 757 psc->sc_ihs[i] = NULL; 758 } 759 760 if (psc->sc_ihs_num > 0) 761 pci_intr_release(psc->sc_pa.pa_pc, psc->sc_ihp, psc->sc_ihs_num); 762 763 if (psc->sc_ihs != NULL) { 764 kmem_free(psc->sc_ihs, sizeof(*psc->sc_ihs) * psc->sc_ihs_num); 765 psc->sc_ihs = NULL; 766 } 767 psc->sc_ihs_num = 0; 768 } 769 770 /* 771 * Interrupt handler. 772 */ 773 static int 774 virtio_pci_intr(void *arg) 775 { 776 struct virtio_softc *sc = arg; 777 struct virtio_pci_softc * const psc = (struct virtio_pci_softc *)sc; 778 int isr, r = 0; 779 780 /* check and ack the interrupt */ 781 isr = bus_space_read_stream_1(psc->sc_iot, psc->sc_ioh, 782 VIRTIO_CONFIG_ISR_STATUS); 783 if (isr == 0) 784 return 0; 785 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 786 (sc->sc_config_change != NULL)) 787 r = (sc->sc_config_change)(sc); 788 if (sc->sc_intrhand != NULL) { 789 if (sc->sc_soft_ih != NULL) 790 softint_schedule(sc->sc_soft_ih); 791 else 792 r |= (sc->sc_intrhand)(sc); 793 } 794 795 return r; 796 } 797 798 static int 799 virtio_pci_msix_queue_intr(void *arg) 800 { 801 struct virtio_softc *sc = arg; 802 int r = 0; 803 804 if (sc->sc_intrhand != NULL) { 805 if (sc->sc_soft_ih != NULL) 806 softint_schedule(sc->sc_soft_ih); 807 else 808 r |= (sc->sc_intrhand)(sc); 809 } 810 811 return r; 812 } 813 814 static int 815 virtio_pci_msix_config_intr(void *arg) 816 { 817 struct virtio_softc *sc = arg; 818 int r = 0; 819 820 if (sc->sc_config_change != NULL) 821 r = (sc->sc_config_change)(sc); 822 return r; 823 } 824 825 MODULE(MODULE_CLASS_DRIVER, virtio_pci, "pci,virtio"); 826 827 #ifdef _MODULE 828 #include "ioconf.c" 829 #endif 830 831 static int 832 virtio_pci_modcmd(modcmd_t cmd, void *opaque) 833 { 834 int error = 0; 835 836 #ifdef _MODULE 837 switch (cmd) { 838 case MODULE_CMD_INIT: 839 error = config_init_component(cfdriver_ioconf_virtio_pci, 840 cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci); 841 break; 842 case MODULE_CMD_FINI: 843 error = config_fini_component(cfdriver_ioconf_virtio_pci, 844 cfattach_ioconf_virtio_pci, cfdata_ioconf_virtio_pci); 845 break; 846 default: 847 error = ENOTTY; 848 break; 849 } 850 #endif 851 852 return error; 853 } 854