1 /*- 2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * $FreeBSD: src/sys/dev/virtio/pci/virtio_pci.c,v 1.3 2012/04/14 05:48:04 grehan Exp $ 27 */ 28 29 /* Driver for the VirtIO PCI interface. */ 30 31 #include <sys/cdefs.h> 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/bus.h> 36 #include <sys/kernel.h> 37 #include <sys/module.h> 38 #include <sys/malloc.h> 39 40 #include <bus/pci/pcivar.h> 41 #include <bus/pci/pcireg.h> 42 43 #include <sys/bus.h> 44 #include <sys/param.h> 45 #include <sys/rman.h> 46 47 #include <dev/virtual/virtio/virtio/virtio.h> 48 #include <dev/virtual/virtio/virtio/virtqueue.h> 49 #include "virtio_pci.h" 50 #include "virtio_if.h" 51 #include "virtio_bus_if.h" 52 53 struct vtpci_softc { 54 device_t vtpci_dev; 55 struct resource *vtpci_res; 56 struct resource *vtpci_msix_res; 57 uint64_t vtpci_features; 58 uint32_t vtpci_flags; 59 int vtpci_irq_type; 60 int vtpci_irq_rid; 61 #define VIRTIO_PCI_FLAG_NO_MSI 0x0001 62 #define VIRTIO_PCI_FLAG_MSI 0x0002 63 #define VIRTIO_PCI_FLAG_NO_MSIX 0x0010 64 #define VIRTIO_PCI_FLAG_MSIX 0x0020 65 #define VIRTIO_PCI_FLAG_SHARED_MSIX 0x0040 66 67 device_t vtpci_child_dev; 68 struct virtio_feature_desc *vtpci_child_feat_desc; 69 70 /* 71 * Ideally, each virtqueue that the driver provides a callback for 72 * will receive its own MSIX vector. If there are not sufficient 73 * vectors available, we will then attempt to have all the VQs 74 * share one vector. Note that when using MSIX, the configuration 75 * changed notifications must be on their own vector. 76 * 77 * If MSIX is not available, we will attempt to have the whole 78 * device share one MSI vector, and then, finally, one legacy 79 * interrupt. 80 */ 81 int vtpci_nvqs; 82 struct vtpci_virtqueue { 83 struct virtqueue *vq; 84 85 /* Index into vtpci_intr_res[] below. Unused, then -1. */ 86 int ires_idx; 87 } vtpci_vqx[VIRTIO_MAX_VIRTQUEUES]; 88 89 /* 90 * When using MSIX interrupts, the first element of vtpci_intr_res[] 91 * is always the configuration changed notifications. The remaining 92 * element(s) are used for the virtqueues. 93 * 94 * With MSI and legacy interrupts, only the first element of 95 * vtpci_intr_res[] is used. 96 */ 97 int vtpci_nintr_res; 98 struct vtpci_intr_resource { 99 struct resource *irq; 100 int rid; 101 void *intrhand; 102 } vtpci_intr_res[1 + VIRTIO_MAX_VIRTQUEUES]; 103 }; 104 105 static int vtpci_probe(device_t); 106 static int vtpci_attach(device_t); 107 static int vtpci_detach(device_t); 108 static int vtpci_suspend(device_t); 109 static int vtpci_resume(device_t); 110 static int vtpci_shutdown(device_t); 111 static void vtpci_driver_added(device_t, driver_t *); 112 static void vtpci_child_detached(device_t, device_t); 113 static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *); 114 static int vtpci_write_ivar(device_t, device_t, int, uintptr_t); 115 116 static uint64_t vtpci_negotiate_features(device_t, uint64_t); 117 static int vtpci_with_feature(device_t, uint64_t); 118 static int vtpci_alloc_virtqueues(device_t, int, int, 119 struct vq_alloc_info *); 120 static int vtpci_setup_intr(device_t); 121 static void vtpci_stop(device_t); 122 static int vtpci_reinit(device_t, uint64_t); 123 static void vtpci_reinit_complete(device_t); 124 static void vtpci_notify_virtqueue(device_t, uint16_t); 125 static uint8_t vtpci_get_status(device_t); 126 static void vtpci_set_status(device_t, uint8_t); 127 static void vtpci_read_dev_config(device_t, bus_size_t, void *, int); 128 static void vtpci_write_dev_config(device_t, bus_size_t, void *, int); 129 130 static void vtpci_describe_features(struct vtpci_softc *, const char *, 131 uint64_t); 132 static void vtpci_probe_and_attach_child(struct vtpci_softc *); 133 134 static int vtpci_alloc_interrupts(struct vtpci_softc *, int, int, 135 struct vq_alloc_info *); 136 static int vtpci_alloc_intr_resources(struct vtpci_softc *, int, 137 struct vq_alloc_info *); 138 static int vtpci_alloc_msi(struct vtpci_softc *); 139 static int vtpci_alloc_msix(struct vtpci_softc *, int); 140 static int vtpci_register_msix_vector(struct vtpci_softc *, int, int); 141 142 static void vtpci_free_interrupts(struct vtpci_softc *); 143 static void vtpci_free_virtqueues(struct vtpci_softc *); 144 static void vtpci_release_child_resources(struct vtpci_softc *); 145 static void vtpci_reset(struct vtpci_softc *); 146 147 static int vtpci_legacy_intr(void *); 148 static int vtpci_vq_shared_intr(void *); 149 static int vtpci_vq_intr(void *); 150 static int vtpci_config_intr(void *); 151 152 /* 153 * I/O port read/write wrappers. 154 */ 155 #define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o)) 156 #define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o)) 157 #define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o)) 158 #define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v)) 159 #define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v)) 160 #define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v)) 161 162 /* Tunables. */ 163 static int vtpci_disable_msix = 0; 164 TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix); 165 166 static device_method_t vtpci_methods[] = { 167 /* Device interface. */ 168 DEVMETHOD(device_probe, vtpci_probe), 169 DEVMETHOD(device_attach, vtpci_attach), 170 DEVMETHOD(device_detach, vtpci_detach), 171 DEVMETHOD(device_suspend, vtpci_suspend), 172 DEVMETHOD(device_resume, vtpci_resume), 173 DEVMETHOD(device_shutdown, vtpci_shutdown), 174 175 /* Bus interface. */ 176 DEVMETHOD(bus_driver_added, vtpci_driver_added), 177 DEVMETHOD(bus_child_detached, vtpci_child_detached), 178 DEVMETHOD(bus_read_ivar, vtpci_read_ivar), 179 DEVMETHOD(bus_write_ivar, vtpci_write_ivar), 180 181 /* VirtIO bus interface. */ 182 DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features), 183 DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature), 184 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues), 185 DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr), 186 DEVMETHOD(virtio_bus_stop, vtpci_stop), 187 DEVMETHOD(virtio_bus_reinit, vtpci_reinit), 188 DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete), 189 DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue), 190 DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config), 191 DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config), 192 193 { 0, 0 } 194 }; 195 196 static driver_t vtpci_driver = { 197 "virtio_pci", 198 vtpci_methods, 199 sizeof(struct vtpci_softc) 200 }; 201 202 devclass_t vtpci_devclass; 203 204 DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0); 205 MODULE_VERSION(virtio_pci, 1); 206 MODULE_DEPEND(virtio_pci, pci, 1, 1, 1); 207 MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1); 208 209 static int 210 vtpci_probe(device_t dev) 211 { 212 char desc[36]; 213 const char *name; 214 215 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) 216 return (ENXIO); 217 218 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || 219 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX) 220 return (ENXIO); 221 222 if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION) 223 return (ENXIO); 224 225 name = virtio_device_name(pci_get_subdevice(dev)); 226 if (name == NULL) 227 name = "Unknown"; 228 229 ksnprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name); 230 device_set_desc_copy(dev, desc); 231 232 return (BUS_PROBE_DEFAULT); 233 } 234 235 static int 236 vtpci_attach(device_t dev) 237 { 238 struct vtpci_softc *sc; 239 device_t child; 240 int rid; 241 242 sc = device_get_softc(dev); 243 sc->vtpci_dev = dev; 244 245 pci_enable_busmaster(dev); 246 247 rid = PCIR_BAR(0); 248 sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, 249 RF_ACTIVE); 250 if (sc->vtpci_res == NULL) { 251 device_printf(dev, "cannot map I/O space\n"); 252 return (ENXIO); 253 } 254 255 if (pci_find_extcap(dev, PCIY_MSI, NULL) != 0) 256 sc->vtpci_flags |= VIRTIO_PCI_FLAG_NO_MSI; 257 /* XXX(vsrinivas): Check out how to get MSI-X */ 258 #if OLD_MSI 259 if (pci_find_extcap(dev, PCIY_MSIX, NULL) == 0) { 260 rid = PCIR_BAR(1); 261 sc->vtpci_msix_res = bus_alloc_resource_any(dev, 262 SYS_RES_MEMORY, &rid, RF_ACTIVE); 263 } 264 #endif 265 if (sc->vtpci_msix_res == NULL) 266 sc->vtpci_flags |= VIRTIO_PCI_FLAG_NO_MSIX; 267 268 vtpci_reset(sc); 269 270 /* Tell the host we've noticed this device. */ 271 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 272 273 if ((child = device_add_child(dev, NULL, -1)) == NULL) { 274 device_printf(dev, "cannot create child device\n"); 275 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 276 vtpci_detach(dev); 277 return (ENOMEM); 278 } 279 280 sc->vtpci_child_dev = child; 281 vtpci_probe_and_attach_child(sc); 282 283 return (0); 284 } 285 286 static int 287 vtpci_detach(device_t dev) 288 { 289 struct vtpci_softc *sc; 290 device_t child; 291 int error; 292 293 sc = device_get_softc(dev); 294 295 if ((child = sc->vtpci_child_dev) != NULL) { 296 error = device_delete_child(dev, child); 297 if (error) 298 return (error); 299 sc->vtpci_child_dev = NULL; 300 } 301 302 vtpci_reset(sc); 303 304 if (sc->vtpci_msix_res != NULL) { 305 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1), 306 sc->vtpci_msix_res); 307 sc->vtpci_msix_res = NULL; 308 } 309 310 if (sc->vtpci_res != NULL) { 311 bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0), 312 sc->vtpci_res); 313 sc->vtpci_res = NULL; 314 } 315 316 return (0); 317 } 318 319 static int 320 vtpci_suspend(device_t dev) 321 { 322 323 return (bus_generic_suspend(dev)); 324 } 325 326 static int 327 vtpci_resume(device_t dev) 328 { 329 330 return (bus_generic_resume(dev)); 331 } 332 333 static int 334 vtpci_shutdown(device_t dev) 335 { 336 337 (void) bus_generic_shutdown(dev); 338 /* Forcibly stop the host device. */ 339 vtpci_stop(dev); 340 341 return (0); 342 } 343 344 static void 345 vtpci_driver_added(device_t dev, driver_t *driver) 346 { 347 struct vtpci_softc *sc; 348 349 sc = device_get_softc(dev); 350 351 vtpci_probe_and_attach_child(sc); 352 } 353 354 static void 355 vtpci_child_detached(device_t dev, device_t child) 356 { 357 struct vtpci_softc *sc; 358 359 sc = device_get_softc(dev); 360 361 vtpci_reset(sc); 362 vtpci_release_child_resources(sc); 363 } 364 365 static int 366 vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) 367 { 368 struct vtpci_softc *sc; 369 370 sc = device_get_softc(dev); 371 372 if (sc->vtpci_child_dev != child) 373 return (ENOENT); 374 375 switch (index) { 376 case VIRTIO_IVAR_DEVTYPE: 377 *result = pci_get_subdevice(dev); 378 break; 379 default: 380 return (ENOENT); 381 } 382 383 return (0); 384 } 385 386 static int 387 vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value) 388 { 389 struct vtpci_softc *sc; 390 391 sc = device_get_softc(dev); 392 393 if (sc->vtpci_child_dev != child) 394 return (ENOENT); 395 396 switch (index) { 397 case VIRTIO_IVAR_FEATURE_DESC: 398 sc->vtpci_child_feat_desc = (void *) value; 399 break; 400 default: 401 return (ENOENT); 402 } 403 404 return (0); 405 } 406 407 static uint64_t 408 vtpci_negotiate_features(device_t dev, uint64_t child_features) 409 { 410 struct vtpci_softc *sc; 411 uint64_t host_features, features; 412 413 sc = device_get_softc(dev); 414 415 host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES); 416 vtpci_describe_features(sc, "host", host_features); 417 418 /* 419 * Limit negotiated features to what the driver, virtqueue, and 420 * host all support. 421 */ 422 features = host_features & child_features; 423 features = virtqueue_filter_features(features); 424 sc->vtpci_features = features; 425 426 vtpci_describe_features(sc, "negotiated", features); 427 vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features); 428 429 return (features); 430 } 431 432 static int 433 vtpci_with_feature(device_t dev, uint64_t feature) 434 { 435 struct vtpci_softc *sc; 436 437 sc = device_get_softc(dev); 438 439 return ((sc->vtpci_features & feature) != 0); 440 } 441 442 static int 443 vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs, 444 struct vq_alloc_info *vq_info) 445 { 446 struct vtpci_softc *sc; 447 struct vtpci_virtqueue *vqx; 448 struct vq_alloc_info *info; 449 int queue, error; 450 uint16_t vq_size; 451 452 sc = device_get_softc(dev); 453 454 if (sc->vtpci_nvqs != 0 || nvqs <= 0 || 455 nvqs > VIRTIO_MAX_VIRTQUEUES) 456 return (EINVAL); 457 458 error = vtpci_alloc_interrupts(sc, flags, nvqs, vq_info); 459 if (error) { 460 device_printf(dev, "cannot allocate interrupts\n"); 461 return (error); 462 } 463 464 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) { 465 error = vtpci_register_msix_vector(sc, 466 VIRTIO_MSI_CONFIG_VECTOR, 0); 467 if (error) 468 return (error); 469 } 470 471 for (queue = 0; queue < nvqs; queue++) { 472 vqx = &sc->vtpci_vqx[queue]; 473 info = &vq_info[queue]; 474 475 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, queue); 476 477 vq_size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); 478 error = virtqueue_alloc(dev, queue, vq_size, 479 VIRTIO_PCI_VRING_ALIGN, 0xFFFFFFFFUL, info, &vqx->vq); 480 if (error) 481 return (error); 482 483 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) { 484 error = vtpci_register_msix_vector(sc, 485 VIRTIO_MSI_QUEUE_VECTOR, vqx->ires_idx); 486 if (error) 487 return (error); 488 } 489 490 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 491 virtqueue_paddr(vqx->vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 492 493 *info->vqai_vq = vqx->vq; 494 sc->vtpci_nvqs++; 495 } 496 497 return (0); 498 } 499 500 static int 501 vtpci_setup_intr(device_t dev) 502 { 503 struct vtpci_softc *sc; 504 struct vtpci_intr_resource *ires; 505 struct vtpci_virtqueue *vqx; 506 int i, flags, error; 507 508 sc = device_get_softc(dev); 509 flags = INTR_MPSAFE; 510 ires = &sc->vtpci_intr_res[0]; 511 512 if ((sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) == 0) { 513 error = bus_setup_intr(dev, ires->irq, flags, 514 (driver_intr_t *) vtpci_legacy_intr, sc, &ires->intrhand, NULL); 515 516 return (error); 517 } 518 519 error = bus_setup_intr(dev, ires->irq, flags,(driver_intr_t *) vtpci_config_intr, 520 sc, &ires->intrhand, NULL); 521 if (error) 522 return (error); 523 524 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_SHARED_MSIX) { 525 ires = &sc->vtpci_intr_res[1]; 526 error = bus_setup_intr(dev, ires->irq, flags, 527 (driver_intr_t *) vtpci_vq_shared_intr, sc, &ires->intrhand, NULL); 528 529 return (error); 530 } 531 532 /* Setup an interrupt handler for each virtqueue. */ 533 for (i = 0; i < sc->vtpci_nvqs; i++) { 534 vqx = &sc->vtpci_vqx[i]; 535 if (vqx->ires_idx < 1) 536 continue; 537 538 ires = &sc->vtpci_intr_res[vqx->ires_idx]; 539 error = bus_setup_intr(dev, ires->irq, flags, 540 (driver_intr_t *) vtpci_vq_intr, vqx->vq, &ires->intrhand, NULL); 541 if (error) 542 return (error); 543 } 544 545 return (0); 546 } 547 548 static void 549 vtpci_stop(device_t dev) 550 { 551 552 vtpci_reset(device_get_softc(dev)); 553 } 554 555 static int 556 vtpci_reinit(device_t dev, uint64_t features) 557 { 558 struct vtpci_softc *sc; 559 struct vtpci_virtqueue *vqx; 560 struct virtqueue *vq; 561 int queue, error; 562 uint16_t vq_size; 563 564 sc = device_get_softc(dev); 565 566 /* 567 * Redrive the device initialization. This is a bit of an abuse 568 * of the specification, but both VirtualBox and QEMU/KVM seem 569 * to play nice. We do not allow the host device to change from 570 * what was originally negotiated beyond what the guest driver 571 * changed (MSIX state should not change, number of virtqueues 572 * and their size remain the same, etc). 573 */ 574 575 if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) 576 vtpci_stop(dev); 577 578 /* 579 * Quickly drive the status through ACK and DRIVER. The device 580 * does not become usable again until vtpci_reinit_complete(). 581 */ 582 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 583 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 584 585 vtpci_negotiate_features(dev, features); 586 587 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) { 588 error = vtpci_register_msix_vector(sc, 589 VIRTIO_MSI_CONFIG_VECTOR, 0); 590 if (error) 591 return (error); 592 } 593 594 for (queue = 0; queue < sc->vtpci_nvqs; queue++) { 595 vqx = &sc->vtpci_vqx[queue]; 596 vq = vqx->vq; 597 598 KASSERT(vq != NULL, ("vq %d not allocated", queue)); 599 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, queue); 600 601 vq_size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); 602 error = virtqueue_reinit(vq, vq_size); 603 if (error) 604 return (error); 605 606 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) { 607 error = vtpci_register_msix_vector(sc, 608 VIRTIO_MSI_QUEUE_VECTOR, vqx->ires_idx); 609 if (error) 610 return (error); 611 } 612 613 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 614 virtqueue_paddr(vqx->vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 615 } 616 617 return (0); 618 } 619 620 static void 621 vtpci_reinit_complete(device_t dev) 622 { 623 624 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 625 } 626 627 static void 628 vtpci_notify_virtqueue(device_t dev, uint16_t queue) 629 { 630 struct vtpci_softc *sc; 631 632 sc = device_get_softc(dev); 633 634 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue); 635 } 636 637 static uint8_t 638 vtpci_get_status(device_t dev) 639 { 640 struct vtpci_softc *sc; 641 642 sc = device_get_softc(dev); 643 644 return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS)); 645 } 646 647 static void 648 vtpci_set_status(device_t dev, uint8_t status) 649 { 650 struct vtpci_softc *sc; 651 652 sc = device_get_softc(dev); 653 654 if (status != VIRTIO_CONFIG_STATUS_RESET) 655 status |= vtpci_get_status(dev); 656 657 vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status); 658 } 659 660 static void 661 vtpci_read_dev_config(device_t dev, bus_size_t offset, 662 void *dst, int length) 663 { 664 struct vtpci_softc *sc; 665 bus_size_t off; 666 uint8_t *d; 667 int size; 668 669 sc = device_get_softc(dev); 670 off = VIRTIO_PCI_CONFIG(sc) + offset; 671 672 for (d = dst; length > 0; d += size, off += size, length -= size) { 673 if (length >= 4) { 674 size = 4; 675 *(uint32_t *)d = vtpci_read_config_4(sc, off); 676 } else if (length >= 2) { 677 size = 2; 678 *(uint16_t *)d = vtpci_read_config_2(sc, off); 679 } else { 680 size = 1; 681 *d = vtpci_read_config_1(sc, off); 682 } 683 } 684 } 685 686 static void 687 vtpci_write_dev_config(device_t dev, bus_size_t offset, 688 void *src, int length) 689 { 690 struct vtpci_softc *sc; 691 bus_size_t off; 692 uint8_t *s; 693 int size; 694 695 sc = device_get_softc(dev); 696 off = VIRTIO_PCI_CONFIG(sc) + offset; 697 698 for (s = src; length > 0; s += size, off += size, length -= size) { 699 if (length >= 4) { 700 size = 4; 701 vtpci_write_config_4(sc, off, *(uint32_t *)s); 702 } else if (length >= 2) { 703 size = 2; 704 vtpci_write_config_2(sc, off, *(uint16_t *)s); 705 } else { 706 size = 1; 707 vtpci_write_config_1(sc, off, *s); 708 } 709 } 710 } 711 712 static void 713 vtpci_describe_features(struct vtpci_softc *sc, const char *msg, 714 uint64_t features) 715 { 716 device_t dev, child; 717 718 dev = sc->vtpci_dev; 719 child = sc->vtpci_child_dev; 720 721 if (device_is_attached(child) && bootverbose == 0) 722 return; 723 724 virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc); 725 } 726 727 static void 728 vtpci_probe_and_attach_child(struct vtpci_softc *sc) 729 { 730 device_t dev, child; 731 732 dev = sc->vtpci_dev; 733 child = sc->vtpci_child_dev; 734 735 if (child == NULL) 736 return; 737 738 if (device_get_state(child) != DS_NOTPRESENT) 739 return; 740 741 if (device_probe_child(dev, child) != 0) 742 return; 743 744 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 745 if (DEVICE_ATTACH(child) != 0) { 746 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 747 vtpci_reset(sc); 748 vtpci_release_child_resources(sc); 749 750 /* Reset status for future attempt. */ 751 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 752 } else 753 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 754 } 755 756 static int 757 vtpci_alloc_interrupts(struct vtpci_softc *sc, int flags, int nvqs, 758 struct vq_alloc_info *vq_info) 759 { 760 int i, nvectors, error; 761 762 /* 763 * Only allocate a vector for virtqueues that are actually 764 * expecting an interrupt. 765 */ 766 for (nvectors = 0, i = 0; i < nvqs; i++) 767 if (vq_info[i].vqai_intr != NULL) 768 nvectors++; 769 770 if (vtpci_disable_msix != 0 || 771 sc->vtpci_flags & VIRTIO_PCI_FLAG_NO_MSIX || 772 flags & VIRTIO_ALLOC_VQS_DISABLE_MSIX || 773 vtpci_alloc_msix(sc, nvectors) != 0) { 774 /* 775 * Use MSI interrupts if available. Otherwise, we fallback 776 * to legacy interrupts. 777 */ 778 if ((sc->vtpci_flags & VIRTIO_PCI_FLAG_NO_MSI) == 0 && 779 vtpci_alloc_msi(sc) == 0) 780 sc->vtpci_flags |= VIRTIO_PCI_FLAG_MSI; 781 782 sc->vtpci_nintr_res = 1; 783 } 784 785 error = vtpci_alloc_intr_resources(sc, nvqs, vq_info); 786 787 return (error); 788 } 789 790 static int 791 vtpci_alloc_intr_resources(struct vtpci_softc *sc, int nvqs, 792 struct vq_alloc_info *vq_info) 793 { 794 device_t dev; 795 struct resource *irq; 796 struct vtpci_virtqueue *vqx; 797 int i, rid, flags, res_idx; 798 799 dev = sc->vtpci_dev; 800 flags = RF_ACTIVE; 801 802 if ((sc->vtpci_flags & 803 (VIRTIO_PCI_FLAG_MSI | VIRTIO_PCI_FLAG_MSIX)) == 0) { 804 rid = 0; 805 flags |= RF_SHAREABLE; 806 } else 807 rid = 1; 808 809 for (i = 0; i < sc->vtpci_nintr_res; i++) { 810 irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, flags); 811 if (irq == NULL) 812 return (ENXIO); 813 814 sc->vtpci_intr_res[i].irq = irq; 815 sc->vtpci_intr_res[i].rid = rid++; 816 } 817 818 /* 819 * Map the virtqueue into the correct index in vq_intr_res[]. Note the 820 * first index is reserved for configuration changes notifications. 821 */ 822 for (i = 0, res_idx = 1; i < nvqs; i++) { 823 vqx = &sc->vtpci_vqx[i]; 824 825 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) { 826 if (vq_info[i].vqai_intr == NULL) 827 vqx->ires_idx = -1; 828 else if (sc->vtpci_flags & VIRTIO_PCI_FLAG_SHARED_MSIX) 829 vqx->ires_idx = res_idx; 830 else 831 vqx->ires_idx = res_idx++; 832 } else 833 vqx->ires_idx = -1; 834 } 835 836 return (0); 837 } 838 839 static int 840 vtpci_alloc_msi(struct vtpci_softc *sc) 841 { 842 device_t dev; 843 int nmsi; 844 u_int irq_flags; 845 846 dev = sc->vtpci_dev; 847 nmsi = pci_msi_count(dev); 848 849 if (nmsi < 1) 850 return (1); 851 852 sc->vtpci_irq_rid = 0; 853 sc->vtpci_irq_type = pci_alloc_1intr(dev, 1, 854 &sc->vtpci_irq_rid, &irq_flags); 855 856 857 return (1); 858 } 859 860 static int 861 vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors) 862 { 863 /* XXX(vsrinivas): Huh? Is this how MSI-X works?*/ 864 /* XXX(vsrinivas): All of this was disabled... */ 865 #ifdef OLD_MSI 866 device_t dev; 867 int nmsix, cnt, required; 868 869 dev = sc->vtpci_dev; 870 871 nmsix = pci_msix_count(dev); 872 if (nmsix < 1) 873 return (1); 874 875 /* An additional vector is needed for the config changes. */ 876 required = nvectors + 1; 877 if (nmsix >= required) { 878 cnt = required; 879 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) 880 goto out; 881 882 pci_release_msi(dev); 883 } 884 885 /* Attempt shared MSIX configuration. */ 886 required = 2; 887 if (nmsix >= required) { 888 cnt = required; 889 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 890 sc->vtpci_flags |= VIRTIO_PCI_FLAG_SHARED_MSIX; 891 goto out; 892 } 893 894 pci_release_msi(dev); 895 } 896 897 return (1); 898 899 out: 900 sc->vtpci_nintr_res = required; 901 sc->vtpci_flags |= VIRTIO_PCI_FLAG_MSIX; 902 903 if (bootverbose) { 904 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_SHARED_MSIX) 905 device_printf(dev, "using shared virtqueue MSIX\n"); 906 else 907 device_printf(dev, "using per virtqueue MSIX\n"); 908 } 909 #endif 910 return (0); 911 } 912 913 static int 914 vtpci_register_msix_vector(struct vtpci_softc *sc, int offset, int res_idx) 915 { 916 device_t dev; 917 uint16_t vector; 918 919 dev = sc->vtpci_dev; 920 921 if (offset != VIRTIO_MSI_CONFIG_VECTOR && 922 offset != VIRTIO_MSI_QUEUE_VECTOR) 923 return (EINVAL); 924 925 if (res_idx != -1) { 926 /* Map from rid to host vector. */ 927 vector = sc->vtpci_intr_res[res_idx].rid - 1; 928 } else 929 vector = VIRTIO_MSI_NO_VECTOR; 930 931 /* The first resource is special; make sure it is used correctly. */ 932 if (res_idx == 0) { 933 KASSERT(vector == 0, ("unexpected config vector")); 934 KASSERT(offset == VIRTIO_MSI_CONFIG_VECTOR, 935 ("unexpected config offset")); 936 } 937 938 vtpci_write_config_2(sc, offset, vector); 939 940 if (vtpci_read_config_2(sc, offset) != vector) { 941 device_printf(dev, "insufficient host resources for " 942 "MSIX interrupts\n"); 943 return (ENODEV); 944 } 945 946 return (0); 947 } 948 949 static void 950 vtpci_free_interrupts(struct vtpci_softc *sc) 951 { 952 device_t dev; 953 struct vtpci_intr_resource *ires; 954 int i; 955 956 dev = sc->vtpci_dev; 957 sc->vtpci_nintr_res = 0; 958 959 if (sc->vtpci_flags & (VIRTIO_PCI_FLAG_MSI | VIRTIO_PCI_FLAG_MSIX)) { 960 pci_release_msi(dev); 961 sc->vtpci_flags &= ~(VIRTIO_PCI_FLAG_MSI | 962 VIRTIO_PCI_FLAG_MSIX | VIRTIO_PCI_FLAG_SHARED_MSIX); 963 } 964 965 for (i = 0; i < 1 + VIRTIO_MAX_VIRTQUEUES; i++) { 966 ires = &sc->vtpci_intr_res[i]; 967 968 if (ires->intrhand != NULL) { 969 bus_teardown_intr(dev, ires->irq, ires->intrhand); 970 ires->intrhand = NULL; 971 } 972 973 if (ires->irq != NULL) { 974 bus_release_resource(dev, SYS_RES_IRQ, ires->rid, 975 ires->irq); 976 ires->irq = NULL; 977 } 978 979 ires->rid = -1; 980 } 981 } 982 983 static void 984 vtpci_free_virtqueues(struct vtpci_softc *sc) 985 { 986 struct vtpci_virtqueue *vqx; 987 int i; 988 989 sc->vtpci_nvqs = 0; 990 991 for (i = 0; i < VIRTIO_MAX_VIRTQUEUES; i++) { 992 vqx = &sc->vtpci_vqx[i]; 993 994 if (vqx->vq != NULL) { 995 virtqueue_free(vqx->vq); 996 vqx->vq = NULL; 997 } 998 } 999 } 1000 1001 static void 1002 vtpci_release_child_resources(struct vtpci_softc *sc) 1003 { 1004 1005 vtpci_free_interrupts(sc); 1006 vtpci_free_virtqueues(sc); 1007 } 1008 1009 static void 1010 vtpci_reset(struct vtpci_softc *sc) 1011 { 1012 1013 /* 1014 * Setting the status to RESET sets the host device to 1015 * the original, uninitialized state. 1016 */ 1017 vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET); 1018 } 1019 1020 static int 1021 vtpci_legacy_intr(void *xsc) 1022 { 1023 struct vtpci_softc *sc; 1024 struct vtpci_virtqueue *vqx; 1025 int i; 1026 uint8_t isr; 1027 1028 sc = xsc; 1029 vqx = &sc->vtpci_vqx[0]; 1030 1031 /* Reading the ISR also clears it. */ 1032 isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR); 1033 1034 if (isr & VIRTIO_PCI_ISR_CONFIG) 1035 vtpci_config_intr(sc); 1036 1037 if (isr & VIRTIO_PCI_ISR_INTR) 1038 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) 1039 virtqueue_intr(vqx->vq); 1040 1041 return isr; 1042 } 1043 1044 static int 1045 vtpci_vq_shared_intr(void *xsc) 1046 { 1047 struct vtpci_softc *sc; 1048 struct vtpci_virtqueue *vqx; 1049 int i, rc; 1050 1051 rc = 0; 1052 sc = xsc; 1053 vqx = &sc->vtpci_vqx[0]; 1054 1055 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) 1056 rc |= virtqueue_intr(vqx->vq); 1057 1058 return rc; 1059 } 1060 1061 static int 1062 vtpci_vq_intr(void *xvq) 1063 { 1064 struct virtqueue *vq; 1065 int rc; 1066 1067 vq = xvq; 1068 rc = virtqueue_intr(vq); 1069 1070 return rc; 1071 } 1072 1073 static int 1074 vtpci_config_intr(void *xsc) 1075 { 1076 struct vtpci_softc *sc; 1077 device_t child; 1078 int rc; 1079 1080 rc = 0; 1081 sc = xsc; 1082 child = sc->vtpci_child_dev; 1083 1084 if (child != NULL) 1085 rc = VIRTIO_CONFIG_CHANGE(child); 1086 1087 return rc; 1088 } 1089