1 /*- 2 * Copyright (c) 2011, Bryan Venteicher <bryanv@daemoninthecloset.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice unmodified, this list of conditions, and the following 10 * disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 25 * 26 * $FreeBSD: src/sys/dev/virtio/pci/virtio_pci.c,v 1.3 2012/04/14 05:48:04 grehan Exp $ 27 */ 28 29 /* Driver for the VirtIO PCI interface. */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/bus.h> 34 #include <sys/kernel.h> 35 #include <sys/module.h> 36 #include <sys/malloc.h> 37 #include <sys/serialize.h> 38 39 #include <bus/pci/pcivar.h> 40 #include <bus/pci/pcireg.h> 41 42 #include <sys/bus.h> 43 #include <sys/param.h> 44 #include <sys/rman.h> 45 46 #include <dev/virtual/virtio/virtio/virtio.h> 47 #include <dev/virtual/virtio/virtio/virtqueue.h> 48 #include "virtio_pci.h" 49 #include "virtio_if.h" 50 #include "virtio_bus_if.h" 51 52 struct vtpci_softc { 53 device_t vtpci_dev; 54 struct resource *vtpci_res; 55 struct resource *vtpci_msix_res; 56 uint64_t vtpci_features; 57 uint32_t vtpci_flags; 58 int vtpci_irq_type; 59 int vtpci_irq_rid; 60 #define VIRTIO_PCI_FLAG_NO_MSI 0x0001 61 #define VIRTIO_PCI_FLAG_MSI 0x0002 62 #define VIRTIO_PCI_FLAG_NO_MSIX 0x0010 63 #define VIRTIO_PCI_FLAG_MSIX 0x0020 64 #define VIRTIO_PCI_FLAG_SHARED_MSIX 0x0040 65 66 device_t vtpci_child_dev; 67 struct virtio_feature_desc *vtpci_child_feat_desc; 68 69 /* 70 * Ideally, each virtqueue that the driver provides a callback for 71 * will receive its own MSIX vector. If there are not sufficient 72 * vectors available, we will then attempt to have all the VQs 73 * share one vector. Note that when using MSIX, the configuration 74 * changed notifications must be on their own vector. 75 * 76 * If MSIX is not available, we will attempt to have the whole 77 * device share one MSI vector, and then, finally, one legacy 78 * interrupt. 79 */ 80 int vtpci_nvqs; 81 struct vtpci_virtqueue { 82 struct virtqueue *vq; 83 84 /* Index into vtpci_intr_res[] below. Unused, then -1. */ 85 int ires_idx; 86 } vtpci_vqx[VIRTIO_MAX_VIRTQUEUES]; 87 88 /* 89 * When using MSIX interrupts, the first element of vtpci_intr_res[] 90 * is always the configuration changed notifications. The remaining 91 * element(s) are used for the virtqueues. 92 * 93 * With MSI and legacy interrupts, only the first element of 94 * vtpci_intr_res[] is used. 95 */ 96 int vtpci_nintr_res; 97 struct vtpci_intr_resource { 98 struct resource *irq; 99 int rid; 100 void *intrhand; 101 } vtpci_intr_res[1 + VIRTIO_MAX_VIRTQUEUES]; 102 }; 103 104 static int vtpci_probe(device_t); 105 static int vtpci_attach(device_t); 106 static int vtpci_detach(device_t); 107 static int vtpci_suspend(device_t); 108 static int vtpci_resume(device_t); 109 static int vtpci_shutdown(device_t); 110 static void vtpci_driver_added(device_t, driver_t *); 111 static void vtpci_child_detached(device_t, device_t); 112 static int vtpci_read_ivar(device_t, device_t, int, uintptr_t *); 113 static int vtpci_write_ivar(device_t, device_t, int, uintptr_t); 114 115 static uint64_t vtpci_negotiate_features(device_t, uint64_t); 116 static int vtpci_with_feature(device_t, uint64_t); 117 static int vtpci_alloc_virtqueues(device_t, int, int, 118 struct vq_alloc_info *); 119 static int vtpci_setup_intr(device_t, lwkt_serialize_t); 120 static void vtpci_stop(device_t); 121 static int vtpci_reinit(device_t, uint64_t); 122 static void vtpci_reinit_complete(device_t); 123 static void vtpci_notify_virtqueue(device_t, uint16_t); 124 static uint8_t vtpci_get_status(device_t); 125 static void vtpci_set_status(device_t, uint8_t); 126 static void vtpci_read_dev_config(device_t, bus_size_t, void *, int); 127 static void vtpci_write_dev_config(device_t, bus_size_t, void *, int); 128 129 static void vtpci_describe_features(struct vtpci_softc *, const char *, 130 uint64_t); 131 static void vtpci_probe_and_attach_child(struct vtpci_softc *); 132 133 static int vtpci_alloc_interrupts(struct vtpci_softc *, int, int, 134 struct vq_alloc_info *); 135 static int vtpci_alloc_intr_resources(struct vtpci_softc *, int, 136 struct vq_alloc_info *); 137 static int vtpci_alloc_msi(struct vtpci_softc *); 138 static int vtpci_alloc_msix(struct vtpci_softc *, int); 139 static int vtpci_register_msix_vector(struct vtpci_softc *, int, int); 140 141 static void vtpci_free_interrupts(struct vtpci_softc *); 142 static void vtpci_free_virtqueues(struct vtpci_softc *); 143 static void vtpci_release_child_resources(struct vtpci_softc *); 144 static void vtpci_reset(struct vtpci_softc *); 145 146 static int vtpci_legacy_intr(void *); 147 static int vtpci_vq_shared_intr(void *); 148 static int vtpci_vq_intr(void *); 149 static int vtpci_config_intr(void *); 150 151 /* 152 * I/O port read/write wrappers. 153 */ 154 #define vtpci_read_config_1(sc, o) bus_read_1((sc)->vtpci_res, (o)) 155 #define vtpci_read_config_2(sc, o) bus_read_2((sc)->vtpci_res, (o)) 156 #define vtpci_read_config_4(sc, o) bus_read_4((sc)->vtpci_res, (o)) 157 #define vtpci_write_config_1(sc, o, v) bus_write_1((sc)->vtpci_res, (o), (v)) 158 #define vtpci_write_config_2(sc, o, v) bus_write_2((sc)->vtpci_res, (o), (v)) 159 #define vtpci_write_config_4(sc, o, v) bus_write_4((sc)->vtpci_res, (o), (v)) 160 161 /* Tunables. */ 162 static int vtpci_disable_msix = 0; 163 TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix); 164 165 static device_method_t vtpci_methods[] = { 166 /* Device interface. */ 167 DEVMETHOD(device_probe, vtpci_probe), 168 DEVMETHOD(device_attach, vtpci_attach), 169 DEVMETHOD(device_detach, vtpci_detach), 170 DEVMETHOD(device_suspend, vtpci_suspend), 171 DEVMETHOD(device_resume, vtpci_resume), 172 DEVMETHOD(device_shutdown, vtpci_shutdown), 173 174 /* Bus interface. */ 175 DEVMETHOD(bus_driver_added, vtpci_driver_added), 176 DEVMETHOD(bus_child_detached, vtpci_child_detached), 177 DEVMETHOD(bus_read_ivar, vtpci_read_ivar), 178 DEVMETHOD(bus_write_ivar, vtpci_write_ivar), 179 180 /* VirtIO bus interface. */ 181 DEVMETHOD(virtio_bus_negotiate_features, vtpci_negotiate_features), 182 DEVMETHOD(virtio_bus_with_feature, vtpci_with_feature), 183 DEVMETHOD(virtio_bus_alloc_virtqueues, vtpci_alloc_virtqueues), 184 DEVMETHOD(virtio_bus_setup_intr, vtpci_setup_intr), 185 DEVMETHOD(virtio_bus_stop, vtpci_stop), 186 DEVMETHOD(virtio_bus_reinit, vtpci_reinit), 187 DEVMETHOD(virtio_bus_reinit_complete, vtpci_reinit_complete), 188 DEVMETHOD(virtio_bus_notify_vq, vtpci_notify_virtqueue), 189 DEVMETHOD(virtio_bus_read_device_config, vtpci_read_dev_config), 190 DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config), 191 192 DEVMETHOD_END 193 }; 194 195 static driver_t vtpci_driver = { 196 "virtio_pci", 197 vtpci_methods, 198 sizeof(struct vtpci_softc) 199 }; 200 201 devclass_t vtpci_devclass; 202 203 DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, NULL, NULL); 204 MODULE_VERSION(virtio_pci, 1); 205 MODULE_DEPEND(virtio_pci, pci, 1, 1, 1); 206 MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1); 207 208 static int 209 vtpci_probe(device_t dev) 210 { 211 char desc[36]; 212 const char *name; 213 214 if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID) 215 return (ENXIO); 216 217 if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN || 218 pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX) 219 return (ENXIO); 220 221 if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION) 222 return (ENXIO); 223 224 name = virtio_device_name(pci_get_subdevice(dev)); 225 if (name == NULL) 226 name = "Unknown"; 227 228 ksnprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name); 229 device_set_desc_copy(dev, desc); 230 231 return (BUS_PROBE_DEFAULT); 232 } 233 234 static int 235 vtpci_attach(device_t dev) 236 { 237 struct vtpci_softc *sc; 238 device_t child; 239 int rid; 240 241 sc = device_get_softc(dev); 242 sc->vtpci_dev = dev; 243 244 pci_enable_busmaster(dev); 245 246 rid = PCIR_BAR(0); 247 sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid, 248 RF_ACTIVE); 249 if (sc->vtpci_res == NULL) { 250 device_printf(dev, "cannot map I/O space\n"); 251 return (ENXIO); 252 } 253 254 if (pci_find_extcap(dev, PCIY_MSI, NULL) != 0) 255 sc->vtpci_flags |= VIRTIO_PCI_FLAG_NO_MSI; 256 /* XXX(vsrinivas): Check out how to get MSI-X */ 257 #ifdef OLD_MSI 258 if (pci_find_extcap(dev, PCIY_MSIX, NULL) == 0) { 259 rid = PCIR_BAR(1); 260 sc->vtpci_msix_res = bus_alloc_resource_any(dev, 261 SYS_RES_MEMORY, &rid, RF_ACTIVE); 262 } 263 #endif 264 if (sc->vtpci_msix_res == NULL) 265 sc->vtpci_flags |= VIRTIO_PCI_FLAG_NO_MSIX; 266 267 vtpci_reset(sc); 268 269 /* Tell the host we've noticed this device. */ 270 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 271 272 if ((child = device_add_child(dev, NULL, -1)) == NULL) { 273 device_printf(dev, "cannot create child device\n"); 274 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 275 vtpci_detach(dev); 276 return (ENOMEM); 277 } 278 279 sc->vtpci_child_dev = child; 280 vtpci_probe_and_attach_child(sc); 281 282 return (0); 283 } 284 285 static int 286 vtpci_detach(device_t dev) 287 { 288 struct vtpci_softc *sc; 289 device_t child; 290 int error; 291 292 sc = device_get_softc(dev); 293 294 if ((child = sc->vtpci_child_dev) != NULL) { 295 error = device_delete_child(dev, child); 296 if (error) 297 return (error); 298 sc->vtpci_child_dev = NULL; 299 } 300 301 vtpci_reset(sc); 302 303 if (sc->vtpci_msix_res != NULL) { 304 bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1), 305 sc->vtpci_msix_res); 306 sc->vtpci_msix_res = NULL; 307 } 308 309 if (sc->vtpci_res != NULL) { 310 bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0), 311 sc->vtpci_res); 312 sc->vtpci_res = NULL; 313 } 314 315 return (0); 316 } 317 318 static int 319 vtpci_suspend(device_t dev) 320 { 321 322 return (bus_generic_suspend(dev)); 323 } 324 325 static int 326 vtpci_resume(device_t dev) 327 { 328 329 return (bus_generic_resume(dev)); 330 } 331 332 static int 333 vtpci_shutdown(device_t dev) 334 { 335 336 (void) bus_generic_shutdown(dev); 337 /* Forcibly stop the host device. */ 338 vtpci_stop(dev); 339 340 return (0); 341 } 342 343 static void 344 vtpci_driver_added(device_t dev, driver_t *driver) 345 { 346 struct vtpci_softc *sc; 347 348 sc = device_get_softc(dev); 349 350 vtpci_probe_and_attach_child(sc); 351 } 352 353 static void 354 vtpci_child_detached(device_t dev, device_t child) 355 { 356 struct vtpci_softc *sc; 357 358 sc = device_get_softc(dev); 359 360 vtpci_reset(sc); 361 vtpci_release_child_resources(sc); 362 } 363 364 static int 365 vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result) 366 { 367 struct vtpci_softc *sc; 368 369 sc = device_get_softc(dev); 370 371 if (sc->vtpci_child_dev != child) 372 return (ENOENT); 373 374 switch (index) { 375 case VIRTIO_IVAR_DEVTYPE: 376 *result = pci_get_subdevice(dev); 377 break; 378 default: 379 return (ENOENT); 380 } 381 382 return (0); 383 } 384 385 static int 386 vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value) 387 { 388 struct vtpci_softc *sc; 389 390 sc = device_get_softc(dev); 391 392 if (sc->vtpci_child_dev != child) 393 return (ENOENT); 394 395 switch (index) { 396 case VIRTIO_IVAR_FEATURE_DESC: 397 sc->vtpci_child_feat_desc = (void *) value; 398 break; 399 default: 400 return (ENOENT); 401 } 402 403 return (0); 404 } 405 406 static uint64_t 407 vtpci_negotiate_features(device_t dev, uint64_t child_features) 408 { 409 struct vtpci_softc *sc; 410 uint64_t host_features, features; 411 412 sc = device_get_softc(dev); 413 414 host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES); 415 vtpci_describe_features(sc, "host", host_features); 416 417 /* 418 * Limit negotiated features to what the driver, virtqueue, and 419 * host all support. 420 */ 421 features = host_features & child_features; 422 features = virtqueue_filter_features(features); 423 sc->vtpci_features = features; 424 425 vtpci_describe_features(sc, "negotiated", features); 426 vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features); 427 428 return (features); 429 } 430 431 static int 432 vtpci_with_feature(device_t dev, uint64_t feature) 433 { 434 struct vtpci_softc *sc; 435 436 sc = device_get_softc(dev); 437 438 return ((sc->vtpci_features & feature) != 0); 439 } 440 441 static int 442 vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs, 443 struct vq_alloc_info *vq_info) 444 { 445 struct vtpci_softc *sc; 446 struct vtpci_virtqueue *vqx; 447 struct vq_alloc_info *info; 448 int queue, error; 449 uint16_t vq_size; 450 451 sc = device_get_softc(dev); 452 453 if (sc->vtpci_nvqs != 0 || nvqs <= 0 || 454 nvqs > VIRTIO_MAX_VIRTQUEUES) 455 return (EINVAL); 456 457 error = vtpci_alloc_interrupts(sc, flags, nvqs, vq_info); 458 if (error) { 459 device_printf(dev, "cannot allocate interrupts\n"); 460 return (error); 461 } 462 463 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) { 464 error = vtpci_register_msix_vector(sc, 465 VIRTIO_MSI_CONFIG_VECTOR, 0); 466 if (error) 467 return (error); 468 } 469 470 for (queue = 0; queue < nvqs; queue++) { 471 vqx = &sc->vtpci_vqx[queue]; 472 info = &vq_info[queue]; 473 474 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, queue); 475 476 vq_size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); 477 error = virtqueue_alloc(dev, queue, vq_size, 478 VIRTIO_PCI_VRING_ALIGN, 0xFFFFFFFFUL, info, &vqx->vq); 479 if (error) 480 return (error); 481 482 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) { 483 error = vtpci_register_msix_vector(sc, 484 VIRTIO_MSI_QUEUE_VECTOR, vqx->ires_idx); 485 if (error) 486 return (error); 487 } 488 489 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 490 virtqueue_paddr(vqx->vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 491 492 *info->vqai_vq = vqx->vq; 493 sc->vtpci_nvqs++; 494 } 495 496 return (0); 497 } 498 499 static int 500 vtpci_setup_intr(device_t dev, lwkt_serialize_t slz) 501 { 502 struct vtpci_softc *sc; 503 struct vtpci_intr_resource *ires; 504 struct vtpci_virtqueue *vqx; 505 int i, flags, error; 506 507 sc = device_get_softc(dev); 508 flags = INTR_MPSAFE; 509 ires = &sc->vtpci_intr_res[0]; 510 511 if ((sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) == 0) { 512 error = bus_setup_intr(dev, ires->irq, flags, 513 (driver_intr_t *) vtpci_legacy_intr, 514 sc, &ires->intrhand, slz); 515 return (error); 516 } 517 518 error = bus_setup_intr(dev, ires->irq, flags, 519 (driver_intr_t *) vtpci_config_intr, 520 sc, &ires->intrhand, slz); 521 if (error) 522 return (error); 523 524 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_SHARED_MSIX) { 525 ires = &sc->vtpci_intr_res[1]; 526 error = bus_setup_intr(dev, ires->irq, flags, 527 (driver_intr_t *) vtpci_vq_shared_intr, 528 sc, &ires->intrhand, slz); 529 530 return (error); 531 } 532 533 /* Setup an interrupt handler for each virtqueue. */ 534 for (i = 0; i < sc->vtpci_nvqs; i++) { 535 vqx = &sc->vtpci_vqx[i]; 536 if (vqx->ires_idx < 1) 537 continue; 538 539 ires = &sc->vtpci_intr_res[vqx->ires_idx]; 540 error = bus_setup_intr(dev, ires->irq, flags, 541 (driver_intr_t *) vtpci_vq_intr, 542 vqx->vq, &ires->intrhand, slz); 543 if (error) 544 return (error); 545 } 546 547 return (0); 548 } 549 550 static void 551 vtpci_stop(device_t dev) 552 { 553 vtpci_reset(device_get_softc(dev)); 554 } 555 556 static int 557 vtpci_reinit(device_t dev, uint64_t features) 558 { 559 struct vtpci_softc *sc; 560 struct vtpci_virtqueue *vqx; 561 struct virtqueue *vq; 562 int queue, error; 563 uint16_t vq_size; 564 565 sc = device_get_softc(dev); 566 567 /* 568 * Redrive the device initialization. This is a bit of an abuse 569 * of the specification, but both VirtualBox and QEMU/KVM seem 570 * to play nice. We do not allow the host device to change from 571 * what was originally negotiated beyond what the guest driver 572 * changed (MSIX state should not change, number of virtqueues 573 * and their size remain the same, etc). 574 */ 575 576 if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET) 577 vtpci_stop(dev); 578 579 /* 580 * Quickly drive the status through ACK and DRIVER. The device 581 * does not become usable again until vtpci_reinit_complete(). 582 */ 583 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 584 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 585 586 vtpci_negotiate_features(dev, features); 587 588 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) { 589 error = vtpci_register_msix_vector(sc, 590 VIRTIO_MSI_CONFIG_VECTOR, 0); 591 if (error) 592 return (error); 593 } 594 595 for (queue = 0; queue < sc->vtpci_nvqs; queue++) { 596 vqx = &sc->vtpci_vqx[queue]; 597 vq = vqx->vq; 598 599 KASSERT(vq != NULL, ("vq %d not allocated", queue)); 600 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, queue); 601 602 vq_size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM); 603 error = virtqueue_reinit(vq, vq_size); 604 if (error) 605 return (error); 606 607 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) { 608 error = vtpci_register_msix_vector(sc, 609 VIRTIO_MSI_QUEUE_VECTOR, vqx->ires_idx); 610 if (error) 611 return (error); 612 } 613 614 vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 615 virtqueue_paddr(vqx->vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT); 616 } 617 618 return (0); 619 } 620 621 static void 622 vtpci_reinit_complete(device_t dev) 623 { 624 625 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 626 } 627 628 static void 629 vtpci_notify_virtqueue(device_t dev, uint16_t queue) 630 { 631 struct vtpci_softc *sc; 632 633 sc = device_get_softc(dev); 634 635 vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue); 636 } 637 638 static uint8_t 639 vtpci_get_status(device_t dev) 640 { 641 struct vtpci_softc *sc; 642 643 sc = device_get_softc(dev); 644 645 return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS)); 646 } 647 648 static void 649 vtpci_set_status(device_t dev, uint8_t status) 650 { 651 struct vtpci_softc *sc; 652 653 sc = device_get_softc(dev); 654 655 if (status != VIRTIO_CONFIG_STATUS_RESET) 656 status |= vtpci_get_status(dev); 657 658 vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status); 659 } 660 661 static void 662 vtpci_read_dev_config(device_t dev, bus_size_t offset, 663 void *dst, int length) 664 { 665 struct vtpci_softc *sc; 666 bus_size_t off; 667 uint8_t *d; 668 int size; 669 670 sc = device_get_softc(dev); 671 off = VIRTIO_PCI_CONFIG(sc) + offset; 672 673 for (d = dst; length > 0; d += size, off += size, length -= size) { 674 if (length >= 4) { 675 size = 4; 676 *(uint32_t *)d = vtpci_read_config_4(sc, off); 677 } else if (length >= 2) { 678 size = 2; 679 *(uint16_t *)d = vtpci_read_config_2(sc, off); 680 } else { 681 size = 1; 682 *d = vtpci_read_config_1(sc, off); 683 } 684 } 685 } 686 687 static void 688 vtpci_write_dev_config(device_t dev, bus_size_t offset, 689 void *src, int length) 690 { 691 struct vtpci_softc *sc; 692 bus_size_t off; 693 uint8_t *s; 694 int size; 695 696 sc = device_get_softc(dev); 697 off = VIRTIO_PCI_CONFIG(sc) + offset; 698 699 for (s = src; length > 0; s += size, off += size, length -= size) { 700 if (length >= 4) { 701 size = 4; 702 vtpci_write_config_4(sc, off, *(uint32_t *)s); 703 } else if (length >= 2) { 704 size = 2; 705 vtpci_write_config_2(sc, off, *(uint16_t *)s); 706 } else { 707 size = 1; 708 vtpci_write_config_1(sc, off, *s); 709 } 710 } 711 } 712 713 static void 714 vtpci_describe_features(struct vtpci_softc *sc, const char *msg, 715 uint64_t features) 716 { 717 device_t dev, child; 718 719 dev = sc->vtpci_dev; 720 child = sc->vtpci_child_dev; 721 722 if (device_is_attached(child) && bootverbose == 0) 723 return; 724 725 virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc); 726 } 727 728 static void 729 vtpci_probe_and_attach_child(struct vtpci_softc *sc) 730 { 731 device_t dev, child; 732 733 dev = sc->vtpci_dev; 734 child = sc->vtpci_child_dev; 735 736 if (child == NULL) 737 return; 738 739 if (device_get_state(child) != DS_NOTPRESENT) 740 return; 741 742 if (device_probe_child(dev, child) != 0) 743 return; 744 745 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER); 746 if (DEVICE_ATTACH(child) != 0) { 747 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED); 748 vtpci_reset(sc); 749 vtpci_release_child_resources(sc); 750 751 /* Reset status for future attempt. */ 752 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK); 753 } else 754 vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK); 755 } 756 757 static int 758 vtpci_alloc_interrupts(struct vtpci_softc *sc, int flags, int nvqs, 759 struct vq_alloc_info *vq_info) 760 { 761 int i, nvectors, error; 762 763 /* 764 * Only allocate a vector for virtqueues that are actually 765 * expecting an interrupt. 766 */ 767 for (nvectors = 0, i = 0; i < nvqs; i++) 768 if (vq_info[i].vqai_intr != NULL) 769 nvectors++; 770 771 if (vtpci_disable_msix != 0 || 772 sc->vtpci_flags & VIRTIO_PCI_FLAG_NO_MSIX || 773 flags & VIRTIO_ALLOC_VQS_DISABLE_MSIX || 774 vtpci_alloc_msix(sc, nvectors) != 0) { 775 /* 776 * Use MSI interrupts if available. Otherwise, we fallback 777 * to legacy interrupts. 778 */ 779 if ((sc->vtpci_flags & VIRTIO_PCI_FLAG_NO_MSI) == 0 && 780 vtpci_alloc_msi(sc) == 0) 781 sc->vtpci_flags |= VIRTIO_PCI_FLAG_MSI; 782 783 sc->vtpci_nintr_res = 1; 784 } 785 786 error = vtpci_alloc_intr_resources(sc, nvqs, vq_info); 787 788 return (error); 789 } 790 791 static int 792 vtpci_alloc_intr_resources(struct vtpci_softc *sc, int nvqs, 793 struct vq_alloc_info *vq_info) 794 { 795 device_t dev; 796 struct resource *irq; 797 struct vtpci_virtqueue *vqx; 798 int i, rid, flags, res_idx; 799 800 dev = sc->vtpci_dev; 801 flags = RF_ACTIVE; 802 803 if ((sc->vtpci_flags & 804 (VIRTIO_PCI_FLAG_MSI | VIRTIO_PCI_FLAG_MSIX)) == 0) { 805 rid = 0; 806 flags |= RF_SHAREABLE; 807 } else 808 rid = 1; 809 810 for (i = 0; i < sc->vtpci_nintr_res; i++) { 811 irq = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, flags); 812 if (irq == NULL) 813 return (ENXIO); 814 815 sc->vtpci_intr_res[i].irq = irq; 816 sc->vtpci_intr_res[i].rid = rid++; 817 } 818 819 /* 820 * Map the virtqueue into the correct index in vq_intr_res[]. Note the 821 * first index is reserved for configuration changes notifications. 822 */ 823 for (i = 0, res_idx = 1; i < nvqs; i++) { 824 vqx = &sc->vtpci_vqx[i]; 825 826 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_MSIX) { 827 if (vq_info[i].vqai_intr == NULL) 828 vqx->ires_idx = -1; 829 else if (sc->vtpci_flags & VIRTIO_PCI_FLAG_SHARED_MSIX) 830 vqx->ires_idx = res_idx; 831 else 832 vqx->ires_idx = res_idx++; 833 } else 834 vqx->ires_idx = -1; 835 } 836 837 return (0); 838 } 839 840 static int 841 vtpci_alloc_msi(struct vtpci_softc *sc) 842 { 843 device_t dev; 844 int nmsi; 845 u_int irq_flags; 846 847 dev = sc->vtpci_dev; 848 nmsi = pci_msi_count(dev); 849 850 if (nmsi < 1) 851 return (1); 852 853 sc->vtpci_irq_rid = 0; 854 sc->vtpci_irq_type = pci_alloc_1intr(dev, 1, 855 &sc->vtpci_irq_rid, &irq_flags); 856 857 858 return (1); 859 } 860 861 static int 862 vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors) 863 { 864 /* XXX(vsrinivas): Huh? Is this how MSI-X works?*/ 865 /* XXX(vsrinivas): All of this was disabled... */ 866 #ifdef OLD_MSI 867 device_t dev; 868 int nmsix, cnt, required; 869 870 dev = sc->vtpci_dev; 871 872 nmsix = pci_msix_count(dev); 873 if (nmsix < 1) 874 return (1); 875 876 /* An additional vector is needed for the config changes. */ 877 required = nvectors + 1; 878 if (nmsix >= required) { 879 cnt = required; 880 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) 881 goto out; 882 883 pci_release_msi(dev); 884 } 885 886 /* Attempt shared MSIX configuration. */ 887 required = 2; 888 if (nmsix >= required) { 889 cnt = required; 890 if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) { 891 sc->vtpci_flags |= VIRTIO_PCI_FLAG_SHARED_MSIX; 892 goto out; 893 } 894 895 pci_release_msi(dev); 896 } 897 898 return (1); 899 900 out: 901 sc->vtpci_nintr_res = required; 902 sc->vtpci_flags |= VIRTIO_PCI_FLAG_MSIX; 903 904 if (bootverbose) { 905 if (sc->vtpci_flags & VIRTIO_PCI_FLAG_SHARED_MSIX) 906 device_printf(dev, "using shared virtqueue MSIX\n"); 907 else 908 device_printf(dev, "using per virtqueue MSIX\n"); 909 } 910 #endif 911 return (0); 912 } 913 914 static int 915 vtpci_register_msix_vector(struct vtpci_softc *sc, int offset, int res_idx) 916 { 917 device_t dev; 918 uint16_t vector; 919 920 dev = sc->vtpci_dev; 921 922 if (offset != VIRTIO_MSI_CONFIG_VECTOR && 923 offset != VIRTIO_MSI_QUEUE_VECTOR) 924 return (EINVAL); 925 926 if (res_idx != -1) { 927 /* Map from rid to host vector. */ 928 vector = sc->vtpci_intr_res[res_idx].rid - 1; 929 } else 930 vector = VIRTIO_MSI_NO_VECTOR; 931 932 /* The first resource is special; make sure it is used correctly. */ 933 if (res_idx == 0) { 934 KASSERT(vector == 0, ("unexpected config vector")); 935 KASSERT(offset == VIRTIO_MSI_CONFIG_VECTOR, 936 ("unexpected config offset")); 937 } 938 939 vtpci_write_config_2(sc, offset, vector); 940 941 if (vtpci_read_config_2(sc, offset) != vector) { 942 device_printf(dev, "insufficient host resources for " 943 "MSIX interrupts\n"); 944 return (ENODEV); 945 } 946 947 return (0); 948 } 949 950 static void 951 vtpci_free_interrupts(struct vtpci_softc *sc) 952 { 953 device_t dev; 954 struct vtpci_intr_resource *ires; 955 int i; 956 957 dev = sc->vtpci_dev; 958 sc->vtpci_nintr_res = 0; 959 960 if (sc->vtpci_flags & (VIRTIO_PCI_FLAG_MSI | VIRTIO_PCI_FLAG_MSIX)) { 961 pci_release_msi(dev); 962 sc->vtpci_flags &= ~(VIRTIO_PCI_FLAG_MSI | 963 VIRTIO_PCI_FLAG_MSIX | VIRTIO_PCI_FLAG_SHARED_MSIX); 964 } 965 966 for (i = 0; i < 1 + VIRTIO_MAX_VIRTQUEUES; i++) { 967 ires = &sc->vtpci_intr_res[i]; 968 969 if (ires->intrhand != NULL) { 970 bus_teardown_intr(dev, ires->irq, ires->intrhand); 971 ires->intrhand = NULL; 972 } 973 974 if (ires->irq != NULL) { 975 bus_release_resource(dev, SYS_RES_IRQ, ires->rid, 976 ires->irq); 977 ires->irq = NULL; 978 } 979 980 ires->rid = -1; 981 } 982 } 983 984 static void 985 vtpci_free_virtqueues(struct vtpci_softc *sc) 986 { 987 struct vtpci_virtqueue *vqx; 988 int i; 989 990 sc->vtpci_nvqs = 0; 991 992 for (i = 0; i < VIRTIO_MAX_VIRTQUEUES; i++) { 993 vqx = &sc->vtpci_vqx[i]; 994 995 if (vqx->vq != NULL) { 996 virtqueue_free(vqx->vq); 997 vqx->vq = NULL; 998 } 999 } 1000 } 1001 1002 static void 1003 vtpci_release_child_resources(struct vtpci_softc *sc) 1004 { 1005 1006 vtpci_free_interrupts(sc); 1007 vtpci_free_virtqueues(sc); 1008 } 1009 1010 static void 1011 vtpci_reset(struct vtpci_softc *sc) 1012 { 1013 1014 /* 1015 * Setting the status to RESET sets the host device to 1016 * the original, uninitialized state. 1017 */ 1018 vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET); 1019 } 1020 1021 static int 1022 vtpci_legacy_intr(void *xsc) 1023 { 1024 struct vtpci_softc *sc; 1025 struct vtpci_virtqueue *vqx; 1026 int i; 1027 uint8_t isr; 1028 1029 sc = xsc; 1030 vqx = &sc->vtpci_vqx[0]; 1031 1032 /* Reading the ISR also clears it. */ 1033 isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR); 1034 1035 if (isr & VIRTIO_PCI_ISR_CONFIG) 1036 vtpci_config_intr(sc); 1037 1038 if (isr & VIRTIO_PCI_ISR_INTR) 1039 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) 1040 virtqueue_intr(vqx->vq); 1041 1042 return isr; 1043 } 1044 1045 static int 1046 vtpci_vq_shared_intr(void *xsc) 1047 { 1048 struct vtpci_softc *sc; 1049 struct vtpci_virtqueue *vqx; 1050 int i, rc; 1051 1052 rc = 0; 1053 sc = xsc; 1054 vqx = &sc->vtpci_vqx[0]; 1055 1056 for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) 1057 rc |= virtqueue_intr(vqx->vq); 1058 1059 return rc; 1060 } 1061 1062 static int 1063 vtpci_vq_intr(void *xvq) 1064 { 1065 struct virtqueue *vq; 1066 int rc; 1067 1068 vq = xvq; 1069 rc = virtqueue_intr(vq); 1070 1071 return rc; 1072 } 1073 1074 static int 1075 vtpci_config_intr(void *xsc) 1076 { 1077 struct vtpci_softc *sc; 1078 device_t child; 1079 int rc; 1080 1081 rc = 0; 1082 sc = xsc; 1083 child = sc->vtpci_child_dev; 1084 1085 if (child != NULL) 1086 rc = VIRTIO_CONFIG_CHANGE(child); 1087 1088 return rc; 1089 } 1090