1 /* $NetBSD: virtio.c,v 1.18 2016/09/27 03:33:32 pgoyette Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.18 2016/09/27 03:33:32 pgoyette Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/atomic.h> 35 #include <sys/bus.h> 36 #include <sys/device.h> 37 #include <sys/kmem.h> 38 #include <sys/module.h> 39 40 #include <dev/pci/pcidevs.h> 41 #include <dev/pci/pcireg.h> 42 #include <dev/pci/pcivar.h> 43 44 #include <dev/pci/virtioreg.h> 45 #include <dev/pci/virtiovar.h> 46 47 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 48 49 static int virtio_match(device_t, cfdata_t, void *); 50 static void virtio_attach(device_t, device_t, void *); 51 static int virtio_rescan(device_t, const char *, const int *); 52 static int virtio_detach(device_t, int); 53 static int virtio_intr(void *arg); 54 static int virtio_msix_queue_intr(void *); 55 static int virtio_msix_config_intr(void *); 56 static int virtio_setup_msix_vectors(struct virtio_softc *); 57 static int virtio_setup_msix_interrupts(struct virtio_softc *, 58 struct pci_attach_args *); 59 static int virtio_setup_intx_interrupt(struct virtio_softc *, 60 struct pci_attach_args *); 61 static int virtio_setup_interrupts(struct virtio_softc *); 62 static void virtio_soft_intr(void *arg); 63 static void virtio_init_vq(struct virtio_softc *, 64 struct virtqueue *, const bool); 65 66 CFATTACH_DECL3_NEW(virtio, sizeof(struct virtio_softc), 67 virtio_match, virtio_attach, virtio_detach, NULL, virtio_rescan, NULL, 68 DVF_DETACH_SHUTDOWN); 69 70 static void 71 virtio_set_status(struct virtio_softc *sc, int status) 72 { 73 int old = 0; 74 75 if (status != 0) 76 old = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 77 VIRTIO_CONFIG_DEVICE_STATUS); 78 bus_space_write_1(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_DEVICE_STATUS, 79 status|old); 80 } 81 82 #define virtio_device_reset(sc) virtio_set_status((sc), 0) 83 84 static int 85 virtio_match(device_t parent, cfdata_t match, void *aux) 86 { 87 struct pci_attach_args *pa; 88 89 pa = (struct pci_attach_args *)aux; 90 switch (PCI_VENDOR(pa->pa_id)) { 91 case PCI_VENDOR_QUMRANET: 92 if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <= 93 PCI_PRODUCT(pa->pa_id)) && 94 (PCI_PRODUCT(pa->pa_id) <= 95 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) 96 return 1; 97 break; 98 } 99 100 return 0; 101 } 102 103 static const char *virtio_device_name[] = { 104 "Unknown (0)", /* 0 */ 105 "Network", /* 1 */ 106 "Block", /* 2 */ 107 "Console", /* 3 */ 108 "Entropy", /* 4 */ 109 "Memory Balloon", /* 5 */ 110 "I/O Memory", /* 6 */ 111 "Remote Processor Messaging", /* 7 */ 112 "SCSI", /* 8 */ 113 "9P Transport", /* 9 */ 114 "mac80211 wlan", /* 10 */ 115 }; 116 #define NDEVNAMES __arraycount(virtio_device_name) 117 118 #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX 0 119 #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX 1 120 121 static int 122 virtio_setup_msix_vectors(struct virtio_softc *sc) 123 { 124 int offset, vector, ret, qid; 125 126 offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR; 127 vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 128 129 bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, vector); 130 ret = bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset); 131 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 132 vector, ret); 133 if (ret != vector) 134 return -1; 135 136 for (qid = 0; qid < sc->sc_nvqs; qid++) { 137 offset = VIRTIO_CONFIG_QUEUE_SELECT; 138 bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, qid); 139 140 offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR; 141 vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 142 143 bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, vector); 144 ret = bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset); 145 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 146 vector, ret); 147 if (ret != vector) 148 return -1; 149 } 150 151 return 0; 152 } 153 154 static int 155 virtio_setup_msix_interrupts(struct virtio_softc *sc, 156 struct pci_attach_args *pa) 157 { 158 device_t self = sc->sc_dev; 159 pci_chipset_tag_t pc = pa->pa_pc; 160 char intrbuf[PCI_INTRSTR_LEN]; 161 char const *intrstr; 162 int idx; 163 164 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 165 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 166 pci_intr_setattr(pc, &sc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 167 168 sc->sc_ihs[idx] = pci_intr_establish_xname(pc, sc->sc_ihp[idx], IPL_NET, 169 virtio_msix_config_intr, sc, device_xname(sc->sc_dev)); 170 if (sc->sc_ihs[idx] == NULL) { 171 aprint_error_dev(self, "couldn't establish MSI-X for config\n"); 172 goto error; 173 } 174 175 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 176 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 177 pci_intr_setattr(pc, &sc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 178 179 sc->sc_ihs[idx] = pci_intr_establish_xname(pc, sc->sc_ihp[idx], IPL_NET, 180 virtio_msix_queue_intr, sc, device_xname(sc->sc_dev)); 181 if (sc->sc_ihs[idx] == NULL) { 182 aprint_error_dev(self, "couldn't establish MSI-X for queues\n"); 183 goto error; 184 } 185 186 if (virtio_setup_msix_vectors(sc) != 0) { 187 aprint_error_dev(self, "couldn't setup MSI-X vectors\n"); 188 goto error; 189 } 190 191 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 192 intrstr = pci_intr_string(pc, sc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 193 aprint_normal_dev(self, "config interrupting at %s\n", intrstr); 194 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 195 intrstr = pci_intr_string(pc, sc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 196 aprint_normal_dev(self, "queues interrupting at %s\n", intrstr); 197 198 return 0; 199 200 error: 201 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 202 if (sc->sc_ihs[idx] != NULL) 203 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[idx]); 204 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 205 if (sc->sc_ihs[idx] != NULL) 206 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[idx]); 207 208 return -1; 209 } 210 211 static int 212 virtio_setup_intx_interrupt(struct virtio_softc *sc, 213 struct pci_attach_args *pa) 214 { 215 device_t self = sc->sc_dev; 216 pci_chipset_tag_t pc = pa->pa_pc; 217 char intrbuf[PCI_INTRSTR_LEN]; 218 char const *intrstr; 219 220 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 221 pci_intr_setattr(pc, &sc->sc_ihp[0], PCI_INTR_MPSAFE, true); 222 223 sc->sc_ihs[0] = pci_intr_establish_xname(pc, sc->sc_ihp[0], 224 IPL_NET, virtio_intr, sc, device_xname(sc->sc_dev)); 225 if (sc->sc_ihs[0] == NULL) { 226 aprint_error_dev(self, "couldn't establish INTx\n"); 227 return -1; 228 } 229 230 intrstr = pci_intr_string(pc, sc->sc_ihp[0], intrbuf, sizeof(intrbuf)); 231 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 232 233 return 0; 234 } 235 236 static int 237 virtio_setup_interrupts(struct virtio_softc *sc) 238 { 239 device_t self = sc->sc_dev; 240 pci_chipset_tag_t pc = sc->sc_pa.pa_pc; 241 int error; 242 int nmsix; 243 int counts[PCI_INTR_TYPE_SIZE]; 244 pci_intr_type_t max_type; 245 246 nmsix = pci_msix_count(sc->sc_pa.pa_pc, sc->sc_pa.pa_tag); 247 aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix); 248 249 /* We need at least two: one for config and the other for queues */ 250 if ((sc->sc_flags & VIRTIO_F_PCI_INTR_MSIX) == 0 || nmsix < 2) { 251 /* Try INTx only */ 252 max_type = PCI_INTR_TYPE_INTX; 253 counts[PCI_INTR_TYPE_INTX] = 1; 254 } else { 255 /* Try MSI-X first and INTx second */ 256 max_type = PCI_INTR_TYPE_MSIX; 257 counts[PCI_INTR_TYPE_MSIX] = 2; 258 counts[PCI_INTR_TYPE_MSI] = 0; 259 counts[PCI_INTR_TYPE_INTX] = 1; 260 } 261 262 retry: 263 error = pci_intr_alloc(&sc->sc_pa, &sc->sc_ihp, counts, max_type); 264 if (error != 0) { 265 aprint_error_dev(self, "couldn't map interrupt\n"); 266 return -1; 267 } 268 269 if (pci_intr_type(pc, sc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) { 270 sc->sc_ihs = kmem_alloc(sizeof(*sc->sc_ihs) * 2, 271 KM_SLEEP); 272 if (sc->sc_ihs == NULL) { 273 pci_intr_release(pc, sc->sc_ihp, 2); 274 275 /* Retry INTx */ 276 max_type = PCI_INTR_TYPE_INTX; 277 counts[PCI_INTR_TYPE_INTX] = 1; 278 goto retry; 279 } 280 281 error = virtio_setup_msix_interrupts(sc, &sc->sc_pa); 282 if (error != 0) { 283 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * 2); 284 pci_intr_release(pc, sc->sc_ihp, 2); 285 286 /* Retry INTx */ 287 max_type = PCI_INTR_TYPE_INTX; 288 counts[PCI_INTR_TYPE_INTX] = 1; 289 goto retry; 290 } 291 292 sc->sc_ihs_num = 2; 293 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI; 294 } else if (pci_intr_type(pc, sc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) { 295 sc->sc_ihs = kmem_alloc(sizeof(*sc->sc_ihs) * 1, 296 KM_SLEEP); 297 if (sc->sc_ihs == NULL) { 298 pci_intr_release(pc, sc->sc_ihp, 1); 299 return -1; 300 } 301 302 error = virtio_setup_intx_interrupt(sc, &sc->sc_pa); 303 if (error != 0) { 304 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * 1); 305 pci_intr_release(pc, sc->sc_ihp, 1); 306 return -1; 307 } 308 309 sc->sc_ihs_num = 1; 310 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 311 } 312 313 return 0; 314 } 315 316 static void 317 virtio_attach(device_t parent, device_t self, void *aux) 318 { 319 struct virtio_softc *sc = device_private(self); 320 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 321 pci_chipset_tag_t pc = pa->pa_pc; 322 pcitag_t tag = pa->pa_tag; 323 int revision; 324 pcireg_t id; 325 326 revision = PCI_REVISION(pa->pa_class); 327 if (revision != 0) { 328 aprint_normal(": unknown revision 0x%02x; giving up\n", 329 revision); 330 return; 331 } 332 aprint_normal("\n"); 333 aprint_naive("\n"); 334 335 /* subsystem ID shows what I am */ 336 id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 337 aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n", 338 (PCI_SUBSYS_ID(id) < NDEVNAMES? 339 virtio_device_name[PCI_SUBSYS_ID(id)] : "Unknown"), 340 revision); 341 342 sc->sc_dev = self; 343 sc->sc_pc = pc; 344 sc->sc_tag = tag; 345 sc->sc_iot = pa->pa_iot; 346 if (pci_dma64_available(pa)) 347 sc->sc_dmat = pa->pa_dmat64; 348 else 349 sc->sc_dmat = pa->pa_dmat; 350 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 351 352 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, 353 &sc->sc_iot, &sc->sc_ioh, NULL, &sc->sc_iosize)) { 354 aprint_error_dev(self, "can't map i/o space\n"); 355 return; 356 } 357 358 virtio_device_reset(sc); 359 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 360 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 361 362 /* XXX: use softc as aux... */ 363 sc->sc_childdevid = PCI_SUBSYS_ID(id); 364 sc->sc_child = NULL; 365 sc->sc_pa = *pa; 366 virtio_rescan(self, "virtio", 0); 367 return; 368 } 369 370 /* ARGSUSED */ 371 static int 372 virtio_rescan(device_t self, const char *attr, const int *scan_flags) 373 { 374 struct virtio_softc *sc; 375 int r; 376 377 sc = device_private(self); 378 if (sc->sc_child) /* Child already attached? */ 379 return 0; 380 config_found_ia(self, attr, sc, NULL); 381 if (sc->sc_child == NULL) { 382 aprint_error_dev(self, 383 "no matching child driver; not configured\n"); 384 return 0; 385 } 386 if (sc->sc_child == (void*)1) { /* this shows error */ 387 aprint_error_dev(self, 388 "virtio configuration failed\n"); 389 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 390 return 0; 391 } 392 393 r = virtio_setup_interrupts(sc); 394 if (r != 0) { 395 aprint_error_dev(self, "failed to setup interrupts\n"); 396 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 397 return 0; 398 } 399 400 sc->sc_soft_ih = NULL; 401 if (sc->sc_flags & VIRTIO_F_PCI_INTR_SOFTINT) { 402 u_int flags = SOFTINT_NET; 403 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 404 flags |= SOFTINT_MPSAFE; 405 406 sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr, sc); 407 if (sc->sc_soft_ih == NULL) 408 aprint_error(": failed to establish soft interrupt\n"); 409 } 410 411 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 412 413 return 0; 414 } 415 416 static int 417 virtio_detach(device_t self, int flags) 418 { 419 struct virtio_softc *sc = device_private(self); 420 int r; 421 int i; 422 423 if (sc->sc_child != 0 && sc->sc_child != (void*)1) { 424 r = config_detach(sc->sc_child, flags); 425 if (r) 426 return r; 427 } 428 KASSERT(sc->sc_child == 0 || sc->sc_child == (void*)1); 429 KASSERT(sc->sc_vqs == 0); 430 for (i = 0; i < sc->sc_ihs_num; i++) { 431 if (sc->sc_ihs[i] == NULL) 432 continue; 433 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]); 434 } 435 pci_intr_release(sc->sc_pc, sc->sc_ihp, sc->sc_ihs_num); 436 if (sc->sc_ihs != NULL) 437 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * sc->sc_ihs_num); 438 sc->sc_ihs_num = 0; 439 if (sc->sc_iosize) 440 bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize); 441 sc->sc_iosize = 0; 442 443 return 0; 444 } 445 446 /* 447 * Reset the device. 448 */ 449 /* 450 * To reset the device to a known state, do following: 451 * virtio_reset(sc); // this will stop the device activity 452 * <dequeue finished requests>; // virtio_dequeue() still can be called 453 * <revoke pending requests in the vqs if any>; 454 * virtio_reinit_begin(sc); // dequeue prohibitted 455 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 456 * <some other initialization>; 457 * virtio_reinit_end(sc); // device activated; enqueue allowed 458 * Once attached, feature negotiation can only be allowed after virtio_reset. 459 */ 460 void 461 virtio_reset(struct virtio_softc *sc) 462 { 463 virtio_device_reset(sc); 464 } 465 466 void 467 virtio_reinit_start(struct virtio_softc *sc) 468 { 469 int i; 470 471 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 472 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 473 for (i = 0; i < sc->sc_nvqs; i++) { 474 int n; 475 struct virtqueue *vq = &sc->sc_vqs[i]; 476 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 477 VIRTIO_CONFIG_QUEUE_SELECT, 478 vq->vq_index); 479 n = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 480 VIRTIO_CONFIG_QUEUE_SIZE); 481 if (n == 0) /* vq disappeared */ 482 continue; 483 if (n != vq->vq_num) { 484 panic("%s: virtqueue size changed, vq index %d\n", 485 device_xname(sc->sc_dev), 486 vq->vq_index); 487 } 488 virtio_init_vq(sc, vq, true); 489 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 490 VIRTIO_CONFIG_QUEUE_ADDRESS, 491 (vq->vq_dmamap->dm_segs[0].ds_addr 492 / VIRTIO_PAGE_SIZE)); 493 } 494 495 /* MSI-X should have more than one handles where INTx has just one */ 496 if (sc->sc_ihs_num > 1) { 497 if (virtio_setup_msix_vectors(sc) != 0) { 498 aprint_error_dev(sc->sc_dev, 499 "couldn't setup MSI-X vectors\n"); 500 return; 501 } 502 } 503 } 504 505 void 506 virtio_reinit_end(struct virtio_softc *sc) 507 { 508 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 509 } 510 511 /* 512 * Feature negotiation. 513 */ 514 uint32_t 515 virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features) 516 { 517 uint32_t r; 518 519 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 520 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 521 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 522 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 523 VIRTIO_CONFIG_DEVICE_FEATURES); 524 r &= guest_features; 525 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 526 VIRTIO_CONFIG_GUEST_FEATURES, r); 527 sc->sc_features = r; 528 if (r & VIRTIO_F_RING_INDIRECT_DESC) 529 sc->sc_indirect = true; 530 else 531 sc->sc_indirect = false; 532 533 return r; 534 } 535 536 /* 537 * Device configuration registers. 538 */ 539 uint8_t 540 virtio_read_device_config_1(struct virtio_softc *sc, int index) 541 { 542 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, 543 sc->sc_config_offset + index); 544 } 545 546 uint16_t 547 virtio_read_device_config_2(struct virtio_softc *sc, int index) 548 { 549 return bus_space_read_2(sc->sc_iot, sc->sc_ioh, 550 sc->sc_config_offset + index); 551 } 552 553 uint32_t 554 virtio_read_device_config_4(struct virtio_softc *sc, int index) 555 { 556 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, 557 sc->sc_config_offset + index); 558 } 559 560 uint64_t 561 virtio_read_device_config_8(struct virtio_softc *sc, int index) 562 { 563 uint64_t r; 564 565 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 566 sc->sc_config_offset + index + sizeof(uint32_t)); 567 r <<= 32; 568 r += bus_space_read_4(sc->sc_iot, sc->sc_ioh, 569 sc->sc_config_offset + index); 570 return r; 571 } 572 573 void 574 virtio_write_device_config_1(struct virtio_softc *sc, 575 int index, uint8_t value) 576 { 577 bus_space_write_1(sc->sc_iot, sc->sc_ioh, 578 sc->sc_config_offset + index, value); 579 } 580 581 void 582 virtio_write_device_config_2(struct virtio_softc *sc, 583 int index, uint16_t value) 584 { 585 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 586 sc->sc_config_offset + index, value); 587 } 588 589 void 590 virtio_write_device_config_4(struct virtio_softc *sc, 591 int index, uint32_t value) 592 { 593 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 594 sc->sc_config_offset + index, value); 595 } 596 597 void 598 virtio_write_device_config_8(struct virtio_softc *sc, 599 int index, uint64_t value) 600 { 601 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 602 sc->sc_config_offset + index, 603 value & 0xffffffff); 604 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 605 sc->sc_config_offset + index + sizeof(uint32_t), 606 value >> 32); 607 } 608 609 /* 610 * Interrupt handler. 611 */ 612 static int 613 virtio_intr(void *arg) 614 { 615 struct virtio_softc *sc = arg; 616 int isr, r = 0; 617 618 /* check and ack the interrupt */ 619 isr = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 620 VIRTIO_CONFIG_ISR_STATUS); 621 if (isr == 0) 622 return 0; 623 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 624 (sc->sc_config_change != NULL)) 625 r = (sc->sc_config_change)(sc); 626 if (sc->sc_intrhand != NULL) { 627 if (sc->sc_soft_ih != NULL) 628 softint_schedule(sc->sc_soft_ih); 629 else 630 r |= (sc->sc_intrhand)(sc); 631 } 632 633 return r; 634 } 635 636 static int 637 virtio_msix_queue_intr(void *arg) 638 { 639 struct virtio_softc *sc = arg; 640 int r = 0; 641 642 if (sc->sc_intrhand != NULL) { 643 if (sc->sc_soft_ih != NULL) 644 softint_schedule(sc->sc_soft_ih); 645 else 646 r |= (sc->sc_intrhand)(sc); 647 } 648 649 return r; 650 } 651 652 static int 653 virtio_msix_config_intr(void *arg) 654 { 655 struct virtio_softc *sc = arg; 656 657 /* TODO: handle events */ 658 aprint_debug_dev(sc->sc_dev, "%s\n", __func__); 659 return 1; 660 } 661 662 static void 663 virtio_soft_intr(void *arg) 664 { 665 struct virtio_softc *sc = arg; 666 667 KASSERT(sc->sc_intrhand != NULL); 668 669 (sc->sc_intrhand)(sc); 670 } 671 672 /* 673 * dmamap sync operations for a virtqueue. 674 */ 675 static inline void 676 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 677 { 678 /* availoffset == sizeof(vring_desc)*vq_num */ 679 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 680 ops); 681 } 682 683 static inline void 684 vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 685 { 686 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 687 vq->vq_availoffset, 688 offsetof(struct vring_avail, ring) 689 + vq->vq_num * sizeof(uint16_t), 690 ops); 691 } 692 693 static inline void 694 vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 695 { 696 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 697 vq->vq_usedoffset, 698 offsetof(struct vring_used, ring) 699 + vq->vq_num * sizeof(struct vring_used_elem), 700 ops); 701 } 702 703 static inline void 704 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 705 int ops) 706 { 707 int offset = vq->vq_indirectoffset 708 + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 709 710 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 711 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, 712 ops); 713 } 714 715 /* 716 * Can be used as sc_intrhand. 717 */ 718 /* 719 * Scan vq, bus_dmamap_sync for the vqs (not for the payload), 720 * and calls (*vq_done)() if some entries are consumed. 721 */ 722 int 723 virtio_vq_intr(struct virtio_softc *sc) 724 { 725 struct virtqueue *vq; 726 int i, r = 0; 727 728 for (i = 0; i < sc->sc_nvqs; i++) { 729 vq = &sc->sc_vqs[i]; 730 if (vq->vq_queued) { 731 vq->vq_queued = 0; 732 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 733 } 734 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 735 membar_consumer(); 736 if (vq->vq_used_idx != vq->vq_used->idx) { 737 if (vq->vq_done) 738 r |= (vq->vq_done)(vq); 739 } 740 } 741 742 return r; 743 } 744 745 /* 746 * Start/stop vq interrupt. No guarantee. 747 */ 748 void 749 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 750 { 751 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 752 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 753 vq->vq_queued++; 754 } 755 756 void 757 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 758 { 759 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 760 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 761 vq->vq_queued++; 762 } 763 764 /* 765 * Initialize vq structure. 766 */ 767 static void 768 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, 769 const bool reinit) 770 { 771 int i, j; 772 int vq_size = vq->vq_num; 773 774 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 775 776 /* build the indirect descriptor chain */ 777 if (vq->vq_indirect != NULL) { 778 struct vring_desc *vd; 779 780 for (i = 0; i < vq_size; i++) { 781 vd = vq->vq_indirect; 782 vd += vq->vq_maxnsegs * i; 783 for (j = 0; j < vq->vq_maxnsegs-1; j++) 784 vd[j].next = j + 1; 785 } 786 } 787 788 /* free slot management */ 789 SIMPLEQ_INIT(&vq->vq_freelist); 790 for (i = 0; i < vq_size; i++) { 791 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, 792 &vq->vq_entries[i], qe_list); 793 vq->vq_entries[i].qe_index = i; 794 } 795 if (!reinit) 796 mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl); 797 798 /* enqueue/dequeue status */ 799 vq->vq_avail_idx = 0; 800 vq->vq_used_idx = 0; 801 vq->vq_queued = 0; 802 if (!reinit) { 803 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 804 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 805 } 806 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 807 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 808 vq->vq_queued++; 809 } 810 811 /* 812 * Allocate/free a vq. 813 */ 814 int 815 virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, int index, 816 int maxsegsize, int maxnsegs, const char *name) 817 { 818 int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0; 819 int rsegs, r; 820 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1))& \ 821 ~(VIRTIO_PAGE_SIZE-1)) 822 823 memset(vq, 0, sizeof(*vq)); 824 825 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 826 VIRTIO_CONFIG_QUEUE_SELECT, index); 827 vq_size = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 828 VIRTIO_CONFIG_QUEUE_SIZE); 829 if (vq_size == 0) { 830 aprint_error_dev(sc->sc_dev, 831 "virtqueue not exist, index %d for %s\n", 832 index, name); 833 goto err; 834 } 835 /* allocsize1: descriptor table + avail ring + pad */ 836 allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size 837 + sizeof(uint16_t)*(2+vq_size)); 838 /* allocsize2: used ring + pad */ 839 allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2 840 + sizeof(struct vring_used_elem)*vq_size); 841 /* allocsize3: indirect table */ 842 if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) 843 allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size; 844 else 845 allocsize3 = 0; 846 allocsize = allocsize1 + allocsize2 + allocsize3; 847 848 /* alloc and map the memory */ 849 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 850 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 851 if (r != 0) { 852 aprint_error_dev(sc->sc_dev, 853 "virtqueue %d for %s allocation failed, " 854 "error code %d\n", index, name, r); 855 goto err; 856 } 857 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize, 858 &vq->vq_vaddr, BUS_DMA_NOWAIT); 859 if (r != 0) { 860 aprint_error_dev(sc->sc_dev, 861 "virtqueue %d for %s map failed, " 862 "error code %d\n", index, name, r); 863 goto err; 864 } 865 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 866 BUS_DMA_NOWAIT, &vq->vq_dmamap); 867 if (r != 0) { 868 aprint_error_dev(sc->sc_dev, 869 "virtqueue %d for %s dmamap creation failed, " 870 "error code %d\n", index, name, r); 871 goto err; 872 } 873 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 874 vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT); 875 if (r != 0) { 876 aprint_error_dev(sc->sc_dev, 877 "virtqueue %d for %s dmamap load failed, " 878 "error code %d\n", index, name, r); 879 goto err; 880 } 881 882 /* set the vq address */ 883 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 884 VIRTIO_CONFIG_QUEUE_ADDRESS, 885 (vq->vq_dmamap->dm_segs[0].ds_addr 886 / VIRTIO_PAGE_SIZE)); 887 888 /* remember addresses and offsets for later use */ 889 vq->vq_owner = sc; 890 vq->vq_num = vq_size; 891 vq->vq_index = index; 892 vq->vq_desc = vq->vq_vaddr; 893 vq->vq_availoffset = sizeof(struct vring_desc)*vq_size; 894 vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset); 895 vq->vq_usedoffset = allocsize1; 896 vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset); 897 if (allocsize3 > 0) { 898 vq->vq_indirectoffset = allocsize1 + allocsize2; 899 vq->vq_indirect = (void*)(((char*)vq->vq_desc) 900 + vq->vq_indirectoffset); 901 } 902 vq->vq_bytesize = allocsize; 903 vq->vq_maxsegsize = maxsegsize; 904 vq->vq_maxnsegs = maxnsegs; 905 906 /* free slot management */ 907 vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size, 908 KM_NOSLEEP); 909 if (vq->vq_entries == NULL) { 910 r = ENOMEM; 911 goto err; 912 } 913 914 virtio_init_vq(sc, vq, false); 915 916 aprint_verbose_dev(sc->sc_dev, 917 "allocated %u byte for virtqueue %d for %s, " 918 "size %d\n", allocsize, index, name, vq_size); 919 if (allocsize3 > 0) 920 aprint_verbose_dev(sc->sc_dev, 921 "using %d byte (%d entries) " 922 "indirect descriptors\n", 923 allocsize3, maxnsegs * vq_size); 924 return 0; 925 926 err: 927 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 928 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 929 if (vq->vq_dmamap) 930 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 931 if (vq->vq_vaddr) 932 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 933 if (vq->vq_segs[0].ds_addr) 934 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 935 memset(vq, 0, sizeof(*vq)); 936 937 return -1; 938 } 939 940 int 941 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 942 { 943 struct vq_entry *qe; 944 int i = 0; 945 946 /* device must be already deactivated */ 947 /* confirm the vq is empty */ 948 SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) { 949 i++; 950 } 951 if (i != vq->vq_num) { 952 printf("%s: freeing non-empty vq, index %d\n", 953 device_xname(sc->sc_dev), vq->vq_index); 954 return EBUSY; 955 } 956 957 /* tell device that there's no virtqueue any longer */ 958 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 959 VIRTIO_CONFIG_QUEUE_SELECT, vq->vq_index); 960 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 961 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 962 963 kmem_free(vq->vq_entries, sizeof(*vq->vq_entries) * vq->vq_num); 964 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 965 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 966 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 967 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 968 mutex_destroy(&vq->vq_freelist_lock); 969 mutex_destroy(&vq->vq_uring_lock); 970 mutex_destroy(&vq->vq_aring_lock); 971 memset(vq, 0, sizeof(*vq)); 972 973 return 0; 974 } 975 976 /* 977 * Free descriptor management. 978 */ 979 static struct vq_entry * 980 vq_alloc_entry(struct virtqueue *vq) 981 { 982 struct vq_entry *qe; 983 984 mutex_enter(&vq->vq_freelist_lock); 985 if (SIMPLEQ_EMPTY(&vq->vq_freelist)) { 986 mutex_exit(&vq->vq_freelist_lock); 987 return NULL; 988 } 989 qe = SIMPLEQ_FIRST(&vq->vq_freelist); 990 SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list); 991 mutex_exit(&vq->vq_freelist_lock); 992 993 return qe; 994 } 995 996 static void 997 vq_free_entry(struct virtqueue *vq, struct vq_entry *qe) 998 { 999 mutex_enter(&vq->vq_freelist_lock); 1000 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list); 1001 mutex_exit(&vq->vq_freelist_lock); 1002 1003 return; 1004 } 1005 1006 /* 1007 * Enqueue several dmamaps as a single request. 1008 */ 1009 /* 1010 * Typical usage: 1011 * <queue size> number of followings are stored in arrays 1012 * - command blocks (in dmamem) should be pre-allocated and mapped 1013 * - dmamaps for command blocks should be pre-allocated and loaded 1014 * - dmamaps for payload should be pre-allocated 1015 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 1016 * if (r) // currently 0 or EAGAIN 1017 * return r; 1018 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 1019 * if (r) { 1020 * virtio_enqueue_abort(sc, vq, slot); 1021 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 1022 * return r; 1023 * } 1024 * r = virtio_enqueue_reserve(sc, vq, slot, 1025 * dmamap_payload[slot]->dm_nsegs+1); 1026 * // ^ +1 for command 1027 * if (r) { // currently 0 or EAGAIN 1028 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 1029 * return r; // do not call abort() 1030 * } 1031 * <setup and prepare commands> 1032 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 1033 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 1034 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 1035 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 1036 * virtio_enqueue_commit(sc, vq, slot, true); 1037 */ 1038 1039 /* 1040 * enqueue_prep: allocate a slot number 1041 */ 1042 int 1043 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 1044 { 1045 struct vq_entry *qe1; 1046 1047 KASSERT(slotp != NULL); 1048 1049 qe1 = vq_alloc_entry(vq); 1050 if (qe1 == NULL) 1051 return EAGAIN; 1052 /* next slot is not allocated yet */ 1053 qe1->qe_next = -1; 1054 *slotp = qe1->qe_index; 1055 1056 return 0; 1057 } 1058 1059 /* 1060 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 1061 */ 1062 int 1063 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 1064 int slot, int nsegs) 1065 { 1066 int indirect; 1067 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1068 1069 KASSERT(qe1->qe_next == -1); 1070 KASSERT(1 <= nsegs && nsegs <= vq->vq_num); 1071 1072 if ((vq->vq_indirect != NULL) && 1073 (nsegs >= MINSEG_INDIRECT) && 1074 (nsegs <= vq->vq_maxnsegs)) 1075 indirect = 1; 1076 else 1077 indirect = 0; 1078 qe1->qe_indirect = indirect; 1079 1080 if (indirect) { 1081 struct vring_desc *vd; 1082 int i; 1083 1084 vd = &vq->vq_desc[qe1->qe_index]; 1085 vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr 1086 + vq->vq_indirectoffset; 1087 vd->addr += sizeof(struct vring_desc) 1088 * vq->vq_maxnsegs * qe1->qe_index; 1089 vd->len = sizeof(struct vring_desc) * nsegs; 1090 vd->flags = VRING_DESC_F_INDIRECT; 1091 1092 vd = vq->vq_indirect; 1093 vd += vq->vq_maxnsegs * qe1->qe_index; 1094 qe1->qe_desc_base = vd; 1095 1096 for (i = 0; i < nsegs-1; i++) { 1097 vd[i].flags = VRING_DESC_F_NEXT; 1098 } 1099 vd[i].flags = 0; 1100 qe1->qe_next = 0; 1101 1102 return 0; 1103 } else { 1104 struct vring_desc *vd; 1105 struct vq_entry *qe; 1106 int i, s; 1107 1108 vd = &vq->vq_desc[0]; 1109 qe1->qe_desc_base = vd; 1110 qe1->qe_next = qe1->qe_index; 1111 s = slot; 1112 for (i = 0; i < nsegs - 1; i++) { 1113 qe = vq_alloc_entry(vq); 1114 if (qe == NULL) { 1115 vd[s].flags = 0; 1116 virtio_enqueue_abort(sc, vq, slot); 1117 return EAGAIN; 1118 } 1119 vd[s].flags = VRING_DESC_F_NEXT; 1120 vd[s].next = qe->qe_index; 1121 s = qe->qe_index; 1122 } 1123 vd[s].flags = 0; 1124 1125 return 0; 1126 } 1127 } 1128 1129 /* 1130 * enqueue: enqueue a single dmamap. 1131 */ 1132 int 1133 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1134 bus_dmamap_t dmamap, bool write) 1135 { 1136 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1137 struct vring_desc *vd = qe1->qe_desc_base; 1138 int i; 1139 int s = qe1->qe_next; 1140 1141 KASSERT(s >= 0); 1142 KASSERT(dmamap->dm_nsegs > 0); 1143 1144 for (i = 0; i < dmamap->dm_nsegs; i++) { 1145 vd[s].addr = dmamap->dm_segs[i].ds_addr; 1146 vd[s].len = dmamap->dm_segs[i].ds_len; 1147 if (!write) 1148 vd[s].flags |= VRING_DESC_F_WRITE; 1149 s = vd[s].next; 1150 } 1151 qe1->qe_next = s; 1152 1153 return 0; 1154 } 1155 1156 int 1157 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1158 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 1159 bool write) 1160 { 1161 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1162 struct vring_desc *vd = qe1->qe_desc_base; 1163 int s = qe1->qe_next; 1164 1165 KASSERT(s >= 0); 1166 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 1167 KASSERT((dmamap->dm_segs[0].ds_len > start) && 1168 (dmamap->dm_segs[0].ds_len >= start + len)); 1169 1170 vd[s].addr = dmamap->dm_segs[0].ds_addr + start; 1171 vd[s].len = len; 1172 if (!write) 1173 vd[s].flags |= VRING_DESC_F_WRITE; 1174 qe1->qe_next = vd[s].next; 1175 1176 return 0; 1177 } 1178 1179 /* 1180 * enqueue_commit: add it to the aring. 1181 */ 1182 int 1183 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1184 bool notifynow) 1185 { 1186 struct vq_entry *qe1; 1187 1188 if (slot < 0) { 1189 mutex_enter(&vq->vq_aring_lock); 1190 goto notify; 1191 } 1192 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 1193 qe1 = &vq->vq_entries[slot]; 1194 if (qe1->qe_indirect) 1195 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 1196 mutex_enter(&vq->vq_aring_lock); 1197 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot; 1198 1199 notify: 1200 if (notifynow) { 1201 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 1202 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 1203 membar_producer(); 1204 vq->vq_avail->idx = vq->vq_avail_idx; 1205 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 1206 membar_producer(); 1207 vq->vq_queued++; 1208 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 1209 membar_consumer(); 1210 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) 1211 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 1212 VIRTIO_CONFIG_QUEUE_NOTIFY, 1213 vq->vq_index); 1214 } 1215 mutex_exit(&vq->vq_aring_lock); 1216 1217 return 0; 1218 } 1219 1220 /* 1221 * enqueue_abort: rollback. 1222 */ 1223 int 1224 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1225 { 1226 struct vq_entry *qe = &vq->vq_entries[slot]; 1227 struct vring_desc *vd; 1228 int s; 1229 1230 if (qe->qe_next < 0) { 1231 vq_free_entry(vq, qe); 1232 return 0; 1233 } 1234 1235 s = slot; 1236 vd = &vq->vq_desc[0]; 1237 while (vd[s].flags & VRING_DESC_F_NEXT) { 1238 s = vd[s].next; 1239 vq_free_entry(vq, qe); 1240 qe = &vq->vq_entries[s]; 1241 } 1242 vq_free_entry(vq, qe); 1243 return 0; 1244 } 1245 1246 /* 1247 * Dequeue a request. 1248 */ 1249 /* 1250 * dequeue: dequeue a request from uring; dmamap_sync for uring is 1251 * already done in the interrupt handler. 1252 */ 1253 int 1254 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 1255 int *slotp, int *lenp) 1256 { 1257 uint16_t slot, usedidx; 1258 struct vq_entry *qe; 1259 1260 if (vq->vq_used_idx == vq->vq_used->idx) 1261 return ENOENT; 1262 mutex_enter(&vq->vq_uring_lock); 1263 usedidx = vq->vq_used_idx++; 1264 mutex_exit(&vq->vq_uring_lock); 1265 usedidx %= vq->vq_num; 1266 slot = vq->vq_used->ring[usedidx].id; 1267 qe = &vq->vq_entries[slot]; 1268 1269 if (qe->qe_indirect) 1270 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 1271 1272 if (slotp) 1273 *slotp = slot; 1274 if (lenp) 1275 *lenp = vq->vq_used->ring[usedidx].len; 1276 1277 return 0; 1278 } 1279 1280 /* 1281 * dequeue_commit: complete dequeue; the slot is recycled for future use. 1282 * if you forget to call this the slot will be leaked. 1283 */ 1284 int 1285 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1286 { 1287 struct vq_entry *qe = &vq->vq_entries[slot]; 1288 struct vring_desc *vd = &vq->vq_desc[0]; 1289 int s = slot; 1290 1291 while (vd[s].flags & VRING_DESC_F_NEXT) { 1292 s = vd[s].next; 1293 vq_free_entry(vq, qe); 1294 qe = &vq->vq_entries[s]; 1295 } 1296 vq_free_entry(vq, qe); 1297 1298 return 0; 1299 } 1300 1301 MODULE(MODULE_CLASS_DRIVER, virtio, "pci"); 1302 1303 #ifdef _MODULE 1304 #include "ioconf.c" 1305 #endif 1306 1307 static int 1308 virtio_modcmd(modcmd_t cmd, void *opaque) 1309 { 1310 int error = 0; 1311 1312 #ifdef _MODULE 1313 switch (cmd) { 1314 case MODULE_CMD_INIT: 1315 error = config_init_component(cfdriver_ioconf_virtio, 1316 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1317 break; 1318 case MODULE_CMD_FINI: 1319 error = config_fini_component(cfdriver_ioconf_virtio, 1320 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1321 break; 1322 default: 1323 error = ENOTTY; 1324 break; 1325 } 1326 #endif 1327 1328 return error; 1329 } 1330