1 /* $NetBSD: virtio.c,v 1.14 2016/01/10 03:07:25 christos Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.14 2016/01/10 03:07:25 christos Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/atomic.h> 35 #include <sys/bus.h> 36 #include <sys/device.h> 37 #include <sys/kmem.h> 38 39 #include <dev/pci/pcidevs.h> 40 #include <dev/pci/pcireg.h> 41 #include <dev/pci/pcivar.h> 42 43 #include <dev/pci/virtioreg.h> 44 #include <dev/pci/virtiovar.h> 45 46 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 47 48 static int virtio_match(device_t, cfdata_t, void *); 49 static void virtio_attach(device_t, device_t, void *); 50 static int virtio_detach(device_t, int); 51 static int virtio_intr(void *arg); 52 static int virtio_msix_queue_intr(void *); 53 static int virtio_msix_config_intr(void *); 54 static int virtio_setup_msix_vectors(struct virtio_softc *); 55 static int virtio_setup_msix_interrupts(struct virtio_softc *, 56 struct pci_attach_args *); 57 static int virtio_setup_intx_interrupt(struct virtio_softc *, 58 struct pci_attach_args *); 59 static int virtio_setup_interrupts(struct virtio_softc *, 60 struct pci_attach_args *); 61 static void virtio_soft_intr(void *arg); 62 static void virtio_init_vq(struct virtio_softc *, 63 struct virtqueue *, const bool); 64 65 CFATTACH_DECL3_NEW(virtio, sizeof(struct virtio_softc), 66 virtio_match, virtio_attach, virtio_detach, NULL, NULL, NULL, 67 DVF_DETACH_SHUTDOWN); 68 69 static void 70 virtio_set_status(struct virtio_softc *sc, int status) 71 { 72 int old = 0; 73 74 if (status != 0) 75 old = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 76 VIRTIO_CONFIG_DEVICE_STATUS); 77 bus_space_write_1(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_DEVICE_STATUS, 78 status|old); 79 } 80 81 #define virtio_device_reset(sc) virtio_set_status((sc), 0) 82 83 static int 84 virtio_match(device_t parent, cfdata_t match, void *aux) 85 { 86 struct pci_attach_args *pa; 87 88 pa = (struct pci_attach_args *)aux; 89 switch (PCI_VENDOR(pa->pa_id)) { 90 case PCI_VENDOR_QUMRANET: 91 if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <= 92 PCI_PRODUCT(pa->pa_id)) && 93 (PCI_PRODUCT(pa->pa_id) <= 94 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) 95 return 1; 96 break; 97 } 98 99 return 0; 100 } 101 102 static const char *virtio_device_name[] = { 103 "Unknown (0)", /* 0 */ 104 "Network", /* 1 */ 105 "Block", /* 2 */ 106 "Console", /* 3 */ 107 "Entropy", /* 4 */ 108 "Memory Balloon", /* 5 */ 109 "I/O Memory", /* 6 */ 110 "Remote Processor Messaging", /* 7 */ 111 "SCSI", /* 8 */ 112 "9P Transport", /* 9 */ 113 "mac80211 wlan", /* 10 */ 114 }; 115 #define NDEVNAMES __arraycount(virtio_device_name) 116 117 #define VIRTIO_MSIX_CONFIG_VECTOR_INDEX 0 118 #define VIRTIO_MSIX_QUEUE_VECTOR_INDEX 1 119 120 static int 121 virtio_setup_msix_vectors(struct virtio_softc *sc) 122 { 123 int offset, vector, ret, qid; 124 125 offset = VIRTIO_CONFIG_MSI_CONFIG_VECTOR; 126 vector = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 127 128 bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, vector); 129 ret = bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset); 130 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 131 vector, ret); 132 if (ret != vector) 133 return -1; 134 135 for (qid = 0; qid < sc->sc_nvqs; qid++) { 136 offset = VIRTIO_CONFIG_QUEUE_SELECT; 137 bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, qid); 138 139 offset = VIRTIO_CONFIG_MSI_QUEUE_VECTOR; 140 vector = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 141 142 bus_space_write_2(sc->sc_iot, sc->sc_ioh, offset, vector); 143 ret = bus_space_read_2(sc->sc_iot, sc->sc_ioh, offset); 144 aprint_debug_dev(sc->sc_dev, "expected=%d, actual=%d\n", 145 vector, ret); 146 if (ret != vector) 147 return -1; 148 } 149 150 return 0; 151 } 152 153 static int 154 virtio_setup_msix_interrupts(struct virtio_softc *sc, 155 struct pci_attach_args *pa) 156 { 157 device_t self = sc->sc_dev; 158 pci_chipset_tag_t pc = pa->pa_pc; 159 char intrbuf[PCI_INTRSTR_LEN]; 160 char const *intrstr; 161 int idx; 162 163 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 164 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 165 pci_intr_setattr(pc, &sc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 166 167 sc->sc_ihs[idx] = pci_intr_establish_xname(pc, sc->sc_ihp[idx], IPL_NET, 168 virtio_msix_config_intr, sc, device_xname(sc->sc_dev)); 169 if (sc->sc_ihs[idx] == NULL) { 170 aprint_error_dev(self, "couldn't establish MSI-X for config\n"); 171 goto error; 172 } 173 174 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 175 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 176 pci_intr_setattr(pc, &sc->sc_ihp[idx], PCI_INTR_MPSAFE, true); 177 178 sc->sc_ihs[idx] = pci_intr_establish_xname(pc, sc->sc_ihp[idx], IPL_NET, 179 virtio_msix_queue_intr, sc, device_xname(sc->sc_dev)); 180 if (sc->sc_ihs[idx] == NULL) { 181 aprint_error_dev(self, "couldn't establish MSI-X for queues\n"); 182 goto error; 183 } 184 185 if (virtio_setup_msix_vectors(sc) != 0) { 186 aprint_error_dev(self, "couldn't setup MSI-X vectors\n"); 187 goto error; 188 } 189 190 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 191 intrstr = pci_intr_string(pc, sc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 192 aprint_normal_dev(self, "config interrupting at %s\n", intrstr); 193 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 194 intrstr = pci_intr_string(pc, sc->sc_ihp[idx], intrbuf, sizeof(intrbuf)); 195 aprint_normal_dev(self, "queues interrupting at %s\n", intrstr); 196 197 return 0; 198 199 error: 200 idx = VIRTIO_MSIX_CONFIG_VECTOR_INDEX; 201 if (sc->sc_ihs[idx] != NULL) 202 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[idx]); 203 idx = VIRTIO_MSIX_QUEUE_VECTOR_INDEX; 204 if (sc->sc_ihs[idx] != NULL) 205 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[idx]); 206 207 return -1; 208 } 209 210 static int 211 virtio_setup_intx_interrupt(struct virtio_softc *sc, struct pci_attach_args *pa) 212 { 213 device_t self = sc->sc_dev; 214 pci_chipset_tag_t pc = pa->pa_pc; 215 char intrbuf[PCI_INTRSTR_LEN]; 216 char const *intrstr; 217 218 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 219 pci_intr_setattr(pc, &sc->sc_ihp[0], PCI_INTR_MPSAFE, true); 220 221 sc->sc_ihs[0] = pci_intr_establish_xname(pc, sc->sc_ihp[0], 222 IPL_NET, virtio_intr, sc, device_xname(sc->sc_dev)); 223 if (sc->sc_ihs[0] == NULL) { 224 aprint_error_dev(self, "couldn't establish INTx\n"); 225 return -1; 226 } 227 228 intrstr = pci_intr_string(pc, sc->sc_ihp[0], intrbuf, sizeof(intrbuf)); 229 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 230 231 return 0; 232 } 233 234 static int 235 virtio_setup_interrupts(struct virtio_softc *sc, struct pci_attach_args *pa) 236 { 237 device_t self = sc->sc_dev; 238 pci_chipset_tag_t pc = pa->pa_pc; 239 int error; 240 int nmsix; 241 int counts[PCI_INTR_TYPE_SIZE]; 242 pci_intr_type_t max_type; 243 244 nmsix = pci_msix_count(pa->pa_pc, pa->pa_tag); 245 aprint_debug_dev(self, "pci_msix_count=%d\n", nmsix); 246 247 /* We need at least two: one for config and the other for queues */ 248 if ((sc->sc_flags & VIRTIO_F_PCI_INTR_MSIX) == 0 || nmsix < 2) { 249 /* Try INTx only */ 250 max_type = PCI_INTR_TYPE_INTX; 251 counts[PCI_INTR_TYPE_INTX] = 1; 252 } else { 253 /* Try MSI-X first and INTx second */ 254 max_type = PCI_INTR_TYPE_MSIX; 255 counts[PCI_INTR_TYPE_MSIX] = 2; 256 counts[PCI_INTR_TYPE_MSI] = 0; 257 counts[PCI_INTR_TYPE_INTX] = 1; 258 } 259 260 retry: 261 error = pci_intr_alloc(pa, &sc->sc_ihp, counts, max_type); 262 if (error != 0) { 263 aprint_error_dev(self, "couldn't map interrupt\n"); 264 return -1; 265 } 266 267 if (pci_intr_type(sc->sc_ihp[0]) == PCI_INTR_TYPE_MSIX) { 268 sc->sc_ihs = kmem_alloc(sizeof(*sc->sc_ihs) * 2, 269 KM_SLEEP); 270 if (sc->sc_ihs == NULL) { 271 pci_intr_release(pc, sc->sc_ihp, 2); 272 273 /* Retry INTx */ 274 max_type = PCI_INTR_TYPE_INTX; 275 counts[PCI_INTR_TYPE_INTX] = 1; 276 goto retry; 277 } 278 279 error = virtio_setup_msix_interrupts(sc, pa); 280 if (error != 0) { 281 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * 2); 282 pci_intr_release(pc, sc->sc_ihp, 2); 283 284 /* Retry INTx */ 285 max_type = PCI_INTR_TYPE_INTX; 286 counts[PCI_INTR_TYPE_INTX] = 1; 287 goto retry; 288 } 289 290 sc->sc_ihs_num = 2; 291 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI; 292 } else if (pci_intr_type(sc->sc_ihp[0]) == PCI_INTR_TYPE_INTX) { 293 sc->sc_ihs = kmem_alloc(sizeof(*sc->sc_ihs) * 1, 294 KM_SLEEP); 295 if (sc->sc_ihs == NULL) { 296 pci_intr_release(pc, sc->sc_ihp, 1); 297 return -1; 298 } 299 300 error = virtio_setup_intx_interrupt(sc, pa); 301 if (error != 0) { 302 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * 1); 303 pci_intr_release(pc, sc->sc_ihp, 1); 304 return -1; 305 } 306 307 sc->sc_ihs_num = 1; 308 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 309 } 310 311 return 0; 312 } 313 314 static void 315 virtio_attach(device_t parent, device_t self, void *aux) 316 { 317 struct virtio_softc *sc = device_private(self); 318 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 319 pci_chipset_tag_t pc = pa->pa_pc; 320 pcitag_t tag = pa->pa_tag; 321 int revision; 322 pcireg_t id; 323 int r; 324 325 revision = PCI_REVISION(pa->pa_class); 326 if (revision != 0) { 327 aprint_normal(": unknown revision 0x%02x; giving up\n", 328 revision); 329 return; 330 } 331 aprint_normal("\n"); 332 aprint_naive("\n"); 333 334 /* subsystem ID shows what I am */ 335 id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 336 aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n", 337 (PCI_SUBSYS_ID(id) < NDEVNAMES? 338 virtio_device_name[PCI_SUBSYS_ID(id)] : "Unknown"), 339 revision); 340 341 sc->sc_dev = self; 342 sc->sc_pc = pc; 343 sc->sc_tag = tag; 344 sc->sc_iot = pa->pa_iot; 345 if (pci_dma64_available(pa)) 346 sc->sc_dmat = pa->pa_dmat64; 347 else 348 sc->sc_dmat = pa->pa_dmat; 349 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 350 351 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, 352 &sc->sc_iot, &sc->sc_ioh, NULL, &sc->sc_iosize)) { 353 aprint_error_dev(self, "can't map i/o space\n"); 354 return; 355 } 356 357 virtio_device_reset(sc); 358 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 359 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 360 361 /* XXX: use softc as aux... */ 362 sc->sc_childdevid = PCI_SUBSYS_ID(id); 363 sc->sc_child = NULL; 364 config_found(self, sc, NULL); 365 if (sc->sc_child == NULL) { 366 aprint_error_dev(self, 367 "no matching child driver; not configured\n"); 368 return; 369 } 370 if (sc->sc_child == (void*)1) { /* this shows error */ 371 aprint_error_dev(self, 372 "virtio configuration failed\n"); 373 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 374 return; 375 } 376 377 r = virtio_setup_interrupts(sc, pa); 378 if (r != 0) { 379 aprint_error_dev(self, "failed to setup interrupts\n"); 380 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 381 return; 382 } 383 384 sc->sc_soft_ih = NULL; 385 if (sc->sc_flags & VIRTIO_F_PCI_INTR_SOFTINT) { 386 u_int flags = SOFTINT_NET; 387 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 388 flags |= SOFTINT_MPSAFE; 389 390 sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr, sc); 391 if (sc->sc_soft_ih == NULL) 392 aprint_error(": failed to establish soft interrupt\n"); 393 } 394 395 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 396 397 return; 398 } 399 400 static int 401 virtio_detach(device_t self, int flags) 402 { 403 struct virtio_softc *sc = device_private(self); 404 int r; 405 int i; 406 407 if (sc->sc_child != 0 && sc->sc_child != (void*)1) { 408 r = config_detach(sc->sc_child, flags); 409 if (r) 410 return r; 411 } 412 KASSERT(sc->sc_child == 0 || sc->sc_child == (void*)1); 413 KASSERT(sc->sc_vqs == 0); 414 for (i = 0; i < sc->sc_ihs_num; i++) { 415 if (sc->sc_ihs[i] == NULL) 416 continue; 417 pci_intr_disestablish(sc->sc_pc, sc->sc_ihs[i]); 418 } 419 pci_intr_release(sc->sc_pc, sc->sc_ihp, sc->sc_ihs_num); 420 kmem_free(sc->sc_ihs, sizeof(*sc->sc_ihs) * sc->sc_ihs_num); 421 sc->sc_ihs_num = 0; 422 if (sc->sc_iosize) 423 bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize); 424 sc->sc_iosize = 0; 425 426 return 0; 427 } 428 429 /* 430 * Reset the device. 431 */ 432 /* 433 * To reset the device to a known state, do following: 434 * virtio_reset(sc); // this will stop the device activity 435 * <dequeue finished requests>; // virtio_dequeue() still can be called 436 * <revoke pending requests in the vqs if any>; 437 * virtio_reinit_begin(sc); // dequeue prohibitted 438 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 439 * <some other initialization>; 440 * virtio_reinit_end(sc); // device activated; enqueue allowed 441 * Once attached, feature negotiation can only be allowed after virtio_reset. 442 */ 443 void 444 virtio_reset(struct virtio_softc *sc) 445 { 446 virtio_device_reset(sc); 447 } 448 449 void 450 virtio_reinit_start(struct virtio_softc *sc) 451 { 452 int i; 453 454 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 455 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 456 for (i = 0; i < sc->sc_nvqs; i++) { 457 int n; 458 struct virtqueue *vq = &sc->sc_vqs[i]; 459 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 460 VIRTIO_CONFIG_QUEUE_SELECT, 461 vq->vq_index); 462 n = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 463 VIRTIO_CONFIG_QUEUE_SIZE); 464 if (n == 0) /* vq disappeared */ 465 continue; 466 if (n != vq->vq_num) { 467 panic("%s: virtqueue size changed, vq index %d\n", 468 device_xname(sc->sc_dev), 469 vq->vq_index); 470 } 471 virtio_init_vq(sc, vq, true); 472 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 473 VIRTIO_CONFIG_QUEUE_ADDRESS, 474 (vq->vq_dmamap->dm_segs[0].ds_addr 475 / VIRTIO_PAGE_SIZE)); 476 } 477 478 /* MSI-X should have more than one handles where INTx has just one */ 479 if (sc->sc_ihs_num > 1) { 480 if (virtio_setup_msix_vectors(sc) != 0) { 481 aprint_error_dev(sc->sc_dev, "couldn't setup MSI-X vectors\n"); 482 return; 483 } 484 } 485 } 486 487 void 488 virtio_reinit_end(struct virtio_softc *sc) 489 { 490 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 491 } 492 493 /* 494 * Feature negotiation. 495 */ 496 uint32_t 497 virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features) 498 { 499 uint32_t r; 500 501 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 502 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 503 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 504 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 505 VIRTIO_CONFIG_DEVICE_FEATURES); 506 r &= guest_features; 507 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 508 VIRTIO_CONFIG_GUEST_FEATURES, r); 509 sc->sc_features = r; 510 if (r & VIRTIO_F_RING_INDIRECT_DESC) 511 sc->sc_indirect = true; 512 else 513 sc->sc_indirect = false; 514 515 return r; 516 } 517 518 /* 519 * Device configuration registers. 520 */ 521 uint8_t 522 virtio_read_device_config_1(struct virtio_softc *sc, int index) 523 { 524 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, 525 sc->sc_config_offset + index); 526 } 527 528 uint16_t 529 virtio_read_device_config_2(struct virtio_softc *sc, int index) 530 { 531 return bus_space_read_2(sc->sc_iot, sc->sc_ioh, 532 sc->sc_config_offset + index); 533 } 534 535 uint32_t 536 virtio_read_device_config_4(struct virtio_softc *sc, int index) 537 { 538 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, 539 sc->sc_config_offset + index); 540 } 541 542 uint64_t 543 virtio_read_device_config_8(struct virtio_softc *sc, int index) 544 { 545 uint64_t r; 546 547 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 548 sc->sc_config_offset + index + sizeof(uint32_t)); 549 r <<= 32; 550 r += bus_space_read_4(sc->sc_iot, sc->sc_ioh, 551 sc->sc_config_offset + index); 552 return r; 553 } 554 555 void 556 virtio_write_device_config_1(struct virtio_softc *sc, 557 int index, uint8_t value) 558 { 559 bus_space_write_1(sc->sc_iot, sc->sc_ioh, 560 sc->sc_config_offset + index, value); 561 } 562 563 void 564 virtio_write_device_config_2(struct virtio_softc *sc, 565 int index, uint16_t value) 566 { 567 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 568 sc->sc_config_offset + index, value); 569 } 570 571 void 572 virtio_write_device_config_4(struct virtio_softc *sc, 573 int index, uint32_t value) 574 { 575 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 576 sc->sc_config_offset + index, value); 577 } 578 579 void 580 virtio_write_device_config_8(struct virtio_softc *sc, 581 int index, uint64_t value) 582 { 583 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 584 sc->sc_config_offset + index, 585 value & 0xffffffff); 586 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 587 sc->sc_config_offset + index + sizeof(uint32_t), 588 value >> 32); 589 } 590 591 /* 592 * Interrupt handler. 593 */ 594 static int 595 virtio_intr(void *arg) 596 { 597 struct virtio_softc *sc = arg; 598 int isr, r = 0; 599 600 /* check and ack the interrupt */ 601 isr = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 602 VIRTIO_CONFIG_ISR_STATUS); 603 if (isr == 0) 604 return 0; 605 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 606 (sc->sc_config_change != NULL)) 607 r = (sc->sc_config_change)(sc); 608 if (sc->sc_intrhand != NULL) { 609 if (sc->sc_soft_ih != NULL) 610 softint_schedule(sc->sc_soft_ih); 611 else 612 r |= (sc->sc_intrhand)(sc); 613 } 614 615 return r; 616 } 617 618 static int 619 virtio_msix_queue_intr(void *arg) 620 { 621 struct virtio_softc *sc = arg; 622 int r = 0; 623 624 if (sc->sc_intrhand != NULL) { 625 if (sc->sc_soft_ih != NULL) 626 softint_schedule(sc->sc_soft_ih); 627 else 628 r |= (sc->sc_intrhand)(sc); 629 } 630 631 return r; 632 } 633 634 static int 635 virtio_msix_config_intr(void *arg) 636 { 637 struct virtio_softc *sc = arg; 638 639 /* TODO: handle events */ 640 aprint_debug_dev(sc->sc_dev, "%s\n", __func__); 641 return 1; 642 } 643 644 static void 645 virtio_soft_intr(void *arg) 646 { 647 struct virtio_softc *sc = arg; 648 649 KASSERT(sc->sc_intrhand != NULL); 650 651 (sc->sc_intrhand)(sc); 652 } 653 654 /* 655 * dmamap sync operations for a virtqueue. 656 */ 657 static inline void 658 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 659 { 660 /* availoffset == sizeof(vring_desc)*vq_num */ 661 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 662 ops); 663 } 664 665 static inline void 666 vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 667 { 668 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 669 vq->vq_availoffset, 670 offsetof(struct vring_avail, ring) 671 + vq->vq_num * sizeof(uint16_t), 672 ops); 673 } 674 675 static inline void 676 vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 677 { 678 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 679 vq->vq_usedoffset, 680 offsetof(struct vring_used, ring) 681 + vq->vq_num * sizeof(struct vring_used_elem), 682 ops); 683 } 684 685 static inline void 686 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 687 int ops) 688 { 689 int offset = vq->vq_indirectoffset 690 + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 691 692 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 693 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, 694 ops); 695 } 696 697 /* 698 * Can be used as sc_intrhand. 699 */ 700 /* 701 * Scan vq, bus_dmamap_sync for the vqs (not for the payload), 702 * and calls (*vq_done)() if some entries are consumed. 703 */ 704 int 705 virtio_vq_intr(struct virtio_softc *sc) 706 { 707 struct virtqueue *vq; 708 int i, r = 0; 709 710 for (i = 0; i < sc->sc_nvqs; i++) { 711 vq = &sc->sc_vqs[i]; 712 if (vq->vq_queued) { 713 vq->vq_queued = 0; 714 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 715 } 716 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 717 membar_consumer(); 718 if (vq->vq_used_idx != vq->vq_used->idx) { 719 if (vq->vq_done) 720 r |= (vq->vq_done)(vq); 721 } 722 } 723 724 return r; 725 } 726 727 /* 728 * Start/stop vq interrupt. No guarantee. 729 */ 730 void 731 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 732 { 733 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 734 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 735 vq->vq_queued++; 736 } 737 738 void 739 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 740 { 741 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 742 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 743 vq->vq_queued++; 744 } 745 746 /* 747 * Initialize vq structure. 748 */ 749 static void 750 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, const bool reinit) 751 { 752 int i, j; 753 int vq_size = vq->vq_num; 754 755 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 756 757 /* build the indirect descriptor chain */ 758 if (vq->vq_indirect != NULL) { 759 struct vring_desc *vd; 760 761 for (i = 0; i < vq_size; i++) { 762 vd = vq->vq_indirect; 763 vd += vq->vq_maxnsegs * i; 764 for (j = 0; j < vq->vq_maxnsegs-1; j++) 765 vd[j].next = j + 1; 766 } 767 } 768 769 /* free slot management */ 770 SIMPLEQ_INIT(&vq->vq_freelist); 771 for (i = 0; i < vq_size; i++) { 772 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, 773 &vq->vq_entries[i], qe_list); 774 vq->vq_entries[i].qe_index = i; 775 } 776 if (!reinit) 777 mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl); 778 779 /* enqueue/dequeue status */ 780 vq->vq_avail_idx = 0; 781 vq->vq_used_idx = 0; 782 vq->vq_queued = 0; 783 if (!reinit) { 784 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 785 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 786 } 787 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 788 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 789 vq->vq_queued++; 790 } 791 792 /* 793 * Allocate/free a vq. 794 */ 795 int 796 virtio_alloc_vq(struct virtio_softc *sc, 797 struct virtqueue *vq, int index, int maxsegsize, int maxnsegs, 798 const char *name) 799 { 800 int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0; 801 int rsegs, r; 802 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1))& \ 803 ~(VIRTIO_PAGE_SIZE-1)) 804 805 memset(vq, 0, sizeof(*vq)); 806 807 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 808 VIRTIO_CONFIG_QUEUE_SELECT, index); 809 vq_size = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 810 VIRTIO_CONFIG_QUEUE_SIZE); 811 if (vq_size == 0) { 812 aprint_error_dev(sc->sc_dev, 813 "virtqueue not exist, index %d for %s\n", 814 index, name); 815 goto err; 816 } 817 /* allocsize1: descriptor table + avail ring + pad */ 818 allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size 819 + sizeof(uint16_t)*(2+vq_size)); 820 /* allocsize2: used ring + pad */ 821 allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2 822 + sizeof(struct vring_used_elem)*vq_size); 823 /* allocsize3: indirect table */ 824 if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) 825 allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size; 826 else 827 allocsize3 = 0; 828 allocsize = allocsize1 + allocsize2 + allocsize3; 829 830 /* alloc and map the memory */ 831 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 832 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 833 if (r != 0) { 834 aprint_error_dev(sc->sc_dev, 835 "virtqueue %d for %s allocation failed, " 836 "error code %d\n", index, name, r); 837 goto err; 838 } 839 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize, 840 &vq->vq_vaddr, BUS_DMA_NOWAIT); 841 if (r != 0) { 842 aprint_error_dev(sc->sc_dev, 843 "virtqueue %d for %s map failed, " 844 "error code %d\n", index, name, r); 845 goto err; 846 } 847 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 848 BUS_DMA_NOWAIT, &vq->vq_dmamap); 849 if (r != 0) { 850 aprint_error_dev(sc->sc_dev, 851 "virtqueue %d for %s dmamap creation failed, " 852 "error code %d\n", index, name, r); 853 goto err; 854 } 855 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 856 vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT); 857 if (r != 0) { 858 aprint_error_dev(sc->sc_dev, 859 "virtqueue %d for %s dmamap load failed, " 860 "error code %d\n", index, name, r); 861 goto err; 862 } 863 864 /* set the vq address */ 865 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 866 VIRTIO_CONFIG_QUEUE_ADDRESS, 867 (vq->vq_dmamap->dm_segs[0].ds_addr 868 / VIRTIO_PAGE_SIZE)); 869 870 /* remember addresses and offsets for later use */ 871 vq->vq_owner = sc; 872 vq->vq_num = vq_size; 873 vq->vq_index = index; 874 vq->vq_desc = vq->vq_vaddr; 875 vq->vq_availoffset = sizeof(struct vring_desc)*vq_size; 876 vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset); 877 vq->vq_usedoffset = allocsize1; 878 vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset); 879 if (allocsize3 > 0) { 880 vq->vq_indirectoffset = allocsize1 + allocsize2; 881 vq->vq_indirect = (void*)(((char*)vq->vq_desc) 882 + vq->vq_indirectoffset); 883 } 884 vq->vq_bytesize = allocsize; 885 vq->vq_maxsegsize = maxsegsize; 886 vq->vq_maxnsegs = maxnsegs; 887 888 /* free slot management */ 889 vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size, 890 KM_NOSLEEP); 891 if (vq->vq_entries == NULL) { 892 r = ENOMEM; 893 goto err; 894 } 895 896 virtio_init_vq(sc, vq, false); 897 898 aprint_verbose_dev(sc->sc_dev, 899 "allocated %u byte for virtqueue %d for %s, " 900 "size %d\n", allocsize, index, name, vq_size); 901 if (allocsize3 > 0) 902 aprint_verbose_dev(sc->sc_dev, 903 "using %d byte (%d entries) " 904 "indirect descriptors\n", 905 allocsize3, maxnsegs * vq_size); 906 return 0; 907 908 err: 909 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 910 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 911 if (vq->vq_dmamap) 912 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 913 if (vq->vq_vaddr) 914 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 915 if (vq->vq_segs[0].ds_addr) 916 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 917 memset(vq, 0, sizeof(*vq)); 918 919 return -1; 920 } 921 922 int 923 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 924 { 925 struct vq_entry *qe; 926 int i = 0; 927 928 /* device must be already deactivated */ 929 /* confirm the vq is empty */ 930 SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) { 931 i++; 932 } 933 if (i != vq->vq_num) { 934 printf("%s: freeing non-empty vq, index %d\n", 935 device_xname(sc->sc_dev), vq->vq_index); 936 return EBUSY; 937 } 938 939 /* tell device that there's no virtqueue any longer */ 940 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 941 VIRTIO_CONFIG_QUEUE_SELECT, vq->vq_index); 942 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 943 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 944 945 kmem_free(vq->vq_entries, sizeof(*vq->vq_entries) * vq->vq_num); 946 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 947 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 948 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 949 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 950 mutex_destroy(&vq->vq_freelist_lock); 951 mutex_destroy(&vq->vq_uring_lock); 952 mutex_destroy(&vq->vq_aring_lock); 953 memset(vq, 0, sizeof(*vq)); 954 955 return 0; 956 } 957 958 /* 959 * Free descriptor management. 960 */ 961 static struct vq_entry * 962 vq_alloc_entry(struct virtqueue *vq) 963 { 964 struct vq_entry *qe; 965 966 mutex_enter(&vq->vq_freelist_lock); 967 if (SIMPLEQ_EMPTY(&vq->vq_freelist)) { 968 mutex_exit(&vq->vq_freelist_lock); 969 return NULL; 970 } 971 qe = SIMPLEQ_FIRST(&vq->vq_freelist); 972 SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list); 973 mutex_exit(&vq->vq_freelist_lock); 974 975 return qe; 976 } 977 978 static void 979 vq_free_entry(struct virtqueue *vq, struct vq_entry *qe) 980 { 981 mutex_enter(&vq->vq_freelist_lock); 982 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list); 983 mutex_exit(&vq->vq_freelist_lock); 984 985 return; 986 } 987 988 /* 989 * Enqueue several dmamaps as a single request. 990 */ 991 /* 992 * Typical usage: 993 * <queue size> number of followings are stored in arrays 994 * - command blocks (in dmamem) should be pre-allocated and mapped 995 * - dmamaps for command blocks should be pre-allocated and loaded 996 * - dmamaps for payload should be pre-allocated 997 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 998 * if (r) // currently 0 or EAGAIN 999 * return r; 1000 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 1001 * if (r) { 1002 * virtio_enqueue_abort(sc, vq, slot); 1003 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 1004 * return r; 1005 * } 1006 * r = virtio_enqueue_reserve(sc, vq, slot, 1007 * dmamap_payload[slot]->dm_nsegs+1); 1008 * // ^ +1 for command 1009 * if (r) { // currently 0 or EAGAIN 1010 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 1011 * return r; // do not call abort() 1012 * } 1013 * <setup and prepare commands> 1014 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 1015 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 1016 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 1017 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 1018 * virtio_enqueue_commit(sc, vq, slot, true); 1019 */ 1020 1021 /* 1022 * enqueue_prep: allocate a slot number 1023 */ 1024 int 1025 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 1026 { 1027 struct vq_entry *qe1; 1028 1029 KASSERT(slotp != NULL); 1030 1031 qe1 = vq_alloc_entry(vq); 1032 if (qe1 == NULL) 1033 return EAGAIN; 1034 /* next slot is not allocated yet */ 1035 qe1->qe_next = -1; 1036 *slotp = qe1->qe_index; 1037 1038 return 0; 1039 } 1040 1041 /* 1042 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 1043 */ 1044 int 1045 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 1046 int slot, int nsegs) 1047 { 1048 int indirect; 1049 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1050 1051 KASSERT(qe1->qe_next == -1); 1052 KASSERT(1 <= nsegs && nsegs <= vq->vq_num); 1053 1054 if ((vq->vq_indirect != NULL) && 1055 (nsegs >= MINSEG_INDIRECT) && 1056 (nsegs <= vq->vq_maxnsegs)) 1057 indirect = 1; 1058 else 1059 indirect = 0; 1060 qe1->qe_indirect = indirect; 1061 1062 if (indirect) { 1063 struct vring_desc *vd; 1064 int i; 1065 1066 vd = &vq->vq_desc[qe1->qe_index]; 1067 vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr 1068 + vq->vq_indirectoffset; 1069 vd->addr += sizeof(struct vring_desc) 1070 * vq->vq_maxnsegs * qe1->qe_index; 1071 vd->len = sizeof(struct vring_desc) * nsegs; 1072 vd->flags = VRING_DESC_F_INDIRECT; 1073 1074 vd = vq->vq_indirect; 1075 vd += vq->vq_maxnsegs * qe1->qe_index; 1076 qe1->qe_desc_base = vd; 1077 1078 for (i = 0; i < nsegs-1; i++) { 1079 vd[i].flags = VRING_DESC_F_NEXT; 1080 } 1081 vd[i].flags = 0; 1082 qe1->qe_next = 0; 1083 1084 return 0; 1085 } else { 1086 struct vring_desc *vd; 1087 struct vq_entry *qe; 1088 int i, s; 1089 1090 vd = &vq->vq_desc[0]; 1091 qe1->qe_desc_base = vd; 1092 qe1->qe_next = qe1->qe_index; 1093 s = slot; 1094 for (i = 0; i < nsegs - 1; i++) { 1095 qe = vq_alloc_entry(vq); 1096 if (qe == NULL) { 1097 vd[s].flags = 0; 1098 virtio_enqueue_abort(sc, vq, slot); 1099 return EAGAIN; 1100 } 1101 vd[s].flags = VRING_DESC_F_NEXT; 1102 vd[s].next = qe->qe_index; 1103 s = qe->qe_index; 1104 } 1105 vd[s].flags = 0; 1106 1107 return 0; 1108 } 1109 } 1110 1111 /* 1112 * enqueue: enqueue a single dmamap. 1113 */ 1114 int 1115 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1116 bus_dmamap_t dmamap, bool write) 1117 { 1118 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1119 struct vring_desc *vd = qe1->qe_desc_base; 1120 int i; 1121 int s = qe1->qe_next; 1122 1123 KASSERT(s >= 0); 1124 KASSERT(dmamap->dm_nsegs > 0); 1125 1126 for (i = 0; i < dmamap->dm_nsegs; i++) { 1127 vd[s].addr = dmamap->dm_segs[i].ds_addr; 1128 vd[s].len = dmamap->dm_segs[i].ds_len; 1129 if (!write) 1130 vd[s].flags |= VRING_DESC_F_WRITE; 1131 s = vd[s].next; 1132 } 1133 qe1->qe_next = s; 1134 1135 return 0; 1136 } 1137 1138 int 1139 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1140 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 1141 bool write) 1142 { 1143 struct vq_entry *qe1 = &vq->vq_entries[slot]; 1144 struct vring_desc *vd = qe1->qe_desc_base; 1145 int s = qe1->qe_next; 1146 1147 KASSERT(s >= 0); 1148 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 1149 KASSERT((dmamap->dm_segs[0].ds_len > start) && 1150 (dmamap->dm_segs[0].ds_len >= start + len)); 1151 1152 vd[s].addr = dmamap->dm_segs[0].ds_addr + start; 1153 vd[s].len = len; 1154 if (!write) 1155 vd[s].flags |= VRING_DESC_F_WRITE; 1156 qe1->qe_next = vd[s].next; 1157 1158 return 0; 1159 } 1160 1161 /* 1162 * enqueue_commit: add it to the aring. 1163 */ 1164 int 1165 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 1166 bool notifynow) 1167 { 1168 struct vq_entry *qe1; 1169 1170 if (slot < 0) { 1171 mutex_enter(&vq->vq_aring_lock); 1172 goto notify; 1173 } 1174 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 1175 qe1 = &vq->vq_entries[slot]; 1176 if (qe1->qe_indirect) 1177 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 1178 mutex_enter(&vq->vq_aring_lock); 1179 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot; 1180 1181 notify: 1182 if (notifynow) { 1183 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 1184 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 1185 membar_producer(); 1186 vq->vq_avail->idx = vq->vq_avail_idx; 1187 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 1188 membar_producer(); 1189 vq->vq_queued++; 1190 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 1191 membar_consumer(); 1192 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) 1193 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 1194 VIRTIO_CONFIG_QUEUE_NOTIFY, 1195 vq->vq_index); 1196 } 1197 mutex_exit(&vq->vq_aring_lock); 1198 1199 return 0; 1200 } 1201 1202 /* 1203 * enqueue_abort: rollback. 1204 */ 1205 int 1206 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1207 { 1208 struct vq_entry *qe = &vq->vq_entries[slot]; 1209 struct vring_desc *vd; 1210 int s; 1211 1212 if (qe->qe_next < 0) { 1213 vq_free_entry(vq, qe); 1214 return 0; 1215 } 1216 1217 s = slot; 1218 vd = &vq->vq_desc[0]; 1219 while (vd[s].flags & VRING_DESC_F_NEXT) { 1220 s = vd[s].next; 1221 vq_free_entry(vq, qe); 1222 qe = &vq->vq_entries[s]; 1223 } 1224 vq_free_entry(vq, qe); 1225 return 0; 1226 } 1227 1228 /* 1229 * Dequeue a request. 1230 */ 1231 /* 1232 * dequeue: dequeue a request from uring; dmamap_sync for uring is 1233 * already done in the interrupt handler. 1234 */ 1235 int 1236 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 1237 int *slotp, int *lenp) 1238 { 1239 uint16_t slot, usedidx; 1240 struct vq_entry *qe; 1241 1242 if (vq->vq_used_idx == vq->vq_used->idx) 1243 return ENOENT; 1244 mutex_enter(&vq->vq_uring_lock); 1245 usedidx = vq->vq_used_idx++; 1246 mutex_exit(&vq->vq_uring_lock); 1247 usedidx %= vq->vq_num; 1248 slot = vq->vq_used->ring[usedidx].id; 1249 qe = &vq->vq_entries[slot]; 1250 1251 if (qe->qe_indirect) 1252 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 1253 1254 if (slotp) 1255 *slotp = slot; 1256 if (lenp) 1257 *lenp = vq->vq_used->ring[usedidx].len; 1258 1259 return 0; 1260 } 1261 1262 /* 1263 * dequeue_commit: complete dequeue; the slot is recycled for future use. 1264 * if you forget to call this the slot will be leaked. 1265 */ 1266 int 1267 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1268 { 1269 struct vq_entry *qe = &vq->vq_entries[slot]; 1270 struct vring_desc *vd = &vq->vq_desc[0]; 1271 int s = slot; 1272 1273 while (vd[s].flags & VRING_DESC_F_NEXT) { 1274 s = vd[s].next; 1275 vq_free_entry(vq, qe); 1276 qe = &vq->vq_entries[s]; 1277 } 1278 vq_free_entry(vq, qe); 1279 1280 return 0; 1281 } 1282