1 /* $NetBSD: virtio.c,v 1.9 2015/05/05 10:58:35 ozaki-r Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.9 2015/05/05 10:58:35 ozaki-r Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/atomic.h> 35 #include <sys/bus.h> 36 #include <sys/device.h> 37 #include <sys/kmem.h> 38 39 #include <dev/pci/pcidevs.h> 40 #include <dev/pci/pcireg.h> 41 #include <dev/pci/pcivar.h> 42 43 #include <dev/pci/virtioreg.h> 44 #include <dev/pci/virtiovar.h> 45 46 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 47 48 static int virtio_match(device_t, cfdata_t, void *); 49 static void virtio_attach(device_t, device_t, void *); 50 static int virtio_detach(device_t, int); 51 static int virtio_intr(void *arg); 52 static void virtio_soft_intr(void *arg); 53 static void virtio_init_vq(struct virtio_softc *, 54 struct virtqueue *, const bool); 55 56 CFATTACH_DECL3_NEW(virtio, sizeof(struct virtio_softc), 57 virtio_match, virtio_attach, virtio_detach, NULL, NULL, NULL, 58 DVF_DETACH_SHUTDOWN); 59 60 static void 61 virtio_set_status(struct virtio_softc *sc, int status) 62 { 63 int old = 0; 64 65 if (status != 0) 66 old = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 67 VIRTIO_CONFIG_DEVICE_STATUS); 68 bus_space_write_1(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_DEVICE_STATUS, 69 status|old); 70 } 71 72 #define virtio_device_reset(sc) virtio_set_status((sc), 0) 73 74 static int 75 virtio_match(device_t parent, cfdata_t match, void *aux) 76 { 77 struct pci_attach_args *pa; 78 79 pa = (struct pci_attach_args *)aux; 80 switch (PCI_VENDOR(pa->pa_id)) { 81 case PCI_VENDOR_QUMRANET: 82 if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <= 83 PCI_PRODUCT(pa->pa_id)) && 84 (PCI_PRODUCT(pa->pa_id) <= 85 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) 86 return 1; 87 break; 88 } 89 90 return 0; 91 } 92 93 static const char *virtio_device_name[] = { 94 "Unknown (0)", /* 0 */ 95 "Network", /* 1 */ 96 "Block", /* 2 */ 97 "Console", /* 3 */ 98 "Entropy", /* 4 */ 99 "Memory Balloon", /* 5 */ 100 "Unknown (6)", /* 6 */ 101 "Unknown (7)", /* 7 */ 102 "Unknown (8)", /* 8 */ 103 "9P Transport" /* 9 */ 104 }; 105 #define NDEVNAMES (sizeof(virtio_device_name)/sizeof(char*)) 106 107 static void 108 virtio_attach(device_t parent, device_t self, void *aux) 109 { 110 struct virtio_softc *sc = device_private(self); 111 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 112 pci_chipset_tag_t pc = pa->pa_pc; 113 pcitag_t tag = pa->pa_tag; 114 int revision; 115 pcireg_t id; 116 char const *intrstr; 117 pci_intr_handle_t ih; 118 char intrbuf[PCI_INTRSTR_LEN]; 119 120 revision = PCI_REVISION(pa->pa_class); 121 if (revision != 0) { 122 aprint_normal(": unknown revision 0x%02x; giving up\n", 123 revision); 124 return; 125 } 126 aprint_normal("\n"); 127 aprint_naive("\n"); 128 129 /* subsystem ID shows what I am */ 130 id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 131 aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n", 132 (PCI_SUBSYS_ID(id) < NDEVNAMES? 133 virtio_device_name[PCI_SUBSYS_ID(id)] : "Unknown"), 134 revision); 135 136 sc->sc_dev = self; 137 sc->sc_pc = pc; 138 sc->sc_tag = tag; 139 sc->sc_iot = pa->pa_iot; 140 sc->sc_dmat = pa->pa_dmat; 141 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 142 143 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, 144 &sc->sc_iot, &sc->sc_ioh, NULL, &sc->sc_iosize)) { 145 aprint_error_dev(self, "can't map i/o space\n"); 146 return; 147 } 148 149 virtio_device_reset(sc); 150 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 151 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 152 153 /* XXX: use softc as aux... */ 154 sc->sc_childdevid = PCI_SUBSYS_ID(id); 155 sc->sc_child = NULL; 156 config_found(self, sc, NULL); 157 if (sc->sc_child == NULL) { 158 aprint_error_dev(self, 159 "no matching child driver; not configured\n"); 160 return; 161 } 162 if (sc->sc_child == (void*)1) { /* this shows error */ 163 aprint_error_dev(self, 164 "virtio configuration failed\n"); 165 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 166 return; 167 } 168 169 if (pci_intr_map(pa, &ih)) { 170 aprint_error_dev(self, "couldn't map interrupt\n"); 171 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 172 return; 173 } 174 175 intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf)); 176 177 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 178 pci_intr_setattr(pc, &ih, PCI_INTR_MPSAFE, true); 179 180 sc->sc_ih = pci_intr_establish(pc, ih, sc->sc_ipl, virtio_intr, sc); 181 182 if (sc->sc_ih == NULL) { 183 aprint_error_dev(self, "couldn't establish interrupt"); 184 if (intrstr != NULL) 185 aprint_error(" at %s", intrstr); 186 aprint_error("\n"); 187 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 188 return; 189 } 190 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 191 192 sc->sc_soft_ih = NULL; 193 if (sc->sc_flags & VIRTIO_F_PCI_INTR_SOFTINT) { 194 u_int flags = SOFTINT_NET; 195 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 196 flags |= SOFTINT_MPSAFE; 197 198 sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr, sc); 199 if (sc->sc_soft_ih == NULL) 200 aprint_error(": failed to establish soft interrupt\n"); 201 } 202 203 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 204 205 return; 206 } 207 208 static int 209 virtio_detach(device_t self, int flags) 210 { 211 struct virtio_softc *sc = device_private(self); 212 int r; 213 214 if (sc->sc_child != 0 && sc->sc_child != (void*)1) { 215 r = config_detach(sc->sc_child, flags); 216 if (r) 217 return r; 218 } 219 KASSERT(sc->sc_child == 0 || sc->sc_child == (void*)1); 220 KASSERT(sc->sc_vqs == 0); 221 if (sc->sc_ih != NULL) { 222 pci_intr_disestablish(sc->sc_pc, sc->sc_ih); 223 sc->sc_ih = NULL; 224 } 225 if (sc->sc_iosize) 226 bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize); 227 sc->sc_iosize = 0; 228 229 return 0; 230 } 231 232 /* 233 * Reset the device. 234 */ 235 /* 236 * To reset the device to a known state, do following: 237 * virtio_reset(sc); // this will stop the device activity 238 * <dequeue finished requests>; // virtio_dequeue() still can be called 239 * <revoke pending requests in the vqs if any>; 240 * virtio_reinit_begin(sc); // dequeue prohibitted 241 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 242 * <some other initialization>; 243 * virtio_reinit_end(sc); // device activated; enqueue allowed 244 * Once attached, feature negotiation can only be allowed after virtio_reset. 245 */ 246 void 247 virtio_reset(struct virtio_softc *sc) 248 { 249 virtio_device_reset(sc); 250 } 251 252 void 253 virtio_reinit_start(struct virtio_softc *sc) 254 { 255 int i; 256 257 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 258 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 259 for (i = 0; i < sc->sc_nvqs; i++) { 260 int n; 261 struct virtqueue *vq = &sc->sc_vqs[i]; 262 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 263 VIRTIO_CONFIG_QUEUE_SELECT, 264 vq->vq_index); 265 n = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 266 VIRTIO_CONFIG_QUEUE_SIZE); 267 if (n == 0) /* vq disappeared */ 268 continue; 269 if (n != vq->vq_num) { 270 panic("%s: virtqueue size changed, vq index %d\n", 271 device_xname(sc->sc_dev), 272 vq->vq_index); 273 } 274 virtio_init_vq(sc, vq, true); 275 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 276 VIRTIO_CONFIG_QUEUE_ADDRESS, 277 (vq->vq_dmamap->dm_segs[0].ds_addr 278 / VIRTIO_PAGE_SIZE)); 279 } 280 } 281 282 void 283 virtio_reinit_end(struct virtio_softc *sc) 284 { 285 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 286 } 287 288 /* 289 * Feature negotiation. 290 */ 291 uint32_t 292 virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features) 293 { 294 uint32_t r; 295 296 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 297 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 298 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 299 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 300 VIRTIO_CONFIG_DEVICE_FEATURES); 301 r &= guest_features; 302 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 303 VIRTIO_CONFIG_GUEST_FEATURES, r); 304 sc->sc_features = r; 305 if (r & VIRTIO_F_RING_INDIRECT_DESC) 306 sc->sc_indirect = true; 307 else 308 sc->sc_indirect = false; 309 310 return r; 311 } 312 313 /* 314 * Device configuration registers. 315 */ 316 uint8_t 317 virtio_read_device_config_1(struct virtio_softc *sc, int index) 318 { 319 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, 320 sc->sc_config_offset + index); 321 } 322 323 uint16_t 324 virtio_read_device_config_2(struct virtio_softc *sc, int index) 325 { 326 return bus_space_read_2(sc->sc_iot, sc->sc_ioh, 327 sc->sc_config_offset + index); 328 } 329 330 uint32_t 331 virtio_read_device_config_4(struct virtio_softc *sc, int index) 332 { 333 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, 334 sc->sc_config_offset + index); 335 } 336 337 uint64_t 338 virtio_read_device_config_8(struct virtio_softc *sc, int index) 339 { 340 uint64_t r; 341 342 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 343 sc->sc_config_offset + index + sizeof(uint32_t)); 344 r <<= 32; 345 r += bus_space_read_4(sc->sc_iot, sc->sc_ioh, 346 sc->sc_config_offset + index); 347 return r; 348 } 349 350 void 351 virtio_write_device_config_1(struct virtio_softc *sc, 352 int index, uint8_t value) 353 { 354 bus_space_write_1(sc->sc_iot, sc->sc_ioh, 355 sc->sc_config_offset + index, value); 356 } 357 358 void 359 virtio_write_device_config_2(struct virtio_softc *sc, 360 int index, uint16_t value) 361 { 362 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 363 sc->sc_config_offset + index, value); 364 } 365 366 void 367 virtio_write_device_config_4(struct virtio_softc *sc, 368 int index, uint32_t value) 369 { 370 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 371 sc->sc_config_offset + index, value); 372 } 373 374 void 375 virtio_write_device_config_8(struct virtio_softc *sc, 376 int index, uint64_t value) 377 { 378 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 379 sc->sc_config_offset + index, 380 value & 0xffffffff); 381 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 382 sc->sc_config_offset + index + sizeof(uint32_t), 383 value >> 32); 384 } 385 386 /* 387 * Interrupt handler. 388 */ 389 static int 390 virtio_intr(void *arg) 391 { 392 struct virtio_softc *sc = arg; 393 int isr, r = 0; 394 395 /* check and ack the interrupt */ 396 isr = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 397 VIRTIO_CONFIG_ISR_STATUS); 398 if (isr == 0) 399 return 0; 400 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 401 (sc->sc_config_change != NULL)) 402 r = (sc->sc_config_change)(sc); 403 if (sc->sc_intrhand != NULL) { 404 if (sc->sc_soft_ih != NULL) 405 softint_schedule(sc->sc_soft_ih); 406 else 407 r |= (sc->sc_intrhand)(sc); 408 } 409 410 return r; 411 } 412 413 static void 414 virtio_soft_intr(void *arg) 415 { 416 struct virtio_softc *sc = arg; 417 418 KASSERT(sc->sc_intrhand != NULL); 419 420 (sc->sc_intrhand)(sc); 421 } 422 423 /* 424 * dmamap sync operations for a virtqueue. 425 */ 426 static inline void 427 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 428 { 429 /* availoffset == sizeof(vring_desc)*vq_num */ 430 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 431 ops); 432 } 433 434 static inline void 435 vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 436 { 437 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 438 vq->vq_availoffset, 439 offsetof(struct vring_avail, ring) 440 + vq->vq_num * sizeof(uint16_t), 441 ops); 442 } 443 444 static inline void 445 vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 446 { 447 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 448 vq->vq_usedoffset, 449 offsetof(struct vring_used, ring) 450 + vq->vq_num * sizeof(struct vring_used_elem), 451 ops); 452 } 453 454 static inline void 455 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 456 int ops) 457 { 458 int offset = vq->vq_indirectoffset 459 + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 460 461 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 462 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, 463 ops); 464 } 465 466 /* 467 * Can be used as sc_intrhand. 468 */ 469 /* 470 * Scan vq, bus_dmamap_sync for the vqs (not for the payload), 471 * and calls (*vq_done)() if some entries are consumed. 472 */ 473 int 474 virtio_vq_intr(struct virtio_softc *sc) 475 { 476 struct virtqueue *vq; 477 int i, r = 0; 478 479 for (i = 0; i < sc->sc_nvqs; i++) { 480 vq = &sc->sc_vqs[i]; 481 if (vq->vq_queued) { 482 vq->vq_queued = 0; 483 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 484 } 485 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 486 membar_consumer(); 487 if (vq->vq_used_idx != vq->vq_used->idx) { 488 if (vq->vq_done) 489 r |= (vq->vq_done)(vq); 490 } 491 } 492 493 return r; 494 } 495 496 /* 497 * Start/stop vq interrupt. No guarantee. 498 */ 499 void 500 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 501 { 502 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 503 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 504 vq->vq_queued++; 505 } 506 507 void 508 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 509 { 510 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 511 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 512 vq->vq_queued++; 513 } 514 515 /* 516 * Initialize vq structure. 517 */ 518 static void 519 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, const bool reinit) 520 { 521 int i, j; 522 int vq_size = vq->vq_num; 523 524 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 525 526 /* build the indirect descriptor chain */ 527 if (vq->vq_indirect != NULL) { 528 struct vring_desc *vd; 529 530 for (i = 0; i < vq_size; i++) { 531 vd = vq->vq_indirect; 532 vd += vq->vq_maxnsegs * i; 533 for (j = 0; j < vq->vq_maxnsegs-1; j++) 534 vd[j].next = j + 1; 535 } 536 } 537 538 /* free slot management */ 539 SIMPLEQ_INIT(&vq->vq_freelist); 540 for (i = 0; i < vq_size; i++) { 541 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, 542 &vq->vq_entries[i], qe_list); 543 vq->vq_entries[i].qe_index = i; 544 } 545 if (!reinit) 546 mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl); 547 548 /* enqueue/dequeue status */ 549 vq->vq_avail_idx = 0; 550 vq->vq_used_idx = 0; 551 vq->vq_queued = 0; 552 if (!reinit) { 553 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 554 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 555 } 556 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 557 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 558 vq->vq_queued++; 559 } 560 561 /* 562 * Allocate/free a vq. 563 */ 564 int 565 virtio_alloc_vq(struct virtio_softc *sc, 566 struct virtqueue *vq, int index, int maxsegsize, int maxnsegs, 567 const char *name) 568 { 569 int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0; 570 int rsegs, r; 571 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1))& \ 572 ~(VIRTIO_PAGE_SIZE-1)) 573 574 memset(vq, 0, sizeof(*vq)); 575 576 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 577 VIRTIO_CONFIG_QUEUE_SELECT, index); 578 vq_size = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 579 VIRTIO_CONFIG_QUEUE_SIZE); 580 if (vq_size == 0) { 581 aprint_error_dev(sc->sc_dev, 582 "virtqueue not exist, index %d for %s\n", 583 index, name); 584 goto err; 585 } 586 /* allocsize1: descriptor table + avail ring + pad */ 587 allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size 588 + sizeof(uint16_t)*(2+vq_size)); 589 /* allocsize2: used ring + pad */ 590 allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2 591 + sizeof(struct vring_used_elem)*vq_size); 592 /* allocsize3: indirect table */ 593 if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) 594 allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size; 595 else 596 allocsize3 = 0; 597 allocsize = allocsize1 + allocsize2 + allocsize3; 598 599 /* alloc and map the memory */ 600 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 601 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 602 if (r != 0) { 603 aprint_error_dev(sc->sc_dev, 604 "virtqueue %d for %s allocation failed, " 605 "error code %d\n", index, name, r); 606 goto err; 607 } 608 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize, 609 &vq->vq_vaddr, BUS_DMA_NOWAIT); 610 if (r != 0) { 611 aprint_error_dev(sc->sc_dev, 612 "virtqueue %d for %s map failed, " 613 "error code %d\n", index, name, r); 614 goto err; 615 } 616 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 617 BUS_DMA_NOWAIT, &vq->vq_dmamap); 618 if (r != 0) { 619 aprint_error_dev(sc->sc_dev, 620 "virtqueue %d for %s dmamap creation failed, " 621 "error code %d\n", index, name, r); 622 goto err; 623 } 624 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 625 vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT); 626 if (r != 0) { 627 aprint_error_dev(sc->sc_dev, 628 "virtqueue %d for %s dmamap load failed, " 629 "error code %d\n", index, name, r); 630 goto err; 631 } 632 633 /* set the vq address */ 634 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 635 VIRTIO_CONFIG_QUEUE_ADDRESS, 636 (vq->vq_dmamap->dm_segs[0].ds_addr 637 / VIRTIO_PAGE_SIZE)); 638 639 /* remember addresses and offsets for later use */ 640 vq->vq_owner = sc; 641 vq->vq_num = vq_size; 642 vq->vq_index = index; 643 vq->vq_desc = vq->vq_vaddr; 644 vq->vq_availoffset = sizeof(struct vring_desc)*vq_size; 645 vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset); 646 vq->vq_usedoffset = allocsize1; 647 vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset); 648 if (allocsize3 > 0) { 649 vq->vq_indirectoffset = allocsize1 + allocsize2; 650 vq->vq_indirect = (void*)(((char*)vq->vq_desc) 651 + vq->vq_indirectoffset); 652 } 653 vq->vq_bytesize = allocsize; 654 vq->vq_maxsegsize = maxsegsize; 655 vq->vq_maxnsegs = maxnsegs; 656 657 /* free slot management */ 658 vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size, 659 KM_NOSLEEP); 660 if (vq->vq_entries == NULL) { 661 r = ENOMEM; 662 goto err; 663 } 664 665 virtio_init_vq(sc, vq, false); 666 667 aprint_verbose_dev(sc->sc_dev, 668 "allocated %u byte for virtqueue %d for %s, " 669 "size %d\n", allocsize, index, name, vq_size); 670 if (allocsize3 > 0) 671 aprint_verbose_dev(sc->sc_dev, 672 "using %d byte (%d entries) " 673 "indirect descriptors\n", 674 allocsize3, maxnsegs * vq_size); 675 return 0; 676 677 err: 678 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 679 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 680 if (vq->vq_dmamap) 681 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 682 if (vq->vq_vaddr) 683 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 684 if (vq->vq_segs[0].ds_addr) 685 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 686 memset(vq, 0, sizeof(*vq)); 687 688 return -1; 689 } 690 691 int 692 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 693 { 694 struct vq_entry *qe; 695 int i = 0; 696 697 /* device must be already deactivated */ 698 /* confirm the vq is empty */ 699 SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) { 700 i++; 701 } 702 if (i != vq->vq_num) { 703 printf("%s: freeing non-empty vq, index %d\n", 704 device_xname(sc->sc_dev), vq->vq_index); 705 return EBUSY; 706 } 707 708 /* tell device that there's no virtqueue any longer */ 709 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 710 VIRTIO_CONFIG_QUEUE_SELECT, vq->vq_index); 711 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 712 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 713 714 kmem_free(vq->vq_entries, vq->vq_bytesize); 715 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 716 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 717 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 718 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 719 mutex_destroy(&vq->vq_freelist_lock); 720 mutex_destroy(&vq->vq_uring_lock); 721 mutex_destroy(&vq->vq_aring_lock); 722 memset(vq, 0, sizeof(*vq)); 723 724 return 0; 725 } 726 727 /* 728 * Free descriptor management. 729 */ 730 static struct vq_entry * 731 vq_alloc_entry(struct virtqueue *vq) 732 { 733 struct vq_entry *qe; 734 735 mutex_enter(&vq->vq_freelist_lock); 736 if (SIMPLEQ_EMPTY(&vq->vq_freelist)) { 737 mutex_exit(&vq->vq_freelist_lock); 738 return NULL; 739 } 740 qe = SIMPLEQ_FIRST(&vq->vq_freelist); 741 SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list); 742 mutex_exit(&vq->vq_freelist_lock); 743 744 return qe; 745 } 746 747 static void 748 vq_free_entry(struct virtqueue *vq, struct vq_entry *qe) 749 { 750 mutex_enter(&vq->vq_freelist_lock); 751 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list); 752 mutex_exit(&vq->vq_freelist_lock); 753 754 return; 755 } 756 757 /* 758 * Enqueue several dmamaps as a single request. 759 */ 760 /* 761 * Typical usage: 762 * <queue size> number of followings are stored in arrays 763 * - command blocks (in dmamem) should be pre-allocated and mapped 764 * - dmamaps for command blocks should be pre-allocated and loaded 765 * - dmamaps for payload should be pre-allocated 766 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 767 * if (r) // currently 0 or EAGAIN 768 * return r; 769 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 770 * if (r) { 771 * virtio_enqueue_abort(sc, vq, slot); 772 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 773 * return r; 774 * } 775 * r = virtio_enqueue_reserve(sc, vq, slot, 776 * dmamap_payload[slot]->dm_nsegs+1); 777 * // ^ +1 for command 778 * if (r) { // currently 0 or EAGAIN 779 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 780 * return r; // do not call abort() 781 * } 782 * <setup and prepare commands> 783 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 784 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 785 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 786 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 787 * virtio_enqueue_commit(sc, vq, slot, true); 788 */ 789 790 /* 791 * enqueue_prep: allocate a slot number 792 */ 793 int 794 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 795 { 796 struct vq_entry *qe1; 797 798 KASSERT(slotp != NULL); 799 800 qe1 = vq_alloc_entry(vq); 801 if (qe1 == NULL) 802 return EAGAIN; 803 /* next slot is not allocated yet */ 804 qe1->qe_next = -1; 805 *slotp = qe1->qe_index; 806 807 return 0; 808 } 809 810 /* 811 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 812 */ 813 int 814 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 815 int slot, int nsegs) 816 { 817 int indirect; 818 struct vq_entry *qe1 = &vq->vq_entries[slot]; 819 820 KASSERT(qe1->qe_next == -1); 821 KASSERT(1 <= nsegs && nsegs <= vq->vq_num); 822 823 if ((vq->vq_indirect != NULL) && 824 (nsegs >= MINSEG_INDIRECT) && 825 (nsegs <= vq->vq_maxnsegs)) 826 indirect = 1; 827 else 828 indirect = 0; 829 qe1->qe_indirect = indirect; 830 831 if (indirect) { 832 struct vring_desc *vd; 833 int i; 834 835 vd = &vq->vq_desc[qe1->qe_index]; 836 vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr 837 + vq->vq_indirectoffset; 838 vd->addr += sizeof(struct vring_desc) 839 * vq->vq_maxnsegs * qe1->qe_index; 840 vd->len = sizeof(struct vring_desc) * nsegs; 841 vd->flags = VRING_DESC_F_INDIRECT; 842 843 vd = vq->vq_indirect; 844 vd += vq->vq_maxnsegs * qe1->qe_index; 845 qe1->qe_desc_base = vd; 846 847 for (i = 0; i < nsegs-1; i++) { 848 vd[i].flags = VRING_DESC_F_NEXT; 849 } 850 vd[i].flags = 0; 851 qe1->qe_next = 0; 852 853 return 0; 854 } else { 855 struct vring_desc *vd; 856 struct vq_entry *qe; 857 int i, s; 858 859 vd = &vq->vq_desc[0]; 860 qe1->qe_desc_base = vd; 861 qe1->qe_next = qe1->qe_index; 862 s = slot; 863 for (i = 0; i < nsegs - 1; i++) { 864 qe = vq_alloc_entry(vq); 865 if (qe == NULL) { 866 vd[s].flags = 0; 867 virtio_enqueue_abort(sc, vq, slot); 868 return EAGAIN; 869 } 870 vd[s].flags = VRING_DESC_F_NEXT; 871 vd[s].next = qe->qe_index; 872 s = qe->qe_index; 873 } 874 vd[s].flags = 0; 875 876 return 0; 877 } 878 } 879 880 /* 881 * enqueue: enqueue a single dmamap. 882 */ 883 int 884 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 885 bus_dmamap_t dmamap, bool write) 886 { 887 struct vq_entry *qe1 = &vq->vq_entries[slot]; 888 struct vring_desc *vd = qe1->qe_desc_base; 889 int i; 890 int s = qe1->qe_next; 891 892 KASSERT(s >= 0); 893 KASSERT(dmamap->dm_nsegs > 0); 894 895 for (i = 0; i < dmamap->dm_nsegs; i++) { 896 vd[s].addr = dmamap->dm_segs[i].ds_addr; 897 vd[s].len = dmamap->dm_segs[i].ds_len; 898 if (!write) 899 vd[s].flags |= VRING_DESC_F_WRITE; 900 s = vd[s].next; 901 } 902 qe1->qe_next = s; 903 904 return 0; 905 } 906 907 int 908 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 909 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 910 bool write) 911 { 912 struct vq_entry *qe1 = &vq->vq_entries[slot]; 913 struct vring_desc *vd = qe1->qe_desc_base; 914 int s = qe1->qe_next; 915 916 KASSERT(s >= 0); 917 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 918 KASSERT((dmamap->dm_segs[0].ds_len > start) && 919 (dmamap->dm_segs[0].ds_len >= start + len)); 920 921 vd[s].addr = dmamap->dm_segs[0].ds_addr + start; 922 vd[s].len = len; 923 if (!write) 924 vd[s].flags |= VRING_DESC_F_WRITE; 925 qe1->qe_next = vd[s].next; 926 927 return 0; 928 } 929 930 /* 931 * enqueue_commit: add it to the aring. 932 */ 933 int 934 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 935 bool notifynow) 936 { 937 struct vq_entry *qe1; 938 939 if (slot < 0) { 940 mutex_enter(&vq->vq_aring_lock); 941 goto notify; 942 } 943 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 944 qe1 = &vq->vq_entries[slot]; 945 if (qe1->qe_indirect) 946 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 947 mutex_enter(&vq->vq_aring_lock); 948 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot; 949 950 notify: 951 if (notifynow) { 952 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 953 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 954 membar_producer(); 955 vq->vq_avail->idx = vq->vq_avail_idx; 956 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 957 membar_producer(); 958 vq->vq_queued++; 959 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 960 membar_consumer(); 961 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) 962 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 963 VIRTIO_CONFIG_QUEUE_NOTIFY, 964 vq->vq_index); 965 } 966 mutex_exit(&vq->vq_aring_lock); 967 968 return 0; 969 } 970 971 /* 972 * enqueue_abort: rollback. 973 */ 974 int 975 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 976 { 977 struct vq_entry *qe = &vq->vq_entries[slot]; 978 struct vring_desc *vd; 979 int s; 980 981 if (qe->qe_next < 0) { 982 vq_free_entry(vq, qe); 983 return 0; 984 } 985 986 s = slot; 987 vd = &vq->vq_desc[0]; 988 while (vd[s].flags & VRING_DESC_F_NEXT) { 989 s = vd[s].next; 990 vq_free_entry(vq, qe); 991 qe = &vq->vq_entries[s]; 992 } 993 vq_free_entry(vq, qe); 994 return 0; 995 } 996 997 /* 998 * Dequeue a request. 999 */ 1000 /* 1001 * dequeue: dequeue a request from uring; dmamap_sync for uring is 1002 * already done in the interrupt handler. 1003 */ 1004 int 1005 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 1006 int *slotp, int *lenp) 1007 { 1008 uint16_t slot, usedidx; 1009 struct vq_entry *qe; 1010 1011 if (vq->vq_used_idx == vq->vq_used->idx) 1012 return ENOENT; 1013 mutex_enter(&vq->vq_uring_lock); 1014 usedidx = vq->vq_used_idx++; 1015 mutex_exit(&vq->vq_uring_lock); 1016 usedidx %= vq->vq_num; 1017 slot = vq->vq_used->ring[usedidx].id; 1018 qe = &vq->vq_entries[slot]; 1019 1020 if (qe->qe_indirect) 1021 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 1022 1023 if (slotp) 1024 *slotp = slot; 1025 if (lenp) 1026 *lenp = vq->vq_used->ring[usedidx].len; 1027 1028 return 0; 1029 } 1030 1031 /* 1032 * dequeue_commit: complete dequeue; the slot is recycled for future use. 1033 * if you forget to call this the slot will be leaked. 1034 */ 1035 int 1036 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1037 { 1038 struct vq_entry *qe = &vq->vq_entries[slot]; 1039 struct vring_desc *vd = &vq->vq_desc[0]; 1040 int s = slot; 1041 1042 while (vd[s].flags & VRING_DESC_F_NEXT) { 1043 s = vd[s].next; 1044 vq_free_entry(vq, qe); 1045 qe = &vq->vq_entries[s]; 1046 } 1047 vq_free_entry(vq, qe); 1048 1049 return 0; 1050 } 1051