1 /* $NetBSD: virtio.c,v 1.3 2011/11/02 23:05:52 njoly Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.3 2011/11/02 23:05:52 njoly Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/atomic.h> 35 #include <sys/bus.h> 36 #include <sys/device.h> 37 #include <sys/kmem.h> 38 39 #include <dev/pci/pcidevs.h> 40 #include <dev/pci/pcireg.h> 41 #include <dev/pci/pcivar.h> 42 43 #include <dev/pci/virtioreg.h> 44 #include <dev/pci/virtiovar.h> 45 46 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 47 48 static int virtio_match(device_t, cfdata_t, void *); 49 static void virtio_attach(device_t, device_t, void *); 50 static int virtio_detach(device_t, int); 51 static int virtio_intr(void *arg); 52 static void virtio_init_vq(struct virtio_softc *, 53 struct virtqueue *, const bool); 54 55 CFATTACH_DECL3_NEW(virtio, sizeof(struct virtio_softc), 56 virtio_match, virtio_attach, virtio_detach, NULL, NULL, NULL, 57 DVF_DETACH_SHUTDOWN); 58 59 static void 60 virtio_set_status(struct virtio_softc *sc, int status) 61 { 62 int old = 0; 63 64 if (status != 0) 65 old = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 66 VIRTIO_CONFIG_DEVICE_STATUS); 67 bus_space_write_1(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_DEVICE_STATUS, 68 status|old); 69 } 70 71 #define virtio_device_reset(sc) virtio_set_status((sc), 0) 72 73 static int 74 virtio_match(device_t parent, cfdata_t match, void *aux) 75 { 76 struct pci_attach_args *pa; 77 78 pa = (struct pci_attach_args *)aux; 79 switch (PCI_VENDOR(pa->pa_id)) { 80 case PCI_VENDOR_QUMRANET: 81 if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <= 82 PCI_PRODUCT(pa->pa_id)) && 83 (PCI_PRODUCT(pa->pa_id) <= 84 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) 85 return 1; 86 break; 87 } 88 89 return 0; 90 } 91 92 static const char *virtio_device_name[] = { 93 "Unknown (0)", /* 0 */ 94 "Network", /* 1 */ 95 "Block", /* 2 */ 96 "Console", /* 3 */ 97 "Entropy", /* 4 */ 98 "Memory Balloon", /* 5 */ 99 "Unknown (6)", /* 6 */ 100 "Unknown (7)", /* 7 */ 101 "Unknown (8)", /* 8 */ 102 "9P Transport" /* 9 */ 103 }; 104 #define NDEVNAMES (sizeof(virtio_device_name)/sizeof(char*)) 105 106 static void 107 virtio_attach(device_t parent, device_t self, void *aux) 108 { 109 struct virtio_softc *sc = device_private(self); 110 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 111 pci_chipset_tag_t pc = pa->pa_pc; 112 pcitag_t tag = pa->pa_tag; 113 int revision; 114 pcireg_t id; 115 char const *intrstr; 116 pci_intr_handle_t ih; 117 118 revision = PCI_REVISION(pa->pa_class); 119 if (revision != 0) { 120 aprint_normal(": unknown revision 0x%02x; giving up\n", 121 revision); 122 return; 123 } 124 aprint_normal("\n"); 125 aprint_naive("\n"); 126 127 /* subsystem ID shows what I am */ 128 id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 129 aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n", 130 (PCI_PRODUCT(id) < NDEVNAMES? 131 virtio_device_name[PCI_PRODUCT(id)] : "Unknown"), 132 revision); 133 134 sc->sc_dev = self; 135 sc->sc_pc = pc; 136 sc->sc_tag = tag; 137 sc->sc_iot = pa->pa_iot; 138 sc->sc_dmat = pa->pa_dmat; 139 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 140 141 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, 142 &sc->sc_iot, &sc->sc_ioh, NULL, &sc->sc_iosize)) { 143 aprint_error_dev(self, "can't map i/o space\n"); 144 return; 145 } 146 147 virtio_device_reset(sc); 148 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 149 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 150 151 /* XXX: use softc as aux... */ 152 sc->sc_childdevid = PCI_PRODUCT(id); 153 sc->sc_child = NULL; 154 config_found(self, sc, NULL); 155 if (sc->sc_child == NULL) { 156 aprint_error_dev(self, 157 "no matching child driver; not configured\n"); 158 return; 159 } 160 if (sc->sc_child == (void*)1) { /* this shows error */ 161 aprint_error_dev(self, 162 "virtio configuration failed\n"); 163 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 164 return; 165 } 166 167 if (pci_intr_map(pa, &ih)) { 168 aprint_error_dev(self, "couldn't map interrupt\n"); 169 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 170 return; 171 } 172 intrstr = pci_intr_string(pc, ih); 173 sc->sc_ih = pci_intr_establish(pc, ih, sc->sc_ipl, virtio_intr, sc); 174 if (sc->sc_ih == NULL) { 175 aprint_error_dev(self, "couldn't establish interrupt"); 176 if (intrstr != NULL) 177 aprint_error(" at %s", intrstr); 178 aprint_error("\n"); 179 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 180 return; 181 } 182 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 183 184 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 185 186 return; 187 } 188 189 static int 190 virtio_detach(device_t self, int flags) 191 { 192 struct virtio_softc *sc = device_private(self); 193 int r; 194 195 if (sc->sc_child != 0 && sc->sc_child != (void*)1) { 196 r = config_detach(sc->sc_child, flags); 197 if (r) 198 return r; 199 } 200 KASSERT(sc->sc_child == 0 || sc->sc_child == (void*)1); 201 KASSERT(sc->sc_vqs == 0); 202 pci_intr_disestablish(sc->sc_pc, sc->sc_ih); 203 sc->sc_ih = 0; 204 if (sc->sc_iosize) 205 bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize); 206 sc->sc_iosize = 0; 207 208 return 0; 209 } 210 211 /* 212 * Reset the device. 213 */ 214 /* 215 * To reset the device to a known state, do following: 216 * virtio_reset(sc); // this will stop the device activity 217 * <dequeue finished requests>; // virtio_dequeue() still can be called 218 * <revoke pending requests in the vqs if any>; 219 * virtio_reinit_begin(sc); // dequeue prohibitted 220 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 221 * <some other initialization>; 222 * virtio_reinit_end(sc); // device activated; enqueue allowed 223 * Once attached, feature negotiation can only be allowed after virtio_reset. 224 */ 225 void 226 virtio_reset(struct virtio_softc *sc) 227 { 228 virtio_device_reset(sc); 229 } 230 231 void 232 virtio_reinit_start(struct virtio_softc *sc) 233 { 234 int i; 235 236 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 237 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 238 for (i = 0; i < sc->sc_nvqs; i++) { 239 int n; 240 struct virtqueue *vq = &sc->sc_vqs[i]; 241 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 242 VIRTIO_CONFIG_QUEUE_SELECT, 243 vq->vq_index); 244 n = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 245 VIRTIO_CONFIG_QUEUE_SIZE); 246 if (n == 0) /* vq disappeared */ 247 continue; 248 if (n != vq->vq_num) { 249 panic("%s: virtqueue size changed, vq index %d\n", 250 device_xname(sc->sc_dev), 251 vq->vq_index); 252 } 253 virtio_init_vq(sc, vq, true); 254 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 255 VIRTIO_CONFIG_QUEUE_ADDRESS, 256 (vq->vq_dmamap->dm_segs[0].ds_addr 257 / VIRTIO_PAGE_SIZE)); 258 } 259 } 260 261 void 262 virtio_reinit_end(struct virtio_softc *sc) 263 { 264 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 265 } 266 267 /* 268 * Feature negotiation. 269 */ 270 uint32_t 271 virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features) 272 { 273 uint32_t r; 274 275 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 276 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 277 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 278 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 279 VIRTIO_CONFIG_DEVICE_FEATURES); 280 r &= guest_features; 281 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 282 VIRTIO_CONFIG_GUEST_FEATURES, r); 283 sc->sc_features = r; 284 if (r & VIRTIO_F_RING_INDIRECT_DESC) 285 sc->sc_indirect = true; 286 else 287 sc->sc_indirect = false; 288 289 return r; 290 } 291 292 /* 293 * Device configuration registers. 294 */ 295 uint8_t 296 virtio_read_device_config_1(struct virtio_softc *sc, int index) 297 { 298 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, 299 sc->sc_config_offset + index); 300 } 301 302 uint16_t 303 virtio_read_device_config_2(struct virtio_softc *sc, int index) 304 { 305 return bus_space_read_2(sc->sc_iot, sc->sc_ioh, 306 sc->sc_config_offset + index); 307 } 308 309 uint32_t 310 virtio_read_device_config_4(struct virtio_softc *sc, int index) 311 { 312 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, 313 sc->sc_config_offset + index); 314 } 315 316 uint64_t 317 virtio_read_device_config_8(struct virtio_softc *sc, int index) 318 { 319 uint64_t r; 320 321 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 322 sc->sc_config_offset + index + sizeof(uint32_t)); 323 r <<= 32; 324 r += bus_space_read_4(sc->sc_iot, sc->sc_ioh, 325 sc->sc_config_offset + index); 326 return r; 327 } 328 329 void 330 virtio_write_device_config_1(struct virtio_softc *sc, 331 int index, uint8_t value) 332 { 333 bus_space_write_1(sc->sc_iot, sc->sc_ioh, 334 sc->sc_config_offset + index, value); 335 } 336 337 void 338 virtio_write_device_config_2(struct virtio_softc *sc, 339 int index, uint16_t value) 340 { 341 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 342 sc->sc_config_offset + index, value); 343 } 344 345 void 346 virtio_write_device_config_4(struct virtio_softc *sc, 347 int index, uint32_t value) 348 { 349 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 350 sc->sc_config_offset + index, value); 351 } 352 353 void 354 virtio_write_device_config_8(struct virtio_softc *sc, 355 int index, uint64_t value) 356 { 357 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 358 sc->sc_config_offset + index, 359 value & 0xffffffff); 360 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 361 sc->sc_config_offset + index + sizeof(uint32_t), 362 value >> 32); 363 } 364 365 /* 366 * Interrupt handler. 367 */ 368 static int 369 virtio_intr(void *arg) 370 { 371 struct virtio_softc *sc = arg; 372 int isr, r = 0; 373 374 /* check and ack the interrupt */ 375 isr = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 376 VIRTIO_CONFIG_ISR_STATUS); 377 if (isr == 0) 378 return 0; 379 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 380 (sc->sc_config_change != NULL)) 381 r = (sc->sc_config_change)(sc); 382 if (sc->sc_intrhand != NULL) 383 r |= (sc->sc_intrhand)(sc); 384 385 return r; 386 } 387 388 /* 389 * dmamap sync operations for a virtqueue. 390 */ 391 static inline void 392 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 393 { 394 /* availoffset == sizeof(vring_desc)*vq_num */ 395 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 396 ops); 397 } 398 399 static inline void 400 vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 401 { 402 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 403 vq->vq_availoffset, 404 offsetof(struct vring_avail, ring) 405 + vq->vq_num * sizeof(uint16_t), 406 ops); 407 } 408 409 static inline void 410 vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 411 { 412 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 413 vq->vq_usedoffset, 414 offsetof(struct vring_used, ring) 415 + vq->vq_num * sizeof(struct vring_used_elem), 416 ops); 417 } 418 419 static inline void 420 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 421 int ops) 422 { 423 int offset = vq->vq_indirectoffset 424 + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 425 426 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 427 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, 428 ops); 429 } 430 431 /* 432 * Can be used as sc_intrhand. 433 */ 434 /* 435 * Scan vq, bus_dmamap_sync for the vqs (not for the payload), 436 * and calls (*vq_done)() if some entries are consumed. 437 */ 438 int 439 virtio_vq_intr(struct virtio_softc *sc) 440 { 441 struct virtqueue *vq; 442 int i, r = 0; 443 444 for (i = 0; i < sc->sc_nvqs; i++) { 445 vq = &sc->sc_vqs[i]; 446 if (vq->vq_queued) { 447 vq->vq_queued = 0; 448 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 449 } 450 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 451 membar_consumer(); 452 if (vq->vq_used_idx != vq->vq_used->idx) { 453 if (vq->vq_done) 454 r |= (vq->vq_done)(vq); 455 } 456 } 457 458 459 return r; 460 } 461 462 /* 463 * Start/stop vq interrupt. No guarantee. 464 */ 465 void 466 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 467 { 468 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 469 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 470 vq->vq_queued++; 471 } 472 473 void 474 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 475 { 476 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 477 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 478 vq->vq_queued++; 479 } 480 481 /* 482 * Initialize vq structure. 483 */ 484 static void 485 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, const bool reinit) 486 { 487 int i, j; 488 int vq_size = vq->vq_num; 489 490 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 491 492 /* build the indirect descriptor chain */ 493 if (vq->vq_indirect != NULL) { 494 struct vring_desc *vd; 495 496 for (i = 0; i < vq_size; i++) { 497 vd = vq->vq_indirect; 498 vd += vq->vq_maxnsegs * i; 499 for (j = 0; j < vq->vq_maxnsegs-1; j++) 500 vd[j].next = j + 1; 501 } 502 } 503 504 /* free slot management */ 505 SIMPLEQ_INIT(&vq->vq_freelist); 506 for (i = 0; i < vq_size; i++) { 507 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, 508 &vq->vq_entries[i], qe_list); 509 vq->vq_entries[i].qe_index = i; 510 } 511 if (!reinit) 512 mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl); 513 514 /* enqueue/dequeue status */ 515 vq->vq_avail_idx = 0; 516 vq->vq_used_idx = 0; 517 vq->vq_queued = 0; 518 if (!reinit) { 519 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 520 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 521 } 522 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 523 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 524 vq->vq_queued++; 525 } 526 527 /* 528 * Allocate/free a vq. 529 */ 530 int 531 virtio_alloc_vq(struct virtio_softc *sc, 532 struct virtqueue *vq, int index, int maxsegsize, int maxnsegs, 533 const char *name) 534 { 535 int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0; 536 int rsegs, r; 537 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1))& \ 538 ~(VIRTIO_PAGE_SIZE-1)) 539 540 memset(vq, 0, sizeof(*vq)); 541 542 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 543 VIRTIO_CONFIG_QUEUE_SELECT, index); 544 vq_size = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 545 VIRTIO_CONFIG_QUEUE_SIZE); 546 if (vq_size == 0) { 547 aprint_error_dev(sc->sc_dev, 548 "virtqueue not exist, index %d for %s\n", 549 index, name); 550 goto err; 551 } 552 /* allocsize1: descriptor table + avail ring + pad */ 553 allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size 554 + sizeof(uint16_t)*(2+vq_size)); 555 /* allocsize2: used ring + pad */ 556 allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2 557 + sizeof(struct vring_used_elem)*vq_size); 558 /* allocsize3: indirect table */ 559 if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) 560 allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size; 561 else 562 allocsize3 = 0; 563 allocsize = allocsize1 + allocsize2 + allocsize3; 564 565 /* alloc and map the memory */ 566 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 567 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 568 if (r != 0) { 569 aprint_error_dev(sc->sc_dev, 570 "virtqueue %d for %s allocation failed, " 571 "error code %d\n", index, name, r); 572 goto err; 573 } 574 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize, 575 &vq->vq_vaddr, BUS_DMA_NOWAIT); 576 if (r != 0) { 577 aprint_error_dev(sc->sc_dev, 578 "virtqueue %d for %s map failed, " 579 "error code %d\n", index, name, r); 580 goto err; 581 } 582 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 583 BUS_DMA_NOWAIT, &vq->vq_dmamap); 584 if (r != 0) { 585 aprint_error_dev(sc->sc_dev, 586 "virtqueue %d for %s dmamap creation failed, " 587 "error code %d\n", index, name, r); 588 goto err; 589 } 590 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 591 vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT); 592 if (r != 0) { 593 aprint_error_dev(sc->sc_dev, 594 "virtqueue %d for %s dmamap load failed, " 595 "error code %d\n", index, name, r); 596 goto err; 597 } 598 599 /* set the vq address */ 600 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 601 VIRTIO_CONFIG_QUEUE_ADDRESS, 602 (vq->vq_dmamap->dm_segs[0].ds_addr 603 / VIRTIO_PAGE_SIZE)); 604 605 /* remember addresses and offsets for later use */ 606 vq->vq_owner = sc; 607 vq->vq_num = vq_size; 608 vq->vq_index = index; 609 vq->vq_desc = vq->vq_vaddr; 610 vq->vq_availoffset = sizeof(struct vring_desc)*vq_size; 611 vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset); 612 vq->vq_usedoffset = allocsize1; 613 vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset); 614 if (allocsize3 > 0) { 615 vq->vq_indirectoffset = allocsize1 + allocsize2; 616 vq->vq_indirect = (void*)(((char*)vq->vq_desc) 617 + vq->vq_indirectoffset); 618 } 619 vq->vq_bytesize = allocsize; 620 vq->vq_maxsegsize = maxsegsize; 621 vq->vq_maxnsegs = maxnsegs; 622 623 /* free slot management */ 624 vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size, 625 KM_NOSLEEP); 626 if (vq->vq_entries == NULL) { 627 r = ENOMEM; 628 goto err; 629 } 630 631 virtio_init_vq(sc, vq, false); 632 633 aprint_verbose_dev(sc->sc_dev, 634 "allocated %u byte for virtqueue %d for %s, " 635 "size %d\n", allocsize, index, name, vq_size); 636 if (allocsize3 > 0) 637 aprint_verbose_dev(sc->sc_dev, 638 "using %d byte (%d entries) " 639 "indirect descriptors\n", 640 allocsize3, maxnsegs * vq_size); 641 return 0; 642 643 err: 644 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 645 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 646 if (vq->vq_dmamap) 647 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 648 if (vq->vq_vaddr) 649 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 650 if (vq->vq_segs[0].ds_addr) 651 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 652 memset(vq, 0, sizeof(*vq)); 653 654 return -1; 655 } 656 657 int 658 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 659 { 660 struct vq_entry *qe; 661 int i = 0; 662 663 /* device must be already deactivated */ 664 /* confirm the vq is empty */ 665 SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) { 666 i++; 667 } 668 if (i != vq->vq_num) { 669 printf("%s: freeing non-empty vq, index %d\n", 670 device_xname(sc->sc_dev), vq->vq_index); 671 return EBUSY; 672 } 673 674 /* tell device that there's no virtqueue any longer */ 675 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 676 VIRTIO_CONFIG_QUEUE_SELECT, vq->vq_index); 677 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 678 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 679 680 kmem_free(vq->vq_entries, vq->vq_bytesize); 681 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 682 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 683 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 684 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 685 mutex_destroy(&vq->vq_freelist_lock); 686 mutex_destroy(&vq->vq_uring_lock); 687 mutex_destroy(&vq->vq_aring_lock); 688 memset(vq, 0, sizeof(*vq)); 689 690 return 0; 691 } 692 693 /* 694 * Free descriptor management. 695 */ 696 static struct vq_entry * 697 vq_alloc_entry(struct virtqueue *vq) 698 { 699 struct vq_entry *qe; 700 701 mutex_enter(&vq->vq_freelist_lock); 702 if (SIMPLEQ_EMPTY(&vq->vq_freelist)) { 703 mutex_exit(&vq->vq_freelist_lock); 704 return NULL; 705 } 706 qe = SIMPLEQ_FIRST(&vq->vq_freelist); 707 SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list); 708 mutex_exit(&vq->vq_freelist_lock); 709 710 return qe; 711 } 712 713 static void 714 vq_free_entry(struct virtqueue *vq, struct vq_entry *qe) 715 { 716 mutex_enter(&vq->vq_freelist_lock); 717 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list); 718 mutex_exit(&vq->vq_freelist_lock); 719 720 return; 721 } 722 723 /* 724 * Enqueue several dmamaps as a single request. 725 */ 726 /* 727 * Typical usage: 728 * <queue size> number of followings are stored in arrays 729 * - command blocks (in dmamem) should be pre-allocated and mapped 730 * - dmamaps for command blocks should be pre-allocated and loaded 731 * - dmamaps for payload should be pre-allocated 732 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 733 * if (r) // currently 0 or EAGAIN 734 * return r; 735 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 736 * if (r) { 737 * virtio_enqueue_abort(sc, vq, slot); 738 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 739 * return r; 740 * } 741 * r = virtio_enqueue_reserve(sc, vq, slot, 742 * dmamap_payload[slot]->dm_nsegs+1); 743 * // ^ +1 for command 744 * if (r) { // currently 0 or EAGAIN 745 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 746 * return r; // do not call abort() 747 * } 748 * <setup and prepare commands> 749 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 750 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 751 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 752 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 753 * virtio_enqueue_commit(sc, vq, slot, true); 754 */ 755 756 /* 757 * enqueue_prep: allocate a slot number 758 */ 759 int 760 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 761 { 762 struct vq_entry *qe1; 763 764 KASSERT(slotp != NULL); 765 766 qe1 = vq_alloc_entry(vq); 767 if (qe1 == NULL) 768 return EAGAIN; 769 /* next slot is not allocated yet */ 770 qe1->qe_next = -1; 771 *slotp = qe1->qe_index; 772 773 return 0; 774 } 775 776 /* 777 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 778 */ 779 int 780 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 781 int slot, int nsegs) 782 { 783 int indirect; 784 struct vq_entry *qe1 = &vq->vq_entries[slot]; 785 786 KASSERT(qe1->qe_next == -1); 787 KASSERT(1 <= nsegs && nsegs <= vq->vq_num); 788 789 if ((vq->vq_indirect != NULL) && 790 (nsegs >= MINSEG_INDIRECT) && 791 (nsegs <= vq->vq_maxnsegs)) 792 indirect = 1; 793 else 794 indirect = 0; 795 qe1->qe_indirect = indirect; 796 797 if (indirect) { 798 struct vring_desc *vd; 799 int i; 800 801 vd = &vq->vq_desc[qe1->qe_index]; 802 vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr 803 + vq->vq_indirectoffset; 804 vd->addr += sizeof(struct vring_desc) 805 * vq->vq_maxnsegs * qe1->qe_index; 806 vd->len = sizeof(struct vring_desc) * nsegs; 807 vd->flags = VRING_DESC_F_INDIRECT; 808 809 vd = vq->vq_indirect; 810 vd += vq->vq_maxnsegs * qe1->qe_index; 811 qe1->qe_desc_base = vd; 812 813 for (i = 0; i < nsegs-1; i++) { 814 vd[i].flags = VRING_DESC_F_NEXT; 815 } 816 vd[i].flags = 0; 817 qe1->qe_next = 0; 818 819 return 0; 820 } else { 821 struct vring_desc *vd; 822 struct vq_entry *qe; 823 int i, s; 824 825 vd = &vq->vq_desc[0]; 826 qe1->qe_desc_base = vd; 827 qe1->qe_next = qe1->qe_index; 828 s = slot; 829 for (i = 0; i < nsegs - 1; i++) { 830 qe = vq_alloc_entry(vq); 831 if (qe == NULL) { 832 vd[s].flags = 0; 833 virtio_enqueue_abort(sc, vq, slot); 834 return EAGAIN; 835 } 836 vd[s].flags = VRING_DESC_F_NEXT; 837 vd[s].next = qe->qe_index; 838 s = qe->qe_index; 839 } 840 vd[s].flags = 0; 841 842 return 0; 843 } 844 } 845 846 /* 847 * enqueue: enqueue a single dmamap. 848 */ 849 int 850 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 851 bus_dmamap_t dmamap, bool write) 852 { 853 struct vq_entry *qe1 = &vq->vq_entries[slot]; 854 struct vring_desc *vd = qe1->qe_desc_base; 855 int i; 856 int s = qe1->qe_next; 857 858 KASSERT(s >= 0); 859 KASSERT(dmamap->dm_nsegs > 0); 860 861 for (i = 0; i < dmamap->dm_nsegs; i++) { 862 vd[s].addr = dmamap->dm_segs[i].ds_addr; 863 vd[s].len = dmamap->dm_segs[i].ds_len; 864 if (!write) 865 vd[s].flags |= VRING_DESC_F_WRITE; 866 s = vd[s].next; 867 } 868 qe1->qe_next = s; 869 870 return 0; 871 } 872 873 int 874 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 875 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 876 bool write) 877 { 878 struct vq_entry *qe1 = &vq->vq_entries[slot]; 879 struct vring_desc *vd = qe1->qe_desc_base; 880 int s = qe1->qe_next; 881 882 KASSERT(s >= 0); 883 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 884 KASSERT((dmamap->dm_segs[0].ds_len > start) && 885 (dmamap->dm_segs[0].ds_len >= start + len)); 886 887 vd[s].addr = dmamap->dm_segs[0].ds_addr + start; 888 vd[s].len = len; 889 if (!write) 890 vd[s].flags |= VRING_DESC_F_WRITE; 891 qe1->qe_next = vd[s].next; 892 893 return 0; 894 } 895 896 /* 897 * enqueue_commit: add it to the aring. 898 */ 899 int 900 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 901 bool notifynow) 902 { 903 struct vq_entry *qe1; 904 905 if (slot < 0) { 906 mutex_enter(&vq->vq_aring_lock); 907 goto notify; 908 } 909 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 910 qe1 = &vq->vq_entries[slot]; 911 if (qe1->qe_indirect) 912 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 913 mutex_enter(&vq->vq_aring_lock); 914 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot; 915 916 notify: 917 if (notifynow) { 918 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 919 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 920 membar_producer(); 921 vq->vq_avail->idx = vq->vq_avail_idx; 922 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 923 membar_producer(); 924 vq->vq_queued++; 925 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 926 membar_consumer(); 927 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) 928 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 929 VIRTIO_CONFIG_QUEUE_NOTIFY, 930 vq->vq_index); 931 } 932 mutex_exit(&vq->vq_aring_lock); 933 934 return 0; 935 } 936 937 /* 938 * enqueue_abort: rollback. 939 */ 940 int 941 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 942 { 943 struct vq_entry *qe = &vq->vq_entries[slot]; 944 struct vring_desc *vd; 945 int s; 946 947 if (qe->qe_next < 0) { 948 vq_free_entry(vq, qe); 949 return 0; 950 } 951 952 s = slot; 953 vd = &vq->vq_desc[0]; 954 while (vd[s].flags & VRING_DESC_F_NEXT) { 955 s = vd[s].next; 956 vq_free_entry(vq, qe); 957 qe = &vq->vq_entries[s]; 958 } 959 vq_free_entry(vq, qe); 960 return 0; 961 } 962 963 /* 964 * Dequeue a request. 965 */ 966 /* 967 * dequeue: dequeue a request from uring; dmamap_sync for uring is 968 * already done in the interrupt handler. 969 */ 970 int 971 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 972 int *slotp, int *lenp) 973 { 974 uint16_t slot, usedidx; 975 struct vq_entry *qe; 976 977 if (vq->vq_used_idx == vq->vq_used->idx) 978 return ENOENT; 979 mutex_enter(&vq->vq_uring_lock); 980 usedidx = vq->vq_used_idx++; 981 mutex_exit(&vq->vq_uring_lock); 982 usedidx %= vq->vq_num; 983 slot = vq->vq_used->ring[usedidx].id; 984 qe = &vq->vq_entries[slot]; 985 986 if (qe->qe_indirect) 987 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 988 989 if (slotp) 990 *slotp = slot; 991 if (lenp) 992 *lenp = vq->vq_used->ring[usedidx].len; 993 994 return 0; 995 } 996 997 /* 998 * dequeue_commit: complete dequeue; the slot is recycled for future use. 999 * if you forget to call this the slot will be leaked. 1000 */ 1001 int 1002 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1003 { 1004 struct vq_entry *qe = &vq->vq_entries[slot]; 1005 struct vring_desc *vd = &vq->vq_desc[0]; 1006 int s = slot; 1007 1008 while (vd[s].flags & VRING_DESC_F_NEXT) { 1009 s = vd[s].next; 1010 vq_free_entry(vq, qe); 1011 qe = &vq->vq_entries[s]; 1012 } 1013 vq_free_entry(vq, qe); 1014 1015 return 0; 1016 } 1017