1 /* $NetBSD: virtio.c,v 1.6 2014/07/22 01:55:54 ozaki-r Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.6 2014/07/22 01:55:54 ozaki-r Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/atomic.h> 35 #include <sys/bus.h> 36 #include <sys/device.h> 37 #include <sys/kmem.h> 38 39 #include <dev/pci/pcidevs.h> 40 #include <dev/pci/pcireg.h> 41 #include <dev/pci/pcivar.h> 42 43 #include <dev/pci/virtioreg.h> 44 #include <dev/pci/virtiovar.h> 45 46 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 47 48 static int virtio_match(device_t, cfdata_t, void *); 49 static void virtio_attach(device_t, device_t, void *); 50 static int virtio_detach(device_t, int); 51 static int virtio_intr(void *arg); 52 static void virtio_init_vq(struct virtio_softc *, 53 struct virtqueue *, const bool); 54 55 CFATTACH_DECL3_NEW(virtio, sizeof(struct virtio_softc), 56 virtio_match, virtio_attach, virtio_detach, NULL, NULL, NULL, 57 DVF_DETACH_SHUTDOWN); 58 59 static void 60 virtio_set_status(struct virtio_softc *sc, int status) 61 { 62 int old = 0; 63 64 if (status != 0) 65 old = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 66 VIRTIO_CONFIG_DEVICE_STATUS); 67 bus_space_write_1(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_DEVICE_STATUS, 68 status|old); 69 } 70 71 #define virtio_device_reset(sc) virtio_set_status((sc), 0) 72 73 static int 74 virtio_match(device_t parent, cfdata_t match, void *aux) 75 { 76 struct pci_attach_args *pa; 77 78 pa = (struct pci_attach_args *)aux; 79 switch (PCI_VENDOR(pa->pa_id)) { 80 case PCI_VENDOR_QUMRANET: 81 if ((PCI_PRODUCT_QUMRANET_VIRTIO_1000 <= 82 PCI_PRODUCT(pa->pa_id)) && 83 (PCI_PRODUCT(pa->pa_id) <= 84 PCI_PRODUCT_QUMRANET_VIRTIO_103F)) 85 return 1; 86 break; 87 } 88 89 return 0; 90 } 91 92 static const char *virtio_device_name[] = { 93 "Unknown (0)", /* 0 */ 94 "Network", /* 1 */ 95 "Block", /* 2 */ 96 "Console", /* 3 */ 97 "Entropy", /* 4 */ 98 "Memory Balloon", /* 5 */ 99 "Unknown (6)", /* 6 */ 100 "Unknown (7)", /* 7 */ 101 "Unknown (8)", /* 8 */ 102 "9P Transport" /* 9 */ 103 }; 104 #define NDEVNAMES (sizeof(virtio_device_name)/sizeof(char*)) 105 106 static void 107 virtio_attach(device_t parent, device_t self, void *aux) 108 { 109 struct virtio_softc *sc = device_private(self); 110 struct pci_attach_args *pa = (struct pci_attach_args *)aux; 111 pci_chipset_tag_t pc = pa->pa_pc; 112 pcitag_t tag = pa->pa_tag; 113 int revision; 114 pcireg_t id; 115 char const *intrstr; 116 pci_intr_handle_t ih; 117 char intrbuf[PCI_INTRSTR_LEN]; 118 119 revision = PCI_REVISION(pa->pa_class); 120 if (revision != 0) { 121 aprint_normal(": unknown revision 0x%02x; giving up\n", 122 revision); 123 return; 124 } 125 aprint_normal("\n"); 126 aprint_naive("\n"); 127 128 /* subsystem ID shows what I am */ 129 id = pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG); 130 aprint_normal_dev(self, "Virtio %s Device (rev. 0x%02x)\n", 131 (PCI_PRODUCT(id) < NDEVNAMES? 132 virtio_device_name[PCI_PRODUCT(id)] : "Unknown"), 133 revision); 134 135 sc->sc_dev = self; 136 sc->sc_pc = pc; 137 sc->sc_tag = tag; 138 sc->sc_iot = pa->pa_iot; 139 sc->sc_dmat = pa->pa_dmat; 140 sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI; 141 142 if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0, 143 &sc->sc_iot, &sc->sc_ioh, NULL, &sc->sc_iosize)) { 144 aprint_error_dev(self, "can't map i/o space\n"); 145 return; 146 } 147 148 virtio_device_reset(sc); 149 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 150 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 151 152 /* XXX: use softc as aux... */ 153 sc->sc_childdevid = PCI_PRODUCT(id); 154 sc->sc_child = NULL; 155 config_found(self, sc, NULL); 156 if (sc->sc_child == NULL) { 157 aprint_error_dev(self, 158 "no matching child driver; not configured\n"); 159 return; 160 } 161 if (sc->sc_child == (void*)1) { /* this shows error */ 162 aprint_error_dev(self, 163 "virtio configuration failed\n"); 164 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 165 return; 166 } 167 168 if (pci_intr_map(pa, &ih)) { 169 aprint_error_dev(self, "couldn't map interrupt\n"); 170 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 171 return; 172 } 173 174 intrstr = pci_intr_string(pc, ih, intrbuf, sizeof(intrbuf)); 175 176 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 177 pci_intr_setattr(pc, &ih, PCI_INTR_MPSAFE, true); 178 179 sc->sc_ih = pci_intr_establish(pc, ih, sc->sc_ipl, virtio_intr, sc); 180 181 if (sc->sc_ih == NULL) { 182 aprint_error_dev(self, "couldn't establish interrupt"); 183 if (intrstr != NULL) 184 aprint_error(" at %s", intrstr); 185 aprint_error("\n"); 186 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 187 return; 188 } 189 aprint_normal_dev(self, "interrupting at %s\n", intrstr); 190 191 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 192 193 return; 194 } 195 196 static int 197 virtio_detach(device_t self, int flags) 198 { 199 struct virtio_softc *sc = device_private(self); 200 int r; 201 202 if (sc->sc_child != 0 && sc->sc_child != (void*)1) { 203 r = config_detach(sc->sc_child, flags); 204 if (r) 205 return r; 206 } 207 KASSERT(sc->sc_child == 0 || sc->sc_child == (void*)1); 208 KASSERT(sc->sc_vqs == 0); 209 if (sc->sc_ih != NULL) { 210 pci_intr_disestablish(sc->sc_pc, sc->sc_ih); 211 sc->sc_ih = NULL; 212 } 213 if (sc->sc_iosize) 214 bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize); 215 sc->sc_iosize = 0; 216 217 return 0; 218 } 219 220 /* 221 * Reset the device. 222 */ 223 /* 224 * To reset the device to a known state, do following: 225 * virtio_reset(sc); // this will stop the device activity 226 * <dequeue finished requests>; // virtio_dequeue() still can be called 227 * <revoke pending requests in the vqs if any>; 228 * virtio_reinit_begin(sc); // dequeue prohibitted 229 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 230 * <some other initialization>; 231 * virtio_reinit_end(sc); // device activated; enqueue allowed 232 * Once attached, feature negotiation can only be allowed after virtio_reset. 233 */ 234 void 235 virtio_reset(struct virtio_softc *sc) 236 { 237 virtio_device_reset(sc); 238 } 239 240 void 241 virtio_reinit_start(struct virtio_softc *sc) 242 { 243 int i; 244 245 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 246 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 247 for (i = 0; i < sc->sc_nvqs; i++) { 248 int n; 249 struct virtqueue *vq = &sc->sc_vqs[i]; 250 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 251 VIRTIO_CONFIG_QUEUE_SELECT, 252 vq->vq_index); 253 n = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 254 VIRTIO_CONFIG_QUEUE_SIZE); 255 if (n == 0) /* vq disappeared */ 256 continue; 257 if (n != vq->vq_num) { 258 panic("%s: virtqueue size changed, vq index %d\n", 259 device_xname(sc->sc_dev), 260 vq->vq_index); 261 } 262 virtio_init_vq(sc, vq, true); 263 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 264 VIRTIO_CONFIG_QUEUE_ADDRESS, 265 (vq->vq_dmamap->dm_segs[0].ds_addr 266 / VIRTIO_PAGE_SIZE)); 267 } 268 } 269 270 void 271 virtio_reinit_end(struct virtio_softc *sc) 272 { 273 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 274 } 275 276 /* 277 * Feature negotiation. 278 */ 279 uint32_t 280 virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features) 281 { 282 uint32_t r; 283 284 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 285 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 286 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 287 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 288 VIRTIO_CONFIG_DEVICE_FEATURES); 289 r &= guest_features; 290 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 291 VIRTIO_CONFIG_GUEST_FEATURES, r); 292 sc->sc_features = r; 293 if (r & VIRTIO_F_RING_INDIRECT_DESC) 294 sc->sc_indirect = true; 295 else 296 sc->sc_indirect = false; 297 298 return r; 299 } 300 301 /* 302 * Device configuration registers. 303 */ 304 uint8_t 305 virtio_read_device_config_1(struct virtio_softc *sc, int index) 306 { 307 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, 308 sc->sc_config_offset + index); 309 } 310 311 uint16_t 312 virtio_read_device_config_2(struct virtio_softc *sc, int index) 313 { 314 return bus_space_read_2(sc->sc_iot, sc->sc_ioh, 315 sc->sc_config_offset + index); 316 } 317 318 uint32_t 319 virtio_read_device_config_4(struct virtio_softc *sc, int index) 320 { 321 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, 322 sc->sc_config_offset + index); 323 } 324 325 uint64_t 326 virtio_read_device_config_8(struct virtio_softc *sc, int index) 327 { 328 uint64_t r; 329 330 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 331 sc->sc_config_offset + index + sizeof(uint32_t)); 332 r <<= 32; 333 r += bus_space_read_4(sc->sc_iot, sc->sc_ioh, 334 sc->sc_config_offset + index); 335 return r; 336 } 337 338 void 339 virtio_write_device_config_1(struct virtio_softc *sc, 340 int index, uint8_t value) 341 { 342 bus_space_write_1(sc->sc_iot, sc->sc_ioh, 343 sc->sc_config_offset + index, value); 344 } 345 346 void 347 virtio_write_device_config_2(struct virtio_softc *sc, 348 int index, uint16_t value) 349 { 350 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 351 sc->sc_config_offset + index, value); 352 } 353 354 void 355 virtio_write_device_config_4(struct virtio_softc *sc, 356 int index, uint32_t value) 357 { 358 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 359 sc->sc_config_offset + index, value); 360 } 361 362 void 363 virtio_write_device_config_8(struct virtio_softc *sc, 364 int index, uint64_t value) 365 { 366 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 367 sc->sc_config_offset + index, 368 value & 0xffffffff); 369 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 370 sc->sc_config_offset + index + sizeof(uint32_t), 371 value >> 32); 372 } 373 374 /* 375 * Interrupt handler. 376 */ 377 static int 378 virtio_intr(void *arg) 379 { 380 struct virtio_softc *sc = arg; 381 int isr, r = 0; 382 383 /* check and ack the interrupt */ 384 isr = bus_space_read_1(sc->sc_iot, sc->sc_ioh, 385 VIRTIO_CONFIG_ISR_STATUS); 386 if (isr == 0) 387 return 0; 388 if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) && 389 (sc->sc_config_change != NULL)) 390 r = (sc->sc_config_change)(sc); 391 if (sc->sc_intrhand != NULL) 392 r |= (sc->sc_intrhand)(sc); 393 394 return r; 395 } 396 397 /* 398 * dmamap sync operations for a virtqueue. 399 */ 400 static inline void 401 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 402 { 403 /* availoffset == sizeof(vring_desc)*vq_num */ 404 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 405 ops); 406 } 407 408 static inline void 409 vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 410 { 411 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 412 vq->vq_availoffset, 413 offsetof(struct vring_avail, ring) 414 + vq->vq_num * sizeof(uint16_t), 415 ops); 416 } 417 418 static inline void 419 vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 420 { 421 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 422 vq->vq_usedoffset, 423 offsetof(struct vring_used, ring) 424 + vq->vq_num * sizeof(struct vring_used_elem), 425 ops); 426 } 427 428 static inline void 429 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 430 int ops) 431 { 432 int offset = vq->vq_indirectoffset 433 + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 434 435 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 436 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, 437 ops); 438 } 439 440 /* 441 * Can be used as sc_intrhand. 442 */ 443 /* 444 * Scan vq, bus_dmamap_sync for the vqs (not for the payload), 445 * and calls (*vq_done)() if some entries are consumed. 446 */ 447 int 448 virtio_vq_intr(struct virtio_softc *sc) 449 { 450 struct virtqueue *vq; 451 int i, r = 0; 452 453 for (i = 0; i < sc->sc_nvqs; i++) { 454 vq = &sc->sc_vqs[i]; 455 if (vq->vq_queued) { 456 vq->vq_queued = 0; 457 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 458 } 459 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 460 membar_consumer(); 461 if (vq->vq_used_idx != vq->vq_used->idx) { 462 if (vq->vq_done) 463 r |= (vq->vq_done)(vq); 464 } 465 } 466 467 468 return r; 469 } 470 471 /* 472 * Start/stop vq interrupt. No guarantee. 473 */ 474 void 475 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 476 { 477 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 478 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 479 vq->vq_queued++; 480 } 481 482 void 483 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 484 { 485 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 486 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 487 vq->vq_queued++; 488 } 489 490 /* 491 * Initialize vq structure. 492 */ 493 static void 494 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, const bool reinit) 495 { 496 int i, j; 497 int vq_size = vq->vq_num; 498 499 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 500 501 /* build the indirect descriptor chain */ 502 if (vq->vq_indirect != NULL) { 503 struct vring_desc *vd; 504 505 for (i = 0; i < vq_size; i++) { 506 vd = vq->vq_indirect; 507 vd += vq->vq_maxnsegs * i; 508 for (j = 0; j < vq->vq_maxnsegs-1; j++) 509 vd[j].next = j + 1; 510 } 511 } 512 513 /* free slot management */ 514 SIMPLEQ_INIT(&vq->vq_freelist); 515 for (i = 0; i < vq_size; i++) { 516 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, 517 &vq->vq_entries[i], qe_list); 518 vq->vq_entries[i].qe_index = i; 519 } 520 if (!reinit) 521 mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl); 522 523 /* enqueue/dequeue status */ 524 vq->vq_avail_idx = 0; 525 vq->vq_used_idx = 0; 526 vq->vq_queued = 0; 527 if (!reinit) { 528 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 529 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 530 } 531 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 532 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 533 vq->vq_queued++; 534 } 535 536 /* 537 * Allocate/free a vq. 538 */ 539 int 540 virtio_alloc_vq(struct virtio_softc *sc, 541 struct virtqueue *vq, int index, int maxsegsize, int maxnsegs, 542 const char *name) 543 { 544 int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0; 545 int rsegs, r; 546 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1))& \ 547 ~(VIRTIO_PAGE_SIZE-1)) 548 549 memset(vq, 0, sizeof(*vq)); 550 551 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 552 VIRTIO_CONFIG_QUEUE_SELECT, index); 553 vq_size = bus_space_read_2(sc->sc_iot, sc->sc_ioh, 554 VIRTIO_CONFIG_QUEUE_SIZE); 555 if (vq_size == 0) { 556 aprint_error_dev(sc->sc_dev, 557 "virtqueue not exist, index %d for %s\n", 558 index, name); 559 goto err; 560 } 561 /* allocsize1: descriptor table + avail ring + pad */ 562 allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size 563 + sizeof(uint16_t)*(2+vq_size)); 564 /* allocsize2: used ring + pad */ 565 allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2 566 + sizeof(struct vring_used_elem)*vq_size); 567 /* allocsize3: indirect table */ 568 if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) 569 allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size; 570 else 571 allocsize3 = 0; 572 allocsize = allocsize1 + allocsize2 + allocsize3; 573 574 /* alloc and map the memory */ 575 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 576 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 577 if (r != 0) { 578 aprint_error_dev(sc->sc_dev, 579 "virtqueue %d for %s allocation failed, " 580 "error code %d\n", index, name, r); 581 goto err; 582 } 583 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize, 584 &vq->vq_vaddr, BUS_DMA_NOWAIT); 585 if (r != 0) { 586 aprint_error_dev(sc->sc_dev, 587 "virtqueue %d for %s map failed, " 588 "error code %d\n", index, name, r); 589 goto err; 590 } 591 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 592 BUS_DMA_NOWAIT, &vq->vq_dmamap); 593 if (r != 0) { 594 aprint_error_dev(sc->sc_dev, 595 "virtqueue %d for %s dmamap creation failed, " 596 "error code %d\n", index, name, r); 597 goto err; 598 } 599 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 600 vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT); 601 if (r != 0) { 602 aprint_error_dev(sc->sc_dev, 603 "virtqueue %d for %s dmamap load failed, " 604 "error code %d\n", index, name, r); 605 goto err; 606 } 607 608 /* set the vq address */ 609 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 610 VIRTIO_CONFIG_QUEUE_ADDRESS, 611 (vq->vq_dmamap->dm_segs[0].ds_addr 612 / VIRTIO_PAGE_SIZE)); 613 614 /* remember addresses and offsets for later use */ 615 vq->vq_owner = sc; 616 vq->vq_num = vq_size; 617 vq->vq_index = index; 618 vq->vq_desc = vq->vq_vaddr; 619 vq->vq_availoffset = sizeof(struct vring_desc)*vq_size; 620 vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset); 621 vq->vq_usedoffset = allocsize1; 622 vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset); 623 if (allocsize3 > 0) { 624 vq->vq_indirectoffset = allocsize1 + allocsize2; 625 vq->vq_indirect = (void*)(((char*)vq->vq_desc) 626 + vq->vq_indirectoffset); 627 } 628 vq->vq_bytesize = allocsize; 629 vq->vq_maxsegsize = maxsegsize; 630 vq->vq_maxnsegs = maxnsegs; 631 632 /* free slot management */ 633 vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size, 634 KM_NOSLEEP); 635 if (vq->vq_entries == NULL) { 636 r = ENOMEM; 637 goto err; 638 } 639 640 virtio_init_vq(sc, vq, false); 641 642 aprint_verbose_dev(sc->sc_dev, 643 "allocated %u byte for virtqueue %d for %s, " 644 "size %d\n", allocsize, index, name, vq_size); 645 if (allocsize3 > 0) 646 aprint_verbose_dev(sc->sc_dev, 647 "using %d byte (%d entries) " 648 "indirect descriptors\n", 649 allocsize3, maxnsegs * vq_size); 650 return 0; 651 652 err: 653 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 654 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 655 if (vq->vq_dmamap) 656 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 657 if (vq->vq_vaddr) 658 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 659 if (vq->vq_segs[0].ds_addr) 660 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 661 memset(vq, 0, sizeof(*vq)); 662 663 return -1; 664 } 665 666 int 667 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 668 { 669 struct vq_entry *qe; 670 int i = 0; 671 672 /* device must be already deactivated */ 673 /* confirm the vq is empty */ 674 SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) { 675 i++; 676 } 677 if (i != vq->vq_num) { 678 printf("%s: freeing non-empty vq, index %d\n", 679 device_xname(sc->sc_dev), vq->vq_index); 680 return EBUSY; 681 } 682 683 /* tell device that there's no virtqueue any longer */ 684 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 685 VIRTIO_CONFIG_QUEUE_SELECT, vq->vq_index); 686 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 687 VIRTIO_CONFIG_QUEUE_ADDRESS, 0); 688 689 kmem_free(vq->vq_entries, vq->vq_bytesize); 690 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 691 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 692 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 693 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 694 mutex_destroy(&vq->vq_freelist_lock); 695 mutex_destroy(&vq->vq_uring_lock); 696 mutex_destroy(&vq->vq_aring_lock); 697 memset(vq, 0, sizeof(*vq)); 698 699 return 0; 700 } 701 702 /* 703 * Free descriptor management. 704 */ 705 static struct vq_entry * 706 vq_alloc_entry(struct virtqueue *vq) 707 { 708 struct vq_entry *qe; 709 710 mutex_enter(&vq->vq_freelist_lock); 711 if (SIMPLEQ_EMPTY(&vq->vq_freelist)) { 712 mutex_exit(&vq->vq_freelist_lock); 713 return NULL; 714 } 715 qe = SIMPLEQ_FIRST(&vq->vq_freelist); 716 SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list); 717 mutex_exit(&vq->vq_freelist_lock); 718 719 return qe; 720 } 721 722 static void 723 vq_free_entry(struct virtqueue *vq, struct vq_entry *qe) 724 { 725 mutex_enter(&vq->vq_freelist_lock); 726 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list); 727 mutex_exit(&vq->vq_freelist_lock); 728 729 return; 730 } 731 732 /* 733 * Enqueue several dmamaps as a single request. 734 */ 735 /* 736 * Typical usage: 737 * <queue size> number of followings are stored in arrays 738 * - command blocks (in dmamem) should be pre-allocated and mapped 739 * - dmamaps for command blocks should be pre-allocated and loaded 740 * - dmamaps for payload should be pre-allocated 741 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 742 * if (r) // currently 0 or EAGAIN 743 * return r; 744 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 745 * if (r) { 746 * virtio_enqueue_abort(sc, vq, slot); 747 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 748 * return r; 749 * } 750 * r = virtio_enqueue_reserve(sc, vq, slot, 751 * dmamap_payload[slot]->dm_nsegs+1); 752 * // ^ +1 for command 753 * if (r) { // currently 0 or EAGAIN 754 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 755 * return r; // do not call abort() 756 * } 757 * <setup and prepare commands> 758 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 759 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 760 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 761 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 762 * virtio_enqueue_commit(sc, vq, slot, true); 763 */ 764 765 /* 766 * enqueue_prep: allocate a slot number 767 */ 768 int 769 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 770 { 771 struct vq_entry *qe1; 772 773 KASSERT(slotp != NULL); 774 775 qe1 = vq_alloc_entry(vq); 776 if (qe1 == NULL) 777 return EAGAIN; 778 /* next slot is not allocated yet */ 779 qe1->qe_next = -1; 780 *slotp = qe1->qe_index; 781 782 return 0; 783 } 784 785 /* 786 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 787 */ 788 int 789 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 790 int slot, int nsegs) 791 { 792 int indirect; 793 struct vq_entry *qe1 = &vq->vq_entries[slot]; 794 795 KASSERT(qe1->qe_next == -1); 796 KASSERT(1 <= nsegs && nsegs <= vq->vq_num); 797 798 if ((vq->vq_indirect != NULL) && 799 (nsegs >= MINSEG_INDIRECT) && 800 (nsegs <= vq->vq_maxnsegs)) 801 indirect = 1; 802 else 803 indirect = 0; 804 qe1->qe_indirect = indirect; 805 806 if (indirect) { 807 struct vring_desc *vd; 808 int i; 809 810 vd = &vq->vq_desc[qe1->qe_index]; 811 vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr 812 + vq->vq_indirectoffset; 813 vd->addr += sizeof(struct vring_desc) 814 * vq->vq_maxnsegs * qe1->qe_index; 815 vd->len = sizeof(struct vring_desc) * nsegs; 816 vd->flags = VRING_DESC_F_INDIRECT; 817 818 vd = vq->vq_indirect; 819 vd += vq->vq_maxnsegs * qe1->qe_index; 820 qe1->qe_desc_base = vd; 821 822 for (i = 0; i < nsegs-1; i++) { 823 vd[i].flags = VRING_DESC_F_NEXT; 824 } 825 vd[i].flags = 0; 826 qe1->qe_next = 0; 827 828 return 0; 829 } else { 830 struct vring_desc *vd; 831 struct vq_entry *qe; 832 int i, s; 833 834 vd = &vq->vq_desc[0]; 835 qe1->qe_desc_base = vd; 836 qe1->qe_next = qe1->qe_index; 837 s = slot; 838 for (i = 0; i < nsegs - 1; i++) { 839 qe = vq_alloc_entry(vq); 840 if (qe == NULL) { 841 vd[s].flags = 0; 842 virtio_enqueue_abort(sc, vq, slot); 843 return EAGAIN; 844 } 845 vd[s].flags = VRING_DESC_F_NEXT; 846 vd[s].next = qe->qe_index; 847 s = qe->qe_index; 848 } 849 vd[s].flags = 0; 850 851 return 0; 852 } 853 } 854 855 /* 856 * enqueue: enqueue a single dmamap. 857 */ 858 int 859 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 860 bus_dmamap_t dmamap, bool write) 861 { 862 struct vq_entry *qe1 = &vq->vq_entries[slot]; 863 struct vring_desc *vd = qe1->qe_desc_base; 864 int i; 865 int s = qe1->qe_next; 866 867 KASSERT(s >= 0); 868 KASSERT(dmamap->dm_nsegs > 0); 869 870 for (i = 0; i < dmamap->dm_nsegs; i++) { 871 vd[s].addr = dmamap->dm_segs[i].ds_addr; 872 vd[s].len = dmamap->dm_segs[i].ds_len; 873 if (!write) 874 vd[s].flags |= VRING_DESC_F_WRITE; 875 s = vd[s].next; 876 } 877 qe1->qe_next = s; 878 879 return 0; 880 } 881 882 int 883 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 884 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 885 bool write) 886 { 887 struct vq_entry *qe1 = &vq->vq_entries[slot]; 888 struct vring_desc *vd = qe1->qe_desc_base; 889 int s = qe1->qe_next; 890 891 KASSERT(s >= 0); 892 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 893 KASSERT((dmamap->dm_segs[0].ds_len > start) && 894 (dmamap->dm_segs[0].ds_len >= start + len)); 895 896 vd[s].addr = dmamap->dm_segs[0].ds_addr + start; 897 vd[s].len = len; 898 if (!write) 899 vd[s].flags |= VRING_DESC_F_WRITE; 900 qe1->qe_next = vd[s].next; 901 902 return 0; 903 } 904 905 /* 906 * enqueue_commit: add it to the aring. 907 */ 908 int 909 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 910 bool notifynow) 911 { 912 struct vq_entry *qe1; 913 914 if (slot < 0) { 915 mutex_enter(&vq->vq_aring_lock); 916 goto notify; 917 } 918 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 919 qe1 = &vq->vq_entries[slot]; 920 if (qe1->qe_indirect) 921 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 922 mutex_enter(&vq->vq_aring_lock); 923 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot; 924 925 notify: 926 if (notifynow) { 927 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 928 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 929 membar_producer(); 930 vq->vq_avail->idx = vq->vq_avail_idx; 931 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 932 membar_producer(); 933 vq->vq_queued++; 934 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 935 membar_consumer(); 936 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) 937 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 938 VIRTIO_CONFIG_QUEUE_NOTIFY, 939 vq->vq_index); 940 } 941 mutex_exit(&vq->vq_aring_lock); 942 943 return 0; 944 } 945 946 /* 947 * enqueue_abort: rollback. 948 */ 949 int 950 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 951 { 952 struct vq_entry *qe = &vq->vq_entries[slot]; 953 struct vring_desc *vd; 954 int s; 955 956 if (qe->qe_next < 0) { 957 vq_free_entry(vq, qe); 958 return 0; 959 } 960 961 s = slot; 962 vd = &vq->vq_desc[0]; 963 while (vd[s].flags & VRING_DESC_F_NEXT) { 964 s = vd[s].next; 965 vq_free_entry(vq, qe); 966 qe = &vq->vq_entries[s]; 967 } 968 vq_free_entry(vq, qe); 969 return 0; 970 } 971 972 /* 973 * Dequeue a request. 974 */ 975 /* 976 * dequeue: dequeue a request from uring; dmamap_sync for uring is 977 * already done in the interrupt handler. 978 */ 979 int 980 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 981 int *slotp, int *lenp) 982 { 983 uint16_t slot, usedidx; 984 struct vq_entry *qe; 985 986 if (vq->vq_used_idx == vq->vq_used->idx) 987 return ENOENT; 988 mutex_enter(&vq->vq_uring_lock); 989 usedidx = vq->vq_used_idx++; 990 mutex_exit(&vq->vq_uring_lock); 991 usedidx %= vq->vq_num; 992 slot = vq->vq_used->ring[usedidx].id; 993 qe = &vq->vq_entries[slot]; 994 995 if (qe->qe_indirect) 996 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 997 998 if (slotp) 999 *slotp = slot; 1000 if (lenp) 1001 *lenp = vq->vq_used->ring[usedidx].len; 1002 1003 return 0; 1004 } 1005 1006 /* 1007 * dequeue_commit: complete dequeue; the slot is recycled for future use. 1008 * if you forget to call this the slot will be leaked. 1009 */ 1010 int 1011 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 1012 { 1013 struct vq_entry *qe = &vq->vq_entries[slot]; 1014 struct vring_desc *vd = &vq->vq_desc[0]; 1015 int s = slot; 1016 1017 while (vd[s].flags & VRING_DESC_F_NEXT) { 1018 s = vd[s].next; 1019 vq_free_entry(vq, qe); 1020 qe = &vq->vq_entries[s]; 1021 } 1022 vq_free_entry(vq, qe); 1023 1024 return 0; 1025 } 1026