1 /* $NetBSD: virtio.c,v 1.41 2020/05/25 07:52:16 yamaguchi Exp $ */ 2 3 /* 4 * Copyright (c) 2010 Minoura Makoto. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __KERNEL_RCSID(0, "$NetBSD: virtio.c,v 1.41 2020/05/25 07:52:16 yamaguchi Exp $"); 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/atomic.h> 35 #include <sys/bus.h> 36 #include <sys/device.h> 37 #include <sys/kmem.h> 38 #include <sys/module.h> 39 40 #define VIRTIO_PRIVATE 41 42 #include <dev/pci/virtioreg.h> /* XXX: move to non-pci */ 43 #include <dev/pci/virtiovar.h> /* XXX: move to non-pci */ 44 45 #define MINSEG_INDIRECT 2 /* use indirect if nsegs >= this value */ 46 47 static void virtio_init_vq(struct virtio_softc *, 48 struct virtqueue *, const bool); 49 50 void 51 virtio_set_status(struct virtio_softc *sc, int status) 52 { 53 sc->sc_ops->set_status(sc, status); 54 } 55 56 /* 57 * Reset the device. 58 */ 59 /* 60 * To reset the device to a known state, do following: 61 * virtio_reset(sc); // this will stop the device activity 62 * <dequeue finished requests>; // virtio_dequeue() still can be called 63 * <revoke pending requests in the vqs if any>; 64 * virtio_reinit_begin(sc); // dequeue prohibitted 65 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 66 * <some other initialization>; 67 * virtio_reinit_end(sc); // device activated; enqueue allowed 68 * Once attached, feature negotiation can only be allowed after virtio_reset. 69 */ 70 void 71 virtio_reset(struct virtio_softc *sc) 72 { 73 virtio_device_reset(sc); 74 } 75 76 void 77 virtio_reinit_start(struct virtio_softc *sc) 78 { 79 int i; 80 81 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 82 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 83 for (i = 0; i < sc->sc_nvqs; i++) { 84 int n; 85 struct virtqueue *vq = &sc->sc_vqs[i]; 86 n = sc->sc_ops->read_queue_size(sc, vq->vq_index); 87 if (n == 0) /* vq disappeared */ 88 continue; 89 if (n != vq->vq_num) { 90 panic("%s: virtqueue size changed, vq index %d\n", 91 device_xname(sc->sc_dev), 92 vq->vq_index); 93 } 94 virtio_init_vq(sc, vq, true); 95 sc->sc_ops->setup_queue(sc, vq->vq_index, 96 vq->vq_dmamap->dm_segs[0].ds_addr / VIRTIO_PAGE_SIZE); 97 } 98 } 99 100 void 101 virtio_reinit_end(struct virtio_softc *sc) 102 { 103 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 104 } 105 106 /* 107 * Feature negotiation. 108 */ 109 uint32_t 110 virtio_negotiate_features(struct virtio_softc *sc, uint32_t guest_features) 111 { 112 uint32_t r; 113 114 if (!(device_cfdata(sc->sc_dev)->cf_flags & 1) && 115 !(device_cfdata(sc->sc_child)->cf_flags & 1)) /* XXX */ 116 guest_features |= VIRTIO_F_RING_INDIRECT_DESC; 117 r = sc->sc_ops->neg_features(sc, guest_features); 118 sc->sc_features = r; 119 if (r & VIRTIO_F_RING_INDIRECT_DESC) 120 sc->sc_indirect = true; 121 else 122 sc->sc_indirect = false; 123 124 return r; 125 } 126 127 /* 128 * Device configuration registers. 129 */ 130 uint8_t 131 virtio_read_device_config_1(struct virtio_softc *sc, int index) 132 { 133 return sc->sc_ops->read_dev_cfg_1(sc, index); 134 } 135 136 uint16_t 137 virtio_read_device_config_2(struct virtio_softc *sc, int index) 138 { 139 return sc->sc_ops->read_dev_cfg_2(sc, index); 140 } 141 142 uint32_t 143 virtio_read_device_config_4(struct virtio_softc *sc, int index) 144 { 145 return sc->sc_ops->read_dev_cfg_4(sc, index); 146 } 147 148 uint64_t 149 virtio_read_device_config_8(struct virtio_softc *sc, int index) 150 { 151 return sc->sc_ops->read_dev_cfg_8(sc, index); 152 } 153 154 void 155 virtio_write_device_config_1(struct virtio_softc *sc, 156 int index, uint8_t value) 157 { 158 return sc->sc_ops->write_dev_cfg_1(sc, index, value); 159 } 160 161 void 162 virtio_write_device_config_2(struct virtio_softc *sc, 163 int index, uint16_t value) 164 { 165 return sc->sc_ops->write_dev_cfg_2(sc, index, value); 166 } 167 168 void 169 virtio_write_device_config_4(struct virtio_softc *sc, 170 int index, uint32_t value) 171 { 172 return sc->sc_ops->write_dev_cfg_4(sc, index, value); 173 } 174 175 void 176 virtio_write_device_config_8(struct virtio_softc *sc, 177 int index, uint64_t value) 178 { 179 return sc->sc_ops->write_dev_cfg_8(sc, index, value); 180 } 181 182 /* 183 * Interrupt handler. 184 */ 185 static void 186 virtio_soft_intr(void *arg) 187 { 188 struct virtio_softc *sc = arg; 189 190 KASSERT(sc->sc_intrhand != NULL); 191 192 (sc->sc_intrhand)(sc); 193 } 194 195 /* 196 * dmamap sync operations for a virtqueue. 197 */ 198 static inline void 199 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 200 { 201 /* availoffset == sizeof(vring_desc)*vq_num */ 202 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 203 ops); 204 } 205 206 static inline void 207 vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 208 { 209 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 210 vq->vq_availoffset, 211 offsetof(struct vring_avail, ring) 212 + vq->vq_num * sizeof(uint16_t), 213 ops); 214 } 215 216 static inline void 217 vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 218 { 219 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 220 vq->vq_usedoffset, 221 offsetof(struct vring_used, ring) 222 + vq->vq_num * sizeof(struct vring_used_elem), 223 ops); 224 } 225 226 static inline void 227 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 228 int ops) 229 { 230 int offset = vq->vq_indirectoffset 231 + sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 232 233 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 234 offset, sizeof(struct vring_desc) * vq->vq_maxnsegs, 235 ops); 236 } 237 238 /* 239 * Can be used as sc_intrhand. 240 */ 241 /* 242 * Scan vq, bus_dmamap_sync for the vqs (not for the payload), 243 * and calls (*vq_done)() if some entries are consumed. 244 */ 245 bool 246 virtio_vq_is_enqueued(struct virtio_softc *sc, struct virtqueue *vq) 247 { 248 249 if (vq->vq_queued) { 250 vq->vq_queued = 0; 251 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 252 } 253 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 254 membar_consumer(); 255 256 return (vq->vq_used_idx != vq->vq_used->idx) ? 1 : 0; 257 } 258 259 int 260 virtio_vq_intr(struct virtio_softc *sc) 261 { 262 struct virtqueue *vq; 263 int i, r = 0; 264 265 for (i = 0; i < sc->sc_nvqs; i++) { 266 vq = &sc->sc_vqs[i]; 267 if (virtio_vq_is_enqueued(sc, vq) == 1) { 268 if (vq->vq_done) 269 r |= (vq->vq_done)(vq); 270 } 271 } 272 273 return r; 274 } 275 276 int 277 virtio_vq_intrhand(struct virtio_softc *sc) 278 { 279 struct virtqueue *vq; 280 int i, r = 0; 281 282 for (i = 0; i < sc->sc_nvqs; i++) { 283 vq = &sc->sc_vqs[i]; 284 r |= (vq->vq_intrhand)(vq->vq_intrhand_arg); 285 } 286 287 return r; 288 } 289 290 /* 291 * Start/stop vq interrupt. No guarantee. 292 */ 293 void 294 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 295 { 296 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 297 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 298 vq->vq_queued++; 299 } 300 301 void 302 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 303 { 304 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 305 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 306 vq->vq_queued++; 307 } 308 309 /* 310 * Initialize vq structure. 311 */ 312 static void 313 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, 314 const bool reinit) 315 { 316 int i, j; 317 int vq_size = vq->vq_num; 318 319 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 320 321 /* build the indirect descriptor chain */ 322 if (vq->vq_indirect != NULL) { 323 struct vring_desc *vd; 324 325 for (i = 0; i < vq_size; i++) { 326 vd = vq->vq_indirect; 327 vd += vq->vq_maxnsegs * i; 328 for (j = 0; j < vq->vq_maxnsegs-1; j++) { 329 vd[j].next = j + 1; 330 } 331 } 332 } 333 334 /* free slot management */ 335 SIMPLEQ_INIT(&vq->vq_freelist); 336 for (i = 0; i < vq_size; i++) { 337 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, 338 &vq->vq_entries[i], qe_list); 339 vq->vq_entries[i].qe_index = i; 340 } 341 if (!reinit) 342 mutex_init(&vq->vq_freelist_lock, MUTEX_SPIN, sc->sc_ipl); 343 344 /* enqueue/dequeue status */ 345 vq->vq_avail_idx = 0; 346 vq->vq_used_idx = 0; 347 vq->vq_queued = 0; 348 if (!reinit) { 349 mutex_init(&vq->vq_aring_lock, MUTEX_SPIN, sc->sc_ipl); 350 mutex_init(&vq->vq_uring_lock, MUTEX_SPIN, sc->sc_ipl); 351 } 352 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 353 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 354 vq->vq_queued++; 355 } 356 357 /* 358 * Allocate/free a vq. 359 */ 360 int 361 virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, int index, 362 int maxsegsize, int maxnsegs, const char *name) 363 { 364 int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0; 365 int rsegs, r; 366 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1))& \ 367 ~(VIRTIO_PAGE_SIZE-1)) 368 369 /* Make sure callers allocate vqs in order */ 370 KASSERT(sc->sc_nvqs == index); 371 372 memset(vq, 0, sizeof(*vq)); 373 374 vq_size = sc->sc_ops->read_queue_size(sc, index); 375 if (vq_size == 0) { 376 aprint_error_dev(sc->sc_dev, 377 "virtqueue not exist, index %d for %s\n", 378 index, name); 379 goto err; 380 } 381 /* allocsize1: descriptor table + avail ring + pad */ 382 allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc)*vq_size 383 + sizeof(uint16_t)*(2+vq_size)); 384 /* allocsize2: used ring + pad */ 385 allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t)*2 386 + sizeof(struct vring_used_elem)*vq_size); 387 /* allocsize3: indirect table */ 388 if (sc->sc_indirect && maxnsegs >= MINSEG_INDIRECT) 389 allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size; 390 else 391 allocsize3 = 0; 392 allocsize = allocsize1 + allocsize2 + allocsize3; 393 394 /* alloc and map the memory */ 395 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 396 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 397 if (r != 0) { 398 aprint_error_dev(sc->sc_dev, 399 "virtqueue %d for %s allocation failed, " 400 "error code %d\n", index, name, r); 401 goto err; 402 } 403 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize, 404 &vq->vq_vaddr, BUS_DMA_NOWAIT); 405 if (r != 0) { 406 aprint_error_dev(sc->sc_dev, 407 "virtqueue %d for %s map failed, " 408 "error code %d\n", index, name, r); 409 goto err; 410 } 411 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 412 BUS_DMA_NOWAIT, &vq->vq_dmamap); 413 if (r != 0) { 414 aprint_error_dev(sc->sc_dev, 415 "virtqueue %d for %s dmamap creation failed, " 416 "error code %d\n", index, name, r); 417 goto err; 418 } 419 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, 420 vq->vq_vaddr, allocsize, NULL, BUS_DMA_NOWAIT); 421 if (r != 0) { 422 aprint_error_dev(sc->sc_dev, 423 "virtqueue %d for %s dmamap load failed, " 424 "error code %d\n", index, name, r); 425 goto err; 426 } 427 428 /* set the vq address */ 429 sc->sc_ops->setup_queue(sc, index, 430 vq->vq_dmamap->dm_segs[0].ds_addr / VIRTIO_PAGE_SIZE); 431 432 /* remember addresses and offsets for later use */ 433 vq->vq_owner = sc; 434 vq->vq_num = vq_size; 435 vq->vq_index = index; 436 vq->vq_desc = vq->vq_vaddr; 437 vq->vq_availoffset = sizeof(struct vring_desc)*vq_size; 438 vq->vq_avail = (void*)(((char*)vq->vq_desc) + vq->vq_availoffset); 439 vq->vq_usedoffset = allocsize1; 440 vq->vq_used = (void*)(((char*)vq->vq_desc) + vq->vq_usedoffset); 441 if (allocsize3 > 0) { 442 vq->vq_indirectoffset = allocsize1 + allocsize2; 443 vq->vq_indirect = (void*)(((char*)vq->vq_desc) 444 + vq->vq_indirectoffset); 445 } 446 vq->vq_bytesize = allocsize; 447 vq->vq_maxsegsize = maxsegsize; 448 vq->vq_maxnsegs = maxnsegs; 449 450 /* free slot management */ 451 vq->vq_entries = kmem_zalloc(sizeof(struct vq_entry)*vq_size, 452 KM_SLEEP); 453 virtio_init_vq(sc, vq, false); 454 455 aprint_verbose_dev(sc->sc_dev, 456 "allocated %u byte for virtqueue %d for %s, " 457 "size %d\n", allocsize, index, name, vq_size); 458 if (allocsize3 > 0) 459 aprint_verbose_dev(sc->sc_dev, 460 "using %d byte (%d entries) " 461 "indirect descriptors\n", 462 allocsize3, maxnsegs * vq_size); 463 464 sc->sc_nvqs++; 465 466 return 0; 467 468 err: 469 sc->sc_ops->setup_queue(sc, index, 0); 470 if (vq->vq_dmamap) 471 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 472 if (vq->vq_vaddr) 473 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 474 if (vq->vq_segs[0].ds_addr) 475 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 476 memset(vq, 0, sizeof(*vq)); 477 478 return -1; 479 } 480 481 int 482 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 483 { 484 struct vq_entry *qe; 485 int i = 0; 486 487 /* device must be already deactivated */ 488 /* confirm the vq is empty */ 489 SIMPLEQ_FOREACH(qe, &vq->vq_freelist, qe_list) { 490 i++; 491 } 492 if (i != vq->vq_num) { 493 printf("%s: freeing non-empty vq, index %d\n", 494 device_xname(sc->sc_dev), vq->vq_index); 495 return EBUSY; 496 } 497 498 /* tell device that there's no virtqueue any longer */ 499 sc->sc_ops->setup_queue(sc, vq->vq_index, 0); 500 501 kmem_free(vq->vq_entries, sizeof(*vq->vq_entries) * vq->vq_num); 502 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 503 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 504 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 505 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 506 mutex_destroy(&vq->vq_freelist_lock); 507 mutex_destroy(&vq->vq_uring_lock); 508 mutex_destroy(&vq->vq_aring_lock); 509 memset(vq, 0, sizeof(*vq)); 510 511 sc->sc_nvqs--; 512 513 return 0; 514 } 515 516 /* 517 * Free descriptor management. 518 */ 519 static struct vq_entry * 520 vq_alloc_entry(struct virtqueue *vq) 521 { 522 struct vq_entry *qe; 523 524 mutex_enter(&vq->vq_freelist_lock); 525 if (SIMPLEQ_EMPTY(&vq->vq_freelist)) { 526 mutex_exit(&vq->vq_freelist_lock); 527 return NULL; 528 } 529 qe = SIMPLEQ_FIRST(&vq->vq_freelist); 530 SIMPLEQ_REMOVE_HEAD(&vq->vq_freelist, qe_list); 531 mutex_exit(&vq->vq_freelist_lock); 532 533 return qe; 534 } 535 536 static void 537 vq_free_entry(struct virtqueue *vq, struct vq_entry *qe) 538 { 539 mutex_enter(&vq->vq_freelist_lock); 540 SIMPLEQ_INSERT_TAIL(&vq->vq_freelist, qe, qe_list); 541 mutex_exit(&vq->vq_freelist_lock); 542 543 return; 544 } 545 546 /* 547 * Enqueue several dmamaps as a single request. 548 */ 549 /* 550 * Typical usage: 551 * <queue size> number of followings are stored in arrays 552 * - command blocks (in dmamem) should be pre-allocated and mapped 553 * - dmamaps for command blocks should be pre-allocated and loaded 554 * - dmamaps for payload should be pre-allocated 555 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 556 * if (r) // currently 0 or EAGAIN 557 * return r; 558 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 559 * if (r) { 560 * virtio_enqueue_abort(sc, vq, slot); 561 * return r; 562 * } 563 * r = virtio_enqueue_reserve(sc, vq, slot, 564 * dmamap_payload[slot]->dm_nsegs+1); 565 * // ^ +1 for command 566 * if (r) { // currently 0 or EAGAIN 567 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 568 * return r; // do not call abort() 569 * } 570 * <setup and prepare commands> 571 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 572 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 573 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], false); 574 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 575 * virtio_enqueue_commit(sc, vq, slot, true); 576 */ 577 578 /* 579 * enqueue_prep: allocate a slot number 580 */ 581 int 582 virtio_enqueue_prep(struct virtio_softc *sc, struct virtqueue *vq, int *slotp) 583 { 584 struct vq_entry *qe1; 585 586 KASSERT(slotp != NULL); 587 588 qe1 = vq_alloc_entry(vq); 589 if (qe1 == NULL) 590 return EAGAIN; 591 /* next slot is not allocated yet */ 592 qe1->qe_next = -1; 593 *slotp = qe1->qe_index; 594 595 return 0; 596 } 597 598 /* 599 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 600 */ 601 int 602 virtio_enqueue_reserve(struct virtio_softc *sc, struct virtqueue *vq, 603 int slot, int nsegs) 604 { 605 int indirect; 606 struct vq_entry *qe1 = &vq->vq_entries[slot]; 607 608 KASSERT(qe1->qe_next == -1); 609 KASSERT(1 <= nsegs && nsegs <= vq->vq_num); 610 611 if ((vq->vq_indirect != NULL) && 612 (nsegs >= MINSEG_INDIRECT) && 613 (nsegs <= vq->vq_maxnsegs)) 614 indirect = 1; 615 else 616 indirect = 0; 617 qe1->qe_indirect = indirect; 618 619 if (indirect) { 620 struct vring_desc *vd; 621 int i; 622 623 vd = &vq->vq_desc[qe1->qe_index]; 624 vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr 625 + vq->vq_indirectoffset; 626 vd->addr += sizeof(struct vring_desc) 627 * vq->vq_maxnsegs * qe1->qe_index; 628 vd->len = sizeof(struct vring_desc) * nsegs; 629 vd->flags = VRING_DESC_F_INDIRECT; 630 631 vd = vq->vq_indirect; 632 vd += vq->vq_maxnsegs * qe1->qe_index; 633 qe1->qe_desc_base = vd; 634 635 for (i = 0; i < nsegs-1; i++) { 636 vd[i].flags = VRING_DESC_F_NEXT; 637 } 638 vd[i].flags = 0; 639 qe1->qe_next = 0; 640 641 return 0; 642 } else { 643 struct vring_desc *vd; 644 struct vq_entry *qe; 645 int i, s; 646 647 vd = &vq->vq_desc[0]; 648 qe1->qe_desc_base = vd; 649 qe1->qe_next = qe1->qe_index; 650 s = slot; 651 for (i = 0; i < nsegs - 1; i++) { 652 qe = vq_alloc_entry(vq); 653 if (qe == NULL) { 654 vd[s].flags = 0; 655 virtio_enqueue_abort(sc, vq, slot); 656 return EAGAIN; 657 } 658 vd[s].flags = VRING_DESC_F_NEXT; 659 vd[s].next = qe->qe_index; 660 s = qe->qe_index; 661 } 662 vd[s].flags = 0; 663 664 return 0; 665 } 666 } 667 668 /* 669 * enqueue: enqueue a single dmamap. 670 */ 671 int 672 virtio_enqueue(struct virtio_softc *sc, struct virtqueue *vq, int slot, 673 bus_dmamap_t dmamap, bool write) 674 { 675 struct vq_entry *qe1 = &vq->vq_entries[slot]; 676 struct vring_desc *vd = qe1->qe_desc_base; 677 int i; 678 int s = qe1->qe_next; 679 680 KASSERT(s >= 0); 681 KASSERT(dmamap->dm_nsegs > 0); 682 683 for (i = 0; i < dmamap->dm_nsegs; i++) { 684 vd[s].addr = dmamap->dm_segs[i].ds_addr; 685 vd[s].len = dmamap->dm_segs[i].ds_len; 686 if (!write) 687 vd[s].flags |= VRING_DESC_F_WRITE; 688 s = vd[s].next; 689 } 690 qe1->qe_next = s; 691 692 return 0; 693 } 694 695 int 696 virtio_enqueue_p(struct virtio_softc *sc, struct virtqueue *vq, int slot, 697 bus_dmamap_t dmamap, bus_addr_t start, bus_size_t len, 698 bool write) 699 { 700 struct vq_entry *qe1 = &vq->vq_entries[slot]; 701 struct vring_desc *vd = qe1->qe_desc_base; 702 int s = qe1->qe_next; 703 704 KASSERT(s >= 0); 705 KASSERT(dmamap->dm_nsegs == 1); /* XXX */ 706 KASSERT((dmamap->dm_segs[0].ds_len > start) && 707 (dmamap->dm_segs[0].ds_len >= start + len)); 708 709 vd[s].addr = dmamap->dm_segs[0].ds_addr + start; 710 vd[s].len = len; 711 if (!write) 712 vd[s].flags |= VRING_DESC_F_WRITE; 713 qe1->qe_next = vd[s].next; 714 715 return 0; 716 } 717 718 /* 719 * enqueue_commit: add it to the aring. 720 */ 721 int 722 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 723 bool notifynow) 724 { 725 struct vq_entry *qe1; 726 727 if (slot < 0) { 728 mutex_enter(&vq->vq_aring_lock); 729 goto notify; 730 } 731 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 732 qe1 = &vq->vq_entries[slot]; 733 if (qe1->qe_indirect) 734 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 735 mutex_enter(&vq->vq_aring_lock); 736 vq->vq_avail->ring[(vq->vq_avail_idx++) % vq->vq_num] = slot; 737 738 notify: 739 if (notifynow) { 740 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 741 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 742 membar_producer(); 743 vq->vq_avail->idx = vq->vq_avail_idx; 744 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 745 membar_producer(); 746 vq->vq_queued++; 747 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 748 membar_consumer(); 749 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) 750 sc->sc_ops->kick(sc, vq->vq_index); 751 } 752 mutex_exit(&vq->vq_aring_lock); 753 754 return 0; 755 } 756 757 /* 758 * enqueue_abort: rollback. 759 */ 760 int 761 virtio_enqueue_abort(struct virtio_softc *sc, struct virtqueue *vq, int slot) 762 { 763 struct vq_entry *qe = &vq->vq_entries[slot]; 764 struct vring_desc *vd; 765 int s; 766 767 if (qe->qe_next < 0) { 768 vq_free_entry(vq, qe); 769 return 0; 770 } 771 772 s = slot; 773 vd = &vq->vq_desc[0]; 774 while (vd[s].flags & VRING_DESC_F_NEXT) { 775 s = vd[s].next; 776 vq_free_entry(vq, qe); 777 qe = &vq->vq_entries[s]; 778 } 779 vq_free_entry(vq, qe); 780 return 0; 781 } 782 783 /* 784 * Dequeue a request. 785 */ 786 /* 787 * dequeue: dequeue a request from uring; dmamap_sync for uring is 788 * already done in the interrupt handler. 789 */ 790 int 791 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 792 int *slotp, int *lenp) 793 { 794 uint16_t slot, usedidx; 795 struct vq_entry *qe; 796 797 if (vq->vq_used_idx == vq->vq_used->idx) 798 return ENOENT; 799 mutex_enter(&vq->vq_uring_lock); 800 usedidx = vq->vq_used_idx++; 801 mutex_exit(&vq->vq_uring_lock); 802 usedidx %= vq->vq_num; 803 slot = vq->vq_used->ring[usedidx].id; 804 qe = &vq->vq_entries[slot]; 805 806 if (qe->qe_indirect) 807 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 808 809 if (slotp) 810 *slotp = slot; 811 if (lenp) 812 *lenp = vq->vq_used->ring[usedidx].len; 813 814 return 0; 815 } 816 817 /* 818 * dequeue_commit: complete dequeue; the slot is recycled for future use. 819 * if you forget to call this the slot will be leaked. 820 */ 821 int 822 virtio_dequeue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot) 823 { 824 struct vq_entry *qe = &vq->vq_entries[slot]; 825 struct vring_desc *vd = &vq->vq_desc[0]; 826 int s = slot; 827 828 while (vd[s].flags & VRING_DESC_F_NEXT) { 829 s = vd[s].next; 830 vq_free_entry(vq, qe); 831 qe = &vq->vq_entries[s]; 832 } 833 vq_free_entry(vq, qe); 834 835 return 0; 836 } 837 838 /* 839 * Attach a child, fill all the members. 840 */ 841 void 842 virtio_child_attach_start(struct virtio_softc *sc, device_t child, int ipl, 843 struct virtqueue *vqs, 844 virtio_callback config_change, 845 virtio_callback intr_hand, 846 int req_flags, int req_features, const char *feat_bits) 847 { 848 char buf[256]; 849 int features; 850 851 sc->sc_child = child; 852 sc->sc_ipl = ipl; 853 sc->sc_vqs = vqs; 854 sc->sc_config_change = config_change; 855 sc->sc_intrhand = intr_hand; 856 sc->sc_flags = req_flags; 857 858 features = virtio_negotiate_features(sc, req_features); 859 snprintb(buf, sizeof(buf), feat_bits, features); 860 aprint_normal(": Features: %s\n", buf); 861 aprint_naive("\n"); 862 } 863 864 void 865 virtio_child_attach_set_vqs(struct virtio_softc *sc, 866 struct virtqueue *vqs, int nvq_pairs) 867 { 868 869 KASSERT(nvq_pairs == 1 || 870 (sc->sc_flags & VIRTIO_F_PCI_INTR_SOFTINT) == 0); 871 if (nvq_pairs > 1) 872 sc->sc_child_mq = true; 873 874 sc->sc_vqs = vqs; 875 } 876 877 int 878 virtio_child_attach_finish(struct virtio_softc *sc) 879 { 880 int r; 881 882 r = sc->sc_ops->setup_interrupts(sc); 883 if (r != 0) { 884 aprint_error_dev(sc->sc_dev, "failed to setup interrupts\n"); 885 goto fail; 886 } 887 888 KASSERT(sc->sc_soft_ih == NULL); 889 if (sc->sc_flags & VIRTIO_F_PCI_INTR_SOFTINT) { 890 u_int flags = SOFTINT_NET; 891 if (sc->sc_flags & VIRTIO_F_PCI_INTR_MPSAFE) 892 flags |= SOFTINT_MPSAFE; 893 894 sc->sc_soft_ih = softint_establish(flags, virtio_soft_intr, sc); 895 if (sc->sc_soft_ih == NULL) { 896 sc->sc_ops->free_interrupts(sc); 897 aprint_error_dev(sc->sc_dev, 898 "failed to establish soft interrupt\n"); 899 goto fail; 900 } 901 } 902 903 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 904 return 0; 905 906 fail: 907 if (sc->sc_soft_ih) { 908 softint_disestablish(sc->sc_soft_ih); 909 sc->sc_soft_ih = NULL; 910 } 911 912 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 913 return 1; 914 } 915 916 void 917 virtio_child_detach(struct virtio_softc *sc) 918 { 919 sc->sc_child = NULL; 920 sc->sc_vqs = NULL; 921 922 virtio_device_reset(sc); 923 924 sc->sc_ops->free_interrupts(sc); 925 926 if (sc->sc_soft_ih) { 927 softint_disestablish(sc->sc_soft_ih); 928 sc->sc_soft_ih = NULL; 929 } 930 } 931 932 void 933 virtio_child_attach_failed(struct virtio_softc *sc) 934 { 935 virtio_child_detach(sc); 936 937 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 938 939 sc->sc_child = VIRTIO_CHILD_FAILED; 940 } 941 942 bus_dma_tag_t 943 virtio_dmat(struct virtio_softc *sc) 944 { 945 return sc->sc_dmat; 946 } 947 948 device_t 949 virtio_child(struct virtio_softc *sc) 950 { 951 return sc->sc_child; 952 } 953 954 int 955 virtio_intrhand(struct virtio_softc *sc) 956 { 957 return (sc->sc_intrhand)(sc); 958 } 959 960 uint32_t 961 virtio_features(struct virtio_softc *sc) 962 { 963 return sc->sc_features; 964 } 965 966 int 967 virtiobusprint(void *aux, const char *pnp) 968 { 969 struct virtio_attach_args * const va = aux; 970 971 if (va->sc_childdevid == 0) 972 return QUIET; /* No device present */ 973 974 if (pnp) 975 aprint_normal("Device ID %d at %s", va->sc_childdevid, pnp); 976 977 return UNCONF; 978 } 979 980 MODULE(MODULE_CLASS_DRIVER, virtio, NULL); 981 982 #ifdef _MODULE 983 #include "ioconf.c" 984 #endif 985 986 static int 987 virtio_modcmd(modcmd_t cmd, void *opaque) 988 { 989 int error = 0; 990 991 #ifdef _MODULE 992 switch (cmd) { 993 case MODULE_CMD_INIT: 994 error = config_init_component(cfdriver_ioconf_virtio, 995 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 996 break; 997 case MODULE_CMD_FINI: 998 error = config_fini_component(cfdriver_ioconf_virtio, 999 cfattach_ioconf_virtio, cfdata_ioconf_virtio); 1000 break; 1001 default: 1002 error = ENOTTY; 1003 break; 1004 } 1005 #endif 1006 1007 return error; 1008 } 1009