1 /* $OpenBSD: virtio.c,v 1.13 2019/01/10 18:06:56 sf Exp $ */ 2 /* $NetBSD: virtio.c,v 1.3 2011/11/02 23:05:52 njoly Exp $ */ 3 4 /* 5 * Copyright (c) 2012 Stefan Fritsch, Alexander Fiveg. 6 * Copyright (c) 2010 Minoura Makoto. 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 #include <sys/param.h> 31 #include <sys/systm.h> 32 #include <sys/kernel.h> 33 #include <sys/device.h> 34 #include <sys/mutex.h> 35 #include <sys/atomic.h> 36 #include <sys/malloc.h> 37 38 #include <dev/pv/virtioreg.h> 39 #include <dev/pv/virtiovar.h> 40 41 #if VIRTIO_DEBUG 42 #define VIRTIO_ASSERT(x) KASSERT(x) 43 #else 44 #define VIRTIO_ASSERT(x) 45 #endif 46 47 void virtio_init_vq(struct virtio_softc *, 48 struct virtqueue *, int); 49 void vq_free_entry(struct virtqueue *, struct vq_entry *); 50 struct vq_entry *vq_alloc_entry(struct virtqueue *); 51 52 struct cfdriver virtio_cd = { 53 NULL, "virtio", DV_DULL 54 }; 55 56 static const char * const virtio_device_name[] = { 57 "Unknown (0)", /* 0 */ 58 "Network", /* 1 */ 59 "Block", /* 2 */ 60 "Console", /* 3 */ 61 "Entropy", /* 4 */ 62 "Memory Balloon", /* 5 */ 63 "IO Memory", /* 6 */ 64 "Rpmsg", /* 7 */ 65 "SCSI host", /* 8 */ 66 "9P Transport", /* 9 */ 67 "mac80211 wlan" /* 10 */ 68 }; 69 #define NDEVNAMES (sizeof(virtio_device_name)/sizeof(char*)) 70 71 const char * 72 virtio_device_string(int id) 73 { 74 return id < NDEVNAMES ? virtio_device_name[id] : "Unknown"; 75 } 76 77 #if VIRTIO_DEBUG 78 static const struct virtio_feature_name transport_feature_names[] = { 79 { VIRTIO_F_NOTIFY_ON_EMPTY, "NotifyOnEmpty"}, 80 { VIRTIO_F_RING_INDIRECT_DESC, "RingIndirectDesc"}, 81 { VIRTIO_F_RING_EVENT_IDX, "RingEventIdx"}, 82 { VIRTIO_F_BAD_FEATURE, "BadFeature"}, 83 { 0, NULL} 84 }; 85 86 void 87 virtio_log_features(uint32_t host, uint32_t neg, 88 const struct virtio_feature_name *guest_feature_names) 89 { 90 const struct virtio_feature_name *namep; 91 int i; 92 char c; 93 uint32_t bit; 94 95 for (i = 0; i < 32; i++) { 96 if (i == 30) { 97 /* 98 * VIRTIO_F_BAD_FEATURE is only used for 99 * checking correct negotiation 100 */ 101 continue; 102 } 103 bit = 1 << i; 104 if ((host&bit) == 0) 105 continue; 106 namep = (i < 24) ? guest_feature_names : 107 transport_feature_names; 108 while (namep->bit && namep->bit != bit) 109 namep++; 110 c = (neg&bit) ? '+' : '-'; 111 if (namep->name) 112 printf(" %c%s", c, namep->name); 113 else 114 printf(" %cUnknown(%d)", c, i); 115 } 116 } 117 #endif 118 119 /* 120 * Reset the device. 121 */ 122 /* 123 * To reset the device to a known state, do following: 124 * virtio_reset(sc); // this will stop the device activity 125 * <dequeue finished requests>; // virtio_dequeue() still can be called 126 * <revoke pending requests in the vqs if any>; 127 * virtio_reinit_start(sc); // dequeue prohibitted 128 * newfeatures = virtio_negotiate_features(sc, requestedfeatures); 129 * <some other initialization>; 130 * virtio_reinit_end(sc); // device activated; enqueue allowed 131 * Once attached, feature negotiation can only be allowed after virtio_reset. 132 */ 133 void 134 virtio_reset(struct virtio_softc *sc) 135 { 136 virtio_device_reset(sc); 137 } 138 139 void 140 virtio_reinit_start(struct virtio_softc *sc) 141 { 142 int i; 143 144 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 145 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 146 for (i = 0; i < sc->sc_nvqs; i++) { 147 int n; 148 struct virtqueue *vq = &sc->sc_vqs[i]; 149 n = virtio_read_queue_size(sc, vq->vq_index); 150 if (n == 0) /* vq disappeared */ 151 continue; 152 if (n != vq->vq_num) { 153 panic("%s: virtqueue size changed, vq index %d\n", 154 sc->sc_dev.dv_xname, vq->vq_index); 155 } 156 virtio_init_vq(sc, vq, 1); 157 virtio_setup_queue(sc, vq->vq_index, 158 vq->vq_dmamap->dm_segs[0].ds_addr / VIRTIO_PAGE_SIZE); 159 } 160 } 161 162 void 163 virtio_reinit_end(struct virtio_softc *sc) 164 { 165 virtio_set_status(sc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK); 166 } 167 168 /* 169 * dmamap sync operations for a virtqueue. 170 */ 171 static inline void 172 vq_sync_descs(struct virtio_softc *sc, struct virtqueue *vq, int ops) 173 { 174 /* availoffset == sizeof(vring_desc)*vq_num */ 175 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, 0, vq->vq_availoffset, 176 ops); 177 } 178 179 static inline void 180 vq_sync_aring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 181 { 182 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, vq->vq_availoffset, 183 offsetof(struct vring_avail, ring) + vq->vq_num * sizeof(uint16_t), 184 ops); 185 } 186 187 static inline void 188 vq_sync_uring(struct virtio_softc *sc, struct virtqueue *vq, int ops) 189 { 190 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, vq->vq_usedoffset, 191 offsetof(struct vring_used, ring) + vq->vq_num * 192 sizeof(struct vring_used_elem), ops); 193 } 194 195 static inline void 196 vq_sync_indirect(struct virtio_softc *sc, struct virtqueue *vq, int slot, 197 int ops) 198 { 199 int offset = vq->vq_indirectoffset + 200 sizeof(struct vring_desc) * vq->vq_maxnsegs * slot; 201 202 bus_dmamap_sync(sc->sc_dmat, vq->vq_dmamap, offset, 203 sizeof(struct vring_desc) * vq->vq_maxnsegs, ops); 204 } 205 206 /* 207 * Scan vq, bus_dmamap_sync for the vqs (not for the payload), 208 * and calls (*vq_done)() if some entries are consumed. 209 * For use in transport specific irq handlers. 210 */ 211 int 212 virtio_check_vqs(struct virtio_softc *sc) 213 { 214 struct virtqueue *vq; 215 int i, r = 0; 216 217 /* going backwards is better for if_vio */ 218 for (i = sc->sc_nvqs - 1; i >= 0; i--) { 219 vq = &sc->sc_vqs[i]; 220 if (vq->vq_queued) { 221 vq->vq_queued = 0; 222 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 223 } 224 vq_sync_uring(sc, vq, BUS_DMASYNC_POSTREAD); 225 if (vq->vq_used_idx != vq->vq_used->idx) { 226 if (vq->vq_done) 227 r |= (vq->vq_done)(vq); 228 } 229 } 230 231 return r; 232 } 233 234 /* 235 * Initialize vq structure. 236 */ 237 void 238 virtio_init_vq(struct virtio_softc *sc, struct virtqueue *vq, int reinit) 239 { 240 int i, j; 241 int vq_size = vq->vq_num; 242 243 memset(vq->vq_vaddr, 0, vq->vq_bytesize); 244 245 /* build the indirect descriptor chain */ 246 if (vq->vq_indirect != NULL) { 247 struct vring_desc *vd; 248 249 for (i = 0; i < vq_size; i++) { 250 vd = vq->vq_indirect; 251 vd += vq->vq_maxnsegs * i; 252 for (j = 0; j < vq->vq_maxnsegs-1; j++) 253 vd[j].next = j + 1; 254 } 255 } 256 257 /* free slot management */ 258 SLIST_INIT(&vq->vq_freelist); 259 /* 260 * virtio_enqueue_trim needs monotonely raising entries, therefore 261 * initialize in reverse order 262 */ 263 for (i = vq_size - 1; i >= 0; i--) { 264 SLIST_INSERT_HEAD(&vq->vq_freelist, &vq->vq_entries[i], 265 qe_list); 266 vq->vq_entries[i].qe_index = i; 267 } 268 269 /* enqueue/dequeue status */ 270 vq->vq_avail_idx = 0; 271 vq->vq_used_idx = 0; 272 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 273 vq_sync_uring(sc, vq, BUS_DMASYNC_PREREAD); 274 vq->vq_queued = 1; 275 } 276 277 /* 278 * Allocate/free a vq. 279 * 280 * maxnsegs denotes how much space should be allocated for indirect 281 * descriptors. maxnsegs == 1 can be used to disable use indirect 282 * descriptors for this queue. 283 */ 284 int 285 virtio_alloc_vq(struct virtio_softc *sc, struct virtqueue *vq, int index, 286 int maxsegsize, int maxnsegs, const char *name) 287 { 288 int vq_size, allocsize1, allocsize2, allocsize3, allocsize = 0; 289 int rsegs, r, hdrlen; 290 #define VIRTQUEUE_ALIGN(n) (((n)+(VIRTIO_PAGE_SIZE-1))& \ 291 ~(VIRTIO_PAGE_SIZE-1)) 292 293 memset(vq, 0, sizeof(*vq)); 294 295 vq_size = virtio_read_queue_size(sc, index); 296 if (vq_size == 0) { 297 printf("virtqueue not exist, index %d for %s\n", index, name); 298 goto err; 299 } 300 if (((vq_size - 1) & vq_size) != 0) 301 panic("vq_size not power of two: %d", vq_size); 302 303 hdrlen = (sc->sc_features & VIRTIO_F_RING_EVENT_IDX) ? 3 : 2; 304 305 /* allocsize1: descriptor table + avail ring + pad */ 306 allocsize1 = VIRTQUEUE_ALIGN(sizeof(struct vring_desc) * vq_size 307 + sizeof(uint16_t) * (hdrlen + vq_size)); 308 /* allocsize2: used ring + pad */ 309 allocsize2 = VIRTQUEUE_ALIGN(sizeof(uint16_t) * hdrlen 310 + sizeof(struct vring_used_elem) * vq_size); 311 /* allocsize3: indirect table */ 312 if (sc->sc_indirect && maxnsegs > 1) 313 allocsize3 = sizeof(struct vring_desc) * maxnsegs * vq_size; 314 else 315 allocsize3 = 0; 316 allocsize = allocsize1 + allocsize2 + allocsize3; 317 318 /* alloc and map the memory */ 319 r = bus_dmamem_alloc(sc->sc_dmat, allocsize, VIRTIO_PAGE_SIZE, 0, 320 &vq->vq_segs[0], 1, &rsegs, BUS_DMA_NOWAIT); 321 if (r != 0) { 322 printf("virtqueue %d for %s allocation failed, error %d\n", 323 index, name, r); 324 goto err; 325 } 326 r = bus_dmamem_map(sc->sc_dmat, &vq->vq_segs[0], 1, allocsize, 327 (caddr_t*)&vq->vq_vaddr, BUS_DMA_NOWAIT); 328 if (r != 0) { 329 printf("virtqueue %d for %s map failed, error %d\n", index, 330 name, r); 331 goto err; 332 } 333 r = bus_dmamap_create(sc->sc_dmat, allocsize, 1, allocsize, 0, 334 BUS_DMA_NOWAIT, &vq->vq_dmamap); 335 if (r != 0) { 336 printf("virtqueue %d for %s dmamap creation failed, " 337 "error %d\n", index, name, r); 338 goto err; 339 } 340 r = bus_dmamap_load(sc->sc_dmat, vq->vq_dmamap, vq->vq_vaddr, 341 allocsize, NULL, BUS_DMA_NOWAIT); 342 if (r != 0) { 343 printf("virtqueue %d for %s dmamap load failed, error %d\n", 344 index, name, r); 345 goto err; 346 } 347 348 virtio_setup_queue(sc, index, 349 vq->vq_dmamap->dm_segs[0].ds_addr / VIRTIO_PAGE_SIZE); 350 351 /* remember addresses and offsets for later use */ 352 vq->vq_owner = sc; 353 vq->vq_num = vq_size; 354 vq->vq_mask = vq_size - 1; 355 vq->vq_index = index; 356 vq->vq_desc = vq->vq_vaddr; 357 vq->vq_availoffset = sizeof(struct vring_desc)*vq_size; 358 vq->vq_avail = (struct vring_avail*)(((char*)vq->vq_desc) + 359 vq->vq_availoffset); 360 vq->vq_usedoffset = allocsize1; 361 vq->vq_used = (struct vring_used*)(((char*)vq->vq_desc) + 362 vq->vq_usedoffset); 363 if (allocsize3 > 0) { 364 vq->vq_indirectoffset = allocsize1 + allocsize2; 365 vq->vq_indirect = (void*)(((char*)vq->vq_desc) 366 + vq->vq_indirectoffset); 367 } 368 vq->vq_bytesize = allocsize; 369 vq->vq_maxnsegs = maxnsegs; 370 371 /* free slot management */ 372 vq->vq_entries = mallocarray(vq_size, sizeof(struct vq_entry), 373 M_DEVBUF, M_NOWAIT | M_ZERO); 374 if (vq->vq_entries == NULL) { 375 r = ENOMEM; 376 goto err; 377 } 378 379 virtio_init_vq(sc, vq, 0); 380 381 #if VIRTIO_DEBUG 382 printf("\nallocated %u byte for virtqueue %d for %s, size %d\n", 383 allocsize, index, name, vq_size); 384 if (allocsize3 > 0) 385 printf("using %d byte (%d entries) indirect descriptors\n", 386 allocsize3, maxnsegs * vq_size); 387 #endif 388 return 0; 389 390 err: 391 virtio_setup_queue(sc, index, 0); 392 if (vq->vq_dmamap) 393 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 394 if (vq->vq_vaddr) 395 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, allocsize); 396 if (vq->vq_segs[0].ds_addr) 397 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 398 memset(vq, 0, sizeof(*vq)); 399 400 return -1; 401 } 402 403 int 404 virtio_free_vq(struct virtio_softc *sc, struct virtqueue *vq) 405 { 406 struct vq_entry *qe; 407 int i = 0; 408 409 /* device must be already deactivated */ 410 /* confirm the vq is empty */ 411 SLIST_FOREACH(qe, &vq->vq_freelist, qe_list) { 412 i++; 413 } 414 if (i != vq->vq_num) { 415 printf("%s: freeing non-empty vq, index %d\n", 416 sc->sc_dev.dv_xname, vq->vq_index); 417 return EBUSY; 418 } 419 420 /* tell device that there's no virtqueue any longer */ 421 virtio_setup_queue(sc, vq->vq_index, 0); 422 423 free(vq->vq_entries, M_DEVBUF, 0); 424 bus_dmamap_unload(sc->sc_dmat, vq->vq_dmamap); 425 bus_dmamap_destroy(sc->sc_dmat, vq->vq_dmamap); 426 bus_dmamem_unmap(sc->sc_dmat, vq->vq_vaddr, vq->vq_bytesize); 427 bus_dmamem_free(sc->sc_dmat, &vq->vq_segs[0], 1); 428 memset(vq, 0, sizeof(*vq)); 429 430 return 0; 431 } 432 433 /* 434 * Free descriptor management. 435 */ 436 struct vq_entry * 437 vq_alloc_entry(struct virtqueue *vq) 438 { 439 struct vq_entry *qe; 440 441 if (SLIST_EMPTY(&vq->vq_freelist)) 442 return NULL; 443 qe = SLIST_FIRST(&vq->vq_freelist); 444 SLIST_REMOVE_HEAD(&vq->vq_freelist, qe_list); 445 446 return qe; 447 } 448 449 void 450 vq_free_entry(struct virtqueue *vq, struct vq_entry *qe) 451 { 452 SLIST_INSERT_HEAD(&vq->vq_freelist, qe, qe_list); 453 } 454 455 /* 456 * Enqueue several dmamaps as a single request. 457 */ 458 /* 459 * Typical usage: 460 * <queue size> number of followings are stored in arrays 461 * - command blocks (in dmamem) should be pre-allocated and mapped 462 * - dmamaps for command blocks should be pre-allocated and loaded 463 * - dmamaps for payload should be pre-allocated 464 * r = virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 465 * if (r) // currently 0 or EAGAIN 466 * return r; 467 * r = bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 468 * if (r) { 469 * virtio_enqueue_abort(sc, vq, slot); 470 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 471 * return r; 472 * } 473 * r = virtio_enqueue_reserve(sc, vq, slot, 474 * dmamap_payload[slot]->dm_nsegs+1); 475 * // ^ +1 for command 476 * if (r) { // currently 0 or EAGAIN 477 * bus_dmamap_unload(dmat, dmamap_payload[slot]); 478 * return r; // do not call abort() 479 * } 480 * <setup and prepare commands> 481 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 482 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 483 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], 0); 484 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 485 * virtio_enqueue_commit(sc, vq, slot, 1); 486 * 487 * Alternative usage with statically allocated slots: 488 * <during initialization> 489 * // while not out of slots, do 490 * virtio_enqueue_prep(sc, vq, &slot); // allocate a slot 491 * virtio_enqueue_reserve(sc, vq, slot, max_segs); // reserve all slots 492 * that may ever be needed 493 * 494 * <when enqueing a request> 495 * // Don't call virtio_enqueue_prep() 496 * bus_dmamap_load(dmat, dmamap_payload[slot], data, count, ..); 497 * bus_dmamap_sync(dmat, dmamap_cmd[slot],... BUS_DMASYNC_PREWRITE); 498 * bus_dmamap_sync(dmat, dmamap_payload[slot],...); 499 * virtio_enqueue_trim(sc, vq, slot, num_segs_needed); 500 * virtio_enqueue(sc, vq, slot, dmamap_cmd[slot], 0); 501 * virtio_enqueue(sc, vq, slot, dmamap_payload[slot], iswrite); 502 * virtio_enqueue_commit(sc, vq, slot, 1); 503 * 504 * <when dequeuing> 505 * // don't call virtio_dequeue_commit() 506 */ 507 508 /* 509 * enqueue_prep: allocate a slot number 510 */ 511 int 512 virtio_enqueue_prep(struct virtqueue *vq, int *slotp) 513 { 514 struct vq_entry *qe1; 515 516 VIRTIO_ASSERT(slotp != NULL); 517 518 qe1 = vq_alloc_entry(vq); 519 if (qe1 == NULL) 520 return EAGAIN; 521 /* next slot is not allocated yet */ 522 qe1->qe_next = -1; 523 *slotp = qe1->qe_index; 524 525 return 0; 526 } 527 528 /* 529 * enqueue_reserve: allocate remaining slots and build the descriptor chain. 530 * Calls virtio_enqueue_abort() on failure. 531 */ 532 int 533 virtio_enqueue_reserve(struct virtqueue *vq, int slot, int nsegs) 534 { 535 struct vq_entry *qe1 = &vq->vq_entries[slot]; 536 537 VIRTIO_ASSERT(qe1->qe_next == -1); 538 VIRTIO_ASSERT(1 <= nsegs && nsegs <= vq->vq_num); 539 540 if (vq->vq_indirect != NULL && nsegs > 1 && nsegs <= vq->vq_maxnsegs) { 541 struct vring_desc *vd; 542 int i; 543 544 qe1->qe_indirect = 1; 545 546 vd = &vq->vq_desc[qe1->qe_index]; 547 vd->addr = vq->vq_dmamap->dm_segs[0].ds_addr + 548 vq->vq_indirectoffset; 549 vd->addr += sizeof(struct vring_desc) * vq->vq_maxnsegs * 550 qe1->qe_index; 551 vd->len = sizeof(struct vring_desc) * nsegs; 552 vd->flags = VRING_DESC_F_INDIRECT; 553 554 vd = vq->vq_indirect; 555 vd += vq->vq_maxnsegs * qe1->qe_index; 556 qe1->qe_desc_base = vd; 557 558 for (i = 0; i < nsegs-1; i++) 559 vd[i].flags = VRING_DESC_F_NEXT; 560 vd[i].flags = 0; 561 qe1->qe_next = 0; 562 563 return 0; 564 } else { 565 struct vring_desc *vd; 566 struct vq_entry *qe; 567 int i, s; 568 569 qe1->qe_indirect = 0; 570 571 vd = &vq->vq_desc[0]; 572 qe1->qe_desc_base = vd; 573 qe1->qe_next = qe1->qe_index; 574 s = slot; 575 for (i = 0; i < nsegs - 1; i++) { 576 qe = vq_alloc_entry(vq); 577 if (qe == NULL) { 578 vd[s].flags = 0; 579 virtio_enqueue_abort(vq, slot); 580 return EAGAIN; 581 } 582 vd[s].flags = VRING_DESC_F_NEXT; 583 vd[s].next = qe->qe_index; 584 s = qe->qe_index; 585 } 586 vd[s].flags = 0; 587 588 return 0; 589 } 590 } 591 592 /* 593 * enqueue: enqueue a single dmamap. 594 */ 595 int 596 virtio_enqueue(struct virtqueue *vq, int slot, bus_dmamap_t dmamap, int write) 597 { 598 struct vq_entry *qe1 = &vq->vq_entries[slot]; 599 struct vring_desc *vd = qe1->qe_desc_base; 600 int i; 601 int s = qe1->qe_next; 602 603 VIRTIO_ASSERT(s >= 0); 604 VIRTIO_ASSERT(dmamap->dm_nsegs > 0); 605 if (dmamap->dm_nsegs > vq->vq_maxnsegs) { 606 #if VIRTIO_DEBUG 607 for (i = 0; i < dmamap->dm_nsegs; i++) { 608 printf(" %d (%d): %p %lx \n", i, write, 609 (void *)dmamap->dm_segs[i].ds_addr, 610 dmamap->dm_segs[i].ds_len); 611 } 612 #endif 613 panic("dmamap->dm_nseg %d > vq->vq_maxnsegs %d\n", 614 dmamap->dm_nsegs, vq->vq_maxnsegs); 615 } 616 617 for (i = 0; i < dmamap->dm_nsegs; i++) { 618 vd[s].addr = dmamap->dm_segs[i].ds_addr; 619 vd[s].len = dmamap->dm_segs[i].ds_len; 620 if (!write) 621 vd[s].flags |= VRING_DESC_F_WRITE; 622 s = vd[s].next; 623 } 624 qe1->qe_next = s; 625 626 return 0; 627 } 628 629 int 630 virtio_enqueue_p(struct virtqueue *vq, int slot, bus_dmamap_t dmamap, 631 bus_addr_t start, bus_size_t len, int write) 632 { 633 struct vq_entry *qe1 = &vq->vq_entries[slot]; 634 struct vring_desc *vd = qe1->qe_desc_base; 635 int s = qe1->qe_next; 636 637 VIRTIO_ASSERT(s >= 0); 638 /* XXX todo: handle more segments */ 639 VIRTIO_ASSERT(dmamap->dm_nsegs == 1); 640 VIRTIO_ASSERT((dmamap->dm_segs[0].ds_len > start) && 641 (dmamap->dm_segs[0].ds_len >= start + len)); 642 643 vd[s].addr = dmamap->dm_segs[0].ds_addr + start; 644 vd[s].len = len; 645 if (!write) 646 vd[s].flags |= VRING_DESC_F_WRITE; 647 qe1->qe_next = vd[s].next; 648 649 return 0; 650 } 651 652 static void 653 publish_avail_idx(struct virtio_softc *sc, struct virtqueue *vq) 654 { 655 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 656 657 virtio_membar_producer(); 658 vq->vq_avail->idx = vq->vq_avail_idx; 659 vq_sync_aring(sc, vq, BUS_DMASYNC_POSTWRITE); 660 vq->vq_queued = 1; 661 } 662 663 /* 664 * enqueue_commit: add it to the aring. 665 */ 666 void 667 virtio_enqueue_commit(struct virtio_softc *sc, struct virtqueue *vq, int slot, 668 int notifynow) 669 { 670 struct vq_entry *qe1; 671 672 if (slot < 0) 673 goto notify; 674 vq_sync_descs(sc, vq, BUS_DMASYNC_PREWRITE); 675 qe1 = &vq->vq_entries[slot]; 676 if (qe1->qe_indirect) 677 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_PREWRITE); 678 vq->vq_avail->ring[(vq->vq_avail_idx++) & vq->vq_mask] = slot; 679 680 notify: 681 if (notifynow) { 682 if (vq->vq_owner->sc_features & VIRTIO_F_RING_EVENT_IDX) { 683 uint16_t o = vq->vq_avail->idx; 684 uint16_t n = vq->vq_avail_idx; 685 uint16_t t; 686 publish_avail_idx(sc, vq); 687 688 virtio_membar_sync(); 689 t = VQ_AVAIL_EVENT(vq) + 1; 690 if ((uint16_t)(n - t) < (uint16_t)(n - o)) 691 sc->sc_ops->kick(sc, vq->vq_index); 692 } else { 693 publish_avail_idx(sc, vq); 694 695 virtio_membar_sync(); 696 if (!(vq->vq_used->flags & VRING_USED_F_NO_NOTIFY)) 697 sc->sc_ops->kick(sc, vq->vq_index); 698 } 699 } 700 } 701 702 /* 703 * enqueue_abort: rollback. 704 */ 705 int 706 virtio_enqueue_abort(struct virtqueue *vq, int slot) 707 { 708 struct vq_entry *qe = &vq->vq_entries[slot]; 709 struct vring_desc *vd; 710 int s; 711 712 if (qe->qe_next < 0) { 713 vq_free_entry(vq, qe); 714 return 0; 715 } 716 717 s = slot; 718 vd = &vq->vq_desc[0]; 719 while (vd[s].flags & VRING_DESC_F_NEXT) { 720 s = vd[s].next; 721 vq_free_entry(vq, qe); 722 qe = &vq->vq_entries[s]; 723 } 724 vq_free_entry(vq, qe); 725 return 0; 726 } 727 728 /* 729 * enqueue_trim: adjust buffer size to given # of segments, a.k.a. 730 * descriptors. 731 */ 732 void 733 virtio_enqueue_trim(struct virtqueue *vq, int slot, int nsegs) 734 { 735 struct vq_entry *qe1 = &vq->vq_entries[slot]; 736 struct vring_desc *vd = &vq->vq_desc[0]; 737 int i; 738 739 if ((vd[slot].flags & VRING_DESC_F_INDIRECT) == 0) { 740 qe1->qe_next = qe1->qe_index; 741 /* 742 * N.B.: the vq_entries are ASSUMED to be a contiguous 743 * block with slot being the index to the first one. 744 */ 745 } else { 746 qe1->qe_next = 0; 747 vd = &vq->vq_desc[qe1->qe_index]; 748 vd->len = sizeof(struct vring_desc) * nsegs; 749 vd = qe1->qe_desc_base; 750 slot = 0; 751 } 752 753 for (i = 0; i < nsegs -1 ; i++) { 754 vd[slot].flags = VRING_DESC_F_NEXT; 755 slot++; 756 } 757 vd[slot].flags = 0; 758 } 759 760 /* 761 * Dequeue a request. 762 */ 763 /* 764 * dequeue: dequeue a request from uring; dmamap_sync for uring is 765 * already done in the interrupt handler. 766 */ 767 int 768 virtio_dequeue(struct virtio_softc *sc, struct virtqueue *vq, 769 int *slotp, int *lenp) 770 { 771 uint16_t slot, usedidx; 772 struct vq_entry *qe; 773 774 if (vq->vq_used_idx == vq->vq_used->idx) 775 return ENOENT; 776 usedidx = vq->vq_used_idx++; 777 usedidx &= vq->vq_mask; 778 779 virtio_membar_consumer(); 780 slot = vq->vq_used->ring[usedidx].id; 781 qe = &vq->vq_entries[slot]; 782 783 if (qe->qe_indirect) 784 vq_sync_indirect(sc, vq, slot, BUS_DMASYNC_POSTWRITE); 785 786 if (slotp) 787 *slotp = slot; 788 if (lenp) 789 *lenp = vq->vq_used->ring[usedidx].len; 790 791 return 0; 792 } 793 794 /* 795 * dequeue_commit: complete dequeue; the slot is recycled for future use. 796 * if you forget to call this the slot will be leaked. 797 * 798 * Don't call this if you use statically allocated slots 799 * and virtio_dequeue_trim(). 800 */ 801 int 802 virtio_dequeue_commit(struct virtqueue *vq, int slot) 803 { 804 struct vq_entry *qe = &vq->vq_entries[slot]; 805 struct vring_desc *vd = &vq->vq_desc[0]; 806 int s = slot; 807 808 while (vd[s].flags & VRING_DESC_F_NEXT) { 809 s = vd[s].next; 810 vq_free_entry(vq, qe); 811 qe = &vq->vq_entries[s]; 812 } 813 vq_free_entry(vq, qe); 814 815 return 0; 816 } 817 818 /* 819 * Increase the event index in order to delay interrupts. 820 * Returns 0 on success; returns 1 if the used ring has already advanced 821 * too far, and the caller must process the queue again (otherewise, no 822 * more interrupts will happen). 823 */ 824 int 825 virtio_postpone_intr(struct virtqueue *vq, uint16_t nslots) 826 { 827 uint16_t idx; 828 829 idx = vq->vq_used_idx + nslots; 830 831 /* set the new event index: avail_ring->used_event = idx */ 832 VQ_USED_EVENT(vq) = idx; 833 virtio_membar_sync(); 834 835 vq_sync_aring(vq->vq_owner, vq, BUS_DMASYNC_PREWRITE); 836 vq->vq_queued++; 837 838 if (nslots < virtio_nused(vq)) 839 return 1; 840 841 return 0; 842 } 843 844 /* 845 * Postpone interrupt until 3/4 of the available descriptors have been 846 * consumed. 847 */ 848 int 849 virtio_postpone_intr_smart(struct virtqueue *vq) 850 { 851 uint16_t nslots; 852 853 nslots = (uint16_t)(vq->vq_avail->idx - vq->vq_used_idx) * 3 / 4; 854 855 return virtio_postpone_intr(vq, nslots); 856 } 857 858 /* 859 * Postpone interrupt until all of the available descriptors have been 860 * consumed. 861 */ 862 int 863 virtio_postpone_intr_far(struct virtqueue *vq) 864 { 865 uint16_t nslots; 866 867 nslots = (uint16_t)(vq->vq_avail->idx - vq->vq_used_idx); 868 869 return virtio_postpone_intr(vq, nslots); 870 } 871 872 873 /* 874 * Start/stop vq interrupt. No guarantee. 875 */ 876 void 877 virtio_stop_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 878 { 879 if ((sc->sc_features & VIRTIO_F_RING_EVENT_IDX)) { 880 /* 881 * No way to disable the interrupt completely with 882 * RingEventIdx. Instead advance used_event by half 883 * the possible value. This won't happen soon and 884 * is far enough in the past to not trigger a spurios 885 * interrupt. 886 */ 887 VQ_USED_EVENT(vq) = vq->vq_used_idx + 0x8000; 888 } else { 889 vq->vq_avail->flags |= VRING_AVAIL_F_NO_INTERRUPT; 890 } 891 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 892 vq->vq_queued++; 893 } 894 895 int 896 virtio_start_vq_intr(struct virtio_softc *sc, struct virtqueue *vq) 897 { 898 /* 899 * If event index feature is negotiated, enabling 900 * interrupts is done through setting the latest 901 * consumed index in the used_event field 902 */ 903 if (sc->sc_features & VIRTIO_F_RING_EVENT_IDX) 904 VQ_USED_EVENT(vq) = vq->vq_used_idx; 905 else 906 vq->vq_avail->flags &= ~VRING_AVAIL_F_NO_INTERRUPT; 907 908 virtio_membar_sync(); 909 910 vq_sync_aring(sc, vq, BUS_DMASYNC_PREWRITE); 911 vq->vq_queued++; 912 913 if (vq->vq_used_idx != vq->vq_used->idx) 914 return 1; 915 916 return 0; 917 } 918 919 /* 920 * Returns a number of slots in the used ring available to 921 * be supplied to the avail ring. 922 */ 923 int 924 virtio_nused(struct virtqueue *vq) 925 { 926 uint16_t n; 927 928 n = (uint16_t)(vq->vq_used->idx - vq->vq_used_idx); 929 VIRTIO_ASSERT(n <= vq->vq_num); 930 931 return n; 932 } 933 934 #if VIRTIO_DEBUG 935 void 936 virtio_vq_dump(struct virtqueue *vq) 937 { 938 /* Common fields */ 939 printf(" + vq num: %d\n", vq->vq_num); 940 printf(" + vq mask: 0x%X\n", vq->vq_mask); 941 printf(" + vq index: %d\n", vq->vq_index); 942 printf(" + vq used idx: %d\n", vq->vq_used_idx); 943 printf(" + vq avail idx: %d\n", vq->vq_avail_idx); 944 printf(" + vq queued: %d\n",vq->vq_queued); 945 /* Avail ring fields */ 946 printf(" + avail flags: 0x%X\n", vq->vq_avail->flags); 947 printf(" + avail idx: %d\n", vq->vq_avail->idx); 948 printf(" + avail event: %d\n", VQ_AVAIL_EVENT(vq)); 949 /* Used ring fields */ 950 printf(" + used flags: 0x%X\n",vq->vq_used->flags); 951 printf(" + used idx: %d\n",vq->vq_used->idx); 952 printf(" + used event: %d\n", VQ_USED_EVENT(vq)); 953 printf(" +++++++++++++++++++++++++++\n"); 954 } 955 #endif 956