xref: /netbsd-src/sys/dev/pci/ld_virtio.c (revision 212397c69a103ae7e5eafa8731ddfae671d2dee7)
1 /*	$NetBSD: ld_virtio.c,v 1.9 2015/10/27 16:03:04 christos Exp $	*/
2 
3 /*
4  * Copyright (c) 2010 Minoura Makoto.
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26  */
27 
28 #include <sys/cdefs.h>
29 __KERNEL_RCSID(0, "$NetBSD: ld_virtio.c,v 1.9 2015/10/27 16:03:04 christos Exp $");
30 
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/kernel.h>
34 #include <sys/buf.h>
35 #include <sys/bus.h>
36 #include <sys/device.h>
37 #include <sys/disk.h>
38 #include <sys/mutex.h>
39 
40 #include <dev/pci/pcidevs.h>
41 #include <dev/pci/pcireg.h>
42 #include <dev/pci/pcivar.h>
43 
44 #include <dev/ldvar.h>
45 #include <dev/pci/virtioreg.h>
46 #include <dev/pci/virtiovar.h>
47 
48 /*
49  * ld_virtioreg:
50  */
51 /* Configuration registers */
52 #define VIRTIO_BLK_CONFIG_CAPACITY	0 /* 64bit */
53 #define VIRTIO_BLK_CONFIG_SIZE_MAX	8 /* 32bit */
54 #define VIRTIO_BLK_CONFIG_SEG_MAX	12 /* 32bit */
55 #define VIRTIO_BLK_CONFIG_GEOMETRY_C	16 /* 16bit */
56 #define VIRTIO_BLK_CONFIG_GEOMETRY_H	18 /* 8bit */
57 #define VIRTIO_BLK_CONFIG_GEOMETRY_S	19 /* 8bit */
58 #define VIRTIO_BLK_CONFIG_BLK_SIZE	20 /* 32bit */
59 
60 /* Feature bits */
61 #define VIRTIO_BLK_F_BARRIER	(1<<0)
62 #define VIRTIO_BLK_F_SIZE_MAX	(1<<1)
63 #define VIRTIO_BLK_F_SEG_MAX	(1<<2)
64 #define VIRTIO_BLK_F_GEOMETRY	(1<<4)
65 #define VIRTIO_BLK_F_RO		(1<<5)
66 #define VIRTIO_BLK_F_BLK_SIZE	(1<<6)
67 #define VIRTIO_BLK_F_SCSI	(1<<7)
68 #define VIRTIO_BLK_F_FLUSH	(1<<9)
69 
70 /*
71  * Each block request uses at least two segments - one for the header
72  * and one for the status.
73 */
74 #define	VIRTIO_BLK_MIN_SEGMENTS	2
75 
76 #define VIRTIO_BLK_FLAG_BITS \
77 	VIRTIO_COMMON_FLAG_BITS \
78 	"\x0a""FLUSH" \
79 	"\x08""SCSI" \
80 	"\x07""BLK_SIZE" \
81 	"\x06""RO" \
82 	"\x05""GEOMETRY" \
83 	"\x03""SEG_MAX" \
84 	"\x02""SIZE_MAX" \
85 	"\x01""BARRIER"
86 
87 /* Command */
88 #define VIRTIO_BLK_T_IN		0
89 #define VIRTIO_BLK_T_OUT	1
90 #define VIRTIO_BLK_T_BARRIER	0x80000000
91 
92 /* Status */
93 #define VIRTIO_BLK_S_OK		0
94 #define VIRTIO_BLK_S_IOERR	1
95 
96 /* Request header structure */
97 struct virtio_blk_req_hdr {
98 	uint32_t	type;	/* VIRTIO_BLK_T_* */
99 	uint32_t	ioprio;
100 	uint64_t	sector;
101 } __packed;
102 /* 512*virtio_blk_req_hdr.sector byte payload and 1 byte status follows */
103 
104 
105 /*
106  * ld_virtiovar:
107  */
108 struct virtio_blk_req {
109 	struct virtio_blk_req_hdr	vr_hdr;
110 	uint8_t				vr_status;
111 	struct buf			*vr_bp;
112 	bus_dmamap_t			vr_cmdsts;
113 	bus_dmamap_t			vr_payload;
114 };
115 
116 struct ld_virtio_softc {
117 	struct ld_softc		sc_ld;
118 	device_t		sc_dev;
119 
120 	struct virtio_softc	*sc_virtio;
121 	struct virtqueue	sc_vq;
122 
123 	struct virtio_blk_req	*sc_reqs;
124 	bus_dma_segment_t	sc_reqs_seg;
125 
126 	int			sc_readonly;
127 };
128 
129 static int	ld_virtio_match(device_t, cfdata_t, void *);
130 static void	ld_virtio_attach(device_t, device_t, void *);
131 static int	ld_virtio_detach(device_t, int);
132 
133 CFATTACH_DECL_NEW(ld_virtio, sizeof(struct ld_virtio_softc),
134     ld_virtio_match, ld_virtio_attach, ld_virtio_detach, NULL);
135 
136 static int
137 ld_virtio_match(device_t parent, cfdata_t match, void *aux)
138 {
139 	struct virtio_softc *va = aux;
140 
141 	if (va->sc_childdevid == PCI_PRODUCT_VIRTIO_BLOCK)
142 		return 1;
143 
144 	return 0;
145 }
146 
147 static int ld_virtio_vq_done(struct virtqueue *);
148 static int ld_virtio_dump(struct ld_softc *, void *, int, int);
149 static int ld_virtio_start(struct ld_softc *, struct buf *);
150 
151 static int
152 ld_virtio_alloc_reqs(struct ld_virtio_softc *sc, int qsize)
153 {
154 	int allocsize, r, rsegs, i;
155 	struct ld_softc *ld = &sc->sc_ld;
156 	void *vaddr;
157 
158 	allocsize = sizeof(struct virtio_blk_req) * qsize;
159 	r = bus_dmamem_alloc(sc->sc_virtio->sc_dmat, allocsize, 0, 0,
160 			     &sc->sc_reqs_seg, 1, &rsegs, BUS_DMA_NOWAIT);
161 	if (r != 0) {
162 		aprint_error_dev(sc->sc_dev,
163 				 "DMA memory allocation failed, size %d, "
164 				 "error code %d\n", allocsize, r);
165 		goto err_none;
166 	}
167 	r = bus_dmamem_map(sc->sc_virtio->sc_dmat,
168 			   &sc->sc_reqs_seg, 1, allocsize,
169 			   &vaddr, BUS_DMA_NOWAIT);
170 	if (r != 0) {
171 		aprint_error_dev(sc->sc_dev,
172 				 "DMA memory map failed, "
173 				 "error code %d\n", r);
174 		goto err_dmamem_alloc;
175 	}
176 	sc->sc_reqs = vaddr;
177 	memset(vaddr, 0, allocsize);
178 	for (i = 0; i < qsize; i++) {
179 		struct virtio_blk_req *vr = &sc->sc_reqs[i];
180 		r = bus_dmamap_create(sc->sc_virtio->sc_dmat,
181 				      offsetof(struct virtio_blk_req, vr_bp),
182 				      1,
183 				      offsetof(struct virtio_blk_req, vr_bp),
184 				      0,
185 				      BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW,
186 				      &vr->vr_cmdsts);
187 		if (r != 0) {
188 			aprint_error_dev(sc->sc_dev,
189 					 "command dmamap creation failed, "
190 					 "error code %d\n", r);
191 			goto err_reqs;
192 		}
193 		r = bus_dmamap_load(sc->sc_virtio->sc_dmat, vr->vr_cmdsts,
194 				    &vr->vr_hdr,
195 				    offsetof(struct virtio_blk_req, vr_bp),
196 				    NULL, BUS_DMA_NOWAIT);
197 		if (r != 0) {
198 			aprint_error_dev(sc->sc_dev,
199 					 "command dmamap load failed, "
200 					 "error code %d\n", r);
201 			goto err_reqs;
202 		}
203 		r = bus_dmamap_create(sc->sc_virtio->sc_dmat,
204 				      ld->sc_maxxfer,
205 				      (ld->sc_maxxfer / NBPG) +
206 				      VIRTIO_BLK_MIN_SEGMENTS,
207 				      ld->sc_maxxfer,
208 				      0,
209 				      BUS_DMA_NOWAIT|BUS_DMA_ALLOCNOW,
210 				      &vr->vr_payload);
211 		if (r != 0) {
212 			aprint_error_dev(sc->sc_dev,
213 					 "payload dmamap creation failed, "
214 					 "error code %d\n", r);
215 			goto err_reqs;
216 		}
217 	}
218 	return 0;
219 
220 err_reqs:
221 	for (i = 0; i < qsize; i++) {
222 		struct virtio_blk_req *vr = &sc->sc_reqs[i];
223 		if (vr->vr_cmdsts) {
224 			bus_dmamap_destroy(sc->sc_virtio->sc_dmat,
225 					   vr->vr_cmdsts);
226 			vr->vr_cmdsts = 0;
227 		}
228 		if (vr->vr_payload) {
229 			bus_dmamap_destroy(sc->sc_virtio->sc_dmat,
230 					   vr->vr_payload);
231 			vr->vr_payload = 0;
232 		}
233 	}
234 	bus_dmamem_unmap(sc->sc_virtio->sc_dmat, sc->sc_reqs, allocsize);
235 err_dmamem_alloc:
236 	bus_dmamem_free(sc->sc_virtio->sc_dmat, &sc->sc_reqs_seg, 1);
237 err_none:
238 	return -1;
239 }
240 
241 static void
242 ld_virtio_attach(device_t parent, device_t self, void *aux)
243 {
244 	struct ld_virtio_softc *sc = device_private(self);
245 	struct ld_softc *ld = &sc->sc_ld;
246 	struct virtio_softc *vsc = device_private(parent);
247 	uint32_t features;
248 	char buf[256];
249 	int qsize, maxxfersize, maxnsegs;
250 
251 	if (vsc->sc_child != NULL) {
252 		aprint_normal(": child already attached for %s; "
253 			      "something wrong...\n",
254 			      device_xname(parent));
255 		return;
256 	}
257 
258 	sc->sc_dev = self;
259 	sc->sc_virtio = vsc;
260 
261 	vsc->sc_child = self;
262 	vsc->sc_ipl = IPL_BIO;
263 	vsc->sc_vqs = &sc->sc_vq;
264 	vsc->sc_nvqs = 1;
265 	vsc->sc_config_change = NULL;
266 	vsc->sc_intrhand = virtio_vq_intr;
267 	vsc->sc_flags = 0;
268 
269 	features = virtio_negotiate_features(vsc,
270 					     (VIRTIO_BLK_F_SIZE_MAX |
271 					      VIRTIO_BLK_F_SEG_MAX |
272 					      VIRTIO_BLK_F_GEOMETRY |
273 					      VIRTIO_BLK_F_RO |
274 					      VIRTIO_BLK_F_BLK_SIZE));
275 	if (features & VIRTIO_BLK_F_RO)
276 		sc->sc_readonly = 1;
277 	else
278 		sc->sc_readonly = 0;
279 
280 	snprintb(buf, sizeof(buf), VIRTIO_BLK_FLAG_BITS, features);
281 	aprint_normal(": Features: %s\n", buf);
282 	aprint_naive("\n");
283 	if (features & VIRTIO_BLK_F_BLK_SIZE) {
284 		ld->sc_secsize = virtio_read_device_config_4(vsc,
285 					VIRTIO_BLK_CONFIG_BLK_SIZE);
286 	} else
287 		ld->sc_secsize = 512;
288 
289 	/* At least genfs_io assumes maxxfer == MAXPHYS. */
290 	if (features & VIRTIO_BLK_F_SIZE_MAX) {
291 		maxxfersize = virtio_read_device_config_4(vsc,
292 		    VIRTIO_BLK_CONFIG_SIZE_MAX);
293 		if (maxxfersize < MAXPHYS) {
294 			aprint_error_dev(sc->sc_dev,
295 			    "Too small SIZE_MAX %dK minimum is %dK\n",
296 			    maxxfersize / 1024, MAXPHYS / 1024);
297 			// goto err;
298 			maxxfersize = MAXPHYS;
299 		} else if (maxxfersize > MAXPHYS) {
300 			aprint_normal_dev(sc->sc_dev,
301 			    "Clip SEG_MAX from %dK to %dK\n",
302 			    maxxfersize / 1024,
303 			    MAXPHYS / 1024);
304 			maxxfersize = MAXPHYS;
305 		}
306 	} else
307 		maxxfersize = MAXPHYS;
308 
309 	if (features & VIRTIO_BLK_F_SEG_MAX) {
310 		maxnsegs = virtio_read_device_config_4(vsc,
311 		    VIRTIO_BLK_CONFIG_SEG_MAX);
312 		if (maxnsegs < VIRTIO_BLK_MIN_SEGMENTS) {
313 			aprint_error_dev(sc->sc_dev,
314 			    "Too small SEG_MAX %d minimum is %d\n",
315 			    maxnsegs, VIRTIO_BLK_MIN_SEGMENTS);
316 			maxnsegs = maxxfersize / NBPG;
317 			// goto err;
318 		}
319 	} else
320 		maxnsegs = maxxfersize / NBPG;
321 
322 	/* 2 for the minimum size */
323 	maxnsegs += VIRTIO_BLK_MIN_SEGMENTS;
324 
325 	if (virtio_alloc_vq(vsc, &sc->sc_vq, 0, maxxfersize, maxnsegs,
326 	    "I/O request") != 0) {
327 		goto err;
328 	}
329 	qsize = sc->sc_vq.vq_num;
330 	sc->sc_vq.vq_done = ld_virtio_vq_done;
331 
332 	ld->sc_dv = self;
333 	ld->sc_secperunit = virtio_read_device_config_8(vsc,
334 				VIRTIO_BLK_CONFIG_CAPACITY);
335 	ld->sc_maxxfer = maxxfersize;
336 	if (features & VIRTIO_BLK_F_GEOMETRY) {
337 		ld->sc_ncylinders = virtio_read_device_config_2(vsc,
338 					VIRTIO_BLK_CONFIG_GEOMETRY_C);
339 		ld->sc_nheads     = virtio_read_device_config_1(vsc,
340 					VIRTIO_BLK_CONFIG_GEOMETRY_H);
341 		ld->sc_nsectors   = virtio_read_device_config_1(vsc,
342 					VIRTIO_BLK_CONFIG_GEOMETRY_S);
343 	}
344 	ld->sc_maxqueuecnt = qsize;
345 
346 	if (ld_virtio_alloc_reqs(sc, qsize) < 0)
347 		goto err;
348 
349 	ld->sc_dump = ld_virtio_dump;
350 	ld->sc_flush = NULL;
351 	ld->sc_start = ld_virtio_start;
352 
353 	ld->sc_flags = LDF_ENABLED;
354 	ldattach(ld);
355 
356 	return;
357 
358 err:
359 	vsc->sc_child = (void*)1;
360 	return;
361 }
362 
363 static int
364 ld_virtio_start(struct ld_softc *ld, struct buf *bp)
365 {
366 	/* splbio */
367 	struct ld_virtio_softc *sc = device_private(ld->sc_dv);
368 	struct virtio_softc *vsc = sc->sc_virtio;
369 	struct virtqueue *vq = &sc->sc_vq;
370 	struct virtio_blk_req *vr;
371 	int r;
372 	int isread = (bp->b_flags & B_READ);
373 	int slot;
374 
375 	if (sc->sc_readonly && !isread)
376 		return EIO;
377 
378 	r = virtio_enqueue_prep(vsc, vq, &slot);
379 	if (r != 0)
380 		return r;
381 
382 	vr = &sc->sc_reqs[slot];
383 	KASSERT(vr->vr_bp == NULL);
384 
385 	r = bus_dmamap_load(vsc->sc_dmat, vr->vr_payload,
386 			    bp->b_data, bp->b_bcount, NULL,
387 			    ((isread?BUS_DMA_READ:BUS_DMA_WRITE)
388 			     |BUS_DMA_NOWAIT));
389 	if (r != 0) {
390 		aprint_error_dev(sc->sc_dev,
391 		    "payload dmamap failed, error code %d\n", r);
392 		virtio_enqueue_abort(vsc, vq, slot);
393 		return r;
394 	}
395 
396 	r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs +
397 	    VIRTIO_BLK_MIN_SEGMENTS);
398 	if (r != 0) {
399 		virtio_enqueue_abort(vsc, vq, slot);
400 		bus_dmamap_unload(vsc->sc_dmat, vr->vr_payload);
401 		return r;
402 	}
403 
404 	vr->vr_bp = bp;
405 	vr->vr_hdr.type = isread?VIRTIO_BLK_T_IN:VIRTIO_BLK_T_OUT;
406 	vr->vr_hdr.ioprio = 0;
407 	vr->vr_hdr.sector = bp->b_rawblkno * sc->sc_ld.sc_secsize / 512;
408 
409 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_cmdsts,
410 			0, sizeof(struct virtio_blk_req_hdr),
411 			BUS_DMASYNC_PREWRITE);
412 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_payload,
413 			0, bp->b_bcount,
414 			isread?BUS_DMASYNC_PREREAD:BUS_DMASYNC_PREWRITE);
415 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_cmdsts,
416 			offsetof(struct virtio_blk_req, vr_status),
417 			sizeof(uint8_t),
418 			BUS_DMASYNC_PREREAD);
419 
420 	virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
421 			 0, sizeof(struct virtio_blk_req_hdr),
422 			 true);
423 	virtio_enqueue(vsc, vq, slot, vr->vr_payload, !isread);
424 	virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
425 			 offsetof(struct virtio_blk_req, vr_status),
426 			 sizeof(uint8_t),
427 			 false);
428 	virtio_enqueue_commit(vsc, vq, slot, true);
429 
430 	return 0;
431 }
432 
433 static void
434 ld_virtio_vq_done1(struct ld_virtio_softc *sc, struct virtio_softc *vsc,
435 		   struct virtqueue *vq, int slot)
436 {
437 	struct virtio_blk_req *vr = &sc->sc_reqs[slot];
438 	struct buf *bp = vr->vr_bp;
439 
440 	vr->vr_bp = NULL;
441 
442 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_cmdsts,
443 			0, sizeof(struct virtio_blk_req_hdr),
444 			BUS_DMASYNC_POSTWRITE);
445 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_payload,
446 			0, bp->b_bcount,
447 			(bp->b_flags & B_READ)?BUS_DMASYNC_POSTREAD
448 					      :BUS_DMASYNC_POSTWRITE);
449 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_cmdsts,
450 			sizeof(struct virtio_blk_req_hdr), sizeof(uint8_t),
451 			BUS_DMASYNC_POSTREAD);
452 
453 	if (vr->vr_status != VIRTIO_BLK_S_OK) {
454 		bp->b_error = EIO;
455 		bp->b_resid = bp->b_bcount;
456 	} else {
457 		bp->b_error = 0;
458 		bp->b_resid = 0;
459 	}
460 
461 	virtio_dequeue_commit(vsc, vq, slot);
462 
463 	lddone(&sc->sc_ld, bp);
464 }
465 
466 static int
467 ld_virtio_vq_done(struct virtqueue *vq)
468 {
469 	struct virtio_softc *vsc = vq->vq_owner;
470 	struct ld_virtio_softc *sc = device_private(vsc->sc_child);
471 	int r = 0;
472 	int slot;
473 
474 again:
475 	if (virtio_dequeue(vsc, vq, &slot, NULL))
476 		return r;
477 	r = 1;
478 
479 	ld_virtio_vq_done1(sc, vsc, vq, slot);
480 	goto again;
481 }
482 
483 static int
484 ld_virtio_dump(struct ld_softc *ld, void *data, int blkno, int blkcnt)
485 {
486 	struct ld_virtio_softc *sc = device_private(ld->sc_dv);
487 	struct virtio_softc *vsc = sc->sc_virtio;
488 	struct virtqueue *vq = &sc->sc_vq;
489 	struct virtio_blk_req *vr;
490 	int slot, r;
491 
492 	if (sc->sc_readonly)
493 		return EIO;
494 
495 	r = virtio_enqueue_prep(vsc, vq, &slot);
496 	if (r != 0) {
497 		if (r == EAGAIN) { /* no free slot; dequeue first */
498 			delay(100);
499 			ld_virtio_vq_done(vq);
500 			r = virtio_enqueue_prep(vsc, vq, &slot);
501 			if (r != 0)
502 				return r;
503 		}
504 		return r;
505 	}
506 	vr = &sc->sc_reqs[slot];
507 	r = bus_dmamap_load(vsc->sc_dmat, vr->vr_payload,
508 			    data, blkcnt*ld->sc_secsize, NULL,
509 			    BUS_DMA_WRITE|BUS_DMA_NOWAIT);
510 	if (r != 0)
511 		return r;
512 
513 	r = virtio_enqueue_reserve(vsc, vq, slot, vr->vr_payload->dm_nsegs +
514 	    VIRTIO_BLK_MIN_SEGMENTS);
515 	if (r != 0) {
516 		bus_dmamap_unload(vsc->sc_dmat, vr->vr_payload);
517 		return r;
518 	}
519 
520 	vr->vr_bp = (void*)0xdeadbeef;
521 	vr->vr_hdr.type = VIRTIO_BLK_T_OUT;
522 	vr->vr_hdr.ioprio = 0;
523 	vr->vr_hdr.sector = (daddr_t) blkno * ld->sc_secsize / 512;
524 
525 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_cmdsts,
526 			0, sizeof(struct virtio_blk_req_hdr),
527 			BUS_DMASYNC_PREWRITE);
528 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_payload,
529 			0, blkcnt*ld->sc_secsize,
530 			BUS_DMASYNC_PREWRITE);
531 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_cmdsts,
532 			offsetof(struct virtio_blk_req, vr_status),
533 			sizeof(uint8_t),
534 			BUS_DMASYNC_PREREAD);
535 
536 	virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
537 			 0, sizeof(struct virtio_blk_req_hdr),
538 			 true);
539 	virtio_enqueue(vsc, vq, slot, vr->vr_payload, true);
540 	virtio_enqueue_p(vsc, vq, slot, vr->vr_cmdsts,
541 			 offsetof(struct virtio_blk_req, vr_status),
542 			 sizeof(uint8_t),
543 			 false);
544 	virtio_enqueue_commit(vsc, vq, slot, true);
545 
546 	for ( ; ; ) {
547 		int dslot;
548 
549 		r = virtio_dequeue(vsc, vq, &dslot, NULL);
550 		if (r != 0)
551 			continue;
552 		if (dslot != slot) {
553 			ld_virtio_vq_done1(sc, vsc, vq, dslot);
554 			continue;
555 		} else
556 			break;
557 	}
558 
559 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_cmdsts,
560 			0, sizeof(struct virtio_blk_req_hdr),
561 			BUS_DMASYNC_POSTWRITE);
562 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_payload,
563 			0, blkcnt*ld->sc_secsize,
564 			BUS_DMASYNC_POSTWRITE);
565 	bus_dmamap_sync(vsc->sc_dmat, vr->vr_cmdsts,
566 			offsetof(struct virtio_blk_req, vr_status),
567 			sizeof(uint8_t),
568 			BUS_DMASYNC_POSTREAD);
569 	if (vr->vr_status == VIRTIO_BLK_S_OK)
570 		r = 0;
571 	else
572 		r = EIO;
573 	virtio_dequeue_commit(vsc, vq, slot);
574 
575 	return r;
576 }
577 
578 static int
579 ld_virtio_detach(device_t self, int flags)
580 {
581 	struct ld_virtio_softc *sc = device_private(self);
582 	struct ld_softc *ld = &sc->sc_ld;
583 	bus_dma_tag_t dmat = sc->sc_virtio->sc_dmat;
584 	int r, i, qsize;
585 
586 	qsize = sc->sc_vq.vq_num;
587 	r = ldbegindetach(ld, flags);
588 	if (r != 0)
589 		return r;
590 	virtio_reset(sc->sc_virtio);
591 	virtio_free_vq(sc->sc_virtio, &sc->sc_vq);
592 
593 	for (i = 0; i < qsize; i++) {
594 		bus_dmamap_destroy(dmat,
595 				   sc->sc_reqs[i].vr_cmdsts);
596 		bus_dmamap_destroy(dmat,
597 				   sc->sc_reqs[i].vr_payload);
598 	}
599 	bus_dmamem_unmap(dmat, sc->sc_reqs,
600 			 sizeof(struct virtio_blk_req) * qsize);
601 	bus_dmamem_free(dmat, &sc->sc_reqs_seg, 1);
602 
603 	ldenddetach(ld);
604 
605 	return 0;
606 }
607