xref: /openbsd-src/sys/dev/pci/virtio_pci.c (revision f2da64fbbbf1b03f09f390ab01267c93dfd77c4c) 
1 /*	$OpenBSD: virtio_pci.c,v 1.15 2016/07/19 02:51:09 sf Exp $	*/
2 /*	$NetBSD: virtio.c,v 1.3 2011/11/02 23:05:52 njoly Exp $	*/
3 
4 /*
5  * Copyright (c) 2012 Stefan Fritsch.
6  * Copyright (c) 2010 Minoura Makoto.
7  * All rights reserved.
8  *
9  * Redistribution and use in source and binary forms, with or without
10  * modification, are permitted provided that the following conditions
11  * are met:
12  * 1. Redistributions of source code must retain the above copyright
13  *    notice, this list of conditions and the following disclaimer.
14  * 2. Redistributions in binary form must reproduce the above copyright
15  *    notice, this list of conditions and the following disclaimer in the
16  *    documentation and/or other materials provided with the distribution.
17  *
18  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
19  * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
20  * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
21  * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
22  * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
23  * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28  */
29 
30 #include <sys/param.h>
31 #include <sys/systm.h>
32 #include <sys/device.h>
33 #include <sys/mutex.h>
34 
35 #include <dev/pci/pcidevs.h>
36 #include <dev/pci/pcireg.h>
37 #include <dev/pci/pcivar.h>
38 
39 #include <dev/pci/virtioreg.h>
40 #include <dev/pci/virtiovar.h>
41 
42 /*
43  * XXX: Before being used on big endian arches, the access to config registers
44  * XXX: needs to be reviewed/fixed. The non-device specific registers are
45  * XXX: PCI-endian while the device specific registers are native endian.
46  */
47 
48 #define MAX_MSIX_VECS	8
49 #define virtio_set_status(sc, s) virtio_pci_set_status(sc, s)
50 #define virtio_device_reset(sc) virtio_set_status((sc), 0)
51 
52 struct virtio_pci_softc;
53 
54 int		virtio_pci_match(struct device *, void *, void *);
55 void		virtio_pci_attach(struct device *, struct device *, void *);
56 int		virtio_pci_detach(struct device *, int);
57 
58 void		virtio_pci_kick(struct virtio_softc *, uint16_t);
59 uint8_t		virtio_pci_read_device_config_1(struct virtio_softc *, int);
60 uint16_t	virtio_pci_read_device_config_2(struct virtio_softc *, int);
61 uint32_t	virtio_pci_read_device_config_4(struct virtio_softc *, int);
62 uint64_t	virtio_pci_read_device_config_8(struct virtio_softc *, int);
63 void		virtio_pci_write_device_config_1(struct virtio_softc *, int, uint8_t);
64 void		virtio_pci_write_device_config_2(struct virtio_softc *, int, uint16_t);
65 void		virtio_pci_write_device_config_4(struct virtio_softc *, int, uint32_t);
66 void		virtio_pci_write_device_config_8(struct virtio_softc *, int, uint64_t);
67 uint16_t	virtio_pci_read_queue_size(struct virtio_softc *, uint16_t);
68 void		virtio_pci_setup_queue(struct virtio_softc *, uint16_t, uint32_t);
69 void		virtio_pci_set_status(struct virtio_softc *, int);
70 uint32_t	virtio_pci_negotiate_features(struct virtio_softc *, uint32_t,
71 					      const struct virtio_feature_name *);
72 int		virtio_pci_msix_establish(struct virtio_pci_softc *, struct pci_attach_args *, int, int (*)(void *), void *);
73 int		virtio_pci_setup_msix(struct virtio_pci_softc *, struct pci_attach_args *, int);
74 void		virtio_pci_free_irqs(struct virtio_pci_softc *);
75 int		virtio_pci_poll_intr(void *);
76 int		virtio_pci_legacy_intr(void *);
77 int		virtio_pci_config_intr(void *);
78 int		virtio_pci_queue_intr(void *);
79 int		virtio_pci_shared_queue_intr(void *);
80 
81 enum irq_type {
82 	IRQ_NO_MSIX,
83 	IRQ_MSIX_SHARED, /* vec 0: config irq, vec 1 shared by all vqs */
84 	IRQ_MSIX_PER_VQ, /* vec 0: config irq, vec n: irq of vq[n-1] */
85 };
86 
87 struct virtio_pci_softc {
88 	struct virtio_softc	sc_sc;
89 	pci_chipset_tag_t	sc_pc;
90 
91 	bus_space_tag_t		sc_iot;
92 	bus_space_handle_t	sc_ioh;
93 	bus_size_t		sc_iosize;
94 
95 	void			*sc_ih[MAX_MSIX_VECS];
96 
97 	int			sc_config_offset;
98 	enum irq_type		sc_irq_type;
99 };
100 
101 struct cfattach virtio_pci_ca = {
102 	sizeof(struct virtio_pci_softc),
103 	virtio_pci_match,
104 	virtio_pci_attach,
105 	virtio_pci_detach,
106 	NULL
107 };
108 
109 struct virtio_ops virtio_pci_ops = {
110 	virtio_pci_kick,
111 	virtio_pci_read_device_config_1,
112 	virtio_pci_read_device_config_2,
113 	virtio_pci_read_device_config_4,
114 	virtio_pci_read_device_config_8,
115 	virtio_pci_write_device_config_1,
116 	virtio_pci_write_device_config_2,
117 	virtio_pci_write_device_config_4,
118 	virtio_pci_write_device_config_8,
119 	virtio_pci_read_queue_size,
120 	virtio_pci_setup_queue,
121 	virtio_pci_set_status,
122 	virtio_pci_negotiate_features,
123 	virtio_pci_poll_intr,
124 };
125 
126 uint16_t
127 virtio_pci_read_queue_size(struct virtio_softc *vsc, uint16_t idx)
128 {
129 	struct virtio_pci_softc *sc = (struct virtio_pci_softc *)vsc;
130 	bus_space_write_2(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_QUEUE_SELECT,
131 	    idx);
132 	return bus_space_read_2(sc->sc_iot, sc->sc_ioh,
133 	    VIRTIO_CONFIG_QUEUE_SIZE);
134 }
135 
136 void
137 virtio_pci_setup_queue(struct virtio_softc *vsc, uint16_t idx, uint32_t addr)
138 {
139 	struct virtio_pci_softc *sc = (struct virtio_pci_softc *)vsc;
140 	bus_space_write_2(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_QUEUE_SELECT,
141 	    idx);
142 	bus_space_write_4(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_QUEUE_ADDRESS,
143 	    addr);
144 
145 	/*
146 	 * This path is only executed if this function is called after
147 	 * the child's attach function has finished. In other cases,
148 	 * it's done in virtio_pci_setup_msix().
149 	 */
150 	if (sc->sc_irq_type != IRQ_NO_MSIX) {
151 		int vec = 1;
152 		if (sc->sc_irq_type == IRQ_MSIX_PER_VQ)
153 		       vec += idx;
154 		bus_space_write_2(sc->sc_iot, sc->sc_ioh,
155 		    VIRTIO_MSI_QUEUE_VECTOR, vec);
156 	}
157 }
158 
159 void
160 virtio_pci_set_status(struct virtio_softc *vsc, int status)
161 {
162 	struct virtio_pci_softc *sc = (struct virtio_pci_softc *)vsc;
163 	int old = 0;
164 
165 	if (status != 0)
166 		old = bus_space_read_1(sc->sc_iot, sc->sc_ioh,
167 				       VIRTIO_CONFIG_DEVICE_STATUS);
168 	bus_space_write_1(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_DEVICE_STATUS,
169 			  status|old);
170 }
171 
172 int
173 virtio_pci_match(struct device *parent, void *match, void *aux)
174 {
175 	struct pci_attach_args *pa;
176 
177 	pa = (struct pci_attach_args *)aux;
178 	if (PCI_VENDOR(pa->pa_id) == PCI_VENDOR_QUMRANET &&
179 	    PCI_PRODUCT(pa->pa_id) >= 0x1000 &&
180 	    PCI_PRODUCT(pa->pa_id) <= 0x103f &&
181 	    PCI_REVISION(pa->pa_class) == 0)
182 		return 1;
183 	return 0;
184 }
185 
186 void
187 virtio_pci_attach(struct device *parent, struct device *self, void *aux)
188 {
189 	struct virtio_pci_softc *sc = (struct virtio_pci_softc *)self;
190 	struct virtio_softc *vsc = &sc->sc_sc;
191 	struct pci_attach_args *pa = (struct pci_attach_args *)aux;
192 	pci_chipset_tag_t pc = pa->pa_pc;
193 	pcitag_t tag = pa->pa_tag;
194 	int revision;
195 	pcireg_t id;
196 	char const *intrstr;
197 	pci_intr_handle_t ih;
198 
199 	revision = PCI_REVISION(pa->pa_class);
200 	if (revision != 0) {
201 		printf("unknown revision 0x%02x; giving up\n", revision);
202 		return;
203 	}
204 
205 	/* subsystem ID shows what I am */
206 	id = PCI_PRODUCT(pci_conf_read(pc, tag, PCI_SUBSYS_ID_REG));
207 
208 	printf("\n");
209 
210 	vsc->sc_ops = &virtio_pci_ops;
211 	sc->sc_pc = pc;
212 	vsc->sc_dmat = pa->pa_dmat;
213 	sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
214 	sc->sc_irq_type = IRQ_NO_MSIX;
215 
216 	/*
217 	 * For virtio, ignore normal MSI black/white-listing depending on the
218 	 * PCI bridge but enable it unconditionally.
219 	 */
220 	pa->pa_flags |= PCI_FLAGS_MSI_ENABLED;
221 
222 	if (pci_mapreg_map(pa, PCI_MAPREG_START, PCI_MAPREG_TYPE_IO, 0,
223 	    &sc->sc_iot, &sc->sc_ioh, NULL, &sc->sc_iosize, 0)) {
224 		printf("%s: can't map i/o space\n", vsc->sc_dev.dv_xname);
225 		return;
226 	}
227 
228 	virtio_device_reset(vsc);
229 	virtio_pci_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_ACK);
230 	virtio_pci_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER);
231 
232 	/* XXX: use softc as aux... */
233 	vsc->sc_childdevid = id;
234 	vsc->sc_child = NULL;
235 	config_found(self, sc, NULL);
236 	if (vsc->sc_child == NULL) {
237 		printf("%s: no matching child driver; not configured\n",
238 		    vsc->sc_dev.dv_xname);
239 		goto fail_1;
240 	}
241 	if (vsc->sc_child == VIRTIO_CHILD_ERROR) {
242 		printf("%s: virtio configuration failed\n",
243 		    vsc->sc_dev.dv_xname);
244 		goto fail_1;
245 	}
246 
247 	if (virtio_pci_setup_msix(sc, pa, 0) == 0) {
248 		sc->sc_irq_type = IRQ_MSIX_PER_VQ;
249 		intrstr = "msix per-VQ";
250 	} else if (virtio_pci_setup_msix(sc, pa, 1) == 0) {
251 		sc->sc_irq_type = IRQ_MSIX_SHARED;
252 		intrstr = "msix shared";
253 	} else {
254 		if (pci_intr_map_msi(pa, &ih) != 0 && pci_intr_map(pa, &ih) != 0) {
255 			printf("%s: couldn't map interrupt\n", vsc->sc_dev.dv_xname);
256 			goto fail_2;
257 		}
258 		intrstr = pci_intr_string(pc, ih);
259 		/*
260 		 * We always set the IPL_MPSAFE flag in order to do the relatively
261 		 * expensive ISR read without lock, and then grab the kernel lock in
262 		 * the interrupt handler.
263 		 * For now, we don't support IPL_MPSAFE vq_done functions.
264 		 */
265 		KASSERT((vsc->sc_ipl & IPL_MPSAFE) == 0);
266 		sc->sc_ih[0] = pci_intr_establish(pc, ih, vsc->sc_ipl | IPL_MPSAFE,
267 		    virtio_pci_legacy_intr, sc, vsc->sc_dev.dv_xname);
268 		if (sc->sc_ih[0] == NULL) {
269 			printf("%s: couldn't establish interrupt", vsc->sc_dev.dv_xname);
270 			if (intrstr != NULL)
271 				printf(" at %s", intrstr);
272 			printf("\n");
273 			goto fail_2;
274 		}
275 	}
276 	printf("%s: %s\n", vsc->sc_dev.dv_xname, intrstr);
277 
278 	virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER_OK);
279 	return;
280 
281 fail_2:
282 	config_detach(vsc->sc_child, 0);
283 fail_1:
284 	/* no pci_mapreg_unmap() or pci_intr_unmap() */
285 	virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED);
286 }
287 
288 int
289 virtio_pci_detach(struct device *self, int flags)
290 {
291 	struct virtio_pci_softc *sc = (struct virtio_pci_softc *)self;
292 	struct virtio_softc *vsc = &sc->sc_sc;
293 	int r;
294 
295 	if (vsc->sc_child != 0 && vsc->sc_child != VIRTIO_CHILD_ERROR) {
296 		r = config_detach(vsc->sc_child, flags);
297 		if (r)
298 			return r;
299 	}
300 	KASSERT(vsc->sc_child == 0 || vsc->sc_child == VIRTIO_CHILD_ERROR);
301 	KASSERT(vsc->sc_vqs == 0);
302 	virtio_pci_free_irqs(sc);
303 	if (sc->sc_iosize)
304 		bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize);
305 	sc->sc_iosize = 0;
306 
307 	return 0;
308 }
309 
310 /*
311  * Feature negotiation.
312  * Prints available / negotiated features if guest_feature_names != NULL and
313  * VIRTIO_DEBUG is 1
314  */
315 uint32_t
316 virtio_pci_negotiate_features(struct virtio_softc *vsc, uint32_t guest_features,
317     const struct virtio_feature_name *guest_feature_names)
318 {
319 	struct virtio_pci_softc *sc = (struct virtio_pci_softc *)vsc;
320 	uint32_t host, neg;
321 
322 	/*
323 	 * indirect descriptors can be switched off by setting bit 1 in the
324 	 * driver flags, see config(8)
325 	 */
326 	if (!(vsc->sc_dev.dv_cfdata->cf_flags & 1) &&
327 	    !(vsc->sc_child->dv_cfdata->cf_flags & 1)) {
328 		guest_features |= VIRTIO_F_RING_INDIRECT_DESC;
329 	} else {
330 		printf("RingIndirectDesc disabled by UKC\n");
331 	}
332 	host = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
333 				VIRTIO_CONFIG_DEVICE_FEATURES);
334 	neg = host & guest_features;
335 #if VIRTIO_DEBUG
336 	if (guest_feature_names)
337 		virtio_log_features(host, neg, guest_feature_names);
338 #endif
339 	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
340 			  VIRTIO_CONFIG_GUEST_FEATURES, neg);
341 	vsc->sc_features = neg;
342 	if (neg & VIRTIO_F_RING_INDIRECT_DESC)
343 		vsc->sc_indirect = 1;
344 	else
345 		vsc->sc_indirect = 0;
346 
347 	return neg;
348 }
349 
350 /*
351  * Device configuration registers.
352  */
353 uint8_t
354 virtio_pci_read_device_config_1(struct virtio_softc *vsc, int index)
355 {
356 	struct virtio_pci_softc *sc = (struct virtio_pci_softc *)vsc;
357 	return bus_space_read_1(sc->sc_iot, sc->sc_ioh,
358 	    sc->sc_config_offset + index);
359 }
360 
361 uint16_t
362 virtio_pci_read_device_config_2(struct virtio_softc *vsc, int index)
363 {
364 	struct virtio_pci_softc *sc = (struct virtio_pci_softc *)vsc;
365 	return bus_space_read_2(sc->sc_iot, sc->sc_ioh,
366 	    sc->sc_config_offset + index);
367 }
368 
369 uint32_t
370 virtio_pci_read_device_config_4(struct virtio_softc *vsc, int index)
371 {
372 	struct virtio_pci_softc *sc = (struct virtio_pci_softc *)vsc;
373 	return bus_space_read_4(sc->sc_iot, sc->sc_ioh,
374 	    sc->sc_config_offset + index);
375 }
376 
377 uint64_t
378 virtio_pci_read_device_config_8(struct virtio_softc *vsc, int index)
379 {
380 	struct virtio_pci_softc *sc = (struct virtio_pci_softc *)vsc;
381 	uint64_t r;
382 
383 	r = bus_space_read_4(sc->sc_iot, sc->sc_ioh,
384 	    sc->sc_config_offset + index + sizeof(uint32_t));
385 	r <<= 32;
386 	r += bus_space_read_4(sc->sc_iot, sc->sc_ioh,
387 	    sc->sc_config_offset + index);
388 	return r;
389 }
390 
391 void
392 virtio_pci_write_device_config_1(struct virtio_softc *vsc, int index,
393     uint8_t value)
394 {
395 	struct virtio_pci_softc *sc = (struct virtio_pci_softc *)vsc;
396 	bus_space_write_1(sc->sc_iot, sc->sc_ioh,
397 	    sc->sc_config_offset + index, value);
398 }
399 
400 void
401 virtio_pci_write_device_config_2(struct virtio_softc *vsc, int index,
402     uint16_t value)
403 {
404 	struct virtio_pci_softc *sc = (struct virtio_pci_softc *)vsc;
405 	bus_space_write_2(sc->sc_iot, sc->sc_ioh,
406 	    sc->sc_config_offset + index, value);
407 }
408 
409 void
410 virtio_pci_write_device_config_4(struct virtio_softc *vsc,
411 			     int index, uint32_t value)
412 {
413 	struct virtio_pci_softc *sc = (struct virtio_pci_softc *)vsc;
414 	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
415 	    sc->sc_config_offset + index, value);
416 }
417 
418 void
419 virtio_pci_write_device_config_8(struct virtio_softc *vsc,
420 			     int index, uint64_t value)
421 {
422 	struct virtio_pci_softc *sc = (struct virtio_pci_softc *)vsc;
423 	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
424 	    sc->sc_config_offset + index, value & 0xffffffff);
425 	bus_space_write_4(sc->sc_iot, sc->sc_ioh,
426 	    sc->sc_config_offset + index + sizeof(uint32_t), value >> 32);
427 }
428 
429 int
430 virtio_pci_msix_establish(struct virtio_pci_softc *sc,
431     struct pci_attach_args *pa, int idx, int (*handler)(void *), void *ih_arg)
432 {
433 	struct virtio_softc *vsc = &sc->sc_sc;
434 	pci_intr_handle_t ih;
435 
436 	if (pci_intr_map_msix(pa, idx, &ih) != 0) {
437 #if VIRTIO_DEBUG
438 		printf("%s[%d]: pci_intr_map_msix failed\n",
439 		    vsc->sc_dev.dv_xname, idx);
440 #endif
441 		return 1;
442 	}
443 	sc->sc_ih[idx] = pci_intr_establish(sc->sc_pc, ih, vsc->sc_ipl,
444 	    handler, ih_arg, vsc->sc_dev.dv_xname);
445 	if (sc->sc_ih[idx] == NULL) {
446 		printf("%s[%d]: couldn't establish msix interrupt\n",
447 		    vsc->sc_dev.dv_xname, idx);
448 		return 1;
449 	}
450 	return 0;
451 }
452 
453 void
454 virtio_pci_free_irqs(struct virtio_pci_softc *sc)
455 {
456 	struct virtio_softc *vsc = &sc->sc_sc;
457 	int i;
458 
459 	if (sc->sc_config_offset == VIRTIO_CONFIG_DEVICE_CONFIG_MSI) {
460 		for (i = 0; i < vsc->sc_nvqs; i++) {
461 			bus_space_write_2(sc->sc_iot, sc->sc_ioh,
462 			    VIRTIO_CONFIG_QUEUE_SELECT, i);
463 			bus_space_write_2(sc->sc_iot, sc->sc_ioh,
464 			    VIRTIO_MSI_QUEUE_VECTOR, VIRTIO_MSI_NO_VECTOR);
465 		}
466 	}
467 
468 	for (i = 0; i < MAX_MSIX_VECS; i++) {
469 		if (sc->sc_ih[i]) {
470 			pci_intr_disestablish(sc->sc_pc, sc->sc_ih[i]);
471 			sc->sc_ih[i] = NULL;
472 		}
473 	}
474 
475 	sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_NOMSI;
476 }
477 
478 int
479 virtio_pci_setup_msix(struct virtio_pci_softc *sc, struct pci_attach_args *pa,
480     int shared)
481 {
482 	struct virtio_softc *vsc = &sc->sc_sc;
483 	int i;
484 
485 	if (virtio_pci_msix_establish(sc, pa, 0, virtio_pci_config_intr, vsc))
486 		return 1;
487 	sc->sc_config_offset = VIRTIO_CONFIG_DEVICE_CONFIG_MSI;
488 	bus_space_write_2(sc->sc_iot, sc->sc_ioh, VIRTIO_MSI_CONFIG_VECTOR, 0);
489 
490 	if (shared) {
491 		if (virtio_pci_msix_establish(sc, pa, 1,
492 		    virtio_pci_shared_queue_intr, vsc)) {
493 			goto fail;
494 		}
495 
496 		for (i = 0; i < vsc->sc_nvqs; i++) {
497 			bus_space_write_2(sc->sc_iot, sc->sc_ioh,
498 			    VIRTIO_CONFIG_QUEUE_SELECT, i);
499 			bus_space_write_2(sc->sc_iot, sc->sc_ioh,
500 			    VIRTIO_MSI_QUEUE_VECTOR, 1);
501 		}
502 	} else {
503 		for (i = 0; i <= vsc->sc_nvqs; i++) {
504 			if (virtio_pci_msix_establish(sc, pa, i + 1,
505 			    virtio_pci_queue_intr, &vsc->sc_vqs[i])) {
506 				goto fail;
507 			}
508 			bus_space_write_2(sc->sc_iot, sc->sc_ioh,
509 			    VIRTIO_CONFIG_QUEUE_SELECT, i);
510 			bus_space_write_2(sc->sc_iot, sc->sc_ioh,
511 			    VIRTIO_MSI_QUEUE_VECTOR, i + 1);
512 		}
513 	}
514 
515 	return 0;
516 fail:
517 	virtio_pci_free_irqs(sc);
518 	return 1;
519 }
520 
521 /*
522  * Interrupt handler.
523  */
524 
525 /*
526  * Only used without MSI-X
527  */
528 int
529 virtio_pci_legacy_intr(void *arg)
530 {
531 	struct virtio_pci_softc *sc = arg;
532 	struct virtio_softc *vsc = &sc->sc_sc;
533 	int isr, r = 0;
534 
535 	/* check and ack the interrupt */
536 	isr = bus_space_read_1(sc->sc_iot, sc->sc_ioh,
537 	    VIRTIO_CONFIG_ISR_STATUS);
538 	if (isr == 0)
539 		return 0;
540 	KERNEL_LOCK();
541 	if ((isr & VIRTIO_CONFIG_ISR_CONFIG_CHANGE) &&
542 	    (vsc->sc_config_change != NULL)) {
543 		r = (vsc->sc_config_change)(vsc);
544 	}
545 	r |= virtio_check_vqs(vsc);
546 	KERNEL_UNLOCK();
547 
548 	return r;
549 }
550 
551 /*
552  * Only used with MSI-X
553  */
554 int
555 virtio_pci_config_intr(void *arg)
556 {
557 	struct virtio_softc *vsc = arg;
558 
559 	if (vsc->sc_config_change != NULL)
560 		return vsc->sc_config_change(vsc);
561 	return 0;
562 }
563 
564 /*
565  * Only used with MSI-X
566  */
567 int
568 virtio_pci_queue_intr(void *arg)
569 {
570 	struct virtqueue *vq = arg;
571 
572 	if (vq->vq_done)
573 		return (vq->vq_done)(vq);
574 	return 0;
575 }
576 
577 int
578 virtio_pci_shared_queue_intr(void *arg)
579 {
580 	struct virtio_softc *vsc = arg;
581 
582 	return virtio_check_vqs(vsc);
583 }
584 
585 /*
586  * Interrupt handler to be used when polling.
587  * We cannot use isr here because it is not defined in MSI-X mode.
588  */
589 int
590 virtio_pci_poll_intr(void *arg)
591 {
592 	struct virtio_pci_softc *sc = arg;
593 	struct virtio_softc *vsc = &sc->sc_sc;
594 	int r = 0;
595 
596 	if (vsc->sc_config_change != NULL)
597 		r = (vsc->sc_config_change)(vsc);
598 
599 	r |= virtio_check_vqs(vsc);
600 
601 	return r;
602 }
603 
604 
605 void
606 virtio_pci_kick(struct virtio_softc *vsc, uint16_t idx)
607 {
608 	struct virtio_pci_softc *sc = (struct virtio_pci_softc *)vsc;
609 	bus_space_write_2(sc->sc_iot, sc->sc_ioh, VIRTIO_CONFIG_QUEUE_NOTIFY,
610 	    idx);
611 }
612