1 /* $OpenBSD: virtio_mmio.c,v 1.16 2024/08/27 19:01:11 sf Exp $ */ 2 /* $NetBSD: virtio.c,v 1.3 2011/11/02 23:05:52 njoly Exp $ */ 3 4 /* 5 * Copyright (c) 2014 Patrick Wildt <patrick@blueri.se> 6 * Copyright (c) 2012 Stefan Fritsch. 7 * Copyright (c) 2010 Minoura Makoto. 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 #include <sys/param.h> 32 #include <sys/systm.h> 33 #include <sys/kernel.h> 34 #include <sys/device.h> 35 #include <sys/mutex.h> 36 37 #include <dev/pv/virtioreg.h> 38 #include <dev/pv/virtiovar.h> 39 40 #include <machine/fdt.h> 41 42 #include <dev/ofw/fdt.h> 43 #include <dev/ofw/openfirm.h> 44 45 #define VIRTIO_MMIO_MAGIC ('v' | 'i' << 8 | 'r' << 16 | 't' << 24) 46 47 #define VIRTIO_MMIO_MAGIC_VALUE 0x000 48 #define VIRTIO_MMIO_VERSION 0x004 49 #define VIRTIO_MMIO_DEVICE_ID 0x008 50 #define VIRTIO_MMIO_VENDOR_ID 0x00c 51 #define VIRTIO_MMIO_HOST_FEATURES 0x010 52 #define VIRTIO_MMIO_HOST_FEATURES_SEL 0x014 53 #define VIRTIO_MMIO_GUEST_FEATURES 0x020 54 #define VIRTIO_MMIO_GUEST_FEATURES_SEL 0x024 55 #define VIRTIO_MMIO_GUEST_PAGE_SIZE 0x028 56 #define VIRTIO_MMIO_QUEUE_SEL 0x030 57 #define VIRTIO_MMIO_QUEUE_NUM_MAX 0x034 58 #define VIRTIO_MMIO_QUEUE_NUM 0x038 59 #define VIRTIO_MMIO_QUEUE_ALIGN 0x03c 60 #define VIRTIO_MMIO_QUEUE_PFN 0x040 61 #define VIRTIO_MMIO_QUEUE_READY 0x044 62 #define VIRTIO_MMIO_QUEUE_NOTIFY 0x050 63 #define VIRTIO_MMIO_INTERRUPT_STATUS 0x060 64 #define VIRTIO_MMIO_INTERRUPT_ACK 0x064 65 #define VIRTIO_MMIO_STATUS 0x070 66 #define VIRTIO_MMIO_QUEUE_DESC_LOW 0x080 67 #define VIRTIO_MMIO_QUEUE_DESC_HIGH 0x084 68 #define VIRTIO_MMIO_QUEUE_AVAIL_LOW 0x090 69 #define VIRTIO_MMIO_QUEUE_AVAIL_HIGH 0x094 70 #define VIRTIO_MMIO_QUEUE_USED_LOW 0x0a0 71 #define VIRTIO_MMIO_QUEUE_USED_HIGH 0x0a4 72 #define VIRTIO_MMIO_CONFIG 0x100 73 74 #define VIRTIO_MMIO_INT_VRING (1 << 0) 75 #define VIRTIO_MMIO_INT_CONFIG (1 << 1) 76 77 #define DEVNAME(sc) (sc)->sc_dev.dv_xname 78 79 /* 80 * XXX: Before being used on big endian arches, the access to config registers 81 * XXX: needs to be reviewed/fixed. The non-device specific registers are 82 * XXX: PCI-endian while the device specific registers are native endian. 83 */ 84 85 int virtio_mmio_match(struct device *, void *, void *); 86 void virtio_mmio_attach(struct device *, struct device *, void *); 87 int virtio_mmio_detach(struct device *, int); 88 89 void virtio_mmio_kick(struct virtio_softc *, uint16_t); 90 uint8_t virtio_mmio_read_device_config_1(struct virtio_softc *, int); 91 uint16_t virtio_mmio_read_device_config_2(struct virtio_softc *, int); 92 uint32_t virtio_mmio_read_device_config_4(struct virtio_softc *, int); 93 uint64_t virtio_mmio_read_device_config_8(struct virtio_softc *, int); 94 void virtio_mmio_write_device_config_1(struct virtio_softc *, int, uint8_t); 95 void virtio_mmio_write_device_config_2(struct virtio_softc *, int, uint16_t); 96 void virtio_mmio_write_device_config_4(struct virtio_softc *, int, uint32_t); 97 void virtio_mmio_write_device_config_8(struct virtio_softc *, int, uint64_t); 98 uint16_t virtio_mmio_read_queue_size(struct virtio_softc *, uint16_t); 99 void virtio_mmio_setup_queue(struct virtio_softc *, struct virtqueue *, uint64_t); 100 int virtio_mmio_get_status(struct virtio_softc *); 101 void virtio_mmio_set_status(struct virtio_softc *, int); 102 int virtio_mmio_negotiate_features(struct virtio_softc *, 103 const struct virtio_feature_name *); 104 int virtio_mmio_intr(void *); 105 106 struct virtio_mmio_softc { 107 struct virtio_softc sc_sc; 108 109 bus_space_tag_t sc_iot; 110 bus_space_handle_t sc_ioh; 111 bus_size_t sc_iosize; 112 bus_dma_tag_t sc_dmat; 113 114 void *sc_ih; 115 116 int sc_config_offset; 117 uint32_t sc_version; 118 }; 119 120 const struct cfattach virtio_mmio_ca = { 121 sizeof(struct virtio_mmio_softc), 122 virtio_mmio_match, 123 virtio_mmio_attach, 124 virtio_mmio_detach, 125 NULL 126 }; 127 128 const struct cfattach virtio_mmio_fdt_ca = { 129 sizeof(struct virtio_mmio_softc), 130 NULL, 131 virtio_mmio_attach, 132 virtio_mmio_detach, 133 NULL 134 }; 135 136 const struct virtio_ops virtio_mmio_ops = { 137 virtio_mmio_kick, 138 virtio_mmio_read_device_config_1, 139 virtio_mmio_read_device_config_2, 140 virtio_mmio_read_device_config_4, 141 virtio_mmio_read_device_config_8, 142 virtio_mmio_write_device_config_1, 143 virtio_mmio_write_device_config_2, 144 virtio_mmio_write_device_config_4, 145 virtio_mmio_write_device_config_8, 146 virtio_mmio_read_queue_size, 147 virtio_mmio_setup_queue, 148 virtio_mmio_get_status, 149 virtio_mmio_set_status, 150 virtio_mmio_negotiate_features, 151 virtio_mmio_intr, 152 }; 153 154 uint16_t 155 virtio_mmio_read_queue_size(struct virtio_softc *vsc, uint16_t idx) 156 { 157 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 158 bus_space_write_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_QUEUE_SEL, idx); 159 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, 160 VIRTIO_MMIO_QUEUE_NUM_MAX); 161 } 162 163 void 164 virtio_mmio_setup_queue(struct virtio_softc *vsc, struct virtqueue *vq, 165 uint64_t addr) 166 { 167 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 168 bus_space_write_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_QUEUE_SEL, 169 vq->vq_index); 170 bus_space_write_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_QUEUE_NUM, 171 bus_space_read_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_QUEUE_NUM_MAX)); 172 if (sc->sc_version == 1) { 173 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 174 VIRTIO_MMIO_QUEUE_ALIGN, PAGE_SIZE); 175 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 176 VIRTIO_MMIO_QUEUE_PFN, addr / VIRTIO_PAGE_SIZE); 177 } else { 178 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 179 VIRTIO_MMIO_QUEUE_DESC_LOW, addr); 180 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 181 VIRTIO_MMIO_QUEUE_DESC_HIGH, addr >> 32); 182 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 183 VIRTIO_MMIO_QUEUE_AVAIL_LOW, 184 addr + vq->vq_availoffset); 185 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 186 VIRTIO_MMIO_QUEUE_AVAIL_HIGH, 187 (addr + vq->vq_availoffset) >> 32); 188 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 189 VIRTIO_MMIO_QUEUE_USED_LOW, 190 addr + vq->vq_usedoffset); 191 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 192 VIRTIO_MMIO_QUEUE_USED_HIGH, 193 (addr + vq->vq_usedoffset) >> 32); 194 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 195 VIRTIO_MMIO_QUEUE_READY, 1); 196 } 197 } 198 199 int 200 virtio_mmio_get_status(struct virtio_softc *vsc) 201 { 202 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 203 204 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, 205 VIRTIO_MMIO_STATUS); 206 } 207 208 void 209 virtio_mmio_set_status(struct virtio_softc *vsc, int status) 210 { 211 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 212 int old = 0; 213 214 if (status == 0) { 215 bus_space_write_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_STATUS, 216 0); 217 while (bus_space_read_4(sc->sc_iot, sc->sc_ioh, 218 VIRTIO_MMIO_STATUS) != 0) { 219 CPU_BUSY_CYCLE(); 220 } 221 } else { 222 old = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 223 VIRTIO_MMIO_STATUS); 224 bus_space_write_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_STATUS, 225 status|old); 226 } 227 } 228 229 int 230 virtio_mmio_match(struct device *parent, void *cfdata, void *aux) 231 { 232 struct fdt_attach_args *faa = aux; 233 234 return OF_is_compatible(faa->fa_node, "virtio,mmio"); 235 } 236 237 void 238 virtio_mmio_attach(struct device *parent, struct device *self, void *aux) 239 { 240 struct fdt_attach_args *faa = aux; 241 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)self; 242 struct virtio_softc *vsc = &sc->sc_sc; 243 uint32_t id, magic; 244 struct virtio_attach_args va = { 0 }; 245 246 if (faa->fa_nreg < 1) { 247 printf(": no register data\n"); 248 return; 249 } 250 251 sc->sc_iosize = faa->fa_reg[0].size; 252 sc->sc_iot = faa->fa_iot; 253 sc->sc_dmat = faa->fa_dmat; 254 if (bus_space_map(sc->sc_iot, faa->fa_reg[0].addr, faa->fa_reg[0].size, 255 0, &sc->sc_ioh)) 256 panic("%s: bus_space_map failed!", __func__); 257 258 magic = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 259 VIRTIO_MMIO_MAGIC_VALUE); 260 if (magic != VIRTIO_MMIO_MAGIC) { 261 printf(": wrong magic value 0x%08x; giving up\n", magic); 262 return; 263 } 264 265 sc->sc_version = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 266 VIRTIO_MMIO_VERSION); 267 if (sc->sc_version < 1 || sc->sc_version > 2) { 268 printf(": unknown version 0x%02x; giving up\n", sc->sc_version); 269 return; 270 } 271 272 id = bus_space_read_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_DEVICE_ID); 273 printf(": Virtio %s Device", virtio_device_string(id)); 274 275 printf("\n"); 276 277 /* No device connected. */ 278 if (id == 0) 279 return; 280 281 if (sc->sc_version == 1) 282 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 283 VIRTIO_MMIO_GUEST_PAGE_SIZE, PAGE_SIZE); 284 285 vsc->sc_ops = &virtio_mmio_ops; 286 vsc->sc_dmat = sc->sc_dmat; 287 sc->sc_config_offset = VIRTIO_MMIO_CONFIG; 288 289 virtio_device_reset(vsc); 290 virtio_mmio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_ACK); 291 virtio_mmio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_DRIVER); 292 293 va.va_devid = id; 294 va.va_nintr = 1; 295 vsc->sc_child = NULL; 296 config_found(self, &va, NULL); 297 if (vsc->sc_child == NULL) { 298 printf("%s: no matching child driver; not configured\n", 299 vsc->sc_dev.dv_xname); 300 goto fail_1; 301 } 302 if (vsc->sc_child == VIRTIO_CHILD_ERROR) { 303 printf("%s: virtio configuration failed\n", 304 vsc->sc_dev.dv_xname); 305 goto fail_1; 306 } 307 308 sc->sc_ih = fdt_intr_establish(faa->fa_node, vsc->sc_ipl, 309 virtio_mmio_intr, sc, vsc->sc_dev.dv_xname); 310 if (sc->sc_ih == NULL) { 311 printf("%s: couldn't establish interrupt\n", 312 vsc->sc_dev.dv_xname); 313 goto fail_2; 314 } 315 316 return; 317 318 fail_2: 319 config_detach(vsc->sc_child, 0); 320 fail_1: 321 /* no mmio_mapreg_unmap() or mmio_intr_unmap() */ 322 virtio_set_status(vsc, VIRTIO_CONFIG_DEVICE_STATUS_FAILED); 323 } 324 325 int 326 virtio_mmio_detach(struct device *self, int flags) 327 { 328 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)self; 329 struct virtio_softc *vsc = &sc->sc_sc; 330 int r; 331 332 if (vsc->sc_child != 0 && vsc->sc_child != VIRTIO_CHILD_ERROR) { 333 r = config_detach(vsc->sc_child, flags); 334 if (r) 335 return r; 336 } 337 KASSERT(vsc->sc_child == 0 || vsc->sc_child == VIRTIO_CHILD_ERROR); 338 KASSERT(vsc->sc_vqs == 0); 339 fdt_intr_disestablish(sc->sc_ih); 340 sc->sc_ih = 0; 341 if (sc->sc_iosize) 342 bus_space_unmap(sc->sc_iot, sc->sc_ioh, sc->sc_iosize); 343 sc->sc_iosize = 0; 344 345 return 0; 346 } 347 348 /* 349 * Feature negotiation. 350 * Prints available / negotiated features if guest_feature_names != NULL and 351 * VIRTIO_DEBUG is 1 352 */ 353 int 354 virtio_mmio_negotiate_features(struct virtio_softc *vsc, 355 const struct virtio_feature_name *guest_feature_names) 356 { 357 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 358 uint64_t host, neg; 359 360 vsc->sc_active_features = 0; 361 362 /* 363 * We enable indirect descriptors by default. They can be switched 364 * off by setting bit 1 in the driver flags, see config(8). 365 */ 366 if (!(vsc->sc_dev.dv_cfdata->cf_flags & VIRTIO_CF_NO_INDIRECT) && 367 !(vsc->sc_child->dv_cfdata->cf_flags & VIRTIO_CF_NO_INDIRECT)) { 368 vsc->sc_driver_features |= VIRTIO_F_RING_INDIRECT_DESC; 369 } else if (guest_feature_names != NULL) { 370 printf("RingIndirectDesc disabled by UKC\n"); 371 } 372 /* 373 * The driver must add VIRTIO_F_RING_EVENT_IDX if it supports it. 374 * If it did, check if it is disabled by bit 2 in the driver flags. 375 */ 376 if ((vsc->sc_driver_features & VIRTIO_F_RING_EVENT_IDX) && 377 ((vsc->sc_dev.dv_cfdata->cf_flags & VIRTIO_CF_NO_EVENT_IDX) || 378 (vsc->sc_child->dv_cfdata->cf_flags & VIRTIO_CF_NO_EVENT_IDX))) { 379 if (guest_feature_names != NULL) 380 printf(" RingEventIdx disabled by UKC"); 381 vsc->sc_driver_features &= ~(VIRTIO_F_RING_EVENT_IDX); 382 } 383 384 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 385 VIRTIO_MMIO_HOST_FEATURES_SEL, 0); 386 host = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 387 VIRTIO_MMIO_HOST_FEATURES); 388 neg = host & vsc->sc_driver_features; 389 #if VIRTIO_DEBUG 390 if (guest_feature_names) 391 virtio_log_features(host, neg, guest_feature_names); 392 #endif 393 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 394 VIRTIO_MMIO_GUEST_FEATURES_SEL, 0); 395 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 396 VIRTIO_MMIO_GUEST_FEATURES, neg); 397 vsc->sc_active_features = neg; 398 if (neg & VIRTIO_F_RING_INDIRECT_DESC) 399 vsc->sc_indirect = 1; 400 else 401 vsc->sc_indirect = 0; 402 403 return 0; 404 } 405 406 /* 407 * Device configuration registers. 408 */ 409 uint8_t 410 virtio_mmio_read_device_config_1(struct virtio_softc *vsc, int index) 411 { 412 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 413 return bus_space_read_1(sc->sc_iot, sc->sc_ioh, 414 sc->sc_config_offset + index); 415 } 416 417 uint16_t 418 virtio_mmio_read_device_config_2(struct virtio_softc *vsc, int index) 419 { 420 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 421 return bus_space_read_2(sc->sc_iot, sc->sc_ioh, 422 sc->sc_config_offset + index); 423 } 424 425 uint32_t 426 virtio_mmio_read_device_config_4(struct virtio_softc *vsc, int index) 427 { 428 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 429 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, 430 sc->sc_config_offset + index); 431 } 432 433 uint64_t 434 virtio_mmio_read_device_config_8(struct virtio_softc *vsc, int index) 435 { 436 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 437 uint64_t r; 438 439 r = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 440 sc->sc_config_offset + index + sizeof(uint32_t)); 441 r <<= 32; 442 r += bus_space_read_4(sc->sc_iot, sc->sc_ioh, 443 sc->sc_config_offset + index); 444 return r; 445 } 446 447 void 448 virtio_mmio_write_device_config_1(struct virtio_softc *vsc, 449 int index, uint8_t value) 450 { 451 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 452 bus_space_write_1(sc->sc_iot, sc->sc_ioh, 453 sc->sc_config_offset + index, value); 454 } 455 456 void 457 virtio_mmio_write_device_config_2(struct virtio_softc *vsc, 458 int index, uint16_t value) 459 { 460 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 461 bus_space_write_2(sc->sc_iot, sc->sc_ioh, 462 sc->sc_config_offset + index, value); 463 } 464 465 void 466 virtio_mmio_write_device_config_4(struct virtio_softc *vsc, 467 int index, uint32_t value) 468 { 469 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 470 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 471 sc->sc_config_offset + index, value); 472 } 473 474 void 475 virtio_mmio_write_device_config_8(struct virtio_softc *vsc, 476 int index, uint64_t value) 477 { 478 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 479 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 480 sc->sc_config_offset + index, 481 value & 0xffffffff); 482 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 483 sc->sc_config_offset + index + sizeof(uint32_t), 484 value >> 32); 485 } 486 487 /* 488 * Interrupt handler. 489 */ 490 int 491 virtio_mmio_intr(void *arg) 492 { 493 struct virtio_mmio_softc *sc = arg; 494 struct virtio_softc *vsc = &sc->sc_sc; 495 int isr, r = 0; 496 497 /* check and ack the interrupt */ 498 isr = bus_space_read_4(sc->sc_iot, sc->sc_ioh, 499 VIRTIO_MMIO_INTERRUPT_STATUS); 500 bus_space_write_4(sc->sc_iot, sc->sc_ioh, 501 VIRTIO_MMIO_INTERRUPT_ACK, isr); 502 if ((isr & VIRTIO_MMIO_INT_CONFIG) && 503 (vsc->sc_config_change != NULL)) 504 r = (vsc->sc_config_change)(vsc); 505 if ((isr & VIRTIO_MMIO_INT_VRING)) 506 r |= virtio_check_vqs(vsc); 507 508 return r; 509 } 510 511 void 512 virtio_mmio_kick(struct virtio_softc *vsc, uint16_t idx) 513 { 514 struct virtio_mmio_softc *sc = (struct virtio_mmio_softc *)vsc; 515 bus_space_write_4(sc->sc_iot, sc->sc_ioh, VIRTIO_MMIO_QUEUE_NOTIFY, 516 idx); 517 } 518