1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/mmio.h" 37 #include "spdk/string.h" 38 #include "spdk/env.h" 39 40 #include "spdk_internal/virtio.h" 41 #include "spdk_internal/memory.h" 42 43 struct virtio_hw { 44 uint8_t use_msix; 45 uint32_t notify_off_multiplier; 46 uint8_t *isr; 47 uint16_t *notify_base; 48 49 struct { 50 /** Mem-mapped resources from given PCI BAR */ 51 void *vaddr; 52 53 /** Length of the address space */ 54 uint32_t len; 55 } pci_bar[6]; 56 57 struct virtio_pci_common_cfg *common_cfg; 58 struct spdk_pci_device *pci_dev; 59 60 /** Device-specific PCI config space */ 61 void *dev_cfg; 62 }; 63 64 struct virtio_pci_probe_ctx { 65 virtio_pci_create_cb enum_cb; 66 void *enum_ctx; 67 uint16_t device_id; 68 }; 69 70 /* 71 * Following macros are derived from linux/pci_regs.h, however, 72 * we can't simply include that header here, as there is no such 73 * file for non-Linux platform. 74 */ 75 #define PCI_CAPABILITY_LIST 0x34 76 #define PCI_CAP_ID_VNDR 0x09 77 #define PCI_CAP_ID_MSIX 0x11 78 79 static inline int 80 check_vq_phys_addr_ok(struct virtqueue *vq) 81 { 82 /* Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit, 83 * and only accepts 32 bit page frame number. 84 * Check if the allocated physical memory exceeds 16TB. 85 */ 86 if ((vq->vq_ring_mem + vq->vq_ring_size - 1) >> 87 (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) { 88 SPDK_ERRLOG("vring address shouldn't be above 16TB!\n"); 89 return 0; 90 } 91 92 return 1; 93 } 94 95 static void 96 free_virtio_hw(struct virtio_hw *hw) 97 { 98 unsigned i; 99 100 for (i = 0; i < 6; ++i) { 101 if (hw->pci_bar[i].vaddr == NULL) { 102 continue; 103 } 104 105 spdk_pci_device_unmap_bar(hw->pci_dev, i, hw->pci_bar[i].vaddr); 106 } 107 108 free(hw); 109 } 110 111 static void 112 pci_dump_json_info(struct virtio_dev *dev, struct spdk_json_write_ctx *w) 113 { 114 struct virtio_hw *hw = dev->ctx; 115 struct spdk_pci_addr pci_addr = spdk_pci_device_get_addr((struct spdk_pci_device *)hw->pci_dev); 116 char addr[32]; 117 118 spdk_json_write_name(w, "type"); 119 if (dev->modern) { 120 spdk_json_write_string(w, "pci-modern"); 121 } else { 122 spdk_json_write_string(w, "pci-legacy"); 123 } 124 125 spdk_json_write_name(w, "pci_address"); 126 spdk_pci_addr_fmt(addr, sizeof(addr), &pci_addr); 127 spdk_json_write_string(w, addr); 128 } 129 130 static void 131 pci_write_json_config(struct virtio_dev *dev, struct spdk_json_write_ctx *w) 132 { 133 struct virtio_hw *hw = dev->ctx; 134 struct spdk_pci_addr pci_addr = spdk_pci_device_get_addr(hw->pci_dev); 135 char addr[32]; 136 137 spdk_pci_addr_fmt(addr, sizeof(addr), &pci_addr); 138 139 spdk_json_write_named_string(w, "trtype", "pci"); 140 spdk_json_write_named_string(w, "traddr", addr); 141 } 142 143 static inline void 144 io_write64_twopart(uint64_t val, uint32_t *lo, uint32_t *hi) 145 { 146 spdk_mmio_write_4(lo, val & ((1ULL << 32) - 1)); 147 spdk_mmio_write_4(hi, val >> 32); 148 } 149 150 static int 151 modern_read_dev_config(struct virtio_dev *dev, size_t offset, 152 void *dst, int length) 153 { 154 struct virtio_hw *hw = dev->ctx; 155 int i; 156 uint8_t *p; 157 uint8_t old_gen, new_gen; 158 159 do { 160 old_gen = spdk_mmio_read_1(&hw->common_cfg->config_generation); 161 162 p = dst; 163 for (i = 0; i < length; i++) { 164 *p++ = spdk_mmio_read_1((uint8_t *)hw->dev_cfg + offset + i); 165 } 166 167 new_gen = spdk_mmio_read_1(&hw->common_cfg->config_generation); 168 } while (old_gen != new_gen); 169 170 return 0; 171 } 172 173 static int 174 modern_write_dev_config(struct virtio_dev *dev, size_t offset, 175 const void *src, int length) 176 { 177 struct virtio_hw *hw = dev->ctx; 178 int i; 179 const uint8_t *p = src; 180 181 for (i = 0; i < length; i++) { 182 spdk_mmio_write_1(((uint8_t *)hw->dev_cfg) + offset + i, *p++); 183 } 184 185 return 0; 186 } 187 188 static uint64_t 189 modern_get_features(struct virtio_dev *dev) 190 { 191 struct virtio_hw *hw = dev->ctx; 192 uint32_t features_lo, features_hi; 193 194 spdk_mmio_write_4(&hw->common_cfg->device_feature_select, 0); 195 features_lo = spdk_mmio_read_4(&hw->common_cfg->device_feature); 196 197 spdk_mmio_write_4(&hw->common_cfg->device_feature_select, 1); 198 features_hi = spdk_mmio_read_4(&hw->common_cfg->device_feature); 199 200 return ((uint64_t)features_hi << 32) | features_lo; 201 } 202 203 static int 204 modern_set_features(struct virtio_dev *dev, uint64_t features) 205 { 206 struct virtio_hw *hw = dev->ctx; 207 208 if ((features & (1ULL << VIRTIO_F_VERSION_1)) == 0) { 209 SPDK_ERRLOG("VIRTIO_F_VERSION_1 feature is not enabled.\n"); 210 return -EINVAL; 211 } 212 213 spdk_mmio_write_4(&hw->common_cfg->guest_feature_select, 0); 214 spdk_mmio_write_4(&hw->common_cfg->guest_feature, features & ((1ULL << 32) - 1)); 215 216 spdk_mmio_write_4(&hw->common_cfg->guest_feature_select, 1); 217 spdk_mmio_write_4(&hw->common_cfg->guest_feature, features >> 32); 218 219 dev->negotiated_features = features; 220 221 return 0; 222 } 223 224 static void 225 modern_destruct_dev(struct virtio_dev *vdev) 226 { 227 struct virtio_hw *hw = vdev->ctx; 228 struct spdk_pci_device *pci_dev = hw->pci_dev; 229 230 free_virtio_hw(hw); 231 spdk_pci_device_detach(pci_dev); 232 } 233 234 static uint8_t 235 modern_get_status(struct virtio_dev *dev) 236 { 237 struct virtio_hw *hw = dev->ctx; 238 239 return spdk_mmio_read_1(&hw->common_cfg->device_status); 240 } 241 242 static void 243 modern_set_status(struct virtio_dev *dev, uint8_t status) 244 { 245 struct virtio_hw *hw = dev->ctx; 246 247 spdk_mmio_write_1(&hw->common_cfg->device_status, status); 248 } 249 250 static uint16_t 251 modern_get_queue_size(struct virtio_dev *dev, uint16_t queue_id) 252 { 253 struct virtio_hw *hw = dev->ctx; 254 255 spdk_mmio_write_2(&hw->common_cfg->queue_select, queue_id); 256 return spdk_mmio_read_2(&hw->common_cfg->queue_size); 257 } 258 259 static int 260 modern_setup_queue(struct virtio_dev *dev, struct virtqueue *vq) 261 { 262 struct virtio_hw *hw = dev->ctx; 263 uint64_t desc_addr, avail_addr, used_addr; 264 uint16_t notify_off; 265 void *queue_mem; 266 uint64_t queue_mem_phys_addr; 267 268 /* To ensure physical address contiguity we make the queue occupy 269 * only a single hugepage (2MB). As of Virtio 1.0, the queue size 270 * always falls within this limit. 271 */ 272 if (vq->vq_ring_size > VALUE_2MB) { 273 return -ENOMEM; 274 } 275 276 queue_mem = spdk_dma_zmalloc(vq->vq_ring_size, VALUE_2MB, &queue_mem_phys_addr); 277 if (queue_mem == NULL) { 278 return -ENOMEM; 279 } 280 281 vq->vq_ring_mem = queue_mem_phys_addr; 282 vq->vq_ring_virt_mem = queue_mem; 283 284 if (!check_vq_phys_addr_ok(vq)) { 285 spdk_dma_free(queue_mem); 286 return -ENOMEM; 287 } 288 289 desc_addr = vq->vq_ring_mem; 290 avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc); 291 used_addr = (avail_addr + offsetof(struct vring_avail, ring[vq->vq_nentries]) 292 + VIRTIO_PCI_VRING_ALIGN - 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1); 293 294 spdk_mmio_write_2(&hw->common_cfg->queue_select, vq->vq_queue_index); 295 296 io_write64_twopart(desc_addr, &hw->common_cfg->queue_desc_lo, 297 &hw->common_cfg->queue_desc_hi); 298 io_write64_twopart(avail_addr, &hw->common_cfg->queue_avail_lo, 299 &hw->common_cfg->queue_avail_hi); 300 io_write64_twopart(used_addr, &hw->common_cfg->queue_used_lo, 301 &hw->common_cfg->queue_used_hi); 302 303 notify_off = spdk_mmio_read_2(&hw->common_cfg->queue_notify_off); 304 vq->notify_addr = (void *)((uint8_t *)hw->notify_base + 305 notify_off * hw->notify_off_multiplier); 306 307 spdk_mmio_write_2(&hw->common_cfg->queue_enable, 1); 308 309 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "queue %"PRIu16" addresses:\n", vq->vq_queue_index); 310 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "\t desc_addr: %" PRIx64 "\n", desc_addr); 311 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "\t aval_addr: %" PRIx64 "\n", avail_addr); 312 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "\t used_addr: %" PRIx64 "\n", used_addr); 313 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "\t notify addr: %p (notify offset: %"PRIu16")\n", 314 vq->notify_addr, notify_off); 315 316 return 0; 317 } 318 319 static void 320 modern_del_queue(struct virtio_dev *dev, struct virtqueue *vq) 321 { 322 struct virtio_hw *hw = dev->ctx; 323 324 spdk_mmio_write_2(&hw->common_cfg->queue_select, vq->vq_queue_index); 325 326 io_write64_twopart(0, &hw->common_cfg->queue_desc_lo, 327 &hw->common_cfg->queue_desc_hi); 328 io_write64_twopart(0, &hw->common_cfg->queue_avail_lo, 329 &hw->common_cfg->queue_avail_hi); 330 io_write64_twopart(0, &hw->common_cfg->queue_used_lo, 331 &hw->common_cfg->queue_used_hi); 332 333 spdk_mmio_write_2(&hw->common_cfg->queue_enable, 0); 334 335 spdk_dma_free(vq->vq_ring_virt_mem); 336 } 337 338 static void 339 modern_notify_queue(struct virtio_dev *dev, struct virtqueue *vq) 340 { 341 spdk_mmio_write_2(vq->notify_addr, vq->vq_queue_index); 342 } 343 344 static const struct virtio_dev_ops modern_ops = { 345 .read_dev_cfg = modern_read_dev_config, 346 .write_dev_cfg = modern_write_dev_config, 347 .get_status = modern_get_status, 348 .set_status = modern_set_status, 349 .get_features = modern_get_features, 350 .set_features = modern_set_features, 351 .destruct_dev = modern_destruct_dev, 352 .get_queue_size = modern_get_queue_size, 353 .setup_queue = modern_setup_queue, 354 .del_queue = modern_del_queue, 355 .notify_queue = modern_notify_queue, 356 .dump_json_info = pci_dump_json_info, 357 .write_json_config = pci_write_json_config, 358 }; 359 360 static void * 361 get_cfg_addr(struct virtio_hw *hw, struct virtio_pci_cap *cap) 362 { 363 uint8_t bar = cap->bar; 364 uint32_t length = cap->length; 365 uint32_t offset = cap->offset; 366 367 if (bar > 5) { 368 SPDK_ERRLOG("invalid bar: %"PRIu8"\n", bar); 369 return NULL; 370 } 371 372 if (offset + length < offset) { 373 SPDK_ERRLOG("offset(%"PRIu32") + length(%"PRIu32") overflows\n", 374 offset, length); 375 return NULL; 376 } 377 378 if (offset + length > hw->pci_bar[bar].len) { 379 SPDK_ERRLOG("invalid cap: overflows bar space: %"PRIu32" > %"PRIu32"\n", 380 offset + length, hw->pci_bar[bar].len); 381 return NULL; 382 } 383 384 if (hw->pci_bar[bar].vaddr == NULL) { 385 SPDK_ERRLOG("bar %"PRIu8" base addr is NULL\n", bar); 386 return NULL; 387 } 388 389 return hw->pci_bar[bar].vaddr + offset; 390 } 391 392 static int 393 virtio_read_caps(struct virtio_hw *hw) 394 { 395 uint8_t pos; 396 struct virtio_pci_cap cap; 397 int ret; 398 399 ret = spdk_pci_device_cfg_read(hw->pci_dev, &pos, 1, PCI_CAPABILITY_LIST); 400 if (ret < 0) { 401 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "failed to read pci capability list\n"); 402 return ret; 403 } 404 405 while (pos) { 406 ret = spdk_pci_device_cfg_read(hw->pci_dev, &cap, sizeof(cap), pos); 407 if (ret < 0) { 408 SPDK_ERRLOG("failed to read pci cap at pos: %"PRIx8"\n", pos); 409 break; 410 } 411 412 if (cap.cap_vndr == PCI_CAP_ID_MSIX) { 413 hw->use_msix = 1; 414 } 415 416 if (cap.cap_vndr != PCI_CAP_ID_VNDR) { 417 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, 418 "[%2"PRIx8"] skipping non VNDR cap id: %02"PRIx8"\n", 419 pos, cap.cap_vndr); 420 goto next; 421 } 422 423 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, 424 "[%2"PRIx8"] cfg type: %"PRIu8", bar: %"PRIu8", offset: %04"PRIx32", len: %"PRIu32"\n", 425 pos, cap.cfg_type, cap.bar, cap.offset, cap.length); 426 427 switch (cap.cfg_type) { 428 case VIRTIO_PCI_CAP_COMMON_CFG: 429 hw->common_cfg = get_cfg_addr(hw, &cap); 430 break; 431 case VIRTIO_PCI_CAP_NOTIFY_CFG: 432 spdk_pci_device_cfg_read(hw->pci_dev, &hw->notify_off_multiplier, 433 4, pos + sizeof(cap)); 434 hw->notify_base = get_cfg_addr(hw, &cap); 435 break; 436 case VIRTIO_PCI_CAP_DEVICE_CFG: 437 hw->dev_cfg = get_cfg_addr(hw, &cap); 438 break; 439 case VIRTIO_PCI_CAP_ISR_CFG: 440 hw->isr = get_cfg_addr(hw, &cap); 441 break; 442 } 443 444 next: 445 pos = cap.cap_next; 446 } 447 448 if (hw->common_cfg == NULL || hw->notify_base == NULL || 449 hw->dev_cfg == NULL || hw->isr == NULL) { 450 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "no modern virtio pci device found.\n"); 451 if (ret < 0) { 452 return ret; 453 } else { 454 return -EINVAL; 455 } 456 } 457 458 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "found modern virtio pci device.\n"); 459 460 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "common cfg mapped at: %p\n", hw->common_cfg); 461 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "device cfg mapped at: %p\n", hw->dev_cfg); 462 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "isr cfg mapped at: %p\n", hw->isr); 463 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "notify base: %p, notify off multiplier: %u\n", 464 hw->notify_base, hw->notify_off_multiplier); 465 466 return 0; 467 } 468 469 static int 470 virtio_pci_dev_probe(struct spdk_pci_device *pci_dev, struct virtio_pci_probe_ctx *ctx) 471 { 472 struct virtio_hw *hw; 473 uint8_t *bar_vaddr; 474 uint64_t bar_paddr, bar_len; 475 int rc; 476 unsigned i; 477 char bdf[32]; 478 struct spdk_pci_addr addr; 479 480 addr = spdk_pci_device_get_addr(pci_dev); 481 rc = spdk_pci_addr_fmt(bdf, sizeof(bdf), &addr); 482 if (rc != 0) { 483 SPDK_ERRLOG("Ignoring a device with non-parseable PCI address\n"); 484 return -1; 485 } 486 487 hw = calloc(1, sizeof(*hw)); 488 if (hw == NULL) { 489 SPDK_ERRLOG("%s: calloc failed\n", bdf); 490 return -1; 491 } 492 493 hw->pci_dev = pci_dev; 494 495 for (i = 0; i < 6; ++i) { 496 rc = spdk_pci_device_map_bar(pci_dev, i, (void *) &bar_vaddr, &bar_paddr, 497 &bar_len); 498 if (rc != 0) { 499 SPDK_ERRLOG("%s: failed to memmap PCI BAR %u\n", bdf, i); 500 free_virtio_hw(hw); 501 return -1; 502 } 503 504 hw->pci_bar[i].vaddr = bar_vaddr; 505 hw->pci_bar[i].len = bar_len; 506 } 507 508 /* Virtio PCI caps exist only on modern PCI devices. 509 * Legacy devices are not supported. 510 */ 511 if (virtio_read_caps(hw) != 0) { 512 SPDK_NOTICELOG("Ignoring legacy PCI device at %s\n", bdf); 513 free_virtio_hw(hw); 514 return -1; 515 } 516 517 rc = ctx->enum_cb((struct virtio_pci_ctx *)hw, ctx->enum_ctx); 518 if (rc != 0) { 519 free_virtio_hw(hw); 520 } 521 522 return rc; 523 } 524 525 static int 526 virtio_pci_dev_probe_cb(void *probe_ctx, struct spdk_pci_device *pci_dev) 527 { 528 struct virtio_pci_probe_ctx *ctx = probe_ctx; 529 uint16_t pci_device_id = spdk_pci_device_get_device_id(pci_dev); 530 531 if (pci_device_id != ctx->device_id) { 532 return 1; 533 } 534 535 return virtio_pci_dev_probe(pci_dev, ctx); 536 } 537 538 int 539 virtio_pci_dev_enumerate(virtio_pci_create_cb enum_cb, void *enum_ctx, 540 uint16_t pci_device_id) 541 { 542 struct virtio_pci_probe_ctx ctx; 543 544 if (!spdk_process_is_primary()) { 545 SPDK_WARNLOG("virtio_pci secondary process support is not implemented yet.\n"); 546 return 0; 547 } 548 549 ctx.enum_cb = enum_cb; 550 ctx.enum_ctx = enum_ctx; 551 ctx.device_id = pci_device_id; 552 553 return spdk_pci_enumerate(spdk_pci_virtio_get_driver(), 554 virtio_pci_dev_probe_cb, &ctx); 555 } 556 557 int 558 virtio_pci_dev_attach(virtio_pci_create_cb enum_cb, void *enum_ctx, 559 uint16_t pci_device_id, struct spdk_pci_addr *pci_address) 560 { 561 struct virtio_pci_probe_ctx ctx; 562 563 if (!spdk_process_is_primary()) { 564 SPDK_WARNLOG("virtio_pci secondary process support is not implemented yet.\n"); 565 return 0; 566 } 567 568 ctx.enum_cb = enum_cb; 569 ctx.enum_ctx = enum_ctx; 570 ctx.device_id = pci_device_id; 571 572 return spdk_pci_device_attach(spdk_pci_virtio_get_driver(), 573 virtio_pci_dev_probe_cb, &ctx, pci_address); 574 } 575 576 int 577 virtio_pci_dev_init(struct virtio_dev *vdev, const char *name, 578 struct virtio_pci_ctx *pci_ctx) 579 { 580 int rc; 581 582 rc = virtio_dev_construct(vdev, name, &modern_ops, pci_ctx); 583 if (rc != 0) { 584 return rc; 585 } 586 587 vdev->is_hw = 1; 588 vdev->modern = 1; 589 590 return 0; 591 } 592 593 SPDK_LOG_REGISTER_COMPONENT("virtio_pci", SPDK_LOG_VIRTIO_PCI) 594