1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/mmio.h" 37 #include "spdk/string.h" 38 #include "spdk/env.h" 39 40 #include "spdk_internal/virtio.h" 41 42 struct virtio_hw { 43 uint8_t use_msix; 44 uint32_t notify_off_multiplier; 45 uint8_t *isr; 46 uint16_t *notify_base; 47 48 struct { 49 /** Mem-mapped resources from given PCI BAR */ 50 void *vaddr; 51 52 /** Length of the address space */ 53 uint32_t len; 54 } pci_bar[6]; 55 56 struct virtio_pci_common_cfg *common_cfg; 57 struct spdk_pci_device *pci_dev; 58 59 /** Device-specific PCI config space */ 60 void *dev_cfg; 61 }; 62 63 struct virtio_pci_probe_ctx { 64 virtio_pci_create_cb enum_cb; 65 void *enum_ctx; 66 uint16_t device_id; 67 }; 68 69 /* 70 * Following macros are derived from linux/pci_regs.h, however, 71 * we can't simply include that header here, as there is no such 72 * file for non-Linux platform. 73 */ 74 #define PCI_CAPABILITY_LIST 0x34 75 #define PCI_CAP_ID_VNDR 0x09 76 #define PCI_CAP_ID_MSIX 0x11 77 78 static inline int 79 check_vq_phys_addr_ok(struct virtqueue *vq) 80 { 81 /* Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit, 82 * and only accepts 32 bit page frame number. 83 * Check if the allocated physical memory exceeds 16TB. 84 */ 85 if ((vq->vq_ring_mem + vq->vq_ring_size - 1) >> 86 (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) { 87 SPDK_ERRLOG("vring address shouldn't be above 16TB!\n"); 88 return 0; 89 } 90 91 return 1; 92 } 93 94 static void 95 free_virtio_hw(struct virtio_hw *hw) 96 { 97 unsigned i; 98 99 for (i = 0; i < 6; ++i) { 100 if (hw->pci_bar[i].vaddr == NULL) { 101 continue; 102 } 103 104 spdk_pci_device_unmap_bar(hw->pci_dev, i, hw->pci_bar[i].vaddr); 105 } 106 107 free(hw); 108 } 109 110 static void 111 pci_dump_json_config(struct virtio_dev *dev, struct spdk_json_write_ctx *w) 112 { 113 struct virtio_hw *hw = dev->ctx; 114 struct spdk_pci_addr pci_addr = spdk_pci_device_get_addr((struct spdk_pci_device *)hw->pci_dev); 115 char addr[32]; 116 117 spdk_json_write_name(w, "type"); 118 if (dev->modern) { 119 spdk_json_write_string(w, "pci-modern"); 120 } else { 121 spdk_json_write_string(w, "pci-legacy"); 122 } 123 124 spdk_json_write_name(w, "pci_address"); 125 spdk_pci_addr_fmt(addr, sizeof(addr), &pci_addr); 126 spdk_json_write_string(w, addr); 127 } 128 129 static inline void 130 io_write64_twopart(uint64_t val, uint32_t *lo, uint32_t *hi) 131 { 132 spdk_mmio_write_4(lo, val & ((1ULL << 32) - 1)); 133 spdk_mmio_write_4(hi, val >> 32); 134 } 135 136 static void 137 modern_read_dev_config(struct virtio_dev *dev, size_t offset, 138 void *dst, int length) 139 { 140 struct virtio_hw *hw = dev->ctx; 141 int i; 142 uint8_t *p; 143 uint8_t old_gen, new_gen; 144 145 do { 146 old_gen = spdk_mmio_read_1(&hw->common_cfg->config_generation); 147 148 p = dst; 149 for (i = 0; i < length; i++) { 150 *p++ = spdk_mmio_read_1((uint8_t *)hw->dev_cfg + offset + i); 151 } 152 153 new_gen = spdk_mmio_read_1(&hw->common_cfg->config_generation); 154 } while (old_gen != new_gen); 155 } 156 157 static void 158 modern_write_dev_config(struct virtio_dev *dev, size_t offset, 159 const void *src, int length) 160 { 161 struct virtio_hw *hw = dev->ctx; 162 int i; 163 const uint8_t *p = src; 164 165 for (i = 0; i < length; i++) { 166 spdk_mmio_write_1(((uint8_t *)hw->dev_cfg) + offset + i, *p++); 167 } 168 } 169 170 static uint64_t 171 modern_get_features(struct virtio_dev *dev) 172 { 173 struct virtio_hw *hw = dev->ctx; 174 uint32_t features_lo, features_hi; 175 176 spdk_mmio_write_4(&hw->common_cfg->device_feature_select, 0); 177 features_lo = spdk_mmio_read_4(&hw->common_cfg->device_feature); 178 179 spdk_mmio_write_4(&hw->common_cfg->device_feature_select, 1); 180 features_hi = spdk_mmio_read_4(&hw->common_cfg->device_feature); 181 182 return ((uint64_t)features_hi << 32) | features_lo; 183 } 184 185 static int 186 modern_set_features(struct virtio_dev *dev, uint64_t features) 187 { 188 struct virtio_hw *hw = dev->ctx; 189 190 if ((features & (1ULL << VIRTIO_F_VERSION_1)) == 0) { 191 SPDK_ERRLOG("VIRTIO_F_VERSION_1 feature is not enabled.\n"); 192 return -1; 193 } 194 195 spdk_mmio_write_4(&hw->common_cfg->guest_feature_select, 0); 196 spdk_mmio_write_4(&hw->common_cfg->guest_feature, features & ((1ULL << 32) - 1)); 197 198 spdk_mmio_write_4(&hw->common_cfg->guest_feature_select, 1); 199 spdk_mmio_write_4(&hw->common_cfg->guest_feature, features >> 32); 200 201 dev->negotiated_features = features; 202 203 return 0; 204 } 205 206 static void 207 modern_destruct_dev(struct virtio_dev *vdev) 208 { 209 struct virtio_hw *hw = vdev->ctx; 210 struct spdk_pci_device *pci_dev = hw->pci_dev; 211 212 free_virtio_hw(hw); 213 spdk_pci_device_detach(pci_dev); 214 } 215 216 static uint8_t 217 modern_get_status(struct virtio_dev *dev) 218 { 219 struct virtio_hw *hw = dev->ctx; 220 221 return spdk_mmio_read_1(&hw->common_cfg->device_status); 222 } 223 224 static void 225 modern_set_status(struct virtio_dev *dev, uint8_t status) 226 { 227 struct virtio_hw *hw = dev->ctx; 228 229 spdk_mmio_write_1(&hw->common_cfg->device_status, status); 230 } 231 232 static uint16_t 233 modern_get_queue_num(struct virtio_dev *dev, uint16_t queue_id) 234 { 235 struct virtio_hw *hw = dev->ctx; 236 237 spdk_mmio_write_2(&hw->common_cfg->queue_select, queue_id); 238 return spdk_mmio_read_2(&hw->common_cfg->queue_size); 239 } 240 241 static int 242 modern_setup_queue(struct virtio_dev *dev, struct virtqueue *vq) 243 { 244 struct virtio_hw *hw = dev->ctx; 245 uint64_t desc_addr, avail_addr, used_addr; 246 uint16_t notify_off; 247 248 if (!check_vq_phys_addr_ok(vq)) { 249 return -1; 250 } 251 252 desc_addr = vq->vq_ring_mem; 253 avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc); 254 used_addr = (avail_addr + offsetof(struct vring_avail, ring[vq->vq_nentries]) 255 + VIRTIO_PCI_VRING_ALIGN - 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1); 256 257 spdk_mmio_write_2(&hw->common_cfg->queue_select, vq->vq_queue_index); 258 259 io_write64_twopart(desc_addr, &hw->common_cfg->queue_desc_lo, 260 &hw->common_cfg->queue_desc_hi); 261 io_write64_twopart(avail_addr, &hw->common_cfg->queue_avail_lo, 262 &hw->common_cfg->queue_avail_hi); 263 io_write64_twopart(used_addr, &hw->common_cfg->queue_used_lo, 264 &hw->common_cfg->queue_used_hi); 265 266 notify_off = spdk_mmio_read_2(&hw->common_cfg->queue_notify_off); 267 vq->notify_addr = (void *)((uint8_t *)hw->notify_base + 268 notify_off * hw->notify_off_multiplier); 269 270 spdk_mmio_write_2(&hw->common_cfg->queue_enable, 1); 271 272 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "queue %"PRIu16" addresses:\n", vq->vq_queue_index); 273 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "\t desc_addr: %" PRIx64 "\n", desc_addr); 274 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "\t aval_addr: %" PRIx64 "\n", avail_addr); 275 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "\t used_addr: %" PRIx64 "\n", used_addr); 276 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "\t notify addr: %p (notify offset: %"PRIu16")\n", 277 vq->notify_addr, notify_off); 278 279 return 0; 280 } 281 282 static void 283 modern_del_queue(struct virtio_dev *dev, struct virtqueue *vq) 284 { 285 struct virtio_hw *hw = dev->ctx; 286 287 spdk_mmio_write_2(&hw->common_cfg->queue_select, vq->vq_queue_index); 288 289 io_write64_twopart(0, &hw->common_cfg->queue_desc_lo, 290 &hw->common_cfg->queue_desc_hi); 291 io_write64_twopart(0, &hw->common_cfg->queue_avail_lo, 292 &hw->common_cfg->queue_avail_hi); 293 io_write64_twopart(0, &hw->common_cfg->queue_used_lo, 294 &hw->common_cfg->queue_used_hi); 295 296 spdk_mmio_write_2(&hw->common_cfg->queue_enable, 0); 297 } 298 299 static void 300 modern_notify_queue(struct virtio_dev *dev, struct virtqueue *vq) 301 { 302 spdk_mmio_write_2(vq->notify_addr, vq->vq_queue_index); 303 } 304 305 static const struct virtio_dev_ops modern_ops = { 306 .read_dev_cfg = modern_read_dev_config, 307 .write_dev_cfg = modern_write_dev_config, 308 .get_status = modern_get_status, 309 .set_status = modern_set_status, 310 .get_features = modern_get_features, 311 .set_features = modern_set_features, 312 .destruct_dev = modern_destruct_dev, 313 .get_queue_num = modern_get_queue_num, 314 .setup_queue = modern_setup_queue, 315 .del_queue = modern_del_queue, 316 .notify_queue = modern_notify_queue, 317 .dump_json_config = pci_dump_json_config, 318 }; 319 320 static void * 321 get_cfg_addr(struct virtio_hw *hw, struct virtio_pci_cap *cap) 322 { 323 uint8_t bar = cap->bar; 324 uint32_t length = cap->length; 325 uint32_t offset = cap->offset; 326 327 if (bar > 5) { 328 SPDK_ERRLOG("invalid bar: %"PRIu8"\n", bar); 329 return NULL; 330 } 331 332 if (offset + length < offset) { 333 SPDK_ERRLOG("offset(%"PRIu32") + length(%"PRIu32") overflows\n", 334 offset, length); 335 return NULL; 336 } 337 338 if (offset + length > hw->pci_bar[bar].len) { 339 SPDK_ERRLOG("invalid cap: overflows bar space: %"PRIu32" > %"PRIu32"\n", 340 offset + length, hw->pci_bar[bar].len); 341 return NULL; 342 } 343 344 if (hw->pci_bar[bar].vaddr == NULL) { 345 SPDK_ERRLOG("bar %"PRIu8" base addr is NULL\n", bar); 346 return NULL; 347 } 348 349 return hw->pci_bar[bar].vaddr + offset; 350 } 351 352 static int 353 virtio_read_caps(struct virtio_hw *hw) 354 { 355 uint8_t pos; 356 struct virtio_pci_cap cap; 357 int ret; 358 359 ret = spdk_pci_device_cfg_read(hw->pci_dev, &pos, 1, PCI_CAPABILITY_LIST); 360 if (ret < 0) { 361 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "failed to read pci capability list\n"); 362 return -1; 363 } 364 365 while (pos) { 366 ret = spdk_pci_device_cfg_read(hw->pci_dev, &cap, sizeof(cap), pos); 367 if (ret < 0) { 368 SPDK_ERRLOG("failed to read pci cap at pos: %"PRIx8"\n", pos); 369 break; 370 } 371 372 if (cap.cap_vndr == PCI_CAP_ID_MSIX) { 373 hw->use_msix = 1; 374 } 375 376 if (cap.cap_vndr != PCI_CAP_ID_VNDR) { 377 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, 378 "[%2"PRIx8"] skipping non VNDR cap id: %02"PRIx8"\n", 379 pos, cap.cap_vndr); 380 goto next; 381 } 382 383 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, 384 "[%2"PRIx8"] cfg type: %"PRIu8", bar: %"PRIu8", offset: %04"PRIx32", len: %"PRIu32"\n", 385 pos, cap.cfg_type, cap.bar, cap.offset, cap.length); 386 387 switch (cap.cfg_type) { 388 case VIRTIO_PCI_CAP_COMMON_CFG: 389 hw->common_cfg = get_cfg_addr(hw, &cap); 390 break; 391 case VIRTIO_PCI_CAP_NOTIFY_CFG: 392 spdk_pci_device_cfg_read(hw->pci_dev, &hw->notify_off_multiplier, 393 4, pos + sizeof(cap)); 394 hw->notify_base = get_cfg_addr(hw, &cap); 395 break; 396 case VIRTIO_PCI_CAP_DEVICE_CFG: 397 hw->dev_cfg = get_cfg_addr(hw, &cap); 398 break; 399 case VIRTIO_PCI_CAP_ISR_CFG: 400 hw->isr = get_cfg_addr(hw, &cap); 401 break; 402 } 403 404 next: 405 pos = cap.cap_next; 406 } 407 408 if (hw->common_cfg == NULL || hw->notify_base == NULL || 409 hw->dev_cfg == NULL || hw->isr == NULL) { 410 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "no modern virtio pci device found.\n"); 411 return -1; 412 } 413 414 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "found modern virtio pci device.\n"); 415 416 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "common cfg mapped at: %p\n", hw->common_cfg); 417 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "device cfg mapped at: %p\n", hw->dev_cfg); 418 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "isr cfg mapped at: %p\n", hw->isr); 419 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "notify base: %p, notify off multiplier: %u\n", 420 hw->notify_base, hw->notify_off_multiplier); 421 422 return 0; 423 } 424 425 static int 426 virtio_pci_dev_probe(struct spdk_pci_device *pci_dev, struct virtio_pci_probe_ctx *ctx) 427 { 428 struct virtio_hw *hw; 429 uint8_t *bar_vaddr; 430 uint64_t bar_paddr, bar_len; 431 int rc; 432 unsigned i; 433 char bdf[32]; 434 struct spdk_pci_addr addr; 435 436 addr = spdk_pci_device_get_addr(pci_dev); 437 rc = spdk_pci_addr_fmt(bdf, sizeof(bdf), &addr); 438 if (rc != 0) { 439 SPDK_ERRLOG("Ignoring a device with non-parseable PCI address\n"); 440 return -1; 441 } 442 443 hw = calloc(1, sizeof(*hw)); 444 if (hw == NULL) { 445 SPDK_ERRLOG("%s: calloc failed\n", bdf); 446 return -1; 447 } 448 449 hw->pci_dev = pci_dev; 450 451 for (i = 0; i < 6; ++i) { 452 rc = spdk_pci_device_map_bar(pci_dev, i, (void *) &bar_vaddr, &bar_paddr, 453 &bar_len); 454 if (rc != 0) { 455 SPDK_ERRLOG("%s: failed to memmap PCI BAR %u\n", bdf, i); 456 free_virtio_hw(hw); 457 return -1; 458 } 459 460 hw->pci_bar[i].vaddr = bar_vaddr; 461 hw->pci_bar[i].len = bar_len; 462 } 463 464 /* Virtio PCI caps exist only on modern PCI devices. 465 * Legacy devices are not supported. 466 */ 467 if (virtio_read_caps(hw) != 0) { 468 SPDK_NOTICELOG("Ignoring legacy PCI device at %s\n", bdf); 469 free_virtio_hw(hw); 470 return -1; 471 } 472 473 rc = ctx->enum_cb((struct virtio_pci_ctx *)hw, ctx->enum_ctx); 474 if (rc != 0) { 475 free_virtio_hw(hw); 476 } 477 478 return rc; 479 } 480 481 static int 482 virtio_pci_dev_probe_cb(void *probe_ctx, struct spdk_pci_device *pci_dev) 483 { 484 struct virtio_pci_probe_ctx *ctx = probe_ctx; 485 uint16_t pci_device_id = spdk_pci_device_get_device_id(pci_dev); 486 487 if (pci_device_id != ctx->device_id) { 488 return 1; 489 } 490 491 return virtio_pci_dev_probe(pci_dev, ctx); 492 } 493 494 int 495 virtio_pci_dev_enumerate(virtio_pci_create_cb enum_cb, void *enum_ctx, 496 uint16_t pci_device_id) 497 { 498 struct virtio_pci_probe_ctx ctx; 499 500 if (!spdk_process_is_primary()) { 501 SPDK_WARNLOG("virtio_pci secondary process support is not implemented yet.\n"); 502 return 0; 503 } 504 505 ctx.enum_cb = enum_cb; 506 ctx.enum_ctx = enum_ctx; 507 ctx.device_id = pci_device_id; 508 509 return spdk_pci_virtio_enumerate(virtio_pci_dev_probe_cb, &ctx); 510 } 511 512 int 513 virtio_pci_dev_attach(virtio_pci_create_cb enum_cb, void *enum_ctx, 514 uint16_t pci_device_id, struct spdk_pci_addr *pci_address) 515 { 516 struct virtio_pci_probe_ctx ctx; 517 518 if (!spdk_process_is_primary()) { 519 SPDK_WARNLOG("virtio_pci secondary process support is not implemented yet.\n"); 520 return 0; 521 } 522 523 ctx.enum_cb = enum_cb; 524 ctx.enum_ctx = enum_ctx; 525 ctx.device_id = pci_device_id; 526 527 return spdk_pci_virtio_device_attach(virtio_pci_dev_probe_cb, &ctx, pci_address); 528 } 529 530 int 531 virtio_pci_dev_init(struct virtio_dev *vdev, const char *name, 532 struct virtio_pci_ctx *pci_ctx) 533 { 534 int rc; 535 536 rc = virtio_dev_construct(vdev, name, &modern_ops, pci_ctx); 537 if (rc != 0) { 538 return -1; 539 } 540 541 vdev->is_hw = 1; 542 vdev->modern = 1; 543 544 return 0; 545 } 546 547 SPDK_LOG_REGISTER_COMPONENT("virtio_pci", SPDK_LOG_VIRTIO_PCI) 548