1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2010-2014 Intel Corporation. All rights reserved. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "spdk/mmio.h" 37 #include "spdk/string.h" 38 #include "spdk/env.h" 39 #include "spdk/pci_ids.h" 40 41 #include "spdk_internal/virtio.h" 42 43 struct virtio_hw { 44 uint8_t use_msix; 45 uint32_t notify_off_multiplier; 46 uint8_t *isr; 47 uint16_t *notify_base; 48 49 struct { 50 /** Mem-mapped resources from given PCI BAR */ 51 void *vaddr; 52 53 /** Length of the address space */ 54 uint32_t len; 55 } pci_bar[6]; 56 57 struct virtio_pci_common_cfg *common_cfg; 58 struct spdk_pci_device *pci_dev; 59 60 /** Device-specific PCI config space */ 61 void *dev_cfg; 62 }; 63 64 /* 65 * Following macros are derived from linux/pci_regs.h, however, 66 * we can't simply include that header here, as there is no such 67 * file for non-Linux platform. 68 */ 69 #define PCI_CAPABILITY_LIST 0x34 70 #define PCI_CAP_ID_VNDR 0x09 71 #define PCI_CAP_ID_MSIX 0x11 72 73 static inline int 74 check_vq_phys_addr_ok(struct virtqueue *vq) 75 { 76 /* Virtio PCI device VIRTIO_PCI_QUEUE_PF register is 32bit, 77 * and only accepts 32 bit page frame number. 78 * Check if the allocated physical memory exceeds 16TB. 79 */ 80 if ((vq->vq_ring_mem + vq->vq_ring_size - 1) >> 81 (VIRTIO_PCI_QUEUE_ADDR_SHIFT + 32)) { 82 SPDK_ERRLOG("vring address shouldn't be above 16TB!\n"); 83 return 0; 84 } 85 86 return 1; 87 } 88 89 static void 90 free_virtio_hw(struct virtio_hw *hw) 91 { 92 unsigned i; 93 94 for (i = 0; i < 6; ++i) { 95 if (hw->pci_bar[i].vaddr == NULL) { 96 continue; 97 } 98 99 spdk_pci_device_unmap_bar(hw->pci_dev, i, hw->pci_bar[i].vaddr); 100 } 101 102 free(hw); 103 } 104 105 static void 106 pci_dump_json_config(struct virtio_dev *dev, struct spdk_json_write_ctx *w) 107 { 108 struct virtio_hw *hw = dev->ctx; 109 struct spdk_pci_addr pci_addr = spdk_pci_device_get_addr((struct spdk_pci_device *)hw->pci_dev); 110 char addr[32]; 111 112 spdk_json_write_name(w, "type"); 113 if (dev->modern) { 114 spdk_json_write_string(w, "pci-modern"); 115 } else { 116 spdk_json_write_string(w, "pci-legacy"); 117 } 118 119 spdk_json_write_name(w, "pci_address"); 120 spdk_pci_addr_fmt(addr, sizeof(addr), &pci_addr); 121 spdk_json_write_string(w, addr); 122 } 123 124 static inline void 125 io_write64_twopart(uint64_t val, uint32_t *lo, uint32_t *hi) 126 { 127 spdk_mmio_write_4(lo, val & ((1ULL << 32) - 1)); 128 spdk_mmio_write_4(hi, val >> 32); 129 } 130 131 static void 132 modern_read_dev_config(struct virtio_dev *dev, size_t offset, 133 void *dst, int length) 134 { 135 struct virtio_hw *hw = dev->ctx; 136 int i; 137 uint8_t *p; 138 uint8_t old_gen, new_gen; 139 140 do { 141 old_gen = spdk_mmio_read_1(&hw->common_cfg->config_generation); 142 143 p = dst; 144 for (i = 0; i < length; i++) { 145 *p++ = spdk_mmio_read_1((uint8_t *)hw->dev_cfg + offset + i); 146 } 147 148 new_gen = spdk_mmio_read_1(&hw->common_cfg->config_generation); 149 } while (old_gen != new_gen); 150 } 151 152 static void 153 modern_write_dev_config(struct virtio_dev *dev, size_t offset, 154 const void *src, int length) 155 { 156 struct virtio_hw *hw = dev->ctx; 157 int i; 158 const uint8_t *p = src; 159 160 for (i = 0; i < length; i++) { 161 spdk_mmio_write_1(((uint8_t *)hw->dev_cfg) + offset + i, *p++); 162 } 163 } 164 165 static uint64_t 166 modern_get_features(struct virtio_dev *dev) 167 { 168 struct virtio_hw *hw = dev->ctx; 169 uint32_t features_lo, features_hi; 170 171 spdk_mmio_write_4(&hw->common_cfg->device_feature_select, 0); 172 features_lo = spdk_mmio_read_4(&hw->common_cfg->device_feature); 173 174 spdk_mmio_write_4(&hw->common_cfg->device_feature_select, 1); 175 features_hi = spdk_mmio_read_4(&hw->common_cfg->device_feature); 176 177 return ((uint64_t)features_hi << 32) | features_lo; 178 } 179 180 static int 181 modern_set_features(struct virtio_dev *dev, uint64_t features) 182 { 183 struct virtio_hw *hw = dev->ctx; 184 185 if ((features & (1ULL << VIRTIO_F_VERSION_1)) == 0) { 186 SPDK_ERRLOG("VIRTIO_F_VERSION_1 feature is not enabled.\n"); 187 return -1; 188 } 189 190 spdk_mmio_write_4(&hw->common_cfg->guest_feature_select, 0); 191 spdk_mmio_write_4(&hw->common_cfg->guest_feature, features & ((1ULL << 32) - 1)); 192 193 spdk_mmio_write_4(&hw->common_cfg->guest_feature_select, 1); 194 spdk_mmio_write_4(&hw->common_cfg->guest_feature, features >> 32); 195 196 dev->negotiated_features = features; 197 198 return 0; 199 } 200 201 static void 202 modern_destruct_dev(struct virtio_dev *vdev) 203 { 204 struct virtio_hw *hw = vdev->ctx; 205 206 free_virtio_hw(hw); 207 free(vdev->name); 208 } 209 210 static uint8_t 211 modern_get_status(struct virtio_dev *dev) 212 { 213 struct virtio_hw *hw = dev->ctx; 214 215 return spdk_mmio_read_1(&hw->common_cfg->device_status); 216 } 217 218 static void 219 modern_set_status(struct virtio_dev *dev, uint8_t status) 220 { 221 struct virtio_hw *hw = dev->ctx; 222 223 spdk_mmio_write_1(&hw->common_cfg->device_status, status); 224 } 225 226 static uint16_t 227 modern_get_queue_num(struct virtio_dev *dev, uint16_t queue_id) 228 { 229 struct virtio_hw *hw = dev->ctx; 230 231 spdk_mmio_write_2(&hw->common_cfg->queue_select, queue_id); 232 return spdk_mmio_read_2(&hw->common_cfg->queue_size); 233 } 234 235 static int 236 modern_setup_queue(struct virtio_dev *dev, struct virtqueue *vq) 237 { 238 struct virtio_hw *hw = dev->ctx; 239 uint64_t desc_addr, avail_addr, used_addr; 240 uint16_t notify_off; 241 242 if (!check_vq_phys_addr_ok(vq)) { 243 return -1; 244 } 245 246 desc_addr = vq->vq_ring_mem; 247 avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc); 248 used_addr = (avail_addr + offsetof(struct vring_avail, ring[vq->vq_nentries]) 249 + VIRTIO_PCI_VRING_ALIGN - 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1); 250 251 spdk_mmio_write_2(&hw->common_cfg->queue_select, vq->vq_queue_index); 252 253 io_write64_twopart(desc_addr, &hw->common_cfg->queue_desc_lo, 254 &hw->common_cfg->queue_desc_hi); 255 io_write64_twopart(avail_addr, &hw->common_cfg->queue_avail_lo, 256 &hw->common_cfg->queue_avail_hi); 257 io_write64_twopart(used_addr, &hw->common_cfg->queue_used_lo, 258 &hw->common_cfg->queue_used_hi); 259 260 notify_off = spdk_mmio_read_2(&hw->common_cfg->queue_notify_off); 261 vq->notify_addr = (void *)((uint8_t *)hw->notify_base + 262 notify_off * hw->notify_off_multiplier); 263 264 spdk_mmio_write_2(&hw->common_cfg->queue_enable, 1); 265 266 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "queue %"PRIu16" addresses:\n", vq->vq_queue_index); 267 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "\t desc_addr: %" PRIx64 "\n", desc_addr); 268 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "\t aval_addr: %" PRIx64 "\n", avail_addr); 269 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "\t used_addr: %" PRIx64 "\n", used_addr); 270 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "\t notify addr: %p (notify offset: %"PRIu16")\n", 271 vq->notify_addr, notify_off); 272 273 return 0; 274 } 275 276 static void 277 modern_del_queue(struct virtio_dev *dev, struct virtqueue *vq) 278 { 279 struct virtio_hw *hw = dev->ctx; 280 281 spdk_mmio_write_2(&hw->common_cfg->queue_select, vq->vq_queue_index); 282 283 io_write64_twopart(0, &hw->common_cfg->queue_desc_lo, 284 &hw->common_cfg->queue_desc_hi); 285 io_write64_twopart(0, &hw->common_cfg->queue_avail_lo, 286 &hw->common_cfg->queue_avail_hi); 287 io_write64_twopart(0, &hw->common_cfg->queue_used_lo, 288 &hw->common_cfg->queue_used_hi); 289 290 spdk_mmio_write_2(&hw->common_cfg->queue_enable, 0); 291 } 292 293 static void 294 modern_notify_queue(struct virtio_dev *dev, struct virtqueue *vq) 295 { 296 spdk_mmio_write_2(vq->notify_addr, vq->vq_queue_index); 297 } 298 299 static const struct virtio_dev_ops modern_ops = { 300 .read_dev_cfg = modern_read_dev_config, 301 .write_dev_cfg = modern_write_dev_config, 302 .get_status = modern_get_status, 303 .set_status = modern_set_status, 304 .get_features = modern_get_features, 305 .set_features = modern_set_features, 306 .destruct_dev = modern_destruct_dev, 307 .get_queue_num = modern_get_queue_num, 308 .setup_queue = modern_setup_queue, 309 .del_queue = modern_del_queue, 310 .notify_queue = modern_notify_queue, 311 .dump_json_config = pci_dump_json_config, 312 }; 313 314 static void * 315 get_cfg_addr(struct virtio_hw *hw, struct virtio_pci_cap *cap) 316 { 317 uint8_t bar = cap->bar; 318 uint32_t length = cap->length; 319 uint32_t offset = cap->offset; 320 321 if (bar > 5) { 322 SPDK_ERRLOG("invalid bar: %"PRIu8"\n", bar); 323 return NULL; 324 } 325 326 if (offset + length < offset) { 327 SPDK_ERRLOG("offset(%"PRIu32") + length(%"PRIu32") overflows\n", 328 offset, length); 329 return NULL; 330 } 331 332 if (offset + length > hw->pci_bar[bar].len) { 333 SPDK_ERRLOG("invalid cap: overflows bar space: %"PRIu32" > %"PRIu32"\n", 334 offset + length, hw->pci_bar[bar].len); 335 return NULL; 336 } 337 338 if (hw->pci_bar[bar].vaddr == NULL) { 339 SPDK_ERRLOG("bar %"PRIu8" base addr is NULL\n", bar); 340 return NULL; 341 } 342 343 return hw->pci_bar[bar].vaddr + offset; 344 } 345 346 static int 347 virtio_read_caps(struct virtio_hw *hw) 348 { 349 uint8_t pos; 350 struct virtio_pci_cap cap; 351 int ret; 352 353 ret = spdk_pci_device_cfg_read(hw->pci_dev, &pos, 1, PCI_CAPABILITY_LIST); 354 if (ret < 0) { 355 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "failed to read pci capability list\n"); 356 return -1; 357 } 358 359 while (pos) { 360 ret = spdk_pci_device_cfg_read(hw->pci_dev, &cap, sizeof(cap), pos); 361 if (ret < 0) { 362 SPDK_ERRLOG("failed to read pci cap at pos: %"PRIx8"\n", pos); 363 break; 364 } 365 366 if (cap.cap_vndr == PCI_CAP_ID_MSIX) { 367 hw->use_msix = 1; 368 } 369 370 if (cap.cap_vndr != PCI_CAP_ID_VNDR) { 371 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, 372 "[%2"PRIx8"] skipping non VNDR cap id: %02"PRIx8"\n", 373 pos, cap.cap_vndr); 374 goto next; 375 } 376 377 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, 378 "[%2"PRIx8"] cfg type: %"PRIu8", bar: %"PRIu8", offset: %04"PRIx32", len: %"PRIu32"\n", 379 pos, cap.cfg_type, cap.bar, cap.offset, cap.length); 380 381 switch (cap.cfg_type) { 382 case VIRTIO_PCI_CAP_COMMON_CFG: 383 hw->common_cfg = get_cfg_addr(hw, &cap); 384 break; 385 case VIRTIO_PCI_CAP_NOTIFY_CFG: 386 spdk_pci_device_cfg_read(hw->pci_dev, &hw->notify_off_multiplier, 387 4, pos + sizeof(cap)); 388 hw->notify_base = get_cfg_addr(hw, &cap); 389 break; 390 case VIRTIO_PCI_CAP_DEVICE_CFG: 391 hw->dev_cfg = get_cfg_addr(hw, &cap); 392 break; 393 case VIRTIO_PCI_CAP_ISR_CFG: 394 hw->isr = get_cfg_addr(hw, &cap); 395 break; 396 } 397 398 next: 399 pos = cap.cap_next; 400 } 401 402 if (hw->common_cfg == NULL || hw->notify_base == NULL || 403 hw->dev_cfg == NULL || hw->isr == NULL) { 404 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "no modern virtio pci device found.\n"); 405 return -1; 406 } 407 408 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "found modern virtio pci device.\n"); 409 410 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "common cfg mapped at: %p\n", hw->common_cfg); 411 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "device cfg mapped at: %p\n", hw->dev_cfg); 412 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "isr cfg mapped at: %p\n", hw->isr); 413 SPDK_DEBUGLOG(SPDK_LOG_VIRTIO_PCI, "notify base: %p, notify off multiplier: %u\n", 414 hw->notify_base, hw->notify_off_multiplier); 415 416 return 0; 417 } 418 419 static int 420 virtio_pci_dev_probe(struct spdk_pci_device *pci_dev, virtio_pci_create_cb enum_cb) 421 { 422 struct virtio_hw *hw; 423 uint8_t *bar_vaddr; 424 uint64_t bar_paddr, bar_len; 425 int rc; 426 unsigned i; 427 428 hw = calloc(1, sizeof(*hw)); 429 if (hw == NULL) { 430 SPDK_ERRLOG("calloc failed\n"); 431 return -1; 432 } 433 434 hw->pci_dev = pci_dev; 435 436 for (i = 0; i < 6; ++i) { 437 rc = spdk_pci_device_map_bar(pci_dev, i, (void *) &bar_vaddr, &bar_paddr, 438 &bar_len); 439 if (rc != 0) { 440 SPDK_ERRLOG("failed to memmap PCI BAR %u\n", i); 441 free_virtio_hw(hw); 442 return -1; 443 } 444 445 hw->pci_bar[i].vaddr = bar_vaddr; 446 hw->pci_bar[i].len = bar_len; 447 } 448 449 /* Virtio PCI caps exist only on modern PCI devices. 450 * Legacy devices are not supported. 451 */ 452 if (virtio_read_caps(hw) != 0) { 453 SPDK_NOTICELOG("Ignoring legacy PCI device.\n"); 454 free_virtio_hw(hw); 455 return -1; 456 } 457 458 rc = enum_cb((struct virtio_pci_ctx *)hw); 459 if (rc != 0) { 460 free_virtio_hw(hw); 461 } 462 463 return rc; 464 } 465 466 static int 467 virtio_pci_scsi_dev_probe_cb(void *ctx, struct spdk_pci_device *pci_dev) 468 { 469 virtio_pci_create_cb enum_cb = ctx; 470 uint16_t pci_device_id = spdk_pci_device_get_device_id(pci_dev); 471 472 if (pci_device_id != PCI_DEVICE_ID_VIRTIO_SCSI_MODERN) { 473 return 1; 474 } 475 476 return virtio_pci_dev_probe(pci_dev, enum_cb); 477 } 478 479 int 480 virtio_pci_scsi_dev_enumerate(virtio_pci_create_cb enum_cb) 481 { 482 if (!spdk_process_is_primary()) { 483 SPDK_WARNLOG("virtio_pci secondary process support is not implemented yet.\n"); 484 return 0; 485 } 486 487 return spdk_pci_virtio_enumerate(virtio_pci_scsi_dev_probe_cb, enum_cb); 488 } 489 490 int 491 virtio_pci_dev_init(struct virtio_dev *vdev, const char *name, 492 struct virtio_pci_ctx *pci_ctx) 493 { 494 int rc; 495 char *name_dup; 496 497 name_dup = strdup(name); 498 if (name_dup == NULL) { 499 return -1; 500 } 501 502 rc = virtio_dev_construct(vdev, &modern_ops, pci_ctx); 503 if (rc != 0) { 504 free(name_dup); 505 return -1; 506 } 507 508 vdev->name = name_dup; 509 vdev->is_hw = 1; 510 vdev->modern = 1; 511 512 return 0; 513 } 514 515 SPDK_LOG_REGISTER_COMPONENT("virtio_pci", SPDK_LOG_VIRTIO_PCI) 516