1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. All rights reserved. 3 */ 4 5 #include "spdk/stdinc.h" 6 #include "spdk/memory.h" 7 #include "spdk/vfio_user_pci.h" 8 9 #include "spdk_internal/virtio.h" 10 11 #include <linux/vfio.h> 12 13 struct virtio_vfio_user_dev { 14 struct vfio_device *ctx; 15 char path[PATH_MAX]; 16 17 uint32_t pci_cap_region; 18 uint32_t pci_cap_common_cfg_offset; 19 uint32_t pci_cap_common_cfg_length; 20 uint32_t pci_cap_device_specific_offset; 21 uint32_t pci_cap_device_specific_length; 22 uint32_t pci_cap_notifications_offset; 23 uint32_t pci_cap_notifications_length; 24 }; 25 26 static int 27 virtio_vfio_user_read_dev_config(struct virtio_dev *vdev, size_t offset, 28 void *dst, int length) 29 { 30 struct virtio_vfio_user_dev *dev = vdev->ctx; 31 32 SPDK_DEBUGLOG(virtio_vfio_user, "offset 0x%lx, length 0x%x\n", offset, length); 33 return spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 34 dev->pci_cap_device_specific_offset + offset, 35 length, dst, false); 36 } 37 38 static int 39 virtio_vfio_user_write_dev_config(struct virtio_dev *vdev, size_t offset, 40 const void *src, int length) 41 { 42 struct virtio_vfio_user_dev *dev = vdev->ctx; 43 44 SPDK_DEBUGLOG(virtio_vfio_user, "offset 0x%lx, length 0x%x\n", offset, length); 45 return spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 46 dev->pci_cap_device_specific_offset + offset, 47 length, (void *)src, true); 48 } 49 50 static uint8_t 51 virtio_vfio_user_get_status(struct virtio_dev *vdev) 52 { 53 struct virtio_vfio_user_dev *dev = vdev->ctx; 54 uint64_t offset; 55 uint8_t status = 0; 56 int rc; 57 58 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_STATUS; 59 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 60 offset, 1, &status, false); 61 if (rc) { 62 SPDK_ERRLOG("Failed to get device status\n"); 63 } 64 65 SPDK_DEBUGLOG(virtio_vfio_user, "device status %x\n", status); 66 67 return status; 68 } 69 70 static void 71 virtio_vfio_user_set_status(struct virtio_dev *vdev, uint8_t status) 72 { 73 struct virtio_vfio_user_dev *dev = vdev->ctx; 74 uint64_t offset; 75 int rc; 76 77 SPDK_DEBUGLOG(virtio_vfio_user, "device status %x\n", status); 78 79 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_STATUS; 80 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 81 offset, 1, &status, true); 82 if (rc) { 83 SPDK_ERRLOG("Failed to set device status\n"); 84 } 85 } 86 87 static uint64_t 88 virtio_vfio_user_get_features(struct virtio_dev *vdev) 89 { 90 struct virtio_vfio_user_dev *dev = vdev->ctx; 91 uint64_t offset; 92 uint32_t features_lo, features_hi, feature_select; 93 int rc; 94 95 feature_select = 0; 96 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_DFSELECT; 97 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 98 offset, 4, &feature_select, true); 99 if (rc) { 100 SPDK_ERRLOG("Failed to set device feature select\n"); 101 } 102 103 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_DF; 104 features_lo = 0; 105 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 106 offset, 4, &features_lo, false); 107 if (rc) { 108 SPDK_ERRLOG("Failed to get device feature low\n"); 109 } 110 111 feature_select = 1; 112 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_DFSELECT; 113 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 114 offset, 4, &feature_select, true); 115 if (rc) { 116 SPDK_ERRLOG("Failed to set device feature select\n"); 117 } 118 119 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_DF; 120 features_hi = 0; 121 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 122 offset, 4, &features_hi, false); 123 if (rc) { 124 SPDK_ERRLOG("Failed to get device feature high\n"); 125 } 126 127 SPDK_DEBUGLOG(virtio_vfio_user, "feature_hi 0x%x, feature_low 0x%x\n", features_hi, features_lo); 128 129 return (((uint64_t)features_hi << 32) | ((uint64_t)features_lo)); 130 } 131 132 static int 133 virtio_vfio_user_set_features(struct virtio_dev *vdev, uint64_t features) 134 { 135 struct virtio_vfio_user_dev *dev = vdev->ctx; 136 uint64_t offset; 137 uint32_t features_lo, features_hi, feature_select; 138 int rc; 139 140 feature_select = 0; 141 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_GFSELECT; 142 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 143 offset, 4, &feature_select, true); 144 if (rc) { 145 SPDK_ERRLOG("Failed to set Guest feature select\n"); 146 return rc; 147 } 148 149 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_GF; 150 features_lo = (uint32_t)features; 151 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 152 offset, 4, &features_lo, true); 153 if (rc) { 154 SPDK_ERRLOG("Failed to set Guest feature low\n"); 155 return rc; 156 } 157 158 feature_select = 1; 159 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_GFSELECT; 160 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 161 offset, 4, &feature_select, true); 162 if (rc) { 163 SPDK_ERRLOG("Failed to set Guest feature select\n"); 164 return rc; 165 } 166 167 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_GF; 168 features_hi = (uint32_t)(features >> 32); 169 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 170 offset, 4, &features_hi, true); 171 if (rc) { 172 SPDK_ERRLOG("Failed to set Guest feature high\n"); 173 } 174 175 vdev->negotiated_features = features; 176 SPDK_DEBUGLOG(virtio_vfio_user, "features 0x%"PRIx64"\n", features); 177 178 return rc; 179 } 180 181 static void 182 virtio_vfio_user_destruct_dev(struct virtio_dev *vdev) 183 { 184 struct virtio_vfio_user_dev *dev = vdev->ctx; 185 186 spdk_vfio_user_release(dev->ctx); 187 free(dev); 188 } 189 190 static uint16_t 191 virtio_vfio_user_get_queue_size(struct virtio_dev *vdev, uint16_t queue_id) 192 { 193 struct virtio_vfio_user_dev *dev = vdev->ctx; 194 uint64_t offset; 195 uint16_t qsize = 0; 196 int rc; 197 198 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_SELECT; 199 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 200 offset, 2, &queue_id, true); 201 if (rc) { 202 SPDK_ERRLOG("Failed to set queue select\n"); 203 return 0; 204 } 205 206 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_SIZE; 207 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 208 offset, 2, &qsize, false); 209 if (rc) { 210 SPDK_ERRLOG("Failed to get queue size\n"); 211 return 0; 212 } 213 214 SPDK_DEBUGLOG(virtio_vfio_user, "queue %u, size %u\n", queue_id, qsize); 215 216 return qsize; 217 } 218 219 static int 220 virtio_vfio_user_setup_queue(struct virtio_dev *vdev, struct virtqueue *vq) 221 { 222 struct virtio_vfio_user_dev *dev = vdev->ctx; 223 uint64_t desc_addr, avail_addr, used_addr, offset; 224 uint32_t addr_lo, addr_hi; 225 uint16_t notify_off, queue_enable; 226 void *queue_mem; 227 uint64_t queue_mem_phys_addr; 228 int rc; 229 230 /* To ensure physical address contiguity we make the queue occupy 231 * only a single hugepage (2MB). As of Virtio 1.0, the queue size 232 * always falls within this limit. 233 */ 234 if (vq->vq_ring_size > VALUE_2MB) { 235 return -ENOMEM; 236 } 237 238 queue_mem = spdk_zmalloc(vq->vq_ring_size, VALUE_2MB, NULL, 239 SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 240 if (queue_mem == NULL) { 241 return -ENOMEM; 242 } 243 244 queue_mem_phys_addr = spdk_vtophys(queue_mem, NULL); 245 if (queue_mem_phys_addr == SPDK_VTOPHYS_ERROR) { 246 spdk_free(queue_mem); 247 return -EFAULT; 248 } 249 250 vq->vq_ring_mem = queue_mem_phys_addr; 251 vq->vq_ring_virt_mem = queue_mem; 252 253 desc_addr = vq->vq_ring_mem; 254 avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc); 255 used_addr = (avail_addr + offsetof(struct vring_avail, ring[vq->vq_nentries]) 256 + VIRTIO_PCI_VRING_ALIGN - 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1); 257 258 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_SELECT; 259 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 260 offset, 2, &vq->vq_queue_index, true); 261 if (rc) { 262 SPDK_ERRLOG("Failed to set queue select\n"); 263 goto err; 264 } 265 266 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_DESCLO; 267 addr_lo = (uint32_t)desc_addr; 268 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 269 offset, 4, &addr_lo, true); 270 if (rc) { 271 SPDK_ERRLOG("Failed to set desc addr low\n"); 272 goto err; 273 } 274 275 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_DESCHI; 276 addr_hi = (uint32_t)(desc_addr >> 32); 277 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 278 offset, 4, &addr_hi, true); 279 if (rc) { 280 SPDK_ERRLOG("Failed to set desc addr high\n"); 281 goto err; 282 } 283 284 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_AVAILLO; 285 addr_lo = (uint32_t)avail_addr; 286 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 287 offset, 4, &addr_lo, true); 288 if (rc) { 289 SPDK_ERRLOG("Failed to set avail addr low\n"); 290 goto err; 291 } 292 293 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_AVAILHI; 294 addr_hi = (uint32_t)(avail_addr >> 32); 295 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 296 offset, 4, &addr_hi, true); 297 if (rc) { 298 SPDK_ERRLOG("Failed to set avail addr high\n"); 299 goto err; 300 } 301 302 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_USEDLO; 303 addr_lo = (uint32_t)used_addr; 304 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 305 offset, 4, &addr_lo, true); 306 if (rc) { 307 SPDK_ERRLOG("Failed to set used addr low\n"); 308 goto err; 309 } 310 311 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_USEDHI; 312 addr_hi = (uint32_t)(used_addr >> 32); 313 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 314 offset, 4, &addr_hi, true); 315 if (rc) { 316 SPDK_ERRLOG("Failed to set used addr high\n"); 317 goto err; 318 } 319 320 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_NOFF; 321 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 322 offset, 2, ¬ify_off, false); 323 if (rc) { 324 SPDK_ERRLOG("Failed to get queue notify off\n"); 325 goto err; 326 } 327 328 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_ENABLE; 329 queue_enable = 1; 330 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 331 offset, 2, &queue_enable, true); 332 if (rc) { 333 SPDK_ERRLOG("Failed to enable queue %u\n", vq->vq_queue_index); 334 goto err; 335 } 336 337 SPDK_DEBUGLOG(virtio_vfio_user, "queue %"PRIu16" addresses:\n", vq->vq_queue_index); 338 SPDK_DEBUGLOG(virtio_vfio_user, "\t desc_addr: %" PRIx64 "\n", desc_addr); 339 SPDK_DEBUGLOG(virtio_vfio_user, "\t aval_addr: %" PRIx64 "\n", avail_addr); 340 SPDK_DEBUGLOG(virtio_vfio_user, "\t used_addr: %" PRIx64 "\n", used_addr); 341 342 return 0; 343 err: 344 spdk_free(queue_mem); 345 return rc; 346 } 347 348 static void 349 virtio_vfio_user_del_queue(struct virtio_dev *vdev, struct virtqueue *vq) 350 { 351 struct virtio_vfio_user_dev *dev = vdev->ctx; 352 uint64_t offset; 353 uint16_t queue_enable = 0; 354 int rc; 355 356 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_SELECT; 357 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 358 offset, 2, &vq->vq_queue_index, true); 359 if (rc) { 360 SPDK_ERRLOG("Failed to select queue %u\n", vq->vq_queue_index); 361 spdk_free(vq->vq_ring_virt_mem); 362 return; 363 } 364 365 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_ENABLE; 366 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 367 offset, 2, &queue_enable, true); 368 if (rc) { 369 SPDK_ERRLOG("Failed to enable queue %u\n", vq->vq_queue_index); 370 } 371 372 spdk_free(vq->vq_ring_virt_mem); 373 /* TODO: clear desc/avail/used address */ 374 } 375 376 static void 377 virtio_vfio_user_notify_queue(struct virtio_dev *vdev, struct virtqueue *vq) 378 { 379 /* we're running in polling mode, no need to write doorbells */ 380 } 381 382 static const struct virtio_dev_ops virtio_vfio_user_ops = { 383 .read_dev_cfg = virtio_vfio_user_read_dev_config, 384 .write_dev_cfg = virtio_vfio_user_write_dev_config, 385 .get_status = virtio_vfio_user_get_status, 386 .set_status = virtio_vfio_user_set_status, 387 .get_features = virtio_vfio_user_get_features, 388 .set_features = virtio_vfio_user_set_features, 389 .destruct_dev = virtio_vfio_user_destruct_dev, 390 .get_queue_size = virtio_vfio_user_get_queue_size, 391 .setup_queue = virtio_vfio_user_setup_queue, 392 .del_queue = virtio_vfio_user_del_queue, 393 .notify_queue = virtio_vfio_user_notify_queue 394 }; 395 396 int 397 virtio_vfio_user_dev_init(struct virtio_dev *vdev, const char *name, const char *path) 398 { 399 struct virtio_vfio_user_dev *dev; 400 uint16_t cmd_reg; 401 int rc; 402 403 if (name == NULL) { 404 SPDK_ERRLOG("No name gived for controller: %s\n", path); 405 return -EINVAL; 406 } 407 408 rc = access(path, F_OK); 409 if (rc != 0) { 410 SPDK_ERRLOG("Access path %s failed\n", path); 411 return -EACCES; 412 } 413 414 dev = calloc(1, sizeof(*dev)); 415 if (dev == NULL) { 416 return -ENOMEM; 417 } 418 419 rc = virtio_dev_construct(vdev, name, &virtio_vfio_user_ops, dev); 420 if (rc != 0) { 421 SPDK_ERRLOG("Failed to init device: %s\n", path); 422 free(dev); 423 return rc; 424 } 425 426 snprintf(dev->path, PATH_MAX, "%s", path); 427 dev->ctx = spdk_vfio_user_setup(path); 428 if (!dev->ctx) { 429 SPDK_ERRLOG("Error to setup %s as vfio device\n", path); 430 virtio_dev_destruct(vdev); 431 free(dev); 432 return -EINVAL; 433 } 434 435 /* Enable PCI busmaster and disable INTx */ 436 rc = spdk_vfio_user_pci_bar_access(dev->ctx, VFIO_PCI_CONFIG_REGION_INDEX, 4, 2, 437 &cmd_reg, false); 438 if (rc != 0) { 439 SPDK_ERRLOG("Read PCI CMD REG failed\n"); 440 virtio_dev_destruct(vdev); 441 spdk_vfio_user_release(dev->ctx); 442 free(dev); 443 return rc; 444 } 445 cmd_reg |= 0x404; 446 rc = spdk_vfio_user_pci_bar_access(dev->ctx, VFIO_PCI_CONFIG_REGION_INDEX, 4, 2, 447 &cmd_reg, true); 448 if (rc != 0) { 449 SPDK_ERRLOG("Write PCI CMD REG failed\n"); 450 virtio_dev_destruct(vdev); 451 spdk_vfio_user_release(dev->ctx); 452 free(dev); 453 return rc; 454 } 455 456 /* TODO: we cat get virtio device PCI common space layout via 457 * iterating vendor capabilities in PCI Configuration space, 458 * while here we use hardcoded layout first, this feature can 459 * be added in future. 460 * 461 * vfio-user emulated virtio device layout in Target: 462 * 463 * region 1: MSI-X Table 464 * region 2: MSI-X PBA 465 * region 4: virtio modern memory 64bits BAR 466 * Common configuration 0x0 - 0x1000 467 * ISR access 0x1000 - 0x2000 468 * Device specific configuration 0x2000 - 0x3000 469 * Notifications 0x3000 - 0x4000 470 */ 471 dev->pci_cap_region = VFIO_PCI_BAR4_REGION_INDEX; 472 dev->pci_cap_common_cfg_offset = 0x0; 473 dev->pci_cap_common_cfg_length = 0x1000; 474 dev->pci_cap_device_specific_offset = 0x2000; 475 dev->pci_cap_device_specific_length = 0x1000; 476 dev->pci_cap_notifications_offset = 0x3000; 477 dev->pci_cap_notifications_length = 0x1000; 478 479 return 0; 480 } 481 482 SPDK_LOG_REGISTER_COMPONENT(virtio_vfio_user) 483