1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. All rights reserved. 3 */ 4 5 #include "spdk/stdinc.h" 6 #include "spdk/memory.h" 7 #include "spdk/vfio_user_pci.h" 8 9 #include "spdk_internal/virtio.h" 10 11 #include <linux/vfio.h> 12 13 struct virtio_vfio_user_dev { 14 struct vfio_device *ctx; 15 char path[PATH_MAX]; 16 17 uint32_t pci_cap_region; 18 uint32_t pci_cap_common_cfg_offset; 19 uint32_t pci_cap_common_cfg_length; 20 uint32_t pci_cap_device_specific_offset; 21 uint32_t pci_cap_device_specific_length; 22 uint32_t pci_cap_notifications_offset; 23 uint32_t pci_cap_notifications_length; 24 }; 25 26 static int 27 virtio_vfio_user_read_dev_config(struct virtio_dev *vdev, size_t offset, 28 void *dst, int length) 29 { 30 struct virtio_vfio_user_dev *dev = vdev->ctx; 31 32 SPDK_DEBUGLOG(virtio_vfio_user, "offset 0x%lx, length 0x%x\n", offset, length); 33 return spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 34 dev->pci_cap_device_specific_offset + offset, 35 length, dst, false); 36 } 37 38 static int 39 virtio_vfio_user_write_dev_config(struct virtio_dev *vdev, size_t offset, 40 const void *src, int length) 41 { 42 struct virtio_vfio_user_dev *dev = vdev->ctx; 43 44 SPDK_DEBUGLOG(virtio_vfio_user, "offset 0x%lx, length 0x%x\n", offset, length); 45 return spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 46 dev->pci_cap_device_specific_offset + offset, 47 length, (void *)src, true); 48 } 49 50 static uint8_t 51 virtio_vfio_user_get_status(struct virtio_dev *vdev) 52 { 53 struct virtio_vfio_user_dev *dev = vdev->ctx; 54 uint64_t offset; 55 uint8_t status = 0; 56 int rc; 57 58 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_STATUS; 59 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 60 offset, 1, &status, false); 61 if (rc) { 62 SPDK_ERRLOG("Failed to get device status\n"); 63 } 64 65 SPDK_DEBUGLOG(virtio_vfio_user, "device status %x\n", status); 66 67 return status; 68 } 69 70 static void 71 virtio_vfio_user_set_status(struct virtio_dev *vdev, uint8_t status) 72 { 73 struct virtio_vfio_user_dev *dev = vdev->ctx; 74 uint64_t offset; 75 int rc; 76 77 SPDK_DEBUGLOG(virtio_vfio_user, "device status %x\n", status); 78 79 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_STATUS; 80 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 81 offset, 1, &status, true); 82 if (rc) { 83 SPDK_ERRLOG("Failed to set device status\n"); 84 } 85 } 86 87 static uint64_t 88 virtio_vfio_user_get_features(struct virtio_dev *vdev) 89 { 90 struct virtio_vfio_user_dev *dev = vdev->ctx; 91 uint64_t offset; 92 uint32_t features_lo, features_hi, feature_select; 93 int rc; 94 95 feature_select = 0; 96 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_DFSELECT; 97 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 98 offset, 4, &feature_select, true); 99 if (rc) { 100 SPDK_ERRLOG("Failed to set device feature select\n"); 101 } 102 103 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_DF; 104 features_lo = 0; 105 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 106 offset, 4, &features_lo, false); 107 if (rc) { 108 SPDK_ERRLOG("Failed to get device feature low\n"); 109 } 110 111 feature_select = 1; 112 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_DFSELECT; 113 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 114 offset, 4, &feature_select, true); 115 if (rc) { 116 SPDK_ERRLOG("Failed to set device feature select\n"); 117 } 118 119 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_DF; 120 features_hi = 0; 121 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 122 offset, 4, &features_hi, false); 123 if (rc) { 124 SPDK_ERRLOG("Failed to get device feature high\n"); 125 } 126 127 SPDK_DEBUGLOG(virtio_vfio_user, "feature_hi 0x%x, feature_low 0x%x\n", features_hi, features_lo); 128 129 return (((uint64_t)features_hi << 32) | ((uint64_t)features_lo)); 130 } 131 132 static int 133 virtio_vfio_user_set_features(struct virtio_dev *vdev, uint64_t features) 134 { 135 struct virtio_vfio_user_dev *dev = vdev->ctx; 136 uint64_t offset; 137 uint32_t features_lo, features_hi, feature_select; 138 int rc; 139 140 feature_select = 0; 141 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_GFSELECT; 142 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 143 offset, 4, &feature_select, true); 144 if (rc) { 145 SPDK_ERRLOG("Failed to set Guest feature select\n"); 146 return rc; 147 } 148 149 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_GF; 150 features_lo = (uint32_t)features; 151 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 152 offset, 4, &features_lo, true); 153 if (rc) { 154 SPDK_ERRLOG("Failed to set Guest feature low\n"); 155 return rc; 156 } 157 158 feature_select = 1; 159 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_GFSELECT; 160 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 161 offset, 4, &feature_select, true); 162 if (rc) { 163 SPDK_ERRLOG("Failed to set Guest feature select\n"); 164 return rc; 165 } 166 167 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_GF; 168 features_hi = (uint32_t)(features >> 32); 169 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 170 offset, 4, &features_hi, true); 171 if (rc) { 172 SPDK_ERRLOG("Failed to set Guest feature high\n"); 173 } 174 175 vdev->negotiated_features = features; 176 SPDK_DEBUGLOG(virtio_vfio_user, "features 0x%"PRIx64"\n", features); 177 178 return rc; 179 } 180 181 static void 182 virtio_vfio_user_destruct_dev(struct virtio_dev *vdev) 183 { 184 struct virtio_vfio_user_dev *dev = vdev->ctx; 185 186 if (dev) { 187 spdk_vfio_user_release(dev->ctx); 188 free(dev); 189 } 190 } 191 192 static uint16_t 193 virtio_vfio_user_get_queue_size(struct virtio_dev *vdev, uint16_t queue_id) 194 { 195 struct virtio_vfio_user_dev *dev = vdev->ctx; 196 uint64_t offset; 197 uint16_t qsize = 0; 198 int rc; 199 200 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_SELECT; 201 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 202 offset, 2, &queue_id, true); 203 if (rc) { 204 SPDK_ERRLOG("Failed to set queue select\n"); 205 return 0; 206 } 207 208 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_SIZE; 209 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 210 offset, 2, &qsize, false); 211 if (rc) { 212 SPDK_ERRLOG("Failed to get queue size\n"); 213 return 0; 214 } 215 216 SPDK_DEBUGLOG(virtio_vfio_user, "queue %u, size %u\n", queue_id, qsize); 217 218 return qsize; 219 } 220 221 static int 222 virtio_vfio_user_setup_queue(struct virtio_dev *vdev, struct virtqueue *vq) 223 { 224 struct virtio_vfio_user_dev *dev = vdev->ctx; 225 uint64_t desc_addr, avail_addr, used_addr, offset; 226 uint32_t addr_lo, addr_hi; 227 uint16_t notify_off, queue_enable; 228 void *queue_mem; 229 uint64_t queue_mem_phys_addr; 230 int rc; 231 232 /* To ensure physical address contiguity we make the queue occupy 233 * only a single hugepage (2MB). As of Virtio 1.0, the queue size 234 * always falls within this limit. 235 */ 236 if (vq->vq_ring_size > VALUE_2MB) { 237 return -ENOMEM; 238 } 239 240 queue_mem = spdk_zmalloc(vq->vq_ring_size, VALUE_2MB, NULL, 241 SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 242 if (queue_mem == NULL) { 243 return -ENOMEM; 244 } 245 246 queue_mem_phys_addr = spdk_vtophys(queue_mem, NULL); 247 if (queue_mem_phys_addr == SPDK_VTOPHYS_ERROR) { 248 spdk_free(queue_mem); 249 return -EFAULT; 250 } 251 252 vq->vq_ring_mem = queue_mem_phys_addr; 253 vq->vq_ring_virt_mem = queue_mem; 254 255 desc_addr = vq->vq_ring_mem; 256 avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc); 257 used_addr = (avail_addr + offsetof(struct vring_avail, ring[vq->vq_nentries]) 258 + VIRTIO_PCI_VRING_ALIGN - 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1); 259 260 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_SELECT; 261 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 262 offset, 2, &vq->vq_queue_index, true); 263 if (rc) { 264 SPDK_ERRLOG("Failed to set queue select\n"); 265 goto err; 266 } 267 268 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_DESCLO; 269 addr_lo = (uint32_t)desc_addr; 270 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 271 offset, 4, &addr_lo, true); 272 if (rc) { 273 SPDK_ERRLOG("Failed to set desc addr low\n"); 274 goto err; 275 } 276 277 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_DESCHI; 278 addr_hi = (uint32_t)(desc_addr >> 32); 279 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 280 offset, 4, &addr_hi, true); 281 if (rc) { 282 SPDK_ERRLOG("Failed to set desc addr high\n"); 283 goto err; 284 } 285 286 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_AVAILLO; 287 addr_lo = (uint32_t)avail_addr; 288 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 289 offset, 4, &addr_lo, true); 290 if (rc) { 291 SPDK_ERRLOG("Failed to set avail addr low\n"); 292 goto err; 293 } 294 295 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_AVAILHI; 296 addr_hi = (uint32_t)(avail_addr >> 32); 297 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 298 offset, 4, &addr_hi, true); 299 if (rc) { 300 SPDK_ERRLOG("Failed to set avail addr high\n"); 301 goto err; 302 } 303 304 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_USEDLO; 305 addr_lo = (uint32_t)used_addr; 306 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 307 offset, 4, &addr_lo, true); 308 if (rc) { 309 SPDK_ERRLOG("Failed to set used addr low\n"); 310 goto err; 311 } 312 313 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_USEDHI; 314 addr_hi = (uint32_t)(used_addr >> 32); 315 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 316 offset, 4, &addr_hi, true); 317 if (rc) { 318 SPDK_ERRLOG("Failed to set used addr high\n"); 319 goto err; 320 } 321 322 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_NOFF; 323 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 324 offset, 2, ¬ify_off, false); 325 if (rc) { 326 SPDK_ERRLOG("Failed to get queue notify off\n"); 327 goto err; 328 } 329 330 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_ENABLE; 331 queue_enable = 1; 332 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 333 offset, 2, &queue_enable, true); 334 if (rc) { 335 SPDK_ERRLOG("Failed to enable queue %u\n", vq->vq_queue_index); 336 goto err; 337 } 338 339 SPDK_DEBUGLOG(virtio_vfio_user, "queue %"PRIu16" addresses:\n", vq->vq_queue_index); 340 SPDK_DEBUGLOG(virtio_vfio_user, "\t desc_addr: %" PRIx64 "\n", desc_addr); 341 SPDK_DEBUGLOG(virtio_vfio_user, "\t aval_addr: %" PRIx64 "\n", avail_addr); 342 SPDK_DEBUGLOG(virtio_vfio_user, "\t used_addr: %" PRIx64 "\n", used_addr); 343 344 return 0; 345 err: 346 spdk_free(queue_mem); 347 return rc; 348 } 349 350 static void 351 virtio_vfio_user_del_queue(struct virtio_dev *vdev, struct virtqueue *vq) 352 { 353 struct virtio_vfio_user_dev *dev = vdev->ctx; 354 uint64_t offset; 355 uint16_t queue_enable = 0; 356 int rc; 357 358 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_SELECT; 359 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 360 offset, 2, &vq->vq_queue_index, true); 361 if (rc) { 362 SPDK_ERRLOG("Failed to select queue %u\n", vq->vq_queue_index); 363 spdk_free(vq->vq_ring_virt_mem); 364 return; 365 } 366 367 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_ENABLE; 368 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 369 offset, 2, &queue_enable, true); 370 if (rc) { 371 SPDK_ERRLOG("Failed to enable queue %u\n", vq->vq_queue_index); 372 } 373 374 spdk_free(vq->vq_ring_virt_mem); 375 /* TODO: clear desc/avail/used address */ 376 } 377 378 static void 379 virtio_vfio_user_notify_queue(struct virtio_dev *vdev, struct virtqueue *vq) 380 { 381 /* we're running in polling mode, no need to write doorbells */ 382 } 383 384 static const struct virtio_dev_ops virtio_vfio_user_ops = { 385 .read_dev_cfg = virtio_vfio_user_read_dev_config, 386 .write_dev_cfg = virtio_vfio_user_write_dev_config, 387 .get_status = virtio_vfio_user_get_status, 388 .set_status = virtio_vfio_user_set_status, 389 .get_features = virtio_vfio_user_get_features, 390 .set_features = virtio_vfio_user_set_features, 391 .destruct_dev = virtio_vfio_user_destruct_dev, 392 .get_queue_size = virtio_vfio_user_get_queue_size, 393 .setup_queue = virtio_vfio_user_setup_queue, 394 .del_queue = virtio_vfio_user_del_queue, 395 .notify_queue = virtio_vfio_user_notify_queue 396 }; 397 398 int 399 virtio_vfio_user_dev_init(struct virtio_dev *vdev, const char *name, const char *path) 400 { 401 struct virtio_vfio_user_dev *dev; 402 uint16_t cmd_reg; 403 int rc; 404 405 if (name == NULL) { 406 SPDK_ERRLOG("No name gived for controller: %s\n", path); 407 return -EINVAL; 408 } 409 410 rc = access(path, F_OK); 411 if (rc != 0) { 412 SPDK_ERRLOG("Access path %s failed\n", path); 413 return -EACCES; 414 } 415 416 dev = calloc(1, sizeof(*dev)); 417 if (dev == NULL) { 418 return -ENOMEM; 419 } 420 421 rc = virtio_dev_construct(vdev, name, &virtio_vfio_user_ops, dev); 422 if (rc != 0) { 423 SPDK_ERRLOG("Failed to init device: %s\n", path); 424 free(dev); 425 return rc; 426 } 427 428 snprintf(dev->path, PATH_MAX, "%s", path); 429 dev->ctx = spdk_vfio_user_setup(path); 430 if (!dev->ctx) { 431 SPDK_ERRLOG("Error to setup %s as vfio device\n", path); 432 virtio_dev_destruct(vdev); 433 return -EINVAL; 434 } 435 436 /* Enable PCI busmaster and disable INTx */ 437 rc = spdk_vfio_user_pci_bar_access(dev->ctx, VFIO_PCI_CONFIG_REGION_INDEX, 4, 2, 438 &cmd_reg, false); 439 if (rc != 0) { 440 SPDK_ERRLOG("Read PCI CMD REG failed\n"); 441 virtio_dev_destruct(vdev); 442 return rc; 443 } 444 cmd_reg |= 0x404; 445 rc = spdk_vfio_user_pci_bar_access(dev->ctx, VFIO_PCI_CONFIG_REGION_INDEX, 4, 2, 446 &cmd_reg, true); 447 if (rc != 0) { 448 SPDK_ERRLOG("Write PCI CMD REG failed\n"); 449 virtio_dev_destruct(vdev); 450 return rc; 451 } 452 453 /* TODO: we cat get virtio device PCI common space layout via 454 * iterating vendor capabilities in PCI Configuration space, 455 * while here we use hardcoded layout first, this feature can 456 * be added in future. 457 * 458 * vfio-user emulated virtio device layout in Target: 459 * 460 * region 1: MSI-X Table 461 * region 2: MSI-X PBA 462 * region 4: virtio modern memory 64bits BAR 463 * Common configuration 0x0 - 0x1000 464 * ISR access 0x1000 - 0x2000 465 * Device specific configuration 0x2000 - 0x3000 466 * Notifications 0x3000 - 0x4000 467 */ 468 dev->pci_cap_region = VFIO_PCI_BAR4_REGION_INDEX; 469 dev->pci_cap_common_cfg_offset = 0x0; 470 dev->pci_cap_common_cfg_length = 0x1000; 471 dev->pci_cap_device_specific_offset = 0x2000; 472 dev->pci_cap_device_specific_length = 0x1000; 473 dev->pci_cap_notifications_offset = 0x3000; 474 dev->pci_cap_notifications_length = 0x1000; 475 476 return 0; 477 } 478 479 SPDK_LOG_REGISTER_COMPONENT(virtio_vfio_user) 480