1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. All rights reserved. 3 */ 4 5 #include "spdk/stdinc.h" 6 #include "spdk/memory.h" 7 #include "spdk/vfio_user_pci.h" 8 9 #include "spdk_internal/virtio.h" 10 11 #include <linux/vfio.h> 12 13 struct virtio_vfio_user_dev { 14 struct vfio_device *ctx; 15 char path[PATH_MAX]; 16 17 uint32_t pci_cap_region; 18 uint32_t pci_cap_common_cfg_offset; 19 uint32_t pci_cap_common_cfg_length; 20 uint32_t pci_cap_device_specific_offset; 21 uint32_t pci_cap_device_specific_length; 22 uint32_t pci_cap_notifications_offset; 23 uint32_t pci_cap_notifications_length; 24 }; 25 26 static int 27 virtio_vfio_user_read_dev_config(struct virtio_dev *vdev, size_t offset, 28 void *dst, int length) 29 { 30 struct virtio_vfio_user_dev *dev = vdev->ctx; 31 32 SPDK_DEBUGLOG(virtio_vfio_user, "offset 0x%lx, length 0x%x\n", offset, length); 33 return spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 34 dev->pci_cap_device_specific_offset + offset, 35 length, dst, false); 36 } 37 38 static int 39 virtio_vfio_user_write_dev_config(struct virtio_dev *vdev, size_t offset, 40 const void *src, int length) 41 { 42 struct virtio_vfio_user_dev *dev = vdev->ctx; 43 44 SPDK_DEBUGLOG(virtio_vfio_user, "offset 0x%lx, length 0x%x\n", offset, length); 45 return spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 46 dev->pci_cap_device_specific_offset + offset, 47 length, (void *)src, true); 48 } 49 50 static uint8_t 51 virtio_vfio_user_get_status(struct virtio_dev *vdev) 52 { 53 struct virtio_vfio_user_dev *dev = vdev->ctx; 54 uint64_t offset; 55 uint8_t status = 0; 56 int rc; 57 58 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_STATUS; 59 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 60 offset, 1, &status, false); 61 if (rc) { 62 SPDK_ERRLOG("Failed to get device status\n"); 63 } 64 65 SPDK_DEBUGLOG(virtio_vfio_user, "device status %x\n", status); 66 67 return status; 68 } 69 70 static void 71 virtio_vfio_user_set_status(struct virtio_dev *vdev, uint8_t status) 72 { 73 struct virtio_vfio_user_dev *dev = vdev->ctx; 74 uint64_t offset; 75 int rc; 76 77 SPDK_DEBUGLOG(virtio_vfio_user, "device status %x\n", status); 78 79 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_STATUS; 80 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 81 offset, 1, &status, true); 82 if (rc) { 83 SPDK_ERRLOG("Failed to set device status\n"); 84 } 85 } 86 87 static uint64_t 88 virtio_vfio_user_get_features(struct virtio_dev *vdev) 89 { 90 struct virtio_vfio_user_dev *dev = vdev->ctx; 91 uint64_t offset; 92 uint32_t features_lo, features_hi, feature_select; 93 int rc; 94 95 feature_select = 0; 96 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_DFSELECT; 97 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 98 offset, 4, &feature_select, true); 99 if (rc) { 100 SPDK_ERRLOG("Failed to set device feature select\n"); 101 } 102 103 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_DF; 104 features_lo = 0; 105 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 106 offset, 4, &features_lo, false); 107 if (rc) { 108 SPDK_ERRLOG("Failed to get device feature low\n"); 109 } 110 111 feature_select = 1; 112 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_DFSELECT; 113 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 114 offset, 4, &feature_select, true); 115 if (rc) { 116 SPDK_ERRLOG("Failed to set device feature select\n"); 117 } 118 119 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_DF; 120 features_hi = 0; 121 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 122 offset, 4, &features_hi, false); 123 if (rc) { 124 SPDK_ERRLOG("Failed to get device feature high\n"); 125 } 126 127 SPDK_DEBUGLOG(virtio_vfio_user, "feature_hi 0x%x, feature_low 0x%x\n", features_hi, features_lo); 128 129 return (((uint64_t)features_hi << 32) | ((uint64_t)features_lo)); 130 } 131 132 static int 133 virtio_vfio_user_set_features(struct virtio_dev *vdev, uint64_t features) 134 { 135 struct virtio_vfio_user_dev *dev = vdev->ctx; 136 uint64_t offset; 137 uint32_t features_lo, features_hi, feature_select; 138 int rc; 139 140 feature_select = 0; 141 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_GFSELECT; 142 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 143 offset, 4, &feature_select, true); 144 if (rc) { 145 SPDK_ERRLOG("Failed to set Guest feature select\n"); 146 return rc; 147 } 148 149 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_GF; 150 features_lo = (uint32_t)features; 151 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 152 offset, 4, &features_lo, true); 153 if (rc) { 154 SPDK_ERRLOG("Failed to set Guest feature low\n"); 155 return rc; 156 } 157 158 feature_select = 1; 159 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_GFSELECT; 160 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 161 offset, 4, &feature_select, true); 162 if (rc) { 163 SPDK_ERRLOG("Failed to set Guest feature select\n"); 164 return rc; 165 } 166 167 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_GF; 168 features_hi = (uint32_t)(features >> 32); 169 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 170 offset, 4, &features_hi, true); 171 if (rc) { 172 SPDK_ERRLOG("Failed to set Guest feature high\n"); 173 } 174 175 vdev->negotiated_features = features; 176 SPDK_DEBUGLOG(virtio_vfio_user, "features 0x%"PRIx64"\n", features); 177 178 return rc; 179 } 180 181 static void 182 virtio_vfio_user_destruct_dev(struct virtio_dev *vdev) 183 { 184 struct virtio_vfio_user_dev *dev = vdev->ctx; 185 186 if (dev) { 187 spdk_vfio_user_release(dev->ctx); 188 free(dev); 189 } 190 } 191 192 static uint16_t 193 virtio_vfio_user_get_queue_size(struct virtio_dev *vdev, uint16_t queue_id) 194 { 195 struct virtio_vfio_user_dev *dev = vdev->ctx; 196 uint64_t offset; 197 uint16_t qsize = 0; 198 int rc; 199 200 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_SELECT; 201 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 202 offset, 2, &queue_id, true); 203 if (rc) { 204 SPDK_ERRLOG("Failed to set queue select\n"); 205 return 0; 206 } 207 208 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_SIZE; 209 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 210 offset, 2, &qsize, false); 211 if (rc) { 212 SPDK_ERRLOG("Failed to get queue size\n"); 213 return 0; 214 } 215 216 SPDK_DEBUGLOG(virtio_vfio_user, "queue %u, size %u\n", queue_id, qsize); 217 218 return qsize; 219 } 220 221 static int 222 virtio_vfio_user_setup_queue(struct virtio_dev *vdev, struct virtqueue *vq) 223 { 224 struct virtio_vfio_user_dev *dev = vdev->ctx; 225 uint64_t desc_addr, avail_addr, used_addr, offset; 226 uint32_t addr_lo, addr_hi; 227 uint16_t notify_off, queue_enable; 228 void *queue_mem; 229 uint64_t queue_mem_phys_addr; 230 int rc; 231 232 queue_mem = spdk_zmalloc(vq->vq_ring_size, VIRTIO_PCI_VRING_ALIGN, NULL, 233 SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 234 if (queue_mem == NULL) { 235 return -ENOMEM; 236 } 237 238 queue_mem_phys_addr = spdk_vtophys(queue_mem, NULL); 239 if (queue_mem_phys_addr == SPDK_VTOPHYS_ERROR) { 240 spdk_free(queue_mem); 241 return -EFAULT; 242 } 243 244 vq->vq_ring_mem = queue_mem_phys_addr; 245 vq->vq_ring_virt_mem = queue_mem; 246 247 desc_addr = vq->vq_ring_mem; 248 avail_addr = desc_addr + vq->vq_nentries * sizeof(struct vring_desc); 249 used_addr = (avail_addr + offsetof(struct vring_avail, ring[vq->vq_nentries]) 250 + VIRTIO_PCI_VRING_ALIGN - 1) & ~(VIRTIO_PCI_VRING_ALIGN - 1); 251 252 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_SELECT; 253 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 254 offset, 2, &vq->vq_queue_index, true); 255 if (rc) { 256 SPDK_ERRLOG("Failed to set queue select\n"); 257 goto err; 258 } 259 260 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_DESCLO; 261 addr_lo = (uint32_t)desc_addr; 262 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 263 offset, 4, &addr_lo, true); 264 if (rc) { 265 SPDK_ERRLOG("Failed to set desc addr low\n"); 266 goto err; 267 } 268 269 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_DESCHI; 270 addr_hi = (uint32_t)(desc_addr >> 32); 271 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 272 offset, 4, &addr_hi, true); 273 if (rc) { 274 SPDK_ERRLOG("Failed to set desc addr high\n"); 275 goto err; 276 } 277 278 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_AVAILLO; 279 addr_lo = (uint32_t)avail_addr; 280 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 281 offset, 4, &addr_lo, true); 282 if (rc) { 283 SPDK_ERRLOG("Failed to set avail addr low\n"); 284 goto err; 285 } 286 287 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_AVAILHI; 288 addr_hi = (uint32_t)(avail_addr >> 32); 289 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 290 offset, 4, &addr_hi, true); 291 if (rc) { 292 SPDK_ERRLOG("Failed to set avail addr high\n"); 293 goto err; 294 } 295 296 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_USEDLO; 297 addr_lo = (uint32_t)used_addr; 298 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 299 offset, 4, &addr_lo, true); 300 if (rc) { 301 SPDK_ERRLOG("Failed to set used addr low\n"); 302 goto err; 303 } 304 305 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_USEDHI; 306 addr_hi = (uint32_t)(used_addr >> 32); 307 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 308 offset, 4, &addr_hi, true); 309 if (rc) { 310 SPDK_ERRLOG("Failed to set used addr high\n"); 311 goto err; 312 } 313 314 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_NOFF; 315 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 316 offset, 2, ¬ify_off, false); 317 if (rc) { 318 SPDK_ERRLOG("Failed to get queue notify off\n"); 319 goto err; 320 } 321 322 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_ENABLE; 323 queue_enable = 1; 324 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 325 offset, 2, &queue_enable, true); 326 if (rc) { 327 SPDK_ERRLOG("Failed to enable queue %u\n", vq->vq_queue_index); 328 goto err; 329 } 330 331 SPDK_DEBUGLOG(virtio_vfio_user, "queue %"PRIu16" addresses:\n", vq->vq_queue_index); 332 SPDK_DEBUGLOG(virtio_vfio_user, "\t desc_addr: %" PRIx64 "\n", desc_addr); 333 SPDK_DEBUGLOG(virtio_vfio_user, "\t aval_addr: %" PRIx64 "\n", avail_addr); 334 SPDK_DEBUGLOG(virtio_vfio_user, "\t used_addr: %" PRIx64 "\n", used_addr); 335 336 return 0; 337 err: 338 spdk_free(queue_mem); 339 return rc; 340 } 341 342 static void 343 virtio_vfio_user_del_queue(struct virtio_dev *vdev, struct virtqueue *vq) 344 { 345 struct virtio_vfio_user_dev *dev = vdev->ctx; 346 uint64_t offset; 347 uint16_t queue_enable = 0; 348 int rc; 349 350 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_SELECT; 351 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 352 offset, 2, &vq->vq_queue_index, true); 353 if (rc) { 354 SPDK_ERRLOG("Failed to select queue %u\n", vq->vq_queue_index); 355 spdk_free(vq->vq_ring_virt_mem); 356 return; 357 } 358 359 offset = dev->pci_cap_common_cfg_offset + VIRTIO_PCI_COMMON_Q_ENABLE; 360 rc = spdk_vfio_user_pci_bar_access(dev->ctx, dev->pci_cap_region, 361 offset, 2, &queue_enable, true); 362 if (rc) { 363 SPDK_ERRLOG("Failed to enable queue %u\n", vq->vq_queue_index); 364 } 365 366 spdk_free(vq->vq_ring_virt_mem); 367 /* TODO: clear desc/avail/used address */ 368 } 369 370 static void 371 virtio_vfio_user_notify_queue(struct virtio_dev *vdev, struct virtqueue *vq) 372 { 373 /* we're running in polling mode, no need to write doorbells */ 374 } 375 376 static const struct virtio_dev_ops virtio_vfio_user_ops = { 377 .read_dev_cfg = virtio_vfio_user_read_dev_config, 378 .write_dev_cfg = virtio_vfio_user_write_dev_config, 379 .get_status = virtio_vfio_user_get_status, 380 .set_status = virtio_vfio_user_set_status, 381 .get_features = virtio_vfio_user_get_features, 382 .set_features = virtio_vfio_user_set_features, 383 .destruct_dev = virtio_vfio_user_destruct_dev, 384 .get_queue_size = virtio_vfio_user_get_queue_size, 385 .setup_queue = virtio_vfio_user_setup_queue, 386 .del_queue = virtio_vfio_user_del_queue, 387 .notify_queue = virtio_vfio_user_notify_queue 388 }; 389 390 int 391 virtio_vfio_user_dev_init(struct virtio_dev *vdev, const char *name, const char *path) 392 { 393 struct virtio_vfio_user_dev *dev; 394 uint16_t cmd_reg; 395 int rc; 396 397 if (name == NULL) { 398 SPDK_ERRLOG("No name gived for controller: %s\n", path); 399 return -EINVAL; 400 } 401 402 rc = access(path, F_OK); 403 if (rc != 0) { 404 SPDK_ERRLOG("Access path %s failed\n", path); 405 return -EACCES; 406 } 407 408 dev = calloc(1, sizeof(*dev)); 409 if (dev == NULL) { 410 return -ENOMEM; 411 } 412 413 rc = virtio_dev_construct(vdev, name, &virtio_vfio_user_ops, dev); 414 if (rc != 0) { 415 SPDK_ERRLOG("Failed to init device: %s\n", path); 416 free(dev); 417 return rc; 418 } 419 420 snprintf(dev->path, PATH_MAX, "%s", path); 421 dev->ctx = spdk_vfio_user_setup(path); 422 if (!dev->ctx) { 423 SPDK_ERRLOG("Error to setup %s as vfio device\n", path); 424 virtio_dev_destruct(vdev); 425 return -EINVAL; 426 } 427 428 /* Enable PCI busmaster and disable INTx */ 429 rc = spdk_vfio_user_pci_bar_access(dev->ctx, VFIO_PCI_CONFIG_REGION_INDEX, 4, 2, 430 &cmd_reg, false); 431 if (rc != 0) { 432 SPDK_ERRLOG("Read PCI CMD REG failed\n"); 433 virtio_dev_destruct(vdev); 434 return rc; 435 } 436 cmd_reg |= 0x404; 437 rc = spdk_vfio_user_pci_bar_access(dev->ctx, VFIO_PCI_CONFIG_REGION_INDEX, 4, 2, 438 &cmd_reg, true); 439 if (rc != 0) { 440 SPDK_ERRLOG("Write PCI CMD REG failed\n"); 441 virtio_dev_destruct(vdev); 442 return rc; 443 } 444 445 /* TODO: we cat get virtio device PCI common space layout via 446 * iterating vendor capabilities in PCI Configuration space, 447 * while here we use hardcoded layout first, this feature can 448 * be added in future. 449 * 450 * vfio-user emulated virtio device layout in Target: 451 * 452 * region 1: MSI-X Table 453 * region 2: MSI-X PBA 454 * region 4: virtio modern memory 64bits BAR 455 * Common configuration 0x0 - 0x1000 456 * ISR access 0x1000 - 0x2000 457 * Device specific configuration 0x2000 - 0x3000 458 * Notifications 0x3000 - 0x4000 459 */ 460 dev->pci_cap_region = VFIO_PCI_BAR4_REGION_INDEX; 461 dev->pci_cap_common_cfg_offset = 0x0; 462 dev->pci_cap_common_cfg_length = 0x1000; 463 dev->pci_cap_device_specific_offset = 0x2000; 464 dev->pci_cap_device_specific_length = 0x1000; 465 dev->pci_cap_notifications_offset = 0x3000; 466 dev->pci_cap_notifications_length = 0x1000; 467 468 return 0; 469 } 470 471 SPDK_LOG_REGISTER_COMPONENT(virtio_vfio_user) 472