1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 /* 7 * vfio-user transport for PCI devices. 8 */ 9 10 #include "spdk/stdinc.h" 11 #include "spdk/log.h" 12 #include "spdk/env.h" 13 #include "spdk/queue.h" 14 #include "spdk/util.h" 15 #include "spdk/vfio_user_pci.h" 16 17 #include "vfio_user_internal.h" 18 19 static uint32_t g_vfio_dev_id; 20 21 int 22 spdk_vfio_user_pci_bar_access(struct vfio_device *dev, uint32_t index, uint64_t offset, 23 size_t len, void *buf, bool is_write) 24 { 25 struct vfio_pci_region *region = &dev->regions[index]; 26 uint32_t i; 27 28 if (offset + len > region->size) { 29 return -EINVAL; 30 } 31 32 if (!region->nr_mmaps || (offset < region->mmaps[0].offset)) { 33 return vfio_user_dev_mmio_access(dev, index, offset, len, buf, is_write); 34 } 35 36 /* SPARSE MMAP */ 37 for (i = 0; i < region->nr_mmaps; i++) { 38 if ((offset >= region->mmaps[i].offset) && 39 (offset + len <= region->mmaps[i].offset + region->mmaps[i].size)) { 40 assert(region->mmaps[i].mem != NULL); 41 void *bar_addr = region->mmaps[i].mem + offset - region->mmaps[i].offset; 42 if (is_write) { 43 memcpy(bar_addr, buf, len); 44 } else { 45 memcpy(buf, bar_addr, len); 46 } 47 return 0; 48 } 49 } 50 51 return -EFAULT; 52 } 53 54 static int 55 vfio_add_mr(struct vfio_device *dev, struct vfio_memory_region *mr) 56 { 57 if (dev->nr_mrs == VFIO_MAXIMUM_MEMORY_REGIONS) { 58 SPDK_ERRLOG("Maximum supported memory regions %d\n", VFIO_MAXIMUM_MEMORY_REGIONS); 59 return -EINVAL; 60 } 61 62 TAILQ_INSERT_TAIL(&dev->mrs_head, mr, link); 63 dev->nr_mrs++; 64 65 SPDK_DEBUGLOG(vfio_pci, "Add memory region: FD %d, VADDR 0x%lx, IOVA 0x%lx, Size 0x%lx\n", 66 mr->fd, mr->vaddr, mr->iova, mr->size); 67 68 return 0; 69 } 70 71 static struct vfio_memory_region * 72 vfio_get_mr(struct vfio_device *dev, uint64_t addr, size_t len) 73 { 74 struct vfio_memory_region *mr, *tmp_mr; 75 76 if (dev->nr_mrs == 0) { 77 return false; 78 } 79 80 TAILQ_FOREACH_SAFE(mr, &dev->mrs_head, link, tmp_mr) { 81 if ((mr->vaddr == addr) || (mr->iova == addr)) { 82 return mr; 83 } 84 } 85 86 return false; 87 } 88 89 static void 90 vfio_remove_mr(struct vfio_device *dev, uint64_t addr, size_t len) 91 { 92 struct vfio_memory_region *mr, *tmp_mr; 93 94 TAILQ_FOREACH_SAFE(mr, &dev->mrs_head, link, tmp_mr) { 95 if ((mr->vaddr == addr) || (mr->iova == addr)) { 96 SPDK_DEBUGLOG(vfio_pci, "Remove memory region: FD %d, VADDR 0x%lx, IOVA 0x%lx, Size 0x%lx\n", 97 mr->fd, mr->vaddr, mr->iova, mr->size); 98 TAILQ_REMOVE(&dev->mrs_head, mr, link); 99 assert(dev->nr_mrs > 0); 100 dev->nr_mrs--; 101 free(mr); 102 return; 103 } 104 } 105 } 106 107 static int 108 vfio_mr_map_notify(void *cb_ctx, struct spdk_mem_map *map, 109 enum spdk_mem_map_notify_action action, 110 void *vaddr, size_t size) 111 { 112 int ret; 113 struct vfio_device *dev = cb_ctx; 114 struct vfio_memory_region *mr; 115 uint64_t offset; 116 117 mr = vfio_get_mr(dev, (uint64_t)vaddr, size); 118 if (action == SPDK_MEM_MAP_NOTIFY_UNREGISTER) { 119 if (!mr) { 120 SPDK_ERRLOG("Memory region VADDR %p doesn't exist\n", vaddr); 121 return -EEXIST; 122 } 123 124 ret = vfio_user_dev_dma_map_unmap(dev, mr, false); 125 /* remove the memory region */ 126 vfio_remove_mr(dev, (uint64_t)vaddr, size); 127 return ret; 128 } 129 130 /* SPDK_MEM_MAP_NOTIFY_REGISTER */ 131 if (mr != NULL) { 132 SPDK_ERRLOG("Memory region VADDR 0x%lx already exist\n", mr->vaddr); 133 return -EEXIST; 134 } 135 136 mr = calloc(1, sizeof(*mr)); 137 if (mr == NULL) { 138 return -ENOMEM; 139 } 140 mr->vaddr = (uint64_t)(uintptr_t)vaddr; 141 mr->iova = mr->vaddr; 142 mr->size = size; 143 mr->fd = spdk_mem_get_fd_and_offset(vaddr, &offset); 144 if (mr->fd < 0) { 145 SPDK_ERRLOG("Error to get the memory map offset\n"); 146 free(mr); 147 return -EFAULT; 148 } 149 mr->offset = offset; 150 151 ret = vfio_add_mr(dev, mr); 152 if (ret) { 153 free(mr); 154 return ret; 155 } 156 157 return vfio_user_dev_dma_map_unmap(dev, mr, true); 158 } 159 160 static int 161 vfio_device_dma_map(struct vfio_device *device) 162 { 163 const struct spdk_mem_map_ops vfio_map_ops = { 164 .notify_cb = vfio_mr_map_notify, 165 .are_contiguous = NULL, 166 }; 167 168 device->map = spdk_mem_map_alloc((uint64_t)NULL, &vfio_map_ops, device); 169 if (device->map == NULL) { 170 SPDK_ERRLOG("Failed to allocate memory map structure\n"); 171 return -EFAULT; 172 } 173 174 return 0; 175 } 176 177 static struct vfio_info_cap_header * 178 vfio_device_get_info_cap(struct vfio_region_info *info, int cap) 179 { 180 struct vfio_info_cap_header *h; 181 size_t offset; 182 183 if ((info->flags & VFIO_REGION_INFO_FLAG_CAPS) == 0) { 184 return NULL; 185 } 186 187 offset = info->cap_offset; 188 while (offset != 0) { 189 h = (struct vfio_info_cap_header *)((uintptr_t)info + offset); 190 if (h->id == cap) { 191 return h; 192 } 193 offset = h->next; 194 } 195 196 return NULL; 197 } 198 199 static int 200 vfio_device_setup_sparse_mmaps(struct vfio_device *device, int index, 201 struct vfio_region_info *info, int *fds) 202 { 203 struct vfio_info_cap_header *hdr; 204 struct vfio_region_info_cap_sparse_mmap *sparse; 205 struct vfio_pci_region *region = &device->regions[index]; 206 uint32_t i, j = 0; 207 int rc = 0, prot = 0; 208 209 hdr = vfio_device_get_info_cap(info, VFIO_REGION_INFO_CAP_SPARSE_MMAP); 210 if (!hdr) { 211 SPDK_NOTICELOG("Device doesn't have sparse mmap\n"); 212 return -EEXIST; 213 } 214 215 sparse = SPDK_CONTAINEROF(hdr, struct vfio_region_info_cap_sparse_mmap, header); 216 for (i = 0; i < sparse->nr_areas; i++) { 217 if (sparse->areas[i].size) { 218 region->mmaps[j].offset = sparse->areas[i].offset; 219 region->mmaps[j].size = sparse->areas[i].size; 220 prot |= info->flags & VFIO_REGION_INFO_FLAG_READ ? PROT_READ : 0; 221 prot |= info->flags & VFIO_REGION_INFO_FLAG_WRITE ? PROT_WRITE : 0; 222 if (*fds) { 223 region->mmaps[j].mem = mmap(NULL, region->mmaps[j].size, prot, MAP_SHARED, 224 fds[i], region->offset + region->mmaps[j].offset); 225 if (region->mmaps[j].mem == MAP_FAILED) { 226 SPDK_ERRLOG("Device SPARSE MMAP failed\n"); 227 rc = -EIO; 228 goto out; 229 } 230 } else { 231 SPDK_DEBUGLOG(vfio_pci, "No valid fd, skip mmap for bar %d region %u\n", index, i); 232 } 233 SPDK_DEBUGLOG(vfio_pci, "Sparse region %u, Size 0x%llx, Offset 0x%llx, Map addr %p\n", 234 i, sparse->areas[i].size, sparse->areas[i].offset, 235 region->mmaps[j].mem); 236 j++; 237 } 238 } 239 device->regions[index].nr_mmaps = j; 240 out: 241 for (i = 0; i < sparse->nr_areas; i++) { 242 close(fds[i]); 243 } 244 245 return rc; 246 } 247 248 static int 249 vfio_device_map_region(struct vfio_device *device, struct vfio_pci_region *region, int fd) 250 { 251 int prot = 0; 252 253 prot |= region->flags & VFIO_REGION_INFO_FLAG_READ ? PROT_READ : 0; 254 prot |= region->flags & VFIO_REGION_INFO_FLAG_WRITE ? PROT_WRITE : 0; 255 256 region->mmaps[0].offset = 0; 257 region->mmaps[0].size = region->size; 258 259 region->mmaps[0].mem = mmap(NULL, region->size, prot, MAP_SHARED, 260 fd, region->offset); 261 close(fd); 262 if (region->mmaps[0].mem == MAP_FAILED) { 263 SPDK_ERRLOG("Device Region MMAP failed\n"); 264 return -EFAULT; 265 } 266 SPDK_DEBUGLOG(vfio_pci, "Memory mapped to %p\n", region->mmaps[0].mem); 267 region->nr_mmaps = 1; 268 269 return 0; 270 } 271 272 static int 273 vfio_device_map_bars_and_config_region(struct vfio_device *device) 274 { 275 uint32_t i; 276 int ret; 277 size_t len = 4096; 278 int fds[VFIO_MAXIMUM_SPARSE_MMAP_REGIONS]; 279 struct vfio_region_info *info; 280 uint8_t *buf; 281 282 buf = calloc(1, len); 283 if (!buf) { 284 return -ENOMEM; 285 } 286 287 info = (struct vfio_region_info *)buf; 288 for (i = 0; i < device->pci_regions; i++) { 289 memset(info, 0, len); 290 memset(fds, 0, sizeof(fds)); 291 292 info->index = i; 293 ret = vfio_user_get_dev_region_info(device, info, len, fds, VFIO_MAXIMUM_SPARSE_MMAP_REGIONS); 294 if (ret) { 295 SPDK_ERRLOG("Device setup bar %d failed\n", ret); 296 free(buf); 297 return ret; 298 } 299 300 device->regions[i].size = info->size; 301 device->regions[i].offset = info->offset; 302 device->regions[i].flags = info->flags; 303 304 SPDK_DEBUGLOG(vfio_pci, "Bar %d, Size 0x%llx, Offset 0x%llx, Flags 0x%x, Cap offset %u\n", 305 i, info->size, info->offset, info->flags, info->cap_offset); 306 307 /* Setup MMAP if any */ 308 if (info->size && (info->flags & VFIO_REGION_INFO_FLAG_MMAP)) { 309 /* try to map sparse memory region first */ 310 ret = vfio_device_setup_sparse_mmaps(device, i, info, fds); 311 if (ret < 0) { 312 ret = vfio_device_map_region(device, &device->regions[i], fds[0]); 313 } 314 315 if (ret != 0) { 316 SPDK_ERRLOG("Setup Device %s region %d failed\n", device->name, i); 317 free(buf); 318 return ret; 319 } 320 } 321 } 322 323 free(buf); 324 return 0; 325 } 326 327 static void 328 vfio_device_unmap_bars(struct vfio_device *dev) 329 { 330 uint32_t i, j; 331 struct vfio_pci_region *region; 332 333 for (i = 0; i < dev->pci_regions; i++) { 334 region = &dev->regions[i]; 335 for (j = 0; j < region->nr_mmaps; j++) { 336 if (region->mmaps[j].mem) { 337 munmap(region->mmaps[j].mem, region->mmaps[j].size); 338 } 339 } 340 } 341 memset(dev->regions, 0, sizeof(dev->regions)); 342 } 343 344 struct vfio_device * 345 spdk_vfio_user_setup(const char *path) 346 { 347 int ret; 348 struct vfio_device *device = NULL; 349 struct vfio_user_device_info dev_info = {}; 350 351 device = calloc(1, sizeof(*device)); 352 if (!device) { 353 return NULL; 354 } 355 TAILQ_INIT(&device->mrs_head); 356 snprintf(device->path, PATH_MAX, "%s", path); 357 snprintf(device->name, sizeof(device->name), "vfio-user%u", g_vfio_dev_id++); 358 359 ret = vfio_user_dev_setup(device); 360 if (ret) { 361 free(device); 362 SPDK_ERRLOG("Error to setup vfio-user via path %s\n", path); 363 return NULL; 364 } 365 366 ret = vfio_user_get_dev_info(device, &dev_info, sizeof(dev_info)); 367 if (ret) { 368 SPDK_ERRLOG("Device get info failed\n"); 369 goto cleanup; 370 } 371 device->pci_regions = dev_info.num_regions; 372 device->flags = dev_info.flags; 373 374 ret = vfio_device_map_bars_and_config_region(device); 375 if (ret) { 376 goto cleanup; 377 } 378 379 /* Register DMA Region */ 380 ret = vfio_device_dma_map(device); 381 if (ret) { 382 SPDK_ERRLOG("Container DMA map failed\n"); 383 goto cleanup; 384 } 385 386 SPDK_DEBUGLOG(vfio_pci, "Device %s, Path %s Setup Successfully\n", device->name, device->path); 387 388 return device; 389 390 cleanup: 391 close(device->fd); 392 free(device); 393 return NULL; 394 } 395 396 void 397 spdk_vfio_user_release(struct vfio_device *dev) 398 { 399 SPDK_DEBUGLOG(vfio_pci, "Release file %s\n", dev->path); 400 401 vfio_device_unmap_bars(dev); 402 if (dev->map) { 403 spdk_mem_map_free(&dev->map); 404 } 405 close(dev->fd); 406 407 free(dev); 408 } 409 410 void * 411 spdk_vfio_user_get_bar_addr(struct vfio_device *dev, uint32_t index, uint64_t offset, uint32_t len) 412 { 413 struct vfio_pci_region *region = &dev->regions[index]; 414 uint32_t i; 415 416 if (!region->size || !(region->flags & VFIO_REGION_INFO_FLAG_MMAP)) { 417 return NULL; 418 } 419 420 for (i = 0; i < region->nr_mmaps; i++) { 421 if (region->mmaps[i].mem && (region->mmaps[i].offset <= offset) && 422 ((offset + len) <= (region->mmaps[i].offset + region->mmaps[i].size))) { 423 return (void *)((uintptr_t)region->mmaps[i].mem + offset - region->mmaps[i].offset); 424 } 425 } 426 427 return NULL; 428 } 429 430 /* For fuzzing only */ 431 int 432 spdk_vfio_user_dev_send_request(struct vfio_device *dev, enum vfio_user_command command, 433 void *arg, size_t arg_len, size_t buf_len, int *fds, 434 int max_fds) 435 { 436 return vfio_user_dev_send_request(dev, command, arg, arg_len, buf_len, fds, max_fds); 437 } 438 439 SPDK_LOG_REGISTER_COMPONENT(vfio_pci) 440