1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 /* 7 * vfio-user transport for PCI devices. 8 */ 9 10 #include "spdk/stdinc.h" 11 #include "spdk/log.h" 12 #include "spdk/env.h" 13 #include "spdk/queue.h" 14 #include "spdk/util.h" 15 #include "spdk/vfio_user_pci.h" 16 17 #include "vfio_user_internal.h" 18 19 static uint32_t g_vfio_dev_id; 20 21 int 22 spdk_vfio_user_pci_bar_access(struct vfio_device *dev, uint32_t index, uint64_t offset, 23 size_t len, void *buf, bool is_write) 24 { 25 struct vfio_pci_region *region = &dev->regions[index]; 26 uint32_t i; 27 28 if (offset + len > region->size) { 29 return -EINVAL; 30 } 31 32 if (!region->nr_mmaps || (offset < region->mmaps[0].offset)) { 33 return vfio_user_dev_mmio_access(dev, index, offset, len, buf, is_write); 34 } 35 36 /* SPARSE MMAP */ 37 for (i = 0; i < region->nr_mmaps; i++) { 38 if ((offset >= region->mmaps[i].offset) && 39 (offset + len <= region->mmaps[i].offset + region->mmaps[i].size)) { 40 assert(region->mmaps[i].mem != NULL); 41 void *bar_addr = region->mmaps[i].mem + offset - region->mmaps[i].offset; 42 if (is_write) { 43 memcpy(bar_addr, buf, len); 44 } else { 45 memcpy(buf, bar_addr, len); 46 } 47 return 0; 48 } 49 } 50 51 return -EFAULT; 52 } 53 54 static int 55 vfio_add_mr(struct vfio_device *dev, struct vfio_memory_region *mr) 56 { 57 if (dev->nr_mrs == VFIO_MAXIMUM_MEMORY_REGIONS) { 58 SPDK_ERRLOG("Maximum supported memory regions %d\n", VFIO_MAXIMUM_MEMORY_REGIONS); 59 return -EINVAL; 60 } 61 62 TAILQ_INSERT_TAIL(&dev->mrs_head, mr, link); 63 dev->nr_mrs++; 64 65 SPDK_DEBUGLOG(vfio_pci, "Add memory region: FD %d, VADDR 0x%lx, IOVA 0x%lx, Size 0x%lx\n", 66 mr->fd, mr->vaddr, mr->iova, mr->size); 67 68 return 0; 69 } 70 71 static struct vfio_memory_region * 72 vfio_get_mr(struct vfio_device *dev, uint64_t addr, size_t len) 73 { 74 struct vfio_memory_region *mr, *tmp_mr; 75 76 if (dev->nr_mrs == 0) { 77 return false; 78 } 79 80 TAILQ_FOREACH_SAFE(mr, &dev->mrs_head, link, tmp_mr) { 81 if ((mr->vaddr == addr) || (mr->iova == addr)) { 82 return mr; 83 } 84 } 85 86 return false; 87 } 88 89 static void 90 vfio_remove_mr(struct vfio_device *dev, uint64_t addr, size_t len) 91 { 92 struct vfio_memory_region *mr, *tmp_mr; 93 94 TAILQ_FOREACH_SAFE(mr, &dev->mrs_head, link, tmp_mr) { 95 if ((mr->vaddr == addr) || (mr->iova == addr)) { 96 SPDK_DEBUGLOG(vfio_pci, "Remove memory region: FD %d, VADDR 0x%lx, IOVA 0x%lx, Size 0x%lx\n", 97 mr->fd, mr->vaddr, mr->iova, mr->size); 98 TAILQ_REMOVE(&dev->mrs_head, mr, link); 99 assert(dev->nr_mrs > 0); 100 dev->nr_mrs--; 101 free(mr); 102 return; 103 } 104 } 105 } 106 107 static int 108 vfio_mr_map_notify(void *cb_ctx, struct spdk_mem_map *map, 109 enum spdk_mem_map_notify_action action, 110 void *vaddr, size_t size) 111 { 112 int ret; 113 struct vfio_device *dev = cb_ctx; 114 struct vfio_memory_region *mr; 115 uint64_t offset; 116 117 mr = vfio_get_mr(dev, (uint64_t)vaddr, size); 118 if (action == SPDK_MEM_MAP_NOTIFY_UNREGISTER) { 119 if (!mr) { 120 SPDK_ERRLOG("Memory region VADDR %p doesn't exist\n", vaddr); 121 return -EEXIST; 122 } 123 124 ret = vfio_user_dev_dma_map_unmap(dev, mr, false); 125 /* remove the memory region */ 126 vfio_remove_mr(dev, (uint64_t)vaddr, size); 127 return ret; 128 } 129 130 /* SPDK_MEM_MAP_NOTIFY_REGISTER */ 131 if (mr != NULL) { 132 SPDK_ERRLOG("Memory region VADDR 0x%lx already exist\n", mr->vaddr); 133 return -EEXIST; 134 } 135 136 mr = calloc(1, sizeof(*mr)); 137 if (mr == NULL) { 138 return -ENOMEM; 139 } 140 mr->vaddr = (uint64_t)(uintptr_t)vaddr; 141 mr->iova = mr->vaddr; 142 mr->size = size; 143 mr->fd = spdk_mem_get_fd_and_offset(vaddr, &offset); 144 if (mr->fd < 0) { 145 SPDK_ERRLOG("Error to get the memory map offset\n"); 146 free(mr); 147 return -EFAULT; 148 } 149 mr->offset = offset; 150 151 ret = vfio_add_mr(dev, mr); 152 if (ret) { 153 free(mr); 154 return ret; 155 } 156 157 return vfio_user_dev_dma_map_unmap(dev, mr, true); 158 } 159 160 static int 161 vfio_device_dma_map(struct vfio_device *device) 162 { 163 const struct spdk_mem_map_ops vfio_map_ops = { 164 .notify_cb = vfio_mr_map_notify, 165 .are_contiguous = NULL, 166 }; 167 168 device->map = spdk_mem_map_alloc((uint64_t)NULL, &vfio_map_ops, device); 169 if (device->map == NULL) { 170 SPDK_ERRLOG("Failed to allocate memory map structure\n"); 171 return -EFAULT; 172 } 173 174 return 0; 175 } 176 177 static struct vfio_info_cap_header * 178 vfio_device_get_info_cap(struct vfio_region_info *info, int cap) 179 { 180 struct vfio_info_cap_header *h; 181 size_t offset; 182 183 if ((info->flags & VFIO_REGION_INFO_FLAG_CAPS) == 0) { 184 return NULL; 185 } 186 187 offset = info->cap_offset; 188 while (offset != 0) { 189 h = (struct vfio_info_cap_header *)((uintptr_t)info + offset); 190 if (h->id == cap) { 191 return h; 192 } 193 offset = h->next; 194 } 195 196 return NULL; 197 } 198 199 static int 200 vfio_device_setup_sparse_mmaps(struct vfio_device *device, int index, 201 struct vfio_region_info *info, int *fds) 202 { 203 struct vfio_info_cap_header *hdr; 204 struct vfio_region_info_cap_sparse_mmap *sparse; 205 struct vfio_pci_region *region = &device->regions[index]; 206 uint32_t i, j = 0; 207 int prot = 0; 208 209 hdr = vfio_device_get_info_cap(info, VFIO_REGION_INFO_CAP_SPARSE_MMAP); 210 if (!hdr) { 211 SPDK_NOTICELOG("Device doesn't have sparse mmap\n"); 212 return -EEXIST; 213 } 214 215 sparse = SPDK_CONTAINEROF(hdr, struct vfio_region_info_cap_sparse_mmap, header); 216 for (i = 0; i < sparse->nr_areas; i++) { 217 if (sparse->areas[i].size) { 218 region->mmaps[j].offset = sparse->areas[i].offset; 219 region->mmaps[j].size = sparse->areas[i].size; 220 prot |= info->flags & VFIO_REGION_INFO_FLAG_READ ? PROT_READ : 0; 221 prot |= info->flags & VFIO_REGION_INFO_FLAG_WRITE ? PROT_WRITE : 0; 222 if (*fds) { 223 region->mmaps[j].mem = mmap(NULL, region->mmaps[j].size, prot, MAP_SHARED, 224 fds[i], region->offset + region->mmaps[j].offset); 225 if (region->mmaps[j].mem == MAP_FAILED) { 226 SPDK_ERRLOG("Device SPARSE MMAP failed\n"); 227 return -EIO; 228 } 229 } else { 230 SPDK_DEBUGLOG(vfio_pci, "No valid fd, skip mmap for bar %d region %u\n", index, i); 231 } 232 SPDK_DEBUGLOG(vfio_pci, "Sparse region %u, Size 0x%llx, Offset 0x%llx, Map addr %p\n", 233 i, sparse->areas[i].size, sparse->areas[i].offset, 234 region->mmaps[j].mem); 235 j++; 236 } 237 } 238 device->regions[index].nr_mmaps = j; 239 240 return 0; 241 } 242 243 static int 244 vfio_device_map_region(struct vfio_device *device, struct vfio_pci_region *region, int fd) 245 { 246 int prot = 0; 247 248 prot |= region->flags & VFIO_REGION_INFO_FLAG_READ ? PROT_READ : 0; 249 prot |= region->flags & VFIO_REGION_INFO_FLAG_WRITE ? PROT_WRITE : 0; 250 251 region->mmaps[0].offset = 0; 252 region->mmaps[0].size = region->size; 253 254 region->mmaps[0].mem = mmap(NULL, region->size, prot, MAP_SHARED, 255 fd, region->offset); 256 if (region->mmaps[0].mem == MAP_FAILED) { 257 SPDK_ERRLOG("Device Region MMAP failed\n"); 258 return -EFAULT; 259 } 260 SPDK_DEBUGLOG(vfio_pci, "Memory mapped to %p\n", region->mmaps[0].mem); 261 region->nr_mmaps = 1; 262 263 return 0; 264 } 265 266 static int 267 vfio_device_map_bars_and_config_region(struct vfio_device *device) 268 { 269 uint32_t i; 270 int ret; 271 size_t len = 4096; 272 int fds[VFIO_MAXIMUM_SPARSE_MMAP_REGIONS]; 273 struct vfio_region_info *info; 274 uint8_t *buf; 275 276 buf = calloc(1, len); 277 if (!buf) { 278 return -ENOMEM; 279 } 280 281 info = (struct vfio_region_info *)buf; 282 for (i = 0; i < device->pci_regions; i++) { 283 memset(info, 0, len); 284 memset(fds, 0, sizeof(fds)); 285 286 info->index = i; 287 ret = vfio_user_get_dev_region_info(device, info, len, fds, VFIO_MAXIMUM_SPARSE_MMAP_REGIONS); 288 if (ret) { 289 SPDK_ERRLOG("Device setup bar %d failed\n", ret); 290 free(buf); 291 return ret; 292 } 293 294 device->regions[i].size = info->size; 295 device->regions[i].offset = info->offset; 296 device->regions[i].flags = info->flags; 297 298 SPDK_DEBUGLOG(vfio_pci, "Bar %d, Size 0x%llx, Offset 0x%llx, Flags 0x%x, Cap offset %u\n", 299 i, info->size, info->offset, info->flags, info->cap_offset); 300 301 /* Setup MMAP if any */ 302 if (info->size && (info->flags & VFIO_REGION_INFO_FLAG_MMAP)) { 303 /* try to map sparse memory region first */ 304 ret = vfio_device_setup_sparse_mmaps(device, i, info, fds); 305 if (ret < 0) { 306 ret = vfio_device_map_region(device, &device->regions[i], fds[0]); 307 } 308 309 if (ret != 0) { 310 SPDK_ERRLOG("Setup Device %s region %d failed\n", device->name, i); 311 free(buf); 312 return ret; 313 } 314 } 315 } 316 317 free(buf); 318 return 0; 319 } 320 321 static void 322 vfio_device_unmap_bars(struct vfio_device *dev) 323 { 324 uint32_t i, j; 325 struct vfio_pci_region *region; 326 327 for (i = 0; i < dev->pci_regions; i++) { 328 region = &dev->regions[i]; 329 for (j = 0; j < region->nr_mmaps; j++) { 330 if (region->mmaps[j].mem) { 331 munmap(region->mmaps[j].mem, region->mmaps[j].size); 332 } 333 } 334 } 335 memset(dev->regions, 0, sizeof(dev->regions)); 336 } 337 338 struct vfio_device * 339 spdk_vfio_user_setup(const char *path) 340 { 341 int ret; 342 struct vfio_device *device = NULL; 343 struct vfio_user_device_info dev_info = {}; 344 345 device = calloc(1, sizeof(*device)); 346 if (!device) { 347 return NULL; 348 } 349 TAILQ_INIT(&device->mrs_head); 350 snprintf(device->path, PATH_MAX, "%s", path); 351 snprintf(device->name, sizeof(device->name), "vfio-user%u", g_vfio_dev_id++); 352 353 ret = vfio_user_dev_setup(device); 354 if (ret) { 355 free(device); 356 SPDK_ERRLOG("Error to setup vfio-user via path %s\n", path); 357 return NULL; 358 } 359 360 ret = vfio_user_get_dev_info(device, &dev_info, sizeof(dev_info)); 361 if (ret) { 362 SPDK_ERRLOG("Device get info failed\n"); 363 goto cleanup; 364 } 365 device->pci_regions = dev_info.num_regions; 366 device->flags = dev_info.flags; 367 368 ret = vfio_device_map_bars_and_config_region(device); 369 if (ret) { 370 goto cleanup; 371 } 372 373 /* Register DMA Region */ 374 ret = vfio_device_dma_map(device); 375 if (ret) { 376 SPDK_ERRLOG("Container DMA map failed\n"); 377 goto cleanup; 378 } 379 380 SPDK_DEBUGLOG(vfio_pci, "Device %s, Path %s Setup Successfully\n", device->name, device->path); 381 382 return device; 383 384 cleanup: 385 close(device->fd); 386 free(device); 387 return NULL; 388 } 389 390 void 391 spdk_vfio_user_release(struct vfio_device *dev) 392 { 393 SPDK_DEBUGLOG(vfio_pci, "Release file %s\n", dev->path); 394 395 vfio_device_unmap_bars(dev); 396 if (dev->map) { 397 spdk_mem_map_free(&dev->map); 398 } 399 close(dev->fd); 400 401 free(dev); 402 } 403 404 void * 405 spdk_vfio_user_get_bar_addr(struct vfio_device *dev, uint32_t index, uint64_t offset, uint32_t len) 406 { 407 struct vfio_pci_region *region = &dev->regions[index]; 408 uint32_t i; 409 410 if (!region->size || !(region->flags & VFIO_REGION_INFO_FLAG_MMAP)) { 411 return NULL; 412 } 413 414 for (i = 0; i < region->nr_mmaps; i++) { 415 if (region->mmaps[i].mem && (region->mmaps[i].offset <= offset) && 416 ((offset + len) <= (region->mmaps[i].offset + region->mmaps[i].size))) { 417 return (void *)((uintptr_t)region->mmaps[i].mem + offset - region->mmaps[i].offset); 418 } 419 } 420 421 return NULL; 422 } 423 424 /* For fuzzing only */ 425 int 426 spdk_vfio_user_dev_send_request(struct vfio_device *dev, enum vfio_user_command command, 427 void *arg, size_t arg_len, size_t buf_len, int *fds, 428 int max_fds) 429 { 430 return vfio_user_dev_send_request(dev, command, arg, arg_len, buf_len, fds, max_fds); 431 } 432 433 SPDK_LOG_REGISTER_COMPONENT(vfio_pci) 434