1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 /* 35 * vfio-user transport for PCI devices. 36 */ 37 38 #include "spdk/stdinc.h" 39 #include "spdk/log.h" 40 #include "spdk/env.h" 41 #include "spdk/queue.h" 42 #include "spdk/util.h" 43 #include "spdk/vfio_user_pci.h" 44 45 #include "vfio_user_internal.h" 46 47 static uint32_t g_vfio_dev_id; 48 49 int 50 spdk_vfio_user_pci_bar_access(struct vfio_device *dev, uint32_t index, uint64_t offset, 51 size_t len, void *buf, bool is_write) 52 { 53 struct vfio_pci_region *region = &dev->regions[index]; 54 uint32_t i; 55 56 if (offset + len > region->size) { 57 return -EINVAL; 58 } 59 60 if (!region->nr_mmaps || (offset < region->mmaps[0].offset)) { 61 return vfio_user_dev_mmio_access(dev, index, offset, len, buf, is_write); 62 } 63 64 /* SPARSE MMAP */ 65 for (i = 0; i < region->nr_mmaps; i++) { 66 if ((offset >= region->mmaps[i].offset) && 67 (offset + len <= region->mmaps[i].offset + region->mmaps[i].size)) { 68 assert(region->mmaps[i].mem != NULL); 69 void *bar_addr = region->mmaps[i].mem + offset; 70 if (is_write) { 71 memcpy(bar_addr, buf, len); 72 } else { 73 memcpy(buf, bar_addr, len); 74 } 75 return 0; 76 } 77 } 78 79 return -EFAULT; 80 } 81 82 static int 83 vfio_add_mr(struct vfio_device *dev, struct vfio_memory_region *mr) 84 { 85 if (dev->nr_mrs == VFIO_MAXIMUM_MEMORY_REGIONS) { 86 SPDK_ERRLOG("Maximum supported memory regions %d\n", VFIO_MAXIMUM_MEMORY_REGIONS); 87 return -EINVAL; 88 } 89 90 TAILQ_INSERT_TAIL(&dev->mrs_head, mr, link); 91 dev->nr_mrs++; 92 93 SPDK_DEBUGLOG(vfio_pci, "Add memory region: FD %d, VADDR 0x%lx, IOVA 0x%lx, Size 0x%lx\n", 94 mr->fd, mr->vaddr, mr->iova, mr->size); 95 96 return 0; 97 } 98 99 static struct vfio_memory_region * 100 vfio_get_mr(struct vfio_device *dev, uint64_t addr, size_t len) 101 { 102 struct vfio_memory_region *mr, *tmp_mr; 103 104 if (dev->nr_mrs == 0) { 105 return false; 106 } 107 108 TAILQ_FOREACH_SAFE(mr, &dev->mrs_head, link, tmp_mr) { 109 if ((mr->vaddr == addr) || (mr->iova == addr)) { 110 return mr; 111 } 112 } 113 114 return false; 115 } 116 117 static void 118 vfio_remove_mr(struct vfio_device *dev, uint64_t addr, size_t len) 119 { 120 struct vfio_memory_region *mr, *tmp_mr; 121 122 TAILQ_FOREACH_SAFE(mr, &dev->mrs_head, link, tmp_mr) { 123 if ((mr->vaddr == addr) || (mr->iova == addr)) { 124 SPDK_DEBUGLOG(vfio_pci, "Remove memory region: FD %d, VADDR 0x%lx, IOVA 0x%lx, Size 0x%lx\n", 125 mr->fd, mr->vaddr, mr->iova, mr->size); 126 TAILQ_REMOVE(&dev->mrs_head, mr, link); 127 assert(dev->nr_mrs > 0); 128 dev->nr_mrs--; 129 free(mr); 130 return; 131 } 132 } 133 } 134 135 static int 136 vfio_mr_map_notify(void *cb_ctx, struct spdk_mem_map *map, 137 enum spdk_mem_map_notify_action action, 138 void *vaddr, size_t size) 139 { 140 int ret; 141 struct vfio_device *dev = cb_ctx; 142 struct vfio_memory_region *mr; 143 uint64_t offset; 144 145 mr = vfio_get_mr(dev, (uint64_t)vaddr, size); 146 if (action == SPDK_MEM_MAP_NOTIFY_UNREGISTER) { 147 if (!mr) { 148 SPDK_ERRLOG("Memory region VADDR %p doesn't exist\n", vaddr); 149 return -EEXIST; 150 } 151 152 ret = vfio_user_dev_dma_map_unmap(dev, mr, false); 153 /* remove the memory region */ 154 vfio_remove_mr(dev, (uint64_t)vaddr, size); 155 return ret; 156 } 157 158 /* SPDK_MEM_MAP_NOTIFY_REGISTER */ 159 if (mr != NULL) { 160 SPDK_ERRLOG("Memory region VADDR 0x%lx already exist\n", mr->vaddr); 161 return -EEXIST; 162 } 163 164 mr = calloc(1, sizeof(*mr)); 165 if (mr == NULL) { 166 return -ENOMEM; 167 } 168 mr->vaddr = (uint64_t)(uintptr_t)vaddr; 169 mr->iova = mr->vaddr; 170 mr->size = size; 171 mr->fd = spdk_mem_get_fd_and_offset(vaddr, &offset); 172 if (mr->fd < 0) { 173 SPDK_ERRLOG("Error to get the memory map offset\n"); 174 free(mr); 175 return -EFAULT; 176 } 177 mr->offset = offset; 178 179 ret = vfio_add_mr(dev, mr); 180 if (ret) { 181 free(mr); 182 return ret; 183 } 184 185 return vfio_user_dev_dma_map_unmap(dev, mr, true); 186 } 187 188 static int 189 vfio_device_dma_map(struct vfio_device *device) 190 { 191 const struct spdk_mem_map_ops vfio_map_ops = { 192 .notify_cb = vfio_mr_map_notify, 193 .are_contiguous = NULL, 194 }; 195 196 device->map = spdk_mem_map_alloc((uint64_t)NULL, &vfio_map_ops, device); 197 if (device->map == NULL) { 198 SPDK_ERRLOG("Failed to allocate memory map structure\n"); 199 return -EFAULT; 200 } 201 202 return 0; 203 } 204 205 static struct vfio_info_cap_header * 206 vfio_device_get_info_cap(struct vfio_region_info *info, int cap) 207 { 208 struct vfio_info_cap_header *h; 209 size_t offset; 210 211 if ((info->flags & VFIO_REGION_INFO_FLAG_CAPS) == 0) { 212 return NULL; 213 } 214 215 offset = info->cap_offset; 216 while (offset != 0) { 217 h = (struct vfio_info_cap_header *)((uintptr_t)info + offset); 218 if (h->id == cap) { 219 return h; 220 } 221 offset = h->next; 222 } 223 224 return NULL; 225 } 226 227 static int 228 vfio_device_setup_sparse_mmaps(struct vfio_device *device, int index, 229 struct vfio_region_info *info, int *fds) 230 { 231 struct vfio_info_cap_header *hdr; 232 struct vfio_region_info_cap_sparse_mmap *sparse; 233 struct vfio_pci_region *region = &device->regions[index]; 234 uint32_t i, j = 0; 235 int prot = 0; 236 237 hdr = vfio_device_get_info_cap(info, VFIO_REGION_INFO_CAP_SPARSE_MMAP); 238 if (!hdr) { 239 SPDK_NOTICELOG("Device doesn't have sparse mmap\n"); 240 return -EEXIST; 241 } 242 243 sparse = SPDK_CONTAINEROF(hdr, struct vfio_region_info_cap_sparse_mmap, header); 244 for (i = 0; i < sparse->nr_areas; i++) { 245 if (sparse->areas[i].size) { 246 region->mmaps[j].offset = sparse->areas[i].offset; 247 region->mmaps[j].size = sparse->areas[i].size; 248 prot |= info->flags & VFIO_REGION_INFO_FLAG_READ ? PROT_READ : 0; 249 prot |= info->flags & VFIO_REGION_INFO_FLAG_WRITE ? PROT_WRITE : 0; 250 if (*fds) { 251 region->mmaps[j].mem = mmap(NULL, region->mmaps[j].size, prot, MAP_SHARED, 252 fds[i], region->offset + region->mmaps[j].offset); 253 if (region->mmaps[j].mem == MAP_FAILED) { 254 SPDK_ERRLOG("Device SPARSE MMAP failed\n"); 255 return -EIO; 256 } 257 } else { 258 SPDK_DEBUGLOG(vfio_pci, "No valid fd, skip mmap for bar %d region %u\n", index, i); 259 } 260 SPDK_DEBUGLOG(vfio_pci, "Sparse region %u, Size 0x%llx, Offset 0x%llx, Map addr %p\n", 261 i, sparse->areas[i].size, sparse->areas[i].offset, 262 region->mmaps[j].mem); 263 j++; 264 } 265 } 266 device->regions[index].nr_mmaps = j; 267 268 return 0; 269 } 270 271 static int 272 vfio_device_map_region(struct vfio_device *device, struct vfio_pci_region *region, int fd) 273 { 274 int prot = 0; 275 276 prot |= region->flags & VFIO_REGION_INFO_FLAG_READ ? PROT_READ : 0; 277 prot |= region->flags & VFIO_REGION_INFO_FLAG_WRITE ? PROT_WRITE : 0; 278 279 region->mmaps[0].offset = 0; 280 region->mmaps[0].size = region->size; 281 282 region->mmaps[0].mem = mmap(NULL, region->size, prot, MAP_SHARED, 283 fd, region->offset); 284 if (region->mmaps[0].mem == MAP_FAILED) { 285 SPDK_ERRLOG("Device Region MMAP failed\n"); 286 return -EFAULT; 287 } 288 SPDK_DEBUGLOG(vfio_pci, "Memory mapped to %p\n", region->mmaps[0].mem); 289 region->nr_mmaps = 1; 290 291 return 0; 292 } 293 294 static int 295 vfio_device_map_bars_and_config_region(struct vfio_device *device) 296 { 297 uint32_t i; 298 int ret; 299 size_t len = 4096; 300 int fds[VFIO_MAXIMUM_SPARSE_MMAP_REGIONS]; 301 struct vfio_region_info *info; 302 uint8_t *buf; 303 304 buf = calloc(1, len); 305 if (!buf) { 306 return -ENOMEM; 307 } 308 309 info = (struct vfio_region_info *)buf; 310 for (i = 0; i < device->pci_regions; i++) { 311 memset(info, 0, len); 312 memset(fds, 0, sizeof(fds)); 313 314 info->index = i; 315 ret = vfio_user_get_dev_region_info(device, info, len, fds, VFIO_MAXIMUM_SPARSE_MMAP_REGIONS); 316 if (ret) { 317 SPDK_ERRLOG("Device setup bar %d failed\n", ret); 318 free(buf); 319 return ret; 320 } 321 322 device->regions[i].size = info->size; 323 device->regions[i].offset = info->offset; 324 device->regions[i].flags = info->flags; 325 326 SPDK_DEBUGLOG(vfio_pci, "Bar %d, Size 0x%llx, Offset 0x%llx, Flags 0x%x, Cap offset %u\n", 327 i, info->size, info->offset, info->flags, info->cap_offset); 328 329 /* Setup MMAP if any */ 330 if (info->size && (info->flags & VFIO_REGION_INFO_FLAG_MMAP)) { 331 /* try to map sparse memory region first */ 332 ret = vfio_device_setup_sparse_mmaps(device, i, info, fds); 333 if (ret < 0) { 334 ret = vfio_device_map_region(device, &device->regions[i], fds[0]); 335 } 336 337 if (ret != 0) { 338 SPDK_ERRLOG("Setup Device %s region %d failed\n", device->name, i); 339 free(buf); 340 return ret; 341 } 342 } 343 } 344 345 free(buf); 346 return 0; 347 } 348 349 static void 350 vfio_device_unmap_bars(struct vfio_device *dev) 351 { 352 uint32_t i, j; 353 struct vfio_pci_region *region; 354 355 for (i = 0; i < dev->pci_regions; i++) { 356 region = &dev->regions[i]; 357 for (j = 0; j < region->nr_mmaps; j++) { 358 if (region->mmaps[j].mem) { 359 munmap(region->mmaps[j].mem, region->mmaps[j].size); 360 } 361 } 362 } 363 memset(dev->regions, 0, sizeof(dev->regions)); 364 } 365 366 struct vfio_device * 367 spdk_vfio_user_setup(const char *path) 368 { 369 int ret; 370 struct vfio_device *device = NULL; 371 struct vfio_user_device_info dev_info = {}; 372 373 device = calloc(1, sizeof(*device)); 374 if (!device) { 375 return NULL; 376 } 377 TAILQ_INIT(&device->mrs_head); 378 snprintf(device->path, PATH_MAX, "%s", path); 379 snprintf(device->name, sizeof(device->name), "vfio-user%u", g_vfio_dev_id++); 380 381 ret = vfio_user_dev_setup(device); 382 if (ret) { 383 free(device); 384 SPDK_ERRLOG("Error to setup vfio-user via path %s\n", path); 385 return NULL; 386 } 387 388 ret = vfio_user_get_dev_info(device, &dev_info, sizeof(dev_info)); 389 if (ret) { 390 SPDK_ERRLOG("Device get info failed\n"); 391 goto cleanup; 392 } 393 device->pci_regions = dev_info.num_regions; 394 device->flags = dev_info.flags; 395 396 ret = vfio_device_map_bars_and_config_region(device); 397 if (ret) { 398 goto cleanup; 399 } 400 401 /* Register DMA Region */ 402 ret = vfio_device_dma_map(device); 403 if (ret) { 404 SPDK_ERRLOG("Container DMA map failed\n"); 405 goto cleanup; 406 } 407 408 SPDK_DEBUGLOG(vfio_pci, "Device %s, Path %s Setup Successfully\n", device->name, device->path); 409 410 return device; 411 412 cleanup: 413 close(device->fd); 414 free(device); 415 return NULL; 416 } 417 418 void 419 spdk_vfio_user_release(struct vfio_device *dev) 420 { 421 SPDK_DEBUGLOG(vfio_pci, "Release file %s\n", dev->path); 422 423 vfio_device_unmap_bars(dev); 424 if (dev->map) { 425 spdk_mem_map_free(&dev->map); 426 } 427 close(dev->fd); 428 429 free(dev); 430 } 431 432 void * 433 spdk_vfio_user_get_bar_addr(struct vfio_device *dev, uint32_t index, uint64_t offset, uint32_t len) 434 { 435 struct vfio_pci_region *region = &dev->regions[index]; 436 uint32_t i; 437 438 if (!region->size || !(region->flags & VFIO_REGION_INFO_FLAG_MMAP)) { 439 return NULL; 440 } 441 442 for (i = 0; i < region->nr_mmaps; i++) { 443 if (region->mmaps[i].mem && (region->mmaps[i].offset <= offset) && 444 ((offset + len) <= (region->mmaps[i].offset + region->mmaps[i].size))) { 445 return (void *)((uintptr_t)region->mmaps[i].mem + offset - region->mmaps[i].offset); 446 } 447 } 448 449 return NULL; 450 } 451 452 SPDK_LOG_REGISTER_COMPONENT(vfio_pci) 453