1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "spdk/stdinc.h" 35 36 #include "env_internal.h" 37 38 #include <rte_config.h> 39 #include <rte_malloc.h> 40 #include <rte_memory.h> 41 #include <rte_eal_memconfig.h> 42 43 #include "spdk_internal/assert.h" 44 #include "spdk_internal/memory.h" 45 46 #include "spdk/assert.h" 47 #include "spdk/likely.h" 48 #include "spdk/queue.h" 49 #include "spdk/util.h" 50 #include "spdk/env_dpdk.h" 51 52 #ifdef __FreeBSD__ 53 #define SPDK_VFIO_ENABLED 0 54 #else 55 #include <linux/version.h> 56 #if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 6, 0) 57 #define SPDK_VFIO_ENABLED 1 58 #include <linux/vfio.h> 59 #include <rte_vfio.h> 60 61 struct spdk_vfio_dma_map { 62 struct vfio_iommu_type1_dma_map map; 63 struct vfio_iommu_type1_dma_unmap unmap; 64 TAILQ_ENTRY(spdk_vfio_dma_map) tailq; 65 }; 66 67 struct vfio_cfg { 68 int fd; 69 bool enabled; 70 bool noiommu_enabled; 71 unsigned device_ref; 72 TAILQ_HEAD(, spdk_vfio_dma_map) maps; 73 pthread_mutex_t mutex; 74 }; 75 76 static struct vfio_cfg g_vfio = { 77 .fd = -1, 78 .enabled = false, 79 .noiommu_enabled = false, 80 .device_ref = 0, 81 .maps = TAILQ_HEAD_INITIALIZER(g_vfio.maps), 82 .mutex = PTHREAD_MUTEX_INITIALIZER 83 }; 84 85 #else 86 #define SPDK_VFIO_ENABLED 0 87 #endif 88 #endif 89 90 #if DEBUG 91 #define DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__) 92 #else 93 #define DEBUG_PRINT(...) 94 #endif 95 96 #define FN_2MB_TO_4KB(fn) (fn << (SHIFT_2MB - SHIFT_4KB)) 97 #define FN_4KB_TO_2MB(fn) (fn >> (SHIFT_2MB - SHIFT_4KB)) 98 99 #define MAP_256TB_IDX(vfn_2mb) ((vfn_2mb) >> (SHIFT_1GB - SHIFT_2MB)) 100 #define MAP_1GB_IDX(vfn_2mb) ((vfn_2mb) & ((1ULL << (SHIFT_1GB - SHIFT_2MB)) - 1)) 101 102 /* Page is registered */ 103 #define REG_MAP_REGISTERED (1ULL << 62) 104 105 /* A notification region barrier. The 2MB translation entry that's marked 106 * with this flag must be unregistered separately. This allows contiguous 107 * regions to be unregistered in the same chunks they were registered. 108 */ 109 #define REG_MAP_NOTIFY_START (1ULL << 63) 110 111 /* Translation of a single 2MB page. */ 112 struct map_2mb { 113 uint64_t translation_2mb; 114 }; 115 116 /* Second-level map table indexed by bits [21..29] of the virtual address. 117 * Each entry contains the address translation or error for entries that haven't 118 * been retrieved yet. 119 */ 120 struct map_1gb { 121 struct map_2mb map[1ULL << (SHIFT_1GB - SHIFT_2MB)]; 122 }; 123 124 /* Top-level map table indexed by bits [30..47] of the virtual address. 125 * Each entry points to a second-level map table or NULL. 126 */ 127 struct map_256tb { 128 struct map_1gb *map[1ULL << (SHIFT_256TB - SHIFT_1GB)]; 129 }; 130 131 /* Page-granularity memory address translation */ 132 struct spdk_mem_map { 133 struct map_256tb map_256tb; 134 pthread_mutex_t mutex; 135 uint64_t default_translation; 136 struct spdk_mem_map_ops ops; 137 void *cb_ctx; 138 TAILQ_ENTRY(spdk_mem_map) tailq; 139 }; 140 141 /* Registrations map. The 64 bit translations are bit fields with the 142 * following layout (starting with the low bits): 143 * 0 - 61 : reserved 144 * 62 - 63 : flags 145 */ 146 static struct spdk_mem_map *g_mem_reg_map; 147 static TAILQ_HEAD(, spdk_mem_map) g_spdk_mem_maps = TAILQ_HEAD_INITIALIZER(g_spdk_mem_maps); 148 static pthread_mutex_t g_spdk_mem_map_mutex = PTHREAD_MUTEX_INITIALIZER; 149 150 static bool g_legacy_mem; 151 152 /* 153 * Walk the currently registered memory via the main memory registration map 154 * and call the new map's notify callback for each virtually contiguous region. 155 */ 156 static int 157 spdk_mem_map_notify_walk(struct spdk_mem_map *map, enum spdk_mem_map_notify_action action) 158 { 159 size_t idx_256tb; 160 uint64_t idx_1gb; 161 uint64_t contig_start = UINT64_MAX; 162 uint64_t contig_end = UINT64_MAX; 163 struct map_1gb *map_1gb; 164 int rc; 165 166 if (!g_mem_reg_map) { 167 return -EINVAL; 168 } 169 170 /* Hold the memory registration map mutex so no new registrations can be added while we are looping. */ 171 pthread_mutex_lock(&g_mem_reg_map->mutex); 172 173 for (idx_256tb = 0; 174 idx_256tb < sizeof(g_mem_reg_map->map_256tb.map) / sizeof(g_mem_reg_map->map_256tb.map[0]); 175 idx_256tb++) { 176 map_1gb = g_mem_reg_map->map_256tb.map[idx_256tb]; 177 178 if (!map_1gb) { 179 if (contig_start != UINT64_MAX) { 180 /* End of of a virtually contiguous range */ 181 rc = map->ops.notify_cb(map->cb_ctx, map, action, 182 (void *)contig_start, 183 contig_end - contig_start + VALUE_2MB); 184 /* Don't bother handling unregister failures. It can't be any worse */ 185 if (rc != 0 && action == SPDK_MEM_MAP_NOTIFY_REGISTER) { 186 goto err_unregister; 187 } 188 } 189 contig_start = UINT64_MAX; 190 continue; 191 } 192 193 for (idx_1gb = 0; idx_1gb < sizeof(map_1gb->map) / sizeof(map_1gb->map[0]); idx_1gb++) { 194 if ((map_1gb->map[idx_1gb].translation_2mb & REG_MAP_REGISTERED) && 195 (contig_start == UINT64_MAX || 196 (map_1gb->map[idx_1gb].translation_2mb & REG_MAP_NOTIFY_START) == 0)) { 197 /* Rebuild the virtual address from the indexes */ 198 uint64_t vaddr = (idx_256tb << SHIFT_1GB) | (idx_1gb << SHIFT_2MB); 199 200 if (contig_start == UINT64_MAX) { 201 contig_start = vaddr; 202 } 203 204 contig_end = vaddr; 205 } else { 206 if (contig_start != UINT64_MAX) { 207 /* End of of a virtually contiguous range */ 208 rc = map->ops.notify_cb(map->cb_ctx, map, action, 209 (void *)contig_start, 210 contig_end - contig_start + VALUE_2MB); 211 /* Don't bother handling unregister failures. It can't be any worse */ 212 if (rc != 0 && action == SPDK_MEM_MAP_NOTIFY_REGISTER) { 213 goto err_unregister; 214 } 215 216 /* This page might be a part of a neighbour region, so process 217 * it again. The idx_1gb will be incremented immediately. 218 */ 219 idx_1gb--; 220 } 221 contig_start = UINT64_MAX; 222 } 223 } 224 } 225 226 pthread_mutex_unlock(&g_mem_reg_map->mutex); 227 return 0; 228 229 err_unregister: 230 /* Unwind to the first empty translation so we don't unregister 231 * a region that just failed to register. 232 */ 233 idx_256tb = MAP_256TB_IDX((contig_start >> SHIFT_2MB) - 1); 234 idx_1gb = MAP_1GB_IDX((contig_start >> SHIFT_2MB) - 1); 235 contig_start = UINT64_MAX; 236 contig_end = UINT64_MAX; 237 238 /* Unregister any memory we managed to register before the failure */ 239 for (; idx_256tb < SIZE_MAX; idx_256tb--) { 240 map_1gb = g_mem_reg_map->map_256tb.map[idx_256tb]; 241 242 if (!map_1gb) { 243 if (contig_end != UINT64_MAX) { 244 /* End of of a virtually contiguous range */ 245 map->ops.notify_cb(map->cb_ctx, map, 246 SPDK_MEM_MAP_NOTIFY_UNREGISTER, 247 (void *)contig_start, 248 contig_end - contig_start + VALUE_2MB); 249 } 250 contig_end = UINT64_MAX; 251 continue; 252 } 253 254 for (; idx_1gb < UINT64_MAX; idx_1gb--) { 255 if ((map_1gb->map[idx_1gb].translation_2mb & REG_MAP_REGISTERED) && 256 (contig_end == UINT64_MAX || (map_1gb->map[idx_1gb].translation_2mb & REG_MAP_NOTIFY_START) == 0)) { 257 /* Rebuild the virtual address from the indexes */ 258 uint64_t vaddr = (idx_256tb << SHIFT_1GB) | (idx_1gb << SHIFT_2MB); 259 260 if (contig_end == UINT64_MAX) { 261 contig_end = vaddr; 262 } 263 contig_start = vaddr; 264 } else { 265 if (contig_end != UINT64_MAX) { 266 /* End of of a virtually contiguous range */ 267 map->ops.notify_cb(map->cb_ctx, map, 268 SPDK_MEM_MAP_NOTIFY_UNREGISTER, 269 (void *)contig_start, 270 contig_end - contig_start + VALUE_2MB); 271 idx_1gb++; 272 } 273 contig_end = UINT64_MAX; 274 } 275 } 276 idx_1gb = sizeof(map_1gb->map) / sizeof(map_1gb->map[0]) - 1; 277 } 278 279 pthread_mutex_unlock(&g_mem_reg_map->mutex); 280 return rc; 281 } 282 283 struct spdk_mem_map * 284 spdk_mem_map_alloc(uint64_t default_translation, const struct spdk_mem_map_ops *ops, void *cb_ctx) 285 { 286 struct spdk_mem_map *map; 287 int rc; 288 289 map = calloc(1, sizeof(*map)); 290 if (map == NULL) { 291 return NULL; 292 } 293 294 if (pthread_mutex_init(&map->mutex, NULL)) { 295 free(map); 296 return NULL; 297 } 298 299 map->default_translation = default_translation; 300 map->cb_ctx = cb_ctx; 301 if (ops) { 302 map->ops = *ops; 303 } 304 305 if (ops && ops->notify_cb) { 306 pthread_mutex_lock(&g_spdk_mem_map_mutex); 307 rc = spdk_mem_map_notify_walk(map, SPDK_MEM_MAP_NOTIFY_REGISTER); 308 if (rc != 0) { 309 pthread_mutex_unlock(&g_spdk_mem_map_mutex); 310 DEBUG_PRINT("Initial mem_map notify failed\n"); 311 pthread_mutex_destroy(&map->mutex); 312 free(map); 313 return NULL; 314 } 315 TAILQ_INSERT_TAIL(&g_spdk_mem_maps, map, tailq); 316 pthread_mutex_unlock(&g_spdk_mem_map_mutex); 317 } 318 319 return map; 320 } 321 322 void 323 spdk_mem_map_free(struct spdk_mem_map **pmap) 324 { 325 struct spdk_mem_map *map; 326 size_t i; 327 328 if (!pmap) { 329 return; 330 } 331 332 map = *pmap; 333 334 if (!map) { 335 return; 336 } 337 338 if (map->ops.notify_cb) { 339 pthread_mutex_lock(&g_spdk_mem_map_mutex); 340 spdk_mem_map_notify_walk(map, SPDK_MEM_MAP_NOTIFY_UNREGISTER); 341 TAILQ_REMOVE(&g_spdk_mem_maps, map, tailq); 342 pthread_mutex_unlock(&g_spdk_mem_map_mutex); 343 } 344 345 for (i = 0; i < sizeof(map->map_256tb.map) / sizeof(map->map_256tb.map[0]); i++) { 346 if (g_legacy_mem) { 347 rte_free(map->map_256tb.map[i]); 348 } else { 349 free(map->map_256tb.map[i]); 350 } 351 } 352 353 pthread_mutex_destroy(&map->mutex); 354 355 free(map); 356 *pmap = NULL; 357 } 358 359 int 360 spdk_mem_register(void *vaddr, size_t len) 361 { 362 struct spdk_mem_map *map; 363 int rc; 364 void *seg_vaddr; 365 size_t seg_len; 366 uint64_t reg; 367 368 if ((uintptr_t)vaddr & ~MASK_256TB) { 369 DEBUG_PRINT("invalid usermode virtual address %p\n", vaddr); 370 return -EINVAL; 371 } 372 373 if (((uintptr_t)vaddr & MASK_2MB) || (len & MASK_2MB)) { 374 DEBUG_PRINT("invalid %s parameters, vaddr=%p len=%ju\n", 375 __func__, vaddr, len); 376 return -EINVAL; 377 } 378 379 if (len == 0) { 380 return 0; 381 } 382 383 pthread_mutex_lock(&g_spdk_mem_map_mutex); 384 385 seg_vaddr = vaddr; 386 seg_len = len; 387 while (seg_len > 0) { 388 reg = spdk_mem_map_translate(g_mem_reg_map, (uint64_t)seg_vaddr, NULL); 389 if (reg & REG_MAP_REGISTERED) { 390 pthread_mutex_unlock(&g_spdk_mem_map_mutex); 391 return -EBUSY; 392 } 393 seg_vaddr += VALUE_2MB; 394 seg_len -= VALUE_2MB; 395 } 396 397 seg_vaddr = vaddr; 398 seg_len = 0; 399 while (len > 0) { 400 spdk_mem_map_set_translation(g_mem_reg_map, (uint64_t)vaddr, VALUE_2MB, 401 seg_len == 0 ? REG_MAP_REGISTERED | REG_MAP_NOTIFY_START : REG_MAP_REGISTERED); 402 seg_len += VALUE_2MB; 403 vaddr += VALUE_2MB; 404 len -= VALUE_2MB; 405 } 406 407 TAILQ_FOREACH(map, &g_spdk_mem_maps, tailq) { 408 rc = map->ops.notify_cb(map->cb_ctx, map, SPDK_MEM_MAP_NOTIFY_REGISTER, seg_vaddr, seg_len); 409 if (rc != 0) { 410 pthread_mutex_unlock(&g_spdk_mem_map_mutex); 411 return rc; 412 } 413 } 414 415 pthread_mutex_unlock(&g_spdk_mem_map_mutex); 416 return 0; 417 } 418 419 int 420 spdk_mem_unregister(void *vaddr, size_t len) 421 { 422 struct spdk_mem_map *map; 423 int rc; 424 void *seg_vaddr; 425 size_t seg_len; 426 uint64_t reg, newreg; 427 428 if ((uintptr_t)vaddr & ~MASK_256TB) { 429 DEBUG_PRINT("invalid usermode virtual address %p\n", vaddr); 430 return -EINVAL; 431 } 432 433 if (((uintptr_t)vaddr & MASK_2MB) || (len & MASK_2MB)) { 434 DEBUG_PRINT("invalid %s parameters, vaddr=%p len=%ju\n", 435 __func__, vaddr, len); 436 return -EINVAL; 437 } 438 439 pthread_mutex_lock(&g_spdk_mem_map_mutex); 440 441 /* The first page must be a start of a region. Also check if it's 442 * registered to make sure we don't return -ERANGE for non-registered 443 * regions. 444 */ 445 reg = spdk_mem_map_translate(g_mem_reg_map, (uint64_t)vaddr, NULL); 446 if ((reg & REG_MAP_REGISTERED) && (reg & REG_MAP_NOTIFY_START) == 0) { 447 pthread_mutex_unlock(&g_spdk_mem_map_mutex); 448 return -ERANGE; 449 } 450 451 seg_vaddr = vaddr; 452 seg_len = len; 453 while (seg_len > 0) { 454 reg = spdk_mem_map_translate(g_mem_reg_map, (uint64_t)seg_vaddr, NULL); 455 if ((reg & REG_MAP_REGISTERED) == 0) { 456 pthread_mutex_unlock(&g_spdk_mem_map_mutex); 457 return -EINVAL; 458 } 459 seg_vaddr += VALUE_2MB; 460 seg_len -= VALUE_2MB; 461 } 462 463 newreg = spdk_mem_map_translate(g_mem_reg_map, (uint64_t)seg_vaddr, NULL); 464 /* If the next page is registered, it must be a start of a region as well, 465 * otherwise we'd be unregistering only a part of a region. 466 */ 467 if ((newreg & REG_MAP_NOTIFY_START) == 0 && (newreg & REG_MAP_REGISTERED)) { 468 pthread_mutex_unlock(&g_spdk_mem_map_mutex); 469 return -ERANGE; 470 } 471 seg_vaddr = vaddr; 472 seg_len = 0; 473 474 while (len > 0) { 475 reg = spdk_mem_map_translate(g_mem_reg_map, (uint64_t)vaddr, NULL); 476 spdk_mem_map_set_translation(g_mem_reg_map, (uint64_t)vaddr, VALUE_2MB, 0); 477 478 if (seg_len > 0 && (reg & REG_MAP_NOTIFY_START)) { 479 TAILQ_FOREACH(map, &g_spdk_mem_maps, tailq) { 480 rc = map->ops.notify_cb(map->cb_ctx, map, SPDK_MEM_MAP_NOTIFY_UNREGISTER, seg_vaddr, seg_len); 481 if (rc != 0) { 482 pthread_mutex_unlock(&g_spdk_mem_map_mutex); 483 return rc; 484 } 485 } 486 487 seg_vaddr = vaddr; 488 seg_len = VALUE_2MB; 489 } else { 490 seg_len += VALUE_2MB; 491 } 492 493 vaddr += VALUE_2MB; 494 len -= VALUE_2MB; 495 } 496 497 if (seg_len > 0) { 498 TAILQ_FOREACH(map, &g_spdk_mem_maps, tailq) { 499 rc = map->ops.notify_cb(map->cb_ctx, map, SPDK_MEM_MAP_NOTIFY_UNREGISTER, seg_vaddr, seg_len); 500 if (rc != 0) { 501 pthread_mutex_unlock(&g_spdk_mem_map_mutex); 502 return rc; 503 } 504 } 505 } 506 507 pthread_mutex_unlock(&g_spdk_mem_map_mutex); 508 return 0; 509 } 510 511 static struct map_1gb * 512 spdk_mem_map_get_map_1gb(struct spdk_mem_map *map, uint64_t vfn_2mb) 513 { 514 struct map_1gb *map_1gb; 515 uint64_t idx_256tb = MAP_256TB_IDX(vfn_2mb); 516 size_t i; 517 518 if (spdk_unlikely(idx_256tb >= SPDK_COUNTOF(map->map_256tb.map))) { 519 return NULL; 520 } 521 522 map_1gb = map->map_256tb.map[idx_256tb]; 523 524 if (!map_1gb) { 525 pthread_mutex_lock(&map->mutex); 526 527 /* Recheck to make sure nobody else got the mutex first. */ 528 map_1gb = map->map_256tb.map[idx_256tb]; 529 if (!map_1gb) { 530 /* Some of the existing apps use TCMalloc hugepage 531 * allocator and register this tcmalloc allocated 532 * hugepage memory with SPDK in the mmap hook. Since 533 * this function is called in the spdk_mem_register 534 * code path we can't do a malloc here otherwise that 535 * would cause a livelock. So we use the dpdk provided 536 * allocator instead, which avoids this cyclic 537 * dependency. Note this is only guaranteed to work when 538 * DPDK dynamic memory allocation is disabled (--legacy-mem), 539 * which then is a requirement for anyone using TCMalloc in 540 * this way. 541 */ 542 if (g_legacy_mem) { 543 map_1gb = rte_malloc(NULL, sizeof(struct map_1gb), 0); 544 } else { 545 map_1gb = malloc(sizeof(struct map_1gb)); 546 } 547 if (map_1gb) { 548 /* initialize all entries to default translation */ 549 for (i = 0; i < SPDK_COUNTOF(map_1gb->map); i++) { 550 map_1gb->map[i].translation_2mb = map->default_translation; 551 } 552 map->map_256tb.map[idx_256tb] = map_1gb; 553 } 554 } 555 556 pthread_mutex_unlock(&map->mutex); 557 558 if (!map_1gb) { 559 DEBUG_PRINT("allocation failed\n"); 560 return NULL; 561 } 562 } 563 564 return map_1gb; 565 } 566 567 int 568 spdk_mem_map_set_translation(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size, 569 uint64_t translation) 570 { 571 uint64_t vfn_2mb; 572 struct map_1gb *map_1gb; 573 uint64_t idx_1gb; 574 struct map_2mb *map_2mb; 575 576 if ((uintptr_t)vaddr & ~MASK_256TB) { 577 DEBUG_PRINT("invalid usermode virtual address %lu\n", vaddr); 578 return -EINVAL; 579 } 580 581 /* For now, only 2 MB-aligned registrations are supported */ 582 if (((uintptr_t)vaddr & MASK_2MB) || (size & MASK_2MB)) { 583 DEBUG_PRINT("invalid %s parameters, vaddr=%lu len=%ju\n", 584 __func__, vaddr, size); 585 return -EINVAL; 586 } 587 588 vfn_2mb = vaddr >> SHIFT_2MB; 589 590 while (size) { 591 map_1gb = spdk_mem_map_get_map_1gb(map, vfn_2mb); 592 if (!map_1gb) { 593 DEBUG_PRINT("could not get %p map\n", (void *)vaddr); 594 return -ENOMEM; 595 } 596 597 idx_1gb = MAP_1GB_IDX(vfn_2mb); 598 map_2mb = &map_1gb->map[idx_1gb]; 599 map_2mb->translation_2mb = translation; 600 601 size -= VALUE_2MB; 602 vfn_2mb++; 603 } 604 605 return 0; 606 } 607 608 int 609 spdk_mem_map_clear_translation(struct spdk_mem_map *map, uint64_t vaddr, uint64_t size) 610 { 611 return spdk_mem_map_set_translation(map, vaddr, size, map->default_translation); 612 } 613 614 inline uint64_t 615 spdk_mem_map_translate(const struct spdk_mem_map *map, uint64_t vaddr, uint64_t *size) 616 { 617 const struct map_1gb *map_1gb; 618 const struct map_2mb *map_2mb; 619 uint64_t idx_256tb; 620 uint64_t idx_1gb; 621 uint64_t vfn_2mb; 622 uint64_t cur_size; 623 uint64_t prev_translation; 624 uint64_t orig_translation; 625 626 if (spdk_unlikely(vaddr & ~MASK_256TB)) { 627 DEBUG_PRINT("invalid usermode virtual address %p\n", (void *)vaddr); 628 return map->default_translation; 629 } 630 631 vfn_2mb = vaddr >> SHIFT_2MB; 632 idx_256tb = MAP_256TB_IDX(vfn_2mb); 633 idx_1gb = MAP_1GB_IDX(vfn_2mb); 634 635 map_1gb = map->map_256tb.map[idx_256tb]; 636 if (spdk_unlikely(!map_1gb)) { 637 return map->default_translation; 638 } 639 640 cur_size = VALUE_2MB - _2MB_OFFSET(vaddr); 641 map_2mb = &map_1gb->map[idx_1gb]; 642 if (size == NULL || map->ops.are_contiguous == NULL || 643 map_2mb->translation_2mb == map->default_translation) { 644 if (size != NULL) { 645 *size = spdk_min(*size, cur_size); 646 } 647 return map_2mb->translation_2mb; 648 } 649 650 orig_translation = map_2mb->translation_2mb; 651 prev_translation = orig_translation; 652 while (cur_size < *size) { 653 vfn_2mb++; 654 idx_256tb = MAP_256TB_IDX(vfn_2mb); 655 idx_1gb = MAP_1GB_IDX(vfn_2mb); 656 657 map_1gb = map->map_256tb.map[idx_256tb]; 658 if (spdk_unlikely(!map_1gb)) { 659 break; 660 } 661 662 map_2mb = &map_1gb->map[idx_1gb]; 663 if (!map->ops.are_contiguous(prev_translation, map_2mb->translation_2mb)) { 664 break; 665 } 666 667 cur_size += VALUE_2MB; 668 prev_translation = map_2mb->translation_2mb; 669 } 670 671 *size = spdk_min(*size, cur_size); 672 return orig_translation; 673 } 674 675 #if RTE_VERSION >= RTE_VERSION_NUM(18, 05, 0, 0) 676 static void 677 memory_hotplug_cb(enum rte_mem_event event_type, 678 const void *addr, size_t len, void *arg) 679 { 680 if (event_type == RTE_MEM_EVENT_ALLOC) { 681 spdk_mem_register((void *)addr, len); 682 683 #if RTE_VERSION >= RTE_VERSION_NUM(19, 02, 0, 0) 684 if (!spdk_env_dpdk_external_init()) { 685 return; 686 } 687 #endif 688 689 /* Prior to DPDK 19.02, we have to worry about DPDK 690 * freeing memory in different units than it was allocated. 691 * That doesn't work with things like RDMA MRs. So for 692 * those versions of DPDK, mark each segment so that DPDK 693 * won't later free it. That ensures we don't have to deal 694 * with that scenario. 695 * 696 * DPDK 19.02 added the --match-allocations RTE flag to 697 * avoid this condition. 698 * 699 * Note: if the user initialized DPDK separately, we can't 700 * be sure that --match-allocations was specified, so need 701 * to still mark the segments so they aren't freed. 702 */ 703 while (len > 0) { 704 struct rte_memseg *seg; 705 706 seg = rte_mem_virt2memseg(addr, NULL); 707 assert(seg != NULL); 708 seg->flags |= RTE_MEMSEG_FLAG_DO_NOT_FREE; 709 addr = (void *)((uintptr_t)addr + seg->hugepage_sz); 710 len -= seg->hugepage_sz; 711 } 712 } else if (event_type == RTE_MEM_EVENT_FREE) { 713 spdk_mem_unregister((void *)addr, len); 714 } 715 } 716 717 static int 718 memory_iter_cb(const struct rte_memseg_list *msl, 719 const struct rte_memseg *ms, size_t len, void *arg) 720 { 721 return spdk_mem_register(ms->addr, len); 722 } 723 #endif 724 725 int 726 spdk_mem_map_init(bool legacy_mem) 727 { 728 g_legacy_mem = legacy_mem; 729 730 g_mem_reg_map = spdk_mem_map_alloc(0, NULL, NULL); 731 if (g_mem_reg_map == NULL) { 732 DEBUG_PRINT("memory registration map allocation failed\n"); 733 return -ENOMEM; 734 } 735 736 /* 737 * Walk all DPDK memory segments and register them 738 * with the master memory map 739 */ 740 #if RTE_VERSION >= RTE_VERSION_NUM(18, 05, 0, 0) 741 rte_mem_event_callback_register("spdk", memory_hotplug_cb, NULL); 742 rte_memseg_contig_walk(memory_iter_cb, NULL); 743 #else 744 struct rte_mem_config *mcfg; 745 size_t seg_idx; 746 747 mcfg = rte_eal_get_configuration()->mem_config; 748 for (seg_idx = 0; seg_idx < RTE_MAX_MEMSEG; seg_idx++) { 749 struct rte_memseg *seg = &mcfg->memseg[seg_idx]; 750 751 if (seg->addr == NULL) { 752 break; 753 } 754 755 spdk_mem_register(seg->addr, seg->len); 756 } 757 #endif 758 return 0; 759 } 760 761 bool 762 spdk_iommu_is_enabled(void) 763 { 764 #if SPDK_VFIO_ENABLED 765 return g_vfio.enabled && !g_vfio.noiommu_enabled; 766 #else 767 return false; 768 #endif 769 } 770 771 struct spdk_vtophys_pci_device { 772 struct rte_pci_device *pci_device; 773 TAILQ_ENTRY(spdk_vtophys_pci_device) tailq; 774 }; 775 776 static pthread_mutex_t g_vtophys_pci_devices_mutex = PTHREAD_MUTEX_INITIALIZER; 777 static TAILQ_HEAD(, spdk_vtophys_pci_device) g_vtophys_pci_devices = 778 TAILQ_HEAD_INITIALIZER(g_vtophys_pci_devices); 779 780 static struct spdk_mem_map *g_vtophys_map; 781 782 #if SPDK_VFIO_ENABLED 783 static int 784 vtophys_iommu_map_dma(uint64_t vaddr, uint64_t iova, uint64_t size) 785 { 786 struct spdk_vfio_dma_map *dma_map; 787 int ret; 788 789 dma_map = calloc(1, sizeof(*dma_map)); 790 if (dma_map == NULL) { 791 return -ENOMEM; 792 } 793 794 dma_map->map.argsz = sizeof(dma_map->map); 795 dma_map->map.flags = VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE; 796 dma_map->map.vaddr = vaddr; 797 dma_map->map.iova = iova; 798 dma_map->map.size = size; 799 800 dma_map->unmap.argsz = sizeof(dma_map->unmap); 801 dma_map->unmap.flags = 0; 802 dma_map->unmap.iova = iova; 803 dma_map->unmap.size = size; 804 805 pthread_mutex_lock(&g_vfio.mutex); 806 if (g_vfio.device_ref == 0) { 807 /* VFIO requires at least one device (IOMMU group) to be added to 808 * a VFIO container before it is possible to perform any IOMMU 809 * operations on that container. This memory will be mapped once 810 * the first device (IOMMU group) is hotplugged. 811 * 812 * Since the vfio container is managed internally by DPDK, it is 813 * also possible that some device is already in that container, but 814 * it's not managed by SPDK - e.g. an NIC attached internally 815 * inside DPDK. We could map the memory straight away in such 816 * scenario, but there's no need to do it. DPDK devices clearly 817 * don't need our mappings and hence we defer the mapping 818 * unconditionally until the first SPDK-managed device is 819 * hotplugged. 820 */ 821 goto out_insert; 822 } 823 824 ret = ioctl(g_vfio.fd, VFIO_IOMMU_MAP_DMA, &dma_map->map); 825 if (ret) { 826 DEBUG_PRINT("Cannot set up DMA mapping, error %d\n", errno); 827 pthread_mutex_unlock(&g_vfio.mutex); 828 free(dma_map); 829 return ret; 830 } 831 832 out_insert: 833 TAILQ_INSERT_TAIL(&g_vfio.maps, dma_map, tailq); 834 pthread_mutex_unlock(&g_vfio.mutex); 835 return 0; 836 } 837 838 static int 839 vtophys_iommu_unmap_dma(uint64_t iova, uint64_t size) 840 { 841 struct spdk_vfio_dma_map *dma_map; 842 int ret; 843 844 pthread_mutex_lock(&g_vfio.mutex); 845 TAILQ_FOREACH(dma_map, &g_vfio.maps, tailq) { 846 if (dma_map->map.iova == iova) { 847 break; 848 } 849 } 850 851 if (dma_map == NULL) { 852 DEBUG_PRINT("Cannot clear DMA mapping for IOVA %"PRIx64" - it's not mapped\n", iova); 853 pthread_mutex_unlock(&g_vfio.mutex); 854 return -ENXIO; 855 } 856 857 /** don't support partial or multiple-page unmap for now */ 858 assert(dma_map->map.size == size); 859 860 if (g_vfio.device_ref == 0) { 861 /* Memory is not mapped anymore, just remove it's references */ 862 goto out_remove; 863 } 864 865 866 ret = ioctl(g_vfio.fd, VFIO_IOMMU_UNMAP_DMA, &dma_map->unmap); 867 if (ret) { 868 DEBUG_PRINT("Cannot clear DMA mapping, error %d\n", errno); 869 pthread_mutex_unlock(&g_vfio.mutex); 870 return ret; 871 } 872 873 out_remove: 874 TAILQ_REMOVE(&g_vfio.maps, dma_map, tailq); 875 pthread_mutex_unlock(&g_vfio.mutex); 876 free(dma_map); 877 return 0; 878 } 879 #endif 880 881 static uint64_t 882 vtophys_get_paddr_memseg(uint64_t vaddr) 883 { 884 uintptr_t paddr; 885 struct rte_memseg *seg; 886 887 #if RTE_VERSION >= RTE_VERSION_NUM(18, 05, 0, 0) 888 seg = rte_mem_virt2memseg((void *)(uintptr_t)vaddr, NULL); 889 if (seg != NULL) { 890 paddr = seg->phys_addr; 891 if (paddr == RTE_BAD_IOVA) { 892 return SPDK_VTOPHYS_ERROR; 893 } 894 paddr += (vaddr - (uintptr_t)seg->addr); 895 return paddr; 896 } 897 #else 898 struct rte_mem_config *mcfg; 899 uint32_t seg_idx; 900 901 mcfg = rte_eal_get_configuration()->mem_config; 902 for (seg_idx = 0; seg_idx < RTE_MAX_MEMSEG; seg_idx++) { 903 seg = &mcfg->memseg[seg_idx]; 904 if (seg->addr == NULL) { 905 break; 906 } 907 908 if (vaddr >= (uintptr_t)seg->addr && 909 vaddr < ((uintptr_t)seg->addr + seg->len)) { 910 paddr = seg->phys_addr; 911 if (paddr == RTE_BAD_IOVA) { 912 return SPDK_VTOPHYS_ERROR; 913 } 914 paddr += (vaddr - (uintptr_t)seg->addr); 915 return paddr; 916 } 917 } 918 #endif 919 920 return SPDK_VTOPHYS_ERROR; 921 } 922 923 /* Try to get the paddr from /proc/self/pagemap */ 924 static uint64_t 925 vtophys_get_paddr_pagemap(uint64_t vaddr) 926 { 927 uintptr_t paddr; 928 929 /* Silence static analyzers */ 930 assert(vaddr != 0); 931 paddr = rte_mem_virt2iova((void *)vaddr); 932 if (paddr == RTE_BAD_IOVA) { 933 /* 934 * The vaddr may be valid but doesn't have a backing page 935 * assigned yet. Touch the page to ensure a backing page 936 * gets assigned, then try to translate again. 937 */ 938 rte_atomic64_read((rte_atomic64_t *)vaddr); 939 paddr = rte_mem_virt2iova((void *)vaddr); 940 } 941 if (paddr == RTE_BAD_IOVA) { 942 /* Unable to get to the physical address. */ 943 return SPDK_VTOPHYS_ERROR; 944 } 945 946 return paddr; 947 } 948 949 /* Try to get the paddr from pci devices */ 950 static uint64_t 951 vtophys_get_paddr_pci(uint64_t vaddr) 952 { 953 struct spdk_vtophys_pci_device *vtophys_dev; 954 uintptr_t paddr; 955 struct rte_pci_device *dev; 956 struct rte_mem_resource *res; 957 unsigned r; 958 959 pthread_mutex_lock(&g_vtophys_pci_devices_mutex); 960 TAILQ_FOREACH(vtophys_dev, &g_vtophys_pci_devices, tailq) { 961 dev = vtophys_dev->pci_device; 962 963 for (r = 0; r < PCI_MAX_RESOURCE; r++) { 964 res = &dev->mem_resource[r]; 965 if (res->phys_addr && vaddr >= (uint64_t)res->addr && 966 vaddr < (uint64_t)res->addr + res->len) { 967 paddr = res->phys_addr + (vaddr - (uint64_t)res->addr); 968 DEBUG_PRINT("%s: %p -> %p\n", __func__, (void *)vaddr, 969 (void *)paddr); 970 pthread_mutex_unlock(&g_vtophys_pci_devices_mutex); 971 return paddr; 972 } 973 } 974 } 975 pthread_mutex_unlock(&g_vtophys_pci_devices_mutex); 976 977 return SPDK_VTOPHYS_ERROR; 978 } 979 980 static int 981 spdk_vtophys_notify(void *cb_ctx, struct spdk_mem_map *map, 982 enum spdk_mem_map_notify_action action, 983 void *vaddr, size_t len) 984 { 985 int rc = 0, pci_phys = 0; 986 uint64_t paddr; 987 988 if ((uintptr_t)vaddr & ~MASK_256TB) { 989 DEBUG_PRINT("invalid usermode virtual address %p\n", vaddr); 990 return -EINVAL; 991 } 992 993 if (((uintptr_t)vaddr & MASK_2MB) || (len & MASK_2MB)) { 994 DEBUG_PRINT("invalid parameters, vaddr=%p len=%ju\n", 995 vaddr, len); 996 return -EINVAL; 997 } 998 999 /* Get the physical address from the DPDK memsegs */ 1000 paddr = vtophys_get_paddr_memseg((uint64_t)vaddr); 1001 1002 switch (action) { 1003 case SPDK_MEM_MAP_NOTIFY_REGISTER: 1004 if (paddr == SPDK_VTOPHYS_ERROR) { 1005 /* This is not an address that DPDK is managing. */ 1006 #if SPDK_VFIO_ENABLED 1007 enum rte_iova_mode iova_mode; 1008 1009 #if RTE_VERSION >= RTE_VERSION_NUM(19, 11, 0, 0) 1010 iova_mode = rte_eal_iova_mode(); 1011 #else 1012 iova_mode = rte_eal_get_configuration()->iova_mode; 1013 #endif 1014 1015 if (spdk_iommu_is_enabled() && iova_mode == RTE_IOVA_VA) { 1016 /* We'll use the virtual address as the iova to match DPDK. */ 1017 paddr = (uint64_t)vaddr; 1018 rc = vtophys_iommu_map_dma((uint64_t)vaddr, paddr, len); 1019 if (rc) { 1020 return -EFAULT; 1021 } 1022 while (len > 0) { 1023 rc = spdk_mem_map_set_translation(map, (uint64_t)vaddr, VALUE_2MB, paddr); 1024 if (rc != 0) { 1025 return rc; 1026 } 1027 vaddr += VALUE_2MB; 1028 paddr += VALUE_2MB; 1029 len -= VALUE_2MB; 1030 } 1031 } else 1032 #endif 1033 { 1034 /* Get the physical address from /proc/self/pagemap. */ 1035 paddr = vtophys_get_paddr_pagemap((uint64_t)vaddr); 1036 if (paddr == SPDK_VTOPHYS_ERROR) { 1037 /* Get the physical address from PCI devices */ 1038 paddr = vtophys_get_paddr_pci((uint64_t)vaddr); 1039 if (paddr == SPDK_VTOPHYS_ERROR) { 1040 DEBUG_PRINT("could not get phys addr for %p\n", vaddr); 1041 return -EFAULT; 1042 } 1043 /* The beginning of this address range points to a PCI resource, 1044 * so the rest must point to a PCI resource as well. 1045 */ 1046 pci_phys = 1; 1047 } 1048 1049 /* Get paddr for each 2MB chunk in this address range */ 1050 while (len > 0) { 1051 /* Get the physical address from /proc/self/pagemap. */ 1052 if (pci_phys) { 1053 paddr = vtophys_get_paddr_pci((uint64_t)vaddr); 1054 } else { 1055 paddr = vtophys_get_paddr_pagemap((uint64_t)vaddr); 1056 } 1057 1058 if (paddr == SPDK_VTOPHYS_ERROR) { 1059 DEBUG_PRINT("could not get phys addr for %p\n", vaddr); 1060 return -EFAULT; 1061 } 1062 1063 /* Since PCI paddr can break the 2MiB physical alignment skip this check for that. */ 1064 if (!pci_phys && (paddr & MASK_2MB)) { 1065 DEBUG_PRINT("invalid paddr 0x%" PRIx64 " - must be 2MB aligned\n", paddr); 1066 return -EINVAL; 1067 } 1068 #if SPDK_VFIO_ENABLED 1069 /* If the IOMMU is on, but DPDK is using iova-mode=pa, we want to register this memory 1070 * with the IOMMU using the physical address to match. */ 1071 if (spdk_iommu_is_enabled()) { 1072 rc = vtophys_iommu_map_dma((uint64_t)vaddr, paddr, VALUE_2MB); 1073 if (rc) { 1074 DEBUG_PRINT("Unable to assign vaddr %p to paddr 0x%" PRIx64 "\n", vaddr, paddr); 1075 return -EFAULT; 1076 } 1077 } 1078 #endif 1079 1080 rc = spdk_mem_map_set_translation(map, (uint64_t)vaddr, VALUE_2MB, paddr); 1081 if (rc != 0) { 1082 return rc; 1083 } 1084 1085 vaddr += VALUE_2MB; 1086 len -= VALUE_2MB; 1087 } 1088 } 1089 } else { 1090 /* This is an address managed by DPDK. Just setup the translations. */ 1091 while (len > 0) { 1092 paddr = vtophys_get_paddr_memseg((uint64_t)vaddr); 1093 if (paddr == SPDK_VTOPHYS_ERROR) { 1094 DEBUG_PRINT("could not get phys addr for %p\n", vaddr); 1095 return -EFAULT; 1096 } 1097 1098 rc = spdk_mem_map_set_translation(map, (uint64_t)vaddr, VALUE_2MB, paddr); 1099 if (rc != 0) { 1100 return rc; 1101 } 1102 1103 vaddr += VALUE_2MB; 1104 len -= VALUE_2MB; 1105 } 1106 } 1107 1108 break; 1109 case SPDK_MEM_MAP_NOTIFY_UNREGISTER: 1110 #if SPDK_VFIO_ENABLED 1111 if (paddr == SPDK_VTOPHYS_ERROR) { 1112 /* 1113 * This is not an address that DPDK is managing. If vfio is enabled, 1114 * we need to unmap the range from the IOMMU 1115 */ 1116 if (spdk_iommu_is_enabled()) { 1117 uint64_t buffer_len = len; 1118 uint8_t *va = vaddr; 1119 enum rte_iova_mode iova_mode; 1120 1121 #if RTE_VERSION >= RTE_VERSION_NUM(19, 11, 0, 0) 1122 iova_mode = rte_eal_iova_mode(); 1123 #else 1124 iova_mode = rte_eal_get_configuration()->iova_mode; 1125 #endif 1126 /* 1127 * In virtual address mode, the region is contiguous and can be done in 1128 * one unmap. 1129 */ 1130 if (iova_mode == RTE_IOVA_VA) { 1131 paddr = spdk_mem_map_translate(map, (uint64_t)va, &buffer_len); 1132 if (buffer_len != len || paddr != (uintptr_t)va) { 1133 DEBUG_PRINT("Unmapping %p with length %lu failed because " 1134 "translation had address 0x%" PRIx64 " and length %lu\n", 1135 va, len, paddr, buffer_len); 1136 return -EINVAL; 1137 } 1138 rc = vtophys_iommu_unmap_dma(paddr, len); 1139 if (rc) { 1140 DEBUG_PRINT("Failed to iommu unmap paddr 0x%" PRIx64 "\n", paddr); 1141 return -EFAULT; 1142 } 1143 } else if (iova_mode == RTE_IOVA_PA) { 1144 /* Get paddr for each 2MB chunk in this address range */ 1145 while (buffer_len > 0) { 1146 paddr = spdk_mem_map_translate(map, (uint64_t)va, NULL); 1147 1148 if (paddr == SPDK_VTOPHYS_ERROR || buffer_len < VALUE_2MB) { 1149 DEBUG_PRINT("could not get phys addr for %p\n", va); 1150 return -EFAULT; 1151 } 1152 1153 rc = vtophys_iommu_unmap_dma(paddr, VALUE_2MB); 1154 if (rc) { 1155 DEBUG_PRINT("Failed to iommu unmap paddr 0x%" PRIx64 "\n", paddr); 1156 return -EFAULT; 1157 } 1158 1159 va += VALUE_2MB; 1160 buffer_len -= VALUE_2MB; 1161 } 1162 } 1163 } 1164 } 1165 #endif 1166 while (len > 0) { 1167 rc = spdk_mem_map_clear_translation(map, (uint64_t)vaddr, VALUE_2MB); 1168 if (rc != 0) { 1169 return rc; 1170 } 1171 1172 vaddr += VALUE_2MB; 1173 len -= VALUE_2MB; 1174 } 1175 1176 break; 1177 default: 1178 SPDK_UNREACHABLE(); 1179 } 1180 1181 return rc; 1182 } 1183 1184 static int 1185 vtophys_check_contiguous_entries(uint64_t paddr1, uint64_t paddr2) 1186 { 1187 /* This function is always called with paddrs for two subsequent 1188 * 2MB chunks in virtual address space, so those chunks will be only 1189 * physically contiguous if the physical addresses are 2MB apart 1190 * from each other as well. 1191 */ 1192 return (paddr2 - paddr1 == VALUE_2MB); 1193 } 1194 1195 #if SPDK_VFIO_ENABLED 1196 1197 static bool 1198 spdk_vfio_enabled(void) 1199 { 1200 return rte_vfio_is_enabled("vfio_pci"); 1201 } 1202 1203 /* Check if IOMMU is enabled on the system */ 1204 static bool 1205 has_iommu_groups(void) 1206 { 1207 struct dirent *d; 1208 int count = 0; 1209 DIR *dir = opendir("/sys/kernel/iommu_groups"); 1210 1211 if (dir == NULL) { 1212 return false; 1213 } 1214 1215 while (count < 3 && (d = readdir(dir)) != NULL) { 1216 count++; 1217 } 1218 1219 closedir(dir); 1220 /* there will always be ./ and ../ entries */ 1221 return count > 2; 1222 } 1223 1224 static bool 1225 spdk_vfio_noiommu_enabled(void) 1226 { 1227 return rte_vfio_noiommu_is_enabled(); 1228 } 1229 1230 static void 1231 spdk_vtophys_iommu_init(void) 1232 { 1233 char proc_fd_path[PATH_MAX + 1]; 1234 char link_path[PATH_MAX + 1]; 1235 const char vfio_path[] = "/dev/vfio/vfio"; 1236 DIR *dir; 1237 struct dirent *d; 1238 1239 if (!spdk_vfio_enabled()) { 1240 return; 1241 } 1242 1243 if (spdk_vfio_noiommu_enabled()) { 1244 g_vfio.noiommu_enabled = true; 1245 } else if (!has_iommu_groups()) { 1246 return; 1247 } 1248 1249 dir = opendir("/proc/self/fd"); 1250 if (!dir) { 1251 DEBUG_PRINT("Failed to open /proc/self/fd (%d)\n", errno); 1252 return; 1253 } 1254 1255 while ((d = readdir(dir)) != NULL) { 1256 if (d->d_type != DT_LNK) { 1257 continue; 1258 } 1259 1260 snprintf(proc_fd_path, sizeof(proc_fd_path), "/proc/self/fd/%s", d->d_name); 1261 if (readlink(proc_fd_path, link_path, sizeof(link_path)) != (sizeof(vfio_path) - 1)) { 1262 continue; 1263 } 1264 1265 if (memcmp(link_path, vfio_path, sizeof(vfio_path) - 1) == 0) { 1266 sscanf(d->d_name, "%d", &g_vfio.fd); 1267 break; 1268 } 1269 } 1270 1271 closedir(dir); 1272 1273 if (g_vfio.fd < 0) { 1274 DEBUG_PRINT("Failed to discover DPDK VFIO container fd.\n"); 1275 return; 1276 } 1277 1278 g_vfio.enabled = true; 1279 1280 return; 1281 } 1282 #endif 1283 1284 void 1285 spdk_vtophys_pci_device_added(struct rte_pci_device *pci_device) 1286 { 1287 struct spdk_vtophys_pci_device *vtophys_dev; 1288 1289 pthread_mutex_lock(&g_vtophys_pci_devices_mutex); 1290 1291 vtophys_dev = calloc(1, sizeof(*vtophys_dev)); 1292 if (vtophys_dev) { 1293 vtophys_dev->pci_device = pci_device; 1294 TAILQ_INSERT_TAIL(&g_vtophys_pci_devices, vtophys_dev, tailq); 1295 } else { 1296 DEBUG_PRINT("Memory allocation error\n"); 1297 } 1298 pthread_mutex_unlock(&g_vtophys_pci_devices_mutex); 1299 1300 #if SPDK_VFIO_ENABLED 1301 struct spdk_vfio_dma_map *dma_map; 1302 int ret; 1303 1304 if (!g_vfio.enabled) { 1305 return; 1306 } 1307 1308 pthread_mutex_lock(&g_vfio.mutex); 1309 g_vfio.device_ref++; 1310 if (g_vfio.device_ref > 1) { 1311 pthread_mutex_unlock(&g_vfio.mutex); 1312 return; 1313 } 1314 1315 /* This is the first SPDK device using DPDK vfio. This means that the first 1316 * IOMMU group might have been just been added to the DPDK vfio container. 1317 * From this point it is certain that the memory can be mapped now. 1318 */ 1319 TAILQ_FOREACH(dma_map, &g_vfio.maps, tailq) { 1320 ret = ioctl(g_vfio.fd, VFIO_IOMMU_MAP_DMA, &dma_map->map); 1321 if (ret) { 1322 DEBUG_PRINT("Cannot update DMA mapping, error %d\n", errno); 1323 break; 1324 } 1325 } 1326 pthread_mutex_unlock(&g_vfio.mutex); 1327 #endif 1328 } 1329 1330 void 1331 spdk_vtophys_pci_device_removed(struct rte_pci_device *pci_device) 1332 { 1333 struct spdk_vtophys_pci_device *vtophys_dev; 1334 1335 pthread_mutex_lock(&g_vtophys_pci_devices_mutex); 1336 TAILQ_FOREACH(vtophys_dev, &g_vtophys_pci_devices, tailq) { 1337 if (vtophys_dev->pci_device == pci_device) { 1338 TAILQ_REMOVE(&g_vtophys_pci_devices, vtophys_dev, tailq); 1339 free(vtophys_dev); 1340 break; 1341 } 1342 } 1343 pthread_mutex_unlock(&g_vtophys_pci_devices_mutex); 1344 1345 #if SPDK_VFIO_ENABLED 1346 struct spdk_vfio_dma_map *dma_map; 1347 int ret; 1348 1349 if (!g_vfio.enabled) { 1350 return; 1351 } 1352 1353 pthread_mutex_lock(&g_vfio.mutex); 1354 assert(g_vfio.device_ref > 0); 1355 g_vfio.device_ref--; 1356 if (g_vfio.device_ref > 0) { 1357 pthread_mutex_unlock(&g_vfio.mutex); 1358 return; 1359 } 1360 1361 /* This is the last SPDK device using DPDK vfio. If DPDK doesn't have 1362 * any additional devices using it's vfio container, all the mappings 1363 * will be automatically removed by the Linux vfio driver. We unmap 1364 * the memory manually to be able to easily re-map it later regardless 1365 * of other, external factors. 1366 */ 1367 TAILQ_FOREACH(dma_map, &g_vfio.maps, tailq) { 1368 ret = ioctl(g_vfio.fd, VFIO_IOMMU_UNMAP_DMA, &dma_map->unmap); 1369 if (ret) { 1370 DEBUG_PRINT("Cannot unmap DMA memory, error %d\n", errno); 1371 break; 1372 } 1373 } 1374 pthread_mutex_unlock(&g_vfio.mutex); 1375 #endif 1376 } 1377 1378 int 1379 spdk_vtophys_init(void) 1380 { 1381 const struct spdk_mem_map_ops vtophys_map_ops = { 1382 .notify_cb = spdk_vtophys_notify, 1383 .are_contiguous = vtophys_check_contiguous_entries, 1384 }; 1385 1386 #if SPDK_VFIO_ENABLED 1387 spdk_vtophys_iommu_init(); 1388 #endif 1389 1390 g_vtophys_map = spdk_mem_map_alloc(SPDK_VTOPHYS_ERROR, &vtophys_map_ops, NULL); 1391 if (g_vtophys_map == NULL) { 1392 DEBUG_PRINT("vtophys map allocation failed\n"); 1393 return -ENOMEM; 1394 } 1395 return 0; 1396 } 1397 1398 uint64_t 1399 spdk_vtophys(void *buf, uint64_t *size) 1400 { 1401 uint64_t vaddr, paddr_2mb; 1402 1403 vaddr = (uint64_t)buf; 1404 paddr_2mb = spdk_mem_map_translate(g_vtophys_map, vaddr, size); 1405 1406 /* 1407 * SPDK_VTOPHYS_ERROR has all bits set, so if the lookup returned SPDK_VTOPHYS_ERROR, 1408 * we will still bitwise-or it with the buf offset below, but the result will still be 1409 * SPDK_VTOPHYS_ERROR. However now that we do + rather than | (due to PCI vtophys being 1410 * unaligned) we must now check the return value before addition. 1411 */ 1412 SPDK_STATIC_ASSERT(SPDK_VTOPHYS_ERROR == UINT64_C(-1), "SPDK_VTOPHYS_ERROR should be all 1s"); 1413 if (paddr_2mb == SPDK_VTOPHYS_ERROR) { 1414 return SPDK_VTOPHYS_ERROR; 1415 } else { 1416 return paddr_2mb + (vaddr & MASK_2MB); 1417 } 1418 } 1419