1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "env_dpdk/memory.c" 7 8 #define UNIT_TEST_NO_VTOPHYS 9 #include "common/lib/test_env.c" 10 #include "spdk_internal/cunit.h" 11 12 #include "spdk/bit_array.h" 13 14 #define PAGE_ARRAY_SIZE (100) 15 static struct spdk_bit_array *g_page_array; 16 static void *g_vaddr_to_fail = (void *)UINT64_MAX; 17 18 DEFINE_STUB(rte_memseg_contig_walk, int, (rte_memseg_contig_walk_t func, void *arg), 0); 19 DEFINE_STUB(rte_mem_virt2memseg, struct rte_memseg *, 20 (const void *virt, const struct rte_memseg_list *msl), NULL); 21 DEFINE_STUB(spdk_env_dpdk_external_init, bool, (void), true); 22 DEFINE_STUB(rte_mem_event_callback_register, int, 23 (const char *name, rte_mem_event_callback_t clb, void *arg), 0); 24 DEFINE_STUB(rte_mem_virt2iova, rte_iova_t, (const void *virtaddr), 0); 25 DEFINE_STUB(rte_eal_iova_mode, enum rte_iova_mode, (void), RTE_IOVA_VA); 26 DEFINE_STUB(rte_vfio_is_enabled, int, (const char *modname), 0); 27 DEFINE_STUB(rte_vfio_noiommu_is_enabled, int, (void), 0); 28 DEFINE_STUB(rte_memseg_get_fd_thread_unsafe, int, (const struct rte_memseg *ms), 0); 29 DEFINE_STUB(rte_memseg_get_fd_offset_thread_unsafe, int, 30 (const struct rte_memseg *ms, size_t *offset), 0); 31 DEFINE_STUB(dpdk_pci_device_get_mem_resource, struct rte_mem_resource *, 32 (struct rte_pci_device *dev, uint32_t bar), 0); 33 34 static int 35 test_mem_map_notify(void *cb_ctx, struct spdk_mem_map *map, 36 enum spdk_mem_map_notify_action action, 37 void *vaddr, size_t len) 38 { 39 uint32_t i, end; 40 41 SPDK_CU_ASSERT_FATAL(((uintptr_t)vaddr & MASK_2MB) == 0); 42 SPDK_CU_ASSERT_FATAL((len & MASK_2MB) == 0); 43 44 /* 45 * This is a test requirement - the bit array we use to verify 46 * pages are valid is only so large. 47 */ 48 SPDK_CU_ASSERT_FATAL((uintptr_t)vaddr < (VALUE_2MB * PAGE_ARRAY_SIZE)); 49 50 i = (uintptr_t)vaddr >> SHIFT_2MB; 51 end = i + (len >> SHIFT_2MB); 52 for (; i < end; i++) { 53 switch (action) { 54 case SPDK_MEM_MAP_NOTIFY_REGISTER: 55 /* This page should not already be registered */ 56 SPDK_CU_ASSERT_FATAL(spdk_bit_array_get(g_page_array, i) == false); 57 SPDK_CU_ASSERT_FATAL(spdk_bit_array_set(g_page_array, i) == 0); 58 break; 59 case SPDK_MEM_MAP_NOTIFY_UNREGISTER: 60 SPDK_CU_ASSERT_FATAL(spdk_bit_array_get(g_page_array, i) == true); 61 spdk_bit_array_clear(g_page_array, i); 62 break; 63 default: 64 SPDK_UNREACHABLE(); 65 } 66 } 67 68 return 0; 69 } 70 71 static int 72 test_mem_map_notify_fail(void *cb_ctx, struct spdk_mem_map *map, 73 enum spdk_mem_map_notify_action action, void *vaddr, size_t size) 74 { 75 struct spdk_mem_map *reg_map = cb_ctx; 76 uint64_t reg_addr; 77 uint64_t reg_size = size; 78 79 switch (action) { 80 case SPDK_MEM_MAP_NOTIFY_REGISTER: 81 if (vaddr == g_vaddr_to_fail) { 82 /* Test the error handling. */ 83 return -1; 84 } 85 86 CU_ASSERT(spdk_mem_map_set_translation(map, (uint64_t)vaddr, (uint64_t)size, (uint64_t)vaddr) == 0); 87 88 break; 89 case SPDK_MEM_MAP_NOTIFY_UNREGISTER: 90 /* validate the start address */ 91 reg_addr = spdk_mem_map_translate(map, (uint64_t)vaddr, ®_size); 92 CU_ASSERT(reg_addr == (uint64_t)vaddr); 93 spdk_mem_map_clear_translation(map, (uint64_t)vaddr, size); 94 95 /* Clear the same region in the other mem_map to be able to 96 * verify that there was no memory left still registered after 97 * the mem_map creation failure. 98 */ 99 spdk_mem_map_clear_translation(reg_map, (uint64_t)vaddr, size); 100 break; 101 } 102 103 return 0; 104 } 105 106 static int 107 test_mem_map_notify_checklen(void *cb_ctx, struct spdk_mem_map *map, 108 enum spdk_mem_map_notify_action action, void *vaddr, size_t size) 109 { 110 size_t *len_arr = cb_ctx; 111 112 /* 113 * This is a test requirement - the len array we use to verify 114 * pages are valid is only so large. 115 */ 116 SPDK_CU_ASSERT_FATAL((uintptr_t)vaddr < (VALUE_2MB * PAGE_ARRAY_SIZE)); 117 118 switch (action) { 119 case SPDK_MEM_MAP_NOTIFY_REGISTER: 120 assert(size == len_arr[(uintptr_t)vaddr / VALUE_2MB]); 121 break; 122 case SPDK_MEM_MAP_NOTIFY_UNREGISTER: 123 CU_ASSERT(size == len_arr[(uintptr_t)vaddr / VALUE_2MB]); 124 break; 125 } 126 127 return 0; 128 } 129 130 static int 131 test_check_regions_contiguous(uint64_t addr1, uint64_t addr2) 132 { 133 return addr1 == addr2; 134 } 135 136 const struct spdk_mem_map_ops test_mem_map_ops = { 137 .notify_cb = test_mem_map_notify, 138 .are_contiguous = test_check_regions_contiguous 139 }; 140 141 const struct spdk_mem_map_ops test_mem_map_ops_no_contig = { 142 .notify_cb = test_mem_map_notify, 143 .are_contiguous = NULL 144 }; 145 146 struct spdk_mem_map_ops test_map_ops_notify_fail = { 147 .notify_cb = test_mem_map_notify_fail, 148 .are_contiguous = NULL 149 }; 150 151 struct spdk_mem_map_ops test_map_ops_notify_checklen = { 152 .notify_cb = test_mem_map_notify_checklen, 153 .are_contiguous = NULL 154 }; 155 156 static void 157 test_mem_map_alloc_free(void) 158 { 159 struct spdk_mem_map *map, *failed_map; 160 uint64_t default_translation = 0xDEADBEEF0BADF00D; 161 int i; 162 163 map = spdk_mem_map_alloc(default_translation, &test_mem_map_ops, NULL); 164 SPDK_CU_ASSERT_FATAL(map != NULL); 165 spdk_mem_map_free(&map); 166 CU_ASSERT(map == NULL); 167 168 map = spdk_mem_map_alloc(default_translation, NULL, NULL); 169 SPDK_CU_ASSERT_FATAL(map != NULL); 170 171 /* Register some memory for the initial memory walk in 172 * spdk_mem_map_alloc(). We'll fail registering the last region 173 * and will check if the mem_map cleaned up all its previously 174 * initialized translations. 175 */ 176 for (i = 0; i < 5; i++) { 177 spdk_mem_register((void *)(uintptr_t)(2 * i * VALUE_2MB), VALUE_2MB); 178 } 179 180 /* The last region */ 181 g_vaddr_to_fail = (void *)(8 * VALUE_2MB); 182 failed_map = spdk_mem_map_alloc(default_translation, &test_map_ops_notify_fail, map); 183 CU_ASSERT(failed_map == NULL); 184 185 for (i = 0; i < 4; i++) { 186 uint64_t reg, size = VALUE_2MB; 187 188 reg = spdk_mem_map_translate(map, 2 * i * VALUE_2MB, &size); 189 /* check if `failed_map` didn't leave any translations behind */ 190 CU_ASSERT(reg == default_translation); 191 } 192 193 for (i = 0; i < 5; i++) { 194 spdk_mem_unregister((void *)(uintptr_t)(2 * i * VALUE_2MB), VALUE_2MB); 195 } 196 197 spdk_mem_map_free(&map); 198 CU_ASSERT(map == NULL); 199 } 200 201 static void 202 test_mem_map_translation(void) 203 { 204 struct spdk_mem_map *map; 205 uint64_t default_translation = 0xDEADBEEF0BADF00D; 206 uint64_t addr; 207 uint64_t mapping_length; 208 int rc; 209 210 map = spdk_mem_map_alloc(default_translation, &test_mem_map_ops, NULL); 211 SPDK_CU_ASSERT_FATAL(map != NULL); 212 213 /* Try to get translation for address with no translation */ 214 addr = spdk_mem_map_translate(map, 10, NULL); 215 CU_ASSERT(addr == default_translation); 216 217 /* Set translation for region of non-2MB multiple size */ 218 rc = spdk_mem_map_set_translation(map, VALUE_2MB, 1234, VALUE_2MB); 219 CU_ASSERT(rc == -EINVAL); 220 221 /* Set translation for vaddr that isn't 2MB aligned */ 222 rc = spdk_mem_map_set_translation(map, 1234, VALUE_2MB, VALUE_2MB); 223 CU_ASSERT(rc == -EINVAL); 224 225 /* Set translation for one 2MB page */ 226 rc = spdk_mem_map_set_translation(map, VALUE_2MB, VALUE_2MB, VALUE_2MB); 227 CU_ASSERT(rc == 0); 228 229 /* Set translation for region that overlaps the previous translation */ 230 rc = spdk_mem_map_set_translation(map, 0, 3 * VALUE_2MB, 0); 231 CU_ASSERT(rc == 0); 232 233 /* Make sure we indicate that the three regions are contiguous */ 234 mapping_length = VALUE_2MB * 3; 235 addr = spdk_mem_map_translate(map, 0, &mapping_length); 236 CU_ASSERT(addr == 0); 237 CU_ASSERT(mapping_length == VALUE_2MB * 3); 238 239 /* Translate an unaligned address */ 240 mapping_length = VALUE_2MB * 3; 241 addr = spdk_mem_map_translate(map, VALUE_4KB, &mapping_length); 242 CU_ASSERT(addr == 0); 243 CU_ASSERT(mapping_length == VALUE_2MB * 3 - VALUE_4KB); 244 245 /* Clear translation for the middle page of the larger region. */ 246 rc = spdk_mem_map_clear_translation(map, VALUE_2MB, VALUE_2MB); 247 CU_ASSERT(rc == 0); 248 249 /* Get translation for first page */ 250 addr = spdk_mem_map_translate(map, 0, NULL); 251 CU_ASSERT(addr == 0); 252 253 /* Make sure we indicate that the three regions are no longer contiguous */ 254 mapping_length = VALUE_2MB * 3; 255 addr = spdk_mem_map_translate(map, 0, &mapping_length); 256 CU_ASSERT(addr == 0); 257 CU_ASSERT(mapping_length == VALUE_2MB); 258 259 /* Get translation for an unallocated block. Make sure size is 0 */ 260 mapping_length = VALUE_2MB * 3; 261 addr = spdk_mem_map_translate(map, VALUE_2MB, &mapping_length); 262 CU_ASSERT(addr == default_translation); 263 CU_ASSERT(mapping_length == VALUE_2MB); 264 265 /* Verify translation for 2nd page is the default */ 266 addr = spdk_mem_map_translate(map, VALUE_2MB, NULL); 267 CU_ASSERT(addr == default_translation); 268 269 /* Get translation for third page */ 270 addr = spdk_mem_map_translate(map, 2 * VALUE_2MB, NULL); 271 /* 272 * Note that addr should be 0, not 4MB. When we set the 273 * translation above, we said the whole 6MB region 274 * should translate to 0. 275 */ 276 CU_ASSERT(addr == 0); 277 278 /* Translate only a subset of a 2MB page */ 279 mapping_length = 543; 280 addr = spdk_mem_map_translate(map, 0, &mapping_length); 281 CU_ASSERT(addr == 0); 282 CU_ASSERT(mapping_length == 543); 283 284 /* Translate another subset of a 2MB page */ 285 mapping_length = 543; 286 addr = spdk_mem_map_translate(map, VALUE_4KB, &mapping_length); 287 CU_ASSERT(addr == 0); 288 CU_ASSERT(mapping_length == 543); 289 290 /* Try to translate an unaligned region that is only partially registered */ 291 mapping_length = 543; 292 addr = spdk_mem_map_translate(map, 3 * VALUE_2MB - 196, &mapping_length); 293 CU_ASSERT(addr == 0); 294 CU_ASSERT(mapping_length == 196); 295 296 /* Clear translation for the first page */ 297 rc = spdk_mem_map_clear_translation(map, 0, VALUE_2MB); 298 CU_ASSERT(rc == 0); 299 300 /* Get translation for the first page */ 301 addr = spdk_mem_map_translate(map, 0, NULL); 302 CU_ASSERT(addr == default_translation); 303 304 /* Clear translation for the third page */ 305 rc = spdk_mem_map_clear_translation(map, 2 * VALUE_2MB, VALUE_2MB); 306 CU_ASSERT(rc == 0); 307 308 /* Get translation for the third page */ 309 addr = spdk_mem_map_translate(map, 2 * VALUE_2MB, NULL); 310 CU_ASSERT(addr == default_translation); 311 312 /* Set translation for the last valid 2MB region */ 313 rc = spdk_mem_map_set_translation(map, 0xffffffe00000ULL, VALUE_2MB, 0x1234); 314 CU_ASSERT(rc == 0); 315 316 /* Verify translation for last valid 2MB region */ 317 addr = spdk_mem_map_translate(map, 0xffffffe00000ULL, NULL); 318 CU_ASSERT(addr == 0x1234); 319 320 /* Attempt to set translation for the first invalid address */ 321 rc = spdk_mem_map_set_translation(map, 0x1000000000000ULL, VALUE_2MB, 0x5678); 322 CU_ASSERT(rc == -EINVAL); 323 324 /* Attempt to set translation starting at a valid address but exceeding the valid range */ 325 rc = spdk_mem_map_set_translation(map, 0xffffffe00000ULL, VALUE_2MB * 2, 0x123123); 326 CU_ASSERT(rc != 0); 327 328 spdk_mem_map_free(&map); 329 CU_ASSERT(map == NULL); 330 331 /* Allocate a map without a contiguous region checker */ 332 map = spdk_mem_map_alloc(default_translation, &test_mem_map_ops_no_contig, NULL); 333 SPDK_CU_ASSERT_FATAL(map != NULL); 334 335 /* map three contiguous regions */ 336 rc = spdk_mem_map_set_translation(map, 0, 3 * VALUE_2MB, 0); 337 CU_ASSERT(rc == 0); 338 339 /* Since we can't check their contiguity, make sure we only return the size of one page */ 340 mapping_length = VALUE_2MB * 3; 341 addr = spdk_mem_map_translate(map, 0, &mapping_length); 342 CU_ASSERT(addr == 0); 343 CU_ASSERT(mapping_length == VALUE_2MB); 344 345 /* Translate only a subset of a 2MB page */ 346 mapping_length = 543; 347 addr = spdk_mem_map_translate(map, 0, &mapping_length); 348 CU_ASSERT(addr == 0); 349 CU_ASSERT(mapping_length == 543); 350 351 /* Clear the translation */ 352 rc = spdk_mem_map_clear_translation(map, 0, VALUE_2MB * 3); 353 CU_ASSERT(rc == 0); 354 355 spdk_mem_map_free(&map); 356 CU_ASSERT(map == NULL); 357 } 358 359 static void 360 test_mem_map_registration(void) 361 { 362 int rc; 363 struct spdk_mem_map *map; 364 uint64_t default_translation = 0xDEADBEEF0BADF00D; 365 366 map = spdk_mem_map_alloc(default_translation, &test_mem_map_ops, NULL); 367 SPDK_CU_ASSERT_FATAL(map != NULL); 368 369 /* Unregister memory region that wasn't previously registered */ 370 rc = spdk_mem_unregister((void *)VALUE_2MB, VALUE_2MB); 371 CU_ASSERT(rc == -EINVAL); 372 373 /* Register non-2MB multiple size */ 374 rc = spdk_mem_register((void *)VALUE_2MB, 1234); 375 CU_ASSERT(rc == -EINVAL); 376 377 /* Register region that isn't 2MB aligned */ 378 rc = spdk_mem_register((void *)1234, VALUE_2MB); 379 CU_ASSERT(rc == -EINVAL); 380 381 /* Register one 2MB page */ 382 rc = spdk_mem_register((void *)VALUE_2MB, VALUE_2MB); 383 CU_ASSERT(rc == 0); 384 385 /* Register an overlapping address range */ 386 rc = spdk_mem_register((void *)0, 3 * VALUE_2MB); 387 CU_ASSERT(rc == -EBUSY); 388 389 /* Unregister a 2MB page */ 390 rc = spdk_mem_unregister((void *)VALUE_2MB, VALUE_2MB); 391 CU_ASSERT(rc == 0); 392 393 /* Register non overlapping address range */ 394 rc = spdk_mem_register((void *)0, 3 * VALUE_2MB); 395 CU_ASSERT(rc == 0); 396 397 /* Unregister the middle page of the larger region. */ 398 rc = spdk_mem_unregister((void *)VALUE_2MB, VALUE_2MB); 399 CU_ASSERT(rc == -ERANGE); 400 401 /* Unregister the first page */ 402 rc = spdk_mem_unregister((void *)0, VALUE_2MB); 403 CU_ASSERT(rc == -ERANGE); 404 405 /* Unregister the third page */ 406 rc = spdk_mem_unregister((void *)(2 * VALUE_2MB), VALUE_2MB); 407 CU_ASSERT(rc == -ERANGE); 408 409 /* Unregister the entire address range */ 410 rc = spdk_mem_unregister((void *)0, 3 * VALUE_2MB); 411 CU_ASSERT(rc == 0); 412 413 spdk_mem_map_free(&map); 414 CU_ASSERT(map == NULL); 415 } 416 417 static void 418 test_mem_map_registration_adjacent(void) 419 { 420 struct spdk_mem_map *map, *newmap; 421 uint64_t default_translation = 0xDEADBEEF0BADF00D; 422 uintptr_t vaddr; 423 unsigned i; 424 size_t notify_len[PAGE_ARRAY_SIZE] = {0}; 425 size_t chunk_len[] = { 2, 1, 3, 2, 1, 1 }; 426 427 map = spdk_mem_map_alloc(default_translation, 428 &test_map_ops_notify_checklen, notify_len); 429 SPDK_CU_ASSERT_FATAL(map != NULL); 430 431 vaddr = 0; 432 for (i = 0; i < SPDK_COUNTOF(chunk_len); i++) { 433 notify_len[vaddr / VALUE_2MB] = chunk_len[i] * VALUE_2MB; 434 spdk_mem_register((void *)vaddr, notify_len[vaddr / VALUE_2MB]); 435 vaddr += notify_len[vaddr / VALUE_2MB]; 436 } 437 438 /* Verify the memory is translated in the same chunks it was registered */ 439 newmap = spdk_mem_map_alloc(default_translation, 440 &test_map_ops_notify_checklen, notify_len); 441 SPDK_CU_ASSERT_FATAL(newmap != NULL); 442 spdk_mem_map_free(&newmap); 443 CU_ASSERT(newmap == NULL); 444 445 vaddr = 0; 446 for (i = 0; i < SPDK_COUNTOF(chunk_len); i++) { 447 notify_len[vaddr / VALUE_2MB] = chunk_len[i] * VALUE_2MB; 448 spdk_mem_unregister((void *)vaddr, notify_len[vaddr / VALUE_2MB]); 449 vaddr += notify_len[vaddr / VALUE_2MB]; 450 } 451 452 /* Register all chunks again just to unregister them again, but this 453 * time with only a single unregister() call. 454 */ 455 vaddr = 0; 456 for (i = 0; i < SPDK_COUNTOF(chunk_len); i++) { 457 notify_len[vaddr / VALUE_2MB] = chunk_len[i] * VALUE_2MB; 458 spdk_mem_register((void *)vaddr, notify_len[vaddr / VALUE_2MB]); 459 vaddr += notify_len[vaddr / VALUE_2MB]; 460 } 461 spdk_mem_unregister(0, vaddr); 462 463 spdk_mem_map_free(&map); 464 CU_ASSERT(map == NULL); 465 } 466 467 int 468 main(int argc, char **argv) 469 { 470 CU_pSuite suite = NULL; 471 unsigned int num_failures; 472 473 /* 474 * These tests can use PAGE_ARRAY_SIZE 2MB pages of memory. 475 * Note that the tests just verify addresses - this memory 476 * is not actually allocated. 477 */ 478 g_page_array = spdk_bit_array_create(PAGE_ARRAY_SIZE); 479 480 /* Initialize the memory map */ 481 if (mem_map_init(false) < 0) { 482 return CUE_NOMEMORY; 483 } 484 485 if (CU_initialize_registry() != CUE_SUCCESS) { 486 return CU_get_error(); 487 } 488 489 suite = CU_add_suite("memory", NULL, NULL); 490 if (suite == NULL) { 491 CU_cleanup_registry(); 492 return CU_get_error(); 493 } 494 495 if ( 496 CU_add_test(suite, "alloc and free memory map", test_mem_map_alloc_free) == NULL || 497 CU_add_test(suite, "mem map translation", test_mem_map_translation) == NULL || 498 CU_add_test(suite, "mem map registration", test_mem_map_registration) == NULL || 499 CU_add_test(suite, "mem map adjacent registrations", test_mem_map_registration_adjacent) == NULL 500 ) { 501 CU_cleanup_registry(); 502 return CU_get_error(); 503 } 504 505 num_failures = spdk_ut_run_tests(argc, argv, NULL); 506 CU_cleanup_registry(); 507 508 spdk_bit_array_free(&g_page_array); 509 510 return num_failures; 511 } 512