1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "env_dpdk/memory.c" 35 36 #define UNIT_TEST_NO_VTOPHYS 37 #define UNIT_TEST_NO_PCI_ADDR 38 #include "common/lib/test_env.c" 39 #include "spdk_cunit.h" 40 41 #include "spdk/bit_array.h" 42 43 #define PAGE_ARRAY_SIZE (100) 44 static struct spdk_bit_array *g_page_array; 45 static void *g_vaddr_to_fail = (void *)UINT64_MAX; 46 47 DEFINE_STUB(rte_memseg_contig_walk, int, (rte_memseg_contig_walk_t func, void *arg), 0); 48 DEFINE_STUB(rte_mem_virt2memseg, struct rte_memseg *, 49 (const void *virt, const struct rte_memseg_list *msl), NULL); 50 DEFINE_STUB(spdk_env_dpdk_external_init, bool, (void), true); 51 DEFINE_STUB(rte_mem_event_callback_register, int, 52 (const char *name, rte_mem_event_callback_t clb, void *arg), 0); 53 DEFINE_STUB(rte_mem_virt2iova, rte_iova_t, (const void *virtaddr), 0); 54 DEFINE_STUB(rte_eal_iova_mode, enum rte_iova_mode, (void), RTE_IOVA_VA); 55 DEFINE_STUB(rte_vfio_is_enabled, int, (const char *modname), 0); 56 DEFINE_STUB(rte_vfio_noiommu_is_enabled, int, (void), 0); 57 DEFINE_STUB(rte_memseg_get_fd_thread_unsafe, int, (const struct rte_memseg *ms), 0); 58 DEFINE_STUB(rte_memseg_get_fd_offset_thread_unsafe, int, 59 (const struct rte_memseg *ms, size_t *offset), 0); 60 61 static int 62 test_mem_map_notify(void *cb_ctx, struct spdk_mem_map *map, 63 enum spdk_mem_map_notify_action action, 64 void *vaddr, size_t len) 65 { 66 uint32_t i, end; 67 68 SPDK_CU_ASSERT_FATAL(((uintptr_t)vaddr & MASK_2MB) == 0); 69 SPDK_CU_ASSERT_FATAL((len & MASK_2MB) == 0); 70 71 /* 72 * This is a test requirement - the bit array we use to verify 73 * pages are valid is only so large. 74 */ 75 SPDK_CU_ASSERT_FATAL((uintptr_t)vaddr < (VALUE_2MB * PAGE_ARRAY_SIZE)); 76 77 i = (uintptr_t)vaddr >> SHIFT_2MB; 78 end = i + (len >> SHIFT_2MB); 79 for (; i < end; i++) { 80 switch (action) { 81 case SPDK_MEM_MAP_NOTIFY_REGISTER: 82 /* This page should not already be registered */ 83 SPDK_CU_ASSERT_FATAL(spdk_bit_array_get(g_page_array, i) == false); 84 SPDK_CU_ASSERT_FATAL(spdk_bit_array_set(g_page_array, i) == 0); 85 break; 86 case SPDK_MEM_MAP_NOTIFY_UNREGISTER: 87 SPDK_CU_ASSERT_FATAL(spdk_bit_array_get(g_page_array, i) == true); 88 spdk_bit_array_clear(g_page_array, i); 89 break; 90 default: 91 SPDK_UNREACHABLE(); 92 } 93 } 94 95 return 0; 96 } 97 98 static int 99 test_mem_map_notify_fail(void *cb_ctx, struct spdk_mem_map *map, 100 enum spdk_mem_map_notify_action action, void *vaddr, size_t size) 101 { 102 struct spdk_mem_map *reg_map = cb_ctx; 103 uint64_t reg_addr; 104 uint64_t reg_size = size; 105 106 switch (action) { 107 case SPDK_MEM_MAP_NOTIFY_REGISTER: 108 if (vaddr == g_vaddr_to_fail) { 109 /* Test the error handling. */ 110 return -1; 111 } 112 113 CU_ASSERT(spdk_mem_map_set_translation(map, (uint64_t)vaddr, (uint64_t)size, (uint64_t)vaddr) == 0); 114 115 break; 116 case SPDK_MEM_MAP_NOTIFY_UNREGISTER: 117 /* validate the start address */ 118 reg_addr = spdk_mem_map_translate(map, (uint64_t)vaddr, ®_size); 119 CU_ASSERT(reg_addr == (uint64_t)vaddr); 120 spdk_mem_map_clear_translation(map, (uint64_t)vaddr, size); 121 122 /* Clear the same region in the other mem_map to be able to 123 * verify that there was no memory left still registered after 124 * the mem_map creation failure. 125 */ 126 spdk_mem_map_clear_translation(reg_map, (uint64_t)vaddr, size); 127 break; 128 } 129 130 return 0; 131 } 132 133 static int 134 test_mem_map_notify_checklen(void *cb_ctx, struct spdk_mem_map *map, 135 enum spdk_mem_map_notify_action action, void *vaddr, size_t size) 136 { 137 size_t *len_arr = cb_ctx; 138 139 /* 140 * This is a test requirement - the len array we use to verify 141 * pages are valid is only so large. 142 */ 143 SPDK_CU_ASSERT_FATAL((uintptr_t)vaddr < (VALUE_2MB * PAGE_ARRAY_SIZE)); 144 145 switch (action) { 146 case SPDK_MEM_MAP_NOTIFY_REGISTER: 147 assert(size == len_arr[(uintptr_t)vaddr / VALUE_2MB]); 148 break; 149 case SPDK_MEM_MAP_NOTIFY_UNREGISTER: 150 CU_ASSERT(size == len_arr[(uintptr_t)vaddr / VALUE_2MB]); 151 break; 152 } 153 154 return 0; 155 } 156 157 static int 158 test_check_regions_contiguous(uint64_t addr1, uint64_t addr2) 159 { 160 return addr1 == addr2; 161 } 162 163 const struct spdk_mem_map_ops test_mem_map_ops = { 164 .notify_cb = test_mem_map_notify, 165 .are_contiguous = test_check_regions_contiguous 166 }; 167 168 const struct spdk_mem_map_ops test_mem_map_ops_no_contig = { 169 .notify_cb = test_mem_map_notify, 170 .are_contiguous = NULL 171 }; 172 173 struct spdk_mem_map_ops test_map_ops_notify_fail = { 174 .notify_cb = test_mem_map_notify_fail, 175 .are_contiguous = NULL 176 }; 177 178 struct spdk_mem_map_ops test_map_ops_notify_checklen = { 179 .notify_cb = test_mem_map_notify_checklen, 180 .are_contiguous = NULL 181 }; 182 183 static void 184 test_mem_map_alloc_free(void) 185 { 186 struct spdk_mem_map *map, *failed_map; 187 uint64_t default_translation = 0xDEADBEEF0BADF00D; 188 int i; 189 190 map = spdk_mem_map_alloc(default_translation, &test_mem_map_ops, NULL); 191 SPDK_CU_ASSERT_FATAL(map != NULL); 192 spdk_mem_map_free(&map); 193 CU_ASSERT(map == NULL); 194 195 map = spdk_mem_map_alloc(default_translation, NULL, NULL); 196 SPDK_CU_ASSERT_FATAL(map != NULL); 197 198 /* Register some memory for the initial memory walk in 199 * spdk_mem_map_alloc(). We'll fail registering the last region 200 * and will check if the mem_map cleaned up all its previously 201 * initialized translations. 202 */ 203 for (i = 0; i < 5; i++) { 204 spdk_mem_register((void *)(uintptr_t)(2 * i * VALUE_2MB), VALUE_2MB); 205 } 206 207 /* The last region */ 208 g_vaddr_to_fail = (void *)(8 * VALUE_2MB); 209 failed_map = spdk_mem_map_alloc(default_translation, &test_map_ops_notify_fail, map); 210 CU_ASSERT(failed_map == NULL); 211 212 for (i = 0; i < 4; i++) { 213 uint64_t reg, size = VALUE_2MB; 214 215 reg = spdk_mem_map_translate(map, 2 * i * VALUE_2MB, &size); 216 /* check if `failed_map` didn't leave any translations behind */ 217 CU_ASSERT(reg == default_translation); 218 } 219 220 for (i = 0; i < 5; i++) { 221 spdk_mem_unregister((void *)(uintptr_t)(2 * i * VALUE_2MB), VALUE_2MB); 222 } 223 224 spdk_mem_map_free(&map); 225 CU_ASSERT(map == NULL); 226 } 227 228 static void 229 test_mem_map_translation(void) 230 { 231 struct spdk_mem_map *map; 232 uint64_t default_translation = 0xDEADBEEF0BADF00D; 233 uint64_t addr; 234 uint64_t mapping_length; 235 int rc; 236 237 map = spdk_mem_map_alloc(default_translation, &test_mem_map_ops, NULL); 238 SPDK_CU_ASSERT_FATAL(map != NULL); 239 240 /* Try to get translation for address with no translation */ 241 addr = spdk_mem_map_translate(map, 10, NULL); 242 CU_ASSERT(addr == default_translation); 243 244 /* Set translation for region of non-2MB multiple size */ 245 rc = spdk_mem_map_set_translation(map, VALUE_2MB, 1234, VALUE_2MB); 246 CU_ASSERT(rc == -EINVAL); 247 248 /* Set translation for vaddr that isn't 2MB aligned */ 249 rc = spdk_mem_map_set_translation(map, 1234, VALUE_2MB, VALUE_2MB); 250 CU_ASSERT(rc == -EINVAL); 251 252 /* Set translation for one 2MB page */ 253 rc = spdk_mem_map_set_translation(map, VALUE_2MB, VALUE_2MB, VALUE_2MB); 254 CU_ASSERT(rc == 0); 255 256 /* Set translation for region that overlaps the previous translation */ 257 rc = spdk_mem_map_set_translation(map, 0, 3 * VALUE_2MB, 0); 258 CU_ASSERT(rc == 0); 259 260 /* Make sure we indicate that the three regions are contiguous */ 261 mapping_length = VALUE_2MB * 3; 262 addr = spdk_mem_map_translate(map, 0, &mapping_length); 263 CU_ASSERT(addr == 0); 264 CU_ASSERT(mapping_length == VALUE_2MB * 3); 265 266 /* Translate an unaligned address */ 267 mapping_length = VALUE_2MB * 3; 268 addr = spdk_mem_map_translate(map, VALUE_4KB, &mapping_length); 269 CU_ASSERT(addr == 0); 270 CU_ASSERT(mapping_length == VALUE_2MB * 3 - VALUE_4KB); 271 272 /* Clear translation for the middle page of the larger region. */ 273 rc = spdk_mem_map_clear_translation(map, VALUE_2MB, VALUE_2MB); 274 CU_ASSERT(rc == 0); 275 276 /* Get translation for first page */ 277 addr = spdk_mem_map_translate(map, 0, NULL); 278 CU_ASSERT(addr == 0); 279 280 /* Make sure we indicate that the three regions are no longer contiguous */ 281 mapping_length = VALUE_2MB * 3; 282 addr = spdk_mem_map_translate(map, 0, &mapping_length); 283 CU_ASSERT(addr == 0); 284 CU_ASSERT(mapping_length == VALUE_2MB); 285 286 /* Get translation for an unallocated block. Make sure size is 0 */ 287 mapping_length = VALUE_2MB * 3; 288 addr = spdk_mem_map_translate(map, VALUE_2MB, &mapping_length); 289 CU_ASSERT(addr == default_translation); 290 CU_ASSERT(mapping_length == VALUE_2MB); 291 292 /* Verify translation for 2nd page is the default */ 293 addr = spdk_mem_map_translate(map, VALUE_2MB, NULL); 294 CU_ASSERT(addr == default_translation); 295 296 /* Get translation for third page */ 297 addr = spdk_mem_map_translate(map, 2 * VALUE_2MB, NULL); 298 /* 299 * Note that addr should be 0, not 4MB. When we set the 300 * translation above, we said the whole 6MB region 301 * should translate to 0. 302 */ 303 CU_ASSERT(addr == 0); 304 305 /* Translate only a subset of a 2MB page */ 306 mapping_length = 543; 307 addr = spdk_mem_map_translate(map, 0, &mapping_length); 308 CU_ASSERT(addr == 0); 309 CU_ASSERT(mapping_length == 543); 310 311 /* Translate another subset of a 2MB page */ 312 mapping_length = 543; 313 addr = spdk_mem_map_translate(map, VALUE_4KB, &mapping_length); 314 CU_ASSERT(addr == 0); 315 CU_ASSERT(mapping_length == 543); 316 317 /* Try to translate an unaligned region that is only partially registered */ 318 mapping_length = 543; 319 addr = spdk_mem_map_translate(map, 3 * VALUE_2MB - 196, &mapping_length); 320 CU_ASSERT(addr == 0); 321 CU_ASSERT(mapping_length == 196); 322 323 /* Clear translation for the first page */ 324 rc = spdk_mem_map_clear_translation(map, 0, VALUE_2MB); 325 CU_ASSERT(rc == 0); 326 327 /* Get translation for the first page */ 328 addr = spdk_mem_map_translate(map, 0, NULL); 329 CU_ASSERT(addr == default_translation); 330 331 /* Clear translation for the third page */ 332 rc = spdk_mem_map_clear_translation(map, 2 * VALUE_2MB, VALUE_2MB); 333 CU_ASSERT(rc == 0); 334 335 /* Get translation for the third page */ 336 addr = spdk_mem_map_translate(map, 2 * VALUE_2MB, NULL); 337 CU_ASSERT(addr == default_translation); 338 339 /* Set translation for the last valid 2MB region */ 340 rc = spdk_mem_map_set_translation(map, 0xffffffe00000ULL, VALUE_2MB, 0x1234); 341 CU_ASSERT(rc == 0); 342 343 /* Verify translation for last valid 2MB region */ 344 addr = spdk_mem_map_translate(map, 0xffffffe00000ULL, NULL); 345 CU_ASSERT(addr == 0x1234); 346 347 /* Attempt to set translation for the first invalid address */ 348 rc = spdk_mem_map_set_translation(map, 0x1000000000000ULL, VALUE_2MB, 0x5678); 349 CU_ASSERT(rc == -EINVAL); 350 351 /* Attempt to set translation starting at a valid address but exceeding the valid range */ 352 rc = spdk_mem_map_set_translation(map, 0xffffffe00000ULL, VALUE_2MB * 2, 0x123123); 353 CU_ASSERT(rc != 0); 354 355 spdk_mem_map_free(&map); 356 CU_ASSERT(map == NULL); 357 358 /* Allocate a map without a contiguous region checker */ 359 map = spdk_mem_map_alloc(default_translation, &test_mem_map_ops_no_contig, NULL); 360 SPDK_CU_ASSERT_FATAL(map != NULL); 361 362 /* map three contiguous regions */ 363 rc = spdk_mem_map_set_translation(map, 0, 3 * VALUE_2MB, 0); 364 CU_ASSERT(rc == 0); 365 366 /* Since we can't check their contiguity, make sure we only return the size of one page */ 367 mapping_length = VALUE_2MB * 3; 368 addr = spdk_mem_map_translate(map, 0, &mapping_length); 369 CU_ASSERT(addr == 0); 370 CU_ASSERT(mapping_length == VALUE_2MB); 371 372 /* Translate only a subset of a 2MB page */ 373 mapping_length = 543; 374 addr = spdk_mem_map_translate(map, 0, &mapping_length); 375 CU_ASSERT(addr == 0); 376 CU_ASSERT(mapping_length == 543); 377 378 /* Clear the translation */ 379 rc = spdk_mem_map_clear_translation(map, 0, VALUE_2MB * 3); 380 CU_ASSERT(rc == 0); 381 382 spdk_mem_map_free(&map); 383 CU_ASSERT(map == NULL); 384 } 385 386 static void 387 test_mem_map_registration(void) 388 { 389 int rc; 390 struct spdk_mem_map *map; 391 uint64_t default_translation = 0xDEADBEEF0BADF00D; 392 393 map = spdk_mem_map_alloc(default_translation, &test_mem_map_ops, NULL); 394 SPDK_CU_ASSERT_FATAL(map != NULL); 395 396 /* Unregister memory region that wasn't previously registered */ 397 rc = spdk_mem_unregister((void *)VALUE_2MB, VALUE_2MB); 398 CU_ASSERT(rc == -EINVAL); 399 400 /* Register non-2MB multiple size */ 401 rc = spdk_mem_register((void *)VALUE_2MB, 1234); 402 CU_ASSERT(rc == -EINVAL); 403 404 /* Register region that isn't 2MB aligned */ 405 rc = spdk_mem_register((void *)1234, VALUE_2MB); 406 CU_ASSERT(rc == -EINVAL); 407 408 /* Register one 2MB page */ 409 rc = spdk_mem_register((void *)VALUE_2MB, VALUE_2MB); 410 CU_ASSERT(rc == 0); 411 412 /* Register an overlapping address range */ 413 rc = spdk_mem_register((void *)0, 3 * VALUE_2MB); 414 CU_ASSERT(rc == -EBUSY); 415 416 /* Unregister a 2MB page */ 417 rc = spdk_mem_unregister((void *)VALUE_2MB, VALUE_2MB); 418 CU_ASSERT(rc == 0); 419 420 /* Register non overlapping address range */ 421 rc = spdk_mem_register((void *)0, 3 * VALUE_2MB); 422 CU_ASSERT(rc == 0); 423 424 /* Unregister the middle page of the larger region. */ 425 rc = spdk_mem_unregister((void *)VALUE_2MB, VALUE_2MB); 426 CU_ASSERT(rc == -ERANGE); 427 428 /* Unregister the first page */ 429 rc = spdk_mem_unregister((void *)0, VALUE_2MB); 430 CU_ASSERT(rc == -ERANGE); 431 432 /* Unregister the third page */ 433 rc = spdk_mem_unregister((void *)(2 * VALUE_2MB), VALUE_2MB); 434 CU_ASSERT(rc == -ERANGE); 435 436 /* Unregister the entire address range */ 437 rc = spdk_mem_unregister((void *)0, 3 * VALUE_2MB); 438 CU_ASSERT(rc == 0); 439 440 spdk_mem_map_free(&map); 441 CU_ASSERT(map == NULL); 442 } 443 444 static void 445 test_mem_map_registration_adjacent(void) 446 { 447 struct spdk_mem_map *map, *newmap; 448 uint64_t default_translation = 0xDEADBEEF0BADF00D; 449 uintptr_t vaddr; 450 unsigned i; 451 size_t notify_len[PAGE_ARRAY_SIZE] = {0}; 452 size_t chunk_len[] = { 2, 1, 3, 2, 1, 1 }; 453 454 map = spdk_mem_map_alloc(default_translation, 455 &test_map_ops_notify_checklen, notify_len); 456 SPDK_CU_ASSERT_FATAL(map != NULL); 457 458 vaddr = 0; 459 for (i = 0; i < SPDK_COUNTOF(chunk_len); i++) { 460 notify_len[vaddr / VALUE_2MB] = chunk_len[i] * VALUE_2MB; 461 spdk_mem_register((void *)vaddr, notify_len[vaddr / VALUE_2MB]); 462 vaddr += notify_len[vaddr / VALUE_2MB]; 463 } 464 465 /* Verify the memory is translated in the same chunks it was registered */ 466 newmap = spdk_mem_map_alloc(default_translation, 467 &test_map_ops_notify_checklen, notify_len); 468 SPDK_CU_ASSERT_FATAL(newmap != NULL); 469 spdk_mem_map_free(&newmap); 470 CU_ASSERT(newmap == NULL); 471 472 vaddr = 0; 473 for (i = 0; i < SPDK_COUNTOF(chunk_len); i++) { 474 notify_len[vaddr / VALUE_2MB] = chunk_len[i] * VALUE_2MB; 475 spdk_mem_unregister((void *)vaddr, notify_len[vaddr / VALUE_2MB]); 476 vaddr += notify_len[vaddr / VALUE_2MB]; 477 } 478 479 /* Register all chunks again just to unregister them again, but this 480 * time with only a single unregister() call. 481 */ 482 vaddr = 0; 483 for (i = 0; i < SPDK_COUNTOF(chunk_len); i++) { 484 notify_len[vaddr / VALUE_2MB] = chunk_len[i] * VALUE_2MB; 485 spdk_mem_register((void *)vaddr, notify_len[vaddr / VALUE_2MB]); 486 vaddr += notify_len[vaddr / VALUE_2MB]; 487 } 488 spdk_mem_unregister(0, vaddr); 489 490 spdk_mem_map_free(&map); 491 CU_ASSERT(map == NULL); 492 } 493 494 int 495 main(int argc, char **argv) 496 { 497 CU_pSuite suite = NULL; 498 unsigned int num_failures; 499 500 /* 501 * These tests can use PAGE_ARRAY_SIZE 2MB pages of memory. 502 * Note that the tests just verify addresses - this memory 503 * is not actually allocated. 504 */ 505 g_page_array = spdk_bit_array_create(PAGE_ARRAY_SIZE); 506 507 /* Initialize the memory map */ 508 if (mem_map_init(false) < 0) { 509 return CUE_NOMEMORY; 510 } 511 512 if (CU_initialize_registry() != CUE_SUCCESS) { 513 return CU_get_error(); 514 } 515 516 suite = CU_add_suite("memory", NULL, NULL); 517 if (suite == NULL) { 518 CU_cleanup_registry(); 519 return CU_get_error(); 520 } 521 522 if ( 523 CU_add_test(suite, "alloc and free memory map", test_mem_map_alloc_free) == NULL || 524 CU_add_test(suite, "mem map translation", test_mem_map_translation) == NULL || 525 CU_add_test(suite, "mem map registration", test_mem_map_registration) == NULL || 526 CU_add_test(suite, "mem map adjacent registrations", test_mem_map_registration_adjacent) == NULL 527 ) { 528 CU_cleanup_registry(); 529 return CU_get_error(); 530 } 531 532 CU_basic_set_mode(CU_BRM_VERBOSE); 533 CU_basic_run_tests(); 534 num_failures = CU_get_number_of_failures(); 535 CU_cleanup_registry(); 536 537 spdk_bit_array_free(&g_page_array); 538 539 return num_failures; 540 } 541