1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) Intel Corporation. 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 11 * * Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * * Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in 15 * the documentation and/or other materials provided with the 16 * distribution. 17 * * Neither the name of Intel Corporation nor the names of its 18 * contributors may be used to endorse or promote products derived 19 * from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 32 */ 33 34 #include "env_dpdk/memory.c" 35 36 #define UNIT_TEST_NO_VTOPHYS 37 #define UNIT_TEST_NO_PCI_ADDR 38 #include "common/lib/test_env.c" 39 #include "spdk_cunit.h" 40 41 #include "spdk/bit_array.h" 42 43 #define PAGE_ARRAY_SIZE (100) 44 static struct spdk_bit_array *g_page_array; 45 static void *g_vaddr_to_fail = (void *)UINT64_MAX; 46 47 DEFINE_STUB(rte_memseg_contig_walk, int, (rte_memseg_contig_walk_t func, void *arg), 0); 48 DEFINE_STUB(rte_mem_virt2memseg, struct rte_memseg *, 49 (const void *virt, const struct rte_memseg_list *msl), NULL); 50 DEFINE_STUB(spdk_env_dpdk_external_init, bool, (void), true); 51 DEFINE_STUB(rte_mem_event_callback_register, int, 52 (const char *name, rte_mem_event_callback_t clb, void *arg), 0); 53 DEFINE_STUB(rte_mem_virt2iova, rte_iova_t, (const void *virtaddr), 0); 54 DEFINE_STUB(rte_eal_iova_mode, enum rte_iova_mode, (void), RTE_IOVA_VA); 55 DEFINE_STUB(rte_vfio_is_enabled, int, (const char *modname), 0); 56 DEFINE_STUB(rte_vfio_noiommu_is_enabled, int, (void), 0); 57 DEFINE_STUB(rte_memseg_get_fd_thread_unsafe, int, (const struct rte_memseg *ms), 0); 58 DEFINE_STUB(rte_memseg_get_fd_offset_thread_unsafe, int, 59 (const struct rte_memseg *ms, size_t *offset), 0); 60 61 static int 62 test_mem_map_notify(void *cb_ctx, struct spdk_mem_map *map, 63 enum spdk_mem_map_notify_action action, 64 void *vaddr, size_t len) 65 { 66 uint32_t i, end; 67 68 SPDK_CU_ASSERT_FATAL(((uintptr_t)vaddr & MASK_2MB) == 0); 69 SPDK_CU_ASSERT_FATAL((len & MASK_2MB) == 0); 70 71 /* 72 * This is a test requirement - the bit array we use to verify 73 * pages are valid is only so large. 74 */ 75 SPDK_CU_ASSERT_FATAL((uintptr_t)vaddr < (VALUE_2MB * PAGE_ARRAY_SIZE)); 76 77 i = (uintptr_t)vaddr >> SHIFT_2MB; 78 end = i + (len >> SHIFT_2MB); 79 for (; i < end; i++) { 80 switch (action) { 81 case SPDK_MEM_MAP_NOTIFY_REGISTER: 82 /* This page should not already be registered */ 83 SPDK_CU_ASSERT_FATAL(spdk_bit_array_get(g_page_array, i) == false); 84 SPDK_CU_ASSERT_FATAL(spdk_bit_array_set(g_page_array, i) == 0); 85 break; 86 case SPDK_MEM_MAP_NOTIFY_UNREGISTER: 87 SPDK_CU_ASSERT_FATAL(spdk_bit_array_get(g_page_array, i) == true); 88 spdk_bit_array_clear(g_page_array, i); 89 break; 90 default: 91 SPDK_UNREACHABLE(); 92 } 93 } 94 95 return 0; 96 } 97 98 static int 99 test_mem_map_notify_fail(void *cb_ctx, struct spdk_mem_map *map, 100 enum spdk_mem_map_notify_action action, void *vaddr, size_t size) 101 { 102 struct spdk_mem_map *reg_map = cb_ctx; 103 104 switch (action) { 105 case SPDK_MEM_MAP_NOTIFY_REGISTER: 106 if (vaddr == g_vaddr_to_fail) { 107 /* Test the error handling. */ 108 return -1; 109 } 110 break; 111 case SPDK_MEM_MAP_NOTIFY_UNREGISTER: 112 /* Clear the same region in the other mem_map to be able to 113 * verify that there was no memory left still registered after 114 * the mem_map creation failure. 115 */ 116 spdk_mem_map_clear_translation(reg_map, (uint64_t)vaddr, size); 117 break; 118 } 119 120 return 0; 121 } 122 123 static int 124 test_mem_map_notify_checklen(void *cb_ctx, struct spdk_mem_map *map, 125 enum spdk_mem_map_notify_action action, void *vaddr, size_t size) 126 { 127 size_t *len_arr = cb_ctx; 128 129 /* 130 * This is a test requirement - the len array we use to verify 131 * pages are valid is only so large. 132 */ 133 SPDK_CU_ASSERT_FATAL((uintptr_t)vaddr < (VALUE_2MB * PAGE_ARRAY_SIZE)); 134 135 switch (action) { 136 case SPDK_MEM_MAP_NOTIFY_REGISTER: 137 assert(size == len_arr[(uintptr_t)vaddr / VALUE_2MB]); 138 break; 139 case SPDK_MEM_MAP_NOTIFY_UNREGISTER: 140 CU_ASSERT(size == len_arr[(uintptr_t)vaddr / VALUE_2MB]); 141 break; 142 } 143 144 return 0; 145 } 146 147 static int 148 test_check_regions_contiguous(uint64_t addr1, uint64_t addr2) 149 { 150 return addr1 == addr2; 151 } 152 153 const struct spdk_mem_map_ops test_mem_map_ops = { 154 .notify_cb = test_mem_map_notify, 155 .are_contiguous = test_check_regions_contiguous 156 }; 157 158 const struct spdk_mem_map_ops test_mem_map_ops_no_contig = { 159 .notify_cb = test_mem_map_notify, 160 .are_contiguous = NULL 161 }; 162 163 struct spdk_mem_map_ops test_map_ops_notify_fail = { 164 .notify_cb = test_mem_map_notify_fail, 165 .are_contiguous = NULL 166 }; 167 168 struct spdk_mem_map_ops test_map_ops_notify_checklen = { 169 .notify_cb = test_mem_map_notify_checklen, 170 .are_contiguous = NULL 171 }; 172 173 static void 174 test_mem_map_alloc_free(void) 175 { 176 struct spdk_mem_map *map, *failed_map; 177 uint64_t default_translation = 0xDEADBEEF0BADF00D; 178 int i; 179 180 map = spdk_mem_map_alloc(default_translation, &test_mem_map_ops, NULL); 181 SPDK_CU_ASSERT_FATAL(map != NULL); 182 spdk_mem_map_free(&map); 183 CU_ASSERT(map == NULL); 184 185 map = spdk_mem_map_alloc(default_translation, NULL, NULL); 186 SPDK_CU_ASSERT_FATAL(map != NULL); 187 188 /* Register some memory for the initial memory walk in 189 * spdk_mem_map_alloc(). We'll fail registering the last region 190 * and will check if the mem_map cleaned up all its previously 191 * initialized translations. 192 */ 193 for (i = 0; i < 5; i++) { 194 spdk_mem_register((void *)(uintptr_t)(2 * i * VALUE_2MB), VALUE_2MB); 195 } 196 197 /* The last region */ 198 g_vaddr_to_fail = (void *)(8 * VALUE_2MB); 199 failed_map = spdk_mem_map_alloc(default_translation, &test_map_ops_notify_fail, map); 200 CU_ASSERT(failed_map == NULL); 201 202 for (i = 0; i < 4; i++) { 203 uint64_t reg, size = VALUE_2MB; 204 205 reg = spdk_mem_map_translate(map, 2 * i * VALUE_2MB, &size); 206 /* check if `failed_map` didn't leave any translations behind */ 207 CU_ASSERT(reg == default_translation); 208 } 209 210 for (i = 0; i < 5; i++) { 211 spdk_mem_unregister((void *)(uintptr_t)(2 * i * VALUE_2MB), VALUE_2MB); 212 } 213 214 spdk_mem_map_free(&map); 215 CU_ASSERT(map == NULL); 216 } 217 218 static void 219 test_mem_map_translation(void) 220 { 221 struct spdk_mem_map *map; 222 uint64_t default_translation = 0xDEADBEEF0BADF00D; 223 uint64_t addr; 224 uint64_t mapping_length; 225 int rc; 226 227 map = spdk_mem_map_alloc(default_translation, &test_mem_map_ops, NULL); 228 SPDK_CU_ASSERT_FATAL(map != NULL); 229 230 /* Try to get translation for address with no translation */ 231 addr = spdk_mem_map_translate(map, 10, NULL); 232 CU_ASSERT(addr == default_translation); 233 234 /* Set translation for region of non-2MB multiple size */ 235 rc = spdk_mem_map_set_translation(map, VALUE_2MB, 1234, VALUE_2MB); 236 CU_ASSERT(rc == -EINVAL); 237 238 /* Set translation for vaddr that isn't 2MB aligned */ 239 rc = spdk_mem_map_set_translation(map, 1234, VALUE_2MB, VALUE_2MB); 240 CU_ASSERT(rc == -EINVAL); 241 242 /* Set translation for one 2MB page */ 243 rc = spdk_mem_map_set_translation(map, VALUE_2MB, VALUE_2MB, VALUE_2MB); 244 CU_ASSERT(rc == 0); 245 246 /* Set translation for region that overlaps the previous translation */ 247 rc = spdk_mem_map_set_translation(map, 0, 3 * VALUE_2MB, 0); 248 CU_ASSERT(rc == 0); 249 250 /* Make sure we indicate that the three regions are contiguous */ 251 mapping_length = VALUE_2MB * 3; 252 addr = spdk_mem_map_translate(map, 0, &mapping_length); 253 CU_ASSERT(addr == 0); 254 CU_ASSERT(mapping_length == VALUE_2MB * 3); 255 256 /* Translate an unaligned address */ 257 mapping_length = VALUE_2MB * 3; 258 addr = spdk_mem_map_translate(map, VALUE_4KB, &mapping_length); 259 CU_ASSERT(addr == 0); 260 CU_ASSERT(mapping_length == VALUE_2MB * 3 - VALUE_4KB); 261 262 /* Clear translation for the middle page of the larger region. */ 263 rc = spdk_mem_map_clear_translation(map, VALUE_2MB, VALUE_2MB); 264 CU_ASSERT(rc == 0); 265 266 /* Get translation for first page */ 267 addr = spdk_mem_map_translate(map, 0, NULL); 268 CU_ASSERT(addr == 0); 269 270 /* Make sure we indicate that the three regions are no longer contiguous */ 271 mapping_length = VALUE_2MB * 3; 272 addr = spdk_mem_map_translate(map, 0, &mapping_length); 273 CU_ASSERT(addr == 0); 274 CU_ASSERT(mapping_length == VALUE_2MB); 275 276 /* Get translation for an unallocated block. Make sure size is 0 */ 277 mapping_length = VALUE_2MB * 3; 278 addr = spdk_mem_map_translate(map, VALUE_2MB, &mapping_length); 279 CU_ASSERT(addr == default_translation); 280 CU_ASSERT(mapping_length == VALUE_2MB); 281 282 /* Verify translation for 2nd page is the default */ 283 addr = spdk_mem_map_translate(map, VALUE_2MB, NULL); 284 CU_ASSERT(addr == default_translation); 285 286 /* Get translation for third page */ 287 addr = spdk_mem_map_translate(map, 2 * VALUE_2MB, NULL); 288 /* 289 * Note that addr should be 0, not 4MB. When we set the 290 * translation above, we said the whole 6MB region 291 * should translate to 0. 292 */ 293 CU_ASSERT(addr == 0); 294 295 /* Translate only a subset of a 2MB page */ 296 mapping_length = 543; 297 addr = spdk_mem_map_translate(map, 0, &mapping_length); 298 CU_ASSERT(addr == 0); 299 CU_ASSERT(mapping_length == 543); 300 301 /* Translate another subset of a 2MB page */ 302 mapping_length = 543; 303 addr = spdk_mem_map_translate(map, VALUE_4KB, &mapping_length); 304 CU_ASSERT(addr == 0); 305 CU_ASSERT(mapping_length == 543); 306 307 /* Try to translate an unaligned region that is only partially registered */ 308 mapping_length = 543; 309 addr = spdk_mem_map_translate(map, 3 * VALUE_2MB - 196, &mapping_length); 310 CU_ASSERT(addr == 0); 311 CU_ASSERT(mapping_length == 196); 312 313 /* Clear translation for the first page */ 314 rc = spdk_mem_map_clear_translation(map, 0, VALUE_2MB); 315 CU_ASSERT(rc == 0); 316 317 /* Get translation for the first page */ 318 addr = spdk_mem_map_translate(map, 0, NULL); 319 CU_ASSERT(addr == default_translation); 320 321 /* Clear translation for the third page */ 322 rc = spdk_mem_map_clear_translation(map, 2 * VALUE_2MB, VALUE_2MB); 323 CU_ASSERT(rc == 0); 324 325 /* Get translation for the third page */ 326 addr = spdk_mem_map_translate(map, 2 * VALUE_2MB, NULL); 327 CU_ASSERT(addr == default_translation); 328 329 /* Set translation for the last valid 2MB region */ 330 rc = spdk_mem_map_set_translation(map, 0xffffffe00000ULL, VALUE_2MB, 0x1234); 331 CU_ASSERT(rc == 0); 332 333 /* Verify translation for last valid 2MB region */ 334 addr = spdk_mem_map_translate(map, 0xffffffe00000ULL, NULL); 335 CU_ASSERT(addr == 0x1234); 336 337 /* Attempt to set translation for the first invalid address */ 338 rc = spdk_mem_map_set_translation(map, 0x1000000000000ULL, VALUE_2MB, 0x5678); 339 CU_ASSERT(rc == -EINVAL); 340 341 /* Attempt to set translation starting at a valid address but exceeding the valid range */ 342 rc = spdk_mem_map_set_translation(map, 0xffffffe00000ULL, VALUE_2MB * 2, 0x123123); 343 CU_ASSERT(rc != 0); 344 345 spdk_mem_map_free(&map); 346 CU_ASSERT(map == NULL); 347 348 /* Allocate a map without a contiguous region checker */ 349 map = spdk_mem_map_alloc(default_translation, &test_mem_map_ops_no_contig, NULL); 350 SPDK_CU_ASSERT_FATAL(map != NULL); 351 352 /* map three contiguous regions */ 353 rc = spdk_mem_map_set_translation(map, 0, 3 * VALUE_2MB, 0); 354 CU_ASSERT(rc == 0); 355 356 /* Since we can't check their contiguity, make sure we only return the size of one page */ 357 mapping_length = VALUE_2MB * 3; 358 addr = spdk_mem_map_translate(map, 0, &mapping_length); 359 CU_ASSERT(addr == 0); 360 CU_ASSERT(mapping_length == VALUE_2MB); 361 362 /* Translate only a subset of a 2MB page */ 363 mapping_length = 543; 364 addr = spdk_mem_map_translate(map, 0, &mapping_length); 365 CU_ASSERT(addr == 0); 366 CU_ASSERT(mapping_length == 543); 367 368 /* Clear the translation */ 369 rc = spdk_mem_map_clear_translation(map, 0, VALUE_2MB * 3); 370 CU_ASSERT(rc == 0); 371 372 spdk_mem_map_free(&map); 373 CU_ASSERT(map == NULL); 374 } 375 376 static void 377 test_mem_map_registration(void) 378 { 379 int rc; 380 struct spdk_mem_map *map; 381 uint64_t default_translation = 0xDEADBEEF0BADF00D; 382 383 map = spdk_mem_map_alloc(default_translation, &test_mem_map_ops, NULL); 384 SPDK_CU_ASSERT_FATAL(map != NULL); 385 386 /* Unregister memory region that wasn't previously registered */ 387 rc = spdk_mem_unregister((void *)VALUE_2MB, VALUE_2MB); 388 CU_ASSERT(rc == -EINVAL); 389 390 /* Register non-2MB multiple size */ 391 rc = spdk_mem_register((void *)VALUE_2MB, 1234); 392 CU_ASSERT(rc == -EINVAL); 393 394 /* Register region that isn't 2MB aligned */ 395 rc = spdk_mem_register((void *)1234, VALUE_2MB); 396 CU_ASSERT(rc == -EINVAL); 397 398 /* Register one 2MB page */ 399 rc = spdk_mem_register((void *)VALUE_2MB, VALUE_2MB); 400 CU_ASSERT(rc == 0); 401 402 /* Register an overlapping address range */ 403 rc = spdk_mem_register((void *)0, 3 * VALUE_2MB); 404 CU_ASSERT(rc == -EBUSY); 405 406 /* Unregister a 2MB page */ 407 rc = spdk_mem_unregister((void *)VALUE_2MB, VALUE_2MB); 408 CU_ASSERT(rc == 0); 409 410 /* Register non overlapping address range */ 411 rc = spdk_mem_register((void *)0, 3 * VALUE_2MB); 412 CU_ASSERT(rc == 0); 413 414 /* Unregister the middle page of the larger region. */ 415 rc = spdk_mem_unregister((void *)VALUE_2MB, VALUE_2MB); 416 CU_ASSERT(rc == -ERANGE); 417 418 /* Unregister the first page */ 419 rc = spdk_mem_unregister((void *)0, VALUE_2MB); 420 CU_ASSERT(rc == -ERANGE); 421 422 /* Unregister the third page */ 423 rc = spdk_mem_unregister((void *)(2 * VALUE_2MB), VALUE_2MB); 424 CU_ASSERT(rc == -ERANGE); 425 426 /* Unregister the entire address range */ 427 rc = spdk_mem_unregister((void *)0, 3 * VALUE_2MB); 428 CU_ASSERT(rc == 0); 429 430 spdk_mem_map_free(&map); 431 CU_ASSERT(map == NULL); 432 } 433 434 static void 435 test_mem_map_registration_adjacent(void) 436 { 437 struct spdk_mem_map *map, *newmap; 438 uint64_t default_translation = 0xDEADBEEF0BADF00D; 439 uintptr_t vaddr; 440 unsigned i; 441 size_t notify_len[PAGE_ARRAY_SIZE] = {0}; 442 size_t chunk_len[] = { 2, 1, 3, 2, 1, 1 }; 443 444 map = spdk_mem_map_alloc(default_translation, 445 &test_map_ops_notify_checklen, notify_len); 446 SPDK_CU_ASSERT_FATAL(map != NULL); 447 448 vaddr = 0; 449 for (i = 0; i < SPDK_COUNTOF(chunk_len); i++) { 450 notify_len[vaddr / VALUE_2MB] = chunk_len[i] * VALUE_2MB; 451 spdk_mem_register((void *)vaddr, notify_len[vaddr / VALUE_2MB]); 452 vaddr += notify_len[vaddr / VALUE_2MB]; 453 } 454 455 /* Verify the memory is translated in the same chunks it was registered */ 456 newmap = spdk_mem_map_alloc(default_translation, 457 &test_map_ops_notify_checklen, notify_len); 458 SPDK_CU_ASSERT_FATAL(newmap != NULL); 459 spdk_mem_map_free(&newmap); 460 CU_ASSERT(newmap == NULL); 461 462 vaddr = 0; 463 for (i = 0; i < SPDK_COUNTOF(chunk_len); i++) { 464 notify_len[vaddr / VALUE_2MB] = chunk_len[i] * VALUE_2MB; 465 spdk_mem_unregister((void *)vaddr, notify_len[vaddr / VALUE_2MB]); 466 vaddr += notify_len[vaddr / VALUE_2MB]; 467 } 468 469 /* Register all chunks again just to unregister them again, but this 470 * time with only a single unregister() call. 471 */ 472 vaddr = 0; 473 for (i = 0; i < SPDK_COUNTOF(chunk_len); i++) { 474 notify_len[vaddr / VALUE_2MB] = chunk_len[i] * VALUE_2MB; 475 spdk_mem_register((void *)vaddr, notify_len[vaddr / VALUE_2MB]); 476 vaddr += notify_len[vaddr / VALUE_2MB]; 477 } 478 spdk_mem_unregister(0, vaddr); 479 480 spdk_mem_map_free(&map); 481 CU_ASSERT(map == NULL); 482 } 483 484 int 485 main(int argc, char **argv) 486 { 487 CU_pSuite suite = NULL; 488 unsigned int num_failures; 489 490 /* 491 * These tests can use PAGE_ARRAY_SIZE 2MB pages of memory. 492 * Note that the tests just verify addresses - this memory 493 * is not actually allocated. 494 */ 495 g_page_array = spdk_bit_array_create(PAGE_ARRAY_SIZE); 496 497 /* Initialize the memory map */ 498 if (mem_map_init(false) < 0) { 499 return CUE_NOMEMORY; 500 } 501 502 if (CU_initialize_registry() != CUE_SUCCESS) { 503 return CU_get_error(); 504 } 505 506 suite = CU_add_suite("memory", NULL, NULL); 507 if (suite == NULL) { 508 CU_cleanup_registry(); 509 return CU_get_error(); 510 } 511 512 if ( 513 CU_add_test(suite, "alloc and free memory map", test_mem_map_alloc_free) == NULL || 514 CU_add_test(suite, "mem map translation", test_mem_map_translation) == NULL || 515 CU_add_test(suite, "mem map registration", test_mem_map_registration) == NULL || 516 CU_add_test(suite, "mem map adjacent registrations", test_mem_map_registration_adjacent) == NULL 517 ) { 518 CU_cleanup_registry(); 519 return CU_get_error(); 520 } 521 522 CU_basic_set_mode(CU_BRM_VERBOSE); 523 CU_basic_run_tests(); 524 num_failures = CU_get_number_of_failures(); 525 CU_cleanup_registry(); 526 527 spdk_bit_array_free(&g_page_array); 528 529 return num_failures; 530 } 531