1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2017 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "env_dpdk/memory.c" 7 8 #define UNIT_TEST_NO_VTOPHYS 9 #define UNIT_TEST_NO_PCI_ADDR 10 #include "common/lib/test_env.c" 11 #include "spdk_cunit.h" 12 13 #include "spdk/bit_array.h" 14 15 #define PAGE_ARRAY_SIZE (100) 16 static struct spdk_bit_array *g_page_array; 17 static void *g_vaddr_to_fail = (void *)UINT64_MAX; 18 19 DEFINE_STUB(rte_memseg_contig_walk, int, (rte_memseg_contig_walk_t func, void *arg), 0); 20 DEFINE_STUB(rte_mem_virt2memseg, struct rte_memseg *, 21 (const void *virt, const struct rte_memseg_list *msl), NULL); 22 DEFINE_STUB(spdk_env_dpdk_external_init, bool, (void), true); 23 DEFINE_STUB(rte_mem_event_callback_register, int, 24 (const char *name, rte_mem_event_callback_t clb, void *arg), 0); 25 DEFINE_STUB(rte_mem_virt2iova, rte_iova_t, (const void *virtaddr), 0); 26 DEFINE_STUB(rte_eal_iova_mode, enum rte_iova_mode, (void), RTE_IOVA_VA); 27 DEFINE_STUB(rte_vfio_is_enabled, int, (const char *modname), 0); 28 DEFINE_STUB(rte_vfio_noiommu_is_enabled, int, (void), 0); 29 DEFINE_STUB(rte_memseg_get_fd_thread_unsafe, int, (const struct rte_memseg *ms), 0); 30 DEFINE_STUB(rte_memseg_get_fd_offset_thread_unsafe, int, 31 (const struct rte_memseg *ms, size_t *offset), 0); 32 DEFINE_STUB(dpdk_pci_device_get_mem_resource, struct rte_mem_resource *, 33 (struct rte_pci_device *dev, uint32_t bar), 0); 34 35 static int 36 test_mem_map_notify(void *cb_ctx, struct spdk_mem_map *map, 37 enum spdk_mem_map_notify_action action, 38 void *vaddr, size_t len) 39 { 40 uint32_t i, end; 41 42 SPDK_CU_ASSERT_FATAL(((uintptr_t)vaddr & MASK_2MB) == 0); 43 SPDK_CU_ASSERT_FATAL((len & MASK_2MB) == 0); 44 45 /* 46 * This is a test requirement - the bit array we use to verify 47 * pages are valid is only so large. 48 */ 49 SPDK_CU_ASSERT_FATAL((uintptr_t)vaddr < (VALUE_2MB * PAGE_ARRAY_SIZE)); 50 51 i = (uintptr_t)vaddr >> SHIFT_2MB; 52 end = i + (len >> SHIFT_2MB); 53 for (; i < end; i++) { 54 switch (action) { 55 case SPDK_MEM_MAP_NOTIFY_REGISTER: 56 /* This page should not already be registered */ 57 SPDK_CU_ASSERT_FATAL(spdk_bit_array_get(g_page_array, i) == false); 58 SPDK_CU_ASSERT_FATAL(spdk_bit_array_set(g_page_array, i) == 0); 59 break; 60 case SPDK_MEM_MAP_NOTIFY_UNREGISTER: 61 SPDK_CU_ASSERT_FATAL(spdk_bit_array_get(g_page_array, i) == true); 62 spdk_bit_array_clear(g_page_array, i); 63 break; 64 default: 65 SPDK_UNREACHABLE(); 66 } 67 } 68 69 return 0; 70 } 71 72 static int 73 test_mem_map_notify_fail(void *cb_ctx, struct spdk_mem_map *map, 74 enum spdk_mem_map_notify_action action, void *vaddr, size_t size) 75 { 76 struct spdk_mem_map *reg_map = cb_ctx; 77 uint64_t reg_addr; 78 uint64_t reg_size = size; 79 80 switch (action) { 81 case SPDK_MEM_MAP_NOTIFY_REGISTER: 82 if (vaddr == g_vaddr_to_fail) { 83 /* Test the error handling. */ 84 return -1; 85 } 86 87 CU_ASSERT(spdk_mem_map_set_translation(map, (uint64_t)vaddr, (uint64_t)size, (uint64_t)vaddr) == 0); 88 89 break; 90 case SPDK_MEM_MAP_NOTIFY_UNREGISTER: 91 /* validate the start address */ 92 reg_addr = spdk_mem_map_translate(map, (uint64_t)vaddr, ®_size); 93 CU_ASSERT(reg_addr == (uint64_t)vaddr); 94 spdk_mem_map_clear_translation(map, (uint64_t)vaddr, size); 95 96 /* Clear the same region in the other mem_map to be able to 97 * verify that there was no memory left still registered after 98 * the mem_map creation failure. 99 */ 100 spdk_mem_map_clear_translation(reg_map, (uint64_t)vaddr, size); 101 break; 102 } 103 104 return 0; 105 } 106 107 static int 108 test_mem_map_notify_checklen(void *cb_ctx, struct spdk_mem_map *map, 109 enum spdk_mem_map_notify_action action, void *vaddr, size_t size) 110 { 111 size_t *len_arr = cb_ctx; 112 113 /* 114 * This is a test requirement - the len array we use to verify 115 * pages are valid is only so large. 116 */ 117 SPDK_CU_ASSERT_FATAL((uintptr_t)vaddr < (VALUE_2MB * PAGE_ARRAY_SIZE)); 118 119 switch (action) { 120 case SPDK_MEM_MAP_NOTIFY_REGISTER: 121 assert(size == len_arr[(uintptr_t)vaddr / VALUE_2MB]); 122 break; 123 case SPDK_MEM_MAP_NOTIFY_UNREGISTER: 124 CU_ASSERT(size == len_arr[(uintptr_t)vaddr / VALUE_2MB]); 125 break; 126 } 127 128 return 0; 129 } 130 131 static int 132 test_check_regions_contiguous(uint64_t addr1, uint64_t addr2) 133 { 134 return addr1 == addr2; 135 } 136 137 const struct spdk_mem_map_ops test_mem_map_ops = { 138 .notify_cb = test_mem_map_notify, 139 .are_contiguous = test_check_regions_contiguous 140 }; 141 142 const struct spdk_mem_map_ops test_mem_map_ops_no_contig = { 143 .notify_cb = test_mem_map_notify, 144 .are_contiguous = NULL 145 }; 146 147 struct spdk_mem_map_ops test_map_ops_notify_fail = { 148 .notify_cb = test_mem_map_notify_fail, 149 .are_contiguous = NULL 150 }; 151 152 struct spdk_mem_map_ops test_map_ops_notify_checklen = { 153 .notify_cb = test_mem_map_notify_checklen, 154 .are_contiguous = NULL 155 }; 156 157 static void 158 test_mem_map_alloc_free(void) 159 { 160 struct spdk_mem_map *map, *failed_map; 161 uint64_t default_translation = 0xDEADBEEF0BADF00D; 162 int i; 163 164 map = spdk_mem_map_alloc(default_translation, &test_mem_map_ops, NULL); 165 SPDK_CU_ASSERT_FATAL(map != NULL); 166 spdk_mem_map_free(&map); 167 CU_ASSERT(map == NULL); 168 169 map = spdk_mem_map_alloc(default_translation, NULL, NULL); 170 SPDK_CU_ASSERT_FATAL(map != NULL); 171 172 /* Register some memory for the initial memory walk in 173 * spdk_mem_map_alloc(). We'll fail registering the last region 174 * and will check if the mem_map cleaned up all its previously 175 * initialized translations. 176 */ 177 for (i = 0; i < 5; i++) { 178 spdk_mem_register((void *)(uintptr_t)(2 * i * VALUE_2MB), VALUE_2MB); 179 } 180 181 /* The last region */ 182 g_vaddr_to_fail = (void *)(8 * VALUE_2MB); 183 failed_map = spdk_mem_map_alloc(default_translation, &test_map_ops_notify_fail, map); 184 CU_ASSERT(failed_map == NULL); 185 186 for (i = 0; i < 4; i++) { 187 uint64_t reg, size = VALUE_2MB; 188 189 reg = spdk_mem_map_translate(map, 2 * i * VALUE_2MB, &size); 190 /* check if `failed_map` didn't leave any translations behind */ 191 CU_ASSERT(reg == default_translation); 192 } 193 194 for (i = 0; i < 5; i++) { 195 spdk_mem_unregister((void *)(uintptr_t)(2 * i * VALUE_2MB), VALUE_2MB); 196 } 197 198 spdk_mem_map_free(&map); 199 CU_ASSERT(map == NULL); 200 } 201 202 static void 203 test_mem_map_translation(void) 204 { 205 struct spdk_mem_map *map; 206 uint64_t default_translation = 0xDEADBEEF0BADF00D; 207 uint64_t addr; 208 uint64_t mapping_length; 209 int rc; 210 211 map = spdk_mem_map_alloc(default_translation, &test_mem_map_ops, NULL); 212 SPDK_CU_ASSERT_FATAL(map != NULL); 213 214 /* Try to get translation for address with no translation */ 215 addr = spdk_mem_map_translate(map, 10, NULL); 216 CU_ASSERT(addr == default_translation); 217 218 /* Set translation for region of non-2MB multiple size */ 219 rc = spdk_mem_map_set_translation(map, VALUE_2MB, 1234, VALUE_2MB); 220 CU_ASSERT(rc == -EINVAL); 221 222 /* Set translation for vaddr that isn't 2MB aligned */ 223 rc = spdk_mem_map_set_translation(map, 1234, VALUE_2MB, VALUE_2MB); 224 CU_ASSERT(rc == -EINVAL); 225 226 /* Set translation for one 2MB page */ 227 rc = spdk_mem_map_set_translation(map, VALUE_2MB, VALUE_2MB, VALUE_2MB); 228 CU_ASSERT(rc == 0); 229 230 /* Set translation for region that overlaps the previous translation */ 231 rc = spdk_mem_map_set_translation(map, 0, 3 * VALUE_2MB, 0); 232 CU_ASSERT(rc == 0); 233 234 /* Make sure we indicate that the three regions are contiguous */ 235 mapping_length = VALUE_2MB * 3; 236 addr = spdk_mem_map_translate(map, 0, &mapping_length); 237 CU_ASSERT(addr == 0); 238 CU_ASSERT(mapping_length == VALUE_2MB * 3); 239 240 /* Translate an unaligned address */ 241 mapping_length = VALUE_2MB * 3; 242 addr = spdk_mem_map_translate(map, VALUE_4KB, &mapping_length); 243 CU_ASSERT(addr == 0); 244 CU_ASSERT(mapping_length == VALUE_2MB * 3 - VALUE_4KB); 245 246 /* Clear translation for the middle page of the larger region. */ 247 rc = spdk_mem_map_clear_translation(map, VALUE_2MB, VALUE_2MB); 248 CU_ASSERT(rc == 0); 249 250 /* Get translation for first page */ 251 addr = spdk_mem_map_translate(map, 0, NULL); 252 CU_ASSERT(addr == 0); 253 254 /* Make sure we indicate that the three regions are no longer contiguous */ 255 mapping_length = VALUE_2MB * 3; 256 addr = spdk_mem_map_translate(map, 0, &mapping_length); 257 CU_ASSERT(addr == 0); 258 CU_ASSERT(mapping_length == VALUE_2MB); 259 260 /* Get translation for an unallocated block. Make sure size is 0 */ 261 mapping_length = VALUE_2MB * 3; 262 addr = spdk_mem_map_translate(map, VALUE_2MB, &mapping_length); 263 CU_ASSERT(addr == default_translation); 264 CU_ASSERT(mapping_length == VALUE_2MB); 265 266 /* Verify translation for 2nd page is the default */ 267 addr = spdk_mem_map_translate(map, VALUE_2MB, NULL); 268 CU_ASSERT(addr == default_translation); 269 270 /* Get translation for third page */ 271 addr = spdk_mem_map_translate(map, 2 * VALUE_2MB, NULL); 272 /* 273 * Note that addr should be 0, not 4MB. When we set the 274 * translation above, we said the whole 6MB region 275 * should translate to 0. 276 */ 277 CU_ASSERT(addr == 0); 278 279 /* Translate only a subset of a 2MB page */ 280 mapping_length = 543; 281 addr = spdk_mem_map_translate(map, 0, &mapping_length); 282 CU_ASSERT(addr == 0); 283 CU_ASSERT(mapping_length == 543); 284 285 /* Translate another subset of a 2MB page */ 286 mapping_length = 543; 287 addr = spdk_mem_map_translate(map, VALUE_4KB, &mapping_length); 288 CU_ASSERT(addr == 0); 289 CU_ASSERT(mapping_length == 543); 290 291 /* Try to translate an unaligned region that is only partially registered */ 292 mapping_length = 543; 293 addr = spdk_mem_map_translate(map, 3 * VALUE_2MB - 196, &mapping_length); 294 CU_ASSERT(addr == 0); 295 CU_ASSERT(mapping_length == 196); 296 297 /* Clear translation for the first page */ 298 rc = spdk_mem_map_clear_translation(map, 0, VALUE_2MB); 299 CU_ASSERT(rc == 0); 300 301 /* Get translation for the first page */ 302 addr = spdk_mem_map_translate(map, 0, NULL); 303 CU_ASSERT(addr == default_translation); 304 305 /* Clear translation for the third page */ 306 rc = spdk_mem_map_clear_translation(map, 2 * VALUE_2MB, VALUE_2MB); 307 CU_ASSERT(rc == 0); 308 309 /* Get translation for the third page */ 310 addr = spdk_mem_map_translate(map, 2 * VALUE_2MB, NULL); 311 CU_ASSERT(addr == default_translation); 312 313 /* Set translation for the last valid 2MB region */ 314 rc = spdk_mem_map_set_translation(map, 0xffffffe00000ULL, VALUE_2MB, 0x1234); 315 CU_ASSERT(rc == 0); 316 317 /* Verify translation for last valid 2MB region */ 318 addr = spdk_mem_map_translate(map, 0xffffffe00000ULL, NULL); 319 CU_ASSERT(addr == 0x1234); 320 321 /* Attempt to set translation for the first invalid address */ 322 rc = spdk_mem_map_set_translation(map, 0x1000000000000ULL, VALUE_2MB, 0x5678); 323 CU_ASSERT(rc == -EINVAL); 324 325 /* Attempt to set translation starting at a valid address but exceeding the valid range */ 326 rc = spdk_mem_map_set_translation(map, 0xffffffe00000ULL, VALUE_2MB * 2, 0x123123); 327 CU_ASSERT(rc != 0); 328 329 spdk_mem_map_free(&map); 330 CU_ASSERT(map == NULL); 331 332 /* Allocate a map without a contiguous region checker */ 333 map = spdk_mem_map_alloc(default_translation, &test_mem_map_ops_no_contig, NULL); 334 SPDK_CU_ASSERT_FATAL(map != NULL); 335 336 /* map three contiguous regions */ 337 rc = spdk_mem_map_set_translation(map, 0, 3 * VALUE_2MB, 0); 338 CU_ASSERT(rc == 0); 339 340 /* Since we can't check their contiguity, make sure we only return the size of one page */ 341 mapping_length = VALUE_2MB * 3; 342 addr = spdk_mem_map_translate(map, 0, &mapping_length); 343 CU_ASSERT(addr == 0); 344 CU_ASSERT(mapping_length == VALUE_2MB); 345 346 /* Translate only a subset of a 2MB page */ 347 mapping_length = 543; 348 addr = spdk_mem_map_translate(map, 0, &mapping_length); 349 CU_ASSERT(addr == 0); 350 CU_ASSERT(mapping_length == 543); 351 352 /* Clear the translation */ 353 rc = spdk_mem_map_clear_translation(map, 0, VALUE_2MB * 3); 354 CU_ASSERT(rc == 0); 355 356 spdk_mem_map_free(&map); 357 CU_ASSERT(map == NULL); 358 } 359 360 static void 361 test_mem_map_registration(void) 362 { 363 int rc; 364 struct spdk_mem_map *map; 365 uint64_t default_translation = 0xDEADBEEF0BADF00D; 366 367 map = spdk_mem_map_alloc(default_translation, &test_mem_map_ops, NULL); 368 SPDK_CU_ASSERT_FATAL(map != NULL); 369 370 /* Unregister memory region that wasn't previously registered */ 371 rc = spdk_mem_unregister((void *)VALUE_2MB, VALUE_2MB); 372 CU_ASSERT(rc == -EINVAL); 373 374 /* Register non-2MB multiple size */ 375 rc = spdk_mem_register((void *)VALUE_2MB, 1234); 376 CU_ASSERT(rc == -EINVAL); 377 378 /* Register region that isn't 2MB aligned */ 379 rc = spdk_mem_register((void *)1234, VALUE_2MB); 380 CU_ASSERT(rc == -EINVAL); 381 382 /* Register one 2MB page */ 383 rc = spdk_mem_register((void *)VALUE_2MB, VALUE_2MB); 384 CU_ASSERT(rc == 0); 385 386 /* Register an overlapping address range */ 387 rc = spdk_mem_register((void *)0, 3 * VALUE_2MB); 388 CU_ASSERT(rc == -EBUSY); 389 390 /* Unregister a 2MB page */ 391 rc = spdk_mem_unregister((void *)VALUE_2MB, VALUE_2MB); 392 CU_ASSERT(rc == 0); 393 394 /* Register non overlapping address range */ 395 rc = spdk_mem_register((void *)0, 3 * VALUE_2MB); 396 CU_ASSERT(rc == 0); 397 398 /* Unregister the middle page of the larger region. */ 399 rc = spdk_mem_unregister((void *)VALUE_2MB, VALUE_2MB); 400 CU_ASSERT(rc == -ERANGE); 401 402 /* Unregister the first page */ 403 rc = spdk_mem_unregister((void *)0, VALUE_2MB); 404 CU_ASSERT(rc == -ERANGE); 405 406 /* Unregister the third page */ 407 rc = spdk_mem_unregister((void *)(2 * VALUE_2MB), VALUE_2MB); 408 CU_ASSERT(rc == -ERANGE); 409 410 /* Unregister the entire address range */ 411 rc = spdk_mem_unregister((void *)0, 3 * VALUE_2MB); 412 CU_ASSERT(rc == 0); 413 414 spdk_mem_map_free(&map); 415 CU_ASSERT(map == NULL); 416 } 417 418 static void 419 test_mem_map_registration_adjacent(void) 420 { 421 struct spdk_mem_map *map, *newmap; 422 uint64_t default_translation = 0xDEADBEEF0BADF00D; 423 uintptr_t vaddr; 424 unsigned i; 425 size_t notify_len[PAGE_ARRAY_SIZE] = {0}; 426 size_t chunk_len[] = { 2, 1, 3, 2, 1, 1 }; 427 428 map = spdk_mem_map_alloc(default_translation, 429 &test_map_ops_notify_checklen, notify_len); 430 SPDK_CU_ASSERT_FATAL(map != NULL); 431 432 vaddr = 0; 433 for (i = 0; i < SPDK_COUNTOF(chunk_len); i++) { 434 notify_len[vaddr / VALUE_2MB] = chunk_len[i] * VALUE_2MB; 435 spdk_mem_register((void *)vaddr, notify_len[vaddr / VALUE_2MB]); 436 vaddr += notify_len[vaddr / VALUE_2MB]; 437 } 438 439 /* Verify the memory is translated in the same chunks it was registered */ 440 newmap = spdk_mem_map_alloc(default_translation, 441 &test_map_ops_notify_checklen, notify_len); 442 SPDK_CU_ASSERT_FATAL(newmap != NULL); 443 spdk_mem_map_free(&newmap); 444 CU_ASSERT(newmap == NULL); 445 446 vaddr = 0; 447 for (i = 0; i < SPDK_COUNTOF(chunk_len); i++) { 448 notify_len[vaddr / VALUE_2MB] = chunk_len[i] * VALUE_2MB; 449 spdk_mem_unregister((void *)vaddr, notify_len[vaddr / VALUE_2MB]); 450 vaddr += notify_len[vaddr / VALUE_2MB]; 451 } 452 453 /* Register all chunks again just to unregister them again, but this 454 * time with only a single unregister() call. 455 */ 456 vaddr = 0; 457 for (i = 0; i < SPDK_COUNTOF(chunk_len); i++) { 458 notify_len[vaddr / VALUE_2MB] = chunk_len[i] * VALUE_2MB; 459 spdk_mem_register((void *)vaddr, notify_len[vaddr / VALUE_2MB]); 460 vaddr += notify_len[vaddr / VALUE_2MB]; 461 } 462 spdk_mem_unregister(0, vaddr); 463 464 spdk_mem_map_free(&map); 465 CU_ASSERT(map == NULL); 466 } 467 468 int 469 main(int argc, char **argv) 470 { 471 CU_pSuite suite = NULL; 472 unsigned int num_failures; 473 474 /* 475 * These tests can use PAGE_ARRAY_SIZE 2MB pages of memory. 476 * Note that the tests just verify addresses - this memory 477 * is not actually allocated. 478 */ 479 g_page_array = spdk_bit_array_create(PAGE_ARRAY_SIZE); 480 481 /* Initialize the memory map */ 482 if (mem_map_init(false) < 0) { 483 return CUE_NOMEMORY; 484 } 485 486 if (CU_initialize_registry() != CUE_SUCCESS) { 487 return CU_get_error(); 488 } 489 490 suite = CU_add_suite("memory", NULL, NULL); 491 if (suite == NULL) { 492 CU_cleanup_registry(); 493 return CU_get_error(); 494 } 495 496 if ( 497 CU_add_test(suite, "alloc and free memory map", test_mem_map_alloc_free) == NULL || 498 CU_add_test(suite, "mem map translation", test_mem_map_translation) == NULL || 499 CU_add_test(suite, "mem map registration", test_mem_map_registration) == NULL || 500 CU_add_test(suite, "mem map adjacent registrations", test_mem_map_registration_adjacent) == NULL 501 ) { 502 CU_cleanup_registry(); 503 return CU_get_error(); 504 } 505 506 CU_basic_set_mode(CU_BRM_VERBOSE); 507 CU_basic_run_tests(); 508 num_failures = CU_get_number_of_failures(); 509 CU_cleanup_registry(); 510 511 spdk_bit_array_free(&g_page_array); 512 513 return num_failures; 514 } 515