1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2021 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk_cunit.h" 8 #include "spdk_internal/mock.h" 9 #include "spdk_internal/accel_module.h" 10 #include "thread/thread_internal.h" 11 #include "common/lib/test_env.c" 12 #include "accel/accel.c" 13 #include "accel/accel_sw.c" 14 #include "unit/lib/json_mock.c" 15 16 #ifdef SPDK_CONFIG_PMDK 17 DEFINE_STUB(pmem_msync, int, (const void *addr, size_t len), 0); 18 DEFINE_STUB(pmem_memcpy_persist, void *, (void *pmemdest, const void *src, size_t len), NULL); 19 DEFINE_STUB(pmem_is_pmem, int, (const void *addr, size_t len), 0); 20 DEFINE_STUB(pmem_memset_persist, void *, (void *pmemdest, int c, size_t len), NULL); 21 #endif 22 23 /* global vars and setup/cleanup functions used for all test functions */ 24 struct spdk_accel_module_if g_module = {}; 25 struct spdk_io_channel *g_ch = NULL; 26 struct accel_io_channel *g_accel_ch = NULL; 27 struct sw_accel_io_channel *g_sw_ch = NULL; 28 struct spdk_io_channel *g_module_ch = NULL; 29 30 static uint64_t g_opc_mask = 0; 31 32 static uint64_t 33 _accel_op_to_bit(enum accel_opcode opc) 34 { 35 return (1 << opc); 36 } 37 38 static bool 39 _supports_opcode(enum accel_opcode opc) 40 { 41 if (_accel_op_to_bit(opc) & g_opc_mask) { 42 return true; 43 } 44 return false; 45 } 46 47 static int 48 test_setup(void) 49 { 50 int i; 51 52 g_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct accel_io_channel)); 53 if (g_ch == NULL) { 54 /* for some reason the assert fatal macro doesn't work in the setup function. */ 55 CU_ASSERT(false); 56 return -1; 57 } 58 g_accel_ch = (struct accel_io_channel *)((char *)g_ch + sizeof(struct spdk_io_channel)); 59 g_module_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct sw_accel_io_channel)); 60 if (g_module_ch == NULL) { 61 CU_ASSERT(false); 62 return -1; 63 } 64 65 g_module.submit_tasks = sw_accel_submit_tasks; 66 g_module.name = "software"; 67 for (i = 0; i < ACCEL_OPC_LAST; i++) { 68 g_accel_ch->module_ch[i] = g_module_ch; 69 g_modules_opc[i] = &g_module; 70 } 71 g_sw_ch = (struct sw_accel_io_channel *)((char *)g_module_ch + sizeof( 72 struct spdk_io_channel)); 73 TAILQ_INIT(&g_sw_ch->tasks_to_complete); 74 g_module.supports_opcode = _supports_opcode; 75 return 0; 76 } 77 78 static int 79 test_cleanup(void) 80 { 81 free(g_ch); 82 free(g_module_ch); 83 84 return 0; 85 } 86 87 #define DUMMY_ARG 0xDEADBEEF 88 static bool g_dummy_cb_called = false; 89 static void 90 dummy_cb_fn(void *cb_arg, int status) 91 { 92 CU_ASSERT(*(uint32_t *)cb_arg == DUMMY_ARG); 93 CU_ASSERT(status == 0); 94 g_dummy_cb_called = true; 95 } 96 97 static void 98 test_spdk_accel_task_complete(void) 99 { 100 struct spdk_accel_task accel_task = {}; 101 struct spdk_accel_task *expected_accel_task = NULL; 102 uint32_t cb_arg = DUMMY_ARG; 103 int status = 0; 104 105 accel_task.accel_ch = g_accel_ch; 106 accel_task.cb_fn = dummy_cb_fn; 107 accel_task.cb_arg = &cb_arg; 108 TAILQ_INIT(&g_accel_ch->task_pool); 109 110 /* Confirm cb is called and task added to list. */ 111 spdk_accel_task_complete(&accel_task, status); 112 CU_ASSERT(g_dummy_cb_called == true); 113 expected_accel_task = TAILQ_FIRST(&g_accel_ch->task_pool); 114 TAILQ_REMOVE(&g_accel_ch->task_pool, expected_accel_task, link); 115 CU_ASSERT(expected_accel_task == &accel_task); 116 } 117 118 static void 119 test_get_task(void) 120 { 121 struct spdk_accel_task *task; 122 struct spdk_accel_task _task; 123 void *cb_arg = NULL; 124 125 TAILQ_INIT(&g_accel_ch->task_pool); 126 127 /* no tasks left, return NULL. */ 128 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 129 CU_ASSERT(task == NULL); 130 131 _task.cb_fn = dummy_cb_fn; 132 _task.cb_arg = cb_arg; 133 _task.accel_ch = g_accel_ch; 134 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &_task, link); 135 136 /* Get a valid task. */ 137 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 138 CU_ASSERT(task == &_task); 139 CU_ASSERT(_task.cb_fn == dummy_cb_fn); 140 CU_ASSERT(_task.cb_arg == cb_arg); 141 CU_ASSERT(_task.accel_ch == g_accel_ch); 142 } 143 144 #define TEST_SUBMIT_SIZE 64 145 static void 146 test_spdk_accel_submit_copy(void) 147 { 148 const uint64_t nbytes = TEST_SUBMIT_SIZE; 149 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 150 uint8_t src[TEST_SUBMIT_SIZE] = {0}; 151 void *cb_arg = NULL; 152 int rc; 153 struct spdk_accel_task task; 154 struct spdk_accel_task *expected_accel_task = NULL; 155 int flags = 0; 156 157 TAILQ_INIT(&g_accel_ch->task_pool); 158 159 /* Fail with no tasks on _get_task() */ 160 rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, flags, NULL, cb_arg); 161 CU_ASSERT(rc == -ENOMEM); 162 163 task.accel_ch = g_accel_ch; 164 task.flags = 1; 165 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 166 167 /* submission OK. */ 168 rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, flags, NULL, cb_arg); 169 CU_ASSERT(rc == 0); 170 CU_ASSERT(task.dst == dst); 171 CU_ASSERT(task.src == src); 172 CU_ASSERT(task.op_code == ACCEL_OPC_COPY); 173 CU_ASSERT(task.nbytes == nbytes); 174 CU_ASSERT(task.flags == 0); 175 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 176 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 177 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 178 CU_ASSERT(expected_accel_task == &task); 179 } 180 181 static void 182 test_spdk_accel_submit_dualcast(void) 183 { 184 void *dst1; 185 void *dst2; 186 void *src; 187 uint32_t align = ALIGN_4K; 188 uint64_t nbytes = TEST_SUBMIT_SIZE; 189 void *cb_arg = NULL; 190 int rc; 191 struct spdk_accel_task task; 192 struct spdk_accel_task *expected_accel_task = NULL; 193 int flags = 0; 194 195 TAILQ_INIT(&g_accel_ch->task_pool); 196 197 /* Dualcast requires 4K alignment on dst addresses, 198 * hence using the hard coded address to test the buffer alignment 199 */ 200 dst1 = (void *)0x5000; 201 dst2 = (void *)0x60f0; 202 src = calloc(1, TEST_SUBMIT_SIZE); 203 SPDK_CU_ASSERT_FATAL(src != NULL); 204 memset(src, 0x5A, TEST_SUBMIT_SIZE); 205 206 /* This should fail since dst2 is not 4k aligned */ 207 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 208 CU_ASSERT(rc == -EINVAL); 209 210 dst1 = (void *)0x7010; 211 dst2 = (void *)0x6000; 212 /* This should fail since dst1 is not 4k aligned */ 213 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 214 CU_ASSERT(rc == -EINVAL); 215 216 /* Dualcast requires 4K alignment on dst addresses */ 217 dst1 = (void *)0x7000; 218 dst2 = (void *)0x6000; 219 /* Fail with no tasks on _get_task() */ 220 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 221 CU_ASSERT(rc == -ENOMEM); 222 223 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 224 225 /* accel submission OK., since we test the SW path , need to use valid memory addresses 226 * cannot hardcode them anymore */ 227 dst1 = spdk_dma_zmalloc(nbytes, align, NULL); 228 SPDK_CU_ASSERT_FATAL(dst1 != NULL); 229 dst2 = spdk_dma_zmalloc(nbytes, align, NULL); 230 SPDK_CU_ASSERT_FATAL(dst2 != NULL); 231 /* SW module does the dualcast. */ 232 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 233 CU_ASSERT(rc == 0); 234 CU_ASSERT(task.dst == dst1); 235 CU_ASSERT(task.dst2 == dst2); 236 CU_ASSERT(task.src == src); 237 CU_ASSERT(task.op_code == ACCEL_OPC_DUALCAST); 238 CU_ASSERT(task.nbytes == nbytes); 239 CU_ASSERT(task.flags == 0); 240 CU_ASSERT(memcmp(dst1, src, TEST_SUBMIT_SIZE) == 0); 241 CU_ASSERT(memcmp(dst2, src, TEST_SUBMIT_SIZE) == 0); 242 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 243 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 244 CU_ASSERT(expected_accel_task == &task); 245 246 free(src); 247 spdk_free(dst1); 248 spdk_free(dst2); 249 } 250 251 static void 252 test_spdk_accel_submit_compare(void) 253 { 254 void *src1; 255 void *src2; 256 uint64_t nbytes = TEST_SUBMIT_SIZE; 257 void *cb_arg = NULL; 258 int rc; 259 struct spdk_accel_task task; 260 struct spdk_accel_task *expected_accel_task = NULL; 261 262 TAILQ_INIT(&g_accel_ch->task_pool); 263 264 src1 = calloc(1, TEST_SUBMIT_SIZE); 265 SPDK_CU_ASSERT_FATAL(src1 != NULL); 266 src2 = calloc(1, TEST_SUBMIT_SIZE); 267 SPDK_CU_ASSERT_FATAL(src2 != NULL); 268 269 /* Fail with no tasks on _get_task() */ 270 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 271 CU_ASSERT(rc == -ENOMEM); 272 273 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 274 275 /* accel submission OK. */ 276 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 277 CU_ASSERT(rc == 0); 278 CU_ASSERT(task.src == src1); 279 CU_ASSERT(task.src2 == src2); 280 CU_ASSERT(task.op_code == ACCEL_OPC_COMPARE); 281 CU_ASSERT(task.nbytes == nbytes); 282 CU_ASSERT(memcmp(src1, src2, TEST_SUBMIT_SIZE) == 0); 283 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 284 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 285 CU_ASSERT(expected_accel_task == &task); 286 287 free(src1); 288 free(src2); 289 } 290 291 static void 292 test_spdk_accel_submit_fill(void) 293 { 294 void *dst; 295 void *src; 296 uint8_t fill = 0xf; 297 uint64_t fill64; 298 uint64_t nbytes = TEST_SUBMIT_SIZE; 299 void *cb_arg = NULL; 300 int rc; 301 struct spdk_accel_task task; 302 struct spdk_accel_task *expected_accel_task = NULL; 303 int flags = 0; 304 305 TAILQ_INIT(&g_accel_ch->task_pool); 306 307 dst = calloc(1, TEST_SUBMIT_SIZE); 308 SPDK_CU_ASSERT_FATAL(dst != NULL); 309 src = calloc(1, TEST_SUBMIT_SIZE); 310 SPDK_CU_ASSERT_FATAL(src != NULL); 311 memset(src, fill, TEST_SUBMIT_SIZE); 312 memset(&fill64, fill, sizeof(uint64_t)); 313 314 /* Fail with no tasks on _get_task() */ 315 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 316 CU_ASSERT(rc == -ENOMEM); 317 318 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 319 320 /* accel submission OK. */ 321 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 322 CU_ASSERT(rc == 0); 323 CU_ASSERT(task.dst == dst); 324 CU_ASSERT(task.fill_pattern == fill64); 325 CU_ASSERT(task.op_code == ACCEL_OPC_FILL); 326 CU_ASSERT(task.nbytes == nbytes); 327 CU_ASSERT(task.flags == 0); 328 329 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 330 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 331 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 332 CU_ASSERT(expected_accel_task == &task); 333 334 free(dst); 335 free(src); 336 } 337 338 static void 339 test_spdk_accel_submit_crc32c(void) 340 { 341 const uint64_t nbytes = TEST_SUBMIT_SIZE; 342 uint32_t crc_dst; 343 uint8_t src[TEST_SUBMIT_SIZE]; 344 uint32_t seed = 1; 345 void *cb_arg = NULL; 346 int rc; 347 struct spdk_accel_task task; 348 struct spdk_accel_task *expected_accel_task = NULL; 349 350 TAILQ_INIT(&g_accel_ch->task_pool); 351 352 /* Fail with no tasks on _get_task() */ 353 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 354 CU_ASSERT(rc == -ENOMEM); 355 356 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 357 358 /* accel submission OK. */ 359 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 360 CU_ASSERT(rc == 0); 361 CU_ASSERT(task.crc_dst == &crc_dst); 362 CU_ASSERT(task.src == src); 363 CU_ASSERT(task.s.iovcnt == 0); 364 CU_ASSERT(task.seed == seed); 365 CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); 366 CU_ASSERT(task.nbytes == nbytes); 367 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 368 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 369 CU_ASSERT(expected_accel_task == &task); 370 } 371 372 static void 373 test_spdk_accel_submit_crc32cv(void) 374 { 375 uint32_t crc_dst; 376 uint32_t seed = 0; 377 uint32_t iov_cnt = 32; 378 void *cb_arg = NULL; 379 int rc; 380 uint32_t i = 0; 381 struct spdk_accel_task task; 382 struct iovec iov[32]; 383 struct spdk_accel_task *expected_accel_task = NULL; 384 385 TAILQ_INIT(&g_accel_ch->task_pool); 386 387 for (i = 0; i < iov_cnt; i++) { 388 iov[i].iov_base = calloc(1, TEST_SUBMIT_SIZE); 389 SPDK_CU_ASSERT_FATAL(iov[i].iov_base != NULL); 390 iov[i].iov_len = TEST_SUBMIT_SIZE; 391 } 392 393 task.nbytes = TEST_SUBMIT_SIZE; 394 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 395 396 /* accel submission OK. */ 397 rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, NULL, cb_arg); 398 CU_ASSERT(rc == 0); 399 CU_ASSERT(task.s.iovs == iov); 400 CU_ASSERT(task.s.iovcnt == iov_cnt); 401 CU_ASSERT(task.crc_dst == &crc_dst); 402 CU_ASSERT(task.seed == seed); 403 CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); 404 CU_ASSERT(task.cb_arg == cb_arg); 405 CU_ASSERT(task.nbytes == iov[0].iov_len); 406 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 407 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 408 CU_ASSERT(expected_accel_task == &task); 409 410 for (i = 0; i < iov_cnt; i++) { 411 free(iov[i].iov_base); 412 } 413 } 414 415 static void 416 test_spdk_accel_submit_copy_crc32c(void) 417 { 418 const uint64_t nbytes = TEST_SUBMIT_SIZE; 419 uint32_t crc_dst; 420 uint8_t dst[TEST_SUBMIT_SIZE]; 421 uint8_t src[TEST_SUBMIT_SIZE]; 422 uint32_t seed = 0; 423 void *cb_arg = NULL; 424 int rc; 425 struct spdk_accel_task task; 426 struct spdk_accel_task *expected_accel_task = NULL; 427 int flags = 0; 428 429 TAILQ_INIT(&g_accel_ch->task_pool); 430 431 /* Fail with no tasks on _get_task() */ 432 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 433 NULL, cb_arg); 434 CU_ASSERT(rc == -ENOMEM); 435 436 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 437 438 /* accel submission OK. */ 439 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 440 NULL, cb_arg); 441 CU_ASSERT(rc == 0); 442 CU_ASSERT(task.dst == dst); 443 CU_ASSERT(task.src == src); 444 CU_ASSERT(task.crc_dst == &crc_dst); 445 CU_ASSERT(task.s.iovcnt == 0); 446 CU_ASSERT(task.seed == seed); 447 CU_ASSERT(task.nbytes == nbytes); 448 CU_ASSERT(task.flags == 0); 449 CU_ASSERT(task.op_code == ACCEL_OPC_COPY_CRC32C); 450 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 451 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 452 CU_ASSERT(expected_accel_task == &task); 453 } 454 455 static void 456 test_spdk_accel_module_find_by_name(void) 457 { 458 struct spdk_accel_module_if mod1 = {}; 459 struct spdk_accel_module_if mod2 = {}; 460 struct spdk_accel_module_if mod3 = {}; 461 struct spdk_accel_module_if *accel_module = NULL; 462 463 mod1.name = "ioat"; 464 mod2.name = "idxd"; 465 mod3.name = "software"; 466 467 TAILQ_INIT(&spdk_accel_module_list); 468 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod1, tailq); 469 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod2, tailq); 470 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod3, tailq); 471 472 /* Now let's find a valid engine */ 473 accel_module = _module_find_by_name("ioat"); 474 CU_ASSERT(accel_module != NULL); 475 476 /* Try to find one that doesn't exist */ 477 accel_module = _module_find_by_name("XXX"); 478 CU_ASSERT(accel_module == NULL); 479 } 480 481 static void 482 test_spdk_accel_module_register(void) 483 { 484 struct spdk_accel_module_if mod1 = {}; 485 struct spdk_accel_module_if mod2 = {}; 486 struct spdk_accel_module_if mod3 = {}; 487 struct spdk_accel_module_if mod4 = {}; 488 struct spdk_accel_module_if *accel_module = NULL; 489 int i = 0; 490 491 mod1.name = "ioat"; 492 mod2.name = "idxd"; 493 mod3.name = "software"; 494 mod4.name = "nothing"; 495 496 TAILQ_INIT(&spdk_accel_module_list); 497 498 spdk_accel_module_list_add(&mod1); 499 spdk_accel_module_list_add(&mod2); 500 spdk_accel_module_list_add(&mod3); 501 spdk_accel_module_list_add(&mod4); 502 503 /* Now confirm they're in the right order. */ 504 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 505 switch (i++) { 506 case 0: 507 CU_ASSERT(strcmp(accel_module->name, "software") == 0); 508 break; 509 case 1: 510 CU_ASSERT(strcmp(accel_module->name, "ioat") == 0); 511 break; 512 case 2: 513 CU_ASSERT(strcmp(accel_module->name, "idxd") == 0); 514 break; 515 case 3: 516 CU_ASSERT(strcmp(accel_module->name, "nothing") == 0); 517 break; 518 default: 519 CU_ASSERT(false); 520 break; 521 } 522 } 523 CU_ASSERT(i == 4); 524 } 525 526 int 527 main(int argc, char **argv) 528 { 529 CU_pSuite suite = NULL; 530 unsigned int num_failures; 531 532 CU_set_error_action(CUEA_ABORT); 533 CU_initialize_registry(); 534 535 suite = CU_add_suite("accel", test_setup, test_cleanup); 536 537 CU_ADD_TEST(suite, test_spdk_accel_task_complete); 538 CU_ADD_TEST(suite, test_get_task); 539 CU_ADD_TEST(suite, test_spdk_accel_submit_copy); 540 CU_ADD_TEST(suite, test_spdk_accel_submit_dualcast); 541 CU_ADD_TEST(suite, test_spdk_accel_submit_compare); 542 CU_ADD_TEST(suite, test_spdk_accel_submit_fill); 543 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c); 544 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32cv); 545 CU_ADD_TEST(suite, test_spdk_accel_submit_copy_crc32c); 546 CU_ADD_TEST(suite, test_spdk_accel_module_find_by_name); 547 CU_ADD_TEST(suite, test_spdk_accel_module_register); 548 549 CU_basic_set_mode(CU_BRM_VERBOSE); 550 CU_basic_run_tests(); 551 num_failures = CU_get_number_of_failures(); 552 CU_cleanup_registry(); 553 554 return num_failures; 555 } 556