1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2021 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk_cunit.h" 8 #include "spdk_internal/mock.h" 9 #include "spdk_internal/accel_module.h" 10 #include "thread/thread_internal.h" 11 #include "common/lib/ut_multithread.c" 12 #include "accel/accel.c" 13 #include "accel/accel_sw.c" 14 #include "unit/lib/json_mock.c" 15 16 #ifdef SPDK_CONFIG_PMDK 17 DEFINE_STUB(pmem_msync, int, (const void *addr, size_t len), 0); 18 DEFINE_STUB(pmem_memcpy_persist, void *, (void *pmemdest, const void *src, size_t len), NULL); 19 DEFINE_STUB(pmem_is_pmem, int, (const void *addr, size_t len), 0); 20 DEFINE_STUB(pmem_memset_persist, void *, (void *pmemdest, int c, size_t len), NULL); 21 #endif 22 DEFINE_STUB_V(spdk_memory_domain_destroy, (struct spdk_memory_domain *domain)); 23 DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, 24 (struct spdk_memory_domain *domain), "UT_DMA"); 25 26 int 27 spdk_memory_domain_create(struct spdk_memory_domain **domain, enum spdk_dma_device_type type, 28 struct spdk_memory_domain_ctx *ctx, const char *id) 29 { 30 *domain = (void *)0xdeadbeef; 31 32 return 0; 33 } 34 35 struct ut_domain_ctx { 36 struct iovec iov; 37 struct iovec expected; 38 int pull_submit_status; 39 int push_submit_status; 40 int pull_complete_status; 41 int push_complete_status; 42 }; 43 44 static struct spdk_memory_domain *g_ut_domain = (void *)0xa55e1; 45 46 int 47 spdk_memory_domain_pull_data(struct spdk_memory_domain *sd, void *sctx, struct iovec *siov, 48 uint32_t siovcnt, struct iovec *diov, uint32_t diovcnt, 49 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 50 { 51 struct ut_domain_ctx *ctx = sctx; 52 53 CU_ASSERT_EQUAL(sd, g_ut_domain); 54 CU_ASSERT_EQUAL(siovcnt, 1); 55 CU_ASSERT_EQUAL(memcmp(siov, &ctx->expected, sizeof(*siov)), 0); 56 57 if (ctx->pull_submit_status != 0) { 58 return ctx->pull_submit_status; 59 } 60 61 if (ctx->pull_complete_status != 0) { 62 cpl_cb(cpl_arg, ctx->pull_complete_status); 63 return 0; 64 } 65 66 spdk_iovcpy(&ctx->iov, 1, diov, diovcnt); 67 cpl_cb(cpl_arg, 0); 68 69 return 0; 70 } 71 72 int 73 spdk_memory_domain_push_data(struct spdk_memory_domain *dd, void *dctx, struct iovec *diov, 74 uint32_t diovcnt, struct iovec *siov, uint32_t siovcnt, 75 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 76 { 77 struct ut_domain_ctx *ctx = dctx; 78 79 CU_ASSERT_EQUAL(dd, g_ut_domain); 80 CU_ASSERT_EQUAL(diovcnt, 1); 81 CU_ASSERT_EQUAL(memcmp(diov, &ctx->expected, sizeof(*diov)), 0); 82 83 if (ctx->push_submit_status != 0) { 84 return ctx->push_submit_status; 85 } 86 87 if (ctx->push_complete_status != 0) { 88 cpl_cb(cpl_arg, ctx->push_complete_status); 89 return 0; 90 } 91 92 spdk_iovcpy(siov, siovcnt, &ctx->iov, 1); 93 cpl_cb(cpl_arg, 0); 94 95 return 0; 96 } 97 98 /* global vars and setup/cleanup functions used for all test functions */ 99 struct spdk_accel_module_if g_module_if = {}; 100 struct accel_module g_module = { .module = &g_module_if }; 101 struct spdk_io_channel *g_ch = NULL; 102 struct accel_io_channel *g_accel_ch = NULL; 103 struct sw_accel_io_channel *g_sw_ch = NULL; 104 struct spdk_io_channel *g_module_ch = NULL; 105 106 static uint64_t g_opc_mask = 0; 107 108 static uint64_t 109 _accel_op_to_bit(enum accel_opcode opc) 110 { 111 return (1 << opc); 112 } 113 114 static bool 115 _supports_opcode(enum accel_opcode opc) 116 { 117 if (_accel_op_to_bit(opc) & g_opc_mask) { 118 return true; 119 } 120 return false; 121 } 122 123 static int 124 test_setup(void) 125 { 126 int i; 127 128 g_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct accel_io_channel)); 129 if (g_ch == NULL) { 130 /* for some reason the assert fatal macro doesn't work in the setup function. */ 131 CU_ASSERT(false); 132 return -1; 133 } 134 g_accel_ch = (struct accel_io_channel *)((char *)g_ch + sizeof(struct spdk_io_channel)); 135 g_module_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct sw_accel_io_channel)); 136 if (g_module_ch == NULL) { 137 CU_ASSERT(false); 138 return -1; 139 } 140 141 g_module_if.submit_tasks = sw_accel_submit_tasks; 142 g_module_if.name = "software"; 143 for (i = 0; i < ACCEL_OPC_LAST; i++) { 144 g_accel_ch->module_ch[i] = g_module_ch; 145 g_modules_opc[i] = g_module; 146 } 147 g_sw_ch = (struct sw_accel_io_channel *)((char *)g_module_ch + sizeof( 148 struct spdk_io_channel)); 149 TAILQ_INIT(&g_sw_ch->tasks_to_complete); 150 g_module_if.supports_opcode = _supports_opcode; 151 return 0; 152 } 153 154 static int 155 test_cleanup(void) 156 { 157 free(g_ch); 158 free(g_module_ch); 159 160 return 0; 161 } 162 163 #define DUMMY_ARG 0xDEADBEEF 164 static bool g_dummy_cb_called = false; 165 static void 166 dummy_cb_fn(void *cb_arg, int status) 167 { 168 CU_ASSERT(*(uint32_t *)cb_arg == DUMMY_ARG); 169 CU_ASSERT(status == 0); 170 g_dummy_cb_called = true; 171 } 172 173 static void 174 test_spdk_accel_task_complete(void) 175 { 176 struct spdk_accel_task accel_task = {}; 177 struct spdk_accel_task *expected_accel_task = NULL; 178 uint32_t cb_arg = DUMMY_ARG; 179 int status = 0; 180 181 accel_task.accel_ch = g_accel_ch; 182 accel_task.cb_fn = dummy_cb_fn; 183 accel_task.cb_arg = &cb_arg; 184 TAILQ_INIT(&g_accel_ch->task_pool); 185 186 /* Confirm cb is called and task added to list. */ 187 spdk_accel_task_complete(&accel_task, status); 188 CU_ASSERT(g_dummy_cb_called == true); 189 expected_accel_task = TAILQ_FIRST(&g_accel_ch->task_pool); 190 TAILQ_REMOVE(&g_accel_ch->task_pool, expected_accel_task, link); 191 CU_ASSERT(expected_accel_task == &accel_task); 192 } 193 194 static void 195 test_get_task(void) 196 { 197 struct spdk_accel_task *task; 198 struct spdk_accel_task _task; 199 void *cb_arg = NULL; 200 201 TAILQ_INIT(&g_accel_ch->task_pool); 202 203 /* no tasks left, return NULL. */ 204 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 205 CU_ASSERT(task == NULL); 206 207 _task.cb_fn = dummy_cb_fn; 208 _task.cb_arg = cb_arg; 209 _task.accel_ch = g_accel_ch; 210 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &_task, link); 211 212 /* Get a valid task. */ 213 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 214 CU_ASSERT(task == &_task); 215 CU_ASSERT(_task.cb_fn == dummy_cb_fn); 216 CU_ASSERT(_task.cb_arg == cb_arg); 217 CU_ASSERT(_task.accel_ch == g_accel_ch); 218 } 219 220 #define TEST_SUBMIT_SIZE 64 221 static void 222 test_spdk_accel_submit_copy(void) 223 { 224 const uint64_t nbytes = TEST_SUBMIT_SIZE; 225 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 226 uint8_t src[TEST_SUBMIT_SIZE] = {0}; 227 void *cb_arg = NULL; 228 int rc; 229 struct spdk_accel_task task; 230 struct spdk_accel_task *expected_accel_task = NULL; 231 int flags = 0; 232 233 TAILQ_INIT(&g_accel_ch->task_pool); 234 235 /* Fail with no tasks on _get_task() */ 236 rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, flags, NULL, cb_arg); 237 CU_ASSERT(rc == -ENOMEM); 238 239 task.accel_ch = g_accel_ch; 240 task.flags = 1; 241 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 242 243 /* submission OK. */ 244 rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, flags, NULL, cb_arg); 245 CU_ASSERT(rc == 0); 246 CU_ASSERT(task.op_code == ACCEL_OPC_COPY); 247 CU_ASSERT(task.flags == 0); 248 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 249 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 250 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 251 CU_ASSERT(expected_accel_task == &task); 252 } 253 254 static void 255 test_spdk_accel_submit_dualcast(void) 256 { 257 void *dst1; 258 void *dst2; 259 void *src; 260 uint32_t align = ALIGN_4K; 261 uint64_t nbytes = TEST_SUBMIT_SIZE; 262 void *cb_arg = NULL; 263 int rc; 264 struct spdk_accel_task task; 265 struct spdk_accel_task *expected_accel_task = NULL; 266 int flags = 0; 267 268 TAILQ_INIT(&g_accel_ch->task_pool); 269 270 /* Dualcast requires 4K alignment on dst addresses, 271 * hence using the hard coded address to test the buffer alignment 272 */ 273 dst1 = (void *)0x5000; 274 dst2 = (void *)0x60f0; 275 src = calloc(1, TEST_SUBMIT_SIZE); 276 SPDK_CU_ASSERT_FATAL(src != NULL); 277 memset(src, 0x5A, TEST_SUBMIT_SIZE); 278 279 /* This should fail since dst2 is not 4k aligned */ 280 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 281 CU_ASSERT(rc == -EINVAL); 282 283 dst1 = (void *)0x7010; 284 dst2 = (void *)0x6000; 285 /* This should fail since dst1 is not 4k aligned */ 286 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 287 CU_ASSERT(rc == -EINVAL); 288 289 /* Dualcast requires 4K alignment on dst addresses */ 290 dst1 = (void *)0x7000; 291 dst2 = (void *)0x6000; 292 /* Fail with no tasks on _get_task() */ 293 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 294 CU_ASSERT(rc == -ENOMEM); 295 296 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 297 298 /* accel submission OK., since we test the SW path , need to use valid memory addresses 299 * cannot hardcode them anymore */ 300 dst1 = spdk_dma_zmalloc(nbytes, align, NULL); 301 SPDK_CU_ASSERT_FATAL(dst1 != NULL); 302 dst2 = spdk_dma_zmalloc(nbytes, align, NULL); 303 SPDK_CU_ASSERT_FATAL(dst2 != NULL); 304 /* SW module does the dualcast. */ 305 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 306 CU_ASSERT(rc == 0); 307 CU_ASSERT(task.op_code == ACCEL_OPC_DUALCAST); 308 CU_ASSERT(task.flags == 0); 309 CU_ASSERT(memcmp(dst1, src, TEST_SUBMIT_SIZE) == 0); 310 CU_ASSERT(memcmp(dst2, src, TEST_SUBMIT_SIZE) == 0); 311 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 312 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 313 CU_ASSERT(expected_accel_task == &task); 314 315 free(src); 316 spdk_free(dst1); 317 spdk_free(dst2); 318 } 319 320 static void 321 test_spdk_accel_submit_compare(void) 322 { 323 void *src1; 324 void *src2; 325 uint64_t nbytes = TEST_SUBMIT_SIZE; 326 void *cb_arg = NULL; 327 int rc; 328 struct spdk_accel_task task; 329 struct spdk_accel_task *expected_accel_task = NULL; 330 331 TAILQ_INIT(&g_accel_ch->task_pool); 332 333 src1 = calloc(1, TEST_SUBMIT_SIZE); 334 SPDK_CU_ASSERT_FATAL(src1 != NULL); 335 src2 = calloc(1, TEST_SUBMIT_SIZE); 336 SPDK_CU_ASSERT_FATAL(src2 != NULL); 337 338 /* Fail with no tasks on _get_task() */ 339 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 340 CU_ASSERT(rc == -ENOMEM); 341 342 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 343 344 /* accel submission OK. */ 345 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 346 CU_ASSERT(rc == 0); 347 CU_ASSERT(task.op_code == ACCEL_OPC_COMPARE); 348 CU_ASSERT(memcmp(src1, src2, TEST_SUBMIT_SIZE) == 0); 349 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 350 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 351 CU_ASSERT(expected_accel_task == &task); 352 353 free(src1); 354 free(src2); 355 } 356 357 static void 358 test_spdk_accel_submit_fill(void) 359 { 360 void *dst; 361 void *src; 362 uint8_t fill = 0xf; 363 uint64_t fill64; 364 uint64_t nbytes = TEST_SUBMIT_SIZE; 365 void *cb_arg = NULL; 366 int rc; 367 struct spdk_accel_task task; 368 struct spdk_accel_task *expected_accel_task = NULL; 369 int flags = 0; 370 371 TAILQ_INIT(&g_accel_ch->task_pool); 372 373 dst = calloc(1, TEST_SUBMIT_SIZE); 374 SPDK_CU_ASSERT_FATAL(dst != NULL); 375 src = calloc(1, TEST_SUBMIT_SIZE); 376 SPDK_CU_ASSERT_FATAL(src != NULL); 377 memset(src, fill, TEST_SUBMIT_SIZE); 378 memset(&fill64, fill, sizeof(uint64_t)); 379 380 /* Fail with no tasks on _get_task() */ 381 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 382 CU_ASSERT(rc == -ENOMEM); 383 384 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 385 386 /* accel submission OK. */ 387 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 388 CU_ASSERT(rc == 0); 389 CU_ASSERT(task.fill_pattern == fill64); 390 CU_ASSERT(task.op_code == ACCEL_OPC_FILL); 391 CU_ASSERT(task.flags == 0); 392 393 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 394 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 395 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 396 CU_ASSERT(expected_accel_task == &task); 397 398 free(dst); 399 free(src); 400 } 401 402 static void 403 test_spdk_accel_submit_crc32c(void) 404 { 405 const uint64_t nbytes = TEST_SUBMIT_SIZE; 406 uint32_t crc_dst; 407 uint8_t src[TEST_SUBMIT_SIZE]; 408 uint32_t seed = 1; 409 void *cb_arg = NULL; 410 int rc; 411 struct spdk_accel_task task; 412 struct spdk_accel_task *expected_accel_task = NULL; 413 414 TAILQ_INIT(&g_accel_ch->task_pool); 415 416 /* Fail with no tasks on _get_task() */ 417 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 418 CU_ASSERT(rc == -ENOMEM); 419 420 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 421 422 /* accel submission OK. */ 423 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 424 CU_ASSERT(rc == 0); 425 CU_ASSERT(task.crc_dst == &crc_dst); 426 CU_ASSERT(task.seed == seed); 427 CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); 428 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 429 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 430 CU_ASSERT(expected_accel_task == &task); 431 } 432 433 static void 434 test_spdk_accel_submit_crc32cv(void) 435 { 436 uint32_t crc_dst; 437 uint32_t seed = 0; 438 uint32_t iov_cnt = 32; 439 void *cb_arg = NULL; 440 int rc; 441 uint32_t i = 0; 442 struct spdk_accel_task task; 443 struct iovec iov[32]; 444 struct spdk_accel_task *expected_accel_task = NULL; 445 446 TAILQ_INIT(&g_accel_ch->task_pool); 447 448 for (i = 0; i < iov_cnt; i++) { 449 iov[i].iov_base = calloc(1, TEST_SUBMIT_SIZE); 450 SPDK_CU_ASSERT_FATAL(iov[i].iov_base != NULL); 451 iov[i].iov_len = TEST_SUBMIT_SIZE; 452 } 453 454 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 455 456 /* accel submission OK. */ 457 rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, NULL, cb_arg); 458 CU_ASSERT(rc == 0); 459 CU_ASSERT(task.s.iovs == iov); 460 CU_ASSERT(task.s.iovcnt == iov_cnt); 461 CU_ASSERT(task.crc_dst == &crc_dst); 462 CU_ASSERT(task.seed == seed); 463 CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); 464 CU_ASSERT(task.cb_arg == cb_arg); 465 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 466 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 467 CU_ASSERT(expected_accel_task == &task); 468 469 for (i = 0; i < iov_cnt; i++) { 470 free(iov[i].iov_base); 471 } 472 } 473 474 static void 475 test_spdk_accel_submit_copy_crc32c(void) 476 { 477 const uint64_t nbytes = TEST_SUBMIT_SIZE; 478 uint32_t crc_dst; 479 uint8_t dst[TEST_SUBMIT_SIZE]; 480 uint8_t src[TEST_SUBMIT_SIZE]; 481 uint32_t seed = 0; 482 void *cb_arg = NULL; 483 int rc; 484 struct spdk_accel_task task; 485 struct spdk_accel_task *expected_accel_task = NULL; 486 int flags = 0; 487 488 TAILQ_INIT(&g_accel_ch->task_pool); 489 490 /* Fail with no tasks on _get_task() */ 491 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 492 NULL, cb_arg); 493 CU_ASSERT(rc == -ENOMEM); 494 495 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 496 497 /* accel submission OK. */ 498 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 499 NULL, cb_arg); 500 CU_ASSERT(rc == 0); 501 CU_ASSERT(task.crc_dst == &crc_dst); 502 CU_ASSERT(task.seed == seed); 503 CU_ASSERT(task.flags == 0); 504 CU_ASSERT(task.op_code == ACCEL_OPC_COPY_CRC32C); 505 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 506 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 507 CU_ASSERT(expected_accel_task == &task); 508 } 509 510 static void 511 test_spdk_accel_module_find_by_name(void) 512 { 513 struct spdk_accel_module_if mod1 = {}; 514 struct spdk_accel_module_if mod2 = {}; 515 struct spdk_accel_module_if mod3 = {}; 516 struct spdk_accel_module_if *accel_module = NULL; 517 518 mod1.name = "ioat"; 519 mod2.name = "idxd"; 520 mod3.name = "software"; 521 522 TAILQ_INIT(&spdk_accel_module_list); 523 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod1, tailq); 524 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod2, tailq); 525 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod3, tailq); 526 527 /* Now let's find a valid engine */ 528 accel_module = _module_find_by_name("ioat"); 529 CU_ASSERT(accel_module != NULL); 530 531 /* Try to find one that doesn't exist */ 532 accel_module = _module_find_by_name("XXX"); 533 CU_ASSERT(accel_module == NULL); 534 } 535 536 static void 537 test_spdk_accel_module_register(void) 538 { 539 struct spdk_accel_module_if mod1 = {}; 540 struct spdk_accel_module_if mod2 = {}; 541 struct spdk_accel_module_if mod3 = {}; 542 struct spdk_accel_module_if mod4 = {}; 543 struct spdk_accel_module_if *accel_module = NULL; 544 int i = 0; 545 546 mod1.name = "ioat"; 547 mod2.name = "idxd"; 548 mod3.name = "software"; 549 mod4.name = "nothing"; 550 551 TAILQ_INIT(&spdk_accel_module_list); 552 553 spdk_accel_module_list_add(&mod1); 554 spdk_accel_module_list_add(&mod2); 555 spdk_accel_module_list_add(&mod3); 556 spdk_accel_module_list_add(&mod4); 557 558 /* Now confirm they're in the right order. */ 559 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 560 switch (i++) { 561 case 0: 562 CU_ASSERT(strcmp(accel_module->name, "software") == 0); 563 break; 564 case 1: 565 CU_ASSERT(strcmp(accel_module->name, "ioat") == 0); 566 break; 567 case 2: 568 CU_ASSERT(strcmp(accel_module->name, "idxd") == 0); 569 break; 570 case 3: 571 CU_ASSERT(strcmp(accel_module->name, "nothing") == 0); 572 break; 573 default: 574 CU_ASSERT(false); 575 break; 576 } 577 } 578 CU_ASSERT(i == 4); 579 } 580 581 struct ut_sequence { 582 bool complete; 583 int status; 584 }; 585 586 static void 587 ut_sequence_step_cb(void *cb_arg) 588 { 589 int *completed = cb_arg; 590 591 (*completed)++; 592 } 593 594 static void 595 ut_sequence_complete_cb(void *cb_arg, int status) 596 { 597 struct ut_sequence *seq = cb_arg; 598 599 seq->complete = true; 600 seq->status = status; 601 } 602 603 static void 604 test_sequence_fill_copy(void) 605 { 606 struct spdk_accel_sequence *seq = NULL; 607 struct spdk_io_channel *ioch; 608 struct ut_sequence ut_seq; 609 char buf[4096], tmp[2][4096], expected[4096]; 610 struct iovec src_iovs[2], dst_iovs[2]; 611 int rc, completed; 612 613 ioch = spdk_accel_get_io_channel(); 614 SPDK_CU_ASSERT_FATAL(ioch != NULL); 615 616 /* First check the simplest case - single task in a sequence */ 617 memset(buf, 0, sizeof(buf)); 618 memset(expected, 0xa5, sizeof(expected)); 619 completed = 0; 620 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 621 ut_sequence_step_cb, &completed); 622 CU_ASSERT_EQUAL(rc, 0); 623 CU_ASSERT_EQUAL(completed, 0); 624 625 ut_seq.complete = false; 626 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 627 CU_ASSERT_EQUAL(rc, 0); 628 629 poll_threads(); 630 CU_ASSERT_EQUAL(completed, 1); 631 CU_ASSERT(ut_seq.complete); 632 CU_ASSERT_EQUAL(ut_seq.status, 0); 633 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 634 635 /* Check a single copy operation */ 636 memset(buf, 0, sizeof(buf)); 637 memset(tmp[0], 0xa5, sizeof(tmp[0])); 638 memset(expected, 0xa5, sizeof(expected)); 639 completed = 0; 640 seq = NULL; 641 642 dst_iovs[0].iov_base = buf; 643 dst_iovs[0].iov_len = sizeof(buf); 644 src_iovs[0].iov_base = tmp[0]; 645 src_iovs[0].iov_len = sizeof(tmp[0]); 646 647 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 648 &src_iovs[0], 1, NULL, NULL, 0, 649 ut_sequence_step_cb, &completed); 650 CU_ASSERT_EQUAL(rc, 0); 651 652 ut_seq.complete = false; 653 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 654 CU_ASSERT_EQUAL(rc, 0); 655 656 poll_threads(); 657 CU_ASSERT_EQUAL(completed, 1); 658 CU_ASSERT(ut_seq.complete); 659 CU_ASSERT_EQUAL(ut_seq.status, 0); 660 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 661 662 /* Check multiple fill operations */ 663 memset(buf, 0, sizeof(buf)); 664 memset(expected, 0xfe, 4096); 665 memset(expected, 0xde, 2048); 666 memset(expected, 0xa5, 1024); 667 seq = NULL; 668 completed = 0; 669 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, NULL, NULL, 0xfe, 0, 670 ut_sequence_step_cb, &completed); 671 CU_ASSERT_EQUAL(rc, 0); 672 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xde, 0, 673 ut_sequence_step_cb, &completed); 674 CU_ASSERT_EQUAL(rc, 0); 675 rc = spdk_accel_append_fill(&seq, ioch, buf, 1024, NULL, NULL, 0xa5, 0, 676 ut_sequence_step_cb, &completed); 677 CU_ASSERT_EQUAL(rc, 0); 678 679 ut_seq.complete = false; 680 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 681 CU_ASSERT_EQUAL(rc, 0); 682 683 poll_threads(); 684 CU_ASSERT_EQUAL(completed, 3); 685 CU_ASSERT(ut_seq.complete); 686 CU_ASSERT_EQUAL(ut_seq.status, 0); 687 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 688 689 /* Check multiple copy operations */ 690 memset(buf, 0, sizeof(buf)); 691 memset(tmp[0], 0, sizeof(tmp[0])); 692 memset(tmp[1], 0, sizeof(tmp[1])); 693 memset(expected, 0xa5, sizeof(expected)); 694 seq = NULL; 695 completed = 0; 696 697 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 698 ut_sequence_step_cb, &completed); 699 CU_ASSERT_EQUAL(rc, 0); 700 701 dst_iovs[0].iov_base = tmp[1]; 702 dst_iovs[0].iov_len = sizeof(tmp[1]); 703 src_iovs[0].iov_base = tmp[0]; 704 src_iovs[0].iov_len = sizeof(tmp[0]); 705 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 706 &src_iovs[0], 1, NULL, NULL, 0, 707 ut_sequence_step_cb, &completed); 708 CU_ASSERT_EQUAL(rc, 0); 709 710 dst_iovs[1].iov_base = buf; 711 dst_iovs[1].iov_len = sizeof(buf); 712 src_iovs[1].iov_base = tmp[1]; 713 src_iovs[1].iov_len = sizeof(tmp[1]); 714 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 715 &src_iovs[1], 1, NULL, NULL, 0, 716 ut_sequence_step_cb, &completed); 717 CU_ASSERT_EQUAL(rc, 0); 718 719 ut_seq.complete = false; 720 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 721 CU_ASSERT_EQUAL(rc, 0); 722 723 poll_threads(); 724 CU_ASSERT_EQUAL(completed, 3); 725 CU_ASSERT(ut_seq.complete); 726 CU_ASSERT_EQUAL(ut_seq.status, 0); 727 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 728 729 /* Check that adding a copy operation at the end will change destination buffer */ 730 memset(buf, 0, sizeof(buf)); 731 memset(tmp[0], 0, sizeof(tmp[0])); 732 memset(expected, 0xa5, sizeof(buf)); 733 seq = NULL; 734 completed = 0; 735 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 736 ut_sequence_step_cb, &completed); 737 CU_ASSERT_EQUAL(rc, 0); 738 739 dst_iovs[0].iov_base = buf; 740 dst_iovs[0].iov_len = sizeof(buf); 741 src_iovs[0].iov_base = tmp[0]; 742 src_iovs[0].iov_len = sizeof(tmp[0]); 743 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 744 &src_iovs[0], 1, NULL, NULL, 0, 745 ut_sequence_step_cb, &completed); 746 CU_ASSERT_EQUAL(rc, 0); 747 748 ut_seq.complete = false; 749 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 750 CU_ASSERT_EQUAL(rc, 0); 751 752 poll_threads(); 753 CU_ASSERT_EQUAL(completed, 2); 754 CU_ASSERT(ut_seq.complete); 755 CU_ASSERT_EQUAL(ut_seq.status, 0); 756 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 757 758 /* Check that it's also possible to add copy operation at the beginning */ 759 memset(buf, 0, sizeof(buf)); 760 memset(tmp[0], 0xde, sizeof(tmp[0])); 761 memset(tmp[1], 0, sizeof(tmp[1])); 762 memset(expected, 0xa5, sizeof(expected)); 763 seq = NULL; 764 completed = 0; 765 766 dst_iovs[0].iov_base = tmp[1]; 767 dst_iovs[0].iov_len = sizeof(tmp[1]); 768 src_iovs[0].iov_base = tmp[0]; 769 src_iovs[0].iov_len = sizeof(tmp[0]); 770 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 771 &src_iovs[0], 1, NULL, NULL, 0, 772 ut_sequence_step_cb, &completed); 773 CU_ASSERT_EQUAL(rc, 0); 774 775 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], sizeof(tmp[1]), NULL, NULL, 0xa5, 0, 776 ut_sequence_step_cb, &completed); 777 CU_ASSERT_EQUAL(rc, 0); 778 779 dst_iovs[1].iov_base = buf; 780 dst_iovs[1].iov_len = sizeof(buf); 781 src_iovs[1].iov_base = tmp[1]; 782 src_iovs[1].iov_len = sizeof(tmp[1]); 783 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 784 &src_iovs[1], 1, NULL, NULL, 0, 785 ut_sequence_step_cb, &completed); 786 CU_ASSERT_EQUAL(rc, 0); 787 788 ut_seq.complete = false; 789 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 790 CU_ASSERT_EQUAL(rc, 0); 791 792 poll_threads(); 793 CU_ASSERT_EQUAL(completed, 3); 794 CU_ASSERT(ut_seq.complete); 795 CU_ASSERT_EQUAL(ut_seq.status, 0); 796 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 797 798 spdk_put_io_channel(ioch); 799 poll_threads(); 800 } 801 802 static void 803 test_sequence_abort(void) 804 { 805 struct spdk_accel_sequence *seq = NULL; 806 struct spdk_io_channel *ioch; 807 char buf[4096], tmp[2][4096], expected[4096]; 808 struct iovec src_iovs[2], dst_iovs[2]; 809 int rc, completed; 810 811 ioch = spdk_accel_get_io_channel(); 812 SPDK_CU_ASSERT_FATAL(ioch != NULL); 813 814 /* Check that aborting a sequence calls operation's callback, the operation is not executed 815 * and the sequence is freed 816 */ 817 memset(buf, 0, sizeof(buf)); 818 memset(expected, 0, sizeof(buf)); 819 completed = 0; 820 seq = NULL; 821 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 822 ut_sequence_step_cb, &completed); 823 CU_ASSERT_EQUAL(rc, 0); 824 825 spdk_accel_sequence_abort(seq); 826 CU_ASSERT_EQUAL(completed, 1); 827 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 828 829 /* Check sequence with multiple operations */ 830 memset(buf, 0, sizeof(buf)); 831 memset(expected, 0, sizeof(buf)); 832 completed = 0; 833 seq = NULL; 834 835 dst_iovs[0].iov_base = tmp[1]; 836 dst_iovs[0].iov_len = sizeof(tmp[1]); 837 src_iovs[0].iov_base = tmp[0]; 838 src_iovs[0].iov_len = sizeof(tmp[0]); 839 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 840 &src_iovs[0], 1, NULL, NULL, 0, 841 ut_sequence_step_cb, &completed); 842 CU_ASSERT_EQUAL(rc, 0); 843 844 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 4096, NULL, NULL, 0xa5, 0, 845 ut_sequence_step_cb, &completed); 846 CU_ASSERT_EQUAL(rc, 0); 847 848 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xde, 0, 849 ut_sequence_step_cb, &completed); 850 CU_ASSERT_EQUAL(rc, 0); 851 852 dst_iovs[1].iov_base = buf; 853 dst_iovs[1].iov_len = sizeof(buf); 854 src_iovs[1].iov_base = tmp[1]; 855 src_iovs[1].iov_len = sizeof(tmp[1]); 856 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 857 &src_iovs[1], 1, NULL, NULL, 0, 858 ut_sequence_step_cb, &completed); 859 CU_ASSERT_EQUAL(rc, 0); 860 861 spdk_accel_sequence_abort(seq); 862 CU_ASSERT_EQUAL(completed, 4); 863 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 864 865 /* This should be a no-op */ 866 spdk_accel_sequence_abort(NULL); 867 868 spdk_put_io_channel(ioch); 869 poll_threads(); 870 } 871 872 static void 873 test_sequence_append_error(void) 874 { 875 struct spdk_accel_sequence *seq = NULL; 876 struct spdk_io_channel *ioch; 877 struct accel_io_channel *accel_ch; 878 struct iovec src_iovs, dst_iovs; 879 char buf[4096]; 880 TAILQ_HEAD(, spdk_accel_task) tasks = TAILQ_HEAD_INITIALIZER(tasks); 881 TAILQ_HEAD(, spdk_accel_sequence) seqs = TAILQ_HEAD_INITIALIZER(seqs); 882 int rc; 883 884 ioch = spdk_accel_get_io_channel(); 885 SPDK_CU_ASSERT_FATAL(ioch != NULL); 886 accel_ch = spdk_io_channel_get_ctx(ioch); 887 888 /* Check that append fails and no sequence object is allocated when there are no more free 889 * tasks */ 890 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 891 892 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 893 ut_sequence_step_cb, NULL); 894 CU_ASSERT_EQUAL(rc, -ENOMEM); 895 CU_ASSERT_PTR_NULL(seq); 896 897 dst_iovs.iov_base = buf; 898 dst_iovs.iov_len = 2048; 899 src_iovs.iov_base = &buf[2048]; 900 src_iovs.iov_len = 2048; 901 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 902 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 903 CU_ASSERT_EQUAL(rc, -ENOMEM); 904 CU_ASSERT_PTR_NULL(seq); 905 906 dst_iovs.iov_base = buf; 907 dst_iovs.iov_len = 2048; 908 src_iovs.iov_base = &buf[2048]; 909 src_iovs.iov_len = 2048; 910 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 911 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 912 CU_ASSERT_EQUAL(rc, -ENOMEM); 913 CU_ASSERT_PTR_NULL(seq); 914 915 /* Check that the same happens when the sequence queue is empty */ 916 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 917 TAILQ_SWAP(&seqs, &accel_ch->seq_pool, spdk_accel_sequence, link); 918 919 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 920 ut_sequence_step_cb, NULL); 921 CU_ASSERT_EQUAL(rc, -ENOMEM); 922 CU_ASSERT_PTR_NULL(seq); 923 924 dst_iovs.iov_base = buf; 925 dst_iovs.iov_len = 2048; 926 src_iovs.iov_base = &buf[2048]; 927 src_iovs.iov_len = 2048; 928 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 929 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 930 CU_ASSERT_EQUAL(rc, -ENOMEM); 931 CU_ASSERT_PTR_NULL(seq); 932 933 dst_iovs.iov_base = buf; 934 dst_iovs.iov_len = 2048; 935 src_iovs.iov_base = &buf[2048]; 936 src_iovs.iov_len = 2048; 937 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 938 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 939 CU_ASSERT_EQUAL(rc, -ENOMEM); 940 CU_ASSERT_PTR_NULL(seq); 941 942 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 943 944 spdk_put_io_channel(ioch); 945 poll_threads(); 946 } 947 948 struct ut_sequence_operation { 949 int complete_status; 950 int submit_status; 951 int count; 952 struct iovec *src_iovs; 953 uint32_t src_iovcnt; 954 struct iovec *dst_iovs; 955 uint32_t dst_iovcnt; 956 int (*submit)(struct spdk_io_channel *ch, struct spdk_accel_task *t); 957 }; 958 959 static struct ut_sequence_operation g_seq_operations[ACCEL_OPC_LAST]; 960 961 static int 962 ut_sequnce_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *task) 963 { 964 struct ut_sequence_operation *op = &g_seq_operations[task->op_code]; 965 966 if (op->submit != NULL) { 967 return op->submit(ch, task); 968 } 969 if (op->src_iovs != NULL) { 970 CU_ASSERT_EQUAL(task->s.iovcnt, op->src_iovcnt); 971 CU_ASSERT_EQUAL(memcmp(task->s.iovs, op->src_iovs, 972 sizeof(struct iovec) * op->src_iovcnt), 0); 973 } 974 if (op->dst_iovs != NULL) { 975 CU_ASSERT_EQUAL(task->d.iovcnt, op->dst_iovcnt); 976 CU_ASSERT_EQUAL(memcmp(task->d.iovs, op->dst_iovs, 977 sizeof(struct iovec) * op->dst_iovcnt), 0); 978 } 979 980 op->count++; 981 if (op->submit_status != 0) { 982 return op->submit_status; 983 } 984 985 spdk_accel_task_complete(task, op->complete_status); 986 987 return 0; 988 } 989 990 static void 991 test_sequence_completion_error(void) 992 { 993 struct spdk_accel_sequence *seq = NULL; 994 struct spdk_io_channel *ioch; 995 struct ut_sequence ut_seq; 996 struct iovec src_iovs, dst_iovs; 997 char buf[4096], tmp[4096]; 998 struct accel_module modules[ACCEL_OPC_LAST]; 999 int i, rc, completed; 1000 1001 ioch = spdk_accel_get_io_channel(); 1002 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1003 1004 /* Override the submit_tasks function */ 1005 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1006 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1007 modules[i] = g_modules_opc[i]; 1008 g_modules_opc[i] = g_module; 1009 } 1010 1011 memset(buf, 0, sizeof(buf)); 1012 memset(tmp, 0, sizeof(tmp)); 1013 1014 /* Check that if the first operation completes with an error, the whole sequence is 1015 * completed with that error and that all operations' completion callbacks are executed 1016 */ 1017 g_seq_operations[ACCEL_OPC_FILL].complete_status = -E2BIG; 1018 completed = 0; 1019 seq = NULL; 1020 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1021 ut_sequence_step_cb, &completed); 1022 CU_ASSERT_EQUAL(rc, 0); 1023 1024 dst_iovs.iov_base = buf; 1025 dst_iovs.iov_len = sizeof(buf); 1026 src_iovs.iov_base = tmp; 1027 src_iovs.iov_len = sizeof(tmp); 1028 1029 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1030 &src_iovs, 1, NULL, NULL, 0, 1031 ut_sequence_step_cb, &completed); 1032 CU_ASSERT_EQUAL(rc, 0); 1033 1034 ut_seq.complete = false; 1035 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1036 CU_ASSERT_EQUAL(rc, 0); 1037 1038 poll_threads(); 1039 CU_ASSERT_EQUAL(completed, 2); 1040 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 1041 1042 /* Check the same with a second operation in the sequence */ 1043 g_seq_operations[ACCEL_OPC_DECOMPRESS].complete_status = -EACCES; 1044 g_seq_operations[ACCEL_OPC_FILL].complete_status = 0; 1045 completed = 0; 1046 seq = NULL; 1047 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1048 ut_sequence_step_cb, &completed); 1049 CU_ASSERT_EQUAL(rc, 0); 1050 1051 dst_iovs.iov_base = buf; 1052 dst_iovs.iov_len = sizeof(buf); 1053 src_iovs.iov_base = tmp; 1054 src_iovs.iov_len = sizeof(tmp); 1055 1056 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1057 &src_iovs, 1, NULL, NULL, 0, 1058 ut_sequence_step_cb, &completed); 1059 CU_ASSERT_EQUAL(rc, 0); 1060 1061 ut_seq.complete = false; 1062 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1063 CU_ASSERT_EQUAL(rc, 0); 1064 1065 poll_threads(); 1066 CU_ASSERT_EQUAL(completed, 2); 1067 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 1068 1069 g_seq_operations[ACCEL_OPC_DECOMPRESS].complete_status = 0; 1070 g_seq_operations[ACCEL_OPC_FILL].complete_status = 0; 1071 1072 /* Check submission failure of the first operation */ 1073 g_seq_operations[ACCEL_OPC_FILL].submit_status = -EADDRINUSE; 1074 completed = 0; 1075 seq = NULL; 1076 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1077 ut_sequence_step_cb, &completed); 1078 CU_ASSERT_EQUAL(rc, 0); 1079 1080 dst_iovs.iov_base = buf; 1081 dst_iovs.iov_len = sizeof(buf); 1082 src_iovs.iov_base = tmp; 1083 src_iovs.iov_len = sizeof(tmp); 1084 1085 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1086 &src_iovs, 1, NULL, NULL, 0, 1087 ut_sequence_step_cb, &completed); 1088 CU_ASSERT_EQUAL(rc, 0); 1089 1090 ut_seq.complete = false; 1091 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1092 CU_ASSERT_EQUAL(rc, 0); 1093 1094 poll_threads(); 1095 CU_ASSERT_EQUAL(completed, 2); 1096 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 1097 1098 /* Check the same with a second operation */ 1099 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit_status = -EADDRNOTAVAIL; 1100 g_seq_operations[ACCEL_OPC_FILL].submit_status = 0; 1101 completed = 0; 1102 seq = NULL; 1103 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1104 ut_sequence_step_cb, &completed); 1105 CU_ASSERT_EQUAL(rc, 0); 1106 1107 dst_iovs.iov_base = buf; 1108 dst_iovs.iov_len = sizeof(buf); 1109 src_iovs.iov_base = tmp; 1110 src_iovs.iov_len = sizeof(tmp); 1111 1112 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1113 &src_iovs, 1, NULL, NULL, 0, 1114 ut_sequence_step_cb, &completed); 1115 CU_ASSERT_EQUAL(rc, 0); 1116 1117 ut_seq.complete = false; 1118 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1119 CU_ASSERT_EQUAL(rc, 0); 1120 1121 poll_threads(); 1122 CU_ASSERT_EQUAL(completed, 2); 1123 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 1124 1125 /* Cleanup module pointers to make subsequent tests work correctly */ 1126 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1127 g_modules_opc[i] = modules[i]; 1128 } 1129 1130 spdk_put_io_channel(ioch); 1131 poll_threads(); 1132 } 1133 1134 #ifdef SPDK_CONFIG_ISAL 1135 static void 1136 ut_compress_cb(void *cb_arg, int status) 1137 { 1138 int *completed = cb_arg; 1139 1140 CU_ASSERT_EQUAL(status, 0); 1141 1142 *completed = 1; 1143 } 1144 1145 static void 1146 test_sequence_decompress(void) 1147 { 1148 struct spdk_accel_sequence *seq = NULL; 1149 struct spdk_io_channel *ioch; 1150 struct ut_sequence ut_seq; 1151 char buf[4096], tmp[2][4096], expected[4096]; 1152 struct iovec src_iovs[2], dst_iovs[2]; 1153 uint32_t compressed_size; 1154 int rc, completed = 0; 1155 1156 ioch = spdk_accel_get_io_channel(); 1157 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1158 1159 memset(expected, 0xa5, sizeof(expected)); 1160 src_iovs[0].iov_base = expected; 1161 src_iovs[0].iov_len = sizeof(expected); 1162 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1163 &compressed_size, 0, ut_compress_cb, &completed); 1164 CU_ASSERT_EQUAL(rc, 0); 1165 1166 while (!completed) { 1167 poll_threads(); 1168 } 1169 1170 /* Check a single decompress operation in a sequence */ 1171 seq = NULL; 1172 completed = 0; 1173 1174 dst_iovs[0].iov_base = buf; 1175 dst_iovs[0].iov_len = sizeof(buf); 1176 src_iovs[0].iov_base = tmp[0]; 1177 src_iovs[0].iov_len = compressed_size; 1178 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1179 &src_iovs[0], 1, NULL, NULL, 0, 1180 ut_sequence_step_cb, &completed); 1181 CU_ASSERT_EQUAL(rc, 0); 1182 1183 ut_seq.complete = false; 1184 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1185 CU_ASSERT_EQUAL(rc, 0); 1186 1187 poll_threads(); 1188 1189 CU_ASSERT_EQUAL(completed, 1); 1190 CU_ASSERT(ut_seq.complete); 1191 CU_ASSERT_EQUAL(ut_seq.status, 0); 1192 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1193 1194 /* Put the decompress operation in the middle of a sequence with a copy operation at the 1195 * beginning and a fill at the end modifying the first 2048B of the buffer. 1196 */ 1197 memset(expected, 0xfe, 2048); 1198 memset(buf, 0, sizeof(buf)); 1199 seq = NULL; 1200 completed = 0; 1201 1202 dst_iovs[0].iov_base = tmp[1]; 1203 dst_iovs[0].iov_len = compressed_size; 1204 src_iovs[0].iov_base = tmp[0]; 1205 src_iovs[0].iov_len = compressed_size; 1206 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1207 &src_iovs[0], 1, NULL, NULL, 0, 1208 ut_sequence_step_cb, &completed); 1209 CU_ASSERT_EQUAL(rc, 0); 1210 1211 dst_iovs[1].iov_base = buf; 1212 dst_iovs[1].iov_len = sizeof(buf); 1213 src_iovs[1].iov_base = tmp[1]; 1214 src_iovs[1].iov_len = compressed_size; 1215 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1216 &src_iovs[1], 1, NULL, NULL, 0, 1217 ut_sequence_step_cb, &completed); 1218 CU_ASSERT_EQUAL(rc, 0); 1219 1220 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1221 ut_sequence_step_cb, &completed); 1222 CU_ASSERT_EQUAL(rc, 0); 1223 1224 ut_seq.complete = false; 1225 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1226 CU_ASSERT_EQUAL(rc, 0); 1227 1228 poll_threads(); 1229 1230 CU_ASSERT_EQUAL(completed, 3); 1231 CU_ASSERT(ut_seq.complete); 1232 CU_ASSERT_EQUAL(ut_seq.status, 0); 1233 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1234 1235 /* Check sequence with decompress at the beginning: decompress -> copy */ 1236 memset(expected, 0xa5, sizeof(expected)); 1237 memset(buf, 0, sizeof(buf)); 1238 seq = NULL; 1239 completed = 0; 1240 1241 dst_iovs[0].iov_base = tmp[1]; 1242 dst_iovs[0].iov_len = sizeof(tmp[1]); 1243 src_iovs[0].iov_base = tmp[0]; 1244 src_iovs[0].iov_len = compressed_size; 1245 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1246 &src_iovs[0], 1, NULL, NULL, 0, 1247 ut_sequence_step_cb, &completed); 1248 CU_ASSERT_EQUAL(rc, 0); 1249 1250 dst_iovs[1].iov_base = buf; 1251 dst_iovs[1].iov_len = sizeof(buf); 1252 src_iovs[1].iov_base = tmp[1]; 1253 src_iovs[1].iov_len = sizeof(tmp[1]); 1254 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1255 &src_iovs[1], 1, NULL, NULL, 0, 1256 ut_sequence_step_cb, &completed); 1257 CU_ASSERT_EQUAL(rc, 0); 1258 1259 ut_seq.complete = false; 1260 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1261 CU_ASSERT_EQUAL(rc, 0); 1262 1263 poll_threads(); 1264 1265 CU_ASSERT_EQUAL(completed, 2); 1266 CU_ASSERT(ut_seq.complete); 1267 CU_ASSERT_EQUAL(ut_seq.status, 0); 1268 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1269 1270 spdk_put_io_channel(ioch); 1271 poll_threads(); 1272 } 1273 1274 static void 1275 test_sequence_reverse(void) 1276 { 1277 struct spdk_accel_sequence *seq = NULL; 1278 struct spdk_io_channel *ioch; 1279 struct ut_sequence ut_seq; 1280 char buf[4096], tmp[2][4096], expected[4096]; 1281 struct iovec src_iovs[2], dst_iovs[2]; 1282 uint32_t compressed_size; 1283 int rc, completed = 0; 1284 1285 ioch = spdk_accel_get_io_channel(); 1286 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1287 1288 memset(expected, 0xa5, sizeof(expected)); 1289 src_iovs[0].iov_base = expected; 1290 src_iovs[0].iov_len = sizeof(expected); 1291 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1292 &compressed_size, 0, ut_compress_cb, &completed); 1293 CU_ASSERT_EQUAL(rc, 0); 1294 1295 while (!completed) { 1296 poll_threads(); 1297 } 1298 1299 /* First check that reversing a sequnce with a single operation is a no-op */ 1300 memset(buf, 0, sizeof(buf)); 1301 seq = NULL; 1302 completed = 0; 1303 1304 dst_iovs[0].iov_base = buf; 1305 dst_iovs[0].iov_len = sizeof(buf); 1306 src_iovs[0].iov_base = tmp[0]; 1307 src_iovs[0].iov_len = compressed_size; 1308 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1309 &src_iovs[0], 1, NULL, NULL, 0, 1310 ut_sequence_step_cb, &completed); 1311 CU_ASSERT_EQUAL(rc, 0); 1312 1313 spdk_accel_sequence_reverse(seq); 1314 1315 ut_seq.complete = false; 1316 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1317 CU_ASSERT_EQUAL(rc, 0); 1318 1319 poll_threads(); 1320 1321 CU_ASSERT_EQUAL(completed, 1); 1322 CU_ASSERT(ut_seq.complete); 1323 CU_ASSERT_EQUAL(ut_seq.status, 0); 1324 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1325 1326 /* Add a copy operation at the end with src set to the compressed data. After reverse(), 1327 * that copy operation should be first, so decompress() should receive compressed data in 1328 * its src buffer. 1329 */ 1330 memset(buf, 0, sizeof(buf)); 1331 memset(tmp[1], 0, sizeof(tmp[1])); 1332 seq = NULL; 1333 completed = 0; 1334 1335 dst_iovs[0].iov_base = buf; 1336 dst_iovs[0].iov_len = sizeof(buf); 1337 src_iovs[0].iov_base = tmp[1]; 1338 src_iovs[0].iov_len = compressed_size; 1339 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1340 &src_iovs[0], 1, NULL, NULL, 0, 1341 ut_sequence_step_cb, &completed); 1342 CU_ASSERT_EQUAL(rc, 0); 1343 1344 dst_iovs[1].iov_base = tmp[1]; 1345 dst_iovs[1].iov_len = compressed_size; 1346 src_iovs[1].iov_base = tmp[0]; 1347 src_iovs[1].iov_len = compressed_size; 1348 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1349 &src_iovs[1], 1, NULL, NULL, 0, 1350 ut_sequence_step_cb, &completed); 1351 CU_ASSERT_EQUAL(rc, 0); 1352 1353 spdk_accel_sequence_reverse(seq); 1354 1355 ut_seq.complete = false; 1356 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1357 CU_ASSERT_EQUAL(rc, 0); 1358 1359 poll_threads(); 1360 1361 CU_ASSERT_EQUAL(completed, 2); 1362 CU_ASSERT(ut_seq.complete); 1363 CU_ASSERT_EQUAL(ut_seq.status, 0); 1364 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1365 1366 /* Check the same, but add an extra fill operation at the beginning that should execute last 1367 * after reverse(). 1368 */ 1369 memset(buf, 0, sizeof(buf)); 1370 memset(tmp[1], 0, sizeof(tmp[1])); 1371 memset(expected, 0xfe, 2048); 1372 seq = NULL; 1373 completed = 0; 1374 1375 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1376 ut_sequence_step_cb, &completed); 1377 CU_ASSERT_EQUAL(rc, 0); 1378 1379 dst_iovs[0].iov_base = buf; 1380 dst_iovs[0].iov_len = sizeof(buf); 1381 src_iovs[0].iov_base = tmp[1]; 1382 src_iovs[0].iov_len = compressed_size; 1383 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1384 &src_iovs[0], 1, NULL, NULL, 0, 1385 ut_sequence_step_cb, &completed); 1386 CU_ASSERT_EQUAL(rc, 0); 1387 1388 dst_iovs[1].iov_base = tmp[1]; 1389 dst_iovs[1].iov_len = compressed_size; 1390 src_iovs[1].iov_base = tmp[0]; 1391 src_iovs[1].iov_len = compressed_size; 1392 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1393 &src_iovs[1], 1, NULL, NULL, 0, 1394 ut_sequence_step_cb, &completed); 1395 CU_ASSERT_EQUAL(rc, 0); 1396 1397 spdk_accel_sequence_reverse(seq); 1398 1399 ut_seq.complete = false; 1400 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1401 CU_ASSERT_EQUAL(rc, 0); 1402 1403 poll_threads(); 1404 1405 CU_ASSERT_EQUAL(completed, 3); 1406 CU_ASSERT(ut_seq.complete); 1407 CU_ASSERT_EQUAL(ut_seq.status, 0); 1408 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1409 1410 /* Build the sequence in order and then reverse it twice */ 1411 memset(buf, 0, sizeof(buf)); 1412 memset(tmp[1], 0, sizeof(tmp[1])); 1413 seq = NULL; 1414 completed = 0; 1415 1416 dst_iovs[0].iov_base = tmp[1]; 1417 dst_iovs[0].iov_len = compressed_size; 1418 src_iovs[0].iov_base = tmp[0]; 1419 src_iovs[0].iov_len = compressed_size; 1420 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1421 &src_iovs[0], 1, NULL, NULL, 0, 1422 ut_sequence_step_cb, &completed); 1423 CU_ASSERT_EQUAL(rc, 0); 1424 1425 dst_iovs[1].iov_base = buf; 1426 dst_iovs[1].iov_len = sizeof(buf); 1427 src_iovs[1].iov_base = tmp[1]; 1428 src_iovs[1].iov_len = compressed_size; 1429 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1430 &src_iovs[1], 1, NULL, NULL, 0, 1431 ut_sequence_step_cb, &completed); 1432 CU_ASSERT_EQUAL(rc, 0); 1433 1434 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1435 ut_sequence_step_cb, &completed); 1436 CU_ASSERT_EQUAL(rc, 0); 1437 1438 spdk_accel_sequence_reverse(seq); 1439 spdk_accel_sequence_reverse(seq); 1440 1441 ut_seq.complete = false; 1442 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1443 CU_ASSERT_EQUAL(rc, 0); 1444 1445 poll_threads(); 1446 1447 CU_ASSERT_EQUAL(completed, 3); 1448 CU_ASSERT(ut_seq.complete); 1449 CU_ASSERT_EQUAL(ut_seq.status, 0); 1450 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1451 1452 spdk_put_io_channel(ioch); 1453 poll_threads(); 1454 } 1455 #endif 1456 1457 static void 1458 test_sequence_copy_elision(void) 1459 { 1460 struct spdk_accel_sequence *seq = NULL; 1461 struct spdk_io_channel *ioch; 1462 struct ut_sequence ut_seq; 1463 struct iovec src_iovs[4], dst_iovs[4], exp_iovs[2]; 1464 char buf[4096], tmp[4][4096]; 1465 struct accel_module modules[ACCEL_OPC_LAST]; 1466 struct spdk_accel_crypto_key key = {}; 1467 int i, rc, completed; 1468 1469 ioch = spdk_accel_get_io_channel(); 1470 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1471 1472 /* Override the submit_tasks function */ 1473 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1474 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1475 g_seq_operations[i].complete_status = 0; 1476 g_seq_operations[i].submit_status = 0; 1477 g_seq_operations[i].count = 0; 1478 1479 modules[i] = g_modules_opc[i]; 1480 g_modules_opc[i] = g_module; 1481 } 1482 1483 /* Check that a copy operation at the beginning is removed */ 1484 seq = NULL; 1485 completed = 0; 1486 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovcnt = 1; 1487 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovcnt = 1; 1488 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1489 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1490 exp_iovs[0].iov_base = tmp[0]; 1491 exp_iovs[0].iov_len = sizeof(tmp[0]); 1492 exp_iovs[1].iov_base = buf; 1493 exp_iovs[1].iov_len = 2048; 1494 1495 dst_iovs[0].iov_base = tmp[1]; 1496 dst_iovs[0].iov_len = sizeof(tmp[1]); 1497 src_iovs[0].iov_base = tmp[0]; 1498 src_iovs[0].iov_len = sizeof(tmp[0]); 1499 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1500 &src_iovs[0], 1, NULL, NULL, 0, 1501 ut_sequence_step_cb, &completed); 1502 CU_ASSERT_EQUAL(rc, 0); 1503 1504 dst_iovs[1].iov_base = buf; 1505 dst_iovs[1].iov_len = 2048; 1506 src_iovs[1].iov_base = tmp[1]; 1507 src_iovs[1].iov_len = sizeof(tmp[1]); 1508 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1509 &src_iovs[1], 1, NULL, NULL, 0, 1510 ut_sequence_step_cb, &completed); 1511 CU_ASSERT_EQUAL(rc, 0); 1512 1513 ut_seq.complete = false; 1514 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1515 CU_ASSERT_EQUAL(rc, 0); 1516 1517 poll_threads(); 1518 1519 CU_ASSERT_EQUAL(completed, 2); 1520 CU_ASSERT(ut_seq.complete); 1521 CU_ASSERT_EQUAL(ut_seq.status, 0); 1522 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1523 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 1524 1525 /* Check that a copy operation at the end is removed too */ 1526 seq = NULL; 1527 completed = 0; 1528 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1529 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 1530 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1531 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1532 exp_iovs[0].iov_base = tmp[0]; 1533 exp_iovs[0].iov_len = sizeof(tmp[0]); 1534 exp_iovs[1].iov_base = buf; 1535 exp_iovs[1].iov_len = 2048; 1536 1537 dst_iovs[0].iov_base = tmp[1]; 1538 dst_iovs[0].iov_len = 2048; 1539 src_iovs[0].iov_base = tmp[0]; 1540 src_iovs[0].iov_len = sizeof(tmp[0]); 1541 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1542 &src_iovs[0], 1, NULL, NULL, 0, 1543 ut_sequence_step_cb, &completed); 1544 CU_ASSERT_EQUAL(rc, 0); 1545 1546 dst_iovs[1].iov_base = buf; 1547 dst_iovs[1].iov_len = 2048; 1548 src_iovs[1].iov_base = tmp[1]; 1549 src_iovs[1].iov_len = 2048; 1550 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1551 &src_iovs[1], 1, NULL, NULL, 0, 1552 ut_sequence_step_cb, &completed); 1553 CU_ASSERT_EQUAL(rc, 0); 1554 1555 ut_seq.complete = false; 1556 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1557 CU_ASSERT_EQUAL(rc, 0); 1558 1559 poll_threads(); 1560 1561 CU_ASSERT_EQUAL(completed, 2); 1562 CU_ASSERT(ut_seq.complete); 1563 CU_ASSERT_EQUAL(ut_seq.status, 0); 1564 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1565 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 1566 1567 /* Check a copy operation both at the beginning and the end */ 1568 seq = NULL; 1569 completed = 0; 1570 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1571 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 1572 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1573 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1574 exp_iovs[0].iov_base = tmp[0]; 1575 exp_iovs[0].iov_len = sizeof(tmp[0]); 1576 exp_iovs[1].iov_base = buf; 1577 exp_iovs[1].iov_len = 2048; 1578 1579 dst_iovs[0].iov_base = tmp[1]; 1580 dst_iovs[0].iov_len = sizeof(tmp[1]); 1581 src_iovs[0].iov_base = tmp[0]; 1582 src_iovs[0].iov_len = sizeof(tmp[0]); 1583 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1584 &src_iovs[0], 1, NULL, NULL, 0, 1585 ut_sequence_step_cb, &completed); 1586 CU_ASSERT_EQUAL(rc, 0); 1587 1588 dst_iovs[1].iov_base = tmp[2]; 1589 dst_iovs[1].iov_len = 2048; 1590 src_iovs[1].iov_base = tmp[1]; 1591 src_iovs[1].iov_len = sizeof(tmp[1]); 1592 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1593 &src_iovs[1], 1, NULL, NULL, 0, 1594 ut_sequence_step_cb, &completed); 1595 CU_ASSERT_EQUAL(rc, 0); 1596 1597 dst_iovs[2].iov_base = buf; 1598 dst_iovs[2].iov_len = 2048; 1599 src_iovs[2].iov_base = tmp[2]; 1600 src_iovs[2].iov_len = 2048; 1601 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1602 &src_iovs[2], 1, NULL, NULL, 0, 1603 ut_sequence_step_cb, &completed); 1604 CU_ASSERT_EQUAL(rc, 0); 1605 1606 ut_seq.complete = false; 1607 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1608 CU_ASSERT_EQUAL(rc, 0); 1609 1610 poll_threads(); 1611 1612 CU_ASSERT_EQUAL(completed, 3); 1613 CU_ASSERT(ut_seq.complete); 1614 CU_ASSERT_EQUAL(ut_seq.status, 0); 1615 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1616 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 1617 1618 /* Check decompress + copy + decompress + copy */ 1619 seq = NULL; 1620 completed = 0; 1621 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1622 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 1623 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = NULL; 1624 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = NULL; 1625 1626 dst_iovs[0].iov_base = tmp[1]; 1627 dst_iovs[0].iov_len = sizeof(tmp[1]); 1628 src_iovs[0].iov_base = tmp[0]; 1629 src_iovs[0].iov_len = sizeof(tmp[0]); 1630 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1631 &src_iovs[0], 1, NULL, NULL, 0, 1632 ut_sequence_step_cb, &completed); 1633 CU_ASSERT_EQUAL(rc, 0); 1634 1635 dst_iovs[1].iov_base = tmp[2]; 1636 dst_iovs[1].iov_len = 2048; 1637 src_iovs[1].iov_base = tmp[1]; 1638 src_iovs[1].iov_len = sizeof(tmp[1]); 1639 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1640 &src_iovs[1], 1, NULL, NULL, 0, 1641 ut_sequence_step_cb, &completed); 1642 CU_ASSERT_EQUAL(rc, 0); 1643 1644 dst_iovs[2].iov_base = tmp[3]; 1645 dst_iovs[2].iov_len = 1024; 1646 src_iovs[2].iov_base = tmp[2]; 1647 src_iovs[2].iov_len = 2048; 1648 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1649 &src_iovs[2], 1, NULL, NULL, 0, 1650 ut_sequence_step_cb, &completed); 1651 CU_ASSERT_EQUAL(rc, 0); 1652 1653 dst_iovs[3].iov_base = buf; 1654 dst_iovs[3].iov_len = 1024; 1655 src_iovs[3].iov_base = tmp[3]; 1656 src_iovs[3].iov_len = 1024; 1657 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 1658 &src_iovs[3], 1, NULL, NULL, 0, 1659 ut_sequence_step_cb, &completed); 1660 CU_ASSERT_EQUAL(rc, 0); 1661 1662 ut_seq.complete = false; 1663 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1664 CU_ASSERT_EQUAL(rc, 0); 1665 1666 poll_threads(); 1667 1668 CU_ASSERT_EQUAL(completed, 4); 1669 CU_ASSERT(ut_seq.complete); 1670 CU_ASSERT_EQUAL(ut_seq.status, 0); 1671 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1672 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 2); 1673 1674 /* Check two copy operations - one of them should be removed, while the other should be 1675 * executed normally */ 1676 seq = NULL; 1677 completed = 0; 1678 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1679 g_seq_operations[ACCEL_OPC_COPY].dst_iovcnt = 1; 1680 g_seq_operations[ACCEL_OPC_COPY].src_iovcnt = 1; 1681 g_seq_operations[ACCEL_OPC_COPY].src_iovs = &exp_iovs[0]; 1682 g_seq_operations[ACCEL_OPC_COPY].dst_iovs = &exp_iovs[1]; 1683 exp_iovs[0].iov_base = tmp[0]; 1684 exp_iovs[0].iov_len = sizeof(tmp[0]); 1685 exp_iovs[1].iov_base = buf; 1686 exp_iovs[1].iov_len = sizeof(buf); 1687 1688 dst_iovs[0].iov_base = tmp[1]; 1689 dst_iovs[0].iov_len = sizeof(tmp[1]); 1690 src_iovs[0].iov_base = tmp[0]; 1691 src_iovs[0].iov_len = sizeof(tmp[0]); 1692 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1693 &src_iovs[0], 1, NULL, NULL, 0, 1694 ut_sequence_step_cb, &completed); 1695 CU_ASSERT_EQUAL(rc, 0); 1696 1697 dst_iovs[1].iov_base = buf; 1698 dst_iovs[1].iov_len = sizeof(buf); 1699 src_iovs[1].iov_base = tmp[1]; 1700 src_iovs[1].iov_len = sizeof(tmp[1]); 1701 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1702 &src_iovs[1], 1, NULL, NULL, 0, 1703 ut_sequence_step_cb, &completed); 1704 CU_ASSERT_EQUAL(rc, 0); 1705 1706 ut_seq.complete = false; 1707 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1708 CU_ASSERT_EQUAL(rc, 0); 1709 1710 poll_threads(); 1711 1712 CU_ASSERT_EQUAL(completed, 2); 1713 CU_ASSERT(ut_seq.complete); 1714 CU_ASSERT_EQUAL(ut_seq.status, 0); 1715 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 1); 1716 1717 /* Check fill + copy */ 1718 seq = NULL; 1719 completed = 0; 1720 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1721 g_seq_operations[ACCEL_OPC_FILL].count = 0; 1722 g_seq_operations[ACCEL_OPC_COPY].src_iovs = NULL; 1723 g_seq_operations[ACCEL_OPC_COPY].dst_iovs = NULL; 1724 g_seq_operations[ACCEL_OPC_FILL].dst_iovcnt = 1; 1725 g_seq_operations[ACCEL_OPC_FILL].dst_iovs = &exp_iovs[0]; 1726 exp_iovs[0].iov_base = buf; 1727 exp_iovs[0].iov_len = sizeof(buf); 1728 1729 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 1730 ut_sequence_step_cb, &completed); 1731 CU_ASSERT_EQUAL(rc, 0); 1732 1733 dst_iovs[0].iov_base = buf; 1734 dst_iovs[0].iov_len = sizeof(buf); 1735 src_iovs[0].iov_base = tmp[0]; 1736 src_iovs[0].iov_len = sizeof(tmp[0]); 1737 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1738 &src_iovs[0], 1, NULL, NULL, 0, 1739 ut_sequence_step_cb, &completed); 1740 CU_ASSERT_EQUAL(rc, 0); 1741 1742 ut_seq.complete = false; 1743 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1744 CU_ASSERT_EQUAL(rc, 0); 1745 1746 poll_threads(); 1747 1748 CU_ASSERT_EQUAL(completed, 2); 1749 CU_ASSERT(ut_seq.complete); 1750 CU_ASSERT_EQUAL(ut_seq.status, 0); 1751 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1752 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 1); 1753 1754 /* Check copy + encrypt + copy */ 1755 seq = NULL; 1756 completed = 0; 1757 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1758 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 1759 g_seq_operations[ACCEL_OPC_ENCRYPT].dst_iovcnt = 1; 1760 g_seq_operations[ACCEL_OPC_ENCRYPT].src_iovcnt = 1; 1761 g_seq_operations[ACCEL_OPC_ENCRYPT].src_iovs = &exp_iovs[0]; 1762 g_seq_operations[ACCEL_OPC_ENCRYPT].dst_iovs = &exp_iovs[1]; 1763 exp_iovs[0].iov_base = tmp[0]; 1764 exp_iovs[0].iov_len = sizeof(tmp[0]); 1765 exp_iovs[1].iov_base = buf; 1766 exp_iovs[1].iov_len = sizeof(buf); 1767 1768 dst_iovs[0].iov_base = tmp[1]; 1769 dst_iovs[0].iov_len = sizeof(tmp[1]); 1770 src_iovs[0].iov_base = tmp[0]; 1771 src_iovs[0].iov_len = sizeof(tmp[0]); 1772 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1773 &src_iovs[0], 1, NULL, NULL, 0, 1774 ut_sequence_step_cb, &completed); 1775 CU_ASSERT_EQUAL(rc, 0); 1776 1777 dst_iovs[1].iov_base = tmp[2]; 1778 dst_iovs[1].iov_len = sizeof(tmp[2]); 1779 src_iovs[1].iov_base = tmp[1]; 1780 src_iovs[1].iov_len = sizeof(tmp[1]); 1781 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1782 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1783 ut_sequence_step_cb, &completed); 1784 CU_ASSERT_EQUAL(rc, 0); 1785 1786 dst_iovs[2].iov_base = buf; 1787 dst_iovs[2].iov_len = sizeof(buf); 1788 src_iovs[2].iov_base = tmp[2]; 1789 src_iovs[2].iov_len = sizeof(tmp[2]); 1790 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1791 &src_iovs[2], 1, NULL, NULL, 0, 1792 ut_sequence_step_cb, &completed); 1793 CU_ASSERT_EQUAL(rc, 0); 1794 1795 ut_seq.complete = false; 1796 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1797 CU_ASSERT_EQUAL(rc, 0); 1798 1799 poll_threads(); 1800 1801 CU_ASSERT_EQUAL(completed, 3); 1802 CU_ASSERT(ut_seq.complete); 1803 CU_ASSERT_EQUAL(ut_seq.status, 0); 1804 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1805 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 1806 1807 /* Check copy + decrypt + copy */ 1808 seq = NULL; 1809 completed = 0; 1810 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1811 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 1812 g_seq_operations[ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1813 g_seq_operations[ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1814 g_seq_operations[ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1815 g_seq_operations[ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1816 exp_iovs[0].iov_base = tmp[0]; 1817 exp_iovs[0].iov_len = sizeof(tmp[0]); 1818 exp_iovs[1].iov_base = buf; 1819 exp_iovs[1].iov_len = sizeof(buf); 1820 1821 dst_iovs[0].iov_base = tmp[1]; 1822 dst_iovs[0].iov_len = sizeof(tmp[1]); 1823 src_iovs[0].iov_base = tmp[0]; 1824 src_iovs[0].iov_len = sizeof(tmp[0]); 1825 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1826 &src_iovs[0], 1, NULL, NULL, 0, 1827 ut_sequence_step_cb, &completed); 1828 CU_ASSERT_EQUAL(rc, 0); 1829 1830 dst_iovs[1].iov_base = tmp[2]; 1831 dst_iovs[1].iov_len = sizeof(tmp[2]); 1832 src_iovs[1].iov_base = tmp[1]; 1833 src_iovs[1].iov_len = sizeof(tmp[1]); 1834 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1835 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1836 ut_sequence_step_cb, &completed); 1837 CU_ASSERT_EQUAL(rc, 0); 1838 1839 dst_iovs[2].iov_base = buf; 1840 dst_iovs[2].iov_len = sizeof(buf); 1841 src_iovs[2].iov_base = tmp[2]; 1842 src_iovs[2].iov_len = sizeof(tmp[2]); 1843 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1844 &src_iovs[2], 1, NULL, NULL, 0, 1845 ut_sequence_step_cb, &completed); 1846 CU_ASSERT_EQUAL(rc, 0); 1847 1848 ut_seq.complete = false; 1849 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1850 CU_ASSERT_EQUAL(rc, 0); 1851 1852 poll_threads(); 1853 1854 CU_ASSERT_EQUAL(completed, 3); 1855 CU_ASSERT(ut_seq.complete); 1856 CU_ASSERT_EQUAL(ut_seq.status, 0); 1857 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1858 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 1859 1860 /* Cleanup module pointers to make subsequent tests work correctly */ 1861 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1862 g_modules_opc[i] = modules[i]; 1863 } 1864 1865 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = NULL; 1866 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = NULL; 1867 g_seq_operations[ACCEL_OPC_ENCRYPT].src_iovs = NULL; 1868 g_seq_operations[ACCEL_OPC_ENCRYPT].src_iovs = NULL; 1869 g_seq_operations[ACCEL_OPC_DECRYPT].dst_iovs = NULL; 1870 g_seq_operations[ACCEL_OPC_DECRYPT].dst_iovs = NULL; 1871 1872 spdk_put_io_channel(ioch); 1873 poll_threads(); 1874 } 1875 1876 static int 1877 ut_submit_decompress(struct spdk_io_channel *ch, struct spdk_accel_task *task) 1878 { 1879 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 1880 1881 spdk_accel_task_complete(task, 0); 1882 1883 return 0; 1884 } 1885 1886 static void 1887 test_sequence_accel_buffers(void) 1888 { 1889 struct spdk_accel_sequence *seq = NULL; 1890 struct spdk_io_channel *ioch; 1891 struct accel_io_channel *accel_ch; 1892 struct ut_sequence ut_seq; 1893 struct iovec src_iovs[3], dst_iovs[3]; 1894 char srcbuf[4096], dstbuf[4096], expected[4096]; 1895 struct accel_module modules[ACCEL_OPC_LAST]; 1896 void *buf[2], *domain_ctx[2], *iobuf_buf; 1897 struct spdk_memory_domain *domain[2]; 1898 struct spdk_iobuf_buffer *cache_entry; 1899 spdk_iobuf_buffer_stailq_t small_cache; 1900 uint32_t small_cache_count; 1901 int i, rc, completed; 1902 1903 ioch = spdk_accel_get_io_channel(); 1904 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1905 1906 /* Override the submit_tasks function */ 1907 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1908 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1909 modules[i] = g_modules_opc[i]; 1910 g_modules_opc[i] = g_module; 1911 } 1912 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 1913 * decompress operations together in one sequence. 1914 */ 1915 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 1916 g_seq_operations[ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 1917 g_seq_operations[ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 1918 1919 /* Check the simplest case: one operation using accel buffer as destination + copy operation 1920 * specifying the actual destination buffer 1921 */ 1922 memset(srcbuf, 0xa5, 4096); 1923 memset(dstbuf, 0x0, 4096); 1924 memset(expected, 0xa5, 4096); 1925 completed = 0; 1926 seq = NULL; 1927 1928 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 1929 CU_ASSERT_EQUAL(rc, 0); 1930 CU_ASSERT_PTR_NOT_NULL(buf[0]); 1931 1932 src_iovs[0].iov_base = srcbuf; 1933 src_iovs[0].iov_len = 4096; 1934 dst_iovs[0].iov_base = buf[0]; 1935 dst_iovs[0].iov_len = 4096; 1936 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 1937 &src_iovs[0], 1, NULL, NULL, 0, 1938 ut_sequence_step_cb, &completed); 1939 CU_ASSERT_EQUAL(rc, 0); 1940 1941 src_iovs[1].iov_base = buf[0]; 1942 src_iovs[1].iov_len = 4096; 1943 dst_iovs[1].iov_base = dstbuf; 1944 dst_iovs[1].iov_len = 4096; 1945 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1946 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 1947 ut_sequence_step_cb, &completed); 1948 CU_ASSERT_EQUAL(rc, 0); 1949 1950 ut_seq.complete = false; 1951 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1952 CU_ASSERT_EQUAL(rc, 0); 1953 1954 poll_threads(); 1955 1956 CU_ASSERT_EQUAL(completed, 2); 1957 CU_ASSERT(ut_seq.complete); 1958 CU_ASSERT_EQUAL(ut_seq.status, 0); 1959 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 1960 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 1961 1962 /* Start with a fill operation using accel buffer, followed by a decompress using another 1963 * accel buffer as dst, followed by a copy operation specifying dst buffer of the whole 1964 * sequence 1965 */ 1966 memset(srcbuf, 0x0, 4096); 1967 memset(dstbuf, 0x0, 4096); 1968 memset(expected, 0x5a, 4096); 1969 completed = 0; 1970 seq = NULL; 1971 1972 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 1973 CU_ASSERT_EQUAL(rc, 0); 1974 CU_ASSERT_PTR_NOT_NULL(buf[0]); 1975 1976 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 1977 ut_sequence_step_cb, &completed); 1978 CU_ASSERT_EQUAL(rc, 0); 1979 1980 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 1981 CU_ASSERT_EQUAL(rc, 0); 1982 CU_ASSERT_PTR_NOT_NULL(buf[1]); 1983 1984 src_iovs[0].iov_base = buf[0]; 1985 src_iovs[0].iov_len = 4096; 1986 dst_iovs[0].iov_base = buf[1]; 1987 dst_iovs[0].iov_len = 4096; 1988 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 1989 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 1990 ut_sequence_step_cb, &completed); 1991 CU_ASSERT_EQUAL(rc, 0); 1992 1993 src_iovs[1].iov_base = buf[1]; 1994 src_iovs[1].iov_len = 4096; 1995 dst_iovs[1].iov_base = dstbuf; 1996 dst_iovs[1].iov_len = 4096; 1997 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1998 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 1999 ut_sequence_step_cb, &completed); 2000 2001 ut_seq.complete = false; 2002 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2003 CU_ASSERT_EQUAL(rc, 0); 2004 2005 poll_threads(); 2006 2007 CU_ASSERT_EQUAL(completed, 3); 2008 CU_ASSERT(ut_seq.complete); 2009 CU_ASSERT_EQUAL(ut_seq.status, 0); 2010 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2011 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2012 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2013 2014 /* Check the same, but with two decompress operations with the first one being in-place */ 2015 memset(srcbuf, 0x0, 4096); 2016 memset(dstbuf, 0x0, 4096); 2017 memset(expected, 0x5a, 4096); 2018 completed = 0; 2019 seq = NULL; 2020 2021 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2022 CU_ASSERT_EQUAL(rc, 0); 2023 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2024 2025 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2026 ut_sequence_step_cb, &completed); 2027 CU_ASSERT_EQUAL(rc, 0); 2028 2029 src_iovs[0].iov_base = buf[0]; 2030 src_iovs[0].iov_len = 4096; 2031 dst_iovs[0].iov_base = buf[0]; 2032 dst_iovs[0].iov_len = 4096; 2033 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2034 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2035 ut_sequence_step_cb, &completed); 2036 CU_ASSERT_EQUAL(rc, 0); 2037 2038 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2039 CU_ASSERT_EQUAL(rc, 0); 2040 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2041 2042 src_iovs[1].iov_base = buf[0]; 2043 src_iovs[1].iov_len = 4096; 2044 dst_iovs[1].iov_base = buf[1]; 2045 dst_iovs[1].iov_len = 4096; 2046 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, domain[1], domain_ctx[1], 2047 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 2048 ut_sequence_step_cb, &completed); 2049 CU_ASSERT_EQUAL(rc, 0); 2050 2051 src_iovs[2].iov_base = buf[1]; 2052 src_iovs[2].iov_len = 4096; 2053 dst_iovs[2].iov_base = dstbuf; 2054 dst_iovs[2].iov_len = 4096; 2055 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 2056 &src_iovs[2], 1, domain[1], domain_ctx[1], 0, 2057 ut_sequence_step_cb, &completed); 2058 2059 ut_seq.complete = false; 2060 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2061 CU_ASSERT_EQUAL(rc, 0); 2062 2063 poll_threads(); 2064 2065 CU_ASSERT_EQUAL(completed, 4); 2066 CU_ASSERT(ut_seq.complete); 2067 CU_ASSERT_EQUAL(ut_seq.status, 0); 2068 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2069 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2070 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2071 2072 /* Check that specifying offsets within accel buffers works correctly */ 2073 memset(srcbuf, 0x0, 4096); 2074 memset(dstbuf, 0x0, 4096); 2075 completed = 0; 2076 seq = NULL; 2077 2078 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2079 CU_ASSERT_EQUAL(rc, 0); 2080 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2081 2082 /* Fill in each 1K of the buffer with different pattern */ 2083 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2084 ut_sequence_step_cb, &completed); 2085 CU_ASSERT_EQUAL(rc, 0); 2086 2087 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2088 0x5a, 0, ut_sequence_step_cb, &completed); 2089 CU_ASSERT_EQUAL(rc, 0); 2090 2091 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2092 0xfe, 0, ut_sequence_step_cb, &completed); 2093 CU_ASSERT_EQUAL(rc, 0); 2094 2095 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2096 0xed, 0, ut_sequence_step_cb, &completed); 2097 CU_ASSERT_EQUAL(rc, 0); 2098 2099 src_iovs[0].iov_base = buf[0]; 2100 src_iovs[0].iov_len = 4096; 2101 dst_iovs[0].iov_base = dstbuf; 2102 dst_iovs[0].iov_len = 4096; 2103 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2104 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2105 ut_sequence_step_cb, &completed); 2106 2107 ut_seq.complete = false; 2108 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2109 CU_ASSERT_EQUAL(rc, 0); 2110 2111 poll_threads(); 2112 2113 memset(&expected[0], 0xa5, 1024); 2114 memset(&expected[1024], 0x5a, 1024); 2115 memset(&expected[2048], 0xfe, 1024); 2116 memset(&expected[3072], 0xed, 1024); 2117 CU_ASSERT_EQUAL(completed, 5); 2118 CU_ASSERT(ut_seq.complete); 2119 CU_ASSERT_EQUAL(ut_seq.status, 0); 2120 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2121 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2122 2123 /* Check the same but this time with a decompress operation on part of the buffer (512B 2124 * offset) */ 2125 memset(srcbuf, 0x0, 4096); 2126 memset(dstbuf, 0x0, 4096); 2127 completed = 0; 2128 seq = NULL; 2129 2130 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2131 CU_ASSERT_EQUAL(rc, 0); 2132 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2133 2134 /* Fill in each 1K of the buffer with different pattern */ 2135 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2136 ut_sequence_step_cb, &completed); 2137 CU_ASSERT_EQUAL(rc, 0); 2138 2139 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2140 0x5a, 0, ut_sequence_step_cb, &completed); 2141 CU_ASSERT_EQUAL(rc, 0); 2142 2143 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2144 0xfe, 0, ut_sequence_step_cb, &completed); 2145 CU_ASSERT_EQUAL(rc, 0); 2146 2147 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2148 0xed, 0, ut_sequence_step_cb, &completed); 2149 CU_ASSERT_EQUAL(rc, 0); 2150 2151 rc = spdk_accel_get_buf(ioch, 3072, &buf[1], &domain[1], &domain_ctx[1]); 2152 CU_ASSERT_EQUAL(rc, 0); 2153 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2154 2155 src_iovs[0].iov_base = (char *)buf[0] + 512; 2156 src_iovs[0].iov_len = 3072; 2157 dst_iovs[0].iov_base = (char *)buf[1] + 256; 2158 dst_iovs[0].iov_len = 3072; 2159 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2160 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2161 ut_sequence_step_cb, &completed); 2162 2163 src_iovs[1].iov_base = (char *)buf[1] + 256; 2164 src_iovs[1].iov_len = 3072; 2165 dst_iovs[1].iov_base = dstbuf; 2166 dst_iovs[1].iov_len = 3072; 2167 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2168 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2169 ut_sequence_step_cb, &completed); 2170 2171 ut_seq.complete = false; 2172 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2173 CU_ASSERT_EQUAL(rc, 0); 2174 2175 poll_threads(); 2176 2177 memset(&expected[0], 0xa5, 512); 2178 memset(&expected[512], 0x5a, 1024); 2179 memset(&expected[1536], 0xfe, 1024); 2180 memset(&expected[2560], 0xed, 512); 2181 CU_ASSERT_EQUAL(completed, 6); 2182 CU_ASSERT(ut_seq.complete); 2183 CU_ASSERT_EQUAL(ut_seq.status, 0); 2184 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 3072), 0); 2185 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2186 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2187 2188 /* Check that if iobuf pool is empty, the sequence processing will wait until a buffer is 2189 * available 2190 */ 2191 accel_ch = spdk_io_channel_get_ctx(ioch); 2192 small_cache_count = accel_ch->iobuf.small.cache_count; 2193 STAILQ_INIT(&small_cache); 2194 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2195 accel_ch->iobuf.small.cache_count = 0; 2196 MOCK_SET(spdk_mempool_get, NULL); 2197 2198 /* First allocate a single buffer used by two operations */ 2199 memset(srcbuf, 0x0, 4096); 2200 memset(dstbuf, 0x0, 4096); 2201 memset(expected, 0xa5, 4096); 2202 completed = 0; 2203 seq = NULL; 2204 2205 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2206 CU_ASSERT_EQUAL(rc, 0); 2207 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2208 2209 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0xa5, 0, 2210 ut_sequence_step_cb, &completed); 2211 CU_ASSERT_EQUAL(rc, 0); 2212 2213 src_iovs[0].iov_base = buf[0]; 2214 src_iovs[0].iov_len = 4096; 2215 dst_iovs[0].iov_base = dstbuf; 2216 dst_iovs[0].iov_len = 4096; 2217 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2218 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2219 ut_sequence_step_cb, &completed); 2220 2221 ut_seq.complete = false; 2222 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2223 CU_ASSERT_EQUAL(rc, 0); 2224 2225 poll_threads(); 2226 2227 CU_ASSERT_EQUAL(completed, 0); 2228 CU_ASSERT(!ut_seq.complete); 2229 2230 /* Get a buffer and return it to the pool to trigger the sequence to finish */ 2231 MOCK_CLEAR(spdk_mempool_get); 2232 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2233 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2234 MOCK_SET(spdk_mempool_get, NULL); 2235 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2236 2237 poll_threads(); 2238 2239 CU_ASSERT_EQUAL(completed, 2); 2240 CU_ASSERT(ut_seq.complete); 2241 CU_ASSERT_EQUAL(ut_seq.status, 0); 2242 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2243 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2244 2245 /* Return the buffers back to the cache */ 2246 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2247 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2248 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2249 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2250 small_cache_count++; 2251 } 2252 accel_ch->iobuf.small.cache_count = 0; 2253 2254 /* Check a bit more complex scenario, with two buffers in the sequence */ 2255 memset(srcbuf, 0x0, 4096); 2256 memset(dstbuf, 0x0, 4096); 2257 memset(expected, 0x5a, 4096); 2258 completed = 0; 2259 seq = NULL; 2260 2261 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2262 CU_ASSERT_EQUAL(rc, 0); 2263 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2264 2265 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2266 ut_sequence_step_cb, &completed); 2267 CU_ASSERT_EQUAL(rc, 0); 2268 2269 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2270 CU_ASSERT_EQUAL(rc, 0); 2271 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2272 2273 src_iovs[0].iov_base = buf[0]; 2274 src_iovs[0].iov_len = 4096; 2275 dst_iovs[0].iov_base = buf[1]; 2276 dst_iovs[0].iov_len = 4096; 2277 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2278 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2279 ut_sequence_step_cb, &completed); 2280 CU_ASSERT_EQUAL(rc, 0); 2281 2282 src_iovs[1].iov_base = buf[1]; 2283 src_iovs[1].iov_len = 4096; 2284 dst_iovs[1].iov_base = dstbuf; 2285 dst_iovs[1].iov_len = 4096; 2286 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2287 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2288 ut_sequence_step_cb, &completed); 2289 CU_ASSERT_EQUAL(rc, 0); 2290 2291 ut_seq.complete = false; 2292 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2293 CU_ASSERT_EQUAL(rc, 0); 2294 2295 poll_threads(); 2296 2297 CU_ASSERT_EQUAL(completed, 0); 2298 CU_ASSERT(!ut_seq.complete); 2299 2300 MOCK_CLEAR(spdk_mempool_get); 2301 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2302 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2303 MOCK_SET(spdk_mempool_get, NULL); 2304 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2305 2306 /* One buffer is not enough to finish the whole sequence */ 2307 poll_threads(); 2308 CU_ASSERT(!ut_seq.complete); 2309 2310 MOCK_CLEAR(spdk_mempool_get); 2311 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2312 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2313 MOCK_SET(spdk_mempool_get, NULL); 2314 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2315 2316 poll_threads(); 2317 2318 CU_ASSERT_EQUAL(completed, 3); 2319 CU_ASSERT(ut_seq.complete); 2320 CU_ASSERT_EQUAL(ut_seq.status, 0); 2321 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2322 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2323 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2324 2325 /* Return the buffers back to the cache */ 2326 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2327 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2328 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2329 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2330 small_cache_count++; 2331 } 2332 accel_ch->iobuf.small.cache_count = 0; 2333 2334 MOCK_CLEAR(spdk_mempool_get); 2335 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2336 accel_ch->iobuf.small.cache_count = small_cache_count; 2337 2338 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = NULL; 2339 g_seq_operations[ACCEL_OPC_COPY].submit = NULL; 2340 g_seq_operations[ACCEL_OPC_FILL].submit = NULL; 2341 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2342 g_modules_opc[i] = modules[i]; 2343 } 2344 2345 spdk_put_io_channel(ioch); 2346 poll_threads(); 2347 } 2348 2349 static void 2350 ut_domain_ctx_init(struct ut_domain_ctx *ctx, void *base, size_t len, struct iovec *expected) 2351 { 2352 ctx->iov.iov_base = base; 2353 ctx->iov.iov_len = len; 2354 ctx->expected = *expected; 2355 ctx->pull_submit_status = 0; 2356 ctx->push_submit_status = 0; 2357 ctx->pull_complete_status = 0; 2358 ctx->push_complete_status = 0; 2359 } 2360 2361 static void 2362 test_sequence_memory_domain(void) 2363 { 2364 struct spdk_accel_sequence *seq = NULL; 2365 struct spdk_io_channel *ioch; 2366 struct accel_io_channel *accel_ch; 2367 struct ut_sequence ut_seq; 2368 struct ut_domain_ctx domctx[4]; 2369 struct spdk_iobuf_buffer *cache_entry; 2370 struct accel_module modules[ACCEL_OPC_LAST]; 2371 struct spdk_memory_domain *accel_domain; 2372 spdk_iobuf_buffer_stailq_t small_cache; 2373 char srcbuf[4096], dstbuf[4096], expected[4096], tmp[4096]; 2374 struct iovec src_iovs[2], dst_iovs[2]; 2375 uint32_t small_cache_count; 2376 void *accel_buf, *accel_domain_ctx, *iobuf_buf; 2377 int i, rc, completed; 2378 2379 ioch = spdk_accel_get_io_channel(); 2380 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2381 2382 /* Override the submit_tasks function */ 2383 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2384 g_module.supports_memory_domains = false; 2385 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2386 modules[i] = g_modules_opc[i]; 2387 g_modules_opc[i] = g_module; 2388 } 2389 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2390 * decompress operations together in one sequence. 2391 */ 2392 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2393 g_seq_operations[ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2394 g_seq_operations[ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2395 2396 /* First check the simplest case - single fill operation with dstbuf in domain */ 2397 memset(expected, 0xa5, sizeof(expected)); 2398 memset(dstbuf, 0x0, sizeof(dstbuf)); 2399 completed = 0; 2400 seq = NULL; 2401 2402 /* Use some garbage pointer as dst and use domain ctx to get the actual dstbuf */ 2403 dst_iovs[0].iov_base = (void *)0xcafebabe; 2404 dst_iovs[0].iov_len = sizeof(dstbuf); 2405 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2406 2407 rc = spdk_accel_append_fill(&seq, ioch, dst_iovs[0].iov_base, dst_iovs[0].iov_len, 2408 g_ut_domain, &domctx[0], 0xa5, 0, 2409 ut_sequence_step_cb, &completed); 2410 CU_ASSERT_EQUAL(rc, 0); 2411 2412 ut_seq.complete = false; 2413 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2414 CU_ASSERT_EQUAL(rc, 0); 2415 2416 poll_threads(); 2417 CU_ASSERT_EQUAL(completed, 1); 2418 CU_ASSERT(ut_seq.complete); 2419 CU_ASSERT_EQUAL(ut_seq.status, 0); 2420 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2421 2422 /* Check operation with both srcbuf and dstbuf in remote memory domain */ 2423 memset(expected, 0x5a, sizeof(expected)); 2424 memset(dstbuf, 0x0, sizeof(dstbuf)); 2425 memset(srcbuf, 0x5a, sizeof(dstbuf)); 2426 completed = 0; 2427 seq = NULL; 2428 2429 src_iovs[0].iov_base = (void *)0xcafebabe; 2430 src_iovs[0].iov_len = sizeof(srcbuf); 2431 dst_iovs[0].iov_base = (void *)0xbeefdead; 2432 dst_iovs[0].iov_len = sizeof(dstbuf); 2433 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2434 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2435 2436 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2437 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2438 ut_sequence_step_cb, &completed); 2439 CU_ASSERT_EQUAL(rc, 0); 2440 2441 ut_seq.complete = false; 2442 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2443 CU_ASSERT_EQUAL(rc, 0); 2444 2445 poll_threads(); 2446 CU_ASSERT_EQUAL(completed, 1); 2447 CU_ASSERT(ut_seq.complete); 2448 CU_ASSERT_EQUAL(ut_seq.status, 0); 2449 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2450 2451 /* Two operations using a buffer in remote domain */ 2452 memset(expected, 0xa5, sizeof(expected)); 2453 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2454 memset(tmp, 0x0, sizeof(tmp)); 2455 memset(dstbuf, 0x0, sizeof(dstbuf)); 2456 completed = 0; 2457 seq = NULL; 2458 2459 src_iovs[0].iov_base = (void *)0xcafebabe; 2460 src_iovs[0].iov_len = sizeof(srcbuf); 2461 dst_iovs[0].iov_base = (void *)0xbeefdead; 2462 dst_iovs[0].iov_len = sizeof(tmp); 2463 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2464 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2465 2466 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2467 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2468 ut_sequence_step_cb, &completed); 2469 CU_ASSERT_EQUAL(rc, 0); 2470 2471 src_iovs[1].iov_base = (void *)0xbeefdead; 2472 src_iovs[1].iov_len = sizeof(tmp); 2473 dst_iovs[1].iov_base = (void *)0xa5a5a5a5; 2474 dst_iovs[1].iov_len = sizeof(dstbuf); 2475 ut_domain_ctx_init(&domctx[2], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2476 ut_domain_ctx_init(&domctx[3], tmp, sizeof(tmp), &src_iovs[1]); 2477 2478 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[2], 2479 &src_iovs[1], 1, g_ut_domain, &domctx[3], 0, 2480 ut_sequence_step_cb, &completed); 2481 CU_ASSERT_EQUAL(rc, 0); 2482 2483 ut_seq.complete = false; 2484 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2485 CU_ASSERT_EQUAL(rc, 0); 2486 2487 poll_threads(); 2488 CU_ASSERT_EQUAL(completed, 2); 2489 CU_ASSERT(ut_seq.complete); 2490 CU_ASSERT_EQUAL(ut_seq.status, 0); 2491 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2492 2493 /* Use buffer in remote memory domain and accel buffer at the same time */ 2494 memset(expected, 0x5a, sizeof(expected)); 2495 memset(srcbuf, 0x5a, sizeof(expected)); 2496 memset(dstbuf, 0x0, sizeof(dstbuf)); 2497 completed = 0; 2498 seq = NULL; 2499 2500 rc = spdk_accel_get_buf(ioch, sizeof(dstbuf), &accel_buf, &accel_domain, &accel_domain_ctx); 2501 CU_ASSERT_EQUAL(rc, 0); 2502 2503 src_iovs[0].iov_base = (void *)0xfeedbeef; 2504 src_iovs[0].iov_len = sizeof(srcbuf); 2505 dst_iovs[0].iov_base = accel_buf; 2506 dst_iovs[0].iov_len = sizeof(dstbuf); 2507 ut_domain_ctx_init(&domctx[0], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2508 2509 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, accel_domain, accel_domain_ctx, 2510 &src_iovs[0], 1, g_ut_domain, &domctx[0], 0, 2511 ut_sequence_step_cb, &completed); 2512 CU_ASSERT_EQUAL(rc, 0); 2513 2514 src_iovs[1].iov_base = accel_buf; 2515 src_iovs[1].iov_len = sizeof(dstbuf); 2516 dst_iovs[1].iov_base = (void *)0xbeeffeed; 2517 dst_iovs[1].iov_len = sizeof(dstbuf); 2518 ut_domain_ctx_init(&domctx[1], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2519 2520 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[1], 2521 &src_iovs[1], 1, accel_domain, accel_domain_ctx, 0, 2522 ut_sequence_step_cb, &completed); 2523 CU_ASSERT_EQUAL(rc, 0); 2524 2525 ut_seq.complete = false; 2526 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2527 CU_ASSERT_EQUAL(rc, 0); 2528 2529 poll_threads(); 2530 CU_ASSERT_EQUAL(completed, 2); 2531 CU_ASSERT(ut_seq.complete); 2532 CU_ASSERT_EQUAL(ut_seq.status, 0); 2533 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2534 spdk_accel_put_buf(ioch, accel_buf, accel_domain, accel_domain_ctx); 2535 2536 /* Check that a sequence with memory domains is correctly executed if buffers are not 2537 * immediately available */ 2538 memset(expected, 0xa5, sizeof(expected)); 2539 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2540 memset(dstbuf, 0x0, sizeof(dstbuf)); 2541 completed = 0; 2542 seq = NULL; 2543 /* Make sure the buffer pool is empty */ 2544 accel_ch = spdk_io_channel_get_ctx(ioch); 2545 small_cache_count = accel_ch->iobuf.small.cache_count; 2546 STAILQ_INIT(&small_cache); 2547 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2548 accel_ch->iobuf.small.cache_count = 0; 2549 MOCK_SET(spdk_mempool_get, NULL); 2550 2551 src_iovs[0].iov_base = (void *)0xdeadbeef; 2552 src_iovs[0].iov_len = sizeof(srcbuf); 2553 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2554 dst_iovs[0].iov_len = sizeof(dstbuf); 2555 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2556 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2557 2558 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2559 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2560 ut_sequence_step_cb, &completed); 2561 CU_ASSERT_EQUAL(rc, 0); 2562 2563 ut_seq.complete = false; 2564 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2565 CU_ASSERT_EQUAL(rc, 0); 2566 2567 poll_threads(); 2568 CU_ASSERT_EQUAL(completed, 0); 2569 CU_ASSERT(!ut_seq.complete); 2570 2571 /* Get a buffer and return it to the pool to trigger the sequence to resume. It shouldn't 2572 * be able to complete, as it needs two buffers */ 2573 MOCK_CLEAR(spdk_mempool_get); 2574 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2575 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2576 MOCK_SET(spdk_mempool_get, NULL); 2577 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2578 2579 CU_ASSERT_EQUAL(completed, 0); 2580 CU_ASSERT(!ut_seq.complete); 2581 2582 /* Return another buffer, this time the sequence should finish */ 2583 MOCK_CLEAR(spdk_mempool_get); 2584 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2585 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2586 MOCK_SET(spdk_mempool_get, NULL); 2587 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2588 2589 poll_threads(); 2590 CU_ASSERT_EQUAL(completed, 1); 2591 CU_ASSERT(ut_seq.complete); 2592 CU_ASSERT_EQUAL(ut_seq.status, 0); 2593 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2594 2595 /* Return the buffers back to the cache */ 2596 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2597 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2598 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2599 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2600 small_cache_count++; 2601 } 2602 accel_ch->iobuf.small.cache_count = 0; 2603 2604 MOCK_CLEAR(spdk_mempool_get); 2605 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2606 accel_ch->iobuf.small.cache_count = small_cache_count; 2607 2608 /* Check error cases, starting with an error from spdk_memory_domain_pull_data() */ 2609 completed = 0; 2610 seq = NULL; 2611 2612 src_iovs[0].iov_base = (void *)0xdeadbeef; 2613 src_iovs[0].iov_len = sizeof(srcbuf); 2614 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2615 dst_iovs[0].iov_len = sizeof(dstbuf); 2616 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2617 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2618 domctx[1].pull_submit_status = -E2BIG; 2619 2620 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2621 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2622 ut_sequence_step_cb, &completed); 2623 CU_ASSERT_EQUAL(rc, 0); 2624 2625 ut_seq.complete = false; 2626 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2627 CU_ASSERT_EQUAL(rc, 0); 2628 2629 CU_ASSERT_EQUAL(completed, 1); 2630 CU_ASSERT(ut_seq.complete); 2631 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 2632 2633 /* Check completion error from spdk_memory_domain_pull_data() */ 2634 completed = 0; 2635 seq = NULL; 2636 2637 src_iovs[0].iov_base = (void *)0xdeadbeef; 2638 src_iovs[0].iov_len = sizeof(srcbuf); 2639 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2640 dst_iovs[0].iov_len = sizeof(dstbuf); 2641 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2642 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2643 domctx[1].pull_complete_status = -EACCES; 2644 2645 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2646 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2647 ut_sequence_step_cb, &completed); 2648 CU_ASSERT_EQUAL(rc, 0); 2649 2650 ut_seq.complete = false; 2651 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2652 CU_ASSERT_EQUAL(rc, 0); 2653 2654 CU_ASSERT_EQUAL(completed, 1); 2655 CU_ASSERT(ut_seq.complete); 2656 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 2657 2658 /* Check submission error from spdk_memory_domain_push_data() */ 2659 completed = 0; 2660 seq = NULL; 2661 2662 src_iovs[0].iov_base = (void *)0xdeadbeef; 2663 src_iovs[0].iov_len = sizeof(srcbuf); 2664 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2665 dst_iovs[0].iov_len = sizeof(dstbuf); 2666 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2667 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2668 domctx[0].push_submit_status = -EADDRINUSE; 2669 2670 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2671 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2672 ut_sequence_step_cb, &completed); 2673 CU_ASSERT_EQUAL(rc, 0); 2674 2675 ut_seq.complete = false; 2676 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2677 CU_ASSERT_EQUAL(rc, 0); 2678 2679 CU_ASSERT_EQUAL(completed, 1); 2680 CU_ASSERT(ut_seq.complete); 2681 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 2682 2683 /* Check completion error from spdk_memory_domain_push_data() */ 2684 completed = 0; 2685 seq = NULL; 2686 2687 src_iovs[0].iov_base = (void *)0xdeadbeef; 2688 src_iovs[0].iov_len = sizeof(srcbuf); 2689 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2690 dst_iovs[0].iov_len = sizeof(dstbuf); 2691 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2692 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2693 domctx[0].push_complete_status = -EADDRNOTAVAIL; 2694 2695 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2696 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2697 ut_sequence_step_cb, &completed); 2698 CU_ASSERT_EQUAL(rc, 0); 2699 2700 ut_seq.complete = false; 2701 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2702 CU_ASSERT_EQUAL(rc, 0); 2703 2704 CU_ASSERT_EQUAL(completed, 1); 2705 CU_ASSERT(ut_seq.complete); 2706 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 2707 2708 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = NULL; 2709 g_seq_operations[ACCEL_OPC_COPY].submit = NULL; 2710 g_seq_operations[ACCEL_OPC_FILL].submit = NULL; 2711 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2712 g_modules_opc[i] = modules[i]; 2713 } 2714 2715 spdk_put_io_channel(ioch); 2716 poll_threads(); 2717 } 2718 2719 static int 2720 ut_submit_decompress_memory_domain(struct spdk_io_channel *ch, struct spdk_accel_task *task) 2721 { 2722 struct ut_domain_ctx *ctx; 2723 struct iovec *src_iovs, *dst_iovs; 2724 uint32_t src_iovcnt, dst_iovcnt; 2725 2726 src_iovs = task->s.iovs; 2727 dst_iovs = task->d.iovs; 2728 src_iovcnt = task->s.iovcnt; 2729 dst_iovcnt = task->d.iovcnt; 2730 2731 if (task->src_domain != NULL) { 2732 ctx = task->src_domain_ctx; 2733 CU_ASSERT_EQUAL(memcmp(task->s.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2734 src_iovs = &ctx->iov; 2735 src_iovcnt = 1; 2736 } 2737 2738 if (task->dst_domain != NULL) { 2739 ctx = task->dst_domain_ctx; 2740 CU_ASSERT_EQUAL(memcmp(task->d.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2741 dst_iovs = &ctx->iov; 2742 dst_iovcnt = 1; 2743 } 2744 2745 spdk_iovcpy(src_iovs, src_iovcnt, dst_iovs, dst_iovcnt); 2746 spdk_accel_task_complete(task, 0); 2747 2748 return 0; 2749 } 2750 2751 static void 2752 test_sequence_module_memory_domain(void) 2753 { 2754 struct spdk_accel_sequence *seq = NULL; 2755 struct spdk_io_channel *ioch; 2756 struct accel_module modules[ACCEL_OPC_LAST]; 2757 struct spdk_memory_domain *accel_domain; 2758 struct ut_sequence ut_seq; 2759 struct ut_domain_ctx domctx[2]; 2760 struct iovec src_iovs[2], dst_iovs[2]; 2761 void *buf, *accel_domain_ctx; 2762 char srcbuf[4096], dstbuf[4096], tmp[4096], expected[4096]; 2763 int i, rc, completed; 2764 2765 ioch = spdk_accel_get_io_channel(); 2766 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2767 2768 /* Override the submit_tasks function */ 2769 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2770 g_module.supports_memory_domains = true; 2771 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2772 modules[i] = g_modules_opc[i]; 2773 g_modules_opc[i] = g_module; 2774 } 2775 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress_memory_domain; 2776 g_seq_operations[ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2777 2778 /* Check a sequence with both buffers in memory domains */ 2779 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2780 memset(expected, 0xa5, sizeof(expected)); 2781 memset(dstbuf, 0, sizeof(dstbuf)); 2782 seq = NULL; 2783 completed = 0; 2784 2785 src_iovs[0].iov_base = (void *)0xcafebabe; 2786 src_iovs[0].iov_len = sizeof(srcbuf); 2787 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2788 dst_iovs[0].iov_len = sizeof(dstbuf); 2789 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2790 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2791 2792 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2793 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2794 ut_sequence_step_cb, &completed); 2795 CU_ASSERT_EQUAL(rc, 0); 2796 2797 ut_seq.complete = false; 2798 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2799 CU_ASSERT_EQUAL(rc, 0); 2800 2801 poll_threads(); 2802 2803 CU_ASSERT_EQUAL(completed, 1); 2804 CU_ASSERT(ut_seq.complete); 2805 CU_ASSERT_EQUAL(ut_seq.status, 0); 2806 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2807 2808 /* Check two operations each with a single buffer in memory domain */ 2809 memset(srcbuf, 0x5a, sizeof(srcbuf)); 2810 memset(expected, 0x5a, sizeof(expected)); 2811 memset(dstbuf, 0, sizeof(dstbuf)); 2812 memset(tmp, 0, sizeof(tmp)); 2813 seq = NULL; 2814 completed = 0; 2815 2816 src_iovs[0].iov_base = srcbuf; 2817 src_iovs[0].iov_len = sizeof(srcbuf); 2818 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2819 dst_iovs[0].iov_len = sizeof(tmp); 2820 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2821 2822 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2823 &src_iovs[0], 1, NULL, NULL, 0, 2824 ut_sequence_step_cb, &completed); 2825 CU_ASSERT_EQUAL(rc, 0); 2826 2827 src_iovs[1].iov_base = (void *)0xfeedbeef; 2828 src_iovs[1].iov_len = sizeof(tmp); 2829 dst_iovs[1].iov_base = dstbuf; 2830 dst_iovs[1].iov_len = sizeof(dstbuf); 2831 ut_domain_ctx_init(&domctx[1], tmp, sizeof(tmp), &src_iovs[1]); 2832 2833 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2834 &src_iovs[1], 1, g_ut_domain, &domctx[1], 0, 2835 ut_sequence_step_cb, &completed); 2836 CU_ASSERT_EQUAL(rc, 0); 2837 2838 ut_seq.complete = false; 2839 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2840 CU_ASSERT_EQUAL(rc, 0); 2841 2842 poll_threads(); 2843 2844 CU_ASSERT_EQUAL(completed, 2); 2845 CU_ASSERT(ut_seq.complete); 2846 CU_ASSERT_EQUAL(ut_seq.status, 0); 2847 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2848 2849 /* Check a sequence with an accel buffer and a buffer in a regular memory domain */ 2850 memset(expected, 0xa5, sizeof(expected)); 2851 memset(dstbuf, 0, sizeof(dstbuf)); 2852 memset(tmp, 0, sizeof(tmp)); 2853 seq = NULL; 2854 completed = 0; 2855 2856 rc = spdk_accel_get_buf(ioch, 4096, &buf, &accel_domain, &accel_domain_ctx); 2857 CU_ASSERT_EQUAL(rc, 0); 2858 2859 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, accel_domain, accel_domain_ctx, 2860 0xa5, 0, ut_sequence_step_cb, &completed); 2861 CU_ASSERT_EQUAL(rc, 0); 2862 2863 src_iovs[0].iov_base = buf; 2864 src_iovs[0].iov_len = 4096; 2865 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2866 dst_iovs[0].iov_len = sizeof(dstbuf); 2867 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2868 2869 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2870 &src_iovs[0], 1, accel_domain, accel_domain_ctx, 0, 2871 ut_sequence_step_cb, &completed); 2872 CU_ASSERT_EQUAL(rc, 0); 2873 2874 ut_seq.complete = false; 2875 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2876 CU_ASSERT_EQUAL(rc, 0); 2877 2878 poll_threads(); 2879 2880 CU_ASSERT_EQUAL(completed, 2); 2881 CU_ASSERT(ut_seq.complete); 2882 CU_ASSERT_EQUAL(ut_seq.status, 0); 2883 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2884 2885 spdk_accel_put_buf(ioch, buf, accel_domain, accel_domain_ctx); 2886 2887 g_module.supports_memory_domains = false; 2888 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = NULL; 2889 g_seq_operations[ACCEL_OPC_FILL].submit = NULL; 2890 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2891 g_modules_opc[i] = modules[i]; 2892 } 2893 2894 spdk_put_io_channel(ioch); 2895 poll_threads(); 2896 } 2897 2898 #ifdef SPDK_CONFIG_ISAL_CRYPTO 2899 static void 2900 ut_encrypt_cb(void *cb_arg, int status) 2901 { 2902 int *completed = cb_arg; 2903 2904 CU_ASSERT_EQUAL(status, 0); 2905 2906 *completed = 1; 2907 } 2908 2909 static void 2910 test_sequence_crypto(void) 2911 { 2912 struct spdk_accel_sequence *seq = NULL; 2913 struct spdk_io_channel *ioch; 2914 struct spdk_accel_crypto_key *key; 2915 struct spdk_accel_crypto_key_create_param key_params = { 2916 .cipher = "AES_XTS", 2917 .hex_key = "00112233445566778899aabbccddeeff", 2918 .hex_key2 = "ffeeddccbbaa99887766554433221100", 2919 .key_name = "ut_key", 2920 }; 2921 struct ut_sequence ut_seq; 2922 unsigned char buf[4096], encrypted[4096] = {}, data[4096], tmp[3][4096]; 2923 struct iovec src_iovs[4], dst_iovs[4]; 2924 int rc, completed = 0; 2925 size_t i; 2926 2927 ioch = spdk_accel_get_io_channel(); 2928 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2929 2930 rc = spdk_accel_crypto_key_create(&key_params); 2931 CU_ASSERT_EQUAL(rc, 0); 2932 key = spdk_accel_crypto_key_get(key_params.key_name); 2933 SPDK_CU_ASSERT_FATAL(key != NULL); 2934 2935 for (i = 0; i < sizeof(data); ++i) { 2936 data[i] = (uint8_t)i & 0xff; 2937 } 2938 2939 dst_iovs[0].iov_base = encrypted; 2940 dst_iovs[0].iov_len = sizeof(encrypted); 2941 src_iovs[0].iov_base = data; 2942 src_iovs[0].iov_len = sizeof(data); 2943 rc = spdk_accel_submit_encrypt(ioch, key, &dst_iovs[0], 1, &src_iovs[0], 1, 0, 4096, 0, 2944 ut_encrypt_cb, &completed); 2945 CU_ASSERT_EQUAL(rc, 0); 2946 2947 while (!completed) { 2948 poll_threads(); 2949 } 2950 2951 /* Verify that encryption operation in a sequence produces the same result */ 2952 seq = NULL; 2953 completed = 0; 2954 2955 dst_iovs[0].iov_base = tmp[0]; 2956 dst_iovs[0].iov_len = sizeof(tmp[0]); 2957 src_iovs[0].iov_base = data; 2958 src_iovs[0].iov_len = sizeof(data); 2959 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2960 &src_iovs[0], 1, NULL, NULL, 0, 2961 ut_sequence_step_cb, &completed); 2962 CU_ASSERT_EQUAL(rc, 0); 2963 2964 dst_iovs[1].iov_base = tmp[1]; 2965 dst_iovs[1].iov_len = sizeof(tmp[1]); 2966 src_iovs[1].iov_base = tmp[0]; 2967 src_iovs[1].iov_len = sizeof(tmp[0]); 2968 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 2969 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 2970 ut_sequence_step_cb, &completed); 2971 CU_ASSERT_EQUAL(rc, 0); 2972 2973 dst_iovs[2].iov_base = buf; 2974 dst_iovs[2].iov_len = sizeof(buf); 2975 src_iovs[2].iov_base = tmp[1]; 2976 src_iovs[2].iov_len = sizeof(tmp[1]); 2977 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 2978 &src_iovs[2], 1, NULL, NULL, 0, 2979 ut_sequence_step_cb, &completed); 2980 CU_ASSERT_EQUAL(rc, 0); 2981 2982 ut_seq.complete = false; 2983 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2984 CU_ASSERT_EQUAL(rc, 0); 2985 2986 poll_threads(); 2987 2988 CU_ASSERT_EQUAL(completed, 3); 2989 CU_ASSERT(ut_seq.complete); 2990 CU_ASSERT_EQUAL(ut_seq.status, 0); 2991 CU_ASSERT_EQUAL(memcmp(buf, encrypted, sizeof(buf)), 0); 2992 2993 /* Check that decryption produces the original buffer */ 2994 seq = NULL; 2995 completed = 0; 2996 memset(buf, 0, sizeof(buf)); 2997 2998 dst_iovs[0].iov_base = tmp[0]; 2999 dst_iovs[0].iov_len = sizeof(tmp[0]); 3000 src_iovs[0].iov_base = encrypted; 3001 src_iovs[0].iov_len = sizeof(encrypted); 3002 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3003 &src_iovs[0], 1, NULL, NULL, 0, 3004 ut_sequence_step_cb, &completed); 3005 CU_ASSERT_EQUAL(rc, 0); 3006 3007 dst_iovs[1].iov_base = tmp[1]; 3008 dst_iovs[1].iov_len = sizeof(tmp[1]); 3009 src_iovs[1].iov_base = tmp[0]; 3010 src_iovs[1].iov_len = sizeof(tmp[0]); 3011 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3012 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3013 ut_sequence_step_cb, &completed); 3014 CU_ASSERT_EQUAL(rc, 0); 3015 3016 dst_iovs[2].iov_base = buf; 3017 dst_iovs[2].iov_len = sizeof(buf); 3018 src_iovs[2].iov_base = tmp[1]; 3019 src_iovs[2].iov_len = sizeof(tmp[1]); 3020 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3021 &src_iovs[2], 1, NULL, NULL, 0, 3022 ut_sequence_step_cb, &completed); 3023 CU_ASSERT_EQUAL(rc, 0); 3024 3025 ut_seq.complete = false; 3026 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3027 CU_ASSERT_EQUAL(rc, 0); 3028 3029 poll_threads(); 3030 3031 CU_ASSERT_EQUAL(completed, 3); 3032 CU_ASSERT(ut_seq.complete); 3033 CU_ASSERT_EQUAL(ut_seq.status, 0); 3034 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3035 3036 /* Check encrypt + decrypt in a single sequence */ 3037 seq = NULL; 3038 completed = 0; 3039 memset(buf, 0, sizeof(buf)); 3040 3041 dst_iovs[0].iov_base = tmp[0]; 3042 dst_iovs[0].iov_len = sizeof(tmp[0]); 3043 src_iovs[0].iov_base = data; 3044 src_iovs[0].iov_len = sizeof(data); 3045 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3046 &src_iovs[0], 1, NULL, NULL, 0, 3047 ut_sequence_step_cb, &completed); 3048 CU_ASSERT_EQUAL(rc, 0); 3049 3050 dst_iovs[1].iov_base = tmp[1]; 3051 dst_iovs[1].iov_len = sizeof(tmp[1]); 3052 src_iovs[1].iov_base = tmp[0]; 3053 src_iovs[1].iov_len = sizeof(tmp[0]); 3054 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3055 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3056 ut_sequence_step_cb, &completed); 3057 CU_ASSERT_EQUAL(rc, 0); 3058 3059 3060 dst_iovs[2].iov_base = tmp[2]; 3061 dst_iovs[2].iov_len = sizeof(tmp[2]); 3062 src_iovs[2].iov_base = tmp[1]; 3063 src_iovs[2].iov_len = sizeof(tmp[1]); 3064 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[2], 1, NULL, NULL, 3065 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3066 ut_sequence_step_cb, &completed); 3067 CU_ASSERT_EQUAL(rc, 0); 3068 3069 dst_iovs[3].iov_base = buf; 3070 dst_iovs[3].iov_len = sizeof(buf); 3071 src_iovs[3].iov_base = tmp[2]; 3072 src_iovs[3].iov_len = sizeof(tmp[2]); 3073 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 3074 &src_iovs[3], 1, NULL, NULL, 0, 3075 ut_sequence_step_cb, &completed); 3076 CU_ASSERT_EQUAL(rc, 0); 3077 3078 ut_seq.complete = false; 3079 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3080 CU_ASSERT_EQUAL(rc, 0); 3081 3082 poll_threads(); 3083 3084 CU_ASSERT_EQUAL(completed, 4); 3085 CU_ASSERT(ut_seq.complete); 3086 CU_ASSERT_EQUAL(ut_seq.status, 0); 3087 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3088 3089 rc = spdk_accel_crypto_key_destroy(key); 3090 CU_ASSERT_EQUAL(rc, 0); 3091 spdk_put_io_channel(ioch); 3092 poll_threads(); 3093 } 3094 #endif /* SPDK_CONFIG_ISAL_CRYPTO */ 3095 3096 static int 3097 test_sequence_setup(void) 3098 { 3099 int rc; 3100 3101 allocate_cores(1); 3102 allocate_threads(1); 3103 set_thread(0); 3104 3105 rc = spdk_iobuf_initialize(); 3106 if (rc != 0) { 3107 CU_ASSERT(false); 3108 return -1; 3109 } 3110 3111 rc = spdk_accel_initialize(); 3112 if (rc != 0) { 3113 CU_ASSERT(false); 3114 return -1; 3115 } 3116 3117 return 0; 3118 } 3119 3120 static void 3121 finish_cb(void *cb_arg) 3122 { 3123 bool *done = cb_arg; 3124 3125 *done = true; 3126 } 3127 3128 static int 3129 test_sequence_cleanup(void) 3130 { 3131 bool done = false; 3132 3133 spdk_accel_finish(finish_cb, &done); 3134 3135 while (!done) { 3136 poll_threads(); 3137 } 3138 3139 done = false; 3140 spdk_iobuf_finish(finish_cb, &done); 3141 while (!done) { 3142 poll_threads(); 3143 } 3144 3145 free_threads(); 3146 free_cores(); 3147 3148 return 0; 3149 } 3150 3151 int 3152 main(int argc, char **argv) 3153 { 3154 CU_pSuite suite = NULL, seq_suite; 3155 unsigned int num_failures; 3156 3157 CU_set_error_action(CUEA_ABORT); 3158 CU_initialize_registry(); 3159 3160 /* Sequence tests require accel to be initialized normally, so run them before the other 3161 * tests which register accel modules which aren't fully implemented, causing accel 3162 * initialization to fail. 3163 */ 3164 seq_suite = CU_add_suite("accel_sequence", test_sequence_setup, test_sequence_cleanup); 3165 CU_ADD_TEST(seq_suite, test_sequence_fill_copy); 3166 CU_ADD_TEST(seq_suite, test_sequence_abort); 3167 CU_ADD_TEST(seq_suite, test_sequence_append_error); 3168 CU_ADD_TEST(seq_suite, test_sequence_completion_error); 3169 #ifdef SPDK_CONFIG_ISAL /* accel_sw requires isa-l for compression */ 3170 CU_ADD_TEST(seq_suite, test_sequence_decompress); 3171 CU_ADD_TEST(seq_suite, test_sequence_reverse); 3172 #endif 3173 CU_ADD_TEST(seq_suite, test_sequence_copy_elision); 3174 CU_ADD_TEST(seq_suite, test_sequence_accel_buffers); 3175 CU_ADD_TEST(seq_suite, test_sequence_memory_domain); 3176 CU_ADD_TEST(seq_suite, test_sequence_module_memory_domain); 3177 #ifdef SPDK_CONFIG_ISAL_CRYPTO /* accel_sw requires isa-l-crypto for crypto operations */ 3178 CU_ADD_TEST(seq_suite, test_sequence_crypto); 3179 #endif 3180 suite = CU_add_suite("accel", test_setup, test_cleanup); 3181 CU_ADD_TEST(suite, test_spdk_accel_task_complete); 3182 CU_ADD_TEST(suite, test_get_task); 3183 CU_ADD_TEST(suite, test_spdk_accel_submit_copy); 3184 CU_ADD_TEST(suite, test_spdk_accel_submit_dualcast); 3185 CU_ADD_TEST(suite, test_spdk_accel_submit_compare); 3186 CU_ADD_TEST(suite, test_spdk_accel_submit_fill); 3187 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c); 3188 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32cv); 3189 CU_ADD_TEST(suite, test_spdk_accel_submit_copy_crc32c); 3190 CU_ADD_TEST(suite, test_spdk_accel_module_find_by_name); 3191 CU_ADD_TEST(suite, test_spdk_accel_module_register); 3192 3193 CU_basic_set_mode(CU_BRM_VERBOSE); 3194 CU_basic_run_tests(); 3195 num_failures = CU_get_number_of_failures(); 3196 CU_cleanup_registry(); 3197 3198 return num_failures; 3199 } 3200