1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2021 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk_cunit.h" 8 #include "spdk_internal/mock.h" 9 #include "spdk_internal/accel_module.h" 10 #include "thread/thread_internal.h" 11 #include "common/lib/ut_multithread.c" 12 #include "accel/accel.c" 13 #include "accel/accel_sw.c" 14 #include "unit/lib/json_mock.c" 15 16 DEFINE_STUB_V(spdk_memory_domain_destroy, (struct spdk_memory_domain *domain)); 17 DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, 18 (struct spdk_memory_domain *domain), "UT_DMA"); 19 20 int 21 spdk_memory_domain_create(struct spdk_memory_domain **domain, enum spdk_dma_device_type type, 22 struct spdk_memory_domain_ctx *ctx, const char *id) 23 { 24 *domain = (void *)0xdeadbeef; 25 26 return 0; 27 } 28 29 struct ut_domain_ctx { 30 struct iovec iov; 31 struct iovec expected; 32 int pull_submit_status; 33 int push_submit_status; 34 int pull_complete_status; 35 int push_complete_status; 36 }; 37 38 static struct spdk_memory_domain *g_ut_domain = (void *)0xa55e1; 39 40 int 41 spdk_memory_domain_pull_data(struct spdk_memory_domain *sd, void *sctx, struct iovec *siov, 42 uint32_t siovcnt, struct iovec *diov, uint32_t diovcnt, 43 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 44 { 45 struct ut_domain_ctx *ctx = sctx; 46 47 CU_ASSERT_EQUAL(sd, g_ut_domain); 48 CU_ASSERT_EQUAL(siovcnt, 1); 49 CU_ASSERT_EQUAL(memcmp(siov, &ctx->expected, sizeof(*siov)), 0); 50 51 if (ctx->pull_submit_status != 0) { 52 return ctx->pull_submit_status; 53 } 54 55 if (ctx->pull_complete_status != 0) { 56 cpl_cb(cpl_arg, ctx->pull_complete_status); 57 return 0; 58 } 59 60 spdk_iovcpy(&ctx->iov, 1, diov, diovcnt); 61 cpl_cb(cpl_arg, 0); 62 63 return 0; 64 } 65 66 int 67 spdk_memory_domain_push_data(struct spdk_memory_domain *dd, void *dctx, struct iovec *diov, 68 uint32_t diovcnt, struct iovec *siov, uint32_t siovcnt, 69 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 70 { 71 struct ut_domain_ctx *ctx = dctx; 72 73 CU_ASSERT_EQUAL(dd, g_ut_domain); 74 CU_ASSERT_EQUAL(diovcnt, 1); 75 CU_ASSERT_EQUAL(memcmp(diov, &ctx->expected, sizeof(*diov)), 0); 76 77 if (ctx->push_submit_status != 0) { 78 return ctx->push_submit_status; 79 } 80 81 if (ctx->push_complete_status != 0) { 82 cpl_cb(cpl_arg, ctx->push_complete_status); 83 return 0; 84 } 85 86 spdk_iovcpy(siov, siovcnt, &ctx->iov, 1); 87 cpl_cb(cpl_arg, 0); 88 89 return 0; 90 } 91 92 /* global vars and setup/cleanup functions used for all test functions */ 93 struct spdk_accel_module_if g_module_if = {}; 94 struct accel_module g_module = { .module = &g_module_if }; 95 struct spdk_io_channel *g_ch = NULL; 96 struct accel_io_channel *g_accel_ch = NULL; 97 struct sw_accel_io_channel *g_sw_ch = NULL; 98 struct spdk_io_channel *g_module_ch = NULL; 99 100 static uint64_t g_opc_mask = 0; 101 102 static uint64_t 103 _accel_op_to_bit(enum accel_opcode opc) 104 { 105 return (1 << opc); 106 } 107 108 static bool 109 _supports_opcode(enum accel_opcode opc) 110 { 111 if (_accel_op_to_bit(opc) & g_opc_mask) { 112 return true; 113 } 114 return false; 115 } 116 117 static int 118 test_setup(void) 119 { 120 int i; 121 122 g_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct accel_io_channel)); 123 if (g_ch == NULL) { 124 /* for some reason the assert fatal macro doesn't work in the setup function. */ 125 CU_ASSERT(false); 126 return -1; 127 } 128 g_accel_ch = (struct accel_io_channel *)((char *)g_ch + sizeof(struct spdk_io_channel)); 129 g_module_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct sw_accel_io_channel)); 130 if (g_module_ch == NULL) { 131 CU_ASSERT(false); 132 return -1; 133 } 134 135 g_module_if.submit_tasks = sw_accel_submit_tasks; 136 g_module_if.name = "software"; 137 for (i = 0; i < ACCEL_OPC_LAST; i++) { 138 g_accel_ch->module_ch[i] = g_module_ch; 139 g_modules_opc[i] = g_module; 140 } 141 g_sw_ch = (struct sw_accel_io_channel *)((char *)g_module_ch + sizeof( 142 struct spdk_io_channel)); 143 TAILQ_INIT(&g_sw_ch->tasks_to_complete); 144 g_module_if.supports_opcode = _supports_opcode; 145 return 0; 146 } 147 148 static int 149 test_cleanup(void) 150 { 151 free(g_ch); 152 free(g_module_ch); 153 154 return 0; 155 } 156 157 #define DUMMY_ARG 0xDEADBEEF 158 static bool g_dummy_cb_called = false; 159 static void 160 dummy_cb_fn(void *cb_arg, int status) 161 { 162 CU_ASSERT(*(uint32_t *)cb_arg == DUMMY_ARG); 163 CU_ASSERT(status == 0); 164 g_dummy_cb_called = true; 165 } 166 167 static void 168 test_spdk_accel_task_complete(void) 169 { 170 struct spdk_accel_task accel_task = {}; 171 struct spdk_accel_task *expected_accel_task = NULL; 172 uint32_t cb_arg = DUMMY_ARG; 173 int status = 0; 174 175 accel_task.accel_ch = g_accel_ch; 176 accel_task.cb_fn = dummy_cb_fn; 177 accel_task.cb_arg = &cb_arg; 178 TAILQ_INIT(&g_accel_ch->task_pool); 179 180 /* Confirm cb is called and task added to list. */ 181 spdk_accel_task_complete(&accel_task, status); 182 CU_ASSERT(g_dummy_cb_called == true); 183 expected_accel_task = TAILQ_FIRST(&g_accel_ch->task_pool); 184 TAILQ_REMOVE(&g_accel_ch->task_pool, expected_accel_task, link); 185 CU_ASSERT(expected_accel_task == &accel_task); 186 } 187 188 static void 189 test_get_task(void) 190 { 191 struct spdk_accel_task *task; 192 struct spdk_accel_task _task; 193 void *cb_arg = NULL; 194 195 TAILQ_INIT(&g_accel_ch->task_pool); 196 197 /* no tasks left, return NULL. */ 198 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 199 CU_ASSERT(task == NULL); 200 201 _task.cb_fn = dummy_cb_fn; 202 _task.cb_arg = cb_arg; 203 _task.accel_ch = g_accel_ch; 204 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &_task, link); 205 206 /* Get a valid task. */ 207 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 208 CU_ASSERT(task == &_task); 209 CU_ASSERT(_task.cb_fn == dummy_cb_fn); 210 CU_ASSERT(_task.cb_arg == cb_arg); 211 CU_ASSERT(_task.accel_ch == g_accel_ch); 212 } 213 214 #define TEST_SUBMIT_SIZE 64 215 static void 216 test_spdk_accel_submit_copy(void) 217 { 218 const uint64_t nbytes = TEST_SUBMIT_SIZE; 219 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 220 uint8_t src[TEST_SUBMIT_SIZE] = {0}; 221 void *cb_arg = NULL; 222 int rc; 223 struct spdk_accel_task task; 224 struct spdk_accel_task *expected_accel_task = NULL; 225 int flags = 0; 226 227 TAILQ_INIT(&g_accel_ch->task_pool); 228 229 /* Fail with no tasks on _get_task() */ 230 rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, flags, NULL, cb_arg); 231 CU_ASSERT(rc == -ENOMEM); 232 233 task.accel_ch = g_accel_ch; 234 task.flags = 1; 235 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 236 237 /* submission OK. */ 238 rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, flags, NULL, cb_arg); 239 CU_ASSERT(rc == 0); 240 CU_ASSERT(task.op_code == ACCEL_OPC_COPY); 241 CU_ASSERT(task.flags == 0); 242 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 243 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 244 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 245 CU_ASSERT(expected_accel_task == &task); 246 } 247 248 static void 249 test_spdk_accel_submit_dualcast(void) 250 { 251 void *dst1; 252 void *dst2; 253 void *src; 254 uint32_t align = ALIGN_4K; 255 uint64_t nbytes = TEST_SUBMIT_SIZE; 256 void *cb_arg = NULL; 257 int rc; 258 struct spdk_accel_task task; 259 struct spdk_accel_task *expected_accel_task = NULL; 260 int flags = 0; 261 262 TAILQ_INIT(&g_accel_ch->task_pool); 263 264 /* Dualcast requires 4K alignment on dst addresses, 265 * hence using the hard coded address to test the buffer alignment 266 */ 267 dst1 = (void *)0x5000; 268 dst2 = (void *)0x60f0; 269 src = calloc(1, TEST_SUBMIT_SIZE); 270 SPDK_CU_ASSERT_FATAL(src != NULL); 271 memset(src, 0x5A, TEST_SUBMIT_SIZE); 272 273 /* This should fail since dst2 is not 4k aligned */ 274 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 275 CU_ASSERT(rc == -EINVAL); 276 277 dst1 = (void *)0x7010; 278 dst2 = (void *)0x6000; 279 /* This should fail since dst1 is not 4k aligned */ 280 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 281 CU_ASSERT(rc == -EINVAL); 282 283 /* Dualcast requires 4K alignment on dst addresses */ 284 dst1 = (void *)0x7000; 285 dst2 = (void *)0x6000; 286 /* Fail with no tasks on _get_task() */ 287 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 288 CU_ASSERT(rc == -ENOMEM); 289 290 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 291 292 /* accel submission OK., since we test the SW path , need to use valid memory addresses 293 * cannot hardcode them anymore */ 294 dst1 = spdk_dma_zmalloc(nbytes, align, NULL); 295 SPDK_CU_ASSERT_FATAL(dst1 != NULL); 296 dst2 = spdk_dma_zmalloc(nbytes, align, NULL); 297 SPDK_CU_ASSERT_FATAL(dst2 != NULL); 298 /* SW module does the dualcast. */ 299 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 300 CU_ASSERT(rc == 0); 301 CU_ASSERT(task.op_code == ACCEL_OPC_DUALCAST); 302 CU_ASSERT(task.flags == 0); 303 CU_ASSERT(memcmp(dst1, src, TEST_SUBMIT_SIZE) == 0); 304 CU_ASSERT(memcmp(dst2, src, TEST_SUBMIT_SIZE) == 0); 305 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 306 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 307 CU_ASSERT(expected_accel_task == &task); 308 309 free(src); 310 spdk_free(dst1); 311 spdk_free(dst2); 312 } 313 314 static void 315 test_spdk_accel_submit_compare(void) 316 { 317 void *src1; 318 void *src2; 319 uint64_t nbytes = TEST_SUBMIT_SIZE; 320 void *cb_arg = NULL; 321 int rc; 322 struct spdk_accel_task task; 323 struct spdk_accel_task *expected_accel_task = NULL; 324 325 TAILQ_INIT(&g_accel_ch->task_pool); 326 327 src1 = calloc(1, TEST_SUBMIT_SIZE); 328 SPDK_CU_ASSERT_FATAL(src1 != NULL); 329 src2 = calloc(1, TEST_SUBMIT_SIZE); 330 SPDK_CU_ASSERT_FATAL(src2 != NULL); 331 332 /* Fail with no tasks on _get_task() */ 333 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 334 CU_ASSERT(rc == -ENOMEM); 335 336 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 337 338 /* accel submission OK. */ 339 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 340 CU_ASSERT(rc == 0); 341 CU_ASSERT(task.op_code == ACCEL_OPC_COMPARE); 342 CU_ASSERT(memcmp(src1, src2, TEST_SUBMIT_SIZE) == 0); 343 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 344 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 345 CU_ASSERT(expected_accel_task == &task); 346 347 free(src1); 348 free(src2); 349 } 350 351 static void 352 test_spdk_accel_submit_fill(void) 353 { 354 void *dst; 355 void *src; 356 uint8_t fill = 0xf; 357 uint64_t fill64; 358 uint64_t nbytes = TEST_SUBMIT_SIZE; 359 void *cb_arg = NULL; 360 int rc; 361 struct spdk_accel_task task; 362 struct spdk_accel_task *expected_accel_task = NULL; 363 int flags = 0; 364 365 TAILQ_INIT(&g_accel_ch->task_pool); 366 367 dst = calloc(1, TEST_SUBMIT_SIZE); 368 SPDK_CU_ASSERT_FATAL(dst != NULL); 369 src = calloc(1, TEST_SUBMIT_SIZE); 370 SPDK_CU_ASSERT_FATAL(src != NULL); 371 memset(src, fill, TEST_SUBMIT_SIZE); 372 memset(&fill64, fill, sizeof(uint64_t)); 373 374 /* Fail with no tasks on _get_task() */ 375 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 376 CU_ASSERT(rc == -ENOMEM); 377 378 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 379 380 /* accel submission OK. */ 381 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 382 CU_ASSERT(rc == 0); 383 CU_ASSERT(task.fill_pattern == fill64); 384 CU_ASSERT(task.op_code == ACCEL_OPC_FILL); 385 CU_ASSERT(task.flags == 0); 386 387 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 388 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 389 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 390 CU_ASSERT(expected_accel_task == &task); 391 392 free(dst); 393 free(src); 394 } 395 396 static void 397 test_spdk_accel_submit_crc32c(void) 398 { 399 const uint64_t nbytes = TEST_SUBMIT_SIZE; 400 uint32_t crc_dst; 401 uint8_t src[TEST_SUBMIT_SIZE]; 402 uint32_t seed = 1; 403 void *cb_arg = NULL; 404 int rc; 405 struct spdk_accel_task task; 406 struct spdk_accel_task *expected_accel_task = NULL; 407 408 TAILQ_INIT(&g_accel_ch->task_pool); 409 410 /* Fail with no tasks on _get_task() */ 411 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 412 CU_ASSERT(rc == -ENOMEM); 413 414 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 415 416 /* accel submission OK. */ 417 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 418 CU_ASSERT(rc == 0); 419 CU_ASSERT(task.crc_dst == &crc_dst); 420 CU_ASSERT(task.seed == seed); 421 CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); 422 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 423 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 424 CU_ASSERT(expected_accel_task == &task); 425 } 426 427 static void 428 test_spdk_accel_submit_crc32cv(void) 429 { 430 uint32_t crc_dst; 431 uint32_t seed = 0; 432 uint32_t iov_cnt = 32; 433 void *cb_arg = NULL; 434 int rc; 435 uint32_t i = 0; 436 struct spdk_accel_task task; 437 struct iovec iov[32]; 438 struct spdk_accel_task *expected_accel_task = NULL; 439 440 TAILQ_INIT(&g_accel_ch->task_pool); 441 442 for (i = 0; i < iov_cnt; i++) { 443 iov[i].iov_base = calloc(1, TEST_SUBMIT_SIZE); 444 SPDK_CU_ASSERT_FATAL(iov[i].iov_base != NULL); 445 iov[i].iov_len = TEST_SUBMIT_SIZE; 446 } 447 448 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 449 450 /* accel submission OK. */ 451 rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, NULL, cb_arg); 452 CU_ASSERT(rc == 0); 453 CU_ASSERT(task.s.iovs == iov); 454 CU_ASSERT(task.s.iovcnt == iov_cnt); 455 CU_ASSERT(task.crc_dst == &crc_dst); 456 CU_ASSERT(task.seed == seed); 457 CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); 458 CU_ASSERT(task.cb_arg == cb_arg); 459 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 460 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 461 CU_ASSERT(expected_accel_task == &task); 462 463 for (i = 0; i < iov_cnt; i++) { 464 free(iov[i].iov_base); 465 } 466 } 467 468 static void 469 test_spdk_accel_submit_copy_crc32c(void) 470 { 471 const uint64_t nbytes = TEST_SUBMIT_SIZE; 472 uint32_t crc_dst; 473 uint8_t dst[TEST_SUBMIT_SIZE]; 474 uint8_t src[TEST_SUBMIT_SIZE]; 475 uint32_t seed = 0; 476 void *cb_arg = NULL; 477 int rc; 478 struct spdk_accel_task task; 479 struct spdk_accel_task *expected_accel_task = NULL; 480 int flags = 0; 481 482 TAILQ_INIT(&g_accel_ch->task_pool); 483 484 /* Fail with no tasks on _get_task() */ 485 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 486 NULL, cb_arg); 487 CU_ASSERT(rc == -ENOMEM); 488 489 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 490 491 /* accel submission OK. */ 492 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 493 NULL, cb_arg); 494 CU_ASSERT(rc == 0); 495 CU_ASSERT(task.crc_dst == &crc_dst); 496 CU_ASSERT(task.seed == seed); 497 CU_ASSERT(task.flags == 0); 498 CU_ASSERT(task.op_code == ACCEL_OPC_COPY_CRC32C); 499 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 500 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 501 CU_ASSERT(expected_accel_task == &task); 502 } 503 504 static void 505 test_spdk_accel_submit_xor(void) 506 { 507 const uint64_t nbytes = TEST_SUBMIT_SIZE; 508 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 509 uint8_t src1[TEST_SUBMIT_SIZE] = {0}; 510 uint8_t src2[TEST_SUBMIT_SIZE] = {0}; 511 void *sources[] = { src1, src2 }; 512 uint32_t nsrcs = SPDK_COUNTOF(sources); 513 int rc; 514 struct spdk_accel_task task; 515 struct spdk_accel_task *expected_accel_task = NULL; 516 517 TAILQ_INIT(&g_accel_ch->task_pool); 518 519 /* Fail with no tasks on _get_task() */ 520 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 521 CU_ASSERT(rc == -ENOMEM); 522 523 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 524 525 /* submission OK. */ 526 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 527 CU_ASSERT(rc == 0); 528 CU_ASSERT(task.nsrcs.srcs == sources); 529 CU_ASSERT(task.nsrcs.cnt == nsrcs); 530 CU_ASSERT(task.d.iovcnt == 1); 531 CU_ASSERT(task.d.iovs[0].iov_base == dst); 532 CU_ASSERT(task.d.iovs[0].iov_len == nbytes); 533 CU_ASSERT(task.op_code == ACCEL_OPC_XOR); 534 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 535 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 536 CU_ASSERT(expected_accel_task == &task); 537 } 538 539 static void 540 test_spdk_accel_module_find_by_name(void) 541 { 542 struct spdk_accel_module_if mod1 = {}; 543 struct spdk_accel_module_if mod2 = {}; 544 struct spdk_accel_module_if mod3 = {}; 545 struct spdk_accel_module_if *accel_module = NULL; 546 547 mod1.name = "ioat"; 548 mod2.name = "idxd"; 549 mod3.name = "software"; 550 551 TAILQ_INIT(&spdk_accel_module_list); 552 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod1, tailq); 553 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod2, tailq); 554 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod3, tailq); 555 556 /* Now let's find a valid engine */ 557 accel_module = _module_find_by_name("ioat"); 558 CU_ASSERT(accel_module != NULL); 559 560 /* Try to find one that doesn't exist */ 561 accel_module = _module_find_by_name("XXX"); 562 CU_ASSERT(accel_module == NULL); 563 } 564 565 static void 566 test_spdk_accel_module_register(void) 567 { 568 struct spdk_accel_module_if mod1 = {}; 569 struct spdk_accel_module_if mod2 = {}; 570 struct spdk_accel_module_if mod3 = {}; 571 struct spdk_accel_module_if mod4 = {}; 572 struct spdk_accel_module_if *accel_module = NULL; 573 int i = 0; 574 575 mod1.name = "ioat"; 576 mod2.name = "idxd"; 577 mod3.name = "software"; 578 mod4.name = "nothing"; 579 580 TAILQ_INIT(&spdk_accel_module_list); 581 582 spdk_accel_module_list_add(&mod1); 583 spdk_accel_module_list_add(&mod2); 584 spdk_accel_module_list_add(&mod3); 585 spdk_accel_module_list_add(&mod4); 586 587 /* Now confirm they're in the right order. */ 588 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 589 switch (i++) { 590 case 0: 591 CU_ASSERT(strcmp(accel_module->name, "software") == 0); 592 break; 593 case 1: 594 CU_ASSERT(strcmp(accel_module->name, "ioat") == 0); 595 break; 596 case 2: 597 CU_ASSERT(strcmp(accel_module->name, "idxd") == 0); 598 break; 599 case 3: 600 CU_ASSERT(strcmp(accel_module->name, "nothing") == 0); 601 break; 602 default: 603 CU_ASSERT(false); 604 break; 605 } 606 } 607 CU_ASSERT(i == 4); 608 } 609 610 struct ut_sequence { 611 bool complete; 612 int status; 613 }; 614 615 static void 616 ut_sequence_step_cb(void *cb_arg) 617 { 618 int *completed = cb_arg; 619 620 (*completed)++; 621 } 622 623 static void 624 ut_sequence_complete_cb(void *cb_arg, int status) 625 { 626 struct ut_sequence *seq = cb_arg; 627 628 seq->complete = true; 629 seq->status = status; 630 } 631 632 static void 633 test_sequence_fill_copy(void) 634 { 635 struct spdk_accel_sequence *seq = NULL; 636 struct spdk_io_channel *ioch; 637 struct ut_sequence ut_seq; 638 char buf[4096], tmp[2][4096], expected[4096]; 639 struct iovec src_iovs[2], dst_iovs[2]; 640 int rc, completed; 641 642 ioch = spdk_accel_get_io_channel(); 643 SPDK_CU_ASSERT_FATAL(ioch != NULL); 644 645 /* First check the simplest case - single task in a sequence */ 646 memset(buf, 0, sizeof(buf)); 647 memset(expected, 0xa5, sizeof(expected)); 648 completed = 0; 649 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 650 ut_sequence_step_cb, &completed); 651 CU_ASSERT_EQUAL(rc, 0); 652 CU_ASSERT_EQUAL(completed, 0); 653 654 ut_seq.complete = false; 655 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 656 CU_ASSERT_EQUAL(rc, 0); 657 658 poll_threads(); 659 CU_ASSERT_EQUAL(completed, 1); 660 CU_ASSERT(ut_seq.complete); 661 CU_ASSERT_EQUAL(ut_seq.status, 0); 662 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 663 664 /* Check a single copy operation */ 665 memset(buf, 0, sizeof(buf)); 666 memset(tmp[0], 0xa5, sizeof(tmp[0])); 667 memset(expected, 0xa5, sizeof(expected)); 668 completed = 0; 669 seq = NULL; 670 671 dst_iovs[0].iov_base = buf; 672 dst_iovs[0].iov_len = sizeof(buf); 673 src_iovs[0].iov_base = tmp[0]; 674 src_iovs[0].iov_len = sizeof(tmp[0]); 675 676 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 677 &src_iovs[0], 1, NULL, NULL, 0, 678 ut_sequence_step_cb, &completed); 679 CU_ASSERT_EQUAL(rc, 0); 680 681 ut_seq.complete = false; 682 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 683 CU_ASSERT_EQUAL(rc, 0); 684 685 poll_threads(); 686 CU_ASSERT_EQUAL(completed, 1); 687 CU_ASSERT(ut_seq.complete); 688 CU_ASSERT_EQUAL(ut_seq.status, 0); 689 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 690 691 /* Check multiple fill operations */ 692 memset(buf, 0, sizeof(buf)); 693 memset(expected, 0xfe, 4096); 694 memset(expected, 0xde, 2048); 695 memset(expected, 0xa5, 1024); 696 seq = NULL; 697 completed = 0; 698 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, NULL, NULL, 0xfe, 0, 699 ut_sequence_step_cb, &completed); 700 CU_ASSERT_EQUAL(rc, 0); 701 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xde, 0, 702 ut_sequence_step_cb, &completed); 703 CU_ASSERT_EQUAL(rc, 0); 704 rc = spdk_accel_append_fill(&seq, ioch, buf, 1024, NULL, NULL, 0xa5, 0, 705 ut_sequence_step_cb, &completed); 706 CU_ASSERT_EQUAL(rc, 0); 707 708 ut_seq.complete = false; 709 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 710 CU_ASSERT_EQUAL(rc, 0); 711 712 poll_threads(); 713 CU_ASSERT_EQUAL(completed, 3); 714 CU_ASSERT(ut_seq.complete); 715 CU_ASSERT_EQUAL(ut_seq.status, 0); 716 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 717 718 /* Check multiple copy operations */ 719 memset(buf, 0, sizeof(buf)); 720 memset(tmp[0], 0, sizeof(tmp[0])); 721 memset(tmp[1], 0, sizeof(tmp[1])); 722 memset(expected, 0xa5, sizeof(expected)); 723 seq = NULL; 724 completed = 0; 725 726 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 727 ut_sequence_step_cb, &completed); 728 CU_ASSERT_EQUAL(rc, 0); 729 730 dst_iovs[0].iov_base = tmp[1]; 731 dst_iovs[0].iov_len = sizeof(tmp[1]); 732 src_iovs[0].iov_base = tmp[0]; 733 src_iovs[0].iov_len = sizeof(tmp[0]); 734 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 735 &src_iovs[0], 1, NULL, NULL, 0, 736 ut_sequence_step_cb, &completed); 737 CU_ASSERT_EQUAL(rc, 0); 738 739 dst_iovs[1].iov_base = buf; 740 dst_iovs[1].iov_len = sizeof(buf); 741 src_iovs[1].iov_base = tmp[1]; 742 src_iovs[1].iov_len = sizeof(tmp[1]); 743 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 744 &src_iovs[1], 1, NULL, NULL, 0, 745 ut_sequence_step_cb, &completed); 746 CU_ASSERT_EQUAL(rc, 0); 747 748 ut_seq.complete = false; 749 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 750 CU_ASSERT_EQUAL(rc, 0); 751 752 poll_threads(); 753 CU_ASSERT_EQUAL(completed, 3); 754 CU_ASSERT(ut_seq.complete); 755 CU_ASSERT_EQUAL(ut_seq.status, 0); 756 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 757 758 /* Check that adding a copy operation at the end will change destination buffer */ 759 memset(buf, 0, sizeof(buf)); 760 memset(tmp[0], 0, sizeof(tmp[0])); 761 memset(expected, 0xa5, sizeof(buf)); 762 seq = NULL; 763 completed = 0; 764 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 765 ut_sequence_step_cb, &completed); 766 CU_ASSERT_EQUAL(rc, 0); 767 768 dst_iovs[0].iov_base = buf; 769 dst_iovs[0].iov_len = sizeof(buf); 770 src_iovs[0].iov_base = tmp[0]; 771 src_iovs[0].iov_len = sizeof(tmp[0]); 772 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 773 &src_iovs[0], 1, NULL, NULL, 0, 774 ut_sequence_step_cb, &completed); 775 CU_ASSERT_EQUAL(rc, 0); 776 777 ut_seq.complete = false; 778 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 779 CU_ASSERT_EQUAL(rc, 0); 780 781 poll_threads(); 782 CU_ASSERT_EQUAL(completed, 2); 783 CU_ASSERT(ut_seq.complete); 784 CU_ASSERT_EQUAL(ut_seq.status, 0); 785 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 786 787 /* Check that it's also possible to add copy operation at the beginning */ 788 memset(buf, 0, sizeof(buf)); 789 memset(tmp[0], 0xde, sizeof(tmp[0])); 790 memset(tmp[1], 0, sizeof(tmp[1])); 791 memset(expected, 0xa5, sizeof(expected)); 792 seq = NULL; 793 completed = 0; 794 795 dst_iovs[0].iov_base = tmp[1]; 796 dst_iovs[0].iov_len = sizeof(tmp[1]); 797 src_iovs[0].iov_base = tmp[0]; 798 src_iovs[0].iov_len = sizeof(tmp[0]); 799 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 800 &src_iovs[0], 1, NULL, NULL, 0, 801 ut_sequence_step_cb, &completed); 802 CU_ASSERT_EQUAL(rc, 0); 803 804 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], sizeof(tmp[1]), NULL, NULL, 0xa5, 0, 805 ut_sequence_step_cb, &completed); 806 CU_ASSERT_EQUAL(rc, 0); 807 808 dst_iovs[1].iov_base = buf; 809 dst_iovs[1].iov_len = sizeof(buf); 810 src_iovs[1].iov_base = tmp[1]; 811 src_iovs[1].iov_len = sizeof(tmp[1]); 812 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 813 &src_iovs[1], 1, NULL, NULL, 0, 814 ut_sequence_step_cb, &completed); 815 CU_ASSERT_EQUAL(rc, 0); 816 817 ut_seq.complete = false; 818 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 819 CU_ASSERT_EQUAL(rc, 0); 820 821 poll_threads(); 822 CU_ASSERT_EQUAL(completed, 3); 823 CU_ASSERT(ut_seq.complete); 824 CU_ASSERT_EQUAL(ut_seq.status, 0); 825 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 826 827 spdk_put_io_channel(ioch); 828 poll_threads(); 829 } 830 831 static void 832 test_sequence_abort(void) 833 { 834 struct spdk_accel_sequence *seq = NULL; 835 struct spdk_io_channel *ioch; 836 char buf[4096], tmp[2][4096], expected[4096]; 837 struct iovec src_iovs[2], dst_iovs[2]; 838 int rc, completed; 839 840 ioch = spdk_accel_get_io_channel(); 841 SPDK_CU_ASSERT_FATAL(ioch != NULL); 842 843 /* Check that aborting a sequence calls operation's callback, the operation is not executed 844 * and the sequence is freed 845 */ 846 memset(buf, 0, sizeof(buf)); 847 memset(expected, 0, sizeof(buf)); 848 completed = 0; 849 seq = NULL; 850 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 851 ut_sequence_step_cb, &completed); 852 CU_ASSERT_EQUAL(rc, 0); 853 854 spdk_accel_sequence_abort(seq); 855 CU_ASSERT_EQUAL(completed, 1); 856 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 857 858 /* Check sequence with multiple operations */ 859 memset(buf, 0, sizeof(buf)); 860 memset(expected, 0, sizeof(buf)); 861 completed = 0; 862 seq = NULL; 863 864 dst_iovs[0].iov_base = tmp[1]; 865 dst_iovs[0].iov_len = sizeof(tmp[1]); 866 src_iovs[0].iov_base = tmp[0]; 867 src_iovs[0].iov_len = sizeof(tmp[0]); 868 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 869 &src_iovs[0], 1, NULL, NULL, 0, 870 ut_sequence_step_cb, &completed); 871 CU_ASSERT_EQUAL(rc, 0); 872 873 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 4096, NULL, NULL, 0xa5, 0, 874 ut_sequence_step_cb, &completed); 875 CU_ASSERT_EQUAL(rc, 0); 876 877 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xde, 0, 878 ut_sequence_step_cb, &completed); 879 CU_ASSERT_EQUAL(rc, 0); 880 881 dst_iovs[1].iov_base = buf; 882 dst_iovs[1].iov_len = sizeof(buf); 883 src_iovs[1].iov_base = tmp[1]; 884 src_iovs[1].iov_len = sizeof(tmp[1]); 885 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 886 &src_iovs[1], 1, NULL, NULL, 0, 887 ut_sequence_step_cb, &completed); 888 CU_ASSERT_EQUAL(rc, 0); 889 890 spdk_accel_sequence_abort(seq); 891 CU_ASSERT_EQUAL(completed, 4); 892 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 893 894 /* This should be a no-op */ 895 spdk_accel_sequence_abort(NULL); 896 897 spdk_put_io_channel(ioch); 898 poll_threads(); 899 } 900 901 static void 902 test_sequence_append_error(void) 903 { 904 struct spdk_accel_sequence *seq = NULL; 905 struct spdk_io_channel *ioch; 906 struct accel_io_channel *accel_ch; 907 struct iovec src_iovs, dst_iovs; 908 char buf[4096]; 909 TAILQ_HEAD(, spdk_accel_task) tasks = TAILQ_HEAD_INITIALIZER(tasks); 910 TAILQ_HEAD(, spdk_accel_sequence) seqs = TAILQ_HEAD_INITIALIZER(seqs); 911 int rc; 912 913 ioch = spdk_accel_get_io_channel(); 914 SPDK_CU_ASSERT_FATAL(ioch != NULL); 915 accel_ch = spdk_io_channel_get_ctx(ioch); 916 917 /* Check that append fails and no sequence object is allocated when there are no more free 918 * tasks */ 919 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 920 921 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 922 ut_sequence_step_cb, NULL); 923 CU_ASSERT_EQUAL(rc, -ENOMEM); 924 CU_ASSERT_PTR_NULL(seq); 925 926 dst_iovs.iov_base = buf; 927 dst_iovs.iov_len = 2048; 928 src_iovs.iov_base = &buf[2048]; 929 src_iovs.iov_len = 2048; 930 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 931 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 932 CU_ASSERT_EQUAL(rc, -ENOMEM); 933 CU_ASSERT_PTR_NULL(seq); 934 935 dst_iovs.iov_base = buf; 936 dst_iovs.iov_len = 2048; 937 src_iovs.iov_base = &buf[2048]; 938 src_iovs.iov_len = 2048; 939 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 940 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 941 CU_ASSERT_EQUAL(rc, -ENOMEM); 942 CU_ASSERT_PTR_NULL(seq); 943 944 /* Check that the same happens when the sequence queue is empty */ 945 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 946 TAILQ_SWAP(&seqs, &accel_ch->seq_pool, spdk_accel_sequence, link); 947 948 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 949 ut_sequence_step_cb, NULL); 950 CU_ASSERT_EQUAL(rc, -ENOMEM); 951 CU_ASSERT_PTR_NULL(seq); 952 953 dst_iovs.iov_base = buf; 954 dst_iovs.iov_len = 2048; 955 src_iovs.iov_base = &buf[2048]; 956 src_iovs.iov_len = 2048; 957 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 958 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 959 CU_ASSERT_EQUAL(rc, -ENOMEM); 960 CU_ASSERT_PTR_NULL(seq); 961 962 dst_iovs.iov_base = buf; 963 dst_iovs.iov_len = 2048; 964 src_iovs.iov_base = &buf[2048]; 965 src_iovs.iov_len = 2048; 966 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 967 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 968 CU_ASSERT_EQUAL(rc, -ENOMEM); 969 CU_ASSERT_PTR_NULL(seq); 970 971 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 972 973 spdk_put_io_channel(ioch); 974 poll_threads(); 975 } 976 977 struct ut_sequence_operation { 978 int complete_status; 979 int submit_status; 980 int count; 981 struct iovec *src_iovs; 982 uint32_t src_iovcnt; 983 struct iovec *dst_iovs; 984 uint32_t dst_iovcnt; 985 int (*submit)(struct spdk_io_channel *ch, struct spdk_accel_task *t); 986 }; 987 988 static struct ut_sequence_operation g_seq_operations[ACCEL_OPC_LAST]; 989 990 static int 991 ut_sequnce_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *task) 992 { 993 struct ut_sequence_operation *op = &g_seq_operations[task->op_code]; 994 995 op->count++; 996 if (op->submit != NULL) { 997 return op->submit(ch, task); 998 } 999 if (op->src_iovs != NULL) { 1000 CU_ASSERT_EQUAL(task->s.iovcnt, op->src_iovcnt); 1001 CU_ASSERT_EQUAL(memcmp(task->s.iovs, op->src_iovs, 1002 sizeof(struct iovec) * op->src_iovcnt), 0); 1003 } 1004 if (op->dst_iovs != NULL) { 1005 CU_ASSERT_EQUAL(task->d.iovcnt, op->dst_iovcnt); 1006 CU_ASSERT_EQUAL(memcmp(task->d.iovs, op->dst_iovs, 1007 sizeof(struct iovec) * op->dst_iovcnt), 0); 1008 } 1009 1010 if (op->submit_status != 0) { 1011 return op->submit_status; 1012 } 1013 1014 spdk_accel_task_complete(task, op->complete_status); 1015 1016 return 0; 1017 } 1018 1019 static void 1020 ut_clear_operations(void) 1021 { 1022 memset(&g_seq_operations, 0, sizeof(g_seq_operations)); 1023 } 1024 1025 static void 1026 test_sequence_completion_error(void) 1027 { 1028 struct spdk_accel_sequence *seq = NULL; 1029 struct spdk_io_channel *ioch; 1030 struct ut_sequence ut_seq; 1031 struct iovec src_iovs, dst_iovs; 1032 char buf[4096], tmp[4096]; 1033 struct accel_module modules[ACCEL_OPC_LAST]; 1034 int i, rc, completed; 1035 1036 ioch = spdk_accel_get_io_channel(); 1037 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1038 1039 /* Override the submit_tasks function */ 1040 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1041 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1042 modules[i] = g_modules_opc[i]; 1043 g_modules_opc[i] = g_module; 1044 } 1045 1046 memset(buf, 0, sizeof(buf)); 1047 memset(tmp, 0, sizeof(tmp)); 1048 1049 /* Check that if the first operation completes with an error, the whole sequence is 1050 * completed with that error and that all operations' completion callbacks are executed 1051 */ 1052 g_seq_operations[ACCEL_OPC_FILL].complete_status = -E2BIG; 1053 completed = 0; 1054 seq = NULL; 1055 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1056 ut_sequence_step_cb, &completed); 1057 CU_ASSERT_EQUAL(rc, 0); 1058 1059 dst_iovs.iov_base = buf; 1060 dst_iovs.iov_len = sizeof(buf); 1061 src_iovs.iov_base = tmp; 1062 src_iovs.iov_len = sizeof(tmp); 1063 1064 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1065 &src_iovs, 1, NULL, NULL, 0, 1066 ut_sequence_step_cb, &completed); 1067 CU_ASSERT_EQUAL(rc, 0); 1068 1069 ut_seq.complete = false; 1070 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1071 CU_ASSERT_EQUAL(rc, 0); 1072 1073 poll_threads(); 1074 CU_ASSERT_EQUAL(completed, 2); 1075 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 1076 1077 /* Check the same with a second operation in the sequence */ 1078 g_seq_operations[ACCEL_OPC_DECOMPRESS].complete_status = -EACCES; 1079 g_seq_operations[ACCEL_OPC_FILL].complete_status = 0; 1080 completed = 0; 1081 seq = NULL; 1082 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1083 ut_sequence_step_cb, &completed); 1084 CU_ASSERT_EQUAL(rc, 0); 1085 1086 dst_iovs.iov_base = buf; 1087 dst_iovs.iov_len = sizeof(buf); 1088 src_iovs.iov_base = tmp; 1089 src_iovs.iov_len = sizeof(tmp); 1090 1091 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1092 &src_iovs, 1, NULL, NULL, 0, 1093 ut_sequence_step_cb, &completed); 1094 CU_ASSERT_EQUAL(rc, 0); 1095 1096 ut_seq.complete = false; 1097 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1098 CU_ASSERT_EQUAL(rc, 0); 1099 1100 poll_threads(); 1101 CU_ASSERT_EQUAL(completed, 2); 1102 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 1103 1104 g_seq_operations[ACCEL_OPC_DECOMPRESS].complete_status = 0; 1105 g_seq_operations[ACCEL_OPC_FILL].complete_status = 0; 1106 1107 /* Check submission failure of the first operation */ 1108 g_seq_operations[ACCEL_OPC_FILL].submit_status = -EADDRINUSE; 1109 completed = 0; 1110 seq = NULL; 1111 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1112 ut_sequence_step_cb, &completed); 1113 CU_ASSERT_EQUAL(rc, 0); 1114 1115 dst_iovs.iov_base = buf; 1116 dst_iovs.iov_len = sizeof(buf); 1117 src_iovs.iov_base = tmp; 1118 src_iovs.iov_len = sizeof(tmp); 1119 1120 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1121 &src_iovs, 1, NULL, NULL, 0, 1122 ut_sequence_step_cb, &completed); 1123 CU_ASSERT_EQUAL(rc, 0); 1124 1125 ut_seq.complete = false; 1126 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1127 CU_ASSERT_EQUAL(rc, 0); 1128 1129 poll_threads(); 1130 CU_ASSERT_EQUAL(completed, 2); 1131 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 1132 1133 /* Check the same with a second operation */ 1134 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit_status = -EADDRNOTAVAIL; 1135 g_seq_operations[ACCEL_OPC_FILL].submit_status = 0; 1136 completed = 0; 1137 seq = NULL; 1138 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1139 ut_sequence_step_cb, &completed); 1140 CU_ASSERT_EQUAL(rc, 0); 1141 1142 dst_iovs.iov_base = buf; 1143 dst_iovs.iov_len = sizeof(buf); 1144 src_iovs.iov_base = tmp; 1145 src_iovs.iov_len = sizeof(tmp); 1146 1147 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1148 &src_iovs, 1, NULL, NULL, 0, 1149 ut_sequence_step_cb, &completed); 1150 CU_ASSERT_EQUAL(rc, 0); 1151 1152 ut_seq.complete = false; 1153 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1154 CU_ASSERT_EQUAL(rc, 0); 1155 1156 poll_threads(); 1157 CU_ASSERT_EQUAL(completed, 2); 1158 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 1159 1160 /* Cleanup module pointers to make subsequent tests work correctly */ 1161 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1162 g_modules_opc[i] = modules[i]; 1163 } 1164 1165 ut_clear_operations(); 1166 spdk_put_io_channel(ioch); 1167 poll_threads(); 1168 } 1169 1170 #ifdef SPDK_CONFIG_ISAL 1171 static void 1172 ut_compress_cb(void *cb_arg, int status) 1173 { 1174 int *completed = cb_arg; 1175 1176 CU_ASSERT_EQUAL(status, 0); 1177 1178 *completed = 1; 1179 } 1180 1181 static void 1182 test_sequence_decompress(void) 1183 { 1184 struct spdk_accel_sequence *seq = NULL; 1185 struct spdk_io_channel *ioch; 1186 struct ut_sequence ut_seq; 1187 char buf[4096], tmp[2][4096], expected[4096]; 1188 struct iovec src_iovs[2], dst_iovs[2]; 1189 uint32_t compressed_size; 1190 int rc, completed = 0; 1191 1192 ioch = spdk_accel_get_io_channel(); 1193 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1194 1195 memset(expected, 0xa5, sizeof(expected)); 1196 src_iovs[0].iov_base = expected; 1197 src_iovs[0].iov_len = sizeof(expected); 1198 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1199 &compressed_size, 0, ut_compress_cb, &completed); 1200 CU_ASSERT_EQUAL(rc, 0); 1201 1202 while (!completed) { 1203 poll_threads(); 1204 } 1205 1206 /* Check a single decompress operation in a sequence */ 1207 seq = NULL; 1208 completed = 0; 1209 1210 dst_iovs[0].iov_base = buf; 1211 dst_iovs[0].iov_len = sizeof(buf); 1212 src_iovs[0].iov_base = tmp[0]; 1213 src_iovs[0].iov_len = compressed_size; 1214 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1215 &src_iovs[0], 1, NULL, NULL, 0, 1216 ut_sequence_step_cb, &completed); 1217 CU_ASSERT_EQUAL(rc, 0); 1218 1219 ut_seq.complete = false; 1220 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1221 CU_ASSERT_EQUAL(rc, 0); 1222 1223 poll_threads(); 1224 1225 CU_ASSERT_EQUAL(completed, 1); 1226 CU_ASSERT(ut_seq.complete); 1227 CU_ASSERT_EQUAL(ut_seq.status, 0); 1228 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1229 1230 /* Put the decompress operation in the middle of a sequence with a copy operation at the 1231 * beginning and a fill at the end modifying the first 2048B of the buffer. 1232 */ 1233 memset(expected, 0xfe, 2048); 1234 memset(buf, 0, sizeof(buf)); 1235 seq = NULL; 1236 completed = 0; 1237 1238 dst_iovs[0].iov_base = tmp[1]; 1239 dst_iovs[0].iov_len = compressed_size; 1240 src_iovs[0].iov_base = tmp[0]; 1241 src_iovs[0].iov_len = compressed_size; 1242 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1243 &src_iovs[0], 1, NULL, NULL, 0, 1244 ut_sequence_step_cb, &completed); 1245 CU_ASSERT_EQUAL(rc, 0); 1246 1247 dst_iovs[1].iov_base = buf; 1248 dst_iovs[1].iov_len = sizeof(buf); 1249 src_iovs[1].iov_base = tmp[1]; 1250 src_iovs[1].iov_len = compressed_size; 1251 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1252 &src_iovs[1], 1, NULL, NULL, 0, 1253 ut_sequence_step_cb, &completed); 1254 CU_ASSERT_EQUAL(rc, 0); 1255 1256 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1257 ut_sequence_step_cb, &completed); 1258 CU_ASSERT_EQUAL(rc, 0); 1259 1260 ut_seq.complete = false; 1261 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1262 CU_ASSERT_EQUAL(rc, 0); 1263 1264 poll_threads(); 1265 1266 CU_ASSERT_EQUAL(completed, 3); 1267 CU_ASSERT(ut_seq.complete); 1268 CU_ASSERT_EQUAL(ut_seq.status, 0); 1269 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1270 1271 /* Check sequence with decompress at the beginning: decompress -> copy */ 1272 memset(expected, 0xa5, sizeof(expected)); 1273 memset(buf, 0, sizeof(buf)); 1274 seq = NULL; 1275 completed = 0; 1276 1277 dst_iovs[0].iov_base = tmp[1]; 1278 dst_iovs[0].iov_len = sizeof(tmp[1]); 1279 src_iovs[0].iov_base = tmp[0]; 1280 src_iovs[0].iov_len = compressed_size; 1281 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1282 &src_iovs[0], 1, NULL, NULL, 0, 1283 ut_sequence_step_cb, &completed); 1284 CU_ASSERT_EQUAL(rc, 0); 1285 1286 dst_iovs[1].iov_base = buf; 1287 dst_iovs[1].iov_len = sizeof(buf); 1288 src_iovs[1].iov_base = tmp[1]; 1289 src_iovs[1].iov_len = sizeof(tmp[1]); 1290 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1291 &src_iovs[1], 1, NULL, NULL, 0, 1292 ut_sequence_step_cb, &completed); 1293 CU_ASSERT_EQUAL(rc, 0); 1294 1295 ut_seq.complete = false; 1296 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1297 CU_ASSERT_EQUAL(rc, 0); 1298 1299 poll_threads(); 1300 1301 CU_ASSERT_EQUAL(completed, 2); 1302 CU_ASSERT(ut_seq.complete); 1303 CU_ASSERT_EQUAL(ut_seq.status, 0); 1304 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1305 1306 spdk_put_io_channel(ioch); 1307 poll_threads(); 1308 } 1309 1310 static void 1311 test_sequence_reverse(void) 1312 { 1313 struct spdk_accel_sequence *seq = NULL; 1314 struct spdk_io_channel *ioch; 1315 struct ut_sequence ut_seq; 1316 char buf[4096], tmp[2][4096], expected[4096]; 1317 struct iovec src_iovs[2], dst_iovs[2]; 1318 uint32_t compressed_size; 1319 int rc, completed = 0; 1320 1321 ioch = spdk_accel_get_io_channel(); 1322 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1323 1324 memset(expected, 0xa5, sizeof(expected)); 1325 src_iovs[0].iov_base = expected; 1326 src_iovs[0].iov_len = sizeof(expected); 1327 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1328 &compressed_size, 0, ut_compress_cb, &completed); 1329 CU_ASSERT_EQUAL(rc, 0); 1330 1331 while (!completed) { 1332 poll_threads(); 1333 } 1334 1335 /* First check that reversing a sequnce with a single operation is a no-op */ 1336 memset(buf, 0, sizeof(buf)); 1337 seq = NULL; 1338 completed = 0; 1339 1340 dst_iovs[0].iov_base = buf; 1341 dst_iovs[0].iov_len = sizeof(buf); 1342 src_iovs[0].iov_base = tmp[0]; 1343 src_iovs[0].iov_len = compressed_size; 1344 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1345 &src_iovs[0], 1, NULL, NULL, 0, 1346 ut_sequence_step_cb, &completed); 1347 CU_ASSERT_EQUAL(rc, 0); 1348 1349 spdk_accel_sequence_reverse(seq); 1350 1351 ut_seq.complete = false; 1352 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1353 CU_ASSERT_EQUAL(rc, 0); 1354 1355 poll_threads(); 1356 1357 CU_ASSERT_EQUAL(completed, 1); 1358 CU_ASSERT(ut_seq.complete); 1359 CU_ASSERT_EQUAL(ut_seq.status, 0); 1360 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1361 1362 /* Add a copy operation at the end with src set to the compressed data. After reverse(), 1363 * that copy operation should be first, so decompress() should receive compressed data in 1364 * its src buffer. 1365 */ 1366 memset(buf, 0, sizeof(buf)); 1367 memset(tmp[1], 0, sizeof(tmp[1])); 1368 seq = NULL; 1369 completed = 0; 1370 1371 dst_iovs[0].iov_base = buf; 1372 dst_iovs[0].iov_len = sizeof(buf); 1373 src_iovs[0].iov_base = tmp[1]; 1374 src_iovs[0].iov_len = compressed_size; 1375 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1376 &src_iovs[0], 1, NULL, NULL, 0, 1377 ut_sequence_step_cb, &completed); 1378 CU_ASSERT_EQUAL(rc, 0); 1379 1380 dst_iovs[1].iov_base = tmp[1]; 1381 dst_iovs[1].iov_len = compressed_size; 1382 src_iovs[1].iov_base = tmp[0]; 1383 src_iovs[1].iov_len = compressed_size; 1384 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1385 &src_iovs[1], 1, NULL, NULL, 0, 1386 ut_sequence_step_cb, &completed); 1387 CU_ASSERT_EQUAL(rc, 0); 1388 1389 spdk_accel_sequence_reverse(seq); 1390 1391 ut_seq.complete = false; 1392 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1393 CU_ASSERT_EQUAL(rc, 0); 1394 1395 poll_threads(); 1396 1397 CU_ASSERT_EQUAL(completed, 2); 1398 CU_ASSERT(ut_seq.complete); 1399 CU_ASSERT_EQUAL(ut_seq.status, 0); 1400 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1401 1402 /* Check the same, but add an extra fill operation at the beginning that should execute last 1403 * after reverse(). 1404 */ 1405 memset(buf, 0, sizeof(buf)); 1406 memset(tmp[1], 0, sizeof(tmp[1])); 1407 memset(expected, 0xfe, 2048); 1408 seq = NULL; 1409 completed = 0; 1410 1411 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1412 ut_sequence_step_cb, &completed); 1413 CU_ASSERT_EQUAL(rc, 0); 1414 1415 dst_iovs[0].iov_base = buf; 1416 dst_iovs[0].iov_len = sizeof(buf); 1417 src_iovs[0].iov_base = tmp[1]; 1418 src_iovs[0].iov_len = compressed_size; 1419 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1420 &src_iovs[0], 1, NULL, NULL, 0, 1421 ut_sequence_step_cb, &completed); 1422 CU_ASSERT_EQUAL(rc, 0); 1423 1424 dst_iovs[1].iov_base = tmp[1]; 1425 dst_iovs[1].iov_len = compressed_size; 1426 src_iovs[1].iov_base = tmp[0]; 1427 src_iovs[1].iov_len = compressed_size; 1428 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1429 &src_iovs[1], 1, NULL, NULL, 0, 1430 ut_sequence_step_cb, &completed); 1431 CU_ASSERT_EQUAL(rc, 0); 1432 1433 spdk_accel_sequence_reverse(seq); 1434 1435 ut_seq.complete = false; 1436 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1437 CU_ASSERT_EQUAL(rc, 0); 1438 1439 poll_threads(); 1440 1441 CU_ASSERT_EQUAL(completed, 3); 1442 CU_ASSERT(ut_seq.complete); 1443 CU_ASSERT_EQUAL(ut_seq.status, 0); 1444 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1445 1446 /* Build the sequence in order and then reverse it twice */ 1447 memset(buf, 0, sizeof(buf)); 1448 memset(tmp[1], 0, sizeof(tmp[1])); 1449 seq = NULL; 1450 completed = 0; 1451 1452 dst_iovs[0].iov_base = tmp[1]; 1453 dst_iovs[0].iov_len = compressed_size; 1454 src_iovs[0].iov_base = tmp[0]; 1455 src_iovs[0].iov_len = compressed_size; 1456 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1457 &src_iovs[0], 1, NULL, NULL, 0, 1458 ut_sequence_step_cb, &completed); 1459 CU_ASSERT_EQUAL(rc, 0); 1460 1461 dst_iovs[1].iov_base = buf; 1462 dst_iovs[1].iov_len = sizeof(buf); 1463 src_iovs[1].iov_base = tmp[1]; 1464 src_iovs[1].iov_len = compressed_size; 1465 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1466 &src_iovs[1], 1, NULL, NULL, 0, 1467 ut_sequence_step_cb, &completed); 1468 CU_ASSERT_EQUAL(rc, 0); 1469 1470 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1471 ut_sequence_step_cb, &completed); 1472 CU_ASSERT_EQUAL(rc, 0); 1473 1474 spdk_accel_sequence_reverse(seq); 1475 spdk_accel_sequence_reverse(seq); 1476 1477 ut_seq.complete = false; 1478 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1479 CU_ASSERT_EQUAL(rc, 0); 1480 1481 poll_threads(); 1482 1483 CU_ASSERT_EQUAL(completed, 3); 1484 CU_ASSERT(ut_seq.complete); 1485 CU_ASSERT_EQUAL(ut_seq.status, 0); 1486 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1487 1488 spdk_put_io_channel(ioch); 1489 poll_threads(); 1490 } 1491 #endif 1492 1493 static void 1494 test_sequence_copy_elision(void) 1495 { 1496 struct spdk_accel_sequence *seq = NULL; 1497 struct spdk_io_channel *ioch; 1498 struct ut_sequence ut_seq; 1499 struct iovec src_iovs[4], dst_iovs[4], exp_iovs[2]; 1500 char buf[4096], tmp[4][4096]; 1501 struct accel_module modules[ACCEL_OPC_LAST]; 1502 struct spdk_accel_crypto_key key = {}; 1503 int i, rc, completed; 1504 1505 ioch = spdk_accel_get_io_channel(); 1506 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1507 1508 /* Override the submit_tasks function */ 1509 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1510 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1511 g_seq_operations[i].complete_status = 0; 1512 g_seq_operations[i].submit_status = 0; 1513 g_seq_operations[i].count = 0; 1514 1515 modules[i] = g_modules_opc[i]; 1516 g_modules_opc[i] = g_module; 1517 } 1518 1519 /* Check that a copy operation at the beginning is removed */ 1520 seq = NULL; 1521 completed = 0; 1522 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovcnt = 1; 1523 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovcnt = 1; 1524 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1525 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1526 exp_iovs[0].iov_base = tmp[0]; 1527 exp_iovs[0].iov_len = sizeof(tmp[0]); 1528 exp_iovs[1].iov_base = buf; 1529 exp_iovs[1].iov_len = 2048; 1530 1531 dst_iovs[0].iov_base = tmp[1]; 1532 dst_iovs[0].iov_len = sizeof(tmp[1]); 1533 src_iovs[0].iov_base = tmp[0]; 1534 src_iovs[0].iov_len = sizeof(tmp[0]); 1535 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1536 &src_iovs[0], 1, NULL, NULL, 0, 1537 ut_sequence_step_cb, &completed); 1538 CU_ASSERT_EQUAL(rc, 0); 1539 1540 dst_iovs[1].iov_base = buf; 1541 dst_iovs[1].iov_len = 2048; 1542 src_iovs[1].iov_base = tmp[1]; 1543 src_iovs[1].iov_len = sizeof(tmp[1]); 1544 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1545 &src_iovs[1], 1, NULL, NULL, 0, 1546 ut_sequence_step_cb, &completed); 1547 CU_ASSERT_EQUAL(rc, 0); 1548 1549 ut_seq.complete = false; 1550 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1551 CU_ASSERT_EQUAL(rc, 0); 1552 1553 poll_threads(); 1554 1555 CU_ASSERT_EQUAL(completed, 2); 1556 CU_ASSERT(ut_seq.complete); 1557 CU_ASSERT_EQUAL(ut_seq.status, 0); 1558 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1559 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 1560 1561 /* Check that a copy operation at the end is removed too */ 1562 seq = NULL; 1563 completed = 0; 1564 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1565 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 1566 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1567 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1568 exp_iovs[0].iov_base = tmp[0]; 1569 exp_iovs[0].iov_len = sizeof(tmp[0]); 1570 exp_iovs[1].iov_base = buf; 1571 exp_iovs[1].iov_len = 2048; 1572 1573 dst_iovs[0].iov_base = tmp[1]; 1574 dst_iovs[0].iov_len = 2048; 1575 src_iovs[0].iov_base = tmp[0]; 1576 src_iovs[0].iov_len = sizeof(tmp[0]); 1577 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1578 &src_iovs[0], 1, NULL, NULL, 0, 1579 ut_sequence_step_cb, &completed); 1580 CU_ASSERT_EQUAL(rc, 0); 1581 1582 dst_iovs[1].iov_base = buf; 1583 dst_iovs[1].iov_len = 2048; 1584 src_iovs[1].iov_base = tmp[1]; 1585 src_iovs[1].iov_len = 2048; 1586 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1587 &src_iovs[1], 1, NULL, NULL, 0, 1588 ut_sequence_step_cb, &completed); 1589 CU_ASSERT_EQUAL(rc, 0); 1590 1591 ut_seq.complete = false; 1592 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1593 CU_ASSERT_EQUAL(rc, 0); 1594 1595 poll_threads(); 1596 1597 CU_ASSERT_EQUAL(completed, 2); 1598 CU_ASSERT(ut_seq.complete); 1599 CU_ASSERT_EQUAL(ut_seq.status, 0); 1600 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1601 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 1602 1603 /* Check a copy operation both at the beginning and the end */ 1604 seq = NULL; 1605 completed = 0; 1606 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1607 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 1608 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1609 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1610 exp_iovs[0].iov_base = tmp[0]; 1611 exp_iovs[0].iov_len = sizeof(tmp[0]); 1612 exp_iovs[1].iov_base = buf; 1613 exp_iovs[1].iov_len = 2048; 1614 1615 dst_iovs[0].iov_base = tmp[1]; 1616 dst_iovs[0].iov_len = sizeof(tmp[1]); 1617 src_iovs[0].iov_base = tmp[0]; 1618 src_iovs[0].iov_len = sizeof(tmp[0]); 1619 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1620 &src_iovs[0], 1, NULL, NULL, 0, 1621 ut_sequence_step_cb, &completed); 1622 CU_ASSERT_EQUAL(rc, 0); 1623 1624 dst_iovs[1].iov_base = tmp[2]; 1625 dst_iovs[1].iov_len = 2048; 1626 src_iovs[1].iov_base = tmp[1]; 1627 src_iovs[1].iov_len = sizeof(tmp[1]); 1628 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1629 &src_iovs[1], 1, NULL, NULL, 0, 1630 ut_sequence_step_cb, &completed); 1631 CU_ASSERT_EQUAL(rc, 0); 1632 1633 dst_iovs[2].iov_base = buf; 1634 dst_iovs[2].iov_len = 2048; 1635 src_iovs[2].iov_base = tmp[2]; 1636 src_iovs[2].iov_len = 2048; 1637 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1638 &src_iovs[2], 1, NULL, NULL, 0, 1639 ut_sequence_step_cb, &completed); 1640 CU_ASSERT_EQUAL(rc, 0); 1641 1642 ut_seq.complete = false; 1643 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1644 CU_ASSERT_EQUAL(rc, 0); 1645 1646 poll_threads(); 1647 1648 CU_ASSERT_EQUAL(completed, 3); 1649 CU_ASSERT(ut_seq.complete); 1650 CU_ASSERT_EQUAL(ut_seq.status, 0); 1651 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1652 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 1653 1654 /* Check decompress + copy + decompress + copy */ 1655 seq = NULL; 1656 completed = 0; 1657 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1658 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 1659 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = NULL; 1660 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = NULL; 1661 1662 dst_iovs[0].iov_base = tmp[1]; 1663 dst_iovs[0].iov_len = sizeof(tmp[1]); 1664 src_iovs[0].iov_base = tmp[0]; 1665 src_iovs[0].iov_len = sizeof(tmp[0]); 1666 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1667 &src_iovs[0], 1, NULL, NULL, 0, 1668 ut_sequence_step_cb, &completed); 1669 CU_ASSERT_EQUAL(rc, 0); 1670 1671 dst_iovs[1].iov_base = tmp[2]; 1672 dst_iovs[1].iov_len = 2048; 1673 src_iovs[1].iov_base = tmp[1]; 1674 src_iovs[1].iov_len = sizeof(tmp[1]); 1675 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1676 &src_iovs[1], 1, NULL, NULL, 0, 1677 ut_sequence_step_cb, &completed); 1678 CU_ASSERT_EQUAL(rc, 0); 1679 1680 dst_iovs[2].iov_base = tmp[3]; 1681 dst_iovs[2].iov_len = 1024; 1682 src_iovs[2].iov_base = tmp[2]; 1683 src_iovs[2].iov_len = 2048; 1684 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1685 &src_iovs[2], 1, NULL, NULL, 0, 1686 ut_sequence_step_cb, &completed); 1687 CU_ASSERT_EQUAL(rc, 0); 1688 1689 dst_iovs[3].iov_base = buf; 1690 dst_iovs[3].iov_len = 1024; 1691 src_iovs[3].iov_base = tmp[3]; 1692 src_iovs[3].iov_len = 1024; 1693 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 1694 &src_iovs[3], 1, NULL, NULL, 0, 1695 ut_sequence_step_cb, &completed); 1696 CU_ASSERT_EQUAL(rc, 0); 1697 1698 ut_seq.complete = false; 1699 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1700 CU_ASSERT_EQUAL(rc, 0); 1701 1702 poll_threads(); 1703 1704 CU_ASSERT_EQUAL(completed, 4); 1705 CU_ASSERT(ut_seq.complete); 1706 CU_ASSERT_EQUAL(ut_seq.status, 0); 1707 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1708 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 2); 1709 1710 /* Check two copy operations - one of them should be removed, while the other should be 1711 * executed normally */ 1712 seq = NULL; 1713 completed = 0; 1714 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1715 g_seq_operations[ACCEL_OPC_COPY].dst_iovcnt = 1; 1716 g_seq_operations[ACCEL_OPC_COPY].src_iovcnt = 1; 1717 g_seq_operations[ACCEL_OPC_COPY].src_iovs = &exp_iovs[0]; 1718 g_seq_operations[ACCEL_OPC_COPY].dst_iovs = &exp_iovs[1]; 1719 exp_iovs[0].iov_base = tmp[0]; 1720 exp_iovs[0].iov_len = sizeof(tmp[0]); 1721 exp_iovs[1].iov_base = buf; 1722 exp_iovs[1].iov_len = sizeof(buf); 1723 1724 dst_iovs[0].iov_base = tmp[1]; 1725 dst_iovs[0].iov_len = sizeof(tmp[1]); 1726 src_iovs[0].iov_base = tmp[0]; 1727 src_iovs[0].iov_len = sizeof(tmp[0]); 1728 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1729 &src_iovs[0], 1, NULL, NULL, 0, 1730 ut_sequence_step_cb, &completed); 1731 CU_ASSERT_EQUAL(rc, 0); 1732 1733 dst_iovs[1].iov_base = buf; 1734 dst_iovs[1].iov_len = sizeof(buf); 1735 src_iovs[1].iov_base = tmp[1]; 1736 src_iovs[1].iov_len = sizeof(tmp[1]); 1737 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1738 &src_iovs[1], 1, NULL, NULL, 0, 1739 ut_sequence_step_cb, &completed); 1740 CU_ASSERT_EQUAL(rc, 0); 1741 1742 ut_seq.complete = false; 1743 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1744 CU_ASSERT_EQUAL(rc, 0); 1745 1746 poll_threads(); 1747 1748 CU_ASSERT_EQUAL(completed, 2); 1749 CU_ASSERT(ut_seq.complete); 1750 CU_ASSERT_EQUAL(ut_seq.status, 0); 1751 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 1); 1752 1753 /* Check fill + copy */ 1754 seq = NULL; 1755 completed = 0; 1756 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1757 g_seq_operations[ACCEL_OPC_FILL].count = 0; 1758 g_seq_operations[ACCEL_OPC_COPY].src_iovs = NULL; 1759 g_seq_operations[ACCEL_OPC_COPY].dst_iovs = NULL; 1760 g_seq_operations[ACCEL_OPC_FILL].dst_iovcnt = 1; 1761 g_seq_operations[ACCEL_OPC_FILL].dst_iovs = &exp_iovs[0]; 1762 exp_iovs[0].iov_base = buf; 1763 exp_iovs[0].iov_len = sizeof(buf); 1764 1765 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 1766 ut_sequence_step_cb, &completed); 1767 CU_ASSERT_EQUAL(rc, 0); 1768 1769 dst_iovs[0].iov_base = buf; 1770 dst_iovs[0].iov_len = sizeof(buf); 1771 src_iovs[0].iov_base = tmp[0]; 1772 src_iovs[0].iov_len = sizeof(tmp[0]); 1773 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1774 &src_iovs[0], 1, NULL, NULL, 0, 1775 ut_sequence_step_cb, &completed); 1776 CU_ASSERT_EQUAL(rc, 0); 1777 1778 ut_seq.complete = false; 1779 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1780 CU_ASSERT_EQUAL(rc, 0); 1781 1782 poll_threads(); 1783 1784 CU_ASSERT_EQUAL(completed, 2); 1785 CU_ASSERT(ut_seq.complete); 1786 CU_ASSERT_EQUAL(ut_seq.status, 0); 1787 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1788 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 1); 1789 1790 /* Check copy + encrypt + copy */ 1791 seq = NULL; 1792 completed = 0; 1793 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1794 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 1795 g_seq_operations[ACCEL_OPC_ENCRYPT].dst_iovcnt = 1; 1796 g_seq_operations[ACCEL_OPC_ENCRYPT].src_iovcnt = 1; 1797 g_seq_operations[ACCEL_OPC_ENCRYPT].src_iovs = &exp_iovs[0]; 1798 g_seq_operations[ACCEL_OPC_ENCRYPT].dst_iovs = &exp_iovs[1]; 1799 exp_iovs[0].iov_base = tmp[0]; 1800 exp_iovs[0].iov_len = sizeof(tmp[0]); 1801 exp_iovs[1].iov_base = buf; 1802 exp_iovs[1].iov_len = sizeof(buf); 1803 1804 dst_iovs[0].iov_base = tmp[1]; 1805 dst_iovs[0].iov_len = sizeof(tmp[1]); 1806 src_iovs[0].iov_base = tmp[0]; 1807 src_iovs[0].iov_len = sizeof(tmp[0]); 1808 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1809 &src_iovs[0], 1, NULL, NULL, 0, 1810 ut_sequence_step_cb, &completed); 1811 CU_ASSERT_EQUAL(rc, 0); 1812 1813 dst_iovs[1].iov_base = tmp[2]; 1814 dst_iovs[1].iov_len = sizeof(tmp[2]); 1815 src_iovs[1].iov_base = tmp[1]; 1816 src_iovs[1].iov_len = sizeof(tmp[1]); 1817 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1818 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1819 ut_sequence_step_cb, &completed); 1820 CU_ASSERT_EQUAL(rc, 0); 1821 1822 dst_iovs[2].iov_base = buf; 1823 dst_iovs[2].iov_len = sizeof(buf); 1824 src_iovs[2].iov_base = tmp[2]; 1825 src_iovs[2].iov_len = sizeof(tmp[2]); 1826 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1827 &src_iovs[2], 1, NULL, NULL, 0, 1828 ut_sequence_step_cb, &completed); 1829 CU_ASSERT_EQUAL(rc, 0); 1830 1831 ut_seq.complete = false; 1832 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1833 CU_ASSERT_EQUAL(rc, 0); 1834 1835 poll_threads(); 1836 1837 CU_ASSERT_EQUAL(completed, 3); 1838 CU_ASSERT(ut_seq.complete); 1839 CU_ASSERT_EQUAL(ut_seq.status, 0); 1840 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1841 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 1842 1843 /* Check copy + decrypt + copy */ 1844 seq = NULL; 1845 completed = 0; 1846 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1847 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 1848 g_seq_operations[ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1849 g_seq_operations[ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1850 g_seq_operations[ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1851 g_seq_operations[ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1852 exp_iovs[0].iov_base = tmp[0]; 1853 exp_iovs[0].iov_len = sizeof(tmp[0]); 1854 exp_iovs[1].iov_base = buf; 1855 exp_iovs[1].iov_len = sizeof(buf); 1856 1857 dst_iovs[0].iov_base = tmp[1]; 1858 dst_iovs[0].iov_len = sizeof(tmp[1]); 1859 src_iovs[0].iov_base = tmp[0]; 1860 src_iovs[0].iov_len = sizeof(tmp[0]); 1861 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1862 &src_iovs[0], 1, NULL, NULL, 0, 1863 ut_sequence_step_cb, &completed); 1864 CU_ASSERT_EQUAL(rc, 0); 1865 1866 dst_iovs[1].iov_base = tmp[2]; 1867 dst_iovs[1].iov_len = sizeof(tmp[2]); 1868 src_iovs[1].iov_base = tmp[1]; 1869 src_iovs[1].iov_len = sizeof(tmp[1]); 1870 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1871 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1872 ut_sequence_step_cb, &completed); 1873 CU_ASSERT_EQUAL(rc, 0); 1874 1875 dst_iovs[2].iov_base = buf; 1876 dst_iovs[2].iov_len = sizeof(buf); 1877 src_iovs[2].iov_base = tmp[2]; 1878 src_iovs[2].iov_len = sizeof(tmp[2]); 1879 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1880 &src_iovs[2], 1, NULL, NULL, 0, 1881 ut_sequence_step_cb, &completed); 1882 CU_ASSERT_EQUAL(rc, 0); 1883 1884 ut_seq.complete = false; 1885 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1886 CU_ASSERT_EQUAL(rc, 0); 1887 1888 poll_threads(); 1889 1890 CU_ASSERT_EQUAL(completed, 3); 1891 CU_ASSERT(ut_seq.complete); 1892 CU_ASSERT_EQUAL(ut_seq.status, 0); 1893 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1894 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 1895 1896 /* Cleanup module pointers to make subsequent tests work correctly */ 1897 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1898 g_modules_opc[i] = modules[i]; 1899 } 1900 1901 ut_clear_operations(); 1902 spdk_put_io_channel(ioch); 1903 poll_threads(); 1904 } 1905 1906 static int 1907 ut_submit_decompress(struct spdk_io_channel *ch, struct spdk_accel_task *task) 1908 { 1909 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 1910 1911 spdk_accel_task_complete(task, 0); 1912 1913 return 0; 1914 } 1915 1916 static void 1917 test_sequence_accel_buffers(void) 1918 { 1919 struct spdk_accel_sequence *seq = NULL; 1920 struct spdk_io_channel *ioch; 1921 struct accel_io_channel *accel_ch; 1922 struct ut_sequence ut_seq; 1923 struct iovec src_iovs[3], dst_iovs[3]; 1924 char srcbuf[4096], dstbuf[4096], expected[4096]; 1925 struct accel_module modules[ACCEL_OPC_LAST]; 1926 void *buf[2], *domain_ctx[2], *iobuf_buf; 1927 struct spdk_memory_domain *domain[2]; 1928 struct spdk_iobuf_buffer *cache_entry; 1929 spdk_iobuf_buffer_stailq_t small_cache; 1930 uint32_t small_cache_count; 1931 int i, rc, completed; 1932 1933 ioch = spdk_accel_get_io_channel(); 1934 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1935 1936 /* Override the submit_tasks function */ 1937 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1938 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1939 modules[i] = g_modules_opc[i]; 1940 g_modules_opc[i] = g_module; 1941 } 1942 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 1943 * decompress operations together in one sequence. 1944 */ 1945 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 1946 g_seq_operations[ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 1947 g_seq_operations[ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 1948 1949 /* Check the simplest case: one operation using accel buffer as destination + copy operation 1950 * specifying the actual destination buffer 1951 */ 1952 memset(srcbuf, 0xa5, 4096); 1953 memset(dstbuf, 0x0, 4096); 1954 memset(expected, 0xa5, 4096); 1955 completed = 0; 1956 seq = NULL; 1957 1958 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 1959 CU_ASSERT_EQUAL(rc, 0); 1960 CU_ASSERT_PTR_NOT_NULL(buf[0]); 1961 1962 src_iovs[0].iov_base = srcbuf; 1963 src_iovs[0].iov_len = 4096; 1964 dst_iovs[0].iov_base = buf[0]; 1965 dst_iovs[0].iov_len = 4096; 1966 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 1967 &src_iovs[0], 1, NULL, NULL, 0, 1968 ut_sequence_step_cb, &completed); 1969 CU_ASSERT_EQUAL(rc, 0); 1970 1971 src_iovs[1].iov_base = buf[0]; 1972 src_iovs[1].iov_len = 4096; 1973 dst_iovs[1].iov_base = dstbuf; 1974 dst_iovs[1].iov_len = 4096; 1975 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1976 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 1977 ut_sequence_step_cb, &completed); 1978 CU_ASSERT_EQUAL(rc, 0); 1979 1980 ut_seq.complete = false; 1981 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1982 CU_ASSERT_EQUAL(rc, 0); 1983 1984 poll_threads(); 1985 1986 CU_ASSERT_EQUAL(completed, 2); 1987 CU_ASSERT(ut_seq.complete); 1988 CU_ASSERT_EQUAL(ut_seq.status, 0); 1989 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 1990 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 1991 1992 /* Start with a fill operation using accel buffer, followed by a decompress using another 1993 * accel buffer as dst, followed by a copy operation specifying dst buffer of the whole 1994 * sequence 1995 */ 1996 memset(srcbuf, 0x0, 4096); 1997 memset(dstbuf, 0x0, 4096); 1998 memset(expected, 0x5a, 4096); 1999 completed = 0; 2000 seq = NULL; 2001 2002 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2003 CU_ASSERT_EQUAL(rc, 0); 2004 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2005 2006 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2007 ut_sequence_step_cb, &completed); 2008 CU_ASSERT_EQUAL(rc, 0); 2009 2010 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2011 CU_ASSERT_EQUAL(rc, 0); 2012 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2013 2014 src_iovs[0].iov_base = buf[0]; 2015 src_iovs[0].iov_len = 4096; 2016 dst_iovs[0].iov_base = buf[1]; 2017 dst_iovs[0].iov_len = 4096; 2018 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2019 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2020 ut_sequence_step_cb, &completed); 2021 CU_ASSERT_EQUAL(rc, 0); 2022 2023 src_iovs[1].iov_base = buf[1]; 2024 src_iovs[1].iov_len = 4096; 2025 dst_iovs[1].iov_base = dstbuf; 2026 dst_iovs[1].iov_len = 4096; 2027 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2028 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2029 ut_sequence_step_cb, &completed); 2030 2031 ut_seq.complete = false; 2032 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2033 CU_ASSERT_EQUAL(rc, 0); 2034 2035 poll_threads(); 2036 2037 CU_ASSERT_EQUAL(completed, 3); 2038 CU_ASSERT(ut_seq.complete); 2039 CU_ASSERT_EQUAL(ut_seq.status, 0); 2040 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2041 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2042 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2043 2044 /* Check the same, but with two decompress operations with the first one being in-place */ 2045 memset(srcbuf, 0x0, 4096); 2046 memset(dstbuf, 0x0, 4096); 2047 memset(expected, 0x5a, 4096); 2048 completed = 0; 2049 seq = NULL; 2050 2051 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2052 CU_ASSERT_EQUAL(rc, 0); 2053 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2054 2055 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2056 ut_sequence_step_cb, &completed); 2057 CU_ASSERT_EQUAL(rc, 0); 2058 2059 src_iovs[0].iov_base = buf[0]; 2060 src_iovs[0].iov_len = 4096; 2061 dst_iovs[0].iov_base = buf[0]; 2062 dst_iovs[0].iov_len = 4096; 2063 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2064 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2065 ut_sequence_step_cb, &completed); 2066 CU_ASSERT_EQUAL(rc, 0); 2067 2068 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2069 CU_ASSERT_EQUAL(rc, 0); 2070 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2071 2072 src_iovs[1].iov_base = buf[0]; 2073 src_iovs[1].iov_len = 4096; 2074 dst_iovs[1].iov_base = buf[1]; 2075 dst_iovs[1].iov_len = 4096; 2076 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, domain[1], domain_ctx[1], 2077 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 2078 ut_sequence_step_cb, &completed); 2079 CU_ASSERT_EQUAL(rc, 0); 2080 2081 src_iovs[2].iov_base = buf[1]; 2082 src_iovs[2].iov_len = 4096; 2083 dst_iovs[2].iov_base = dstbuf; 2084 dst_iovs[2].iov_len = 4096; 2085 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 2086 &src_iovs[2], 1, domain[1], domain_ctx[1], 0, 2087 ut_sequence_step_cb, &completed); 2088 2089 ut_seq.complete = false; 2090 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2091 CU_ASSERT_EQUAL(rc, 0); 2092 2093 poll_threads(); 2094 2095 CU_ASSERT_EQUAL(completed, 4); 2096 CU_ASSERT(ut_seq.complete); 2097 CU_ASSERT_EQUAL(ut_seq.status, 0); 2098 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2099 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2100 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2101 2102 /* Check that specifying offsets within accel buffers works correctly */ 2103 memset(srcbuf, 0x0, 4096); 2104 memset(dstbuf, 0x0, 4096); 2105 completed = 0; 2106 seq = NULL; 2107 2108 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2109 CU_ASSERT_EQUAL(rc, 0); 2110 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2111 2112 /* Fill in each 1K of the buffer with different pattern */ 2113 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2114 ut_sequence_step_cb, &completed); 2115 CU_ASSERT_EQUAL(rc, 0); 2116 2117 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2118 0x5a, 0, ut_sequence_step_cb, &completed); 2119 CU_ASSERT_EQUAL(rc, 0); 2120 2121 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2122 0xfe, 0, ut_sequence_step_cb, &completed); 2123 CU_ASSERT_EQUAL(rc, 0); 2124 2125 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2126 0xed, 0, ut_sequence_step_cb, &completed); 2127 CU_ASSERT_EQUAL(rc, 0); 2128 2129 src_iovs[0].iov_base = buf[0]; 2130 src_iovs[0].iov_len = 4096; 2131 dst_iovs[0].iov_base = dstbuf; 2132 dst_iovs[0].iov_len = 4096; 2133 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2134 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2135 ut_sequence_step_cb, &completed); 2136 2137 ut_seq.complete = false; 2138 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2139 CU_ASSERT_EQUAL(rc, 0); 2140 2141 poll_threads(); 2142 2143 memset(&expected[0], 0xa5, 1024); 2144 memset(&expected[1024], 0x5a, 1024); 2145 memset(&expected[2048], 0xfe, 1024); 2146 memset(&expected[3072], 0xed, 1024); 2147 CU_ASSERT_EQUAL(completed, 5); 2148 CU_ASSERT(ut_seq.complete); 2149 CU_ASSERT_EQUAL(ut_seq.status, 0); 2150 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2151 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2152 2153 /* Check the same but this time with a decompress operation on part of the buffer (512B 2154 * offset) */ 2155 memset(srcbuf, 0x0, 4096); 2156 memset(dstbuf, 0x0, 4096); 2157 completed = 0; 2158 seq = NULL; 2159 2160 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2161 CU_ASSERT_EQUAL(rc, 0); 2162 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2163 2164 /* Fill in each 1K of the buffer with different pattern */ 2165 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2166 ut_sequence_step_cb, &completed); 2167 CU_ASSERT_EQUAL(rc, 0); 2168 2169 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2170 0x5a, 0, ut_sequence_step_cb, &completed); 2171 CU_ASSERT_EQUAL(rc, 0); 2172 2173 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2174 0xfe, 0, ut_sequence_step_cb, &completed); 2175 CU_ASSERT_EQUAL(rc, 0); 2176 2177 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2178 0xed, 0, ut_sequence_step_cb, &completed); 2179 CU_ASSERT_EQUAL(rc, 0); 2180 2181 rc = spdk_accel_get_buf(ioch, 3072, &buf[1], &domain[1], &domain_ctx[1]); 2182 CU_ASSERT_EQUAL(rc, 0); 2183 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2184 2185 src_iovs[0].iov_base = (char *)buf[0] + 512; 2186 src_iovs[0].iov_len = 3072; 2187 dst_iovs[0].iov_base = (char *)buf[1] + 256; 2188 dst_iovs[0].iov_len = 3072; 2189 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2190 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2191 ut_sequence_step_cb, &completed); 2192 2193 src_iovs[1].iov_base = (char *)buf[1] + 256; 2194 src_iovs[1].iov_len = 3072; 2195 dst_iovs[1].iov_base = dstbuf; 2196 dst_iovs[1].iov_len = 3072; 2197 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2198 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2199 ut_sequence_step_cb, &completed); 2200 2201 ut_seq.complete = false; 2202 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2203 CU_ASSERT_EQUAL(rc, 0); 2204 2205 poll_threads(); 2206 2207 memset(&expected[0], 0xa5, 512); 2208 memset(&expected[512], 0x5a, 1024); 2209 memset(&expected[1536], 0xfe, 1024); 2210 memset(&expected[2560], 0xed, 512); 2211 CU_ASSERT_EQUAL(completed, 6); 2212 CU_ASSERT(ut_seq.complete); 2213 CU_ASSERT_EQUAL(ut_seq.status, 0); 2214 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 3072), 0); 2215 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2216 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2217 2218 /* Check that if iobuf pool is empty, the sequence processing will wait until a buffer is 2219 * available 2220 */ 2221 accel_ch = spdk_io_channel_get_ctx(ioch); 2222 small_cache_count = accel_ch->iobuf.small.cache_count; 2223 STAILQ_INIT(&small_cache); 2224 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2225 accel_ch->iobuf.small.cache_count = 0; 2226 MOCK_SET(spdk_mempool_get, NULL); 2227 2228 /* First allocate a single buffer used by two operations */ 2229 memset(srcbuf, 0x0, 4096); 2230 memset(dstbuf, 0x0, 4096); 2231 memset(expected, 0xa5, 4096); 2232 completed = 0; 2233 seq = NULL; 2234 2235 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2236 CU_ASSERT_EQUAL(rc, 0); 2237 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2238 2239 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0xa5, 0, 2240 ut_sequence_step_cb, &completed); 2241 CU_ASSERT_EQUAL(rc, 0); 2242 2243 src_iovs[0].iov_base = buf[0]; 2244 src_iovs[0].iov_len = 4096; 2245 dst_iovs[0].iov_base = dstbuf; 2246 dst_iovs[0].iov_len = 4096; 2247 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2248 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2249 ut_sequence_step_cb, &completed); 2250 2251 ut_seq.complete = false; 2252 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2253 CU_ASSERT_EQUAL(rc, 0); 2254 2255 poll_threads(); 2256 2257 CU_ASSERT_EQUAL(completed, 0); 2258 CU_ASSERT(!ut_seq.complete); 2259 2260 /* Get a buffer and return it to the pool to trigger the sequence to finish */ 2261 MOCK_CLEAR(spdk_mempool_get); 2262 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2263 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2264 MOCK_SET(spdk_mempool_get, NULL); 2265 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2266 2267 poll_threads(); 2268 2269 CU_ASSERT_EQUAL(completed, 2); 2270 CU_ASSERT(ut_seq.complete); 2271 CU_ASSERT_EQUAL(ut_seq.status, 0); 2272 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2273 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2274 2275 /* Return the buffers back to the cache */ 2276 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2277 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2278 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2279 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2280 small_cache_count++; 2281 } 2282 accel_ch->iobuf.small.cache_count = 0; 2283 2284 /* Check a bit more complex scenario, with two buffers in the sequence */ 2285 memset(srcbuf, 0x0, 4096); 2286 memset(dstbuf, 0x0, 4096); 2287 memset(expected, 0x5a, 4096); 2288 completed = 0; 2289 seq = NULL; 2290 2291 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2292 CU_ASSERT_EQUAL(rc, 0); 2293 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2294 2295 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2296 ut_sequence_step_cb, &completed); 2297 CU_ASSERT_EQUAL(rc, 0); 2298 2299 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2300 CU_ASSERT_EQUAL(rc, 0); 2301 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2302 2303 src_iovs[0].iov_base = buf[0]; 2304 src_iovs[0].iov_len = 4096; 2305 dst_iovs[0].iov_base = buf[1]; 2306 dst_iovs[0].iov_len = 4096; 2307 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2308 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2309 ut_sequence_step_cb, &completed); 2310 CU_ASSERT_EQUAL(rc, 0); 2311 2312 src_iovs[1].iov_base = buf[1]; 2313 src_iovs[1].iov_len = 4096; 2314 dst_iovs[1].iov_base = dstbuf; 2315 dst_iovs[1].iov_len = 4096; 2316 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2317 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2318 ut_sequence_step_cb, &completed); 2319 CU_ASSERT_EQUAL(rc, 0); 2320 2321 ut_seq.complete = false; 2322 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2323 CU_ASSERT_EQUAL(rc, 0); 2324 2325 poll_threads(); 2326 2327 CU_ASSERT_EQUAL(completed, 0); 2328 CU_ASSERT(!ut_seq.complete); 2329 2330 MOCK_CLEAR(spdk_mempool_get); 2331 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2332 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2333 MOCK_SET(spdk_mempool_get, NULL); 2334 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2335 2336 /* One buffer is not enough to finish the whole sequence */ 2337 poll_threads(); 2338 CU_ASSERT(!ut_seq.complete); 2339 2340 MOCK_CLEAR(spdk_mempool_get); 2341 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2342 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2343 MOCK_SET(spdk_mempool_get, NULL); 2344 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2345 2346 poll_threads(); 2347 2348 CU_ASSERT_EQUAL(completed, 3); 2349 CU_ASSERT(ut_seq.complete); 2350 CU_ASSERT_EQUAL(ut_seq.status, 0); 2351 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2352 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2353 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2354 2355 /* Return the buffers back to the cache */ 2356 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2357 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2358 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2359 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2360 small_cache_count++; 2361 } 2362 accel_ch->iobuf.small.cache_count = 0; 2363 2364 MOCK_CLEAR(spdk_mempool_get); 2365 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2366 accel_ch->iobuf.small.cache_count = small_cache_count; 2367 2368 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2369 g_modules_opc[i] = modules[i]; 2370 } 2371 2372 ut_clear_operations(); 2373 spdk_put_io_channel(ioch); 2374 poll_threads(); 2375 } 2376 2377 static void 2378 ut_domain_ctx_init(struct ut_domain_ctx *ctx, void *base, size_t len, struct iovec *expected) 2379 { 2380 ctx->iov.iov_base = base; 2381 ctx->iov.iov_len = len; 2382 ctx->expected = *expected; 2383 ctx->pull_submit_status = 0; 2384 ctx->push_submit_status = 0; 2385 ctx->pull_complete_status = 0; 2386 ctx->push_complete_status = 0; 2387 } 2388 2389 static void 2390 test_sequence_memory_domain(void) 2391 { 2392 struct spdk_accel_sequence *seq = NULL; 2393 struct spdk_io_channel *ioch; 2394 struct accel_io_channel *accel_ch; 2395 struct ut_sequence ut_seq; 2396 struct ut_domain_ctx domctx[4]; 2397 struct spdk_iobuf_buffer *cache_entry; 2398 struct accel_module modules[ACCEL_OPC_LAST]; 2399 struct spdk_memory_domain *accel_domain; 2400 spdk_iobuf_buffer_stailq_t small_cache; 2401 char srcbuf[4096], dstbuf[4096], expected[4096], tmp[4096]; 2402 struct iovec src_iovs[2], dst_iovs[2]; 2403 uint32_t small_cache_count; 2404 void *accel_buf, *accel_domain_ctx, *iobuf_buf; 2405 int i, rc, completed; 2406 2407 ioch = spdk_accel_get_io_channel(); 2408 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2409 2410 /* Override the submit_tasks function */ 2411 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2412 g_module.supports_memory_domains = false; 2413 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2414 modules[i] = g_modules_opc[i]; 2415 g_modules_opc[i] = g_module; 2416 } 2417 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2418 * decompress operations together in one sequence. 2419 */ 2420 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2421 g_seq_operations[ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2422 g_seq_operations[ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2423 2424 /* First check the simplest case - single fill operation with dstbuf in domain */ 2425 memset(expected, 0xa5, sizeof(expected)); 2426 memset(dstbuf, 0x0, sizeof(dstbuf)); 2427 completed = 0; 2428 seq = NULL; 2429 2430 /* Use some garbage pointer as dst and use domain ctx to get the actual dstbuf */ 2431 dst_iovs[0].iov_base = (void *)0xcafebabe; 2432 dst_iovs[0].iov_len = sizeof(dstbuf); 2433 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2434 2435 rc = spdk_accel_append_fill(&seq, ioch, dst_iovs[0].iov_base, dst_iovs[0].iov_len, 2436 g_ut_domain, &domctx[0], 0xa5, 0, 2437 ut_sequence_step_cb, &completed); 2438 CU_ASSERT_EQUAL(rc, 0); 2439 2440 ut_seq.complete = false; 2441 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2442 CU_ASSERT_EQUAL(rc, 0); 2443 2444 poll_threads(); 2445 CU_ASSERT_EQUAL(completed, 1); 2446 CU_ASSERT(ut_seq.complete); 2447 CU_ASSERT_EQUAL(ut_seq.status, 0); 2448 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2449 2450 /* Check operation with both srcbuf and dstbuf in remote memory domain */ 2451 memset(expected, 0x5a, sizeof(expected)); 2452 memset(dstbuf, 0x0, sizeof(dstbuf)); 2453 memset(srcbuf, 0x5a, sizeof(dstbuf)); 2454 completed = 0; 2455 seq = NULL; 2456 2457 src_iovs[0].iov_base = (void *)0xcafebabe; 2458 src_iovs[0].iov_len = sizeof(srcbuf); 2459 dst_iovs[0].iov_base = (void *)0xbeefdead; 2460 dst_iovs[0].iov_len = sizeof(dstbuf); 2461 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2462 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2463 2464 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2465 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2466 ut_sequence_step_cb, &completed); 2467 CU_ASSERT_EQUAL(rc, 0); 2468 2469 ut_seq.complete = false; 2470 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2471 CU_ASSERT_EQUAL(rc, 0); 2472 2473 poll_threads(); 2474 CU_ASSERT_EQUAL(completed, 1); 2475 CU_ASSERT(ut_seq.complete); 2476 CU_ASSERT_EQUAL(ut_seq.status, 0); 2477 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2478 2479 /* Two operations using a buffer in remote domain */ 2480 memset(expected, 0xa5, sizeof(expected)); 2481 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2482 memset(tmp, 0x0, sizeof(tmp)); 2483 memset(dstbuf, 0x0, sizeof(dstbuf)); 2484 completed = 0; 2485 seq = NULL; 2486 2487 src_iovs[0].iov_base = (void *)0xcafebabe; 2488 src_iovs[0].iov_len = sizeof(srcbuf); 2489 dst_iovs[0].iov_base = (void *)0xbeefdead; 2490 dst_iovs[0].iov_len = sizeof(tmp); 2491 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2492 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2493 2494 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2495 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2496 ut_sequence_step_cb, &completed); 2497 CU_ASSERT_EQUAL(rc, 0); 2498 2499 src_iovs[1].iov_base = (void *)0xbeefdead; 2500 src_iovs[1].iov_len = sizeof(tmp); 2501 dst_iovs[1].iov_base = (void *)0xa5a5a5a5; 2502 dst_iovs[1].iov_len = sizeof(dstbuf); 2503 ut_domain_ctx_init(&domctx[2], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2504 ut_domain_ctx_init(&domctx[3], tmp, sizeof(tmp), &src_iovs[1]); 2505 2506 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[2], 2507 &src_iovs[1], 1, g_ut_domain, &domctx[3], 0, 2508 ut_sequence_step_cb, &completed); 2509 CU_ASSERT_EQUAL(rc, 0); 2510 2511 ut_seq.complete = false; 2512 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2513 CU_ASSERT_EQUAL(rc, 0); 2514 2515 poll_threads(); 2516 CU_ASSERT_EQUAL(completed, 2); 2517 CU_ASSERT(ut_seq.complete); 2518 CU_ASSERT_EQUAL(ut_seq.status, 0); 2519 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2520 2521 /* Use buffer in remote memory domain and accel buffer at the same time */ 2522 memset(expected, 0x5a, sizeof(expected)); 2523 memset(srcbuf, 0x5a, sizeof(expected)); 2524 memset(dstbuf, 0x0, sizeof(dstbuf)); 2525 completed = 0; 2526 seq = NULL; 2527 2528 rc = spdk_accel_get_buf(ioch, sizeof(dstbuf), &accel_buf, &accel_domain, &accel_domain_ctx); 2529 CU_ASSERT_EQUAL(rc, 0); 2530 2531 src_iovs[0].iov_base = (void *)0xfeedbeef; 2532 src_iovs[0].iov_len = sizeof(srcbuf); 2533 dst_iovs[0].iov_base = accel_buf; 2534 dst_iovs[0].iov_len = sizeof(dstbuf); 2535 ut_domain_ctx_init(&domctx[0], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2536 2537 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, accel_domain, accel_domain_ctx, 2538 &src_iovs[0], 1, g_ut_domain, &domctx[0], 0, 2539 ut_sequence_step_cb, &completed); 2540 CU_ASSERT_EQUAL(rc, 0); 2541 2542 src_iovs[1].iov_base = accel_buf; 2543 src_iovs[1].iov_len = sizeof(dstbuf); 2544 dst_iovs[1].iov_base = (void *)0xbeeffeed; 2545 dst_iovs[1].iov_len = sizeof(dstbuf); 2546 ut_domain_ctx_init(&domctx[1], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2547 2548 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[1], 2549 &src_iovs[1], 1, accel_domain, accel_domain_ctx, 0, 2550 ut_sequence_step_cb, &completed); 2551 CU_ASSERT_EQUAL(rc, 0); 2552 2553 ut_seq.complete = false; 2554 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2555 CU_ASSERT_EQUAL(rc, 0); 2556 2557 poll_threads(); 2558 CU_ASSERT_EQUAL(completed, 2); 2559 CU_ASSERT(ut_seq.complete); 2560 CU_ASSERT_EQUAL(ut_seq.status, 0); 2561 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2562 spdk_accel_put_buf(ioch, accel_buf, accel_domain, accel_domain_ctx); 2563 2564 /* Check that a sequence with memory domains is correctly executed if buffers are not 2565 * immediately available */ 2566 memset(expected, 0xa5, sizeof(expected)); 2567 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2568 memset(dstbuf, 0x0, sizeof(dstbuf)); 2569 completed = 0; 2570 seq = NULL; 2571 /* Make sure the buffer pool is empty */ 2572 accel_ch = spdk_io_channel_get_ctx(ioch); 2573 small_cache_count = accel_ch->iobuf.small.cache_count; 2574 STAILQ_INIT(&small_cache); 2575 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2576 accel_ch->iobuf.small.cache_count = 0; 2577 MOCK_SET(spdk_mempool_get, NULL); 2578 2579 src_iovs[0].iov_base = (void *)0xdeadbeef; 2580 src_iovs[0].iov_len = sizeof(srcbuf); 2581 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2582 dst_iovs[0].iov_len = sizeof(dstbuf); 2583 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2584 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2585 2586 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2587 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2588 ut_sequence_step_cb, &completed); 2589 CU_ASSERT_EQUAL(rc, 0); 2590 2591 ut_seq.complete = false; 2592 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2593 CU_ASSERT_EQUAL(rc, 0); 2594 2595 poll_threads(); 2596 CU_ASSERT_EQUAL(completed, 0); 2597 CU_ASSERT(!ut_seq.complete); 2598 2599 /* Get a buffer and return it to the pool to trigger the sequence to resume. It shouldn't 2600 * be able to complete, as it needs two buffers */ 2601 MOCK_CLEAR(spdk_mempool_get); 2602 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2603 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2604 MOCK_SET(spdk_mempool_get, NULL); 2605 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2606 2607 CU_ASSERT_EQUAL(completed, 0); 2608 CU_ASSERT(!ut_seq.complete); 2609 2610 /* Return another buffer, this time the sequence should finish */ 2611 MOCK_CLEAR(spdk_mempool_get); 2612 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2613 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2614 MOCK_SET(spdk_mempool_get, NULL); 2615 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2616 2617 poll_threads(); 2618 CU_ASSERT_EQUAL(completed, 1); 2619 CU_ASSERT(ut_seq.complete); 2620 CU_ASSERT_EQUAL(ut_seq.status, 0); 2621 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2622 2623 /* Return the buffers back to the cache */ 2624 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2625 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2626 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2627 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2628 small_cache_count++; 2629 } 2630 accel_ch->iobuf.small.cache_count = 0; 2631 2632 MOCK_CLEAR(spdk_mempool_get); 2633 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2634 accel_ch->iobuf.small.cache_count = small_cache_count; 2635 2636 /* Check error cases, starting with an error from spdk_memory_domain_pull_data() */ 2637 completed = 0; 2638 seq = NULL; 2639 2640 src_iovs[0].iov_base = (void *)0xdeadbeef; 2641 src_iovs[0].iov_len = sizeof(srcbuf); 2642 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2643 dst_iovs[0].iov_len = sizeof(dstbuf); 2644 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2645 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2646 domctx[1].pull_submit_status = -E2BIG; 2647 2648 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2649 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2650 ut_sequence_step_cb, &completed); 2651 CU_ASSERT_EQUAL(rc, 0); 2652 2653 ut_seq.complete = false; 2654 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2655 CU_ASSERT_EQUAL(rc, 0); 2656 2657 CU_ASSERT_EQUAL(completed, 1); 2658 CU_ASSERT(ut_seq.complete); 2659 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 2660 2661 /* Check completion error from spdk_memory_domain_pull_data() */ 2662 completed = 0; 2663 seq = NULL; 2664 2665 src_iovs[0].iov_base = (void *)0xdeadbeef; 2666 src_iovs[0].iov_len = sizeof(srcbuf); 2667 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2668 dst_iovs[0].iov_len = sizeof(dstbuf); 2669 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2670 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2671 domctx[1].pull_complete_status = -EACCES; 2672 2673 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2674 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2675 ut_sequence_step_cb, &completed); 2676 CU_ASSERT_EQUAL(rc, 0); 2677 2678 ut_seq.complete = false; 2679 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2680 CU_ASSERT_EQUAL(rc, 0); 2681 2682 CU_ASSERT_EQUAL(completed, 1); 2683 CU_ASSERT(ut_seq.complete); 2684 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 2685 2686 /* Check submission error from spdk_memory_domain_push_data() */ 2687 completed = 0; 2688 seq = NULL; 2689 2690 src_iovs[0].iov_base = (void *)0xdeadbeef; 2691 src_iovs[0].iov_len = sizeof(srcbuf); 2692 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2693 dst_iovs[0].iov_len = sizeof(dstbuf); 2694 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2695 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2696 domctx[0].push_submit_status = -EADDRINUSE; 2697 2698 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2699 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2700 ut_sequence_step_cb, &completed); 2701 CU_ASSERT_EQUAL(rc, 0); 2702 2703 ut_seq.complete = false; 2704 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2705 CU_ASSERT_EQUAL(rc, 0); 2706 2707 CU_ASSERT_EQUAL(completed, 1); 2708 CU_ASSERT(ut_seq.complete); 2709 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 2710 2711 /* Check completion error from spdk_memory_domain_push_data() */ 2712 completed = 0; 2713 seq = NULL; 2714 2715 src_iovs[0].iov_base = (void *)0xdeadbeef; 2716 src_iovs[0].iov_len = sizeof(srcbuf); 2717 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2718 dst_iovs[0].iov_len = sizeof(dstbuf); 2719 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2720 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2721 domctx[0].push_complete_status = -EADDRNOTAVAIL; 2722 2723 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2724 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2725 ut_sequence_step_cb, &completed); 2726 CU_ASSERT_EQUAL(rc, 0); 2727 2728 ut_seq.complete = false; 2729 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2730 CU_ASSERT_EQUAL(rc, 0); 2731 2732 CU_ASSERT_EQUAL(completed, 1); 2733 CU_ASSERT(ut_seq.complete); 2734 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 2735 2736 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2737 g_modules_opc[i] = modules[i]; 2738 } 2739 2740 ut_clear_operations(); 2741 spdk_put_io_channel(ioch); 2742 poll_threads(); 2743 } 2744 2745 static int 2746 ut_submit_decompress_memory_domain(struct spdk_io_channel *ch, struct spdk_accel_task *task) 2747 { 2748 struct ut_domain_ctx *ctx; 2749 struct iovec *src_iovs, *dst_iovs; 2750 uint32_t src_iovcnt, dst_iovcnt; 2751 2752 src_iovs = task->s.iovs; 2753 dst_iovs = task->d.iovs; 2754 src_iovcnt = task->s.iovcnt; 2755 dst_iovcnt = task->d.iovcnt; 2756 2757 if (task->src_domain != NULL) { 2758 ctx = task->src_domain_ctx; 2759 CU_ASSERT_EQUAL(memcmp(task->s.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2760 src_iovs = &ctx->iov; 2761 src_iovcnt = 1; 2762 } 2763 2764 if (task->dst_domain != NULL) { 2765 ctx = task->dst_domain_ctx; 2766 CU_ASSERT_EQUAL(memcmp(task->d.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2767 dst_iovs = &ctx->iov; 2768 dst_iovcnt = 1; 2769 } 2770 2771 spdk_iovcpy(src_iovs, src_iovcnt, dst_iovs, dst_iovcnt); 2772 spdk_accel_task_complete(task, 0); 2773 2774 return 0; 2775 } 2776 2777 static void 2778 test_sequence_module_memory_domain(void) 2779 { 2780 struct spdk_accel_sequence *seq = NULL; 2781 struct spdk_io_channel *ioch; 2782 struct accel_module modules[ACCEL_OPC_LAST]; 2783 struct spdk_memory_domain *accel_domain; 2784 struct ut_sequence ut_seq; 2785 struct ut_domain_ctx domctx[2]; 2786 struct iovec src_iovs[2], dst_iovs[2]; 2787 void *buf, *accel_domain_ctx; 2788 char srcbuf[4096], dstbuf[4096], tmp[4096], expected[4096]; 2789 int i, rc, completed; 2790 2791 ioch = spdk_accel_get_io_channel(); 2792 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2793 2794 /* Override the submit_tasks function */ 2795 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2796 g_module.supports_memory_domains = true; 2797 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2798 modules[i] = g_modules_opc[i]; 2799 g_modules_opc[i] = g_module; 2800 } 2801 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress_memory_domain; 2802 g_seq_operations[ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2803 2804 /* Check a sequence with both buffers in memory domains */ 2805 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2806 memset(expected, 0xa5, sizeof(expected)); 2807 memset(dstbuf, 0, sizeof(dstbuf)); 2808 seq = NULL; 2809 completed = 0; 2810 2811 src_iovs[0].iov_base = (void *)0xcafebabe; 2812 src_iovs[0].iov_len = sizeof(srcbuf); 2813 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2814 dst_iovs[0].iov_len = sizeof(dstbuf); 2815 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2816 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2817 2818 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2819 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2820 ut_sequence_step_cb, &completed); 2821 CU_ASSERT_EQUAL(rc, 0); 2822 2823 ut_seq.complete = false; 2824 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2825 CU_ASSERT_EQUAL(rc, 0); 2826 2827 poll_threads(); 2828 2829 CU_ASSERT_EQUAL(completed, 1); 2830 CU_ASSERT(ut_seq.complete); 2831 CU_ASSERT_EQUAL(ut_seq.status, 0); 2832 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2833 2834 /* Check two operations each with a single buffer in memory domain */ 2835 memset(srcbuf, 0x5a, sizeof(srcbuf)); 2836 memset(expected, 0x5a, sizeof(expected)); 2837 memset(dstbuf, 0, sizeof(dstbuf)); 2838 memset(tmp, 0, sizeof(tmp)); 2839 seq = NULL; 2840 completed = 0; 2841 2842 src_iovs[0].iov_base = srcbuf; 2843 src_iovs[0].iov_len = sizeof(srcbuf); 2844 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2845 dst_iovs[0].iov_len = sizeof(tmp); 2846 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2847 2848 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2849 &src_iovs[0], 1, NULL, NULL, 0, 2850 ut_sequence_step_cb, &completed); 2851 CU_ASSERT_EQUAL(rc, 0); 2852 2853 src_iovs[1].iov_base = (void *)0xfeedbeef; 2854 src_iovs[1].iov_len = sizeof(tmp); 2855 dst_iovs[1].iov_base = dstbuf; 2856 dst_iovs[1].iov_len = sizeof(dstbuf); 2857 ut_domain_ctx_init(&domctx[1], tmp, sizeof(tmp), &src_iovs[1]); 2858 2859 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2860 &src_iovs[1], 1, g_ut_domain, &domctx[1], 0, 2861 ut_sequence_step_cb, &completed); 2862 CU_ASSERT_EQUAL(rc, 0); 2863 2864 ut_seq.complete = false; 2865 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2866 CU_ASSERT_EQUAL(rc, 0); 2867 2868 poll_threads(); 2869 2870 CU_ASSERT_EQUAL(completed, 2); 2871 CU_ASSERT(ut_seq.complete); 2872 CU_ASSERT_EQUAL(ut_seq.status, 0); 2873 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2874 2875 /* Check a sequence with an accel buffer and a buffer in a regular memory domain */ 2876 memset(expected, 0xa5, sizeof(expected)); 2877 memset(dstbuf, 0, sizeof(dstbuf)); 2878 memset(tmp, 0, sizeof(tmp)); 2879 seq = NULL; 2880 completed = 0; 2881 2882 rc = spdk_accel_get_buf(ioch, 4096, &buf, &accel_domain, &accel_domain_ctx); 2883 CU_ASSERT_EQUAL(rc, 0); 2884 2885 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, accel_domain, accel_domain_ctx, 2886 0xa5, 0, ut_sequence_step_cb, &completed); 2887 CU_ASSERT_EQUAL(rc, 0); 2888 2889 src_iovs[0].iov_base = buf; 2890 src_iovs[0].iov_len = 4096; 2891 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2892 dst_iovs[0].iov_len = sizeof(dstbuf); 2893 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2894 2895 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2896 &src_iovs[0], 1, accel_domain, accel_domain_ctx, 0, 2897 ut_sequence_step_cb, &completed); 2898 CU_ASSERT_EQUAL(rc, 0); 2899 2900 ut_seq.complete = false; 2901 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2902 CU_ASSERT_EQUAL(rc, 0); 2903 2904 poll_threads(); 2905 2906 CU_ASSERT_EQUAL(completed, 2); 2907 CU_ASSERT(ut_seq.complete); 2908 CU_ASSERT_EQUAL(ut_seq.status, 0); 2909 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2910 2911 spdk_accel_put_buf(ioch, buf, accel_domain, accel_domain_ctx); 2912 2913 g_module.supports_memory_domains = false; 2914 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2915 g_modules_opc[i] = modules[i]; 2916 } 2917 2918 ut_clear_operations(); 2919 spdk_put_io_channel(ioch); 2920 poll_threads(); 2921 } 2922 2923 #ifdef SPDK_CONFIG_ISAL_CRYPTO 2924 static void 2925 ut_encrypt_cb(void *cb_arg, int status) 2926 { 2927 int *completed = cb_arg; 2928 2929 CU_ASSERT_EQUAL(status, 0); 2930 2931 *completed = 1; 2932 } 2933 2934 static void 2935 test_sequence_crypto(void) 2936 { 2937 struct spdk_accel_sequence *seq = NULL; 2938 struct spdk_io_channel *ioch; 2939 struct spdk_accel_crypto_key *key; 2940 struct spdk_accel_crypto_key_create_param key_params = { 2941 .cipher = "AES_XTS", 2942 .hex_key = "00112233445566778899aabbccddeeff", 2943 .hex_key2 = "ffeeddccbbaa99887766554433221100", 2944 .key_name = "ut_key", 2945 }; 2946 struct ut_sequence ut_seq; 2947 unsigned char buf[4096], encrypted[4096] = {}, data[4096], tmp[3][4096]; 2948 struct iovec src_iovs[4], dst_iovs[4]; 2949 int rc, completed = 0; 2950 size_t i; 2951 2952 ioch = spdk_accel_get_io_channel(); 2953 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2954 2955 rc = spdk_accel_crypto_key_create(&key_params); 2956 CU_ASSERT_EQUAL(rc, 0); 2957 key = spdk_accel_crypto_key_get(key_params.key_name); 2958 SPDK_CU_ASSERT_FATAL(key != NULL); 2959 2960 for (i = 0; i < sizeof(data); ++i) { 2961 data[i] = (uint8_t)i & 0xff; 2962 } 2963 2964 dst_iovs[0].iov_base = encrypted; 2965 dst_iovs[0].iov_len = sizeof(encrypted); 2966 src_iovs[0].iov_base = data; 2967 src_iovs[0].iov_len = sizeof(data); 2968 rc = spdk_accel_submit_encrypt(ioch, key, &dst_iovs[0], 1, &src_iovs[0], 1, 0, 4096, 0, 2969 ut_encrypt_cb, &completed); 2970 CU_ASSERT_EQUAL(rc, 0); 2971 2972 while (!completed) { 2973 poll_threads(); 2974 } 2975 2976 /* Verify that encryption operation in a sequence produces the same result */ 2977 seq = NULL; 2978 completed = 0; 2979 2980 dst_iovs[0].iov_base = tmp[0]; 2981 dst_iovs[0].iov_len = sizeof(tmp[0]); 2982 src_iovs[0].iov_base = data; 2983 src_iovs[0].iov_len = sizeof(data); 2984 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2985 &src_iovs[0], 1, NULL, NULL, 0, 2986 ut_sequence_step_cb, &completed); 2987 CU_ASSERT_EQUAL(rc, 0); 2988 2989 dst_iovs[1].iov_base = tmp[1]; 2990 dst_iovs[1].iov_len = sizeof(tmp[1]); 2991 src_iovs[1].iov_base = tmp[0]; 2992 src_iovs[1].iov_len = sizeof(tmp[0]); 2993 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 2994 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 2995 ut_sequence_step_cb, &completed); 2996 CU_ASSERT_EQUAL(rc, 0); 2997 2998 dst_iovs[2].iov_base = buf; 2999 dst_iovs[2].iov_len = sizeof(buf); 3000 src_iovs[2].iov_base = tmp[1]; 3001 src_iovs[2].iov_len = sizeof(tmp[1]); 3002 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3003 &src_iovs[2], 1, NULL, NULL, 0, 3004 ut_sequence_step_cb, &completed); 3005 CU_ASSERT_EQUAL(rc, 0); 3006 3007 ut_seq.complete = false; 3008 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3009 CU_ASSERT_EQUAL(rc, 0); 3010 3011 poll_threads(); 3012 3013 CU_ASSERT_EQUAL(completed, 3); 3014 CU_ASSERT(ut_seq.complete); 3015 CU_ASSERT_EQUAL(ut_seq.status, 0); 3016 CU_ASSERT_EQUAL(memcmp(buf, encrypted, sizeof(buf)), 0); 3017 3018 /* Check that decryption produces the original buffer */ 3019 seq = NULL; 3020 completed = 0; 3021 memset(buf, 0, sizeof(buf)); 3022 3023 dst_iovs[0].iov_base = tmp[0]; 3024 dst_iovs[0].iov_len = sizeof(tmp[0]); 3025 src_iovs[0].iov_base = encrypted; 3026 src_iovs[0].iov_len = sizeof(encrypted); 3027 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3028 &src_iovs[0], 1, NULL, NULL, 0, 3029 ut_sequence_step_cb, &completed); 3030 CU_ASSERT_EQUAL(rc, 0); 3031 3032 dst_iovs[1].iov_base = tmp[1]; 3033 dst_iovs[1].iov_len = sizeof(tmp[1]); 3034 src_iovs[1].iov_base = tmp[0]; 3035 src_iovs[1].iov_len = sizeof(tmp[0]); 3036 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3037 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3038 ut_sequence_step_cb, &completed); 3039 CU_ASSERT_EQUAL(rc, 0); 3040 3041 dst_iovs[2].iov_base = buf; 3042 dst_iovs[2].iov_len = sizeof(buf); 3043 src_iovs[2].iov_base = tmp[1]; 3044 src_iovs[2].iov_len = sizeof(tmp[1]); 3045 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3046 &src_iovs[2], 1, NULL, NULL, 0, 3047 ut_sequence_step_cb, &completed); 3048 CU_ASSERT_EQUAL(rc, 0); 3049 3050 ut_seq.complete = false; 3051 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3052 CU_ASSERT_EQUAL(rc, 0); 3053 3054 poll_threads(); 3055 3056 CU_ASSERT_EQUAL(completed, 3); 3057 CU_ASSERT(ut_seq.complete); 3058 CU_ASSERT_EQUAL(ut_seq.status, 0); 3059 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3060 3061 /* Check encrypt + decrypt in a single sequence */ 3062 seq = NULL; 3063 completed = 0; 3064 memset(buf, 0, sizeof(buf)); 3065 3066 dst_iovs[0].iov_base = tmp[0]; 3067 dst_iovs[0].iov_len = sizeof(tmp[0]); 3068 src_iovs[0].iov_base = data; 3069 src_iovs[0].iov_len = sizeof(data); 3070 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3071 &src_iovs[0], 1, NULL, NULL, 0, 3072 ut_sequence_step_cb, &completed); 3073 CU_ASSERT_EQUAL(rc, 0); 3074 3075 dst_iovs[1].iov_base = tmp[1]; 3076 dst_iovs[1].iov_len = sizeof(tmp[1]); 3077 src_iovs[1].iov_base = tmp[0]; 3078 src_iovs[1].iov_len = sizeof(tmp[0]); 3079 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3080 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3081 ut_sequence_step_cb, &completed); 3082 CU_ASSERT_EQUAL(rc, 0); 3083 3084 3085 dst_iovs[2].iov_base = tmp[2]; 3086 dst_iovs[2].iov_len = sizeof(tmp[2]); 3087 src_iovs[2].iov_base = tmp[1]; 3088 src_iovs[2].iov_len = sizeof(tmp[1]); 3089 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[2], 1, NULL, NULL, 3090 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3091 ut_sequence_step_cb, &completed); 3092 CU_ASSERT_EQUAL(rc, 0); 3093 3094 dst_iovs[3].iov_base = buf; 3095 dst_iovs[3].iov_len = sizeof(buf); 3096 src_iovs[3].iov_base = tmp[2]; 3097 src_iovs[3].iov_len = sizeof(tmp[2]); 3098 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 3099 &src_iovs[3], 1, NULL, NULL, 0, 3100 ut_sequence_step_cb, &completed); 3101 CU_ASSERT_EQUAL(rc, 0); 3102 3103 ut_seq.complete = false; 3104 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3105 CU_ASSERT_EQUAL(rc, 0); 3106 3107 poll_threads(); 3108 3109 CU_ASSERT_EQUAL(completed, 4); 3110 CU_ASSERT(ut_seq.complete); 3111 CU_ASSERT_EQUAL(ut_seq.status, 0); 3112 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3113 3114 rc = spdk_accel_crypto_key_destroy(key); 3115 CU_ASSERT_EQUAL(rc, 0); 3116 spdk_put_io_channel(ioch); 3117 poll_threads(); 3118 } 3119 #endif /* SPDK_CONFIG_ISAL_CRYPTO */ 3120 3121 static int 3122 ut_submit_crypto(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3123 { 3124 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3125 3126 spdk_accel_task_complete(task, 0); 3127 3128 return 0; 3129 } 3130 3131 struct ut_driver_operation { 3132 int complete_status; 3133 int submit_status; 3134 int count; 3135 bool supported; 3136 }; 3137 3138 static struct ut_driver_operation g_drv_operations[ACCEL_OPC_LAST]; 3139 3140 static int 3141 ut_driver_execute_sequence(struct spdk_accel_sequence *seq) 3142 { 3143 struct spdk_accel_task *task; 3144 struct ut_driver_operation *drv_ops; 3145 3146 while ((task = spdk_accel_sequence_first_task(seq)) != NULL) { 3147 drv_ops = &g_drv_operations[task->op_code]; 3148 if (!drv_ops->supported) { 3149 break; 3150 } 3151 3152 drv_ops->count++; 3153 if (drv_ops->submit_status != 0) { 3154 return drv_ops->submit_status; 3155 } 3156 3157 if (drv_ops->complete_status != 0) { 3158 spdk_accel_task_complete(task, drv_ops->complete_status); 3159 break; 3160 } 3161 3162 switch (task->op_code) { 3163 case ACCEL_OPC_DECOMPRESS: 3164 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3165 break; 3166 case ACCEL_OPC_FILL: 3167 spdk_iov_memset(task->d.iovs, task->d.iovcnt, 3168 (int)(task->fill_pattern & 0xff)); 3169 break; 3170 default: 3171 CU_ASSERT(0 && "unexpected opcode"); 3172 break; 3173 } 3174 3175 spdk_accel_task_complete(task, 0); 3176 } 3177 3178 spdk_accel_sequence_continue(seq); 3179 3180 return 0; 3181 } 3182 3183 static struct spdk_accel_driver g_ut_driver = { 3184 .name = "ut", 3185 .execute_sequence = ut_driver_execute_sequence, 3186 }; 3187 3188 SPDK_ACCEL_DRIVER_REGISTER(ut, &g_ut_driver); 3189 3190 static void 3191 test_sequence_driver(void) 3192 { 3193 struct spdk_accel_sequence *seq = NULL; 3194 struct spdk_io_channel *ioch; 3195 struct spdk_accel_crypto_key key = {}; 3196 struct ut_sequence ut_seq; 3197 struct accel_module modules[ACCEL_OPC_LAST]; 3198 char buf[4096], tmp[3][4096], expected[4096]; 3199 struct iovec src_iovs[3], dst_iovs[3]; 3200 int i, rc, completed = 0; 3201 3202 ioch = spdk_accel_get_io_channel(); 3203 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3204 rc = spdk_accel_set_driver("ut"); 3205 SPDK_CU_ASSERT_FATAL(rc == 0); 3206 3207 /* Override the submit_tasks function */ 3208 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3209 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3210 modules[i] = g_modules_opc[i]; 3211 g_modules_opc[i] = g_module; 3212 } 3213 /* Intercept crypto operations, as they should be executed by an accel module */ 3214 g_seq_operations[ACCEL_OPC_ENCRYPT].submit = ut_submit_crypto; 3215 g_seq_operations[ACCEL_OPC_DECRYPT].submit = ut_submit_crypto; 3216 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3217 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3218 g_seq_operations[ACCEL_OPC_FILL].count = 0; 3219 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3220 3221 g_drv_operations[ACCEL_OPC_FILL].supported = true; 3222 g_drv_operations[ACCEL_OPC_DECOMPRESS].supported = true; 3223 3224 /* First check a sequence that is fully executed using a driver, with the copy at the end 3225 * being removed */ 3226 seq = NULL; 3227 completed = 0; 3228 memset(buf, 0, sizeof(buf)); 3229 memset(tmp[0], 0, sizeof(tmp[0])); 3230 memset(tmp[1], 0, sizeof(tmp[1])); 3231 memset(&expected[0], 0xa5, 2048); 3232 memset(&expected[2048], 0xbe, 2048); 3233 3234 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], 2048, NULL, NULL, 0xa5, 0, 3235 ut_sequence_step_cb, &completed); 3236 CU_ASSERT_EQUAL(rc, 0); 3237 rc = spdk_accel_append_fill(&seq, ioch, &tmp[0][2048], 2048, NULL, NULL, 0xbe, 0, 3238 ut_sequence_step_cb, &completed); 3239 CU_ASSERT_EQUAL(rc, 0); 3240 3241 dst_iovs[0].iov_base = tmp[1]; 3242 dst_iovs[0].iov_len = sizeof(tmp[1]); 3243 src_iovs[0].iov_base = tmp[0]; 3244 src_iovs[0].iov_len = sizeof(tmp[0]); 3245 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3246 &src_iovs[0], 1, NULL, NULL, 0, 3247 ut_sequence_step_cb, &completed); 3248 CU_ASSERT_EQUAL(rc, 0); 3249 3250 dst_iovs[1].iov_base = buf; 3251 dst_iovs[1].iov_len = sizeof(buf); 3252 src_iovs[1].iov_base = tmp[1]; 3253 src_iovs[1].iov_len = sizeof(tmp[1]); 3254 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3255 &src_iovs[1], 1, NULL, NULL, 0, 3256 ut_sequence_step_cb, &completed); 3257 CU_ASSERT_EQUAL(rc, 0); 3258 3259 ut_seq.complete = false; 3260 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3261 CU_ASSERT_EQUAL(rc, 0); 3262 3263 poll_threads(); 3264 3265 CU_ASSERT_EQUAL(completed, 4); 3266 CU_ASSERT(ut_seq.complete); 3267 CU_ASSERT_EQUAL(ut_seq.status, 0); 3268 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3269 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3270 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 3271 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 2); 3272 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3273 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3274 3275 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3276 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3277 3278 /* Check a sequence when the first two operations are executed by a driver, while the rest 3279 * is executed via modules */ 3280 seq = NULL; 3281 completed = 0; 3282 memset(buf, 0, sizeof(buf)); 3283 memset(tmp[0], 0, sizeof(tmp[0])); 3284 memset(tmp[1], 0, sizeof(tmp[1])); 3285 memset(tmp[2], 0, sizeof(tmp[2])); 3286 memset(&expected[0], 0xfe, 4096); 3287 3288 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 0, 3289 ut_sequence_step_cb, &completed); 3290 CU_ASSERT_EQUAL(rc, 0); 3291 3292 dst_iovs[0].iov_base = tmp[1]; 3293 dst_iovs[0].iov_len = sizeof(tmp[1]); 3294 src_iovs[0].iov_base = tmp[0]; 3295 src_iovs[0].iov_len = sizeof(tmp[0]); 3296 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3297 &src_iovs[0], 1, NULL, NULL, 0, 3298 ut_sequence_step_cb, &completed); 3299 CU_ASSERT_EQUAL(rc, 0); 3300 3301 dst_iovs[1].iov_base = tmp[2]; 3302 dst_iovs[1].iov_len = sizeof(tmp[2]); 3303 src_iovs[1].iov_base = tmp[1]; 3304 src_iovs[1].iov_len = sizeof(tmp[1]); 3305 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3306 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3307 ut_sequence_step_cb, &completed); 3308 CU_ASSERT_EQUAL(rc, 0); 3309 3310 dst_iovs[2].iov_base = buf; 3311 dst_iovs[2].iov_len = sizeof(buf); 3312 src_iovs[2].iov_base = tmp[2]; 3313 src_iovs[2].iov_len = sizeof(tmp[2]); 3314 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3315 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3316 ut_sequence_step_cb, &completed); 3317 CU_ASSERT_EQUAL(rc, 0); 3318 3319 ut_seq.complete = false; 3320 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3321 CU_ASSERT_EQUAL(rc, 0); 3322 3323 poll_threads(); 3324 3325 CU_ASSERT_EQUAL(completed, 4); 3326 CU_ASSERT(ut_seq.complete); 3327 CU_ASSERT_EQUAL(ut_seq.status, 0); 3328 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3329 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3330 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3331 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3332 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3333 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3334 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3335 3336 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3337 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3338 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3339 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3340 3341 /* Check sequence when the first and last operations are executed through modules, while the 3342 * ones in the middle are executed by the driver */ 3343 seq = NULL; 3344 completed = 0; 3345 memset(buf, 0, sizeof(buf)); 3346 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3347 memset(tmp[1], 0, sizeof(tmp[1])); 3348 memset(tmp[2], 0, sizeof(tmp[2])); 3349 memset(&expected[0], 0xfe, 2048); 3350 memset(&expected[2048], 0xa5, 2048); 3351 3352 dst_iovs[0].iov_base = tmp[1]; 3353 dst_iovs[0].iov_len = sizeof(tmp[1]); 3354 src_iovs[0].iov_base = tmp[0]; 3355 src_iovs[0].iov_len = sizeof(tmp[0]); 3356 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3357 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3358 ut_sequence_step_cb, &completed); 3359 CU_ASSERT_EQUAL(rc, 0); 3360 3361 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xfe, 0, 3362 ut_sequence_step_cb, &completed); 3363 CU_ASSERT_EQUAL(rc, 0); 3364 3365 dst_iovs[1].iov_base = tmp[2]; 3366 dst_iovs[1].iov_len = sizeof(tmp[2]); 3367 src_iovs[1].iov_base = tmp[1]; 3368 src_iovs[1].iov_len = sizeof(tmp[1]); 3369 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3370 &src_iovs[1], 1, NULL, NULL, 0, 3371 ut_sequence_step_cb, &completed); 3372 CU_ASSERT_EQUAL(rc, 0); 3373 3374 dst_iovs[2].iov_base = buf; 3375 dst_iovs[2].iov_len = sizeof(buf); 3376 src_iovs[2].iov_base = tmp[2]; 3377 src_iovs[2].iov_len = sizeof(tmp[2]); 3378 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3379 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3380 ut_sequence_step_cb, &completed); 3381 CU_ASSERT_EQUAL(rc, 0); 3382 3383 ut_seq.complete = false; 3384 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3385 CU_ASSERT_EQUAL(rc, 0); 3386 3387 poll_threads(); 3388 3389 CU_ASSERT_EQUAL(completed, 4); 3390 CU_ASSERT(ut_seq.complete); 3391 CU_ASSERT_EQUAL(ut_seq.status, 0); 3392 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3393 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3394 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3395 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3396 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3397 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3398 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3399 3400 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3401 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3402 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3403 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3404 3405 /* Check a sequence with operations executed by: module, driver, module, driver */ 3406 seq = NULL; 3407 completed = 0; 3408 memset(buf, 0, sizeof(buf)); 3409 memset(tmp[0], 0x5a, sizeof(tmp[0])); 3410 memset(tmp[1], 0, sizeof(tmp[1])); 3411 memset(tmp[2], 0, sizeof(tmp[2])); 3412 memset(&expected[0], 0xef, 2048); 3413 memset(&expected[2048], 0x5a, 2048); 3414 3415 dst_iovs[0].iov_base = tmp[1]; 3416 dst_iovs[0].iov_len = sizeof(tmp[1]); 3417 src_iovs[0].iov_base = tmp[0]; 3418 src_iovs[0].iov_len = sizeof(tmp[0]); 3419 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3420 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3421 ut_sequence_step_cb, &completed); 3422 CU_ASSERT_EQUAL(rc, 0); 3423 3424 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3425 ut_sequence_step_cb, &completed); 3426 CU_ASSERT_EQUAL(rc, 0); 3427 3428 dst_iovs[1].iov_base = tmp[2]; 3429 dst_iovs[1].iov_len = sizeof(tmp[2]); 3430 src_iovs[1].iov_base = tmp[1]; 3431 src_iovs[1].iov_len = sizeof(tmp[1]); 3432 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3433 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3434 ut_sequence_step_cb, &completed); 3435 CU_ASSERT_EQUAL(rc, 0); 3436 3437 dst_iovs[2].iov_base = buf; 3438 dst_iovs[2].iov_len = sizeof(buf); 3439 src_iovs[2].iov_base = tmp[2]; 3440 src_iovs[2].iov_len = sizeof(tmp[2]); 3441 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3442 &src_iovs[2], 1, NULL, NULL, 0, 3443 ut_sequence_step_cb, &completed); 3444 CU_ASSERT_EQUAL(rc, 0); 3445 3446 ut_seq.complete = false; 3447 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3448 CU_ASSERT_EQUAL(rc, 0); 3449 3450 poll_threads(); 3451 3452 CU_ASSERT_EQUAL(completed, 4); 3453 CU_ASSERT(ut_seq.complete); 3454 CU_ASSERT_EQUAL(ut_seq.status, 0); 3455 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3456 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3457 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3458 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3459 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3460 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3461 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3462 3463 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3464 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3465 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3466 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3467 3468 /* Check that an error returned from driver's execute_sequence() will fail the whole 3469 * sequence and any subsequent operations won't be processed */ 3470 seq = NULL; 3471 completed = 0; 3472 memset(buf, 0, sizeof(buf)); 3473 memset(expected, 0, sizeof(expected)); 3474 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3475 g_drv_operations[ACCEL_OPC_FILL].submit_status = -EPERM; 3476 3477 dst_iovs[0].iov_base = tmp[1]; 3478 dst_iovs[0].iov_len = sizeof(tmp[1]); 3479 src_iovs[0].iov_base = tmp[0]; 3480 src_iovs[0].iov_len = sizeof(tmp[0]); 3481 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3482 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3483 ut_sequence_step_cb, &completed); 3484 CU_ASSERT_EQUAL(rc, 0); 3485 3486 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3487 ut_sequence_step_cb, &completed); 3488 CU_ASSERT_EQUAL(rc, 0); 3489 3490 dst_iovs[1].iov_base = buf; 3491 dst_iovs[1].iov_len = sizeof(buf); 3492 src_iovs[1].iov_base = tmp[1]; 3493 src_iovs[1].iov_len = sizeof(tmp[1]); 3494 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3495 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3496 ut_sequence_step_cb, &completed); 3497 CU_ASSERT_EQUAL(rc, 0); 3498 3499 ut_seq.complete = false; 3500 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3501 CU_ASSERT_EQUAL(rc, 0); 3502 3503 poll_threads(); 3504 3505 CU_ASSERT_EQUAL(completed, 3); 3506 CU_ASSERT(ut_seq.complete); 3507 CU_ASSERT_EQUAL(ut_seq.status, -EPERM); 3508 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3509 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3510 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 0); 3511 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3512 CU_ASSERT_EQUAL(memcmp(buf, expected, 4096), 0); 3513 3514 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3515 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3516 g_drv_operations[ACCEL_OPC_FILL].submit_status = 0; 3517 3518 /* Check that a failed task completed by a driver will cause the whole sequence to be failed 3519 * and any subsequent operations won't be processed */ 3520 seq = NULL; 3521 completed = 0; 3522 memset(buf, 0, sizeof(buf)); 3523 memset(expected, 0, sizeof(expected)); 3524 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3525 g_drv_operations[ACCEL_OPC_FILL].complete_status = -ENOENT; 3526 3527 dst_iovs[0].iov_base = tmp[1]; 3528 dst_iovs[0].iov_len = sizeof(tmp[1]); 3529 src_iovs[0].iov_base = tmp[0]; 3530 src_iovs[0].iov_len = sizeof(tmp[0]); 3531 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3532 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3533 ut_sequence_step_cb, &completed); 3534 CU_ASSERT_EQUAL(rc, 0); 3535 3536 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3537 ut_sequence_step_cb, &completed); 3538 CU_ASSERT_EQUAL(rc, 0); 3539 3540 dst_iovs[1].iov_base = buf; 3541 dst_iovs[1].iov_len = sizeof(buf); 3542 src_iovs[1].iov_base = tmp[1]; 3543 src_iovs[1].iov_len = sizeof(tmp[1]); 3544 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3545 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3546 ut_sequence_step_cb, &completed); 3547 CU_ASSERT_EQUAL(rc, 0); 3548 3549 ut_seq.complete = false; 3550 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3551 CU_ASSERT_EQUAL(rc, 0); 3552 3553 poll_threads(); 3554 3555 CU_ASSERT_EQUAL(completed, 3); 3556 CU_ASSERT(ut_seq.complete); 3557 CU_ASSERT_EQUAL(ut_seq.status, -ENOENT); 3558 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3559 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3560 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 0); 3561 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3562 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3563 3564 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3565 g_modules_opc[i] = modules[i]; 3566 } 3567 3568 /* Clear the driver so that other tests won't use it */ 3569 g_accel_driver = NULL; 3570 memset(&g_drv_operations, 0, sizeof(g_drv_operations)); 3571 3572 ut_clear_operations(); 3573 spdk_put_io_channel(ioch); 3574 poll_threads(); 3575 } 3576 3577 struct ut_saved_iovs { 3578 struct iovec src; 3579 struct iovec dst; 3580 }; 3581 3582 static struct ut_saved_iovs g_seq_saved_iovs[ACCEL_OPC_LAST]; 3583 3584 static int 3585 ut_submit_save_iovs(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3586 { 3587 SPDK_CU_ASSERT_FATAL(task->s.iovcnt == 1); 3588 SPDK_CU_ASSERT_FATAL(task->d.iovcnt == 1); 3589 3590 g_seq_saved_iovs[task->op_code].src = task->s.iovs[0]; 3591 g_seq_saved_iovs[task->op_code].dst = task->d.iovs[0]; 3592 3593 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3594 3595 spdk_accel_task_complete(task, 0); 3596 3597 return 0; 3598 } 3599 3600 static void 3601 test_sequence_same_iovs(void) 3602 { 3603 struct spdk_accel_sequence *seq = NULL; 3604 struct spdk_io_channel *ioch; 3605 struct spdk_accel_crypto_key key = {}; 3606 struct ut_sequence ut_seq; 3607 struct accel_module modules[ACCEL_OPC_LAST]; 3608 char buf[4096], tmp[4096], expected[4096]; 3609 struct iovec iovs[3], expected_siov, expected_diov; 3610 struct spdk_memory_domain *domain; 3611 void *accel_buf, *domain_ctx; 3612 int i, rc, completed = 0; 3613 3614 ioch = spdk_accel_get_io_channel(); 3615 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3616 3617 /* Override the submit_tasks function */ 3618 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3619 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3620 modules[i] = g_modules_opc[i]; 3621 g_modules_opc[i] = g_module; 3622 } 3623 /* Intercept crypto operations, as they should be executed by an accel module */ 3624 g_seq_operations[ACCEL_OPC_ENCRYPT].submit = ut_submit_save_iovs; 3625 g_seq_operations[ACCEL_OPC_DECRYPT].submit = ut_submit_save_iovs; 3626 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3627 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3628 3629 /* Check that it's possible to use the same iovec ptr for different operations */ 3630 seq = NULL; 3631 completed = 0; 3632 memset(buf, 0, sizeof(buf)); 3633 memset(expected, 0xa5, sizeof(expected)); 3634 3635 iovs[0].iov_base = expected; 3636 iovs[0].iov_len = sizeof(expected); 3637 iovs[1].iov_base = tmp; 3638 iovs[1].iov_len = sizeof(tmp); 3639 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, NULL, NULL, 3640 &iovs[0], 1, NULL, NULL, 0, 4096, 0, 3641 ut_sequence_step_cb, &completed); 3642 CU_ASSERT_EQUAL(rc, 0); 3643 /* Reuse iov[1] as src */ 3644 iovs[2].iov_base = buf; 3645 iovs[2].iov_len = sizeof(buf); 3646 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3647 &iovs[1], 1, NULL, NULL, 0, 4096, 0, 3648 ut_sequence_step_cb, &completed); 3649 CU_ASSERT_EQUAL(rc, 0); 3650 3651 ut_seq.complete = false; 3652 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3653 CU_ASSERT_EQUAL(rc, 0); 3654 3655 poll_threads(); 3656 3657 CU_ASSERT_EQUAL(completed, 2); 3658 CU_ASSERT(ut_seq.complete); 3659 CU_ASSERT_EQUAL(ut_seq.status, 0); 3660 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3661 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3662 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3663 expected_siov.iov_base = expected; 3664 expected_siov.iov_len = sizeof(expected); 3665 expected_diov.iov_base = tmp; 3666 expected_diov.iov_len = sizeof(tmp); 3667 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_ENCRYPT].src, 3668 &expected_siov, sizeof(expected_siov)), 0); 3669 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_ENCRYPT].dst, 3670 &expected_diov, sizeof(expected_diov)), 0); 3671 expected_siov.iov_base = tmp; 3672 expected_siov.iov_len = sizeof(tmp); 3673 expected_diov.iov_base = buf; 3674 expected_diov.iov_len = sizeof(buf); 3675 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_DECRYPT].src, 3676 &expected_siov, sizeof(expected_siov)), 0); 3677 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_DECRYPT].dst, 3678 &expected_diov, sizeof(expected_diov)), 0); 3679 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3680 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3681 3682 /* Check the same with an accel buffer */ 3683 seq = NULL; 3684 completed = 0; 3685 memset(buf, 0, sizeof(buf)); 3686 memset(expected, 0x5a, sizeof(expected)); 3687 3688 rc = spdk_accel_get_buf(ioch, sizeof(buf), &accel_buf, &domain, &domain_ctx); 3689 CU_ASSERT_EQUAL(rc, 0); 3690 3691 iovs[0].iov_base = expected; 3692 iovs[0].iov_len = sizeof(expected); 3693 iovs[1].iov_base = accel_buf; 3694 iovs[1].iov_len = sizeof(buf); 3695 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, domain, domain_ctx, 3696 &iovs[0], 1, NULL, NULL, 0, 4096, 0, 3697 ut_sequence_step_cb, &completed); 3698 CU_ASSERT_EQUAL(rc, 0); 3699 /* Reuse iov[1] as src */ 3700 iovs[2].iov_base = buf; 3701 iovs[2].iov_len = sizeof(buf); 3702 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3703 &iovs[1], 1, domain, domain_ctx, 0, 4096, 0, 3704 ut_sequence_step_cb, &completed); 3705 CU_ASSERT_EQUAL(rc, 0); 3706 3707 ut_seq.complete = false; 3708 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3709 CU_ASSERT_EQUAL(rc, 0); 3710 3711 poll_threads(); 3712 3713 CU_ASSERT_EQUAL(completed, 2); 3714 CU_ASSERT(ut_seq.complete); 3715 CU_ASSERT_EQUAL(ut_seq.status, 0); 3716 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3717 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3718 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3719 expected_siov.iov_base = expected; 3720 expected_siov.iov_len = sizeof(expected); 3721 expected_diov.iov_base = buf; 3722 expected_diov.iov_len = sizeof(buf); 3723 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_ENCRYPT].src, 3724 &expected_siov, sizeof(expected_siov)), 0); 3725 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_DECRYPT].dst, 3726 &expected_diov, sizeof(expected_diov)), 0); 3727 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_ENCRYPT].dst, 3728 &g_seq_saved_iovs[ACCEL_OPC_DECRYPT].src, 3729 sizeof(struct iovec)), 0); 3730 spdk_accel_put_buf(ioch, accel_buf, domain, domain_ctx); 3731 3732 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3733 g_modules_opc[i] = modules[i]; 3734 } 3735 3736 ut_clear_operations(); 3737 spdk_put_io_channel(ioch); 3738 poll_threads(); 3739 } 3740 3741 static int 3742 test_sequence_setup(void) 3743 { 3744 int rc; 3745 3746 allocate_cores(1); 3747 allocate_threads(1); 3748 set_thread(0); 3749 3750 rc = spdk_iobuf_initialize(); 3751 if (rc != 0) { 3752 CU_ASSERT(false); 3753 return -1; 3754 } 3755 3756 rc = spdk_accel_initialize(); 3757 if (rc != 0) { 3758 CU_ASSERT(false); 3759 return -1; 3760 } 3761 3762 return 0; 3763 } 3764 3765 static void 3766 finish_cb(void *cb_arg) 3767 { 3768 bool *done = cb_arg; 3769 3770 *done = true; 3771 } 3772 3773 static int 3774 test_sequence_cleanup(void) 3775 { 3776 bool done = false; 3777 3778 spdk_accel_finish(finish_cb, &done); 3779 3780 while (!done) { 3781 poll_threads(); 3782 } 3783 3784 done = false; 3785 spdk_iobuf_finish(finish_cb, &done); 3786 while (!done) { 3787 poll_threads(); 3788 } 3789 3790 free_threads(); 3791 free_cores(); 3792 3793 return 0; 3794 } 3795 3796 int 3797 main(int argc, char **argv) 3798 { 3799 CU_pSuite suite = NULL, seq_suite; 3800 unsigned int num_failures; 3801 3802 CU_set_error_action(CUEA_ABORT); 3803 CU_initialize_registry(); 3804 3805 /* Sequence tests require accel to be initialized normally, so run them before the other 3806 * tests which register accel modules which aren't fully implemented, causing accel 3807 * initialization to fail. 3808 */ 3809 seq_suite = CU_add_suite("accel_sequence", test_sequence_setup, test_sequence_cleanup); 3810 CU_ADD_TEST(seq_suite, test_sequence_fill_copy); 3811 CU_ADD_TEST(seq_suite, test_sequence_abort); 3812 CU_ADD_TEST(seq_suite, test_sequence_append_error); 3813 CU_ADD_TEST(seq_suite, test_sequence_completion_error); 3814 #ifdef SPDK_CONFIG_ISAL /* accel_sw requires isa-l for compression */ 3815 CU_ADD_TEST(seq_suite, test_sequence_decompress); 3816 CU_ADD_TEST(seq_suite, test_sequence_reverse); 3817 #endif 3818 CU_ADD_TEST(seq_suite, test_sequence_copy_elision); 3819 CU_ADD_TEST(seq_suite, test_sequence_accel_buffers); 3820 CU_ADD_TEST(seq_suite, test_sequence_memory_domain); 3821 CU_ADD_TEST(seq_suite, test_sequence_module_memory_domain); 3822 #ifdef SPDK_CONFIG_ISAL_CRYPTO /* accel_sw requires isa-l-crypto for crypto operations */ 3823 CU_ADD_TEST(seq_suite, test_sequence_crypto); 3824 #endif 3825 CU_ADD_TEST(seq_suite, test_sequence_driver); 3826 CU_ADD_TEST(seq_suite, test_sequence_same_iovs); 3827 3828 suite = CU_add_suite("accel", test_setup, test_cleanup); 3829 CU_ADD_TEST(suite, test_spdk_accel_task_complete); 3830 CU_ADD_TEST(suite, test_get_task); 3831 CU_ADD_TEST(suite, test_spdk_accel_submit_copy); 3832 CU_ADD_TEST(suite, test_spdk_accel_submit_dualcast); 3833 CU_ADD_TEST(suite, test_spdk_accel_submit_compare); 3834 CU_ADD_TEST(suite, test_spdk_accel_submit_fill); 3835 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c); 3836 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32cv); 3837 CU_ADD_TEST(suite, test_spdk_accel_submit_copy_crc32c); 3838 CU_ADD_TEST(suite, test_spdk_accel_submit_xor); 3839 CU_ADD_TEST(suite, test_spdk_accel_module_find_by_name); 3840 CU_ADD_TEST(suite, test_spdk_accel_module_register); 3841 3842 CU_basic_set_mode(CU_BRM_VERBOSE); 3843 CU_basic_run_tests(); 3844 num_failures = CU_get_number_of_failures(); 3845 CU_cleanup_registry(); 3846 3847 return num_failures; 3848 } 3849