1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2021 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk_internal/cunit.h" 8 #include "spdk_internal/mock.h" 9 #include "spdk/accel_module.h" 10 #include "thread/thread_internal.h" 11 #include "common/lib/ut_multithread.c" 12 #include "common/lib/test_iobuf.c" 13 #include "accel/accel.c" 14 #include "accel/accel_sw.c" 15 #include "unit/lib/json_mock.c" 16 17 DEFINE_STUB_V(spdk_memory_domain_destroy, (struct spdk_memory_domain *domain)); 18 DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, 19 (struct spdk_memory_domain *domain), "UT_DMA"); 20 DEFINE_STUB(spdk_memory_domain_get_system_domain, struct spdk_memory_domain *, (void), 21 (void *)0xabc); 22 DEFINE_STUB_V(spdk_memory_domain_set_invalidate, (struct spdk_memory_domain *domain, 23 spdk_memory_domain_invalidate_data_cb invalidate_cb)); 24 DEFINE_STUB_V(spdk_memory_domain_set_translation, (struct spdk_memory_domain *domain, 25 spdk_memory_domain_translate_memory_cb translate_cb)); 26 27 int 28 spdk_memory_domain_create(struct spdk_memory_domain **domain, enum spdk_dma_device_type type, 29 struct spdk_memory_domain_ctx *ctx, const char *id) 30 { 31 *domain = (void *)0xdeadbeef; 32 33 return 0; 34 } 35 36 struct ut_domain_ctx { 37 struct iovec iov; 38 struct iovec expected; 39 int pull_submit_status; 40 int push_submit_status; 41 int pull_complete_status; 42 int push_complete_status; 43 }; 44 45 static struct spdk_memory_domain *g_ut_domain = (void *)0xa55e1; 46 47 int 48 spdk_memory_domain_pull_data(struct spdk_memory_domain *sd, void *sctx, struct iovec *siov, 49 uint32_t siovcnt, struct iovec *diov, uint32_t diovcnt, 50 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 51 { 52 struct ut_domain_ctx *ctx = sctx; 53 54 CU_ASSERT_EQUAL(sd, g_ut_domain); 55 CU_ASSERT_EQUAL(siovcnt, 1); 56 CU_ASSERT_EQUAL(memcmp(siov, &ctx->expected, sizeof(*siov)), 0); 57 58 if (ctx->pull_submit_status != 0) { 59 return ctx->pull_submit_status; 60 } 61 62 if (ctx->pull_complete_status != 0) { 63 cpl_cb(cpl_arg, ctx->pull_complete_status); 64 return 0; 65 } 66 67 spdk_iovcpy(&ctx->iov, 1, diov, diovcnt); 68 cpl_cb(cpl_arg, 0); 69 70 return 0; 71 } 72 73 int 74 spdk_memory_domain_push_data(struct spdk_memory_domain *dd, void *dctx, struct iovec *diov, 75 uint32_t diovcnt, struct iovec *siov, uint32_t siovcnt, 76 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 77 { 78 struct ut_domain_ctx *ctx = dctx; 79 80 CU_ASSERT_EQUAL(dd, g_ut_domain); 81 CU_ASSERT_EQUAL(diovcnt, 1); 82 CU_ASSERT_EQUAL(memcmp(diov, &ctx->expected, sizeof(*diov)), 0); 83 84 if (ctx->push_submit_status != 0) { 85 return ctx->push_submit_status; 86 } 87 88 if (ctx->push_complete_status != 0) { 89 cpl_cb(cpl_arg, ctx->push_complete_status); 90 return 0; 91 } 92 93 spdk_iovcpy(siov, siovcnt, &ctx->iov, 1); 94 cpl_cb(cpl_arg, 0); 95 96 return 0; 97 } 98 99 /* global vars and setup/cleanup functions used for all test functions */ 100 struct spdk_accel_module_if g_module_if = {}; 101 struct accel_module g_module = { .module = &g_module_if }; 102 struct spdk_io_channel *g_ch = NULL; 103 struct accel_io_channel *g_accel_ch = NULL; 104 struct sw_accel_io_channel *g_sw_ch = NULL; 105 struct spdk_io_channel *g_module_ch = NULL; 106 107 static uint64_t g_opc_mask = 0; 108 109 static uint64_t 110 _accel_op_to_bit(enum spdk_accel_opcode opc) 111 { 112 return (1 << opc); 113 } 114 115 static bool 116 _supports_opcode(enum spdk_accel_opcode opc) 117 { 118 if (_accel_op_to_bit(opc) & g_opc_mask) { 119 return true; 120 } 121 return false; 122 } 123 124 static int 125 test_setup(void) 126 { 127 int i; 128 129 g_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct accel_io_channel)); 130 if (g_ch == NULL) { 131 /* for some reason the assert fatal macro doesn't work in the setup function. */ 132 CU_ASSERT(false); 133 return -1; 134 } 135 g_accel_ch = (struct accel_io_channel *)((char *)g_ch + sizeof(struct spdk_io_channel)); 136 g_module_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct sw_accel_io_channel)); 137 if (g_module_ch == NULL) { 138 CU_ASSERT(false); 139 return -1; 140 } 141 142 g_module_if.submit_tasks = sw_accel_submit_tasks; 143 g_module_if.name = "software"; 144 for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) { 145 g_accel_ch->module_ch[i] = g_module_ch; 146 g_modules_opc[i] = g_module; 147 } 148 g_sw_ch = (struct sw_accel_io_channel *)((char *)g_module_ch + sizeof( 149 struct spdk_io_channel)); 150 /* Prevent lazy initialization of poller. */ 151 g_sw_ch->completion_poller = (void *)0xdeadbeef; 152 STAILQ_INIT(&g_sw_ch->tasks_to_complete); 153 g_module_if.supports_opcode = _supports_opcode; 154 return 0; 155 } 156 157 static int 158 test_cleanup(void) 159 { 160 free(g_ch); 161 free(g_module_ch); 162 163 return 0; 164 } 165 166 #define DUMMY_ARG 0xDEADBEEF 167 static bool g_dummy_cb_called = false; 168 static void 169 dummy_cb_fn(void *cb_arg, int status) 170 { 171 CU_ASSERT(*(uint32_t *)cb_arg == DUMMY_ARG); 172 CU_ASSERT(status == 0); 173 g_dummy_cb_called = true; 174 } 175 176 static void 177 test_spdk_accel_task_complete(void) 178 { 179 struct spdk_accel_task accel_task = {}; 180 struct spdk_accel_task *expected_accel_task = NULL; 181 uint32_t cb_arg = DUMMY_ARG; 182 int status = 0; 183 184 accel_task.accel_ch = g_accel_ch; 185 accel_task.cb_fn = dummy_cb_fn; 186 accel_task.cb_arg = &cb_arg; 187 STAILQ_INIT(&g_accel_ch->task_pool); 188 189 /* Confirm cb is called and task added to list. */ 190 spdk_accel_task_complete(&accel_task, status); 191 CU_ASSERT(g_dummy_cb_called == true); 192 expected_accel_task = STAILQ_FIRST(&g_accel_ch->task_pool); 193 STAILQ_REMOVE_HEAD(&g_accel_ch->task_pool, link); 194 CU_ASSERT(expected_accel_task == &accel_task); 195 } 196 197 static void 198 test_get_task(void) 199 { 200 struct spdk_accel_task *task; 201 struct spdk_accel_task _task; 202 void *cb_arg = NULL; 203 204 STAILQ_INIT(&g_accel_ch->task_pool); 205 206 /* no tasks left, return NULL. */ 207 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 208 CU_ASSERT(task == NULL); 209 210 _task.cb_fn = dummy_cb_fn; 211 _task.cb_arg = cb_arg; 212 _task.accel_ch = g_accel_ch; 213 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &_task, link); 214 215 /* Get a valid task. */ 216 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 217 CU_ASSERT(task == &_task); 218 CU_ASSERT(_task.cb_fn == dummy_cb_fn); 219 CU_ASSERT(_task.cb_arg == cb_arg); 220 CU_ASSERT(_task.accel_ch == g_accel_ch); 221 } 222 223 #define TEST_SUBMIT_SIZE 64 224 static void 225 test_spdk_accel_submit_copy(void) 226 { 227 const uint64_t nbytes = TEST_SUBMIT_SIZE; 228 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 229 uint8_t src[TEST_SUBMIT_SIZE] = {0}; 230 void *cb_arg = NULL; 231 int rc; 232 struct spdk_accel_task task; 233 struct spdk_accel_task_aux_data task_aux; 234 struct spdk_accel_task *expected_accel_task = NULL; 235 int flags = 0; 236 237 STAILQ_INIT(&g_accel_ch->task_pool); 238 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 239 240 /* Fail with no tasks on _get_task() */ 241 rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, flags, NULL, cb_arg); 242 CU_ASSERT(rc == -ENOMEM); 243 244 task.accel_ch = g_accel_ch; 245 task.flags = 1; 246 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 247 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 248 249 /* submission OK. */ 250 rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, flags, NULL, cb_arg); 251 CU_ASSERT(rc == 0); 252 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COPY); 253 CU_ASSERT(task.flags == 0); 254 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 255 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 256 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 257 CU_ASSERT(expected_accel_task == &task); 258 } 259 260 static void 261 test_spdk_accel_submit_dualcast(void) 262 { 263 void *dst1; 264 void *dst2; 265 void *src; 266 uint32_t align = ALIGN_4K; 267 uint64_t nbytes = TEST_SUBMIT_SIZE; 268 void *cb_arg = NULL; 269 int rc; 270 struct spdk_accel_task task; 271 struct spdk_accel_task_aux_data task_aux; 272 struct spdk_accel_task *expected_accel_task = NULL; 273 int flags = 0; 274 275 STAILQ_INIT(&g_accel_ch->task_pool); 276 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 277 278 /* Dualcast requires 4K alignment on dst addresses, 279 * hence using the hard coded address to test the buffer alignment 280 */ 281 dst1 = (void *)0x5000; 282 dst2 = (void *)0x60f0; 283 src = calloc(1, TEST_SUBMIT_SIZE); 284 SPDK_CU_ASSERT_FATAL(src != NULL); 285 memset(src, 0x5A, TEST_SUBMIT_SIZE); 286 287 /* This should fail since dst2 is not 4k aligned */ 288 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 289 CU_ASSERT(rc == -EINVAL); 290 291 dst1 = (void *)0x7010; 292 dst2 = (void *)0x6000; 293 /* This should fail since dst1 is not 4k aligned */ 294 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 295 CU_ASSERT(rc == -EINVAL); 296 297 /* Dualcast requires 4K alignment on dst addresses */ 298 dst1 = (void *)0x7000; 299 dst2 = (void *)0x6000; 300 /* Fail with no tasks on _get_task() */ 301 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 302 CU_ASSERT(rc == -ENOMEM); 303 304 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 305 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 306 307 /* accel submission OK., since we test the SW path , need to use valid memory addresses 308 * cannot hardcode them anymore */ 309 dst1 = spdk_dma_zmalloc(nbytes, align, NULL); 310 SPDK_CU_ASSERT_FATAL(dst1 != NULL); 311 dst2 = spdk_dma_zmalloc(nbytes, align, NULL); 312 SPDK_CU_ASSERT_FATAL(dst2 != NULL); 313 /* SW module does the dualcast. */ 314 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 315 CU_ASSERT(rc == 0); 316 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_DUALCAST); 317 CU_ASSERT(task.flags == 0); 318 CU_ASSERT(memcmp(dst1, src, TEST_SUBMIT_SIZE) == 0); 319 CU_ASSERT(memcmp(dst2, src, TEST_SUBMIT_SIZE) == 0); 320 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 321 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 322 CU_ASSERT(expected_accel_task == &task); 323 324 free(src); 325 spdk_free(dst1); 326 spdk_free(dst2); 327 } 328 329 static void 330 test_spdk_accel_submit_compare(void) 331 { 332 void *src1; 333 void *src2; 334 uint64_t nbytes = TEST_SUBMIT_SIZE; 335 void *cb_arg = NULL; 336 int rc; 337 struct spdk_accel_task task; 338 struct spdk_accel_task_aux_data task_aux; 339 struct spdk_accel_task *expected_accel_task = NULL; 340 341 STAILQ_INIT(&g_accel_ch->task_pool); 342 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 343 344 src1 = calloc(1, TEST_SUBMIT_SIZE); 345 SPDK_CU_ASSERT_FATAL(src1 != NULL); 346 src2 = calloc(1, TEST_SUBMIT_SIZE); 347 SPDK_CU_ASSERT_FATAL(src2 != NULL); 348 349 /* Fail with no tasks on _get_task() */ 350 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 351 CU_ASSERT(rc == -ENOMEM); 352 353 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 354 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 355 356 /* accel submission OK. */ 357 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 358 CU_ASSERT(rc == 0); 359 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COMPARE); 360 CU_ASSERT(memcmp(src1, src2, TEST_SUBMIT_SIZE) == 0); 361 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 362 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 363 CU_ASSERT(expected_accel_task == &task); 364 365 free(src1); 366 free(src2); 367 } 368 369 static void 370 test_spdk_accel_submit_fill(void) 371 { 372 void *dst; 373 void *src; 374 uint8_t fill = 0xf; 375 uint64_t fill64; 376 uint64_t nbytes = TEST_SUBMIT_SIZE; 377 void *cb_arg = NULL; 378 int rc; 379 struct spdk_accel_task task; 380 struct spdk_accel_task_aux_data task_aux; 381 struct spdk_accel_task *expected_accel_task = NULL; 382 int flags = 0; 383 384 STAILQ_INIT(&g_accel_ch->task_pool); 385 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 386 387 dst = calloc(1, TEST_SUBMIT_SIZE); 388 SPDK_CU_ASSERT_FATAL(dst != NULL); 389 src = calloc(1, TEST_SUBMIT_SIZE); 390 SPDK_CU_ASSERT_FATAL(src != NULL); 391 memset(src, fill, TEST_SUBMIT_SIZE); 392 memset(&fill64, fill, sizeof(uint64_t)); 393 394 /* Fail with no tasks on _get_task() */ 395 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 396 CU_ASSERT(rc == -ENOMEM); 397 398 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 399 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 400 401 /* accel submission OK. */ 402 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 403 CU_ASSERT(rc == 0); 404 CU_ASSERT(task.fill_pattern == fill64); 405 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_FILL); 406 CU_ASSERT(task.flags == 0); 407 408 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 409 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 410 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 411 CU_ASSERT(expected_accel_task == &task); 412 413 free(dst); 414 free(src); 415 } 416 417 static void 418 test_spdk_accel_submit_crc32c(void) 419 { 420 const uint64_t nbytes = TEST_SUBMIT_SIZE; 421 uint32_t crc_dst; 422 uint8_t src[TEST_SUBMIT_SIZE]; 423 uint32_t seed = 1; 424 void *cb_arg = NULL; 425 int rc; 426 struct spdk_accel_task task; 427 struct spdk_accel_task_aux_data task_aux; 428 struct spdk_accel_task *expected_accel_task = NULL; 429 430 STAILQ_INIT(&g_accel_ch->task_pool); 431 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 432 433 /* Fail with no tasks on _get_task() */ 434 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 435 CU_ASSERT(rc == -ENOMEM); 436 437 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 438 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 439 440 /* accel submission OK. */ 441 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 442 CU_ASSERT(rc == 0); 443 CU_ASSERT(task.crc_dst == &crc_dst); 444 CU_ASSERT(task.seed == seed); 445 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_CRC32C); 446 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 447 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 448 CU_ASSERT(expected_accel_task == &task); 449 } 450 451 static void 452 test_spdk_accel_submit_crc32cv(void) 453 { 454 uint32_t crc_dst; 455 uint32_t seed = 0; 456 uint32_t iov_cnt = 32; 457 void *cb_arg = NULL; 458 int rc; 459 uint32_t i = 0; 460 struct spdk_accel_task task; 461 struct spdk_accel_task_aux_data task_aux; 462 struct iovec iov[32]; 463 struct spdk_accel_task *expected_accel_task = NULL; 464 465 STAILQ_INIT(&g_accel_ch->task_pool); 466 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 467 468 for (i = 0; i < iov_cnt; i++) { 469 iov[i].iov_base = calloc(1, TEST_SUBMIT_SIZE); 470 SPDK_CU_ASSERT_FATAL(iov[i].iov_base != NULL); 471 iov[i].iov_len = TEST_SUBMIT_SIZE; 472 } 473 474 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 475 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 476 477 /* accel submission OK. */ 478 rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, NULL, cb_arg); 479 CU_ASSERT(rc == 0); 480 CU_ASSERT(task.s.iovs == iov); 481 CU_ASSERT(task.s.iovcnt == iov_cnt); 482 CU_ASSERT(task.crc_dst == &crc_dst); 483 CU_ASSERT(task.seed == seed); 484 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_CRC32C); 485 CU_ASSERT(task.cb_arg == cb_arg); 486 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 487 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 488 CU_ASSERT(expected_accel_task == &task); 489 490 for (i = 0; i < iov_cnt; i++) { 491 free(iov[i].iov_base); 492 } 493 } 494 495 static void 496 test_spdk_accel_submit_copy_crc32c(void) 497 { 498 const uint64_t nbytes = TEST_SUBMIT_SIZE; 499 uint32_t crc_dst; 500 uint8_t dst[TEST_SUBMIT_SIZE]; 501 uint8_t src[TEST_SUBMIT_SIZE]; 502 uint32_t seed = 0; 503 void *cb_arg = NULL; 504 int rc; 505 struct spdk_accel_task task; 506 struct spdk_accel_task_aux_data task_aux; 507 struct spdk_accel_task *expected_accel_task = NULL; 508 int flags = 0; 509 510 STAILQ_INIT(&g_accel_ch->task_pool); 511 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 512 513 /* Fail with no tasks on _get_task() */ 514 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 515 NULL, cb_arg); 516 CU_ASSERT(rc == -ENOMEM); 517 518 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 519 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 520 521 /* accel submission OK. */ 522 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 523 NULL, cb_arg); 524 CU_ASSERT(rc == 0); 525 CU_ASSERT(task.crc_dst == &crc_dst); 526 CU_ASSERT(task.seed == seed); 527 CU_ASSERT(task.flags == 0); 528 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COPY_CRC32C); 529 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 530 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 531 CU_ASSERT(expected_accel_task == &task); 532 } 533 534 static void 535 test_spdk_accel_submit_xor(void) 536 { 537 const uint64_t nbytes = TEST_SUBMIT_SIZE; 538 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 539 uint8_t src1[TEST_SUBMIT_SIZE] = {0}; 540 uint8_t src2[TEST_SUBMIT_SIZE] = {0}; 541 void *sources[] = { src1, src2 }; 542 uint32_t nsrcs = SPDK_COUNTOF(sources); 543 int rc; 544 struct spdk_accel_task task; 545 struct spdk_accel_task_aux_data task_aux; 546 struct spdk_accel_task *expected_accel_task = NULL; 547 548 STAILQ_INIT(&g_accel_ch->task_pool); 549 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 550 551 /* Fail with no tasks on _get_task() */ 552 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 553 CU_ASSERT(rc == -ENOMEM); 554 555 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 556 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 557 558 /* submission OK. */ 559 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 560 CU_ASSERT(rc == 0); 561 CU_ASSERT(task.nsrcs.srcs == sources); 562 CU_ASSERT(task.nsrcs.cnt == nsrcs); 563 CU_ASSERT(task.d.iovcnt == 1); 564 CU_ASSERT(task.d.iovs[0].iov_base == dst); 565 CU_ASSERT(task.d.iovs[0].iov_len == nbytes); 566 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_XOR); 567 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 568 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 569 CU_ASSERT(expected_accel_task == &task); 570 } 571 572 static void 573 test_spdk_accel_module_find_by_name(void) 574 { 575 struct spdk_accel_module_if mod1 = {}; 576 struct spdk_accel_module_if mod2 = {}; 577 struct spdk_accel_module_if mod3 = {}; 578 struct spdk_accel_module_if *accel_module = NULL; 579 580 mod1.name = "ioat"; 581 mod2.name = "idxd"; 582 mod3.name = "software"; 583 584 TAILQ_INIT(&spdk_accel_module_list); 585 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod1, tailq); 586 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod2, tailq); 587 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod3, tailq); 588 589 /* Now let's find a valid engine */ 590 accel_module = _module_find_by_name("ioat"); 591 CU_ASSERT(accel_module != NULL); 592 593 /* Try to find one that doesn't exist */ 594 accel_module = _module_find_by_name("XXX"); 595 CU_ASSERT(accel_module == NULL); 596 } 597 598 static int 599 ut_module_init_nop(void) 600 { 601 return 0; 602 } 603 604 static bool 605 ut_supports_opcode_all(enum spdk_accel_opcode opcode) 606 { 607 return true; 608 } 609 610 static void 611 ut_accel_module_priority_finish_done(void *done) 612 { 613 *(int *)done = 1; 614 } 615 616 static void 617 test_spdk_accel_module_register(void) 618 { 619 struct spdk_accel_module_if mods[] = { 620 { .name = "mod1", .priority = 1, }, 621 { .name = "mod3", .priority = 3, }, 622 { .name = "mod0", .priority = 0, }, 623 { .name = "mod2", .priority = 2, }, 624 }; 625 int rc, done = 0; 626 const char *modname = NULL; 627 size_t i; 628 629 allocate_cores(1); 630 allocate_threads(1); 631 set_thread(0); 632 633 TAILQ_INIT(&spdk_accel_module_list); 634 for (i = 0; i < SPDK_COUNTOF(mods); ++i) { 635 mods[i].module_init = ut_module_init_nop; 636 mods[i].supports_opcode = ut_supports_opcode_all; 637 spdk_accel_module_list_add(&mods[i]); 638 } 639 640 rc = spdk_accel_initialize(); 641 CU_ASSERT_EQUAL(rc, 0); 642 643 /* All opcodes should be assigned to the module with highest prio - mod3 */ 644 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 645 rc = spdk_accel_get_opc_module_name((enum spdk_accel_opcode)i, &modname); 646 CU_ASSERT_EQUAL(rc, 0); 647 CU_ASSERT_STRING_EQUAL(modname, "mod3"); 648 } 649 650 spdk_accel_finish(ut_accel_module_priority_finish_done, &done); 651 while (!done) { 652 poll_threads(); 653 } 654 655 TAILQ_INIT(&spdk_accel_module_list); 656 free_threads(); 657 free_cores(); 658 } 659 660 struct ut_sequence { 661 bool complete; 662 int status; 663 }; 664 665 static void 666 ut_sequence_step_cb(void *cb_arg) 667 { 668 int *completed = cb_arg; 669 670 (*completed)++; 671 } 672 673 static void 674 ut_sequence_complete_cb(void *cb_arg, int status) 675 { 676 struct ut_sequence *seq = cb_arg; 677 678 seq->complete = true; 679 seq->status = status; 680 } 681 682 static void 683 test_sequence_fill_copy(void) 684 { 685 struct spdk_accel_sequence *seq = NULL; 686 struct spdk_io_channel *ioch; 687 struct ut_sequence ut_seq; 688 char buf[4096], tmp[2][4096], expected[4096]; 689 struct iovec src_iovs[2], dst_iovs[2]; 690 int rc, completed; 691 692 ioch = spdk_accel_get_io_channel(); 693 SPDK_CU_ASSERT_FATAL(ioch != NULL); 694 695 /* First check the simplest case - single task in a sequence */ 696 memset(buf, 0, sizeof(buf)); 697 memset(expected, 0xa5, sizeof(expected)); 698 completed = 0; 699 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 700 ut_sequence_step_cb, &completed); 701 CU_ASSERT_EQUAL(rc, 0); 702 CU_ASSERT_EQUAL(completed, 0); 703 704 ut_seq.complete = false; 705 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 706 707 poll_threads(); 708 CU_ASSERT_EQUAL(completed, 1); 709 CU_ASSERT(ut_seq.complete); 710 CU_ASSERT_EQUAL(ut_seq.status, 0); 711 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 712 713 /* Check a single copy operation */ 714 memset(buf, 0, sizeof(buf)); 715 memset(tmp[0], 0xa5, sizeof(tmp[0])); 716 memset(expected, 0xa5, sizeof(expected)); 717 completed = 0; 718 seq = NULL; 719 720 dst_iovs[0].iov_base = buf; 721 dst_iovs[0].iov_len = sizeof(buf); 722 src_iovs[0].iov_base = tmp[0]; 723 src_iovs[0].iov_len = sizeof(tmp[0]); 724 725 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 726 &src_iovs[0], 1, NULL, NULL, 0, 727 ut_sequence_step_cb, &completed); 728 CU_ASSERT_EQUAL(rc, 0); 729 730 ut_seq.complete = false; 731 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 732 733 poll_threads(); 734 CU_ASSERT_EQUAL(completed, 1); 735 CU_ASSERT(ut_seq.complete); 736 CU_ASSERT_EQUAL(ut_seq.status, 0); 737 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 738 739 /* Check multiple fill operations */ 740 memset(buf, 0, sizeof(buf)); 741 memset(expected, 0xfe, 4096); 742 memset(expected, 0xde, 2048); 743 memset(expected, 0xa5, 1024); 744 seq = NULL; 745 completed = 0; 746 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, NULL, NULL, 0xfe, 0, 747 ut_sequence_step_cb, &completed); 748 CU_ASSERT_EQUAL(rc, 0); 749 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xde, 0, 750 ut_sequence_step_cb, &completed); 751 CU_ASSERT_EQUAL(rc, 0); 752 rc = spdk_accel_append_fill(&seq, ioch, buf, 1024, NULL, NULL, 0xa5, 0, 753 ut_sequence_step_cb, &completed); 754 CU_ASSERT_EQUAL(rc, 0); 755 756 ut_seq.complete = false; 757 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 758 759 poll_threads(); 760 CU_ASSERT_EQUAL(completed, 3); 761 CU_ASSERT(ut_seq.complete); 762 CU_ASSERT_EQUAL(ut_seq.status, 0); 763 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 764 765 /* Check multiple copy operations */ 766 memset(buf, 0, sizeof(buf)); 767 memset(tmp[0], 0, sizeof(tmp[0])); 768 memset(tmp[1], 0, sizeof(tmp[1])); 769 memset(expected, 0xa5, sizeof(expected)); 770 seq = NULL; 771 completed = 0; 772 773 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 774 ut_sequence_step_cb, &completed); 775 CU_ASSERT_EQUAL(rc, 0); 776 777 dst_iovs[0].iov_base = tmp[1]; 778 dst_iovs[0].iov_len = sizeof(tmp[1]); 779 src_iovs[0].iov_base = tmp[0]; 780 src_iovs[0].iov_len = sizeof(tmp[0]); 781 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 782 &src_iovs[0], 1, NULL, NULL, 0, 783 ut_sequence_step_cb, &completed); 784 CU_ASSERT_EQUAL(rc, 0); 785 786 dst_iovs[1].iov_base = buf; 787 dst_iovs[1].iov_len = sizeof(buf); 788 src_iovs[1].iov_base = tmp[1]; 789 src_iovs[1].iov_len = sizeof(tmp[1]); 790 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 791 &src_iovs[1], 1, NULL, NULL, 0, 792 ut_sequence_step_cb, &completed); 793 CU_ASSERT_EQUAL(rc, 0); 794 795 ut_seq.complete = false; 796 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 797 798 poll_threads(); 799 CU_ASSERT_EQUAL(completed, 3); 800 CU_ASSERT(ut_seq.complete); 801 CU_ASSERT_EQUAL(ut_seq.status, 0); 802 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 803 804 /* Check that adding a copy operation at the end will change destination buffer */ 805 memset(buf, 0, sizeof(buf)); 806 memset(tmp[0], 0, sizeof(tmp[0])); 807 memset(expected, 0xa5, sizeof(buf)); 808 seq = NULL; 809 completed = 0; 810 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 811 ut_sequence_step_cb, &completed); 812 CU_ASSERT_EQUAL(rc, 0); 813 814 dst_iovs[0].iov_base = buf; 815 dst_iovs[0].iov_len = sizeof(buf); 816 src_iovs[0].iov_base = tmp[0]; 817 src_iovs[0].iov_len = sizeof(tmp[0]); 818 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 819 &src_iovs[0], 1, NULL, NULL, 0, 820 ut_sequence_step_cb, &completed); 821 CU_ASSERT_EQUAL(rc, 0); 822 823 ut_seq.complete = false; 824 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 825 826 poll_threads(); 827 CU_ASSERT_EQUAL(completed, 2); 828 CU_ASSERT(ut_seq.complete); 829 CU_ASSERT_EQUAL(ut_seq.status, 0); 830 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 831 832 /* Check that it's also possible to add copy operation at the beginning */ 833 memset(buf, 0, sizeof(buf)); 834 memset(tmp[0], 0xde, sizeof(tmp[0])); 835 memset(tmp[1], 0, sizeof(tmp[1])); 836 memset(expected, 0xa5, sizeof(expected)); 837 seq = NULL; 838 completed = 0; 839 840 dst_iovs[0].iov_base = tmp[1]; 841 dst_iovs[0].iov_len = sizeof(tmp[1]); 842 src_iovs[0].iov_base = tmp[0]; 843 src_iovs[0].iov_len = sizeof(tmp[0]); 844 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 845 &src_iovs[0], 1, NULL, NULL, 0, 846 ut_sequence_step_cb, &completed); 847 CU_ASSERT_EQUAL(rc, 0); 848 849 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], sizeof(tmp[1]), NULL, NULL, 0xa5, 0, 850 ut_sequence_step_cb, &completed); 851 CU_ASSERT_EQUAL(rc, 0); 852 853 dst_iovs[1].iov_base = buf; 854 dst_iovs[1].iov_len = sizeof(buf); 855 src_iovs[1].iov_base = tmp[1]; 856 src_iovs[1].iov_len = sizeof(tmp[1]); 857 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 858 &src_iovs[1], 1, NULL, NULL, 0, 859 ut_sequence_step_cb, &completed); 860 CU_ASSERT_EQUAL(rc, 0); 861 862 ut_seq.complete = false; 863 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 864 865 poll_threads(); 866 CU_ASSERT_EQUAL(completed, 3); 867 CU_ASSERT(ut_seq.complete); 868 CU_ASSERT_EQUAL(ut_seq.status, 0); 869 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 870 871 spdk_put_io_channel(ioch); 872 poll_threads(); 873 } 874 875 static void 876 test_sequence_abort(void) 877 { 878 struct spdk_accel_sequence *seq = NULL; 879 struct spdk_io_channel *ioch; 880 char buf[4096], tmp[2][4096], expected[4096]; 881 struct iovec src_iovs[2], dst_iovs[2]; 882 int rc, completed; 883 884 ioch = spdk_accel_get_io_channel(); 885 SPDK_CU_ASSERT_FATAL(ioch != NULL); 886 887 /* Check that aborting a sequence calls operation's callback, the operation is not executed 888 * and the sequence is freed 889 */ 890 memset(buf, 0, sizeof(buf)); 891 memset(expected, 0, sizeof(buf)); 892 completed = 0; 893 seq = NULL; 894 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 895 ut_sequence_step_cb, &completed); 896 CU_ASSERT_EQUAL(rc, 0); 897 898 spdk_accel_sequence_abort(seq); 899 CU_ASSERT_EQUAL(completed, 1); 900 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 901 902 /* Check sequence with multiple operations */ 903 memset(buf, 0, sizeof(buf)); 904 memset(expected, 0, sizeof(buf)); 905 completed = 0; 906 seq = NULL; 907 908 dst_iovs[0].iov_base = tmp[1]; 909 dst_iovs[0].iov_len = sizeof(tmp[1]); 910 src_iovs[0].iov_base = tmp[0]; 911 src_iovs[0].iov_len = sizeof(tmp[0]); 912 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 913 &src_iovs[0], 1, NULL, NULL, 0, 914 ut_sequence_step_cb, &completed); 915 CU_ASSERT_EQUAL(rc, 0); 916 917 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 4096, NULL, NULL, 0xa5, 0, 918 ut_sequence_step_cb, &completed); 919 CU_ASSERT_EQUAL(rc, 0); 920 921 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xde, 0, 922 ut_sequence_step_cb, &completed); 923 CU_ASSERT_EQUAL(rc, 0); 924 925 dst_iovs[1].iov_base = buf; 926 dst_iovs[1].iov_len = sizeof(buf); 927 src_iovs[1].iov_base = tmp[1]; 928 src_iovs[1].iov_len = sizeof(tmp[1]); 929 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 930 &src_iovs[1], 1, NULL, NULL, 0, 931 ut_sequence_step_cb, &completed); 932 CU_ASSERT_EQUAL(rc, 0); 933 934 spdk_accel_sequence_abort(seq); 935 CU_ASSERT_EQUAL(completed, 4); 936 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 937 938 /* This should be a no-op */ 939 spdk_accel_sequence_abort(NULL); 940 941 spdk_put_io_channel(ioch); 942 poll_threads(); 943 } 944 945 static void 946 test_sequence_append_error(void) 947 { 948 struct spdk_accel_sequence *seq = NULL; 949 struct spdk_io_channel *ioch; 950 struct accel_io_channel *accel_ch; 951 struct iovec src_iovs, dst_iovs; 952 char buf[4096]; 953 STAILQ_HEAD(, spdk_accel_task) tasks = STAILQ_HEAD_INITIALIZER(tasks); 954 SLIST_HEAD(, spdk_accel_sequence) seqs = SLIST_HEAD_INITIALIZER(seqs); 955 int rc; 956 957 ioch = spdk_accel_get_io_channel(); 958 SPDK_CU_ASSERT_FATAL(ioch != NULL); 959 accel_ch = spdk_io_channel_get_ctx(ioch); 960 961 /* Check that append fails and no sequence object is allocated when there are no more free 962 * tasks */ 963 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 964 965 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 966 ut_sequence_step_cb, NULL); 967 CU_ASSERT_EQUAL(rc, -ENOMEM); 968 CU_ASSERT_PTR_NULL(seq); 969 970 dst_iovs.iov_base = buf; 971 dst_iovs.iov_len = 2048; 972 src_iovs.iov_base = &buf[2048]; 973 src_iovs.iov_len = 2048; 974 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 975 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 976 CU_ASSERT_EQUAL(rc, -ENOMEM); 977 CU_ASSERT_PTR_NULL(seq); 978 979 dst_iovs.iov_base = buf; 980 dst_iovs.iov_len = 2048; 981 src_iovs.iov_base = &buf[2048]; 982 src_iovs.iov_len = 2048; 983 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 984 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 985 CU_ASSERT_EQUAL(rc, -ENOMEM); 986 CU_ASSERT_PTR_NULL(seq); 987 988 /* Check that the same happens when the sequence queue is empty */ 989 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 990 SLIST_SWAP(&seqs, &accel_ch->seq_pool, spdk_accel_sequence); 991 992 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 993 ut_sequence_step_cb, NULL); 994 CU_ASSERT_EQUAL(rc, -ENOMEM); 995 CU_ASSERT_PTR_NULL(seq); 996 997 dst_iovs.iov_base = buf; 998 dst_iovs.iov_len = 2048; 999 src_iovs.iov_base = &buf[2048]; 1000 src_iovs.iov_len = 2048; 1001 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1002 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 1003 CU_ASSERT_EQUAL(rc, -ENOMEM); 1004 CU_ASSERT_PTR_NULL(seq); 1005 1006 dst_iovs.iov_base = buf; 1007 dst_iovs.iov_len = 2048; 1008 src_iovs.iov_base = &buf[2048]; 1009 src_iovs.iov_len = 2048; 1010 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1011 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 1012 CU_ASSERT_EQUAL(rc, -ENOMEM); 1013 CU_ASSERT_PTR_NULL(seq); 1014 1015 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 1016 1017 spdk_put_io_channel(ioch); 1018 poll_threads(); 1019 } 1020 1021 struct ut_sequence_operation { 1022 int complete_status; 1023 int submit_status; 1024 int count; 1025 struct iovec *src_iovs; 1026 uint32_t src_iovcnt; 1027 struct spdk_memory_domain *src_domain; 1028 void *src_domain_ctx; 1029 struct iovec *dst_iovs; 1030 uint32_t dst_iovcnt; 1031 struct spdk_memory_domain *dst_domain; 1032 void *dst_domain_ctx; 1033 int (*submit)(struct spdk_io_channel *ch, struct spdk_accel_task *t); 1034 }; 1035 1036 static struct ut_sequence_operation g_seq_operations[SPDK_ACCEL_OPC_LAST]; 1037 1038 static int 1039 ut_sequnce_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *task) 1040 { 1041 struct ut_sequence_operation *op = &g_seq_operations[task->op_code]; 1042 1043 op->count++; 1044 if (op->submit != NULL) { 1045 return op->submit(ch, task); 1046 } 1047 if (op->src_iovs != NULL) { 1048 CU_ASSERT_EQUAL(task->s.iovcnt, op->src_iovcnt); 1049 CU_ASSERT_EQUAL(memcmp(task->s.iovs, op->src_iovs, 1050 sizeof(struct iovec) * op->src_iovcnt), 0); 1051 } 1052 if (op->dst_iovs != NULL) { 1053 CU_ASSERT_EQUAL(task->d.iovcnt, op->dst_iovcnt); 1054 CU_ASSERT_EQUAL(memcmp(task->d.iovs, op->dst_iovs, 1055 sizeof(struct iovec) * op->dst_iovcnt), 0); 1056 } 1057 1058 CU_ASSERT_EQUAL(task->src_domain, op->src_domain); 1059 CU_ASSERT_EQUAL(task->dst_domain, op->dst_domain); 1060 if (op->src_domain != NULL) { 1061 CU_ASSERT_EQUAL(task->src_domain_ctx, op->src_domain_ctx); 1062 } 1063 if (op->dst_domain != NULL) { 1064 CU_ASSERT_EQUAL(task->dst_domain_ctx, op->dst_domain_ctx); 1065 } 1066 1067 if (op->submit_status != 0) { 1068 return op->submit_status; 1069 } 1070 1071 spdk_accel_task_complete(task, op->complete_status); 1072 1073 return 0; 1074 } 1075 1076 static void 1077 ut_clear_operations(void) 1078 { 1079 memset(&g_seq_operations, 0, sizeof(g_seq_operations)); 1080 } 1081 1082 static void 1083 test_sequence_completion_error(void) 1084 { 1085 struct spdk_accel_sequence *seq = NULL; 1086 struct spdk_io_channel *ioch; 1087 struct ut_sequence ut_seq; 1088 struct iovec src_iovs, dst_iovs; 1089 char buf[4096], tmp[4096]; 1090 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 1091 int i, rc, completed; 1092 1093 ioch = spdk_accel_get_io_channel(); 1094 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1095 1096 /* Override the submit_tasks function */ 1097 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1098 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1099 modules[i] = g_modules_opc[i]; 1100 g_modules_opc[i] = g_module; 1101 } 1102 1103 memset(buf, 0, sizeof(buf)); 1104 memset(tmp, 0, sizeof(tmp)); 1105 1106 /* Check that if the first operation completes with an error, the whole sequence is 1107 * completed with that error and that all operations' completion callbacks are executed 1108 */ 1109 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = -E2BIG; 1110 completed = 0; 1111 seq = NULL; 1112 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1113 ut_sequence_step_cb, &completed); 1114 CU_ASSERT_EQUAL(rc, 0); 1115 1116 dst_iovs.iov_base = buf; 1117 dst_iovs.iov_len = sizeof(buf); 1118 src_iovs.iov_base = tmp; 1119 src_iovs.iov_len = sizeof(tmp); 1120 1121 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1122 &src_iovs, 1, NULL, NULL, 0, 1123 ut_sequence_step_cb, &completed); 1124 CU_ASSERT_EQUAL(rc, 0); 1125 1126 ut_seq.complete = false; 1127 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1128 1129 poll_threads(); 1130 CU_ASSERT_EQUAL(completed, 2); 1131 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 1132 1133 /* Check the same with a second operation in the sequence */ 1134 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].complete_status = -EACCES; 1135 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 1136 completed = 0; 1137 seq = NULL; 1138 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1139 ut_sequence_step_cb, &completed); 1140 CU_ASSERT_EQUAL(rc, 0); 1141 1142 dst_iovs.iov_base = buf; 1143 dst_iovs.iov_len = sizeof(buf); 1144 src_iovs.iov_base = tmp; 1145 src_iovs.iov_len = sizeof(tmp); 1146 1147 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1148 &src_iovs, 1, NULL, NULL, 0, 1149 ut_sequence_step_cb, &completed); 1150 CU_ASSERT_EQUAL(rc, 0); 1151 1152 ut_seq.complete = false; 1153 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1154 1155 poll_threads(); 1156 CU_ASSERT_EQUAL(completed, 2); 1157 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 1158 1159 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].complete_status = 0; 1160 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 1161 1162 /* Check submission failure of the first operation */ 1163 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit_status = -EADDRINUSE; 1164 completed = 0; 1165 seq = NULL; 1166 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1167 ut_sequence_step_cb, &completed); 1168 CU_ASSERT_EQUAL(rc, 0); 1169 1170 dst_iovs.iov_base = buf; 1171 dst_iovs.iov_len = sizeof(buf); 1172 src_iovs.iov_base = tmp; 1173 src_iovs.iov_len = sizeof(tmp); 1174 1175 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1176 &src_iovs, 1, NULL, NULL, 0, 1177 ut_sequence_step_cb, &completed); 1178 CU_ASSERT_EQUAL(rc, 0); 1179 1180 ut_seq.complete = false; 1181 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1182 1183 poll_threads(); 1184 CU_ASSERT_EQUAL(completed, 2); 1185 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 1186 1187 /* Check the same with a second operation */ 1188 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit_status = -EADDRNOTAVAIL; 1189 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit_status = 0; 1190 completed = 0; 1191 seq = NULL; 1192 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1193 ut_sequence_step_cb, &completed); 1194 CU_ASSERT_EQUAL(rc, 0); 1195 1196 dst_iovs.iov_base = buf; 1197 dst_iovs.iov_len = sizeof(buf); 1198 src_iovs.iov_base = tmp; 1199 src_iovs.iov_len = sizeof(tmp); 1200 1201 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1202 &src_iovs, 1, NULL, NULL, 0, 1203 ut_sequence_step_cb, &completed); 1204 CU_ASSERT_EQUAL(rc, 0); 1205 1206 ut_seq.complete = false; 1207 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1208 1209 poll_threads(); 1210 CU_ASSERT_EQUAL(completed, 2); 1211 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 1212 1213 /* Cleanup module pointers to make subsequent tests work correctly */ 1214 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1215 g_modules_opc[i] = modules[i]; 1216 } 1217 1218 ut_clear_operations(); 1219 spdk_put_io_channel(ioch); 1220 poll_threads(); 1221 } 1222 1223 #ifdef SPDK_CONFIG_ISAL 1224 static void 1225 ut_compress_cb(void *cb_arg, int status) 1226 { 1227 int *completed = cb_arg; 1228 1229 CU_ASSERT_EQUAL(status, 0); 1230 1231 *completed = 1; 1232 } 1233 1234 static void 1235 test_sequence_decompress(void) 1236 { 1237 struct spdk_accel_sequence *seq = NULL; 1238 struct spdk_io_channel *ioch; 1239 struct ut_sequence ut_seq; 1240 char buf[4096], tmp[2][4096], expected[4096]; 1241 struct iovec src_iovs[2], dst_iovs[2]; 1242 uint32_t compressed_size; 1243 int rc, completed = 0; 1244 1245 ioch = spdk_accel_get_io_channel(); 1246 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1247 1248 memset(expected, 0xa5, sizeof(expected)); 1249 src_iovs[0].iov_base = expected; 1250 src_iovs[0].iov_len = sizeof(expected); 1251 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1252 &compressed_size, 0, ut_compress_cb, &completed); 1253 CU_ASSERT_EQUAL(rc, 0); 1254 1255 while (!completed) { 1256 poll_threads(); 1257 } 1258 1259 /* Check a single decompress operation in a sequence */ 1260 seq = NULL; 1261 completed = 0; 1262 1263 dst_iovs[0].iov_base = buf; 1264 dst_iovs[0].iov_len = sizeof(buf); 1265 src_iovs[0].iov_base = tmp[0]; 1266 src_iovs[0].iov_len = compressed_size; 1267 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1268 &src_iovs[0], 1, NULL, NULL, 0, 1269 ut_sequence_step_cb, &completed); 1270 CU_ASSERT_EQUAL(rc, 0); 1271 1272 ut_seq.complete = false; 1273 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1274 1275 poll_threads(); 1276 1277 CU_ASSERT_EQUAL(completed, 1); 1278 CU_ASSERT(ut_seq.complete); 1279 CU_ASSERT_EQUAL(ut_seq.status, 0); 1280 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1281 1282 /* Put the decompress operation in the middle of a sequence with a copy operation at the 1283 * beginning and a fill at the end modifying the first 2048B of the buffer. 1284 */ 1285 memset(expected, 0xfe, 2048); 1286 memset(buf, 0, sizeof(buf)); 1287 seq = NULL; 1288 completed = 0; 1289 1290 dst_iovs[0].iov_base = tmp[1]; 1291 dst_iovs[0].iov_len = compressed_size; 1292 src_iovs[0].iov_base = tmp[0]; 1293 src_iovs[0].iov_len = compressed_size; 1294 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1295 &src_iovs[0], 1, NULL, NULL, 0, 1296 ut_sequence_step_cb, &completed); 1297 CU_ASSERT_EQUAL(rc, 0); 1298 1299 dst_iovs[1].iov_base = buf; 1300 dst_iovs[1].iov_len = sizeof(buf); 1301 src_iovs[1].iov_base = tmp[1]; 1302 src_iovs[1].iov_len = compressed_size; 1303 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1304 &src_iovs[1], 1, NULL, NULL, 0, 1305 ut_sequence_step_cb, &completed); 1306 CU_ASSERT_EQUAL(rc, 0); 1307 1308 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1309 ut_sequence_step_cb, &completed); 1310 CU_ASSERT_EQUAL(rc, 0); 1311 1312 ut_seq.complete = false; 1313 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1314 1315 poll_threads(); 1316 1317 CU_ASSERT_EQUAL(completed, 3); 1318 CU_ASSERT(ut_seq.complete); 1319 CU_ASSERT_EQUAL(ut_seq.status, 0); 1320 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1321 1322 /* Check sequence with decompress at the beginning: decompress -> copy */ 1323 memset(expected, 0xa5, sizeof(expected)); 1324 memset(buf, 0, sizeof(buf)); 1325 seq = NULL; 1326 completed = 0; 1327 1328 dst_iovs[0].iov_base = tmp[1]; 1329 dst_iovs[0].iov_len = sizeof(tmp[1]); 1330 src_iovs[0].iov_base = tmp[0]; 1331 src_iovs[0].iov_len = compressed_size; 1332 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1333 &src_iovs[0], 1, NULL, NULL, 0, 1334 ut_sequence_step_cb, &completed); 1335 CU_ASSERT_EQUAL(rc, 0); 1336 1337 dst_iovs[1].iov_base = buf; 1338 dst_iovs[1].iov_len = sizeof(buf); 1339 src_iovs[1].iov_base = tmp[1]; 1340 src_iovs[1].iov_len = sizeof(tmp[1]); 1341 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1342 &src_iovs[1], 1, NULL, NULL, 0, 1343 ut_sequence_step_cb, &completed); 1344 CU_ASSERT_EQUAL(rc, 0); 1345 1346 ut_seq.complete = false; 1347 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1348 1349 poll_threads(); 1350 1351 CU_ASSERT_EQUAL(completed, 2); 1352 CU_ASSERT(ut_seq.complete); 1353 CU_ASSERT_EQUAL(ut_seq.status, 0); 1354 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1355 1356 spdk_put_io_channel(ioch); 1357 poll_threads(); 1358 } 1359 1360 static void 1361 test_sequence_reverse(void) 1362 { 1363 struct spdk_accel_sequence *seq = NULL; 1364 struct spdk_io_channel *ioch; 1365 struct ut_sequence ut_seq; 1366 char buf[4096], tmp[2][4096], expected[4096]; 1367 struct iovec src_iovs[2], dst_iovs[2]; 1368 uint32_t compressed_size; 1369 int rc, completed = 0; 1370 1371 ioch = spdk_accel_get_io_channel(); 1372 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1373 1374 memset(expected, 0xa5, sizeof(expected)); 1375 src_iovs[0].iov_base = expected; 1376 src_iovs[0].iov_len = sizeof(expected); 1377 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1378 &compressed_size, 0, ut_compress_cb, &completed); 1379 CU_ASSERT_EQUAL(rc, 0); 1380 1381 while (!completed) { 1382 poll_threads(); 1383 } 1384 1385 /* First check that reversing a sequnce with a single operation is a no-op */ 1386 memset(buf, 0, sizeof(buf)); 1387 seq = NULL; 1388 completed = 0; 1389 1390 dst_iovs[0].iov_base = buf; 1391 dst_iovs[0].iov_len = sizeof(buf); 1392 src_iovs[0].iov_base = tmp[0]; 1393 src_iovs[0].iov_len = compressed_size; 1394 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1395 &src_iovs[0], 1, NULL, NULL, 0, 1396 ut_sequence_step_cb, &completed); 1397 CU_ASSERT_EQUAL(rc, 0); 1398 1399 spdk_accel_sequence_reverse(seq); 1400 1401 ut_seq.complete = false; 1402 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1403 1404 poll_threads(); 1405 1406 CU_ASSERT_EQUAL(completed, 1); 1407 CU_ASSERT(ut_seq.complete); 1408 CU_ASSERT_EQUAL(ut_seq.status, 0); 1409 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1410 1411 /* Add a copy operation at the end with src set to the compressed data. After reverse(), 1412 * that copy operation should be first, so decompress() should receive compressed data in 1413 * its src buffer. 1414 */ 1415 memset(buf, 0, sizeof(buf)); 1416 memset(tmp[1], 0, sizeof(tmp[1])); 1417 seq = NULL; 1418 completed = 0; 1419 1420 dst_iovs[0].iov_base = buf; 1421 dst_iovs[0].iov_len = sizeof(buf); 1422 src_iovs[0].iov_base = tmp[1]; 1423 src_iovs[0].iov_len = compressed_size; 1424 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1425 &src_iovs[0], 1, NULL, NULL, 0, 1426 ut_sequence_step_cb, &completed); 1427 CU_ASSERT_EQUAL(rc, 0); 1428 1429 dst_iovs[1].iov_base = tmp[1]; 1430 dst_iovs[1].iov_len = compressed_size; 1431 src_iovs[1].iov_base = tmp[0]; 1432 src_iovs[1].iov_len = compressed_size; 1433 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1434 &src_iovs[1], 1, NULL, NULL, 0, 1435 ut_sequence_step_cb, &completed); 1436 CU_ASSERT_EQUAL(rc, 0); 1437 1438 spdk_accel_sequence_reverse(seq); 1439 1440 ut_seq.complete = false; 1441 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1442 1443 poll_threads(); 1444 1445 CU_ASSERT_EQUAL(completed, 2); 1446 CU_ASSERT(ut_seq.complete); 1447 CU_ASSERT_EQUAL(ut_seq.status, 0); 1448 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1449 1450 /* Check the same, but add an extra fill operation at the beginning that should execute last 1451 * after reverse(). 1452 */ 1453 memset(buf, 0, sizeof(buf)); 1454 memset(tmp[1], 0, sizeof(tmp[1])); 1455 memset(expected, 0xfe, 2048); 1456 seq = NULL; 1457 completed = 0; 1458 1459 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1460 ut_sequence_step_cb, &completed); 1461 CU_ASSERT_EQUAL(rc, 0); 1462 1463 dst_iovs[0].iov_base = buf; 1464 dst_iovs[0].iov_len = sizeof(buf); 1465 src_iovs[0].iov_base = tmp[1]; 1466 src_iovs[0].iov_len = compressed_size; 1467 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1468 &src_iovs[0], 1, NULL, NULL, 0, 1469 ut_sequence_step_cb, &completed); 1470 CU_ASSERT_EQUAL(rc, 0); 1471 1472 dst_iovs[1].iov_base = tmp[1]; 1473 dst_iovs[1].iov_len = compressed_size; 1474 src_iovs[1].iov_base = tmp[0]; 1475 src_iovs[1].iov_len = compressed_size; 1476 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1477 &src_iovs[1], 1, NULL, NULL, 0, 1478 ut_sequence_step_cb, &completed); 1479 CU_ASSERT_EQUAL(rc, 0); 1480 1481 spdk_accel_sequence_reverse(seq); 1482 1483 ut_seq.complete = false; 1484 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1485 1486 poll_threads(); 1487 1488 CU_ASSERT_EQUAL(completed, 3); 1489 CU_ASSERT(ut_seq.complete); 1490 CU_ASSERT_EQUAL(ut_seq.status, 0); 1491 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1492 1493 /* Build the sequence in order and then reverse it twice */ 1494 memset(buf, 0, sizeof(buf)); 1495 memset(tmp[1], 0, sizeof(tmp[1])); 1496 seq = NULL; 1497 completed = 0; 1498 1499 dst_iovs[0].iov_base = tmp[1]; 1500 dst_iovs[0].iov_len = compressed_size; 1501 src_iovs[0].iov_base = tmp[0]; 1502 src_iovs[0].iov_len = compressed_size; 1503 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1504 &src_iovs[0], 1, NULL, NULL, 0, 1505 ut_sequence_step_cb, &completed); 1506 CU_ASSERT_EQUAL(rc, 0); 1507 1508 dst_iovs[1].iov_base = buf; 1509 dst_iovs[1].iov_len = sizeof(buf); 1510 src_iovs[1].iov_base = tmp[1]; 1511 src_iovs[1].iov_len = compressed_size; 1512 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1513 &src_iovs[1], 1, NULL, NULL, 0, 1514 ut_sequence_step_cb, &completed); 1515 CU_ASSERT_EQUAL(rc, 0); 1516 1517 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1518 ut_sequence_step_cb, &completed); 1519 CU_ASSERT_EQUAL(rc, 0); 1520 1521 spdk_accel_sequence_reverse(seq); 1522 spdk_accel_sequence_reverse(seq); 1523 1524 ut_seq.complete = false; 1525 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1526 1527 poll_threads(); 1528 1529 CU_ASSERT_EQUAL(completed, 3); 1530 CU_ASSERT(ut_seq.complete); 1531 CU_ASSERT_EQUAL(ut_seq.status, 0); 1532 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1533 1534 spdk_put_io_channel(ioch); 1535 poll_threads(); 1536 } 1537 #endif 1538 1539 static void 1540 test_sequence_copy_elision(void) 1541 { 1542 struct spdk_accel_sequence *seq = NULL; 1543 struct spdk_io_channel *ioch; 1544 struct ut_sequence ut_seq; 1545 struct iovec src_iovs[4], dst_iovs[4], exp_iovs[2]; 1546 char buf[4096], tmp[4][4096]; 1547 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 1548 struct spdk_accel_crypto_key key = {}; 1549 int i, rc, completed; 1550 1551 ioch = spdk_accel_get_io_channel(); 1552 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1553 1554 /* Override the submit_tasks function */ 1555 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1556 g_module.supports_memory_domains = true; 1557 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1558 g_seq_operations[i].complete_status = 0; 1559 g_seq_operations[i].submit_status = 0; 1560 g_seq_operations[i].count = 0; 1561 1562 modules[i] = g_modules_opc[i]; 1563 g_modules_opc[i] = g_module; 1564 } 1565 1566 /* Check that a copy operation at the beginning is removed */ 1567 seq = NULL; 1568 completed = 0; 1569 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovcnt = 1; 1570 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovcnt = 1; 1571 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1572 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1573 exp_iovs[0].iov_base = tmp[0]; 1574 exp_iovs[0].iov_len = sizeof(tmp[0]); 1575 exp_iovs[1].iov_base = buf; 1576 exp_iovs[1].iov_len = 2048; 1577 1578 dst_iovs[0].iov_base = tmp[1]; 1579 dst_iovs[0].iov_len = sizeof(tmp[1]); 1580 src_iovs[0].iov_base = tmp[0]; 1581 src_iovs[0].iov_len = sizeof(tmp[0]); 1582 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1583 &src_iovs[0], 1, NULL, NULL, 0, 1584 ut_sequence_step_cb, &completed); 1585 CU_ASSERT_EQUAL(rc, 0); 1586 1587 dst_iovs[1].iov_base = buf; 1588 dst_iovs[1].iov_len = 2048; 1589 src_iovs[1].iov_base = tmp[1]; 1590 src_iovs[1].iov_len = sizeof(tmp[1]); 1591 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1592 &src_iovs[1], 1, NULL, NULL, 0, 1593 ut_sequence_step_cb, &completed); 1594 CU_ASSERT_EQUAL(rc, 0); 1595 1596 ut_seq.complete = false; 1597 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1598 1599 poll_threads(); 1600 1601 CU_ASSERT_EQUAL(completed, 2); 1602 CU_ASSERT(ut_seq.complete); 1603 CU_ASSERT_EQUAL(ut_seq.status, 0); 1604 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1605 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1606 1607 /* Check that a copy operation at the end is removed too */ 1608 seq = NULL; 1609 completed = 0; 1610 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1611 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1612 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1613 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1614 exp_iovs[0].iov_base = tmp[0]; 1615 exp_iovs[0].iov_len = sizeof(tmp[0]); 1616 exp_iovs[1].iov_base = buf; 1617 exp_iovs[1].iov_len = 2048; 1618 1619 dst_iovs[0].iov_base = tmp[1]; 1620 dst_iovs[0].iov_len = 2048; 1621 src_iovs[0].iov_base = tmp[0]; 1622 src_iovs[0].iov_len = sizeof(tmp[0]); 1623 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1624 &src_iovs[0], 1, NULL, NULL, 0, 1625 ut_sequence_step_cb, &completed); 1626 CU_ASSERT_EQUAL(rc, 0); 1627 1628 dst_iovs[1].iov_base = buf; 1629 dst_iovs[1].iov_len = 2048; 1630 src_iovs[1].iov_base = tmp[1]; 1631 src_iovs[1].iov_len = 2048; 1632 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1633 &src_iovs[1], 1, NULL, NULL, 0, 1634 ut_sequence_step_cb, &completed); 1635 CU_ASSERT_EQUAL(rc, 0); 1636 1637 ut_seq.complete = false; 1638 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1639 1640 poll_threads(); 1641 1642 CU_ASSERT_EQUAL(completed, 2); 1643 CU_ASSERT(ut_seq.complete); 1644 CU_ASSERT_EQUAL(ut_seq.status, 0); 1645 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1646 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1647 1648 /* Check a copy operation both at the beginning and the end */ 1649 seq = NULL; 1650 completed = 0; 1651 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1652 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1653 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1654 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1655 exp_iovs[0].iov_base = tmp[0]; 1656 exp_iovs[0].iov_len = sizeof(tmp[0]); 1657 exp_iovs[1].iov_base = buf; 1658 exp_iovs[1].iov_len = 2048; 1659 1660 dst_iovs[0].iov_base = tmp[1]; 1661 dst_iovs[0].iov_len = sizeof(tmp[1]); 1662 src_iovs[0].iov_base = tmp[0]; 1663 src_iovs[0].iov_len = sizeof(tmp[0]); 1664 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1665 &src_iovs[0], 1, NULL, NULL, 0, 1666 ut_sequence_step_cb, &completed); 1667 CU_ASSERT_EQUAL(rc, 0); 1668 1669 dst_iovs[1].iov_base = tmp[2]; 1670 dst_iovs[1].iov_len = 2048; 1671 src_iovs[1].iov_base = tmp[1]; 1672 src_iovs[1].iov_len = sizeof(tmp[1]); 1673 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1674 &src_iovs[1], 1, NULL, NULL, 0, 1675 ut_sequence_step_cb, &completed); 1676 CU_ASSERT_EQUAL(rc, 0); 1677 1678 dst_iovs[2].iov_base = buf; 1679 dst_iovs[2].iov_len = 2048; 1680 src_iovs[2].iov_base = tmp[2]; 1681 src_iovs[2].iov_len = 2048; 1682 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1683 &src_iovs[2], 1, NULL, NULL, 0, 1684 ut_sequence_step_cb, &completed); 1685 CU_ASSERT_EQUAL(rc, 0); 1686 1687 ut_seq.complete = false; 1688 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1689 1690 poll_threads(); 1691 1692 CU_ASSERT_EQUAL(completed, 3); 1693 CU_ASSERT(ut_seq.complete); 1694 CU_ASSERT_EQUAL(ut_seq.status, 0); 1695 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1696 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1697 1698 /* Check decompress + copy + decompress + copy */ 1699 seq = NULL; 1700 completed = 0; 1701 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1702 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1703 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = NULL; 1704 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = NULL; 1705 1706 dst_iovs[0].iov_base = tmp[1]; 1707 dst_iovs[0].iov_len = sizeof(tmp[1]); 1708 src_iovs[0].iov_base = tmp[0]; 1709 src_iovs[0].iov_len = sizeof(tmp[0]); 1710 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1711 &src_iovs[0], 1, NULL, NULL, 0, 1712 ut_sequence_step_cb, &completed); 1713 CU_ASSERT_EQUAL(rc, 0); 1714 1715 dst_iovs[1].iov_base = tmp[2]; 1716 dst_iovs[1].iov_len = 2048; 1717 src_iovs[1].iov_base = tmp[1]; 1718 src_iovs[1].iov_len = sizeof(tmp[1]); 1719 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1720 &src_iovs[1], 1, NULL, NULL, 0, 1721 ut_sequence_step_cb, &completed); 1722 CU_ASSERT_EQUAL(rc, 0); 1723 1724 dst_iovs[2].iov_base = tmp[3]; 1725 dst_iovs[2].iov_len = 1024; 1726 src_iovs[2].iov_base = tmp[2]; 1727 src_iovs[2].iov_len = 2048; 1728 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1729 &src_iovs[2], 1, NULL, NULL, 0, 1730 ut_sequence_step_cb, &completed); 1731 CU_ASSERT_EQUAL(rc, 0); 1732 1733 dst_iovs[3].iov_base = buf; 1734 dst_iovs[3].iov_len = 1024; 1735 src_iovs[3].iov_base = tmp[3]; 1736 src_iovs[3].iov_len = 1024; 1737 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 1738 &src_iovs[3], 1, NULL, NULL, 0, 1739 ut_sequence_step_cb, &completed); 1740 CU_ASSERT_EQUAL(rc, 0); 1741 1742 ut_seq.complete = false; 1743 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1744 1745 poll_threads(); 1746 1747 CU_ASSERT_EQUAL(completed, 4); 1748 CU_ASSERT(ut_seq.complete); 1749 CU_ASSERT_EQUAL(ut_seq.status, 0); 1750 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1751 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 2); 1752 1753 /* Check two copy operations - one of them should be removed, while the other should be 1754 * executed normally */ 1755 seq = NULL; 1756 completed = 0; 1757 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1758 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovcnt = 1; 1759 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovcnt = 1; 1760 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovs = &exp_iovs[0]; 1761 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovs = &exp_iovs[1]; 1762 exp_iovs[0].iov_base = tmp[0]; 1763 exp_iovs[0].iov_len = sizeof(tmp[0]); 1764 exp_iovs[1].iov_base = buf; 1765 exp_iovs[1].iov_len = sizeof(buf); 1766 1767 dst_iovs[0].iov_base = tmp[1]; 1768 dst_iovs[0].iov_len = sizeof(tmp[1]); 1769 src_iovs[0].iov_base = tmp[0]; 1770 src_iovs[0].iov_len = sizeof(tmp[0]); 1771 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1772 &src_iovs[0], 1, NULL, NULL, 0, 1773 ut_sequence_step_cb, &completed); 1774 CU_ASSERT_EQUAL(rc, 0); 1775 1776 dst_iovs[1].iov_base = buf; 1777 dst_iovs[1].iov_len = sizeof(buf); 1778 src_iovs[1].iov_base = tmp[1]; 1779 src_iovs[1].iov_len = sizeof(tmp[1]); 1780 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1781 &src_iovs[1], 1, NULL, NULL, 0, 1782 ut_sequence_step_cb, &completed); 1783 CU_ASSERT_EQUAL(rc, 0); 1784 1785 ut_seq.complete = false; 1786 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1787 1788 poll_threads(); 1789 1790 CU_ASSERT_EQUAL(completed, 2); 1791 CU_ASSERT(ut_seq.complete); 1792 CU_ASSERT_EQUAL(ut_seq.status, 0); 1793 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 1794 1795 /* Check fill + copy */ 1796 seq = NULL; 1797 completed = 0; 1798 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1799 g_seq_operations[SPDK_ACCEL_OPC_FILL].count = 0; 1800 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovs = NULL; 1801 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovs = NULL; 1802 g_seq_operations[SPDK_ACCEL_OPC_FILL].dst_iovcnt = 1; 1803 g_seq_operations[SPDK_ACCEL_OPC_FILL].dst_iovs = &exp_iovs[0]; 1804 exp_iovs[0].iov_base = buf; 1805 exp_iovs[0].iov_len = sizeof(buf); 1806 1807 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 1808 ut_sequence_step_cb, &completed); 1809 CU_ASSERT_EQUAL(rc, 0); 1810 1811 dst_iovs[0].iov_base = buf; 1812 dst_iovs[0].iov_len = sizeof(buf); 1813 src_iovs[0].iov_base = tmp[0]; 1814 src_iovs[0].iov_len = sizeof(tmp[0]); 1815 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1816 &src_iovs[0], 1, NULL, NULL, 0, 1817 ut_sequence_step_cb, &completed); 1818 CU_ASSERT_EQUAL(rc, 0); 1819 1820 ut_seq.complete = false; 1821 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1822 1823 poll_threads(); 1824 1825 CU_ASSERT_EQUAL(completed, 2); 1826 CU_ASSERT(ut_seq.complete); 1827 CU_ASSERT_EQUAL(ut_seq.status, 0); 1828 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1829 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 1); 1830 1831 /* Check copy + encrypt + copy */ 1832 seq = NULL; 1833 completed = 0; 1834 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1835 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 1836 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovcnt = 1; 1837 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovcnt = 1; 1838 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovs = &exp_iovs[0]; 1839 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovs = &exp_iovs[1]; 1840 exp_iovs[0].iov_base = tmp[0]; 1841 exp_iovs[0].iov_len = sizeof(tmp[0]); 1842 exp_iovs[1].iov_base = buf; 1843 exp_iovs[1].iov_len = sizeof(buf); 1844 1845 dst_iovs[0].iov_base = tmp[1]; 1846 dst_iovs[0].iov_len = sizeof(tmp[1]); 1847 src_iovs[0].iov_base = tmp[0]; 1848 src_iovs[0].iov_len = sizeof(tmp[0]); 1849 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1850 &src_iovs[0], 1, NULL, NULL, 0, 1851 ut_sequence_step_cb, &completed); 1852 CU_ASSERT_EQUAL(rc, 0); 1853 1854 dst_iovs[1].iov_base = tmp[2]; 1855 dst_iovs[1].iov_len = sizeof(tmp[2]); 1856 src_iovs[1].iov_base = tmp[1]; 1857 src_iovs[1].iov_len = sizeof(tmp[1]); 1858 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1859 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1860 ut_sequence_step_cb, &completed); 1861 CU_ASSERT_EQUAL(rc, 0); 1862 1863 dst_iovs[2].iov_base = buf; 1864 dst_iovs[2].iov_len = sizeof(buf); 1865 src_iovs[2].iov_base = tmp[2]; 1866 src_iovs[2].iov_len = sizeof(tmp[2]); 1867 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1868 &src_iovs[2], 1, NULL, NULL, 0, 1869 ut_sequence_step_cb, &completed); 1870 CU_ASSERT_EQUAL(rc, 0); 1871 1872 ut_seq.complete = false; 1873 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1874 1875 poll_threads(); 1876 1877 CU_ASSERT_EQUAL(completed, 3); 1878 CU_ASSERT(ut_seq.complete); 1879 CU_ASSERT_EQUAL(ut_seq.status, 0); 1880 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1881 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 1882 1883 /* Check copy + decrypt + copy */ 1884 seq = NULL; 1885 completed = 0; 1886 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1887 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 1888 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1889 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1890 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1891 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1892 exp_iovs[0].iov_base = tmp[0]; 1893 exp_iovs[0].iov_len = sizeof(tmp[0]); 1894 exp_iovs[1].iov_base = buf; 1895 exp_iovs[1].iov_len = sizeof(buf); 1896 1897 dst_iovs[0].iov_base = tmp[1]; 1898 dst_iovs[0].iov_len = sizeof(tmp[1]); 1899 src_iovs[0].iov_base = tmp[0]; 1900 src_iovs[0].iov_len = sizeof(tmp[0]); 1901 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1902 &src_iovs[0], 1, NULL, NULL, 0, 1903 ut_sequence_step_cb, &completed); 1904 CU_ASSERT_EQUAL(rc, 0); 1905 1906 dst_iovs[1].iov_base = tmp[2]; 1907 dst_iovs[1].iov_len = sizeof(tmp[2]); 1908 src_iovs[1].iov_base = tmp[1]; 1909 src_iovs[1].iov_len = sizeof(tmp[1]); 1910 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1911 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1912 ut_sequence_step_cb, &completed); 1913 CU_ASSERT_EQUAL(rc, 0); 1914 1915 dst_iovs[2].iov_base = buf; 1916 dst_iovs[2].iov_len = sizeof(buf); 1917 src_iovs[2].iov_base = tmp[2]; 1918 src_iovs[2].iov_len = sizeof(tmp[2]); 1919 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1920 &src_iovs[2], 1, NULL, NULL, 0, 1921 ut_sequence_step_cb, &completed); 1922 CU_ASSERT_EQUAL(rc, 0); 1923 1924 ut_seq.complete = false; 1925 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1926 1927 poll_threads(); 1928 1929 CU_ASSERT_EQUAL(completed, 3); 1930 CU_ASSERT(ut_seq.complete); 1931 CU_ASSERT_EQUAL(ut_seq.status, 0); 1932 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1933 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 1934 1935 /* Check a sequence with memory domains and verify their pointers (and their contexts) are 1936 * correctly set on the next/prev task when a copy is removed */ 1937 seq = NULL; 1938 completed = 0; 1939 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1940 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 1941 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1942 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1943 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1944 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1945 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_domain = (void *)0x1; 1946 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_domain_ctx = (void *)0x2; 1947 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_domain = (void *)0x3; 1948 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_domain_ctx = (void *)0x4; 1949 exp_iovs[0].iov_base = tmp[0]; 1950 exp_iovs[0].iov_len = sizeof(tmp[0]); 1951 exp_iovs[1].iov_base = buf; 1952 exp_iovs[1].iov_len = sizeof(buf); 1953 1954 dst_iovs[0].iov_base = tmp[1]; 1955 dst_iovs[0].iov_len = sizeof(tmp[1]); 1956 src_iovs[0].iov_base = tmp[0]; 1957 src_iovs[0].iov_len = sizeof(tmp[0]); 1958 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, (void *)0xdead, (void *)0xbeef, 1959 &src_iovs[0], 1, (void *)0x3, (void *)0x4, 0, 1960 ut_sequence_step_cb, &completed); 1961 CU_ASSERT_EQUAL(rc, 0); 1962 1963 dst_iovs[1].iov_base = tmp[2]; 1964 dst_iovs[1].iov_len = sizeof(tmp[2]); 1965 src_iovs[1].iov_base = tmp[1]; 1966 src_iovs[1].iov_len = sizeof(tmp[1]); 1967 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 1968 &src_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 0, 1969 sizeof(tmp[2]), 0, ut_sequence_step_cb, &completed); 1970 CU_ASSERT_EQUAL(rc, 0); 1971 1972 dst_iovs[2].iov_base = buf; 1973 dst_iovs[2].iov_len = sizeof(buf); 1974 src_iovs[2].iov_base = tmp[2]; 1975 src_iovs[2].iov_len = sizeof(tmp[2]); 1976 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, (void *)0x1, (void *)0x2, 1977 &src_iovs[2], 1, (void *)0xdead, (void *)0xbeef, 0, 1978 ut_sequence_step_cb, &completed); 1979 CU_ASSERT_EQUAL(rc, 0); 1980 1981 ut_seq.complete = false; 1982 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1983 1984 poll_threads(); 1985 1986 CU_ASSERT_EQUAL(completed, 3); 1987 CU_ASSERT(ut_seq.complete); 1988 CU_ASSERT_EQUAL(ut_seq.status, 0); 1989 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1990 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 1991 1992 /* Cleanup module pointers to make subsequent tests work correctly */ 1993 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1994 g_modules_opc[i] = modules[i]; 1995 } 1996 1997 g_module.supports_memory_domains = false; 1998 ut_clear_operations(); 1999 spdk_put_io_channel(ioch); 2000 poll_threads(); 2001 } 2002 2003 static int 2004 ut_submit_decompress(struct spdk_io_channel *ch, struct spdk_accel_task *task) 2005 { 2006 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 2007 2008 spdk_accel_task_complete(task, 0); 2009 2010 return 0; 2011 } 2012 2013 static void 2014 test_sequence_accel_buffers(void) 2015 { 2016 struct spdk_accel_sequence *seq = NULL; 2017 struct spdk_io_channel *ioch; 2018 struct accel_io_channel *accel_ch; 2019 struct ut_sequence ut_seq; 2020 struct iovec src_iovs[3], dst_iovs[3]; 2021 char srcbuf[4096], dstbuf[4096], expected[4096]; 2022 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2023 void *buf[2], *domain_ctx[2], *iobuf_buf; 2024 struct spdk_memory_domain *domain[2]; 2025 struct spdk_iobuf_buffer *cache_entry; 2026 spdk_iobuf_buffer_stailq_t small_cache; 2027 uint32_t small_cache_count; 2028 int i, rc, completed; 2029 struct spdk_iobuf_opts opts_iobuf = {}; 2030 2031 /* Set up the iobuf to always use the "small" pool */ 2032 opts_iobuf.large_bufsize = 0x20000; 2033 opts_iobuf.large_pool_count = 0; 2034 opts_iobuf.small_bufsize = 0x10000; 2035 opts_iobuf.small_pool_count = 32; 2036 2037 rc = spdk_iobuf_set_opts(&opts_iobuf); 2038 CU_ASSERT(rc == 0); 2039 2040 ioch = spdk_accel_get_io_channel(); 2041 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2042 2043 /* Override the submit_tasks function */ 2044 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2045 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2046 modules[i] = g_modules_opc[i]; 2047 g_modules_opc[i] = g_module; 2048 } 2049 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2050 * decompress operations together in one sequence. 2051 */ 2052 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2053 g_seq_operations[SPDK_ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2054 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2055 2056 /* Check the simplest case: one operation using accel buffer as destination + copy operation 2057 * specifying the actual destination buffer 2058 */ 2059 memset(srcbuf, 0xa5, 4096); 2060 memset(dstbuf, 0x0, 4096); 2061 memset(expected, 0xa5, 4096); 2062 completed = 0; 2063 seq = NULL; 2064 2065 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2066 CU_ASSERT_EQUAL(rc, 0); 2067 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2068 2069 src_iovs[0].iov_base = srcbuf; 2070 src_iovs[0].iov_len = 4096; 2071 dst_iovs[0].iov_base = buf[0]; 2072 dst_iovs[0].iov_len = 4096; 2073 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2074 &src_iovs[0], 1, NULL, NULL, 0, 2075 ut_sequence_step_cb, &completed); 2076 CU_ASSERT_EQUAL(rc, 0); 2077 2078 src_iovs[1].iov_base = buf[0]; 2079 src_iovs[1].iov_len = 4096; 2080 dst_iovs[1].iov_base = dstbuf; 2081 dst_iovs[1].iov_len = 4096; 2082 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2083 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 2084 ut_sequence_step_cb, &completed); 2085 CU_ASSERT_EQUAL(rc, 0); 2086 2087 ut_seq.complete = false; 2088 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2089 2090 poll_threads(); 2091 2092 CU_ASSERT_EQUAL(completed, 2); 2093 CU_ASSERT(ut_seq.complete); 2094 CU_ASSERT_EQUAL(ut_seq.status, 0); 2095 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2096 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2097 2098 /* Start with a fill operation using accel buffer, followed by a decompress using another 2099 * accel buffer as dst, followed by a copy operation specifying dst buffer of the whole 2100 * sequence 2101 */ 2102 memset(srcbuf, 0x0, 4096); 2103 memset(dstbuf, 0x0, 4096); 2104 memset(expected, 0x5a, 4096); 2105 completed = 0; 2106 seq = NULL; 2107 2108 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2109 CU_ASSERT_EQUAL(rc, 0); 2110 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2111 2112 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2113 ut_sequence_step_cb, &completed); 2114 CU_ASSERT_EQUAL(rc, 0); 2115 2116 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2117 CU_ASSERT_EQUAL(rc, 0); 2118 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2119 2120 src_iovs[0].iov_base = buf[0]; 2121 src_iovs[0].iov_len = 4096; 2122 dst_iovs[0].iov_base = buf[1]; 2123 dst_iovs[0].iov_len = 4096; 2124 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2125 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2126 ut_sequence_step_cb, &completed); 2127 CU_ASSERT_EQUAL(rc, 0); 2128 2129 src_iovs[1].iov_base = buf[1]; 2130 src_iovs[1].iov_len = 4096; 2131 dst_iovs[1].iov_base = dstbuf; 2132 dst_iovs[1].iov_len = 4096; 2133 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2134 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2135 ut_sequence_step_cb, &completed); 2136 2137 ut_seq.complete = false; 2138 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2139 2140 poll_threads(); 2141 2142 CU_ASSERT_EQUAL(completed, 3); 2143 CU_ASSERT(ut_seq.complete); 2144 CU_ASSERT_EQUAL(ut_seq.status, 0); 2145 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2146 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2147 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2148 2149 /* Check the same, but with two decompress operations with the first one being in-place */ 2150 memset(srcbuf, 0x0, 4096); 2151 memset(dstbuf, 0x0, 4096); 2152 memset(expected, 0x5a, 4096); 2153 completed = 0; 2154 seq = NULL; 2155 2156 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2157 CU_ASSERT_EQUAL(rc, 0); 2158 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2159 2160 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2161 ut_sequence_step_cb, &completed); 2162 CU_ASSERT_EQUAL(rc, 0); 2163 2164 src_iovs[0].iov_base = buf[0]; 2165 src_iovs[0].iov_len = 4096; 2166 dst_iovs[0].iov_base = buf[0]; 2167 dst_iovs[0].iov_len = 4096; 2168 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2169 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2170 ut_sequence_step_cb, &completed); 2171 CU_ASSERT_EQUAL(rc, 0); 2172 2173 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2174 CU_ASSERT_EQUAL(rc, 0); 2175 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2176 2177 src_iovs[1].iov_base = buf[0]; 2178 src_iovs[1].iov_len = 4096; 2179 dst_iovs[1].iov_base = buf[1]; 2180 dst_iovs[1].iov_len = 4096; 2181 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, domain[1], domain_ctx[1], 2182 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 2183 ut_sequence_step_cb, &completed); 2184 CU_ASSERT_EQUAL(rc, 0); 2185 2186 src_iovs[2].iov_base = buf[1]; 2187 src_iovs[2].iov_len = 4096; 2188 dst_iovs[2].iov_base = dstbuf; 2189 dst_iovs[2].iov_len = 4096; 2190 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 2191 &src_iovs[2], 1, domain[1], domain_ctx[1], 0, 2192 ut_sequence_step_cb, &completed); 2193 2194 ut_seq.complete = false; 2195 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2196 2197 poll_threads(); 2198 2199 CU_ASSERT_EQUAL(completed, 4); 2200 CU_ASSERT(ut_seq.complete); 2201 CU_ASSERT_EQUAL(ut_seq.status, 0); 2202 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2203 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2204 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2205 2206 /* Check that specifying offsets within accel buffers works correctly */ 2207 memset(srcbuf, 0x0, 4096); 2208 memset(dstbuf, 0x0, 4096); 2209 completed = 0; 2210 seq = NULL; 2211 2212 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2213 CU_ASSERT_EQUAL(rc, 0); 2214 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2215 2216 /* Fill in each 1K of the buffer with different pattern */ 2217 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2218 ut_sequence_step_cb, &completed); 2219 CU_ASSERT_EQUAL(rc, 0); 2220 2221 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2222 0x5a, 0, ut_sequence_step_cb, &completed); 2223 CU_ASSERT_EQUAL(rc, 0); 2224 2225 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2226 0xfe, 0, ut_sequence_step_cb, &completed); 2227 CU_ASSERT_EQUAL(rc, 0); 2228 2229 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2230 0xed, 0, ut_sequence_step_cb, &completed); 2231 CU_ASSERT_EQUAL(rc, 0); 2232 2233 src_iovs[0].iov_base = buf[0]; 2234 src_iovs[0].iov_len = 4096; 2235 dst_iovs[0].iov_base = dstbuf; 2236 dst_iovs[0].iov_len = 4096; 2237 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2238 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2239 ut_sequence_step_cb, &completed); 2240 2241 ut_seq.complete = false; 2242 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2243 2244 poll_threads(); 2245 2246 memset(&expected[0], 0xa5, 1024); 2247 memset(&expected[1024], 0x5a, 1024); 2248 memset(&expected[2048], 0xfe, 1024); 2249 memset(&expected[3072], 0xed, 1024); 2250 CU_ASSERT_EQUAL(completed, 5); 2251 CU_ASSERT(ut_seq.complete); 2252 CU_ASSERT_EQUAL(ut_seq.status, 0); 2253 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2254 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2255 2256 /* Check the same but this time with a decompress operation on part of the buffer (512B 2257 * offset) */ 2258 memset(srcbuf, 0x0, 4096); 2259 memset(dstbuf, 0x0, 4096); 2260 completed = 0; 2261 seq = NULL; 2262 2263 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2264 CU_ASSERT_EQUAL(rc, 0); 2265 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2266 2267 /* Fill in each 1K of the buffer with different pattern */ 2268 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2269 ut_sequence_step_cb, &completed); 2270 CU_ASSERT_EQUAL(rc, 0); 2271 2272 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2273 0x5a, 0, ut_sequence_step_cb, &completed); 2274 CU_ASSERT_EQUAL(rc, 0); 2275 2276 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2277 0xfe, 0, ut_sequence_step_cb, &completed); 2278 CU_ASSERT_EQUAL(rc, 0); 2279 2280 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2281 0xed, 0, ut_sequence_step_cb, &completed); 2282 CU_ASSERT_EQUAL(rc, 0); 2283 2284 rc = spdk_accel_get_buf(ioch, 3072, &buf[1], &domain[1], &domain_ctx[1]); 2285 CU_ASSERT_EQUAL(rc, 0); 2286 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2287 2288 src_iovs[0].iov_base = (char *)buf[0] + 512; 2289 src_iovs[0].iov_len = 3072; 2290 dst_iovs[0].iov_base = (char *)buf[1] + 256; 2291 dst_iovs[0].iov_len = 3072; 2292 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2293 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2294 ut_sequence_step_cb, &completed); 2295 2296 src_iovs[1].iov_base = (char *)buf[1] + 256; 2297 src_iovs[1].iov_len = 3072; 2298 dst_iovs[1].iov_base = dstbuf; 2299 dst_iovs[1].iov_len = 3072; 2300 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2301 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2302 ut_sequence_step_cb, &completed); 2303 2304 ut_seq.complete = false; 2305 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2306 2307 poll_threads(); 2308 2309 memset(&expected[0], 0xa5, 512); 2310 memset(&expected[512], 0x5a, 1024); 2311 memset(&expected[1536], 0xfe, 1024); 2312 memset(&expected[2560], 0xed, 512); 2313 CU_ASSERT_EQUAL(completed, 6); 2314 CU_ASSERT(ut_seq.complete); 2315 CU_ASSERT_EQUAL(ut_seq.status, 0); 2316 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 3072), 0); 2317 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2318 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2319 2320 /* Check that if iobuf pool is empty, the sequence processing will wait until a buffer is 2321 * available 2322 */ 2323 accel_ch = spdk_io_channel_get_ctx(ioch); 2324 small_cache_count = accel_ch->iobuf.small.cache_count; 2325 STAILQ_INIT(&small_cache); 2326 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2327 accel_ch->iobuf.small.cache_count = 0; 2328 accel_ch->iobuf.small.cache_size = 0; 2329 g_iobuf.small_pool_count = 0; 2330 2331 /* First allocate a single buffer used by two operations */ 2332 memset(srcbuf, 0x0, 4096); 2333 memset(dstbuf, 0x0, 4096); 2334 memset(expected, 0xa5, 4096); 2335 completed = 0; 2336 seq = NULL; 2337 2338 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2339 CU_ASSERT_EQUAL(rc, 0); 2340 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2341 2342 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0xa5, 0, 2343 ut_sequence_step_cb, &completed); 2344 CU_ASSERT_EQUAL(rc, 0); 2345 2346 src_iovs[0].iov_base = buf[0]; 2347 src_iovs[0].iov_len = 4096; 2348 dst_iovs[0].iov_base = dstbuf; 2349 dst_iovs[0].iov_len = 4096; 2350 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2351 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2352 ut_sequence_step_cb, &completed); 2353 2354 ut_seq.complete = false; 2355 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2356 2357 poll_threads(); 2358 2359 CU_ASSERT_EQUAL(completed, 0); 2360 CU_ASSERT(!ut_seq.complete); 2361 2362 /* Get a buffer and return it to the pool to trigger the sequence to finish */ 2363 g_iobuf.small_pool_count = 1; 2364 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2365 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2366 CU_ASSERT(g_iobuf.small_pool_count == 0); 2367 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2368 2369 poll_threads(); 2370 2371 CU_ASSERT_EQUAL(completed, 2); 2372 CU_ASSERT(ut_seq.complete); 2373 CU_ASSERT_EQUAL(ut_seq.status, 0); 2374 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2375 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2376 2377 /* Return the buffers back to the cache */ 2378 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2379 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2380 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2381 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2382 small_cache_count++; 2383 } 2384 accel_ch->iobuf.small.cache_count = 0; 2385 g_iobuf.small_pool_count = 0; 2386 2387 /* Check a bit more complex scenario, with two buffers in the sequence */ 2388 memset(srcbuf, 0x0, 4096); 2389 memset(dstbuf, 0x0, 4096); 2390 memset(expected, 0x5a, 4096); 2391 completed = 0; 2392 seq = NULL; 2393 2394 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2395 CU_ASSERT_EQUAL(rc, 0); 2396 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2397 2398 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2399 ut_sequence_step_cb, &completed); 2400 CU_ASSERT_EQUAL(rc, 0); 2401 2402 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2403 CU_ASSERT_EQUAL(rc, 0); 2404 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2405 2406 src_iovs[0].iov_base = buf[0]; 2407 src_iovs[0].iov_len = 4096; 2408 dst_iovs[0].iov_base = buf[1]; 2409 dst_iovs[0].iov_len = 4096; 2410 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2411 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2412 ut_sequence_step_cb, &completed); 2413 CU_ASSERT_EQUAL(rc, 0); 2414 2415 src_iovs[1].iov_base = buf[1]; 2416 src_iovs[1].iov_len = 4096; 2417 dst_iovs[1].iov_base = dstbuf; 2418 dst_iovs[1].iov_len = 4096; 2419 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2420 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2421 ut_sequence_step_cb, &completed); 2422 CU_ASSERT_EQUAL(rc, 0); 2423 2424 ut_seq.complete = false; 2425 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2426 2427 poll_threads(); 2428 2429 CU_ASSERT_EQUAL(completed, 0); 2430 CU_ASSERT(!ut_seq.complete); 2431 2432 g_iobuf.small_pool_count = 1; 2433 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2434 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2435 g_iobuf.small_pool_count = 0; 2436 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2437 2438 /* One buffer is not enough to finish the whole sequence */ 2439 poll_threads(); 2440 CU_ASSERT(!ut_seq.complete); 2441 2442 g_iobuf.small_pool_count = 1; 2443 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2444 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2445 g_iobuf.small_pool_count = 0; 2446 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2447 2448 poll_threads(); 2449 2450 CU_ASSERT_EQUAL(completed, 3); 2451 CU_ASSERT(ut_seq.complete); 2452 CU_ASSERT_EQUAL(ut_seq.status, 0); 2453 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2454 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2455 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2456 2457 /* Return the buffers back to the cache */ 2458 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2459 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2460 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2461 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2462 small_cache_count++; 2463 } 2464 accel_ch->iobuf.small.cache_count = 0; 2465 2466 g_iobuf.small_pool_count = 32; 2467 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2468 accel_ch->iobuf.small.cache_count = small_cache_count; 2469 2470 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2471 g_modules_opc[i] = modules[i]; 2472 } 2473 2474 ut_clear_operations(); 2475 spdk_put_io_channel(ioch); 2476 poll_threads(); 2477 } 2478 2479 static void 2480 ut_domain_ctx_init(struct ut_domain_ctx *ctx, void *base, size_t len, struct iovec *expected) 2481 { 2482 ctx->iov.iov_base = base; 2483 ctx->iov.iov_len = len; 2484 ctx->expected = *expected; 2485 ctx->pull_submit_status = 0; 2486 ctx->push_submit_status = 0; 2487 ctx->pull_complete_status = 0; 2488 ctx->push_complete_status = 0; 2489 } 2490 2491 static void 2492 test_sequence_memory_domain(void) 2493 { 2494 struct spdk_accel_sequence *seq = NULL; 2495 struct spdk_io_channel *ioch; 2496 struct accel_io_channel *accel_ch; 2497 struct ut_sequence ut_seq; 2498 struct ut_domain_ctx domctx[4]; 2499 struct spdk_iobuf_buffer *cache_entry; 2500 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2501 struct spdk_memory_domain *accel_domain; 2502 spdk_iobuf_buffer_stailq_t small_cache; 2503 char srcbuf[4096], dstbuf[4096], expected[4096], tmp[4096]; 2504 struct iovec src_iovs[2], dst_iovs[2]; 2505 uint32_t small_cache_count; 2506 void *accel_buf, *accel_domain_ctx, *iobuf_buf; 2507 int i, rc, completed; 2508 2509 ioch = spdk_accel_get_io_channel(); 2510 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2511 2512 /* Override the submit_tasks function */ 2513 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2514 g_module.supports_memory_domains = false; 2515 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2516 modules[i] = g_modules_opc[i]; 2517 g_modules_opc[i] = g_module; 2518 } 2519 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2520 * decompress operations together in one sequence. 2521 */ 2522 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2523 g_seq_operations[SPDK_ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2524 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2525 2526 /* First check the simplest case - single fill operation with dstbuf in domain */ 2527 memset(expected, 0xa5, sizeof(expected)); 2528 memset(dstbuf, 0x0, sizeof(dstbuf)); 2529 completed = 0; 2530 seq = NULL; 2531 2532 /* Use some garbage pointer as dst and use domain ctx to get the actual dstbuf */ 2533 dst_iovs[0].iov_base = (void *)0xcafebabe; 2534 dst_iovs[0].iov_len = sizeof(dstbuf); 2535 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2536 2537 rc = spdk_accel_append_fill(&seq, ioch, dst_iovs[0].iov_base, dst_iovs[0].iov_len, 2538 g_ut_domain, &domctx[0], 0xa5, 0, 2539 ut_sequence_step_cb, &completed); 2540 CU_ASSERT_EQUAL(rc, 0); 2541 2542 ut_seq.complete = false; 2543 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2544 2545 poll_threads(); 2546 CU_ASSERT_EQUAL(completed, 1); 2547 CU_ASSERT(ut_seq.complete); 2548 CU_ASSERT_EQUAL(ut_seq.status, 0); 2549 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2550 2551 /* Check operation with both srcbuf and dstbuf in remote memory domain */ 2552 memset(expected, 0x5a, sizeof(expected)); 2553 memset(dstbuf, 0x0, sizeof(dstbuf)); 2554 memset(srcbuf, 0x5a, sizeof(dstbuf)); 2555 completed = 0; 2556 seq = NULL; 2557 2558 src_iovs[0].iov_base = (void *)0xcafebabe; 2559 src_iovs[0].iov_len = sizeof(srcbuf); 2560 dst_iovs[0].iov_base = (void *)0xbeefdead; 2561 dst_iovs[0].iov_len = sizeof(dstbuf); 2562 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2563 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2564 2565 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2566 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2567 ut_sequence_step_cb, &completed); 2568 CU_ASSERT_EQUAL(rc, 0); 2569 2570 ut_seq.complete = false; 2571 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2572 2573 poll_threads(); 2574 CU_ASSERT_EQUAL(completed, 1); 2575 CU_ASSERT(ut_seq.complete); 2576 CU_ASSERT_EQUAL(ut_seq.status, 0); 2577 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2578 2579 /* Two operations using a buffer in remote domain */ 2580 memset(expected, 0xa5, sizeof(expected)); 2581 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2582 memset(tmp, 0x0, sizeof(tmp)); 2583 memset(dstbuf, 0x0, sizeof(dstbuf)); 2584 completed = 0; 2585 seq = NULL; 2586 2587 src_iovs[0].iov_base = (void *)0xcafebabe; 2588 src_iovs[0].iov_len = sizeof(srcbuf); 2589 dst_iovs[0].iov_base = (void *)0xbeefdead; 2590 dst_iovs[0].iov_len = sizeof(tmp); 2591 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2592 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2593 2594 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2595 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2596 ut_sequence_step_cb, &completed); 2597 CU_ASSERT_EQUAL(rc, 0); 2598 2599 src_iovs[1].iov_base = (void *)0xbeefdead; 2600 src_iovs[1].iov_len = sizeof(tmp); 2601 dst_iovs[1].iov_base = (void *)0xa5a5a5a5; 2602 dst_iovs[1].iov_len = sizeof(dstbuf); 2603 ut_domain_ctx_init(&domctx[2], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2604 ut_domain_ctx_init(&domctx[3], tmp, sizeof(tmp), &src_iovs[1]); 2605 2606 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[2], 2607 &src_iovs[1], 1, g_ut_domain, &domctx[3], 0, 2608 ut_sequence_step_cb, &completed); 2609 CU_ASSERT_EQUAL(rc, 0); 2610 2611 ut_seq.complete = false; 2612 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2613 2614 poll_threads(); 2615 CU_ASSERT_EQUAL(completed, 2); 2616 CU_ASSERT(ut_seq.complete); 2617 CU_ASSERT_EQUAL(ut_seq.status, 0); 2618 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2619 2620 /* Use buffer in remote memory domain and accel buffer at the same time */ 2621 memset(expected, 0x5a, sizeof(expected)); 2622 memset(srcbuf, 0x5a, sizeof(expected)); 2623 memset(dstbuf, 0x0, sizeof(dstbuf)); 2624 completed = 0; 2625 seq = NULL; 2626 2627 rc = spdk_accel_get_buf(ioch, sizeof(dstbuf), &accel_buf, &accel_domain, &accel_domain_ctx); 2628 CU_ASSERT_EQUAL(rc, 0); 2629 2630 src_iovs[0].iov_base = (void *)0xfeedbeef; 2631 src_iovs[0].iov_len = sizeof(srcbuf); 2632 dst_iovs[0].iov_base = accel_buf; 2633 dst_iovs[0].iov_len = sizeof(dstbuf); 2634 ut_domain_ctx_init(&domctx[0], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2635 2636 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, accel_domain, accel_domain_ctx, 2637 &src_iovs[0], 1, g_ut_domain, &domctx[0], 0, 2638 ut_sequence_step_cb, &completed); 2639 CU_ASSERT_EQUAL(rc, 0); 2640 2641 src_iovs[1].iov_base = accel_buf; 2642 src_iovs[1].iov_len = sizeof(dstbuf); 2643 dst_iovs[1].iov_base = (void *)0xbeeffeed; 2644 dst_iovs[1].iov_len = sizeof(dstbuf); 2645 ut_domain_ctx_init(&domctx[1], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2646 2647 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[1], 2648 &src_iovs[1], 1, accel_domain, accel_domain_ctx, 0, 2649 ut_sequence_step_cb, &completed); 2650 CU_ASSERT_EQUAL(rc, 0); 2651 2652 ut_seq.complete = false; 2653 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2654 2655 poll_threads(); 2656 CU_ASSERT_EQUAL(completed, 2); 2657 CU_ASSERT(ut_seq.complete); 2658 CU_ASSERT_EQUAL(ut_seq.status, 0); 2659 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2660 spdk_accel_put_buf(ioch, accel_buf, accel_domain, accel_domain_ctx); 2661 2662 /* Check that a sequence with memory domains is correctly executed if buffers are not 2663 * immediately available */ 2664 memset(expected, 0xa5, sizeof(expected)); 2665 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2666 memset(dstbuf, 0x0, sizeof(dstbuf)); 2667 completed = 0; 2668 seq = NULL; 2669 /* Make sure the buffer pool is empty */ 2670 accel_ch = spdk_io_channel_get_ctx(ioch); 2671 small_cache_count = accel_ch->iobuf.small.cache_count; 2672 STAILQ_INIT(&small_cache); 2673 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2674 accel_ch->iobuf.small.cache_count = 0; 2675 g_iobuf.small_pool_count = 0; 2676 2677 src_iovs[0].iov_base = (void *)0xdeadbeef; 2678 src_iovs[0].iov_len = sizeof(srcbuf); 2679 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2680 dst_iovs[0].iov_len = sizeof(dstbuf); 2681 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2682 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2683 2684 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2685 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2686 ut_sequence_step_cb, &completed); 2687 CU_ASSERT_EQUAL(rc, 0); 2688 2689 ut_seq.complete = false; 2690 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2691 2692 poll_threads(); 2693 CU_ASSERT_EQUAL(completed, 0); 2694 CU_ASSERT(!ut_seq.complete); 2695 2696 /* Get a buffer and return it to the pool to trigger the sequence to resume. It shouldn't 2697 * be able to complete, as it needs two buffers */ 2698 g_iobuf.small_pool_count = 1; 2699 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2700 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2701 g_iobuf.small_pool_count = 0; 2702 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2703 2704 CU_ASSERT_EQUAL(completed, 0); 2705 CU_ASSERT(!ut_seq.complete); 2706 2707 /* Return another buffer, this time the sequence should finish */ 2708 g_iobuf.small_pool_count = 1; 2709 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2710 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2711 g_iobuf.small_pool_count = 0; 2712 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2713 2714 poll_threads(); 2715 CU_ASSERT_EQUAL(completed, 1); 2716 CU_ASSERT(ut_seq.complete); 2717 CU_ASSERT_EQUAL(ut_seq.status, 0); 2718 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2719 2720 /* Return the buffers back to the cache */ 2721 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2722 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2723 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2724 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2725 small_cache_count++; 2726 } 2727 accel_ch->iobuf.small.cache_count = 0; 2728 2729 g_iobuf.small_pool_count = 32; 2730 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2731 accel_ch->iobuf.small.cache_count = small_cache_count; 2732 2733 /* Check error cases, starting with an error from spdk_memory_domain_pull_data() */ 2734 completed = 0; 2735 seq = NULL; 2736 2737 src_iovs[0].iov_base = (void *)0xdeadbeef; 2738 src_iovs[0].iov_len = sizeof(srcbuf); 2739 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2740 dst_iovs[0].iov_len = sizeof(dstbuf); 2741 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2742 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2743 domctx[1].pull_submit_status = -E2BIG; 2744 2745 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2746 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2747 ut_sequence_step_cb, &completed); 2748 CU_ASSERT_EQUAL(rc, 0); 2749 2750 ut_seq.complete = false; 2751 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2752 2753 CU_ASSERT_EQUAL(completed, 1); 2754 CU_ASSERT(ut_seq.complete); 2755 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 2756 2757 /* Check completion error from spdk_memory_domain_pull_data() */ 2758 completed = 0; 2759 seq = NULL; 2760 2761 src_iovs[0].iov_base = (void *)0xdeadbeef; 2762 src_iovs[0].iov_len = sizeof(srcbuf); 2763 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2764 dst_iovs[0].iov_len = sizeof(dstbuf); 2765 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2766 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2767 domctx[1].pull_complete_status = -EACCES; 2768 2769 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2770 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2771 ut_sequence_step_cb, &completed); 2772 CU_ASSERT_EQUAL(rc, 0); 2773 2774 ut_seq.complete = false; 2775 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2776 2777 CU_ASSERT_EQUAL(completed, 1); 2778 CU_ASSERT(ut_seq.complete); 2779 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 2780 2781 /* Check submission error from spdk_memory_domain_push_data() */ 2782 completed = 0; 2783 seq = NULL; 2784 2785 src_iovs[0].iov_base = (void *)0xdeadbeef; 2786 src_iovs[0].iov_len = sizeof(srcbuf); 2787 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2788 dst_iovs[0].iov_len = sizeof(dstbuf); 2789 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2790 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2791 domctx[0].push_submit_status = -EADDRINUSE; 2792 2793 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2794 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2795 ut_sequence_step_cb, &completed); 2796 CU_ASSERT_EQUAL(rc, 0); 2797 2798 ut_seq.complete = false; 2799 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2800 2801 CU_ASSERT_EQUAL(completed, 1); 2802 CU_ASSERT(ut_seq.complete); 2803 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 2804 2805 /* Check completion error from spdk_memory_domain_push_data() */ 2806 completed = 0; 2807 seq = NULL; 2808 2809 src_iovs[0].iov_base = (void *)0xdeadbeef; 2810 src_iovs[0].iov_len = sizeof(srcbuf); 2811 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2812 dst_iovs[0].iov_len = sizeof(dstbuf); 2813 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2814 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2815 domctx[0].push_complete_status = -EADDRNOTAVAIL; 2816 2817 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2818 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2819 ut_sequence_step_cb, &completed); 2820 CU_ASSERT_EQUAL(rc, 0); 2821 2822 ut_seq.complete = false; 2823 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2824 2825 CU_ASSERT_EQUAL(completed, 1); 2826 CU_ASSERT(ut_seq.complete); 2827 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 2828 2829 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2830 g_modules_opc[i] = modules[i]; 2831 } 2832 2833 ut_clear_operations(); 2834 spdk_put_io_channel(ioch); 2835 poll_threads(); 2836 } 2837 2838 static int 2839 ut_submit_decompress_memory_domain(struct spdk_io_channel *ch, struct spdk_accel_task *task) 2840 { 2841 struct ut_domain_ctx *ctx; 2842 struct iovec *src_iovs, *dst_iovs; 2843 uint32_t src_iovcnt, dst_iovcnt; 2844 2845 src_iovs = task->s.iovs; 2846 dst_iovs = task->d.iovs; 2847 src_iovcnt = task->s.iovcnt; 2848 dst_iovcnt = task->d.iovcnt; 2849 2850 if (task->src_domain != NULL) { 2851 ctx = task->src_domain_ctx; 2852 CU_ASSERT_EQUAL(memcmp(task->s.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2853 src_iovs = &ctx->iov; 2854 src_iovcnt = 1; 2855 } 2856 2857 if (task->dst_domain != NULL) { 2858 ctx = task->dst_domain_ctx; 2859 CU_ASSERT_EQUAL(memcmp(task->d.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2860 dst_iovs = &ctx->iov; 2861 dst_iovcnt = 1; 2862 } 2863 2864 spdk_iovcpy(src_iovs, src_iovcnt, dst_iovs, dst_iovcnt); 2865 spdk_accel_task_complete(task, 0); 2866 2867 return 0; 2868 } 2869 2870 static void 2871 test_sequence_module_memory_domain(void) 2872 { 2873 struct spdk_accel_sequence *seq = NULL; 2874 struct spdk_io_channel *ioch; 2875 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2876 struct spdk_memory_domain *accel_domain; 2877 struct ut_sequence ut_seq; 2878 struct ut_domain_ctx domctx[2]; 2879 struct iovec src_iovs[2], dst_iovs[2]; 2880 void *buf, *accel_domain_ctx; 2881 char srcbuf[4096], dstbuf[4096], tmp[4096], expected[4096]; 2882 int i, rc, completed; 2883 2884 ioch = spdk_accel_get_io_channel(); 2885 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2886 2887 /* Override the submit_tasks function */ 2888 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2889 g_module.supports_memory_domains = true; 2890 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2891 modules[i] = g_modules_opc[i]; 2892 g_modules_opc[i] = g_module; 2893 } 2894 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress_memory_domain; 2895 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2896 2897 /* Check a sequence with both buffers in memory domains */ 2898 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2899 memset(expected, 0xa5, sizeof(expected)); 2900 memset(dstbuf, 0, sizeof(dstbuf)); 2901 seq = NULL; 2902 completed = 0; 2903 2904 src_iovs[0].iov_base = (void *)0xcafebabe; 2905 src_iovs[0].iov_len = sizeof(srcbuf); 2906 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2907 dst_iovs[0].iov_len = sizeof(dstbuf); 2908 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2909 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2910 2911 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2912 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2913 ut_sequence_step_cb, &completed); 2914 CU_ASSERT_EQUAL(rc, 0); 2915 2916 ut_seq.complete = false; 2917 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2918 2919 poll_threads(); 2920 2921 CU_ASSERT_EQUAL(completed, 1); 2922 CU_ASSERT(ut_seq.complete); 2923 CU_ASSERT_EQUAL(ut_seq.status, 0); 2924 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2925 2926 /* Check two operations each with a single buffer in memory domain */ 2927 memset(srcbuf, 0x5a, sizeof(srcbuf)); 2928 memset(expected, 0x5a, sizeof(expected)); 2929 memset(dstbuf, 0, sizeof(dstbuf)); 2930 memset(tmp, 0, sizeof(tmp)); 2931 seq = NULL; 2932 completed = 0; 2933 2934 src_iovs[0].iov_base = srcbuf; 2935 src_iovs[0].iov_len = sizeof(srcbuf); 2936 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2937 dst_iovs[0].iov_len = sizeof(tmp); 2938 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2939 2940 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2941 &src_iovs[0], 1, NULL, NULL, 0, 2942 ut_sequence_step_cb, &completed); 2943 CU_ASSERT_EQUAL(rc, 0); 2944 2945 src_iovs[1].iov_base = (void *)0xfeedbeef; 2946 src_iovs[1].iov_len = sizeof(tmp); 2947 dst_iovs[1].iov_base = dstbuf; 2948 dst_iovs[1].iov_len = sizeof(dstbuf); 2949 ut_domain_ctx_init(&domctx[1], tmp, sizeof(tmp), &src_iovs[1]); 2950 2951 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2952 &src_iovs[1], 1, g_ut_domain, &domctx[1], 0, 2953 ut_sequence_step_cb, &completed); 2954 CU_ASSERT_EQUAL(rc, 0); 2955 2956 ut_seq.complete = false; 2957 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2958 2959 poll_threads(); 2960 2961 CU_ASSERT_EQUAL(completed, 2); 2962 CU_ASSERT(ut_seq.complete); 2963 CU_ASSERT_EQUAL(ut_seq.status, 0); 2964 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2965 2966 /* Check a sequence with an accel buffer and a buffer in a regular memory domain */ 2967 memset(expected, 0xa5, sizeof(expected)); 2968 memset(dstbuf, 0, sizeof(dstbuf)); 2969 memset(tmp, 0, sizeof(tmp)); 2970 seq = NULL; 2971 completed = 0; 2972 2973 rc = spdk_accel_get_buf(ioch, 4096, &buf, &accel_domain, &accel_domain_ctx); 2974 CU_ASSERT_EQUAL(rc, 0); 2975 2976 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, accel_domain, accel_domain_ctx, 2977 0xa5, 0, ut_sequence_step_cb, &completed); 2978 CU_ASSERT_EQUAL(rc, 0); 2979 2980 src_iovs[0].iov_base = buf; 2981 src_iovs[0].iov_len = 4096; 2982 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2983 dst_iovs[0].iov_len = sizeof(dstbuf); 2984 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2985 2986 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2987 &src_iovs[0], 1, accel_domain, accel_domain_ctx, 0, 2988 ut_sequence_step_cb, &completed); 2989 CU_ASSERT_EQUAL(rc, 0); 2990 2991 ut_seq.complete = false; 2992 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2993 2994 poll_threads(); 2995 2996 CU_ASSERT_EQUAL(completed, 2); 2997 CU_ASSERT(ut_seq.complete); 2998 CU_ASSERT_EQUAL(ut_seq.status, 0); 2999 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 3000 3001 spdk_accel_put_buf(ioch, buf, accel_domain, accel_domain_ctx); 3002 3003 g_module.supports_memory_domains = false; 3004 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3005 g_modules_opc[i] = modules[i]; 3006 } 3007 3008 ut_clear_operations(); 3009 spdk_put_io_channel(ioch); 3010 poll_threads(); 3011 } 3012 3013 #ifdef SPDK_CONFIG_ISAL_CRYPTO 3014 static void 3015 ut_encrypt_cb(void *cb_arg, int status) 3016 { 3017 int *completed = cb_arg; 3018 3019 CU_ASSERT_EQUAL(status, 0); 3020 3021 *completed = 1; 3022 } 3023 3024 static void 3025 test_sequence_crypto(void) 3026 { 3027 struct spdk_accel_sequence *seq = NULL; 3028 struct spdk_io_channel *ioch; 3029 struct spdk_accel_crypto_key *key; 3030 struct spdk_accel_crypto_key_create_param key_params = { 3031 .cipher = "AES_XTS", 3032 .hex_key = "00112233445566778899aabbccddeeff", 3033 .hex_key2 = "ffeeddccbbaa99887766554433221100", 3034 .key_name = "ut_key", 3035 }; 3036 struct ut_sequence ut_seq; 3037 unsigned char buf[4096], encrypted[4096] = {}, data[4096], tmp[3][4096]; 3038 struct iovec src_iovs[4], dst_iovs[4]; 3039 int rc, completed = 0; 3040 size_t i; 3041 3042 ioch = spdk_accel_get_io_channel(); 3043 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3044 3045 rc = spdk_accel_crypto_key_create(&key_params); 3046 CU_ASSERT_EQUAL(rc, 0); 3047 key = spdk_accel_crypto_key_get(key_params.key_name); 3048 SPDK_CU_ASSERT_FATAL(key != NULL); 3049 3050 for (i = 0; i < sizeof(data); ++i) { 3051 data[i] = (uint8_t)i & 0xff; 3052 } 3053 3054 dst_iovs[0].iov_base = encrypted; 3055 dst_iovs[0].iov_len = sizeof(encrypted); 3056 src_iovs[0].iov_base = data; 3057 src_iovs[0].iov_len = sizeof(data); 3058 rc = spdk_accel_submit_encrypt(ioch, key, &dst_iovs[0], 1, &src_iovs[0], 1, 0, 4096, 0, 3059 ut_encrypt_cb, &completed); 3060 CU_ASSERT_EQUAL(rc, 0); 3061 3062 while (!completed) { 3063 poll_threads(); 3064 } 3065 3066 /* Verify that encryption operation in a sequence produces the same result */ 3067 seq = NULL; 3068 completed = 0; 3069 3070 dst_iovs[0].iov_base = tmp[0]; 3071 dst_iovs[0].iov_len = sizeof(tmp[0]); 3072 src_iovs[0].iov_base = data; 3073 src_iovs[0].iov_len = sizeof(data); 3074 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3075 &src_iovs[0], 1, NULL, NULL, 0, 3076 ut_sequence_step_cb, &completed); 3077 CU_ASSERT_EQUAL(rc, 0); 3078 3079 dst_iovs[1].iov_base = tmp[1]; 3080 dst_iovs[1].iov_len = sizeof(tmp[1]); 3081 src_iovs[1].iov_base = tmp[0]; 3082 src_iovs[1].iov_len = sizeof(tmp[0]); 3083 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3084 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3085 ut_sequence_step_cb, &completed); 3086 CU_ASSERT_EQUAL(rc, 0); 3087 3088 dst_iovs[2].iov_base = buf; 3089 dst_iovs[2].iov_len = sizeof(buf); 3090 src_iovs[2].iov_base = tmp[1]; 3091 src_iovs[2].iov_len = sizeof(tmp[1]); 3092 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3093 &src_iovs[2], 1, NULL, NULL, 0, 3094 ut_sequence_step_cb, &completed); 3095 CU_ASSERT_EQUAL(rc, 0); 3096 3097 ut_seq.complete = false; 3098 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3099 3100 poll_threads(); 3101 3102 CU_ASSERT_EQUAL(completed, 3); 3103 CU_ASSERT(ut_seq.complete); 3104 CU_ASSERT_EQUAL(ut_seq.status, 0); 3105 CU_ASSERT_EQUAL(memcmp(buf, encrypted, sizeof(buf)), 0); 3106 3107 /* Check that decryption produces the original buffer */ 3108 seq = NULL; 3109 completed = 0; 3110 memset(buf, 0, sizeof(buf)); 3111 3112 dst_iovs[0].iov_base = tmp[0]; 3113 dst_iovs[0].iov_len = sizeof(tmp[0]); 3114 src_iovs[0].iov_base = encrypted; 3115 src_iovs[0].iov_len = sizeof(encrypted); 3116 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3117 &src_iovs[0], 1, NULL, NULL, 0, 3118 ut_sequence_step_cb, &completed); 3119 CU_ASSERT_EQUAL(rc, 0); 3120 3121 dst_iovs[1].iov_base = tmp[1]; 3122 dst_iovs[1].iov_len = sizeof(tmp[1]); 3123 src_iovs[1].iov_base = tmp[0]; 3124 src_iovs[1].iov_len = sizeof(tmp[0]); 3125 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3126 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3127 ut_sequence_step_cb, &completed); 3128 CU_ASSERT_EQUAL(rc, 0); 3129 3130 dst_iovs[2].iov_base = buf; 3131 dst_iovs[2].iov_len = sizeof(buf); 3132 src_iovs[2].iov_base = tmp[1]; 3133 src_iovs[2].iov_len = sizeof(tmp[1]); 3134 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3135 &src_iovs[2], 1, NULL, NULL, 0, 3136 ut_sequence_step_cb, &completed); 3137 CU_ASSERT_EQUAL(rc, 0); 3138 3139 ut_seq.complete = false; 3140 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3141 3142 poll_threads(); 3143 3144 CU_ASSERT_EQUAL(completed, 3); 3145 CU_ASSERT(ut_seq.complete); 3146 CU_ASSERT_EQUAL(ut_seq.status, 0); 3147 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3148 3149 /* Check encrypt + decrypt in a single sequence */ 3150 seq = NULL; 3151 completed = 0; 3152 memset(buf, 0, sizeof(buf)); 3153 3154 dst_iovs[0].iov_base = tmp[0]; 3155 dst_iovs[0].iov_len = sizeof(tmp[0]); 3156 src_iovs[0].iov_base = data; 3157 src_iovs[0].iov_len = sizeof(data); 3158 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3159 &src_iovs[0], 1, NULL, NULL, 0, 3160 ut_sequence_step_cb, &completed); 3161 CU_ASSERT_EQUAL(rc, 0); 3162 3163 dst_iovs[1].iov_base = tmp[1]; 3164 dst_iovs[1].iov_len = sizeof(tmp[1]); 3165 src_iovs[1].iov_base = tmp[0]; 3166 src_iovs[1].iov_len = sizeof(tmp[0]); 3167 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3168 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3169 ut_sequence_step_cb, &completed); 3170 CU_ASSERT_EQUAL(rc, 0); 3171 3172 3173 dst_iovs[2].iov_base = tmp[2]; 3174 dst_iovs[2].iov_len = sizeof(tmp[2]); 3175 src_iovs[2].iov_base = tmp[1]; 3176 src_iovs[2].iov_len = sizeof(tmp[1]); 3177 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[2], 1, NULL, NULL, 3178 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3179 ut_sequence_step_cb, &completed); 3180 CU_ASSERT_EQUAL(rc, 0); 3181 3182 dst_iovs[3].iov_base = buf; 3183 dst_iovs[3].iov_len = sizeof(buf); 3184 src_iovs[3].iov_base = tmp[2]; 3185 src_iovs[3].iov_len = sizeof(tmp[2]); 3186 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 3187 &src_iovs[3], 1, NULL, NULL, 0, 3188 ut_sequence_step_cb, &completed); 3189 CU_ASSERT_EQUAL(rc, 0); 3190 3191 ut_seq.complete = false; 3192 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3193 3194 poll_threads(); 3195 3196 CU_ASSERT_EQUAL(completed, 4); 3197 CU_ASSERT(ut_seq.complete); 3198 CU_ASSERT_EQUAL(ut_seq.status, 0); 3199 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3200 3201 rc = spdk_accel_crypto_key_destroy(key); 3202 CU_ASSERT_EQUAL(rc, 0); 3203 spdk_put_io_channel(ioch); 3204 poll_threads(); 3205 } 3206 #endif /* SPDK_CONFIG_ISAL_CRYPTO */ 3207 3208 static int 3209 ut_submit_crypto(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3210 { 3211 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3212 3213 spdk_accel_task_complete(task, 0); 3214 3215 return 0; 3216 } 3217 3218 struct ut_driver_operation { 3219 int complete_status; 3220 int submit_status; 3221 int count; 3222 bool supported; 3223 }; 3224 3225 static struct ut_driver_operation g_drv_operations[SPDK_ACCEL_OPC_LAST]; 3226 static bool g_ut_driver_async_continue; 3227 3228 static void 3229 ut_driver_async_continue(void *arg) 3230 { 3231 struct spdk_accel_sequence *seq = arg; 3232 3233 spdk_accel_sequence_continue(seq); 3234 } 3235 3236 static int 3237 ut_driver_execute_sequence(struct spdk_io_channel *ch, struct spdk_accel_sequence *seq) 3238 { 3239 struct spdk_accel_task *task; 3240 struct ut_driver_operation *drv_ops; 3241 3242 while ((task = spdk_accel_sequence_first_task(seq)) != NULL) { 3243 drv_ops = &g_drv_operations[task->op_code]; 3244 if (!drv_ops->supported) { 3245 break; 3246 } 3247 3248 drv_ops->count++; 3249 if (drv_ops->submit_status != 0) { 3250 return drv_ops->submit_status; 3251 } 3252 3253 if (drv_ops->complete_status != 0) { 3254 spdk_accel_task_complete(task, drv_ops->complete_status); 3255 break; 3256 } 3257 3258 switch (task->op_code) { 3259 case SPDK_ACCEL_OPC_DECOMPRESS: 3260 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3261 break; 3262 case SPDK_ACCEL_OPC_FILL: 3263 spdk_iov_memset(task->d.iovs, task->d.iovcnt, 3264 (int)(task->fill_pattern & 0xff)); 3265 break; 3266 default: 3267 CU_ASSERT(0 && "unexpected opcode"); 3268 break; 3269 } 3270 3271 spdk_accel_task_complete(task, 0); 3272 } 3273 3274 if (g_ut_driver_async_continue) { 3275 spdk_thread_send_msg(spdk_get_thread(), ut_driver_async_continue, seq); 3276 } else { 3277 spdk_accel_sequence_continue(seq); 3278 } 3279 3280 return 0; 3281 } 3282 3283 static struct spdk_io_channel * 3284 ut_driver_get_io_channel(void) 3285 { 3286 return (void *)0xdeadbeef; 3287 } 3288 3289 static struct spdk_accel_driver g_ut_driver = { 3290 .name = "ut", 3291 .execute_sequence = ut_driver_execute_sequence, 3292 .get_io_channel = ut_driver_get_io_channel, 3293 }; 3294 3295 SPDK_ACCEL_DRIVER_REGISTER(ut, &g_ut_driver); 3296 3297 static void 3298 test_sequence_driver(void) 3299 { 3300 struct spdk_accel_sequence *seq = NULL; 3301 struct spdk_io_channel *ioch; 3302 struct spdk_accel_crypto_key key = {}; 3303 struct ut_sequence ut_seq; 3304 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3305 char buf[4096], tmp[3][4096], expected[4096]; 3306 struct iovec src_iovs[3], dst_iovs[3]; 3307 int i, rc, completed = 0; 3308 3309 ioch = spdk_accel_get_io_channel(); 3310 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3311 rc = spdk_accel_set_driver("ut"); 3312 SPDK_CU_ASSERT_FATAL(rc == 0); 3313 3314 /* Override the submit_tasks function */ 3315 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3316 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3317 modules[i] = g_modules_opc[i]; 3318 g_modules_opc[i] = g_module; 3319 } 3320 /* Intercept crypto operations, as they should be executed by an accel module */ 3321 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].submit = ut_submit_crypto; 3322 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].submit = ut_submit_crypto; 3323 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3324 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3325 g_seq_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3326 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3327 3328 g_drv_operations[SPDK_ACCEL_OPC_FILL].supported = true; 3329 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].supported = true; 3330 3331 /* First check a sequence that is fully executed using a driver, with the copy at the end 3332 * being removed */ 3333 seq = NULL; 3334 completed = 0; 3335 memset(buf, 0, sizeof(buf)); 3336 memset(tmp[0], 0, sizeof(tmp[0])); 3337 memset(tmp[1], 0, sizeof(tmp[1])); 3338 memset(&expected[0], 0xa5, 2048); 3339 memset(&expected[2048], 0xbe, 2048); 3340 3341 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], 2048, NULL, NULL, 0xa5, 0, 3342 ut_sequence_step_cb, &completed); 3343 CU_ASSERT_EQUAL(rc, 0); 3344 rc = spdk_accel_append_fill(&seq, ioch, &tmp[0][2048], 2048, NULL, NULL, 0xbe, 0, 3345 ut_sequence_step_cb, &completed); 3346 CU_ASSERT_EQUAL(rc, 0); 3347 3348 dst_iovs[0].iov_base = tmp[1]; 3349 dst_iovs[0].iov_len = sizeof(tmp[1]); 3350 src_iovs[0].iov_base = tmp[0]; 3351 src_iovs[0].iov_len = sizeof(tmp[0]); 3352 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3353 &src_iovs[0], 1, NULL, NULL, 0, 3354 ut_sequence_step_cb, &completed); 3355 CU_ASSERT_EQUAL(rc, 0); 3356 3357 dst_iovs[1].iov_base = buf; 3358 dst_iovs[1].iov_len = sizeof(buf); 3359 src_iovs[1].iov_base = tmp[1]; 3360 src_iovs[1].iov_len = sizeof(tmp[1]); 3361 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3362 &src_iovs[1], 1, NULL, NULL, 0, 3363 ut_sequence_step_cb, &completed); 3364 CU_ASSERT_EQUAL(rc, 0); 3365 3366 ut_seq.complete = false; 3367 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3368 3369 poll_threads(); 3370 3371 CU_ASSERT_EQUAL(completed, 4); 3372 CU_ASSERT(ut_seq.complete); 3373 CU_ASSERT_EQUAL(ut_seq.status, 0); 3374 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3375 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3376 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 3377 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 2); 3378 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3379 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3380 3381 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3382 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3383 3384 /* Check a sequence when the first two operations are executed by a driver, while the rest 3385 * is executed via modules */ 3386 seq = NULL; 3387 completed = 0; 3388 memset(buf, 0, sizeof(buf)); 3389 memset(tmp[0], 0, sizeof(tmp[0])); 3390 memset(tmp[1], 0, sizeof(tmp[1])); 3391 memset(tmp[2], 0, sizeof(tmp[2])); 3392 memset(&expected[0], 0xfe, 4096); 3393 3394 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 0, 3395 ut_sequence_step_cb, &completed); 3396 CU_ASSERT_EQUAL(rc, 0); 3397 3398 dst_iovs[0].iov_base = tmp[1]; 3399 dst_iovs[0].iov_len = sizeof(tmp[1]); 3400 src_iovs[0].iov_base = tmp[0]; 3401 src_iovs[0].iov_len = sizeof(tmp[0]); 3402 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3403 &src_iovs[0], 1, NULL, NULL, 0, 3404 ut_sequence_step_cb, &completed); 3405 CU_ASSERT_EQUAL(rc, 0); 3406 3407 dst_iovs[1].iov_base = tmp[2]; 3408 dst_iovs[1].iov_len = sizeof(tmp[2]); 3409 src_iovs[1].iov_base = tmp[1]; 3410 src_iovs[1].iov_len = sizeof(tmp[1]); 3411 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3412 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3413 ut_sequence_step_cb, &completed); 3414 CU_ASSERT_EQUAL(rc, 0); 3415 3416 dst_iovs[2].iov_base = buf; 3417 dst_iovs[2].iov_len = sizeof(buf); 3418 src_iovs[2].iov_base = tmp[2]; 3419 src_iovs[2].iov_len = sizeof(tmp[2]); 3420 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3421 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3422 ut_sequence_step_cb, &completed); 3423 CU_ASSERT_EQUAL(rc, 0); 3424 3425 ut_seq.complete = false; 3426 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3427 3428 poll_threads(); 3429 3430 CU_ASSERT_EQUAL(completed, 4); 3431 CU_ASSERT(ut_seq.complete); 3432 CU_ASSERT_EQUAL(ut_seq.status, 0); 3433 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3434 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3435 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3436 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3437 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3438 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3439 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3440 3441 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3442 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3443 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3444 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3445 3446 /* Check sequence when the first and last operations are executed through modules, while the 3447 * ones in the middle are executed by the driver */ 3448 seq = NULL; 3449 completed = 0; 3450 memset(buf, 0, sizeof(buf)); 3451 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3452 memset(tmp[1], 0, sizeof(tmp[1])); 3453 memset(tmp[2], 0, sizeof(tmp[2])); 3454 memset(&expected[0], 0xfe, 2048); 3455 memset(&expected[2048], 0xa5, 2048); 3456 3457 dst_iovs[0].iov_base = tmp[1]; 3458 dst_iovs[0].iov_len = sizeof(tmp[1]); 3459 src_iovs[0].iov_base = tmp[0]; 3460 src_iovs[0].iov_len = sizeof(tmp[0]); 3461 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3462 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3463 ut_sequence_step_cb, &completed); 3464 CU_ASSERT_EQUAL(rc, 0); 3465 3466 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xfe, 0, 3467 ut_sequence_step_cb, &completed); 3468 CU_ASSERT_EQUAL(rc, 0); 3469 3470 dst_iovs[1].iov_base = tmp[2]; 3471 dst_iovs[1].iov_len = sizeof(tmp[2]); 3472 src_iovs[1].iov_base = tmp[1]; 3473 src_iovs[1].iov_len = sizeof(tmp[1]); 3474 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3475 &src_iovs[1], 1, NULL, NULL, 0, 3476 ut_sequence_step_cb, &completed); 3477 CU_ASSERT_EQUAL(rc, 0); 3478 3479 dst_iovs[2].iov_base = buf; 3480 dst_iovs[2].iov_len = sizeof(buf); 3481 src_iovs[2].iov_base = tmp[2]; 3482 src_iovs[2].iov_len = sizeof(tmp[2]); 3483 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3484 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3485 ut_sequence_step_cb, &completed); 3486 CU_ASSERT_EQUAL(rc, 0); 3487 3488 ut_seq.complete = false; 3489 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3490 3491 poll_threads(); 3492 3493 CU_ASSERT_EQUAL(completed, 4); 3494 CU_ASSERT(ut_seq.complete); 3495 CU_ASSERT_EQUAL(ut_seq.status, 0); 3496 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3497 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3498 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3499 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3500 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3501 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3502 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3503 3504 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3505 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3506 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3507 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3508 3509 /* Check a sequence with operations executed by: module, driver, module, driver */ 3510 seq = NULL; 3511 completed = 0; 3512 memset(buf, 0, sizeof(buf)); 3513 memset(tmp[0], 0x5a, sizeof(tmp[0])); 3514 memset(tmp[1], 0, sizeof(tmp[1])); 3515 memset(tmp[2], 0, sizeof(tmp[2])); 3516 memset(&expected[0], 0xef, 2048); 3517 memset(&expected[2048], 0x5a, 2048); 3518 3519 dst_iovs[0].iov_base = tmp[1]; 3520 dst_iovs[0].iov_len = sizeof(tmp[1]); 3521 src_iovs[0].iov_base = tmp[0]; 3522 src_iovs[0].iov_len = sizeof(tmp[0]); 3523 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3524 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3525 ut_sequence_step_cb, &completed); 3526 CU_ASSERT_EQUAL(rc, 0); 3527 3528 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3529 ut_sequence_step_cb, &completed); 3530 CU_ASSERT_EQUAL(rc, 0); 3531 3532 dst_iovs[1].iov_base = tmp[2]; 3533 dst_iovs[1].iov_len = sizeof(tmp[2]); 3534 src_iovs[1].iov_base = tmp[1]; 3535 src_iovs[1].iov_len = sizeof(tmp[1]); 3536 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3537 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3538 ut_sequence_step_cb, &completed); 3539 CU_ASSERT_EQUAL(rc, 0); 3540 3541 dst_iovs[2].iov_base = buf; 3542 dst_iovs[2].iov_len = sizeof(buf); 3543 src_iovs[2].iov_base = tmp[2]; 3544 src_iovs[2].iov_len = sizeof(tmp[2]); 3545 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3546 &src_iovs[2], 1, NULL, NULL, 0, 3547 ut_sequence_step_cb, &completed); 3548 CU_ASSERT_EQUAL(rc, 0); 3549 3550 ut_seq.complete = false; 3551 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3552 3553 poll_threads(); 3554 3555 CU_ASSERT_EQUAL(completed, 4); 3556 CU_ASSERT(ut_seq.complete); 3557 CU_ASSERT_EQUAL(ut_seq.status, 0); 3558 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3559 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3560 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3561 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3562 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3563 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3564 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3565 3566 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3567 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3568 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3569 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3570 3571 /* Check that an error returned from driver's execute_sequence() will fail the whole 3572 * sequence and any subsequent operations won't be processed */ 3573 seq = NULL; 3574 completed = 0; 3575 memset(buf, 0, sizeof(buf)); 3576 memset(expected, 0, sizeof(expected)); 3577 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3578 g_drv_operations[SPDK_ACCEL_OPC_FILL].submit_status = -EPERM; 3579 3580 dst_iovs[0].iov_base = tmp[1]; 3581 dst_iovs[0].iov_len = sizeof(tmp[1]); 3582 src_iovs[0].iov_base = tmp[0]; 3583 src_iovs[0].iov_len = sizeof(tmp[0]); 3584 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3585 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3586 ut_sequence_step_cb, &completed); 3587 CU_ASSERT_EQUAL(rc, 0); 3588 3589 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3590 ut_sequence_step_cb, &completed); 3591 CU_ASSERT_EQUAL(rc, 0); 3592 3593 dst_iovs[1].iov_base = buf; 3594 dst_iovs[1].iov_len = sizeof(buf); 3595 src_iovs[1].iov_base = tmp[1]; 3596 src_iovs[1].iov_len = sizeof(tmp[1]); 3597 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3598 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3599 ut_sequence_step_cb, &completed); 3600 CU_ASSERT_EQUAL(rc, 0); 3601 3602 ut_seq.complete = false; 3603 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3604 3605 poll_threads(); 3606 3607 CU_ASSERT_EQUAL(completed, 3); 3608 CU_ASSERT(ut_seq.complete); 3609 CU_ASSERT_EQUAL(ut_seq.status, -EPERM); 3610 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3611 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3612 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 0); 3613 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3614 CU_ASSERT_EQUAL(memcmp(buf, expected, 4096), 0); 3615 3616 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3617 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3618 g_drv_operations[SPDK_ACCEL_OPC_FILL].submit_status = 0; 3619 3620 /* Check that a failed task completed by a driver will cause the whole sequence to be failed 3621 * and any subsequent operations won't be processed */ 3622 seq = NULL; 3623 completed = 0; 3624 memset(buf, 0, sizeof(buf)); 3625 memset(expected, 0, sizeof(expected)); 3626 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3627 g_drv_operations[SPDK_ACCEL_OPC_FILL].complete_status = -ENOENT; 3628 3629 dst_iovs[0].iov_base = tmp[1]; 3630 dst_iovs[0].iov_len = sizeof(tmp[1]); 3631 src_iovs[0].iov_base = tmp[0]; 3632 src_iovs[0].iov_len = sizeof(tmp[0]); 3633 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3634 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3635 ut_sequence_step_cb, &completed); 3636 CU_ASSERT_EQUAL(rc, 0); 3637 3638 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3639 ut_sequence_step_cb, &completed); 3640 CU_ASSERT_EQUAL(rc, 0); 3641 3642 dst_iovs[1].iov_base = buf; 3643 dst_iovs[1].iov_len = sizeof(buf); 3644 src_iovs[1].iov_base = tmp[1]; 3645 src_iovs[1].iov_len = sizeof(tmp[1]); 3646 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3647 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3648 ut_sequence_step_cb, &completed); 3649 CU_ASSERT_EQUAL(rc, 0); 3650 3651 ut_seq.complete = false; 3652 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3653 3654 poll_threads(); 3655 3656 CU_ASSERT_EQUAL(completed, 3); 3657 CU_ASSERT(ut_seq.complete); 3658 CU_ASSERT_EQUAL(ut_seq.status, -ENOENT); 3659 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3660 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3661 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 0); 3662 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3663 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3664 3665 g_drv_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 3666 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3667 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3668 3669 /* Check asynchronous spdk_accel_sequence_continue() */ 3670 g_ut_driver_async_continue = true; 3671 seq = NULL; 3672 completed = 0; 3673 memset(buf, 0, sizeof(buf)); 3674 memset(tmp[0], 0, sizeof(tmp[0])); 3675 memset(&expected[0], 0xfe, 4096); 3676 3677 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 0, 3678 ut_sequence_step_cb, &completed); 3679 CU_ASSERT_EQUAL(rc, 0); 3680 3681 dst_iovs[0].iov_base = buf; 3682 dst_iovs[0].iov_len = sizeof(buf); 3683 src_iovs[0].iov_base = tmp[0]; 3684 src_iovs[0].iov_len = sizeof(tmp[0]); 3685 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3686 &src_iovs[0], 1, NULL, NULL, 0, 3687 ut_sequence_step_cb, &completed); 3688 CU_ASSERT_EQUAL(rc, 0); 3689 3690 ut_seq.complete = false; 3691 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3692 3693 poll_threads(); 3694 3695 CU_ASSERT_EQUAL(completed, 2); 3696 CU_ASSERT(ut_seq.complete); 3697 CU_ASSERT_EQUAL(ut_seq.status, 0); 3698 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3699 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3700 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3701 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3702 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3703 3704 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3705 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3706 3707 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3708 g_modules_opc[i] = modules[i]; 3709 } 3710 3711 /* Clear the driver so that other tests won't use it */ 3712 g_accel_driver = NULL; 3713 memset(&g_drv_operations, 0, sizeof(g_drv_operations)); 3714 3715 ut_clear_operations(); 3716 spdk_put_io_channel(ioch); 3717 poll_threads(); 3718 } 3719 3720 struct ut_saved_iovs { 3721 struct iovec src; 3722 struct iovec dst; 3723 }; 3724 3725 static struct ut_saved_iovs g_seq_saved_iovs[SPDK_ACCEL_OPC_LAST]; 3726 3727 static int 3728 ut_submit_save_iovs(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3729 { 3730 SPDK_CU_ASSERT_FATAL(task->s.iovcnt == 1); 3731 SPDK_CU_ASSERT_FATAL(task->d.iovcnt == 1); 3732 3733 g_seq_saved_iovs[task->op_code].src = task->s.iovs[0]; 3734 g_seq_saved_iovs[task->op_code].dst = task->d.iovs[0]; 3735 3736 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3737 3738 spdk_accel_task_complete(task, 0); 3739 3740 return 0; 3741 } 3742 3743 static void 3744 test_sequence_same_iovs(void) 3745 { 3746 struct spdk_accel_sequence *seq = NULL; 3747 struct spdk_io_channel *ioch; 3748 struct spdk_accel_crypto_key key = {}; 3749 struct ut_sequence ut_seq; 3750 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3751 char buf[4096], tmp[4096], expected[4096]; 3752 struct iovec iovs[3], expected_siov, expected_diov; 3753 struct spdk_memory_domain *domain; 3754 void *accel_buf, *domain_ctx; 3755 int i, rc, completed = 0; 3756 3757 ioch = spdk_accel_get_io_channel(); 3758 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3759 3760 /* Override the submit_tasks function */ 3761 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3762 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3763 modules[i] = g_modules_opc[i]; 3764 g_modules_opc[i] = g_module; 3765 } 3766 /* Intercept crypto operations, as they should be executed by an accel module */ 3767 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].submit = ut_submit_save_iovs; 3768 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].submit = ut_submit_save_iovs; 3769 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3770 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3771 3772 /* Check that it's possible to use the same iovec ptr for different operations */ 3773 seq = NULL; 3774 completed = 0; 3775 memset(buf, 0, sizeof(buf)); 3776 memset(expected, 0xa5, sizeof(expected)); 3777 3778 iovs[0].iov_base = expected; 3779 iovs[0].iov_len = sizeof(expected); 3780 iovs[1].iov_base = tmp; 3781 iovs[1].iov_len = sizeof(tmp); 3782 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, NULL, NULL, 3783 &iovs[0], 1, NULL, NULL, 0, 4096, 0, 3784 ut_sequence_step_cb, &completed); 3785 CU_ASSERT_EQUAL(rc, 0); 3786 /* Reuse iov[1] as src */ 3787 iovs[2].iov_base = buf; 3788 iovs[2].iov_len = sizeof(buf); 3789 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3790 &iovs[1], 1, NULL, NULL, 0, 4096, 0, 3791 ut_sequence_step_cb, &completed); 3792 CU_ASSERT_EQUAL(rc, 0); 3793 3794 ut_seq.complete = false; 3795 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3796 3797 poll_threads(); 3798 3799 CU_ASSERT_EQUAL(completed, 2); 3800 CU_ASSERT(ut_seq.complete); 3801 CU_ASSERT_EQUAL(ut_seq.status, 0); 3802 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3803 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3804 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3805 expected_siov.iov_base = expected; 3806 expected_siov.iov_len = sizeof(expected); 3807 expected_diov.iov_base = tmp; 3808 expected_diov.iov_len = sizeof(tmp); 3809 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].src, 3810 &expected_siov, sizeof(expected_siov)), 0); 3811 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].dst, 3812 &expected_diov, sizeof(expected_diov)), 0); 3813 expected_siov.iov_base = tmp; 3814 expected_siov.iov_len = sizeof(tmp); 3815 expected_diov.iov_base = buf; 3816 expected_diov.iov_len = sizeof(buf); 3817 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].src, 3818 &expected_siov, sizeof(expected_siov)), 0); 3819 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].dst, 3820 &expected_diov, sizeof(expected_diov)), 0); 3821 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3822 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3823 3824 /* Check the same with an accel buffer */ 3825 seq = NULL; 3826 completed = 0; 3827 memset(buf, 0, sizeof(buf)); 3828 memset(expected, 0x5a, sizeof(expected)); 3829 3830 rc = spdk_accel_get_buf(ioch, sizeof(buf), &accel_buf, &domain, &domain_ctx); 3831 CU_ASSERT_EQUAL(rc, 0); 3832 3833 iovs[0].iov_base = expected; 3834 iovs[0].iov_len = sizeof(expected); 3835 iovs[1].iov_base = accel_buf; 3836 iovs[1].iov_len = sizeof(buf); 3837 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, domain, domain_ctx, 3838 &iovs[0], 1, NULL, NULL, 0, 4096, 0, 3839 ut_sequence_step_cb, &completed); 3840 CU_ASSERT_EQUAL(rc, 0); 3841 /* Reuse iov[1] as src */ 3842 iovs[2].iov_base = buf; 3843 iovs[2].iov_len = sizeof(buf); 3844 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3845 &iovs[1], 1, domain, domain_ctx, 0, 4096, 0, 3846 ut_sequence_step_cb, &completed); 3847 CU_ASSERT_EQUAL(rc, 0); 3848 3849 ut_seq.complete = false; 3850 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3851 3852 poll_threads(); 3853 3854 CU_ASSERT_EQUAL(completed, 2); 3855 CU_ASSERT(ut_seq.complete); 3856 CU_ASSERT_EQUAL(ut_seq.status, 0); 3857 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3858 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3859 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3860 expected_siov.iov_base = expected; 3861 expected_siov.iov_len = sizeof(expected); 3862 expected_diov.iov_base = buf; 3863 expected_diov.iov_len = sizeof(buf); 3864 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].src, 3865 &expected_siov, sizeof(expected_siov)), 0); 3866 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].dst, 3867 &expected_diov, sizeof(expected_diov)), 0); 3868 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].dst, 3869 &g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].src, 3870 sizeof(struct iovec)), 0); 3871 spdk_accel_put_buf(ioch, accel_buf, domain, domain_ctx); 3872 3873 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3874 g_modules_opc[i] = modules[i]; 3875 } 3876 3877 ut_clear_operations(); 3878 spdk_put_io_channel(ioch); 3879 poll_threads(); 3880 } 3881 3882 static void 3883 test_sequence_crc32(void) 3884 { 3885 struct spdk_accel_sequence *seq = NULL; 3886 struct spdk_io_channel *ioch; 3887 struct ut_sequence ut_seq; 3888 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3889 char buf[4096], tmp[3][4096]; 3890 struct iovec src_iovs[4], dst_iovs[4]; 3891 uint32_t crc, crc2; 3892 int i, rc, completed; 3893 3894 ioch = spdk_accel_get_io_channel(); 3895 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3896 3897 /* Override the submit_tasks function */ 3898 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3899 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3900 g_seq_operations[i].submit = sw_accel_submit_tasks; 3901 modules[i] = g_modules_opc[i]; 3902 g_modules_opc[i] = g_module; 3903 } 3904 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 3905 3906 /* First check the simplest case - single crc32c operation */ 3907 seq = NULL; 3908 completed = 0; 3909 crc = 0; 3910 memset(buf, 0xa5, sizeof(buf)); 3911 3912 src_iovs[0].iov_base = buf; 3913 src_iovs[0].iov_len = sizeof(buf); 3914 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3915 ut_sequence_step_cb, &completed); 3916 CU_ASSERT_EQUAL(rc, 0); 3917 3918 ut_seq.complete = false; 3919 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3920 3921 poll_threads(); 3922 CU_ASSERT_EQUAL(completed, 1); 3923 CU_ASSERT(ut_seq.complete); 3924 CU_ASSERT_EQUAL(ut_seq.status, 0); 3925 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3926 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3927 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3928 3929 /* Now check copy+crc - this should remove the copy operation and change the buffer for the 3930 * crc operation */ 3931 seq = NULL; 3932 completed = 0; 3933 crc = 0; 3934 memset(buf, 0x5a, sizeof(buf)); 3935 memset(&tmp[0], 0, sizeof(tmp[0])); 3936 3937 dst_iovs[0].iov_base = tmp[0]; 3938 dst_iovs[0].iov_len = sizeof(tmp[0]); 3939 src_iovs[0].iov_base = buf; 3940 src_iovs[0].iov_len = sizeof(buf); 3941 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3942 &src_iovs[0], 1, NULL, NULL, 0, 3943 ut_sequence_step_cb, &completed); 3944 CU_ASSERT_EQUAL(rc, 0); 3945 3946 src_iovs[1].iov_base = tmp[0]; 3947 src_iovs[1].iov_len = sizeof(tmp[0]); 3948 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 3949 ut_sequence_step_cb, &completed); 3950 CU_ASSERT_EQUAL(rc, 0); 3951 3952 ut_seq.complete = false; 3953 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3954 3955 poll_threads(); 3956 CU_ASSERT_EQUAL(completed, 2); 3957 CU_ASSERT(ut_seq.complete); 3958 CU_ASSERT_EQUAL(ut_seq.status, 0); 3959 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3960 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 3961 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3962 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3963 3964 /* Check crc+copy - this time the copy cannot be removed, because there's no operation 3965 * before crc to change the buffer */ 3966 seq = NULL; 3967 completed = 0; 3968 crc = 0; 3969 memset(buf, 0, sizeof(buf)); 3970 memset(&tmp[0], 0xa5, sizeof(tmp[0])); 3971 3972 src_iovs[0].iov_base = tmp[0]; 3973 src_iovs[0].iov_len = sizeof(tmp[0]); 3974 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3975 ut_sequence_step_cb, &completed); 3976 CU_ASSERT_EQUAL(rc, 0); 3977 3978 dst_iovs[1].iov_base = buf; 3979 dst_iovs[1].iov_len = sizeof(buf); 3980 src_iovs[1].iov_base = tmp[0]; 3981 src_iovs[1].iov_len = sizeof(tmp[0]); 3982 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3983 &src_iovs[1], 1, NULL, NULL, 0, 3984 ut_sequence_step_cb, &completed); 3985 CU_ASSERT_EQUAL(rc, 0); 3986 3987 ut_seq.complete = false; 3988 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3989 3990 poll_threads(); 3991 CU_ASSERT_EQUAL(completed, 2); 3992 CU_ASSERT(ut_seq.complete); 3993 CU_ASSERT_EQUAL(ut_seq.status, 0); 3994 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3995 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 3996 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 3997 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 3998 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3999 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 4000 4001 /* Check a sequence with an operation at the beginning that can have its buffer changed, two 4002 * crc operations and a copy at the end. The copy should be removed and the dst buffer of 4003 * the first operation and the src buffer of the crc operations should be changed. 4004 */ 4005 seq = NULL; 4006 completed = 0; 4007 crc = crc2 = 0; 4008 memset(buf, 0, sizeof(buf)); 4009 memset(&tmp[0], 0x5a, sizeof(tmp[0])); 4010 dst_iovs[0].iov_base = tmp[1]; 4011 dst_iovs[0].iov_len = sizeof(tmp[1]); 4012 src_iovs[0].iov_base = tmp[0]; 4013 src_iovs[0].iov_len = sizeof(tmp[0]); 4014 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 4015 &src_iovs[0], 1, NULL, NULL, 0, 4016 ut_sequence_step_cb, &completed); 4017 CU_ASSERT_EQUAL(rc, 0); 4018 4019 src_iovs[1].iov_base = tmp[1]; 4020 src_iovs[1].iov_len = sizeof(tmp[1]); 4021 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 4022 ut_sequence_step_cb, &completed); 4023 CU_ASSERT_EQUAL(rc, 0); 4024 4025 src_iovs[2].iov_base = tmp[1]; 4026 src_iovs[2].iov_len = sizeof(tmp[1]); 4027 rc = spdk_accel_append_crc32c(&seq, ioch, &crc2, &src_iovs[2], 1, NULL, NULL, 0, 4028 ut_sequence_step_cb, &completed); 4029 CU_ASSERT_EQUAL(rc, 0); 4030 4031 dst_iovs[3].iov_base = buf; 4032 dst_iovs[3].iov_len = sizeof(buf); 4033 src_iovs[3].iov_base = tmp[1]; 4034 src_iovs[3].iov_len = sizeof(tmp[1]); 4035 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 4036 &src_iovs[3], 1, NULL, NULL, 0, 4037 ut_sequence_step_cb, &completed); 4038 CU_ASSERT_EQUAL(rc, 0); 4039 4040 ut_seq.complete = false; 4041 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4042 4043 poll_threads(); 4044 CU_ASSERT_EQUAL(completed, 4); 4045 CU_ASSERT(ut_seq.complete); 4046 CU_ASSERT_EQUAL(ut_seq.status, 0); 4047 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 4048 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 2); 4049 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 4050 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 4051 CU_ASSERT_EQUAL(crc, crc2); 4052 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 4053 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 4054 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 4055 4056 /* Check that a copy won't be removed if the buffers don't match */ 4057 seq = NULL; 4058 completed = 0; 4059 crc = 0; 4060 memset(buf, 0, sizeof(buf)); 4061 memset(&tmp[0], 0xa5, 2048); 4062 memset(&tmp[1], 0xfe, sizeof(tmp[1])); 4063 memset(&tmp[2], 0xfe, sizeof(tmp[1])); 4064 dst_iovs[0].iov_base = &tmp[1][2048]; 4065 dst_iovs[0].iov_len = 2048; 4066 src_iovs[0].iov_base = tmp[0]; 4067 src_iovs[0].iov_len = 2048; 4068 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 4069 &src_iovs[0], 1, NULL, NULL, 0, 4070 ut_sequence_step_cb, &completed); 4071 CU_ASSERT_EQUAL(rc, 0); 4072 4073 src_iovs[1].iov_base = &tmp[1][2048]; 4074 src_iovs[1].iov_len = 2048; 4075 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 4076 ut_sequence_step_cb, &completed); 4077 CU_ASSERT_EQUAL(rc, 0); 4078 4079 dst_iovs[2].iov_base = buf; 4080 dst_iovs[2].iov_len = sizeof(buf); 4081 src_iovs[2].iov_base = tmp[1]; 4082 src_iovs[2].iov_len = sizeof(tmp[1]); 4083 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 4084 &src_iovs[2], 1, NULL, NULL, 0, 4085 ut_sequence_step_cb, &completed); 4086 CU_ASSERT_EQUAL(rc, 0); 4087 4088 ut_seq.complete = false; 4089 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4090 4091 poll_threads(); 4092 CU_ASSERT_EQUAL(completed, 3); 4093 CU_ASSERT(ut_seq.complete); 4094 CU_ASSERT_EQUAL(ut_seq.status, 0); 4095 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 4096 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 4097 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 4098 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], 2048, ~0u)); 4099 CU_ASSERT_EQUAL(memcmp(buf, tmp[2], 2048), 0); 4100 CU_ASSERT_EQUAL(memcmp(&buf[2048], tmp[0], 2048), 0); 4101 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 4102 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 4103 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 4104 4105 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 4106 g_modules_opc[i] = modules[i]; 4107 } 4108 4109 ut_clear_operations(); 4110 spdk_put_io_channel(ioch); 4111 poll_threads(); 4112 } 4113 4114 static int 4115 test_sequence_setup(void) 4116 { 4117 int rc; 4118 4119 allocate_cores(1); 4120 allocate_threads(1); 4121 set_thread(0); 4122 4123 rc = spdk_iobuf_initialize(); 4124 if (rc != 0) { 4125 CU_ASSERT(false); 4126 return -1; 4127 } 4128 4129 rc = spdk_accel_initialize(); 4130 if (rc != 0) { 4131 CU_ASSERT(false); 4132 return -1; 4133 } 4134 4135 return 0; 4136 } 4137 4138 static void 4139 finish_cb(void *cb_arg) 4140 { 4141 bool *done = cb_arg; 4142 4143 *done = true; 4144 } 4145 4146 static int 4147 test_sequence_cleanup(void) 4148 { 4149 bool done = false; 4150 4151 spdk_accel_finish(finish_cb, &done); 4152 4153 while (!done) { 4154 poll_threads(); 4155 } 4156 4157 done = false; 4158 spdk_iobuf_finish(finish_cb, &done); 4159 while (!done) { 4160 poll_threads(); 4161 } 4162 4163 free_threads(); 4164 free_cores(); 4165 4166 return 0; 4167 } 4168 4169 int 4170 main(int argc, char **argv) 4171 { 4172 CU_pSuite suite = NULL, seq_suite; 4173 unsigned int num_failures; 4174 4175 CU_initialize_registry(); 4176 4177 /* Sequence tests require accel to be initialized normally, so run them before the other 4178 * tests which register accel modules which aren't fully implemented, causing accel 4179 * initialization to fail. 4180 */ 4181 seq_suite = CU_add_suite("accel_sequence", test_sequence_setup, test_sequence_cleanup); 4182 CU_ADD_TEST(seq_suite, test_sequence_fill_copy); 4183 CU_ADD_TEST(seq_suite, test_sequence_abort); 4184 CU_ADD_TEST(seq_suite, test_sequence_append_error); 4185 CU_ADD_TEST(seq_suite, test_sequence_completion_error); 4186 #ifdef SPDK_CONFIG_ISAL /* accel_sw requires isa-l for compression */ 4187 CU_ADD_TEST(seq_suite, test_sequence_decompress); 4188 CU_ADD_TEST(seq_suite, test_sequence_reverse); 4189 #endif 4190 CU_ADD_TEST(seq_suite, test_sequence_copy_elision); 4191 CU_ADD_TEST(seq_suite, test_sequence_accel_buffers); 4192 CU_ADD_TEST(seq_suite, test_sequence_memory_domain); 4193 CU_ADD_TEST(seq_suite, test_sequence_module_memory_domain); 4194 #ifdef SPDK_CONFIG_ISAL_CRYPTO /* accel_sw requires isa-l-crypto for crypto operations */ 4195 CU_ADD_TEST(seq_suite, test_sequence_crypto); 4196 #endif 4197 CU_ADD_TEST(seq_suite, test_sequence_driver); 4198 CU_ADD_TEST(seq_suite, test_sequence_same_iovs); 4199 CU_ADD_TEST(seq_suite, test_sequence_crc32); 4200 4201 suite = CU_add_suite("accel", test_setup, test_cleanup); 4202 CU_ADD_TEST(suite, test_spdk_accel_task_complete); 4203 CU_ADD_TEST(suite, test_get_task); 4204 CU_ADD_TEST(suite, test_spdk_accel_submit_copy); 4205 CU_ADD_TEST(suite, test_spdk_accel_submit_dualcast); 4206 CU_ADD_TEST(suite, test_spdk_accel_submit_compare); 4207 CU_ADD_TEST(suite, test_spdk_accel_submit_fill); 4208 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c); 4209 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32cv); 4210 CU_ADD_TEST(suite, test_spdk_accel_submit_copy_crc32c); 4211 CU_ADD_TEST(suite, test_spdk_accel_submit_xor); 4212 CU_ADD_TEST(suite, test_spdk_accel_module_find_by_name); 4213 CU_ADD_TEST(suite, test_spdk_accel_module_register); 4214 4215 num_failures = spdk_ut_run_tests(argc, argv, NULL); 4216 CU_cleanup_registry(); 4217 4218 return num_failures; 4219 } 4220