1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2021 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk_internal/cunit.h" 8 #include "spdk_internal/mock.h" 9 #include "spdk/accel_module.h" 10 #include "thread/thread_internal.h" 11 #include "common/lib/ut_multithread.c" 12 #include "common/lib/test_iobuf.c" 13 #include "accel/accel.c" 14 #include "accel/accel_sw.c" 15 #include "unit/lib/json_mock.c" 16 17 DEFINE_STUB_V(spdk_memory_domain_destroy, (struct spdk_memory_domain *domain)); 18 DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, 19 (struct spdk_memory_domain *domain), "UT_DMA"); 20 DEFINE_STUB(spdk_memory_domain_get_system_domain, struct spdk_memory_domain *, (void), 21 (void *)0xabc); 22 DEFINE_STUB_V(spdk_memory_domain_set_invalidate, (struct spdk_memory_domain *domain, 23 spdk_memory_domain_invalidate_data_cb invalidate_cb)); 24 DEFINE_STUB_V(spdk_memory_domain_set_translation, (struct spdk_memory_domain *domain, 25 spdk_memory_domain_translate_memory_cb translate_cb)); 26 27 int 28 spdk_memory_domain_create(struct spdk_memory_domain **domain, enum spdk_dma_device_type type, 29 struct spdk_memory_domain_ctx *ctx, const char *id) 30 { 31 *domain = (void *)0xdeadbeef; 32 33 return 0; 34 } 35 36 struct ut_domain_ctx { 37 struct iovec iov; 38 struct iovec expected; 39 int pull_submit_status; 40 int push_submit_status; 41 int pull_complete_status; 42 int push_complete_status; 43 }; 44 45 static struct spdk_memory_domain *g_ut_domain = (void *)0xa55e1; 46 47 int 48 spdk_memory_domain_pull_data(struct spdk_memory_domain *sd, void *sctx, struct iovec *siov, 49 uint32_t siovcnt, struct iovec *diov, uint32_t diovcnt, 50 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 51 { 52 struct ut_domain_ctx *ctx = sctx; 53 54 CU_ASSERT_EQUAL(sd, g_ut_domain); 55 CU_ASSERT_EQUAL(siovcnt, 1); 56 CU_ASSERT_EQUAL(memcmp(siov, &ctx->expected, sizeof(*siov)), 0); 57 58 if (ctx->pull_submit_status != 0) { 59 return ctx->pull_submit_status; 60 } 61 62 if (ctx->pull_complete_status != 0) { 63 cpl_cb(cpl_arg, ctx->pull_complete_status); 64 return 0; 65 } 66 67 spdk_iovcpy(&ctx->iov, 1, diov, diovcnt); 68 cpl_cb(cpl_arg, 0); 69 70 return 0; 71 } 72 73 int 74 spdk_memory_domain_push_data(struct spdk_memory_domain *dd, void *dctx, struct iovec *diov, 75 uint32_t diovcnt, struct iovec *siov, uint32_t siovcnt, 76 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 77 { 78 struct ut_domain_ctx *ctx = dctx; 79 80 CU_ASSERT_EQUAL(dd, g_ut_domain); 81 CU_ASSERT_EQUAL(diovcnt, 1); 82 CU_ASSERT_EQUAL(memcmp(diov, &ctx->expected, sizeof(*diov)), 0); 83 84 if (ctx->push_submit_status != 0) { 85 return ctx->push_submit_status; 86 } 87 88 if (ctx->push_complete_status != 0) { 89 cpl_cb(cpl_arg, ctx->push_complete_status); 90 return 0; 91 } 92 93 spdk_iovcpy(siov, siovcnt, &ctx->iov, 1); 94 cpl_cb(cpl_arg, 0); 95 96 return 0; 97 } 98 99 /* global vars and setup/cleanup functions used for all test functions */ 100 struct spdk_accel_module_if g_module_if = {}; 101 struct accel_module g_module = { .module = &g_module_if }; 102 struct spdk_io_channel *g_ch = NULL; 103 struct accel_io_channel *g_accel_ch = NULL; 104 struct sw_accel_io_channel *g_sw_ch = NULL; 105 struct spdk_io_channel *g_module_ch = NULL; 106 107 static uint64_t g_opc_mask = 0; 108 109 static uint64_t 110 _accel_op_to_bit(enum spdk_accel_opcode opc) 111 { 112 return (1 << opc); 113 } 114 115 static bool 116 _supports_opcode(enum spdk_accel_opcode opc) 117 { 118 if (_accel_op_to_bit(opc) & g_opc_mask) { 119 return true; 120 } 121 return false; 122 } 123 124 static int 125 test_setup(void) 126 { 127 int i; 128 129 g_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct accel_io_channel)); 130 if (g_ch == NULL) { 131 /* for some reason the assert fatal macro doesn't work in the setup function. */ 132 CU_ASSERT(false); 133 return -1; 134 } 135 g_accel_ch = (struct accel_io_channel *)((char *)g_ch + sizeof(struct spdk_io_channel)); 136 g_module_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct sw_accel_io_channel)); 137 if (g_module_ch == NULL) { 138 CU_ASSERT(false); 139 return -1; 140 } 141 142 g_module_if.submit_tasks = sw_accel_submit_tasks; 143 g_module_if.name = "software"; 144 for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) { 145 g_accel_ch->module_ch[i] = g_module_ch; 146 g_modules_opc[i] = g_module; 147 } 148 g_sw_ch = (struct sw_accel_io_channel *)((char *)g_module_ch + sizeof( 149 struct spdk_io_channel)); 150 STAILQ_INIT(&g_sw_ch->tasks_to_complete); 151 g_module_if.supports_opcode = _supports_opcode; 152 return 0; 153 } 154 155 static int 156 test_cleanup(void) 157 { 158 free(g_ch); 159 free(g_module_ch); 160 161 return 0; 162 } 163 164 #define DUMMY_ARG 0xDEADBEEF 165 static bool g_dummy_cb_called = false; 166 static void 167 dummy_cb_fn(void *cb_arg, int status) 168 { 169 CU_ASSERT(*(uint32_t *)cb_arg == DUMMY_ARG); 170 CU_ASSERT(status == 0); 171 g_dummy_cb_called = true; 172 } 173 174 static void 175 test_spdk_accel_task_complete(void) 176 { 177 struct spdk_accel_task accel_task = {}; 178 struct spdk_accel_task *expected_accel_task = NULL; 179 uint32_t cb_arg = DUMMY_ARG; 180 int status = 0; 181 182 accel_task.accel_ch = g_accel_ch; 183 accel_task.cb_fn = dummy_cb_fn; 184 accel_task.cb_arg = &cb_arg; 185 STAILQ_INIT(&g_accel_ch->task_pool); 186 187 /* Confirm cb is called and task added to list. */ 188 spdk_accel_task_complete(&accel_task, status); 189 CU_ASSERT(g_dummy_cb_called == true); 190 expected_accel_task = STAILQ_FIRST(&g_accel_ch->task_pool); 191 STAILQ_REMOVE_HEAD(&g_accel_ch->task_pool, link); 192 CU_ASSERT(expected_accel_task == &accel_task); 193 } 194 195 static void 196 test_get_task(void) 197 { 198 struct spdk_accel_task *task; 199 struct spdk_accel_task _task; 200 void *cb_arg = NULL; 201 202 STAILQ_INIT(&g_accel_ch->task_pool); 203 204 /* no tasks left, return NULL. */ 205 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 206 CU_ASSERT(task == NULL); 207 208 _task.cb_fn = dummy_cb_fn; 209 _task.cb_arg = cb_arg; 210 _task.accel_ch = g_accel_ch; 211 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &_task, link); 212 213 /* Get a valid task. */ 214 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 215 CU_ASSERT(task == &_task); 216 CU_ASSERT(_task.cb_fn == dummy_cb_fn); 217 CU_ASSERT(_task.cb_arg == cb_arg); 218 CU_ASSERT(_task.accel_ch == g_accel_ch); 219 } 220 221 #define TEST_SUBMIT_SIZE 64 222 static void 223 test_spdk_accel_submit_copy(void) 224 { 225 const uint64_t nbytes = TEST_SUBMIT_SIZE; 226 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 227 uint8_t src[TEST_SUBMIT_SIZE] = {0}; 228 void *cb_arg = NULL; 229 int rc; 230 struct spdk_accel_task task; 231 struct spdk_accel_task_aux_data task_aux; 232 struct spdk_accel_task *expected_accel_task = NULL; 233 int flags = 0; 234 235 STAILQ_INIT(&g_accel_ch->task_pool); 236 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 237 238 /* Fail with no tasks on _get_task() */ 239 rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, flags, NULL, cb_arg); 240 CU_ASSERT(rc == -ENOMEM); 241 242 task.accel_ch = g_accel_ch; 243 task.flags = 1; 244 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 245 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 246 247 /* submission OK. */ 248 rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, flags, NULL, cb_arg); 249 CU_ASSERT(rc == 0); 250 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COPY); 251 CU_ASSERT(task.flags == 0); 252 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 253 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 254 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 255 CU_ASSERT(expected_accel_task == &task); 256 } 257 258 static void 259 test_spdk_accel_submit_dualcast(void) 260 { 261 void *dst1; 262 void *dst2; 263 void *src; 264 uint32_t align = ALIGN_4K; 265 uint64_t nbytes = TEST_SUBMIT_SIZE; 266 void *cb_arg = NULL; 267 int rc; 268 struct spdk_accel_task task; 269 struct spdk_accel_task_aux_data task_aux; 270 struct spdk_accel_task *expected_accel_task = NULL; 271 int flags = 0; 272 273 STAILQ_INIT(&g_accel_ch->task_pool); 274 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 275 276 /* Dualcast requires 4K alignment on dst addresses, 277 * hence using the hard coded address to test the buffer alignment 278 */ 279 dst1 = (void *)0x5000; 280 dst2 = (void *)0x60f0; 281 src = calloc(1, TEST_SUBMIT_SIZE); 282 SPDK_CU_ASSERT_FATAL(src != NULL); 283 memset(src, 0x5A, TEST_SUBMIT_SIZE); 284 285 /* This should fail since dst2 is not 4k aligned */ 286 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 287 CU_ASSERT(rc == -EINVAL); 288 289 dst1 = (void *)0x7010; 290 dst2 = (void *)0x6000; 291 /* This should fail since dst1 is not 4k aligned */ 292 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 293 CU_ASSERT(rc == -EINVAL); 294 295 /* Dualcast requires 4K alignment on dst addresses */ 296 dst1 = (void *)0x7000; 297 dst2 = (void *)0x6000; 298 /* Fail with no tasks on _get_task() */ 299 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 300 CU_ASSERT(rc == -ENOMEM); 301 302 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 303 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 304 305 /* accel submission OK., since we test the SW path , need to use valid memory addresses 306 * cannot hardcode them anymore */ 307 dst1 = spdk_dma_zmalloc(nbytes, align, NULL); 308 SPDK_CU_ASSERT_FATAL(dst1 != NULL); 309 dst2 = spdk_dma_zmalloc(nbytes, align, NULL); 310 SPDK_CU_ASSERT_FATAL(dst2 != NULL); 311 /* SW module does the dualcast. */ 312 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 313 CU_ASSERT(rc == 0); 314 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_DUALCAST); 315 CU_ASSERT(task.flags == 0); 316 CU_ASSERT(memcmp(dst1, src, TEST_SUBMIT_SIZE) == 0); 317 CU_ASSERT(memcmp(dst2, src, TEST_SUBMIT_SIZE) == 0); 318 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 319 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 320 CU_ASSERT(expected_accel_task == &task); 321 322 free(src); 323 spdk_free(dst1); 324 spdk_free(dst2); 325 } 326 327 static void 328 test_spdk_accel_submit_compare(void) 329 { 330 void *src1; 331 void *src2; 332 uint64_t nbytes = TEST_SUBMIT_SIZE; 333 void *cb_arg = NULL; 334 int rc; 335 struct spdk_accel_task task; 336 struct spdk_accel_task_aux_data task_aux; 337 struct spdk_accel_task *expected_accel_task = NULL; 338 339 STAILQ_INIT(&g_accel_ch->task_pool); 340 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 341 342 src1 = calloc(1, TEST_SUBMIT_SIZE); 343 SPDK_CU_ASSERT_FATAL(src1 != NULL); 344 src2 = calloc(1, TEST_SUBMIT_SIZE); 345 SPDK_CU_ASSERT_FATAL(src2 != NULL); 346 347 /* Fail with no tasks on _get_task() */ 348 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 349 CU_ASSERT(rc == -ENOMEM); 350 351 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 352 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 353 354 /* accel submission OK. */ 355 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 356 CU_ASSERT(rc == 0); 357 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COMPARE); 358 CU_ASSERT(memcmp(src1, src2, TEST_SUBMIT_SIZE) == 0); 359 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 360 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 361 CU_ASSERT(expected_accel_task == &task); 362 363 free(src1); 364 free(src2); 365 } 366 367 static void 368 test_spdk_accel_submit_fill(void) 369 { 370 void *dst; 371 void *src; 372 uint8_t fill = 0xf; 373 uint64_t fill64; 374 uint64_t nbytes = TEST_SUBMIT_SIZE; 375 void *cb_arg = NULL; 376 int rc; 377 struct spdk_accel_task task; 378 struct spdk_accel_task_aux_data task_aux; 379 struct spdk_accel_task *expected_accel_task = NULL; 380 int flags = 0; 381 382 STAILQ_INIT(&g_accel_ch->task_pool); 383 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 384 385 dst = calloc(1, TEST_SUBMIT_SIZE); 386 SPDK_CU_ASSERT_FATAL(dst != NULL); 387 src = calloc(1, TEST_SUBMIT_SIZE); 388 SPDK_CU_ASSERT_FATAL(src != NULL); 389 memset(src, fill, TEST_SUBMIT_SIZE); 390 memset(&fill64, fill, sizeof(uint64_t)); 391 392 /* Fail with no tasks on _get_task() */ 393 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 394 CU_ASSERT(rc == -ENOMEM); 395 396 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 397 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 398 399 /* accel submission OK. */ 400 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 401 CU_ASSERT(rc == 0); 402 CU_ASSERT(task.fill_pattern == fill64); 403 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_FILL); 404 CU_ASSERT(task.flags == 0); 405 406 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 407 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 408 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 409 CU_ASSERT(expected_accel_task == &task); 410 411 free(dst); 412 free(src); 413 } 414 415 static void 416 test_spdk_accel_submit_crc32c(void) 417 { 418 const uint64_t nbytes = TEST_SUBMIT_SIZE; 419 uint32_t crc_dst; 420 uint8_t src[TEST_SUBMIT_SIZE]; 421 uint32_t seed = 1; 422 void *cb_arg = NULL; 423 int rc; 424 struct spdk_accel_task task; 425 struct spdk_accel_task_aux_data task_aux; 426 struct spdk_accel_task *expected_accel_task = NULL; 427 428 STAILQ_INIT(&g_accel_ch->task_pool); 429 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 430 431 /* Fail with no tasks on _get_task() */ 432 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 433 CU_ASSERT(rc == -ENOMEM); 434 435 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 436 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 437 438 /* accel submission OK. */ 439 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 440 CU_ASSERT(rc == 0); 441 CU_ASSERT(task.crc_dst == &crc_dst); 442 CU_ASSERT(task.seed == seed); 443 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_CRC32C); 444 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 445 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 446 CU_ASSERT(expected_accel_task == &task); 447 } 448 449 static void 450 test_spdk_accel_submit_crc32cv(void) 451 { 452 uint32_t crc_dst; 453 uint32_t seed = 0; 454 uint32_t iov_cnt = 32; 455 void *cb_arg = NULL; 456 int rc; 457 uint32_t i = 0; 458 struct spdk_accel_task task; 459 struct spdk_accel_task_aux_data task_aux; 460 struct iovec iov[32]; 461 struct spdk_accel_task *expected_accel_task = NULL; 462 463 STAILQ_INIT(&g_accel_ch->task_pool); 464 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 465 466 for (i = 0; i < iov_cnt; i++) { 467 iov[i].iov_base = calloc(1, TEST_SUBMIT_SIZE); 468 SPDK_CU_ASSERT_FATAL(iov[i].iov_base != NULL); 469 iov[i].iov_len = TEST_SUBMIT_SIZE; 470 } 471 472 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 473 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 474 475 /* accel submission OK. */ 476 rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, NULL, cb_arg); 477 CU_ASSERT(rc == 0); 478 CU_ASSERT(task.s.iovs == iov); 479 CU_ASSERT(task.s.iovcnt == iov_cnt); 480 CU_ASSERT(task.crc_dst == &crc_dst); 481 CU_ASSERT(task.seed == seed); 482 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_CRC32C); 483 CU_ASSERT(task.cb_arg == cb_arg); 484 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 485 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 486 CU_ASSERT(expected_accel_task == &task); 487 488 for (i = 0; i < iov_cnt; i++) { 489 free(iov[i].iov_base); 490 } 491 } 492 493 static void 494 test_spdk_accel_submit_copy_crc32c(void) 495 { 496 const uint64_t nbytes = TEST_SUBMIT_SIZE; 497 uint32_t crc_dst; 498 uint8_t dst[TEST_SUBMIT_SIZE]; 499 uint8_t src[TEST_SUBMIT_SIZE]; 500 uint32_t seed = 0; 501 void *cb_arg = NULL; 502 int rc; 503 struct spdk_accel_task task; 504 struct spdk_accel_task_aux_data task_aux; 505 struct spdk_accel_task *expected_accel_task = NULL; 506 int flags = 0; 507 508 STAILQ_INIT(&g_accel_ch->task_pool); 509 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 510 511 /* Fail with no tasks on _get_task() */ 512 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 513 NULL, cb_arg); 514 CU_ASSERT(rc == -ENOMEM); 515 516 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 517 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 518 519 /* accel submission OK. */ 520 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 521 NULL, cb_arg); 522 CU_ASSERT(rc == 0); 523 CU_ASSERT(task.crc_dst == &crc_dst); 524 CU_ASSERT(task.seed == seed); 525 CU_ASSERT(task.flags == 0); 526 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COPY_CRC32C); 527 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 528 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 529 CU_ASSERT(expected_accel_task == &task); 530 } 531 532 static void 533 test_spdk_accel_submit_xor(void) 534 { 535 const uint64_t nbytes = TEST_SUBMIT_SIZE; 536 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 537 uint8_t src1[TEST_SUBMIT_SIZE] = {0}; 538 uint8_t src2[TEST_SUBMIT_SIZE] = {0}; 539 void *sources[] = { src1, src2 }; 540 uint32_t nsrcs = SPDK_COUNTOF(sources); 541 int rc; 542 struct spdk_accel_task task; 543 struct spdk_accel_task_aux_data task_aux; 544 struct spdk_accel_task *expected_accel_task = NULL; 545 546 STAILQ_INIT(&g_accel_ch->task_pool); 547 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 548 549 /* Fail with no tasks on _get_task() */ 550 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 551 CU_ASSERT(rc == -ENOMEM); 552 553 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 554 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 555 556 /* submission OK. */ 557 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 558 CU_ASSERT(rc == 0); 559 CU_ASSERT(task.nsrcs.srcs == sources); 560 CU_ASSERT(task.nsrcs.cnt == nsrcs); 561 CU_ASSERT(task.d.iovcnt == 1); 562 CU_ASSERT(task.d.iovs[0].iov_base == dst); 563 CU_ASSERT(task.d.iovs[0].iov_len == nbytes); 564 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_XOR); 565 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 566 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 567 CU_ASSERT(expected_accel_task == &task); 568 } 569 570 static void 571 test_spdk_accel_module_find_by_name(void) 572 { 573 struct spdk_accel_module_if mod1 = {}; 574 struct spdk_accel_module_if mod2 = {}; 575 struct spdk_accel_module_if mod3 = {}; 576 struct spdk_accel_module_if *accel_module = NULL; 577 578 mod1.name = "ioat"; 579 mod2.name = "idxd"; 580 mod3.name = "software"; 581 582 TAILQ_INIT(&spdk_accel_module_list); 583 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod1, tailq); 584 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod2, tailq); 585 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod3, tailq); 586 587 /* Now let's find a valid engine */ 588 accel_module = _module_find_by_name("ioat"); 589 CU_ASSERT(accel_module != NULL); 590 591 /* Try to find one that doesn't exist */ 592 accel_module = _module_find_by_name("XXX"); 593 CU_ASSERT(accel_module == NULL); 594 } 595 596 static int 597 ut_module_init_nop(void) 598 { 599 return 0; 600 } 601 602 static bool 603 ut_supports_opcode_all(enum spdk_accel_opcode opcode) 604 { 605 return true; 606 } 607 608 static void 609 ut_accel_module_priority_finish_done(void *done) 610 { 611 *(int *)done = 1; 612 } 613 614 static void 615 test_spdk_accel_module_register(void) 616 { 617 struct spdk_accel_module_if mods[] = { 618 { .name = "mod1", .priority = 1, }, 619 { .name = "mod3", .priority = 3, }, 620 { .name = "mod0", .priority = 0, }, 621 { .name = "mod2", .priority = 2, }, 622 }; 623 int rc, done = 0; 624 const char *modname = NULL; 625 size_t i; 626 627 allocate_cores(1); 628 allocate_threads(1); 629 set_thread(0); 630 631 TAILQ_INIT(&spdk_accel_module_list); 632 for (i = 0; i < SPDK_COUNTOF(mods); ++i) { 633 mods[i].module_init = ut_module_init_nop; 634 mods[i].supports_opcode = ut_supports_opcode_all; 635 spdk_accel_module_list_add(&mods[i]); 636 } 637 638 rc = spdk_accel_initialize(); 639 CU_ASSERT_EQUAL(rc, 0); 640 641 /* All opcodes should be assigned to the module with highest prio - mod3 */ 642 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 643 rc = spdk_accel_get_opc_module_name((enum spdk_accel_opcode)i, &modname); 644 CU_ASSERT_EQUAL(rc, 0); 645 CU_ASSERT_STRING_EQUAL(modname, "mod3"); 646 } 647 648 spdk_accel_finish(ut_accel_module_priority_finish_done, &done); 649 while (!done) { 650 poll_threads(); 651 } 652 653 TAILQ_INIT(&spdk_accel_module_list); 654 free_threads(); 655 free_cores(); 656 } 657 658 struct ut_sequence { 659 bool complete; 660 int status; 661 }; 662 663 static void 664 ut_sequence_step_cb(void *cb_arg) 665 { 666 int *completed = cb_arg; 667 668 (*completed)++; 669 } 670 671 static void 672 ut_sequence_complete_cb(void *cb_arg, int status) 673 { 674 struct ut_sequence *seq = cb_arg; 675 676 seq->complete = true; 677 seq->status = status; 678 } 679 680 static void 681 test_sequence_fill_copy(void) 682 { 683 struct spdk_accel_sequence *seq = NULL; 684 struct spdk_io_channel *ioch; 685 struct ut_sequence ut_seq; 686 char buf[4096], tmp[2][4096], expected[4096]; 687 struct iovec src_iovs[2], dst_iovs[2]; 688 int rc, completed; 689 690 ioch = spdk_accel_get_io_channel(); 691 SPDK_CU_ASSERT_FATAL(ioch != NULL); 692 693 /* First check the simplest case - single task in a sequence */ 694 memset(buf, 0, sizeof(buf)); 695 memset(expected, 0xa5, sizeof(expected)); 696 completed = 0; 697 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 698 ut_sequence_step_cb, &completed); 699 CU_ASSERT_EQUAL(rc, 0); 700 CU_ASSERT_EQUAL(completed, 0); 701 702 ut_seq.complete = false; 703 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 704 705 poll_threads(); 706 CU_ASSERT_EQUAL(completed, 1); 707 CU_ASSERT(ut_seq.complete); 708 CU_ASSERT_EQUAL(ut_seq.status, 0); 709 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 710 711 /* Check a single copy operation */ 712 memset(buf, 0, sizeof(buf)); 713 memset(tmp[0], 0xa5, sizeof(tmp[0])); 714 memset(expected, 0xa5, sizeof(expected)); 715 completed = 0; 716 seq = NULL; 717 718 dst_iovs[0].iov_base = buf; 719 dst_iovs[0].iov_len = sizeof(buf); 720 src_iovs[0].iov_base = tmp[0]; 721 src_iovs[0].iov_len = sizeof(tmp[0]); 722 723 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 724 &src_iovs[0], 1, NULL, NULL, 0, 725 ut_sequence_step_cb, &completed); 726 CU_ASSERT_EQUAL(rc, 0); 727 728 ut_seq.complete = false; 729 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 730 731 poll_threads(); 732 CU_ASSERT_EQUAL(completed, 1); 733 CU_ASSERT(ut_seq.complete); 734 CU_ASSERT_EQUAL(ut_seq.status, 0); 735 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 736 737 /* Check multiple fill operations */ 738 memset(buf, 0, sizeof(buf)); 739 memset(expected, 0xfe, 4096); 740 memset(expected, 0xde, 2048); 741 memset(expected, 0xa5, 1024); 742 seq = NULL; 743 completed = 0; 744 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, NULL, NULL, 0xfe, 0, 745 ut_sequence_step_cb, &completed); 746 CU_ASSERT_EQUAL(rc, 0); 747 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xde, 0, 748 ut_sequence_step_cb, &completed); 749 CU_ASSERT_EQUAL(rc, 0); 750 rc = spdk_accel_append_fill(&seq, ioch, buf, 1024, NULL, NULL, 0xa5, 0, 751 ut_sequence_step_cb, &completed); 752 CU_ASSERT_EQUAL(rc, 0); 753 754 ut_seq.complete = false; 755 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 756 757 poll_threads(); 758 CU_ASSERT_EQUAL(completed, 3); 759 CU_ASSERT(ut_seq.complete); 760 CU_ASSERT_EQUAL(ut_seq.status, 0); 761 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 762 763 /* Check multiple copy operations */ 764 memset(buf, 0, sizeof(buf)); 765 memset(tmp[0], 0, sizeof(tmp[0])); 766 memset(tmp[1], 0, sizeof(tmp[1])); 767 memset(expected, 0xa5, sizeof(expected)); 768 seq = NULL; 769 completed = 0; 770 771 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 772 ut_sequence_step_cb, &completed); 773 CU_ASSERT_EQUAL(rc, 0); 774 775 dst_iovs[0].iov_base = tmp[1]; 776 dst_iovs[0].iov_len = sizeof(tmp[1]); 777 src_iovs[0].iov_base = tmp[0]; 778 src_iovs[0].iov_len = sizeof(tmp[0]); 779 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 780 &src_iovs[0], 1, NULL, NULL, 0, 781 ut_sequence_step_cb, &completed); 782 CU_ASSERT_EQUAL(rc, 0); 783 784 dst_iovs[1].iov_base = buf; 785 dst_iovs[1].iov_len = sizeof(buf); 786 src_iovs[1].iov_base = tmp[1]; 787 src_iovs[1].iov_len = sizeof(tmp[1]); 788 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 789 &src_iovs[1], 1, NULL, NULL, 0, 790 ut_sequence_step_cb, &completed); 791 CU_ASSERT_EQUAL(rc, 0); 792 793 ut_seq.complete = false; 794 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 795 796 poll_threads(); 797 CU_ASSERT_EQUAL(completed, 3); 798 CU_ASSERT(ut_seq.complete); 799 CU_ASSERT_EQUAL(ut_seq.status, 0); 800 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 801 802 /* Check that adding a copy operation at the end will change destination buffer */ 803 memset(buf, 0, sizeof(buf)); 804 memset(tmp[0], 0, sizeof(tmp[0])); 805 memset(expected, 0xa5, sizeof(buf)); 806 seq = NULL; 807 completed = 0; 808 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 809 ut_sequence_step_cb, &completed); 810 CU_ASSERT_EQUAL(rc, 0); 811 812 dst_iovs[0].iov_base = buf; 813 dst_iovs[0].iov_len = sizeof(buf); 814 src_iovs[0].iov_base = tmp[0]; 815 src_iovs[0].iov_len = sizeof(tmp[0]); 816 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 817 &src_iovs[0], 1, NULL, NULL, 0, 818 ut_sequence_step_cb, &completed); 819 CU_ASSERT_EQUAL(rc, 0); 820 821 ut_seq.complete = false; 822 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 823 824 poll_threads(); 825 CU_ASSERT_EQUAL(completed, 2); 826 CU_ASSERT(ut_seq.complete); 827 CU_ASSERT_EQUAL(ut_seq.status, 0); 828 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 829 830 /* Check that it's also possible to add copy operation at the beginning */ 831 memset(buf, 0, sizeof(buf)); 832 memset(tmp[0], 0xde, sizeof(tmp[0])); 833 memset(tmp[1], 0, sizeof(tmp[1])); 834 memset(expected, 0xa5, sizeof(expected)); 835 seq = NULL; 836 completed = 0; 837 838 dst_iovs[0].iov_base = tmp[1]; 839 dst_iovs[0].iov_len = sizeof(tmp[1]); 840 src_iovs[0].iov_base = tmp[0]; 841 src_iovs[0].iov_len = sizeof(tmp[0]); 842 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 843 &src_iovs[0], 1, NULL, NULL, 0, 844 ut_sequence_step_cb, &completed); 845 CU_ASSERT_EQUAL(rc, 0); 846 847 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], sizeof(tmp[1]), NULL, NULL, 0xa5, 0, 848 ut_sequence_step_cb, &completed); 849 CU_ASSERT_EQUAL(rc, 0); 850 851 dst_iovs[1].iov_base = buf; 852 dst_iovs[1].iov_len = sizeof(buf); 853 src_iovs[1].iov_base = tmp[1]; 854 src_iovs[1].iov_len = sizeof(tmp[1]); 855 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 856 &src_iovs[1], 1, NULL, NULL, 0, 857 ut_sequence_step_cb, &completed); 858 CU_ASSERT_EQUAL(rc, 0); 859 860 ut_seq.complete = false; 861 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 862 863 poll_threads(); 864 CU_ASSERT_EQUAL(completed, 3); 865 CU_ASSERT(ut_seq.complete); 866 CU_ASSERT_EQUAL(ut_seq.status, 0); 867 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 868 869 spdk_put_io_channel(ioch); 870 poll_threads(); 871 } 872 873 static void 874 test_sequence_abort(void) 875 { 876 struct spdk_accel_sequence *seq = NULL; 877 struct spdk_io_channel *ioch; 878 char buf[4096], tmp[2][4096], expected[4096]; 879 struct iovec src_iovs[2], dst_iovs[2]; 880 int rc, completed; 881 882 ioch = spdk_accel_get_io_channel(); 883 SPDK_CU_ASSERT_FATAL(ioch != NULL); 884 885 /* Check that aborting a sequence calls operation's callback, the operation is not executed 886 * and the sequence is freed 887 */ 888 memset(buf, 0, sizeof(buf)); 889 memset(expected, 0, sizeof(buf)); 890 completed = 0; 891 seq = NULL; 892 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 893 ut_sequence_step_cb, &completed); 894 CU_ASSERT_EQUAL(rc, 0); 895 896 spdk_accel_sequence_abort(seq); 897 CU_ASSERT_EQUAL(completed, 1); 898 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 899 900 /* Check sequence with multiple operations */ 901 memset(buf, 0, sizeof(buf)); 902 memset(expected, 0, sizeof(buf)); 903 completed = 0; 904 seq = NULL; 905 906 dst_iovs[0].iov_base = tmp[1]; 907 dst_iovs[0].iov_len = sizeof(tmp[1]); 908 src_iovs[0].iov_base = tmp[0]; 909 src_iovs[0].iov_len = sizeof(tmp[0]); 910 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 911 &src_iovs[0], 1, NULL, NULL, 0, 912 ut_sequence_step_cb, &completed); 913 CU_ASSERT_EQUAL(rc, 0); 914 915 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 4096, NULL, NULL, 0xa5, 0, 916 ut_sequence_step_cb, &completed); 917 CU_ASSERT_EQUAL(rc, 0); 918 919 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xde, 0, 920 ut_sequence_step_cb, &completed); 921 CU_ASSERT_EQUAL(rc, 0); 922 923 dst_iovs[1].iov_base = buf; 924 dst_iovs[1].iov_len = sizeof(buf); 925 src_iovs[1].iov_base = tmp[1]; 926 src_iovs[1].iov_len = sizeof(tmp[1]); 927 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 928 &src_iovs[1], 1, NULL, NULL, 0, 929 ut_sequence_step_cb, &completed); 930 CU_ASSERT_EQUAL(rc, 0); 931 932 spdk_accel_sequence_abort(seq); 933 CU_ASSERT_EQUAL(completed, 4); 934 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 935 936 /* This should be a no-op */ 937 spdk_accel_sequence_abort(NULL); 938 939 spdk_put_io_channel(ioch); 940 poll_threads(); 941 } 942 943 static void 944 test_sequence_append_error(void) 945 { 946 struct spdk_accel_sequence *seq = NULL; 947 struct spdk_io_channel *ioch; 948 struct accel_io_channel *accel_ch; 949 struct iovec src_iovs, dst_iovs; 950 char buf[4096]; 951 STAILQ_HEAD(, spdk_accel_task) tasks = STAILQ_HEAD_INITIALIZER(tasks); 952 SLIST_HEAD(, spdk_accel_sequence) seqs = SLIST_HEAD_INITIALIZER(seqs); 953 int rc; 954 955 ioch = spdk_accel_get_io_channel(); 956 SPDK_CU_ASSERT_FATAL(ioch != NULL); 957 accel_ch = spdk_io_channel_get_ctx(ioch); 958 959 /* Check that append fails and no sequence object is allocated when there are no more free 960 * tasks */ 961 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 962 963 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 964 ut_sequence_step_cb, NULL); 965 CU_ASSERT_EQUAL(rc, -ENOMEM); 966 CU_ASSERT_PTR_NULL(seq); 967 968 dst_iovs.iov_base = buf; 969 dst_iovs.iov_len = 2048; 970 src_iovs.iov_base = &buf[2048]; 971 src_iovs.iov_len = 2048; 972 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 973 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 974 CU_ASSERT_EQUAL(rc, -ENOMEM); 975 CU_ASSERT_PTR_NULL(seq); 976 977 dst_iovs.iov_base = buf; 978 dst_iovs.iov_len = 2048; 979 src_iovs.iov_base = &buf[2048]; 980 src_iovs.iov_len = 2048; 981 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 982 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 983 CU_ASSERT_EQUAL(rc, -ENOMEM); 984 CU_ASSERT_PTR_NULL(seq); 985 986 /* Check that the same happens when the sequence queue is empty */ 987 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 988 SLIST_SWAP(&seqs, &accel_ch->seq_pool, spdk_accel_sequence); 989 990 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 991 ut_sequence_step_cb, NULL); 992 CU_ASSERT_EQUAL(rc, -ENOMEM); 993 CU_ASSERT_PTR_NULL(seq); 994 995 dst_iovs.iov_base = buf; 996 dst_iovs.iov_len = 2048; 997 src_iovs.iov_base = &buf[2048]; 998 src_iovs.iov_len = 2048; 999 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1000 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 1001 CU_ASSERT_EQUAL(rc, -ENOMEM); 1002 CU_ASSERT_PTR_NULL(seq); 1003 1004 dst_iovs.iov_base = buf; 1005 dst_iovs.iov_len = 2048; 1006 src_iovs.iov_base = &buf[2048]; 1007 src_iovs.iov_len = 2048; 1008 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1009 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 1010 CU_ASSERT_EQUAL(rc, -ENOMEM); 1011 CU_ASSERT_PTR_NULL(seq); 1012 1013 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 1014 1015 spdk_put_io_channel(ioch); 1016 poll_threads(); 1017 } 1018 1019 struct ut_sequence_operation { 1020 int complete_status; 1021 int submit_status; 1022 int count; 1023 struct iovec *src_iovs; 1024 uint32_t src_iovcnt; 1025 struct spdk_memory_domain *src_domain; 1026 void *src_domain_ctx; 1027 struct iovec *dst_iovs; 1028 uint32_t dst_iovcnt; 1029 struct spdk_memory_domain *dst_domain; 1030 void *dst_domain_ctx; 1031 int (*submit)(struct spdk_io_channel *ch, struct spdk_accel_task *t); 1032 }; 1033 1034 static struct ut_sequence_operation g_seq_operations[SPDK_ACCEL_OPC_LAST]; 1035 1036 static int 1037 ut_sequnce_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *task) 1038 { 1039 struct ut_sequence_operation *op = &g_seq_operations[task->op_code]; 1040 1041 op->count++; 1042 if (op->submit != NULL) { 1043 return op->submit(ch, task); 1044 } 1045 if (op->src_iovs != NULL) { 1046 CU_ASSERT_EQUAL(task->s.iovcnt, op->src_iovcnt); 1047 CU_ASSERT_EQUAL(memcmp(task->s.iovs, op->src_iovs, 1048 sizeof(struct iovec) * op->src_iovcnt), 0); 1049 } 1050 if (op->dst_iovs != NULL) { 1051 CU_ASSERT_EQUAL(task->d.iovcnt, op->dst_iovcnt); 1052 CU_ASSERT_EQUAL(memcmp(task->d.iovs, op->dst_iovs, 1053 sizeof(struct iovec) * op->dst_iovcnt), 0); 1054 } 1055 1056 CU_ASSERT_EQUAL(task->src_domain, op->src_domain); 1057 CU_ASSERT_EQUAL(task->dst_domain, op->dst_domain); 1058 if (op->src_domain != NULL) { 1059 CU_ASSERT_EQUAL(task->src_domain_ctx, op->src_domain_ctx); 1060 } 1061 if (op->dst_domain != NULL) { 1062 CU_ASSERT_EQUAL(task->dst_domain_ctx, op->dst_domain_ctx); 1063 } 1064 1065 if (op->submit_status != 0) { 1066 return op->submit_status; 1067 } 1068 1069 spdk_accel_task_complete(task, op->complete_status); 1070 1071 return 0; 1072 } 1073 1074 static void 1075 ut_clear_operations(void) 1076 { 1077 memset(&g_seq_operations, 0, sizeof(g_seq_operations)); 1078 } 1079 1080 static void 1081 test_sequence_completion_error(void) 1082 { 1083 struct spdk_accel_sequence *seq = NULL; 1084 struct spdk_io_channel *ioch; 1085 struct ut_sequence ut_seq; 1086 struct iovec src_iovs, dst_iovs; 1087 char buf[4096], tmp[4096]; 1088 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 1089 int i, rc, completed; 1090 1091 ioch = spdk_accel_get_io_channel(); 1092 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1093 1094 /* Override the submit_tasks function */ 1095 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1096 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1097 modules[i] = g_modules_opc[i]; 1098 g_modules_opc[i] = g_module; 1099 } 1100 1101 memset(buf, 0, sizeof(buf)); 1102 memset(tmp, 0, sizeof(tmp)); 1103 1104 /* Check that if the first operation completes with an error, the whole sequence is 1105 * completed with that error and that all operations' completion callbacks are executed 1106 */ 1107 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = -E2BIG; 1108 completed = 0; 1109 seq = NULL; 1110 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1111 ut_sequence_step_cb, &completed); 1112 CU_ASSERT_EQUAL(rc, 0); 1113 1114 dst_iovs.iov_base = buf; 1115 dst_iovs.iov_len = sizeof(buf); 1116 src_iovs.iov_base = tmp; 1117 src_iovs.iov_len = sizeof(tmp); 1118 1119 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1120 &src_iovs, 1, NULL, NULL, 0, 1121 ut_sequence_step_cb, &completed); 1122 CU_ASSERT_EQUAL(rc, 0); 1123 1124 ut_seq.complete = false; 1125 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1126 1127 poll_threads(); 1128 CU_ASSERT_EQUAL(completed, 2); 1129 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 1130 1131 /* Check the same with a second operation in the sequence */ 1132 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].complete_status = -EACCES; 1133 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 1134 completed = 0; 1135 seq = NULL; 1136 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1137 ut_sequence_step_cb, &completed); 1138 CU_ASSERT_EQUAL(rc, 0); 1139 1140 dst_iovs.iov_base = buf; 1141 dst_iovs.iov_len = sizeof(buf); 1142 src_iovs.iov_base = tmp; 1143 src_iovs.iov_len = sizeof(tmp); 1144 1145 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1146 &src_iovs, 1, NULL, NULL, 0, 1147 ut_sequence_step_cb, &completed); 1148 CU_ASSERT_EQUAL(rc, 0); 1149 1150 ut_seq.complete = false; 1151 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1152 1153 poll_threads(); 1154 CU_ASSERT_EQUAL(completed, 2); 1155 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 1156 1157 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].complete_status = 0; 1158 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 1159 1160 /* Check submission failure of the first operation */ 1161 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit_status = -EADDRINUSE; 1162 completed = 0; 1163 seq = NULL; 1164 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1165 ut_sequence_step_cb, &completed); 1166 CU_ASSERT_EQUAL(rc, 0); 1167 1168 dst_iovs.iov_base = buf; 1169 dst_iovs.iov_len = sizeof(buf); 1170 src_iovs.iov_base = tmp; 1171 src_iovs.iov_len = sizeof(tmp); 1172 1173 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1174 &src_iovs, 1, NULL, NULL, 0, 1175 ut_sequence_step_cb, &completed); 1176 CU_ASSERT_EQUAL(rc, 0); 1177 1178 ut_seq.complete = false; 1179 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1180 1181 poll_threads(); 1182 CU_ASSERT_EQUAL(completed, 2); 1183 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 1184 1185 /* Check the same with a second operation */ 1186 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit_status = -EADDRNOTAVAIL; 1187 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit_status = 0; 1188 completed = 0; 1189 seq = NULL; 1190 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1191 ut_sequence_step_cb, &completed); 1192 CU_ASSERT_EQUAL(rc, 0); 1193 1194 dst_iovs.iov_base = buf; 1195 dst_iovs.iov_len = sizeof(buf); 1196 src_iovs.iov_base = tmp; 1197 src_iovs.iov_len = sizeof(tmp); 1198 1199 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1200 &src_iovs, 1, NULL, NULL, 0, 1201 ut_sequence_step_cb, &completed); 1202 CU_ASSERT_EQUAL(rc, 0); 1203 1204 ut_seq.complete = false; 1205 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1206 1207 poll_threads(); 1208 CU_ASSERT_EQUAL(completed, 2); 1209 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 1210 1211 /* Cleanup module pointers to make subsequent tests work correctly */ 1212 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1213 g_modules_opc[i] = modules[i]; 1214 } 1215 1216 ut_clear_operations(); 1217 spdk_put_io_channel(ioch); 1218 poll_threads(); 1219 } 1220 1221 #ifdef SPDK_CONFIG_ISAL 1222 static void 1223 ut_compress_cb(void *cb_arg, int status) 1224 { 1225 int *completed = cb_arg; 1226 1227 CU_ASSERT_EQUAL(status, 0); 1228 1229 *completed = 1; 1230 } 1231 1232 static void 1233 test_sequence_decompress(void) 1234 { 1235 struct spdk_accel_sequence *seq = NULL; 1236 struct spdk_io_channel *ioch; 1237 struct ut_sequence ut_seq; 1238 char buf[4096], tmp[2][4096], expected[4096]; 1239 struct iovec src_iovs[2], dst_iovs[2]; 1240 uint32_t compressed_size; 1241 int rc, completed = 0; 1242 1243 ioch = spdk_accel_get_io_channel(); 1244 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1245 1246 memset(expected, 0xa5, sizeof(expected)); 1247 src_iovs[0].iov_base = expected; 1248 src_iovs[0].iov_len = sizeof(expected); 1249 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1250 &compressed_size, 0, ut_compress_cb, &completed); 1251 CU_ASSERT_EQUAL(rc, 0); 1252 1253 while (!completed) { 1254 poll_threads(); 1255 } 1256 1257 /* Check a single decompress operation in a sequence */ 1258 seq = NULL; 1259 completed = 0; 1260 1261 dst_iovs[0].iov_base = buf; 1262 dst_iovs[0].iov_len = sizeof(buf); 1263 src_iovs[0].iov_base = tmp[0]; 1264 src_iovs[0].iov_len = compressed_size; 1265 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1266 &src_iovs[0], 1, NULL, NULL, 0, 1267 ut_sequence_step_cb, &completed); 1268 CU_ASSERT_EQUAL(rc, 0); 1269 1270 ut_seq.complete = false; 1271 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1272 1273 poll_threads(); 1274 1275 CU_ASSERT_EQUAL(completed, 1); 1276 CU_ASSERT(ut_seq.complete); 1277 CU_ASSERT_EQUAL(ut_seq.status, 0); 1278 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1279 1280 /* Put the decompress operation in the middle of a sequence with a copy operation at the 1281 * beginning and a fill at the end modifying the first 2048B of the buffer. 1282 */ 1283 memset(expected, 0xfe, 2048); 1284 memset(buf, 0, sizeof(buf)); 1285 seq = NULL; 1286 completed = 0; 1287 1288 dst_iovs[0].iov_base = tmp[1]; 1289 dst_iovs[0].iov_len = compressed_size; 1290 src_iovs[0].iov_base = tmp[0]; 1291 src_iovs[0].iov_len = compressed_size; 1292 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1293 &src_iovs[0], 1, NULL, NULL, 0, 1294 ut_sequence_step_cb, &completed); 1295 CU_ASSERT_EQUAL(rc, 0); 1296 1297 dst_iovs[1].iov_base = buf; 1298 dst_iovs[1].iov_len = sizeof(buf); 1299 src_iovs[1].iov_base = tmp[1]; 1300 src_iovs[1].iov_len = compressed_size; 1301 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1302 &src_iovs[1], 1, NULL, NULL, 0, 1303 ut_sequence_step_cb, &completed); 1304 CU_ASSERT_EQUAL(rc, 0); 1305 1306 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1307 ut_sequence_step_cb, &completed); 1308 CU_ASSERT_EQUAL(rc, 0); 1309 1310 ut_seq.complete = false; 1311 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1312 1313 poll_threads(); 1314 1315 CU_ASSERT_EQUAL(completed, 3); 1316 CU_ASSERT(ut_seq.complete); 1317 CU_ASSERT_EQUAL(ut_seq.status, 0); 1318 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1319 1320 /* Check sequence with decompress at the beginning: decompress -> copy */ 1321 memset(expected, 0xa5, sizeof(expected)); 1322 memset(buf, 0, sizeof(buf)); 1323 seq = NULL; 1324 completed = 0; 1325 1326 dst_iovs[0].iov_base = tmp[1]; 1327 dst_iovs[0].iov_len = sizeof(tmp[1]); 1328 src_iovs[0].iov_base = tmp[0]; 1329 src_iovs[0].iov_len = compressed_size; 1330 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1331 &src_iovs[0], 1, NULL, NULL, 0, 1332 ut_sequence_step_cb, &completed); 1333 CU_ASSERT_EQUAL(rc, 0); 1334 1335 dst_iovs[1].iov_base = buf; 1336 dst_iovs[1].iov_len = sizeof(buf); 1337 src_iovs[1].iov_base = tmp[1]; 1338 src_iovs[1].iov_len = sizeof(tmp[1]); 1339 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1340 &src_iovs[1], 1, NULL, NULL, 0, 1341 ut_sequence_step_cb, &completed); 1342 CU_ASSERT_EQUAL(rc, 0); 1343 1344 ut_seq.complete = false; 1345 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1346 1347 poll_threads(); 1348 1349 CU_ASSERT_EQUAL(completed, 2); 1350 CU_ASSERT(ut_seq.complete); 1351 CU_ASSERT_EQUAL(ut_seq.status, 0); 1352 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1353 1354 spdk_put_io_channel(ioch); 1355 poll_threads(); 1356 } 1357 1358 static void 1359 test_sequence_reverse(void) 1360 { 1361 struct spdk_accel_sequence *seq = NULL; 1362 struct spdk_io_channel *ioch; 1363 struct ut_sequence ut_seq; 1364 char buf[4096], tmp[2][4096], expected[4096]; 1365 struct iovec src_iovs[2], dst_iovs[2]; 1366 uint32_t compressed_size; 1367 int rc, completed = 0; 1368 1369 ioch = spdk_accel_get_io_channel(); 1370 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1371 1372 memset(expected, 0xa5, sizeof(expected)); 1373 src_iovs[0].iov_base = expected; 1374 src_iovs[0].iov_len = sizeof(expected); 1375 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1376 &compressed_size, 0, ut_compress_cb, &completed); 1377 CU_ASSERT_EQUAL(rc, 0); 1378 1379 while (!completed) { 1380 poll_threads(); 1381 } 1382 1383 /* First check that reversing a sequnce with a single operation is a no-op */ 1384 memset(buf, 0, sizeof(buf)); 1385 seq = NULL; 1386 completed = 0; 1387 1388 dst_iovs[0].iov_base = buf; 1389 dst_iovs[0].iov_len = sizeof(buf); 1390 src_iovs[0].iov_base = tmp[0]; 1391 src_iovs[0].iov_len = compressed_size; 1392 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1393 &src_iovs[0], 1, NULL, NULL, 0, 1394 ut_sequence_step_cb, &completed); 1395 CU_ASSERT_EQUAL(rc, 0); 1396 1397 spdk_accel_sequence_reverse(seq); 1398 1399 ut_seq.complete = false; 1400 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1401 1402 poll_threads(); 1403 1404 CU_ASSERT_EQUAL(completed, 1); 1405 CU_ASSERT(ut_seq.complete); 1406 CU_ASSERT_EQUAL(ut_seq.status, 0); 1407 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1408 1409 /* Add a copy operation at the end with src set to the compressed data. After reverse(), 1410 * that copy operation should be first, so decompress() should receive compressed data in 1411 * its src buffer. 1412 */ 1413 memset(buf, 0, sizeof(buf)); 1414 memset(tmp[1], 0, sizeof(tmp[1])); 1415 seq = NULL; 1416 completed = 0; 1417 1418 dst_iovs[0].iov_base = buf; 1419 dst_iovs[0].iov_len = sizeof(buf); 1420 src_iovs[0].iov_base = tmp[1]; 1421 src_iovs[0].iov_len = compressed_size; 1422 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1423 &src_iovs[0], 1, NULL, NULL, 0, 1424 ut_sequence_step_cb, &completed); 1425 CU_ASSERT_EQUAL(rc, 0); 1426 1427 dst_iovs[1].iov_base = tmp[1]; 1428 dst_iovs[1].iov_len = compressed_size; 1429 src_iovs[1].iov_base = tmp[0]; 1430 src_iovs[1].iov_len = compressed_size; 1431 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1432 &src_iovs[1], 1, NULL, NULL, 0, 1433 ut_sequence_step_cb, &completed); 1434 CU_ASSERT_EQUAL(rc, 0); 1435 1436 spdk_accel_sequence_reverse(seq); 1437 1438 ut_seq.complete = false; 1439 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1440 1441 poll_threads(); 1442 1443 CU_ASSERT_EQUAL(completed, 2); 1444 CU_ASSERT(ut_seq.complete); 1445 CU_ASSERT_EQUAL(ut_seq.status, 0); 1446 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1447 1448 /* Check the same, but add an extra fill operation at the beginning that should execute last 1449 * after reverse(). 1450 */ 1451 memset(buf, 0, sizeof(buf)); 1452 memset(tmp[1], 0, sizeof(tmp[1])); 1453 memset(expected, 0xfe, 2048); 1454 seq = NULL; 1455 completed = 0; 1456 1457 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1458 ut_sequence_step_cb, &completed); 1459 CU_ASSERT_EQUAL(rc, 0); 1460 1461 dst_iovs[0].iov_base = buf; 1462 dst_iovs[0].iov_len = sizeof(buf); 1463 src_iovs[0].iov_base = tmp[1]; 1464 src_iovs[0].iov_len = compressed_size; 1465 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1466 &src_iovs[0], 1, NULL, NULL, 0, 1467 ut_sequence_step_cb, &completed); 1468 CU_ASSERT_EQUAL(rc, 0); 1469 1470 dst_iovs[1].iov_base = tmp[1]; 1471 dst_iovs[1].iov_len = compressed_size; 1472 src_iovs[1].iov_base = tmp[0]; 1473 src_iovs[1].iov_len = compressed_size; 1474 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1475 &src_iovs[1], 1, NULL, NULL, 0, 1476 ut_sequence_step_cb, &completed); 1477 CU_ASSERT_EQUAL(rc, 0); 1478 1479 spdk_accel_sequence_reverse(seq); 1480 1481 ut_seq.complete = false; 1482 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1483 1484 poll_threads(); 1485 1486 CU_ASSERT_EQUAL(completed, 3); 1487 CU_ASSERT(ut_seq.complete); 1488 CU_ASSERT_EQUAL(ut_seq.status, 0); 1489 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1490 1491 /* Build the sequence in order and then reverse it twice */ 1492 memset(buf, 0, sizeof(buf)); 1493 memset(tmp[1], 0, sizeof(tmp[1])); 1494 seq = NULL; 1495 completed = 0; 1496 1497 dst_iovs[0].iov_base = tmp[1]; 1498 dst_iovs[0].iov_len = compressed_size; 1499 src_iovs[0].iov_base = tmp[0]; 1500 src_iovs[0].iov_len = compressed_size; 1501 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1502 &src_iovs[0], 1, NULL, NULL, 0, 1503 ut_sequence_step_cb, &completed); 1504 CU_ASSERT_EQUAL(rc, 0); 1505 1506 dst_iovs[1].iov_base = buf; 1507 dst_iovs[1].iov_len = sizeof(buf); 1508 src_iovs[1].iov_base = tmp[1]; 1509 src_iovs[1].iov_len = compressed_size; 1510 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1511 &src_iovs[1], 1, NULL, NULL, 0, 1512 ut_sequence_step_cb, &completed); 1513 CU_ASSERT_EQUAL(rc, 0); 1514 1515 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1516 ut_sequence_step_cb, &completed); 1517 CU_ASSERT_EQUAL(rc, 0); 1518 1519 spdk_accel_sequence_reverse(seq); 1520 spdk_accel_sequence_reverse(seq); 1521 1522 ut_seq.complete = false; 1523 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1524 1525 poll_threads(); 1526 1527 CU_ASSERT_EQUAL(completed, 3); 1528 CU_ASSERT(ut_seq.complete); 1529 CU_ASSERT_EQUAL(ut_seq.status, 0); 1530 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1531 1532 spdk_put_io_channel(ioch); 1533 poll_threads(); 1534 } 1535 #endif 1536 1537 static void 1538 test_sequence_copy_elision(void) 1539 { 1540 struct spdk_accel_sequence *seq = NULL; 1541 struct spdk_io_channel *ioch; 1542 struct ut_sequence ut_seq; 1543 struct iovec src_iovs[4], dst_iovs[4], exp_iovs[2]; 1544 char buf[4096], tmp[4][4096]; 1545 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 1546 struct spdk_accel_crypto_key key = {}; 1547 int i, rc, completed; 1548 1549 ioch = spdk_accel_get_io_channel(); 1550 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1551 1552 /* Override the submit_tasks function */ 1553 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1554 g_module.supports_memory_domains = true; 1555 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1556 g_seq_operations[i].complete_status = 0; 1557 g_seq_operations[i].submit_status = 0; 1558 g_seq_operations[i].count = 0; 1559 1560 modules[i] = g_modules_opc[i]; 1561 g_modules_opc[i] = g_module; 1562 } 1563 1564 /* Check that a copy operation at the beginning is removed */ 1565 seq = NULL; 1566 completed = 0; 1567 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovcnt = 1; 1568 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovcnt = 1; 1569 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1570 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1571 exp_iovs[0].iov_base = tmp[0]; 1572 exp_iovs[0].iov_len = sizeof(tmp[0]); 1573 exp_iovs[1].iov_base = buf; 1574 exp_iovs[1].iov_len = 2048; 1575 1576 dst_iovs[0].iov_base = tmp[1]; 1577 dst_iovs[0].iov_len = sizeof(tmp[1]); 1578 src_iovs[0].iov_base = tmp[0]; 1579 src_iovs[0].iov_len = sizeof(tmp[0]); 1580 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1581 &src_iovs[0], 1, NULL, NULL, 0, 1582 ut_sequence_step_cb, &completed); 1583 CU_ASSERT_EQUAL(rc, 0); 1584 1585 dst_iovs[1].iov_base = buf; 1586 dst_iovs[1].iov_len = 2048; 1587 src_iovs[1].iov_base = tmp[1]; 1588 src_iovs[1].iov_len = sizeof(tmp[1]); 1589 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1590 &src_iovs[1], 1, NULL, NULL, 0, 1591 ut_sequence_step_cb, &completed); 1592 CU_ASSERT_EQUAL(rc, 0); 1593 1594 ut_seq.complete = false; 1595 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1596 1597 poll_threads(); 1598 1599 CU_ASSERT_EQUAL(completed, 2); 1600 CU_ASSERT(ut_seq.complete); 1601 CU_ASSERT_EQUAL(ut_seq.status, 0); 1602 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1603 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1604 1605 /* Check that a copy operation at the end is removed too */ 1606 seq = NULL; 1607 completed = 0; 1608 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1609 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1610 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1611 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1612 exp_iovs[0].iov_base = tmp[0]; 1613 exp_iovs[0].iov_len = sizeof(tmp[0]); 1614 exp_iovs[1].iov_base = buf; 1615 exp_iovs[1].iov_len = 2048; 1616 1617 dst_iovs[0].iov_base = tmp[1]; 1618 dst_iovs[0].iov_len = 2048; 1619 src_iovs[0].iov_base = tmp[0]; 1620 src_iovs[0].iov_len = sizeof(tmp[0]); 1621 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1622 &src_iovs[0], 1, NULL, NULL, 0, 1623 ut_sequence_step_cb, &completed); 1624 CU_ASSERT_EQUAL(rc, 0); 1625 1626 dst_iovs[1].iov_base = buf; 1627 dst_iovs[1].iov_len = 2048; 1628 src_iovs[1].iov_base = tmp[1]; 1629 src_iovs[1].iov_len = 2048; 1630 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1631 &src_iovs[1], 1, NULL, NULL, 0, 1632 ut_sequence_step_cb, &completed); 1633 CU_ASSERT_EQUAL(rc, 0); 1634 1635 ut_seq.complete = false; 1636 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1637 1638 poll_threads(); 1639 1640 CU_ASSERT_EQUAL(completed, 2); 1641 CU_ASSERT(ut_seq.complete); 1642 CU_ASSERT_EQUAL(ut_seq.status, 0); 1643 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1644 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1645 1646 /* Check a copy operation both at the beginning and the end */ 1647 seq = NULL; 1648 completed = 0; 1649 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1650 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1651 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1652 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1653 exp_iovs[0].iov_base = tmp[0]; 1654 exp_iovs[0].iov_len = sizeof(tmp[0]); 1655 exp_iovs[1].iov_base = buf; 1656 exp_iovs[1].iov_len = 2048; 1657 1658 dst_iovs[0].iov_base = tmp[1]; 1659 dst_iovs[0].iov_len = sizeof(tmp[1]); 1660 src_iovs[0].iov_base = tmp[0]; 1661 src_iovs[0].iov_len = sizeof(tmp[0]); 1662 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1663 &src_iovs[0], 1, NULL, NULL, 0, 1664 ut_sequence_step_cb, &completed); 1665 CU_ASSERT_EQUAL(rc, 0); 1666 1667 dst_iovs[1].iov_base = tmp[2]; 1668 dst_iovs[1].iov_len = 2048; 1669 src_iovs[1].iov_base = tmp[1]; 1670 src_iovs[1].iov_len = sizeof(tmp[1]); 1671 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1672 &src_iovs[1], 1, NULL, NULL, 0, 1673 ut_sequence_step_cb, &completed); 1674 CU_ASSERT_EQUAL(rc, 0); 1675 1676 dst_iovs[2].iov_base = buf; 1677 dst_iovs[2].iov_len = 2048; 1678 src_iovs[2].iov_base = tmp[2]; 1679 src_iovs[2].iov_len = 2048; 1680 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1681 &src_iovs[2], 1, NULL, NULL, 0, 1682 ut_sequence_step_cb, &completed); 1683 CU_ASSERT_EQUAL(rc, 0); 1684 1685 ut_seq.complete = false; 1686 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1687 1688 poll_threads(); 1689 1690 CU_ASSERT_EQUAL(completed, 3); 1691 CU_ASSERT(ut_seq.complete); 1692 CU_ASSERT_EQUAL(ut_seq.status, 0); 1693 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1694 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1695 1696 /* Check decompress + copy + decompress + copy */ 1697 seq = NULL; 1698 completed = 0; 1699 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1700 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1701 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = NULL; 1702 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = NULL; 1703 1704 dst_iovs[0].iov_base = tmp[1]; 1705 dst_iovs[0].iov_len = sizeof(tmp[1]); 1706 src_iovs[0].iov_base = tmp[0]; 1707 src_iovs[0].iov_len = sizeof(tmp[0]); 1708 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1709 &src_iovs[0], 1, NULL, NULL, 0, 1710 ut_sequence_step_cb, &completed); 1711 CU_ASSERT_EQUAL(rc, 0); 1712 1713 dst_iovs[1].iov_base = tmp[2]; 1714 dst_iovs[1].iov_len = 2048; 1715 src_iovs[1].iov_base = tmp[1]; 1716 src_iovs[1].iov_len = sizeof(tmp[1]); 1717 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1718 &src_iovs[1], 1, NULL, NULL, 0, 1719 ut_sequence_step_cb, &completed); 1720 CU_ASSERT_EQUAL(rc, 0); 1721 1722 dst_iovs[2].iov_base = tmp[3]; 1723 dst_iovs[2].iov_len = 1024; 1724 src_iovs[2].iov_base = tmp[2]; 1725 src_iovs[2].iov_len = 2048; 1726 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1727 &src_iovs[2], 1, NULL, NULL, 0, 1728 ut_sequence_step_cb, &completed); 1729 CU_ASSERT_EQUAL(rc, 0); 1730 1731 dst_iovs[3].iov_base = buf; 1732 dst_iovs[3].iov_len = 1024; 1733 src_iovs[3].iov_base = tmp[3]; 1734 src_iovs[3].iov_len = 1024; 1735 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 1736 &src_iovs[3], 1, NULL, NULL, 0, 1737 ut_sequence_step_cb, &completed); 1738 CU_ASSERT_EQUAL(rc, 0); 1739 1740 ut_seq.complete = false; 1741 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1742 1743 poll_threads(); 1744 1745 CU_ASSERT_EQUAL(completed, 4); 1746 CU_ASSERT(ut_seq.complete); 1747 CU_ASSERT_EQUAL(ut_seq.status, 0); 1748 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1749 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 2); 1750 1751 /* Check two copy operations - one of them should be removed, while the other should be 1752 * executed normally */ 1753 seq = NULL; 1754 completed = 0; 1755 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1756 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovcnt = 1; 1757 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovcnt = 1; 1758 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovs = &exp_iovs[0]; 1759 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovs = &exp_iovs[1]; 1760 exp_iovs[0].iov_base = tmp[0]; 1761 exp_iovs[0].iov_len = sizeof(tmp[0]); 1762 exp_iovs[1].iov_base = buf; 1763 exp_iovs[1].iov_len = sizeof(buf); 1764 1765 dst_iovs[0].iov_base = tmp[1]; 1766 dst_iovs[0].iov_len = sizeof(tmp[1]); 1767 src_iovs[0].iov_base = tmp[0]; 1768 src_iovs[0].iov_len = sizeof(tmp[0]); 1769 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1770 &src_iovs[0], 1, NULL, NULL, 0, 1771 ut_sequence_step_cb, &completed); 1772 CU_ASSERT_EQUAL(rc, 0); 1773 1774 dst_iovs[1].iov_base = buf; 1775 dst_iovs[1].iov_len = sizeof(buf); 1776 src_iovs[1].iov_base = tmp[1]; 1777 src_iovs[1].iov_len = sizeof(tmp[1]); 1778 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1779 &src_iovs[1], 1, NULL, NULL, 0, 1780 ut_sequence_step_cb, &completed); 1781 CU_ASSERT_EQUAL(rc, 0); 1782 1783 ut_seq.complete = false; 1784 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1785 1786 poll_threads(); 1787 1788 CU_ASSERT_EQUAL(completed, 2); 1789 CU_ASSERT(ut_seq.complete); 1790 CU_ASSERT_EQUAL(ut_seq.status, 0); 1791 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 1792 1793 /* Check fill + copy */ 1794 seq = NULL; 1795 completed = 0; 1796 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1797 g_seq_operations[SPDK_ACCEL_OPC_FILL].count = 0; 1798 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovs = NULL; 1799 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovs = NULL; 1800 g_seq_operations[SPDK_ACCEL_OPC_FILL].dst_iovcnt = 1; 1801 g_seq_operations[SPDK_ACCEL_OPC_FILL].dst_iovs = &exp_iovs[0]; 1802 exp_iovs[0].iov_base = buf; 1803 exp_iovs[0].iov_len = sizeof(buf); 1804 1805 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 1806 ut_sequence_step_cb, &completed); 1807 CU_ASSERT_EQUAL(rc, 0); 1808 1809 dst_iovs[0].iov_base = buf; 1810 dst_iovs[0].iov_len = sizeof(buf); 1811 src_iovs[0].iov_base = tmp[0]; 1812 src_iovs[0].iov_len = sizeof(tmp[0]); 1813 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1814 &src_iovs[0], 1, NULL, NULL, 0, 1815 ut_sequence_step_cb, &completed); 1816 CU_ASSERT_EQUAL(rc, 0); 1817 1818 ut_seq.complete = false; 1819 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1820 1821 poll_threads(); 1822 1823 CU_ASSERT_EQUAL(completed, 2); 1824 CU_ASSERT(ut_seq.complete); 1825 CU_ASSERT_EQUAL(ut_seq.status, 0); 1826 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1827 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 1); 1828 1829 /* Check copy + encrypt + copy */ 1830 seq = NULL; 1831 completed = 0; 1832 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1833 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 1834 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovcnt = 1; 1835 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovcnt = 1; 1836 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovs = &exp_iovs[0]; 1837 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovs = &exp_iovs[1]; 1838 exp_iovs[0].iov_base = tmp[0]; 1839 exp_iovs[0].iov_len = sizeof(tmp[0]); 1840 exp_iovs[1].iov_base = buf; 1841 exp_iovs[1].iov_len = sizeof(buf); 1842 1843 dst_iovs[0].iov_base = tmp[1]; 1844 dst_iovs[0].iov_len = sizeof(tmp[1]); 1845 src_iovs[0].iov_base = tmp[0]; 1846 src_iovs[0].iov_len = sizeof(tmp[0]); 1847 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1848 &src_iovs[0], 1, NULL, NULL, 0, 1849 ut_sequence_step_cb, &completed); 1850 CU_ASSERT_EQUAL(rc, 0); 1851 1852 dst_iovs[1].iov_base = tmp[2]; 1853 dst_iovs[1].iov_len = sizeof(tmp[2]); 1854 src_iovs[1].iov_base = tmp[1]; 1855 src_iovs[1].iov_len = sizeof(tmp[1]); 1856 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1857 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1858 ut_sequence_step_cb, &completed); 1859 CU_ASSERT_EQUAL(rc, 0); 1860 1861 dst_iovs[2].iov_base = buf; 1862 dst_iovs[2].iov_len = sizeof(buf); 1863 src_iovs[2].iov_base = tmp[2]; 1864 src_iovs[2].iov_len = sizeof(tmp[2]); 1865 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1866 &src_iovs[2], 1, NULL, NULL, 0, 1867 ut_sequence_step_cb, &completed); 1868 CU_ASSERT_EQUAL(rc, 0); 1869 1870 ut_seq.complete = false; 1871 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1872 1873 poll_threads(); 1874 1875 CU_ASSERT_EQUAL(completed, 3); 1876 CU_ASSERT(ut_seq.complete); 1877 CU_ASSERT_EQUAL(ut_seq.status, 0); 1878 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1879 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 1880 1881 /* Check copy + decrypt + copy */ 1882 seq = NULL; 1883 completed = 0; 1884 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1885 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 1886 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1887 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1888 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1889 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1890 exp_iovs[0].iov_base = tmp[0]; 1891 exp_iovs[0].iov_len = sizeof(tmp[0]); 1892 exp_iovs[1].iov_base = buf; 1893 exp_iovs[1].iov_len = sizeof(buf); 1894 1895 dst_iovs[0].iov_base = tmp[1]; 1896 dst_iovs[0].iov_len = sizeof(tmp[1]); 1897 src_iovs[0].iov_base = tmp[0]; 1898 src_iovs[0].iov_len = sizeof(tmp[0]); 1899 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1900 &src_iovs[0], 1, NULL, NULL, 0, 1901 ut_sequence_step_cb, &completed); 1902 CU_ASSERT_EQUAL(rc, 0); 1903 1904 dst_iovs[1].iov_base = tmp[2]; 1905 dst_iovs[1].iov_len = sizeof(tmp[2]); 1906 src_iovs[1].iov_base = tmp[1]; 1907 src_iovs[1].iov_len = sizeof(tmp[1]); 1908 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1909 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1910 ut_sequence_step_cb, &completed); 1911 CU_ASSERT_EQUAL(rc, 0); 1912 1913 dst_iovs[2].iov_base = buf; 1914 dst_iovs[2].iov_len = sizeof(buf); 1915 src_iovs[2].iov_base = tmp[2]; 1916 src_iovs[2].iov_len = sizeof(tmp[2]); 1917 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1918 &src_iovs[2], 1, NULL, NULL, 0, 1919 ut_sequence_step_cb, &completed); 1920 CU_ASSERT_EQUAL(rc, 0); 1921 1922 ut_seq.complete = false; 1923 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1924 1925 poll_threads(); 1926 1927 CU_ASSERT_EQUAL(completed, 3); 1928 CU_ASSERT(ut_seq.complete); 1929 CU_ASSERT_EQUAL(ut_seq.status, 0); 1930 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1931 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 1932 1933 /* Check a sequence with memory domains and verify their pointers (and their contexts) are 1934 * correctly set on the next/prev task when a copy is removed */ 1935 seq = NULL; 1936 completed = 0; 1937 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1938 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 1939 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1940 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1941 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1942 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1943 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_domain = (void *)0x1; 1944 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_domain_ctx = (void *)0x2; 1945 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_domain = (void *)0x3; 1946 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_domain_ctx = (void *)0x4; 1947 exp_iovs[0].iov_base = tmp[0]; 1948 exp_iovs[0].iov_len = sizeof(tmp[0]); 1949 exp_iovs[1].iov_base = buf; 1950 exp_iovs[1].iov_len = sizeof(buf); 1951 1952 dst_iovs[0].iov_base = tmp[1]; 1953 dst_iovs[0].iov_len = sizeof(tmp[1]); 1954 src_iovs[0].iov_base = tmp[0]; 1955 src_iovs[0].iov_len = sizeof(tmp[0]); 1956 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, (void *)0xdead, (void *)0xbeef, 1957 &src_iovs[0], 1, (void *)0x3, (void *)0x4, 0, 1958 ut_sequence_step_cb, &completed); 1959 CU_ASSERT_EQUAL(rc, 0); 1960 1961 dst_iovs[1].iov_base = tmp[2]; 1962 dst_iovs[1].iov_len = sizeof(tmp[2]); 1963 src_iovs[1].iov_base = tmp[1]; 1964 src_iovs[1].iov_len = sizeof(tmp[1]); 1965 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 1966 &src_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 0, 1967 sizeof(tmp[2]), 0, ut_sequence_step_cb, &completed); 1968 CU_ASSERT_EQUAL(rc, 0); 1969 1970 dst_iovs[2].iov_base = buf; 1971 dst_iovs[2].iov_len = sizeof(buf); 1972 src_iovs[2].iov_base = tmp[2]; 1973 src_iovs[2].iov_len = sizeof(tmp[2]); 1974 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, (void *)0x1, (void *)0x2, 1975 &src_iovs[2], 1, (void *)0xdead, (void *)0xbeef, 0, 1976 ut_sequence_step_cb, &completed); 1977 CU_ASSERT_EQUAL(rc, 0); 1978 1979 ut_seq.complete = false; 1980 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1981 1982 poll_threads(); 1983 1984 CU_ASSERT_EQUAL(completed, 3); 1985 CU_ASSERT(ut_seq.complete); 1986 CU_ASSERT_EQUAL(ut_seq.status, 0); 1987 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1988 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 1989 1990 /* Cleanup module pointers to make subsequent tests work correctly */ 1991 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1992 g_modules_opc[i] = modules[i]; 1993 } 1994 1995 g_module.supports_memory_domains = false; 1996 ut_clear_operations(); 1997 spdk_put_io_channel(ioch); 1998 poll_threads(); 1999 } 2000 2001 static int 2002 ut_submit_decompress(struct spdk_io_channel *ch, struct spdk_accel_task *task) 2003 { 2004 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 2005 2006 spdk_accel_task_complete(task, 0); 2007 2008 return 0; 2009 } 2010 2011 static void 2012 test_sequence_accel_buffers(void) 2013 { 2014 struct spdk_accel_sequence *seq = NULL; 2015 struct spdk_io_channel *ioch; 2016 struct accel_io_channel *accel_ch; 2017 struct ut_sequence ut_seq; 2018 struct iovec src_iovs[3], dst_iovs[3]; 2019 char srcbuf[4096], dstbuf[4096], expected[4096]; 2020 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2021 void *buf[2], *domain_ctx[2], *iobuf_buf; 2022 struct spdk_memory_domain *domain[2]; 2023 struct spdk_iobuf_buffer *cache_entry; 2024 spdk_iobuf_buffer_stailq_t small_cache; 2025 uint32_t small_cache_count; 2026 int i, rc, completed; 2027 struct spdk_iobuf_opts opts_iobuf = {}; 2028 2029 /* Set up the iobuf to always use the "small" pool */ 2030 opts_iobuf.large_bufsize = 0x20000; 2031 opts_iobuf.large_pool_count = 0; 2032 opts_iobuf.small_bufsize = 0x10000; 2033 opts_iobuf.small_pool_count = 32; 2034 2035 rc = spdk_iobuf_set_opts(&opts_iobuf); 2036 CU_ASSERT(rc == 0); 2037 2038 ioch = spdk_accel_get_io_channel(); 2039 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2040 2041 /* Override the submit_tasks function */ 2042 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2043 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2044 modules[i] = g_modules_opc[i]; 2045 g_modules_opc[i] = g_module; 2046 } 2047 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2048 * decompress operations together in one sequence. 2049 */ 2050 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2051 g_seq_operations[SPDK_ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2052 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2053 2054 /* Check the simplest case: one operation using accel buffer as destination + copy operation 2055 * specifying the actual destination buffer 2056 */ 2057 memset(srcbuf, 0xa5, 4096); 2058 memset(dstbuf, 0x0, 4096); 2059 memset(expected, 0xa5, 4096); 2060 completed = 0; 2061 seq = NULL; 2062 2063 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2064 CU_ASSERT_EQUAL(rc, 0); 2065 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2066 2067 src_iovs[0].iov_base = srcbuf; 2068 src_iovs[0].iov_len = 4096; 2069 dst_iovs[0].iov_base = buf[0]; 2070 dst_iovs[0].iov_len = 4096; 2071 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2072 &src_iovs[0], 1, NULL, NULL, 0, 2073 ut_sequence_step_cb, &completed); 2074 CU_ASSERT_EQUAL(rc, 0); 2075 2076 src_iovs[1].iov_base = buf[0]; 2077 src_iovs[1].iov_len = 4096; 2078 dst_iovs[1].iov_base = dstbuf; 2079 dst_iovs[1].iov_len = 4096; 2080 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2081 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 2082 ut_sequence_step_cb, &completed); 2083 CU_ASSERT_EQUAL(rc, 0); 2084 2085 ut_seq.complete = false; 2086 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2087 2088 poll_threads(); 2089 2090 CU_ASSERT_EQUAL(completed, 2); 2091 CU_ASSERT(ut_seq.complete); 2092 CU_ASSERT_EQUAL(ut_seq.status, 0); 2093 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2094 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2095 2096 /* Start with a fill operation using accel buffer, followed by a decompress using another 2097 * accel buffer as dst, followed by a copy operation specifying dst buffer of the whole 2098 * sequence 2099 */ 2100 memset(srcbuf, 0x0, 4096); 2101 memset(dstbuf, 0x0, 4096); 2102 memset(expected, 0x5a, 4096); 2103 completed = 0; 2104 seq = NULL; 2105 2106 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2107 CU_ASSERT_EQUAL(rc, 0); 2108 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2109 2110 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2111 ut_sequence_step_cb, &completed); 2112 CU_ASSERT_EQUAL(rc, 0); 2113 2114 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2115 CU_ASSERT_EQUAL(rc, 0); 2116 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2117 2118 src_iovs[0].iov_base = buf[0]; 2119 src_iovs[0].iov_len = 4096; 2120 dst_iovs[0].iov_base = buf[1]; 2121 dst_iovs[0].iov_len = 4096; 2122 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2123 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2124 ut_sequence_step_cb, &completed); 2125 CU_ASSERT_EQUAL(rc, 0); 2126 2127 src_iovs[1].iov_base = buf[1]; 2128 src_iovs[1].iov_len = 4096; 2129 dst_iovs[1].iov_base = dstbuf; 2130 dst_iovs[1].iov_len = 4096; 2131 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2132 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2133 ut_sequence_step_cb, &completed); 2134 2135 ut_seq.complete = false; 2136 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2137 2138 poll_threads(); 2139 2140 CU_ASSERT_EQUAL(completed, 3); 2141 CU_ASSERT(ut_seq.complete); 2142 CU_ASSERT_EQUAL(ut_seq.status, 0); 2143 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2144 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2145 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2146 2147 /* Check the same, but with two decompress operations with the first one being in-place */ 2148 memset(srcbuf, 0x0, 4096); 2149 memset(dstbuf, 0x0, 4096); 2150 memset(expected, 0x5a, 4096); 2151 completed = 0; 2152 seq = NULL; 2153 2154 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2155 CU_ASSERT_EQUAL(rc, 0); 2156 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2157 2158 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2159 ut_sequence_step_cb, &completed); 2160 CU_ASSERT_EQUAL(rc, 0); 2161 2162 src_iovs[0].iov_base = buf[0]; 2163 src_iovs[0].iov_len = 4096; 2164 dst_iovs[0].iov_base = buf[0]; 2165 dst_iovs[0].iov_len = 4096; 2166 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2167 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2168 ut_sequence_step_cb, &completed); 2169 CU_ASSERT_EQUAL(rc, 0); 2170 2171 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2172 CU_ASSERT_EQUAL(rc, 0); 2173 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2174 2175 src_iovs[1].iov_base = buf[0]; 2176 src_iovs[1].iov_len = 4096; 2177 dst_iovs[1].iov_base = buf[1]; 2178 dst_iovs[1].iov_len = 4096; 2179 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, domain[1], domain_ctx[1], 2180 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 2181 ut_sequence_step_cb, &completed); 2182 CU_ASSERT_EQUAL(rc, 0); 2183 2184 src_iovs[2].iov_base = buf[1]; 2185 src_iovs[2].iov_len = 4096; 2186 dst_iovs[2].iov_base = dstbuf; 2187 dst_iovs[2].iov_len = 4096; 2188 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 2189 &src_iovs[2], 1, domain[1], domain_ctx[1], 0, 2190 ut_sequence_step_cb, &completed); 2191 2192 ut_seq.complete = false; 2193 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2194 2195 poll_threads(); 2196 2197 CU_ASSERT_EQUAL(completed, 4); 2198 CU_ASSERT(ut_seq.complete); 2199 CU_ASSERT_EQUAL(ut_seq.status, 0); 2200 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2201 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2202 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2203 2204 /* Check that specifying offsets within accel buffers works correctly */ 2205 memset(srcbuf, 0x0, 4096); 2206 memset(dstbuf, 0x0, 4096); 2207 completed = 0; 2208 seq = NULL; 2209 2210 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2211 CU_ASSERT_EQUAL(rc, 0); 2212 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2213 2214 /* Fill in each 1K of the buffer with different pattern */ 2215 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2216 ut_sequence_step_cb, &completed); 2217 CU_ASSERT_EQUAL(rc, 0); 2218 2219 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2220 0x5a, 0, ut_sequence_step_cb, &completed); 2221 CU_ASSERT_EQUAL(rc, 0); 2222 2223 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2224 0xfe, 0, ut_sequence_step_cb, &completed); 2225 CU_ASSERT_EQUAL(rc, 0); 2226 2227 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2228 0xed, 0, ut_sequence_step_cb, &completed); 2229 CU_ASSERT_EQUAL(rc, 0); 2230 2231 src_iovs[0].iov_base = buf[0]; 2232 src_iovs[0].iov_len = 4096; 2233 dst_iovs[0].iov_base = dstbuf; 2234 dst_iovs[0].iov_len = 4096; 2235 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2236 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2237 ut_sequence_step_cb, &completed); 2238 2239 ut_seq.complete = false; 2240 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2241 2242 poll_threads(); 2243 2244 memset(&expected[0], 0xa5, 1024); 2245 memset(&expected[1024], 0x5a, 1024); 2246 memset(&expected[2048], 0xfe, 1024); 2247 memset(&expected[3072], 0xed, 1024); 2248 CU_ASSERT_EQUAL(completed, 5); 2249 CU_ASSERT(ut_seq.complete); 2250 CU_ASSERT_EQUAL(ut_seq.status, 0); 2251 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2252 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2253 2254 /* Check the same but this time with a decompress operation on part of the buffer (512B 2255 * offset) */ 2256 memset(srcbuf, 0x0, 4096); 2257 memset(dstbuf, 0x0, 4096); 2258 completed = 0; 2259 seq = NULL; 2260 2261 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2262 CU_ASSERT_EQUAL(rc, 0); 2263 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2264 2265 /* Fill in each 1K of the buffer with different pattern */ 2266 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2267 ut_sequence_step_cb, &completed); 2268 CU_ASSERT_EQUAL(rc, 0); 2269 2270 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2271 0x5a, 0, ut_sequence_step_cb, &completed); 2272 CU_ASSERT_EQUAL(rc, 0); 2273 2274 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2275 0xfe, 0, ut_sequence_step_cb, &completed); 2276 CU_ASSERT_EQUAL(rc, 0); 2277 2278 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2279 0xed, 0, ut_sequence_step_cb, &completed); 2280 CU_ASSERT_EQUAL(rc, 0); 2281 2282 rc = spdk_accel_get_buf(ioch, 3072, &buf[1], &domain[1], &domain_ctx[1]); 2283 CU_ASSERT_EQUAL(rc, 0); 2284 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2285 2286 src_iovs[0].iov_base = (char *)buf[0] + 512; 2287 src_iovs[0].iov_len = 3072; 2288 dst_iovs[0].iov_base = (char *)buf[1] + 256; 2289 dst_iovs[0].iov_len = 3072; 2290 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2291 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2292 ut_sequence_step_cb, &completed); 2293 2294 src_iovs[1].iov_base = (char *)buf[1] + 256; 2295 src_iovs[1].iov_len = 3072; 2296 dst_iovs[1].iov_base = dstbuf; 2297 dst_iovs[1].iov_len = 3072; 2298 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2299 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2300 ut_sequence_step_cb, &completed); 2301 2302 ut_seq.complete = false; 2303 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2304 2305 poll_threads(); 2306 2307 memset(&expected[0], 0xa5, 512); 2308 memset(&expected[512], 0x5a, 1024); 2309 memset(&expected[1536], 0xfe, 1024); 2310 memset(&expected[2560], 0xed, 512); 2311 CU_ASSERT_EQUAL(completed, 6); 2312 CU_ASSERT(ut_seq.complete); 2313 CU_ASSERT_EQUAL(ut_seq.status, 0); 2314 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 3072), 0); 2315 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2316 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2317 2318 /* Check that if iobuf pool is empty, the sequence processing will wait until a buffer is 2319 * available 2320 */ 2321 accel_ch = spdk_io_channel_get_ctx(ioch); 2322 small_cache_count = accel_ch->iobuf.small.cache_count; 2323 STAILQ_INIT(&small_cache); 2324 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2325 accel_ch->iobuf.small.cache_count = 0; 2326 accel_ch->iobuf.small.cache_size = 0; 2327 g_iobuf.small_pool_count = 0; 2328 2329 /* First allocate a single buffer used by two operations */ 2330 memset(srcbuf, 0x0, 4096); 2331 memset(dstbuf, 0x0, 4096); 2332 memset(expected, 0xa5, 4096); 2333 completed = 0; 2334 seq = NULL; 2335 2336 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2337 CU_ASSERT_EQUAL(rc, 0); 2338 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2339 2340 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0xa5, 0, 2341 ut_sequence_step_cb, &completed); 2342 CU_ASSERT_EQUAL(rc, 0); 2343 2344 src_iovs[0].iov_base = buf[0]; 2345 src_iovs[0].iov_len = 4096; 2346 dst_iovs[0].iov_base = dstbuf; 2347 dst_iovs[0].iov_len = 4096; 2348 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2349 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2350 ut_sequence_step_cb, &completed); 2351 2352 ut_seq.complete = false; 2353 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2354 2355 poll_threads(); 2356 2357 CU_ASSERT_EQUAL(completed, 0); 2358 CU_ASSERT(!ut_seq.complete); 2359 2360 /* Get a buffer and return it to the pool to trigger the sequence to finish */ 2361 g_iobuf.small_pool_count = 1; 2362 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2363 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2364 CU_ASSERT(g_iobuf.small_pool_count == 0); 2365 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2366 2367 poll_threads(); 2368 2369 CU_ASSERT_EQUAL(completed, 2); 2370 CU_ASSERT(ut_seq.complete); 2371 CU_ASSERT_EQUAL(ut_seq.status, 0); 2372 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2373 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2374 2375 /* Return the buffers back to the cache */ 2376 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2377 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2378 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2379 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2380 small_cache_count++; 2381 } 2382 accel_ch->iobuf.small.cache_count = 0; 2383 g_iobuf.small_pool_count = 0; 2384 2385 /* Check a bit more complex scenario, with two buffers in the sequence */ 2386 memset(srcbuf, 0x0, 4096); 2387 memset(dstbuf, 0x0, 4096); 2388 memset(expected, 0x5a, 4096); 2389 completed = 0; 2390 seq = NULL; 2391 2392 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2393 CU_ASSERT_EQUAL(rc, 0); 2394 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2395 2396 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2397 ut_sequence_step_cb, &completed); 2398 CU_ASSERT_EQUAL(rc, 0); 2399 2400 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2401 CU_ASSERT_EQUAL(rc, 0); 2402 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2403 2404 src_iovs[0].iov_base = buf[0]; 2405 src_iovs[0].iov_len = 4096; 2406 dst_iovs[0].iov_base = buf[1]; 2407 dst_iovs[0].iov_len = 4096; 2408 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2409 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2410 ut_sequence_step_cb, &completed); 2411 CU_ASSERT_EQUAL(rc, 0); 2412 2413 src_iovs[1].iov_base = buf[1]; 2414 src_iovs[1].iov_len = 4096; 2415 dst_iovs[1].iov_base = dstbuf; 2416 dst_iovs[1].iov_len = 4096; 2417 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2418 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2419 ut_sequence_step_cb, &completed); 2420 CU_ASSERT_EQUAL(rc, 0); 2421 2422 ut_seq.complete = false; 2423 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2424 2425 poll_threads(); 2426 2427 CU_ASSERT_EQUAL(completed, 0); 2428 CU_ASSERT(!ut_seq.complete); 2429 2430 g_iobuf.small_pool_count = 1; 2431 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2432 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2433 g_iobuf.small_pool_count = 0; 2434 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2435 2436 /* One buffer is not enough to finish the whole sequence */ 2437 poll_threads(); 2438 CU_ASSERT(!ut_seq.complete); 2439 2440 g_iobuf.small_pool_count = 1; 2441 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2442 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2443 g_iobuf.small_pool_count = 0; 2444 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2445 2446 poll_threads(); 2447 2448 CU_ASSERT_EQUAL(completed, 3); 2449 CU_ASSERT(ut_seq.complete); 2450 CU_ASSERT_EQUAL(ut_seq.status, 0); 2451 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2452 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2453 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2454 2455 /* Return the buffers back to the cache */ 2456 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2457 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2458 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2459 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2460 small_cache_count++; 2461 } 2462 accel_ch->iobuf.small.cache_count = 0; 2463 2464 g_iobuf.small_pool_count = 32; 2465 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2466 accel_ch->iobuf.small.cache_count = small_cache_count; 2467 2468 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2469 g_modules_opc[i] = modules[i]; 2470 } 2471 2472 ut_clear_operations(); 2473 spdk_put_io_channel(ioch); 2474 poll_threads(); 2475 } 2476 2477 static void 2478 ut_domain_ctx_init(struct ut_domain_ctx *ctx, void *base, size_t len, struct iovec *expected) 2479 { 2480 ctx->iov.iov_base = base; 2481 ctx->iov.iov_len = len; 2482 ctx->expected = *expected; 2483 ctx->pull_submit_status = 0; 2484 ctx->push_submit_status = 0; 2485 ctx->pull_complete_status = 0; 2486 ctx->push_complete_status = 0; 2487 } 2488 2489 static void 2490 test_sequence_memory_domain(void) 2491 { 2492 struct spdk_accel_sequence *seq = NULL; 2493 struct spdk_io_channel *ioch; 2494 struct accel_io_channel *accel_ch; 2495 struct ut_sequence ut_seq; 2496 struct ut_domain_ctx domctx[4]; 2497 struct spdk_iobuf_buffer *cache_entry; 2498 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2499 struct spdk_memory_domain *accel_domain; 2500 spdk_iobuf_buffer_stailq_t small_cache; 2501 char srcbuf[4096], dstbuf[4096], expected[4096], tmp[4096]; 2502 struct iovec src_iovs[2], dst_iovs[2]; 2503 uint32_t small_cache_count; 2504 void *accel_buf, *accel_domain_ctx, *iobuf_buf; 2505 int i, rc, completed; 2506 2507 ioch = spdk_accel_get_io_channel(); 2508 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2509 2510 /* Override the submit_tasks function */ 2511 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2512 g_module.supports_memory_domains = false; 2513 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2514 modules[i] = g_modules_opc[i]; 2515 g_modules_opc[i] = g_module; 2516 } 2517 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2518 * decompress operations together in one sequence. 2519 */ 2520 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2521 g_seq_operations[SPDK_ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2522 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2523 2524 /* First check the simplest case - single fill operation with dstbuf in domain */ 2525 memset(expected, 0xa5, sizeof(expected)); 2526 memset(dstbuf, 0x0, sizeof(dstbuf)); 2527 completed = 0; 2528 seq = NULL; 2529 2530 /* Use some garbage pointer as dst and use domain ctx to get the actual dstbuf */ 2531 dst_iovs[0].iov_base = (void *)0xcafebabe; 2532 dst_iovs[0].iov_len = sizeof(dstbuf); 2533 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2534 2535 rc = spdk_accel_append_fill(&seq, ioch, dst_iovs[0].iov_base, dst_iovs[0].iov_len, 2536 g_ut_domain, &domctx[0], 0xa5, 0, 2537 ut_sequence_step_cb, &completed); 2538 CU_ASSERT_EQUAL(rc, 0); 2539 2540 ut_seq.complete = false; 2541 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2542 2543 poll_threads(); 2544 CU_ASSERT_EQUAL(completed, 1); 2545 CU_ASSERT(ut_seq.complete); 2546 CU_ASSERT_EQUAL(ut_seq.status, 0); 2547 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2548 2549 /* Check operation with both srcbuf and dstbuf in remote memory domain */ 2550 memset(expected, 0x5a, sizeof(expected)); 2551 memset(dstbuf, 0x0, sizeof(dstbuf)); 2552 memset(srcbuf, 0x5a, sizeof(dstbuf)); 2553 completed = 0; 2554 seq = NULL; 2555 2556 src_iovs[0].iov_base = (void *)0xcafebabe; 2557 src_iovs[0].iov_len = sizeof(srcbuf); 2558 dst_iovs[0].iov_base = (void *)0xbeefdead; 2559 dst_iovs[0].iov_len = sizeof(dstbuf); 2560 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2561 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2562 2563 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2564 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2565 ut_sequence_step_cb, &completed); 2566 CU_ASSERT_EQUAL(rc, 0); 2567 2568 ut_seq.complete = false; 2569 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2570 2571 poll_threads(); 2572 CU_ASSERT_EQUAL(completed, 1); 2573 CU_ASSERT(ut_seq.complete); 2574 CU_ASSERT_EQUAL(ut_seq.status, 0); 2575 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2576 2577 /* Two operations using a buffer in remote domain */ 2578 memset(expected, 0xa5, sizeof(expected)); 2579 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2580 memset(tmp, 0x0, sizeof(tmp)); 2581 memset(dstbuf, 0x0, sizeof(dstbuf)); 2582 completed = 0; 2583 seq = NULL; 2584 2585 src_iovs[0].iov_base = (void *)0xcafebabe; 2586 src_iovs[0].iov_len = sizeof(srcbuf); 2587 dst_iovs[0].iov_base = (void *)0xbeefdead; 2588 dst_iovs[0].iov_len = sizeof(tmp); 2589 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2590 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2591 2592 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2593 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2594 ut_sequence_step_cb, &completed); 2595 CU_ASSERT_EQUAL(rc, 0); 2596 2597 src_iovs[1].iov_base = (void *)0xbeefdead; 2598 src_iovs[1].iov_len = sizeof(tmp); 2599 dst_iovs[1].iov_base = (void *)0xa5a5a5a5; 2600 dst_iovs[1].iov_len = sizeof(dstbuf); 2601 ut_domain_ctx_init(&domctx[2], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2602 ut_domain_ctx_init(&domctx[3], tmp, sizeof(tmp), &src_iovs[1]); 2603 2604 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[2], 2605 &src_iovs[1], 1, g_ut_domain, &domctx[3], 0, 2606 ut_sequence_step_cb, &completed); 2607 CU_ASSERT_EQUAL(rc, 0); 2608 2609 ut_seq.complete = false; 2610 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2611 2612 poll_threads(); 2613 CU_ASSERT_EQUAL(completed, 2); 2614 CU_ASSERT(ut_seq.complete); 2615 CU_ASSERT_EQUAL(ut_seq.status, 0); 2616 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2617 2618 /* Use buffer in remote memory domain and accel buffer at the same time */ 2619 memset(expected, 0x5a, sizeof(expected)); 2620 memset(srcbuf, 0x5a, sizeof(expected)); 2621 memset(dstbuf, 0x0, sizeof(dstbuf)); 2622 completed = 0; 2623 seq = NULL; 2624 2625 rc = spdk_accel_get_buf(ioch, sizeof(dstbuf), &accel_buf, &accel_domain, &accel_domain_ctx); 2626 CU_ASSERT_EQUAL(rc, 0); 2627 2628 src_iovs[0].iov_base = (void *)0xfeedbeef; 2629 src_iovs[0].iov_len = sizeof(srcbuf); 2630 dst_iovs[0].iov_base = accel_buf; 2631 dst_iovs[0].iov_len = sizeof(dstbuf); 2632 ut_domain_ctx_init(&domctx[0], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2633 2634 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, accel_domain, accel_domain_ctx, 2635 &src_iovs[0], 1, g_ut_domain, &domctx[0], 0, 2636 ut_sequence_step_cb, &completed); 2637 CU_ASSERT_EQUAL(rc, 0); 2638 2639 src_iovs[1].iov_base = accel_buf; 2640 src_iovs[1].iov_len = sizeof(dstbuf); 2641 dst_iovs[1].iov_base = (void *)0xbeeffeed; 2642 dst_iovs[1].iov_len = sizeof(dstbuf); 2643 ut_domain_ctx_init(&domctx[1], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2644 2645 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[1], 2646 &src_iovs[1], 1, accel_domain, accel_domain_ctx, 0, 2647 ut_sequence_step_cb, &completed); 2648 CU_ASSERT_EQUAL(rc, 0); 2649 2650 ut_seq.complete = false; 2651 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2652 2653 poll_threads(); 2654 CU_ASSERT_EQUAL(completed, 2); 2655 CU_ASSERT(ut_seq.complete); 2656 CU_ASSERT_EQUAL(ut_seq.status, 0); 2657 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2658 spdk_accel_put_buf(ioch, accel_buf, accel_domain, accel_domain_ctx); 2659 2660 /* Check that a sequence with memory domains is correctly executed if buffers are not 2661 * immediately available */ 2662 memset(expected, 0xa5, sizeof(expected)); 2663 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2664 memset(dstbuf, 0x0, sizeof(dstbuf)); 2665 completed = 0; 2666 seq = NULL; 2667 /* Make sure the buffer pool is empty */ 2668 accel_ch = spdk_io_channel_get_ctx(ioch); 2669 small_cache_count = accel_ch->iobuf.small.cache_count; 2670 STAILQ_INIT(&small_cache); 2671 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2672 accel_ch->iobuf.small.cache_count = 0; 2673 g_iobuf.small_pool_count = 0; 2674 2675 src_iovs[0].iov_base = (void *)0xdeadbeef; 2676 src_iovs[0].iov_len = sizeof(srcbuf); 2677 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2678 dst_iovs[0].iov_len = sizeof(dstbuf); 2679 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2680 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2681 2682 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2683 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2684 ut_sequence_step_cb, &completed); 2685 CU_ASSERT_EQUAL(rc, 0); 2686 2687 ut_seq.complete = false; 2688 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2689 2690 poll_threads(); 2691 CU_ASSERT_EQUAL(completed, 0); 2692 CU_ASSERT(!ut_seq.complete); 2693 2694 /* Get a buffer and return it to the pool to trigger the sequence to resume. It shouldn't 2695 * be able to complete, as it needs two buffers */ 2696 g_iobuf.small_pool_count = 1; 2697 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2698 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2699 g_iobuf.small_pool_count = 0; 2700 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2701 2702 CU_ASSERT_EQUAL(completed, 0); 2703 CU_ASSERT(!ut_seq.complete); 2704 2705 /* Return another buffer, this time the sequence should finish */ 2706 g_iobuf.small_pool_count = 1; 2707 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2708 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2709 g_iobuf.small_pool_count = 0; 2710 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2711 2712 poll_threads(); 2713 CU_ASSERT_EQUAL(completed, 1); 2714 CU_ASSERT(ut_seq.complete); 2715 CU_ASSERT_EQUAL(ut_seq.status, 0); 2716 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2717 2718 /* Return the buffers back to the cache */ 2719 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2720 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2721 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2722 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2723 small_cache_count++; 2724 } 2725 accel_ch->iobuf.small.cache_count = 0; 2726 2727 g_iobuf.small_pool_count = 32; 2728 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2729 accel_ch->iobuf.small.cache_count = small_cache_count; 2730 2731 /* Check error cases, starting with an error from spdk_memory_domain_pull_data() */ 2732 completed = 0; 2733 seq = NULL; 2734 2735 src_iovs[0].iov_base = (void *)0xdeadbeef; 2736 src_iovs[0].iov_len = sizeof(srcbuf); 2737 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2738 dst_iovs[0].iov_len = sizeof(dstbuf); 2739 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2740 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2741 domctx[1].pull_submit_status = -E2BIG; 2742 2743 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2744 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2745 ut_sequence_step_cb, &completed); 2746 CU_ASSERT_EQUAL(rc, 0); 2747 2748 ut_seq.complete = false; 2749 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2750 2751 CU_ASSERT_EQUAL(completed, 1); 2752 CU_ASSERT(ut_seq.complete); 2753 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 2754 2755 /* Check completion error from spdk_memory_domain_pull_data() */ 2756 completed = 0; 2757 seq = NULL; 2758 2759 src_iovs[0].iov_base = (void *)0xdeadbeef; 2760 src_iovs[0].iov_len = sizeof(srcbuf); 2761 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2762 dst_iovs[0].iov_len = sizeof(dstbuf); 2763 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2764 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2765 domctx[1].pull_complete_status = -EACCES; 2766 2767 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2768 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2769 ut_sequence_step_cb, &completed); 2770 CU_ASSERT_EQUAL(rc, 0); 2771 2772 ut_seq.complete = false; 2773 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2774 2775 CU_ASSERT_EQUAL(completed, 1); 2776 CU_ASSERT(ut_seq.complete); 2777 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 2778 2779 /* Check submission error from spdk_memory_domain_push_data() */ 2780 completed = 0; 2781 seq = NULL; 2782 2783 src_iovs[0].iov_base = (void *)0xdeadbeef; 2784 src_iovs[0].iov_len = sizeof(srcbuf); 2785 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2786 dst_iovs[0].iov_len = sizeof(dstbuf); 2787 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2788 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2789 domctx[0].push_submit_status = -EADDRINUSE; 2790 2791 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2792 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2793 ut_sequence_step_cb, &completed); 2794 CU_ASSERT_EQUAL(rc, 0); 2795 2796 ut_seq.complete = false; 2797 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2798 2799 CU_ASSERT_EQUAL(completed, 1); 2800 CU_ASSERT(ut_seq.complete); 2801 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 2802 2803 /* Check completion error from spdk_memory_domain_push_data() */ 2804 completed = 0; 2805 seq = NULL; 2806 2807 src_iovs[0].iov_base = (void *)0xdeadbeef; 2808 src_iovs[0].iov_len = sizeof(srcbuf); 2809 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2810 dst_iovs[0].iov_len = sizeof(dstbuf); 2811 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2812 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2813 domctx[0].push_complete_status = -EADDRNOTAVAIL; 2814 2815 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2816 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2817 ut_sequence_step_cb, &completed); 2818 CU_ASSERT_EQUAL(rc, 0); 2819 2820 ut_seq.complete = false; 2821 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2822 2823 CU_ASSERT_EQUAL(completed, 1); 2824 CU_ASSERT(ut_seq.complete); 2825 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 2826 2827 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2828 g_modules_opc[i] = modules[i]; 2829 } 2830 2831 ut_clear_operations(); 2832 spdk_put_io_channel(ioch); 2833 poll_threads(); 2834 } 2835 2836 static int 2837 ut_submit_decompress_memory_domain(struct spdk_io_channel *ch, struct spdk_accel_task *task) 2838 { 2839 struct ut_domain_ctx *ctx; 2840 struct iovec *src_iovs, *dst_iovs; 2841 uint32_t src_iovcnt, dst_iovcnt; 2842 2843 src_iovs = task->s.iovs; 2844 dst_iovs = task->d.iovs; 2845 src_iovcnt = task->s.iovcnt; 2846 dst_iovcnt = task->d.iovcnt; 2847 2848 if (task->src_domain != NULL) { 2849 ctx = task->src_domain_ctx; 2850 CU_ASSERT_EQUAL(memcmp(task->s.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2851 src_iovs = &ctx->iov; 2852 src_iovcnt = 1; 2853 } 2854 2855 if (task->dst_domain != NULL) { 2856 ctx = task->dst_domain_ctx; 2857 CU_ASSERT_EQUAL(memcmp(task->d.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2858 dst_iovs = &ctx->iov; 2859 dst_iovcnt = 1; 2860 } 2861 2862 spdk_iovcpy(src_iovs, src_iovcnt, dst_iovs, dst_iovcnt); 2863 spdk_accel_task_complete(task, 0); 2864 2865 return 0; 2866 } 2867 2868 static void 2869 test_sequence_module_memory_domain(void) 2870 { 2871 struct spdk_accel_sequence *seq = NULL; 2872 struct spdk_io_channel *ioch; 2873 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2874 struct spdk_memory_domain *accel_domain; 2875 struct ut_sequence ut_seq; 2876 struct ut_domain_ctx domctx[2]; 2877 struct iovec src_iovs[2], dst_iovs[2]; 2878 void *buf, *accel_domain_ctx; 2879 char srcbuf[4096], dstbuf[4096], tmp[4096], expected[4096]; 2880 int i, rc, completed; 2881 2882 ioch = spdk_accel_get_io_channel(); 2883 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2884 2885 /* Override the submit_tasks function */ 2886 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2887 g_module.supports_memory_domains = true; 2888 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2889 modules[i] = g_modules_opc[i]; 2890 g_modules_opc[i] = g_module; 2891 } 2892 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress_memory_domain; 2893 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2894 2895 /* Check a sequence with both buffers in memory domains */ 2896 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2897 memset(expected, 0xa5, sizeof(expected)); 2898 memset(dstbuf, 0, sizeof(dstbuf)); 2899 seq = NULL; 2900 completed = 0; 2901 2902 src_iovs[0].iov_base = (void *)0xcafebabe; 2903 src_iovs[0].iov_len = sizeof(srcbuf); 2904 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2905 dst_iovs[0].iov_len = sizeof(dstbuf); 2906 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2907 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2908 2909 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2910 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2911 ut_sequence_step_cb, &completed); 2912 CU_ASSERT_EQUAL(rc, 0); 2913 2914 ut_seq.complete = false; 2915 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2916 2917 poll_threads(); 2918 2919 CU_ASSERT_EQUAL(completed, 1); 2920 CU_ASSERT(ut_seq.complete); 2921 CU_ASSERT_EQUAL(ut_seq.status, 0); 2922 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2923 2924 /* Check two operations each with a single buffer in memory domain */ 2925 memset(srcbuf, 0x5a, sizeof(srcbuf)); 2926 memset(expected, 0x5a, sizeof(expected)); 2927 memset(dstbuf, 0, sizeof(dstbuf)); 2928 memset(tmp, 0, sizeof(tmp)); 2929 seq = NULL; 2930 completed = 0; 2931 2932 src_iovs[0].iov_base = srcbuf; 2933 src_iovs[0].iov_len = sizeof(srcbuf); 2934 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2935 dst_iovs[0].iov_len = sizeof(tmp); 2936 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2937 2938 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2939 &src_iovs[0], 1, NULL, NULL, 0, 2940 ut_sequence_step_cb, &completed); 2941 CU_ASSERT_EQUAL(rc, 0); 2942 2943 src_iovs[1].iov_base = (void *)0xfeedbeef; 2944 src_iovs[1].iov_len = sizeof(tmp); 2945 dst_iovs[1].iov_base = dstbuf; 2946 dst_iovs[1].iov_len = sizeof(dstbuf); 2947 ut_domain_ctx_init(&domctx[1], tmp, sizeof(tmp), &src_iovs[1]); 2948 2949 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2950 &src_iovs[1], 1, g_ut_domain, &domctx[1], 0, 2951 ut_sequence_step_cb, &completed); 2952 CU_ASSERT_EQUAL(rc, 0); 2953 2954 ut_seq.complete = false; 2955 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2956 2957 poll_threads(); 2958 2959 CU_ASSERT_EQUAL(completed, 2); 2960 CU_ASSERT(ut_seq.complete); 2961 CU_ASSERT_EQUAL(ut_seq.status, 0); 2962 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2963 2964 /* Check a sequence with an accel buffer and a buffer in a regular memory domain */ 2965 memset(expected, 0xa5, sizeof(expected)); 2966 memset(dstbuf, 0, sizeof(dstbuf)); 2967 memset(tmp, 0, sizeof(tmp)); 2968 seq = NULL; 2969 completed = 0; 2970 2971 rc = spdk_accel_get_buf(ioch, 4096, &buf, &accel_domain, &accel_domain_ctx); 2972 CU_ASSERT_EQUAL(rc, 0); 2973 2974 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, accel_domain, accel_domain_ctx, 2975 0xa5, 0, ut_sequence_step_cb, &completed); 2976 CU_ASSERT_EQUAL(rc, 0); 2977 2978 src_iovs[0].iov_base = buf; 2979 src_iovs[0].iov_len = 4096; 2980 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2981 dst_iovs[0].iov_len = sizeof(dstbuf); 2982 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2983 2984 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2985 &src_iovs[0], 1, accel_domain, accel_domain_ctx, 0, 2986 ut_sequence_step_cb, &completed); 2987 CU_ASSERT_EQUAL(rc, 0); 2988 2989 ut_seq.complete = false; 2990 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2991 2992 poll_threads(); 2993 2994 CU_ASSERT_EQUAL(completed, 2); 2995 CU_ASSERT(ut_seq.complete); 2996 CU_ASSERT_EQUAL(ut_seq.status, 0); 2997 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2998 2999 spdk_accel_put_buf(ioch, buf, accel_domain, accel_domain_ctx); 3000 3001 g_module.supports_memory_domains = false; 3002 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3003 g_modules_opc[i] = modules[i]; 3004 } 3005 3006 ut_clear_operations(); 3007 spdk_put_io_channel(ioch); 3008 poll_threads(); 3009 } 3010 3011 #ifdef SPDK_CONFIG_ISAL_CRYPTO 3012 static void 3013 ut_encrypt_cb(void *cb_arg, int status) 3014 { 3015 int *completed = cb_arg; 3016 3017 CU_ASSERT_EQUAL(status, 0); 3018 3019 *completed = 1; 3020 } 3021 3022 static void 3023 test_sequence_crypto(void) 3024 { 3025 struct spdk_accel_sequence *seq = NULL; 3026 struct spdk_io_channel *ioch; 3027 struct spdk_accel_crypto_key *key; 3028 struct spdk_accel_crypto_key_create_param key_params = { 3029 .cipher = "AES_XTS", 3030 .hex_key = "00112233445566778899aabbccddeeff", 3031 .hex_key2 = "ffeeddccbbaa99887766554433221100", 3032 .key_name = "ut_key", 3033 }; 3034 struct ut_sequence ut_seq; 3035 unsigned char buf[4096], encrypted[4096] = {}, data[4096], tmp[3][4096]; 3036 struct iovec src_iovs[4], dst_iovs[4]; 3037 int rc, completed = 0; 3038 size_t i; 3039 3040 ioch = spdk_accel_get_io_channel(); 3041 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3042 3043 rc = spdk_accel_crypto_key_create(&key_params); 3044 CU_ASSERT_EQUAL(rc, 0); 3045 key = spdk_accel_crypto_key_get(key_params.key_name); 3046 SPDK_CU_ASSERT_FATAL(key != NULL); 3047 3048 for (i = 0; i < sizeof(data); ++i) { 3049 data[i] = (uint8_t)i & 0xff; 3050 } 3051 3052 dst_iovs[0].iov_base = encrypted; 3053 dst_iovs[0].iov_len = sizeof(encrypted); 3054 src_iovs[0].iov_base = data; 3055 src_iovs[0].iov_len = sizeof(data); 3056 rc = spdk_accel_submit_encrypt(ioch, key, &dst_iovs[0], 1, &src_iovs[0], 1, 0, 4096, 0, 3057 ut_encrypt_cb, &completed); 3058 CU_ASSERT_EQUAL(rc, 0); 3059 3060 while (!completed) { 3061 poll_threads(); 3062 } 3063 3064 /* Verify that encryption operation in a sequence produces the same result */ 3065 seq = NULL; 3066 completed = 0; 3067 3068 dst_iovs[0].iov_base = tmp[0]; 3069 dst_iovs[0].iov_len = sizeof(tmp[0]); 3070 src_iovs[0].iov_base = data; 3071 src_iovs[0].iov_len = sizeof(data); 3072 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3073 &src_iovs[0], 1, NULL, NULL, 0, 3074 ut_sequence_step_cb, &completed); 3075 CU_ASSERT_EQUAL(rc, 0); 3076 3077 dst_iovs[1].iov_base = tmp[1]; 3078 dst_iovs[1].iov_len = sizeof(tmp[1]); 3079 src_iovs[1].iov_base = tmp[0]; 3080 src_iovs[1].iov_len = sizeof(tmp[0]); 3081 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3082 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3083 ut_sequence_step_cb, &completed); 3084 CU_ASSERT_EQUAL(rc, 0); 3085 3086 dst_iovs[2].iov_base = buf; 3087 dst_iovs[2].iov_len = sizeof(buf); 3088 src_iovs[2].iov_base = tmp[1]; 3089 src_iovs[2].iov_len = sizeof(tmp[1]); 3090 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3091 &src_iovs[2], 1, NULL, NULL, 0, 3092 ut_sequence_step_cb, &completed); 3093 CU_ASSERT_EQUAL(rc, 0); 3094 3095 ut_seq.complete = false; 3096 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3097 3098 poll_threads(); 3099 3100 CU_ASSERT_EQUAL(completed, 3); 3101 CU_ASSERT(ut_seq.complete); 3102 CU_ASSERT_EQUAL(ut_seq.status, 0); 3103 CU_ASSERT_EQUAL(memcmp(buf, encrypted, sizeof(buf)), 0); 3104 3105 /* Check that decryption produces the original buffer */ 3106 seq = NULL; 3107 completed = 0; 3108 memset(buf, 0, sizeof(buf)); 3109 3110 dst_iovs[0].iov_base = tmp[0]; 3111 dst_iovs[0].iov_len = sizeof(tmp[0]); 3112 src_iovs[0].iov_base = encrypted; 3113 src_iovs[0].iov_len = sizeof(encrypted); 3114 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3115 &src_iovs[0], 1, NULL, NULL, 0, 3116 ut_sequence_step_cb, &completed); 3117 CU_ASSERT_EQUAL(rc, 0); 3118 3119 dst_iovs[1].iov_base = tmp[1]; 3120 dst_iovs[1].iov_len = sizeof(tmp[1]); 3121 src_iovs[1].iov_base = tmp[0]; 3122 src_iovs[1].iov_len = sizeof(tmp[0]); 3123 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3124 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3125 ut_sequence_step_cb, &completed); 3126 CU_ASSERT_EQUAL(rc, 0); 3127 3128 dst_iovs[2].iov_base = buf; 3129 dst_iovs[2].iov_len = sizeof(buf); 3130 src_iovs[2].iov_base = tmp[1]; 3131 src_iovs[2].iov_len = sizeof(tmp[1]); 3132 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3133 &src_iovs[2], 1, NULL, NULL, 0, 3134 ut_sequence_step_cb, &completed); 3135 CU_ASSERT_EQUAL(rc, 0); 3136 3137 ut_seq.complete = false; 3138 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3139 3140 poll_threads(); 3141 3142 CU_ASSERT_EQUAL(completed, 3); 3143 CU_ASSERT(ut_seq.complete); 3144 CU_ASSERT_EQUAL(ut_seq.status, 0); 3145 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3146 3147 /* Check encrypt + decrypt in a single sequence */ 3148 seq = NULL; 3149 completed = 0; 3150 memset(buf, 0, sizeof(buf)); 3151 3152 dst_iovs[0].iov_base = tmp[0]; 3153 dst_iovs[0].iov_len = sizeof(tmp[0]); 3154 src_iovs[0].iov_base = data; 3155 src_iovs[0].iov_len = sizeof(data); 3156 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3157 &src_iovs[0], 1, NULL, NULL, 0, 3158 ut_sequence_step_cb, &completed); 3159 CU_ASSERT_EQUAL(rc, 0); 3160 3161 dst_iovs[1].iov_base = tmp[1]; 3162 dst_iovs[1].iov_len = sizeof(tmp[1]); 3163 src_iovs[1].iov_base = tmp[0]; 3164 src_iovs[1].iov_len = sizeof(tmp[0]); 3165 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3166 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3167 ut_sequence_step_cb, &completed); 3168 CU_ASSERT_EQUAL(rc, 0); 3169 3170 3171 dst_iovs[2].iov_base = tmp[2]; 3172 dst_iovs[2].iov_len = sizeof(tmp[2]); 3173 src_iovs[2].iov_base = tmp[1]; 3174 src_iovs[2].iov_len = sizeof(tmp[1]); 3175 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[2], 1, NULL, NULL, 3176 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3177 ut_sequence_step_cb, &completed); 3178 CU_ASSERT_EQUAL(rc, 0); 3179 3180 dst_iovs[3].iov_base = buf; 3181 dst_iovs[3].iov_len = sizeof(buf); 3182 src_iovs[3].iov_base = tmp[2]; 3183 src_iovs[3].iov_len = sizeof(tmp[2]); 3184 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 3185 &src_iovs[3], 1, NULL, NULL, 0, 3186 ut_sequence_step_cb, &completed); 3187 CU_ASSERT_EQUAL(rc, 0); 3188 3189 ut_seq.complete = false; 3190 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3191 3192 poll_threads(); 3193 3194 CU_ASSERT_EQUAL(completed, 4); 3195 CU_ASSERT(ut_seq.complete); 3196 CU_ASSERT_EQUAL(ut_seq.status, 0); 3197 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3198 3199 rc = spdk_accel_crypto_key_destroy(key); 3200 CU_ASSERT_EQUAL(rc, 0); 3201 spdk_put_io_channel(ioch); 3202 poll_threads(); 3203 } 3204 #endif /* SPDK_CONFIG_ISAL_CRYPTO */ 3205 3206 static int 3207 ut_submit_crypto(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3208 { 3209 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3210 3211 spdk_accel_task_complete(task, 0); 3212 3213 return 0; 3214 } 3215 3216 struct ut_driver_operation { 3217 int complete_status; 3218 int submit_status; 3219 int count; 3220 bool supported; 3221 }; 3222 3223 static struct ut_driver_operation g_drv_operations[SPDK_ACCEL_OPC_LAST]; 3224 static bool g_ut_driver_async_continue; 3225 3226 static void 3227 ut_driver_async_continue(void *arg) 3228 { 3229 struct spdk_accel_sequence *seq = arg; 3230 3231 spdk_accel_sequence_continue(seq); 3232 } 3233 3234 static int 3235 ut_driver_execute_sequence(struct spdk_io_channel *ch, struct spdk_accel_sequence *seq) 3236 { 3237 struct spdk_accel_task *task; 3238 struct ut_driver_operation *drv_ops; 3239 3240 while ((task = spdk_accel_sequence_first_task(seq)) != NULL) { 3241 drv_ops = &g_drv_operations[task->op_code]; 3242 if (!drv_ops->supported) { 3243 break; 3244 } 3245 3246 drv_ops->count++; 3247 if (drv_ops->submit_status != 0) { 3248 return drv_ops->submit_status; 3249 } 3250 3251 if (drv_ops->complete_status != 0) { 3252 spdk_accel_task_complete(task, drv_ops->complete_status); 3253 break; 3254 } 3255 3256 switch (task->op_code) { 3257 case SPDK_ACCEL_OPC_DECOMPRESS: 3258 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3259 break; 3260 case SPDK_ACCEL_OPC_FILL: 3261 spdk_iov_memset(task->d.iovs, task->d.iovcnt, 3262 (int)(task->fill_pattern & 0xff)); 3263 break; 3264 default: 3265 CU_ASSERT(0 && "unexpected opcode"); 3266 break; 3267 } 3268 3269 spdk_accel_task_complete(task, 0); 3270 } 3271 3272 if (g_ut_driver_async_continue) { 3273 spdk_thread_send_msg(spdk_get_thread(), ut_driver_async_continue, seq); 3274 } else { 3275 spdk_accel_sequence_continue(seq); 3276 } 3277 3278 return 0; 3279 } 3280 3281 static struct spdk_io_channel * 3282 ut_driver_get_io_channel(void) 3283 { 3284 return (void *)0xdeadbeef; 3285 } 3286 3287 static struct spdk_accel_driver g_ut_driver = { 3288 .name = "ut", 3289 .execute_sequence = ut_driver_execute_sequence, 3290 .get_io_channel = ut_driver_get_io_channel, 3291 }; 3292 3293 SPDK_ACCEL_DRIVER_REGISTER(ut, &g_ut_driver); 3294 3295 static void 3296 test_sequence_driver(void) 3297 { 3298 struct spdk_accel_sequence *seq = NULL; 3299 struct spdk_io_channel *ioch; 3300 struct spdk_accel_crypto_key key = {}; 3301 struct ut_sequence ut_seq; 3302 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3303 char buf[4096], tmp[3][4096], expected[4096]; 3304 struct iovec src_iovs[3], dst_iovs[3]; 3305 int i, rc, completed = 0; 3306 3307 ioch = spdk_accel_get_io_channel(); 3308 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3309 rc = spdk_accel_set_driver("ut"); 3310 SPDK_CU_ASSERT_FATAL(rc == 0); 3311 3312 /* Override the submit_tasks function */ 3313 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3314 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3315 modules[i] = g_modules_opc[i]; 3316 g_modules_opc[i] = g_module; 3317 } 3318 /* Intercept crypto operations, as they should be executed by an accel module */ 3319 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].submit = ut_submit_crypto; 3320 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].submit = ut_submit_crypto; 3321 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3322 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3323 g_seq_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3324 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3325 3326 g_drv_operations[SPDK_ACCEL_OPC_FILL].supported = true; 3327 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].supported = true; 3328 3329 /* First check a sequence that is fully executed using a driver, with the copy at the end 3330 * being removed */ 3331 seq = NULL; 3332 completed = 0; 3333 memset(buf, 0, sizeof(buf)); 3334 memset(tmp[0], 0, sizeof(tmp[0])); 3335 memset(tmp[1], 0, sizeof(tmp[1])); 3336 memset(&expected[0], 0xa5, 2048); 3337 memset(&expected[2048], 0xbe, 2048); 3338 3339 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], 2048, NULL, NULL, 0xa5, 0, 3340 ut_sequence_step_cb, &completed); 3341 CU_ASSERT_EQUAL(rc, 0); 3342 rc = spdk_accel_append_fill(&seq, ioch, &tmp[0][2048], 2048, NULL, NULL, 0xbe, 0, 3343 ut_sequence_step_cb, &completed); 3344 CU_ASSERT_EQUAL(rc, 0); 3345 3346 dst_iovs[0].iov_base = tmp[1]; 3347 dst_iovs[0].iov_len = sizeof(tmp[1]); 3348 src_iovs[0].iov_base = tmp[0]; 3349 src_iovs[0].iov_len = sizeof(tmp[0]); 3350 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3351 &src_iovs[0], 1, NULL, NULL, 0, 3352 ut_sequence_step_cb, &completed); 3353 CU_ASSERT_EQUAL(rc, 0); 3354 3355 dst_iovs[1].iov_base = buf; 3356 dst_iovs[1].iov_len = sizeof(buf); 3357 src_iovs[1].iov_base = tmp[1]; 3358 src_iovs[1].iov_len = sizeof(tmp[1]); 3359 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3360 &src_iovs[1], 1, NULL, NULL, 0, 3361 ut_sequence_step_cb, &completed); 3362 CU_ASSERT_EQUAL(rc, 0); 3363 3364 ut_seq.complete = false; 3365 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3366 3367 poll_threads(); 3368 3369 CU_ASSERT_EQUAL(completed, 4); 3370 CU_ASSERT(ut_seq.complete); 3371 CU_ASSERT_EQUAL(ut_seq.status, 0); 3372 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3373 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3374 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 3375 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 2); 3376 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3377 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3378 3379 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3380 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3381 3382 /* Check a sequence when the first two operations are executed by a driver, while the rest 3383 * is executed via modules */ 3384 seq = NULL; 3385 completed = 0; 3386 memset(buf, 0, sizeof(buf)); 3387 memset(tmp[0], 0, sizeof(tmp[0])); 3388 memset(tmp[1], 0, sizeof(tmp[1])); 3389 memset(tmp[2], 0, sizeof(tmp[2])); 3390 memset(&expected[0], 0xfe, 4096); 3391 3392 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 0, 3393 ut_sequence_step_cb, &completed); 3394 CU_ASSERT_EQUAL(rc, 0); 3395 3396 dst_iovs[0].iov_base = tmp[1]; 3397 dst_iovs[0].iov_len = sizeof(tmp[1]); 3398 src_iovs[0].iov_base = tmp[0]; 3399 src_iovs[0].iov_len = sizeof(tmp[0]); 3400 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3401 &src_iovs[0], 1, NULL, NULL, 0, 3402 ut_sequence_step_cb, &completed); 3403 CU_ASSERT_EQUAL(rc, 0); 3404 3405 dst_iovs[1].iov_base = tmp[2]; 3406 dst_iovs[1].iov_len = sizeof(tmp[2]); 3407 src_iovs[1].iov_base = tmp[1]; 3408 src_iovs[1].iov_len = sizeof(tmp[1]); 3409 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3410 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3411 ut_sequence_step_cb, &completed); 3412 CU_ASSERT_EQUAL(rc, 0); 3413 3414 dst_iovs[2].iov_base = buf; 3415 dst_iovs[2].iov_len = sizeof(buf); 3416 src_iovs[2].iov_base = tmp[2]; 3417 src_iovs[2].iov_len = sizeof(tmp[2]); 3418 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3419 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3420 ut_sequence_step_cb, &completed); 3421 CU_ASSERT_EQUAL(rc, 0); 3422 3423 ut_seq.complete = false; 3424 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3425 3426 poll_threads(); 3427 3428 CU_ASSERT_EQUAL(completed, 4); 3429 CU_ASSERT(ut_seq.complete); 3430 CU_ASSERT_EQUAL(ut_seq.status, 0); 3431 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3432 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3433 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3434 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3435 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3436 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3437 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3438 3439 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3440 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3441 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3442 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3443 3444 /* Check sequence when the first and last operations are executed through modules, while the 3445 * ones in the middle are executed by the driver */ 3446 seq = NULL; 3447 completed = 0; 3448 memset(buf, 0, sizeof(buf)); 3449 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3450 memset(tmp[1], 0, sizeof(tmp[1])); 3451 memset(tmp[2], 0, sizeof(tmp[2])); 3452 memset(&expected[0], 0xfe, 2048); 3453 memset(&expected[2048], 0xa5, 2048); 3454 3455 dst_iovs[0].iov_base = tmp[1]; 3456 dst_iovs[0].iov_len = sizeof(tmp[1]); 3457 src_iovs[0].iov_base = tmp[0]; 3458 src_iovs[0].iov_len = sizeof(tmp[0]); 3459 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3460 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3461 ut_sequence_step_cb, &completed); 3462 CU_ASSERT_EQUAL(rc, 0); 3463 3464 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xfe, 0, 3465 ut_sequence_step_cb, &completed); 3466 CU_ASSERT_EQUAL(rc, 0); 3467 3468 dst_iovs[1].iov_base = tmp[2]; 3469 dst_iovs[1].iov_len = sizeof(tmp[2]); 3470 src_iovs[1].iov_base = tmp[1]; 3471 src_iovs[1].iov_len = sizeof(tmp[1]); 3472 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3473 &src_iovs[1], 1, NULL, NULL, 0, 3474 ut_sequence_step_cb, &completed); 3475 CU_ASSERT_EQUAL(rc, 0); 3476 3477 dst_iovs[2].iov_base = buf; 3478 dst_iovs[2].iov_len = sizeof(buf); 3479 src_iovs[2].iov_base = tmp[2]; 3480 src_iovs[2].iov_len = sizeof(tmp[2]); 3481 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3482 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3483 ut_sequence_step_cb, &completed); 3484 CU_ASSERT_EQUAL(rc, 0); 3485 3486 ut_seq.complete = false; 3487 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3488 3489 poll_threads(); 3490 3491 CU_ASSERT_EQUAL(completed, 4); 3492 CU_ASSERT(ut_seq.complete); 3493 CU_ASSERT_EQUAL(ut_seq.status, 0); 3494 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3495 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3496 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3497 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3498 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3499 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3500 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3501 3502 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3503 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3504 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3505 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3506 3507 /* Check a sequence with operations executed by: module, driver, module, driver */ 3508 seq = NULL; 3509 completed = 0; 3510 memset(buf, 0, sizeof(buf)); 3511 memset(tmp[0], 0x5a, sizeof(tmp[0])); 3512 memset(tmp[1], 0, sizeof(tmp[1])); 3513 memset(tmp[2], 0, sizeof(tmp[2])); 3514 memset(&expected[0], 0xef, 2048); 3515 memset(&expected[2048], 0x5a, 2048); 3516 3517 dst_iovs[0].iov_base = tmp[1]; 3518 dst_iovs[0].iov_len = sizeof(tmp[1]); 3519 src_iovs[0].iov_base = tmp[0]; 3520 src_iovs[0].iov_len = sizeof(tmp[0]); 3521 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3522 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3523 ut_sequence_step_cb, &completed); 3524 CU_ASSERT_EQUAL(rc, 0); 3525 3526 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3527 ut_sequence_step_cb, &completed); 3528 CU_ASSERT_EQUAL(rc, 0); 3529 3530 dst_iovs[1].iov_base = tmp[2]; 3531 dst_iovs[1].iov_len = sizeof(tmp[2]); 3532 src_iovs[1].iov_base = tmp[1]; 3533 src_iovs[1].iov_len = sizeof(tmp[1]); 3534 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3535 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3536 ut_sequence_step_cb, &completed); 3537 CU_ASSERT_EQUAL(rc, 0); 3538 3539 dst_iovs[2].iov_base = buf; 3540 dst_iovs[2].iov_len = sizeof(buf); 3541 src_iovs[2].iov_base = tmp[2]; 3542 src_iovs[2].iov_len = sizeof(tmp[2]); 3543 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3544 &src_iovs[2], 1, NULL, NULL, 0, 3545 ut_sequence_step_cb, &completed); 3546 CU_ASSERT_EQUAL(rc, 0); 3547 3548 ut_seq.complete = false; 3549 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3550 3551 poll_threads(); 3552 3553 CU_ASSERT_EQUAL(completed, 4); 3554 CU_ASSERT(ut_seq.complete); 3555 CU_ASSERT_EQUAL(ut_seq.status, 0); 3556 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3557 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3558 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3559 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3560 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3561 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3562 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3563 3564 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3565 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3566 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3567 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3568 3569 /* Check that an error returned from driver's execute_sequence() will fail the whole 3570 * sequence and any subsequent operations won't be processed */ 3571 seq = NULL; 3572 completed = 0; 3573 memset(buf, 0, sizeof(buf)); 3574 memset(expected, 0, sizeof(expected)); 3575 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3576 g_drv_operations[SPDK_ACCEL_OPC_FILL].submit_status = -EPERM; 3577 3578 dst_iovs[0].iov_base = tmp[1]; 3579 dst_iovs[0].iov_len = sizeof(tmp[1]); 3580 src_iovs[0].iov_base = tmp[0]; 3581 src_iovs[0].iov_len = sizeof(tmp[0]); 3582 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3583 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3584 ut_sequence_step_cb, &completed); 3585 CU_ASSERT_EQUAL(rc, 0); 3586 3587 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3588 ut_sequence_step_cb, &completed); 3589 CU_ASSERT_EQUAL(rc, 0); 3590 3591 dst_iovs[1].iov_base = buf; 3592 dst_iovs[1].iov_len = sizeof(buf); 3593 src_iovs[1].iov_base = tmp[1]; 3594 src_iovs[1].iov_len = sizeof(tmp[1]); 3595 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3596 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3597 ut_sequence_step_cb, &completed); 3598 CU_ASSERT_EQUAL(rc, 0); 3599 3600 ut_seq.complete = false; 3601 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3602 3603 poll_threads(); 3604 3605 CU_ASSERT_EQUAL(completed, 3); 3606 CU_ASSERT(ut_seq.complete); 3607 CU_ASSERT_EQUAL(ut_seq.status, -EPERM); 3608 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3609 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3610 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 0); 3611 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3612 CU_ASSERT_EQUAL(memcmp(buf, expected, 4096), 0); 3613 3614 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3615 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3616 g_drv_operations[SPDK_ACCEL_OPC_FILL].submit_status = 0; 3617 3618 /* Check that a failed task completed by a driver will cause the whole sequence to be failed 3619 * and any subsequent operations won't be processed */ 3620 seq = NULL; 3621 completed = 0; 3622 memset(buf, 0, sizeof(buf)); 3623 memset(expected, 0, sizeof(expected)); 3624 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3625 g_drv_operations[SPDK_ACCEL_OPC_FILL].complete_status = -ENOENT; 3626 3627 dst_iovs[0].iov_base = tmp[1]; 3628 dst_iovs[0].iov_len = sizeof(tmp[1]); 3629 src_iovs[0].iov_base = tmp[0]; 3630 src_iovs[0].iov_len = sizeof(tmp[0]); 3631 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3632 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3633 ut_sequence_step_cb, &completed); 3634 CU_ASSERT_EQUAL(rc, 0); 3635 3636 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3637 ut_sequence_step_cb, &completed); 3638 CU_ASSERT_EQUAL(rc, 0); 3639 3640 dst_iovs[1].iov_base = buf; 3641 dst_iovs[1].iov_len = sizeof(buf); 3642 src_iovs[1].iov_base = tmp[1]; 3643 src_iovs[1].iov_len = sizeof(tmp[1]); 3644 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3645 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3646 ut_sequence_step_cb, &completed); 3647 CU_ASSERT_EQUAL(rc, 0); 3648 3649 ut_seq.complete = false; 3650 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3651 3652 poll_threads(); 3653 3654 CU_ASSERT_EQUAL(completed, 3); 3655 CU_ASSERT(ut_seq.complete); 3656 CU_ASSERT_EQUAL(ut_seq.status, -ENOENT); 3657 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3658 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3659 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 0); 3660 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3661 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3662 3663 g_drv_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 3664 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3665 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3666 3667 /* Check asynchronous spdk_accel_sequence_continue() */ 3668 g_ut_driver_async_continue = true; 3669 seq = NULL; 3670 completed = 0; 3671 memset(buf, 0, sizeof(buf)); 3672 memset(tmp[0], 0, sizeof(tmp[0])); 3673 memset(&expected[0], 0xfe, 4096); 3674 3675 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 0, 3676 ut_sequence_step_cb, &completed); 3677 CU_ASSERT_EQUAL(rc, 0); 3678 3679 dst_iovs[0].iov_base = buf; 3680 dst_iovs[0].iov_len = sizeof(buf); 3681 src_iovs[0].iov_base = tmp[0]; 3682 src_iovs[0].iov_len = sizeof(tmp[0]); 3683 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3684 &src_iovs[0], 1, NULL, NULL, 0, 3685 ut_sequence_step_cb, &completed); 3686 CU_ASSERT_EQUAL(rc, 0); 3687 3688 ut_seq.complete = false; 3689 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3690 3691 poll_threads(); 3692 3693 CU_ASSERT_EQUAL(completed, 2); 3694 CU_ASSERT(ut_seq.complete); 3695 CU_ASSERT_EQUAL(ut_seq.status, 0); 3696 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3697 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3698 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3699 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3700 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3701 3702 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3703 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3704 3705 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3706 g_modules_opc[i] = modules[i]; 3707 } 3708 3709 /* Clear the driver so that other tests won't use it */ 3710 g_accel_driver = NULL; 3711 memset(&g_drv_operations, 0, sizeof(g_drv_operations)); 3712 3713 ut_clear_operations(); 3714 spdk_put_io_channel(ioch); 3715 poll_threads(); 3716 } 3717 3718 struct ut_saved_iovs { 3719 struct iovec src; 3720 struct iovec dst; 3721 }; 3722 3723 static struct ut_saved_iovs g_seq_saved_iovs[SPDK_ACCEL_OPC_LAST]; 3724 3725 static int 3726 ut_submit_save_iovs(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3727 { 3728 SPDK_CU_ASSERT_FATAL(task->s.iovcnt == 1); 3729 SPDK_CU_ASSERT_FATAL(task->d.iovcnt == 1); 3730 3731 g_seq_saved_iovs[task->op_code].src = task->s.iovs[0]; 3732 g_seq_saved_iovs[task->op_code].dst = task->d.iovs[0]; 3733 3734 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3735 3736 spdk_accel_task_complete(task, 0); 3737 3738 return 0; 3739 } 3740 3741 static void 3742 test_sequence_same_iovs(void) 3743 { 3744 struct spdk_accel_sequence *seq = NULL; 3745 struct spdk_io_channel *ioch; 3746 struct spdk_accel_crypto_key key = {}; 3747 struct ut_sequence ut_seq; 3748 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3749 char buf[4096], tmp[4096], expected[4096]; 3750 struct iovec iovs[3], expected_siov, expected_diov; 3751 struct spdk_memory_domain *domain; 3752 void *accel_buf, *domain_ctx; 3753 int i, rc, completed = 0; 3754 3755 ioch = spdk_accel_get_io_channel(); 3756 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3757 3758 /* Override the submit_tasks function */ 3759 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3760 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3761 modules[i] = g_modules_opc[i]; 3762 g_modules_opc[i] = g_module; 3763 } 3764 /* Intercept crypto operations, as they should be executed by an accel module */ 3765 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].submit = ut_submit_save_iovs; 3766 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].submit = ut_submit_save_iovs; 3767 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3768 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3769 3770 /* Check that it's possible to use the same iovec ptr for different operations */ 3771 seq = NULL; 3772 completed = 0; 3773 memset(buf, 0, sizeof(buf)); 3774 memset(expected, 0xa5, sizeof(expected)); 3775 3776 iovs[0].iov_base = expected; 3777 iovs[0].iov_len = sizeof(expected); 3778 iovs[1].iov_base = tmp; 3779 iovs[1].iov_len = sizeof(tmp); 3780 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, NULL, NULL, 3781 &iovs[0], 1, NULL, NULL, 0, 4096, 0, 3782 ut_sequence_step_cb, &completed); 3783 CU_ASSERT_EQUAL(rc, 0); 3784 /* Reuse iov[1] as src */ 3785 iovs[2].iov_base = buf; 3786 iovs[2].iov_len = sizeof(buf); 3787 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3788 &iovs[1], 1, NULL, NULL, 0, 4096, 0, 3789 ut_sequence_step_cb, &completed); 3790 CU_ASSERT_EQUAL(rc, 0); 3791 3792 ut_seq.complete = false; 3793 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3794 3795 poll_threads(); 3796 3797 CU_ASSERT_EQUAL(completed, 2); 3798 CU_ASSERT(ut_seq.complete); 3799 CU_ASSERT_EQUAL(ut_seq.status, 0); 3800 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3801 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3802 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3803 expected_siov.iov_base = expected; 3804 expected_siov.iov_len = sizeof(expected); 3805 expected_diov.iov_base = tmp; 3806 expected_diov.iov_len = sizeof(tmp); 3807 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].src, 3808 &expected_siov, sizeof(expected_siov)), 0); 3809 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].dst, 3810 &expected_diov, sizeof(expected_diov)), 0); 3811 expected_siov.iov_base = tmp; 3812 expected_siov.iov_len = sizeof(tmp); 3813 expected_diov.iov_base = buf; 3814 expected_diov.iov_len = sizeof(buf); 3815 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].src, 3816 &expected_siov, sizeof(expected_siov)), 0); 3817 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].dst, 3818 &expected_diov, sizeof(expected_diov)), 0); 3819 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3820 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3821 3822 /* Check the same with an accel buffer */ 3823 seq = NULL; 3824 completed = 0; 3825 memset(buf, 0, sizeof(buf)); 3826 memset(expected, 0x5a, sizeof(expected)); 3827 3828 rc = spdk_accel_get_buf(ioch, sizeof(buf), &accel_buf, &domain, &domain_ctx); 3829 CU_ASSERT_EQUAL(rc, 0); 3830 3831 iovs[0].iov_base = expected; 3832 iovs[0].iov_len = sizeof(expected); 3833 iovs[1].iov_base = accel_buf; 3834 iovs[1].iov_len = sizeof(buf); 3835 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, domain, domain_ctx, 3836 &iovs[0], 1, NULL, NULL, 0, 4096, 0, 3837 ut_sequence_step_cb, &completed); 3838 CU_ASSERT_EQUAL(rc, 0); 3839 /* Reuse iov[1] as src */ 3840 iovs[2].iov_base = buf; 3841 iovs[2].iov_len = sizeof(buf); 3842 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3843 &iovs[1], 1, domain, domain_ctx, 0, 4096, 0, 3844 ut_sequence_step_cb, &completed); 3845 CU_ASSERT_EQUAL(rc, 0); 3846 3847 ut_seq.complete = false; 3848 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3849 3850 poll_threads(); 3851 3852 CU_ASSERT_EQUAL(completed, 2); 3853 CU_ASSERT(ut_seq.complete); 3854 CU_ASSERT_EQUAL(ut_seq.status, 0); 3855 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3856 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3857 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3858 expected_siov.iov_base = expected; 3859 expected_siov.iov_len = sizeof(expected); 3860 expected_diov.iov_base = buf; 3861 expected_diov.iov_len = sizeof(buf); 3862 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].src, 3863 &expected_siov, sizeof(expected_siov)), 0); 3864 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].dst, 3865 &expected_diov, sizeof(expected_diov)), 0); 3866 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].dst, 3867 &g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].src, 3868 sizeof(struct iovec)), 0); 3869 spdk_accel_put_buf(ioch, accel_buf, domain, domain_ctx); 3870 3871 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3872 g_modules_opc[i] = modules[i]; 3873 } 3874 3875 ut_clear_operations(); 3876 spdk_put_io_channel(ioch); 3877 poll_threads(); 3878 } 3879 3880 static void 3881 test_sequence_crc32(void) 3882 { 3883 struct spdk_accel_sequence *seq = NULL; 3884 struct spdk_io_channel *ioch; 3885 struct ut_sequence ut_seq; 3886 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3887 char buf[4096], tmp[3][4096]; 3888 struct iovec src_iovs[4], dst_iovs[4]; 3889 uint32_t crc, crc2; 3890 int i, rc, completed; 3891 3892 ioch = spdk_accel_get_io_channel(); 3893 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3894 3895 /* Override the submit_tasks function */ 3896 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3897 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3898 g_seq_operations[i].submit = sw_accel_submit_tasks; 3899 modules[i] = g_modules_opc[i]; 3900 g_modules_opc[i] = g_module; 3901 } 3902 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 3903 3904 /* First check the simplest case - single crc32c operation */ 3905 seq = NULL; 3906 completed = 0; 3907 crc = 0; 3908 memset(buf, 0xa5, sizeof(buf)); 3909 3910 src_iovs[0].iov_base = buf; 3911 src_iovs[0].iov_len = sizeof(buf); 3912 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3913 ut_sequence_step_cb, &completed); 3914 CU_ASSERT_EQUAL(rc, 0); 3915 3916 ut_seq.complete = false; 3917 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3918 3919 poll_threads(); 3920 CU_ASSERT_EQUAL(completed, 1); 3921 CU_ASSERT(ut_seq.complete); 3922 CU_ASSERT_EQUAL(ut_seq.status, 0); 3923 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3924 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3925 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3926 3927 /* Now check copy+crc - this should remove the copy operation and change the buffer for the 3928 * crc operation */ 3929 seq = NULL; 3930 completed = 0; 3931 crc = 0; 3932 memset(buf, 0x5a, sizeof(buf)); 3933 memset(&tmp[0], 0, sizeof(tmp[0])); 3934 3935 dst_iovs[0].iov_base = tmp[0]; 3936 dst_iovs[0].iov_len = sizeof(tmp[0]); 3937 src_iovs[0].iov_base = buf; 3938 src_iovs[0].iov_len = sizeof(buf); 3939 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3940 &src_iovs[0], 1, NULL, NULL, 0, 3941 ut_sequence_step_cb, &completed); 3942 CU_ASSERT_EQUAL(rc, 0); 3943 3944 src_iovs[1].iov_base = tmp[0]; 3945 src_iovs[1].iov_len = sizeof(tmp[0]); 3946 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 3947 ut_sequence_step_cb, &completed); 3948 CU_ASSERT_EQUAL(rc, 0); 3949 3950 ut_seq.complete = false; 3951 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3952 3953 poll_threads(); 3954 CU_ASSERT_EQUAL(completed, 2); 3955 CU_ASSERT(ut_seq.complete); 3956 CU_ASSERT_EQUAL(ut_seq.status, 0); 3957 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3958 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 3959 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3960 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3961 3962 /* Check crc+copy - this time the copy cannot be removed, because there's no operation 3963 * before crc to change the buffer */ 3964 seq = NULL; 3965 completed = 0; 3966 crc = 0; 3967 memset(buf, 0, sizeof(buf)); 3968 memset(&tmp[0], 0xa5, sizeof(tmp[0])); 3969 3970 src_iovs[0].iov_base = tmp[0]; 3971 src_iovs[0].iov_len = sizeof(tmp[0]); 3972 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3973 ut_sequence_step_cb, &completed); 3974 CU_ASSERT_EQUAL(rc, 0); 3975 3976 dst_iovs[1].iov_base = buf; 3977 dst_iovs[1].iov_len = sizeof(buf); 3978 src_iovs[1].iov_base = tmp[0]; 3979 src_iovs[1].iov_len = sizeof(tmp[0]); 3980 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3981 &src_iovs[1], 1, NULL, NULL, 0, 3982 ut_sequence_step_cb, &completed); 3983 CU_ASSERT_EQUAL(rc, 0); 3984 3985 ut_seq.complete = false; 3986 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3987 3988 poll_threads(); 3989 CU_ASSERT_EQUAL(completed, 2); 3990 CU_ASSERT(ut_seq.complete); 3991 CU_ASSERT_EQUAL(ut_seq.status, 0); 3992 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3993 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 3994 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 3995 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 3996 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3997 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 3998 3999 /* Check a sequence with an operation at the beginning that can have its buffer changed, two 4000 * crc operations and a copy at the end. The copy should be removed and the dst buffer of 4001 * the first operation and the src buffer of the crc operations should be changed. 4002 */ 4003 seq = NULL; 4004 completed = 0; 4005 crc = crc2 = 0; 4006 memset(buf, 0, sizeof(buf)); 4007 memset(&tmp[0], 0x5a, sizeof(tmp[0])); 4008 dst_iovs[0].iov_base = tmp[1]; 4009 dst_iovs[0].iov_len = sizeof(tmp[1]); 4010 src_iovs[0].iov_base = tmp[0]; 4011 src_iovs[0].iov_len = sizeof(tmp[0]); 4012 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 4013 &src_iovs[0], 1, NULL, NULL, 0, 4014 ut_sequence_step_cb, &completed); 4015 CU_ASSERT_EQUAL(rc, 0); 4016 4017 src_iovs[1].iov_base = tmp[1]; 4018 src_iovs[1].iov_len = sizeof(tmp[1]); 4019 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 4020 ut_sequence_step_cb, &completed); 4021 CU_ASSERT_EQUAL(rc, 0); 4022 4023 src_iovs[2].iov_base = tmp[1]; 4024 src_iovs[2].iov_len = sizeof(tmp[1]); 4025 rc = spdk_accel_append_crc32c(&seq, ioch, &crc2, &src_iovs[2], 1, NULL, NULL, 0, 4026 ut_sequence_step_cb, &completed); 4027 CU_ASSERT_EQUAL(rc, 0); 4028 4029 dst_iovs[3].iov_base = buf; 4030 dst_iovs[3].iov_len = sizeof(buf); 4031 src_iovs[3].iov_base = tmp[1]; 4032 src_iovs[3].iov_len = sizeof(tmp[1]); 4033 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 4034 &src_iovs[3], 1, NULL, NULL, 0, 4035 ut_sequence_step_cb, &completed); 4036 CU_ASSERT_EQUAL(rc, 0); 4037 4038 ut_seq.complete = false; 4039 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4040 4041 poll_threads(); 4042 CU_ASSERT_EQUAL(completed, 4); 4043 CU_ASSERT(ut_seq.complete); 4044 CU_ASSERT_EQUAL(ut_seq.status, 0); 4045 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 4046 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 2); 4047 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 4048 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 4049 CU_ASSERT_EQUAL(crc, crc2); 4050 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 4051 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 4052 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 4053 4054 /* Check that a copy won't be removed if the buffers don't match */ 4055 seq = NULL; 4056 completed = 0; 4057 crc = 0; 4058 memset(buf, 0, sizeof(buf)); 4059 memset(&tmp[0], 0xa5, 2048); 4060 memset(&tmp[1], 0xfe, sizeof(tmp[1])); 4061 memset(&tmp[2], 0xfe, sizeof(tmp[1])); 4062 dst_iovs[0].iov_base = &tmp[1][2048]; 4063 dst_iovs[0].iov_len = 2048; 4064 src_iovs[0].iov_base = tmp[0]; 4065 src_iovs[0].iov_len = 2048; 4066 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 4067 &src_iovs[0], 1, NULL, NULL, 0, 4068 ut_sequence_step_cb, &completed); 4069 CU_ASSERT_EQUAL(rc, 0); 4070 4071 src_iovs[1].iov_base = &tmp[1][2048]; 4072 src_iovs[1].iov_len = 2048; 4073 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 4074 ut_sequence_step_cb, &completed); 4075 CU_ASSERT_EQUAL(rc, 0); 4076 4077 dst_iovs[2].iov_base = buf; 4078 dst_iovs[2].iov_len = sizeof(buf); 4079 src_iovs[2].iov_base = tmp[1]; 4080 src_iovs[2].iov_len = sizeof(tmp[1]); 4081 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 4082 &src_iovs[2], 1, NULL, NULL, 0, 4083 ut_sequence_step_cb, &completed); 4084 CU_ASSERT_EQUAL(rc, 0); 4085 4086 ut_seq.complete = false; 4087 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4088 4089 poll_threads(); 4090 CU_ASSERT_EQUAL(completed, 3); 4091 CU_ASSERT(ut_seq.complete); 4092 CU_ASSERT_EQUAL(ut_seq.status, 0); 4093 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 4094 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 4095 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 4096 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], 2048, ~0u)); 4097 CU_ASSERT_EQUAL(memcmp(buf, tmp[2], 2048), 0); 4098 CU_ASSERT_EQUAL(memcmp(&buf[2048], tmp[0], 2048), 0); 4099 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 4100 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 4101 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 4102 4103 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 4104 g_modules_opc[i] = modules[i]; 4105 } 4106 4107 ut_clear_operations(); 4108 spdk_put_io_channel(ioch); 4109 poll_threads(); 4110 } 4111 4112 static int 4113 test_sequence_setup(void) 4114 { 4115 int rc; 4116 4117 allocate_cores(1); 4118 allocate_threads(1); 4119 set_thread(0); 4120 4121 rc = spdk_iobuf_initialize(); 4122 if (rc != 0) { 4123 CU_ASSERT(false); 4124 return -1; 4125 } 4126 4127 rc = spdk_accel_initialize(); 4128 if (rc != 0) { 4129 CU_ASSERT(false); 4130 return -1; 4131 } 4132 4133 return 0; 4134 } 4135 4136 static void 4137 finish_cb(void *cb_arg) 4138 { 4139 bool *done = cb_arg; 4140 4141 *done = true; 4142 } 4143 4144 static int 4145 test_sequence_cleanup(void) 4146 { 4147 bool done = false; 4148 4149 spdk_accel_finish(finish_cb, &done); 4150 4151 while (!done) { 4152 poll_threads(); 4153 } 4154 4155 done = false; 4156 spdk_iobuf_finish(finish_cb, &done); 4157 while (!done) { 4158 poll_threads(); 4159 } 4160 4161 free_threads(); 4162 free_cores(); 4163 4164 return 0; 4165 } 4166 4167 int 4168 main(int argc, char **argv) 4169 { 4170 CU_pSuite suite = NULL, seq_suite; 4171 unsigned int num_failures; 4172 4173 CU_initialize_registry(); 4174 4175 /* Sequence tests require accel to be initialized normally, so run them before the other 4176 * tests which register accel modules which aren't fully implemented, causing accel 4177 * initialization to fail. 4178 */ 4179 seq_suite = CU_add_suite("accel_sequence", test_sequence_setup, test_sequence_cleanup); 4180 CU_ADD_TEST(seq_suite, test_sequence_fill_copy); 4181 CU_ADD_TEST(seq_suite, test_sequence_abort); 4182 CU_ADD_TEST(seq_suite, test_sequence_append_error); 4183 CU_ADD_TEST(seq_suite, test_sequence_completion_error); 4184 #ifdef SPDK_CONFIG_ISAL /* accel_sw requires isa-l for compression */ 4185 CU_ADD_TEST(seq_suite, test_sequence_decompress); 4186 CU_ADD_TEST(seq_suite, test_sequence_reverse); 4187 #endif 4188 CU_ADD_TEST(seq_suite, test_sequence_copy_elision); 4189 CU_ADD_TEST(seq_suite, test_sequence_accel_buffers); 4190 CU_ADD_TEST(seq_suite, test_sequence_memory_domain); 4191 CU_ADD_TEST(seq_suite, test_sequence_module_memory_domain); 4192 #ifdef SPDK_CONFIG_ISAL_CRYPTO /* accel_sw requires isa-l-crypto for crypto operations */ 4193 CU_ADD_TEST(seq_suite, test_sequence_crypto); 4194 #endif 4195 CU_ADD_TEST(seq_suite, test_sequence_driver); 4196 CU_ADD_TEST(seq_suite, test_sequence_same_iovs); 4197 CU_ADD_TEST(seq_suite, test_sequence_crc32); 4198 4199 suite = CU_add_suite("accel", test_setup, test_cleanup); 4200 CU_ADD_TEST(suite, test_spdk_accel_task_complete); 4201 CU_ADD_TEST(suite, test_get_task); 4202 CU_ADD_TEST(suite, test_spdk_accel_submit_copy); 4203 CU_ADD_TEST(suite, test_spdk_accel_submit_dualcast); 4204 CU_ADD_TEST(suite, test_spdk_accel_submit_compare); 4205 CU_ADD_TEST(suite, test_spdk_accel_submit_fill); 4206 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c); 4207 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32cv); 4208 CU_ADD_TEST(suite, test_spdk_accel_submit_copy_crc32c); 4209 CU_ADD_TEST(suite, test_spdk_accel_submit_xor); 4210 CU_ADD_TEST(suite, test_spdk_accel_module_find_by_name); 4211 CU_ADD_TEST(suite, test_spdk_accel_module_register); 4212 4213 num_failures = spdk_ut_run_tests(argc, argv, NULL); 4214 CU_cleanup_registry(); 4215 4216 return num_failures; 4217 } 4218