1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2021 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk_internal/cunit.h" 8 #include "spdk_internal/mock.h" 9 #include "spdk/accel_module.h" 10 #include "thread/thread_internal.h" 11 #include "common/lib/ut_multithread.c" 12 #include "common/lib/test_iobuf.c" 13 #include "accel/accel.c" 14 #include "accel/accel_sw.c" 15 #include "unit/lib/json_mock.c" 16 17 DEFINE_STUB_V(spdk_memory_domain_destroy, (struct spdk_memory_domain *domain)); 18 DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, 19 (struct spdk_memory_domain *domain), "UT_DMA"); 20 DEFINE_STUB(spdk_memory_domain_get_system_domain, struct spdk_memory_domain *, (void), 21 (void *)0xabc); 22 DEFINE_STUB_V(spdk_memory_domain_set_invalidate, (struct spdk_memory_domain *domain, 23 spdk_memory_domain_invalidate_data_cb invalidate_cb)); 24 DEFINE_STUB_V(spdk_memory_domain_set_translation, (struct spdk_memory_domain *domain, 25 spdk_memory_domain_translate_memory_cb translate_cb)); 26 27 int 28 spdk_memory_domain_create(struct spdk_memory_domain **domain, enum spdk_dma_device_type type, 29 struct spdk_memory_domain_ctx *ctx, const char *id) 30 { 31 *domain = (void *)0xdeadbeef; 32 33 return 0; 34 } 35 36 struct ut_domain_ctx { 37 struct iovec iov; 38 struct iovec expected; 39 int pull_submit_status; 40 int push_submit_status; 41 int pull_complete_status; 42 int push_complete_status; 43 }; 44 45 static struct spdk_memory_domain *g_ut_domain = (void *)0xa55e1; 46 47 int 48 spdk_memory_domain_pull_data(struct spdk_memory_domain *sd, void *sctx, struct iovec *siov, 49 uint32_t siovcnt, struct iovec *diov, uint32_t diovcnt, 50 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 51 { 52 struct ut_domain_ctx *ctx = sctx; 53 54 CU_ASSERT_EQUAL(sd, g_ut_domain); 55 CU_ASSERT_EQUAL(siovcnt, 1); 56 CU_ASSERT_EQUAL(memcmp(siov, &ctx->expected, sizeof(*siov)), 0); 57 58 if (ctx->pull_submit_status != 0) { 59 return ctx->pull_submit_status; 60 } 61 62 if (ctx->pull_complete_status != 0) { 63 cpl_cb(cpl_arg, ctx->pull_complete_status); 64 return 0; 65 } 66 67 spdk_iovcpy(&ctx->iov, 1, diov, diovcnt); 68 cpl_cb(cpl_arg, 0); 69 70 return 0; 71 } 72 73 int 74 spdk_memory_domain_push_data(struct spdk_memory_domain *dd, void *dctx, struct iovec *diov, 75 uint32_t diovcnt, struct iovec *siov, uint32_t siovcnt, 76 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 77 { 78 struct ut_domain_ctx *ctx = dctx; 79 80 CU_ASSERT_EQUAL(dd, g_ut_domain); 81 CU_ASSERT_EQUAL(diovcnt, 1); 82 CU_ASSERT_EQUAL(memcmp(diov, &ctx->expected, sizeof(*diov)), 0); 83 84 if (ctx->push_submit_status != 0) { 85 return ctx->push_submit_status; 86 } 87 88 if (ctx->push_complete_status != 0) { 89 cpl_cb(cpl_arg, ctx->push_complete_status); 90 return 0; 91 } 92 93 spdk_iovcpy(siov, siovcnt, &ctx->iov, 1); 94 cpl_cb(cpl_arg, 0); 95 96 return 0; 97 } 98 99 /* global vars and setup/cleanup functions used for all test functions */ 100 struct spdk_accel_module_if g_module_if = {}; 101 struct accel_module g_module = { .module = &g_module_if }; 102 struct spdk_io_channel *g_ch = NULL; 103 struct accel_io_channel *g_accel_ch = NULL; 104 struct sw_accel_io_channel *g_sw_ch = NULL; 105 struct spdk_io_channel *g_module_ch = NULL; 106 107 static uint64_t g_opc_mask = 0; 108 109 static uint64_t 110 _accel_op_to_bit(enum spdk_accel_opcode opc) 111 { 112 return (1 << opc); 113 } 114 115 static bool 116 _supports_opcode(enum spdk_accel_opcode opc) 117 { 118 if (_accel_op_to_bit(opc) & g_opc_mask) { 119 return true; 120 } 121 return false; 122 } 123 124 static bool 125 _supports_algo(enum spdk_accel_comp_algo algo) 126 { 127 return true; 128 } 129 130 static int 131 _get_compress_level_range(enum spdk_accel_comp_algo algo, uint32_t *min_level, uint32_t *max_level) 132 { 133 *min_level = 0; 134 *max_level = UINT32_MAX; 135 136 return 0; 137 } 138 139 static int 140 test_setup(void) 141 { 142 int i; 143 144 g_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct accel_io_channel)); 145 if (g_ch == NULL) { 146 /* for some reason the assert fatal macro doesn't work in the setup function. */ 147 CU_ASSERT(false); 148 return -1; 149 } 150 g_accel_ch = (struct accel_io_channel *)((char *)g_ch + sizeof(struct spdk_io_channel)); 151 g_module_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct sw_accel_io_channel)); 152 if (g_module_ch == NULL) { 153 CU_ASSERT(false); 154 return -1; 155 } 156 157 g_module_if.submit_tasks = sw_accel_submit_tasks; 158 g_module_if.name = "software"; 159 g_module_if.compress_supports_algo = _supports_algo; 160 g_module_if.get_compress_level_range = _get_compress_level_range; 161 for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) { 162 g_accel_ch->module_ch[i] = g_module_ch; 163 g_modules_opc[i] = g_module; 164 } 165 g_sw_ch = (struct sw_accel_io_channel *)((char *)g_module_ch + sizeof( 166 struct spdk_io_channel)); 167 /* Prevent lazy initialization of poller. */ 168 g_sw_ch->completion_poller = (void *)0xdeadbeef; 169 STAILQ_INIT(&g_sw_ch->tasks_to_complete); 170 g_module_if.supports_opcode = _supports_opcode; 171 return 0; 172 } 173 174 static int 175 test_cleanup(void) 176 { 177 free(g_ch); 178 free(g_module_ch); 179 180 return 0; 181 } 182 183 #define DUMMY_ARG 0xDEADBEEF 184 static bool g_dummy_cb_called = false; 185 static void 186 dummy_cb_fn(void *cb_arg, int status) 187 { 188 CU_ASSERT(*(uint32_t *)cb_arg == DUMMY_ARG); 189 CU_ASSERT(status == 0); 190 g_dummy_cb_called = true; 191 } 192 193 static void 194 test_spdk_accel_task_complete(void) 195 { 196 struct spdk_accel_task accel_task = {}; 197 struct spdk_accel_task *expected_accel_task = NULL; 198 uint32_t cb_arg = DUMMY_ARG; 199 int status = 0; 200 201 accel_task.accel_ch = g_accel_ch; 202 accel_task.cb_fn = dummy_cb_fn; 203 accel_task.cb_arg = &cb_arg; 204 STAILQ_INIT(&g_accel_ch->task_pool); 205 206 /* Confirm cb is called and task added to list. */ 207 spdk_accel_task_complete(&accel_task, status); 208 CU_ASSERT(g_dummy_cb_called == true); 209 expected_accel_task = STAILQ_FIRST(&g_accel_ch->task_pool); 210 STAILQ_REMOVE_HEAD(&g_accel_ch->task_pool, link); 211 CU_ASSERT(expected_accel_task == &accel_task); 212 } 213 214 static void 215 test_get_task(void) 216 { 217 struct spdk_accel_task *task; 218 struct spdk_accel_task _task; 219 void *cb_arg = NULL; 220 221 STAILQ_INIT(&g_accel_ch->task_pool); 222 223 /* no tasks left, return NULL. */ 224 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 225 CU_ASSERT(task == NULL); 226 227 _task.cb_fn = dummy_cb_fn; 228 _task.cb_arg = cb_arg; 229 _task.accel_ch = g_accel_ch; 230 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &_task, link); 231 232 /* Get a valid task. */ 233 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 234 CU_ASSERT(task == &_task); 235 CU_ASSERT(_task.cb_fn == dummy_cb_fn); 236 CU_ASSERT(_task.cb_arg == cb_arg); 237 CU_ASSERT(_task.accel_ch == g_accel_ch); 238 } 239 240 #define TEST_SUBMIT_SIZE 64 241 static void 242 test_spdk_accel_submit_copy(void) 243 { 244 const uint64_t nbytes = TEST_SUBMIT_SIZE; 245 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 246 uint8_t src[TEST_SUBMIT_SIZE] = {0}; 247 void *cb_arg = NULL; 248 int rc; 249 struct spdk_accel_task task; 250 struct spdk_accel_task_aux_data task_aux; 251 struct spdk_accel_task *expected_accel_task = NULL; 252 253 STAILQ_INIT(&g_accel_ch->task_pool); 254 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 255 256 /* Fail with no tasks on _get_task() */ 257 rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, NULL, cb_arg); 258 CU_ASSERT(rc == -ENOMEM); 259 260 task.accel_ch = g_accel_ch; 261 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 262 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 263 264 /* submission OK. */ 265 rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, NULL, cb_arg); 266 CU_ASSERT(rc == 0); 267 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COPY); 268 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 269 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 270 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 271 CU_ASSERT(expected_accel_task == &task); 272 } 273 274 static void 275 test_spdk_accel_submit_dualcast(void) 276 { 277 void *dst1; 278 void *dst2; 279 void *src; 280 uint32_t align = ALIGN_4K; 281 uint64_t nbytes = TEST_SUBMIT_SIZE; 282 void *cb_arg = NULL; 283 int rc; 284 struct spdk_accel_task task; 285 struct spdk_accel_task_aux_data task_aux; 286 struct spdk_accel_task *expected_accel_task = NULL; 287 288 STAILQ_INIT(&g_accel_ch->task_pool); 289 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 290 291 /* Dualcast requires 4K alignment on dst addresses, 292 * hence using the hard coded address to test the buffer alignment 293 */ 294 dst1 = (void *)0x5000; 295 dst2 = (void *)0x60f0; 296 src = calloc(1, TEST_SUBMIT_SIZE); 297 SPDK_CU_ASSERT_FATAL(src != NULL); 298 memset(src, 0x5A, TEST_SUBMIT_SIZE); 299 300 /* This should fail since dst2 is not 4k aligned */ 301 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, NULL, cb_arg); 302 CU_ASSERT(rc == -EINVAL); 303 304 dst1 = (void *)0x7010; 305 dst2 = (void *)0x6000; 306 /* This should fail since dst1 is not 4k aligned */ 307 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, NULL, cb_arg); 308 CU_ASSERT(rc == -EINVAL); 309 310 /* Dualcast requires 4K alignment on dst addresses */ 311 dst1 = (void *)0x7000; 312 dst2 = (void *)0x6000; 313 /* Fail with no tasks on _get_task() */ 314 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, NULL, cb_arg); 315 CU_ASSERT(rc == -ENOMEM); 316 317 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 318 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 319 320 /* accel submission OK., since we test the SW path , need to use valid memory addresses 321 * cannot hardcode them anymore */ 322 dst1 = spdk_dma_zmalloc(nbytes, align, NULL); 323 SPDK_CU_ASSERT_FATAL(dst1 != NULL); 324 dst2 = spdk_dma_zmalloc(nbytes, align, NULL); 325 SPDK_CU_ASSERT_FATAL(dst2 != NULL); 326 /* SW module does the dualcast. */ 327 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, NULL, cb_arg); 328 CU_ASSERT(rc == 0); 329 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_DUALCAST); 330 CU_ASSERT(memcmp(dst1, src, TEST_SUBMIT_SIZE) == 0); 331 CU_ASSERT(memcmp(dst2, src, TEST_SUBMIT_SIZE) == 0); 332 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 333 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 334 CU_ASSERT(expected_accel_task == &task); 335 336 free(src); 337 spdk_free(dst1); 338 spdk_free(dst2); 339 } 340 341 static void 342 test_spdk_accel_submit_compare(void) 343 { 344 void *src1; 345 void *src2; 346 uint64_t nbytes = TEST_SUBMIT_SIZE; 347 void *cb_arg = NULL; 348 int rc; 349 struct spdk_accel_task task; 350 struct spdk_accel_task_aux_data task_aux; 351 struct spdk_accel_task *expected_accel_task = NULL; 352 353 STAILQ_INIT(&g_accel_ch->task_pool); 354 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 355 356 src1 = calloc(1, TEST_SUBMIT_SIZE); 357 SPDK_CU_ASSERT_FATAL(src1 != NULL); 358 src2 = calloc(1, TEST_SUBMIT_SIZE); 359 SPDK_CU_ASSERT_FATAL(src2 != NULL); 360 361 /* Fail with no tasks on _get_task() */ 362 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 363 CU_ASSERT(rc == -ENOMEM); 364 365 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 366 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 367 368 /* accel submission OK. */ 369 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 370 CU_ASSERT(rc == 0); 371 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COMPARE); 372 CU_ASSERT(memcmp(src1, src2, TEST_SUBMIT_SIZE) == 0); 373 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 374 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 375 CU_ASSERT(expected_accel_task == &task); 376 377 free(src1); 378 free(src2); 379 } 380 381 static void 382 test_spdk_accel_submit_fill(void) 383 { 384 void *dst; 385 void *src; 386 uint8_t fill = 0xf; 387 uint64_t fill64; 388 uint64_t nbytes = TEST_SUBMIT_SIZE; 389 void *cb_arg = NULL; 390 int rc; 391 struct spdk_accel_task task; 392 struct spdk_accel_task_aux_data task_aux; 393 struct spdk_accel_task *expected_accel_task = NULL; 394 395 STAILQ_INIT(&g_accel_ch->task_pool); 396 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 397 398 dst = calloc(1, TEST_SUBMIT_SIZE); 399 SPDK_CU_ASSERT_FATAL(dst != NULL); 400 src = calloc(1, TEST_SUBMIT_SIZE); 401 SPDK_CU_ASSERT_FATAL(src != NULL); 402 memset(src, fill, TEST_SUBMIT_SIZE); 403 memset(&fill64, fill, sizeof(uint64_t)); 404 405 /* Fail with no tasks on _get_task() */ 406 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, NULL, cb_arg); 407 CU_ASSERT(rc == -ENOMEM); 408 409 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 410 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 411 412 /* accel submission OK. */ 413 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, NULL, cb_arg); 414 CU_ASSERT(rc == 0); 415 CU_ASSERT(task.fill_pattern == fill64); 416 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_FILL); 417 418 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 419 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 420 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 421 CU_ASSERT(expected_accel_task == &task); 422 423 free(dst); 424 free(src); 425 } 426 427 static void 428 test_spdk_accel_submit_crc32c(void) 429 { 430 const uint64_t nbytes = TEST_SUBMIT_SIZE; 431 uint32_t crc_dst; 432 uint8_t src[TEST_SUBMIT_SIZE]; 433 uint32_t seed = 1; 434 void *cb_arg = NULL; 435 int rc; 436 struct spdk_accel_task task; 437 struct spdk_accel_task_aux_data task_aux; 438 struct spdk_accel_task *expected_accel_task = NULL; 439 440 STAILQ_INIT(&g_accel_ch->task_pool); 441 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 442 443 /* Fail with no tasks on _get_task() */ 444 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 445 CU_ASSERT(rc == -ENOMEM); 446 447 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 448 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 449 450 /* accel submission OK. */ 451 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 452 CU_ASSERT(rc == 0); 453 CU_ASSERT(task.crc_dst == &crc_dst); 454 CU_ASSERT(task.seed == seed); 455 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_CRC32C); 456 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 457 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 458 CU_ASSERT(expected_accel_task == &task); 459 } 460 461 static void 462 test_spdk_accel_submit_crc32cv(void) 463 { 464 uint32_t crc_dst; 465 uint32_t seed = 0; 466 uint32_t iov_cnt = 32; 467 void *cb_arg = NULL; 468 int rc; 469 uint32_t i = 0; 470 struct spdk_accel_task task; 471 struct spdk_accel_task_aux_data task_aux; 472 struct iovec iov[32]; 473 struct spdk_accel_task *expected_accel_task = NULL; 474 475 STAILQ_INIT(&g_accel_ch->task_pool); 476 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 477 478 for (i = 0; i < iov_cnt; i++) { 479 iov[i].iov_base = calloc(1, TEST_SUBMIT_SIZE); 480 SPDK_CU_ASSERT_FATAL(iov[i].iov_base != NULL); 481 iov[i].iov_len = TEST_SUBMIT_SIZE; 482 } 483 484 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 485 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 486 487 /* accel submission OK. */ 488 rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, NULL, cb_arg); 489 CU_ASSERT(rc == 0); 490 CU_ASSERT(task.s.iovs == iov); 491 CU_ASSERT(task.s.iovcnt == iov_cnt); 492 CU_ASSERT(task.crc_dst == &crc_dst); 493 CU_ASSERT(task.seed == seed); 494 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_CRC32C); 495 CU_ASSERT(task.cb_arg == cb_arg); 496 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 497 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 498 CU_ASSERT(expected_accel_task == &task); 499 500 for (i = 0; i < iov_cnt; i++) { 501 free(iov[i].iov_base); 502 } 503 } 504 505 static void 506 test_spdk_accel_submit_copy_crc32c(void) 507 { 508 const uint64_t nbytes = TEST_SUBMIT_SIZE; 509 uint32_t crc_dst; 510 uint8_t dst[TEST_SUBMIT_SIZE]; 511 uint8_t src[TEST_SUBMIT_SIZE]; 512 uint32_t seed = 0; 513 void *cb_arg = NULL; 514 int rc; 515 struct spdk_accel_task task; 516 struct spdk_accel_task_aux_data task_aux; 517 struct spdk_accel_task *expected_accel_task = NULL; 518 519 STAILQ_INIT(&g_accel_ch->task_pool); 520 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 521 522 /* Fail with no tasks on _get_task() */ 523 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, NULL, cb_arg); 524 CU_ASSERT(rc == -ENOMEM); 525 526 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 527 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 528 529 /* accel submission OK. */ 530 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, NULL, cb_arg); 531 CU_ASSERT(rc == 0); 532 CU_ASSERT(task.crc_dst == &crc_dst); 533 CU_ASSERT(task.seed == seed); 534 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COPY_CRC32C); 535 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 536 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 537 CU_ASSERT(expected_accel_task == &task); 538 } 539 540 static void 541 test_spdk_accel_submit_xor(void) 542 { 543 const uint64_t nbytes = TEST_SUBMIT_SIZE; 544 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 545 uint8_t src1[TEST_SUBMIT_SIZE] = {0}; 546 uint8_t src2[TEST_SUBMIT_SIZE] = {0}; 547 void *sources[] = { src1, src2 }; 548 uint32_t nsrcs = SPDK_COUNTOF(sources); 549 int rc; 550 struct spdk_accel_task task; 551 struct spdk_accel_task_aux_data task_aux; 552 struct spdk_accel_task *expected_accel_task = NULL; 553 554 STAILQ_INIT(&g_accel_ch->task_pool); 555 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 556 557 /* Fail with no tasks on _get_task() */ 558 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 559 CU_ASSERT(rc == -ENOMEM); 560 561 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 562 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 563 564 /* submission OK. */ 565 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 566 CU_ASSERT(rc == 0); 567 CU_ASSERT(task.nsrcs.srcs == sources); 568 CU_ASSERT(task.nsrcs.cnt == nsrcs); 569 CU_ASSERT(task.d.iovcnt == 1); 570 CU_ASSERT(task.d.iovs[0].iov_base == dst); 571 CU_ASSERT(task.d.iovs[0].iov_len == nbytes); 572 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_XOR); 573 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 574 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 575 CU_ASSERT(expected_accel_task == &task); 576 } 577 578 static void 579 test_spdk_accel_module_find_by_name(void) 580 { 581 struct spdk_accel_module_if mod1 = {}; 582 struct spdk_accel_module_if mod2 = {}; 583 struct spdk_accel_module_if mod3 = {}; 584 struct spdk_accel_module_if *accel_module = NULL; 585 586 mod1.name = "ioat"; 587 mod2.name = "idxd"; 588 mod3.name = "software"; 589 590 TAILQ_INIT(&spdk_accel_module_list); 591 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod1, tailq); 592 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod2, tailq); 593 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod3, tailq); 594 595 /* Now let's find a valid engine */ 596 accel_module = _module_find_by_name("ioat"); 597 CU_ASSERT(accel_module != NULL); 598 599 /* Try to find one that doesn't exist */ 600 accel_module = _module_find_by_name("XXX"); 601 CU_ASSERT(accel_module == NULL); 602 } 603 604 static int 605 ut_module_init_nop(void) 606 { 607 return 0; 608 } 609 610 static bool 611 ut_supports_opcode_all(enum spdk_accel_opcode opcode) 612 { 613 return true; 614 } 615 616 static void 617 ut_accel_module_priority_finish_done(void *done) 618 { 619 *(int *)done = 1; 620 } 621 622 static void 623 test_spdk_accel_module_register(void) 624 { 625 struct spdk_accel_module_if mods[] = { 626 { .name = "mod1", .priority = 1, }, 627 { .name = "mod3", .priority = 3, }, 628 { .name = "mod0", .priority = 0, }, 629 { .name = "mod2", .priority = 2, }, 630 }; 631 int rc, done = 0; 632 const char *modname = NULL; 633 size_t i; 634 635 allocate_cores(1); 636 allocate_threads(1); 637 set_thread(0); 638 639 TAILQ_INIT(&spdk_accel_module_list); 640 for (i = 0; i < SPDK_COUNTOF(mods); ++i) { 641 mods[i].module_init = ut_module_init_nop; 642 mods[i].supports_opcode = ut_supports_opcode_all; 643 spdk_accel_module_list_add(&mods[i]); 644 } 645 646 rc = spdk_accel_initialize(); 647 CU_ASSERT_EQUAL(rc, 0); 648 649 /* All opcodes should be assigned to the module with highest prio - mod3 */ 650 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 651 rc = spdk_accel_get_opc_module_name((enum spdk_accel_opcode)i, &modname); 652 CU_ASSERT_EQUAL(rc, 0); 653 CU_ASSERT_STRING_EQUAL(modname, "mod3"); 654 } 655 656 spdk_accel_finish(ut_accel_module_priority_finish_done, &done); 657 while (!done) { 658 poll_threads(); 659 } 660 661 TAILQ_INIT(&spdk_accel_module_list); 662 free_threads(); 663 free_cores(); 664 } 665 666 struct ut_sequence { 667 bool complete; 668 int status; 669 }; 670 671 static void 672 ut_sequence_step_cb(void *cb_arg) 673 { 674 int *completed = cb_arg; 675 676 (*completed)++; 677 } 678 679 static void 680 ut_sequence_complete_cb(void *cb_arg, int status) 681 { 682 struct ut_sequence *seq = cb_arg; 683 684 seq->complete = true; 685 seq->status = status; 686 } 687 688 static void 689 test_sequence_fill_copy(void) 690 { 691 struct spdk_accel_sequence *seq = NULL; 692 struct spdk_io_channel *ioch; 693 struct ut_sequence ut_seq; 694 char buf[4096], tmp[2][4096], expected[4096]; 695 struct iovec src_iovs[2], dst_iovs[2]; 696 int rc, completed; 697 698 ioch = spdk_accel_get_io_channel(); 699 SPDK_CU_ASSERT_FATAL(ioch != NULL); 700 701 /* First check the simplest case - single task in a sequence */ 702 memset(buf, 0, sizeof(buf)); 703 memset(expected, 0xa5, sizeof(expected)); 704 completed = 0; 705 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 706 ut_sequence_step_cb, &completed); 707 CU_ASSERT_EQUAL(rc, 0); 708 CU_ASSERT_EQUAL(completed, 0); 709 710 ut_seq.complete = false; 711 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 712 713 poll_threads(); 714 CU_ASSERT_EQUAL(completed, 1); 715 CU_ASSERT(ut_seq.complete); 716 CU_ASSERT_EQUAL(ut_seq.status, 0); 717 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 718 719 /* Check a single copy operation */ 720 memset(buf, 0, sizeof(buf)); 721 memset(tmp[0], 0xa5, sizeof(tmp[0])); 722 memset(expected, 0xa5, sizeof(expected)); 723 completed = 0; 724 seq = NULL; 725 726 dst_iovs[0].iov_base = buf; 727 dst_iovs[0].iov_len = sizeof(buf); 728 src_iovs[0].iov_base = tmp[0]; 729 src_iovs[0].iov_len = sizeof(tmp[0]); 730 731 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 732 &src_iovs[0], 1, NULL, NULL, 733 ut_sequence_step_cb, &completed); 734 CU_ASSERT_EQUAL(rc, 0); 735 736 ut_seq.complete = false; 737 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 738 739 poll_threads(); 740 CU_ASSERT_EQUAL(completed, 1); 741 CU_ASSERT(ut_seq.complete); 742 CU_ASSERT_EQUAL(ut_seq.status, 0); 743 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 744 745 /* Check multiple fill operations */ 746 memset(buf, 0, sizeof(buf)); 747 memset(expected, 0xfe, 4096); 748 memset(expected, 0xde, 2048); 749 memset(expected, 0xa5, 1024); 750 seq = NULL; 751 completed = 0; 752 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, NULL, NULL, 0xfe, 753 ut_sequence_step_cb, &completed); 754 CU_ASSERT_EQUAL(rc, 0); 755 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xde, 756 ut_sequence_step_cb, &completed); 757 CU_ASSERT_EQUAL(rc, 0); 758 rc = spdk_accel_append_fill(&seq, ioch, buf, 1024, NULL, NULL, 0xa5, 759 ut_sequence_step_cb, &completed); 760 CU_ASSERT_EQUAL(rc, 0); 761 762 ut_seq.complete = false; 763 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 764 765 poll_threads(); 766 CU_ASSERT_EQUAL(completed, 3); 767 CU_ASSERT(ut_seq.complete); 768 CU_ASSERT_EQUAL(ut_seq.status, 0); 769 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 770 771 /* Check multiple copy operations */ 772 memset(buf, 0, sizeof(buf)); 773 memset(tmp[0], 0, sizeof(tmp[0])); 774 memset(tmp[1], 0, sizeof(tmp[1])); 775 memset(expected, 0xa5, sizeof(expected)); 776 seq = NULL; 777 completed = 0; 778 779 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 780 ut_sequence_step_cb, &completed); 781 CU_ASSERT_EQUAL(rc, 0); 782 783 dst_iovs[0].iov_base = tmp[1]; 784 dst_iovs[0].iov_len = sizeof(tmp[1]); 785 src_iovs[0].iov_base = tmp[0]; 786 src_iovs[0].iov_len = sizeof(tmp[0]); 787 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 788 &src_iovs[0], 1, NULL, NULL, 789 ut_sequence_step_cb, &completed); 790 CU_ASSERT_EQUAL(rc, 0); 791 792 dst_iovs[1].iov_base = buf; 793 dst_iovs[1].iov_len = sizeof(buf); 794 src_iovs[1].iov_base = tmp[1]; 795 src_iovs[1].iov_len = sizeof(tmp[1]); 796 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 797 &src_iovs[1], 1, NULL, NULL, 798 ut_sequence_step_cb, &completed); 799 CU_ASSERT_EQUAL(rc, 0); 800 801 ut_seq.complete = false; 802 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 803 804 poll_threads(); 805 CU_ASSERT_EQUAL(completed, 3); 806 CU_ASSERT(ut_seq.complete); 807 CU_ASSERT_EQUAL(ut_seq.status, 0); 808 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 809 810 /* Check that adding a copy operation at the end will change destination buffer */ 811 memset(buf, 0, sizeof(buf)); 812 memset(tmp[0], 0, sizeof(tmp[0])); 813 memset(expected, 0xa5, sizeof(buf)); 814 seq = NULL; 815 completed = 0; 816 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 817 ut_sequence_step_cb, &completed); 818 CU_ASSERT_EQUAL(rc, 0); 819 820 dst_iovs[0].iov_base = buf; 821 dst_iovs[0].iov_len = sizeof(buf); 822 src_iovs[0].iov_base = tmp[0]; 823 src_iovs[0].iov_len = sizeof(tmp[0]); 824 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 825 &src_iovs[0], 1, NULL, NULL, 826 ut_sequence_step_cb, &completed); 827 CU_ASSERT_EQUAL(rc, 0); 828 829 ut_seq.complete = false; 830 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 831 832 poll_threads(); 833 CU_ASSERT_EQUAL(completed, 2); 834 CU_ASSERT(ut_seq.complete); 835 CU_ASSERT_EQUAL(ut_seq.status, 0); 836 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 837 838 /* Check that it's also possible to add copy operation at the beginning */ 839 memset(buf, 0, sizeof(buf)); 840 memset(tmp[0], 0xde, sizeof(tmp[0])); 841 memset(tmp[1], 0, sizeof(tmp[1])); 842 memset(expected, 0xa5, sizeof(expected)); 843 seq = NULL; 844 completed = 0; 845 846 dst_iovs[0].iov_base = tmp[1]; 847 dst_iovs[0].iov_len = sizeof(tmp[1]); 848 src_iovs[0].iov_base = tmp[0]; 849 src_iovs[0].iov_len = sizeof(tmp[0]); 850 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 851 &src_iovs[0], 1, NULL, NULL, 852 ut_sequence_step_cb, &completed); 853 CU_ASSERT_EQUAL(rc, 0); 854 855 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], sizeof(tmp[1]), NULL, NULL, 0xa5, 856 ut_sequence_step_cb, &completed); 857 CU_ASSERT_EQUAL(rc, 0); 858 859 dst_iovs[1].iov_base = buf; 860 dst_iovs[1].iov_len = sizeof(buf); 861 src_iovs[1].iov_base = tmp[1]; 862 src_iovs[1].iov_len = sizeof(tmp[1]); 863 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 864 &src_iovs[1], 1, NULL, NULL, 865 ut_sequence_step_cb, &completed); 866 CU_ASSERT_EQUAL(rc, 0); 867 868 ut_seq.complete = false; 869 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 870 871 poll_threads(); 872 CU_ASSERT_EQUAL(completed, 3); 873 CU_ASSERT(ut_seq.complete); 874 CU_ASSERT_EQUAL(ut_seq.status, 0); 875 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 876 877 spdk_put_io_channel(ioch); 878 poll_threads(); 879 } 880 881 static void 882 test_sequence_abort(void) 883 { 884 struct spdk_accel_sequence *seq = NULL; 885 struct spdk_io_channel *ioch; 886 char buf[4096], tmp[2][4096], expected[4096]; 887 struct iovec src_iovs[2], dst_iovs[2]; 888 int rc, completed; 889 890 ioch = spdk_accel_get_io_channel(); 891 SPDK_CU_ASSERT_FATAL(ioch != NULL); 892 893 /* Check that aborting a sequence calls operation's callback, the operation is not executed 894 * and the sequence is freed 895 */ 896 memset(buf, 0, sizeof(buf)); 897 memset(expected, 0, sizeof(buf)); 898 completed = 0; 899 seq = NULL; 900 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 901 ut_sequence_step_cb, &completed); 902 CU_ASSERT_EQUAL(rc, 0); 903 904 spdk_accel_sequence_abort(seq); 905 CU_ASSERT_EQUAL(completed, 1); 906 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 907 908 /* Check sequence with multiple operations */ 909 memset(buf, 0, sizeof(buf)); 910 memset(expected, 0, sizeof(buf)); 911 completed = 0; 912 seq = NULL; 913 914 dst_iovs[0].iov_base = tmp[1]; 915 dst_iovs[0].iov_len = sizeof(tmp[1]); 916 src_iovs[0].iov_base = tmp[0]; 917 src_iovs[0].iov_len = sizeof(tmp[0]); 918 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 919 &src_iovs[0], 1, NULL, NULL, 920 ut_sequence_step_cb, &completed); 921 CU_ASSERT_EQUAL(rc, 0); 922 923 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 4096, NULL, NULL, 0xa5, 924 ut_sequence_step_cb, &completed); 925 CU_ASSERT_EQUAL(rc, 0); 926 927 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xde, 928 ut_sequence_step_cb, &completed); 929 CU_ASSERT_EQUAL(rc, 0); 930 931 dst_iovs[1].iov_base = buf; 932 dst_iovs[1].iov_len = sizeof(buf); 933 src_iovs[1].iov_base = tmp[1]; 934 src_iovs[1].iov_len = sizeof(tmp[1]); 935 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 936 &src_iovs[1], 1, NULL, NULL, 937 ut_sequence_step_cb, &completed); 938 CU_ASSERT_EQUAL(rc, 0); 939 940 spdk_accel_sequence_abort(seq); 941 CU_ASSERT_EQUAL(completed, 4); 942 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 943 944 /* This should be a no-op */ 945 spdk_accel_sequence_abort(NULL); 946 947 spdk_put_io_channel(ioch); 948 poll_threads(); 949 } 950 951 static void 952 test_sequence_append_error(void) 953 { 954 struct spdk_accel_sequence *seq = NULL; 955 struct spdk_io_channel *ioch; 956 struct accel_io_channel *accel_ch; 957 struct iovec src_iovs, dst_iovs; 958 char buf[4096]; 959 STAILQ_HEAD(, spdk_accel_task) tasks = STAILQ_HEAD_INITIALIZER(tasks); 960 SLIST_HEAD(, spdk_accel_sequence) seqs = SLIST_HEAD_INITIALIZER(seqs); 961 int rc; 962 963 ioch = spdk_accel_get_io_channel(); 964 SPDK_CU_ASSERT_FATAL(ioch != NULL); 965 accel_ch = spdk_io_channel_get_ctx(ioch); 966 967 /* Check that append fails and no sequence object is allocated when there are no more free 968 * tasks */ 969 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 970 971 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 972 ut_sequence_step_cb, NULL); 973 CU_ASSERT_EQUAL(rc, -ENOMEM); 974 CU_ASSERT_PTR_NULL(seq); 975 976 dst_iovs.iov_base = buf; 977 dst_iovs.iov_len = 2048; 978 src_iovs.iov_base = &buf[2048]; 979 src_iovs.iov_len = 2048; 980 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 981 &src_iovs, 1, NULL, NULL, ut_sequence_step_cb, NULL); 982 CU_ASSERT_EQUAL(rc, -ENOMEM); 983 CU_ASSERT_PTR_NULL(seq); 984 985 dst_iovs.iov_base = buf; 986 dst_iovs.iov_len = 2048; 987 src_iovs.iov_base = &buf[2048]; 988 src_iovs.iov_len = 2048; 989 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 990 &src_iovs, 1, NULL, NULL, ut_sequence_step_cb, NULL); 991 CU_ASSERT_EQUAL(rc, -ENOMEM); 992 CU_ASSERT_PTR_NULL(seq); 993 994 /* Check that the same happens when the sequence queue is empty */ 995 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 996 SLIST_SWAP(&seqs, &accel_ch->seq_pool, spdk_accel_sequence); 997 998 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 999 ut_sequence_step_cb, NULL); 1000 CU_ASSERT_EQUAL(rc, -ENOMEM); 1001 CU_ASSERT_PTR_NULL(seq); 1002 1003 dst_iovs.iov_base = buf; 1004 dst_iovs.iov_len = 2048; 1005 src_iovs.iov_base = &buf[2048]; 1006 src_iovs.iov_len = 2048; 1007 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1008 &src_iovs, 1, NULL, NULL, ut_sequence_step_cb, NULL); 1009 CU_ASSERT_EQUAL(rc, -ENOMEM); 1010 CU_ASSERT_PTR_NULL(seq); 1011 1012 dst_iovs.iov_base = buf; 1013 dst_iovs.iov_len = 2048; 1014 src_iovs.iov_base = &buf[2048]; 1015 src_iovs.iov_len = 2048; 1016 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1017 &src_iovs, 1, NULL, NULL, ut_sequence_step_cb, NULL); 1018 CU_ASSERT_EQUAL(rc, -ENOMEM); 1019 CU_ASSERT_PTR_NULL(seq); 1020 1021 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 1022 1023 spdk_put_io_channel(ioch); 1024 poll_threads(); 1025 } 1026 1027 struct ut_sequence_operation { 1028 int complete_status; 1029 int submit_status; 1030 int count; 1031 struct iovec *src_iovs; 1032 uint32_t src_iovcnt; 1033 struct spdk_memory_domain *src_domain; 1034 void *src_domain_ctx; 1035 struct iovec *dst_iovs; 1036 uint32_t dst_iovcnt; 1037 struct spdk_memory_domain *dst_domain; 1038 void *dst_domain_ctx; 1039 int (*submit)(struct spdk_io_channel *ch, struct spdk_accel_task *t); 1040 }; 1041 1042 static struct ut_sequence_operation g_seq_operations[SPDK_ACCEL_OPC_LAST]; 1043 1044 static int 1045 ut_sequence_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *task) 1046 { 1047 struct ut_sequence_operation *op = &g_seq_operations[task->op_code]; 1048 1049 op->count++; 1050 if (op->submit != NULL) { 1051 return op->submit(ch, task); 1052 } 1053 if (op->src_iovs != NULL) { 1054 CU_ASSERT_EQUAL(task->s.iovcnt, op->src_iovcnt); 1055 CU_ASSERT_EQUAL(memcmp(task->s.iovs, op->src_iovs, 1056 sizeof(struct iovec) * op->src_iovcnt), 0); 1057 } 1058 if (op->dst_iovs != NULL) { 1059 CU_ASSERT_EQUAL(task->d.iovcnt, op->dst_iovcnt); 1060 CU_ASSERT_EQUAL(memcmp(task->d.iovs, op->dst_iovs, 1061 sizeof(struct iovec) * op->dst_iovcnt), 0); 1062 } 1063 1064 CU_ASSERT_EQUAL(task->src_domain, op->src_domain); 1065 CU_ASSERT_EQUAL(task->dst_domain, op->dst_domain); 1066 if (op->src_domain != NULL) { 1067 CU_ASSERT_EQUAL(task->src_domain_ctx, op->src_domain_ctx); 1068 } 1069 if (op->dst_domain != NULL) { 1070 CU_ASSERT_EQUAL(task->dst_domain_ctx, op->dst_domain_ctx); 1071 } 1072 1073 if (op->submit_status != 0) { 1074 return op->submit_status; 1075 } 1076 1077 spdk_accel_task_complete(task, op->complete_status); 1078 1079 return 0; 1080 } 1081 1082 static void 1083 ut_clear_operations(void) 1084 { 1085 memset(&g_seq_operations, 0, sizeof(g_seq_operations)); 1086 } 1087 1088 static void 1089 test_sequence_completion_error(void) 1090 { 1091 struct spdk_accel_sequence *seq = NULL; 1092 struct spdk_io_channel *ioch; 1093 struct ut_sequence ut_seq; 1094 struct iovec src_iovs, dst_iovs; 1095 char buf[4096], tmp[4096]; 1096 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 1097 int i, rc, completed; 1098 1099 ioch = spdk_accel_get_io_channel(); 1100 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1101 1102 /* Override the submit_tasks function */ 1103 g_module_if.submit_tasks = ut_sequence_submit_tasks; 1104 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1105 modules[i] = g_modules_opc[i]; 1106 g_modules_opc[i] = g_module; 1107 } 1108 1109 memset(buf, 0, sizeof(buf)); 1110 memset(tmp, 0, sizeof(tmp)); 1111 1112 /* Check that if the first operation completes with an error, the whole sequence is 1113 * completed with that error and that all operations' completion callbacks are executed 1114 */ 1115 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = -E2BIG; 1116 completed = 0; 1117 seq = NULL; 1118 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 1119 ut_sequence_step_cb, &completed); 1120 CU_ASSERT_EQUAL(rc, 0); 1121 1122 dst_iovs.iov_base = buf; 1123 dst_iovs.iov_len = sizeof(buf); 1124 src_iovs.iov_base = tmp; 1125 src_iovs.iov_len = sizeof(tmp); 1126 1127 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1128 &src_iovs, 1, NULL, NULL, 1129 ut_sequence_step_cb, &completed); 1130 CU_ASSERT_EQUAL(rc, 0); 1131 1132 ut_seq.complete = false; 1133 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1134 1135 poll_threads(); 1136 CU_ASSERT_EQUAL(completed, 2); 1137 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 1138 1139 /* Check the same with a second operation in the sequence */ 1140 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].complete_status = -EACCES; 1141 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 1142 completed = 0; 1143 seq = NULL; 1144 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 1145 ut_sequence_step_cb, &completed); 1146 CU_ASSERT_EQUAL(rc, 0); 1147 1148 dst_iovs.iov_base = buf; 1149 dst_iovs.iov_len = sizeof(buf); 1150 src_iovs.iov_base = tmp; 1151 src_iovs.iov_len = sizeof(tmp); 1152 1153 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1154 &src_iovs, 1, NULL, NULL, 1155 ut_sequence_step_cb, &completed); 1156 CU_ASSERT_EQUAL(rc, 0); 1157 1158 ut_seq.complete = false; 1159 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1160 1161 poll_threads(); 1162 CU_ASSERT_EQUAL(completed, 2); 1163 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 1164 1165 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].complete_status = 0; 1166 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 1167 1168 /* Check submission failure of the first operation */ 1169 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit_status = -EADDRINUSE; 1170 completed = 0; 1171 seq = NULL; 1172 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 1173 ut_sequence_step_cb, &completed); 1174 CU_ASSERT_EQUAL(rc, 0); 1175 1176 dst_iovs.iov_base = buf; 1177 dst_iovs.iov_len = sizeof(buf); 1178 src_iovs.iov_base = tmp; 1179 src_iovs.iov_len = sizeof(tmp); 1180 1181 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1182 &src_iovs, 1, NULL, NULL, 1183 ut_sequence_step_cb, &completed); 1184 CU_ASSERT_EQUAL(rc, 0); 1185 1186 ut_seq.complete = false; 1187 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1188 1189 poll_threads(); 1190 CU_ASSERT_EQUAL(completed, 2); 1191 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 1192 1193 /* Check the same with a second operation */ 1194 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit_status = -EADDRNOTAVAIL; 1195 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit_status = 0; 1196 completed = 0; 1197 seq = NULL; 1198 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 1199 ut_sequence_step_cb, &completed); 1200 CU_ASSERT_EQUAL(rc, 0); 1201 1202 dst_iovs.iov_base = buf; 1203 dst_iovs.iov_len = sizeof(buf); 1204 src_iovs.iov_base = tmp; 1205 src_iovs.iov_len = sizeof(tmp); 1206 1207 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1208 &src_iovs, 1, NULL, NULL, 1209 ut_sequence_step_cb, &completed); 1210 CU_ASSERT_EQUAL(rc, 0); 1211 1212 ut_seq.complete = false; 1213 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1214 1215 poll_threads(); 1216 CU_ASSERT_EQUAL(completed, 2); 1217 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 1218 1219 /* Cleanup module pointers to make subsequent tests work correctly */ 1220 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1221 g_modules_opc[i] = modules[i]; 1222 } 1223 1224 ut_clear_operations(); 1225 spdk_put_io_channel(ioch); 1226 poll_threads(); 1227 } 1228 1229 #ifdef SPDK_CONFIG_ISAL 1230 static void 1231 ut_compress_cb(void *cb_arg, int status) 1232 { 1233 int *completed = cb_arg; 1234 1235 CU_ASSERT_EQUAL(status, 0); 1236 1237 *completed = 1; 1238 } 1239 1240 static void 1241 test_sequence_decompress(void) 1242 { 1243 struct spdk_accel_sequence *seq = NULL; 1244 struct spdk_io_channel *ioch; 1245 struct ut_sequence ut_seq; 1246 char buf[4096], tmp[2][4096], expected[4096]; 1247 struct iovec src_iovs[2], dst_iovs[2]; 1248 uint32_t compressed_size; 1249 int rc, completed = 0; 1250 1251 ioch = spdk_accel_get_io_channel(); 1252 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1253 1254 memset(expected, 0xa5, sizeof(expected)); 1255 src_iovs[0].iov_base = expected; 1256 src_iovs[0].iov_len = sizeof(expected); 1257 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1258 &compressed_size, ut_compress_cb, &completed); 1259 CU_ASSERT_EQUAL(rc, 0); 1260 1261 while (!completed) { 1262 poll_threads(); 1263 } 1264 1265 /* Check a single decompress operation in a sequence */ 1266 seq = NULL; 1267 completed = 0; 1268 1269 dst_iovs[0].iov_base = buf; 1270 dst_iovs[0].iov_len = sizeof(buf); 1271 src_iovs[0].iov_base = tmp[0]; 1272 src_iovs[0].iov_len = compressed_size; 1273 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1274 &src_iovs[0], 1, NULL, NULL, 1275 ut_sequence_step_cb, &completed); 1276 CU_ASSERT_EQUAL(rc, 0); 1277 1278 ut_seq.complete = false; 1279 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1280 1281 poll_threads(); 1282 1283 CU_ASSERT_EQUAL(completed, 1); 1284 CU_ASSERT(ut_seq.complete); 1285 CU_ASSERT_EQUAL(ut_seq.status, 0); 1286 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1287 1288 /* Put the decompress operation in the middle of a sequence with a copy operation at the 1289 * beginning and a fill at the end modifying the first 2048B of the buffer. 1290 */ 1291 memset(expected, 0xfe, 2048); 1292 memset(buf, 0, sizeof(buf)); 1293 seq = NULL; 1294 completed = 0; 1295 1296 dst_iovs[0].iov_base = tmp[1]; 1297 dst_iovs[0].iov_len = compressed_size; 1298 src_iovs[0].iov_base = tmp[0]; 1299 src_iovs[0].iov_len = compressed_size; 1300 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1301 &src_iovs[0], 1, NULL, NULL, 1302 ut_sequence_step_cb, &completed); 1303 CU_ASSERT_EQUAL(rc, 0); 1304 1305 dst_iovs[1].iov_base = buf; 1306 dst_iovs[1].iov_len = sizeof(buf); 1307 src_iovs[1].iov_base = tmp[1]; 1308 src_iovs[1].iov_len = compressed_size; 1309 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1310 &src_iovs[1], 1, NULL, NULL, 1311 ut_sequence_step_cb, &completed); 1312 CU_ASSERT_EQUAL(rc, 0); 1313 1314 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 1315 ut_sequence_step_cb, &completed); 1316 CU_ASSERT_EQUAL(rc, 0); 1317 1318 ut_seq.complete = false; 1319 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1320 1321 poll_threads(); 1322 1323 CU_ASSERT_EQUAL(completed, 3); 1324 CU_ASSERT(ut_seq.complete); 1325 CU_ASSERT_EQUAL(ut_seq.status, 0); 1326 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1327 1328 /* Check sequence with decompress at the beginning: decompress -> copy */ 1329 memset(expected, 0xa5, sizeof(expected)); 1330 memset(buf, 0, sizeof(buf)); 1331 seq = NULL; 1332 completed = 0; 1333 1334 dst_iovs[0].iov_base = tmp[1]; 1335 dst_iovs[0].iov_len = sizeof(tmp[1]); 1336 src_iovs[0].iov_base = tmp[0]; 1337 src_iovs[0].iov_len = compressed_size; 1338 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1339 &src_iovs[0], 1, NULL, NULL, 1340 ut_sequence_step_cb, &completed); 1341 CU_ASSERT_EQUAL(rc, 0); 1342 1343 dst_iovs[1].iov_base = buf; 1344 dst_iovs[1].iov_len = sizeof(buf); 1345 src_iovs[1].iov_base = tmp[1]; 1346 src_iovs[1].iov_len = sizeof(tmp[1]); 1347 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1348 &src_iovs[1], 1, NULL, NULL, 1349 ut_sequence_step_cb, &completed); 1350 CU_ASSERT_EQUAL(rc, 0); 1351 1352 ut_seq.complete = false; 1353 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1354 1355 poll_threads(); 1356 1357 CU_ASSERT_EQUAL(completed, 2); 1358 CU_ASSERT(ut_seq.complete); 1359 CU_ASSERT_EQUAL(ut_seq.status, 0); 1360 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1361 1362 spdk_put_io_channel(ioch); 1363 poll_threads(); 1364 } 1365 1366 static void 1367 test_sequence_reverse(void) 1368 { 1369 struct spdk_accel_sequence *seq = NULL; 1370 struct spdk_io_channel *ioch; 1371 struct ut_sequence ut_seq; 1372 char buf[4096], tmp[2][4096], expected[4096]; 1373 struct iovec src_iovs[2], dst_iovs[2]; 1374 uint32_t compressed_size; 1375 int rc, completed = 0; 1376 1377 ioch = spdk_accel_get_io_channel(); 1378 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1379 1380 memset(expected, 0xa5, sizeof(expected)); 1381 src_iovs[0].iov_base = expected; 1382 src_iovs[0].iov_len = sizeof(expected); 1383 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1384 &compressed_size, ut_compress_cb, &completed); 1385 CU_ASSERT_EQUAL(rc, 0); 1386 1387 while (!completed) { 1388 poll_threads(); 1389 } 1390 1391 /* First check that reversing a sequence with a single operation is a no-op */ 1392 memset(buf, 0, sizeof(buf)); 1393 seq = NULL; 1394 completed = 0; 1395 1396 dst_iovs[0].iov_base = buf; 1397 dst_iovs[0].iov_len = sizeof(buf); 1398 src_iovs[0].iov_base = tmp[0]; 1399 src_iovs[0].iov_len = compressed_size; 1400 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1401 &src_iovs[0], 1, NULL, NULL, 1402 ut_sequence_step_cb, &completed); 1403 CU_ASSERT_EQUAL(rc, 0); 1404 1405 spdk_accel_sequence_reverse(seq); 1406 1407 ut_seq.complete = false; 1408 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1409 1410 poll_threads(); 1411 1412 CU_ASSERT_EQUAL(completed, 1); 1413 CU_ASSERT(ut_seq.complete); 1414 CU_ASSERT_EQUAL(ut_seq.status, 0); 1415 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1416 1417 /* Add a copy operation at the end with src set to the compressed data. After reverse(), 1418 * that copy operation should be first, so decompress() should receive compressed data in 1419 * its src buffer. 1420 */ 1421 memset(buf, 0, sizeof(buf)); 1422 memset(tmp[1], 0, sizeof(tmp[1])); 1423 seq = NULL; 1424 completed = 0; 1425 1426 dst_iovs[0].iov_base = buf; 1427 dst_iovs[0].iov_len = sizeof(buf); 1428 src_iovs[0].iov_base = tmp[1]; 1429 src_iovs[0].iov_len = compressed_size; 1430 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1431 &src_iovs[0], 1, NULL, NULL, 1432 ut_sequence_step_cb, &completed); 1433 CU_ASSERT_EQUAL(rc, 0); 1434 1435 dst_iovs[1].iov_base = tmp[1]; 1436 dst_iovs[1].iov_len = compressed_size; 1437 src_iovs[1].iov_base = tmp[0]; 1438 src_iovs[1].iov_len = compressed_size; 1439 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1440 &src_iovs[1], 1, NULL, NULL, 1441 ut_sequence_step_cb, &completed); 1442 CU_ASSERT_EQUAL(rc, 0); 1443 1444 spdk_accel_sequence_reverse(seq); 1445 1446 ut_seq.complete = false; 1447 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1448 1449 poll_threads(); 1450 1451 CU_ASSERT_EQUAL(completed, 2); 1452 CU_ASSERT(ut_seq.complete); 1453 CU_ASSERT_EQUAL(ut_seq.status, 0); 1454 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1455 1456 /* Check the same, but add an extra fill operation at the beginning that should execute last 1457 * after reverse(). 1458 */ 1459 memset(buf, 0, sizeof(buf)); 1460 memset(tmp[1], 0, sizeof(tmp[1])); 1461 memset(expected, 0xfe, 2048); 1462 seq = NULL; 1463 completed = 0; 1464 1465 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 1466 ut_sequence_step_cb, &completed); 1467 CU_ASSERT_EQUAL(rc, 0); 1468 1469 dst_iovs[0].iov_base = buf; 1470 dst_iovs[0].iov_len = sizeof(buf); 1471 src_iovs[0].iov_base = tmp[1]; 1472 src_iovs[0].iov_len = compressed_size; 1473 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1474 &src_iovs[0], 1, NULL, NULL, 1475 ut_sequence_step_cb, &completed); 1476 CU_ASSERT_EQUAL(rc, 0); 1477 1478 dst_iovs[1].iov_base = tmp[1]; 1479 dst_iovs[1].iov_len = compressed_size; 1480 src_iovs[1].iov_base = tmp[0]; 1481 src_iovs[1].iov_len = compressed_size; 1482 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1483 &src_iovs[1], 1, NULL, NULL, 1484 ut_sequence_step_cb, &completed); 1485 CU_ASSERT_EQUAL(rc, 0); 1486 1487 spdk_accel_sequence_reverse(seq); 1488 1489 ut_seq.complete = false; 1490 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1491 1492 poll_threads(); 1493 1494 CU_ASSERT_EQUAL(completed, 3); 1495 CU_ASSERT(ut_seq.complete); 1496 CU_ASSERT_EQUAL(ut_seq.status, 0); 1497 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1498 1499 /* Build the sequence in order and then reverse it twice */ 1500 memset(buf, 0, sizeof(buf)); 1501 memset(tmp[1], 0, sizeof(tmp[1])); 1502 seq = NULL; 1503 completed = 0; 1504 1505 dst_iovs[0].iov_base = tmp[1]; 1506 dst_iovs[0].iov_len = compressed_size; 1507 src_iovs[0].iov_base = tmp[0]; 1508 src_iovs[0].iov_len = compressed_size; 1509 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1510 &src_iovs[0], 1, NULL, NULL, 1511 ut_sequence_step_cb, &completed); 1512 CU_ASSERT_EQUAL(rc, 0); 1513 1514 dst_iovs[1].iov_base = buf; 1515 dst_iovs[1].iov_len = sizeof(buf); 1516 src_iovs[1].iov_base = tmp[1]; 1517 src_iovs[1].iov_len = compressed_size; 1518 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1519 &src_iovs[1], 1, NULL, NULL, 1520 ut_sequence_step_cb, &completed); 1521 CU_ASSERT_EQUAL(rc, 0); 1522 1523 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 1524 ut_sequence_step_cb, &completed); 1525 CU_ASSERT_EQUAL(rc, 0); 1526 1527 spdk_accel_sequence_reverse(seq); 1528 spdk_accel_sequence_reverse(seq); 1529 1530 ut_seq.complete = false; 1531 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1532 1533 poll_threads(); 1534 1535 CU_ASSERT_EQUAL(completed, 3); 1536 CU_ASSERT(ut_seq.complete); 1537 CU_ASSERT_EQUAL(ut_seq.status, 0); 1538 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1539 1540 spdk_put_io_channel(ioch); 1541 poll_threads(); 1542 } 1543 #endif 1544 1545 static void 1546 test_sequence_copy_elision(void) 1547 { 1548 struct spdk_accel_sequence *seq = NULL; 1549 struct spdk_io_channel *ioch; 1550 struct ut_sequence ut_seq; 1551 struct iovec src_iovs[4], dst_iovs[4], exp_iovs[2]; 1552 char buf[4096], tmp[4][4096]; 1553 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 1554 struct spdk_accel_crypto_key key = {}; 1555 int i, rc, completed; 1556 1557 ioch = spdk_accel_get_io_channel(); 1558 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1559 1560 /* Override the submit_tasks function */ 1561 g_module_if.submit_tasks = ut_sequence_submit_tasks; 1562 g_module.supports_memory_domains = true; 1563 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1564 g_seq_operations[i].complete_status = 0; 1565 g_seq_operations[i].submit_status = 0; 1566 g_seq_operations[i].count = 0; 1567 1568 modules[i] = g_modules_opc[i]; 1569 g_modules_opc[i] = g_module; 1570 } 1571 1572 /* Check that a copy operation at the beginning is removed */ 1573 seq = NULL; 1574 completed = 0; 1575 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovcnt = 1; 1576 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovcnt = 1; 1577 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1578 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1579 exp_iovs[0].iov_base = tmp[0]; 1580 exp_iovs[0].iov_len = sizeof(tmp[0]); 1581 exp_iovs[1].iov_base = buf; 1582 exp_iovs[1].iov_len = 2048; 1583 1584 dst_iovs[0].iov_base = tmp[1]; 1585 dst_iovs[0].iov_len = sizeof(tmp[1]); 1586 src_iovs[0].iov_base = tmp[0]; 1587 src_iovs[0].iov_len = sizeof(tmp[0]); 1588 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1589 &src_iovs[0], 1, NULL, NULL, 1590 ut_sequence_step_cb, &completed); 1591 CU_ASSERT_EQUAL(rc, 0); 1592 1593 dst_iovs[1].iov_base = buf; 1594 dst_iovs[1].iov_len = 2048; 1595 src_iovs[1].iov_base = tmp[1]; 1596 src_iovs[1].iov_len = sizeof(tmp[1]); 1597 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1598 &src_iovs[1], 1, NULL, NULL, 1599 ut_sequence_step_cb, &completed); 1600 CU_ASSERT_EQUAL(rc, 0); 1601 1602 ut_seq.complete = false; 1603 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1604 1605 poll_threads(); 1606 1607 CU_ASSERT_EQUAL(completed, 2); 1608 CU_ASSERT(ut_seq.complete); 1609 CU_ASSERT_EQUAL(ut_seq.status, 0); 1610 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1611 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1612 1613 /* Check that a copy operation at the end is removed too */ 1614 seq = NULL; 1615 completed = 0; 1616 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1617 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1618 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1619 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1620 exp_iovs[0].iov_base = tmp[0]; 1621 exp_iovs[0].iov_len = sizeof(tmp[0]); 1622 exp_iovs[1].iov_base = buf; 1623 exp_iovs[1].iov_len = 2048; 1624 1625 dst_iovs[0].iov_base = tmp[1]; 1626 dst_iovs[0].iov_len = 2048; 1627 src_iovs[0].iov_base = tmp[0]; 1628 src_iovs[0].iov_len = sizeof(tmp[0]); 1629 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1630 &src_iovs[0], 1, NULL, NULL, 1631 ut_sequence_step_cb, &completed); 1632 CU_ASSERT_EQUAL(rc, 0); 1633 1634 dst_iovs[1].iov_base = buf; 1635 dst_iovs[1].iov_len = 2048; 1636 src_iovs[1].iov_base = tmp[1]; 1637 src_iovs[1].iov_len = 2048; 1638 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1639 &src_iovs[1], 1, NULL, NULL, 1640 ut_sequence_step_cb, &completed); 1641 CU_ASSERT_EQUAL(rc, 0); 1642 1643 ut_seq.complete = false; 1644 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1645 1646 poll_threads(); 1647 1648 CU_ASSERT_EQUAL(completed, 2); 1649 CU_ASSERT(ut_seq.complete); 1650 CU_ASSERT_EQUAL(ut_seq.status, 0); 1651 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1652 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1653 1654 /* Check a copy operation both at the beginning and the end */ 1655 seq = NULL; 1656 completed = 0; 1657 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1658 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1659 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1660 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1661 exp_iovs[0].iov_base = tmp[0]; 1662 exp_iovs[0].iov_len = sizeof(tmp[0]); 1663 exp_iovs[1].iov_base = buf; 1664 exp_iovs[1].iov_len = 2048; 1665 1666 dst_iovs[0].iov_base = tmp[1]; 1667 dst_iovs[0].iov_len = sizeof(tmp[1]); 1668 src_iovs[0].iov_base = tmp[0]; 1669 src_iovs[0].iov_len = sizeof(tmp[0]); 1670 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1671 &src_iovs[0], 1, NULL, NULL, 1672 ut_sequence_step_cb, &completed); 1673 CU_ASSERT_EQUAL(rc, 0); 1674 1675 dst_iovs[1].iov_base = tmp[2]; 1676 dst_iovs[1].iov_len = 2048; 1677 src_iovs[1].iov_base = tmp[1]; 1678 src_iovs[1].iov_len = sizeof(tmp[1]); 1679 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1680 &src_iovs[1], 1, NULL, NULL, 1681 ut_sequence_step_cb, &completed); 1682 CU_ASSERT_EQUAL(rc, 0); 1683 1684 dst_iovs[2].iov_base = buf; 1685 dst_iovs[2].iov_len = 2048; 1686 src_iovs[2].iov_base = tmp[2]; 1687 src_iovs[2].iov_len = 2048; 1688 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1689 &src_iovs[2], 1, NULL, NULL, 1690 ut_sequence_step_cb, &completed); 1691 CU_ASSERT_EQUAL(rc, 0); 1692 1693 ut_seq.complete = false; 1694 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1695 1696 poll_threads(); 1697 1698 CU_ASSERT_EQUAL(completed, 3); 1699 CU_ASSERT(ut_seq.complete); 1700 CU_ASSERT_EQUAL(ut_seq.status, 0); 1701 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1702 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1703 1704 /* Check decompress + copy + decompress + copy */ 1705 seq = NULL; 1706 completed = 0; 1707 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1708 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1709 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = NULL; 1710 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = NULL; 1711 1712 dst_iovs[0].iov_base = tmp[1]; 1713 dst_iovs[0].iov_len = sizeof(tmp[1]); 1714 src_iovs[0].iov_base = tmp[0]; 1715 src_iovs[0].iov_len = sizeof(tmp[0]); 1716 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1717 &src_iovs[0], 1, NULL, NULL, 1718 ut_sequence_step_cb, &completed); 1719 CU_ASSERT_EQUAL(rc, 0); 1720 1721 dst_iovs[1].iov_base = tmp[2]; 1722 dst_iovs[1].iov_len = 2048; 1723 src_iovs[1].iov_base = tmp[1]; 1724 src_iovs[1].iov_len = sizeof(tmp[1]); 1725 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1726 &src_iovs[1], 1, NULL, NULL, 1727 ut_sequence_step_cb, &completed); 1728 CU_ASSERT_EQUAL(rc, 0); 1729 1730 dst_iovs[2].iov_base = tmp[3]; 1731 dst_iovs[2].iov_len = 1024; 1732 src_iovs[2].iov_base = tmp[2]; 1733 src_iovs[2].iov_len = 2048; 1734 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1735 &src_iovs[2], 1, NULL, NULL, 1736 ut_sequence_step_cb, &completed); 1737 CU_ASSERT_EQUAL(rc, 0); 1738 1739 dst_iovs[3].iov_base = buf; 1740 dst_iovs[3].iov_len = 1024; 1741 src_iovs[3].iov_base = tmp[3]; 1742 src_iovs[3].iov_len = 1024; 1743 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 1744 &src_iovs[3], 1, NULL, NULL, 1745 ut_sequence_step_cb, &completed); 1746 CU_ASSERT_EQUAL(rc, 0); 1747 1748 ut_seq.complete = false; 1749 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1750 1751 poll_threads(); 1752 1753 CU_ASSERT_EQUAL(completed, 4); 1754 CU_ASSERT(ut_seq.complete); 1755 CU_ASSERT_EQUAL(ut_seq.status, 0); 1756 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1757 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 2); 1758 1759 /* Check two copy operations - one of them should be removed, while the other should be 1760 * executed normally */ 1761 seq = NULL; 1762 completed = 0; 1763 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1764 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovcnt = 1; 1765 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovcnt = 1; 1766 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovs = &exp_iovs[0]; 1767 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovs = &exp_iovs[1]; 1768 exp_iovs[0].iov_base = tmp[0]; 1769 exp_iovs[0].iov_len = sizeof(tmp[0]); 1770 exp_iovs[1].iov_base = buf; 1771 exp_iovs[1].iov_len = sizeof(buf); 1772 1773 dst_iovs[0].iov_base = tmp[1]; 1774 dst_iovs[0].iov_len = sizeof(tmp[1]); 1775 src_iovs[0].iov_base = tmp[0]; 1776 src_iovs[0].iov_len = sizeof(tmp[0]); 1777 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1778 &src_iovs[0], 1, NULL, NULL, 1779 ut_sequence_step_cb, &completed); 1780 CU_ASSERT_EQUAL(rc, 0); 1781 1782 dst_iovs[1].iov_base = buf; 1783 dst_iovs[1].iov_len = sizeof(buf); 1784 src_iovs[1].iov_base = tmp[1]; 1785 src_iovs[1].iov_len = sizeof(tmp[1]); 1786 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1787 &src_iovs[1], 1, NULL, NULL, 1788 ut_sequence_step_cb, &completed); 1789 CU_ASSERT_EQUAL(rc, 0); 1790 1791 ut_seq.complete = false; 1792 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1793 1794 poll_threads(); 1795 1796 CU_ASSERT_EQUAL(completed, 2); 1797 CU_ASSERT(ut_seq.complete); 1798 CU_ASSERT_EQUAL(ut_seq.status, 0); 1799 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 1800 1801 /* Check fill + copy */ 1802 seq = NULL; 1803 completed = 0; 1804 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1805 g_seq_operations[SPDK_ACCEL_OPC_FILL].count = 0; 1806 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovs = NULL; 1807 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovs = NULL; 1808 g_seq_operations[SPDK_ACCEL_OPC_FILL].dst_iovcnt = 1; 1809 g_seq_operations[SPDK_ACCEL_OPC_FILL].dst_iovs = &exp_iovs[0]; 1810 exp_iovs[0].iov_base = buf; 1811 exp_iovs[0].iov_len = sizeof(buf); 1812 1813 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 1814 ut_sequence_step_cb, &completed); 1815 CU_ASSERT_EQUAL(rc, 0); 1816 1817 dst_iovs[0].iov_base = buf; 1818 dst_iovs[0].iov_len = sizeof(buf); 1819 src_iovs[0].iov_base = tmp[0]; 1820 src_iovs[0].iov_len = sizeof(tmp[0]); 1821 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1822 &src_iovs[0], 1, NULL, NULL, 1823 ut_sequence_step_cb, &completed); 1824 CU_ASSERT_EQUAL(rc, 0); 1825 1826 ut_seq.complete = false; 1827 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1828 1829 poll_threads(); 1830 1831 CU_ASSERT_EQUAL(completed, 2); 1832 CU_ASSERT(ut_seq.complete); 1833 CU_ASSERT_EQUAL(ut_seq.status, 0); 1834 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1835 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 1); 1836 1837 /* Check copy + encrypt + copy */ 1838 seq = NULL; 1839 completed = 0; 1840 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1841 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 1842 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovcnt = 1; 1843 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovcnt = 1; 1844 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovs = &exp_iovs[0]; 1845 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovs = &exp_iovs[1]; 1846 exp_iovs[0].iov_base = tmp[0]; 1847 exp_iovs[0].iov_len = sizeof(tmp[0]); 1848 exp_iovs[1].iov_base = buf; 1849 exp_iovs[1].iov_len = sizeof(buf); 1850 1851 dst_iovs[0].iov_base = tmp[1]; 1852 dst_iovs[0].iov_len = sizeof(tmp[1]); 1853 src_iovs[0].iov_base = tmp[0]; 1854 src_iovs[0].iov_len = sizeof(tmp[0]); 1855 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1856 &src_iovs[0], 1, NULL, NULL, 1857 ut_sequence_step_cb, &completed); 1858 CU_ASSERT_EQUAL(rc, 0); 1859 1860 dst_iovs[1].iov_base = tmp[2]; 1861 dst_iovs[1].iov_len = sizeof(tmp[2]); 1862 src_iovs[1].iov_base = tmp[1]; 1863 src_iovs[1].iov_len = sizeof(tmp[1]); 1864 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1865 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 1866 ut_sequence_step_cb, &completed); 1867 CU_ASSERT_EQUAL(rc, 0); 1868 1869 dst_iovs[2].iov_base = buf; 1870 dst_iovs[2].iov_len = sizeof(buf); 1871 src_iovs[2].iov_base = tmp[2]; 1872 src_iovs[2].iov_len = sizeof(tmp[2]); 1873 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1874 &src_iovs[2], 1, NULL, NULL, 1875 ut_sequence_step_cb, &completed); 1876 CU_ASSERT_EQUAL(rc, 0); 1877 1878 ut_seq.complete = false; 1879 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1880 1881 poll_threads(); 1882 1883 CU_ASSERT_EQUAL(completed, 3); 1884 CU_ASSERT(ut_seq.complete); 1885 CU_ASSERT_EQUAL(ut_seq.status, 0); 1886 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1887 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 1888 1889 /* Check copy + decrypt + copy */ 1890 seq = NULL; 1891 completed = 0; 1892 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1893 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 1894 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1895 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1896 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1897 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1898 exp_iovs[0].iov_base = tmp[0]; 1899 exp_iovs[0].iov_len = sizeof(tmp[0]); 1900 exp_iovs[1].iov_base = buf; 1901 exp_iovs[1].iov_len = sizeof(buf); 1902 1903 dst_iovs[0].iov_base = tmp[1]; 1904 dst_iovs[0].iov_len = sizeof(tmp[1]); 1905 src_iovs[0].iov_base = tmp[0]; 1906 src_iovs[0].iov_len = sizeof(tmp[0]); 1907 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1908 &src_iovs[0], 1, NULL, NULL, 1909 ut_sequence_step_cb, &completed); 1910 CU_ASSERT_EQUAL(rc, 0); 1911 1912 dst_iovs[1].iov_base = tmp[2]; 1913 dst_iovs[1].iov_len = sizeof(tmp[2]); 1914 src_iovs[1].iov_base = tmp[1]; 1915 src_iovs[1].iov_len = sizeof(tmp[1]); 1916 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1917 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 1918 ut_sequence_step_cb, &completed); 1919 CU_ASSERT_EQUAL(rc, 0); 1920 1921 dst_iovs[2].iov_base = buf; 1922 dst_iovs[2].iov_len = sizeof(buf); 1923 src_iovs[2].iov_base = tmp[2]; 1924 src_iovs[2].iov_len = sizeof(tmp[2]); 1925 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1926 &src_iovs[2], 1, NULL, NULL, 1927 ut_sequence_step_cb, &completed); 1928 CU_ASSERT_EQUAL(rc, 0); 1929 1930 ut_seq.complete = false; 1931 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1932 1933 poll_threads(); 1934 1935 CU_ASSERT_EQUAL(completed, 3); 1936 CU_ASSERT(ut_seq.complete); 1937 CU_ASSERT_EQUAL(ut_seq.status, 0); 1938 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1939 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 1940 1941 /* Check a sequence with memory domains and verify their pointers (and their contexts) are 1942 * correctly set on the next/prev task when a copy is removed */ 1943 seq = NULL; 1944 completed = 0; 1945 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1946 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 1947 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1948 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1949 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1950 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1951 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_domain = (void *)0x1; 1952 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_domain_ctx = (void *)0x2; 1953 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_domain = (void *)0x3; 1954 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_domain_ctx = (void *)0x4; 1955 exp_iovs[0].iov_base = tmp[0]; 1956 exp_iovs[0].iov_len = sizeof(tmp[0]); 1957 exp_iovs[1].iov_base = buf; 1958 exp_iovs[1].iov_len = sizeof(buf); 1959 1960 dst_iovs[0].iov_base = tmp[1]; 1961 dst_iovs[0].iov_len = sizeof(tmp[1]); 1962 src_iovs[0].iov_base = tmp[0]; 1963 src_iovs[0].iov_len = sizeof(tmp[0]); 1964 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, (void *)0xdead, (void *)0xbeef, 1965 &src_iovs[0], 1, (void *)0x3, (void *)0x4, 1966 ut_sequence_step_cb, &completed); 1967 CU_ASSERT_EQUAL(rc, 0); 1968 1969 dst_iovs[1].iov_base = tmp[2]; 1970 dst_iovs[1].iov_len = sizeof(tmp[2]); 1971 src_iovs[1].iov_base = tmp[1]; 1972 src_iovs[1].iov_len = sizeof(tmp[1]); 1973 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 1974 &src_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 0, 1975 sizeof(tmp[2]), ut_sequence_step_cb, &completed); 1976 CU_ASSERT_EQUAL(rc, 0); 1977 1978 dst_iovs[2].iov_base = buf; 1979 dst_iovs[2].iov_len = sizeof(buf); 1980 src_iovs[2].iov_base = tmp[2]; 1981 src_iovs[2].iov_len = sizeof(tmp[2]); 1982 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, (void *)0x1, (void *)0x2, 1983 &src_iovs[2], 1, (void *)0xdead, (void *)0xbeef, 1984 ut_sequence_step_cb, &completed); 1985 CU_ASSERT_EQUAL(rc, 0); 1986 1987 ut_seq.complete = false; 1988 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1989 1990 poll_threads(); 1991 1992 CU_ASSERT_EQUAL(completed, 3); 1993 CU_ASSERT(ut_seq.complete); 1994 CU_ASSERT_EQUAL(ut_seq.status, 0); 1995 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1996 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 1997 1998 /* Cleanup module pointers to make subsequent tests work correctly */ 1999 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2000 g_modules_opc[i] = modules[i]; 2001 } 2002 2003 g_module.supports_memory_domains = false; 2004 ut_clear_operations(); 2005 spdk_put_io_channel(ioch); 2006 poll_threads(); 2007 } 2008 2009 static int 2010 ut_submit_decompress(struct spdk_io_channel *ch, struct spdk_accel_task *task) 2011 { 2012 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 2013 2014 spdk_accel_task_complete(task, 0); 2015 2016 return 0; 2017 } 2018 2019 static void 2020 test_sequence_accel_buffers(void) 2021 { 2022 struct spdk_accel_sequence *seq = NULL; 2023 struct spdk_io_channel *ioch; 2024 struct accel_io_channel *accel_ch; 2025 struct ut_sequence ut_seq; 2026 struct iovec src_iovs[3], dst_iovs[3]; 2027 char srcbuf[4096], dstbuf[4096], expected[4096]; 2028 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2029 void *buf[2], *domain_ctx[2], *iobuf_buf; 2030 struct spdk_memory_domain *domain[2]; 2031 struct spdk_iobuf_buffer *cache_entry; 2032 struct spdk_iobuf_pool_cache *small; 2033 spdk_iobuf_buffer_stailq_t small_cache; 2034 uint32_t small_cache_count; 2035 int i, rc, completed; 2036 struct spdk_iobuf_opts opts_iobuf = {}; 2037 2038 /* Set up the iobuf to always use the "small" pool */ 2039 opts_iobuf.large_bufsize = 0x20000; 2040 opts_iobuf.large_pool_count = 0; 2041 opts_iobuf.small_bufsize = 0x10000; 2042 opts_iobuf.small_pool_count = 32; 2043 2044 rc = spdk_iobuf_set_opts(&opts_iobuf); 2045 CU_ASSERT(rc == 0); 2046 2047 ioch = spdk_accel_get_io_channel(); 2048 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2049 2050 /* Override the submit_tasks function */ 2051 g_module_if.submit_tasks = ut_sequence_submit_tasks; 2052 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2053 modules[i] = g_modules_opc[i]; 2054 g_modules_opc[i] = g_module; 2055 } 2056 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2057 * decompress operations together in one sequence. 2058 */ 2059 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2060 g_seq_operations[SPDK_ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2061 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2062 2063 /* Check the simplest case: one operation using accel buffer as destination + copy operation 2064 * specifying the actual destination buffer 2065 */ 2066 memset(srcbuf, 0xa5, 4096); 2067 memset(dstbuf, 0x0, 4096); 2068 memset(expected, 0xa5, 4096); 2069 completed = 0; 2070 seq = NULL; 2071 2072 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2073 CU_ASSERT_EQUAL(rc, 0); 2074 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2075 2076 src_iovs[0].iov_base = srcbuf; 2077 src_iovs[0].iov_len = 4096; 2078 dst_iovs[0].iov_base = buf[0]; 2079 dst_iovs[0].iov_len = 4096; 2080 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2081 &src_iovs[0], 1, NULL, NULL, 2082 ut_sequence_step_cb, &completed); 2083 CU_ASSERT_EQUAL(rc, 0); 2084 2085 src_iovs[1].iov_base = buf[0]; 2086 src_iovs[1].iov_len = 4096; 2087 dst_iovs[1].iov_base = dstbuf; 2088 dst_iovs[1].iov_len = 4096; 2089 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2090 &src_iovs[1], 1, domain[0], domain_ctx[0], 2091 ut_sequence_step_cb, &completed); 2092 CU_ASSERT_EQUAL(rc, 0); 2093 2094 ut_seq.complete = false; 2095 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2096 2097 poll_threads(); 2098 2099 CU_ASSERT_EQUAL(completed, 2); 2100 CU_ASSERT(ut_seq.complete); 2101 CU_ASSERT_EQUAL(ut_seq.status, 0); 2102 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2103 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2104 2105 /* Start with a fill operation using accel buffer, followed by a decompress using another 2106 * accel buffer as dst, followed by a copy operation specifying dst buffer of the whole 2107 * sequence 2108 */ 2109 memset(srcbuf, 0x0, 4096); 2110 memset(dstbuf, 0x0, 4096); 2111 memset(expected, 0x5a, 4096); 2112 completed = 0; 2113 seq = NULL; 2114 2115 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2116 CU_ASSERT_EQUAL(rc, 0); 2117 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2118 2119 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 2120 ut_sequence_step_cb, &completed); 2121 CU_ASSERT_EQUAL(rc, 0); 2122 2123 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2124 CU_ASSERT_EQUAL(rc, 0); 2125 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2126 2127 src_iovs[0].iov_base = buf[0]; 2128 src_iovs[0].iov_len = 4096; 2129 dst_iovs[0].iov_base = buf[1]; 2130 dst_iovs[0].iov_len = 4096; 2131 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2132 &src_iovs[0], 1, domain[0], domain_ctx[0], 2133 ut_sequence_step_cb, &completed); 2134 CU_ASSERT_EQUAL(rc, 0); 2135 2136 src_iovs[1].iov_base = buf[1]; 2137 src_iovs[1].iov_len = 4096; 2138 dst_iovs[1].iov_base = dstbuf; 2139 dst_iovs[1].iov_len = 4096; 2140 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2141 &src_iovs[1], 1, domain[1], domain_ctx[1], 2142 ut_sequence_step_cb, &completed); 2143 2144 ut_seq.complete = false; 2145 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2146 2147 poll_threads(); 2148 2149 CU_ASSERT_EQUAL(completed, 3); 2150 CU_ASSERT(ut_seq.complete); 2151 CU_ASSERT_EQUAL(ut_seq.status, 0); 2152 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2153 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2154 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2155 2156 /* Check the same, but with two decompress operations with the first one being in-place */ 2157 memset(srcbuf, 0x0, 4096); 2158 memset(dstbuf, 0x0, 4096); 2159 memset(expected, 0x5a, 4096); 2160 completed = 0; 2161 seq = NULL; 2162 2163 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2164 CU_ASSERT_EQUAL(rc, 0); 2165 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2166 2167 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 2168 ut_sequence_step_cb, &completed); 2169 CU_ASSERT_EQUAL(rc, 0); 2170 2171 src_iovs[0].iov_base = buf[0]; 2172 src_iovs[0].iov_len = 4096; 2173 dst_iovs[0].iov_base = buf[0]; 2174 dst_iovs[0].iov_len = 4096; 2175 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2176 &src_iovs[0], 1, domain[0], domain_ctx[0], 2177 ut_sequence_step_cb, &completed); 2178 CU_ASSERT_EQUAL(rc, 0); 2179 2180 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2181 CU_ASSERT_EQUAL(rc, 0); 2182 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2183 2184 src_iovs[1].iov_base = buf[0]; 2185 src_iovs[1].iov_len = 4096; 2186 dst_iovs[1].iov_base = buf[1]; 2187 dst_iovs[1].iov_len = 4096; 2188 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, domain[1], domain_ctx[1], 2189 &src_iovs[1], 1, domain[0], domain_ctx[0], 2190 ut_sequence_step_cb, &completed); 2191 CU_ASSERT_EQUAL(rc, 0); 2192 2193 src_iovs[2].iov_base = buf[1]; 2194 src_iovs[2].iov_len = 4096; 2195 dst_iovs[2].iov_base = dstbuf; 2196 dst_iovs[2].iov_len = 4096; 2197 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 2198 &src_iovs[2], 1, domain[1], domain_ctx[1], 2199 ut_sequence_step_cb, &completed); 2200 2201 ut_seq.complete = false; 2202 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2203 2204 poll_threads(); 2205 2206 CU_ASSERT_EQUAL(completed, 4); 2207 CU_ASSERT(ut_seq.complete); 2208 CU_ASSERT_EQUAL(ut_seq.status, 0); 2209 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2210 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2211 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2212 2213 /* Check that specifying offsets within accel buffers works correctly */ 2214 memset(srcbuf, 0x0, 4096); 2215 memset(dstbuf, 0x0, 4096); 2216 completed = 0; 2217 seq = NULL; 2218 2219 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2220 CU_ASSERT_EQUAL(rc, 0); 2221 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2222 2223 /* Fill in each 1K of the buffer with different pattern */ 2224 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 2225 ut_sequence_step_cb, &completed); 2226 CU_ASSERT_EQUAL(rc, 0); 2227 2228 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2229 0x5a, ut_sequence_step_cb, &completed); 2230 CU_ASSERT_EQUAL(rc, 0); 2231 2232 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2233 0xfe, ut_sequence_step_cb, &completed); 2234 CU_ASSERT_EQUAL(rc, 0); 2235 2236 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2237 0xed, ut_sequence_step_cb, &completed); 2238 CU_ASSERT_EQUAL(rc, 0); 2239 2240 src_iovs[0].iov_base = buf[0]; 2241 src_iovs[0].iov_len = 4096; 2242 dst_iovs[0].iov_base = dstbuf; 2243 dst_iovs[0].iov_len = 4096; 2244 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2245 &src_iovs[0], 1, domain[0], domain_ctx[0], 2246 ut_sequence_step_cb, &completed); 2247 2248 ut_seq.complete = false; 2249 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2250 2251 poll_threads(); 2252 2253 memset(&expected[0], 0xa5, 1024); 2254 memset(&expected[1024], 0x5a, 1024); 2255 memset(&expected[2048], 0xfe, 1024); 2256 memset(&expected[3072], 0xed, 1024); 2257 CU_ASSERT_EQUAL(completed, 5); 2258 CU_ASSERT(ut_seq.complete); 2259 CU_ASSERT_EQUAL(ut_seq.status, 0); 2260 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2261 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2262 2263 /* Check the same but this time with a decompress operation on part of the buffer (512B 2264 * offset) */ 2265 memset(srcbuf, 0x0, 4096); 2266 memset(dstbuf, 0x0, 4096); 2267 completed = 0; 2268 seq = NULL; 2269 2270 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2271 CU_ASSERT_EQUAL(rc, 0); 2272 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2273 2274 /* Fill in each 1K of the buffer with different pattern */ 2275 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 2276 ut_sequence_step_cb, &completed); 2277 CU_ASSERT_EQUAL(rc, 0); 2278 2279 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2280 0x5a, ut_sequence_step_cb, &completed); 2281 CU_ASSERT_EQUAL(rc, 0); 2282 2283 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2284 0xfe, ut_sequence_step_cb, &completed); 2285 CU_ASSERT_EQUAL(rc, 0); 2286 2287 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2288 0xed, ut_sequence_step_cb, &completed); 2289 CU_ASSERT_EQUAL(rc, 0); 2290 2291 rc = spdk_accel_get_buf(ioch, 3072, &buf[1], &domain[1], &domain_ctx[1]); 2292 CU_ASSERT_EQUAL(rc, 0); 2293 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2294 2295 src_iovs[0].iov_base = (char *)buf[0] + 512; 2296 src_iovs[0].iov_len = 3072; 2297 dst_iovs[0].iov_base = (char *)buf[1] + 256; 2298 dst_iovs[0].iov_len = 3072; 2299 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2300 &src_iovs[0], 1, domain[0], domain_ctx[0], 2301 ut_sequence_step_cb, &completed); 2302 2303 src_iovs[1].iov_base = (char *)buf[1] + 256; 2304 src_iovs[1].iov_len = 3072; 2305 dst_iovs[1].iov_base = dstbuf; 2306 dst_iovs[1].iov_len = 3072; 2307 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2308 &src_iovs[1], 1, domain[1], domain_ctx[1], 2309 ut_sequence_step_cb, &completed); 2310 2311 ut_seq.complete = false; 2312 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2313 2314 poll_threads(); 2315 2316 memset(&expected[0], 0xa5, 512); 2317 memset(&expected[512], 0x5a, 1024); 2318 memset(&expected[1536], 0xfe, 1024); 2319 memset(&expected[2560], 0xed, 512); 2320 CU_ASSERT_EQUAL(completed, 6); 2321 CU_ASSERT(ut_seq.complete); 2322 CU_ASSERT_EQUAL(ut_seq.status, 0); 2323 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 3072), 0); 2324 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2325 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2326 2327 /* Check that if iobuf pool is empty, the sequence processing will wait until a buffer is 2328 * available 2329 */ 2330 accel_ch = spdk_io_channel_get_ctx(ioch); 2331 small = &accel_ch->iobuf.cache[0].small; 2332 small_cache_count = small->cache_count; 2333 STAILQ_INIT(&small_cache); 2334 STAILQ_SWAP(&small->cache, &small_cache, spdk_iobuf_buffer); 2335 small->cache_count = 0; 2336 small->cache_size = 0; 2337 g_iobuf.small_pool_count = 0; 2338 2339 /* First allocate a single buffer used by two operations */ 2340 memset(srcbuf, 0x0, 4096); 2341 memset(dstbuf, 0x0, 4096); 2342 memset(expected, 0xa5, 4096); 2343 completed = 0; 2344 seq = NULL; 2345 2346 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2347 CU_ASSERT_EQUAL(rc, 0); 2348 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2349 2350 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0xa5, 2351 ut_sequence_step_cb, &completed); 2352 CU_ASSERT_EQUAL(rc, 0); 2353 2354 src_iovs[0].iov_base = buf[0]; 2355 src_iovs[0].iov_len = 4096; 2356 dst_iovs[0].iov_base = dstbuf; 2357 dst_iovs[0].iov_len = 4096; 2358 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2359 &src_iovs[0], 1, domain[0], domain_ctx[0], 2360 ut_sequence_step_cb, &completed); 2361 2362 ut_seq.complete = false; 2363 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2364 2365 poll_threads(); 2366 2367 CU_ASSERT_EQUAL(completed, 0); 2368 CU_ASSERT(!ut_seq.complete); 2369 2370 /* Get a buffer and return it to the pool to trigger the sequence to finish */ 2371 g_iobuf.small_pool_count = 1; 2372 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2373 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2374 CU_ASSERT(g_iobuf.small_pool_count == 0); 2375 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2376 2377 poll_threads(); 2378 2379 CU_ASSERT_EQUAL(completed, 2); 2380 CU_ASSERT(ut_seq.complete); 2381 CU_ASSERT_EQUAL(ut_seq.status, 0); 2382 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2383 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2384 2385 /* Return the buffers back to the cache */ 2386 while (!STAILQ_EMPTY(&small->cache)) { 2387 cache_entry = STAILQ_FIRST(&small->cache); 2388 STAILQ_REMOVE_HEAD(&small->cache, stailq); 2389 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2390 small_cache_count++; 2391 } 2392 small->cache_count = 0; 2393 g_iobuf.small_pool_count = 0; 2394 2395 /* Check a bit more complex scenario, with two buffers in the sequence */ 2396 memset(srcbuf, 0x0, 4096); 2397 memset(dstbuf, 0x0, 4096); 2398 memset(expected, 0x5a, 4096); 2399 completed = 0; 2400 seq = NULL; 2401 2402 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2403 CU_ASSERT_EQUAL(rc, 0); 2404 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2405 2406 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 2407 ut_sequence_step_cb, &completed); 2408 CU_ASSERT_EQUAL(rc, 0); 2409 2410 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2411 CU_ASSERT_EQUAL(rc, 0); 2412 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2413 2414 src_iovs[0].iov_base = buf[0]; 2415 src_iovs[0].iov_len = 4096; 2416 dst_iovs[0].iov_base = buf[1]; 2417 dst_iovs[0].iov_len = 4096; 2418 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2419 &src_iovs[0], 1, domain[0], domain_ctx[0], 2420 ut_sequence_step_cb, &completed); 2421 CU_ASSERT_EQUAL(rc, 0); 2422 2423 src_iovs[1].iov_base = buf[1]; 2424 src_iovs[1].iov_len = 4096; 2425 dst_iovs[1].iov_base = dstbuf; 2426 dst_iovs[1].iov_len = 4096; 2427 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2428 &src_iovs[1], 1, domain[1], domain_ctx[1], 2429 ut_sequence_step_cb, &completed); 2430 CU_ASSERT_EQUAL(rc, 0); 2431 2432 ut_seq.complete = false; 2433 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2434 2435 poll_threads(); 2436 2437 CU_ASSERT_EQUAL(completed, 0); 2438 CU_ASSERT(!ut_seq.complete); 2439 2440 g_iobuf.small_pool_count = 1; 2441 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2442 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2443 g_iobuf.small_pool_count = 0; 2444 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2445 2446 /* One buffer is not enough to finish the whole sequence */ 2447 poll_threads(); 2448 CU_ASSERT(!ut_seq.complete); 2449 2450 g_iobuf.small_pool_count = 1; 2451 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2452 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2453 g_iobuf.small_pool_count = 0; 2454 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2455 2456 poll_threads(); 2457 2458 CU_ASSERT_EQUAL(completed, 3); 2459 CU_ASSERT(ut_seq.complete); 2460 CU_ASSERT_EQUAL(ut_seq.status, 0); 2461 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2462 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2463 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2464 2465 /* Return the buffers back to the cache */ 2466 while (!STAILQ_EMPTY(&small->cache)) { 2467 cache_entry = STAILQ_FIRST(&small->cache); 2468 STAILQ_REMOVE_HEAD(&small->cache, stailq); 2469 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2470 small_cache_count++; 2471 } 2472 small->cache_count = 0; 2473 2474 g_iobuf.small_pool_count = 32; 2475 STAILQ_SWAP(&small->cache, &small_cache, spdk_iobuf_buffer); 2476 small->cache_count = small_cache_count; 2477 2478 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2479 g_modules_opc[i] = modules[i]; 2480 } 2481 2482 ut_clear_operations(); 2483 spdk_put_io_channel(ioch); 2484 poll_threads(); 2485 } 2486 2487 static void 2488 ut_domain_ctx_init(struct ut_domain_ctx *ctx, void *base, size_t len, struct iovec *expected) 2489 { 2490 ctx->iov.iov_base = base; 2491 ctx->iov.iov_len = len; 2492 ctx->expected = *expected; 2493 ctx->pull_submit_status = 0; 2494 ctx->push_submit_status = 0; 2495 ctx->pull_complete_status = 0; 2496 ctx->push_complete_status = 0; 2497 } 2498 2499 static void 2500 test_sequence_memory_domain(void) 2501 { 2502 struct spdk_accel_sequence *seq = NULL; 2503 struct spdk_io_channel *ioch; 2504 struct accel_io_channel *accel_ch; 2505 struct ut_sequence ut_seq; 2506 struct ut_domain_ctx domctx[4]; 2507 struct spdk_iobuf_buffer *cache_entry; 2508 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2509 struct spdk_memory_domain *accel_domain; 2510 struct spdk_iobuf_pool_cache *small; 2511 spdk_iobuf_buffer_stailq_t small_cache; 2512 char srcbuf[4096], dstbuf[4096], expected[4096], tmp[4096]; 2513 struct iovec src_iovs[2], dst_iovs[2]; 2514 uint32_t small_cache_count; 2515 void *accel_buf, *accel_domain_ctx, *iobuf_buf; 2516 int i, rc, completed; 2517 2518 ioch = spdk_accel_get_io_channel(); 2519 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2520 2521 /* Override the submit_tasks function */ 2522 g_module_if.submit_tasks = ut_sequence_submit_tasks; 2523 g_module.supports_memory_domains = false; 2524 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2525 modules[i] = g_modules_opc[i]; 2526 g_modules_opc[i] = g_module; 2527 } 2528 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2529 * decompress operations together in one sequence. 2530 */ 2531 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2532 g_seq_operations[SPDK_ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2533 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2534 2535 /* First check the simplest case - single fill operation with dstbuf in domain */ 2536 memset(expected, 0xa5, sizeof(expected)); 2537 memset(dstbuf, 0x0, sizeof(dstbuf)); 2538 completed = 0; 2539 seq = NULL; 2540 2541 /* Use some garbage pointer as dst and use domain ctx to get the actual dstbuf */ 2542 dst_iovs[0].iov_base = (void *)0xcafebabe; 2543 dst_iovs[0].iov_len = sizeof(dstbuf); 2544 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2545 2546 rc = spdk_accel_append_fill(&seq, ioch, dst_iovs[0].iov_base, dst_iovs[0].iov_len, 2547 g_ut_domain, &domctx[0], 0xa5, 2548 ut_sequence_step_cb, &completed); 2549 CU_ASSERT_EQUAL(rc, 0); 2550 2551 ut_seq.complete = false; 2552 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2553 2554 poll_threads(); 2555 CU_ASSERT_EQUAL(completed, 1); 2556 CU_ASSERT(ut_seq.complete); 2557 CU_ASSERT_EQUAL(ut_seq.status, 0); 2558 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2559 2560 /* Check operation with both srcbuf and dstbuf in remote memory domain */ 2561 memset(expected, 0x5a, sizeof(expected)); 2562 memset(dstbuf, 0x0, sizeof(dstbuf)); 2563 memset(srcbuf, 0x5a, sizeof(dstbuf)); 2564 completed = 0; 2565 seq = NULL; 2566 2567 src_iovs[0].iov_base = (void *)0xcafebabe; 2568 src_iovs[0].iov_len = sizeof(srcbuf); 2569 dst_iovs[0].iov_base = (void *)0xbeefdead; 2570 dst_iovs[0].iov_len = sizeof(dstbuf); 2571 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2572 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2573 2574 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2575 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2576 ut_sequence_step_cb, &completed); 2577 CU_ASSERT_EQUAL(rc, 0); 2578 2579 ut_seq.complete = false; 2580 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2581 2582 poll_threads(); 2583 CU_ASSERT_EQUAL(completed, 1); 2584 CU_ASSERT(ut_seq.complete); 2585 CU_ASSERT_EQUAL(ut_seq.status, 0); 2586 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2587 2588 /* Two operations using a buffer in remote domain */ 2589 memset(expected, 0xa5, sizeof(expected)); 2590 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2591 memset(tmp, 0x0, sizeof(tmp)); 2592 memset(dstbuf, 0x0, sizeof(dstbuf)); 2593 completed = 0; 2594 seq = NULL; 2595 2596 src_iovs[0].iov_base = (void *)0xcafebabe; 2597 src_iovs[0].iov_len = sizeof(srcbuf); 2598 dst_iovs[0].iov_base = (void *)0xbeefdead; 2599 dst_iovs[0].iov_len = sizeof(tmp); 2600 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2601 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2602 2603 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2604 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2605 ut_sequence_step_cb, &completed); 2606 CU_ASSERT_EQUAL(rc, 0); 2607 2608 src_iovs[1].iov_base = (void *)0xbeefdead; 2609 src_iovs[1].iov_len = sizeof(tmp); 2610 dst_iovs[1].iov_base = (void *)0xa5a5a5a5; 2611 dst_iovs[1].iov_len = sizeof(dstbuf); 2612 ut_domain_ctx_init(&domctx[2], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2613 ut_domain_ctx_init(&domctx[3], tmp, sizeof(tmp), &src_iovs[1]); 2614 2615 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[2], 2616 &src_iovs[1], 1, g_ut_domain, &domctx[3], 2617 ut_sequence_step_cb, &completed); 2618 CU_ASSERT_EQUAL(rc, 0); 2619 2620 ut_seq.complete = false; 2621 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2622 2623 poll_threads(); 2624 CU_ASSERT_EQUAL(completed, 2); 2625 CU_ASSERT(ut_seq.complete); 2626 CU_ASSERT_EQUAL(ut_seq.status, 0); 2627 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2628 2629 /* Use buffer in remote memory domain and accel buffer at the same time */ 2630 memset(expected, 0x5a, sizeof(expected)); 2631 memset(srcbuf, 0x5a, sizeof(expected)); 2632 memset(dstbuf, 0x0, sizeof(dstbuf)); 2633 completed = 0; 2634 seq = NULL; 2635 2636 rc = spdk_accel_get_buf(ioch, sizeof(dstbuf), &accel_buf, &accel_domain, &accel_domain_ctx); 2637 CU_ASSERT_EQUAL(rc, 0); 2638 2639 src_iovs[0].iov_base = (void *)0xfeedbeef; 2640 src_iovs[0].iov_len = sizeof(srcbuf); 2641 dst_iovs[0].iov_base = accel_buf; 2642 dst_iovs[0].iov_len = sizeof(dstbuf); 2643 ut_domain_ctx_init(&domctx[0], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2644 2645 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, accel_domain, accel_domain_ctx, 2646 &src_iovs[0], 1, g_ut_domain, &domctx[0], 2647 ut_sequence_step_cb, &completed); 2648 CU_ASSERT_EQUAL(rc, 0); 2649 2650 src_iovs[1].iov_base = accel_buf; 2651 src_iovs[1].iov_len = sizeof(dstbuf); 2652 dst_iovs[1].iov_base = (void *)0xbeeffeed; 2653 dst_iovs[1].iov_len = sizeof(dstbuf); 2654 ut_domain_ctx_init(&domctx[1], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2655 2656 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[1], 2657 &src_iovs[1], 1, accel_domain, accel_domain_ctx, 2658 ut_sequence_step_cb, &completed); 2659 CU_ASSERT_EQUAL(rc, 0); 2660 2661 ut_seq.complete = false; 2662 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2663 2664 poll_threads(); 2665 CU_ASSERT_EQUAL(completed, 2); 2666 CU_ASSERT(ut_seq.complete); 2667 CU_ASSERT_EQUAL(ut_seq.status, 0); 2668 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2669 spdk_accel_put_buf(ioch, accel_buf, accel_domain, accel_domain_ctx); 2670 2671 /* Check that a sequence with memory domains is correctly executed if buffers are not 2672 * immediately available */ 2673 memset(expected, 0xa5, sizeof(expected)); 2674 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2675 memset(dstbuf, 0x0, sizeof(dstbuf)); 2676 completed = 0; 2677 seq = NULL; 2678 /* Make sure the buffer pool is empty */ 2679 accel_ch = spdk_io_channel_get_ctx(ioch); 2680 small = &accel_ch->iobuf.cache[0].small; 2681 small_cache_count = small->cache_count; 2682 STAILQ_INIT(&small_cache); 2683 STAILQ_SWAP(&small->cache, &small_cache, spdk_iobuf_buffer); 2684 small->cache_count = 0; 2685 g_iobuf.small_pool_count = 0; 2686 2687 src_iovs[0].iov_base = (void *)0xdeadbeef; 2688 src_iovs[0].iov_len = sizeof(srcbuf); 2689 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2690 dst_iovs[0].iov_len = sizeof(dstbuf); 2691 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2692 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2693 2694 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2695 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2696 ut_sequence_step_cb, &completed); 2697 CU_ASSERT_EQUAL(rc, 0); 2698 2699 ut_seq.complete = false; 2700 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2701 2702 poll_threads(); 2703 CU_ASSERT_EQUAL(completed, 0); 2704 CU_ASSERT(!ut_seq.complete); 2705 2706 /* Get a buffer and return it to the pool to trigger the sequence to resume. It shouldn't 2707 * be able to complete, as it needs two buffers */ 2708 g_iobuf.small_pool_count = 1; 2709 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2710 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2711 g_iobuf.small_pool_count = 0; 2712 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2713 2714 CU_ASSERT_EQUAL(completed, 0); 2715 CU_ASSERT(!ut_seq.complete); 2716 2717 /* Return another buffer, this time the sequence should finish */ 2718 g_iobuf.small_pool_count = 1; 2719 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2720 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2721 g_iobuf.small_pool_count = 0; 2722 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2723 2724 poll_threads(); 2725 CU_ASSERT_EQUAL(completed, 1); 2726 CU_ASSERT(ut_seq.complete); 2727 CU_ASSERT_EQUAL(ut_seq.status, 0); 2728 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2729 2730 /* Return the buffers back to the cache */ 2731 while (!STAILQ_EMPTY(&small->cache)) { 2732 cache_entry = STAILQ_FIRST(&small->cache); 2733 STAILQ_REMOVE_HEAD(&small->cache, stailq); 2734 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2735 small_cache_count++; 2736 } 2737 small->cache_count = 0; 2738 2739 g_iobuf.small_pool_count = 32; 2740 STAILQ_SWAP(&small->cache, &small_cache, spdk_iobuf_buffer); 2741 small->cache_count = small_cache_count; 2742 2743 /* Check error cases, starting with an error from spdk_memory_domain_pull_data() */ 2744 completed = 0; 2745 seq = NULL; 2746 2747 src_iovs[0].iov_base = (void *)0xdeadbeef; 2748 src_iovs[0].iov_len = sizeof(srcbuf); 2749 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2750 dst_iovs[0].iov_len = sizeof(dstbuf); 2751 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2752 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2753 domctx[1].pull_submit_status = -E2BIG; 2754 2755 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2756 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2757 ut_sequence_step_cb, &completed); 2758 CU_ASSERT_EQUAL(rc, 0); 2759 2760 ut_seq.complete = false; 2761 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2762 2763 CU_ASSERT_EQUAL(completed, 1); 2764 CU_ASSERT(ut_seq.complete); 2765 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 2766 2767 /* Check completion error from spdk_memory_domain_pull_data() */ 2768 completed = 0; 2769 seq = NULL; 2770 2771 src_iovs[0].iov_base = (void *)0xdeadbeef; 2772 src_iovs[0].iov_len = sizeof(srcbuf); 2773 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2774 dst_iovs[0].iov_len = sizeof(dstbuf); 2775 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2776 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2777 domctx[1].pull_complete_status = -EACCES; 2778 2779 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2780 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2781 ut_sequence_step_cb, &completed); 2782 CU_ASSERT_EQUAL(rc, 0); 2783 2784 ut_seq.complete = false; 2785 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2786 2787 CU_ASSERT_EQUAL(completed, 1); 2788 CU_ASSERT(ut_seq.complete); 2789 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 2790 2791 /* Check submission error from spdk_memory_domain_push_data() */ 2792 completed = 0; 2793 seq = NULL; 2794 2795 src_iovs[0].iov_base = (void *)0xdeadbeef; 2796 src_iovs[0].iov_len = sizeof(srcbuf); 2797 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2798 dst_iovs[0].iov_len = sizeof(dstbuf); 2799 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2800 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2801 domctx[0].push_submit_status = -EADDRINUSE; 2802 2803 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2804 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2805 ut_sequence_step_cb, &completed); 2806 CU_ASSERT_EQUAL(rc, 0); 2807 2808 ut_seq.complete = false; 2809 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2810 2811 CU_ASSERT_EQUAL(completed, 1); 2812 CU_ASSERT(ut_seq.complete); 2813 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 2814 2815 /* Check completion error from spdk_memory_domain_push_data() */ 2816 completed = 0; 2817 seq = NULL; 2818 2819 src_iovs[0].iov_base = (void *)0xdeadbeef; 2820 src_iovs[0].iov_len = sizeof(srcbuf); 2821 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2822 dst_iovs[0].iov_len = sizeof(dstbuf); 2823 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2824 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2825 domctx[0].push_complete_status = -EADDRNOTAVAIL; 2826 2827 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2828 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2829 ut_sequence_step_cb, &completed); 2830 CU_ASSERT_EQUAL(rc, 0); 2831 2832 ut_seq.complete = false; 2833 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2834 2835 CU_ASSERT_EQUAL(completed, 1); 2836 CU_ASSERT(ut_seq.complete); 2837 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 2838 2839 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2840 g_modules_opc[i] = modules[i]; 2841 } 2842 2843 ut_clear_operations(); 2844 spdk_put_io_channel(ioch); 2845 poll_threads(); 2846 } 2847 2848 static int 2849 ut_submit_decompress_memory_domain(struct spdk_io_channel *ch, struct spdk_accel_task *task) 2850 { 2851 struct ut_domain_ctx *ctx; 2852 struct iovec *src_iovs, *dst_iovs; 2853 uint32_t src_iovcnt, dst_iovcnt; 2854 2855 src_iovs = task->s.iovs; 2856 dst_iovs = task->d.iovs; 2857 src_iovcnt = task->s.iovcnt; 2858 dst_iovcnt = task->d.iovcnt; 2859 2860 if (task->src_domain != NULL) { 2861 ctx = task->src_domain_ctx; 2862 CU_ASSERT_EQUAL(memcmp(task->s.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2863 src_iovs = &ctx->iov; 2864 src_iovcnt = 1; 2865 } 2866 2867 if (task->dst_domain != NULL) { 2868 ctx = task->dst_domain_ctx; 2869 CU_ASSERT_EQUAL(memcmp(task->d.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2870 dst_iovs = &ctx->iov; 2871 dst_iovcnt = 1; 2872 } 2873 2874 spdk_iovcpy(src_iovs, src_iovcnt, dst_iovs, dst_iovcnt); 2875 spdk_accel_task_complete(task, 0); 2876 2877 return 0; 2878 } 2879 2880 static void 2881 test_sequence_module_memory_domain(void) 2882 { 2883 struct spdk_accel_sequence *seq = NULL; 2884 struct spdk_io_channel *ioch; 2885 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2886 struct spdk_memory_domain *accel_domain; 2887 struct ut_sequence ut_seq; 2888 struct ut_domain_ctx domctx[2]; 2889 struct iovec src_iovs[2], dst_iovs[2]; 2890 void *buf, *accel_domain_ctx; 2891 char srcbuf[4096], dstbuf[4096], tmp[4096], expected[4096]; 2892 int i, rc, completed; 2893 2894 ioch = spdk_accel_get_io_channel(); 2895 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2896 2897 /* Override the submit_tasks function */ 2898 g_module_if.submit_tasks = ut_sequence_submit_tasks; 2899 g_module.supports_memory_domains = true; 2900 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2901 modules[i] = g_modules_opc[i]; 2902 g_modules_opc[i] = g_module; 2903 } 2904 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress_memory_domain; 2905 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2906 2907 /* Check a sequence with both buffers in memory domains */ 2908 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2909 memset(expected, 0xa5, sizeof(expected)); 2910 memset(dstbuf, 0, sizeof(dstbuf)); 2911 seq = NULL; 2912 completed = 0; 2913 2914 src_iovs[0].iov_base = (void *)0xcafebabe; 2915 src_iovs[0].iov_len = sizeof(srcbuf); 2916 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2917 dst_iovs[0].iov_len = sizeof(dstbuf); 2918 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2919 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2920 2921 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2922 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2923 ut_sequence_step_cb, &completed); 2924 CU_ASSERT_EQUAL(rc, 0); 2925 2926 ut_seq.complete = false; 2927 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2928 2929 poll_threads(); 2930 2931 CU_ASSERT_EQUAL(completed, 1); 2932 CU_ASSERT(ut_seq.complete); 2933 CU_ASSERT_EQUAL(ut_seq.status, 0); 2934 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2935 2936 /* Check two operations each with a single buffer in memory domain */ 2937 memset(srcbuf, 0x5a, sizeof(srcbuf)); 2938 memset(expected, 0x5a, sizeof(expected)); 2939 memset(dstbuf, 0, sizeof(dstbuf)); 2940 memset(tmp, 0, sizeof(tmp)); 2941 seq = NULL; 2942 completed = 0; 2943 2944 src_iovs[0].iov_base = srcbuf; 2945 src_iovs[0].iov_len = sizeof(srcbuf); 2946 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2947 dst_iovs[0].iov_len = sizeof(tmp); 2948 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2949 2950 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2951 &src_iovs[0], 1, NULL, NULL, 2952 ut_sequence_step_cb, &completed); 2953 CU_ASSERT_EQUAL(rc, 0); 2954 2955 src_iovs[1].iov_base = (void *)0xfeedbeef; 2956 src_iovs[1].iov_len = sizeof(tmp); 2957 dst_iovs[1].iov_base = dstbuf; 2958 dst_iovs[1].iov_len = sizeof(dstbuf); 2959 ut_domain_ctx_init(&domctx[1], tmp, sizeof(tmp), &src_iovs[1]); 2960 2961 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2962 &src_iovs[1], 1, g_ut_domain, &domctx[1], 2963 ut_sequence_step_cb, &completed); 2964 CU_ASSERT_EQUAL(rc, 0); 2965 2966 ut_seq.complete = false; 2967 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2968 2969 poll_threads(); 2970 2971 CU_ASSERT_EQUAL(completed, 2); 2972 CU_ASSERT(ut_seq.complete); 2973 CU_ASSERT_EQUAL(ut_seq.status, 0); 2974 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2975 2976 /* Check a sequence with an accel buffer and a buffer in a regular memory domain */ 2977 memset(expected, 0xa5, sizeof(expected)); 2978 memset(dstbuf, 0, sizeof(dstbuf)); 2979 memset(tmp, 0, sizeof(tmp)); 2980 seq = NULL; 2981 completed = 0; 2982 2983 rc = spdk_accel_get_buf(ioch, 4096, &buf, &accel_domain, &accel_domain_ctx); 2984 CU_ASSERT_EQUAL(rc, 0); 2985 2986 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, accel_domain, accel_domain_ctx, 2987 0xa5, ut_sequence_step_cb, &completed); 2988 CU_ASSERT_EQUAL(rc, 0); 2989 2990 src_iovs[0].iov_base = buf; 2991 src_iovs[0].iov_len = 4096; 2992 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2993 dst_iovs[0].iov_len = sizeof(dstbuf); 2994 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2995 2996 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2997 &src_iovs[0], 1, accel_domain, accel_domain_ctx, 2998 ut_sequence_step_cb, &completed); 2999 CU_ASSERT_EQUAL(rc, 0); 3000 3001 ut_seq.complete = false; 3002 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3003 3004 poll_threads(); 3005 3006 CU_ASSERT_EQUAL(completed, 2); 3007 CU_ASSERT(ut_seq.complete); 3008 CU_ASSERT_EQUAL(ut_seq.status, 0); 3009 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 3010 3011 spdk_accel_put_buf(ioch, buf, accel_domain, accel_domain_ctx); 3012 3013 g_module.supports_memory_domains = false; 3014 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3015 g_modules_opc[i] = modules[i]; 3016 } 3017 3018 ut_clear_operations(); 3019 spdk_put_io_channel(ioch); 3020 poll_threads(); 3021 } 3022 3023 #ifdef SPDK_CONFIG_ISAL_CRYPTO 3024 static void 3025 ut_encrypt_cb(void *cb_arg, int status) 3026 { 3027 int *completed = cb_arg; 3028 3029 CU_ASSERT_EQUAL(status, 0); 3030 3031 *completed = 1; 3032 } 3033 3034 static void 3035 test_sequence_crypto(void) 3036 { 3037 struct spdk_accel_sequence *seq = NULL; 3038 struct spdk_io_channel *ioch; 3039 struct spdk_accel_crypto_key *key; 3040 struct spdk_accel_crypto_key_create_param key_params = { 3041 .cipher = "AES_XTS", 3042 .hex_key = "00112233445566778899aabbccddeeff", 3043 .hex_key2 = "ffeeddccbbaa99887766554433221100", 3044 .key_name = "ut_key", 3045 }; 3046 struct ut_sequence ut_seq; 3047 unsigned char buf[4096], encrypted[4096] = {}, data[4096], tmp[3][4096]; 3048 struct iovec src_iovs[4], dst_iovs[4]; 3049 int rc, completed = 0; 3050 size_t i; 3051 3052 ioch = spdk_accel_get_io_channel(); 3053 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3054 3055 rc = spdk_accel_crypto_key_create(&key_params); 3056 CU_ASSERT_EQUAL(rc, 0); 3057 key = spdk_accel_crypto_key_get(key_params.key_name); 3058 SPDK_CU_ASSERT_FATAL(key != NULL); 3059 3060 for (i = 0; i < sizeof(data); ++i) { 3061 data[i] = (uint8_t)i & 0xff; 3062 } 3063 3064 dst_iovs[0].iov_base = encrypted; 3065 dst_iovs[0].iov_len = sizeof(encrypted); 3066 src_iovs[0].iov_base = data; 3067 src_iovs[0].iov_len = sizeof(data); 3068 rc = spdk_accel_submit_encrypt(ioch, key, &dst_iovs[0], 1, &src_iovs[0], 1, 0, 4096, 3069 ut_encrypt_cb, &completed); 3070 CU_ASSERT_EQUAL(rc, 0); 3071 3072 while (!completed) { 3073 poll_threads(); 3074 } 3075 3076 /* Verify that encryption operation in a sequence produces the same result */ 3077 seq = NULL; 3078 completed = 0; 3079 3080 dst_iovs[0].iov_base = tmp[0]; 3081 dst_iovs[0].iov_len = sizeof(tmp[0]); 3082 src_iovs[0].iov_base = data; 3083 src_iovs[0].iov_len = sizeof(data); 3084 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3085 &src_iovs[0], 1, NULL, NULL, 3086 ut_sequence_step_cb, &completed); 3087 CU_ASSERT_EQUAL(rc, 0); 3088 3089 dst_iovs[1].iov_base = tmp[1]; 3090 dst_iovs[1].iov_len = sizeof(tmp[1]); 3091 src_iovs[1].iov_base = tmp[0]; 3092 src_iovs[1].iov_len = sizeof(tmp[0]); 3093 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3094 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3095 ut_sequence_step_cb, &completed); 3096 CU_ASSERT_EQUAL(rc, 0); 3097 3098 dst_iovs[2].iov_base = buf; 3099 dst_iovs[2].iov_len = sizeof(buf); 3100 src_iovs[2].iov_base = tmp[1]; 3101 src_iovs[2].iov_len = sizeof(tmp[1]); 3102 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3103 &src_iovs[2], 1, NULL, NULL, 3104 ut_sequence_step_cb, &completed); 3105 CU_ASSERT_EQUAL(rc, 0); 3106 3107 ut_seq.complete = false; 3108 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3109 3110 poll_threads(); 3111 3112 CU_ASSERT_EQUAL(completed, 3); 3113 CU_ASSERT(ut_seq.complete); 3114 CU_ASSERT_EQUAL(ut_seq.status, 0); 3115 CU_ASSERT_EQUAL(memcmp(buf, encrypted, sizeof(buf)), 0); 3116 3117 /* Check that decryption produces the original buffer */ 3118 seq = NULL; 3119 completed = 0; 3120 memset(buf, 0, sizeof(buf)); 3121 3122 dst_iovs[0].iov_base = tmp[0]; 3123 dst_iovs[0].iov_len = sizeof(tmp[0]); 3124 src_iovs[0].iov_base = encrypted; 3125 src_iovs[0].iov_len = sizeof(encrypted); 3126 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3127 &src_iovs[0], 1, NULL, NULL, 3128 ut_sequence_step_cb, &completed); 3129 CU_ASSERT_EQUAL(rc, 0); 3130 3131 dst_iovs[1].iov_base = tmp[1]; 3132 dst_iovs[1].iov_len = sizeof(tmp[1]); 3133 src_iovs[1].iov_base = tmp[0]; 3134 src_iovs[1].iov_len = sizeof(tmp[0]); 3135 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3136 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3137 ut_sequence_step_cb, &completed); 3138 CU_ASSERT_EQUAL(rc, 0); 3139 3140 dst_iovs[2].iov_base = buf; 3141 dst_iovs[2].iov_len = sizeof(buf); 3142 src_iovs[2].iov_base = tmp[1]; 3143 src_iovs[2].iov_len = sizeof(tmp[1]); 3144 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3145 &src_iovs[2], 1, NULL, NULL, 3146 ut_sequence_step_cb, &completed); 3147 CU_ASSERT_EQUAL(rc, 0); 3148 3149 ut_seq.complete = false; 3150 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3151 3152 poll_threads(); 3153 3154 CU_ASSERT_EQUAL(completed, 3); 3155 CU_ASSERT(ut_seq.complete); 3156 CU_ASSERT_EQUAL(ut_seq.status, 0); 3157 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3158 3159 /* Check encrypt + decrypt in a single sequence */ 3160 seq = NULL; 3161 completed = 0; 3162 memset(buf, 0, sizeof(buf)); 3163 3164 dst_iovs[0].iov_base = tmp[0]; 3165 dst_iovs[0].iov_len = sizeof(tmp[0]); 3166 src_iovs[0].iov_base = data; 3167 src_iovs[0].iov_len = sizeof(data); 3168 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3169 &src_iovs[0], 1, NULL, NULL, 3170 ut_sequence_step_cb, &completed); 3171 CU_ASSERT_EQUAL(rc, 0); 3172 3173 dst_iovs[1].iov_base = tmp[1]; 3174 dst_iovs[1].iov_len = sizeof(tmp[1]); 3175 src_iovs[1].iov_base = tmp[0]; 3176 src_iovs[1].iov_len = sizeof(tmp[0]); 3177 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3178 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3179 ut_sequence_step_cb, &completed); 3180 CU_ASSERT_EQUAL(rc, 0); 3181 3182 3183 dst_iovs[2].iov_base = tmp[2]; 3184 dst_iovs[2].iov_len = sizeof(tmp[2]); 3185 src_iovs[2].iov_base = tmp[1]; 3186 src_iovs[2].iov_len = sizeof(tmp[1]); 3187 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[2], 1, NULL, NULL, 3188 &src_iovs[2], 1, NULL, NULL, 0, 4096, 3189 ut_sequence_step_cb, &completed); 3190 CU_ASSERT_EQUAL(rc, 0); 3191 3192 dst_iovs[3].iov_base = buf; 3193 dst_iovs[3].iov_len = sizeof(buf); 3194 src_iovs[3].iov_base = tmp[2]; 3195 src_iovs[3].iov_len = sizeof(tmp[2]); 3196 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 3197 &src_iovs[3], 1, NULL, NULL, 3198 ut_sequence_step_cb, &completed); 3199 CU_ASSERT_EQUAL(rc, 0); 3200 3201 ut_seq.complete = false; 3202 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3203 3204 poll_threads(); 3205 3206 CU_ASSERT_EQUAL(completed, 4); 3207 CU_ASSERT(ut_seq.complete); 3208 CU_ASSERT_EQUAL(ut_seq.status, 0); 3209 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3210 3211 rc = spdk_accel_crypto_key_destroy(key); 3212 CU_ASSERT_EQUAL(rc, 0); 3213 spdk_put_io_channel(ioch); 3214 poll_threads(); 3215 } 3216 #endif /* SPDK_CONFIG_ISAL_CRYPTO */ 3217 3218 static int 3219 ut_submit_crypto(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3220 { 3221 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3222 3223 spdk_accel_task_complete(task, 0); 3224 3225 return 0; 3226 } 3227 3228 struct ut_driver_operation { 3229 int complete_status; 3230 int submit_status; 3231 int count; 3232 bool supported; 3233 }; 3234 3235 static struct ut_driver_operation g_drv_operations[SPDK_ACCEL_OPC_LAST]; 3236 static bool g_ut_driver_async_continue; 3237 3238 static void 3239 ut_driver_async_continue(void *arg) 3240 { 3241 struct spdk_accel_sequence *seq = arg; 3242 3243 spdk_accel_sequence_continue(seq); 3244 } 3245 3246 static int 3247 ut_driver_execute_sequence(struct spdk_io_channel *ch, struct spdk_accel_sequence *seq) 3248 { 3249 struct spdk_accel_task *task; 3250 struct ut_driver_operation *drv_ops; 3251 3252 while ((task = spdk_accel_sequence_first_task(seq)) != NULL) { 3253 drv_ops = &g_drv_operations[task->op_code]; 3254 if (!drv_ops->supported) { 3255 break; 3256 } 3257 3258 drv_ops->count++; 3259 if (drv_ops->submit_status != 0) { 3260 return drv_ops->submit_status; 3261 } 3262 3263 if (drv_ops->complete_status != 0) { 3264 spdk_accel_task_complete(task, drv_ops->complete_status); 3265 break; 3266 } 3267 3268 switch (task->op_code) { 3269 case SPDK_ACCEL_OPC_DECOMPRESS: 3270 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3271 break; 3272 case SPDK_ACCEL_OPC_FILL: 3273 spdk_iov_memset(task->d.iovs, task->d.iovcnt, 3274 (int)(task->fill_pattern & 0xff)); 3275 break; 3276 default: 3277 CU_ASSERT(0 && "unexpected opcode"); 3278 break; 3279 } 3280 3281 spdk_accel_task_complete(task, 0); 3282 } 3283 3284 if (g_ut_driver_async_continue) { 3285 spdk_thread_send_msg(spdk_get_thread(), ut_driver_async_continue, seq); 3286 } else { 3287 spdk_accel_sequence_continue(seq); 3288 } 3289 3290 return 0; 3291 } 3292 3293 static struct spdk_io_channel * 3294 ut_driver_get_io_channel(void) 3295 { 3296 return (void *)0xdeadbeef; 3297 } 3298 3299 static struct spdk_accel_driver g_ut_driver = { 3300 .name = "ut", 3301 .execute_sequence = ut_driver_execute_sequence, 3302 .get_io_channel = ut_driver_get_io_channel, 3303 }; 3304 3305 SPDK_ACCEL_DRIVER_REGISTER(ut, &g_ut_driver); 3306 3307 static void 3308 test_sequence_driver(void) 3309 { 3310 struct spdk_accel_sequence *seq = NULL; 3311 struct spdk_io_channel *ioch; 3312 struct spdk_accel_crypto_key key = {}; 3313 struct ut_sequence ut_seq; 3314 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3315 char buf[4096], tmp[3][4096], expected[4096]; 3316 struct iovec src_iovs[3], dst_iovs[3]; 3317 int i, rc, completed = 0; 3318 3319 ioch = spdk_accel_get_io_channel(); 3320 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3321 rc = spdk_accel_set_driver("ut"); 3322 SPDK_CU_ASSERT_FATAL(rc == 0); 3323 3324 /* Override the submit_tasks function */ 3325 g_module_if.submit_tasks = ut_sequence_submit_tasks; 3326 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3327 modules[i] = g_modules_opc[i]; 3328 g_modules_opc[i] = g_module; 3329 } 3330 /* Intercept crypto operations, as they should be executed by an accel module */ 3331 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].submit = ut_submit_crypto; 3332 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].submit = ut_submit_crypto; 3333 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3334 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3335 g_seq_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3336 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3337 3338 g_drv_operations[SPDK_ACCEL_OPC_FILL].supported = true; 3339 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].supported = true; 3340 3341 /* First check a sequence that is fully executed using a driver, with the copy at the end 3342 * being removed */ 3343 seq = NULL; 3344 completed = 0; 3345 memset(buf, 0, sizeof(buf)); 3346 memset(tmp[0], 0, sizeof(tmp[0])); 3347 memset(tmp[1], 0, sizeof(tmp[1])); 3348 memset(&expected[0], 0xa5, 2048); 3349 memset(&expected[2048], 0xbe, 2048); 3350 3351 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], 2048, NULL, NULL, 0xa5, 3352 ut_sequence_step_cb, &completed); 3353 CU_ASSERT_EQUAL(rc, 0); 3354 rc = spdk_accel_append_fill(&seq, ioch, &tmp[0][2048], 2048, NULL, NULL, 0xbe, 3355 ut_sequence_step_cb, &completed); 3356 CU_ASSERT_EQUAL(rc, 0); 3357 3358 dst_iovs[0].iov_base = tmp[1]; 3359 dst_iovs[0].iov_len = sizeof(tmp[1]); 3360 src_iovs[0].iov_base = tmp[0]; 3361 src_iovs[0].iov_len = sizeof(tmp[0]); 3362 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3363 &src_iovs[0], 1, NULL, NULL, 3364 ut_sequence_step_cb, &completed); 3365 CU_ASSERT_EQUAL(rc, 0); 3366 3367 dst_iovs[1].iov_base = buf; 3368 dst_iovs[1].iov_len = sizeof(buf); 3369 src_iovs[1].iov_base = tmp[1]; 3370 src_iovs[1].iov_len = sizeof(tmp[1]); 3371 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3372 &src_iovs[1], 1, NULL, NULL, 3373 ut_sequence_step_cb, &completed); 3374 CU_ASSERT_EQUAL(rc, 0); 3375 3376 ut_seq.complete = false; 3377 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3378 3379 poll_threads(); 3380 3381 CU_ASSERT_EQUAL(completed, 4); 3382 CU_ASSERT(ut_seq.complete); 3383 CU_ASSERT_EQUAL(ut_seq.status, 0); 3384 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3385 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3386 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 3387 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 2); 3388 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3389 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3390 3391 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3392 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3393 3394 /* Check a sequence when the first two operations are executed by a driver, while the rest 3395 * is executed via modules */ 3396 seq = NULL; 3397 completed = 0; 3398 memset(buf, 0, sizeof(buf)); 3399 memset(tmp[0], 0, sizeof(tmp[0])); 3400 memset(tmp[1], 0, sizeof(tmp[1])); 3401 memset(tmp[2], 0, sizeof(tmp[2])); 3402 memset(&expected[0], 0xfe, 4096); 3403 3404 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 3405 ut_sequence_step_cb, &completed); 3406 CU_ASSERT_EQUAL(rc, 0); 3407 3408 dst_iovs[0].iov_base = tmp[1]; 3409 dst_iovs[0].iov_len = sizeof(tmp[1]); 3410 src_iovs[0].iov_base = tmp[0]; 3411 src_iovs[0].iov_len = sizeof(tmp[0]); 3412 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3413 &src_iovs[0], 1, NULL, NULL, 3414 ut_sequence_step_cb, &completed); 3415 CU_ASSERT_EQUAL(rc, 0); 3416 3417 dst_iovs[1].iov_base = tmp[2]; 3418 dst_iovs[1].iov_len = sizeof(tmp[2]); 3419 src_iovs[1].iov_base = tmp[1]; 3420 src_iovs[1].iov_len = sizeof(tmp[1]); 3421 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3422 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3423 ut_sequence_step_cb, &completed); 3424 CU_ASSERT_EQUAL(rc, 0); 3425 3426 dst_iovs[2].iov_base = buf; 3427 dst_iovs[2].iov_len = sizeof(buf); 3428 src_iovs[2].iov_base = tmp[2]; 3429 src_iovs[2].iov_len = sizeof(tmp[2]); 3430 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3431 &src_iovs[2], 1, NULL, NULL, 0, 4096, 3432 ut_sequence_step_cb, &completed); 3433 CU_ASSERT_EQUAL(rc, 0); 3434 3435 ut_seq.complete = false; 3436 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3437 3438 poll_threads(); 3439 3440 CU_ASSERT_EQUAL(completed, 4); 3441 CU_ASSERT(ut_seq.complete); 3442 CU_ASSERT_EQUAL(ut_seq.status, 0); 3443 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3444 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3445 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3446 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3447 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3448 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3449 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3450 3451 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3452 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3453 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3454 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3455 3456 /* Check sequence when the first and last operations are executed through modules, while the 3457 * ones in the middle are executed by the driver */ 3458 seq = NULL; 3459 completed = 0; 3460 memset(buf, 0, sizeof(buf)); 3461 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3462 memset(tmp[1], 0, sizeof(tmp[1])); 3463 memset(tmp[2], 0, sizeof(tmp[2])); 3464 memset(&expected[0], 0xfe, 2048); 3465 memset(&expected[2048], 0xa5, 2048); 3466 3467 dst_iovs[0].iov_base = tmp[1]; 3468 dst_iovs[0].iov_len = sizeof(tmp[1]); 3469 src_iovs[0].iov_base = tmp[0]; 3470 src_iovs[0].iov_len = sizeof(tmp[0]); 3471 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3472 &src_iovs[0], 1, NULL, NULL, 0, 4096, 3473 ut_sequence_step_cb, &completed); 3474 CU_ASSERT_EQUAL(rc, 0); 3475 3476 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xfe, 3477 ut_sequence_step_cb, &completed); 3478 CU_ASSERT_EQUAL(rc, 0); 3479 3480 dst_iovs[1].iov_base = tmp[2]; 3481 dst_iovs[1].iov_len = sizeof(tmp[2]); 3482 src_iovs[1].iov_base = tmp[1]; 3483 src_iovs[1].iov_len = sizeof(tmp[1]); 3484 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3485 &src_iovs[1], 1, NULL, NULL, 3486 ut_sequence_step_cb, &completed); 3487 CU_ASSERT_EQUAL(rc, 0); 3488 3489 dst_iovs[2].iov_base = buf; 3490 dst_iovs[2].iov_len = sizeof(buf); 3491 src_iovs[2].iov_base = tmp[2]; 3492 src_iovs[2].iov_len = sizeof(tmp[2]); 3493 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3494 &src_iovs[2], 1, NULL, NULL, 0, 4096, 3495 ut_sequence_step_cb, &completed); 3496 CU_ASSERT_EQUAL(rc, 0); 3497 3498 ut_seq.complete = false; 3499 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3500 3501 poll_threads(); 3502 3503 CU_ASSERT_EQUAL(completed, 4); 3504 CU_ASSERT(ut_seq.complete); 3505 CU_ASSERT_EQUAL(ut_seq.status, 0); 3506 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3507 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3508 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3509 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3510 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3511 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3512 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3513 3514 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3515 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3516 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3517 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3518 3519 /* Check a sequence with operations executed by: module, driver, module, driver */ 3520 seq = NULL; 3521 completed = 0; 3522 memset(buf, 0, sizeof(buf)); 3523 memset(tmp[0], 0x5a, sizeof(tmp[0])); 3524 memset(tmp[1], 0, sizeof(tmp[1])); 3525 memset(tmp[2], 0, sizeof(tmp[2])); 3526 memset(&expected[0], 0xef, 2048); 3527 memset(&expected[2048], 0x5a, 2048); 3528 3529 dst_iovs[0].iov_base = tmp[1]; 3530 dst_iovs[0].iov_len = sizeof(tmp[1]); 3531 src_iovs[0].iov_base = tmp[0]; 3532 src_iovs[0].iov_len = sizeof(tmp[0]); 3533 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3534 &src_iovs[0], 1, NULL, NULL, 0, 4096, 3535 ut_sequence_step_cb, &completed); 3536 CU_ASSERT_EQUAL(rc, 0); 3537 3538 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 3539 ut_sequence_step_cb, &completed); 3540 CU_ASSERT_EQUAL(rc, 0); 3541 3542 dst_iovs[1].iov_base = tmp[2]; 3543 dst_iovs[1].iov_len = sizeof(tmp[2]); 3544 src_iovs[1].iov_base = tmp[1]; 3545 src_iovs[1].iov_len = sizeof(tmp[1]); 3546 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3547 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3548 ut_sequence_step_cb, &completed); 3549 CU_ASSERT_EQUAL(rc, 0); 3550 3551 dst_iovs[2].iov_base = buf; 3552 dst_iovs[2].iov_len = sizeof(buf); 3553 src_iovs[2].iov_base = tmp[2]; 3554 src_iovs[2].iov_len = sizeof(tmp[2]); 3555 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3556 &src_iovs[2], 1, NULL, NULL, 3557 ut_sequence_step_cb, &completed); 3558 CU_ASSERT_EQUAL(rc, 0); 3559 3560 ut_seq.complete = false; 3561 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3562 3563 poll_threads(); 3564 3565 CU_ASSERT_EQUAL(completed, 4); 3566 CU_ASSERT(ut_seq.complete); 3567 CU_ASSERT_EQUAL(ut_seq.status, 0); 3568 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3569 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3570 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3571 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3572 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3573 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3574 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3575 3576 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3577 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3578 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3579 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3580 3581 /* Check that an error returned from driver's execute_sequence() will fail the whole 3582 * sequence and any subsequent operations won't be processed */ 3583 seq = NULL; 3584 completed = 0; 3585 memset(buf, 0, sizeof(buf)); 3586 memset(expected, 0, sizeof(expected)); 3587 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3588 g_drv_operations[SPDK_ACCEL_OPC_FILL].submit_status = -EPERM; 3589 3590 dst_iovs[0].iov_base = tmp[1]; 3591 dst_iovs[0].iov_len = sizeof(tmp[1]); 3592 src_iovs[0].iov_base = tmp[0]; 3593 src_iovs[0].iov_len = sizeof(tmp[0]); 3594 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3595 &src_iovs[0], 1, NULL, NULL, 0, 4096, 3596 ut_sequence_step_cb, &completed); 3597 CU_ASSERT_EQUAL(rc, 0); 3598 3599 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 3600 ut_sequence_step_cb, &completed); 3601 CU_ASSERT_EQUAL(rc, 0); 3602 3603 dst_iovs[1].iov_base = buf; 3604 dst_iovs[1].iov_len = sizeof(buf); 3605 src_iovs[1].iov_base = tmp[1]; 3606 src_iovs[1].iov_len = sizeof(tmp[1]); 3607 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3608 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3609 ut_sequence_step_cb, &completed); 3610 CU_ASSERT_EQUAL(rc, 0); 3611 3612 ut_seq.complete = false; 3613 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3614 3615 poll_threads(); 3616 3617 CU_ASSERT_EQUAL(completed, 3); 3618 CU_ASSERT(ut_seq.complete); 3619 CU_ASSERT_EQUAL(ut_seq.status, -EPERM); 3620 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3621 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3622 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 0); 3623 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3624 CU_ASSERT_EQUAL(memcmp(buf, expected, 4096), 0); 3625 3626 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3627 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3628 g_drv_operations[SPDK_ACCEL_OPC_FILL].submit_status = 0; 3629 3630 /* Check that a failed task completed by a driver will cause the whole sequence to be failed 3631 * and any subsequent operations won't be processed */ 3632 seq = NULL; 3633 completed = 0; 3634 memset(buf, 0, sizeof(buf)); 3635 memset(expected, 0, sizeof(expected)); 3636 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3637 g_drv_operations[SPDK_ACCEL_OPC_FILL].complete_status = -ENOENT; 3638 3639 dst_iovs[0].iov_base = tmp[1]; 3640 dst_iovs[0].iov_len = sizeof(tmp[1]); 3641 src_iovs[0].iov_base = tmp[0]; 3642 src_iovs[0].iov_len = sizeof(tmp[0]); 3643 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3644 &src_iovs[0], 1, NULL, NULL, 0, 4096, 3645 ut_sequence_step_cb, &completed); 3646 CU_ASSERT_EQUAL(rc, 0); 3647 3648 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 3649 ut_sequence_step_cb, &completed); 3650 CU_ASSERT_EQUAL(rc, 0); 3651 3652 dst_iovs[1].iov_base = buf; 3653 dst_iovs[1].iov_len = sizeof(buf); 3654 src_iovs[1].iov_base = tmp[1]; 3655 src_iovs[1].iov_len = sizeof(tmp[1]); 3656 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3657 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3658 ut_sequence_step_cb, &completed); 3659 CU_ASSERT_EQUAL(rc, 0); 3660 3661 ut_seq.complete = false; 3662 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3663 3664 poll_threads(); 3665 3666 CU_ASSERT_EQUAL(completed, 3); 3667 CU_ASSERT(ut_seq.complete); 3668 CU_ASSERT_EQUAL(ut_seq.status, -ENOENT); 3669 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3670 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3671 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 0); 3672 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3673 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3674 3675 g_drv_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 3676 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3677 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3678 3679 /* Check asynchronous spdk_accel_sequence_continue() */ 3680 g_ut_driver_async_continue = true; 3681 seq = NULL; 3682 completed = 0; 3683 memset(buf, 0, sizeof(buf)); 3684 memset(tmp[0], 0, sizeof(tmp[0])); 3685 memset(&expected[0], 0xfe, 4096); 3686 3687 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 3688 ut_sequence_step_cb, &completed); 3689 CU_ASSERT_EQUAL(rc, 0); 3690 3691 dst_iovs[0].iov_base = buf; 3692 dst_iovs[0].iov_len = sizeof(buf); 3693 src_iovs[0].iov_base = tmp[0]; 3694 src_iovs[0].iov_len = sizeof(tmp[0]); 3695 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3696 &src_iovs[0], 1, NULL, NULL, 3697 ut_sequence_step_cb, &completed); 3698 CU_ASSERT_EQUAL(rc, 0); 3699 3700 ut_seq.complete = false; 3701 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3702 3703 poll_threads(); 3704 3705 CU_ASSERT_EQUAL(completed, 2); 3706 CU_ASSERT(ut_seq.complete); 3707 CU_ASSERT_EQUAL(ut_seq.status, 0); 3708 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3709 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3710 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3711 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3712 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3713 3714 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3715 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3716 3717 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3718 g_modules_opc[i] = modules[i]; 3719 } 3720 3721 /* Clear the driver so that other tests won't use it */ 3722 g_accel_driver = NULL; 3723 memset(&g_drv_operations, 0, sizeof(g_drv_operations)); 3724 3725 ut_clear_operations(); 3726 spdk_put_io_channel(ioch); 3727 poll_threads(); 3728 } 3729 3730 struct ut_saved_iovs { 3731 struct iovec src; 3732 struct iovec dst; 3733 }; 3734 3735 static struct ut_saved_iovs g_seq_saved_iovs[SPDK_ACCEL_OPC_LAST]; 3736 3737 static int 3738 ut_submit_save_iovs(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3739 { 3740 SPDK_CU_ASSERT_FATAL(task->s.iovcnt == 1); 3741 SPDK_CU_ASSERT_FATAL(task->d.iovcnt == 1); 3742 3743 g_seq_saved_iovs[task->op_code].src = task->s.iovs[0]; 3744 g_seq_saved_iovs[task->op_code].dst = task->d.iovs[0]; 3745 3746 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3747 3748 spdk_accel_task_complete(task, 0); 3749 3750 return 0; 3751 } 3752 3753 static void 3754 test_sequence_same_iovs(void) 3755 { 3756 struct spdk_accel_sequence *seq = NULL; 3757 struct spdk_io_channel *ioch; 3758 struct spdk_accel_crypto_key key = {}; 3759 struct ut_sequence ut_seq; 3760 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3761 char buf[4096], tmp[4096], expected[4096]; 3762 struct iovec iovs[3], expected_siov, expected_diov; 3763 struct spdk_memory_domain *domain; 3764 void *accel_buf, *domain_ctx; 3765 int i, rc, completed = 0; 3766 3767 ioch = spdk_accel_get_io_channel(); 3768 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3769 3770 /* Override the submit_tasks function */ 3771 g_module_if.submit_tasks = ut_sequence_submit_tasks; 3772 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3773 modules[i] = g_modules_opc[i]; 3774 g_modules_opc[i] = g_module; 3775 } 3776 /* Intercept crypto operations, as they should be executed by an accel module */ 3777 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].submit = ut_submit_save_iovs; 3778 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].submit = ut_submit_save_iovs; 3779 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3780 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3781 3782 /* Check that it's possible to use the same iovec ptr for different operations */ 3783 seq = NULL; 3784 completed = 0; 3785 memset(buf, 0, sizeof(buf)); 3786 memset(expected, 0xa5, sizeof(expected)); 3787 3788 iovs[0].iov_base = expected; 3789 iovs[0].iov_len = sizeof(expected); 3790 iovs[1].iov_base = tmp; 3791 iovs[1].iov_len = sizeof(tmp); 3792 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, NULL, NULL, 3793 &iovs[0], 1, NULL, NULL, 0, 4096, 3794 ut_sequence_step_cb, &completed); 3795 CU_ASSERT_EQUAL(rc, 0); 3796 /* Reuse iov[1] as src */ 3797 iovs[2].iov_base = buf; 3798 iovs[2].iov_len = sizeof(buf); 3799 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3800 &iovs[1], 1, NULL, NULL, 0, 4096, 3801 ut_sequence_step_cb, &completed); 3802 CU_ASSERT_EQUAL(rc, 0); 3803 3804 ut_seq.complete = false; 3805 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3806 3807 poll_threads(); 3808 3809 CU_ASSERT_EQUAL(completed, 2); 3810 CU_ASSERT(ut_seq.complete); 3811 CU_ASSERT_EQUAL(ut_seq.status, 0); 3812 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3813 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3814 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3815 expected_siov.iov_base = expected; 3816 expected_siov.iov_len = sizeof(expected); 3817 expected_diov.iov_base = tmp; 3818 expected_diov.iov_len = sizeof(tmp); 3819 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].src, 3820 &expected_siov, sizeof(expected_siov)), 0); 3821 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].dst, 3822 &expected_diov, sizeof(expected_diov)), 0); 3823 expected_siov.iov_base = tmp; 3824 expected_siov.iov_len = sizeof(tmp); 3825 expected_diov.iov_base = buf; 3826 expected_diov.iov_len = sizeof(buf); 3827 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].src, 3828 &expected_siov, sizeof(expected_siov)), 0); 3829 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].dst, 3830 &expected_diov, sizeof(expected_diov)), 0); 3831 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3832 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3833 3834 /* Check the same with an accel buffer */ 3835 seq = NULL; 3836 completed = 0; 3837 memset(buf, 0, sizeof(buf)); 3838 memset(expected, 0x5a, sizeof(expected)); 3839 3840 rc = spdk_accel_get_buf(ioch, sizeof(buf), &accel_buf, &domain, &domain_ctx); 3841 CU_ASSERT_EQUAL(rc, 0); 3842 3843 iovs[0].iov_base = expected; 3844 iovs[0].iov_len = sizeof(expected); 3845 iovs[1].iov_base = accel_buf; 3846 iovs[1].iov_len = sizeof(buf); 3847 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, domain, domain_ctx, 3848 &iovs[0], 1, NULL, NULL, 0, 4096, 3849 ut_sequence_step_cb, &completed); 3850 CU_ASSERT_EQUAL(rc, 0); 3851 /* Reuse iov[1] as src */ 3852 iovs[2].iov_base = buf; 3853 iovs[2].iov_len = sizeof(buf); 3854 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3855 &iovs[1], 1, domain, domain_ctx, 0, 4096, 3856 ut_sequence_step_cb, &completed); 3857 CU_ASSERT_EQUAL(rc, 0); 3858 3859 ut_seq.complete = false; 3860 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3861 3862 poll_threads(); 3863 3864 CU_ASSERT_EQUAL(completed, 2); 3865 CU_ASSERT(ut_seq.complete); 3866 CU_ASSERT_EQUAL(ut_seq.status, 0); 3867 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3868 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3869 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3870 expected_siov.iov_base = expected; 3871 expected_siov.iov_len = sizeof(expected); 3872 expected_diov.iov_base = buf; 3873 expected_diov.iov_len = sizeof(buf); 3874 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].src, 3875 &expected_siov, sizeof(expected_siov)), 0); 3876 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].dst, 3877 &expected_diov, sizeof(expected_diov)), 0); 3878 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].dst, 3879 &g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].src, 3880 sizeof(struct iovec)), 0); 3881 spdk_accel_put_buf(ioch, accel_buf, domain, domain_ctx); 3882 3883 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3884 g_modules_opc[i] = modules[i]; 3885 } 3886 3887 ut_clear_operations(); 3888 spdk_put_io_channel(ioch); 3889 poll_threads(); 3890 } 3891 3892 static void 3893 test_sequence_crc32(void) 3894 { 3895 struct spdk_accel_sequence *seq = NULL; 3896 struct spdk_io_channel *ioch; 3897 struct ut_sequence ut_seq; 3898 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3899 char buf[4096], tmp[3][4096]; 3900 struct iovec src_iovs[4], dst_iovs[4]; 3901 uint32_t crc, crc2; 3902 int i, rc, completed; 3903 3904 ioch = spdk_accel_get_io_channel(); 3905 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3906 3907 /* Override the submit_tasks function */ 3908 g_module_if.submit_tasks = ut_sequence_submit_tasks; 3909 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3910 g_seq_operations[i].submit = sw_accel_submit_tasks; 3911 modules[i] = g_modules_opc[i]; 3912 g_modules_opc[i] = g_module; 3913 } 3914 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 3915 3916 /* First check the simplest case - single crc32c operation */ 3917 seq = NULL; 3918 completed = 0; 3919 crc = 0; 3920 memset(buf, 0xa5, sizeof(buf)); 3921 3922 src_iovs[0].iov_base = buf; 3923 src_iovs[0].iov_len = sizeof(buf); 3924 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3925 ut_sequence_step_cb, &completed); 3926 CU_ASSERT_EQUAL(rc, 0); 3927 3928 ut_seq.complete = false; 3929 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3930 3931 poll_threads(); 3932 CU_ASSERT_EQUAL(completed, 1); 3933 CU_ASSERT(ut_seq.complete); 3934 CU_ASSERT_EQUAL(ut_seq.status, 0); 3935 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3936 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3937 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3938 3939 /* Now check copy+crc - This should not remove the copy. Otherwise the data does not 3940 * end up where the user expected it to be. */ 3941 seq = NULL; 3942 completed = 0; 3943 crc = 0; 3944 memset(buf, 0x5a, sizeof(buf)); 3945 memset(&tmp[0], 0, sizeof(tmp[0])); 3946 3947 dst_iovs[0].iov_base = tmp[0]; 3948 dst_iovs[0].iov_len = sizeof(tmp[0]); 3949 src_iovs[0].iov_base = buf; 3950 src_iovs[0].iov_len = sizeof(buf); 3951 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3952 &src_iovs[0], 1, NULL, NULL, 3953 ut_sequence_step_cb, &completed); 3954 CU_ASSERT_EQUAL(rc, 0); 3955 3956 src_iovs[1].iov_base = tmp[0]; 3957 src_iovs[1].iov_len = sizeof(tmp[0]); 3958 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 3959 ut_sequence_step_cb, &completed); 3960 CU_ASSERT_EQUAL(rc, 0); 3961 3962 ut_seq.complete = false; 3963 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3964 3965 poll_threads(); 3966 CU_ASSERT_EQUAL(completed, 2); 3967 CU_ASSERT(ut_seq.complete); 3968 CU_ASSERT_EQUAL(ut_seq.status, 0); 3969 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3970 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 3971 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 3972 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3973 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3974 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 3975 3976 /* Check crc+copy - Again, the copy cannot be removed. */ 3977 seq = NULL; 3978 completed = 0; 3979 crc = 0; 3980 memset(buf, 0, sizeof(buf)); 3981 memset(&tmp[0], 0xa5, sizeof(tmp[0])); 3982 3983 src_iovs[0].iov_base = tmp[0]; 3984 src_iovs[0].iov_len = sizeof(tmp[0]); 3985 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3986 ut_sequence_step_cb, &completed); 3987 CU_ASSERT_EQUAL(rc, 0); 3988 3989 dst_iovs[1].iov_base = buf; 3990 dst_iovs[1].iov_len = sizeof(buf); 3991 src_iovs[1].iov_base = tmp[0]; 3992 src_iovs[1].iov_len = sizeof(tmp[0]); 3993 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3994 &src_iovs[1], 1, NULL, NULL, 3995 ut_sequence_step_cb, &completed); 3996 CU_ASSERT_EQUAL(rc, 0); 3997 3998 ut_seq.complete = false; 3999 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4000 4001 poll_threads(); 4002 CU_ASSERT_EQUAL(completed, 2); 4003 CU_ASSERT(ut_seq.complete); 4004 CU_ASSERT_EQUAL(ut_seq.status, 0); 4005 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 4006 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 4007 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 4008 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 4009 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 4010 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 4011 4012 /* Check a sequence with an operation at the beginning that can have its buffer changed, two 4013 * crc operations and a copy at the end. The copy should be removed and the dst buffer of 4014 * the first operation and the src buffer of the crc operations should be changed. 4015 */ 4016 seq = NULL; 4017 completed = 0; 4018 crc = crc2 = 0; 4019 memset(buf, 0, sizeof(buf)); 4020 memset(&tmp[0], 0x5a, sizeof(tmp[0])); 4021 dst_iovs[0].iov_base = tmp[1]; 4022 dst_iovs[0].iov_len = sizeof(tmp[1]); 4023 src_iovs[0].iov_base = tmp[0]; 4024 src_iovs[0].iov_len = sizeof(tmp[0]); 4025 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 4026 &src_iovs[0], 1, NULL, NULL, 4027 ut_sequence_step_cb, &completed); 4028 CU_ASSERT_EQUAL(rc, 0); 4029 4030 src_iovs[1].iov_base = tmp[1]; 4031 src_iovs[1].iov_len = sizeof(tmp[1]); 4032 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 4033 ut_sequence_step_cb, &completed); 4034 CU_ASSERT_EQUAL(rc, 0); 4035 4036 src_iovs[2].iov_base = tmp[1]; 4037 src_iovs[2].iov_len = sizeof(tmp[1]); 4038 rc = spdk_accel_append_crc32c(&seq, ioch, &crc2, &src_iovs[2], 1, NULL, NULL, 0, 4039 ut_sequence_step_cb, &completed); 4040 CU_ASSERT_EQUAL(rc, 0); 4041 4042 dst_iovs[3].iov_base = buf; 4043 dst_iovs[3].iov_len = sizeof(buf); 4044 src_iovs[3].iov_base = tmp[1]; 4045 src_iovs[3].iov_len = sizeof(tmp[1]); 4046 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 4047 &src_iovs[3], 1, NULL, NULL, 4048 ut_sequence_step_cb, &completed); 4049 CU_ASSERT_EQUAL(rc, 0); 4050 4051 ut_seq.complete = false; 4052 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4053 4054 poll_threads(); 4055 CU_ASSERT_EQUAL(completed, 4); 4056 CU_ASSERT(ut_seq.complete); 4057 CU_ASSERT_EQUAL(ut_seq.status, 0); 4058 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 4059 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 2); 4060 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 4061 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 4062 CU_ASSERT_EQUAL(crc, crc2); 4063 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 4064 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 4065 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 4066 4067 /* Check that a copy won't be removed if the buffers don't match */ 4068 seq = NULL; 4069 completed = 0; 4070 crc = 0; 4071 memset(buf, 0, sizeof(buf)); 4072 memset(&tmp[0], 0xa5, 2048); 4073 memset(&tmp[1], 0xfe, sizeof(tmp[1])); 4074 memset(&tmp[2], 0xfe, sizeof(tmp[1])); 4075 dst_iovs[0].iov_base = &tmp[1][2048]; 4076 dst_iovs[0].iov_len = 2048; 4077 src_iovs[0].iov_base = tmp[0]; 4078 src_iovs[0].iov_len = 2048; 4079 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 4080 &src_iovs[0], 1, NULL, NULL, 4081 ut_sequence_step_cb, &completed); 4082 CU_ASSERT_EQUAL(rc, 0); 4083 4084 src_iovs[1].iov_base = &tmp[1][2048]; 4085 src_iovs[1].iov_len = 2048; 4086 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 4087 ut_sequence_step_cb, &completed); 4088 CU_ASSERT_EQUAL(rc, 0); 4089 4090 dst_iovs[2].iov_base = buf; 4091 dst_iovs[2].iov_len = sizeof(buf); 4092 src_iovs[2].iov_base = tmp[1]; 4093 src_iovs[2].iov_len = sizeof(tmp[1]); 4094 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 4095 &src_iovs[2], 1, NULL, NULL, 4096 ut_sequence_step_cb, &completed); 4097 CU_ASSERT_EQUAL(rc, 0); 4098 4099 ut_seq.complete = false; 4100 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4101 4102 poll_threads(); 4103 CU_ASSERT_EQUAL(completed, 3); 4104 CU_ASSERT(ut_seq.complete); 4105 CU_ASSERT_EQUAL(ut_seq.status, 0); 4106 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 4107 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 4108 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 4109 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], 2048, ~0u)); 4110 CU_ASSERT_EQUAL(memcmp(buf, tmp[2], 2048), 0); 4111 CU_ASSERT_EQUAL(memcmp(&buf[2048], tmp[0], 2048), 0); 4112 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 4113 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 4114 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 4115 4116 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 4117 g_modules_opc[i] = modules[i]; 4118 } 4119 4120 ut_clear_operations(); 4121 spdk_put_io_channel(ioch); 4122 poll_threads(); 4123 } 4124 4125 static void 4126 test_sequence_dix_generate_verify(void) 4127 { 4128 struct spdk_accel_sequence *seq; 4129 struct spdk_io_channel *ioch; 4130 struct ut_sequence ut_seq = {}; 4131 char srcbuf[3][4096], dstbuf[3][4096]; 4132 struct iovec src_iovs[3], dst_iovs[3]; 4133 uint32_t block_size = 512, md_size = 8; 4134 struct spdk_dif_ctx_init_ext_opts dif_opts; 4135 struct spdk_dif_ctx dif_ctx = {}; 4136 struct spdk_dif_error dif_err; 4137 int i, rc, completed = 0; 4138 4139 ioch = spdk_accel_get_io_channel(); 4140 SPDK_CU_ASSERT_FATAL(ioch != NULL); 4141 4142 for (i = 0; i < 3; i++) { 4143 memset(srcbuf[i], 0xdead, sizeof(srcbuf[i])); 4144 memset(dstbuf[i], 0, sizeof(dstbuf[i])); 4145 src_iovs[i].iov_base = srcbuf[i]; 4146 src_iovs[i].iov_len = sizeof(srcbuf[i]); 4147 dst_iovs[i].iov_base = dstbuf[i]; 4148 dst_iovs[i].iov_len = sizeof(dstbuf[i]); 4149 } 4150 4151 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format); 4152 dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16; 4153 4154 rc = spdk_dif_ctx_init(&dif_ctx, block_size, md_size, false, true, SPDK_DIF_TYPE1, 4155 SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_APPTAG_CHECK | 4156 SPDK_DIF_FLAGS_REFTAG_CHECK, 10, 0xFFFF, 20, 0, 0, &dif_opts); 4157 SPDK_CU_ASSERT_FATAL(rc == 0); 4158 4159 seq = NULL; 4160 completed = 0; 4161 for (i = 0; i < 3; i++) { 4162 rc = spdk_accel_append_dix_generate(&seq, ioch, &dst_iovs[i], 1, NULL, NULL, 4163 &src_iovs[i], NULL, NULL, 4164 src_iovs[i].iov_len / block_size, &dif_ctx, 4165 ut_sequence_step_cb, &completed); 4166 CU_ASSERT_EQUAL(rc, 0); 4167 rc = spdk_accel_append_dix_verify(&seq, ioch, &dst_iovs[i], 1, NULL, NULL, 4168 &src_iovs[i], NULL, NULL, 4169 src_iovs[i].iov_len / block_size, &dif_ctx, 4170 &dif_err, ut_sequence_step_cb, &completed); 4171 CU_ASSERT_EQUAL(rc, 0); 4172 } 4173 4174 ut_seq.complete = false; 4175 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4176 4177 poll_threads(); 4178 CU_ASSERT_EQUAL(completed, 6); 4179 CU_ASSERT(ut_seq.complete); 4180 CU_ASSERT_EQUAL(ut_seq.status, 0); 4181 4182 /* DIX metadata should be equal for the same source buffers and tags */ 4183 CU_ASSERT_EQUAL(memcmp(dstbuf[0], dstbuf[1], 4096), 0); 4184 CU_ASSERT_EQUAL(memcmp(dstbuf[0], dstbuf[2], 4096), 0); 4185 4186 ut_clear_operations(); 4187 spdk_put_io_channel(ioch); 4188 poll_threads(); 4189 } 4190 4191 static void 4192 test_sequence_dix(void) 4193 { 4194 struct spdk_accel_sequence *seq = NULL; 4195 struct spdk_io_channel *ioch; 4196 struct ut_sequence ut_seq = {}; 4197 char buf[3][4096], md_buf[64]; 4198 struct iovec iovs[3], md_iov; 4199 uint32_t block_size = 512, md_size = 8; 4200 struct spdk_dif_ctx_init_ext_opts dif_opts; 4201 struct spdk_dif_ctx dif_ctx = {}; 4202 struct spdk_dif_error dif_err; 4203 struct spdk_accel_crypto_key *key; 4204 struct spdk_accel_crypto_key_create_param key_params = { 4205 .cipher = "AES_XTS", 4206 .hex_key = "00112233445566778899aabbccddeeff", 4207 .hex_key2 = "ffeeddccbbaa99887766554433221100", 4208 .key_name = "ut_key", 4209 }; 4210 int i, rc, completed = 0; 4211 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 4212 4213 ioch = spdk_accel_get_io_channel(); 4214 SPDK_CU_ASSERT_FATAL(ioch != NULL); 4215 4216 rc = spdk_accel_crypto_key_create(&key_params); 4217 CU_ASSERT_EQUAL(rc, 0); 4218 key = spdk_accel_crypto_key_get(key_params.key_name); 4219 SPDK_CU_ASSERT_FATAL(key != NULL); 4220 4221 /* Override the submit_tasks function. */ 4222 g_module_if.submit_tasks = ut_sequence_submit_tasks; 4223 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 4224 g_seq_operations[i].complete_status = 0; 4225 g_seq_operations[i].submit_status = 0; 4226 g_seq_operations[i].count = 0; 4227 4228 modules[i] = g_modules_opc[i]; 4229 g_modules_opc[i] = g_module; 4230 } 4231 4232 for (i = 0; i < 3; i++) { 4233 memset(buf[i], 0, sizeof(buf[i])); 4234 iovs[i].iov_base = buf[i]; 4235 iovs[i].iov_len = sizeof(buf[i]); 4236 } 4237 memset(md_buf, 0, sizeof(md_buf)); 4238 md_iov.iov_base = md_buf; 4239 md_iov.iov_len = sizeof(md_buf); 4240 4241 /* Prepare first source buffer. */ 4242 memset(iovs[0].iov_base, 0xde, iovs[0].iov_len); 4243 4244 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 4245 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 4246 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovcnt = 1; 4247 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovcnt = 1; 4248 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovs = &iovs[0]; 4249 /* Copy will be skipped, so the destination buffer of encrypt will change. */ 4250 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovs = &iovs[2]; 4251 4252 rc = spdk_accel_append_encrypt(&seq, ioch, key, &iovs[1], 1, NULL, NULL, 4253 &iovs[0], 1, NULL, NULL, 0, block_size, 4254 ut_sequence_step_cb, &completed); 4255 CU_ASSERT_EQUAL(rc, 0); 4256 4257 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format); 4258 dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16; 4259 4260 rc = spdk_dif_ctx_init(&dif_ctx, block_size, md_size, false, true, SPDK_DIF_TYPE1, 4261 SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_APPTAG_CHECK | 4262 SPDK_DIF_FLAGS_REFTAG_CHECK, 10, 0xFFFF, 20, 0, 0, &dif_opts); 4263 SPDK_CU_ASSERT_FATAL(rc == 0); 4264 4265 rc = spdk_accel_append_dix_generate(&seq, ioch, &iovs[1], 1, NULL, NULL, 4266 &md_iov, NULL, NULL, iovs[1].iov_len / block_size, 4267 &dif_ctx, ut_sequence_step_cb, &completed); 4268 CU_ASSERT_EQUAL(rc, 0); 4269 4270 rc = spdk_accel_append_copy(&seq, ioch, &iovs[2], 1, NULL, NULL, 4271 &iovs[1], 1, NULL, NULL, 4272 ut_sequence_step_cb, &completed); 4273 CU_ASSERT_EQUAL(rc, 0); 4274 4275 ut_seq.complete = false; 4276 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4277 4278 poll_threads(); 4279 4280 CU_ASSERT_EQUAL(completed, 3); 4281 CU_ASSERT(ut_seq.complete); 4282 CU_ASSERT_EQUAL(ut_seq.status, 0); 4283 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 4284 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 4285 4286 seq = NULL; 4287 completed = 0; 4288 4289 /* This time we start with first iovec containing encrypted data from previous sequence. */ 4290 memcpy(iovs[0].iov_base, iovs[2].iov_base, iovs[0].iov_len); 4291 4292 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 4293 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 4294 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 4295 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1; 4296 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &iovs[0]; 4297 /* Copy will be skipped, so the destination buffer of decrypt will change. */ 4298 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &iovs[2]; 4299 4300 rc = spdk_accel_append_decrypt(&seq, ioch, key, &iovs[1], 1, NULL, NULL, 4301 &iovs[0], 1, NULL, NULL, 0, block_size, 4302 ut_sequence_step_cb, &completed); 4303 CU_ASSERT_EQUAL(rc, 0); 4304 4305 rc = spdk_accel_append_dix_verify(&seq, ioch, &iovs[1], 1, NULL, NULL, &md_iov, NULL, NULL, 4306 iovs[1].iov_len / block_size, &dif_ctx, &dif_err, 4307 ut_sequence_step_cb, &completed); 4308 CU_ASSERT_EQUAL(rc, 0); 4309 4310 rc = spdk_accel_append_copy(&seq, ioch, &iovs[2], 1, NULL, NULL, &iovs[1], 1, NULL, NULL, 4311 ut_sequence_step_cb, &completed); 4312 CU_ASSERT_EQUAL(rc, 0); 4313 4314 ut_seq.complete = false; 4315 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4316 4317 poll_threads(); 4318 4319 CU_ASSERT_EQUAL(completed, 3); 4320 CU_ASSERT(ut_seq.complete); 4321 CU_ASSERT_EQUAL(ut_seq.status, 0); 4322 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 4323 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 4324 4325 ut_clear_operations(); 4326 4327 /* Cleanup module pointers to make subsequent tests work correctly. */ 4328 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 4329 g_modules_opc[i] = modules[i]; 4330 } 4331 4332 rc = spdk_accel_crypto_key_destroy(key); 4333 CU_ASSERT_EQUAL(rc, 0); 4334 4335 spdk_put_io_channel(ioch); 4336 poll_threads(); 4337 } 4338 4339 static int 4340 test_sequence_setup(void) 4341 { 4342 int rc; 4343 4344 allocate_cores(1); 4345 allocate_threads(1); 4346 set_thread(0); 4347 4348 rc = spdk_iobuf_initialize(); 4349 if (rc != 0) { 4350 CU_ASSERT(false); 4351 return -1; 4352 } 4353 4354 rc = spdk_accel_initialize(); 4355 if (rc != 0) { 4356 CU_ASSERT(false); 4357 return -1; 4358 } 4359 4360 g_module_if.name = "software"; 4361 g_module_if.compress_supports_algo = _supports_algo; 4362 g_module_if.get_compress_level_range = _get_compress_level_range; 4363 4364 return 0; 4365 } 4366 4367 static void 4368 finish_cb(void *cb_arg) 4369 { 4370 bool *done = cb_arg; 4371 4372 *done = true; 4373 } 4374 4375 static int 4376 test_sequence_cleanup(void) 4377 { 4378 bool done = false; 4379 4380 spdk_accel_finish(finish_cb, &done); 4381 4382 while (!done) { 4383 poll_threads(); 4384 } 4385 4386 done = false; 4387 spdk_iobuf_finish(finish_cb, &done); 4388 while (!done) { 4389 poll_threads(); 4390 } 4391 4392 free_threads(); 4393 free_cores(); 4394 4395 return 0; 4396 } 4397 4398 int 4399 main(int argc, char **argv) 4400 { 4401 CU_pSuite suite = NULL, seq_suite; 4402 unsigned int num_failures; 4403 4404 CU_initialize_registry(); 4405 4406 /* Sequence tests require accel to be initialized normally, so run them before the other 4407 * tests which register accel modules which aren't fully implemented, causing accel 4408 * initialization to fail. 4409 */ 4410 seq_suite = CU_add_suite("accel_sequence", test_sequence_setup, test_sequence_cleanup); 4411 CU_ADD_TEST(seq_suite, test_sequence_fill_copy); 4412 CU_ADD_TEST(seq_suite, test_sequence_abort); 4413 CU_ADD_TEST(seq_suite, test_sequence_append_error); 4414 CU_ADD_TEST(seq_suite, test_sequence_completion_error); 4415 #ifdef SPDK_CONFIG_ISAL /* accel_sw requires isa-l for compression */ 4416 CU_ADD_TEST(seq_suite, test_sequence_decompress); 4417 CU_ADD_TEST(seq_suite, test_sequence_reverse); 4418 #endif 4419 CU_ADD_TEST(seq_suite, test_sequence_copy_elision); 4420 CU_ADD_TEST(seq_suite, test_sequence_accel_buffers); 4421 CU_ADD_TEST(seq_suite, test_sequence_memory_domain); 4422 CU_ADD_TEST(seq_suite, test_sequence_module_memory_domain); 4423 #ifdef SPDK_CONFIG_ISAL_CRYPTO /* accel_sw requires isa-l-crypto for crypto operations */ 4424 CU_ADD_TEST(seq_suite, test_sequence_crypto); 4425 #endif 4426 CU_ADD_TEST(seq_suite, test_sequence_driver); 4427 CU_ADD_TEST(seq_suite, test_sequence_same_iovs); 4428 CU_ADD_TEST(seq_suite, test_sequence_crc32); 4429 CU_ADD_TEST(seq_suite, test_sequence_dix_generate_verify); 4430 CU_ADD_TEST(seq_suite, test_sequence_dix); 4431 4432 suite = CU_add_suite("accel", test_setup, test_cleanup); 4433 CU_ADD_TEST(suite, test_spdk_accel_task_complete); 4434 CU_ADD_TEST(suite, test_get_task); 4435 CU_ADD_TEST(suite, test_spdk_accel_submit_copy); 4436 CU_ADD_TEST(suite, test_spdk_accel_submit_dualcast); 4437 CU_ADD_TEST(suite, test_spdk_accel_submit_compare); 4438 CU_ADD_TEST(suite, test_spdk_accel_submit_fill); 4439 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c); 4440 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32cv); 4441 CU_ADD_TEST(suite, test_spdk_accel_submit_copy_crc32c); 4442 CU_ADD_TEST(suite, test_spdk_accel_submit_xor); 4443 CU_ADD_TEST(suite, test_spdk_accel_module_find_by_name); 4444 CU_ADD_TEST(suite, test_spdk_accel_module_register); 4445 4446 num_failures = spdk_ut_run_tests(argc, argv, NULL); 4447 CU_cleanup_registry(); 4448 4449 return num_failures; 4450 } 4451