1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2021 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk_internal/cunit.h" 8 #include "spdk_internal/mock.h" 9 #include "spdk/accel_module.h" 10 #include "thread/thread_internal.h" 11 #include "common/lib/ut_multithread.c" 12 #include "common/lib/test_iobuf.c" 13 #include "accel/accel.c" 14 #include "accel/accel_sw.c" 15 #include "unit/lib/json_mock.c" 16 17 DEFINE_STUB_V(spdk_memory_domain_destroy, (struct spdk_memory_domain *domain)); 18 DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, 19 (struct spdk_memory_domain *domain), "UT_DMA"); 20 DEFINE_STUB(spdk_memory_domain_get_system_domain, struct spdk_memory_domain *, (void), 21 (void *)0xabc); 22 DEFINE_STUB_V(spdk_memory_domain_set_invalidate, (struct spdk_memory_domain *domain, 23 spdk_memory_domain_invalidate_data_cb invalidate_cb)); 24 DEFINE_STUB_V(spdk_memory_domain_set_translation, (struct spdk_memory_domain *domain, 25 spdk_memory_domain_translate_memory_cb translate_cb)); 26 27 int 28 spdk_memory_domain_create(struct spdk_memory_domain **domain, enum spdk_dma_device_type type, 29 struct spdk_memory_domain_ctx *ctx, const char *id) 30 { 31 *domain = (void *)0xdeadbeef; 32 33 return 0; 34 } 35 36 struct ut_domain_ctx { 37 struct iovec iov; 38 struct iovec expected; 39 int pull_submit_status; 40 int push_submit_status; 41 int pull_complete_status; 42 int push_complete_status; 43 }; 44 45 static struct spdk_memory_domain *g_ut_domain = (void *)0xa55e1; 46 47 int 48 spdk_memory_domain_pull_data(struct spdk_memory_domain *sd, void *sctx, struct iovec *siov, 49 uint32_t siovcnt, struct iovec *diov, uint32_t diovcnt, 50 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 51 { 52 struct ut_domain_ctx *ctx = sctx; 53 54 CU_ASSERT_EQUAL(sd, g_ut_domain); 55 CU_ASSERT_EQUAL(siovcnt, 1); 56 CU_ASSERT_EQUAL(memcmp(siov, &ctx->expected, sizeof(*siov)), 0); 57 58 if (ctx->pull_submit_status != 0) { 59 return ctx->pull_submit_status; 60 } 61 62 if (ctx->pull_complete_status != 0) { 63 cpl_cb(cpl_arg, ctx->pull_complete_status); 64 return 0; 65 } 66 67 spdk_iovcpy(&ctx->iov, 1, diov, diovcnt); 68 cpl_cb(cpl_arg, 0); 69 70 return 0; 71 } 72 73 int 74 spdk_memory_domain_push_data(struct spdk_memory_domain *dd, void *dctx, struct iovec *diov, 75 uint32_t diovcnt, struct iovec *siov, uint32_t siovcnt, 76 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 77 { 78 struct ut_domain_ctx *ctx = dctx; 79 80 CU_ASSERT_EQUAL(dd, g_ut_domain); 81 CU_ASSERT_EQUAL(diovcnt, 1); 82 CU_ASSERT_EQUAL(memcmp(diov, &ctx->expected, sizeof(*diov)), 0); 83 84 if (ctx->push_submit_status != 0) { 85 return ctx->push_submit_status; 86 } 87 88 if (ctx->push_complete_status != 0) { 89 cpl_cb(cpl_arg, ctx->push_complete_status); 90 return 0; 91 } 92 93 spdk_iovcpy(siov, siovcnt, &ctx->iov, 1); 94 cpl_cb(cpl_arg, 0); 95 96 return 0; 97 } 98 99 /* global vars and setup/cleanup functions used for all test functions */ 100 struct spdk_accel_module_if g_module_if = {}; 101 struct accel_module g_module = { .module = &g_module_if }; 102 struct spdk_io_channel *g_ch = NULL; 103 struct accel_io_channel *g_accel_ch = NULL; 104 struct sw_accel_io_channel *g_sw_ch = NULL; 105 struct spdk_io_channel *g_module_ch = NULL; 106 107 static uint64_t g_opc_mask = 0; 108 109 static uint64_t 110 _accel_op_to_bit(enum spdk_accel_opcode opc) 111 { 112 return (1 << opc); 113 } 114 115 static bool 116 _supports_opcode(enum spdk_accel_opcode opc) 117 { 118 if (_accel_op_to_bit(opc) & g_opc_mask) { 119 return true; 120 } 121 return false; 122 } 123 124 static bool 125 _supports_algo(enum spdk_accel_comp_algo algo) 126 { 127 return true; 128 } 129 130 static int 131 _get_compress_level_range(enum spdk_accel_comp_algo algo, uint32_t *min_level, uint32_t *max_level) 132 { 133 *min_level = 0; 134 *max_level = UINT32_MAX; 135 136 return 0; 137 } 138 139 static int 140 test_setup(void) 141 { 142 int i; 143 144 g_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct accel_io_channel)); 145 if (g_ch == NULL) { 146 /* for some reason the assert fatal macro doesn't work in the setup function. */ 147 CU_ASSERT(false); 148 return -1; 149 } 150 g_accel_ch = (struct accel_io_channel *)((char *)g_ch + sizeof(struct spdk_io_channel)); 151 g_module_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct sw_accel_io_channel)); 152 if (g_module_ch == NULL) { 153 CU_ASSERT(false); 154 return -1; 155 } 156 157 g_module_if.submit_tasks = sw_accel_submit_tasks; 158 g_module_if.name = "software"; 159 g_module_if.compress_supports_algo = _supports_algo; 160 g_module_if.get_compress_level_range = _get_compress_level_range; 161 for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) { 162 g_accel_ch->module_ch[i] = g_module_ch; 163 g_modules_opc[i] = g_module; 164 } 165 g_sw_ch = (struct sw_accel_io_channel *)((char *)g_module_ch + sizeof( 166 struct spdk_io_channel)); 167 /* Prevent lazy initialization of poller. */ 168 g_sw_ch->completion_poller = (void *)0xdeadbeef; 169 STAILQ_INIT(&g_sw_ch->tasks_to_complete); 170 g_module_if.supports_opcode = _supports_opcode; 171 return 0; 172 } 173 174 static int 175 test_cleanup(void) 176 { 177 free(g_ch); 178 free(g_module_ch); 179 180 return 0; 181 } 182 183 #define DUMMY_ARG 0xDEADBEEF 184 static bool g_dummy_cb_called = false; 185 static void 186 dummy_cb_fn(void *cb_arg, int status) 187 { 188 CU_ASSERT(*(uint32_t *)cb_arg == DUMMY_ARG); 189 CU_ASSERT(status == 0); 190 g_dummy_cb_called = true; 191 } 192 193 static void 194 test_spdk_accel_task_complete(void) 195 { 196 struct spdk_accel_task accel_task = {}; 197 struct spdk_accel_task *expected_accel_task = NULL; 198 uint32_t cb_arg = DUMMY_ARG; 199 int status = 0; 200 201 accel_task.accel_ch = g_accel_ch; 202 accel_task.cb_fn = dummy_cb_fn; 203 accel_task.cb_arg = &cb_arg; 204 STAILQ_INIT(&g_accel_ch->task_pool); 205 206 /* Confirm cb is called and task added to list. */ 207 spdk_accel_task_complete(&accel_task, status); 208 CU_ASSERT(g_dummy_cb_called == true); 209 expected_accel_task = STAILQ_FIRST(&g_accel_ch->task_pool); 210 STAILQ_REMOVE_HEAD(&g_accel_ch->task_pool, link); 211 CU_ASSERT(expected_accel_task == &accel_task); 212 } 213 214 static void 215 test_get_task(void) 216 { 217 struct spdk_accel_task *task; 218 struct spdk_accel_task _task; 219 void *cb_arg = NULL; 220 221 STAILQ_INIT(&g_accel_ch->task_pool); 222 223 /* no tasks left, return NULL. */ 224 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 225 CU_ASSERT(task == NULL); 226 227 _task.cb_fn = dummy_cb_fn; 228 _task.cb_arg = cb_arg; 229 _task.accel_ch = g_accel_ch; 230 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &_task, link); 231 232 /* Get a valid task. */ 233 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 234 CU_ASSERT(task == &_task); 235 CU_ASSERT(_task.cb_fn == dummy_cb_fn); 236 CU_ASSERT(_task.cb_arg == cb_arg); 237 CU_ASSERT(_task.accel_ch == g_accel_ch); 238 } 239 240 #define TEST_SUBMIT_SIZE 64 241 static void 242 test_spdk_accel_submit_copy(void) 243 { 244 const uint64_t nbytes = TEST_SUBMIT_SIZE; 245 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 246 uint8_t src[TEST_SUBMIT_SIZE] = {0}; 247 void *cb_arg = NULL; 248 int rc; 249 struct spdk_accel_task task; 250 struct spdk_accel_task_aux_data task_aux; 251 struct spdk_accel_task *expected_accel_task = NULL; 252 253 STAILQ_INIT(&g_accel_ch->task_pool); 254 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 255 256 /* Fail with no tasks on _get_task() */ 257 rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, NULL, cb_arg); 258 CU_ASSERT(rc == -ENOMEM); 259 260 task.accel_ch = g_accel_ch; 261 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 262 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 263 264 /* submission OK. */ 265 rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, NULL, cb_arg); 266 CU_ASSERT(rc == 0); 267 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COPY); 268 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 269 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 270 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 271 CU_ASSERT(expected_accel_task == &task); 272 } 273 274 static void 275 test_spdk_accel_submit_dualcast(void) 276 { 277 void *dst1; 278 void *dst2; 279 void *src; 280 uint32_t align = ALIGN_4K; 281 uint64_t nbytes = TEST_SUBMIT_SIZE; 282 void *cb_arg = NULL; 283 int rc; 284 struct spdk_accel_task task; 285 struct spdk_accel_task_aux_data task_aux; 286 struct spdk_accel_task *expected_accel_task = NULL; 287 288 STAILQ_INIT(&g_accel_ch->task_pool); 289 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 290 291 /* Dualcast requires 4K alignment on dst addresses, 292 * hence using the hard coded address to test the buffer alignment 293 */ 294 dst1 = (void *)0x5000; 295 dst2 = (void *)0x60f0; 296 src = calloc(1, TEST_SUBMIT_SIZE); 297 SPDK_CU_ASSERT_FATAL(src != NULL); 298 memset(src, 0x5A, TEST_SUBMIT_SIZE); 299 300 /* This should fail since dst2 is not 4k aligned */ 301 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, NULL, cb_arg); 302 CU_ASSERT(rc == -EINVAL); 303 304 dst1 = (void *)0x7010; 305 dst2 = (void *)0x6000; 306 /* This should fail since dst1 is not 4k aligned */ 307 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, NULL, cb_arg); 308 CU_ASSERT(rc == -EINVAL); 309 310 /* Dualcast requires 4K alignment on dst addresses */ 311 dst1 = (void *)0x7000; 312 dst2 = (void *)0x6000; 313 /* Fail with no tasks on _get_task() */ 314 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, NULL, cb_arg); 315 CU_ASSERT(rc == -ENOMEM); 316 317 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 318 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 319 320 /* accel submission OK., since we test the SW path , need to use valid memory addresses 321 * cannot hardcode them anymore */ 322 dst1 = spdk_dma_zmalloc(nbytes, align, NULL); 323 SPDK_CU_ASSERT_FATAL(dst1 != NULL); 324 dst2 = spdk_dma_zmalloc(nbytes, align, NULL); 325 SPDK_CU_ASSERT_FATAL(dst2 != NULL); 326 /* SW module does the dualcast. */ 327 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, NULL, cb_arg); 328 CU_ASSERT(rc == 0); 329 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_DUALCAST); 330 CU_ASSERT(memcmp(dst1, src, TEST_SUBMIT_SIZE) == 0); 331 CU_ASSERT(memcmp(dst2, src, TEST_SUBMIT_SIZE) == 0); 332 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 333 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 334 CU_ASSERT(expected_accel_task == &task); 335 336 free(src); 337 spdk_free(dst1); 338 spdk_free(dst2); 339 } 340 341 static void 342 test_spdk_accel_submit_compare(void) 343 { 344 void *src1; 345 void *src2; 346 uint64_t nbytes = TEST_SUBMIT_SIZE; 347 void *cb_arg = NULL; 348 int rc; 349 struct spdk_accel_task task; 350 struct spdk_accel_task_aux_data task_aux; 351 struct spdk_accel_task *expected_accel_task = NULL; 352 353 STAILQ_INIT(&g_accel_ch->task_pool); 354 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 355 356 src1 = calloc(1, TEST_SUBMIT_SIZE); 357 SPDK_CU_ASSERT_FATAL(src1 != NULL); 358 src2 = calloc(1, TEST_SUBMIT_SIZE); 359 SPDK_CU_ASSERT_FATAL(src2 != NULL); 360 361 /* Fail with no tasks on _get_task() */ 362 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 363 CU_ASSERT(rc == -ENOMEM); 364 365 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 366 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 367 368 /* accel submission OK. */ 369 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 370 CU_ASSERT(rc == 0); 371 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COMPARE); 372 CU_ASSERT(memcmp(src1, src2, TEST_SUBMIT_SIZE) == 0); 373 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 374 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 375 CU_ASSERT(expected_accel_task == &task); 376 377 free(src1); 378 free(src2); 379 } 380 381 static void 382 test_spdk_accel_submit_fill(void) 383 { 384 void *dst; 385 void *src; 386 uint8_t fill = 0xf; 387 uint64_t fill64; 388 uint64_t nbytes = TEST_SUBMIT_SIZE; 389 void *cb_arg = NULL; 390 int rc; 391 struct spdk_accel_task task; 392 struct spdk_accel_task_aux_data task_aux; 393 struct spdk_accel_task *expected_accel_task = NULL; 394 395 STAILQ_INIT(&g_accel_ch->task_pool); 396 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 397 398 dst = calloc(1, TEST_SUBMIT_SIZE); 399 SPDK_CU_ASSERT_FATAL(dst != NULL); 400 src = calloc(1, TEST_SUBMIT_SIZE); 401 SPDK_CU_ASSERT_FATAL(src != NULL); 402 memset(src, fill, TEST_SUBMIT_SIZE); 403 memset(&fill64, fill, sizeof(uint64_t)); 404 405 /* Fail with no tasks on _get_task() */ 406 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, NULL, cb_arg); 407 CU_ASSERT(rc == -ENOMEM); 408 409 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 410 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 411 412 /* accel submission OK. */ 413 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, NULL, cb_arg); 414 CU_ASSERT(rc == 0); 415 CU_ASSERT(task.fill_pattern == fill64); 416 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_FILL); 417 418 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 419 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 420 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 421 CU_ASSERT(expected_accel_task == &task); 422 423 free(dst); 424 free(src); 425 } 426 427 static void 428 test_spdk_accel_submit_crc32c(void) 429 { 430 const uint64_t nbytes = TEST_SUBMIT_SIZE; 431 uint32_t crc_dst; 432 uint8_t src[TEST_SUBMIT_SIZE]; 433 uint32_t seed = 1; 434 void *cb_arg = NULL; 435 int rc; 436 struct spdk_accel_task task; 437 struct spdk_accel_task_aux_data task_aux; 438 struct spdk_accel_task *expected_accel_task = NULL; 439 440 STAILQ_INIT(&g_accel_ch->task_pool); 441 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 442 443 /* Fail with no tasks on _get_task() */ 444 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 445 CU_ASSERT(rc == -ENOMEM); 446 447 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 448 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 449 450 /* accel submission OK. */ 451 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 452 CU_ASSERT(rc == 0); 453 CU_ASSERT(task.crc_dst == &crc_dst); 454 CU_ASSERT(task.seed == seed); 455 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_CRC32C); 456 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 457 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 458 CU_ASSERT(expected_accel_task == &task); 459 } 460 461 static void 462 test_spdk_accel_submit_crc32cv(void) 463 { 464 uint32_t crc_dst; 465 uint32_t seed = 0; 466 uint32_t iov_cnt = 32; 467 void *cb_arg = NULL; 468 int rc; 469 uint32_t i = 0; 470 struct spdk_accel_task task; 471 struct spdk_accel_task_aux_data task_aux; 472 struct iovec iov[32]; 473 struct spdk_accel_task *expected_accel_task = NULL; 474 475 STAILQ_INIT(&g_accel_ch->task_pool); 476 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 477 478 for (i = 0; i < iov_cnt; i++) { 479 iov[i].iov_base = calloc(1, TEST_SUBMIT_SIZE); 480 SPDK_CU_ASSERT_FATAL(iov[i].iov_base != NULL); 481 iov[i].iov_len = TEST_SUBMIT_SIZE; 482 } 483 484 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 485 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 486 487 /* accel submission OK. */ 488 rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, NULL, cb_arg); 489 CU_ASSERT(rc == 0); 490 CU_ASSERT(task.s.iovs == iov); 491 CU_ASSERT(task.s.iovcnt == iov_cnt); 492 CU_ASSERT(task.crc_dst == &crc_dst); 493 CU_ASSERT(task.seed == seed); 494 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_CRC32C); 495 CU_ASSERT(task.cb_arg == cb_arg); 496 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 497 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 498 CU_ASSERT(expected_accel_task == &task); 499 500 for (i = 0; i < iov_cnt; i++) { 501 free(iov[i].iov_base); 502 } 503 } 504 505 static void 506 test_spdk_accel_submit_copy_crc32c(void) 507 { 508 const uint64_t nbytes = TEST_SUBMIT_SIZE; 509 uint32_t crc_dst; 510 uint8_t dst[TEST_SUBMIT_SIZE]; 511 uint8_t src[TEST_SUBMIT_SIZE]; 512 uint32_t seed = 0; 513 void *cb_arg = NULL; 514 int rc; 515 struct spdk_accel_task task; 516 struct spdk_accel_task_aux_data task_aux; 517 struct spdk_accel_task *expected_accel_task = NULL; 518 519 STAILQ_INIT(&g_accel_ch->task_pool); 520 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 521 522 /* Fail with no tasks on _get_task() */ 523 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, NULL, cb_arg); 524 CU_ASSERT(rc == -ENOMEM); 525 526 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 527 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 528 529 /* accel submission OK. */ 530 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, NULL, cb_arg); 531 CU_ASSERT(rc == 0); 532 CU_ASSERT(task.crc_dst == &crc_dst); 533 CU_ASSERT(task.seed == seed); 534 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COPY_CRC32C); 535 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 536 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 537 CU_ASSERT(expected_accel_task == &task); 538 } 539 540 static void 541 test_spdk_accel_submit_xor(void) 542 { 543 const uint64_t nbytes = TEST_SUBMIT_SIZE; 544 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 545 uint8_t src1[TEST_SUBMIT_SIZE] = {0}; 546 uint8_t src2[TEST_SUBMIT_SIZE] = {0}; 547 void *sources[] = { src1, src2 }; 548 uint32_t nsrcs = SPDK_COUNTOF(sources); 549 int rc; 550 struct spdk_accel_task task; 551 struct spdk_accel_task_aux_data task_aux; 552 struct spdk_accel_task *expected_accel_task = NULL; 553 554 STAILQ_INIT(&g_accel_ch->task_pool); 555 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 556 557 /* Fail with no tasks on _get_task() */ 558 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 559 CU_ASSERT(rc == -ENOMEM); 560 561 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 562 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 563 564 /* submission OK. */ 565 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 566 CU_ASSERT(rc == 0); 567 CU_ASSERT(task.nsrcs.srcs == sources); 568 CU_ASSERT(task.nsrcs.cnt == nsrcs); 569 CU_ASSERT(task.d.iovcnt == 1); 570 CU_ASSERT(task.d.iovs[0].iov_base == dst); 571 CU_ASSERT(task.d.iovs[0].iov_len == nbytes); 572 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_XOR); 573 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 574 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 575 CU_ASSERT(expected_accel_task == &task); 576 } 577 578 static void 579 test_spdk_accel_module_find_by_name(void) 580 { 581 struct spdk_accel_module_if mod1 = {}; 582 struct spdk_accel_module_if mod2 = {}; 583 struct spdk_accel_module_if mod3 = {}; 584 struct spdk_accel_module_if *accel_module = NULL; 585 586 mod1.name = "ioat"; 587 mod2.name = "idxd"; 588 mod3.name = "software"; 589 590 TAILQ_INIT(&spdk_accel_module_list); 591 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod1, tailq); 592 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod2, tailq); 593 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod3, tailq); 594 595 /* Now let's find a valid engine */ 596 accel_module = _module_find_by_name("ioat"); 597 CU_ASSERT(accel_module != NULL); 598 599 /* Try to find one that doesn't exist */ 600 accel_module = _module_find_by_name("XXX"); 601 CU_ASSERT(accel_module == NULL); 602 } 603 604 static int 605 ut_module_init_nop(void) 606 { 607 return 0; 608 } 609 610 static bool 611 ut_supports_opcode_all(enum spdk_accel_opcode opcode) 612 { 613 return true; 614 } 615 616 static void 617 ut_accel_module_priority_finish_done(void *done) 618 { 619 *(int *)done = 1; 620 } 621 622 static void 623 test_spdk_accel_module_register(void) 624 { 625 struct spdk_accel_module_if mods[] = { 626 { .name = "mod1", .priority = 1, }, 627 { .name = "mod3", .priority = 3, }, 628 { .name = "mod0", .priority = 0, }, 629 { .name = "mod2", .priority = 2, }, 630 }; 631 int rc, done = 0; 632 const char *modname = NULL; 633 size_t i; 634 635 allocate_cores(1); 636 allocate_threads(1); 637 set_thread(0); 638 639 TAILQ_INIT(&spdk_accel_module_list); 640 for (i = 0; i < SPDK_COUNTOF(mods); ++i) { 641 mods[i].module_init = ut_module_init_nop; 642 mods[i].supports_opcode = ut_supports_opcode_all; 643 spdk_accel_module_list_add(&mods[i]); 644 } 645 646 rc = spdk_accel_initialize(); 647 CU_ASSERT_EQUAL(rc, 0); 648 649 /* All opcodes should be assigned to the module with highest prio - mod3 */ 650 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 651 rc = spdk_accel_get_opc_module_name((enum spdk_accel_opcode)i, &modname); 652 CU_ASSERT_EQUAL(rc, 0); 653 CU_ASSERT_STRING_EQUAL(modname, "mod3"); 654 } 655 656 spdk_accel_finish(ut_accel_module_priority_finish_done, &done); 657 while (!done) { 658 poll_threads(); 659 } 660 661 TAILQ_INIT(&spdk_accel_module_list); 662 free_threads(); 663 free_cores(); 664 } 665 666 struct ut_sequence { 667 bool complete; 668 int status; 669 }; 670 671 static void 672 ut_sequence_step_cb(void *cb_arg) 673 { 674 int *completed = cb_arg; 675 676 (*completed)++; 677 } 678 679 static void 680 ut_sequence_complete_cb(void *cb_arg, int status) 681 { 682 struct ut_sequence *seq = cb_arg; 683 684 seq->complete = true; 685 seq->status = status; 686 } 687 688 static void 689 test_sequence_fill_copy(void) 690 { 691 struct spdk_accel_sequence *seq = NULL; 692 struct spdk_io_channel *ioch; 693 struct ut_sequence ut_seq; 694 char buf[4096], tmp[2][4096], expected[4096]; 695 struct iovec src_iovs[2], dst_iovs[2]; 696 int rc, completed; 697 698 ioch = spdk_accel_get_io_channel(); 699 SPDK_CU_ASSERT_FATAL(ioch != NULL); 700 701 /* First check the simplest case - single task in a sequence */ 702 memset(buf, 0, sizeof(buf)); 703 memset(expected, 0xa5, sizeof(expected)); 704 completed = 0; 705 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 706 ut_sequence_step_cb, &completed); 707 CU_ASSERT_EQUAL(rc, 0); 708 CU_ASSERT_EQUAL(completed, 0); 709 710 ut_seq.complete = false; 711 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 712 713 poll_threads(); 714 CU_ASSERT_EQUAL(completed, 1); 715 CU_ASSERT(ut_seq.complete); 716 CU_ASSERT_EQUAL(ut_seq.status, 0); 717 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 718 719 /* Check a single copy operation */ 720 memset(buf, 0, sizeof(buf)); 721 memset(tmp[0], 0xa5, sizeof(tmp[0])); 722 memset(expected, 0xa5, sizeof(expected)); 723 completed = 0; 724 seq = NULL; 725 726 dst_iovs[0].iov_base = buf; 727 dst_iovs[0].iov_len = sizeof(buf); 728 src_iovs[0].iov_base = tmp[0]; 729 src_iovs[0].iov_len = sizeof(tmp[0]); 730 731 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 732 &src_iovs[0], 1, NULL, NULL, 733 ut_sequence_step_cb, &completed); 734 CU_ASSERT_EQUAL(rc, 0); 735 736 ut_seq.complete = false; 737 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 738 739 poll_threads(); 740 CU_ASSERT_EQUAL(completed, 1); 741 CU_ASSERT(ut_seq.complete); 742 CU_ASSERT_EQUAL(ut_seq.status, 0); 743 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 744 745 /* Check multiple fill operations */ 746 memset(buf, 0, sizeof(buf)); 747 memset(expected, 0xfe, 4096); 748 memset(expected, 0xde, 2048); 749 memset(expected, 0xa5, 1024); 750 seq = NULL; 751 completed = 0; 752 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, NULL, NULL, 0xfe, 753 ut_sequence_step_cb, &completed); 754 CU_ASSERT_EQUAL(rc, 0); 755 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xde, 756 ut_sequence_step_cb, &completed); 757 CU_ASSERT_EQUAL(rc, 0); 758 rc = spdk_accel_append_fill(&seq, ioch, buf, 1024, NULL, NULL, 0xa5, 759 ut_sequence_step_cb, &completed); 760 CU_ASSERT_EQUAL(rc, 0); 761 762 ut_seq.complete = false; 763 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 764 765 poll_threads(); 766 CU_ASSERT_EQUAL(completed, 3); 767 CU_ASSERT(ut_seq.complete); 768 CU_ASSERT_EQUAL(ut_seq.status, 0); 769 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 770 771 /* Check multiple copy operations */ 772 memset(buf, 0, sizeof(buf)); 773 memset(tmp[0], 0, sizeof(tmp[0])); 774 memset(tmp[1], 0, sizeof(tmp[1])); 775 memset(expected, 0xa5, sizeof(expected)); 776 seq = NULL; 777 completed = 0; 778 779 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 780 ut_sequence_step_cb, &completed); 781 CU_ASSERT_EQUAL(rc, 0); 782 783 dst_iovs[0].iov_base = tmp[1]; 784 dst_iovs[0].iov_len = sizeof(tmp[1]); 785 src_iovs[0].iov_base = tmp[0]; 786 src_iovs[0].iov_len = sizeof(tmp[0]); 787 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 788 &src_iovs[0], 1, NULL, NULL, 789 ut_sequence_step_cb, &completed); 790 CU_ASSERT_EQUAL(rc, 0); 791 792 dst_iovs[1].iov_base = buf; 793 dst_iovs[1].iov_len = sizeof(buf); 794 src_iovs[1].iov_base = tmp[1]; 795 src_iovs[1].iov_len = sizeof(tmp[1]); 796 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 797 &src_iovs[1], 1, NULL, NULL, 798 ut_sequence_step_cb, &completed); 799 CU_ASSERT_EQUAL(rc, 0); 800 801 ut_seq.complete = false; 802 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 803 804 poll_threads(); 805 CU_ASSERT_EQUAL(completed, 3); 806 CU_ASSERT(ut_seq.complete); 807 CU_ASSERT_EQUAL(ut_seq.status, 0); 808 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 809 810 /* Check that adding a copy operation at the end will change destination buffer */ 811 memset(buf, 0, sizeof(buf)); 812 memset(tmp[0], 0, sizeof(tmp[0])); 813 memset(expected, 0xa5, sizeof(buf)); 814 seq = NULL; 815 completed = 0; 816 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 817 ut_sequence_step_cb, &completed); 818 CU_ASSERT_EQUAL(rc, 0); 819 820 dst_iovs[0].iov_base = buf; 821 dst_iovs[0].iov_len = sizeof(buf); 822 src_iovs[0].iov_base = tmp[0]; 823 src_iovs[0].iov_len = sizeof(tmp[0]); 824 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 825 &src_iovs[0], 1, NULL, NULL, 826 ut_sequence_step_cb, &completed); 827 CU_ASSERT_EQUAL(rc, 0); 828 829 ut_seq.complete = false; 830 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 831 832 poll_threads(); 833 CU_ASSERT_EQUAL(completed, 2); 834 CU_ASSERT(ut_seq.complete); 835 CU_ASSERT_EQUAL(ut_seq.status, 0); 836 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 837 838 /* Check that it's also possible to add copy operation at the beginning */ 839 memset(buf, 0, sizeof(buf)); 840 memset(tmp[0], 0xde, sizeof(tmp[0])); 841 memset(tmp[1], 0, sizeof(tmp[1])); 842 memset(expected, 0xa5, sizeof(expected)); 843 seq = NULL; 844 completed = 0; 845 846 dst_iovs[0].iov_base = tmp[1]; 847 dst_iovs[0].iov_len = sizeof(tmp[1]); 848 src_iovs[0].iov_base = tmp[0]; 849 src_iovs[0].iov_len = sizeof(tmp[0]); 850 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 851 &src_iovs[0], 1, NULL, NULL, 852 ut_sequence_step_cb, &completed); 853 CU_ASSERT_EQUAL(rc, 0); 854 855 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], sizeof(tmp[1]), NULL, NULL, 0xa5, 856 ut_sequence_step_cb, &completed); 857 CU_ASSERT_EQUAL(rc, 0); 858 859 dst_iovs[1].iov_base = buf; 860 dst_iovs[1].iov_len = sizeof(buf); 861 src_iovs[1].iov_base = tmp[1]; 862 src_iovs[1].iov_len = sizeof(tmp[1]); 863 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 864 &src_iovs[1], 1, NULL, NULL, 865 ut_sequence_step_cb, &completed); 866 CU_ASSERT_EQUAL(rc, 0); 867 868 ut_seq.complete = false; 869 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 870 871 poll_threads(); 872 CU_ASSERT_EQUAL(completed, 3); 873 CU_ASSERT(ut_seq.complete); 874 CU_ASSERT_EQUAL(ut_seq.status, 0); 875 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 876 877 spdk_put_io_channel(ioch); 878 poll_threads(); 879 } 880 881 static void 882 test_sequence_abort(void) 883 { 884 struct spdk_accel_sequence *seq = NULL; 885 struct spdk_io_channel *ioch; 886 char buf[4096], tmp[2][4096], expected[4096]; 887 struct iovec src_iovs[2], dst_iovs[2]; 888 int rc, completed; 889 890 ioch = spdk_accel_get_io_channel(); 891 SPDK_CU_ASSERT_FATAL(ioch != NULL); 892 893 /* Check that aborting a sequence calls operation's callback, the operation is not executed 894 * and the sequence is freed 895 */ 896 memset(buf, 0, sizeof(buf)); 897 memset(expected, 0, sizeof(buf)); 898 completed = 0; 899 seq = NULL; 900 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 901 ut_sequence_step_cb, &completed); 902 CU_ASSERT_EQUAL(rc, 0); 903 904 spdk_accel_sequence_abort(seq); 905 CU_ASSERT_EQUAL(completed, 1); 906 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 907 908 /* Check sequence with multiple operations */ 909 memset(buf, 0, sizeof(buf)); 910 memset(expected, 0, sizeof(buf)); 911 completed = 0; 912 seq = NULL; 913 914 dst_iovs[0].iov_base = tmp[1]; 915 dst_iovs[0].iov_len = sizeof(tmp[1]); 916 src_iovs[0].iov_base = tmp[0]; 917 src_iovs[0].iov_len = sizeof(tmp[0]); 918 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 919 &src_iovs[0], 1, NULL, NULL, 920 ut_sequence_step_cb, &completed); 921 CU_ASSERT_EQUAL(rc, 0); 922 923 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 4096, NULL, NULL, 0xa5, 924 ut_sequence_step_cb, &completed); 925 CU_ASSERT_EQUAL(rc, 0); 926 927 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xde, 928 ut_sequence_step_cb, &completed); 929 CU_ASSERT_EQUAL(rc, 0); 930 931 dst_iovs[1].iov_base = buf; 932 dst_iovs[1].iov_len = sizeof(buf); 933 src_iovs[1].iov_base = tmp[1]; 934 src_iovs[1].iov_len = sizeof(tmp[1]); 935 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 936 &src_iovs[1], 1, NULL, NULL, 937 ut_sequence_step_cb, &completed); 938 CU_ASSERT_EQUAL(rc, 0); 939 940 spdk_accel_sequence_abort(seq); 941 CU_ASSERT_EQUAL(completed, 4); 942 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 943 944 /* This should be a no-op */ 945 spdk_accel_sequence_abort(NULL); 946 947 spdk_put_io_channel(ioch); 948 poll_threads(); 949 } 950 951 static void 952 test_sequence_append_error(void) 953 { 954 struct spdk_accel_sequence *seq = NULL; 955 struct spdk_io_channel *ioch; 956 struct accel_io_channel *accel_ch; 957 struct iovec src_iovs, dst_iovs; 958 char buf[4096]; 959 STAILQ_HEAD(, spdk_accel_task) tasks = STAILQ_HEAD_INITIALIZER(tasks); 960 SLIST_HEAD(, spdk_accel_sequence) seqs = SLIST_HEAD_INITIALIZER(seqs); 961 int rc; 962 963 ioch = spdk_accel_get_io_channel(); 964 SPDK_CU_ASSERT_FATAL(ioch != NULL); 965 accel_ch = spdk_io_channel_get_ctx(ioch); 966 967 /* Check that append fails and no sequence object is allocated when there are no more free 968 * tasks */ 969 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 970 971 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 972 ut_sequence_step_cb, NULL); 973 CU_ASSERT_EQUAL(rc, -ENOMEM); 974 CU_ASSERT_PTR_NULL(seq); 975 976 dst_iovs.iov_base = buf; 977 dst_iovs.iov_len = 2048; 978 src_iovs.iov_base = &buf[2048]; 979 src_iovs.iov_len = 2048; 980 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 981 &src_iovs, 1, NULL, NULL, ut_sequence_step_cb, NULL); 982 CU_ASSERT_EQUAL(rc, -ENOMEM); 983 CU_ASSERT_PTR_NULL(seq); 984 985 dst_iovs.iov_base = buf; 986 dst_iovs.iov_len = 2048; 987 src_iovs.iov_base = &buf[2048]; 988 src_iovs.iov_len = 2048; 989 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 990 &src_iovs, 1, NULL, NULL, ut_sequence_step_cb, NULL); 991 CU_ASSERT_EQUAL(rc, -ENOMEM); 992 CU_ASSERT_PTR_NULL(seq); 993 994 /* Check that the same happens when the sequence queue is empty */ 995 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 996 SLIST_SWAP(&seqs, &accel_ch->seq_pool, spdk_accel_sequence); 997 998 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 999 ut_sequence_step_cb, NULL); 1000 CU_ASSERT_EQUAL(rc, -ENOMEM); 1001 CU_ASSERT_PTR_NULL(seq); 1002 1003 dst_iovs.iov_base = buf; 1004 dst_iovs.iov_len = 2048; 1005 src_iovs.iov_base = &buf[2048]; 1006 src_iovs.iov_len = 2048; 1007 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1008 &src_iovs, 1, NULL, NULL, ut_sequence_step_cb, NULL); 1009 CU_ASSERT_EQUAL(rc, -ENOMEM); 1010 CU_ASSERT_PTR_NULL(seq); 1011 1012 dst_iovs.iov_base = buf; 1013 dst_iovs.iov_len = 2048; 1014 src_iovs.iov_base = &buf[2048]; 1015 src_iovs.iov_len = 2048; 1016 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1017 &src_iovs, 1, NULL, NULL, ut_sequence_step_cb, NULL); 1018 CU_ASSERT_EQUAL(rc, -ENOMEM); 1019 CU_ASSERT_PTR_NULL(seq); 1020 1021 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 1022 1023 spdk_put_io_channel(ioch); 1024 poll_threads(); 1025 } 1026 1027 struct ut_sequence_operation { 1028 int complete_status; 1029 int submit_status; 1030 int count; 1031 struct iovec *src_iovs; 1032 uint32_t src_iovcnt; 1033 struct spdk_memory_domain *src_domain; 1034 void *src_domain_ctx; 1035 struct iovec *dst_iovs; 1036 uint32_t dst_iovcnt; 1037 struct spdk_memory_domain *dst_domain; 1038 void *dst_domain_ctx; 1039 int (*submit)(struct spdk_io_channel *ch, struct spdk_accel_task *t); 1040 }; 1041 1042 static struct ut_sequence_operation g_seq_operations[SPDK_ACCEL_OPC_LAST]; 1043 1044 static int 1045 ut_sequence_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *task) 1046 { 1047 struct ut_sequence_operation *op = &g_seq_operations[task->op_code]; 1048 1049 op->count++; 1050 if (op->submit != NULL) { 1051 return op->submit(ch, task); 1052 } 1053 if (op->src_iovs != NULL) { 1054 CU_ASSERT_EQUAL(task->s.iovcnt, op->src_iovcnt); 1055 CU_ASSERT_EQUAL(memcmp(task->s.iovs, op->src_iovs, 1056 sizeof(struct iovec) * op->src_iovcnt), 0); 1057 } 1058 if (op->dst_iovs != NULL) { 1059 CU_ASSERT_EQUAL(task->d.iovcnt, op->dst_iovcnt); 1060 CU_ASSERT_EQUAL(memcmp(task->d.iovs, op->dst_iovs, 1061 sizeof(struct iovec) * op->dst_iovcnt), 0); 1062 } 1063 1064 CU_ASSERT_EQUAL(task->src_domain, op->src_domain); 1065 CU_ASSERT_EQUAL(task->dst_domain, op->dst_domain); 1066 if (op->src_domain != NULL) { 1067 CU_ASSERT_EQUAL(task->src_domain_ctx, op->src_domain_ctx); 1068 } 1069 if (op->dst_domain != NULL) { 1070 CU_ASSERT_EQUAL(task->dst_domain_ctx, op->dst_domain_ctx); 1071 } 1072 1073 if (op->submit_status != 0) { 1074 return op->submit_status; 1075 } 1076 1077 spdk_accel_task_complete(task, op->complete_status); 1078 1079 return 0; 1080 } 1081 1082 static void 1083 ut_clear_operations(void) 1084 { 1085 memset(&g_seq_operations, 0, sizeof(g_seq_operations)); 1086 } 1087 1088 static void 1089 test_sequence_completion_error(void) 1090 { 1091 struct spdk_accel_sequence *seq = NULL; 1092 struct spdk_io_channel *ioch; 1093 struct ut_sequence ut_seq; 1094 struct iovec src_iovs, dst_iovs; 1095 char buf[4096], tmp[4096]; 1096 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 1097 int i, rc, completed; 1098 1099 ioch = spdk_accel_get_io_channel(); 1100 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1101 1102 /* Override the submit_tasks function */ 1103 g_module_if.submit_tasks = ut_sequence_submit_tasks; 1104 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1105 modules[i] = g_modules_opc[i]; 1106 g_modules_opc[i] = g_module; 1107 } 1108 1109 memset(buf, 0, sizeof(buf)); 1110 memset(tmp, 0, sizeof(tmp)); 1111 1112 /* Check that if the first operation completes with an error, the whole sequence is 1113 * completed with that error and that all operations' completion callbacks are executed 1114 */ 1115 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = -E2BIG; 1116 completed = 0; 1117 seq = NULL; 1118 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 1119 ut_sequence_step_cb, &completed); 1120 CU_ASSERT_EQUAL(rc, 0); 1121 1122 dst_iovs.iov_base = buf; 1123 dst_iovs.iov_len = sizeof(buf); 1124 src_iovs.iov_base = tmp; 1125 src_iovs.iov_len = sizeof(tmp); 1126 1127 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1128 &src_iovs, 1, NULL, NULL, 1129 ut_sequence_step_cb, &completed); 1130 CU_ASSERT_EQUAL(rc, 0); 1131 1132 ut_seq.complete = false; 1133 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1134 1135 poll_threads(); 1136 CU_ASSERT_EQUAL(completed, 2); 1137 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 1138 1139 /* Check the same with a second operation in the sequence */ 1140 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].complete_status = -EACCES; 1141 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 1142 completed = 0; 1143 seq = NULL; 1144 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 1145 ut_sequence_step_cb, &completed); 1146 CU_ASSERT_EQUAL(rc, 0); 1147 1148 dst_iovs.iov_base = buf; 1149 dst_iovs.iov_len = sizeof(buf); 1150 src_iovs.iov_base = tmp; 1151 src_iovs.iov_len = sizeof(tmp); 1152 1153 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1154 &src_iovs, 1, NULL, NULL, 1155 ut_sequence_step_cb, &completed); 1156 CU_ASSERT_EQUAL(rc, 0); 1157 1158 ut_seq.complete = false; 1159 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1160 1161 poll_threads(); 1162 CU_ASSERT_EQUAL(completed, 2); 1163 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 1164 1165 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].complete_status = 0; 1166 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 1167 1168 /* Check submission failure of the first operation */ 1169 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit_status = -EADDRINUSE; 1170 completed = 0; 1171 seq = NULL; 1172 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 1173 ut_sequence_step_cb, &completed); 1174 CU_ASSERT_EQUAL(rc, 0); 1175 1176 dst_iovs.iov_base = buf; 1177 dst_iovs.iov_len = sizeof(buf); 1178 src_iovs.iov_base = tmp; 1179 src_iovs.iov_len = sizeof(tmp); 1180 1181 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1182 &src_iovs, 1, NULL, NULL, 1183 ut_sequence_step_cb, &completed); 1184 CU_ASSERT_EQUAL(rc, 0); 1185 1186 ut_seq.complete = false; 1187 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1188 1189 poll_threads(); 1190 CU_ASSERT_EQUAL(completed, 2); 1191 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 1192 1193 /* Check the same with a second operation */ 1194 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit_status = -EADDRNOTAVAIL; 1195 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit_status = 0; 1196 completed = 0; 1197 seq = NULL; 1198 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 1199 ut_sequence_step_cb, &completed); 1200 CU_ASSERT_EQUAL(rc, 0); 1201 1202 dst_iovs.iov_base = buf; 1203 dst_iovs.iov_len = sizeof(buf); 1204 src_iovs.iov_base = tmp; 1205 src_iovs.iov_len = sizeof(tmp); 1206 1207 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1208 &src_iovs, 1, NULL, NULL, 1209 ut_sequence_step_cb, &completed); 1210 CU_ASSERT_EQUAL(rc, 0); 1211 1212 ut_seq.complete = false; 1213 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1214 1215 poll_threads(); 1216 CU_ASSERT_EQUAL(completed, 2); 1217 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 1218 1219 /* Cleanup module pointers to make subsequent tests work correctly */ 1220 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1221 g_modules_opc[i] = modules[i]; 1222 } 1223 1224 ut_clear_operations(); 1225 spdk_put_io_channel(ioch); 1226 poll_threads(); 1227 } 1228 1229 #ifdef SPDK_CONFIG_ISAL 1230 static void 1231 ut_compress_cb(void *cb_arg, int status) 1232 { 1233 int *completed = cb_arg; 1234 1235 CU_ASSERT_EQUAL(status, 0); 1236 1237 *completed = 1; 1238 } 1239 1240 static void 1241 test_sequence_decompress(void) 1242 { 1243 struct spdk_accel_sequence *seq = NULL; 1244 struct spdk_io_channel *ioch; 1245 struct ut_sequence ut_seq; 1246 char buf[4096], tmp[2][4096], expected[4096]; 1247 struct iovec src_iovs[2], dst_iovs[2]; 1248 uint32_t compressed_size; 1249 int rc, completed = 0; 1250 1251 ioch = spdk_accel_get_io_channel(); 1252 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1253 1254 memset(expected, 0xa5, sizeof(expected)); 1255 src_iovs[0].iov_base = expected; 1256 src_iovs[0].iov_len = sizeof(expected); 1257 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1258 &compressed_size, ut_compress_cb, &completed); 1259 CU_ASSERT_EQUAL(rc, 0); 1260 1261 while (!completed) { 1262 poll_threads(); 1263 } 1264 1265 /* Check a single decompress operation in a sequence */ 1266 seq = NULL; 1267 completed = 0; 1268 1269 dst_iovs[0].iov_base = buf; 1270 dst_iovs[0].iov_len = sizeof(buf); 1271 src_iovs[0].iov_base = tmp[0]; 1272 src_iovs[0].iov_len = compressed_size; 1273 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1274 &src_iovs[0], 1, NULL, NULL, 1275 ut_sequence_step_cb, &completed); 1276 CU_ASSERT_EQUAL(rc, 0); 1277 1278 ut_seq.complete = false; 1279 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1280 1281 poll_threads(); 1282 1283 CU_ASSERT_EQUAL(completed, 1); 1284 CU_ASSERT(ut_seq.complete); 1285 CU_ASSERT_EQUAL(ut_seq.status, 0); 1286 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1287 1288 /* Put the decompress operation in the middle of a sequence with a copy operation at the 1289 * beginning and a fill at the end modifying the first 2048B of the buffer. 1290 */ 1291 memset(expected, 0xfe, 2048); 1292 memset(buf, 0, sizeof(buf)); 1293 seq = NULL; 1294 completed = 0; 1295 1296 dst_iovs[0].iov_base = tmp[1]; 1297 dst_iovs[0].iov_len = compressed_size; 1298 src_iovs[0].iov_base = tmp[0]; 1299 src_iovs[0].iov_len = compressed_size; 1300 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1301 &src_iovs[0], 1, NULL, NULL, 1302 ut_sequence_step_cb, &completed); 1303 CU_ASSERT_EQUAL(rc, 0); 1304 1305 dst_iovs[1].iov_base = buf; 1306 dst_iovs[1].iov_len = sizeof(buf); 1307 src_iovs[1].iov_base = tmp[1]; 1308 src_iovs[1].iov_len = compressed_size; 1309 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1310 &src_iovs[1], 1, NULL, NULL, 1311 ut_sequence_step_cb, &completed); 1312 CU_ASSERT_EQUAL(rc, 0); 1313 1314 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 1315 ut_sequence_step_cb, &completed); 1316 CU_ASSERT_EQUAL(rc, 0); 1317 1318 ut_seq.complete = false; 1319 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1320 1321 poll_threads(); 1322 1323 CU_ASSERT_EQUAL(completed, 3); 1324 CU_ASSERT(ut_seq.complete); 1325 CU_ASSERT_EQUAL(ut_seq.status, 0); 1326 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1327 1328 /* Check sequence with decompress at the beginning: decompress -> copy */ 1329 memset(expected, 0xa5, sizeof(expected)); 1330 memset(buf, 0, sizeof(buf)); 1331 seq = NULL; 1332 completed = 0; 1333 1334 dst_iovs[0].iov_base = tmp[1]; 1335 dst_iovs[0].iov_len = sizeof(tmp[1]); 1336 src_iovs[0].iov_base = tmp[0]; 1337 src_iovs[0].iov_len = compressed_size; 1338 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1339 &src_iovs[0], 1, NULL, NULL, 1340 ut_sequence_step_cb, &completed); 1341 CU_ASSERT_EQUAL(rc, 0); 1342 1343 dst_iovs[1].iov_base = buf; 1344 dst_iovs[1].iov_len = sizeof(buf); 1345 src_iovs[1].iov_base = tmp[1]; 1346 src_iovs[1].iov_len = sizeof(tmp[1]); 1347 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1348 &src_iovs[1], 1, NULL, NULL, 1349 ut_sequence_step_cb, &completed); 1350 CU_ASSERT_EQUAL(rc, 0); 1351 1352 ut_seq.complete = false; 1353 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1354 1355 poll_threads(); 1356 1357 CU_ASSERT_EQUAL(completed, 2); 1358 CU_ASSERT(ut_seq.complete); 1359 CU_ASSERT_EQUAL(ut_seq.status, 0); 1360 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1361 1362 spdk_put_io_channel(ioch); 1363 poll_threads(); 1364 } 1365 1366 static void 1367 test_sequence_reverse(void) 1368 { 1369 struct spdk_accel_sequence *seq = NULL; 1370 struct spdk_io_channel *ioch; 1371 struct ut_sequence ut_seq; 1372 char buf[4096], tmp[2][4096], expected[4096]; 1373 struct iovec src_iovs[2], dst_iovs[2]; 1374 uint32_t compressed_size; 1375 int rc, completed = 0; 1376 1377 ioch = spdk_accel_get_io_channel(); 1378 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1379 1380 memset(expected, 0xa5, sizeof(expected)); 1381 src_iovs[0].iov_base = expected; 1382 src_iovs[0].iov_len = sizeof(expected); 1383 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1384 &compressed_size, ut_compress_cb, &completed); 1385 CU_ASSERT_EQUAL(rc, 0); 1386 1387 while (!completed) { 1388 poll_threads(); 1389 } 1390 1391 /* First check that reversing a sequence with a single operation is a no-op */ 1392 memset(buf, 0, sizeof(buf)); 1393 seq = NULL; 1394 completed = 0; 1395 1396 dst_iovs[0].iov_base = buf; 1397 dst_iovs[0].iov_len = sizeof(buf); 1398 src_iovs[0].iov_base = tmp[0]; 1399 src_iovs[0].iov_len = compressed_size; 1400 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1401 &src_iovs[0], 1, NULL, NULL, 1402 ut_sequence_step_cb, &completed); 1403 CU_ASSERT_EQUAL(rc, 0); 1404 1405 spdk_accel_sequence_reverse(seq); 1406 1407 ut_seq.complete = false; 1408 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1409 1410 poll_threads(); 1411 1412 CU_ASSERT_EQUAL(completed, 1); 1413 CU_ASSERT(ut_seq.complete); 1414 CU_ASSERT_EQUAL(ut_seq.status, 0); 1415 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1416 1417 /* Add a copy operation at the end with src set to the compressed data. After reverse(), 1418 * that copy operation should be first, so decompress() should receive compressed data in 1419 * its src buffer. 1420 */ 1421 memset(buf, 0, sizeof(buf)); 1422 memset(tmp[1], 0, sizeof(tmp[1])); 1423 seq = NULL; 1424 completed = 0; 1425 1426 dst_iovs[0].iov_base = buf; 1427 dst_iovs[0].iov_len = sizeof(buf); 1428 src_iovs[0].iov_base = tmp[1]; 1429 src_iovs[0].iov_len = compressed_size; 1430 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1431 &src_iovs[0], 1, NULL, NULL, 1432 ut_sequence_step_cb, &completed); 1433 CU_ASSERT_EQUAL(rc, 0); 1434 1435 dst_iovs[1].iov_base = tmp[1]; 1436 dst_iovs[1].iov_len = compressed_size; 1437 src_iovs[1].iov_base = tmp[0]; 1438 src_iovs[1].iov_len = compressed_size; 1439 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1440 &src_iovs[1], 1, NULL, NULL, 1441 ut_sequence_step_cb, &completed); 1442 CU_ASSERT_EQUAL(rc, 0); 1443 1444 spdk_accel_sequence_reverse(seq); 1445 1446 ut_seq.complete = false; 1447 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1448 1449 poll_threads(); 1450 1451 CU_ASSERT_EQUAL(completed, 2); 1452 CU_ASSERT(ut_seq.complete); 1453 CU_ASSERT_EQUAL(ut_seq.status, 0); 1454 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1455 1456 /* Check the same, but add an extra fill operation at the beginning that should execute last 1457 * after reverse(). 1458 */ 1459 memset(buf, 0, sizeof(buf)); 1460 memset(tmp[1], 0, sizeof(tmp[1])); 1461 memset(expected, 0xfe, 2048); 1462 seq = NULL; 1463 completed = 0; 1464 1465 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 1466 ut_sequence_step_cb, &completed); 1467 CU_ASSERT_EQUAL(rc, 0); 1468 1469 dst_iovs[0].iov_base = buf; 1470 dst_iovs[0].iov_len = sizeof(buf); 1471 src_iovs[0].iov_base = tmp[1]; 1472 src_iovs[0].iov_len = compressed_size; 1473 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1474 &src_iovs[0], 1, NULL, NULL, 1475 ut_sequence_step_cb, &completed); 1476 CU_ASSERT_EQUAL(rc, 0); 1477 1478 dst_iovs[1].iov_base = tmp[1]; 1479 dst_iovs[1].iov_len = compressed_size; 1480 src_iovs[1].iov_base = tmp[0]; 1481 src_iovs[1].iov_len = compressed_size; 1482 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1483 &src_iovs[1], 1, NULL, NULL, 1484 ut_sequence_step_cb, &completed); 1485 CU_ASSERT_EQUAL(rc, 0); 1486 1487 spdk_accel_sequence_reverse(seq); 1488 1489 ut_seq.complete = false; 1490 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1491 1492 poll_threads(); 1493 1494 CU_ASSERT_EQUAL(completed, 3); 1495 CU_ASSERT(ut_seq.complete); 1496 CU_ASSERT_EQUAL(ut_seq.status, 0); 1497 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1498 1499 /* Build the sequence in order and then reverse it twice */ 1500 memset(buf, 0, sizeof(buf)); 1501 memset(tmp[1], 0, sizeof(tmp[1])); 1502 seq = NULL; 1503 completed = 0; 1504 1505 dst_iovs[0].iov_base = tmp[1]; 1506 dst_iovs[0].iov_len = compressed_size; 1507 src_iovs[0].iov_base = tmp[0]; 1508 src_iovs[0].iov_len = compressed_size; 1509 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1510 &src_iovs[0], 1, NULL, NULL, 1511 ut_sequence_step_cb, &completed); 1512 CU_ASSERT_EQUAL(rc, 0); 1513 1514 dst_iovs[1].iov_base = buf; 1515 dst_iovs[1].iov_len = sizeof(buf); 1516 src_iovs[1].iov_base = tmp[1]; 1517 src_iovs[1].iov_len = compressed_size; 1518 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1519 &src_iovs[1], 1, NULL, NULL, 1520 ut_sequence_step_cb, &completed); 1521 CU_ASSERT_EQUAL(rc, 0); 1522 1523 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 1524 ut_sequence_step_cb, &completed); 1525 CU_ASSERT_EQUAL(rc, 0); 1526 1527 spdk_accel_sequence_reverse(seq); 1528 spdk_accel_sequence_reverse(seq); 1529 1530 ut_seq.complete = false; 1531 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1532 1533 poll_threads(); 1534 1535 CU_ASSERT_EQUAL(completed, 3); 1536 CU_ASSERT(ut_seq.complete); 1537 CU_ASSERT_EQUAL(ut_seq.status, 0); 1538 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1539 1540 spdk_put_io_channel(ioch); 1541 poll_threads(); 1542 } 1543 #endif 1544 1545 static void 1546 test_sequence_copy_elision(void) 1547 { 1548 struct spdk_accel_sequence *seq = NULL; 1549 struct spdk_io_channel *ioch; 1550 struct ut_sequence ut_seq; 1551 struct iovec src_iovs[4], dst_iovs[4], exp_iovs[2]; 1552 char buf[4096], tmp[4][4096]; 1553 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 1554 struct spdk_accel_crypto_key key = {}; 1555 int i, rc, completed; 1556 1557 ioch = spdk_accel_get_io_channel(); 1558 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1559 1560 /* Override the submit_tasks function */ 1561 g_module_if.submit_tasks = ut_sequence_submit_tasks; 1562 g_module.supports_memory_domains = true; 1563 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1564 g_seq_operations[i].complete_status = 0; 1565 g_seq_operations[i].submit_status = 0; 1566 g_seq_operations[i].count = 0; 1567 1568 modules[i] = g_modules_opc[i]; 1569 g_modules_opc[i] = g_module; 1570 } 1571 1572 /* Check that a copy operation at the beginning is removed */ 1573 seq = NULL; 1574 completed = 0; 1575 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovcnt = 1; 1576 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovcnt = 1; 1577 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1578 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1579 exp_iovs[0].iov_base = tmp[0]; 1580 exp_iovs[0].iov_len = sizeof(tmp[0]); 1581 exp_iovs[1].iov_base = buf; 1582 exp_iovs[1].iov_len = 2048; 1583 1584 dst_iovs[0].iov_base = tmp[1]; 1585 dst_iovs[0].iov_len = sizeof(tmp[1]); 1586 src_iovs[0].iov_base = tmp[0]; 1587 src_iovs[0].iov_len = sizeof(tmp[0]); 1588 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1589 &src_iovs[0], 1, NULL, NULL, 1590 ut_sequence_step_cb, &completed); 1591 CU_ASSERT_EQUAL(rc, 0); 1592 1593 dst_iovs[1].iov_base = buf; 1594 dst_iovs[1].iov_len = 2048; 1595 src_iovs[1].iov_base = tmp[1]; 1596 src_iovs[1].iov_len = sizeof(tmp[1]); 1597 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1598 &src_iovs[1], 1, NULL, NULL, 1599 ut_sequence_step_cb, &completed); 1600 CU_ASSERT_EQUAL(rc, 0); 1601 1602 ut_seq.complete = false; 1603 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1604 1605 poll_threads(); 1606 1607 CU_ASSERT_EQUAL(completed, 2); 1608 CU_ASSERT(ut_seq.complete); 1609 CU_ASSERT_EQUAL(ut_seq.status, 0); 1610 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1611 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1612 1613 /* Check that a copy operation at the end is removed too */ 1614 seq = NULL; 1615 completed = 0; 1616 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1617 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1618 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1619 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1620 exp_iovs[0].iov_base = tmp[0]; 1621 exp_iovs[0].iov_len = sizeof(tmp[0]); 1622 exp_iovs[1].iov_base = buf; 1623 exp_iovs[1].iov_len = 2048; 1624 1625 dst_iovs[0].iov_base = tmp[1]; 1626 dst_iovs[0].iov_len = 2048; 1627 src_iovs[0].iov_base = tmp[0]; 1628 src_iovs[0].iov_len = sizeof(tmp[0]); 1629 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1630 &src_iovs[0], 1, NULL, NULL, 1631 ut_sequence_step_cb, &completed); 1632 CU_ASSERT_EQUAL(rc, 0); 1633 1634 dst_iovs[1].iov_base = buf; 1635 dst_iovs[1].iov_len = 2048; 1636 src_iovs[1].iov_base = tmp[1]; 1637 src_iovs[1].iov_len = 2048; 1638 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1639 &src_iovs[1], 1, NULL, NULL, 1640 ut_sequence_step_cb, &completed); 1641 CU_ASSERT_EQUAL(rc, 0); 1642 1643 ut_seq.complete = false; 1644 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1645 1646 poll_threads(); 1647 1648 CU_ASSERT_EQUAL(completed, 2); 1649 CU_ASSERT(ut_seq.complete); 1650 CU_ASSERT_EQUAL(ut_seq.status, 0); 1651 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1652 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1653 1654 /* Check a copy operation both at the beginning and the end */ 1655 seq = NULL; 1656 completed = 0; 1657 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1658 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1659 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1660 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1661 exp_iovs[0].iov_base = tmp[0]; 1662 exp_iovs[0].iov_len = sizeof(tmp[0]); 1663 exp_iovs[1].iov_base = buf; 1664 exp_iovs[1].iov_len = 2048; 1665 1666 dst_iovs[0].iov_base = tmp[1]; 1667 dst_iovs[0].iov_len = sizeof(tmp[1]); 1668 src_iovs[0].iov_base = tmp[0]; 1669 src_iovs[0].iov_len = sizeof(tmp[0]); 1670 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1671 &src_iovs[0], 1, NULL, NULL, 1672 ut_sequence_step_cb, &completed); 1673 CU_ASSERT_EQUAL(rc, 0); 1674 1675 dst_iovs[1].iov_base = tmp[2]; 1676 dst_iovs[1].iov_len = 2048; 1677 src_iovs[1].iov_base = tmp[1]; 1678 src_iovs[1].iov_len = sizeof(tmp[1]); 1679 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1680 &src_iovs[1], 1, NULL, NULL, 1681 ut_sequence_step_cb, &completed); 1682 CU_ASSERT_EQUAL(rc, 0); 1683 1684 dst_iovs[2].iov_base = buf; 1685 dst_iovs[2].iov_len = 2048; 1686 src_iovs[2].iov_base = tmp[2]; 1687 src_iovs[2].iov_len = 2048; 1688 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1689 &src_iovs[2], 1, NULL, NULL, 1690 ut_sequence_step_cb, &completed); 1691 CU_ASSERT_EQUAL(rc, 0); 1692 1693 ut_seq.complete = false; 1694 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1695 1696 poll_threads(); 1697 1698 CU_ASSERT_EQUAL(completed, 3); 1699 CU_ASSERT(ut_seq.complete); 1700 CU_ASSERT_EQUAL(ut_seq.status, 0); 1701 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1702 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1703 1704 /* Check decompress + copy + decompress + copy */ 1705 seq = NULL; 1706 completed = 0; 1707 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1708 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1709 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = NULL; 1710 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = NULL; 1711 1712 dst_iovs[0].iov_base = tmp[1]; 1713 dst_iovs[0].iov_len = sizeof(tmp[1]); 1714 src_iovs[0].iov_base = tmp[0]; 1715 src_iovs[0].iov_len = sizeof(tmp[0]); 1716 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1717 &src_iovs[0], 1, NULL, NULL, 1718 ut_sequence_step_cb, &completed); 1719 CU_ASSERT_EQUAL(rc, 0); 1720 1721 dst_iovs[1].iov_base = tmp[2]; 1722 dst_iovs[1].iov_len = 2048; 1723 src_iovs[1].iov_base = tmp[1]; 1724 src_iovs[1].iov_len = sizeof(tmp[1]); 1725 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1726 &src_iovs[1], 1, NULL, NULL, 1727 ut_sequence_step_cb, &completed); 1728 CU_ASSERT_EQUAL(rc, 0); 1729 1730 dst_iovs[2].iov_base = tmp[3]; 1731 dst_iovs[2].iov_len = 1024; 1732 src_iovs[2].iov_base = tmp[2]; 1733 src_iovs[2].iov_len = 2048; 1734 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1735 &src_iovs[2], 1, NULL, NULL, 1736 ut_sequence_step_cb, &completed); 1737 CU_ASSERT_EQUAL(rc, 0); 1738 1739 dst_iovs[3].iov_base = buf; 1740 dst_iovs[3].iov_len = 1024; 1741 src_iovs[3].iov_base = tmp[3]; 1742 src_iovs[3].iov_len = 1024; 1743 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 1744 &src_iovs[3], 1, NULL, NULL, 1745 ut_sequence_step_cb, &completed); 1746 CU_ASSERT_EQUAL(rc, 0); 1747 1748 ut_seq.complete = false; 1749 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1750 1751 poll_threads(); 1752 1753 CU_ASSERT_EQUAL(completed, 4); 1754 CU_ASSERT(ut_seq.complete); 1755 CU_ASSERT_EQUAL(ut_seq.status, 0); 1756 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1757 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 2); 1758 1759 /* Check two copy operations - one of them should be removed, while the other should be 1760 * executed normally */ 1761 seq = NULL; 1762 completed = 0; 1763 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1764 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovcnt = 1; 1765 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovcnt = 1; 1766 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovs = &exp_iovs[0]; 1767 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovs = &exp_iovs[1]; 1768 exp_iovs[0].iov_base = tmp[0]; 1769 exp_iovs[0].iov_len = sizeof(tmp[0]); 1770 exp_iovs[1].iov_base = buf; 1771 exp_iovs[1].iov_len = sizeof(buf); 1772 1773 dst_iovs[0].iov_base = tmp[1]; 1774 dst_iovs[0].iov_len = sizeof(tmp[1]); 1775 src_iovs[0].iov_base = tmp[0]; 1776 src_iovs[0].iov_len = sizeof(tmp[0]); 1777 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1778 &src_iovs[0], 1, NULL, NULL, 1779 ut_sequence_step_cb, &completed); 1780 CU_ASSERT_EQUAL(rc, 0); 1781 1782 dst_iovs[1].iov_base = buf; 1783 dst_iovs[1].iov_len = sizeof(buf); 1784 src_iovs[1].iov_base = tmp[1]; 1785 src_iovs[1].iov_len = sizeof(tmp[1]); 1786 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1787 &src_iovs[1], 1, NULL, NULL, 1788 ut_sequence_step_cb, &completed); 1789 CU_ASSERT_EQUAL(rc, 0); 1790 1791 ut_seq.complete = false; 1792 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1793 1794 poll_threads(); 1795 1796 CU_ASSERT_EQUAL(completed, 2); 1797 CU_ASSERT(ut_seq.complete); 1798 CU_ASSERT_EQUAL(ut_seq.status, 0); 1799 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 1800 1801 /* Check fill + copy */ 1802 seq = NULL; 1803 completed = 0; 1804 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1805 g_seq_operations[SPDK_ACCEL_OPC_FILL].count = 0; 1806 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovs = NULL; 1807 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovs = NULL; 1808 g_seq_operations[SPDK_ACCEL_OPC_FILL].dst_iovcnt = 1; 1809 g_seq_operations[SPDK_ACCEL_OPC_FILL].dst_iovs = &exp_iovs[0]; 1810 exp_iovs[0].iov_base = buf; 1811 exp_iovs[0].iov_len = sizeof(buf); 1812 1813 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 1814 ut_sequence_step_cb, &completed); 1815 CU_ASSERT_EQUAL(rc, 0); 1816 1817 dst_iovs[0].iov_base = buf; 1818 dst_iovs[0].iov_len = sizeof(buf); 1819 src_iovs[0].iov_base = tmp[0]; 1820 src_iovs[0].iov_len = sizeof(tmp[0]); 1821 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1822 &src_iovs[0], 1, NULL, NULL, 1823 ut_sequence_step_cb, &completed); 1824 CU_ASSERT_EQUAL(rc, 0); 1825 1826 ut_seq.complete = false; 1827 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1828 1829 poll_threads(); 1830 1831 CU_ASSERT_EQUAL(completed, 2); 1832 CU_ASSERT(ut_seq.complete); 1833 CU_ASSERT_EQUAL(ut_seq.status, 0); 1834 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1835 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 1); 1836 1837 /* Check copy + encrypt + copy */ 1838 seq = NULL; 1839 completed = 0; 1840 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1841 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 1842 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovcnt = 1; 1843 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovcnt = 1; 1844 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovs = &exp_iovs[0]; 1845 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovs = &exp_iovs[1]; 1846 exp_iovs[0].iov_base = tmp[0]; 1847 exp_iovs[0].iov_len = sizeof(tmp[0]); 1848 exp_iovs[1].iov_base = buf; 1849 exp_iovs[1].iov_len = sizeof(buf); 1850 1851 dst_iovs[0].iov_base = tmp[1]; 1852 dst_iovs[0].iov_len = sizeof(tmp[1]); 1853 src_iovs[0].iov_base = tmp[0]; 1854 src_iovs[0].iov_len = sizeof(tmp[0]); 1855 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1856 &src_iovs[0], 1, NULL, NULL, 1857 ut_sequence_step_cb, &completed); 1858 CU_ASSERT_EQUAL(rc, 0); 1859 1860 dst_iovs[1].iov_base = tmp[2]; 1861 dst_iovs[1].iov_len = sizeof(tmp[2]); 1862 src_iovs[1].iov_base = tmp[1]; 1863 src_iovs[1].iov_len = sizeof(tmp[1]); 1864 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1865 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 1866 ut_sequence_step_cb, &completed); 1867 CU_ASSERT_EQUAL(rc, 0); 1868 1869 dst_iovs[2].iov_base = buf; 1870 dst_iovs[2].iov_len = sizeof(buf); 1871 src_iovs[2].iov_base = tmp[2]; 1872 src_iovs[2].iov_len = sizeof(tmp[2]); 1873 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1874 &src_iovs[2], 1, NULL, NULL, 1875 ut_sequence_step_cb, &completed); 1876 CU_ASSERT_EQUAL(rc, 0); 1877 1878 ut_seq.complete = false; 1879 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1880 1881 poll_threads(); 1882 1883 CU_ASSERT_EQUAL(completed, 3); 1884 CU_ASSERT(ut_seq.complete); 1885 CU_ASSERT_EQUAL(ut_seq.status, 0); 1886 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1887 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 1888 1889 /* Check copy + decrypt + copy */ 1890 seq = NULL; 1891 completed = 0; 1892 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1893 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 1894 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1895 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1896 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1897 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1898 exp_iovs[0].iov_base = tmp[0]; 1899 exp_iovs[0].iov_len = sizeof(tmp[0]); 1900 exp_iovs[1].iov_base = buf; 1901 exp_iovs[1].iov_len = sizeof(buf); 1902 1903 dst_iovs[0].iov_base = tmp[1]; 1904 dst_iovs[0].iov_len = sizeof(tmp[1]); 1905 src_iovs[0].iov_base = tmp[0]; 1906 src_iovs[0].iov_len = sizeof(tmp[0]); 1907 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1908 &src_iovs[0], 1, NULL, NULL, 1909 ut_sequence_step_cb, &completed); 1910 CU_ASSERT_EQUAL(rc, 0); 1911 1912 dst_iovs[1].iov_base = tmp[2]; 1913 dst_iovs[1].iov_len = sizeof(tmp[2]); 1914 src_iovs[1].iov_base = tmp[1]; 1915 src_iovs[1].iov_len = sizeof(tmp[1]); 1916 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1917 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 1918 ut_sequence_step_cb, &completed); 1919 CU_ASSERT_EQUAL(rc, 0); 1920 1921 dst_iovs[2].iov_base = buf; 1922 dst_iovs[2].iov_len = sizeof(buf); 1923 src_iovs[2].iov_base = tmp[2]; 1924 src_iovs[2].iov_len = sizeof(tmp[2]); 1925 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1926 &src_iovs[2], 1, NULL, NULL, 1927 ut_sequence_step_cb, &completed); 1928 CU_ASSERT_EQUAL(rc, 0); 1929 1930 ut_seq.complete = false; 1931 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1932 1933 poll_threads(); 1934 1935 CU_ASSERT_EQUAL(completed, 3); 1936 CU_ASSERT(ut_seq.complete); 1937 CU_ASSERT_EQUAL(ut_seq.status, 0); 1938 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1939 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 1940 1941 /* Check a sequence with memory domains and verify their pointers (and their contexts) are 1942 * correctly set on the next/prev task when a copy is removed */ 1943 seq = NULL; 1944 completed = 0; 1945 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1946 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 1947 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1948 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1949 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1950 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1951 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_domain = (void *)0x1; 1952 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_domain_ctx = (void *)0x2; 1953 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_domain = (void *)0x3; 1954 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_domain_ctx = (void *)0x4; 1955 exp_iovs[0].iov_base = tmp[0]; 1956 exp_iovs[0].iov_len = sizeof(tmp[0]); 1957 exp_iovs[1].iov_base = buf; 1958 exp_iovs[1].iov_len = sizeof(buf); 1959 1960 dst_iovs[0].iov_base = tmp[1]; 1961 dst_iovs[0].iov_len = sizeof(tmp[1]); 1962 src_iovs[0].iov_base = tmp[0]; 1963 src_iovs[0].iov_len = sizeof(tmp[0]); 1964 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, (void *)0xdead, (void *)0xbeef, 1965 &src_iovs[0], 1, (void *)0x3, (void *)0x4, 1966 ut_sequence_step_cb, &completed); 1967 CU_ASSERT_EQUAL(rc, 0); 1968 1969 dst_iovs[1].iov_base = tmp[2]; 1970 dst_iovs[1].iov_len = sizeof(tmp[2]); 1971 src_iovs[1].iov_base = tmp[1]; 1972 src_iovs[1].iov_len = sizeof(tmp[1]); 1973 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 1974 &src_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 0, 1975 sizeof(tmp[2]), ut_sequence_step_cb, &completed); 1976 CU_ASSERT_EQUAL(rc, 0); 1977 1978 dst_iovs[2].iov_base = buf; 1979 dst_iovs[2].iov_len = sizeof(buf); 1980 src_iovs[2].iov_base = tmp[2]; 1981 src_iovs[2].iov_len = sizeof(tmp[2]); 1982 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, (void *)0x1, (void *)0x2, 1983 &src_iovs[2], 1, (void *)0xdead, (void *)0xbeef, 1984 ut_sequence_step_cb, &completed); 1985 CU_ASSERT_EQUAL(rc, 0); 1986 1987 ut_seq.complete = false; 1988 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1989 1990 poll_threads(); 1991 1992 CU_ASSERT_EQUAL(completed, 3); 1993 CU_ASSERT(ut_seq.complete); 1994 CU_ASSERT_EQUAL(ut_seq.status, 0); 1995 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1996 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 1997 1998 /* Cleanup module pointers to make subsequent tests work correctly */ 1999 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2000 g_modules_opc[i] = modules[i]; 2001 } 2002 2003 g_module.supports_memory_domains = false; 2004 ut_clear_operations(); 2005 spdk_put_io_channel(ioch); 2006 poll_threads(); 2007 } 2008 2009 static int 2010 ut_submit_decompress(struct spdk_io_channel *ch, struct spdk_accel_task *task) 2011 { 2012 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 2013 2014 spdk_accel_task_complete(task, 0); 2015 2016 return 0; 2017 } 2018 2019 static void 2020 test_sequence_accel_buffers(void) 2021 { 2022 struct spdk_accel_sequence *seq = NULL; 2023 struct spdk_io_channel *ioch; 2024 struct accel_io_channel *accel_ch; 2025 struct ut_sequence ut_seq; 2026 struct iovec src_iovs[3], dst_iovs[3]; 2027 char srcbuf[4096], dstbuf[4096], expected[4096]; 2028 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2029 void *buf[2], *domain_ctx[2], *iobuf_buf; 2030 struct spdk_memory_domain *domain[2]; 2031 struct spdk_iobuf_buffer *cache_entry; 2032 spdk_iobuf_buffer_stailq_t small_cache; 2033 uint32_t small_cache_count; 2034 int i, rc, completed; 2035 struct spdk_iobuf_opts opts_iobuf = {}; 2036 2037 /* Set up the iobuf to always use the "small" pool */ 2038 opts_iobuf.large_bufsize = 0x20000; 2039 opts_iobuf.large_pool_count = 0; 2040 opts_iobuf.small_bufsize = 0x10000; 2041 opts_iobuf.small_pool_count = 32; 2042 2043 rc = spdk_iobuf_set_opts(&opts_iobuf); 2044 CU_ASSERT(rc == 0); 2045 2046 ioch = spdk_accel_get_io_channel(); 2047 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2048 2049 /* Override the submit_tasks function */ 2050 g_module_if.submit_tasks = ut_sequence_submit_tasks; 2051 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2052 modules[i] = g_modules_opc[i]; 2053 g_modules_opc[i] = g_module; 2054 } 2055 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2056 * decompress operations together in one sequence. 2057 */ 2058 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2059 g_seq_operations[SPDK_ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2060 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2061 2062 /* Check the simplest case: one operation using accel buffer as destination + copy operation 2063 * specifying the actual destination buffer 2064 */ 2065 memset(srcbuf, 0xa5, 4096); 2066 memset(dstbuf, 0x0, 4096); 2067 memset(expected, 0xa5, 4096); 2068 completed = 0; 2069 seq = NULL; 2070 2071 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2072 CU_ASSERT_EQUAL(rc, 0); 2073 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2074 2075 src_iovs[0].iov_base = srcbuf; 2076 src_iovs[0].iov_len = 4096; 2077 dst_iovs[0].iov_base = buf[0]; 2078 dst_iovs[0].iov_len = 4096; 2079 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2080 &src_iovs[0], 1, NULL, NULL, 2081 ut_sequence_step_cb, &completed); 2082 CU_ASSERT_EQUAL(rc, 0); 2083 2084 src_iovs[1].iov_base = buf[0]; 2085 src_iovs[1].iov_len = 4096; 2086 dst_iovs[1].iov_base = dstbuf; 2087 dst_iovs[1].iov_len = 4096; 2088 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2089 &src_iovs[1], 1, domain[0], domain_ctx[0], 2090 ut_sequence_step_cb, &completed); 2091 CU_ASSERT_EQUAL(rc, 0); 2092 2093 ut_seq.complete = false; 2094 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2095 2096 poll_threads(); 2097 2098 CU_ASSERT_EQUAL(completed, 2); 2099 CU_ASSERT(ut_seq.complete); 2100 CU_ASSERT_EQUAL(ut_seq.status, 0); 2101 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2102 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2103 2104 /* Start with a fill operation using accel buffer, followed by a decompress using another 2105 * accel buffer as dst, followed by a copy operation specifying dst buffer of the whole 2106 * sequence 2107 */ 2108 memset(srcbuf, 0x0, 4096); 2109 memset(dstbuf, 0x0, 4096); 2110 memset(expected, 0x5a, 4096); 2111 completed = 0; 2112 seq = NULL; 2113 2114 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2115 CU_ASSERT_EQUAL(rc, 0); 2116 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2117 2118 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 2119 ut_sequence_step_cb, &completed); 2120 CU_ASSERT_EQUAL(rc, 0); 2121 2122 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2123 CU_ASSERT_EQUAL(rc, 0); 2124 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2125 2126 src_iovs[0].iov_base = buf[0]; 2127 src_iovs[0].iov_len = 4096; 2128 dst_iovs[0].iov_base = buf[1]; 2129 dst_iovs[0].iov_len = 4096; 2130 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2131 &src_iovs[0], 1, domain[0], domain_ctx[0], 2132 ut_sequence_step_cb, &completed); 2133 CU_ASSERT_EQUAL(rc, 0); 2134 2135 src_iovs[1].iov_base = buf[1]; 2136 src_iovs[1].iov_len = 4096; 2137 dst_iovs[1].iov_base = dstbuf; 2138 dst_iovs[1].iov_len = 4096; 2139 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2140 &src_iovs[1], 1, domain[1], domain_ctx[1], 2141 ut_sequence_step_cb, &completed); 2142 2143 ut_seq.complete = false; 2144 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2145 2146 poll_threads(); 2147 2148 CU_ASSERT_EQUAL(completed, 3); 2149 CU_ASSERT(ut_seq.complete); 2150 CU_ASSERT_EQUAL(ut_seq.status, 0); 2151 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2152 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2153 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2154 2155 /* Check the same, but with two decompress operations with the first one being in-place */ 2156 memset(srcbuf, 0x0, 4096); 2157 memset(dstbuf, 0x0, 4096); 2158 memset(expected, 0x5a, 4096); 2159 completed = 0; 2160 seq = NULL; 2161 2162 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2163 CU_ASSERT_EQUAL(rc, 0); 2164 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2165 2166 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 2167 ut_sequence_step_cb, &completed); 2168 CU_ASSERT_EQUAL(rc, 0); 2169 2170 src_iovs[0].iov_base = buf[0]; 2171 src_iovs[0].iov_len = 4096; 2172 dst_iovs[0].iov_base = buf[0]; 2173 dst_iovs[0].iov_len = 4096; 2174 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2175 &src_iovs[0], 1, domain[0], domain_ctx[0], 2176 ut_sequence_step_cb, &completed); 2177 CU_ASSERT_EQUAL(rc, 0); 2178 2179 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2180 CU_ASSERT_EQUAL(rc, 0); 2181 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2182 2183 src_iovs[1].iov_base = buf[0]; 2184 src_iovs[1].iov_len = 4096; 2185 dst_iovs[1].iov_base = buf[1]; 2186 dst_iovs[1].iov_len = 4096; 2187 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, domain[1], domain_ctx[1], 2188 &src_iovs[1], 1, domain[0], domain_ctx[0], 2189 ut_sequence_step_cb, &completed); 2190 CU_ASSERT_EQUAL(rc, 0); 2191 2192 src_iovs[2].iov_base = buf[1]; 2193 src_iovs[2].iov_len = 4096; 2194 dst_iovs[2].iov_base = dstbuf; 2195 dst_iovs[2].iov_len = 4096; 2196 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 2197 &src_iovs[2], 1, domain[1], domain_ctx[1], 2198 ut_sequence_step_cb, &completed); 2199 2200 ut_seq.complete = false; 2201 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2202 2203 poll_threads(); 2204 2205 CU_ASSERT_EQUAL(completed, 4); 2206 CU_ASSERT(ut_seq.complete); 2207 CU_ASSERT_EQUAL(ut_seq.status, 0); 2208 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2209 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2210 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2211 2212 /* Check that specifying offsets within accel buffers works correctly */ 2213 memset(srcbuf, 0x0, 4096); 2214 memset(dstbuf, 0x0, 4096); 2215 completed = 0; 2216 seq = NULL; 2217 2218 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2219 CU_ASSERT_EQUAL(rc, 0); 2220 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2221 2222 /* Fill in each 1K of the buffer with different pattern */ 2223 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 2224 ut_sequence_step_cb, &completed); 2225 CU_ASSERT_EQUAL(rc, 0); 2226 2227 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2228 0x5a, ut_sequence_step_cb, &completed); 2229 CU_ASSERT_EQUAL(rc, 0); 2230 2231 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2232 0xfe, ut_sequence_step_cb, &completed); 2233 CU_ASSERT_EQUAL(rc, 0); 2234 2235 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2236 0xed, ut_sequence_step_cb, &completed); 2237 CU_ASSERT_EQUAL(rc, 0); 2238 2239 src_iovs[0].iov_base = buf[0]; 2240 src_iovs[0].iov_len = 4096; 2241 dst_iovs[0].iov_base = dstbuf; 2242 dst_iovs[0].iov_len = 4096; 2243 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2244 &src_iovs[0], 1, domain[0], domain_ctx[0], 2245 ut_sequence_step_cb, &completed); 2246 2247 ut_seq.complete = false; 2248 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2249 2250 poll_threads(); 2251 2252 memset(&expected[0], 0xa5, 1024); 2253 memset(&expected[1024], 0x5a, 1024); 2254 memset(&expected[2048], 0xfe, 1024); 2255 memset(&expected[3072], 0xed, 1024); 2256 CU_ASSERT_EQUAL(completed, 5); 2257 CU_ASSERT(ut_seq.complete); 2258 CU_ASSERT_EQUAL(ut_seq.status, 0); 2259 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2260 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2261 2262 /* Check the same but this time with a decompress operation on part of the buffer (512B 2263 * offset) */ 2264 memset(srcbuf, 0x0, 4096); 2265 memset(dstbuf, 0x0, 4096); 2266 completed = 0; 2267 seq = NULL; 2268 2269 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2270 CU_ASSERT_EQUAL(rc, 0); 2271 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2272 2273 /* Fill in each 1K of the buffer with different pattern */ 2274 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 2275 ut_sequence_step_cb, &completed); 2276 CU_ASSERT_EQUAL(rc, 0); 2277 2278 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2279 0x5a, ut_sequence_step_cb, &completed); 2280 CU_ASSERT_EQUAL(rc, 0); 2281 2282 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2283 0xfe, ut_sequence_step_cb, &completed); 2284 CU_ASSERT_EQUAL(rc, 0); 2285 2286 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2287 0xed, ut_sequence_step_cb, &completed); 2288 CU_ASSERT_EQUAL(rc, 0); 2289 2290 rc = spdk_accel_get_buf(ioch, 3072, &buf[1], &domain[1], &domain_ctx[1]); 2291 CU_ASSERT_EQUAL(rc, 0); 2292 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2293 2294 src_iovs[0].iov_base = (char *)buf[0] + 512; 2295 src_iovs[0].iov_len = 3072; 2296 dst_iovs[0].iov_base = (char *)buf[1] + 256; 2297 dst_iovs[0].iov_len = 3072; 2298 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2299 &src_iovs[0], 1, domain[0], domain_ctx[0], 2300 ut_sequence_step_cb, &completed); 2301 2302 src_iovs[1].iov_base = (char *)buf[1] + 256; 2303 src_iovs[1].iov_len = 3072; 2304 dst_iovs[1].iov_base = dstbuf; 2305 dst_iovs[1].iov_len = 3072; 2306 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2307 &src_iovs[1], 1, domain[1], domain_ctx[1], 2308 ut_sequence_step_cb, &completed); 2309 2310 ut_seq.complete = false; 2311 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2312 2313 poll_threads(); 2314 2315 memset(&expected[0], 0xa5, 512); 2316 memset(&expected[512], 0x5a, 1024); 2317 memset(&expected[1536], 0xfe, 1024); 2318 memset(&expected[2560], 0xed, 512); 2319 CU_ASSERT_EQUAL(completed, 6); 2320 CU_ASSERT(ut_seq.complete); 2321 CU_ASSERT_EQUAL(ut_seq.status, 0); 2322 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 3072), 0); 2323 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2324 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2325 2326 /* Check that if iobuf pool is empty, the sequence processing will wait until a buffer is 2327 * available 2328 */ 2329 accel_ch = spdk_io_channel_get_ctx(ioch); 2330 small_cache_count = accel_ch->iobuf.small.cache_count; 2331 STAILQ_INIT(&small_cache); 2332 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2333 accel_ch->iobuf.small.cache_count = 0; 2334 accel_ch->iobuf.small.cache_size = 0; 2335 g_iobuf.small_pool_count = 0; 2336 2337 /* First allocate a single buffer used by two operations */ 2338 memset(srcbuf, 0x0, 4096); 2339 memset(dstbuf, 0x0, 4096); 2340 memset(expected, 0xa5, 4096); 2341 completed = 0; 2342 seq = NULL; 2343 2344 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2345 CU_ASSERT_EQUAL(rc, 0); 2346 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2347 2348 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0xa5, 2349 ut_sequence_step_cb, &completed); 2350 CU_ASSERT_EQUAL(rc, 0); 2351 2352 src_iovs[0].iov_base = buf[0]; 2353 src_iovs[0].iov_len = 4096; 2354 dst_iovs[0].iov_base = dstbuf; 2355 dst_iovs[0].iov_len = 4096; 2356 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2357 &src_iovs[0], 1, domain[0], domain_ctx[0], 2358 ut_sequence_step_cb, &completed); 2359 2360 ut_seq.complete = false; 2361 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2362 2363 poll_threads(); 2364 2365 CU_ASSERT_EQUAL(completed, 0); 2366 CU_ASSERT(!ut_seq.complete); 2367 2368 /* Get a buffer and return it to the pool to trigger the sequence to finish */ 2369 g_iobuf.small_pool_count = 1; 2370 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2371 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2372 CU_ASSERT(g_iobuf.small_pool_count == 0); 2373 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2374 2375 poll_threads(); 2376 2377 CU_ASSERT_EQUAL(completed, 2); 2378 CU_ASSERT(ut_seq.complete); 2379 CU_ASSERT_EQUAL(ut_seq.status, 0); 2380 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2381 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2382 2383 /* Return the buffers back to the cache */ 2384 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2385 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2386 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2387 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2388 small_cache_count++; 2389 } 2390 accel_ch->iobuf.small.cache_count = 0; 2391 g_iobuf.small_pool_count = 0; 2392 2393 /* Check a bit more complex scenario, with two buffers in the sequence */ 2394 memset(srcbuf, 0x0, 4096); 2395 memset(dstbuf, 0x0, 4096); 2396 memset(expected, 0x5a, 4096); 2397 completed = 0; 2398 seq = NULL; 2399 2400 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2401 CU_ASSERT_EQUAL(rc, 0); 2402 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2403 2404 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 2405 ut_sequence_step_cb, &completed); 2406 CU_ASSERT_EQUAL(rc, 0); 2407 2408 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2409 CU_ASSERT_EQUAL(rc, 0); 2410 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2411 2412 src_iovs[0].iov_base = buf[0]; 2413 src_iovs[0].iov_len = 4096; 2414 dst_iovs[0].iov_base = buf[1]; 2415 dst_iovs[0].iov_len = 4096; 2416 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2417 &src_iovs[0], 1, domain[0], domain_ctx[0], 2418 ut_sequence_step_cb, &completed); 2419 CU_ASSERT_EQUAL(rc, 0); 2420 2421 src_iovs[1].iov_base = buf[1]; 2422 src_iovs[1].iov_len = 4096; 2423 dst_iovs[1].iov_base = dstbuf; 2424 dst_iovs[1].iov_len = 4096; 2425 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2426 &src_iovs[1], 1, domain[1], domain_ctx[1], 2427 ut_sequence_step_cb, &completed); 2428 CU_ASSERT_EQUAL(rc, 0); 2429 2430 ut_seq.complete = false; 2431 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2432 2433 poll_threads(); 2434 2435 CU_ASSERT_EQUAL(completed, 0); 2436 CU_ASSERT(!ut_seq.complete); 2437 2438 g_iobuf.small_pool_count = 1; 2439 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2440 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2441 g_iobuf.small_pool_count = 0; 2442 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2443 2444 /* One buffer is not enough to finish the whole sequence */ 2445 poll_threads(); 2446 CU_ASSERT(!ut_seq.complete); 2447 2448 g_iobuf.small_pool_count = 1; 2449 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2450 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2451 g_iobuf.small_pool_count = 0; 2452 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2453 2454 poll_threads(); 2455 2456 CU_ASSERT_EQUAL(completed, 3); 2457 CU_ASSERT(ut_seq.complete); 2458 CU_ASSERT_EQUAL(ut_seq.status, 0); 2459 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2460 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2461 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2462 2463 /* Return the buffers back to the cache */ 2464 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2465 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2466 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2467 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2468 small_cache_count++; 2469 } 2470 accel_ch->iobuf.small.cache_count = 0; 2471 2472 g_iobuf.small_pool_count = 32; 2473 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2474 accel_ch->iobuf.small.cache_count = small_cache_count; 2475 2476 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2477 g_modules_opc[i] = modules[i]; 2478 } 2479 2480 ut_clear_operations(); 2481 spdk_put_io_channel(ioch); 2482 poll_threads(); 2483 } 2484 2485 static void 2486 ut_domain_ctx_init(struct ut_domain_ctx *ctx, void *base, size_t len, struct iovec *expected) 2487 { 2488 ctx->iov.iov_base = base; 2489 ctx->iov.iov_len = len; 2490 ctx->expected = *expected; 2491 ctx->pull_submit_status = 0; 2492 ctx->push_submit_status = 0; 2493 ctx->pull_complete_status = 0; 2494 ctx->push_complete_status = 0; 2495 } 2496 2497 static void 2498 test_sequence_memory_domain(void) 2499 { 2500 struct spdk_accel_sequence *seq = NULL; 2501 struct spdk_io_channel *ioch; 2502 struct accel_io_channel *accel_ch; 2503 struct ut_sequence ut_seq; 2504 struct ut_domain_ctx domctx[4]; 2505 struct spdk_iobuf_buffer *cache_entry; 2506 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2507 struct spdk_memory_domain *accel_domain; 2508 spdk_iobuf_buffer_stailq_t small_cache; 2509 char srcbuf[4096], dstbuf[4096], expected[4096], tmp[4096]; 2510 struct iovec src_iovs[2], dst_iovs[2]; 2511 uint32_t small_cache_count; 2512 void *accel_buf, *accel_domain_ctx, *iobuf_buf; 2513 int i, rc, completed; 2514 2515 ioch = spdk_accel_get_io_channel(); 2516 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2517 2518 /* Override the submit_tasks function */ 2519 g_module_if.submit_tasks = ut_sequence_submit_tasks; 2520 g_module.supports_memory_domains = false; 2521 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2522 modules[i] = g_modules_opc[i]; 2523 g_modules_opc[i] = g_module; 2524 } 2525 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2526 * decompress operations together in one sequence. 2527 */ 2528 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2529 g_seq_operations[SPDK_ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2530 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2531 2532 /* First check the simplest case - single fill operation with dstbuf in domain */ 2533 memset(expected, 0xa5, sizeof(expected)); 2534 memset(dstbuf, 0x0, sizeof(dstbuf)); 2535 completed = 0; 2536 seq = NULL; 2537 2538 /* Use some garbage pointer as dst and use domain ctx to get the actual dstbuf */ 2539 dst_iovs[0].iov_base = (void *)0xcafebabe; 2540 dst_iovs[0].iov_len = sizeof(dstbuf); 2541 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2542 2543 rc = spdk_accel_append_fill(&seq, ioch, dst_iovs[0].iov_base, dst_iovs[0].iov_len, 2544 g_ut_domain, &domctx[0], 0xa5, 2545 ut_sequence_step_cb, &completed); 2546 CU_ASSERT_EQUAL(rc, 0); 2547 2548 ut_seq.complete = false; 2549 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2550 2551 poll_threads(); 2552 CU_ASSERT_EQUAL(completed, 1); 2553 CU_ASSERT(ut_seq.complete); 2554 CU_ASSERT_EQUAL(ut_seq.status, 0); 2555 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2556 2557 /* Check operation with both srcbuf and dstbuf in remote memory domain */ 2558 memset(expected, 0x5a, sizeof(expected)); 2559 memset(dstbuf, 0x0, sizeof(dstbuf)); 2560 memset(srcbuf, 0x5a, sizeof(dstbuf)); 2561 completed = 0; 2562 seq = NULL; 2563 2564 src_iovs[0].iov_base = (void *)0xcafebabe; 2565 src_iovs[0].iov_len = sizeof(srcbuf); 2566 dst_iovs[0].iov_base = (void *)0xbeefdead; 2567 dst_iovs[0].iov_len = sizeof(dstbuf); 2568 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2569 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2570 2571 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2572 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2573 ut_sequence_step_cb, &completed); 2574 CU_ASSERT_EQUAL(rc, 0); 2575 2576 ut_seq.complete = false; 2577 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2578 2579 poll_threads(); 2580 CU_ASSERT_EQUAL(completed, 1); 2581 CU_ASSERT(ut_seq.complete); 2582 CU_ASSERT_EQUAL(ut_seq.status, 0); 2583 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2584 2585 /* Two operations using a buffer in remote domain */ 2586 memset(expected, 0xa5, sizeof(expected)); 2587 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2588 memset(tmp, 0x0, sizeof(tmp)); 2589 memset(dstbuf, 0x0, sizeof(dstbuf)); 2590 completed = 0; 2591 seq = NULL; 2592 2593 src_iovs[0].iov_base = (void *)0xcafebabe; 2594 src_iovs[0].iov_len = sizeof(srcbuf); 2595 dst_iovs[0].iov_base = (void *)0xbeefdead; 2596 dst_iovs[0].iov_len = sizeof(tmp); 2597 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2598 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2599 2600 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2601 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2602 ut_sequence_step_cb, &completed); 2603 CU_ASSERT_EQUAL(rc, 0); 2604 2605 src_iovs[1].iov_base = (void *)0xbeefdead; 2606 src_iovs[1].iov_len = sizeof(tmp); 2607 dst_iovs[1].iov_base = (void *)0xa5a5a5a5; 2608 dst_iovs[1].iov_len = sizeof(dstbuf); 2609 ut_domain_ctx_init(&domctx[2], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2610 ut_domain_ctx_init(&domctx[3], tmp, sizeof(tmp), &src_iovs[1]); 2611 2612 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[2], 2613 &src_iovs[1], 1, g_ut_domain, &domctx[3], 2614 ut_sequence_step_cb, &completed); 2615 CU_ASSERT_EQUAL(rc, 0); 2616 2617 ut_seq.complete = false; 2618 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2619 2620 poll_threads(); 2621 CU_ASSERT_EQUAL(completed, 2); 2622 CU_ASSERT(ut_seq.complete); 2623 CU_ASSERT_EQUAL(ut_seq.status, 0); 2624 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2625 2626 /* Use buffer in remote memory domain and accel buffer at the same time */ 2627 memset(expected, 0x5a, sizeof(expected)); 2628 memset(srcbuf, 0x5a, sizeof(expected)); 2629 memset(dstbuf, 0x0, sizeof(dstbuf)); 2630 completed = 0; 2631 seq = NULL; 2632 2633 rc = spdk_accel_get_buf(ioch, sizeof(dstbuf), &accel_buf, &accel_domain, &accel_domain_ctx); 2634 CU_ASSERT_EQUAL(rc, 0); 2635 2636 src_iovs[0].iov_base = (void *)0xfeedbeef; 2637 src_iovs[0].iov_len = sizeof(srcbuf); 2638 dst_iovs[0].iov_base = accel_buf; 2639 dst_iovs[0].iov_len = sizeof(dstbuf); 2640 ut_domain_ctx_init(&domctx[0], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2641 2642 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, accel_domain, accel_domain_ctx, 2643 &src_iovs[0], 1, g_ut_domain, &domctx[0], 2644 ut_sequence_step_cb, &completed); 2645 CU_ASSERT_EQUAL(rc, 0); 2646 2647 src_iovs[1].iov_base = accel_buf; 2648 src_iovs[1].iov_len = sizeof(dstbuf); 2649 dst_iovs[1].iov_base = (void *)0xbeeffeed; 2650 dst_iovs[1].iov_len = sizeof(dstbuf); 2651 ut_domain_ctx_init(&domctx[1], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2652 2653 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[1], 2654 &src_iovs[1], 1, accel_domain, accel_domain_ctx, 2655 ut_sequence_step_cb, &completed); 2656 CU_ASSERT_EQUAL(rc, 0); 2657 2658 ut_seq.complete = false; 2659 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2660 2661 poll_threads(); 2662 CU_ASSERT_EQUAL(completed, 2); 2663 CU_ASSERT(ut_seq.complete); 2664 CU_ASSERT_EQUAL(ut_seq.status, 0); 2665 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2666 spdk_accel_put_buf(ioch, accel_buf, accel_domain, accel_domain_ctx); 2667 2668 /* Check that a sequence with memory domains is correctly executed if buffers are not 2669 * immediately available */ 2670 memset(expected, 0xa5, sizeof(expected)); 2671 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2672 memset(dstbuf, 0x0, sizeof(dstbuf)); 2673 completed = 0; 2674 seq = NULL; 2675 /* Make sure the buffer pool is empty */ 2676 accel_ch = spdk_io_channel_get_ctx(ioch); 2677 small_cache_count = accel_ch->iobuf.small.cache_count; 2678 STAILQ_INIT(&small_cache); 2679 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2680 accel_ch->iobuf.small.cache_count = 0; 2681 g_iobuf.small_pool_count = 0; 2682 2683 src_iovs[0].iov_base = (void *)0xdeadbeef; 2684 src_iovs[0].iov_len = sizeof(srcbuf); 2685 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2686 dst_iovs[0].iov_len = sizeof(dstbuf); 2687 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2688 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2689 2690 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2691 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2692 ut_sequence_step_cb, &completed); 2693 CU_ASSERT_EQUAL(rc, 0); 2694 2695 ut_seq.complete = false; 2696 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2697 2698 poll_threads(); 2699 CU_ASSERT_EQUAL(completed, 0); 2700 CU_ASSERT(!ut_seq.complete); 2701 2702 /* Get a buffer and return it to the pool to trigger the sequence to resume. It shouldn't 2703 * be able to complete, as it needs two buffers */ 2704 g_iobuf.small_pool_count = 1; 2705 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2706 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2707 g_iobuf.small_pool_count = 0; 2708 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2709 2710 CU_ASSERT_EQUAL(completed, 0); 2711 CU_ASSERT(!ut_seq.complete); 2712 2713 /* Return another buffer, this time the sequence should finish */ 2714 g_iobuf.small_pool_count = 1; 2715 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2716 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2717 g_iobuf.small_pool_count = 0; 2718 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2719 2720 poll_threads(); 2721 CU_ASSERT_EQUAL(completed, 1); 2722 CU_ASSERT(ut_seq.complete); 2723 CU_ASSERT_EQUAL(ut_seq.status, 0); 2724 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2725 2726 /* Return the buffers back to the cache */ 2727 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2728 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2729 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2730 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2731 small_cache_count++; 2732 } 2733 accel_ch->iobuf.small.cache_count = 0; 2734 2735 g_iobuf.small_pool_count = 32; 2736 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2737 accel_ch->iobuf.small.cache_count = small_cache_count; 2738 2739 /* Check error cases, starting with an error from spdk_memory_domain_pull_data() */ 2740 completed = 0; 2741 seq = NULL; 2742 2743 src_iovs[0].iov_base = (void *)0xdeadbeef; 2744 src_iovs[0].iov_len = sizeof(srcbuf); 2745 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2746 dst_iovs[0].iov_len = sizeof(dstbuf); 2747 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2748 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2749 domctx[1].pull_submit_status = -E2BIG; 2750 2751 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2752 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2753 ut_sequence_step_cb, &completed); 2754 CU_ASSERT_EQUAL(rc, 0); 2755 2756 ut_seq.complete = false; 2757 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2758 2759 CU_ASSERT_EQUAL(completed, 1); 2760 CU_ASSERT(ut_seq.complete); 2761 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 2762 2763 /* Check completion error from spdk_memory_domain_pull_data() */ 2764 completed = 0; 2765 seq = NULL; 2766 2767 src_iovs[0].iov_base = (void *)0xdeadbeef; 2768 src_iovs[0].iov_len = sizeof(srcbuf); 2769 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2770 dst_iovs[0].iov_len = sizeof(dstbuf); 2771 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2772 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2773 domctx[1].pull_complete_status = -EACCES; 2774 2775 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2776 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2777 ut_sequence_step_cb, &completed); 2778 CU_ASSERT_EQUAL(rc, 0); 2779 2780 ut_seq.complete = false; 2781 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2782 2783 CU_ASSERT_EQUAL(completed, 1); 2784 CU_ASSERT(ut_seq.complete); 2785 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 2786 2787 /* Check submission error from spdk_memory_domain_push_data() */ 2788 completed = 0; 2789 seq = NULL; 2790 2791 src_iovs[0].iov_base = (void *)0xdeadbeef; 2792 src_iovs[0].iov_len = sizeof(srcbuf); 2793 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2794 dst_iovs[0].iov_len = sizeof(dstbuf); 2795 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2796 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2797 domctx[0].push_submit_status = -EADDRINUSE; 2798 2799 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2800 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2801 ut_sequence_step_cb, &completed); 2802 CU_ASSERT_EQUAL(rc, 0); 2803 2804 ut_seq.complete = false; 2805 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2806 2807 CU_ASSERT_EQUAL(completed, 1); 2808 CU_ASSERT(ut_seq.complete); 2809 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 2810 2811 /* Check completion error from spdk_memory_domain_push_data() */ 2812 completed = 0; 2813 seq = NULL; 2814 2815 src_iovs[0].iov_base = (void *)0xdeadbeef; 2816 src_iovs[0].iov_len = sizeof(srcbuf); 2817 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2818 dst_iovs[0].iov_len = sizeof(dstbuf); 2819 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2820 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2821 domctx[0].push_complete_status = -EADDRNOTAVAIL; 2822 2823 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2824 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2825 ut_sequence_step_cb, &completed); 2826 CU_ASSERT_EQUAL(rc, 0); 2827 2828 ut_seq.complete = false; 2829 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2830 2831 CU_ASSERT_EQUAL(completed, 1); 2832 CU_ASSERT(ut_seq.complete); 2833 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 2834 2835 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2836 g_modules_opc[i] = modules[i]; 2837 } 2838 2839 ut_clear_operations(); 2840 spdk_put_io_channel(ioch); 2841 poll_threads(); 2842 } 2843 2844 static int 2845 ut_submit_decompress_memory_domain(struct spdk_io_channel *ch, struct spdk_accel_task *task) 2846 { 2847 struct ut_domain_ctx *ctx; 2848 struct iovec *src_iovs, *dst_iovs; 2849 uint32_t src_iovcnt, dst_iovcnt; 2850 2851 src_iovs = task->s.iovs; 2852 dst_iovs = task->d.iovs; 2853 src_iovcnt = task->s.iovcnt; 2854 dst_iovcnt = task->d.iovcnt; 2855 2856 if (task->src_domain != NULL) { 2857 ctx = task->src_domain_ctx; 2858 CU_ASSERT_EQUAL(memcmp(task->s.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2859 src_iovs = &ctx->iov; 2860 src_iovcnt = 1; 2861 } 2862 2863 if (task->dst_domain != NULL) { 2864 ctx = task->dst_domain_ctx; 2865 CU_ASSERT_EQUAL(memcmp(task->d.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2866 dst_iovs = &ctx->iov; 2867 dst_iovcnt = 1; 2868 } 2869 2870 spdk_iovcpy(src_iovs, src_iovcnt, dst_iovs, dst_iovcnt); 2871 spdk_accel_task_complete(task, 0); 2872 2873 return 0; 2874 } 2875 2876 static void 2877 test_sequence_module_memory_domain(void) 2878 { 2879 struct spdk_accel_sequence *seq = NULL; 2880 struct spdk_io_channel *ioch; 2881 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2882 struct spdk_memory_domain *accel_domain; 2883 struct ut_sequence ut_seq; 2884 struct ut_domain_ctx domctx[2]; 2885 struct iovec src_iovs[2], dst_iovs[2]; 2886 void *buf, *accel_domain_ctx; 2887 char srcbuf[4096], dstbuf[4096], tmp[4096], expected[4096]; 2888 int i, rc, completed; 2889 2890 ioch = spdk_accel_get_io_channel(); 2891 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2892 2893 /* Override the submit_tasks function */ 2894 g_module_if.submit_tasks = ut_sequence_submit_tasks; 2895 g_module.supports_memory_domains = true; 2896 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2897 modules[i] = g_modules_opc[i]; 2898 g_modules_opc[i] = g_module; 2899 } 2900 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress_memory_domain; 2901 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2902 2903 /* Check a sequence with both buffers in memory domains */ 2904 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2905 memset(expected, 0xa5, sizeof(expected)); 2906 memset(dstbuf, 0, sizeof(dstbuf)); 2907 seq = NULL; 2908 completed = 0; 2909 2910 src_iovs[0].iov_base = (void *)0xcafebabe; 2911 src_iovs[0].iov_len = sizeof(srcbuf); 2912 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2913 dst_iovs[0].iov_len = sizeof(dstbuf); 2914 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2915 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2916 2917 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2918 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2919 ut_sequence_step_cb, &completed); 2920 CU_ASSERT_EQUAL(rc, 0); 2921 2922 ut_seq.complete = false; 2923 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2924 2925 poll_threads(); 2926 2927 CU_ASSERT_EQUAL(completed, 1); 2928 CU_ASSERT(ut_seq.complete); 2929 CU_ASSERT_EQUAL(ut_seq.status, 0); 2930 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2931 2932 /* Check two operations each with a single buffer in memory domain */ 2933 memset(srcbuf, 0x5a, sizeof(srcbuf)); 2934 memset(expected, 0x5a, sizeof(expected)); 2935 memset(dstbuf, 0, sizeof(dstbuf)); 2936 memset(tmp, 0, sizeof(tmp)); 2937 seq = NULL; 2938 completed = 0; 2939 2940 src_iovs[0].iov_base = srcbuf; 2941 src_iovs[0].iov_len = sizeof(srcbuf); 2942 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2943 dst_iovs[0].iov_len = sizeof(tmp); 2944 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2945 2946 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2947 &src_iovs[0], 1, NULL, NULL, 2948 ut_sequence_step_cb, &completed); 2949 CU_ASSERT_EQUAL(rc, 0); 2950 2951 src_iovs[1].iov_base = (void *)0xfeedbeef; 2952 src_iovs[1].iov_len = sizeof(tmp); 2953 dst_iovs[1].iov_base = dstbuf; 2954 dst_iovs[1].iov_len = sizeof(dstbuf); 2955 ut_domain_ctx_init(&domctx[1], tmp, sizeof(tmp), &src_iovs[1]); 2956 2957 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2958 &src_iovs[1], 1, g_ut_domain, &domctx[1], 2959 ut_sequence_step_cb, &completed); 2960 CU_ASSERT_EQUAL(rc, 0); 2961 2962 ut_seq.complete = false; 2963 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2964 2965 poll_threads(); 2966 2967 CU_ASSERT_EQUAL(completed, 2); 2968 CU_ASSERT(ut_seq.complete); 2969 CU_ASSERT_EQUAL(ut_seq.status, 0); 2970 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2971 2972 /* Check a sequence with an accel buffer and a buffer in a regular memory domain */ 2973 memset(expected, 0xa5, sizeof(expected)); 2974 memset(dstbuf, 0, sizeof(dstbuf)); 2975 memset(tmp, 0, sizeof(tmp)); 2976 seq = NULL; 2977 completed = 0; 2978 2979 rc = spdk_accel_get_buf(ioch, 4096, &buf, &accel_domain, &accel_domain_ctx); 2980 CU_ASSERT_EQUAL(rc, 0); 2981 2982 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, accel_domain, accel_domain_ctx, 2983 0xa5, ut_sequence_step_cb, &completed); 2984 CU_ASSERT_EQUAL(rc, 0); 2985 2986 src_iovs[0].iov_base = buf; 2987 src_iovs[0].iov_len = 4096; 2988 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2989 dst_iovs[0].iov_len = sizeof(dstbuf); 2990 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2991 2992 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2993 &src_iovs[0], 1, accel_domain, accel_domain_ctx, 2994 ut_sequence_step_cb, &completed); 2995 CU_ASSERT_EQUAL(rc, 0); 2996 2997 ut_seq.complete = false; 2998 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2999 3000 poll_threads(); 3001 3002 CU_ASSERT_EQUAL(completed, 2); 3003 CU_ASSERT(ut_seq.complete); 3004 CU_ASSERT_EQUAL(ut_seq.status, 0); 3005 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 3006 3007 spdk_accel_put_buf(ioch, buf, accel_domain, accel_domain_ctx); 3008 3009 g_module.supports_memory_domains = false; 3010 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3011 g_modules_opc[i] = modules[i]; 3012 } 3013 3014 ut_clear_operations(); 3015 spdk_put_io_channel(ioch); 3016 poll_threads(); 3017 } 3018 3019 #ifdef SPDK_CONFIG_ISAL_CRYPTO 3020 static void 3021 ut_encrypt_cb(void *cb_arg, int status) 3022 { 3023 int *completed = cb_arg; 3024 3025 CU_ASSERT_EQUAL(status, 0); 3026 3027 *completed = 1; 3028 } 3029 3030 static void 3031 test_sequence_crypto(void) 3032 { 3033 struct spdk_accel_sequence *seq = NULL; 3034 struct spdk_io_channel *ioch; 3035 struct spdk_accel_crypto_key *key; 3036 struct spdk_accel_crypto_key_create_param key_params = { 3037 .cipher = "AES_XTS", 3038 .hex_key = "00112233445566778899aabbccddeeff", 3039 .hex_key2 = "ffeeddccbbaa99887766554433221100", 3040 .key_name = "ut_key", 3041 }; 3042 struct ut_sequence ut_seq; 3043 unsigned char buf[4096], encrypted[4096] = {}, data[4096], tmp[3][4096]; 3044 struct iovec src_iovs[4], dst_iovs[4]; 3045 int rc, completed = 0; 3046 size_t i; 3047 3048 ioch = spdk_accel_get_io_channel(); 3049 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3050 3051 rc = spdk_accel_crypto_key_create(&key_params); 3052 CU_ASSERT_EQUAL(rc, 0); 3053 key = spdk_accel_crypto_key_get(key_params.key_name); 3054 SPDK_CU_ASSERT_FATAL(key != NULL); 3055 3056 for (i = 0; i < sizeof(data); ++i) { 3057 data[i] = (uint8_t)i & 0xff; 3058 } 3059 3060 dst_iovs[0].iov_base = encrypted; 3061 dst_iovs[0].iov_len = sizeof(encrypted); 3062 src_iovs[0].iov_base = data; 3063 src_iovs[0].iov_len = sizeof(data); 3064 rc = spdk_accel_submit_encrypt(ioch, key, &dst_iovs[0], 1, &src_iovs[0], 1, 0, 4096, 3065 ut_encrypt_cb, &completed); 3066 CU_ASSERT_EQUAL(rc, 0); 3067 3068 while (!completed) { 3069 poll_threads(); 3070 } 3071 3072 /* Verify that encryption operation in a sequence produces the same result */ 3073 seq = NULL; 3074 completed = 0; 3075 3076 dst_iovs[0].iov_base = tmp[0]; 3077 dst_iovs[0].iov_len = sizeof(tmp[0]); 3078 src_iovs[0].iov_base = data; 3079 src_iovs[0].iov_len = sizeof(data); 3080 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3081 &src_iovs[0], 1, NULL, NULL, 3082 ut_sequence_step_cb, &completed); 3083 CU_ASSERT_EQUAL(rc, 0); 3084 3085 dst_iovs[1].iov_base = tmp[1]; 3086 dst_iovs[1].iov_len = sizeof(tmp[1]); 3087 src_iovs[1].iov_base = tmp[0]; 3088 src_iovs[1].iov_len = sizeof(tmp[0]); 3089 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3090 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3091 ut_sequence_step_cb, &completed); 3092 CU_ASSERT_EQUAL(rc, 0); 3093 3094 dst_iovs[2].iov_base = buf; 3095 dst_iovs[2].iov_len = sizeof(buf); 3096 src_iovs[2].iov_base = tmp[1]; 3097 src_iovs[2].iov_len = sizeof(tmp[1]); 3098 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3099 &src_iovs[2], 1, NULL, NULL, 3100 ut_sequence_step_cb, &completed); 3101 CU_ASSERT_EQUAL(rc, 0); 3102 3103 ut_seq.complete = false; 3104 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3105 3106 poll_threads(); 3107 3108 CU_ASSERT_EQUAL(completed, 3); 3109 CU_ASSERT(ut_seq.complete); 3110 CU_ASSERT_EQUAL(ut_seq.status, 0); 3111 CU_ASSERT_EQUAL(memcmp(buf, encrypted, sizeof(buf)), 0); 3112 3113 /* Check that decryption produces the original buffer */ 3114 seq = NULL; 3115 completed = 0; 3116 memset(buf, 0, sizeof(buf)); 3117 3118 dst_iovs[0].iov_base = tmp[0]; 3119 dst_iovs[0].iov_len = sizeof(tmp[0]); 3120 src_iovs[0].iov_base = encrypted; 3121 src_iovs[0].iov_len = sizeof(encrypted); 3122 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3123 &src_iovs[0], 1, NULL, NULL, 3124 ut_sequence_step_cb, &completed); 3125 CU_ASSERT_EQUAL(rc, 0); 3126 3127 dst_iovs[1].iov_base = tmp[1]; 3128 dst_iovs[1].iov_len = sizeof(tmp[1]); 3129 src_iovs[1].iov_base = tmp[0]; 3130 src_iovs[1].iov_len = sizeof(tmp[0]); 3131 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3132 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3133 ut_sequence_step_cb, &completed); 3134 CU_ASSERT_EQUAL(rc, 0); 3135 3136 dst_iovs[2].iov_base = buf; 3137 dst_iovs[2].iov_len = sizeof(buf); 3138 src_iovs[2].iov_base = tmp[1]; 3139 src_iovs[2].iov_len = sizeof(tmp[1]); 3140 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3141 &src_iovs[2], 1, NULL, NULL, 3142 ut_sequence_step_cb, &completed); 3143 CU_ASSERT_EQUAL(rc, 0); 3144 3145 ut_seq.complete = false; 3146 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3147 3148 poll_threads(); 3149 3150 CU_ASSERT_EQUAL(completed, 3); 3151 CU_ASSERT(ut_seq.complete); 3152 CU_ASSERT_EQUAL(ut_seq.status, 0); 3153 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3154 3155 /* Check encrypt + decrypt in a single sequence */ 3156 seq = NULL; 3157 completed = 0; 3158 memset(buf, 0, sizeof(buf)); 3159 3160 dst_iovs[0].iov_base = tmp[0]; 3161 dst_iovs[0].iov_len = sizeof(tmp[0]); 3162 src_iovs[0].iov_base = data; 3163 src_iovs[0].iov_len = sizeof(data); 3164 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3165 &src_iovs[0], 1, NULL, NULL, 3166 ut_sequence_step_cb, &completed); 3167 CU_ASSERT_EQUAL(rc, 0); 3168 3169 dst_iovs[1].iov_base = tmp[1]; 3170 dst_iovs[1].iov_len = sizeof(tmp[1]); 3171 src_iovs[1].iov_base = tmp[0]; 3172 src_iovs[1].iov_len = sizeof(tmp[0]); 3173 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3174 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3175 ut_sequence_step_cb, &completed); 3176 CU_ASSERT_EQUAL(rc, 0); 3177 3178 3179 dst_iovs[2].iov_base = tmp[2]; 3180 dst_iovs[2].iov_len = sizeof(tmp[2]); 3181 src_iovs[2].iov_base = tmp[1]; 3182 src_iovs[2].iov_len = sizeof(tmp[1]); 3183 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[2], 1, NULL, NULL, 3184 &src_iovs[2], 1, NULL, NULL, 0, 4096, 3185 ut_sequence_step_cb, &completed); 3186 CU_ASSERT_EQUAL(rc, 0); 3187 3188 dst_iovs[3].iov_base = buf; 3189 dst_iovs[3].iov_len = sizeof(buf); 3190 src_iovs[3].iov_base = tmp[2]; 3191 src_iovs[3].iov_len = sizeof(tmp[2]); 3192 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 3193 &src_iovs[3], 1, NULL, NULL, 3194 ut_sequence_step_cb, &completed); 3195 CU_ASSERT_EQUAL(rc, 0); 3196 3197 ut_seq.complete = false; 3198 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3199 3200 poll_threads(); 3201 3202 CU_ASSERT_EQUAL(completed, 4); 3203 CU_ASSERT(ut_seq.complete); 3204 CU_ASSERT_EQUAL(ut_seq.status, 0); 3205 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3206 3207 rc = spdk_accel_crypto_key_destroy(key); 3208 CU_ASSERT_EQUAL(rc, 0); 3209 spdk_put_io_channel(ioch); 3210 poll_threads(); 3211 } 3212 #endif /* SPDK_CONFIG_ISAL_CRYPTO */ 3213 3214 static int 3215 ut_submit_crypto(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3216 { 3217 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3218 3219 spdk_accel_task_complete(task, 0); 3220 3221 return 0; 3222 } 3223 3224 struct ut_driver_operation { 3225 int complete_status; 3226 int submit_status; 3227 int count; 3228 bool supported; 3229 }; 3230 3231 static struct ut_driver_operation g_drv_operations[SPDK_ACCEL_OPC_LAST]; 3232 static bool g_ut_driver_async_continue; 3233 3234 static void 3235 ut_driver_async_continue(void *arg) 3236 { 3237 struct spdk_accel_sequence *seq = arg; 3238 3239 spdk_accel_sequence_continue(seq); 3240 } 3241 3242 static int 3243 ut_driver_execute_sequence(struct spdk_io_channel *ch, struct spdk_accel_sequence *seq) 3244 { 3245 struct spdk_accel_task *task; 3246 struct ut_driver_operation *drv_ops; 3247 3248 while ((task = spdk_accel_sequence_first_task(seq)) != NULL) { 3249 drv_ops = &g_drv_operations[task->op_code]; 3250 if (!drv_ops->supported) { 3251 break; 3252 } 3253 3254 drv_ops->count++; 3255 if (drv_ops->submit_status != 0) { 3256 return drv_ops->submit_status; 3257 } 3258 3259 if (drv_ops->complete_status != 0) { 3260 spdk_accel_task_complete(task, drv_ops->complete_status); 3261 break; 3262 } 3263 3264 switch (task->op_code) { 3265 case SPDK_ACCEL_OPC_DECOMPRESS: 3266 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3267 break; 3268 case SPDK_ACCEL_OPC_FILL: 3269 spdk_iov_memset(task->d.iovs, task->d.iovcnt, 3270 (int)(task->fill_pattern & 0xff)); 3271 break; 3272 default: 3273 CU_ASSERT(0 && "unexpected opcode"); 3274 break; 3275 } 3276 3277 spdk_accel_task_complete(task, 0); 3278 } 3279 3280 if (g_ut_driver_async_continue) { 3281 spdk_thread_send_msg(spdk_get_thread(), ut_driver_async_continue, seq); 3282 } else { 3283 spdk_accel_sequence_continue(seq); 3284 } 3285 3286 return 0; 3287 } 3288 3289 static struct spdk_io_channel * 3290 ut_driver_get_io_channel(void) 3291 { 3292 return (void *)0xdeadbeef; 3293 } 3294 3295 static struct spdk_accel_driver g_ut_driver = { 3296 .name = "ut", 3297 .execute_sequence = ut_driver_execute_sequence, 3298 .get_io_channel = ut_driver_get_io_channel, 3299 }; 3300 3301 SPDK_ACCEL_DRIVER_REGISTER(ut, &g_ut_driver); 3302 3303 static void 3304 test_sequence_driver(void) 3305 { 3306 struct spdk_accel_sequence *seq = NULL; 3307 struct spdk_io_channel *ioch; 3308 struct spdk_accel_crypto_key key = {}; 3309 struct ut_sequence ut_seq; 3310 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3311 char buf[4096], tmp[3][4096], expected[4096]; 3312 struct iovec src_iovs[3], dst_iovs[3]; 3313 int i, rc, completed = 0; 3314 3315 ioch = spdk_accel_get_io_channel(); 3316 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3317 rc = spdk_accel_set_driver("ut"); 3318 SPDK_CU_ASSERT_FATAL(rc == 0); 3319 3320 /* Override the submit_tasks function */ 3321 g_module_if.submit_tasks = ut_sequence_submit_tasks; 3322 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3323 modules[i] = g_modules_opc[i]; 3324 g_modules_opc[i] = g_module; 3325 } 3326 /* Intercept crypto operations, as they should be executed by an accel module */ 3327 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].submit = ut_submit_crypto; 3328 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].submit = ut_submit_crypto; 3329 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3330 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3331 g_seq_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3332 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3333 3334 g_drv_operations[SPDK_ACCEL_OPC_FILL].supported = true; 3335 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].supported = true; 3336 3337 /* First check a sequence that is fully executed using a driver, with the copy at the end 3338 * being removed */ 3339 seq = NULL; 3340 completed = 0; 3341 memset(buf, 0, sizeof(buf)); 3342 memset(tmp[0], 0, sizeof(tmp[0])); 3343 memset(tmp[1], 0, sizeof(tmp[1])); 3344 memset(&expected[0], 0xa5, 2048); 3345 memset(&expected[2048], 0xbe, 2048); 3346 3347 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], 2048, NULL, NULL, 0xa5, 3348 ut_sequence_step_cb, &completed); 3349 CU_ASSERT_EQUAL(rc, 0); 3350 rc = spdk_accel_append_fill(&seq, ioch, &tmp[0][2048], 2048, NULL, NULL, 0xbe, 3351 ut_sequence_step_cb, &completed); 3352 CU_ASSERT_EQUAL(rc, 0); 3353 3354 dst_iovs[0].iov_base = tmp[1]; 3355 dst_iovs[0].iov_len = sizeof(tmp[1]); 3356 src_iovs[0].iov_base = tmp[0]; 3357 src_iovs[0].iov_len = sizeof(tmp[0]); 3358 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3359 &src_iovs[0], 1, NULL, NULL, 3360 ut_sequence_step_cb, &completed); 3361 CU_ASSERT_EQUAL(rc, 0); 3362 3363 dst_iovs[1].iov_base = buf; 3364 dst_iovs[1].iov_len = sizeof(buf); 3365 src_iovs[1].iov_base = tmp[1]; 3366 src_iovs[1].iov_len = sizeof(tmp[1]); 3367 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3368 &src_iovs[1], 1, NULL, NULL, 3369 ut_sequence_step_cb, &completed); 3370 CU_ASSERT_EQUAL(rc, 0); 3371 3372 ut_seq.complete = false; 3373 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3374 3375 poll_threads(); 3376 3377 CU_ASSERT_EQUAL(completed, 4); 3378 CU_ASSERT(ut_seq.complete); 3379 CU_ASSERT_EQUAL(ut_seq.status, 0); 3380 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3381 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3382 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 3383 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 2); 3384 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3385 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3386 3387 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3388 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3389 3390 /* Check a sequence when the first two operations are executed by a driver, while the rest 3391 * is executed via modules */ 3392 seq = NULL; 3393 completed = 0; 3394 memset(buf, 0, sizeof(buf)); 3395 memset(tmp[0], 0, sizeof(tmp[0])); 3396 memset(tmp[1], 0, sizeof(tmp[1])); 3397 memset(tmp[2], 0, sizeof(tmp[2])); 3398 memset(&expected[0], 0xfe, 4096); 3399 3400 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 3401 ut_sequence_step_cb, &completed); 3402 CU_ASSERT_EQUAL(rc, 0); 3403 3404 dst_iovs[0].iov_base = tmp[1]; 3405 dst_iovs[0].iov_len = sizeof(tmp[1]); 3406 src_iovs[0].iov_base = tmp[0]; 3407 src_iovs[0].iov_len = sizeof(tmp[0]); 3408 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3409 &src_iovs[0], 1, NULL, NULL, 3410 ut_sequence_step_cb, &completed); 3411 CU_ASSERT_EQUAL(rc, 0); 3412 3413 dst_iovs[1].iov_base = tmp[2]; 3414 dst_iovs[1].iov_len = sizeof(tmp[2]); 3415 src_iovs[1].iov_base = tmp[1]; 3416 src_iovs[1].iov_len = sizeof(tmp[1]); 3417 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3418 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3419 ut_sequence_step_cb, &completed); 3420 CU_ASSERT_EQUAL(rc, 0); 3421 3422 dst_iovs[2].iov_base = buf; 3423 dst_iovs[2].iov_len = sizeof(buf); 3424 src_iovs[2].iov_base = tmp[2]; 3425 src_iovs[2].iov_len = sizeof(tmp[2]); 3426 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3427 &src_iovs[2], 1, NULL, NULL, 0, 4096, 3428 ut_sequence_step_cb, &completed); 3429 CU_ASSERT_EQUAL(rc, 0); 3430 3431 ut_seq.complete = false; 3432 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3433 3434 poll_threads(); 3435 3436 CU_ASSERT_EQUAL(completed, 4); 3437 CU_ASSERT(ut_seq.complete); 3438 CU_ASSERT_EQUAL(ut_seq.status, 0); 3439 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3440 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3441 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3442 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3443 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3444 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3445 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3446 3447 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3448 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3449 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3450 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3451 3452 /* Check sequence when the first and last operations are executed through modules, while the 3453 * ones in the middle are executed by the driver */ 3454 seq = NULL; 3455 completed = 0; 3456 memset(buf, 0, sizeof(buf)); 3457 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3458 memset(tmp[1], 0, sizeof(tmp[1])); 3459 memset(tmp[2], 0, sizeof(tmp[2])); 3460 memset(&expected[0], 0xfe, 2048); 3461 memset(&expected[2048], 0xa5, 2048); 3462 3463 dst_iovs[0].iov_base = tmp[1]; 3464 dst_iovs[0].iov_len = sizeof(tmp[1]); 3465 src_iovs[0].iov_base = tmp[0]; 3466 src_iovs[0].iov_len = sizeof(tmp[0]); 3467 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3468 &src_iovs[0], 1, NULL, NULL, 0, 4096, 3469 ut_sequence_step_cb, &completed); 3470 CU_ASSERT_EQUAL(rc, 0); 3471 3472 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xfe, 3473 ut_sequence_step_cb, &completed); 3474 CU_ASSERT_EQUAL(rc, 0); 3475 3476 dst_iovs[1].iov_base = tmp[2]; 3477 dst_iovs[1].iov_len = sizeof(tmp[2]); 3478 src_iovs[1].iov_base = tmp[1]; 3479 src_iovs[1].iov_len = sizeof(tmp[1]); 3480 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3481 &src_iovs[1], 1, NULL, NULL, 3482 ut_sequence_step_cb, &completed); 3483 CU_ASSERT_EQUAL(rc, 0); 3484 3485 dst_iovs[2].iov_base = buf; 3486 dst_iovs[2].iov_len = sizeof(buf); 3487 src_iovs[2].iov_base = tmp[2]; 3488 src_iovs[2].iov_len = sizeof(tmp[2]); 3489 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3490 &src_iovs[2], 1, NULL, NULL, 0, 4096, 3491 ut_sequence_step_cb, &completed); 3492 CU_ASSERT_EQUAL(rc, 0); 3493 3494 ut_seq.complete = false; 3495 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3496 3497 poll_threads(); 3498 3499 CU_ASSERT_EQUAL(completed, 4); 3500 CU_ASSERT(ut_seq.complete); 3501 CU_ASSERT_EQUAL(ut_seq.status, 0); 3502 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3503 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3504 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3505 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3506 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3507 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3508 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3509 3510 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3511 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3512 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3513 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3514 3515 /* Check a sequence with operations executed by: module, driver, module, driver */ 3516 seq = NULL; 3517 completed = 0; 3518 memset(buf, 0, sizeof(buf)); 3519 memset(tmp[0], 0x5a, sizeof(tmp[0])); 3520 memset(tmp[1], 0, sizeof(tmp[1])); 3521 memset(tmp[2], 0, sizeof(tmp[2])); 3522 memset(&expected[0], 0xef, 2048); 3523 memset(&expected[2048], 0x5a, 2048); 3524 3525 dst_iovs[0].iov_base = tmp[1]; 3526 dst_iovs[0].iov_len = sizeof(tmp[1]); 3527 src_iovs[0].iov_base = tmp[0]; 3528 src_iovs[0].iov_len = sizeof(tmp[0]); 3529 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3530 &src_iovs[0], 1, NULL, NULL, 0, 4096, 3531 ut_sequence_step_cb, &completed); 3532 CU_ASSERT_EQUAL(rc, 0); 3533 3534 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 3535 ut_sequence_step_cb, &completed); 3536 CU_ASSERT_EQUAL(rc, 0); 3537 3538 dst_iovs[1].iov_base = tmp[2]; 3539 dst_iovs[1].iov_len = sizeof(tmp[2]); 3540 src_iovs[1].iov_base = tmp[1]; 3541 src_iovs[1].iov_len = sizeof(tmp[1]); 3542 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3543 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3544 ut_sequence_step_cb, &completed); 3545 CU_ASSERT_EQUAL(rc, 0); 3546 3547 dst_iovs[2].iov_base = buf; 3548 dst_iovs[2].iov_len = sizeof(buf); 3549 src_iovs[2].iov_base = tmp[2]; 3550 src_iovs[2].iov_len = sizeof(tmp[2]); 3551 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3552 &src_iovs[2], 1, NULL, NULL, 3553 ut_sequence_step_cb, &completed); 3554 CU_ASSERT_EQUAL(rc, 0); 3555 3556 ut_seq.complete = false; 3557 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3558 3559 poll_threads(); 3560 3561 CU_ASSERT_EQUAL(completed, 4); 3562 CU_ASSERT(ut_seq.complete); 3563 CU_ASSERT_EQUAL(ut_seq.status, 0); 3564 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3565 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3566 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3567 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3568 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3569 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3570 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3571 3572 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3573 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3574 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3575 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3576 3577 /* Check that an error returned from driver's execute_sequence() will fail the whole 3578 * sequence and any subsequent operations won't be processed */ 3579 seq = NULL; 3580 completed = 0; 3581 memset(buf, 0, sizeof(buf)); 3582 memset(expected, 0, sizeof(expected)); 3583 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3584 g_drv_operations[SPDK_ACCEL_OPC_FILL].submit_status = -EPERM; 3585 3586 dst_iovs[0].iov_base = tmp[1]; 3587 dst_iovs[0].iov_len = sizeof(tmp[1]); 3588 src_iovs[0].iov_base = tmp[0]; 3589 src_iovs[0].iov_len = sizeof(tmp[0]); 3590 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3591 &src_iovs[0], 1, NULL, NULL, 0, 4096, 3592 ut_sequence_step_cb, &completed); 3593 CU_ASSERT_EQUAL(rc, 0); 3594 3595 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 3596 ut_sequence_step_cb, &completed); 3597 CU_ASSERT_EQUAL(rc, 0); 3598 3599 dst_iovs[1].iov_base = buf; 3600 dst_iovs[1].iov_len = sizeof(buf); 3601 src_iovs[1].iov_base = tmp[1]; 3602 src_iovs[1].iov_len = sizeof(tmp[1]); 3603 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3604 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3605 ut_sequence_step_cb, &completed); 3606 CU_ASSERT_EQUAL(rc, 0); 3607 3608 ut_seq.complete = false; 3609 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3610 3611 poll_threads(); 3612 3613 CU_ASSERT_EQUAL(completed, 3); 3614 CU_ASSERT(ut_seq.complete); 3615 CU_ASSERT_EQUAL(ut_seq.status, -EPERM); 3616 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3617 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3618 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 0); 3619 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3620 CU_ASSERT_EQUAL(memcmp(buf, expected, 4096), 0); 3621 3622 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3623 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3624 g_drv_operations[SPDK_ACCEL_OPC_FILL].submit_status = 0; 3625 3626 /* Check that a failed task completed by a driver will cause the whole sequence to be failed 3627 * and any subsequent operations won't be processed */ 3628 seq = NULL; 3629 completed = 0; 3630 memset(buf, 0, sizeof(buf)); 3631 memset(expected, 0, sizeof(expected)); 3632 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3633 g_drv_operations[SPDK_ACCEL_OPC_FILL].complete_status = -ENOENT; 3634 3635 dst_iovs[0].iov_base = tmp[1]; 3636 dst_iovs[0].iov_len = sizeof(tmp[1]); 3637 src_iovs[0].iov_base = tmp[0]; 3638 src_iovs[0].iov_len = sizeof(tmp[0]); 3639 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3640 &src_iovs[0], 1, NULL, NULL, 0, 4096, 3641 ut_sequence_step_cb, &completed); 3642 CU_ASSERT_EQUAL(rc, 0); 3643 3644 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 3645 ut_sequence_step_cb, &completed); 3646 CU_ASSERT_EQUAL(rc, 0); 3647 3648 dst_iovs[1].iov_base = buf; 3649 dst_iovs[1].iov_len = sizeof(buf); 3650 src_iovs[1].iov_base = tmp[1]; 3651 src_iovs[1].iov_len = sizeof(tmp[1]); 3652 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3653 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3654 ut_sequence_step_cb, &completed); 3655 CU_ASSERT_EQUAL(rc, 0); 3656 3657 ut_seq.complete = false; 3658 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3659 3660 poll_threads(); 3661 3662 CU_ASSERT_EQUAL(completed, 3); 3663 CU_ASSERT(ut_seq.complete); 3664 CU_ASSERT_EQUAL(ut_seq.status, -ENOENT); 3665 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3666 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3667 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 0); 3668 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3669 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3670 3671 g_drv_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 3672 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3673 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3674 3675 /* Check asynchronous spdk_accel_sequence_continue() */ 3676 g_ut_driver_async_continue = true; 3677 seq = NULL; 3678 completed = 0; 3679 memset(buf, 0, sizeof(buf)); 3680 memset(tmp[0], 0, sizeof(tmp[0])); 3681 memset(&expected[0], 0xfe, 4096); 3682 3683 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 3684 ut_sequence_step_cb, &completed); 3685 CU_ASSERT_EQUAL(rc, 0); 3686 3687 dst_iovs[0].iov_base = buf; 3688 dst_iovs[0].iov_len = sizeof(buf); 3689 src_iovs[0].iov_base = tmp[0]; 3690 src_iovs[0].iov_len = sizeof(tmp[0]); 3691 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3692 &src_iovs[0], 1, NULL, NULL, 3693 ut_sequence_step_cb, &completed); 3694 CU_ASSERT_EQUAL(rc, 0); 3695 3696 ut_seq.complete = false; 3697 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3698 3699 poll_threads(); 3700 3701 CU_ASSERT_EQUAL(completed, 2); 3702 CU_ASSERT(ut_seq.complete); 3703 CU_ASSERT_EQUAL(ut_seq.status, 0); 3704 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3705 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3706 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3707 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3708 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3709 3710 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3711 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3712 3713 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3714 g_modules_opc[i] = modules[i]; 3715 } 3716 3717 /* Clear the driver so that other tests won't use it */ 3718 g_accel_driver = NULL; 3719 memset(&g_drv_operations, 0, sizeof(g_drv_operations)); 3720 3721 ut_clear_operations(); 3722 spdk_put_io_channel(ioch); 3723 poll_threads(); 3724 } 3725 3726 struct ut_saved_iovs { 3727 struct iovec src; 3728 struct iovec dst; 3729 }; 3730 3731 static struct ut_saved_iovs g_seq_saved_iovs[SPDK_ACCEL_OPC_LAST]; 3732 3733 static int 3734 ut_submit_save_iovs(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3735 { 3736 SPDK_CU_ASSERT_FATAL(task->s.iovcnt == 1); 3737 SPDK_CU_ASSERT_FATAL(task->d.iovcnt == 1); 3738 3739 g_seq_saved_iovs[task->op_code].src = task->s.iovs[0]; 3740 g_seq_saved_iovs[task->op_code].dst = task->d.iovs[0]; 3741 3742 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3743 3744 spdk_accel_task_complete(task, 0); 3745 3746 return 0; 3747 } 3748 3749 static void 3750 test_sequence_same_iovs(void) 3751 { 3752 struct spdk_accel_sequence *seq = NULL; 3753 struct spdk_io_channel *ioch; 3754 struct spdk_accel_crypto_key key = {}; 3755 struct ut_sequence ut_seq; 3756 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3757 char buf[4096], tmp[4096], expected[4096]; 3758 struct iovec iovs[3], expected_siov, expected_diov; 3759 struct spdk_memory_domain *domain; 3760 void *accel_buf, *domain_ctx; 3761 int i, rc, completed = 0; 3762 3763 ioch = spdk_accel_get_io_channel(); 3764 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3765 3766 /* Override the submit_tasks function */ 3767 g_module_if.submit_tasks = ut_sequence_submit_tasks; 3768 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3769 modules[i] = g_modules_opc[i]; 3770 g_modules_opc[i] = g_module; 3771 } 3772 /* Intercept crypto operations, as they should be executed by an accel module */ 3773 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].submit = ut_submit_save_iovs; 3774 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].submit = ut_submit_save_iovs; 3775 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3776 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3777 3778 /* Check that it's possible to use the same iovec ptr for different operations */ 3779 seq = NULL; 3780 completed = 0; 3781 memset(buf, 0, sizeof(buf)); 3782 memset(expected, 0xa5, sizeof(expected)); 3783 3784 iovs[0].iov_base = expected; 3785 iovs[0].iov_len = sizeof(expected); 3786 iovs[1].iov_base = tmp; 3787 iovs[1].iov_len = sizeof(tmp); 3788 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, NULL, NULL, 3789 &iovs[0], 1, NULL, NULL, 0, 4096, 3790 ut_sequence_step_cb, &completed); 3791 CU_ASSERT_EQUAL(rc, 0); 3792 /* Reuse iov[1] as src */ 3793 iovs[2].iov_base = buf; 3794 iovs[2].iov_len = sizeof(buf); 3795 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3796 &iovs[1], 1, NULL, NULL, 0, 4096, 3797 ut_sequence_step_cb, &completed); 3798 CU_ASSERT_EQUAL(rc, 0); 3799 3800 ut_seq.complete = false; 3801 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3802 3803 poll_threads(); 3804 3805 CU_ASSERT_EQUAL(completed, 2); 3806 CU_ASSERT(ut_seq.complete); 3807 CU_ASSERT_EQUAL(ut_seq.status, 0); 3808 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3809 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3810 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3811 expected_siov.iov_base = expected; 3812 expected_siov.iov_len = sizeof(expected); 3813 expected_diov.iov_base = tmp; 3814 expected_diov.iov_len = sizeof(tmp); 3815 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].src, 3816 &expected_siov, sizeof(expected_siov)), 0); 3817 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].dst, 3818 &expected_diov, sizeof(expected_diov)), 0); 3819 expected_siov.iov_base = tmp; 3820 expected_siov.iov_len = sizeof(tmp); 3821 expected_diov.iov_base = buf; 3822 expected_diov.iov_len = sizeof(buf); 3823 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].src, 3824 &expected_siov, sizeof(expected_siov)), 0); 3825 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].dst, 3826 &expected_diov, sizeof(expected_diov)), 0); 3827 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3828 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3829 3830 /* Check the same with an accel buffer */ 3831 seq = NULL; 3832 completed = 0; 3833 memset(buf, 0, sizeof(buf)); 3834 memset(expected, 0x5a, sizeof(expected)); 3835 3836 rc = spdk_accel_get_buf(ioch, sizeof(buf), &accel_buf, &domain, &domain_ctx); 3837 CU_ASSERT_EQUAL(rc, 0); 3838 3839 iovs[0].iov_base = expected; 3840 iovs[0].iov_len = sizeof(expected); 3841 iovs[1].iov_base = accel_buf; 3842 iovs[1].iov_len = sizeof(buf); 3843 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, domain, domain_ctx, 3844 &iovs[0], 1, NULL, NULL, 0, 4096, 3845 ut_sequence_step_cb, &completed); 3846 CU_ASSERT_EQUAL(rc, 0); 3847 /* Reuse iov[1] as src */ 3848 iovs[2].iov_base = buf; 3849 iovs[2].iov_len = sizeof(buf); 3850 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3851 &iovs[1], 1, domain, domain_ctx, 0, 4096, 3852 ut_sequence_step_cb, &completed); 3853 CU_ASSERT_EQUAL(rc, 0); 3854 3855 ut_seq.complete = false; 3856 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3857 3858 poll_threads(); 3859 3860 CU_ASSERT_EQUAL(completed, 2); 3861 CU_ASSERT(ut_seq.complete); 3862 CU_ASSERT_EQUAL(ut_seq.status, 0); 3863 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3864 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3865 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3866 expected_siov.iov_base = expected; 3867 expected_siov.iov_len = sizeof(expected); 3868 expected_diov.iov_base = buf; 3869 expected_diov.iov_len = sizeof(buf); 3870 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].src, 3871 &expected_siov, sizeof(expected_siov)), 0); 3872 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].dst, 3873 &expected_diov, sizeof(expected_diov)), 0); 3874 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].dst, 3875 &g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].src, 3876 sizeof(struct iovec)), 0); 3877 spdk_accel_put_buf(ioch, accel_buf, domain, domain_ctx); 3878 3879 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3880 g_modules_opc[i] = modules[i]; 3881 } 3882 3883 ut_clear_operations(); 3884 spdk_put_io_channel(ioch); 3885 poll_threads(); 3886 } 3887 3888 static void 3889 test_sequence_crc32(void) 3890 { 3891 struct spdk_accel_sequence *seq = NULL; 3892 struct spdk_io_channel *ioch; 3893 struct ut_sequence ut_seq; 3894 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3895 char buf[4096], tmp[3][4096]; 3896 struct iovec src_iovs[4], dst_iovs[4]; 3897 uint32_t crc, crc2; 3898 int i, rc, completed; 3899 3900 ioch = spdk_accel_get_io_channel(); 3901 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3902 3903 /* Override the submit_tasks function */ 3904 g_module_if.submit_tasks = ut_sequence_submit_tasks; 3905 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3906 g_seq_operations[i].submit = sw_accel_submit_tasks; 3907 modules[i] = g_modules_opc[i]; 3908 g_modules_opc[i] = g_module; 3909 } 3910 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 3911 3912 /* First check the simplest case - single crc32c operation */ 3913 seq = NULL; 3914 completed = 0; 3915 crc = 0; 3916 memset(buf, 0xa5, sizeof(buf)); 3917 3918 src_iovs[0].iov_base = buf; 3919 src_iovs[0].iov_len = sizeof(buf); 3920 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3921 ut_sequence_step_cb, &completed); 3922 CU_ASSERT_EQUAL(rc, 0); 3923 3924 ut_seq.complete = false; 3925 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3926 3927 poll_threads(); 3928 CU_ASSERT_EQUAL(completed, 1); 3929 CU_ASSERT(ut_seq.complete); 3930 CU_ASSERT_EQUAL(ut_seq.status, 0); 3931 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3932 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3933 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3934 3935 /* Now check copy+crc - This should not remove the copy. Otherwise the data does not 3936 * end up where the user expected it to be. */ 3937 seq = NULL; 3938 completed = 0; 3939 crc = 0; 3940 memset(buf, 0x5a, sizeof(buf)); 3941 memset(&tmp[0], 0, sizeof(tmp[0])); 3942 3943 dst_iovs[0].iov_base = tmp[0]; 3944 dst_iovs[0].iov_len = sizeof(tmp[0]); 3945 src_iovs[0].iov_base = buf; 3946 src_iovs[0].iov_len = sizeof(buf); 3947 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3948 &src_iovs[0], 1, NULL, NULL, 3949 ut_sequence_step_cb, &completed); 3950 CU_ASSERT_EQUAL(rc, 0); 3951 3952 src_iovs[1].iov_base = tmp[0]; 3953 src_iovs[1].iov_len = sizeof(tmp[0]); 3954 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 3955 ut_sequence_step_cb, &completed); 3956 CU_ASSERT_EQUAL(rc, 0); 3957 3958 ut_seq.complete = false; 3959 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3960 3961 poll_threads(); 3962 CU_ASSERT_EQUAL(completed, 2); 3963 CU_ASSERT(ut_seq.complete); 3964 CU_ASSERT_EQUAL(ut_seq.status, 0); 3965 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3966 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 3967 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 3968 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3969 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3970 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 3971 3972 /* Check crc+copy - Again, the copy cannot be removed. */ 3973 seq = NULL; 3974 completed = 0; 3975 crc = 0; 3976 memset(buf, 0, sizeof(buf)); 3977 memset(&tmp[0], 0xa5, sizeof(tmp[0])); 3978 3979 src_iovs[0].iov_base = tmp[0]; 3980 src_iovs[0].iov_len = sizeof(tmp[0]); 3981 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3982 ut_sequence_step_cb, &completed); 3983 CU_ASSERT_EQUAL(rc, 0); 3984 3985 dst_iovs[1].iov_base = buf; 3986 dst_iovs[1].iov_len = sizeof(buf); 3987 src_iovs[1].iov_base = tmp[0]; 3988 src_iovs[1].iov_len = sizeof(tmp[0]); 3989 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3990 &src_iovs[1], 1, NULL, NULL, 3991 ut_sequence_step_cb, &completed); 3992 CU_ASSERT_EQUAL(rc, 0); 3993 3994 ut_seq.complete = false; 3995 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3996 3997 poll_threads(); 3998 CU_ASSERT_EQUAL(completed, 2); 3999 CU_ASSERT(ut_seq.complete); 4000 CU_ASSERT_EQUAL(ut_seq.status, 0); 4001 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 4002 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 4003 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 4004 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 4005 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 4006 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 4007 4008 /* Check a sequence with an operation at the beginning that can have its buffer changed, two 4009 * crc operations and a copy at the end. The copy should be removed and the dst buffer of 4010 * the first operation and the src buffer of the crc operations should be changed. 4011 */ 4012 seq = NULL; 4013 completed = 0; 4014 crc = crc2 = 0; 4015 memset(buf, 0, sizeof(buf)); 4016 memset(&tmp[0], 0x5a, sizeof(tmp[0])); 4017 dst_iovs[0].iov_base = tmp[1]; 4018 dst_iovs[0].iov_len = sizeof(tmp[1]); 4019 src_iovs[0].iov_base = tmp[0]; 4020 src_iovs[0].iov_len = sizeof(tmp[0]); 4021 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 4022 &src_iovs[0], 1, NULL, NULL, 4023 ut_sequence_step_cb, &completed); 4024 CU_ASSERT_EQUAL(rc, 0); 4025 4026 src_iovs[1].iov_base = tmp[1]; 4027 src_iovs[1].iov_len = sizeof(tmp[1]); 4028 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 4029 ut_sequence_step_cb, &completed); 4030 CU_ASSERT_EQUAL(rc, 0); 4031 4032 src_iovs[2].iov_base = tmp[1]; 4033 src_iovs[2].iov_len = sizeof(tmp[1]); 4034 rc = spdk_accel_append_crc32c(&seq, ioch, &crc2, &src_iovs[2], 1, NULL, NULL, 0, 4035 ut_sequence_step_cb, &completed); 4036 CU_ASSERT_EQUAL(rc, 0); 4037 4038 dst_iovs[3].iov_base = buf; 4039 dst_iovs[3].iov_len = sizeof(buf); 4040 src_iovs[3].iov_base = tmp[1]; 4041 src_iovs[3].iov_len = sizeof(tmp[1]); 4042 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 4043 &src_iovs[3], 1, NULL, NULL, 4044 ut_sequence_step_cb, &completed); 4045 CU_ASSERT_EQUAL(rc, 0); 4046 4047 ut_seq.complete = false; 4048 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4049 4050 poll_threads(); 4051 CU_ASSERT_EQUAL(completed, 4); 4052 CU_ASSERT(ut_seq.complete); 4053 CU_ASSERT_EQUAL(ut_seq.status, 0); 4054 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 4055 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 2); 4056 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 4057 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 4058 CU_ASSERT_EQUAL(crc, crc2); 4059 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 4060 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 4061 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 4062 4063 /* Check that a copy won't be removed if the buffers don't match */ 4064 seq = NULL; 4065 completed = 0; 4066 crc = 0; 4067 memset(buf, 0, sizeof(buf)); 4068 memset(&tmp[0], 0xa5, 2048); 4069 memset(&tmp[1], 0xfe, sizeof(tmp[1])); 4070 memset(&tmp[2], 0xfe, sizeof(tmp[1])); 4071 dst_iovs[0].iov_base = &tmp[1][2048]; 4072 dst_iovs[0].iov_len = 2048; 4073 src_iovs[0].iov_base = tmp[0]; 4074 src_iovs[0].iov_len = 2048; 4075 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 4076 &src_iovs[0], 1, NULL, NULL, 4077 ut_sequence_step_cb, &completed); 4078 CU_ASSERT_EQUAL(rc, 0); 4079 4080 src_iovs[1].iov_base = &tmp[1][2048]; 4081 src_iovs[1].iov_len = 2048; 4082 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 4083 ut_sequence_step_cb, &completed); 4084 CU_ASSERT_EQUAL(rc, 0); 4085 4086 dst_iovs[2].iov_base = buf; 4087 dst_iovs[2].iov_len = sizeof(buf); 4088 src_iovs[2].iov_base = tmp[1]; 4089 src_iovs[2].iov_len = sizeof(tmp[1]); 4090 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 4091 &src_iovs[2], 1, NULL, NULL, 4092 ut_sequence_step_cb, &completed); 4093 CU_ASSERT_EQUAL(rc, 0); 4094 4095 ut_seq.complete = false; 4096 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4097 4098 poll_threads(); 4099 CU_ASSERT_EQUAL(completed, 3); 4100 CU_ASSERT(ut_seq.complete); 4101 CU_ASSERT_EQUAL(ut_seq.status, 0); 4102 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 4103 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 4104 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 4105 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], 2048, ~0u)); 4106 CU_ASSERT_EQUAL(memcmp(buf, tmp[2], 2048), 0); 4107 CU_ASSERT_EQUAL(memcmp(&buf[2048], tmp[0], 2048), 0); 4108 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 4109 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 4110 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 4111 4112 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 4113 g_modules_opc[i] = modules[i]; 4114 } 4115 4116 ut_clear_operations(); 4117 spdk_put_io_channel(ioch); 4118 poll_threads(); 4119 } 4120 4121 static void 4122 test_sequence_dix_generate_verify(void) 4123 { 4124 struct spdk_accel_sequence *seq; 4125 struct spdk_io_channel *ioch; 4126 struct ut_sequence ut_seq = {}; 4127 char srcbuf[3][4096], dstbuf[3][4096]; 4128 struct iovec src_iovs[3], dst_iovs[3]; 4129 uint32_t block_size = 512, md_size = 8; 4130 struct spdk_dif_ctx_init_ext_opts dif_opts; 4131 struct spdk_dif_ctx dif_ctx = {}; 4132 struct spdk_dif_error dif_err; 4133 int i, rc, completed = 0; 4134 4135 ioch = spdk_accel_get_io_channel(); 4136 SPDK_CU_ASSERT_FATAL(ioch != NULL); 4137 4138 for (i = 0; i < 3; i++) { 4139 memset(srcbuf[i], 0xdead, sizeof(srcbuf[i])); 4140 memset(dstbuf[i], 0, sizeof(dstbuf[i])); 4141 src_iovs[i].iov_base = srcbuf[i]; 4142 src_iovs[i].iov_len = sizeof(srcbuf[i]); 4143 dst_iovs[i].iov_base = dstbuf[i]; 4144 dst_iovs[i].iov_len = sizeof(dstbuf[i]); 4145 } 4146 4147 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format); 4148 dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16; 4149 4150 rc = spdk_dif_ctx_init(&dif_ctx, block_size, md_size, false, true, SPDK_DIF_TYPE1, 4151 SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_APPTAG_CHECK | 4152 SPDK_DIF_FLAGS_REFTAG_CHECK, 10, 0xFFFF, 20, 0, 0, &dif_opts); 4153 SPDK_CU_ASSERT_FATAL(rc == 0); 4154 4155 seq = NULL; 4156 completed = 0; 4157 for (i = 0; i < 3; i++) { 4158 rc = spdk_accel_append_dix_generate(&seq, ioch, &dst_iovs[i], 1, NULL, NULL, 4159 &src_iovs[i], NULL, NULL, 4160 src_iovs[i].iov_len / block_size, &dif_ctx, 4161 ut_sequence_step_cb, &completed); 4162 CU_ASSERT_EQUAL(rc, 0); 4163 rc = spdk_accel_append_dix_verify(&seq, ioch, &dst_iovs[i], 1, NULL, NULL, 4164 &src_iovs[i], NULL, NULL, 4165 src_iovs[i].iov_len / block_size, &dif_ctx, 4166 &dif_err, ut_sequence_step_cb, &completed); 4167 CU_ASSERT_EQUAL(rc, 0); 4168 } 4169 4170 ut_seq.complete = false; 4171 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4172 4173 poll_threads(); 4174 CU_ASSERT_EQUAL(completed, 6); 4175 CU_ASSERT(ut_seq.complete); 4176 CU_ASSERT_EQUAL(ut_seq.status, 0); 4177 4178 /* DIX metadata should be equal for the same source buffers and tags */ 4179 CU_ASSERT_EQUAL(memcmp(dstbuf[0], dstbuf[1], 4096), 0); 4180 CU_ASSERT_EQUAL(memcmp(dstbuf[0], dstbuf[2], 4096), 0); 4181 4182 ut_clear_operations(); 4183 spdk_put_io_channel(ioch); 4184 poll_threads(); 4185 } 4186 4187 static void 4188 test_sequence_dix(void) 4189 { 4190 struct spdk_accel_sequence *seq = NULL; 4191 struct spdk_io_channel *ioch; 4192 struct ut_sequence ut_seq = {}; 4193 char buf[3][4096], md_buf[64]; 4194 struct iovec iovs[3], md_iov; 4195 uint32_t block_size = 512, md_size = 8; 4196 struct spdk_dif_ctx_init_ext_opts dif_opts; 4197 struct spdk_dif_ctx dif_ctx = {}; 4198 struct spdk_dif_error dif_err; 4199 struct spdk_accel_crypto_key *key; 4200 struct spdk_accel_crypto_key_create_param key_params = { 4201 .cipher = "AES_XTS", 4202 .hex_key = "00112233445566778899aabbccddeeff", 4203 .hex_key2 = "ffeeddccbbaa99887766554433221100", 4204 .key_name = "ut_key", 4205 }; 4206 int i, rc, completed = 0; 4207 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 4208 4209 ioch = spdk_accel_get_io_channel(); 4210 SPDK_CU_ASSERT_FATAL(ioch != NULL); 4211 4212 rc = spdk_accel_crypto_key_create(&key_params); 4213 CU_ASSERT_EQUAL(rc, 0); 4214 key = spdk_accel_crypto_key_get(key_params.key_name); 4215 SPDK_CU_ASSERT_FATAL(key != NULL); 4216 4217 /* Override the submit_tasks function. */ 4218 g_module_if.submit_tasks = ut_sequence_submit_tasks; 4219 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 4220 g_seq_operations[i].complete_status = 0; 4221 g_seq_operations[i].submit_status = 0; 4222 g_seq_operations[i].count = 0; 4223 4224 modules[i] = g_modules_opc[i]; 4225 g_modules_opc[i] = g_module; 4226 } 4227 4228 for (i = 0; i < 3; i++) { 4229 memset(buf[i], 0, sizeof(buf[i])); 4230 iovs[i].iov_base = buf[i]; 4231 iovs[i].iov_len = sizeof(buf[i]); 4232 } 4233 memset(md_buf, 0, sizeof(md_buf)); 4234 md_iov.iov_base = md_buf; 4235 md_iov.iov_len = sizeof(md_buf); 4236 4237 /* Prepare first source buffer. */ 4238 memset(iovs[0].iov_base, 0xde, iovs[0].iov_len); 4239 4240 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 4241 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 4242 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovcnt = 1; 4243 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovcnt = 1; 4244 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovs = &iovs[0]; 4245 /* Copy will be skipped, so the destination buffer of encrypt will change. */ 4246 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovs = &iovs[2]; 4247 4248 rc = spdk_accel_append_encrypt(&seq, ioch, key, &iovs[1], 1, NULL, NULL, 4249 &iovs[0], 1, NULL, NULL, 0, block_size, 4250 ut_sequence_step_cb, &completed); 4251 CU_ASSERT_EQUAL(rc, 0); 4252 4253 dif_opts.size = SPDK_SIZEOF(&dif_opts, dif_pi_format); 4254 dif_opts.dif_pi_format = SPDK_DIF_PI_FORMAT_16; 4255 4256 rc = spdk_dif_ctx_init(&dif_ctx, block_size, md_size, false, true, SPDK_DIF_TYPE1, 4257 SPDK_DIF_FLAGS_GUARD_CHECK | SPDK_DIF_FLAGS_APPTAG_CHECK | 4258 SPDK_DIF_FLAGS_REFTAG_CHECK, 10, 0xFFFF, 20, 0, 0, &dif_opts); 4259 SPDK_CU_ASSERT_FATAL(rc == 0); 4260 4261 rc = spdk_accel_append_dix_generate(&seq, ioch, &iovs[1], 1, NULL, NULL, 4262 &md_iov, NULL, NULL, iovs[1].iov_len / block_size, 4263 &dif_ctx, ut_sequence_step_cb, &completed); 4264 CU_ASSERT_EQUAL(rc, 0); 4265 4266 rc = spdk_accel_append_copy(&seq, ioch, &iovs[2], 1, NULL, NULL, 4267 &iovs[1], 1, NULL, NULL, 4268 ut_sequence_step_cb, &completed); 4269 CU_ASSERT_EQUAL(rc, 0); 4270 4271 ut_seq.complete = false; 4272 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4273 4274 poll_threads(); 4275 4276 CU_ASSERT_EQUAL(completed, 3); 4277 CU_ASSERT(ut_seq.complete); 4278 CU_ASSERT_EQUAL(ut_seq.status, 0); 4279 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 4280 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 4281 4282 seq = NULL; 4283 completed = 0; 4284 4285 /* This time we start with first iovec containing encrypted data from previous sequence. */ 4286 memcpy(iovs[0].iov_base, iovs[2].iov_base, iovs[0].iov_len); 4287 4288 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 4289 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 4290 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 4291 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1; 4292 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &iovs[0]; 4293 /* Copy will be skipped, so the destination buffer of decrypt will change. */ 4294 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &iovs[2]; 4295 4296 rc = spdk_accel_append_decrypt(&seq, ioch, key, &iovs[1], 1, NULL, NULL, 4297 &iovs[0], 1, NULL, NULL, 0, block_size, 4298 ut_sequence_step_cb, &completed); 4299 CU_ASSERT_EQUAL(rc, 0); 4300 4301 rc = spdk_accel_append_dix_verify(&seq, ioch, &iovs[1], 1, NULL, NULL, &md_iov, NULL, NULL, 4302 iovs[1].iov_len / block_size, &dif_ctx, &dif_err, 4303 ut_sequence_step_cb, &completed); 4304 CU_ASSERT_EQUAL(rc, 0); 4305 4306 rc = spdk_accel_append_copy(&seq, ioch, &iovs[2], 1, NULL, NULL, &iovs[1], 1, NULL, NULL, 4307 ut_sequence_step_cb, &completed); 4308 CU_ASSERT_EQUAL(rc, 0); 4309 4310 ut_seq.complete = false; 4311 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4312 4313 poll_threads(); 4314 4315 CU_ASSERT_EQUAL(completed, 3); 4316 CU_ASSERT(ut_seq.complete); 4317 CU_ASSERT_EQUAL(ut_seq.status, 0); 4318 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 4319 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 4320 4321 ut_clear_operations(); 4322 4323 /* Cleanup module pointers to make subsequent tests work correctly. */ 4324 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 4325 g_modules_opc[i] = modules[i]; 4326 } 4327 4328 rc = spdk_accel_crypto_key_destroy(key); 4329 CU_ASSERT_EQUAL(rc, 0); 4330 4331 spdk_put_io_channel(ioch); 4332 poll_threads(); 4333 } 4334 4335 static int 4336 test_sequence_setup(void) 4337 { 4338 int rc; 4339 4340 allocate_cores(1); 4341 allocate_threads(1); 4342 set_thread(0); 4343 4344 rc = spdk_iobuf_initialize(); 4345 if (rc != 0) { 4346 CU_ASSERT(false); 4347 return -1; 4348 } 4349 4350 rc = spdk_accel_initialize(); 4351 if (rc != 0) { 4352 CU_ASSERT(false); 4353 return -1; 4354 } 4355 4356 g_module_if.name = "software"; 4357 g_module_if.compress_supports_algo = _supports_algo; 4358 g_module_if.get_compress_level_range = _get_compress_level_range; 4359 4360 return 0; 4361 } 4362 4363 static void 4364 finish_cb(void *cb_arg) 4365 { 4366 bool *done = cb_arg; 4367 4368 *done = true; 4369 } 4370 4371 static int 4372 test_sequence_cleanup(void) 4373 { 4374 bool done = false; 4375 4376 spdk_accel_finish(finish_cb, &done); 4377 4378 while (!done) { 4379 poll_threads(); 4380 } 4381 4382 done = false; 4383 spdk_iobuf_finish(finish_cb, &done); 4384 while (!done) { 4385 poll_threads(); 4386 } 4387 4388 free_threads(); 4389 free_cores(); 4390 4391 return 0; 4392 } 4393 4394 int 4395 main(int argc, char **argv) 4396 { 4397 CU_pSuite suite = NULL, seq_suite; 4398 unsigned int num_failures; 4399 4400 CU_initialize_registry(); 4401 4402 /* Sequence tests require accel to be initialized normally, so run them before the other 4403 * tests which register accel modules which aren't fully implemented, causing accel 4404 * initialization to fail. 4405 */ 4406 seq_suite = CU_add_suite("accel_sequence", test_sequence_setup, test_sequence_cleanup); 4407 CU_ADD_TEST(seq_suite, test_sequence_fill_copy); 4408 CU_ADD_TEST(seq_suite, test_sequence_abort); 4409 CU_ADD_TEST(seq_suite, test_sequence_append_error); 4410 CU_ADD_TEST(seq_suite, test_sequence_completion_error); 4411 #ifdef SPDK_CONFIG_ISAL /* accel_sw requires isa-l for compression */ 4412 CU_ADD_TEST(seq_suite, test_sequence_decompress); 4413 CU_ADD_TEST(seq_suite, test_sequence_reverse); 4414 #endif 4415 CU_ADD_TEST(seq_suite, test_sequence_copy_elision); 4416 CU_ADD_TEST(seq_suite, test_sequence_accel_buffers); 4417 CU_ADD_TEST(seq_suite, test_sequence_memory_domain); 4418 CU_ADD_TEST(seq_suite, test_sequence_module_memory_domain); 4419 #ifdef SPDK_CONFIG_ISAL_CRYPTO /* accel_sw requires isa-l-crypto for crypto operations */ 4420 CU_ADD_TEST(seq_suite, test_sequence_crypto); 4421 #endif 4422 CU_ADD_TEST(seq_suite, test_sequence_driver); 4423 CU_ADD_TEST(seq_suite, test_sequence_same_iovs); 4424 CU_ADD_TEST(seq_suite, test_sequence_crc32); 4425 CU_ADD_TEST(seq_suite, test_sequence_dix_generate_verify); 4426 CU_ADD_TEST(seq_suite, test_sequence_dix); 4427 4428 suite = CU_add_suite("accel", test_setup, test_cleanup); 4429 CU_ADD_TEST(suite, test_spdk_accel_task_complete); 4430 CU_ADD_TEST(suite, test_get_task); 4431 CU_ADD_TEST(suite, test_spdk_accel_submit_copy); 4432 CU_ADD_TEST(suite, test_spdk_accel_submit_dualcast); 4433 CU_ADD_TEST(suite, test_spdk_accel_submit_compare); 4434 CU_ADD_TEST(suite, test_spdk_accel_submit_fill); 4435 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c); 4436 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32cv); 4437 CU_ADD_TEST(suite, test_spdk_accel_submit_copy_crc32c); 4438 CU_ADD_TEST(suite, test_spdk_accel_submit_xor); 4439 CU_ADD_TEST(suite, test_spdk_accel_module_find_by_name); 4440 CU_ADD_TEST(suite, test_spdk_accel_module_register); 4441 4442 num_failures = spdk_ut_run_tests(argc, argv, NULL); 4443 CU_cleanup_registry(); 4444 4445 return num_failures; 4446 } 4447