1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2021 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk_internal/cunit.h" 8 #include "spdk_internal/mock.h" 9 #include "spdk/accel_module.h" 10 #include "thread/thread_internal.h" 11 #include "common/lib/ut_multithread.c" 12 #include "common/lib/test_iobuf.c" 13 #include "accel/accel.c" 14 #include "accel/accel_sw.c" 15 #include "unit/lib/json_mock.c" 16 17 DEFINE_STUB_V(spdk_memory_domain_destroy, (struct spdk_memory_domain *domain)); 18 DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, 19 (struct spdk_memory_domain *domain), "UT_DMA"); 20 21 int 22 spdk_memory_domain_create(struct spdk_memory_domain **domain, enum spdk_dma_device_type type, 23 struct spdk_memory_domain_ctx *ctx, const char *id) 24 { 25 *domain = (void *)0xdeadbeef; 26 27 return 0; 28 } 29 30 struct ut_domain_ctx { 31 struct iovec iov; 32 struct iovec expected; 33 int pull_submit_status; 34 int push_submit_status; 35 int pull_complete_status; 36 int push_complete_status; 37 }; 38 39 static struct spdk_memory_domain *g_ut_domain = (void *)0xa55e1; 40 41 int 42 spdk_memory_domain_pull_data(struct spdk_memory_domain *sd, void *sctx, struct iovec *siov, 43 uint32_t siovcnt, struct iovec *diov, uint32_t diovcnt, 44 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 45 { 46 struct ut_domain_ctx *ctx = sctx; 47 48 CU_ASSERT_EQUAL(sd, g_ut_domain); 49 CU_ASSERT_EQUAL(siovcnt, 1); 50 CU_ASSERT_EQUAL(memcmp(siov, &ctx->expected, sizeof(*siov)), 0); 51 52 if (ctx->pull_submit_status != 0) { 53 return ctx->pull_submit_status; 54 } 55 56 if (ctx->pull_complete_status != 0) { 57 cpl_cb(cpl_arg, ctx->pull_complete_status); 58 return 0; 59 } 60 61 spdk_iovcpy(&ctx->iov, 1, diov, diovcnt); 62 cpl_cb(cpl_arg, 0); 63 64 return 0; 65 } 66 67 int 68 spdk_memory_domain_push_data(struct spdk_memory_domain *dd, void *dctx, struct iovec *diov, 69 uint32_t diovcnt, struct iovec *siov, uint32_t siovcnt, 70 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 71 { 72 struct ut_domain_ctx *ctx = dctx; 73 74 CU_ASSERT_EQUAL(dd, g_ut_domain); 75 CU_ASSERT_EQUAL(diovcnt, 1); 76 CU_ASSERT_EQUAL(memcmp(diov, &ctx->expected, sizeof(*diov)), 0); 77 78 if (ctx->push_submit_status != 0) { 79 return ctx->push_submit_status; 80 } 81 82 if (ctx->push_complete_status != 0) { 83 cpl_cb(cpl_arg, ctx->push_complete_status); 84 return 0; 85 } 86 87 spdk_iovcpy(siov, siovcnt, &ctx->iov, 1); 88 cpl_cb(cpl_arg, 0); 89 90 return 0; 91 } 92 93 /* global vars and setup/cleanup functions used for all test functions */ 94 struct spdk_accel_module_if g_module_if = {}; 95 struct accel_module g_module = { .module = &g_module_if }; 96 struct spdk_io_channel *g_ch = NULL; 97 struct accel_io_channel *g_accel_ch = NULL; 98 struct sw_accel_io_channel *g_sw_ch = NULL; 99 struct spdk_io_channel *g_module_ch = NULL; 100 101 static uint64_t g_opc_mask = 0; 102 103 static uint64_t 104 _accel_op_to_bit(enum spdk_accel_opcode opc) 105 { 106 return (1 << opc); 107 } 108 109 static bool 110 _supports_opcode(enum spdk_accel_opcode opc) 111 { 112 if (_accel_op_to_bit(opc) & g_opc_mask) { 113 return true; 114 } 115 return false; 116 } 117 118 static int 119 test_setup(void) 120 { 121 int i; 122 123 g_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct accel_io_channel)); 124 if (g_ch == NULL) { 125 /* for some reason the assert fatal macro doesn't work in the setup function. */ 126 CU_ASSERT(false); 127 return -1; 128 } 129 g_accel_ch = (struct accel_io_channel *)((char *)g_ch + sizeof(struct spdk_io_channel)); 130 g_module_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct sw_accel_io_channel)); 131 if (g_module_ch == NULL) { 132 CU_ASSERT(false); 133 return -1; 134 } 135 136 g_module_if.submit_tasks = sw_accel_submit_tasks; 137 g_module_if.name = "software"; 138 for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) { 139 g_accel_ch->module_ch[i] = g_module_ch; 140 g_modules_opc[i] = g_module; 141 } 142 g_sw_ch = (struct sw_accel_io_channel *)((char *)g_module_ch + sizeof( 143 struct spdk_io_channel)); 144 TAILQ_INIT(&g_sw_ch->tasks_to_complete); 145 g_module_if.supports_opcode = _supports_opcode; 146 return 0; 147 } 148 149 static int 150 test_cleanup(void) 151 { 152 free(g_ch); 153 free(g_module_ch); 154 155 return 0; 156 } 157 158 #define DUMMY_ARG 0xDEADBEEF 159 static bool g_dummy_cb_called = false; 160 static void 161 dummy_cb_fn(void *cb_arg, int status) 162 { 163 CU_ASSERT(*(uint32_t *)cb_arg == DUMMY_ARG); 164 CU_ASSERT(status == 0); 165 g_dummy_cb_called = true; 166 } 167 168 static void 169 test_spdk_accel_task_complete(void) 170 { 171 struct spdk_accel_task accel_task = {}; 172 struct spdk_accel_task *expected_accel_task = NULL; 173 uint32_t cb_arg = DUMMY_ARG; 174 int status = 0; 175 176 accel_task.accel_ch = g_accel_ch; 177 accel_task.cb_fn = dummy_cb_fn; 178 accel_task.cb_arg = &cb_arg; 179 TAILQ_INIT(&g_accel_ch->task_pool); 180 181 /* Confirm cb is called and task added to list. */ 182 spdk_accel_task_complete(&accel_task, status); 183 CU_ASSERT(g_dummy_cb_called == true); 184 expected_accel_task = TAILQ_FIRST(&g_accel_ch->task_pool); 185 TAILQ_REMOVE(&g_accel_ch->task_pool, expected_accel_task, link); 186 CU_ASSERT(expected_accel_task == &accel_task); 187 } 188 189 static void 190 test_get_task(void) 191 { 192 struct spdk_accel_task *task; 193 struct spdk_accel_task _task; 194 void *cb_arg = NULL; 195 196 TAILQ_INIT(&g_accel_ch->task_pool); 197 198 /* no tasks left, return NULL. */ 199 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 200 CU_ASSERT(task == NULL); 201 202 _task.cb_fn = dummy_cb_fn; 203 _task.cb_arg = cb_arg; 204 _task.accel_ch = g_accel_ch; 205 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &_task, link); 206 207 /* Get a valid task. */ 208 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 209 CU_ASSERT(task == &_task); 210 CU_ASSERT(_task.cb_fn == dummy_cb_fn); 211 CU_ASSERT(_task.cb_arg == cb_arg); 212 CU_ASSERT(_task.accel_ch == g_accel_ch); 213 } 214 215 #define TEST_SUBMIT_SIZE 64 216 static void 217 test_spdk_accel_submit_copy(void) 218 { 219 const uint64_t nbytes = TEST_SUBMIT_SIZE; 220 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 221 uint8_t src[TEST_SUBMIT_SIZE] = {0}; 222 void *cb_arg = NULL; 223 int rc; 224 struct spdk_accel_task task; 225 struct spdk_accel_task *expected_accel_task = NULL; 226 int flags = 0; 227 228 TAILQ_INIT(&g_accel_ch->task_pool); 229 230 /* Fail with no tasks on _get_task() */ 231 rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, flags, NULL, cb_arg); 232 CU_ASSERT(rc == -ENOMEM); 233 234 task.accel_ch = g_accel_ch; 235 task.flags = 1; 236 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 237 238 /* submission OK. */ 239 rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, flags, NULL, cb_arg); 240 CU_ASSERT(rc == 0); 241 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COPY); 242 CU_ASSERT(task.flags == 0); 243 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 244 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 245 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 246 CU_ASSERT(expected_accel_task == &task); 247 } 248 249 static void 250 test_spdk_accel_submit_dualcast(void) 251 { 252 void *dst1; 253 void *dst2; 254 void *src; 255 uint32_t align = ALIGN_4K; 256 uint64_t nbytes = TEST_SUBMIT_SIZE; 257 void *cb_arg = NULL; 258 int rc; 259 struct spdk_accel_task task; 260 struct spdk_accel_task *expected_accel_task = NULL; 261 int flags = 0; 262 263 TAILQ_INIT(&g_accel_ch->task_pool); 264 265 /* Dualcast requires 4K alignment on dst addresses, 266 * hence using the hard coded address to test the buffer alignment 267 */ 268 dst1 = (void *)0x5000; 269 dst2 = (void *)0x60f0; 270 src = calloc(1, TEST_SUBMIT_SIZE); 271 SPDK_CU_ASSERT_FATAL(src != NULL); 272 memset(src, 0x5A, TEST_SUBMIT_SIZE); 273 274 /* This should fail since dst2 is not 4k aligned */ 275 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 276 CU_ASSERT(rc == -EINVAL); 277 278 dst1 = (void *)0x7010; 279 dst2 = (void *)0x6000; 280 /* This should fail since dst1 is not 4k aligned */ 281 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 282 CU_ASSERT(rc == -EINVAL); 283 284 /* Dualcast requires 4K alignment on dst addresses */ 285 dst1 = (void *)0x7000; 286 dst2 = (void *)0x6000; 287 /* Fail with no tasks on _get_task() */ 288 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 289 CU_ASSERT(rc == -ENOMEM); 290 291 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 292 293 /* accel submission OK., since we test the SW path , need to use valid memory addresses 294 * cannot hardcode them anymore */ 295 dst1 = spdk_dma_zmalloc(nbytes, align, NULL); 296 SPDK_CU_ASSERT_FATAL(dst1 != NULL); 297 dst2 = spdk_dma_zmalloc(nbytes, align, NULL); 298 SPDK_CU_ASSERT_FATAL(dst2 != NULL); 299 /* SW module does the dualcast. */ 300 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 301 CU_ASSERT(rc == 0); 302 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_DUALCAST); 303 CU_ASSERT(task.flags == 0); 304 CU_ASSERT(memcmp(dst1, src, TEST_SUBMIT_SIZE) == 0); 305 CU_ASSERT(memcmp(dst2, src, TEST_SUBMIT_SIZE) == 0); 306 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 307 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 308 CU_ASSERT(expected_accel_task == &task); 309 310 free(src); 311 spdk_free(dst1); 312 spdk_free(dst2); 313 } 314 315 static void 316 test_spdk_accel_submit_compare(void) 317 { 318 void *src1; 319 void *src2; 320 uint64_t nbytes = TEST_SUBMIT_SIZE; 321 void *cb_arg = NULL; 322 int rc; 323 struct spdk_accel_task task; 324 struct spdk_accel_task *expected_accel_task = NULL; 325 326 TAILQ_INIT(&g_accel_ch->task_pool); 327 328 src1 = calloc(1, TEST_SUBMIT_SIZE); 329 SPDK_CU_ASSERT_FATAL(src1 != NULL); 330 src2 = calloc(1, TEST_SUBMIT_SIZE); 331 SPDK_CU_ASSERT_FATAL(src2 != NULL); 332 333 /* Fail with no tasks on _get_task() */ 334 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 335 CU_ASSERT(rc == -ENOMEM); 336 337 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 338 339 /* accel submission OK. */ 340 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 341 CU_ASSERT(rc == 0); 342 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COMPARE); 343 CU_ASSERT(memcmp(src1, src2, TEST_SUBMIT_SIZE) == 0); 344 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 345 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 346 CU_ASSERT(expected_accel_task == &task); 347 348 free(src1); 349 free(src2); 350 } 351 352 static void 353 test_spdk_accel_submit_fill(void) 354 { 355 void *dst; 356 void *src; 357 uint8_t fill = 0xf; 358 uint64_t fill64; 359 uint64_t nbytes = TEST_SUBMIT_SIZE; 360 void *cb_arg = NULL; 361 int rc; 362 struct spdk_accel_task task; 363 struct spdk_accel_task *expected_accel_task = NULL; 364 int flags = 0; 365 366 TAILQ_INIT(&g_accel_ch->task_pool); 367 368 dst = calloc(1, TEST_SUBMIT_SIZE); 369 SPDK_CU_ASSERT_FATAL(dst != NULL); 370 src = calloc(1, TEST_SUBMIT_SIZE); 371 SPDK_CU_ASSERT_FATAL(src != NULL); 372 memset(src, fill, TEST_SUBMIT_SIZE); 373 memset(&fill64, fill, sizeof(uint64_t)); 374 375 /* Fail with no tasks on _get_task() */ 376 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 377 CU_ASSERT(rc == -ENOMEM); 378 379 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 380 381 /* accel submission OK. */ 382 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 383 CU_ASSERT(rc == 0); 384 CU_ASSERT(task.fill_pattern == fill64); 385 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_FILL); 386 CU_ASSERT(task.flags == 0); 387 388 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 389 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 390 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 391 CU_ASSERT(expected_accel_task == &task); 392 393 free(dst); 394 free(src); 395 } 396 397 static void 398 test_spdk_accel_submit_crc32c(void) 399 { 400 const uint64_t nbytes = TEST_SUBMIT_SIZE; 401 uint32_t crc_dst; 402 uint8_t src[TEST_SUBMIT_SIZE]; 403 uint32_t seed = 1; 404 void *cb_arg = NULL; 405 int rc; 406 struct spdk_accel_task task; 407 struct spdk_accel_task *expected_accel_task = NULL; 408 409 TAILQ_INIT(&g_accel_ch->task_pool); 410 411 /* Fail with no tasks on _get_task() */ 412 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 413 CU_ASSERT(rc == -ENOMEM); 414 415 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 416 417 /* accel submission OK. */ 418 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 419 CU_ASSERT(rc == 0); 420 CU_ASSERT(task.crc_dst == &crc_dst); 421 CU_ASSERT(task.seed == seed); 422 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_CRC32C); 423 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 424 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 425 CU_ASSERT(expected_accel_task == &task); 426 } 427 428 static void 429 test_spdk_accel_submit_crc32cv(void) 430 { 431 uint32_t crc_dst; 432 uint32_t seed = 0; 433 uint32_t iov_cnt = 32; 434 void *cb_arg = NULL; 435 int rc; 436 uint32_t i = 0; 437 struct spdk_accel_task task; 438 struct iovec iov[32]; 439 struct spdk_accel_task *expected_accel_task = NULL; 440 441 TAILQ_INIT(&g_accel_ch->task_pool); 442 443 for (i = 0; i < iov_cnt; i++) { 444 iov[i].iov_base = calloc(1, TEST_SUBMIT_SIZE); 445 SPDK_CU_ASSERT_FATAL(iov[i].iov_base != NULL); 446 iov[i].iov_len = TEST_SUBMIT_SIZE; 447 } 448 449 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 450 451 /* accel submission OK. */ 452 rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, NULL, cb_arg); 453 CU_ASSERT(rc == 0); 454 CU_ASSERT(task.s.iovs == iov); 455 CU_ASSERT(task.s.iovcnt == iov_cnt); 456 CU_ASSERT(task.crc_dst == &crc_dst); 457 CU_ASSERT(task.seed == seed); 458 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_CRC32C); 459 CU_ASSERT(task.cb_arg == cb_arg); 460 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 461 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 462 CU_ASSERT(expected_accel_task == &task); 463 464 for (i = 0; i < iov_cnt; i++) { 465 free(iov[i].iov_base); 466 } 467 } 468 469 static void 470 test_spdk_accel_submit_copy_crc32c(void) 471 { 472 const uint64_t nbytes = TEST_SUBMIT_SIZE; 473 uint32_t crc_dst; 474 uint8_t dst[TEST_SUBMIT_SIZE]; 475 uint8_t src[TEST_SUBMIT_SIZE]; 476 uint32_t seed = 0; 477 void *cb_arg = NULL; 478 int rc; 479 struct spdk_accel_task task; 480 struct spdk_accel_task *expected_accel_task = NULL; 481 int flags = 0; 482 483 TAILQ_INIT(&g_accel_ch->task_pool); 484 485 /* Fail with no tasks on _get_task() */ 486 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 487 NULL, cb_arg); 488 CU_ASSERT(rc == -ENOMEM); 489 490 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 491 492 /* accel submission OK. */ 493 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 494 NULL, cb_arg); 495 CU_ASSERT(rc == 0); 496 CU_ASSERT(task.crc_dst == &crc_dst); 497 CU_ASSERT(task.seed == seed); 498 CU_ASSERT(task.flags == 0); 499 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COPY_CRC32C); 500 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 501 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 502 CU_ASSERT(expected_accel_task == &task); 503 } 504 505 static void 506 test_spdk_accel_submit_xor(void) 507 { 508 const uint64_t nbytes = TEST_SUBMIT_SIZE; 509 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 510 uint8_t src1[TEST_SUBMIT_SIZE] = {0}; 511 uint8_t src2[TEST_SUBMIT_SIZE] = {0}; 512 void *sources[] = { src1, src2 }; 513 uint32_t nsrcs = SPDK_COUNTOF(sources); 514 int rc; 515 struct spdk_accel_task task; 516 struct spdk_accel_task *expected_accel_task = NULL; 517 518 TAILQ_INIT(&g_accel_ch->task_pool); 519 520 /* Fail with no tasks on _get_task() */ 521 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 522 CU_ASSERT(rc == -ENOMEM); 523 524 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 525 526 /* submission OK. */ 527 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 528 CU_ASSERT(rc == 0); 529 CU_ASSERT(task.nsrcs.srcs == sources); 530 CU_ASSERT(task.nsrcs.cnt == nsrcs); 531 CU_ASSERT(task.d.iovcnt == 1); 532 CU_ASSERT(task.d.iovs[0].iov_base == dst); 533 CU_ASSERT(task.d.iovs[0].iov_len == nbytes); 534 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_XOR); 535 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 536 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 537 CU_ASSERT(expected_accel_task == &task); 538 } 539 540 static void 541 test_spdk_accel_module_find_by_name(void) 542 { 543 struct spdk_accel_module_if mod1 = {}; 544 struct spdk_accel_module_if mod2 = {}; 545 struct spdk_accel_module_if mod3 = {}; 546 struct spdk_accel_module_if *accel_module = NULL; 547 548 mod1.name = "ioat"; 549 mod2.name = "idxd"; 550 mod3.name = "software"; 551 552 TAILQ_INIT(&spdk_accel_module_list); 553 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod1, tailq); 554 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod2, tailq); 555 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod3, tailq); 556 557 /* Now let's find a valid engine */ 558 accel_module = _module_find_by_name("ioat"); 559 CU_ASSERT(accel_module != NULL); 560 561 /* Try to find one that doesn't exist */ 562 accel_module = _module_find_by_name("XXX"); 563 CU_ASSERT(accel_module == NULL); 564 } 565 566 static int 567 ut_module_init_nop(void) 568 { 569 return 0; 570 } 571 572 static bool 573 ut_supports_opcode_all(enum spdk_accel_opcode opcode) 574 { 575 return true; 576 } 577 578 static void 579 ut_accel_module_priority_finish_done(void *done) 580 { 581 *(int *)done = 1; 582 } 583 584 static void 585 test_spdk_accel_module_register(void) 586 { 587 struct spdk_accel_module_if mods[] = { 588 { .name = "mod1", .priority = 1, }, 589 { .name = "mod3", .priority = 3, }, 590 { .name = "mod0", .priority = 0, }, 591 { .name = "mod2", .priority = 2, }, 592 }; 593 int rc, done = 0; 594 const char *modname = NULL; 595 size_t i; 596 597 allocate_cores(1); 598 allocate_threads(1); 599 set_thread(0); 600 601 TAILQ_INIT(&spdk_accel_module_list); 602 for (i = 0; i < SPDK_COUNTOF(mods); ++i) { 603 mods[i].module_init = ut_module_init_nop; 604 mods[i].supports_opcode = ut_supports_opcode_all; 605 spdk_accel_module_list_add(&mods[i]); 606 } 607 608 rc = spdk_accel_initialize(); 609 CU_ASSERT_EQUAL(rc, 0); 610 611 /* All opcodes should be assigned to the module with highest prio - mod3 */ 612 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 613 rc = spdk_accel_get_opc_module_name((enum spdk_accel_opcode)i, &modname); 614 CU_ASSERT_EQUAL(rc, 0); 615 CU_ASSERT_STRING_EQUAL(modname, "mod3"); 616 } 617 618 spdk_accel_finish(ut_accel_module_priority_finish_done, &done); 619 while (!done) { 620 poll_threads(); 621 } 622 623 TAILQ_INIT(&spdk_accel_module_list); 624 free_threads(); 625 free_cores(); 626 } 627 628 struct ut_sequence { 629 bool complete; 630 int status; 631 }; 632 633 static void 634 ut_sequence_step_cb(void *cb_arg) 635 { 636 int *completed = cb_arg; 637 638 (*completed)++; 639 } 640 641 static void 642 ut_sequence_complete_cb(void *cb_arg, int status) 643 { 644 struct ut_sequence *seq = cb_arg; 645 646 seq->complete = true; 647 seq->status = status; 648 } 649 650 static void 651 test_sequence_fill_copy(void) 652 { 653 struct spdk_accel_sequence *seq = NULL; 654 struct spdk_io_channel *ioch; 655 struct ut_sequence ut_seq; 656 char buf[4096], tmp[2][4096], expected[4096]; 657 struct iovec src_iovs[2], dst_iovs[2]; 658 int rc, completed; 659 660 ioch = spdk_accel_get_io_channel(); 661 SPDK_CU_ASSERT_FATAL(ioch != NULL); 662 663 /* First check the simplest case - single task in a sequence */ 664 memset(buf, 0, sizeof(buf)); 665 memset(expected, 0xa5, sizeof(expected)); 666 completed = 0; 667 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 668 ut_sequence_step_cb, &completed); 669 CU_ASSERT_EQUAL(rc, 0); 670 CU_ASSERT_EQUAL(completed, 0); 671 672 ut_seq.complete = false; 673 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 674 675 poll_threads(); 676 CU_ASSERT_EQUAL(completed, 1); 677 CU_ASSERT(ut_seq.complete); 678 CU_ASSERT_EQUAL(ut_seq.status, 0); 679 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 680 681 /* Check a single copy operation */ 682 memset(buf, 0, sizeof(buf)); 683 memset(tmp[0], 0xa5, sizeof(tmp[0])); 684 memset(expected, 0xa5, sizeof(expected)); 685 completed = 0; 686 seq = NULL; 687 688 dst_iovs[0].iov_base = buf; 689 dst_iovs[0].iov_len = sizeof(buf); 690 src_iovs[0].iov_base = tmp[0]; 691 src_iovs[0].iov_len = sizeof(tmp[0]); 692 693 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 694 &src_iovs[0], 1, NULL, NULL, 0, 695 ut_sequence_step_cb, &completed); 696 CU_ASSERT_EQUAL(rc, 0); 697 698 ut_seq.complete = false; 699 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 700 701 poll_threads(); 702 CU_ASSERT_EQUAL(completed, 1); 703 CU_ASSERT(ut_seq.complete); 704 CU_ASSERT_EQUAL(ut_seq.status, 0); 705 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 706 707 /* Check multiple fill operations */ 708 memset(buf, 0, sizeof(buf)); 709 memset(expected, 0xfe, 4096); 710 memset(expected, 0xde, 2048); 711 memset(expected, 0xa5, 1024); 712 seq = NULL; 713 completed = 0; 714 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, NULL, NULL, 0xfe, 0, 715 ut_sequence_step_cb, &completed); 716 CU_ASSERT_EQUAL(rc, 0); 717 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xde, 0, 718 ut_sequence_step_cb, &completed); 719 CU_ASSERT_EQUAL(rc, 0); 720 rc = spdk_accel_append_fill(&seq, ioch, buf, 1024, NULL, NULL, 0xa5, 0, 721 ut_sequence_step_cb, &completed); 722 CU_ASSERT_EQUAL(rc, 0); 723 724 ut_seq.complete = false; 725 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 726 727 poll_threads(); 728 CU_ASSERT_EQUAL(completed, 3); 729 CU_ASSERT(ut_seq.complete); 730 CU_ASSERT_EQUAL(ut_seq.status, 0); 731 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 732 733 /* Check multiple copy operations */ 734 memset(buf, 0, sizeof(buf)); 735 memset(tmp[0], 0, sizeof(tmp[0])); 736 memset(tmp[1], 0, sizeof(tmp[1])); 737 memset(expected, 0xa5, sizeof(expected)); 738 seq = NULL; 739 completed = 0; 740 741 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 742 ut_sequence_step_cb, &completed); 743 CU_ASSERT_EQUAL(rc, 0); 744 745 dst_iovs[0].iov_base = tmp[1]; 746 dst_iovs[0].iov_len = sizeof(tmp[1]); 747 src_iovs[0].iov_base = tmp[0]; 748 src_iovs[0].iov_len = sizeof(tmp[0]); 749 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 750 &src_iovs[0], 1, NULL, NULL, 0, 751 ut_sequence_step_cb, &completed); 752 CU_ASSERT_EQUAL(rc, 0); 753 754 dst_iovs[1].iov_base = buf; 755 dst_iovs[1].iov_len = sizeof(buf); 756 src_iovs[1].iov_base = tmp[1]; 757 src_iovs[1].iov_len = sizeof(tmp[1]); 758 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 759 &src_iovs[1], 1, NULL, NULL, 0, 760 ut_sequence_step_cb, &completed); 761 CU_ASSERT_EQUAL(rc, 0); 762 763 ut_seq.complete = false; 764 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 765 766 poll_threads(); 767 CU_ASSERT_EQUAL(completed, 3); 768 CU_ASSERT(ut_seq.complete); 769 CU_ASSERT_EQUAL(ut_seq.status, 0); 770 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 771 772 /* Check that adding a copy operation at the end will change destination buffer */ 773 memset(buf, 0, sizeof(buf)); 774 memset(tmp[0], 0, sizeof(tmp[0])); 775 memset(expected, 0xa5, sizeof(buf)); 776 seq = NULL; 777 completed = 0; 778 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 779 ut_sequence_step_cb, &completed); 780 CU_ASSERT_EQUAL(rc, 0); 781 782 dst_iovs[0].iov_base = buf; 783 dst_iovs[0].iov_len = sizeof(buf); 784 src_iovs[0].iov_base = tmp[0]; 785 src_iovs[0].iov_len = sizeof(tmp[0]); 786 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 787 &src_iovs[0], 1, NULL, NULL, 0, 788 ut_sequence_step_cb, &completed); 789 CU_ASSERT_EQUAL(rc, 0); 790 791 ut_seq.complete = false; 792 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 793 794 poll_threads(); 795 CU_ASSERT_EQUAL(completed, 2); 796 CU_ASSERT(ut_seq.complete); 797 CU_ASSERT_EQUAL(ut_seq.status, 0); 798 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 799 800 /* Check that it's also possible to add copy operation at the beginning */ 801 memset(buf, 0, sizeof(buf)); 802 memset(tmp[0], 0xde, sizeof(tmp[0])); 803 memset(tmp[1], 0, sizeof(tmp[1])); 804 memset(expected, 0xa5, sizeof(expected)); 805 seq = NULL; 806 completed = 0; 807 808 dst_iovs[0].iov_base = tmp[1]; 809 dst_iovs[0].iov_len = sizeof(tmp[1]); 810 src_iovs[0].iov_base = tmp[0]; 811 src_iovs[0].iov_len = sizeof(tmp[0]); 812 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 813 &src_iovs[0], 1, NULL, NULL, 0, 814 ut_sequence_step_cb, &completed); 815 CU_ASSERT_EQUAL(rc, 0); 816 817 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], sizeof(tmp[1]), NULL, NULL, 0xa5, 0, 818 ut_sequence_step_cb, &completed); 819 CU_ASSERT_EQUAL(rc, 0); 820 821 dst_iovs[1].iov_base = buf; 822 dst_iovs[1].iov_len = sizeof(buf); 823 src_iovs[1].iov_base = tmp[1]; 824 src_iovs[1].iov_len = sizeof(tmp[1]); 825 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 826 &src_iovs[1], 1, NULL, NULL, 0, 827 ut_sequence_step_cb, &completed); 828 CU_ASSERT_EQUAL(rc, 0); 829 830 ut_seq.complete = false; 831 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 832 833 poll_threads(); 834 CU_ASSERT_EQUAL(completed, 3); 835 CU_ASSERT(ut_seq.complete); 836 CU_ASSERT_EQUAL(ut_seq.status, 0); 837 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 838 839 spdk_put_io_channel(ioch); 840 poll_threads(); 841 } 842 843 static void 844 test_sequence_abort(void) 845 { 846 struct spdk_accel_sequence *seq = NULL; 847 struct spdk_io_channel *ioch; 848 char buf[4096], tmp[2][4096], expected[4096]; 849 struct iovec src_iovs[2], dst_iovs[2]; 850 int rc, completed; 851 852 ioch = spdk_accel_get_io_channel(); 853 SPDK_CU_ASSERT_FATAL(ioch != NULL); 854 855 /* Check that aborting a sequence calls operation's callback, the operation is not executed 856 * and the sequence is freed 857 */ 858 memset(buf, 0, sizeof(buf)); 859 memset(expected, 0, sizeof(buf)); 860 completed = 0; 861 seq = NULL; 862 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 863 ut_sequence_step_cb, &completed); 864 CU_ASSERT_EQUAL(rc, 0); 865 866 spdk_accel_sequence_abort(seq); 867 CU_ASSERT_EQUAL(completed, 1); 868 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 869 870 /* Check sequence with multiple operations */ 871 memset(buf, 0, sizeof(buf)); 872 memset(expected, 0, sizeof(buf)); 873 completed = 0; 874 seq = NULL; 875 876 dst_iovs[0].iov_base = tmp[1]; 877 dst_iovs[0].iov_len = sizeof(tmp[1]); 878 src_iovs[0].iov_base = tmp[0]; 879 src_iovs[0].iov_len = sizeof(tmp[0]); 880 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 881 &src_iovs[0], 1, NULL, NULL, 0, 882 ut_sequence_step_cb, &completed); 883 CU_ASSERT_EQUAL(rc, 0); 884 885 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 4096, NULL, NULL, 0xa5, 0, 886 ut_sequence_step_cb, &completed); 887 CU_ASSERT_EQUAL(rc, 0); 888 889 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xde, 0, 890 ut_sequence_step_cb, &completed); 891 CU_ASSERT_EQUAL(rc, 0); 892 893 dst_iovs[1].iov_base = buf; 894 dst_iovs[1].iov_len = sizeof(buf); 895 src_iovs[1].iov_base = tmp[1]; 896 src_iovs[1].iov_len = sizeof(tmp[1]); 897 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 898 &src_iovs[1], 1, NULL, NULL, 0, 899 ut_sequence_step_cb, &completed); 900 CU_ASSERT_EQUAL(rc, 0); 901 902 spdk_accel_sequence_abort(seq); 903 CU_ASSERT_EQUAL(completed, 4); 904 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 905 906 /* This should be a no-op */ 907 spdk_accel_sequence_abort(NULL); 908 909 spdk_put_io_channel(ioch); 910 poll_threads(); 911 } 912 913 static void 914 test_sequence_append_error(void) 915 { 916 struct spdk_accel_sequence *seq = NULL; 917 struct spdk_io_channel *ioch; 918 struct accel_io_channel *accel_ch; 919 struct iovec src_iovs, dst_iovs; 920 char buf[4096]; 921 TAILQ_HEAD(, spdk_accel_task) tasks = TAILQ_HEAD_INITIALIZER(tasks); 922 TAILQ_HEAD(, spdk_accel_sequence) seqs = TAILQ_HEAD_INITIALIZER(seqs); 923 int rc; 924 925 ioch = spdk_accel_get_io_channel(); 926 SPDK_CU_ASSERT_FATAL(ioch != NULL); 927 accel_ch = spdk_io_channel_get_ctx(ioch); 928 929 /* Check that append fails and no sequence object is allocated when there are no more free 930 * tasks */ 931 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 932 933 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 934 ut_sequence_step_cb, NULL); 935 CU_ASSERT_EQUAL(rc, -ENOMEM); 936 CU_ASSERT_PTR_NULL(seq); 937 938 dst_iovs.iov_base = buf; 939 dst_iovs.iov_len = 2048; 940 src_iovs.iov_base = &buf[2048]; 941 src_iovs.iov_len = 2048; 942 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 943 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 944 CU_ASSERT_EQUAL(rc, -ENOMEM); 945 CU_ASSERT_PTR_NULL(seq); 946 947 dst_iovs.iov_base = buf; 948 dst_iovs.iov_len = 2048; 949 src_iovs.iov_base = &buf[2048]; 950 src_iovs.iov_len = 2048; 951 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 952 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 953 CU_ASSERT_EQUAL(rc, -ENOMEM); 954 CU_ASSERT_PTR_NULL(seq); 955 956 /* Check that the same happens when the sequence queue is empty */ 957 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 958 TAILQ_SWAP(&seqs, &accel_ch->seq_pool, spdk_accel_sequence, link); 959 960 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 961 ut_sequence_step_cb, NULL); 962 CU_ASSERT_EQUAL(rc, -ENOMEM); 963 CU_ASSERT_PTR_NULL(seq); 964 965 dst_iovs.iov_base = buf; 966 dst_iovs.iov_len = 2048; 967 src_iovs.iov_base = &buf[2048]; 968 src_iovs.iov_len = 2048; 969 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 970 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 971 CU_ASSERT_EQUAL(rc, -ENOMEM); 972 CU_ASSERT_PTR_NULL(seq); 973 974 dst_iovs.iov_base = buf; 975 dst_iovs.iov_len = 2048; 976 src_iovs.iov_base = &buf[2048]; 977 src_iovs.iov_len = 2048; 978 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 979 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 980 CU_ASSERT_EQUAL(rc, -ENOMEM); 981 CU_ASSERT_PTR_NULL(seq); 982 983 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 984 985 spdk_put_io_channel(ioch); 986 poll_threads(); 987 } 988 989 struct ut_sequence_operation { 990 int complete_status; 991 int submit_status; 992 int count; 993 struct iovec *src_iovs; 994 uint32_t src_iovcnt; 995 struct spdk_memory_domain *src_domain; 996 void *src_domain_ctx; 997 struct iovec *dst_iovs; 998 uint32_t dst_iovcnt; 999 struct spdk_memory_domain *dst_domain; 1000 void *dst_domain_ctx; 1001 int (*submit)(struct spdk_io_channel *ch, struct spdk_accel_task *t); 1002 }; 1003 1004 static struct ut_sequence_operation g_seq_operations[SPDK_ACCEL_OPC_LAST]; 1005 1006 static int 1007 ut_sequnce_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *task) 1008 { 1009 struct ut_sequence_operation *op = &g_seq_operations[task->op_code]; 1010 1011 op->count++; 1012 if (op->submit != NULL) { 1013 return op->submit(ch, task); 1014 } 1015 if (op->src_iovs != NULL) { 1016 CU_ASSERT_EQUAL(task->s.iovcnt, op->src_iovcnt); 1017 CU_ASSERT_EQUAL(memcmp(task->s.iovs, op->src_iovs, 1018 sizeof(struct iovec) * op->src_iovcnt), 0); 1019 } 1020 if (op->dst_iovs != NULL) { 1021 CU_ASSERT_EQUAL(task->d.iovcnt, op->dst_iovcnt); 1022 CU_ASSERT_EQUAL(memcmp(task->d.iovs, op->dst_iovs, 1023 sizeof(struct iovec) * op->dst_iovcnt), 0); 1024 } 1025 1026 CU_ASSERT_EQUAL(task->src_domain, op->src_domain); 1027 CU_ASSERT_EQUAL(task->dst_domain, op->dst_domain); 1028 if (op->src_domain != NULL) { 1029 CU_ASSERT_EQUAL(task->src_domain_ctx, op->src_domain_ctx); 1030 } 1031 if (op->dst_domain != NULL) { 1032 CU_ASSERT_EQUAL(task->dst_domain_ctx, op->dst_domain_ctx); 1033 } 1034 1035 if (op->submit_status != 0) { 1036 return op->submit_status; 1037 } 1038 1039 spdk_accel_task_complete(task, op->complete_status); 1040 1041 return 0; 1042 } 1043 1044 static void 1045 ut_clear_operations(void) 1046 { 1047 memset(&g_seq_operations, 0, sizeof(g_seq_operations)); 1048 } 1049 1050 static void 1051 test_sequence_completion_error(void) 1052 { 1053 struct spdk_accel_sequence *seq = NULL; 1054 struct spdk_io_channel *ioch; 1055 struct ut_sequence ut_seq; 1056 struct iovec src_iovs, dst_iovs; 1057 char buf[4096], tmp[4096]; 1058 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 1059 int i, rc, completed; 1060 1061 ioch = spdk_accel_get_io_channel(); 1062 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1063 1064 /* Override the submit_tasks function */ 1065 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1066 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1067 modules[i] = g_modules_opc[i]; 1068 g_modules_opc[i] = g_module; 1069 } 1070 1071 memset(buf, 0, sizeof(buf)); 1072 memset(tmp, 0, sizeof(tmp)); 1073 1074 /* Check that if the first operation completes with an error, the whole sequence is 1075 * completed with that error and that all operations' completion callbacks are executed 1076 */ 1077 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = -E2BIG; 1078 completed = 0; 1079 seq = NULL; 1080 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1081 ut_sequence_step_cb, &completed); 1082 CU_ASSERT_EQUAL(rc, 0); 1083 1084 dst_iovs.iov_base = buf; 1085 dst_iovs.iov_len = sizeof(buf); 1086 src_iovs.iov_base = tmp; 1087 src_iovs.iov_len = sizeof(tmp); 1088 1089 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1090 &src_iovs, 1, NULL, NULL, 0, 1091 ut_sequence_step_cb, &completed); 1092 CU_ASSERT_EQUAL(rc, 0); 1093 1094 ut_seq.complete = false; 1095 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1096 1097 poll_threads(); 1098 CU_ASSERT_EQUAL(completed, 2); 1099 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 1100 1101 /* Check the same with a second operation in the sequence */ 1102 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].complete_status = -EACCES; 1103 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 1104 completed = 0; 1105 seq = NULL; 1106 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1107 ut_sequence_step_cb, &completed); 1108 CU_ASSERT_EQUAL(rc, 0); 1109 1110 dst_iovs.iov_base = buf; 1111 dst_iovs.iov_len = sizeof(buf); 1112 src_iovs.iov_base = tmp; 1113 src_iovs.iov_len = sizeof(tmp); 1114 1115 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1116 &src_iovs, 1, NULL, NULL, 0, 1117 ut_sequence_step_cb, &completed); 1118 CU_ASSERT_EQUAL(rc, 0); 1119 1120 ut_seq.complete = false; 1121 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1122 1123 poll_threads(); 1124 CU_ASSERT_EQUAL(completed, 2); 1125 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 1126 1127 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].complete_status = 0; 1128 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 1129 1130 /* Check submission failure of the first operation */ 1131 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit_status = -EADDRINUSE; 1132 completed = 0; 1133 seq = NULL; 1134 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1135 ut_sequence_step_cb, &completed); 1136 CU_ASSERT_EQUAL(rc, 0); 1137 1138 dst_iovs.iov_base = buf; 1139 dst_iovs.iov_len = sizeof(buf); 1140 src_iovs.iov_base = tmp; 1141 src_iovs.iov_len = sizeof(tmp); 1142 1143 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1144 &src_iovs, 1, NULL, NULL, 0, 1145 ut_sequence_step_cb, &completed); 1146 CU_ASSERT_EQUAL(rc, 0); 1147 1148 ut_seq.complete = false; 1149 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1150 1151 poll_threads(); 1152 CU_ASSERT_EQUAL(completed, 2); 1153 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 1154 1155 /* Check the same with a second operation */ 1156 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit_status = -EADDRNOTAVAIL; 1157 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit_status = 0; 1158 completed = 0; 1159 seq = NULL; 1160 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1161 ut_sequence_step_cb, &completed); 1162 CU_ASSERT_EQUAL(rc, 0); 1163 1164 dst_iovs.iov_base = buf; 1165 dst_iovs.iov_len = sizeof(buf); 1166 src_iovs.iov_base = tmp; 1167 src_iovs.iov_len = sizeof(tmp); 1168 1169 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1170 &src_iovs, 1, NULL, NULL, 0, 1171 ut_sequence_step_cb, &completed); 1172 CU_ASSERT_EQUAL(rc, 0); 1173 1174 ut_seq.complete = false; 1175 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1176 1177 poll_threads(); 1178 CU_ASSERT_EQUAL(completed, 2); 1179 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 1180 1181 /* Cleanup module pointers to make subsequent tests work correctly */ 1182 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1183 g_modules_opc[i] = modules[i]; 1184 } 1185 1186 ut_clear_operations(); 1187 spdk_put_io_channel(ioch); 1188 poll_threads(); 1189 } 1190 1191 #ifdef SPDK_CONFIG_ISAL 1192 static void 1193 ut_compress_cb(void *cb_arg, int status) 1194 { 1195 int *completed = cb_arg; 1196 1197 CU_ASSERT_EQUAL(status, 0); 1198 1199 *completed = 1; 1200 } 1201 1202 static void 1203 test_sequence_decompress(void) 1204 { 1205 struct spdk_accel_sequence *seq = NULL; 1206 struct spdk_io_channel *ioch; 1207 struct ut_sequence ut_seq; 1208 char buf[4096], tmp[2][4096], expected[4096]; 1209 struct iovec src_iovs[2], dst_iovs[2]; 1210 uint32_t compressed_size; 1211 int rc, completed = 0; 1212 1213 ioch = spdk_accel_get_io_channel(); 1214 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1215 1216 memset(expected, 0xa5, sizeof(expected)); 1217 src_iovs[0].iov_base = expected; 1218 src_iovs[0].iov_len = sizeof(expected); 1219 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1220 &compressed_size, 0, ut_compress_cb, &completed); 1221 CU_ASSERT_EQUAL(rc, 0); 1222 1223 while (!completed) { 1224 poll_threads(); 1225 } 1226 1227 /* Check a single decompress operation in a sequence */ 1228 seq = NULL; 1229 completed = 0; 1230 1231 dst_iovs[0].iov_base = buf; 1232 dst_iovs[0].iov_len = sizeof(buf); 1233 src_iovs[0].iov_base = tmp[0]; 1234 src_iovs[0].iov_len = compressed_size; 1235 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1236 &src_iovs[0], 1, NULL, NULL, 0, 1237 ut_sequence_step_cb, &completed); 1238 CU_ASSERT_EQUAL(rc, 0); 1239 1240 ut_seq.complete = false; 1241 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1242 1243 poll_threads(); 1244 1245 CU_ASSERT_EQUAL(completed, 1); 1246 CU_ASSERT(ut_seq.complete); 1247 CU_ASSERT_EQUAL(ut_seq.status, 0); 1248 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1249 1250 /* Put the decompress operation in the middle of a sequence with a copy operation at the 1251 * beginning and a fill at the end modifying the first 2048B of the buffer. 1252 */ 1253 memset(expected, 0xfe, 2048); 1254 memset(buf, 0, sizeof(buf)); 1255 seq = NULL; 1256 completed = 0; 1257 1258 dst_iovs[0].iov_base = tmp[1]; 1259 dst_iovs[0].iov_len = compressed_size; 1260 src_iovs[0].iov_base = tmp[0]; 1261 src_iovs[0].iov_len = compressed_size; 1262 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1263 &src_iovs[0], 1, NULL, NULL, 0, 1264 ut_sequence_step_cb, &completed); 1265 CU_ASSERT_EQUAL(rc, 0); 1266 1267 dst_iovs[1].iov_base = buf; 1268 dst_iovs[1].iov_len = sizeof(buf); 1269 src_iovs[1].iov_base = tmp[1]; 1270 src_iovs[1].iov_len = compressed_size; 1271 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1272 &src_iovs[1], 1, NULL, NULL, 0, 1273 ut_sequence_step_cb, &completed); 1274 CU_ASSERT_EQUAL(rc, 0); 1275 1276 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1277 ut_sequence_step_cb, &completed); 1278 CU_ASSERT_EQUAL(rc, 0); 1279 1280 ut_seq.complete = false; 1281 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1282 1283 poll_threads(); 1284 1285 CU_ASSERT_EQUAL(completed, 3); 1286 CU_ASSERT(ut_seq.complete); 1287 CU_ASSERT_EQUAL(ut_seq.status, 0); 1288 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1289 1290 /* Check sequence with decompress at the beginning: decompress -> copy */ 1291 memset(expected, 0xa5, sizeof(expected)); 1292 memset(buf, 0, sizeof(buf)); 1293 seq = NULL; 1294 completed = 0; 1295 1296 dst_iovs[0].iov_base = tmp[1]; 1297 dst_iovs[0].iov_len = sizeof(tmp[1]); 1298 src_iovs[0].iov_base = tmp[0]; 1299 src_iovs[0].iov_len = compressed_size; 1300 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1301 &src_iovs[0], 1, NULL, NULL, 0, 1302 ut_sequence_step_cb, &completed); 1303 CU_ASSERT_EQUAL(rc, 0); 1304 1305 dst_iovs[1].iov_base = buf; 1306 dst_iovs[1].iov_len = sizeof(buf); 1307 src_iovs[1].iov_base = tmp[1]; 1308 src_iovs[1].iov_len = sizeof(tmp[1]); 1309 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1310 &src_iovs[1], 1, NULL, NULL, 0, 1311 ut_sequence_step_cb, &completed); 1312 CU_ASSERT_EQUAL(rc, 0); 1313 1314 ut_seq.complete = false; 1315 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1316 1317 poll_threads(); 1318 1319 CU_ASSERT_EQUAL(completed, 2); 1320 CU_ASSERT(ut_seq.complete); 1321 CU_ASSERT_EQUAL(ut_seq.status, 0); 1322 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1323 1324 spdk_put_io_channel(ioch); 1325 poll_threads(); 1326 } 1327 1328 static void 1329 test_sequence_reverse(void) 1330 { 1331 struct spdk_accel_sequence *seq = NULL; 1332 struct spdk_io_channel *ioch; 1333 struct ut_sequence ut_seq; 1334 char buf[4096], tmp[2][4096], expected[4096]; 1335 struct iovec src_iovs[2], dst_iovs[2]; 1336 uint32_t compressed_size; 1337 int rc, completed = 0; 1338 1339 ioch = spdk_accel_get_io_channel(); 1340 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1341 1342 memset(expected, 0xa5, sizeof(expected)); 1343 src_iovs[0].iov_base = expected; 1344 src_iovs[0].iov_len = sizeof(expected); 1345 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1346 &compressed_size, 0, ut_compress_cb, &completed); 1347 CU_ASSERT_EQUAL(rc, 0); 1348 1349 while (!completed) { 1350 poll_threads(); 1351 } 1352 1353 /* First check that reversing a sequnce with a single operation is a no-op */ 1354 memset(buf, 0, sizeof(buf)); 1355 seq = NULL; 1356 completed = 0; 1357 1358 dst_iovs[0].iov_base = buf; 1359 dst_iovs[0].iov_len = sizeof(buf); 1360 src_iovs[0].iov_base = tmp[0]; 1361 src_iovs[0].iov_len = compressed_size; 1362 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1363 &src_iovs[0], 1, NULL, NULL, 0, 1364 ut_sequence_step_cb, &completed); 1365 CU_ASSERT_EQUAL(rc, 0); 1366 1367 spdk_accel_sequence_reverse(seq); 1368 1369 ut_seq.complete = false; 1370 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1371 1372 poll_threads(); 1373 1374 CU_ASSERT_EQUAL(completed, 1); 1375 CU_ASSERT(ut_seq.complete); 1376 CU_ASSERT_EQUAL(ut_seq.status, 0); 1377 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1378 1379 /* Add a copy operation at the end with src set to the compressed data. After reverse(), 1380 * that copy operation should be first, so decompress() should receive compressed data in 1381 * its src buffer. 1382 */ 1383 memset(buf, 0, sizeof(buf)); 1384 memset(tmp[1], 0, sizeof(tmp[1])); 1385 seq = NULL; 1386 completed = 0; 1387 1388 dst_iovs[0].iov_base = buf; 1389 dst_iovs[0].iov_len = sizeof(buf); 1390 src_iovs[0].iov_base = tmp[1]; 1391 src_iovs[0].iov_len = compressed_size; 1392 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1393 &src_iovs[0], 1, NULL, NULL, 0, 1394 ut_sequence_step_cb, &completed); 1395 CU_ASSERT_EQUAL(rc, 0); 1396 1397 dst_iovs[1].iov_base = tmp[1]; 1398 dst_iovs[1].iov_len = compressed_size; 1399 src_iovs[1].iov_base = tmp[0]; 1400 src_iovs[1].iov_len = compressed_size; 1401 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1402 &src_iovs[1], 1, NULL, NULL, 0, 1403 ut_sequence_step_cb, &completed); 1404 CU_ASSERT_EQUAL(rc, 0); 1405 1406 spdk_accel_sequence_reverse(seq); 1407 1408 ut_seq.complete = false; 1409 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1410 1411 poll_threads(); 1412 1413 CU_ASSERT_EQUAL(completed, 2); 1414 CU_ASSERT(ut_seq.complete); 1415 CU_ASSERT_EQUAL(ut_seq.status, 0); 1416 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1417 1418 /* Check the same, but add an extra fill operation at the beginning that should execute last 1419 * after reverse(). 1420 */ 1421 memset(buf, 0, sizeof(buf)); 1422 memset(tmp[1], 0, sizeof(tmp[1])); 1423 memset(expected, 0xfe, 2048); 1424 seq = NULL; 1425 completed = 0; 1426 1427 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1428 ut_sequence_step_cb, &completed); 1429 CU_ASSERT_EQUAL(rc, 0); 1430 1431 dst_iovs[0].iov_base = buf; 1432 dst_iovs[0].iov_len = sizeof(buf); 1433 src_iovs[0].iov_base = tmp[1]; 1434 src_iovs[0].iov_len = compressed_size; 1435 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1436 &src_iovs[0], 1, NULL, NULL, 0, 1437 ut_sequence_step_cb, &completed); 1438 CU_ASSERT_EQUAL(rc, 0); 1439 1440 dst_iovs[1].iov_base = tmp[1]; 1441 dst_iovs[1].iov_len = compressed_size; 1442 src_iovs[1].iov_base = tmp[0]; 1443 src_iovs[1].iov_len = compressed_size; 1444 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1445 &src_iovs[1], 1, NULL, NULL, 0, 1446 ut_sequence_step_cb, &completed); 1447 CU_ASSERT_EQUAL(rc, 0); 1448 1449 spdk_accel_sequence_reverse(seq); 1450 1451 ut_seq.complete = false; 1452 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1453 1454 poll_threads(); 1455 1456 CU_ASSERT_EQUAL(completed, 3); 1457 CU_ASSERT(ut_seq.complete); 1458 CU_ASSERT_EQUAL(ut_seq.status, 0); 1459 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1460 1461 /* Build the sequence in order and then reverse it twice */ 1462 memset(buf, 0, sizeof(buf)); 1463 memset(tmp[1], 0, sizeof(tmp[1])); 1464 seq = NULL; 1465 completed = 0; 1466 1467 dst_iovs[0].iov_base = tmp[1]; 1468 dst_iovs[0].iov_len = compressed_size; 1469 src_iovs[0].iov_base = tmp[0]; 1470 src_iovs[0].iov_len = compressed_size; 1471 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1472 &src_iovs[0], 1, NULL, NULL, 0, 1473 ut_sequence_step_cb, &completed); 1474 CU_ASSERT_EQUAL(rc, 0); 1475 1476 dst_iovs[1].iov_base = buf; 1477 dst_iovs[1].iov_len = sizeof(buf); 1478 src_iovs[1].iov_base = tmp[1]; 1479 src_iovs[1].iov_len = compressed_size; 1480 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1481 &src_iovs[1], 1, NULL, NULL, 0, 1482 ut_sequence_step_cb, &completed); 1483 CU_ASSERT_EQUAL(rc, 0); 1484 1485 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1486 ut_sequence_step_cb, &completed); 1487 CU_ASSERT_EQUAL(rc, 0); 1488 1489 spdk_accel_sequence_reverse(seq); 1490 spdk_accel_sequence_reverse(seq); 1491 1492 ut_seq.complete = false; 1493 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1494 1495 poll_threads(); 1496 1497 CU_ASSERT_EQUAL(completed, 3); 1498 CU_ASSERT(ut_seq.complete); 1499 CU_ASSERT_EQUAL(ut_seq.status, 0); 1500 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1501 1502 spdk_put_io_channel(ioch); 1503 poll_threads(); 1504 } 1505 #endif 1506 1507 static void 1508 test_sequence_copy_elision(void) 1509 { 1510 struct spdk_accel_sequence *seq = NULL; 1511 struct spdk_io_channel *ioch; 1512 struct ut_sequence ut_seq; 1513 struct iovec src_iovs[4], dst_iovs[4], exp_iovs[2]; 1514 char buf[4096], tmp[4][4096]; 1515 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 1516 struct spdk_accel_crypto_key key = {}; 1517 int i, rc, completed; 1518 1519 ioch = spdk_accel_get_io_channel(); 1520 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1521 1522 /* Override the submit_tasks function */ 1523 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1524 g_module.supports_memory_domains = true; 1525 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1526 g_seq_operations[i].complete_status = 0; 1527 g_seq_operations[i].submit_status = 0; 1528 g_seq_operations[i].count = 0; 1529 1530 modules[i] = g_modules_opc[i]; 1531 g_modules_opc[i] = g_module; 1532 } 1533 1534 /* Check that a copy operation at the beginning is removed */ 1535 seq = NULL; 1536 completed = 0; 1537 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovcnt = 1; 1538 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovcnt = 1; 1539 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1540 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1541 exp_iovs[0].iov_base = tmp[0]; 1542 exp_iovs[0].iov_len = sizeof(tmp[0]); 1543 exp_iovs[1].iov_base = buf; 1544 exp_iovs[1].iov_len = 2048; 1545 1546 dst_iovs[0].iov_base = tmp[1]; 1547 dst_iovs[0].iov_len = sizeof(tmp[1]); 1548 src_iovs[0].iov_base = tmp[0]; 1549 src_iovs[0].iov_len = sizeof(tmp[0]); 1550 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1551 &src_iovs[0], 1, NULL, NULL, 0, 1552 ut_sequence_step_cb, &completed); 1553 CU_ASSERT_EQUAL(rc, 0); 1554 1555 dst_iovs[1].iov_base = buf; 1556 dst_iovs[1].iov_len = 2048; 1557 src_iovs[1].iov_base = tmp[1]; 1558 src_iovs[1].iov_len = sizeof(tmp[1]); 1559 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1560 &src_iovs[1], 1, NULL, NULL, 0, 1561 ut_sequence_step_cb, &completed); 1562 CU_ASSERT_EQUAL(rc, 0); 1563 1564 ut_seq.complete = false; 1565 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1566 1567 poll_threads(); 1568 1569 CU_ASSERT_EQUAL(completed, 2); 1570 CU_ASSERT(ut_seq.complete); 1571 CU_ASSERT_EQUAL(ut_seq.status, 0); 1572 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1573 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1574 1575 /* Check that a copy operation at the end is removed too */ 1576 seq = NULL; 1577 completed = 0; 1578 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1579 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1580 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1581 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1582 exp_iovs[0].iov_base = tmp[0]; 1583 exp_iovs[0].iov_len = sizeof(tmp[0]); 1584 exp_iovs[1].iov_base = buf; 1585 exp_iovs[1].iov_len = 2048; 1586 1587 dst_iovs[0].iov_base = tmp[1]; 1588 dst_iovs[0].iov_len = 2048; 1589 src_iovs[0].iov_base = tmp[0]; 1590 src_iovs[0].iov_len = sizeof(tmp[0]); 1591 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1592 &src_iovs[0], 1, NULL, NULL, 0, 1593 ut_sequence_step_cb, &completed); 1594 CU_ASSERT_EQUAL(rc, 0); 1595 1596 dst_iovs[1].iov_base = buf; 1597 dst_iovs[1].iov_len = 2048; 1598 src_iovs[1].iov_base = tmp[1]; 1599 src_iovs[1].iov_len = 2048; 1600 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1601 &src_iovs[1], 1, NULL, NULL, 0, 1602 ut_sequence_step_cb, &completed); 1603 CU_ASSERT_EQUAL(rc, 0); 1604 1605 ut_seq.complete = false; 1606 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1607 1608 poll_threads(); 1609 1610 CU_ASSERT_EQUAL(completed, 2); 1611 CU_ASSERT(ut_seq.complete); 1612 CU_ASSERT_EQUAL(ut_seq.status, 0); 1613 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1614 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1615 1616 /* Check a copy operation both at the beginning and the end */ 1617 seq = NULL; 1618 completed = 0; 1619 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1620 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1621 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1622 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1623 exp_iovs[0].iov_base = tmp[0]; 1624 exp_iovs[0].iov_len = sizeof(tmp[0]); 1625 exp_iovs[1].iov_base = buf; 1626 exp_iovs[1].iov_len = 2048; 1627 1628 dst_iovs[0].iov_base = tmp[1]; 1629 dst_iovs[0].iov_len = sizeof(tmp[1]); 1630 src_iovs[0].iov_base = tmp[0]; 1631 src_iovs[0].iov_len = sizeof(tmp[0]); 1632 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1633 &src_iovs[0], 1, NULL, NULL, 0, 1634 ut_sequence_step_cb, &completed); 1635 CU_ASSERT_EQUAL(rc, 0); 1636 1637 dst_iovs[1].iov_base = tmp[2]; 1638 dst_iovs[1].iov_len = 2048; 1639 src_iovs[1].iov_base = tmp[1]; 1640 src_iovs[1].iov_len = sizeof(tmp[1]); 1641 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1642 &src_iovs[1], 1, NULL, NULL, 0, 1643 ut_sequence_step_cb, &completed); 1644 CU_ASSERT_EQUAL(rc, 0); 1645 1646 dst_iovs[2].iov_base = buf; 1647 dst_iovs[2].iov_len = 2048; 1648 src_iovs[2].iov_base = tmp[2]; 1649 src_iovs[2].iov_len = 2048; 1650 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1651 &src_iovs[2], 1, NULL, NULL, 0, 1652 ut_sequence_step_cb, &completed); 1653 CU_ASSERT_EQUAL(rc, 0); 1654 1655 ut_seq.complete = false; 1656 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1657 1658 poll_threads(); 1659 1660 CU_ASSERT_EQUAL(completed, 3); 1661 CU_ASSERT(ut_seq.complete); 1662 CU_ASSERT_EQUAL(ut_seq.status, 0); 1663 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1664 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1665 1666 /* Check decompress + copy + decompress + copy */ 1667 seq = NULL; 1668 completed = 0; 1669 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1670 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1671 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = NULL; 1672 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = NULL; 1673 1674 dst_iovs[0].iov_base = tmp[1]; 1675 dst_iovs[0].iov_len = sizeof(tmp[1]); 1676 src_iovs[0].iov_base = tmp[0]; 1677 src_iovs[0].iov_len = sizeof(tmp[0]); 1678 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1679 &src_iovs[0], 1, NULL, NULL, 0, 1680 ut_sequence_step_cb, &completed); 1681 CU_ASSERT_EQUAL(rc, 0); 1682 1683 dst_iovs[1].iov_base = tmp[2]; 1684 dst_iovs[1].iov_len = 2048; 1685 src_iovs[1].iov_base = tmp[1]; 1686 src_iovs[1].iov_len = sizeof(tmp[1]); 1687 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1688 &src_iovs[1], 1, NULL, NULL, 0, 1689 ut_sequence_step_cb, &completed); 1690 CU_ASSERT_EQUAL(rc, 0); 1691 1692 dst_iovs[2].iov_base = tmp[3]; 1693 dst_iovs[2].iov_len = 1024; 1694 src_iovs[2].iov_base = tmp[2]; 1695 src_iovs[2].iov_len = 2048; 1696 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1697 &src_iovs[2], 1, NULL, NULL, 0, 1698 ut_sequence_step_cb, &completed); 1699 CU_ASSERT_EQUAL(rc, 0); 1700 1701 dst_iovs[3].iov_base = buf; 1702 dst_iovs[3].iov_len = 1024; 1703 src_iovs[3].iov_base = tmp[3]; 1704 src_iovs[3].iov_len = 1024; 1705 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 1706 &src_iovs[3], 1, NULL, NULL, 0, 1707 ut_sequence_step_cb, &completed); 1708 CU_ASSERT_EQUAL(rc, 0); 1709 1710 ut_seq.complete = false; 1711 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1712 1713 poll_threads(); 1714 1715 CU_ASSERT_EQUAL(completed, 4); 1716 CU_ASSERT(ut_seq.complete); 1717 CU_ASSERT_EQUAL(ut_seq.status, 0); 1718 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1719 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 2); 1720 1721 /* Check two copy operations - one of them should be removed, while the other should be 1722 * executed normally */ 1723 seq = NULL; 1724 completed = 0; 1725 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1726 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovcnt = 1; 1727 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovcnt = 1; 1728 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovs = &exp_iovs[0]; 1729 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovs = &exp_iovs[1]; 1730 exp_iovs[0].iov_base = tmp[0]; 1731 exp_iovs[0].iov_len = sizeof(tmp[0]); 1732 exp_iovs[1].iov_base = buf; 1733 exp_iovs[1].iov_len = sizeof(buf); 1734 1735 dst_iovs[0].iov_base = tmp[1]; 1736 dst_iovs[0].iov_len = sizeof(tmp[1]); 1737 src_iovs[0].iov_base = tmp[0]; 1738 src_iovs[0].iov_len = sizeof(tmp[0]); 1739 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1740 &src_iovs[0], 1, NULL, NULL, 0, 1741 ut_sequence_step_cb, &completed); 1742 CU_ASSERT_EQUAL(rc, 0); 1743 1744 dst_iovs[1].iov_base = buf; 1745 dst_iovs[1].iov_len = sizeof(buf); 1746 src_iovs[1].iov_base = tmp[1]; 1747 src_iovs[1].iov_len = sizeof(tmp[1]); 1748 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1749 &src_iovs[1], 1, NULL, NULL, 0, 1750 ut_sequence_step_cb, &completed); 1751 CU_ASSERT_EQUAL(rc, 0); 1752 1753 ut_seq.complete = false; 1754 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1755 1756 poll_threads(); 1757 1758 CU_ASSERT_EQUAL(completed, 2); 1759 CU_ASSERT(ut_seq.complete); 1760 CU_ASSERT_EQUAL(ut_seq.status, 0); 1761 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 1762 1763 /* Check fill + copy */ 1764 seq = NULL; 1765 completed = 0; 1766 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1767 g_seq_operations[SPDK_ACCEL_OPC_FILL].count = 0; 1768 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovs = NULL; 1769 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovs = NULL; 1770 g_seq_operations[SPDK_ACCEL_OPC_FILL].dst_iovcnt = 1; 1771 g_seq_operations[SPDK_ACCEL_OPC_FILL].dst_iovs = &exp_iovs[0]; 1772 exp_iovs[0].iov_base = buf; 1773 exp_iovs[0].iov_len = sizeof(buf); 1774 1775 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 1776 ut_sequence_step_cb, &completed); 1777 CU_ASSERT_EQUAL(rc, 0); 1778 1779 dst_iovs[0].iov_base = buf; 1780 dst_iovs[0].iov_len = sizeof(buf); 1781 src_iovs[0].iov_base = tmp[0]; 1782 src_iovs[0].iov_len = sizeof(tmp[0]); 1783 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1784 &src_iovs[0], 1, NULL, NULL, 0, 1785 ut_sequence_step_cb, &completed); 1786 CU_ASSERT_EQUAL(rc, 0); 1787 1788 ut_seq.complete = false; 1789 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1790 1791 poll_threads(); 1792 1793 CU_ASSERT_EQUAL(completed, 2); 1794 CU_ASSERT(ut_seq.complete); 1795 CU_ASSERT_EQUAL(ut_seq.status, 0); 1796 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1797 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 1); 1798 1799 /* Check copy + encrypt + copy */ 1800 seq = NULL; 1801 completed = 0; 1802 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1803 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 1804 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovcnt = 1; 1805 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovcnt = 1; 1806 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovs = &exp_iovs[0]; 1807 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovs = &exp_iovs[1]; 1808 exp_iovs[0].iov_base = tmp[0]; 1809 exp_iovs[0].iov_len = sizeof(tmp[0]); 1810 exp_iovs[1].iov_base = buf; 1811 exp_iovs[1].iov_len = sizeof(buf); 1812 1813 dst_iovs[0].iov_base = tmp[1]; 1814 dst_iovs[0].iov_len = sizeof(tmp[1]); 1815 src_iovs[0].iov_base = tmp[0]; 1816 src_iovs[0].iov_len = sizeof(tmp[0]); 1817 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1818 &src_iovs[0], 1, NULL, NULL, 0, 1819 ut_sequence_step_cb, &completed); 1820 CU_ASSERT_EQUAL(rc, 0); 1821 1822 dst_iovs[1].iov_base = tmp[2]; 1823 dst_iovs[1].iov_len = sizeof(tmp[2]); 1824 src_iovs[1].iov_base = tmp[1]; 1825 src_iovs[1].iov_len = sizeof(tmp[1]); 1826 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1827 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1828 ut_sequence_step_cb, &completed); 1829 CU_ASSERT_EQUAL(rc, 0); 1830 1831 dst_iovs[2].iov_base = buf; 1832 dst_iovs[2].iov_len = sizeof(buf); 1833 src_iovs[2].iov_base = tmp[2]; 1834 src_iovs[2].iov_len = sizeof(tmp[2]); 1835 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1836 &src_iovs[2], 1, NULL, NULL, 0, 1837 ut_sequence_step_cb, &completed); 1838 CU_ASSERT_EQUAL(rc, 0); 1839 1840 ut_seq.complete = false; 1841 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1842 1843 poll_threads(); 1844 1845 CU_ASSERT_EQUAL(completed, 3); 1846 CU_ASSERT(ut_seq.complete); 1847 CU_ASSERT_EQUAL(ut_seq.status, 0); 1848 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1849 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 1850 1851 /* Check copy + decrypt + copy */ 1852 seq = NULL; 1853 completed = 0; 1854 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1855 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 1856 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1857 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1858 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1859 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1860 exp_iovs[0].iov_base = tmp[0]; 1861 exp_iovs[0].iov_len = sizeof(tmp[0]); 1862 exp_iovs[1].iov_base = buf; 1863 exp_iovs[1].iov_len = sizeof(buf); 1864 1865 dst_iovs[0].iov_base = tmp[1]; 1866 dst_iovs[0].iov_len = sizeof(tmp[1]); 1867 src_iovs[0].iov_base = tmp[0]; 1868 src_iovs[0].iov_len = sizeof(tmp[0]); 1869 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1870 &src_iovs[0], 1, NULL, NULL, 0, 1871 ut_sequence_step_cb, &completed); 1872 CU_ASSERT_EQUAL(rc, 0); 1873 1874 dst_iovs[1].iov_base = tmp[2]; 1875 dst_iovs[1].iov_len = sizeof(tmp[2]); 1876 src_iovs[1].iov_base = tmp[1]; 1877 src_iovs[1].iov_len = sizeof(tmp[1]); 1878 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1879 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1880 ut_sequence_step_cb, &completed); 1881 CU_ASSERT_EQUAL(rc, 0); 1882 1883 dst_iovs[2].iov_base = buf; 1884 dst_iovs[2].iov_len = sizeof(buf); 1885 src_iovs[2].iov_base = tmp[2]; 1886 src_iovs[2].iov_len = sizeof(tmp[2]); 1887 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1888 &src_iovs[2], 1, NULL, NULL, 0, 1889 ut_sequence_step_cb, &completed); 1890 CU_ASSERT_EQUAL(rc, 0); 1891 1892 ut_seq.complete = false; 1893 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1894 1895 poll_threads(); 1896 1897 CU_ASSERT_EQUAL(completed, 3); 1898 CU_ASSERT(ut_seq.complete); 1899 CU_ASSERT_EQUAL(ut_seq.status, 0); 1900 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1901 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 1902 1903 /* Check a sequence with memory domains and verify their pointers (and their contexts) are 1904 * correctly set on the next/prev task when a copy is removed */ 1905 seq = NULL; 1906 completed = 0; 1907 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1908 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 1909 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1910 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1911 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1912 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1913 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_domain = (void *)0x1; 1914 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_domain_ctx = (void *)0x2; 1915 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_domain = (void *)0x3; 1916 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_domain_ctx = (void *)0x4; 1917 exp_iovs[0].iov_base = tmp[0]; 1918 exp_iovs[0].iov_len = sizeof(tmp[0]); 1919 exp_iovs[1].iov_base = buf; 1920 exp_iovs[1].iov_len = sizeof(buf); 1921 1922 dst_iovs[0].iov_base = tmp[1]; 1923 dst_iovs[0].iov_len = sizeof(tmp[1]); 1924 src_iovs[0].iov_base = tmp[0]; 1925 src_iovs[0].iov_len = sizeof(tmp[0]); 1926 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, (void *)0xdead, (void *)0xbeef, 1927 &src_iovs[0], 1, (void *)0x3, (void *)0x4, 0, 1928 ut_sequence_step_cb, &completed); 1929 CU_ASSERT_EQUAL(rc, 0); 1930 1931 dst_iovs[1].iov_base = tmp[2]; 1932 dst_iovs[1].iov_len = sizeof(tmp[2]); 1933 src_iovs[1].iov_base = tmp[1]; 1934 src_iovs[1].iov_len = sizeof(tmp[1]); 1935 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 1936 &src_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 0, 1937 sizeof(tmp[2]), 0, ut_sequence_step_cb, &completed); 1938 CU_ASSERT_EQUAL(rc, 0); 1939 1940 dst_iovs[2].iov_base = buf; 1941 dst_iovs[2].iov_len = sizeof(buf); 1942 src_iovs[2].iov_base = tmp[2]; 1943 src_iovs[2].iov_len = sizeof(tmp[2]); 1944 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, (void *)0x1, (void *)0x2, 1945 &src_iovs[2], 1, (void *)0xdead, (void *)0xbeef, 0, 1946 ut_sequence_step_cb, &completed); 1947 CU_ASSERT_EQUAL(rc, 0); 1948 1949 ut_seq.complete = false; 1950 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1951 1952 poll_threads(); 1953 1954 CU_ASSERT_EQUAL(completed, 3); 1955 CU_ASSERT(ut_seq.complete); 1956 CU_ASSERT_EQUAL(ut_seq.status, 0); 1957 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1958 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 1959 1960 /* Cleanup module pointers to make subsequent tests work correctly */ 1961 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1962 g_modules_opc[i] = modules[i]; 1963 } 1964 1965 g_module.supports_memory_domains = false; 1966 ut_clear_operations(); 1967 spdk_put_io_channel(ioch); 1968 poll_threads(); 1969 } 1970 1971 static int 1972 ut_submit_decompress(struct spdk_io_channel *ch, struct spdk_accel_task *task) 1973 { 1974 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 1975 1976 spdk_accel_task_complete(task, 0); 1977 1978 return 0; 1979 } 1980 1981 static void 1982 test_sequence_accel_buffers(void) 1983 { 1984 struct spdk_accel_sequence *seq = NULL; 1985 struct spdk_io_channel *ioch; 1986 struct accel_io_channel *accel_ch; 1987 struct ut_sequence ut_seq; 1988 struct iovec src_iovs[3], dst_iovs[3]; 1989 char srcbuf[4096], dstbuf[4096], expected[4096]; 1990 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 1991 void *buf[2], *domain_ctx[2], *iobuf_buf; 1992 struct spdk_memory_domain *domain[2]; 1993 struct spdk_iobuf_buffer *cache_entry; 1994 spdk_iobuf_buffer_stailq_t small_cache; 1995 uint32_t small_cache_count; 1996 int i, rc, completed; 1997 struct spdk_iobuf_opts opts_iobuf = {}; 1998 1999 /* Set up the iobuf to always use the "small" pool */ 2000 opts_iobuf.large_bufsize = 0x20000; 2001 opts_iobuf.large_pool_count = 0; 2002 opts_iobuf.small_bufsize = 0x10000; 2003 opts_iobuf.small_pool_count = 32; 2004 2005 rc = spdk_iobuf_set_opts(&opts_iobuf); 2006 CU_ASSERT(rc == 0); 2007 2008 ioch = spdk_accel_get_io_channel(); 2009 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2010 2011 /* Override the submit_tasks function */ 2012 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2013 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2014 modules[i] = g_modules_opc[i]; 2015 g_modules_opc[i] = g_module; 2016 } 2017 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2018 * decompress operations together in one sequence. 2019 */ 2020 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2021 g_seq_operations[SPDK_ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2022 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2023 2024 /* Check the simplest case: one operation using accel buffer as destination + copy operation 2025 * specifying the actual destination buffer 2026 */ 2027 memset(srcbuf, 0xa5, 4096); 2028 memset(dstbuf, 0x0, 4096); 2029 memset(expected, 0xa5, 4096); 2030 completed = 0; 2031 seq = NULL; 2032 2033 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2034 CU_ASSERT_EQUAL(rc, 0); 2035 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2036 2037 src_iovs[0].iov_base = srcbuf; 2038 src_iovs[0].iov_len = 4096; 2039 dst_iovs[0].iov_base = buf[0]; 2040 dst_iovs[0].iov_len = 4096; 2041 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2042 &src_iovs[0], 1, NULL, NULL, 0, 2043 ut_sequence_step_cb, &completed); 2044 CU_ASSERT_EQUAL(rc, 0); 2045 2046 src_iovs[1].iov_base = buf[0]; 2047 src_iovs[1].iov_len = 4096; 2048 dst_iovs[1].iov_base = dstbuf; 2049 dst_iovs[1].iov_len = 4096; 2050 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2051 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 2052 ut_sequence_step_cb, &completed); 2053 CU_ASSERT_EQUAL(rc, 0); 2054 2055 ut_seq.complete = false; 2056 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2057 2058 poll_threads(); 2059 2060 CU_ASSERT_EQUAL(completed, 2); 2061 CU_ASSERT(ut_seq.complete); 2062 CU_ASSERT_EQUAL(ut_seq.status, 0); 2063 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2064 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2065 2066 /* Start with a fill operation using accel buffer, followed by a decompress using another 2067 * accel buffer as dst, followed by a copy operation specifying dst buffer of the whole 2068 * sequence 2069 */ 2070 memset(srcbuf, 0x0, 4096); 2071 memset(dstbuf, 0x0, 4096); 2072 memset(expected, 0x5a, 4096); 2073 completed = 0; 2074 seq = NULL; 2075 2076 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2077 CU_ASSERT_EQUAL(rc, 0); 2078 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2079 2080 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2081 ut_sequence_step_cb, &completed); 2082 CU_ASSERT_EQUAL(rc, 0); 2083 2084 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2085 CU_ASSERT_EQUAL(rc, 0); 2086 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2087 2088 src_iovs[0].iov_base = buf[0]; 2089 src_iovs[0].iov_len = 4096; 2090 dst_iovs[0].iov_base = buf[1]; 2091 dst_iovs[0].iov_len = 4096; 2092 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2093 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2094 ut_sequence_step_cb, &completed); 2095 CU_ASSERT_EQUAL(rc, 0); 2096 2097 src_iovs[1].iov_base = buf[1]; 2098 src_iovs[1].iov_len = 4096; 2099 dst_iovs[1].iov_base = dstbuf; 2100 dst_iovs[1].iov_len = 4096; 2101 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2102 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2103 ut_sequence_step_cb, &completed); 2104 2105 ut_seq.complete = false; 2106 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2107 2108 poll_threads(); 2109 2110 CU_ASSERT_EQUAL(completed, 3); 2111 CU_ASSERT(ut_seq.complete); 2112 CU_ASSERT_EQUAL(ut_seq.status, 0); 2113 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2114 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2115 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2116 2117 /* Check the same, but with two decompress operations with the first one being in-place */ 2118 memset(srcbuf, 0x0, 4096); 2119 memset(dstbuf, 0x0, 4096); 2120 memset(expected, 0x5a, 4096); 2121 completed = 0; 2122 seq = NULL; 2123 2124 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2125 CU_ASSERT_EQUAL(rc, 0); 2126 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2127 2128 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2129 ut_sequence_step_cb, &completed); 2130 CU_ASSERT_EQUAL(rc, 0); 2131 2132 src_iovs[0].iov_base = buf[0]; 2133 src_iovs[0].iov_len = 4096; 2134 dst_iovs[0].iov_base = buf[0]; 2135 dst_iovs[0].iov_len = 4096; 2136 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2137 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2138 ut_sequence_step_cb, &completed); 2139 CU_ASSERT_EQUAL(rc, 0); 2140 2141 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2142 CU_ASSERT_EQUAL(rc, 0); 2143 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2144 2145 src_iovs[1].iov_base = buf[0]; 2146 src_iovs[1].iov_len = 4096; 2147 dst_iovs[1].iov_base = buf[1]; 2148 dst_iovs[1].iov_len = 4096; 2149 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, domain[1], domain_ctx[1], 2150 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 2151 ut_sequence_step_cb, &completed); 2152 CU_ASSERT_EQUAL(rc, 0); 2153 2154 src_iovs[2].iov_base = buf[1]; 2155 src_iovs[2].iov_len = 4096; 2156 dst_iovs[2].iov_base = dstbuf; 2157 dst_iovs[2].iov_len = 4096; 2158 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 2159 &src_iovs[2], 1, domain[1], domain_ctx[1], 0, 2160 ut_sequence_step_cb, &completed); 2161 2162 ut_seq.complete = false; 2163 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2164 2165 poll_threads(); 2166 2167 CU_ASSERT_EQUAL(completed, 4); 2168 CU_ASSERT(ut_seq.complete); 2169 CU_ASSERT_EQUAL(ut_seq.status, 0); 2170 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2171 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2172 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2173 2174 /* Check that specifying offsets within accel buffers works correctly */ 2175 memset(srcbuf, 0x0, 4096); 2176 memset(dstbuf, 0x0, 4096); 2177 completed = 0; 2178 seq = NULL; 2179 2180 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2181 CU_ASSERT_EQUAL(rc, 0); 2182 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2183 2184 /* Fill in each 1K of the buffer with different pattern */ 2185 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2186 ut_sequence_step_cb, &completed); 2187 CU_ASSERT_EQUAL(rc, 0); 2188 2189 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2190 0x5a, 0, ut_sequence_step_cb, &completed); 2191 CU_ASSERT_EQUAL(rc, 0); 2192 2193 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2194 0xfe, 0, ut_sequence_step_cb, &completed); 2195 CU_ASSERT_EQUAL(rc, 0); 2196 2197 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2198 0xed, 0, ut_sequence_step_cb, &completed); 2199 CU_ASSERT_EQUAL(rc, 0); 2200 2201 src_iovs[0].iov_base = buf[0]; 2202 src_iovs[0].iov_len = 4096; 2203 dst_iovs[0].iov_base = dstbuf; 2204 dst_iovs[0].iov_len = 4096; 2205 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2206 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2207 ut_sequence_step_cb, &completed); 2208 2209 ut_seq.complete = false; 2210 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2211 2212 poll_threads(); 2213 2214 memset(&expected[0], 0xa5, 1024); 2215 memset(&expected[1024], 0x5a, 1024); 2216 memset(&expected[2048], 0xfe, 1024); 2217 memset(&expected[3072], 0xed, 1024); 2218 CU_ASSERT_EQUAL(completed, 5); 2219 CU_ASSERT(ut_seq.complete); 2220 CU_ASSERT_EQUAL(ut_seq.status, 0); 2221 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2222 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2223 2224 /* Check the same but this time with a decompress operation on part of the buffer (512B 2225 * offset) */ 2226 memset(srcbuf, 0x0, 4096); 2227 memset(dstbuf, 0x0, 4096); 2228 completed = 0; 2229 seq = NULL; 2230 2231 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2232 CU_ASSERT_EQUAL(rc, 0); 2233 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2234 2235 /* Fill in each 1K of the buffer with different pattern */ 2236 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2237 ut_sequence_step_cb, &completed); 2238 CU_ASSERT_EQUAL(rc, 0); 2239 2240 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2241 0x5a, 0, ut_sequence_step_cb, &completed); 2242 CU_ASSERT_EQUAL(rc, 0); 2243 2244 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2245 0xfe, 0, ut_sequence_step_cb, &completed); 2246 CU_ASSERT_EQUAL(rc, 0); 2247 2248 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2249 0xed, 0, ut_sequence_step_cb, &completed); 2250 CU_ASSERT_EQUAL(rc, 0); 2251 2252 rc = spdk_accel_get_buf(ioch, 3072, &buf[1], &domain[1], &domain_ctx[1]); 2253 CU_ASSERT_EQUAL(rc, 0); 2254 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2255 2256 src_iovs[0].iov_base = (char *)buf[0] + 512; 2257 src_iovs[0].iov_len = 3072; 2258 dst_iovs[0].iov_base = (char *)buf[1] + 256; 2259 dst_iovs[0].iov_len = 3072; 2260 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2261 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2262 ut_sequence_step_cb, &completed); 2263 2264 src_iovs[1].iov_base = (char *)buf[1] + 256; 2265 src_iovs[1].iov_len = 3072; 2266 dst_iovs[1].iov_base = dstbuf; 2267 dst_iovs[1].iov_len = 3072; 2268 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2269 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2270 ut_sequence_step_cb, &completed); 2271 2272 ut_seq.complete = false; 2273 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2274 2275 poll_threads(); 2276 2277 memset(&expected[0], 0xa5, 512); 2278 memset(&expected[512], 0x5a, 1024); 2279 memset(&expected[1536], 0xfe, 1024); 2280 memset(&expected[2560], 0xed, 512); 2281 CU_ASSERT_EQUAL(completed, 6); 2282 CU_ASSERT(ut_seq.complete); 2283 CU_ASSERT_EQUAL(ut_seq.status, 0); 2284 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 3072), 0); 2285 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2286 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2287 2288 /* Check that if iobuf pool is empty, the sequence processing will wait until a buffer is 2289 * available 2290 */ 2291 accel_ch = spdk_io_channel_get_ctx(ioch); 2292 small_cache_count = accel_ch->iobuf.small.cache_count; 2293 STAILQ_INIT(&small_cache); 2294 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2295 accel_ch->iobuf.small.cache_count = 0; 2296 accel_ch->iobuf.small.cache_size = 0; 2297 g_iobuf.small_pool_count = 0; 2298 2299 /* First allocate a single buffer used by two operations */ 2300 memset(srcbuf, 0x0, 4096); 2301 memset(dstbuf, 0x0, 4096); 2302 memset(expected, 0xa5, 4096); 2303 completed = 0; 2304 seq = NULL; 2305 2306 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2307 CU_ASSERT_EQUAL(rc, 0); 2308 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2309 2310 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0xa5, 0, 2311 ut_sequence_step_cb, &completed); 2312 CU_ASSERT_EQUAL(rc, 0); 2313 2314 src_iovs[0].iov_base = buf[0]; 2315 src_iovs[0].iov_len = 4096; 2316 dst_iovs[0].iov_base = dstbuf; 2317 dst_iovs[0].iov_len = 4096; 2318 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2319 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2320 ut_sequence_step_cb, &completed); 2321 2322 ut_seq.complete = false; 2323 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2324 2325 poll_threads(); 2326 2327 CU_ASSERT_EQUAL(completed, 0); 2328 CU_ASSERT(!ut_seq.complete); 2329 2330 /* Get a buffer and return it to the pool to trigger the sequence to finish */ 2331 g_iobuf.small_pool_count = 1; 2332 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2333 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2334 CU_ASSERT(g_iobuf.small_pool_count == 0); 2335 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2336 2337 poll_threads(); 2338 2339 CU_ASSERT_EQUAL(completed, 2); 2340 CU_ASSERT(ut_seq.complete); 2341 CU_ASSERT_EQUAL(ut_seq.status, 0); 2342 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2343 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2344 2345 /* Return the buffers back to the cache */ 2346 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2347 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2348 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2349 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2350 small_cache_count++; 2351 } 2352 accel_ch->iobuf.small.cache_count = 0; 2353 g_iobuf.small_pool_count = 0; 2354 2355 /* Check a bit more complex scenario, with two buffers in the sequence */ 2356 memset(srcbuf, 0x0, 4096); 2357 memset(dstbuf, 0x0, 4096); 2358 memset(expected, 0x5a, 4096); 2359 completed = 0; 2360 seq = NULL; 2361 2362 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2363 CU_ASSERT_EQUAL(rc, 0); 2364 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2365 2366 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2367 ut_sequence_step_cb, &completed); 2368 CU_ASSERT_EQUAL(rc, 0); 2369 2370 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2371 CU_ASSERT_EQUAL(rc, 0); 2372 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2373 2374 src_iovs[0].iov_base = buf[0]; 2375 src_iovs[0].iov_len = 4096; 2376 dst_iovs[0].iov_base = buf[1]; 2377 dst_iovs[0].iov_len = 4096; 2378 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2379 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2380 ut_sequence_step_cb, &completed); 2381 CU_ASSERT_EQUAL(rc, 0); 2382 2383 src_iovs[1].iov_base = buf[1]; 2384 src_iovs[1].iov_len = 4096; 2385 dst_iovs[1].iov_base = dstbuf; 2386 dst_iovs[1].iov_len = 4096; 2387 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2388 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2389 ut_sequence_step_cb, &completed); 2390 CU_ASSERT_EQUAL(rc, 0); 2391 2392 ut_seq.complete = false; 2393 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2394 2395 poll_threads(); 2396 2397 CU_ASSERT_EQUAL(completed, 0); 2398 CU_ASSERT(!ut_seq.complete); 2399 2400 g_iobuf.small_pool_count = 1; 2401 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2402 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2403 g_iobuf.small_pool_count = 0; 2404 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2405 2406 /* One buffer is not enough to finish the whole sequence */ 2407 poll_threads(); 2408 CU_ASSERT(!ut_seq.complete); 2409 2410 g_iobuf.small_pool_count = 1; 2411 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2412 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2413 g_iobuf.small_pool_count = 0; 2414 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2415 2416 poll_threads(); 2417 2418 CU_ASSERT_EQUAL(completed, 3); 2419 CU_ASSERT(ut_seq.complete); 2420 CU_ASSERT_EQUAL(ut_seq.status, 0); 2421 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2422 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2423 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2424 2425 /* Return the buffers back to the cache */ 2426 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2427 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2428 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2429 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2430 small_cache_count++; 2431 } 2432 accel_ch->iobuf.small.cache_count = 0; 2433 2434 g_iobuf.small_pool_count = 32; 2435 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2436 accel_ch->iobuf.small.cache_count = small_cache_count; 2437 2438 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2439 g_modules_opc[i] = modules[i]; 2440 } 2441 2442 ut_clear_operations(); 2443 spdk_put_io_channel(ioch); 2444 poll_threads(); 2445 } 2446 2447 static void 2448 ut_domain_ctx_init(struct ut_domain_ctx *ctx, void *base, size_t len, struct iovec *expected) 2449 { 2450 ctx->iov.iov_base = base; 2451 ctx->iov.iov_len = len; 2452 ctx->expected = *expected; 2453 ctx->pull_submit_status = 0; 2454 ctx->push_submit_status = 0; 2455 ctx->pull_complete_status = 0; 2456 ctx->push_complete_status = 0; 2457 } 2458 2459 static void 2460 test_sequence_memory_domain(void) 2461 { 2462 struct spdk_accel_sequence *seq = NULL; 2463 struct spdk_io_channel *ioch; 2464 struct accel_io_channel *accel_ch; 2465 struct ut_sequence ut_seq; 2466 struct ut_domain_ctx domctx[4]; 2467 struct spdk_iobuf_buffer *cache_entry; 2468 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2469 struct spdk_memory_domain *accel_domain; 2470 spdk_iobuf_buffer_stailq_t small_cache; 2471 char srcbuf[4096], dstbuf[4096], expected[4096], tmp[4096]; 2472 struct iovec src_iovs[2], dst_iovs[2]; 2473 uint32_t small_cache_count; 2474 void *accel_buf, *accel_domain_ctx, *iobuf_buf; 2475 int i, rc, completed; 2476 2477 ioch = spdk_accel_get_io_channel(); 2478 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2479 2480 /* Override the submit_tasks function */ 2481 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2482 g_module.supports_memory_domains = false; 2483 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2484 modules[i] = g_modules_opc[i]; 2485 g_modules_opc[i] = g_module; 2486 } 2487 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2488 * decompress operations together in one sequence. 2489 */ 2490 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2491 g_seq_operations[SPDK_ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2492 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2493 2494 /* First check the simplest case - single fill operation with dstbuf in domain */ 2495 memset(expected, 0xa5, sizeof(expected)); 2496 memset(dstbuf, 0x0, sizeof(dstbuf)); 2497 completed = 0; 2498 seq = NULL; 2499 2500 /* Use some garbage pointer as dst and use domain ctx to get the actual dstbuf */ 2501 dst_iovs[0].iov_base = (void *)0xcafebabe; 2502 dst_iovs[0].iov_len = sizeof(dstbuf); 2503 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2504 2505 rc = spdk_accel_append_fill(&seq, ioch, dst_iovs[0].iov_base, dst_iovs[0].iov_len, 2506 g_ut_domain, &domctx[0], 0xa5, 0, 2507 ut_sequence_step_cb, &completed); 2508 CU_ASSERT_EQUAL(rc, 0); 2509 2510 ut_seq.complete = false; 2511 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2512 2513 poll_threads(); 2514 CU_ASSERT_EQUAL(completed, 1); 2515 CU_ASSERT(ut_seq.complete); 2516 CU_ASSERT_EQUAL(ut_seq.status, 0); 2517 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2518 2519 /* Check operation with both srcbuf and dstbuf in remote memory domain */ 2520 memset(expected, 0x5a, sizeof(expected)); 2521 memset(dstbuf, 0x0, sizeof(dstbuf)); 2522 memset(srcbuf, 0x5a, sizeof(dstbuf)); 2523 completed = 0; 2524 seq = NULL; 2525 2526 src_iovs[0].iov_base = (void *)0xcafebabe; 2527 src_iovs[0].iov_len = sizeof(srcbuf); 2528 dst_iovs[0].iov_base = (void *)0xbeefdead; 2529 dst_iovs[0].iov_len = sizeof(dstbuf); 2530 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2531 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2532 2533 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2534 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2535 ut_sequence_step_cb, &completed); 2536 CU_ASSERT_EQUAL(rc, 0); 2537 2538 ut_seq.complete = false; 2539 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2540 2541 poll_threads(); 2542 CU_ASSERT_EQUAL(completed, 1); 2543 CU_ASSERT(ut_seq.complete); 2544 CU_ASSERT_EQUAL(ut_seq.status, 0); 2545 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2546 2547 /* Two operations using a buffer in remote domain */ 2548 memset(expected, 0xa5, sizeof(expected)); 2549 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2550 memset(tmp, 0x0, sizeof(tmp)); 2551 memset(dstbuf, 0x0, sizeof(dstbuf)); 2552 completed = 0; 2553 seq = NULL; 2554 2555 src_iovs[0].iov_base = (void *)0xcafebabe; 2556 src_iovs[0].iov_len = sizeof(srcbuf); 2557 dst_iovs[0].iov_base = (void *)0xbeefdead; 2558 dst_iovs[0].iov_len = sizeof(tmp); 2559 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2560 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2561 2562 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2563 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2564 ut_sequence_step_cb, &completed); 2565 CU_ASSERT_EQUAL(rc, 0); 2566 2567 src_iovs[1].iov_base = (void *)0xbeefdead; 2568 src_iovs[1].iov_len = sizeof(tmp); 2569 dst_iovs[1].iov_base = (void *)0xa5a5a5a5; 2570 dst_iovs[1].iov_len = sizeof(dstbuf); 2571 ut_domain_ctx_init(&domctx[2], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2572 ut_domain_ctx_init(&domctx[3], tmp, sizeof(tmp), &src_iovs[1]); 2573 2574 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[2], 2575 &src_iovs[1], 1, g_ut_domain, &domctx[3], 0, 2576 ut_sequence_step_cb, &completed); 2577 CU_ASSERT_EQUAL(rc, 0); 2578 2579 ut_seq.complete = false; 2580 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2581 2582 poll_threads(); 2583 CU_ASSERT_EQUAL(completed, 2); 2584 CU_ASSERT(ut_seq.complete); 2585 CU_ASSERT_EQUAL(ut_seq.status, 0); 2586 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2587 2588 /* Use buffer in remote memory domain and accel buffer at the same time */ 2589 memset(expected, 0x5a, sizeof(expected)); 2590 memset(srcbuf, 0x5a, sizeof(expected)); 2591 memset(dstbuf, 0x0, sizeof(dstbuf)); 2592 completed = 0; 2593 seq = NULL; 2594 2595 rc = spdk_accel_get_buf(ioch, sizeof(dstbuf), &accel_buf, &accel_domain, &accel_domain_ctx); 2596 CU_ASSERT_EQUAL(rc, 0); 2597 2598 src_iovs[0].iov_base = (void *)0xfeedbeef; 2599 src_iovs[0].iov_len = sizeof(srcbuf); 2600 dst_iovs[0].iov_base = accel_buf; 2601 dst_iovs[0].iov_len = sizeof(dstbuf); 2602 ut_domain_ctx_init(&domctx[0], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2603 2604 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, accel_domain, accel_domain_ctx, 2605 &src_iovs[0], 1, g_ut_domain, &domctx[0], 0, 2606 ut_sequence_step_cb, &completed); 2607 CU_ASSERT_EQUAL(rc, 0); 2608 2609 src_iovs[1].iov_base = accel_buf; 2610 src_iovs[1].iov_len = sizeof(dstbuf); 2611 dst_iovs[1].iov_base = (void *)0xbeeffeed; 2612 dst_iovs[1].iov_len = sizeof(dstbuf); 2613 ut_domain_ctx_init(&domctx[1], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2614 2615 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[1], 2616 &src_iovs[1], 1, accel_domain, accel_domain_ctx, 0, 2617 ut_sequence_step_cb, &completed); 2618 CU_ASSERT_EQUAL(rc, 0); 2619 2620 ut_seq.complete = false; 2621 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2622 2623 poll_threads(); 2624 CU_ASSERT_EQUAL(completed, 2); 2625 CU_ASSERT(ut_seq.complete); 2626 CU_ASSERT_EQUAL(ut_seq.status, 0); 2627 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2628 spdk_accel_put_buf(ioch, accel_buf, accel_domain, accel_domain_ctx); 2629 2630 /* Check that a sequence with memory domains is correctly executed if buffers are not 2631 * immediately available */ 2632 memset(expected, 0xa5, sizeof(expected)); 2633 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2634 memset(dstbuf, 0x0, sizeof(dstbuf)); 2635 completed = 0; 2636 seq = NULL; 2637 /* Make sure the buffer pool is empty */ 2638 accel_ch = spdk_io_channel_get_ctx(ioch); 2639 small_cache_count = accel_ch->iobuf.small.cache_count; 2640 STAILQ_INIT(&small_cache); 2641 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2642 accel_ch->iobuf.small.cache_count = 0; 2643 g_iobuf.small_pool_count = 0; 2644 2645 src_iovs[0].iov_base = (void *)0xdeadbeef; 2646 src_iovs[0].iov_len = sizeof(srcbuf); 2647 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2648 dst_iovs[0].iov_len = sizeof(dstbuf); 2649 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2650 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2651 2652 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2653 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2654 ut_sequence_step_cb, &completed); 2655 CU_ASSERT_EQUAL(rc, 0); 2656 2657 ut_seq.complete = false; 2658 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2659 2660 poll_threads(); 2661 CU_ASSERT_EQUAL(completed, 0); 2662 CU_ASSERT(!ut_seq.complete); 2663 2664 /* Get a buffer and return it to the pool to trigger the sequence to resume. It shouldn't 2665 * be able to complete, as it needs two buffers */ 2666 g_iobuf.small_pool_count = 1; 2667 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2668 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2669 g_iobuf.small_pool_count = 0; 2670 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2671 2672 CU_ASSERT_EQUAL(completed, 0); 2673 CU_ASSERT(!ut_seq.complete); 2674 2675 /* Return another buffer, this time the sequence should finish */ 2676 g_iobuf.small_pool_count = 1; 2677 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2678 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2679 g_iobuf.small_pool_count = 0; 2680 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2681 2682 poll_threads(); 2683 CU_ASSERT_EQUAL(completed, 1); 2684 CU_ASSERT(ut_seq.complete); 2685 CU_ASSERT_EQUAL(ut_seq.status, 0); 2686 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2687 2688 /* Return the buffers back to the cache */ 2689 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2690 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2691 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2692 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2693 small_cache_count++; 2694 } 2695 accel_ch->iobuf.small.cache_count = 0; 2696 2697 g_iobuf.small_pool_count = 32; 2698 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2699 accel_ch->iobuf.small.cache_count = small_cache_count; 2700 2701 /* Check error cases, starting with an error from spdk_memory_domain_pull_data() */ 2702 completed = 0; 2703 seq = NULL; 2704 2705 src_iovs[0].iov_base = (void *)0xdeadbeef; 2706 src_iovs[0].iov_len = sizeof(srcbuf); 2707 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2708 dst_iovs[0].iov_len = sizeof(dstbuf); 2709 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2710 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2711 domctx[1].pull_submit_status = -E2BIG; 2712 2713 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2714 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2715 ut_sequence_step_cb, &completed); 2716 CU_ASSERT_EQUAL(rc, 0); 2717 2718 ut_seq.complete = false; 2719 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2720 2721 CU_ASSERT_EQUAL(completed, 1); 2722 CU_ASSERT(ut_seq.complete); 2723 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 2724 2725 /* Check completion error from spdk_memory_domain_pull_data() */ 2726 completed = 0; 2727 seq = NULL; 2728 2729 src_iovs[0].iov_base = (void *)0xdeadbeef; 2730 src_iovs[0].iov_len = sizeof(srcbuf); 2731 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2732 dst_iovs[0].iov_len = sizeof(dstbuf); 2733 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2734 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2735 domctx[1].pull_complete_status = -EACCES; 2736 2737 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2738 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2739 ut_sequence_step_cb, &completed); 2740 CU_ASSERT_EQUAL(rc, 0); 2741 2742 ut_seq.complete = false; 2743 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2744 2745 CU_ASSERT_EQUAL(completed, 1); 2746 CU_ASSERT(ut_seq.complete); 2747 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 2748 2749 /* Check submission error from spdk_memory_domain_push_data() */ 2750 completed = 0; 2751 seq = NULL; 2752 2753 src_iovs[0].iov_base = (void *)0xdeadbeef; 2754 src_iovs[0].iov_len = sizeof(srcbuf); 2755 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2756 dst_iovs[0].iov_len = sizeof(dstbuf); 2757 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2758 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2759 domctx[0].push_submit_status = -EADDRINUSE; 2760 2761 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2762 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2763 ut_sequence_step_cb, &completed); 2764 CU_ASSERT_EQUAL(rc, 0); 2765 2766 ut_seq.complete = false; 2767 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2768 2769 CU_ASSERT_EQUAL(completed, 1); 2770 CU_ASSERT(ut_seq.complete); 2771 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 2772 2773 /* Check completion error from spdk_memory_domain_push_data() */ 2774 completed = 0; 2775 seq = NULL; 2776 2777 src_iovs[0].iov_base = (void *)0xdeadbeef; 2778 src_iovs[0].iov_len = sizeof(srcbuf); 2779 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2780 dst_iovs[0].iov_len = sizeof(dstbuf); 2781 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2782 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2783 domctx[0].push_complete_status = -EADDRNOTAVAIL; 2784 2785 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2786 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2787 ut_sequence_step_cb, &completed); 2788 CU_ASSERT_EQUAL(rc, 0); 2789 2790 ut_seq.complete = false; 2791 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2792 2793 CU_ASSERT_EQUAL(completed, 1); 2794 CU_ASSERT(ut_seq.complete); 2795 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 2796 2797 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2798 g_modules_opc[i] = modules[i]; 2799 } 2800 2801 ut_clear_operations(); 2802 spdk_put_io_channel(ioch); 2803 poll_threads(); 2804 } 2805 2806 static int 2807 ut_submit_decompress_memory_domain(struct spdk_io_channel *ch, struct spdk_accel_task *task) 2808 { 2809 struct ut_domain_ctx *ctx; 2810 struct iovec *src_iovs, *dst_iovs; 2811 uint32_t src_iovcnt, dst_iovcnt; 2812 2813 src_iovs = task->s.iovs; 2814 dst_iovs = task->d.iovs; 2815 src_iovcnt = task->s.iovcnt; 2816 dst_iovcnt = task->d.iovcnt; 2817 2818 if (task->src_domain != NULL) { 2819 ctx = task->src_domain_ctx; 2820 CU_ASSERT_EQUAL(memcmp(task->s.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2821 src_iovs = &ctx->iov; 2822 src_iovcnt = 1; 2823 } 2824 2825 if (task->dst_domain != NULL) { 2826 ctx = task->dst_domain_ctx; 2827 CU_ASSERT_EQUAL(memcmp(task->d.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2828 dst_iovs = &ctx->iov; 2829 dst_iovcnt = 1; 2830 } 2831 2832 spdk_iovcpy(src_iovs, src_iovcnt, dst_iovs, dst_iovcnt); 2833 spdk_accel_task_complete(task, 0); 2834 2835 return 0; 2836 } 2837 2838 static void 2839 test_sequence_module_memory_domain(void) 2840 { 2841 struct spdk_accel_sequence *seq = NULL; 2842 struct spdk_io_channel *ioch; 2843 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2844 struct spdk_memory_domain *accel_domain; 2845 struct ut_sequence ut_seq; 2846 struct ut_domain_ctx domctx[2]; 2847 struct iovec src_iovs[2], dst_iovs[2]; 2848 void *buf, *accel_domain_ctx; 2849 char srcbuf[4096], dstbuf[4096], tmp[4096], expected[4096]; 2850 int i, rc, completed; 2851 2852 ioch = spdk_accel_get_io_channel(); 2853 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2854 2855 /* Override the submit_tasks function */ 2856 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2857 g_module.supports_memory_domains = true; 2858 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2859 modules[i] = g_modules_opc[i]; 2860 g_modules_opc[i] = g_module; 2861 } 2862 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress_memory_domain; 2863 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2864 2865 /* Check a sequence with both buffers in memory domains */ 2866 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2867 memset(expected, 0xa5, sizeof(expected)); 2868 memset(dstbuf, 0, sizeof(dstbuf)); 2869 seq = NULL; 2870 completed = 0; 2871 2872 src_iovs[0].iov_base = (void *)0xcafebabe; 2873 src_iovs[0].iov_len = sizeof(srcbuf); 2874 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2875 dst_iovs[0].iov_len = sizeof(dstbuf); 2876 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2877 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2878 2879 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2880 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2881 ut_sequence_step_cb, &completed); 2882 CU_ASSERT_EQUAL(rc, 0); 2883 2884 ut_seq.complete = false; 2885 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2886 2887 poll_threads(); 2888 2889 CU_ASSERT_EQUAL(completed, 1); 2890 CU_ASSERT(ut_seq.complete); 2891 CU_ASSERT_EQUAL(ut_seq.status, 0); 2892 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2893 2894 /* Check two operations each with a single buffer in memory domain */ 2895 memset(srcbuf, 0x5a, sizeof(srcbuf)); 2896 memset(expected, 0x5a, sizeof(expected)); 2897 memset(dstbuf, 0, sizeof(dstbuf)); 2898 memset(tmp, 0, sizeof(tmp)); 2899 seq = NULL; 2900 completed = 0; 2901 2902 src_iovs[0].iov_base = srcbuf; 2903 src_iovs[0].iov_len = sizeof(srcbuf); 2904 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2905 dst_iovs[0].iov_len = sizeof(tmp); 2906 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2907 2908 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2909 &src_iovs[0], 1, NULL, NULL, 0, 2910 ut_sequence_step_cb, &completed); 2911 CU_ASSERT_EQUAL(rc, 0); 2912 2913 src_iovs[1].iov_base = (void *)0xfeedbeef; 2914 src_iovs[1].iov_len = sizeof(tmp); 2915 dst_iovs[1].iov_base = dstbuf; 2916 dst_iovs[1].iov_len = sizeof(dstbuf); 2917 ut_domain_ctx_init(&domctx[1], tmp, sizeof(tmp), &src_iovs[1]); 2918 2919 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2920 &src_iovs[1], 1, g_ut_domain, &domctx[1], 0, 2921 ut_sequence_step_cb, &completed); 2922 CU_ASSERT_EQUAL(rc, 0); 2923 2924 ut_seq.complete = false; 2925 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2926 2927 poll_threads(); 2928 2929 CU_ASSERT_EQUAL(completed, 2); 2930 CU_ASSERT(ut_seq.complete); 2931 CU_ASSERT_EQUAL(ut_seq.status, 0); 2932 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2933 2934 /* Check a sequence with an accel buffer and a buffer in a regular memory domain */ 2935 memset(expected, 0xa5, sizeof(expected)); 2936 memset(dstbuf, 0, sizeof(dstbuf)); 2937 memset(tmp, 0, sizeof(tmp)); 2938 seq = NULL; 2939 completed = 0; 2940 2941 rc = spdk_accel_get_buf(ioch, 4096, &buf, &accel_domain, &accel_domain_ctx); 2942 CU_ASSERT_EQUAL(rc, 0); 2943 2944 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, accel_domain, accel_domain_ctx, 2945 0xa5, 0, ut_sequence_step_cb, &completed); 2946 CU_ASSERT_EQUAL(rc, 0); 2947 2948 src_iovs[0].iov_base = buf; 2949 src_iovs[0].iov_len = 4096; 2950 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2951 dst_iovs[0].iov_len = sizeof(dstbuf); 2952 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2953 2954 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2955 &src_iovs[0], 1, accel_domain, accel_domain_ctx, 0, 2956 ut_sequence_step_cb, &completed); 2957 CU_ASSERT_EQUAL(rc, 0); 2958 2959 ut_seq.complete = false; 2960 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2961 2962 poll_threads(); 2963 2964 CU_ASSERT_EQUAL(completed, 2); 2965 CU_ASSERT(ut_seq.complete); 2966 CU_ASSERT_EQUAL(ut_seq.status, 0); 2967 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2968 2969 spdk_accel_put_buf(ioch, buf, accel_domain, accel_domain_ctx); 2970 2971 g_module.supports_memory_domains = false; 2972 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2973 g_modules_opc[i] = modules[i]; 2974 } 2975 2976 ut_clear_operations(); 2977 spdk_put_io_channel(ioch); 2978 poll_threads(); 2979 } 2980 2981 #ifdef SPDK_CONFIG_ISAL_CRYPTO 2982 static void 2983 ut_encrypt_cb(void *cb_arg, int status) 2984 { 2985 int *completed = cb_arg; 2986 2987 CU_ASSERT_EQUAL(status, 0); 2988 2989 *completed = 1; 2990 } 2991 2992 static void 2993 test_sequence_crypto(void) 2994 { 2995 struct spdk_accel_sequence *seq = NULL; 2996 struct spdk_io_channel *ioch; 2997 struct spdk_accel_crypto_key *key; 2998 struct spdk_accel_crypto_key_create_param key_params = { 2999 .cipher = "AES_XTS", 3000 .hex_key = "00112233445566778899aabbccddeeff", 3001 .hex_key2 = "ffeeddccbbaa99887766554433221100", 3002 .key_name = "ut_key", 3003 }; 3004 struct ut_sequence ut_seq; 3005 unsigned char buf[4096], encrypted[4096] = {}, data[4096], tmp[3][4096]; 3006 struct iovec src_iovs[4], dst_iovs[4]; 3007 int rc, completed = 0; 3008 size_t i; 3009 3010 ioch = spdk_accel_get_io_channel(); 3011 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3012 3013 rc = spdk_accel_crypto_key_create(&key_params); 3014 CU_ASSERT_EQUAL(rc, 0); 3015 key = spdk_accel_crypto_key_get(key_params.key_name); 3016 SPDK_CU_ASSERT_FATAL(key != NULL); 3017 3018 for (i = 0; i < sizeof(data); ++i) { 3019 data[i] = (uint8_t)i & 0xff; 3020 } 3021 3022 dst_iovs[0].iov_base = encrypted; 3023 dst_iovs[0].iov_len = sizeof(encrypted); 3024 src_iovs[0].iov_base = data; 3025 src_iovs[0].iov_len = sizeof(data); 3026 rc = spdk_accel_submit_encrypt(ioch, key, &dst_iovs[0], 1, &src_iovs[0], 1, 0, 4096, 0, 3027 ut_encrypt_cb, &completed); 3028 CU_ASSERT_EQUAL(rc, 0); 3029 3030 while (!completed) { 3031 poll_threads(); 3032 } 3033 3034 /* Verify that encryption operation in a sequence produces the same result */ 3035 seq = NULL; 3036 completed = 0; 3037 3038 dst_iovs[0].iov_base = tmp[0]; 3039 dst_iovs[0].iov_len = sizeof(tmp[0]); 3040 src_iovs[0].iov_base = data; 3041 src_iovs[0].iov_len = sizeof(data); 3042 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3043 &src_iovs[0], 1, NULL, NULL, 0, 3044 ut_sequence_step_cb, &completed); 3045 CU_ASSERT_EQUAL(rc, 0); 3046 3047 dst_iovs[1].iov_base = tmp[1]; 3048 dst_iovs[1].iov_len = sizeof(tmp[1]); 3049 src_iovs[1].iov_base = tmp[0]; 3050 src_iovs[1].iov_len = sizeof(tmp[0]); 3051 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3052 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3053 ut_sequence_step_cb, &completed); 3054 CU_ASSERT_EQUAL(rc, 0); 3055 3056 dst_iovs[2].iov_base = buf; 3057 dst_iovs[2].iov_len = sizeof(buf); 3058 src_iovs[2].iov_base = tmp[1]; 3059 src_iovs[2].iov_len = sizeof(tmp[1]); 3060 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3061 &src_iovs[2], 1, NULL, NULL, 0, 3062 ut_sequence_step_cb, &completed); 3063 CU_ASSERT_EQUAL(rc, 0); 3064 3065 ut_seq.complete = false; 3066 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3067 3068 poll_threads(); 3069 3070 CU_ASSERT_EQUAL(completed, 3); 3071 CU_ASSERT(ut_seq.complete); 3072 CU_ASSERT_EQUAL(ut_seq.status, 0); 3073 CU_ASSERT_EQUAL(memcmp(buf, encrypted, sizeof(buf)), 0); 3074 3075 /* Check that decryption produces the original buffer */ 3076 seq = NULL; 3077 completed = 0; 3078 memset(buf, 0, sizeof(buf)); 3079 3080 dst_iovs[0].iov_base = tmp[0]; 3081 dst_iovs[0].iov_len = sizeof(tmp[0]); 3082 src_iovs[0].iov_base = encrypted; 3083 src_iovs[0].iov_len = sizeof(encrypted); 3084 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3085 &src_iovs[0], 1, NULL, NULL, 0, 3086 ut_sequence_step_cb, &completed); 3087 CU_ASSERT_EQUAL(rc, 0); 3088 3089 dst_iovs[1].iov_base = tmp[1]; 3090 dst_iovs[1].iov_len = sizeof(tmp[1]); 3091 src_iovs[1].iov_base = tmp[0]; 3092 src_iovs[1].iov_len = sizeof(tmp[0]); 3093 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3094 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3095 ut_sequence_step_cb, &completed); 3096 CU_ASSERT_EQUAL(rc, 0); 3097 3098 dst_iovs[2].iov_base = buf; 3099 dst_iovs[2].iov_len = sizeof(buf); 3100 src_iovs[2].iov_base = tmp[1]; 3101 src_iovs[2].iov_len = sizeof(tmp[1]); 3102 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3103 &src_iovs[2], 1, NULL, NULL, 0, 3104 ut_sequence_step_cb, &completed); 3105 CU_ASSERT_EQUAL(rc, 0); 3106 3107 ut_seq.complete = false; 3108 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3109 3110 poll_threads(); 3111 3112 CU_ASSERT_EQUAL(completed, 3); 3113 CU_ASSERT(ut_seq.complete); 3114 CU_ASSERT_EQUAL(ut_seq.status, 0); 3115 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3116 3117 /* Check encrypt + decrypt in a single sequence */ 3118 seq = NULL; 3119 completed = 0; 3120 memset(buf, 0, sizeof(buf)); 3121 3122 dst_iovs[0].iov_base = tmp[0]; 3123 dst_iovs[0].iov_len = sizeof(tmp[0]); 3124 src_iovs[0].iov_base = data; 3125 src_iovs[0].iov_len = sizeof(data); 3126 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3127 &src_iovs[0], 1, NULL, NULL, 0, 3128 ut_sequence_step_cb, &completed); 3129 CU_ASSERT_EQUAL(rc, 0); 3130 3131 dst_iovs[1].iov_base = tmp[1]; 3132 dst_iovs[1].iov_len = sizeof(tmp[1]); 3133 src_iovs[1].iov_base = tmp[0]; 3134 src_iovs[1].iov_len = sizeof(tmp[0]); 3135 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3136 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3137 ut_sequence_step_cb, &completed); 3138 CU_ASSERT_EQUAL(rc, 0); 3139 3140 3141 dst_iovs[2].iov_base = tmp[2]; 3142 dst_iovs[2].iov_len = sizeof(tmp[2]); 3143 src_iovs[2].iov_base = tmp[1]; 3144 src_iovs[2].iov_len = sizeof(tmp[1]); 3145 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[2], 1, NULL, NULL, 3146 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3147 ut_sequence_step_cb, &completed); 3148 CU_ASSERT_EQUAL(rc, 0); 3149 3150 dst_iovs[3].iov_base = buf; 3151 dst_iovs[3].iov_len = sizeof(buf); 3152 src_iovs[3].iov_base = tmp[2]; 3153 src_iovs[3].iov_len = sizeof(tmp[2]); 3154 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 3155 &src_iovs[3], 1, NULL, NULL, 0, 3156 ut_sequence_step_cb, &completed); 3157 CU_ASSERT_EQUAL(rc, 0); 3158 3159 ut_seq.complete = false; 3160 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3161 3162 poll_threads(); 3163 3164 CU_ASSERT_EQUAL(completed, 4); 3165 CU_ASSERT(ut_seq.complete); 3166 CU_ASSERT_EQUAL(ut_seq.status, 0); 3167 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3168 3169 rc = spdk_accel_crypto_key_destroy(key); 3170 CU_ASSERT_EQUAL(rc, 0); 3171 spdk_put_io_channel(ioch); 3172 poll_threads(); 3173 } 3174 #endif /* SPDK_CONFIG_ISAL_CRYPTO */ 3175 3176 static int 3177 ut_submit_crypto(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3178 { 3179 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3180 3181 spdk_accel_task_complete(task, 0); 3182 3183 return 0; 3184 } 3185 3186 struct ut_driver_operation { 3187 int complete_status; 3188 int submit_status; 3189 int count; 3190 bool supported; 3191 }; 3192 3193 static struct ut_driver_operation g_drv_operations[SPDK_ACCEL_OPC_LAST]; 3194 static bool g_ut_driver_async_continue; 3195 3196 static void 3197 ut_driver_async_continue(void *arg) 3198 { 3199 struct spdk_accel_sequence *seq = arg; 3200 3201 spdk_accel_sequence_continue(seq); 3202 } 3203 3204 static int 3205 ut_driver_execute_sequence(struct spdk_io_channel *ch, struct spdk_accel_sequence *seq) 3206 { 3207 struct spdk_accel_task *task; 3208 struct ut_driver_operation *drv_ops; 3209 3210 while ((task = spdk_accel_sequence_first_task(seq)) != NULL) { 3211 drv_ops = &g_drv_operations[task->op_code]; 3212 if (!drv_ops->supported) { 3213 break; 3214 } 3215 3216 drv_ops->count++; 3217 if (drv_ops->submit_status != 0) { 3218 return drv_ops->submit_status; 3219 } 3220 3221 if (drv_ops->complete_status != 0) { 3222 spdk_accel_task_complete(task, drv_ops->complete_status); 3223 break; 3224 } 3225 3226 switch (task->op_code) { 3227 case SPDK_ACCEL_OPC_DECOMPRESS: 3228 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3229 break; 3230 case SPDK_ACCEL_OPC_FILL: 3231 spdk_iov_memset(task->d.iovs, task->d.iovcnt, 3232 (int)(task->fill_pattern & 0xff)); 3233 break; 3234 default: 3235 CU_ASSERT(0 && "unexpected opcode"); 3236 break; 3237 } 3238 3239 spdk_accel_task_complete(task, 0); 3240 } 3241 3242 if (g_ut_driver_async_continue) { 3243 spdk_thread_send_msg(spdk_get_thread(), ut_driver_async_continue, seq); 3244 } else { 3245 spdk_accel_sequence_continue(seq); 3246 } 3247 3248 return 0; 3249 } 3250 3251 static struct spdk_io_channel * 3252 ut_driver_get_io_channel(void) 3253 { 3254 return (void *)0xdeadbeef; 3255 } 3256 3257 static struct spdk_accel_driver g_ut_driver = { 3258 .name = "ut", 3259 .execute_sequence = ut_driver_execute_sequence, 3260 .get_io_channel = ut_driver_get_io_channel, 3261 }; 3262 3263 SPDK_ACCEL_DRIVER_REGISTER(ut, &g_ut_driver); 3264 3265 static void 3266 test_sequence_driver(void) 3267 { 3268 struct spdk_accel_sequence *seq = NULL; 3269 struct spdk_io_channel *ioch; 3270 struct spdk_accel_crypto_key key = {}; 3271 struct ut_sequence ut_seq; 3272 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3273 char buf[4096], tmp[3][4096], expected[4096]; 3274 struct iovec src_iovs[3], dst_iovs[3]; 3275 int i, rc, completed = 0; 3276 3277 ioch = spdk_accel_get_io_channel(); 3278 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3279 rc = spdk_accel_set_driver("ut"); 3280 SPDK_CU_ASSERT_FATAL(rc == 0); 3281 3282 /* Override the submit_tasks function */ 3283 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3284 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3285 modules[i] = g_modules_opc[i]; 3286 g_modules_opc[i] = g_module; 3287 } 3288 /* Intercept crypto operations, as they should be executed by an accel module */ 3289 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].submit = ut_submit_crypto; 3290 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].submit = ut_submit_crypto; 3291 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3292 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3293 g_seq_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3294 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3295 3296 g_drv_operations[SPDK_ACCEL_OPC_FILL].supported = true; 3297 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].supported = true; 3298 3299 /* First check a sequence that is fully executed using a driver, with the copy at the end 3300 * being removed */ 3301 seq = NULL; 3302 completed = 0; 3303 memset(buf, 0, sizeof(buf)); 3304 memset(tmp[0], 0, sizeof(tmp[0])); 3305 memset(tmp[1], 0, sizeof(tmp[1])); 3306 memset(&expected[0], 0xa5, 2048); 3307 memset(&expected[2048], 0xbe, 2048); 3308 3309 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], 2048, NULL, NULL, 0xa5, 0, 3310 ut_sequence_step_cb, &completed); 3311 CU_ASSERT_EQUAL(rc, 0); 3312 rc = spdk_accel_append_fill(&seq, ioch, &tmp[0][2048], 2048, NULL, NULL, 0xbe, 0, 3313 ut_sequence_step_cb, &completed); 3314 CU_ASSERT_EQUAL(rc, 0); 3315 3316 dst_iovs[0].iov_base = tmp[1]; 3317 dst_iovs[0].iov_len = sizeof(tmp[1]); 3318 src_iovs[0].iov_base = tmp[0]; 3319 src_iovs[0].iov_len = sizeof(tmp[0]); 3320 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3321 &src_iovs[0], 1, NULL, NULL, 0, 3322 ut_sequence_step_cb, &completed); 3323 CU_ASSERT_EQUAL(rc, 0); 3324 3325 dst_iovs[1].iov_base = buf; 3326 dst_iovs[1].iov_len = sizeof(buf); 3327 src_iovs[1].iov_base = tmp[1]; 3328 src_iovs[1].iov_len = sizeof(tmp[1]); 3329 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3330 &src_iovs[1], 1, NULL, NULL, 0, 3331 ut_sequence_step_cb, &completed); 3332 CU_ASSERT_EQUAL(rc, 0); 3333 3334 ut_seq.complete = false; 3335 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3336 3337 poll_threads(); 3338 3339 CU_ASSERT_EQUAL(completed, 4); 3340 CU_ASSERT(ut_seq.complete); 3341 CU_ASSERT_EQUAL(ut_seq.status, 0); 3342 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3343 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3344 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 3345 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 2); 3346 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3347 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3348 3349 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3350 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3351 3352 /* Check a sequence when the first two operations are executed by a driver, while the rest 3353 * is executed via modules */ 3354 seq = NULL; 3355 completed = 0; 3356 memset(buf, 0, sizeof(buf)); 3357 memset(tmp[0], 0, sizeof(tmp[0])); 3358 memset(tmp[1], 0, sizeof(tmp[1])); 3359 memset(tmp[2], 0, sizeof(tmp[2])); 3360 memset(&expected[0], 0xfe, 4096); 3361 3362 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 0, 3363 ut_sequence_step_cb, &completed); 3364 CU_ASSERT_EQUAL(rc, 0); 3365 3366 dst_iovs[0].iov_base = tmp[1]; 3367 dst_iovs[0].iov_len = sizeof(tmp[1]); 3368 src_iovs[0].iov_base = tmp[0]; 3369 src_iovs[0].iov_len = sizeof(tmp[0]); 3370 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3371 &src_iovs[0], 1, NULL, NULL, 0, 3372 ut_sequence_step_cb, &completed); 3373 CU_ASSERT_EQUAL(rc, 0); 3374 3375 dst_iovs[1].iov_base = tmp[2]; 3376 dst_iovs[1].iov_len = sizeof(tmp[2]); 3377 src_iovs[1].iov_base = tmp[1]; 3378 src_iovs[1].iov_len = sizeof(tmp[1]); 3379 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3380 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3381 ut_sequence_step_cb, &completed); 3382 CU_ASSERT_EQUAL(rc, 0); 3383 3384 dst_iovs[2].iov_base = buf; 3385 dst_iovs[2].iov_len = sizeof(buf); 3386 src_iovs[2].iov_base = tmp[2]; 3387 src_iovs[2].iov_len = sizeof(tmp[2]); 3388 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3389 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3390 ut_sequence_step_cb, &completed); 3391 CU_ASSERT_EQUAL(rc, 0); 3392 3393 ut_seq.complete = false; 3394 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3395 3396 poll_threads(); 3397 3398 CU_ASSERT_EQUAL(completed, 4); 3399 CU_ASSERT(ut_seq.complete); 3400 CU_ASSERT_EQUAL(ut_seq.status, 0); 3401 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3402 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3403 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3404 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3405 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3406 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3407 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3408 3409 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3410 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3411 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3412 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3413 3414 /* Check sequence when the first and last operations are executed through modules, while the 3415 * ones in the middle are executed by the driver */ 3416 seq = NULL; 3417 completed = 0; 3418 memset(buf, 0, sizeof(buf)); 3419 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3420 memset(tmp[1], 0, sizeof(tmp[1])); 3421 memset(tmp[2], 0, sizeof(tmp[2])); 3422 memset(&expected[0], 0xfe, 2048); 3423 memset(&expected[2048], 0xa5, 2048); 3424 3425 dst_iovs[0].iov_base = tmp[1]; 3426 dst_iovs[0].iov_len = sizeof(tmp[1]); 3427 src_iovs[0].iov_base = tmp[0]; 3428 src_iovs[0].iov_len = sizeof(tmp[0]); 3429 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3430 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3431 ut_sequence_step_cb, &completed); 3432 CU_ASSERT_EQUAL(rc, 0); 3433 3434 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xfe, 0, 3435 ut_sequence_step_cb, &completed); 3436 CU_ASSERT_EQUAL(rc, 0); 3437 3438 dst_iovs[1].iov_base = tmp[2]; 3439 dst_iovs[1].iov_len = sizeof(tmp[2]); 3440 src_iovs[1].iov_base = tmp[1]; 3441 src_iovs[1].iov_len = sizeof(tmp[1]); 3442 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3443 &src_iovs[1], 1, NULL, NULL, 0, 3444 ut_sequence_step_cb, &completed); 3445 CU_ASSERT_EQUAL(rc, 0); 3446 3447 dst_iovs[2].iov_base = buf; 3448 dst_iovs[2].iov_len = sizeof(buf); 3449 src_iovs[2].iov_base = tmp[2]; 3450 src_iovs[2].iov_len = sizeof(tmp[2]); 3451 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3452 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3453 ut_sequence_step_cb, &completed); 3454 CU_ASSERT_EQUAL(rc, 0); 3455 3456 ut_seq.complete = false; 3457 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3458 3459 poll_threads(); 3460 3461 CU_ASSERT_EQUAL(completed, 4); 3462 CU_ASSERT(ut_seq.complete); 3463 CU_ASSERT_EQUAL(ut_seq.status, 0); 3464 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3465 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3466 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3467 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3468 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3469 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3470 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3471 3472 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3473 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3474 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3475 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3476 3477 /* Check a sequence with operations executed by: module, driver, module, driver */ 3478 seq = NULL; 3479 completed = 0; 3480 memset(buf, 0, sizeof(buf)); 3481 memset(tmp[0], 0x5a, sizeof(tmp[0])); 3482 memset(tmp[1], 0, sizeof(tmp[1])); 3483 memset(tmp[2], 0, sizeof(tmp[2])); 3484 memset(&expected[0], 0xef, 2048); 3485 memset(&expected[2048], 0x5a, 2048); 3486 3487 dst_iovs[0].iov_base = tmp[1]; 3488 dst_iovs[0].iov_len = sizeof(tmp[1]); 3489 src_iovs[0].iov_base = tmp[0]; 3490 src_iovs[0].iov_len = sizeof(tmp[0]); 3491 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3492 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3493 ut_sequence_step_cb, &completed); 3494 CU_ASSERT_EQUAL(rc, 0); 3495 3496 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3497 ut_sequence_step_cb, &completed); 3498 CU_ASSERT_EQUAL(rc, 0); 3499 3500 dst_iovs[1].iov_base = tmp[2]; 3501 dst_iovs[1].iov_len = sizeof(tmp[2]); 3502 src_iovs[1].iov_base = tmp[1]; 3503 src_iovs[1].iov_len = sizeof(tmp[1]); 3504 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3505 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3506 ut_sequence_step_cb, &completed); 3507 CU_ASSERT_EQUAL(rc, 0); 3508 3509 dst_iovs[2].iov_base = buf; 3510 dst_iovs[2].iov_len = sizeof(buf); 3511 src_iovs[2].iov_base = tmp[2]; 3512 src_iovs[2].iov_len = sizeof(tmp[2]); 3513 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3514 &src_iovs[2], 1, NULL, NULL, 0, 3515 ut_sequence_step_cb, &completed); 3516 CU_ASSERT_EQUAL(rc, 0); 3517 3518 ut_seq.complete = false; 3519 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3520 3521 poll_threads(); 3522 3523 CU_ASSERT_EQUAL(completed, 4); 3524 CU_ASSERT(ut_seq.complete); 3525 CU_ASSERT_EQUAL(ut_seq.status, 0); 3526 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3527 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3528 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3529 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3530 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3531 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3532 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3533 3534 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3535 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3536 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3537 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3538 3539 /* Check that an error returned from driver's execute_sequence() will fail the whole 3540 * sequence and any subsequent operations won't be processed */ 3541 seq = NULL; 3542 completed = 0; 3543 memset(buf, 0, sizeof(buf)); 3544 memset(expected, 0, sizeof(expected)); 3545 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3546 g_drv_operations[SPDK_ACCEL_OPC_FILL].submit_status = -EPERM; 3547 3548 dst_iovs[0].iov_base = tmp[1]; 3549 dst_iovs[0].iov_len = sizeof(tmp[1]); 3550 src_iovs[0].iov_base = tmp[0]; 3551 src_iovs[0].iov_len = sizeof(tmp[0]); 3552 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3553 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3554 ut_sequence_step_cb, &completed); 3555 CU_ASSERT_EQUAL(rc, 0); 3556 3557 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3558 ut_sequence_step_cb, &completed); 3559 CU_ASSERT_EQUAL(rc, 0); 3560 3561 dst_iovs[1].iov_base = buf; 3562 dst_iovs[1].iov_len = sizeof(buf); 3563 src_iovs[1].iov_base = tmp[1]; 3564 src_iovs[1].iov_len = sizeof(tmp[1]); 3565 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3566 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3567 ut_sequence_step_cb, &completed); 3568 CU_ASSERT_EQUAL(rc, 0); 3569 3570 ut_seq.complete = false; 3571 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3572 3573 poll_threads(); 3574 3575 CU_ASSERT_EQUAL(completed, 3); 3576 CU_ASSERT(ut_seq.complete); 3577 CU_ASSERT_EQUAL(ut_seq.status, -EPERM); 3578 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3579 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3580 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 0); 3581 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3582 CU_ASSERT_EQUAL(memcmp(buf, expected, 4096), 0); 3583 3584 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3585 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3586 g_drv_operations[SPDK_ACCEL_OPC_FILL].submit_status = 0; 3587 3588 /* Check that a failed task completed by a driver will cause the whole sequence to be failed 3589 * and any subsequent operations won't be processed */ 3590 seq = NULL; 3591 completed = 0; 3592 memset(buf, 0, sizeof(buf)); 3593 memset(expected, 0, sizeof(expected)); 3594 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3595 g_drv_operations[SPDK_ACCEL_OPC_FILL].complete_status = -ENOENT; 3596 3597 dst_iovs[0].iov_base = tmp[1]; 3598 dst_iovs[0].iov_len = sizeof(tmp[1]); 3599 src_iovs[0].iov_base = tmp[0]; 3600 src_iovs[0].iov_len = sizeof(tmp[0]); 3601 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3602 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3603 ut_sequence_step_cb, &completed); 3604 CU_ASSERT_EQUAL(rc, 0); 3605 3606 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3607 ut_sequence_step_cb, &completed); 3608 CU_ASSERT_EQUAL(rc, 0); 3609 3610 dst_iovs[1].iov_base = buf; 3611 dst_iovs[1].iov_len = sizeof(buf); 3612 src_iovs[1].iov_base = tmp[1]; 3613 src_iovs[1].iov_len = sizeof(tmp[1]); 3614 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3615 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3616 ut_sequence_step_cb, &completed); 3617 CU_ASSERT_EQUAL(rc, 0); 3618 3619 ut_seq.complete = false; 3620 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3621 3622 poll_threads(); 3623 3624 CU_ASSERT_EQUAL(completed, 3); 3625 CU_ASSERT(ut_seq.complete); 3626 CU_ASSERT_EQUAL(ut_seq.status, -ENOENT); 3627 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3628 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3629 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 0); 3630 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3631 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3632 3633 g_drv_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 3634 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3635 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3636 3637 /* Check asynchronous spdk_accel_sequence_continue() */ 3638 g_ut_driver_async_continue = true; 3639 seq = NULL; 3640 completed = 0; 3641 memset(buf, 0, sizeof(buf)); 3642 memset(tmp[0], 0, sizeof(tmp[0])); 3643 memset(&expected[0], 0xfe, 4096); 3644 3645 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 0, 3646 ut_sequence_step_cb, &completed); 3647 CU_ASSERT_EQUAL(rc, 0); 3648 3649 dst_iovs[0].iov_base = buf; 3650 dst_iovs[0].iov_len = sizeof(buf); 3651 src_iovs[0].iov_base = tmp[0]; 3652 src_iovs[0].iov_len = sizeof(tmp[0]); 3653 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3654 &src_iovs[0], 1, NULL, NULL, 0, 3655 ut_sequence_step_cb, &completed); 3656 CU_ASSERT_EQUAL(rc, 0); 3657 3658 ut_seq.complete = false; 3659 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3660 3661 poll_threads(); 3662 3663 CU_ASSERT_EQUAL(completed, 2); 3664 CU_ASSERT(ut_seq.complete); 3665 CU_ASSERT_EQUAL(ut_seq.status, 0); 3666 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3667 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3668 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3669 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3670 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3671 3672 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3673 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3674 3675 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3676 g_modules_opc[i] = modules[i]; 3677 } 3678 3679 /* Clear the driver so that other tests won't use it */ 3680 g_accel_driver = NULL; 3681 memset(&g_drv_operations, 0, sizeof(g_drv_operations)); 3682 3683 ut_clear_operations(); 3684 spdk_put_io_channel(ioch); 3685 poll_threads(); 3686 } 3687 3688 struct ut_saved_iovs { 3689 struct iovec src; 3690 struct iovec dst; 3691 }; 3692 3693 static struct ut_saved_iovs g_seq_saved_iovs[SPDK_ACCEL_OPC_LAST]; 3694 3695 static int 3696 ut_submit_save_iovs(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3697 { 3698 SPDK_CU_ASSERT_FATAL(task->s.iovcnt == 1); 3699 SPDK_CU_ASSERT_FATAL(task->d.iovcnt == 1); 3700 3701 g_seq_saved_iovs[task->op_code].src = task->s.iovs[0]; 3702 g_seq_saved_iovs[task->op_code].dst = task->d.iovs[0]; 3703 3704 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3705 3706 spdk_accel_task_complete(task, 0); 3707 3708 return 0; 3709 } 3710 3711 static void 3712 test_sequence_same_iovs(void) 3713 { 3714 struct spdk_accel_sequence *seq = NULL; 3715 struct spdk_io_channel *ioch; 3716 struct spdk_accel_crypto_key key = {}; 3717 struct ut_sequence ut_seq; 3718 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3719 char buf[4096], tmp[4096], expected[4096]; 3720 struct iovec iovs[3], expected_siov, expected_diov; 3721 struct spdk_memory_domain *domain; 3722 void *accel_buf, *domain_ctx; 3723 int i, rc, completed = 0; 3724 3725 ioch = spdk_accel_get_io_channel(); 3726 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3727 3728 /* Override the submit_tasks function */ 3729 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3730 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3731 modules[i] = g_modules_opc[i]; 3732 g_modules_opc[i] = g_module; 3733 } 3734 /* Intercept crypto operations, as they should be executed by an accel module */ 3735 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].submit = ut_submit_save_iovs; 3736 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].submit = ut_submit_save_iovs; 3737 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3738 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3739 3740 /* Check that it's possible to use the same iovec ptr for different operations */ 3741 seq = NULL; 3742 completed = 0; 3743 memset(buf, 0, sizeof(buf)); 3744 memset(expected, 0xa5, sizeof(expected)); 3745 3746 iovs[0].iov_base = expected; 3747 iovs[0].iov_len = sizeof(expected); 3748 iovs[1].iov_base = tmp; 3749 iovs[1].iov_len = sizeof(tmp); 3750 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, NULL, NULL, 3751 &iovs[0], 1, NULL, NULL, 0, 4096, 0, 3752 ut_sequence_step_cb, &completed); 3753 CU_ASSERT_EQUAL(rc, 0); 3754 /* Reuse iov[1] as src */ 3755 iovs[2].iov_base = buf; 3756 iovs[2].iov_len = sizeof(buf); 3757 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3758 &iovs[1], 1, NULL, NULL, 0, 4096, 0, 3759 ut_sequence_step_cb, &completed); 3760 CU_ASSERT_EQUAL(rc, 0); 3761 3762 ut_seq.complete = false; 3763 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3764 3765 poll_threads(); 3766 3767 CU_ASSERT_EQUAL(completed, 2); 3768 CU_ASSERT(ut_seq.complete); 3769 CU_ASSERT_EQUAL(ut_seq.status, 0); 3770 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3771 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3772 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3773 expected_siov.iov_base = expected; 3774 expected_siov.iov_len = sizeof(expected); 3775 expected_diov.iov_base = tmp; 3776 expected_diov.iov_len = sizeof(tmp); 3777 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].src, 3778 &expected_siov, sizeof(expected_siov)), 0); 3779 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].dst, 3780 &expected_diov, sizeof(expected_diov)), 0); 3781 expected_siov.iov_base = tmp; 3782 expected_siov.iov_len = sizeof(tmp); 3783 expected_diov.iov_base = buf; 3784 expected_diov.iov_len = sizeof(buf); 3785 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].src, 3786 &expected_siov, sizeof(expected_siov)), 0); 3787 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].dst, 3788 &expected_diov, sizeof(expected_diov)), 0); 3789 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3790 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3791 3792 /* Check the same with an accel buffer */ 3793 seq = NULL; 3794 completed = 0; 3795 memset(buf, 0, sizeof(buf)); 3796 memset(expected, 0x5a, sizeof(expected)); 3797 3798 rc = spdk_accel_get_buf(ioch, sizeof(buf), &accel_buf, &domain, &domain_ctx); 3799 CU_ASSERT_EQUAL(rc, 0); 3800 3801 iovs[0].iov_base = expected; 3802 iovs[0].iov_len = sizeof(expected); 3803 iovs[1].iov_base = accel_buf; 3804 iovs[1].iov_len = sizeof(buf); 3805 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, domain, domain_ctx, 3806 &iovs[0], 1, NULL, NULL, 0, 4096, 0, 3807 ut_sequence_step_cb, &completed); 3808 CU_ASSERT_EQUAL(rc, 0); 3809 /* Reuse iov[1] as src */ 3810 iovs[2].iov_base = buf; 3811 iovs[2].iov_len = sizeof(buf); 3812 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3813 &iovs[1], 1, domain, domain_ctx, 0, 4096, 0, 3814 ut_sequence_step_cb, &completed); 3815 CU_ASSERT_EQUAL(rc, 0); 3816 3817 ut_seq.complete = false; 3818 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3819 3820 poll_threads(); 3821 3822 CU_ASSERT_EQUAL(completed, 2); 3823 CU_ASSERT(ut_seq.complete); 3824 CU_ASSERT_EQUAL(ut_seq.status, 0); 3825 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3826 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3827 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3828 expected_siov.iov_base = expected; 3829 expected_siov.iov_len = sizeof(expected); 3830 expected_diov.iov_base = buf; 3831 expected_diov.iov_len = sizeof(buf); 3832 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].src, 3833 &expected_siov, sizeof(expected_siov)), 0); 3834 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].dst, 3835 &expected_diov, sizeof(expected_diov)), 0); 3836 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].dst, 3837 &g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].src, 3838 sizeof(struct iovec)), 0); 3839 spdk_accel_put_buf(ioch, accel_buf, domain, domain_ctx); 3840 3841 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3842 g_modules_opc[i] = modules[i]; 3843 } 3844 3845 ut_clear_operations(); 3846 spdk_put_io_channel(ioch); 3847 poll_threads(); 3848 } 3849 3850 static void 3851 test_sequence_crc32(void) 3852 { 3853 struct spdk_accel_sequence *seq = NULL; 3854 struct spdk_io_channel *ioch; 3855 struct ut_sequence ut_seq; 3856 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3857 char buf[4096], tmp[3][4096]; 3858 struct iovec src_iovs[4], dst_iovs[4]; 3859 uint32_t crc, crc2; 3860 int i, rc, completed; 3861 3862 ioch = spdk_accel_get_io_channel(); 3863 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3864 3865 /* Override the submit_tasks function */ 3866 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3867 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3868 g_seq_operations[i].submit = sw_accel_submit_tasks; 3869 modules[i] = g_modules_opc[i]; 3870 g_modules_opc[i] = g_module; 3871 } 3872 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 3873 3874 /* First check the simplest case - single crc32c operation */ 3875 seq = NULL; 3876 completed = 0; 3877 crc = 0; 3878 memset(buf, 0xa5, sizeof(buf)); 3879 3880 src_iovs[0].iov_base = buf; 3881 src_iovs[0].iov_len = sizeof(buf); 3882 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3883 ut_sequence_step_cb, &completed); 3884 CU_ASSERT_EQUAL(rc, 0); 3885 3886 ut_seq.complete = false; 3887 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3888 3889 poll_threads(); 3890 CU_ASSERT_EQUAL(completed, 1); 3891 CU_ASSERT(ut_seq.complete); 3892 CU_ASSERT_EQUAL(ut_seq.status, 0); 3893 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3894 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3895 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3896 3897 /* Now check copy+crc - this should remove the copy operation and change the buffer for the 3898 * crc operation */ 3899 seq = NULL; 3900 completed = 0; 3901 crc = 0; 3902 memset(buf, 0x5a, sizeof(buf)); 3903 memset(&tmp[0], 0, sizeof(tmp[0])); 3904 3905 dst_iovs[0].iov_base = tmp[0]; 3906 dst_iovs[0].iov_len = sizeof(tmp[0]); 3907 src_iovs[0].iov_base = buf; 3908 src_iovs[0].iov_len = sizeof(buf); 3909 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3910 &src_iovs[0], 1, NULL, NULL, 0, 3911 ut_sequence_step_cb, &completed); 3912 CU_ASSERT_EQUAL(rc, 0); 3913 3914 src_iovs[1].iov_base = tmp[0]; 3915 src_iovs[1].iov_len = sizeof(tmp[0]); 3916 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 3917 ut_sequence_step_cb, &completed); 3918 CU_ASSERT_EQUAL(rc, 0); 3919 3920 ut_seq.complete = false; 3921 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3922 3923 poll_threads(); 3924 CU_ASSERT_EQUAL(completed, 2); 3925 CU_ASSERT(ut_seq.complete); 3926 CU_ASSERT_EQUAL(ut_seq.status, 0); 3927 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3928 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 3929 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3930 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3931 3932 /* Check crc+copy - this time the copy cannot be removed, because there's no operation 3933 * before crc to change the buffer */ 3934 seq = NULL; 3935 completed = 0; 3936 crc = 0; 3937 memset(buf, 0, sizeof(buf)); 3938 memset(&tmp[0], 0xa5, sizeof(tmp[0])); 3939 3940 src_iovs[0].iov_base = tmp[0]; 3941 src_iovs[0].iov_len = sizeof(tmp[0]); 3942 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3943 ut_sequence_step_cb, &completed); 3944 CU_ASSERT_EQUAL(rc, 0); 3945 3946 dst_iovs[1].iov_base = buf; 3947 dst_iovs[1].iov_len = sizeof(buf); 3948 src_iovs[1].iov_base = tmp[0]; 3949 src_iovs[1].iov_len = sizeof(tmp[0]); 3950 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3951 &src_iovs[1], 1, NULL, NULL, 0, 3952 ut_sequence_step_cb, &completed); 3953 CU_ASSERT_EQUAL(rc, 0); 3954 3955 ut_seq.complete = false; 3956 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3957 3958 poll_threads(); 3959 CU_ASSERT_EQUAL(completed, 2); 3960 CU_ASSERT(ut_seq.complete); 3961 CU_ASSERT_EQUAL(ut_seq.status, 0); 3962 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3963 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 3964 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 3965 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 3966 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3967 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 3968 3969 /* Check a sequence with an operation at the beginning that can have its buffer changed, two 3970 * crc operations and a copy at the end. The copy should be removed and the dst buffer of 3971 * the first operation and the src buffer of the crc operations should be changed. 3972 */ 3973 seq = NULL; 3974 completed = 0; 3975 crc = crc2 = 0; 3976 memset(buf, 0, sizeof(buf)); 3977 memset(&tmp[0], 0x5a, sizeof(tmp[0])); 3978 dst_iovs[0].iov_base = tmp[1]; 3979 dst_iovs[0].iov_len = sizeof(tmp[1]); 3980 src_iovs[0].iov_base = tmp[0]; 3981 src_iovs[0].iov_len = sizeof(tmp[0]); 3982 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3983 &src_iovs[0], 1, NULL, NULL, 0, 3984 ut_sequence_step_cb, &completed); 3985 CU_ASSERT_EQUAL(rc, 0); 3986 3987 src_iovs[1].iov_base = tmp[1]; 3988 src_iovs[1].iov_len = sizeof(tmp[1]); 3989 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 3990 ut_sequence_step_cb, &completed); 3991 CU_ASSERT_EQUAL(rc, 0); 3992 3993 src_iovs[2].iov_base = tmp[1]; 3994 src_iovs[2].iov_len = sizeof(tmp[1]); 3995 rc = spdk_accel_append_crc32c(&seq, ioch, &crc2, &src_iovs[2], 1, NULL, NULL, 0, 3996 ut_sequence_step_cb, &completed); 3997 CU_ASSERT_EQUAL(rc, 0); 3998 3999 dst_iovs[3].iov_base = buf; 4000 dst_iovs[3].iov_len = sizeof(buf); 4001 src_iovs[3].iov_base = tmp[1]; 4002 src_iovs[3].iov_len = sizeof(tmp[1]); 4003 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 4004 &src_iovs[3], 1, NULL, NULL, 0, 4005 ut_sequence_step_cb, &completed); 4006 CU_ASSERT_EQUAL(rc, 0); 4007 4008 ut_seq.complete = false; 4009 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4010 4011 poll_threads(); 4012 CU_ASSERT_EQUAL(completed, 4); 4013 CU_ASSERT(ut_seq.complete); 4014 CU_ASSERT_EQUAL(ut_seq.status, 0); 4015 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 4016 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 2); 4017 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 4018 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 4019 CU_ASSERT_EQUAL(crc, crc2); 4020 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 4021 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 4022 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 4023 4024 /* Check that a copy won't be removed if the buffers don't match */ 4025 seq = NULL; 4026 completed = 0; 4027 crc = 0; 4028 memset(buf, 0, sizeof(buf)); 4029 memset(&tmp[0], 0xa5, 2048); 4030 memset(&tmp[1], 0xfe, sizeof(tmp[1])); 4031 memset(&tmp[2], 0xfe, sizeof(tmp[1])); 4032 dst_iovs[0].iov_base = &tmp[1][2048]; 4033 dst_iovs[0].iov_len = 2048; 4034 src_iovs[0].iov_base = tmp[0]; 4035 src_iovs[0].iov_len = 2048; 4036 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 4037 &src_iovs[0], 1, NULL, NULL, 0, 4038 ut_sequence_step_cb, &completed); 4039 CU_ASSERT_EQUAL(rc, 0); 4040 4041 src_iovs[1].iov_base = &tmp[1][2048]; 4042 src_iovs[1].iov_len = 2048; 4043 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 4044 ut_sequence_step_cb, &completed); 4045 CU_ASSERT_EQUAL(rc, 0); 4046 4047 dst_iovs[2].iov_base = buf; 4048 dst_iovs[2].iov_len = sizeof(buf); 4049 src_iovs[2].iov_base = tmp[1]; 4050 src_iovs[2].iov_len = sizeof(tmp[1]); 4051 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 4052 &src_iovs[2], 1, NULL, NULL, 0, 4053 ut_sequence_step_cb, &completed); 4054 CU_ASSERT_EQUAL(rc, 0); 4055 4056 ut_seq.complete = false; 4057 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4058 4059 poll_threads(); 4060 CU_ASSERT_EQUAL(completed, 3); 4061 CU_ASSERT(ut_seq.complete); 4062 CU_ASSERT_EQUAL(ut_seq.status, 0); 4063 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 4064 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 4065 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 4066 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], 2048, ~0u)); 4067 CU_ASSERT_EQUAL(memcmp(buf, tmp[2], 2048), 0); 4068 CU_ASSERT_EQUAL(memcmp(&buf[2048], tmp[0], 2048), 0); 4069 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 4070 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 4071 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 4072 4073 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 4074 g_modules_opc[i] = modules[i]; 4075 } 4076 4077 ut_clear_operations(); 4078 spdk_put_io_channel(ioch); 4079 poll_threads(); 4080 } 4081 4082 static int 4083 test_sequence_setup(void) 4084 { 4085 int rc; 4086 4087 allocate_cores(1); 4088 allocate_threads(1); 4089 set_thread(0); 4090 4091 rc = spdk_iobuf_initialize(); 4092 if (rc != 0) { 4093 CU_ASSERT(false); 4094 return -1; 4095 } 4096 4097 rc = spdk_accel_initialize(); 4098 if (rc != 0) { 4099 CU_ASSERT(false); 4100 return -1; 4101 } 4102 4103 return 0; 4104 } 4105 4106 static void 4107 finish_cb(void *cb_arg) 4108 { 4109 bool *done = cb_arg; 4110 4111 *done = true; 4112 } 4113 4114 static int 4115 test_sequence_cleanup(void) 4116 { 4117 bool done = false; 4118 4119 spdk_accel_finish(finish_cb, &done); 4120 4121 while (!done) { 4122 poll_threads(); 4123 } 4124 4125 done = false; 4126 spdk_iobuf_finish(finish_cb, &done); 4127 while (!done) { 4128 poll_threads(); 4129 } 4130 4131 free_threads(); 4132 free_cores(); 4133 4134 return 0; 4135 } 4136 4137 int 4138 main(int argc, char **argv) 4139 { 4140 CU_pSuite suite = NULL, seq_suite; 4141 unsigned int num_failures; 4142 4143 CU_initialize_registry(); 4144 4145 /* Sequence tests require accel to be initialized normally, so run them before the other 4146 * tests which register accel modules which aren't fully implemented, causing accel 4147 * initialization to fail. 4148 */ 4149 seq_suite = CU_add_suite("accel_sequence", test_sequence_setup, test_sequence_cleanup); 4150 CU_ADD_TEST(seq_suite, test_sequence_fill_copy); 4151 CU_ADD_TEST(seq_suite, test_sequence_abort); 4152 CU_ADD_TEST(seq_suite, test_sequence_append_error); 4153 CU_ADD_TEST(seq_suite, test_sequence_completion_error); 4154 #ifdef SPDK_CONFIG_ISAL /* accel_sw requires isa-l for compression */ 4155 CU_ADD_TEST(seq_suite, test_sequence_decompress); 4156 CU_ADD_TEST(seq_suite, test_sequence_reverse); 4157 #endif 4158 CU_ADD_TEST(seq_suite, test_sequence_copy_elision); 4159 CU_ADD_TEST(seq_suite, test_sequence_accel_buffers); 4160 CU_ADD_TEST(seq_suite, test_sequence_memory_domain); 4161 CU_ADD_TEST(seq_suite, test_sequence_module_memory_domain); 4162 #ifdef SPDK_CONFIG_ISAL_CRYPTO /* accel_sw requires isa-l-crypto for crypto operations */ 4163 CU_ADD_TEST(seq_suite, test_sequence_crypto); 4164 #endif 4165 CU_ADD_TEST(seq_suite, test_sequence_driver); 4166 CU_ADD_TEST(seq_suite, test_sequence_same_iovs); 4167 CU_ADD_TEST(seq_suite, test_sequence_crc32); 4168 4169 suite = CU_add_suite("accel", test_setup, test_cleanup); 4170 CU_ADD_TEST(suite, test_spdk_accel_task_complete); 4171 CU_ADD_TEST(suite, test_get_task); 4172 CU_ADD_TEST(suite, test_spdk_accel_submit_copy); 4173 CU_ADD_TEST(suite, test_spdk_accel_submit_dualcast); 4174 CU_ADD_TEST(suite, test_spdk_accel_submit_compare); 4175 CU_ADD_TEST(suite, test_spdk_accel_submit_fill); 4176 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c); 4177 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32cv); 4178 CU_ADD_TEST(suite, test_spdk_accel_submit_copy_crc32c); 4179 CU_ADD_TEST(suite, test_spdk_accel_submit_xor); 4180 CU_ADD_TEST(suite, test_spdk_accel_module_find_by_name); 4181 CU_ADD_TEST(suite, test_spdk_accel_module_register); 4182 4183 num_failures = spdk_ut_run_tests(argc, argv, NULL); 4184 CU_cleanup_registry(); 4185 4186 return num_failures; 4187 } 4188