1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2021 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk_cunit.h" 8 #include "spdk_internal/mock.h" 9 #include "spdk_internal/accel_module.h" 10 #include "thread/thread_internal.h" 11 #include "common/lib/ut_multithread.c" 12 #include "common/lib/test_iobuf.c" 13 #include "accel/accel.c" 14 #include "accel/accel_sw.c" 15 #include "unit/lib/json_mock.c" 16 17 DEFINE_STUB_V(spdk_memory_domain_destroy, (struct spdk_memory_domain *domain)); 18 DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, 19 (struct spdk_memory_domain *domain), "UT_DMA"); 20 21 int 22 spdk_memory_domain_create(struct spdk_memory_domain **domain, enum spdk_dma_device_type type, 23 struct spdk_memory_domain_ctx *ctx, const char *id) 24 { 25 *domain = (void *)0xdeadbeef; 26 27 return 0; 28 } 29 30 struct ut_domain_ctx { 31 struct iovec iov; 32 struct iovec expected; 33 int pull_submit_status; 34 int push_submit_status; 35 int pull_complete_status; 36 int push_complete_status; 37 }; 38 39 static struct spdk_memory_domain *g_ut_domain = (void *)0xa55e1; 40 41 int 42 spdk_memory_domain_pull_data(struct spdk_memory_domain *sd, void *sctx, struct iovec *siov, 43 uint32_t siovcnt, struct iovec *diov, uint32_t diovcnt, 44 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 45 { 46 struct ut_domain_ctx *ctx = sctx; 47 48 CU_ASSERT_EQUAL(sd, g_ut_domain); 49 CU_ASSERT_EQUAL(siovcnt, 1); 50 CU_ASSERT_EQUAL(memcmp(siov, &ctx->expected, sizeof(*siov)), 0); 51 52 if (ctx->pull_submit_status != 0) { 53 return ctx->pull_submit_status; 54 } 55 56 if (ctx->pull_complete_status != 0) { 57 cpl_cb(cpl_arg, ctx->pull_complete_status); 58 return 0; 59 } 60 61 spdk_iovcpy(&ctx->iov, 1, diov, diovcnt); 62 cpl_cb(cpl_arg, 0); 63 64 return 0; 65 } 66 67 int 68 spdk_memory_domain_push_data(struct spdk_memory_domain *dd, void *dctx, struct iovec *diov, 69 uint32_t diovcnt, struct iovec *siov, uint32_t siovcnt, 70 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 71 { 72 struct ut_domain_ctx *ctx = dctx; 73 74 CU_ASSERT_EQUAL(dd, g_ut_domain); 75 CU_ASSERT_EQUAL(diovcnt, 1); 76 CU_ASSERT_EQUAL(memcmp(diov, &ctx->expected, sizeof(*diov)), 0); 77 78 if (ctx->push_submit_status != 0) { 79 return ctx->push_submit_status; 80 } 81 82 if (ctx->push_complete_status != 0) { 83 cpl_cb(cpl_arg, ctx->push_complete_status); 84 return 0; 85 } 86 87 spdk_iovcpy(siov, siovcnt, &ctx->iov, 1); 88 cpl_cb(cpl_arg, 0); 89 90 return 0; 91 } 92 93 /* global vars and setup/cleanup functions used for all test functions */ 94 struct spdk_accel_module_if g_module_if = {}; 95 struct accel_module g_module = { .module = &g_module_if }; 96 struct spdk_io_channel *g_ch = NULL; 97 struct accel_io_channel *g_accel_ch = NULL; 98 struct sw_accel_io_channel *g_sw_ch = NULL; 99 struct spdk_io_channel *g_module_ch = NULL; 100 101 static uint64_t g_opc_mask = 0; 102 103 static uint64_t 104 _accel_op_to_bit(enum accel_opcode opc) 105 { 106 return (1 << opc); 107 } 108 109 static bool 110 _supports_opcode(enum accel_opcode opc) 111 { 112 if (_accel_op_to_bit(opc) & g_opc_mask) { 113 return true; 114 } 115 return false; 116 } 117 118 static int 119 test_setup(void) 120 { 121 int i; 122 123 g_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct accel_io_channel)); 124 if (g_ch == NULL) { 125 /* for some reason the assert fatal macro doesn't work in the setup function. */ 126 CU_ASSERT(false); 127 return -1; 128 } 129 g_accel_ch = (struct accel_io_channel *)((char *)g_ch + sizeof(struct spdk_io_channel)); 130 g_module_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct sw_accel_io_channel)); 131 if (g_module_ch == NULL) { 132 CU_ASSERT(false); 133 return -1; 134 } 135 136 g_module_if.submit_tasks = sw_accel_submit_tasks; 137 g_module_if.name = "software"; 138 for (i = 0; i < ACCEL_OPC_LAST; i++) { 139 g_accel_ch->module_ch[i] = g_module_ch; 140 g_modules_opc[i] = g_module; 141 } 142 g_sw_ch = (struct sw_accel_io_channel *)((char *)g_module_ch + sizeof( 143 struct spdk_io_channel)); 144 TAILQ_INIT(&g_sw_ch->tasks_to_complete); 145 g_module_if.supports_opcode = _supports_opcode; 146 return 0; 147 } 148 149 static int 150 test_cleanup(void) 151 { 152 free(g_ch); 153 free(g_module_ch); 154 155 return 0; 156 } 157 158 #define DUMMY_ARG 0xDEADBEEF 159 static bool g_dummy_cb_called = false; 160 static void 161 dummy_cb_fn(void *cb_arg, int status) 162 { 163 CU_ASSERT(*(uint32_t *)cb_arg == DUMMY_ARG); 164 CU_ASSERT(status == 0); 165 g_dummy_cb_called = true; 166 } 167 168 static void 169 test_spdk_accel_task_complete(void) 170 { 171 struct spdk_accel_task accel_task = {}; 172 struct spdk_accel_task *expected_accel_task = NULL; 173 uint32_t cb_arg = DUMMY_ARG; 174 int status = 0; 175 176 accel_task.accel_ch = g_accel_ch; 177 accel_task.cb_fn = dummy_cb_fn; 178 accel_task.cb_arg = &cb_arg; 179 TAILQ_INIT(&g_accel_ch->task_pool); 180 181 /* Confirm cb is called and task added to list. */ 182 spdk_accel_task_complete(&accel_task, status); 183 CU_ASSERT(g_dummy_cb_called == true); 184 expected_accel_task = TAILQ_FIRST(&g_accel_ch->task_pool); 185 TAILQ_REMOVE(&g_accel_ch->task_pool, expected_accel_task, link); 186 CU_ASSERT(expected_accel_task == &accel_task); 187 } 188 189 static void 190 test_get_task(void) 191 { 192 struct spdk_accel_task *task; 193 struct spdk_accel_task _task; 194 void *cb_arg = NULL; 195 196 TAILQ_INIT(&g_accel_ch->task_pool); 197 198 /* no tasks left, return NULL. */ 199 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 200 CU_ASSERT(task == NULL); 201 202 _task.cb_fn = dummy_cb_fn; 203 _task.cb_arg = cb_arg; 204 _task.accel_ch = g_accel_ch; 205 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &_task, link); 206 207 /* Get a valid task. */ 208 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 209 CU_ASSERT(task == &_task); 210 CU_ASSERT(_task.cb_fn == dummy_cb_fn); 211 CU_ASSERT(_task.cb_arg == cb_arg); 212 CU_ASSERT(_task.accel_ch == g_accel_ch); 213 } 214 215 #define TEST_SUBMIT_SIZE 64 216 static void 217 test_spdk_accel_submit_copy(void) 218 { 219 const uint64_t nbytes = TEST_SUBMIT_SIZE; 220 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 221 uint8_t src[TEST_SUBMIT_SIZE] = {0}; 222 void *cb_arg = NULL; 223 int rc; 224 struct spdk_accel_task task; 225 struct spdk_accel_task *expected_accel_task = NULL; 226 int flags = 0; 227 228 TAILQ_INIT(&g_accel_ch->task_pool); 229 230 /* Fail with no tasks on _get_task() */ 231 rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, flags, NULL, cb_arg); 232 CU_ASSERT(rc == -ENOMEM); 233 234 task.accel_ch = g_accel_ch; 235 task.flags = 1; 236 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 237 238 /* submission OK. */ 239 rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, flags, NULL, cb_arg); 240 CU_ASSERT(rc == 0); 241 CU_ASSERT(task.op_code == ACCEL_OPC_COPY); 242 CU_ASSERT(task.flags == 0); 243 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 244 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 245 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 246 CU_ASSERT(expected_accel_task == &task); 247 } 248 249 static void 250 test_spdk_accel_submit_dualcast(void) 251 { 252 void *dst1; 253 void *dst2; 254 void *src; 255 uint32_t align = ALIGN_4K; 256 uint64_t nbytes = TEST_SUBMIT_SIZE; 257 void *cb_arg = NULL; 258 int rc; 259 struct spdk_accel_task task; 260 struct spdk_accel_task *expected_accel_task = NULL; 261 int flags = 0; 262 263 TAILQ_INIT(&g_accel_ch->task_pool); 264 265 /* Dualcast requires 4K alignment on dst addresses, 266 * hence using the hard coded address to test the buffer alignment 267 */ 268 dst1 = (void *)0x5000; 269 dst2 = (void *)0x60f0; 270 src = calloc(1, TEST_SUBMIT_SIZE); 271 SPDK_CU_ASSERT_FATAL(src != NULL); 272 memset(src, 0x5A, TEST_SUBMIT_SIZE); 273 274 /* This should fail since dst2 is not 4k aligned */ 275 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 276 CU_ASSERT(rc == -EINVAL); 277 278 dst1 = (void *)0x7010; 279 dst2 = (void *)0x6000; 280 /* This should fail since dst1 is not 4k aligned */ 281 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 282 CU_ASSERT(rc == -EINVAL); 283 284 /* Dualcast requires 4K alignment on dst addresses */ 285 dst1 = (void *)0x7000; 286 dst2 = (void *)0x6000; 287 /* Fail with no tasks on _get_task() */ 288 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 289 CU_ASSERT(rc == -ENOMEM); 290 291 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 292 293 /* accel submission OK., since we test the SW path , need to use valid memory addresses 294 * cannot hardcode them anymore */ 295 dst1 = spdk_dma_zmalloc(nbytes, align, NULL); 296 SPDK_CU_ASSERT_FATAL(dst1 != NULL); 297 dst2 = spdk_dma_zmalloc(nbytes, align, NULL); 298 SPDK_CU_ASSERT_FATAL(dst2 != NULL); 299 /* SW module does the dualcast. */ 300 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 301 CU_ASSERT(rc == 0); 302 CU_ASSERT(task.op_code == ACCEL_OPC_DUALCAST); 303 CU_ASSERT(task.flags == 0); 304 CU_ASSERT(memcmp(dst1, src, TEST_SUBMIT_SIZE) == 0); 305 CU_ASSERT(memcmp(dst2, src, TEST_SUBMIT_SIZE) == 0); 306 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 307 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 308 CU_ASSERT(expected_accel_task == &task); 309 310 free(src); 311 spdk_free(dst1); 312 spdk_free(dst2); 313 } 314 315 static void 316 test_spdk_accel_submit_compare(void) 317 { 318 void *src1; 319 void *src2; 320 uint64_t nbytes = TEST_SUBMIT_SIZE; 321 void *cb_arg = NULL; 322 int rc; 323 struct spdk_accel_task task; 324 struct spdk_accel_task *expected_accel_task = NULL; 325 326 TAILQ_INIT(&g_accel_ch->task_pool); 327 328 src1 = calloc(1, TEST_SUBMIT_SIZE); 329 SPDK_CU_ASSERT_FATAL(src1 != NULL); 330 src2 = calloc(1, TEST_SUBMIT_SIZE); 331 SPDK_CU_ASSERT_FATAL(src2 != NULL); 332 333 /* Fail with no tasks on _get_task() */ 334 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 335 CU_ASSERT(rc == -ENOMEM); 336 337 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 338 339 /* accel submission OK. */ 340 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 341 CU_ASSERT(rc == 0); 342 CU_ASSERT(task.op_code == ACCEL_OPC_COMPARE); 343 CU_ASSERT(memcmp(src1, src2, TEST_SUBMIT_SIZE) == 0); 344 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 345 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 346 CU_ASSERT(expected_accel_task == &task); 347 348 free(src1); 349 free(src2); 350 } 351 352 static void 353 test_spdk_accel_submit_fill(void) 354 { 355 void *dst; 356 void *src; 357 uint8_t fill = 0xf; 358 uint64_t fill64; 359 uint64_t nbytes = TEST_SUBMIT_SIZE; 360 void *cb_arg = NULL; 361 int rc; 362 struct spdk_accel_task task; 363 struct spdk_accel_task *expected_accel_task = NULL; 364 int flags = 0; 365 366 TAILQ_INIT(&g_accel_ch->task_pool); 367 368 dst = calloc(1, TEST_SUBMIT_SIZE); 369 SPDK_CU_ASSERT_FATAL(dst != NULL); 370 src = calloc(1, TEST_SUBMIT_SIZE); 371 SPDK_CU_ASSERT_FATAL(src != NULL); 372 memset(src, fill, TEST_SUBMIT_SIZE); 373 memset(&fill64, fill, sizeof(uint64_t)); 374 375 /* Fail with no tasks on _get_task() */ 376 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 377 CU_ASSERT(rc == -ENOMEM); 378 379 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 380 381 /* accel submission OK. */ 382 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 383 CU_ASSERT(rc == 0); 384 CU_ASSERT(task.fill_pattern == fill64); 385 CU_ASSERT(task.op_code == ACCEL_OPC_FILL); 386 CU_ASSERT(task.flags == 0); 387 388 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 389 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 390 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 391 CU_ASSERT(expected_accel_task == &task); 392 393 free(dst); 394 free(src); 395 } 396 397 static void 398 test_spdk_accel_submit_crc32c(void) 399 { 400 const uint64_t nbytes = TEST_SUBMIT_SIZE; 401 uint32_t crc_dst; 402 uint8_t src[TEST_SUBMIT_SIZE]; 403 uint32_t seed = 1; 404 void *cb_arg = NULL; 405 int rc; 406 struct spdk_accel_task task; 407 struct spdk_accel_task *expected_accel_task = NULL; 408 409 TAILQ_INIT(&g_accel_ch->task_pool); 410 411 /* Fail with no tasks on _get_task() */ 412 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 413 CU_ASSERT(rc == -ENOMEM); 414 415 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 416 417 /* accel submission OK. */ 418 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 419 CU_ASSERT(rc == 0); 420 CU_ASSERT(task.crc_dst == &crc_dst); 421 CU_ASSERT(task.seed == seed); 422 CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); 423 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 424 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 425 CU_ASSERT(expected_accel_task == &task); 426 } 427 428 static void 429 test_spdk_accel_submit_crc32cv(void) 430 { 431 uint32_t crc_dst; 432 uint32_t seed = 0; 433 uint32_t iov_cnt = 32; 434 void *cb_arg = NULL; 435 int rc; 436 uint32_t i = 0; 437 struct spdk_accel_task task; 438 struct iovec iov[32]; 439 struct spdk_accel_task *expected_accel_task = NULL; 440 441 TAILQ_INIT(&g_accel_ch->task_pool); 442 443 for (i = 0; i < iov_cnt; i++) { 444 iov[i].iov_base = calloc(1, TEST_SUBMIT_SIZE); 445 SPDK_CU_ASSERT_FATAL(iov[i].iov_base != NULL); 446 iov[i].iov_len = TEST_SUBMIT_SIZE; 447 } 448 449 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 450 451 /* accel submission OK. */ 452 rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, NULL, cb_arg); 453 CU_ASSERT(rc == 0); 454 CU_ASSERT(task.s.iovs == iov); 455 CU_ASSERT(task.s.iovcnt == iov_cnt); 456 CU_ASSERT(task.crc_dst == &crc_dst); 457 CU_ASSERT(task.seed == seed); 458 CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); 459 CU_ASSERT(task.cb_arg == cb_arg); 460 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 461 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 462 CU_ASSERT(expected_accel_task == &task); 463 464 for (i = 0; i < iov_cnt; i++) { 465 free(iov[i].iov_base); 466 } 467 } 468 469 static void 470 test_spdk_accel_submit_copy_crc32c(void) 471 { 472 const uint64_t nbytes = TEST_SUBMIT_SIZE; 473 uint32_t crc_dst; 474 uint8_t dst[TEST_SUBMIT_SIZE]; 475 uint8_t src[TEST_SUBMIT_SIZE]; 476 uint32_t seed = 0; 477 void *cb_arg = NULL; 478 int rc; 479 struct spdk_accel_task task; 480 struct spdk_accel_task *expected_accel_task = NULL; 481 int flags = 0; 482 483 TAILQ_INIT(&g_accel_ch->task_pool); 484 485 /* Fail with no tasks on _get_task() */ 486 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 487 NULL, cb_arg); 488 CU_ASSERT(rc == -ENOMEM); 489 490 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 491 492 /* accel submission OK. */ 493 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 494 NULL, cb_arg); 495 CU_ASSERT(rc == 0); 496 CU_ASSERT(task.crc_dst == &crc_dst); 497 CU_ASSERT(task.seed == seed); 498 CU_ASSERT(task.flags == 0); 499 CU_ASSERT(task.op_code == ACCEL_OPC_COPY_CRC32C); 500 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 501 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 502 CU_ASSERT(expected_accel_task == &task); 503 } 504 505 static void 506 test_spdk_accel_submit_xor(void) 507 { 508 const uint64_t nbytes = TEST_SUBMIT_SIZE; 509 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 510 uint8_t src1[TEST_SUBMIT_SIZE] = {0}; 511 uint8_t src2[TEST_SUBMIT_SIZE] = {0}; 512 void *sources[] = { src1, src2 }; 513 uint32_t nsrcs = SPDK_COUNTOF(sources); 514 int rc; 515 struct spdk_accel_task task; 516 struct spdk_accel_task *expected_accel_task = NULL; 517 518 TAILQ_INIT(&g_accel_ch->task_pool); 519 520 /* Fail with no tasks on _get_task() */ 521 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 522 CU_ASSERT(rc == -ENOMEM); 523 524 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 525 526 /* submission OK. */ 527 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 528 CU_ASSERT(rc == 0); 529 CU_ASSERT(task.nsrcs.srcs == sources); 530 CU_ASSERT(task.nsrcs.cnt == nsrcs); 531 CU_ASSERT(task.d.iovcnt == 1); 532 CU_ASSERT(task.d.iovs[0].iov_base == dst); 533 CU_ASSERT(task.d.iovs[0].iov_len == nbytes); 534 CU_ASSERT(task.op_code == ACCEL_OPC_XOR); 535 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 536 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 537 CU_ASSERT(expected_accel_task == &task); 538 } 539 540 static void 541 test_spdk_accel_module_find_by_name(void) 542 { 543 struct spdk_accel_module_if mod1 = {}; 544 struct spdk_accel_module_if mod2 = {}; 545 struct spdk_accel_module_if mod3 = {}; 546 struct spdk_accel_module_if *accel_module = NULL; 547 548 mod1.name = "ioat"; 549 mod2.name = "idxd"; 550 mod3.name = "software"; 551 552 TAILQ_INIT(&spdk_accel_module_list); 553 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod1, tailq); 554 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod2, tailq); 555 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod3, tailq); 556 557 /* Now let's find a valid engine */ 558 accel_module = _module_find_by_name("ioat"); 559 CU_ASSERT(accel_module != NULL); 560 561 /* Try to find one that doesn't exist */ 562 accel_module = _module_find_by_name("XXX"); 563 CU_ASSERT(accel_module == NULL); 564 } 565 566 static void 567 test_spdk_accel_module_register(void) 568 { 569 struct spdk_accel_module_if mod1 = {}; 570 struct spdk_accel_module_if mod2 = {}; 571 struct spdk_accel_module_if mod3 = {}; 572 struct spdk_accel_module_if mod4 = {}; 573 struct spdk_accel_module_if *accel_module = NULL; 574 int i = 0; 575 576 mod1.name = "ioat"; 577 mod2.name = "idxd"; 578 mod3.name = "software"; 579 mod4.name = "nothing"; 580 581 TAILQ_INIT(&spdk_accel_module_list); 582 583 spdk_accel_module_list_add(&mod1); 584 spdk_accel_module_list_add(&mod2); 585 spdk_accel_module_list_add(&mod3); 586 spdk_accel_module_list_add(&mod4); 587 588 /* Now confirm they're in the right order. */ 589 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 590 switch (i++) { 591 case 0: 592 CU_ASSERT(strcmp(accel_module->name, "software") == 0); 593 break; 594 case 1: 595 CU_ASSERT(strcmp(accel_module->name, "ioat") == 0); 596 break; 597 case 2: 598 CU_ASSERT(strcmp(accel_module->name, "idxd") == 0); 599 break; 600 case 3: 601 CU_ASSERT(strcmp(accel_module->name, "nothing") == 0); 602 break; 603 default: 604 CU_ASSERT(false); 605 break; 606 } 607 } 608 CU_ASSERT(i == 4); 609 } 610 611 struct ut_sequence { 612 bool complete; 613 int status; 614 }; 615 616 static void 617 ut_sequence_step_cb(void *cb_arg) 618 { 619 int *completed = cb_arg; 620 621 (*completed)++; 622 } 623 624 static void 625 ut_sequence_complete_cb(void *cb_arg, int status) 626 { 627 struct ut_sequence *seq = cb_arg; 628 629 seq->complete = true; 630 seq->status = status; 631 } 632 633 static void 634 test_sequence_fill_copy(void) 635 { 636 struct spdk_accel_sequence *seq = NULL; 637 struct spdk_io_channel *ioch; 638 struct ut_sequence ut_seq; 639 char buf[4096], tmp[2][4096], expected[4096]; 640 struct iovec src_iovs[2], dst_iovs[2]; 641 int rc, completed; 642 643 ioch = spdk_accel_get_io_channel(); 644 SPDK_CU_ASSERT_FATAL(ioch != NULL); 645 646 /* First check the simplest case - single task in a sequence */ 647 memset(buf, 0, sizeof(buf)); 648 memset(expected, 0xa5, sizeof(expected)); 649 completed = 0; 650 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 651 ut_sequence_step_cb, &completed); 652 CU_ASSERT_EQUAL(rc, 0); 653 CU_ASSERT_EQUAL(completed, 0); 654 655 ut_seq.complete = false; 656 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 657 CU_ASSERT_EQUAL(rc, 0); 658 659 poll_threads(); 660 CU_ASSERT_EQUAL(completed, 1); 661 CU_ASSERT(ut_seq.complete); 662 CU_ASSERT_EQUAL(ut_seq.status, 0); 663 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 664 665 /* Check a single copy operation */ 666 memset(buf, 0, sizeof(buf)); 667 memset(tmp[0], 0xa5, sizeof(tmp[0])); 668 memset(expected, 0xa5, sizeof(expected)); 669 completed = 0; 670 seq = NULL; 671 672 dst_iovs[0].iov_base = buf; 673 dst_iovs[0].iov_len = sizeof(buf); 674 src_iovs[0].iov_base = tmp[0]; 675 src_iovs[0].iov_len = sizeof(tmp[0]); 676 677 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 678 &src_iovs[0], 1, NULL, NULL, 0, 679 ut_sequence_step_cb, &completed); 680 CU_ASSERT_EQUAL(rc, 0); 681 682 ut_seq.complete = false; 683 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 684 CU_ASSERT_EQUAL(rc, 0); 685 686 poll_threads(); 687 CU_ASSERT_EQUAL(completed, 1); 688 CU_ASSERT(ut_seq.complete); 689 CU_ASSERT_EQUAL(ut_seq.status, 0); 690 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 691 692 /* Check multiple fill operations */ 693 memset(buf, 0, sizeof(buf)); 694 memset(expected, 0xfe, 4096); 695 memset(expected, 0xde, 2048); 696 memset(expected, 0xa5, 1024); 697 seq = NULL; 698 completed = 0; 699 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, NULL, NULL, 0xfe, 0, 700 ut_sequence_step_cb, &completed); 701 CU_ASSERT_EQUAL(rc, 0); 702 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xde, 0, 703 ut_sequence_step_cb, &completed); 704 CU_ASSERT_EQUAL(rc, 0); 705 rc = spdk_accel_append_fill(&seq, ioch, buf, 1024, NULL, NULL, 0xa5, 0, 706 ut_sequence_step_cb, &completed); 707 CU_ASSERT_EQUAL(rc, 0); 708 709 ut_seq.complete = false; 710 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 711 CU_ASSERT_EQUAL(rc, 0); 712 713 poll_threads(); 714 CU_ASSERT_EQUAL(completed, 3); 715 CU_ASSERT(ut_seq.complete); 716 CU_ASSERT_EQUAL(ut_seq.status, 0); 717 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 718 719 /* Check multiple copy operations */ 720 memset(buf, 0, sizeof(buf)); 721 memset(tmp[0], 0, sizeof(tmp[0])); 722 memset(tmp[1], 0, sizeof(tmp[1])); 723 memset(expected, 0xa5, sizeof(expected)); 724 seq = NULL; 725 completed = 0; 726 727 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 728 ut_sequence_step_cb, &completed); 729 CU_ASSERT_EQUAL(rc, 0); 730 731 dst_iovs[0].iov_base = tmp[1]; 732 dst_iovs[0].iov_len = sizeof(tmp[1]); 733 src_iovs[0].iov_base = tmp[0]; 734 src_iovs[0].iov_len = sizeof(tmp[0]); 735 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 736 &src_iovs[0], 1, NULL, NULL, 0, 737 ut_sequence_step_cb, &completed); 738 CU_ASSERT_EQUAL(rc, 0); 739 740 dst_iovs[1].iov_base = buf; 741 dst_iovs[1].iov_len = sizeof(buf); 742 src_iovs[1].iov_base = tmp[1]; 743 src_iovs[1].iov_len = sizeof(tmp[1]); 744 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 745 &src_iovs[1], 1, NULL, NULL, 0, 746 ut_sequence_step_cb, &completed); 747 CU_ASSERT_EQUAL(rc, 0); 748 749 ut_seq.complete = false; 750 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 751 CU_ASSERT_EQUAL(rc, 0); 752 753 poll_threads(); 754 CU_ASSERT_EQUAL(completed, 3); 755 CU_ASSERT(ut_seq.complete); 756 CU_ASSERT_EQUAL(ut_seq.status, 0); 757 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 758 759 /* Check that adding a copy operation at the end will change destination buffer */ 760 memset(buf, 0, sizeof(buf)); 761 memset(tmp[0], 0, sizeof(tmp[0])); 762 memset(expected, 0xa5, sizeof(buf)); 763 seq = NULL; 764 completed = 0; 765 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 766 ut_sequence_step_cb, &completed); 767 CU_ASSERT_EQUAL(rc, 0); 768 769 dst_iovs[0].iov_base = buf; 770 dst_iovs[0].iov_len = sizeof(buf); 771 src_iovs[0].iov_base = tmp[0]; 772 src_iovs[0].iov_len = sizeof(tmp[0]); 773 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 774 &src_iovs[0], 1, NULL, NULL, 0, 775 ut_sequence_step_cb, &completed); 776 CU_ASSERT_EQUAL(rc, 0); 777 778 ut_seq.complete = false; 779 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 780 CU_ASSERT_EQUAL(rc, 0); 781 782 poll_threads(); 783 CU_ASSERT_EQUAL(completed, 2); 784 CU_ASSERT(ut_seq.complete); 785 CU_ASSERT_EQUAL(ut_seq.status, 0); 786 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 787 788 /* Check that it's also possible to add copy operation at the beginning */ 789 memset(buf, 0, sizeof(buf)); 790 memset(tmp[0], 0xde, sizeof(tmp[0])); 791 memset(tmp[1], 0, sizeof(tmp[1])); 792 memset(expected, 0xa5, sizeof(expected)); 793 seq = NULL; 794 completed = 0; 795 796 dst_iovs[0].iov_base = tmp[1]; 797 dst_iovs[0].iov_len = sizeof(tmp[1]); 798 src_iovs[0].iov_base = tmp[0]; 799 src_iovs[0].iov_len = sizeof(tmp[0]); 800 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 801 &src_iovs[0], 1, NULL, NULL, 0, 802 ut_sequence_step_cb, &completed); 803 CU_ASSERT_EQUAL(rc, 0); 804 805 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], sizeof(tmp[1]), NULL, NULL, 0xa5, 0, 806 ut_sequence_step_cb, &completed); 807 CU_ASSERT_EQUAL(rc, 0); 808 809 dst_iovs[1].iov_base = buf; 810 dst_iovs[1].iov_len = sizeof(buf); 811 src_iovs[1].iov_base = tmp[1]; 812 src_iovs[1].iov_len = sizeof(tmp[1]); 813 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 814 &src_iovs[1], 1, NULL, NULL, 0, 815 ut_sequence_step_cb, &completed); 816 CU_ASSERT_EQUAL(rc, 0); 817 818 ut_seq.complete = false; 819 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 820 CU_ASSERT_EQUAL(rc, 0); 821 822 poll_threads(); 823 CU_ASSERT_EQUAL(completed, 3); 824 CU_ASSERT(ut_seq.complete); 825 CU_ASSERT_EQUAL(ut_seq.status, 0); 826 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 827 828 spdk_put_io_channel(ioch); 829 poll_threads(); 830 } 831 832 static void 833 test_sequence_abort(void) 834 { 835 struct spdk_accel_sequence *seq = NULL; 836 struct spdk_io_channel *ioch; 837 char buf[4096], tmp[2][4096], expected[4096]; 838 struct iovec src_iovs[2], dst_iovs[2]; 839 int rc, completed; 840 841 ioch = spdk_accel_get_io_channel(); 842 SPDK_CU_ASSERT_FATAL(ioch != NULL); 843 844 /* Check that aborting a sequence calls operation's callback, the operation is not executed 845 * and the sequence is freed 846 */ 847 memset(buf, 0, sizeof(buf)); 848 memset(expected, 0, sizeof(buf)); 849 completed = 0; 850 seq = NULL; 851 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 852 ut_sequence_step_cb, &completed); 853 CU_ASSERT_EQUAL(rc, 0); 854 855 spdk_accel_sequence_abort(seq); 856 CU_ASSERT_EQUAL(completed, 1); 857 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 858 859 /* Check sequence with multiple operations */ 860 memset(buf, 0, sizeof(buf)); 861 memset(expected, 0, sizeof(buf)); 862 completed = 0; 863 seq = NULL; 864 865 dst_iovs[0].iov_base = tmp[1]; 866 dst_iovs[0].iov_len = sizeof(tmp[1]); 867 src_iovs[0].iov_base = tmp[0]; 868 src_iovs[0].iov_len = sizeof(tmp[0]); 869 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 870 &src_iovs[0], 1, NULL, NULL, 0, 871 ut_sequence_step_cb, &completed); 872 CU_ASSERT_EQUAL(rc, 0); 873 874 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 4096, NULL, NULL, 0xa5, 0, 875 ut_sequence_step_cb, &completed); 876 CU_ASSERT_EQUAL(rc, 0); 877 878 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xde, 0, 879 ut_sequence_step_cb, &completed); 880 CU_ASSERT_EQUAL(rc, 0); 881 882 dst_iovs[1].iov_base = buf; 883 dst_iovs[1].iov_len = sizeof(buf); 884 src_iovs[1].iov_base = tmp[1]; 885 src_iovs[1].iov_len = sizeof(tmp[1]); 886 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 887 &src_iovs[1], 1, NULL, NULL, 0, 888 ut_sequence_step_cb, &completed); 889 CU_ASSERT_EQUAL(rc, 0); 890 891 spdk_accel_sequence_abort(seq); 892 CU_ASSERT_EQUAL(completed, 4); 893 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 894 895 /* This should be a no-op */ 896 spdk_accel_sequence_abort(NULL); 897 898 spdk_put_io_channel(ioch); 899 poll_threads(); 900 } 901 902 static void 903 test_sequence_append_error(void) 904 { 905 struct spdk_accel_sequence *seq = NULL; 906 struct spdk_io_channel *ioch; 907 struct accel_io_channel *accel_ch; 908 struct iovec src_iovs, dst_iovs; 909 char buf[4096]; 910 TAILQ_HEAD(, spdk_accel_task) tasks = TAILQ_HEAD_INITIALIZER(tasks); 911 TAILQ_HEAD(, spdk_accel_sequence) seqs = TAILQ_HEAD_INITIALIZER(seqs); 912 int rc; 913 914 ioch = spdk_accel_get_io_channel(); 915 SPDK_CU_ASSERT_FATAL(ioch != NULL); 916 accel_ch = spdk_io_channel_get_ctx(ioch); 917 918 /* Check that append fails and no sequence object is allocated when there are no more free 919 * tasks */ 920 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 921 922 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 923 ut_sequence_step_cb, NULL); 924 CU_ASSERT_EQUAL(rc, -ENOMEM); 925 CU_ASSERT_PTR_NULL(seq); 926 927 dst_iovs.iov_base = buf; 928 dst_iovs.iov_len = 2048; 929 src_iovs.iov_base = &buf[2048]; 930 src_iovs.iov_len = 2048; 931 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 932 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 933 CU_ASSERT_EQUAL(rc, -ENOMEM); 934 CU_ASSERT_PTR_NULL(seq); 935 936 dst_iovs.iov_base = buf; 937 dst_iovs.iov_len = 2048; 938 src_iovs.iov_base = &buf[2048]; 939 src_iovs.iov_len = 2048; 940 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 941 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 942 CU_ASSERT_EQUAL(rc, -ENOMEM); 943 CU_ASSERT_PTR_NULL(seq); 944 945 /* Check that the same happens when the sequence queue is empty */ 946 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 947 TAILQ_SWAP(&seqs, &accel_ch->seq_pool, spdk_accel_sequence, link); 948 949 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 950 ut_sequence_step_cb, NULL); 951 CU_ASSERT_EQUAL(rc, -ENOMEM); 952 CU_ASSERT_PTR_NULL(seq); 953 954 dst_iovs.iov_base = buf; 955 dst_iovs.iov_len = 2048; 956 src_iovs.iov_base = &buf[2048]; 957 src_iovs.iov_len = 2048; 958 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 959 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 960 CU_ASSERT_EQUAL(rc, -ENOMEM); 961 CU_ASSERT_PTR_NULL(seq); 962 963 dst_iovs.iov_base = buf; 964 dst_iovs.iov_len = 2048; 965 src_iovs.iov_base = &buf[2048]; 966 src_iovs.iov_len = 2048; 967 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 968 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 969 CU_ASSERT_EQUAL(rc, -ENOMEM); 970 CU_ASSERT_PTR_NULL(seq); 971 972 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 973 974 spdk_put_io_channel(ioch); 975 poll_threads(); 976 } 977 978 struct ut_sequence_operation { 979 int complete_status; 980 int submit_status; 981 int count; 982 struct iovec *src_iovs; 983 uint32_t src_iovcnt; 984 struct iovec *dst_iovs; 985 uint32_t dst_iovcnt; 986 int (*submit)(struct spdk_io_channel *ch, struct spdk_accel_task *t); 987 }; 988 989 static struct ut_sequence_operation g_seq_operations[ACCEL_OPC_LAST]; 990 991 static int 992 ut_sequnce_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *task) 993 { 994 struct ut_sequence_operation *op = &g_seq_operations[task->op_code]; 995 996 op->count++; 997 if (op->submit != NULL) { 998 return op->submit(ch, task); 999 } 1000 if (op->src_iovs != NULL) { 1001 CU_ASSERT_EQUAL(task->s.iovcnt, op->src_iovcnt); 1002 CU_ASSERT_EQUAL(memcmp(task->s.iovs, op->src_iovs, 1003 sizeof(struct iovec) * op->src_iovcnt), 0); 1004 } 1005 if (op->dst_iovs != NULL) { 1006 CU_ASSERT_EQUAL(task->d.iovcnt, op->dst_iovcnt); 1007 CU_ASSERT_EQUAL(memcmp(task->d.iovs, op->dst_iovs, 1008 sizeof(struct iovec) * op->dst_iovcnt), 0); 1009 } 1010 1011 if (op->submit_status != 0) { 1012 return op->submit_status; 1013 } 1014 1015 spdk_accel_task_complete(task, op->complete_status); 1016 1017 return 0; 1018 } 1019 1020 static void 1021 ut_clear_operations(void) 1022 { 1023 memset(&g_seq_operations, 0, sizeof(g_seq_operations)); 1024 } 1025 1026 static void 1027 test_sequence_completion_error(void) 1028 { 1029 struct spdk_accel_sequence *seq = NULL; 1030 struct spdk_io_channel *ioch; 1031 struct ut_sequence ut_seq; 1032 struct iovec src_iovs, dst_iovs; 1033 char buf[4096], tmp[4096]; 1034 struct accel_module modules[ACCEL_OPC_LAST]; 1035 int i, rc, completed; 1036 1037 ioch = spdk_accel_get_io_channel(); 1038 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1039 1040 /* Override the submit_tasks function */ 1041 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1042 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1043 modules[i] = g_modules_opc[i]; 1044 g_modules_opc[i] = g_module; 1045 } 1046 1047 memset(buf, 0, sizeof(buf)); 1048 memset(tmp, 0, sizeof(tmp)); 1049 1050 /* Check that if the first operation completes with an error, the whole sequence is 1051 * completed with that error and that all operations' completion callbacks are executed 1052 */ 1053 g_seq_operations[ACCEL_OPC_FILL].complete_status = -E2BIG; 1054 completed = 0; 1055 seq = NULL; 1056 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1057 ut_sequence_step_cb, &completed); 1058 CU_ASSERT_EQUAL(rc, 0); 1059 1060 dst_iovs.iov_base = buf; 1061 dst_iovs.iov_len = sizeof(buf); 1062 src_iovs.iov_base = tmp; 1063 src_iovs.iov_len = sizeof(tmp); 1064 1065 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1066 &src_iovs, 1, NULL, NULL, 0, 1067 ut_sequence_step_cb, &completed); 1068 CU_ASSERT_EQUAL(rc, 0); 1069 1070 ut_seq.complete = false; 1071 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1072 CU_ASSERT_EQUAL(rc, 0); 1073 1074 poll_threads(); 1075 CU_ASSERT_EQUAL(completed, 2); 1076 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 1077 1078 /* Check the same with a second operation in the sequence */ 1079 g_seq_operations[ACCEL_OPC_DECOMPRESS].complete_status = -EACCES; 1080 g_seq_operations[ACCEL_OPC_FILL].complete_status = 0; 1081 completed = 0; 1082 seq = NULL; 1083 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1084 ut_sequence_step_cb, &completed); 1085 CU_ASSERT_EQUAL(rc, 0); 1086 1087 dst_iovs.iov_base = buf; 1088 dst_iovs.iov_len = sizeof(buf); 1089 src_iovs.iov_base = tmp; 1090 src_iovs.iov_len = sizeof(tmp); 1091 1092 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1093 &src_iovs, 1, NULL, NULL, 0, 1094 ut_sequence_step_cb, &completed); 1095 CU_ASSERT_EQUAL(rc, 0); 1096 1097 ut_seq.complete = false; 1098 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1099 CU_ASSERT_EQUAL(rc, 0); 1100 1101 poll_threads(); 1102 CU_ASSERT_EQUAL(completed, 2); 1103 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 1104 1105 g_seq_operations[ACCEL_OPC_DECOMPRESS].complete_status = 0; 1106 g_seq_operations[ACCEL_OPC_FILL].complete_status = 0; 1107 1108 /* Check submission failure of the first operation */ 1109 g_seq_operations[ACCEL_OPC_FILL].submit_status = -EADDRINUSE; 1110 completed = 0; 1111 seq = NULL; 1112 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1113 ut_sequence_step_cb, &completed); 1114 CU_ASSERT_EQUAL(rc, 0); 1115 1116 dst_iovs.iov_base = buf; 1117 dst_iovs.iov_len = sizeof(buf); 1118 src_iovs.iov_base = tmp; 1119 src_iovs.iov_len = sizeof(tmp); 1120 1121 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1122 &src_iovs, 1, NULL, NULL, 0, 1123 ut_sequence_step_cb, &completed); 1124 CU_ASSERT_EQUAL(rc, 0); 1125 1126 ut_seq.complete = false; 1127 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1128 CU_ASSERT_EQUAL(rc, 0); 1129 1130 poll_threads(); 1131 CU_ASSERT_EQUAL(completed, 2); 1132 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 1133 1134 /* Check the same with a second operation */ 1135 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit_status = -EADDRNOTAVAIL; 1136 g_seq_operations[ACCEL_OPC_FILL].submit_status = 0; 1137 completed = 0; 1138 seq = NULL; 1139 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1140 ut_sequence_step_cb, &completed); 1141 CU_ASSERT_EQUAL(rc, 0); 1142 1143 dst_iovs.iov_base = buf; 1144 dst_iovs.iov_len = sizeof(buf); 1145 src_iovs.iov_base = tmp; 1146 src_iovs.iov_len = sizeof(tmp); 1147 1148 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1149 &src_iovs, 1, NULL, NULL, 0, 1150 ut_sequence_step_cb, &completed); 1151 CU_ASSERT_EQUAL(rc, 0); 1152 1153 ut_seq.complete = false; 1154 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1155 CU_ASSERT_EQUAL(rc, 0); 1156 1157 poll_threads(); 1158 CU_ASSERT_EQUAL(completed, 2); 1159 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 1160 1161 /* Cleanup module pointers to make subsequent tests work correctly */ 1162 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1163 g_modules_opc[i] = modules[i]; 1164 } 1165 1166 ut_clear_operations(); 1167 spdk_put_io_channel(ioch); 1168 poll_threads(); 1169 } 1170 1171 #ifdef SPDK_CONFIG_ISAL 1172 static void 1173 ut_compress_cb(void *cb_arg, int status) 1174 { 1175 int *completed = cb_arg; 1176 1177 CU_ASSERT_EQUAL(status, 0); 1178 1179 *completed = 1; 1180 } 1181 1182 static void 1183 test_sequence_decompress(void) 1184 { 1185 struct spdk_accel_sequence *seq = NULL; 1186 struct spdk_io_channel *ioch; 1187 struct ut_sequence ut_seq; 1188 char buf[4096], tmp[2][4096], expected[4096]; 1189 struct iovec src_iovs[2], dst_iovs[2]; 1190 uint32_t compressed_size; 1191 int rc, completed = 0; 1192 1193 ioch = spdk_accel_get_io_channel(); 1194 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1195 1196 memset(expected, 0xa5, sizeof(expected)); 1197 src_iovs[0].iov_base = expected; 1198 src_iovs[0].iov_len = sizeof(expected); 1199 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1200 &compressed_size, 0, ut_compress_cb, &completed); 1201 CU_ASSERT_EQUAL(rc, 0); 1202 1203 while (!completed) { 1204 poll_threads(); 1205 } 1206 1207 /* Check a single decompress operation in a sequence */ 1208 seq = NULL; 1209 completed = 0; 1210 1211 dst_iovs[0].iov_base = buf; 1212 dst_iovs[0].iov_len = sizeof(buf); 1213 src_iovs[0].iov_base = tmp[0]; 1214 src_iovs[0].iov_len = compressed_size; 1215 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1216 &src_iovs[0], 1, NULL, NULL, 0, 1217 ut_sequence_step_cb, &completed); 1218 CU_ASSERT_EQUAL(rc, 0); 1219 1220 ut_seq.complete = false; 1221 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1222 CU_ASSERT_EQUAL(rc, 0); 1223 1224 poll_threads(); 1225 1226 CU_ASSERT_EQUAL(completed, 1); 1227 CU_ASSERT(ut_seq.complete); 1228 CU_ASSERT_EQUAL(ut_seq.status, 0); 1229 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1230 1231 /* Put the decompress operation in the middle of a sequence with a copy operation at the 1232 * beginning and a fill at the end modifying the first 2048B of the buffer. 1233 */ 1234 memset(expected, 0xfe, 2048); 1235 memset(buf, 0, sizeof(buf)); 1236 seq = NULL; 1237 completed = 0; 1238 1239 dst_iovs[0].iov_base = tmp[1]; 1240 dst_iovs[0].iov_len = compressed_size; 1241 src_iovs[0].iov_base = tmp[0]; 1242 src_iovs[0].iov_len = compressed_size; 1243 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1244 &src_iovs[0], 1, NULL, NULL, 0, 1245 ut_sequence_step_cb, &completed); 1246 CU_ASSERT_EQUAL(rc, 0); 1247 1248 dst_iovs[1].iov_base = buf; 1249 dst_iovs[1].iov_len = sizeof(buf); 1250 src_iovs[1].iov_base = tmp[1]; 1251 src_iovs[1].iov_len = compressed_size; 1252 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1253 &src_iovs[1], 1, NULL, NULL, 0, 1254 ut_sequence_step_cb, &completed); 1255 CU_ASSERT_EQUAL(rc, 0); 1256 1257 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1258 ut_sequence_step_cb, &completed); 1259 CU_ASSERT_EQUAL(rc, 0); 1260 1261 ut_seq.complete = false; 1262 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1263 CU_ASSERT_EQUAL(rc, 0); 1264 1265 poll_threads(); 1266 1267 CU_ASSERT_EQUAL(completed, 3); 1268 CU_ASSERT(ut_seq.complete); 1269 CU_ASSERT_EQUAL(ut_seq.status, 0); 1270 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1271 1272 /* Check sequence with decompress at the beginning: decompress -> copy */ 1273 memset(expected, 0xa5, sizeof(expected)); 1274 memset(buf, 0, sizeof(buf)); 1275 seq = NULL; 1276 completed = 0; 1277 1278 dst_iovs[0].iov_base = tmp[1]; 1279 dst_iovs[0].iov_len = sizeof(tmp[1]); 1280 src_iovs[0].iov_base = tmp[0]; 1281 src_iovs[0].iov_len = compressed_size; 1282 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1283 &src_iovs[0], 1, NULL, NULL, 0, 1284 ut_sequence_step_cb, &completed); 1285 CU_ASSERT_EQUAL(rc, 0); 1286 1287 dst_iovs[1].iov_base = buf; 1288 dst_iovs[1].iov_len = sizeof(buf); 1289 src_iovs[1].iov_base = tmp[1]; 1290 src_iovs[1].iov_len = sizeof(tmp[1]); 1291 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1292 &src_iovs[1], 1, NULL, NULL, 0, 1293 ut_sequence_step_cb, &completed); 1294 CU_ASSERT_EQUAL(rc, 0); 1295 1296 ut_seq.complete = false; 1297 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1298 CU_ASSERT_EQUAL(rc, 0); 1299 1300 poll_threads(); 1301 1302 CU_ASSERT_EQUAL(completed, 2); 1303 CU_ASSERT(ut_seq.complete); 1304 CU_ASSERT_EQUAL(ut_seq.status, 0); 1305 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1306 1307 spdk_put_io_channel(ioch); 1308 poll_threads(); 1309 } 1310 1311 static void 1312 test_sequence_reverse(void) 1313 { 1314 struct spdk_accel_sequence *seq = NULL; 1315 struct spdk_io_channel *ioch; 1316 struct ut_sequence ut_seq; 1317 char buf[4096], tmp[2][4096], expected[4096]; 1318 struct iovec src_iovs[2], dst_iovs[2]; 1319 uint32_t compressed_size; 1320 int rc, completed = 0; 1321 1322 ioch = spdk_accel_get_io_channel(); 1323 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1324 1325 memset(expected, 0xa5, sizeof(expected)); 1326 src_iovs[0].iov_base = expected; 1327 src_iovs[0].iov_len = sizeof(expected); 1328 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1329 &compressed_size, 0, ut_compress_cb, &completed); 1330 CU_ASSERT_EQUAL(rc, 0); 1331 1332 while (!completed) { 1333 poll_threads(); 1334 } 1335 1336 /* First check that reversing a sequnce with a single operation is a no-op */ 1337 memset(buf, 0, sizeof(buf)); 1338 seq = NULL; 1339 completed = 0; 1340 1341 dst_iovs[0].iov_base = buf; 1342 dst_iovs[0].iov_len = sizeof(buf); 1343 src_iovs[0].iov_base = tmp[0]; 1344 src_iovs[0].iov_len = compressed_size; 1345 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1346 &src_iovs[0], 1, NULL, NULL, 0, 1347 ut_sequence_step_cb, &completed); 1348 CU_ASSERT_EQUAL(rc, 0); 1349 1350 spdk_accel_sequence_reverse(seq); 1351 1352 ut_seq.complete = false; 1353 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1354 CU_ASSERT_EQUAL(rc, 0); 1355 1356 poll_threads(); 1357 1358 CU_ASSERT_EQUAL(completed, 1); 1359 CU_ASSERT(ut_seq.complete); 1360 CU_ASSERT_EQUAL(ut_seq.status, 0); 1361 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1362 1363 /* Add a copy operation at the end with src set to the compressed data. After reverse(), 1364 * that copy operation should be first, so decompress() should receive compressed data in 1365 * its src buffer. 1366 */ 1367 memset(buf, 0, sizeof(buf)); 1368 memset(tmp[1], 0, sizeof(tmp[1])); 1369 seq = NULL; 1370 completed = 0; 1371 1372 dst_iovs[0].iov_base = buf; 1373 dst_iovs[0].iov_len = sizeof(buf); 1374 src_iovs[0].iov_base = tmp[1]; 1375 src_iovs[0].iov_len = compressed_size; 1376 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1377 &src_iovs[0], 1, NULL, NULL, 0, 1378 ut_sequence_step_cb, &completed); 1379 CU_ASSERT_EQUAL(rc, 0); 1380 1381 dst_iovs[1].iov_base = tmp[1]; 1382 dst_iovs[1].iov_len = compressed_size; 1383 src_iovs[1].iov_base = tmp[0]; 1384 src_iovs[1].iov_len = compressed_size; 1385 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1386 &src_iovs[1], 1, NULL, NULL, 0, 1387 ut_sequence_step_cb, &completed); 1388 CU_ASSERT_EQUAL(rc, 0); 1389 1390 spdk_accel_sequence_reverse(seq); 1391 1392 ut_seq.complete = false; 1393 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1394 CU_ASSERT_EQUAL(rc, 0); 1395 1396 poll_threads(); 1397 1398 CU_ASSERT_EQUAL(completed, 2); 1399 CU_ASSERT(ut_seq.complete); 1400 CU_ASSERT_EQUAL(ut_seq.status, 0); 1401 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1402 1403 /* Check the same, but add an extra fill operation at the beginning that should execute last 1404 * after reverse(). 1405 */ 1406 memset(buf, 0, sizeof(buf)); 1407 memset(tmp[1], 0, sizeof(tmp[1])); 1408 memset(expected, 0xfe, 2048); 1409 seq = NULL; 1410 completed = 0; 1411 1412 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1413 ut_sequence_step_cb, &completed); 1414 CU_ASSERT_EQUAL(rc, 0); 1415 1416 dst_iovs[0].iov_base = buf; 1417 dst_iovs[0].iov_len = sizeof(buf); 1418 src_iovs[0].iov_base = tmp[1]; 1419 src_iovs[0].iov_len = compressed_size; 1420 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1421 &src_iovs[0], 1, NULL, NULL, 0, 1422 ut_sequence_step_cb, &completed); 1423 CU_ASSERT_EQUAL(rc, 0); 1424 1425 dst_iovs[1].iov_base = tmp[1]; 1426 dst_iovs[1].iov_len = compressed_size; 1427 src_iovs[1].iov_base = tmp[0]; 1428 src_iovs[1].iov_len = compressed_size; 1429 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1430 &src_iovs[1], 1, NULL, NULL, 0, 1431 ut_sequence_step_cb, &completed); 1432 CU_ASSERT_EQUAL(rc, 0); 1433 1434 spdk_accel_sequence_reverse(seq); 1435 1436 ut_seq.complete = false; 1437 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1438 CU_ASSERT_EQUAL(rc, 0); 1439 1440 poll_threads(); 1441 1442 CU_ASSERT_EQUAL(completed, 3); 1443 CU_ASSERT(ut_seq.complete); 1444 CU_ASSERT_EQUAL(ut_seq.status, 0); 1445 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1446 1447 /* Build the sequence in order and then reverse it twice */ 1448 memset(buf, 0, sizeof(buf)); 1449 memset(tmp[1], 0, sizeof(tmp[1])); 1450 seq = NULL; 1451 completed = 0; 1452 1453 dst_iovs[0].iov_base = tmp[1]; 1454 dst_iovs[0].iov_len = compressed_size; 1455 src_iovs[0].iov_base = tmp[0]; 1456 src_iovs[0].iov_len = compressed_size; 1457 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1458 &src_iovs[0], 1, NULL, NULL, 0, 1459 ut_sequence_step_cb, &completed); 1460 CU_ASSERT_EQUAL(rc, 0); 1461 1462 dst_iovs[1].iov_base = buf; 1463 dst_iovs[1].iov_len = sizeof(buf); 1464 src_iovs[1].iov_base = tmp[1]; 1465 src_iovs[1].iov_len = compressed_size; 1466 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1467 &src_iovs[1], 1, NULL, NULL, 0, 1468 ut_sequence_step_cb, &completed); 1469 CU_ASSERT_EQUAL(rc, 0); 1470 1471 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1472 ut_sequence_step_cb, &completed); 1473 CU_ASSERT_EQUAL(rc, 0); 1474 1475 spdk_accel_sequence_reverse(seq); 1476 spdk_accel_sequence_reverse(seq); 1477 1478 ut_seq.complete = false; 1479 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1480 CU_ASSERT_EQUAL(rc, 0); 1481 1482 poll_threads(); 1483 1484 CU_ASSERT_EQUAL(completed, 3); 1485 CU_ASSERT(ut_seq.complete); 1486 CU_ASSERT_EQUAL(ut_seq.status, 0); 1487 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1488 1489 spdk_put_io_channel(ioch); 1490 poll_threads(); 1491 } 1492 #endif 1493 1494 static void 1495 test_sequence_copy_elision(void) 1496 { 1497 struct spdk_accel_sequence *seq = NULL; 1498 struct spdk_io_channel *ioch; 1499 struct ut_sequence ut_seq; 1500 struct iovec src_iovs[4], dst_iovs[4], exp_iovs[2]; 1501 char buf[4096], tmp[4][4096]; 1502 struct accel_module modules[ACCEL_OPC_LAST]; 1503 struct spdk_accel_crypto_key key = {}; 1504 int i, rc, completed; 1505 1506 ioch = spdk_accel_get_io_channel(); 1507 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1508 1509 /* Override the submit_tasks function */ 1510 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1511 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1512 g_seq_operations[i].complete_status = 0; 1513 g_seq_operations[i].submit_status = 0; 1514 g_seq_operations[i].count = 0; 1515 1516 modules[i] = g_modules_opc[i]; 1517 g_modules_opc[i] = g_module; 1518 } 1519 1520 /* Check that a copy operation at the beginning is removed */ 1521 seq = NULL; 1522 completed = 0; 1523 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovcnt = 1; 1524 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovcnt = 1; 1525 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1526 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1527 exp_iovs[0].iov_base = tmp[0]; 1528 exp_iovs[0].iov_len = sizeof(tmp[0]); 1529 exp_iovs[1].iov_base = buf; 1530 exp_iovs[1].iov_len = 2048; 1531 1532 dst_iovs[0].iov_base = tmp[1]; 1533 dst_iovs[0].iov_len = sizeof(tmp[1]); 1534 src_iovs[0].iov_base = tmp[0]; 1535 src_iovs[0].iov_len = sizeof(tmp[0]); 1536 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1537 &src_iovs[0], 1, NULL, NULL, 0, 1538 ut_sequence_step_cb, &completed); 1539 CU_ASSERT_EQUAL(rc, 0); 1540 1541 dst_iovs[1].iov_base = buf; 1542 dst_iovs[1].iov_len = 2048; 1543 src_iovs[1].iov_base = tmp[1]; 1544 src_iovs[1].iov_len = sizeof(tmp[1]); 1545 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1546 &src_iovs[1], 1, NULL, NULL, 0, 1547 ut_sequence_step_cb, &completed); 1548 CU_ASSERT_EQUAL(rc, 0); 1549 1550 ut_seq.complete = false; 1551 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1552 CU_ASSERT_EQUAL(rc, 0); 1553 1554 poll_threads(); 1555 1556 CU_ASSERT_EQUAL(completed, 2); 1557 CU_ASSERT(ut_seq.complete); 1558 CU_ASSERT_EQUAL(ut_seq.status, 0); 1559 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1560 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 1561 1562 /* Check that a copy operation at the end is removed too */ 1563 seq = NULL; 1564 completed = 0; 1565 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1566 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 1567 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1568 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1569 exp_iovs[0].iov_base = tmp[0]; 1570 exp_iovs[0].iov_len = sizeof(tmp[0]); 1571 exp_iovs[1].iov_base = buf; 1572 exp_iovs[1].iov_len = 2048; 1573 1574 dst_iovs[0].iov_base = tmp[1]; 1575 dst_iovs[0].iov_len = 2048; 1576 src_iovs[0].iov_base = tmp[0]; 1577 src_iovs[0].iov_len = sizeof(tmp[0]); 1578 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1579 &src_iovs[0], 1, NULL, NULL, 0, 1580 ut_sequence_step_cb, &completed); 1581 CU_ASSERT_EQUAL(rc, 0); 1582 1583 dst_iovs[1].iov_base = buf; 1584 dst_iovs[1].iov_len = 2048; 1585 src_iovs[1].iov_base = tmp[1]; 1586 src_iovs[1].iov_len = 2048; 1587 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1588 &src_iovs[1], 1, NULL, NULL, 0, 1589 ut_sequence_step_cb, &completed); 1590 CU_ASSERT_EQUAL(rc, 0); 1591 1592 ut_seq.complete = false; 1593 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1594 CU_ASSERT_EQUAL(rc, 0); 1595 1596 poll_threads(); 1597 1598 CU_ASSERT_EQUAL(completed, 2); 1599 CU_ASSERT(ut_seq.complete); 1600 CU_ASSERT_EQUAL(ut_seq.status, 0); 1601 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1602 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 1603 1604 /* Check a copy operation both at the beginning and the end */ 1605 seq = NULL; 1606 completed = 0; 1607 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1608 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 1609 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1610 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1611 exp_iovs[0].iov_base = tmp[0]; 1612 exp_iovs[0].iov_len = sizeof(tmp[0]); 1613 exp_iovs[1].iov_base = buf; 1614 exp_iovs[1].iov_len = 2048; 1615 1616 dst_iovs[0].iov_base = tmp[1]; 1617 dst_iovs[0].iov_len = sizeof(tmp[1]); 1618 src_iovs[0].iov_base = tmp[0]; 1619 src_iovs[0].iov_len = sizeof(tmp[0]); 1620 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1621 &src_iovs[0], 1, NULL, NULL, 0, 1622 ut_sequence_step_cb, &completed); 1623 CU_ASSERT_EQUAL(rc, 0); 1624 1625 dst_iovs[1].iov_base = tmp[2]; 1626 dst_iovs[1].iov_len = 2048; 1627 src_iovs[1].iov_base = tmp[1]; 1628 src_iovs[1].iov_len = sizeof(tmp[1]); 1629 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1630 &src_iovs[1], 1, NULL, NULL, 0, 1631 ut_sequence_step_cb, &completed); 1632 CU_ASSERT_EQUAL(rc, 0); 1633 1634 dst_iovs[2].iov_base = buf; 1635 dst_iovs[2].iov_len = 2048; 1636 src_iovs[2].iov_base = tmp[2]; 1637 src_iovs[2].iov_len = 2048; 1638 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1639 &src_iovs[2], 1, NULL, NULL, 0, 1640 ut_sequence_step_cb, &completed); 1641 CU_ASSERT_EQUAL(rc, 0); 1642 1643 ut_seq.complete = false; 1644 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1645 CU_ASSERT_EQUAL(rc, 0); 1646 1647 poll_threads(); 1648 1649 CU_ASSERT_EQUAL(completed, 3); 1650 CU_ASSERT(ut_seq.complete); 1651 CU_ASSERT_EQUAL(ut_seq.status, 0); 1652 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1653 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 1654 1655 /* Check decompress + copy + decompress + copy */ 1656 seq = NULL; 1657 completed = 0; 1658 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1659 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 1660 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = NULL; 1661 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = NULL; 1662 1663 dst_iovs[0].iov_base = tmp[1]; 1664 dst_iovs[0].iov_len = sizeof(tmp[1]); 1665 src_iovs[0].iov_base = tmp[0]; 1666 src_iovs[0].iov_len = sizeof(tmp[0]); 1667 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1668 &src_iovs[0], 1, NULL, NULL, 0, 1669 ut_sequence_step_cb, &completed); 1670 CU_ASSERT_EQUAL(rc, 0); 1671 1672 dst_iovs[1].iov_base = tmp[2]; 1673 dst_iovs[1].iov_len = 2048; 1674 src_iovs[1].iov_base = tmp[1]; 1675 src_iovs[1].iov_len = sizeof(tmp[1]); 1676 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1677 &src_iovs[1], 1, NULL, NULL, 0, 1678 ut_sequence_step_cb, &completed); 1679 CU_ASSERT_EQUAL(rc, 0); 1680 1681 dst_iovs[2].iov_base = tmp[3]; 1682 dst_iovs[2].iov_len = 1024; 1683 src_iovs[2].iov_base = tmp[2]; 1684 src_iovs[2].iov_len = 2048; 1685 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1686 &src_iovs[2], 1, NULL, NULL, 0, 1687 ut_sequence_step_cb, &completed); 1688 CU_ASSERT_EQUAL(rc, 0); 1689 1690 dst_iovs[3].iov_base = buf; 1691 dst_iovs[3].iov_len = 1024; 1692 src_iovs[3].iov_base = tmp[3]; 1693 src_iovs[3].iov_len = 1024; 1694 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 1695 &src_iovs[3], 1, NULL, NULL, 0, 1696 ut_sequence_step_cb, &completed); 1697 CU_ASSERT_EQUAL(rc, 0); 1698 1699 ut_seq.complete = false; 1700 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1701 CU_ASSERT_EQUAL(rc, 0); 1702 1703 poll_threads(); 1704 1705 CU_ASSERT_EQUAL(completed, 4); 1706 CU_ASSERT(ut_seq.complete); 1707 CU_ASSERT_EQUAL(ut_seq.status, 0); 1708 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1709 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 2); 1710 1711 /* Check two copy operations - one of them should be removed, while the other should be 1712 * executed normally */ 1713 seq = NULL; 1714 completed = 0; 1715 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1716 g_seq_operations[ACCEL_OPC_COPY].dst_iovcnt = 1; 1717 g_seq_operations[ACCEL_OPC_COPY].src_iovcnt = 1; 1718 g_seq_operations[ACCEL_OPC_COPY].src_iovs = &exp_iovs[0]; 1719 g_seq_operations[ACCEL_OPC_COPY].dst_iovs = &exp_iovs[1]; 1720 exp_iovs[0].iov_base = tmp[0]; 1721 exp_iovs[0].iov_len = sizeof(tmp[0]); 1722 exp_iovs[1].iov_base = buf; 1723 exp_iovs[1].iov_len = sizeof(buf); 1724 1725 dst_iovs[0].iov_base = tmp[1]; 1726 dst_iovs[0].iov_len = sizeof(tmp[1]); 1727 src_iovs[0].iov_base = tmp[0]; 1728 src_iovs[0].iov_len = sizeof(tmp[0]); 1729 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1730 &src_iovs[0], 1, NULL, NULL, 0, 1731 ut_sequence_step_cb, &completed); 1732 CU_ASSERT_EQUAL(rc, 0); 1733 1734 dst_iovs[1].iov_base = buf; 1735 dst_iovs[1].iov_len = sizeof(buf); 1736 src_iovs[1].iov_base = tmp[1]; 1737 src_iovs[1].iov_len = sizeof(tmp[1]); 1738 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1739 &src_iovs[1], 1, NULL, NULL, 0, 1740 ut_sequence_step_cb, &completed); 1741 CU_ASSERT_EQUAL(rc, 0); 1742 1743 ut_seq.complete = false; 1744 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1745 CU_ASSERT_EQUAL(rc, 0); 1746 1747 poll_threads(); 1748 1749 CU_ASSERT_EQUAL(completed, 2); 1750 CU_ASSERT(ut_seq.complete); 1751 CU_ASSERT_EQUAL(ut_seq.status, 0); 1752 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 1); 1753 1754 /* Check fill + copy */ 1755 seq = NULL; 1756 completed = 0; 1757 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1758 g_seq_operations[ACCEL_OPC_FILL].count = 0; 1759 g_seq_operations[ACCEL_OPC_COPY].src_iovs = NULL; 1760 g_seq_operations[ACCEL_OPC_COPY].dst_iovs = NULL; 1761 g_seq_operations[ACCEL_OPC_FILL].dst_iovcnt = 1; 1762 g_seq_operations[ACCEL_OPC_FILL].dst_iovs = &exp_iovs[0]; 1763 exp_iovs[0].iov_base = buf; 1764 exp_iovs[0].iov_len = sizeof(buf); 1765 1766 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 1767 ut_sequence_step_cb, &completed); 1768 CU_ASSERT_EQUAL(rc, 0); 1769 1770 dst_iovs[0].iov_base = buf; 1771 dst_iovs[0].iov_len = sizeof(buf); 1772 src_iovs[0].iov_base = tmp[0]; 1773 src_iovs[0].iov_len = sizeof(tmp[0]); 1774 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1775 &src_iovs[0], 1, NULL, NULL, 0, 1776 ut_sequence_step_cb, &completed); 1777 CU_ASSERT_EQUAL(rc, 0); 1778 1779 ut_seq.complete = false; 1780 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1781 CU_ASSERT_EQUAL(rc, 0); 1782 1783 poll_threads(); 1784 1785 CU_ASSERT_EQUAL(completed, 2); 1786 CU_ASSERT(ut_seq.complete); 1787 CU_ASSERT_EQUAL(ut_seq.status, 0); 1788 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1789 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 1); 1790 1791 /* Check copy + encrypt + copy */ 1792 seq = NULL; 1793 completed = 0; 1794 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1795 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 1796 g_seq_operations[ACCEL_OPC_ENCRYPT].dst_iovcnt = 1; 1797 g_seq_operations[ACCEL_OPC_ENCRYPT].src_iovcnt = 1; 1798 g_seq_operations[ACCEL_OPC_ENCRYPT].src_iovs = &exp_iovs[0]; 1799 g_seq_operations[ACCEL_OPC_ENCRYPT].dst_iovs = &exp_iovs[1]; 1800 exp_iovs[0].iov_base = tmp[0]; 1801 exp_iovs[0].iov_len = sizeof(tmp[0]); 1802 exp_iovs[1].iov_base = buf; 1803 exp_iovs[1].iov_len = sizeof(buf); 1804 1805 dst_iovs[0].iov_base = tmp[1]; 1806 dst_iovs[0].iov_len = sizeof(tmp[1]); 1807 src_iovs[0].iov_base = tmp[0]; 1808 src_iovs[0].iov_len = sizeof(tmp[0]); 1809 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1810 &src_iovs[0], 1, NULL, NULL, 0, 1811 ut_sequence_step_cb, &completed); 1812 CU_ASSERT_EQUAL(rc, 0); 1813 1814 dst_iovs[1].iov_base = tmp[2]; 1815 dst_iovs[1].iov_len = sizeof(tmp[2]); 1816 src_iovs[1].iov_base = tmp[1]; 1817 src_iovs[1].iov_len = sizeof(tmp[1]); 1818 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1819 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1820 ut_sequence_step_cb, &completed); 1821 CU_ASSERT_EQUAL(rc, 0); 1822 1823 dst_iovs[2].iov_base = buf; 1824 dst_iovs[2].iov_len = sizeof(buf); 1825 src_iovs[2].iov_base = tmp[2]; 1826 src_iovs[2].iov_len = sizeof(tmp[2]); 1827 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1828 &src_iovs[2], 1, NULL, NULL, 0, 1829 ut_sequence_step_cb, &completed); 1830 CU_ASSERT_EQUAL(rc, 0); 1831 1832 ut_seq.complete = false; 1833 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1834 CU_ASSERT_EQUAL(rc, 0); 1835 1836 poll_threads(); 1837 1838 CU_ASSERT_EQUAL(completed, 3); 1839 CU_ASSERT(ut_seq.complete); 1840 CU_ASSERT_EQUAL(ut_seq.status, 0); 1841 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1842 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 1843 1844 /* Check copy + decrypt + copy */ 1845 seq = NULL; 1846 completed = 0; 1847 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1848 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 1849 g_seq_operations[ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1850 g_seq_operations[ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1851 g_seq_operations[ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1852 g_seq_operations[ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1853 exp_iovs[0].iov_base = tmp[0]; 1854 exp_iovs[0].iov_len = sizeof(tmp[0]); 1855 exp_iovs[1].iov_base = buf; 1856 exp_iovs[1].iov_len = sizeof(buf); 1857 1858 dst_iovs[0].iov_base = tmp[1]; 1859 dst_iovs[0].iov_len = sizeof(tmp[1]); 1860 src_iovs[0].iov_base = tmp[0]; 1861 src_iovs[0].iov_len = sizeof(tmp[0]); 1862 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1863 &src_iovs[0], 1, NULL, NULL, 0, 1864 ut_sequence_step_cb, &completed); 1865 CU_ASSERT_EQUAL(rc, 0); 1866 1867 dst_iovs[1].iov_base = tmp[2]; 1868 dst_iovs[1].iov_len = sizeof(tmp[2]); 1869 src_iovs[1].iov_base = tmp[1]; 1870 src_iovs[1].iov_len = sizeof(tmp[1]); 1871 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1872 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1873 ut_sequence_step_cb, &completed); 1874 CU_ASSERT_EQUAL(rc, 0); 1875 1876 dst_iovs[2].iov_base = buf; 1877 dst_iovs[2].iov_len = sizeof(buf); 1878 src_iovs[2].iov_base = tmp[2]; 1879 src_iovs[2].iov_len = sizeof(tmp[2]); 1880 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1881 &src_iovs[2], 1, NULL, NULL, 0, 1882 ut_sequence_step_cb, &completed); 1883 CU_ASSERT_EQUAL(rc, 0); 1884 1885 ut_seq.complete = false; 1886 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1887 CU_ASSERT_EQUAL(rc, 0); 1888 1889 poll_threads(); 1890 1891 CU_ASSERT_EQUAL(completed, 3); 1892 CU_ASSERT(ut_seq.complete); 1893 CU_ASSERT_EQUAL(ut_seq.status, 0); 1894 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1895 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 1896 1897 /* Cleanup module pointers to make subsequent tests work correctly */ 1898 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1899 g_modules_opc[i] = modules[i]; 1900 } 1901 1902 ut_clear_operations(); 1903 spdk_put_io_channel(ioch); 1904 poll_threads(); 1905 } 1906 1907 static int 1908 ut_submit_decompress(struct spdk_io_channel *ch, struct spdk_accel_task *task) 1909 { 1910 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 1911 1912 spdk_accel_task_complete(task, 0); 1913 1914 return 0; 1915 } 1916 1917 static void 1918 test_sequence_accel_buffers(void) 1919 { 1920 struct spdk_accel_sequence *seq = NULL; 1921 struct spdk_io_channel *ioch; 1922 struct accel_io_channel *accel_ch; 1923 struct ut_sequence ut_seq; 1924 struct iovec src_iovs[3], dst_iovs[3]; 1925 char srcbuf[4096], dstbuf[4096], expected[4096]; 1926 struct accel_module modules[ACCEL_OPC_LAST]; 1927 void *buf[2], *domain_ctx[2], *iobuf_buf; 1928 struct spdk_memory_domain *domain[2]; 1929 struct spdk_iobuf_buffer *cache_entry; 1930 spdk_iobuf_buffer_stailq_t small_cache; 1931 uint32_t small_cache_count; 1932 int i, rc, completed; 1933 struct spdk_iobuf_opts opts_iobuf = {}; 1934 1935 /* Set up the iobuf to always use the "small" pool */ 1936 opts_iobuf.large_bufsize = 0x20000; 1937 opts_iobuf.large_pool_count = 0; 1938 opts_iobuf.small_bufsize = 0x10000; 1939 opts_iobuf.small_pool_count = 32; 1940 1941 rc = spdk_iobuf_set_opts(&opts_iobuf); 1942 CU_ASSERT(rc == 0); 1943 1944 ioch = spdk_accel_get_io_channel(); 1945 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1946 1947 /* Override the submit_tasks function */ 1948 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1949 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1950 modules[i] = g_modules_opc[i]; 1951 g_modules_opc[i] = g_module; 1952 } 1953 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 1954 * decompress operations together in one sequence. 1955 */ 1956 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 1957 g_seq_operations[ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 1958 g_seq_operations[ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 1959 1960 /* Check the simplest case: one operation using accel buffer as destination + copy operation 1961 * specifying the actual destination buffer 1962 */ 1963 memset(srcbuf, 0xa5, 4096); 1964 memset(dstbuf, 0x0, 4096); 1965 memset(expected, 0xa5, 4096); 1966 completed = 0; 1967 seq = NULL; 1968 1969 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 1970 CU_ASSERT_EQUAL(rc, 0); 1971 CU_ASSERT_PTR_NOT_NULL(buf[0]); 1972 1973 src_iovs[0].iov_base = srcbuf; 1974 src_iovs[0].iov_len = 4096; 1975 dst_iovs[0].iov_base = buf[0]; 1976 dst_iovs[0].iov_len = 4096; 1977 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 1978 &src_iovs[0], 1, NULL, NULL, 0, 1979 ut_sequence_step_cb, &completed); 1980 CU_ASSERT_EQUAL(rc, 0); 1981 1982 src_iovs[1].iov_base = buf[0]; 1983 src_iovs[1].iov_len = 4096; 1984 dst_iovs[1].iov_base = dstbuf; 1985 dst_iovs[1].iov_len = 4096; 1986 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1987 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 1988 ut_sequence_step_cb, &completed); 1989 CU_ASSERT_EQUAL(rc, 0); 1990 1991 ut_seq.complete = false; 1992 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1993 CU_ASSERT_EQUAL(rc, 0); 1994 1995 poll_threads(); 1996 1997 CU_ASSERT_EQUAL(completed, 2); 1998 CU_ASSERT(ut_seq.complete); 1999 CU_ASSERT_EQUAL(ut_seq.status, 0); 2000 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2001 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2002 2003 /* Start with a fill operation using accel buffer, followed by a decompress using another 2004 * accel buffer as dst, followed by a copy operation specifying dst buffer of the whole 2005 * sequence 2006 */ 2007 memset(srcbuf, 0x0, 4096); 2008 memset(dstbuf, 0x0, 4096); 2009 memset(expected, 0x5a, 4096); 2010 completed = 0; 2011 seq = NULL; 2012 2013 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2014 CU_ASSERT_EQUAL(rc, 0); 2015 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2016 2017 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2018 ut_sequence_step_cb, &completed); 2019 CU_ASSERT_EQUAL(rc, 0); 2020 2021 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2022 CU_ASSERT_EQUAL(rc, 0); 2023 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2024 2025 src_iovs[0].iov_base = buf[0]; 2026 src_iovs[0].iov_len = 4096; 2027 dst_iovs[0].iov_base = buf[1]; 2028 dst_iovs[0].iov_len = 4096; 2029 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2030 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2031 ut_sequence_step_cb, &completed); 2032 CU_ASSERT_EQUAL(rc, 0); 2033 2034 src_iovs[1].iov_base = buf[1]; 2035 src_iovs[1].iov_len = 4096; 2036 dst_iovs[1].iov_base = dstbuf; 2037 dst_iovs[1].iov_len = 4096; 2038 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2039 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2040 ut_sequence_step_cb, &completed); 2041 2042 ut_seq.complete = false; 2043 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2044 CU_ASSERT_EQUAL(rc, 0); 2045 2046 poll_threads(); 2047 2048 CU_ASSERT_EQUAL(completed, 3); 2049 CU_ASSERT(ut_seq.complete); 2050 CU_ASSERT_EQUAL(ut_seq.status, 0); 2051 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2052 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2053 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2054 2055 /* Check the same, but with two decompress operations with the first one being in-place */ 2056 memset(srcbuf, 0x0, 4096); 2057 memset(dstbuf, 0x0, 4096); 2058 memset(expected, 0x5a, 4096); 2059 completed = 0; 2060 seq = NULL; 2061 2062 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2063 CU_ASSERT_EQUAL(rc, 0); 2064 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2065 2066 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2067 ut_sequence_step_cb, &completed); 2068 CU_ASSERT_EQUAL(rc, 0); 2069 2070 src_iovs[0].iov_base = buf[0]; 2071 src_iovs[0].iov_len = 4096; 2072 dst_iovs[0].iov_base = buf[0]; 2073 dst_iovs[0].iov_len = 4096; 2074 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2075 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2076 ut_sequence_step_cb, &completed); 2077 CU_ASSERT_EQUAL(rc, 0); 2078 2079 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2080 CU_ASSERT_EQUAL(rc, 0); 2081 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2082 2083 src_iovs[1].iov_base = buf[0]; 2084 src_iovs[1].iov_len = 4096; 2085 dst_iovs[1].iov_base = buf[1]; 2086 dst_iovs[1].iov_len = 4096; 2087 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, domain[1], domain_ctx[1], 2088 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 2089 ut_sequence_step_cb, &completed); 2090 CU_ASSERT_EQUAL(rc, 0); 2091 2092 src_iovs[2].iov_base = buf[1]; 2093 src_iovs[2].iov_len = 4096; 2094 dst_iovs[2].iov_base = dstbuf; 2095 dst_iovs[2].iov_len = 4096; 2096 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 2097 &src_iovs[2], 1, domain[1], domain_ctx[1], 0, 2098 ut_sequence_step_cb, &completed); 2099 2100 ut_seq.complete = false; 2101 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2102 CU_ASSERT_EQUAL(rc, 0); 2103 2104 poll_threads(); 2105 2106 CU_ASSERT_EQUAL(completed, 4); 2107 CU_ASSERT(ut_seq.complete); 2108 CU_ASSERT_EQUAL(ut_seq.status, 0); 2109 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2110 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2111 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2112 2113 /* Check that specifying offsets within accel buffers works correctly */ 2114 memset(srcbuf, 0x0, 4096); 2115 memset(dstbuf, 0x0, 4096); 2116 completed = 0; 2117 seq = NULL; 2118 2119 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2120 CU_ASSERT_EQUAL(rc, 0); 2121 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2122 2123 /* Fill in each 1K of the buffer with different pattern */ 2124 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2125 ut_sequence_step_cb, &completed); 2126 CU_ASSERT_EQUAL(rc, 0); 2127 2128 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2129 0x5a, 0, ut_sequence_step_cb, &completed); 2130 CU_ASSERT_EQUAL(rc, 0); 2131 2132 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2133 0xfe, 0, ut_sequence_step_cb, &completed); 2134 CU_ASSERT_EQUAL(rc, 0); 2135 2136 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2137 0xed, 0, ut_sequence_step_cb, &completed); 2138 CU_ASSERT_EQUAL(rc, 0); 2139 2140 src_iovs[0].iov_base = buf[0]; 2141 src_iovs[0].iov_len = 4096; 2142 dst_iovs[0].iov_base = dstbuf; 2143 dst_iovs[0].iov_len = 4096; 2144 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2145 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2146 ut_sequence_step_cb, &completed); 2147 2148 ut_seq.complete = false; 2149 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2150 CU_ASSERT_EQUAL(rc, 0); 2151 2152 poll_threads(); 2153 2154 memset(&expected[0], 0xa5, 1024); 2155 memset(&expected[1024], 0x5a, 1024); 2156 memset(&expected[2048], 0xfe, 1024); 2157 memset(&expected[3072], 0xed, 1024); 2158 CU_ASSERT_EQUAL(completed, 5); 2159 CU_ASSERT(ut_seq.complete); 2160 CU_ASSERT_EQUAL(ut_seq.status, 0); 2161 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2162 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2163 2164 /* Check the same but this time with a decompress operation on part of the buffer (512B 2165 * offset) */ 2166 memset(srcbuf, 0x0, 4096); 2167 memset(dstbuf, 0x0, 4096); 2168 completed = 0; 2169 seq = NULL; 2170 2171 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2172 CU_ASSERT_EQUAL(rc, 0); 2173 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2174 2175 /* Fill in each 1K of the buffer with different pattern */ 2176 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2177 ut_sequence_step_cb, &completed); 2178 CU_ASSERT_EQUAL(rc, 0); 2179 2180 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2181 0x5a, 0, ut_sequence_step_cb, &completed); 2182 CU_ASSERT_EQUAL(rc, 0); 2183 2184 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2185 0xfe, 0, ut_sequence_step_cb, &completed); 2186 CU_ASSERT_EQUAL(rc, 0); 2187 2188 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2189 0xed, 0, ut_sequence_step_cb, &completed); 2190 CU_ASSERT_EQUAL(rc, 0); 2191 2192 rc = spdk_accel_get_buf(ioch, 3072, &buf[1], &domain[1], &domain_ctx[1]); 2193 CU_ASSERT_EQUAL(rc, 0); 2194 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2195 2196 src_iovs[0].iov_base = (char *)buf[0] + 512; 2197 src_iovs[0].iov_len = 3072; 2198 dst_iovs[0].iov_base = (char *)buf[1] + 256; 2199 dst_iovs[0].iov_len = 3072; 2200 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2201 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2202 ut_sequence_step_cb, &completed); 2203 2204 src_iovs[1].iov_base = (char *)buf[1] + 256; 2205 src_iovs[1].iov_len = 3072; 2206 dst_iovs[1].iov_base = dstbuf; 2207 dst_iovs[1].iov_len = 3072; 2208 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2209 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2210 ut_sequence_step_cb, &completed); 2211 2212 ut_seq.complete = false; 2213 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2214 CU_ASSERT_EQUAL(rc, 0); 2215 2216 poll_threads(); 2217 2218 memset(&expected[0], 0xa5, 512); 2219 memset(&expected[512], 0x5a, 1024); 2220 memset(&expected[1536], 0xfe, 1024); 2221 memset(&expected[2560], 0xed, 512); 2222 CU_ASSERT_EQUAL(completed, 6); 2223 CU_ASSERT(ut_seq.complete); 2224 CU_ASSERT_EQUAL(ut_seq.status, 0); 2225 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 3072), 0); 2226 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2227 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2228 2229 /* Check that if iobuf pool is empty, the sequence processing will wait until a buffer is 2230 * available 2231 */ 2232 accel_ch = spdk_io_channel_get_ctx(ioch); 2233 small_cache_count = accel_ch->iobuf.small.cache_count; 2234 STAILQ_INIT(&small_cache); 2235 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2236 accel_ch->iobuf.small.cache_count = 0; 2237 accel_ch->iobuf.small.cache_size = 0; 2238 g_iobuf.small_pool_count = 0; 2239 2240 /* First allocate a single buffer used by two operations */ 2241 memset(srcbuf, 0x0, 4096); 2242 memset(dstbuf, 0x0, 4096); 2243 memset(expected, 0xa5, 4096); 2244 completed = 0; 2245 seq = NULL; 2246 2247 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2248 CU_ASSERT_EQUAL(rc, 0); 2249 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2250 2251 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0xa5, 0, 2252 ut_sequence_step_cb, &completed); 2253 CU_ASSERT_EQUAL(rc, 0); 2254 2255 src_iovs[0].iov_base = buf[0]; 2256 src_iovs[0].iov_len = 4096; 2257 dst_iovs[0].iov_base = dstbuf; 2258 dst_iovs[0].iov_len = 4096; 2259 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2260 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2261 ut_sequence_step_cb, &completed); 2262 2263 ut_seq.complete = false; 2264 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2265 CU_ASSERT_EQUAL(rc, 0); 2266 2267 poll_threads(); 2268 2269 CU_ASSERT_EQUAL(completed, 0); 2270 CU_ASSERT(!ut_seq.complete); 2271 2272 /* Get a buffer and return it to the pool to trigger the sequence to finish */ 2273 g_iobuf.small_pool_count = 1; 2274 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2275 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2276 CU_ASSERT(g_iobuf.small_pool_count == 0); 2277 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2278 2279 poll_threads(); 2280 2281 CU_ASSERT_EQUAL(completed, 2); 2282 CU_ASSERT(ut_seq.complete); 2283 CU_ASSERT_EQUAL(ut_seq.status, 0); 2284 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2285 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2286 2287 /* Return the buffers back to the cache */ 2288 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2289 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2290 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2291 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2292 small_cache_count++; 2293 } 2294 accel_ch->iobuf.small.cache_count = 0; 2295 g_iobuf.small_pool_count = 0; 2296 2297 /* Check a bit more complex scenario, with two buffers in the sequence */ 2298 memset(srcbuf, 0x0, 4096); 2299 memset(dstbuf, 0x0, 4096); 2300 memset(expected, 0x5a, 4096); 2301 completed = 0; 2302 seq = NULL; 2303 2304 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2305 CU_ASSERT_EQUAL(rc, 0); 2306 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2307 2308 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2309 ut_sequence_step_cb, &completed); 2310 CU_ASSERT_EQUAL(rc, 0); 2311 2312 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2313 CU_ASSERT_EQUAL(rc, 0); 2314 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2315 2316 src_iovs[0].iov_base = buf[0]; 2317 src_iovs[0].iov_len = 4096; 2318 dst_iovs[0].iov_base = buf[1]; 2319 dst_iovs[0].iov_len = 4096; 2320 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2321 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2322 ut_sequence_step_cb, &completed); 2323 CU_ASSERT_EQUAL(rc, 0); 2324 2325 src_iovs[1].iov_base = buf[1]; 2326 src_iovs[1].iov_len = 4096; 2327 dst_iovs[1].iov_base = dstbuf; 2328 dst_iovs[1].iov_len = 4096; 2329 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2330 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2331 ut_sequence_step_cb, &completed); 2332 CU_ASSERT_EQUAL(rc, 0); 2333 2334 ut_seq.complete = false; 2335 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2336 CU_ASSERT_EQUAL(rc, 0); 2337 2338 poll_threads(); 2339 2340 CU_ASSERT_EQUAL(completed, 0); 2341 CU_ASSERT(!ut_seq.complete); 2342 2343 g_iobuf.small_pool_count = 1; 2344 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2345 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2346 g_iobuf.small_pool_count = 0; 2347 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2348 2349 /* One buffer is not enough to finish the whole sequence */ 2350 poll_threads(); 2351 CU_ASSERT(!ut_seq.complete); 2352 2353 g_iobuf.small_pool_count = 1; 2354 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2355 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2356 g_iobuf.small_pool_count = 0; 2357 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2358 2359 poll_threads(); 2360 2361 CU_ASSERT_EQUAL(completed, 3); 2362 CU_ASSERT(ut_seq.complete); 2363 CU_ASSERT_EQUAL(ut_seq.status, 0); 2364 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2365 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2366 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2367 2368 /* Return the buffers back to the cache */ 2369 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2370 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2371 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2372 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2373 small_cache_count++; 2374 } 2375 accel_ch->iobuf.small.cache_count = 0; 2376 2377 g_iobuf.small_pool_count = 32; 2378 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2379 accel_ch->iobuf.small.cache_count = small_cache_count; 2380 2381 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2382 g_modules_opc[i] = modules[i]; 2383 } 2384 2385 ut_clear_operations(); 2386 spdk_put_io_channel(ioch); 2387 poll_threads(); 2388 } 2389 2390 static void 2391 ut_domain_ctx_init(struct ut_domain_ctx *ctx, void *base, size_t len, struct iovec *expected) 2392 { 2393 ctx->iov.iov_base = base; 2394 ctx->iov.iov_len = len; 2395 ctx->expected = *expected; 2396 ctx->pull_submit_status = 0; 2397 ctx->push_submit_status = 0; 2398 ctx->pull_complete_status = 0; 2399 ctx->push_complete_status = 0; 2400 } 2401 2402 static void 2403 test_sequence_memory_domain(void) 2404 { 2405 struct spdk_accel_sequence *seq = NULL; 2406 struct spdk_io_channel *ioch; 2407 struct accel_io_channel *accel_ch; 2408 struct ut_sequence ut_seq; 2409 struct ut_domain_ctx domctx[4]; 2410 struct spdk_iobuf_buffer *cache_entry; 2411 struct accel_module modules[ACCEL_OPC_LAST]; 2412 struct spdk_memory_domain *accel_domain; 2413 spdk_iobuf_buffer_stailq_t small_cache; 2414 char srcbuf[4096], dstbuf[4096], expected[4096], tmp[4096]; 2415 struct iovec src_iovs[2], dst_iovs[2]; 2416 uint32_t small_cache_count; 2417 void *accel_buf, *accel_domain_ctx, *iobuf_buf; 2418 int i, rc, completed; 2419 2420 ioch = spdk_accel_get_io_channel(); 2421 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2422 2423 /* Override the submit_tasks function */ 2424 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2425 g_module.supports_memory_domains = false; 2426 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2427 modules[i] = g_modules_opc[i]; 2428 g_modules_opc[i] = g_module; 2429 } 2430 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2431 * decompress operations together in one sequence. 2432 */ 2433 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2434 g_seq_operations[ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2435 g_seq_operations[ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2436 2437 /* First check the simplest case - single fill operation with dstbuf in domain */ 2438 memset(expected, 0xa5, sizeof(expected)); 2439 memset(dstbuf, 0x0, sizeof(dstbuf)); 2440 completed = 0; 2441 seq = NULL; 2442 2443 /* Use some garbage pointer as dst and use domain ctx to get the actual dstbuf */ 2444 dst_iovs[0].iov_base = (void *)0xcafebabe; 2445 dst_iovs[0].iov_len = sizeof(dstbuf); 2446 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2447 2448 rc = spdk_accel_append_fill(&seq, ioch, dst_iovs[0].iov_base, dst_iovs[0].iov_len, 2449 g_ut_domain, &domctx[0], 0xa5, 0, 2450 ut_sequence_step_cb, &completed); 2451 CU_ASSERT_EQUAL(rc, 0); 2452 2453 ut_seq.complete = false; 2454 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2455 CU_ASSERT_EQUAL(rc, 0); 2456 2457 poll_threads(); 2458 CU_ASSERT_EQUAL(completed, 1); 2459 CU_ASSERT(ut_seq.complete); 2460 CU_ASSERT_EQUAL(ut_seq.status, 0); 2461 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2462 2463 /* Check operation with both srcbuf and dstbuf in remote memory domain */ 2464 memset(expected, 0x5a, sizeof(expected)); 2465 memset(dstbuf, 0x0, sizeof(dstbuf)); 2466 memset(srcbuf, 0x5a, sizeof(dstbuf)); 2467 completed = 0; 2468 seq = NULL; 2469 2470 src_iovs[0].iov_base = (void *)0xcafebabe; 2471 src_iovs[0].iov_len = sizeof(srcbuf); 2472 dst_iovs[0].iov_base = (void *)0xbeefdead; 2473 dst_iovs[0].iov_len = sizeof(dstbuf); 2474 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2475 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2476 2477 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2478 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2479 ut_sequence_step_cb, &completed); 2480 CU_ASSERT_EQUAL(rc, 0); 2481 2482 ut_seq.complete = false; 2483 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2484 CU_ASSERT_EQUAL(rc, 0); 2485 2486 poll_threads(); 2487 CU_ASSERT_EQUAL(completed, 1); 2488 CU_ASSERT(ut_seq.complete); 2489 CU_ASSERT_EQUAL(ut_seq.status, 0); 2490 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2491 2492 /* Two operations using a buffer in remote domain */ 2493 memset(expected, 0xa5, sizeof(expected)); 2494 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2495 memset(tmp, 0x0, sizeof(tmp)); 2496 memset(dstbuf, 0x0, sizeof(dstbuf)); 2497 completed = 0; 2498 seq = NULL; 2499 2500 src_iovs[0].iov_base = (void *)0xcafebabe; 2501 src_iovs[0].iov_len = sizeof(srcbuf); 2502 dst_iovs[0].iov_base = (void *)0xbeefdead; 2503 dst_iovs[0].iov_len = sizeof(tmp); 2504 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2505 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2506 2507 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2508 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2509 ut_sequence_step_cb, &completed); 2510 CU_ASSERT_EQUAL(rc, 0); 2511 2512 src_iovs[1].iov_base = (void *)0xbeefdead; 2513 src_iovs[1].iov_len = sizeof(tmp); 2514 dst_iovs[1].iov_base = (void *)0xa5a5a5a5; 2515 dst_iovs[1].iov_len = sizeof(dstbuf); 2516 ut_domain_ctx_init(&domctx[2], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2517 ut_domain_ctx_init(&domctx[3], tmp, sizeof(tmp), &src_iovs[1]); 2518 2519 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[2], 2520 &src_iovs[1], 1, g_ut_domain, &domctx[3], 0, 2521 ut_sequence_step_cb, &completed); 2522 CU_ASSERT_EQUAL(rc, 0); 2523 2524 ut_seq.complete = false; 2525 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2526 CU_ASSERT_EQUAL(rc, 0); 2527 2528 poll_threads(); 2529 CU_ASSERT_EQUAL(completed, 2); 2530 CU_ASSERT(ut_seq.complete); 2531 CU_ASSERT_EQUAL(ut_seq.status, 0); 2532 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2533 2534 /* Use buffer in remote memory domain and accel buffer at the same time */ 2535 memset(expected, 0x5a, sizeof(expected)); 2536 memset(srcbuf, 0x5a, sizeof(expected)); 2537 memset(dstbuf, 0x0, sizeof(dstbuf)); 2538 completed = 0; 2539 seq = NULL; 2540 2541 rc = spdk_accel_get_buf(ioch, sizeof(dstbuf), &accel_buf, &accel_domain, &accel_domain_ctx); 2542 CU_ASSERT_EQUAL(rc, 0); 2543 2544 src_iovs[0].iov_base = (void *)0xfeedbeef; 2545 src_iovs[0].iov_len = sizeof(srcbuf); 2546 dst_iovs[0].iov_base = accel_buf; 2547 dst_iovs[0].iov_len = sizeof(dstbuf); 2548 ut_domain_ctx_init(&domctx[0], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2549 2550 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, accel_domain, accel_domain_ctx, 2551 &src_iovs[0], 1, g_ut_domain, &domctx[0], 0, 2552 ut_sequence_step_cb, &completed); 2553 CU_ASSERT_EQUAL(rc, 0); 2554 2555 src_iovs[1].iov_base = accel_buf; 2556 src_iovs[1].iov_len = sizeof(dstbuf); 2557 dst_iovs[1].iov_base = (void *)0xbeeffeed; 2558 dst_iovs[1].iov_len = sizeof(dstbuf); 2559 ut_domain_ctx_init(&domctx[1], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2560 2561 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[1], 2562 &src_iovs[1], 1, accel_domain, accel_domain_ctx, 0, 2563 ut_sequence_step_cb, &completed); 2564 CU_ASSERT_EQUAL(rc, 0); 2565 2566 ut_seq.complete = false; 2567 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2568 CU_ASSERT_EQUAL(rc, 0); 2569 2570 poll_threads(); 2571 CU_ASSERT_EQUAL(completed, 2); 2572 CU_ASSERT(ut_seq.complete); 2573 CU_ASSERT_EQUAL(ut_seq.status, 0); 2574 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2575 spdk_accel_put_buf(ioch, accel_buf, accel_domain, accel_domain_ctx); 2576 2577 /* Check that a sequence with memory domains is correctly executed if buffers are not 2578 * immediately available */ 2579 memset(expected, 0xa5, sizeof(expected)); 2580 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2581 memset(dstbuf, 0x0, sizeof(dstbuf)); 2582 completed = 0; 2583 seq = NULL; 2584 /* Make sure the buffer pool is empty */ 2585 accel_ch = spdk_io_channel_get_ctx(ioch); 2586 small_cache_count = accel_ch->iobuf.small.cache_count; 2587 STAILQ_INIT(&small_cache); 2588 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2589 accel_ch->iobuf.small.cache_count = 0; 2590 g_iobuf.small_pool_count = 0; 2591 2592 src_iovs[0].iov_base = (void *)0xdeadbeef; 2593 src_iovs[0].iov_len = sizeof(srcbuf); 2594 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2595 dst_iovs[0].iov_len = sizeof(dstbuf); 2596 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2597 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2598 2599 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2600 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2601 ut_sequence_step_cb, &completed); 2602 CU_ASSERT_EQUAL(rc, 0); 2603 2604 ut_seq.complete = false; 2605 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2606 CU_ASSERT_EQUAL(rc, 0); 2607 2608 poll_threads(); 2609 CU_ASSERT_EQUAL(completed, 0); 2610 CU_ASSERT(!ut_seq.complete); 2611 2612 /* Get a buffer and return it to the pool to trigger the sequence to resume. It shouldn't 2613 * be able to complete, as it needs two buffers */ 2614 g_iobuf.small_pool_count = 1; 2615 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2616 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2617 g_iobuf.small_pool_count = 0; 2618 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2619 2620 CU_ASSERT_EQUAL(completed, 0); 2621 CU_ASSERT(!ut_seq.complete); 2622 2623 /* Return another buffer, this time the sequence should finish */ 2624 g_iobuf.small_pool_count = 1; 2625 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2626 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2627 g_iobuf.small_pool_count = 0; 2628 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2629 2630 poll_threads(); 2631 CU_ASSERT_EQUAL(completed, 1); 2632 CU_ASSERT(ut_seq.complete); 2633 CU_ASSERT_EQUAL(ut_seq.status, 0); 2634 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2635 2636 /* Return the buffers back to the cache */ 2637 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2638 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2639 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2640 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2641 small_cache_count++; 2642 } 2643 accel_ch->iobuf.small.cache_count = 0; 2644 2645 g_iobuf.small_pool_count = 32; 2646 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2647 accel_ch->iobuf.small.cache_count = small_cache_count; 2648 2649 /* Check error cases, starting with an error from spdk_memory_domain_pull_data() */ 2650 completed = 0; 2651 seq = NULL; 2652 2653 src_iovs[0].iov_base = (void *)0xdeadbeef; 2654 src_iovs[0].iov_len = sizeof(srcbuf); 2655 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2656 dst_iovs[0].iov_len = sizeof(dstbuf); 2657 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2658 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2659 domctx[1].pull_submit_status = -E2BIG; 2660 2661 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2662 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2663 ut_sequence_step_cb, &completed); 2664 CU_ASSERT_EQUAL(rc, 0); 2665 2666 ut_seq.complete = false; 2667 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2668 CU_ASSERT_EQUAL(rc, 0); 2669 2670 CU_ASSERT_EQUAL(completed, 1); 2671 CU_ASSERT(ut_seq.complete); 2672 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 2673 2674 /* Check completion error from spdk_memory_domain_pull_data() */ 2675 completed = 0; 2676 seq = NULL; 2677 2678 src_iovs[0].iov_base = (void *)0xdeadbeef; 2679 src_iovs[0].iov_len = sizeof(srcbuf); 2680 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2681 dst_iovs[0].iov_len = sizeof(dstbuf); 2682 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2683 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2684 domctx[1].pull_complete_status = -EACCES; 2685 2686 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2687 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2688 ut_sequence_step_cb, &completed); 2689 CU_ASSERT_EQUAL(rc, 0); 2690 2691 ut_seq.complete = false; 2692 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2693 CU_ASSERT_EQUAL(rc, 0); 2694 2695 CU_ASSERT_EQUAL(completed, 1); 2696 CU_ASSERT(ut_seq.complete); 2697 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 2698 2699 /* Check submission error from spdk_memory_domain_push_data() */ 2700 completed = 0; 2701 seq = NULL; 2702 2703 src_iovs[0].iov_base = (void *)0xdeadbeef; 2704 src_iovs[0].iov_len = sizeof(srcbuf); 2705 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2706 dst_iovs[0].iov_len = sizeof(dstbuf); 2707 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2708 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2709 domctx[0].push_submit_status = -EADDRINUSE; 2710 2711 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2712 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2713 ut_sequence_step_cb, &completed); 2714 CU_ASSERT_EQUAL(rc, 0); 2715 2716 ut_seq.complete = false; 2717 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2718 CU_ASSERT_EQUAL(rc, 0); 2719 2720 CU_ASSERT_EQUAL(completed, 1); 2721 CU_ASSERT(ut_seq.complete); 2722 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 2723 2724 /* Check completion error from spdk_memory_domain_push_data() */ 2725 completed = 0; 2726 seq = NULL; 2727 2728 src_iovs[0].iov_base = (void *)0xdeadbeef; 2729 src_iovs[0].iov_len = sizeof(srcbuf); 2730 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2731 dst_iovs[0].iov_len = sizeof(dstbuf); 2732 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2733 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2734 domctx[0].push_complete_status = -EADDRNOTAVAIL; 2735 2736 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2737 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2738 ut_sequence_step_cb, &completed); 2739 CU_ASSERT_EQUAL(rc, 0); 2740 2741 ut_seq.complete = false; 2742 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2743 CU_ASSERT_EQUAL(rc, 0); 2744 2745 CU_ASSERT_EQUAL(completed, 1); 2746 CU_ASSERT(ut_seq.complete); 2747 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 2748 2749 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2750 g_modules_opc[i] = modules[i]; 2751 } 2752 2753 ut_clear_operations(); 2754 spdk_put_io_channel(ioch); 2755 poll_threads(); 2756 } 2757 2758 static int 2759 ut_submit_decompress_memory_domain(struct spdk_io_channel *ch, struct spdk_accel_task *task) 2760 { 2761 struct ut_domain_ctx *ctx; 2762 struct iovec *src_iovs, *dst_iovs; 2763 uint32_t src_iovcnt, dst_iovcnt; 2764 2765 src_iovs = task->s.iovs; 2766 dst_iovs = task->d.iovs; 2767 src_iovcnt = task->s.iovcnt; 2768 dst_iovcnt = task->d.iovcnt; 2769 2770 if (task->src_domain != NULL) { 2771 ctx = task->src_domain_ctx; 2772 CU_ASSERT_EQUAL(memcmp(task->s.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2773 src_iovs = &ctx->iov; 2774 src_iovcnt = 1; 2775 } 2776 2777 if (task->dst_domain != NULL) { 2778 ctx = task->dst_domain_ctx; 2779 CU_ASSERT_EQUAL(memcmp(task->d.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2780 dst_iovs = &ctx->iov; 2781 dst_iovcnt = 1; 2782 } 2783 2784 spdk_iovcpy(src_iovs, src_iovcnt, dst_iovs, dst_iovcnt); 2785 spdk_accel_task_complete(task, 0); 2786 2787 return 0; 2788 } 2789 2790 static void 2791 test_sequence_module_memory_domain(void) 2792 { 2793 struct spdk_accel_sequence *seq = NULL; 2794 struct spdk_io_channel *ioch; 2795 struct accel_module modules[ACCEL_OPC_LAST]; 2796 struct spdk_memory_domain *accel_domain; 2797 struct ut_sequence ut_seq; 2798 struct ut_domain_ctx domctx[2]; 2799 struct iovec src_iovs[2], dst_iovs[2]; 2800 void *buf, *accel_domain_ctx; 2801 char srcbuf[4096], dstbuf[4096], tmp[4096], expected[4096]; 2802 int i, rc, completed; 2803 2804 ioch = spdk_accel_get_io_channel(); 2805 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2806 2807 /* Override the submit_tasks function */ 2808 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2809 g_module.supports_memory_domains = true; 2810 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2811 modules[i] = g_modules_opc[i]; 2812 g_modules_opc[i] = g_module; 2813 } 2814 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress_memory_domain; 2815 g_seq_operations[ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2816 2817 /* Check a sequence with both buffers in memory domains */ 2818 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2819 memset(expected, 0xa5, sizeof(expected)); 2820 memset(dstbuf, 0, sizeof(dstbuf)); 2821 seq = NULL; 2822 completed = 0; 2823 2824 src_iovs[0].iov_base = (void *)0xcafebabe; 2825 src_iovs[0].iov_len = sizeof(srcbuf); 2826 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2827 dst_iovs[0].iov_len = sizeof(dstbuf); 2828 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2829 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2830 2831 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2832 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2833 ut_sequence_step_cb, &completed); 2834 CU_ASSERT_EQUAL(rc, 0); 2835 2836 ut_seq.complete = false; 2837 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2838 CU_ASSERT_EQUAL(rc, 0); 2839 2840 poll_threads(); 2841 2842 CU_ASSERT_EQUAL(completed, 1); 2843 CU_ASSERT(ut_seq.complete); 2844 CU_ASSERT_EQUAL(ut_seq.status, 0); 2845 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2846 2847 /* Check two operations each with a single buffer in memory domain */ 2848 memset(srcbuf, 0x5a, sizeof(srcbuf)); 2849 memset(expected, 0x5a, sizeof(expected)); 2850 memset(dstbuf, 0, sizeof(dstbuf)); 2851 memset(tmp, 0, sizeof(tmp)); 2852 seq = NULL; 2853 completed = 0; 2854 2855 src_iovs[0].iov_base = srcbuf; 2856 src_iovs[0].iov_len = sizeof(srcbuf); 2857 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2858 dst_iovs[0].iov_len = sizeof(tmp); 2859 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2860 2861 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2862 &src_iovs[0], 1, NULL, NULL, 0, 2863 ut_sequence_step_cb, &completed); 2864 CU_ASSERT_EQUAL(rc, 0); 2865 2866 src_iovs[1].iov_base = (void *)0xfeedbeef; 2867 src_iovs[1].iov_len = sizeof(tmp); 2868 dst_iovs[1].iov_base = dstbuf; 2869 dst_iovs[1].iov_len = sizeof(dstbuf); 2870 ut_domain_ctx_init(&domctx[1], tmp, sizeof(tmp), &src_iovs[1]); 2871 2872 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2873 &src_iovs[1], 1, g_ut_domain, &domctx[1], 0, 2874 ut_sequence_step_cb, &completed); 2875 CU_ASSERT_EQUAL(rc, 0); 2876 2877 ut_seq.complete = false; 2878 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2879 CU_ASSERT_EQUAL(rc, 0); 2880 2881 poll_threads(); 2882 2883 CU_ASSERT_EQUAL(completed, 2); 2884 CU_ASSERT(ut_seq.complete); 2885 CU_ASSERT_EQUAL(ut_seq.status, 0); 2886 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2887 2888 /* Check a sequence with an accel buffer and a buffer in a regular memory domain */ 2889 memset(expected, 0xa5, sizeof(expected)); 2890 memset(dstbuf, 0, sizeof(dstbuf)); 2891 memset(tmp, 0, sizeof(tmp)); 2892 seq = NULL; 2893 completed = 0; 2894 2895 rc = spdk_accel_get_buf(ioch, 4096, &buf, &accel_domain, &accel_domain_ctx); 2896 CU_ASSERT_EQUAL(rc, 0); 2897 2898 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, accel_domain, accel_domain_ctx, 2899 0xa5, 0, ut_sequence_step_cb, &completed); 2900 CU_ASSERT_EQUAL(rc, 0); 2901 2902 src_iovs[0].iov_base = buf; 2903 src_iovs[0].iov_len = 4096; 2904 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2905 dst_iovs[0].iov_len = sizeof(dstbuf); 2906 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2907 2908 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2909 &src_iovs[0], 1, accel_domain, accel_domain_ctx, 0, 2910 ut_sequence_step_cb, &completed); 2911 CU_ASSERT_EQUAL(rc, 0); 2912 2913 ut_seq.complete = false; 2914 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2915 CU_ASSERT_EQUAL(rc, 0); 2916 2917 poll_threads(); 2918 2919 CU_ASSERT_EQUAL(completed, 2); 2920 CU_ASSERT(ut_seq.complete); 2921 CU_ASSERT_EQUAL(ut_seq.status, 0); 2922 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2923 2924 spdk_accel_put_buf(ioch, buf, accel_domain, accel_domain_ctx); 2925 2926 g_module.supports_memory_domains = false; 2927 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2928 g_modules_opc[i] = modules[i]; 2929 } 2930 2931 ut_clear_operations(); 2932 spdk_put_io_channel(ioch); 2933 poll_threads(); 2934 } 2935 2936 #ifdef SPDK_CONFIG_ISAL_CRYPTO 2937 static void 2938 ut_encrypt_cb(void *cb_arg, int status) 2939 { 2940 int *completed = cb_arg; 2941 2942 CU_ASSERT_EQUAL(status, 0); 2943 2944 *completed = 1; 2945 } 2946 2947 static void 2948 test_sequence_crypto(void) 2949 { 2950 struct spdk_accel_sequence *seq = NULL; 2951 struct spdk_io_channel *ioch; 2952 struct spdk_accel_crypto_key *key; 2953 struct spdk_accel_crypto_key_create_param key_params = { 2954 .cipher = "AES_XTS", 2955 .hex_key = "00112233445566778899aabbccddeeff", 2956 .hex_key2 = "ffeeddccbbaa99887766554433221100", 2957 .key_name = "ut_key", 2958 }; 2959 struct ut_sequence ut_seq; 2960 unsigned char buf[4096], encrypted[4096] = {}, data[4096], tmp[3][4096]; 2961 struct iovec src_iovs[4], dst_iovs[4]; 2962 int rc, completed = 0; 2963 size_t i; 2964 2965 ioch = spdk_accel_get_io_channel(); 2966 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2967 2968 rc = spdk_accel_crypto_key_create(&key_params); 2969 CU_ASSERT_EQUAL(rc, 0); 2970 key = spdk_accel_crypto_key_get(key_params.key_name); 2971 SPDK_CU_ASSERT_FATAL(key != NULL); 2972 2973 for (i = 0; i < sizeof(data); ++i) { 2974 data[i] = (uint8_t)i & 0xff; 2975 } 2976 2977 dst_iovs[0].iov_base = encrypted; 2978 dst_iovs[0].iov_len = sizeof(encrypted); 2979 src_iovs[0].iov_base = data; 2980 src_iovs[0].iov_len = sizeof(data); 2981 rc = spdk_accel_submit_encrypt(ioch, key, &dst_iovs[0], 1, &src_iovs[0], 1, 0, 4096, 0, 2982 ut_encrypt_cb, &completed); 2983 CU_ASSERT_EQUAL(rc, 0); 2984 2985 while (!completed) { 2986 poll_threads(); 2987 } 2988 2989 /* Verify that encryption operation in a sequence produces the same result */ 2990 seq = NULL; 2991 completed = 0; 2992 2993 dst_iovs[0].iov_base = tmp[0]; 2994 dst_iovs[0].iov_len = sizeof(tmp[0]); 2995 src_iovs[0].iov_base = data; 2996 src_iovs[0].iov_len = sizeof(data); 2997 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2998 &src_iovs[0], 1, NULL, NULL, 0, 2999 ut_sequence_step_cb, &completed); 3000 CU_ASSERT_EQUAL(rc, 0); 3001 3002 dst_iovs[1].iov_base = tmp[1]; 3003 dst_iovs[1].iov_len = sizeof(tmp[1]); 3004 src_iovs[1].iov_base = tmp[0]; 3005 src_iovs[1].iov_len = sizeof(tmp[0]); 3006 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3007 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3008 ut_sequence_step_cb, &completed); 3009 CU_ASSERT_EQUAL(rc, 0); 3010 3011 dst_iovs[2].iov_base = buf; 3012 dst_iovs[2].iov_len = sizeof(buf); 3013 src_iovs[2].iov_base = tmp[1]; 3014 src_iovs[2].iov_len = sizeof(tmp[1]); 3015 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3016 &src_iovs[2], 1, NULL, NULL, 0, 3017 ut_sequence_step_cb, &completed); 3018 CU_ASSERT_EQUAL(rc, 0); 3019 3020 ut_seq.complete = false; 3021 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3022 CU_ASSERT_EQUAL(rc, 0); 3023 3024 poll_threads(); 3025 3026 CU_ASSERT_EQUAL(completed, 3); 3027 CU_ASSERT(ut_seq.complete); 3028 CU_ASSERT_EQUAL(ut_seq.status, 0); 3029 CU_ASSERT_EQUAL(memcmp(buf, encrypted, sizeof(buf)), 0); 3030 3031 /* Check that decryption produces the original buffer */ 3032 seq = NULL; 3033 completed = 0; 3034 memset(buf, 0, sizeof(buf)); 3035 3036 dst_iovs[0].iov_base = tmp[0]; 3037 dst_iovs[0].iov_len = sizeof(tmp[0]); 3038 src_iovs[0].iov_base = encrypted; 3039 src_iovs[0].iov_len = sizeof(encrypted); 3040 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3041 &src_iovs[0], 1, NULL, NULL, 0, 3042 ut_sequence_step_cb, &completed); 3043 CU_ASSERT_EQUAL(rc, 0); 3044 3045 dst_iovs[1].iov_base = tmp[1]; 3046 dst_iovs[1].iov_len = sizeof(tmp[1]); 3047 src_iovs[1].iov_base = tmp[0]; 3048 src_iovs[1].iov_len = sizeof(tmp[0]); 3049 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3050 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3051 ut_sequence_step_cb, &completed); 3052 CU_ASSERT_EQUAL(rc, 0); 3053 3054 dst_iovs[2].iov_base = buf; 3055 dst_iovs[2].iov_len = sizeof(buf); 3056 src_iovs[2].iov_base = tmp[1]; 3057 src_iovs[2].iov_len = sizeof(tmp[1]); 3058 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3059 &src_iovs[2], 1, NULL, NULL, 0, 3060 ut_sequence_step_cb, &completed); 3061 CU_ASSERT_EQUAL(rc, 0); 3062 3063 ut_seq.complete = false; 3064 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3065 CU_ASSERT_EQUAL(rc, 0); 3066 3067 poll_threads(); 3068 3069 CU_ASSERT_EQUAL(completed, 3); 3070 CU_ASSERT(ut_seq.complete); 3071 CU_ASSERT_EQUAL(ut_seq.status, 0); 3072 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3073 3074 /* Check encrypt + decrypt in a single sequence */ 3075 seq = NULL; 3076 completed = 0; 3077 memset(buf, 0, sizeof(buf)); 3078 3079 dst_iovs[0].iov_base = tmp[0]; 3080 dst_iovs[0].iov_len = sizeof(tmp[0]); 3081 src_iovs[0].iov_base = data; 3082 src_iovs[0].iov_len = sizeof(data); 3083 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3084 &src_iovs[0], 1, NULL, NULL, 0, 3085 ut_sequence_step_cb, &completed); 3086 CU_ASSERT_EQUAL(rc, 0); 3087 3088 dst_iovs[1].iov_base = tmp[1]; 3089 dst_iovs[1].iov_len = sizeof(tmp[1]); 3090 src_iovs[1].iov_base = tmp[0]; 3091 src_iovs[1].iov_len = sizeof(tmp[0]); 3092 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3093 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3094 ut_sequence_step_cb, &completed); 3095 CU_ASSERT_EQUAL(rc, 0); 3096 3097 3098 dst_iovs[2].iov_base = tmp[2]; 3099 dst_iovs[2].iov_len = sizeof(tmp[2]); 3100 src_iovs[2].iov_base = tmp[1]; 3101 src_iovs[2].iov_len = sizeof(tmp[1]); 3102 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[2], 1, NULL, NULL, 3103 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3104 ut_sequence_step_cb, &completed); 3105 CU_ASSERT_EQUAL(rc, 0); 3106 3107 dst_iovs[3].iov_base = buf; 3108 dst_iovs[3].iov_len = sizeof(buf); 3109 src_iovs[3].iov_base = tmp[2]; 3110 src_iovs[3].iov_len = sizeof(tmp[2]); 3111 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 3112 &src_iovs[3], 1, NULL, NULL, 0, 3113 ut_sequence_step_cb, &completed); 3114 CU_ASSERT_EQUAL(rc, 0); 3115 3116 ut_seq.complete = false; 3117 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3118 CU_ASSERT_EQUAL(rc, 0); 3119 3120 poll_threads(); 3121 3122 CU_ASSERT_EQUAL(completed, 4); 3123 CU_ASSERT(ut_seq.complete); 3124 CU_ASSERT_EQUAL(ut_seq.status, 0); 3125 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3126 3127 rc = spdk_accel_crypto_key_destroy(key); 3128 CU_ASSERT_EQUAL(rc, 0); 3129 spdk_put_io_channel(ioch); 3130 poll_threads(); 3131 } 3132 #endif /* SPDK_CONFIG_ISAL_CRYPTO */ 3133 3134 static int 3135 ut_submit_crypto(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3136 { 3137 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3138 3139 spdk_accel_task_complete(task, 0); 3140 3141 return 0; 3142 } 3143 3144 struct ut_driver_operation { 3145 int complete_status; 3146 int submit_status; 3147 int count; 3148 bool supported; 3149 }; 3150 3151 static struct ut_driver_operation g_drv_operations[ACCEL_OPC_LAST]; 3152 3153 static int 3154 ut_driver_execute_sequence(struct spdk_accel_sequence *seq) 3155 { 3156 struct spdk_accel_task *task; 3157 struct ut_driver_operation *drv_ops; 3158 3159 while ((task = spdk_accel_sequence_first_task(seq)) != NULL) { 3160 drv_ops = &g_drv_operations[task->op_code]; 3161 if (!drv_ops->supported) { 3162 break; 3163 } 3164 3165 drv_ops->count++; 3166 if (drv_ops->submit_status != 0) { 3167 return drv_ops->submit_status; 3168 } 3169 3170 if (drv_ops->complete_status != 0) { 3171 spdk_accel_task_complete(task, drv_ops->complete_status); 3172 break; 3173 } 3174 3175 switch (task->op_code) { 3176 case ACCEL_OPC_DECOMPRESS: 3177 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3178 break; 3179 case ACCEL_OPC_FILL: 3180 spdk_iov_memset(task->d.iovs, task->d.iovcnt, 3181 (int)(task->fill_pattern & 0xff)); 3182 break; 3183 default: 3184 CU_ASSERT(0 && "unexpected opcode"); 3185 break; 3186 } 3187 3188 spdk_accel_task_complete(task, 0); 3189 } 3190 3191 spdk_accel_sequence_continue(seq); 3192 3193 return 0; 3194 } 3195 3196 static struct spdk_accel_driver g_ut_driver = { 3197 .name = "ut", 3198 .execute_sequence = ut_driver_execute_sequence, 3199 }; 3200 3201 SPDK_ACCEL_DRIVER_REGISTER(ut, &g_ut_driver); 3202 3203 static void 3204 test_sequence_driver(void) 3205 { 3206 struct spdk_accel_sequence *seq = NULL; 3207 struct spdk_io_channel *ioch; 3208 struct spdk_accel_crypto_key key = {}; 3209 struct ut_sequence ut_seq; 3210 struct accel_module modules[ACCEL_OPC_LAST]; 3211 char buf[4096], tmp[3][4096], expected[4096]; 3212 struct iovec src_iovs[3], dst_iovs[3]; 3213 int i, rc, completed = 0; 3214 3215 ioch = spdk_accel_get_io_channel(); 3216 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3217 rc = spdk_accel_set_driver("ut"); 3218 SPDK_CU_ASSERT_FATAL(rc == 0); 3219 3220 /* Override the submit_tasks function */ 3221 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3222 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3223 modules[i] = g_modules_opc[i]; 3224 g_modules_opc[i] = g_module; 3225 } 3226 /* Intercept crypto operations, as they should be executed by an accel module */ 3227 g_seq_operations[ACCEL_OPC_ENCRYPT].submit = ut_submit_crypto; 3228 g_seq_operations[ACCEL_OPC_DECRYPT].submit = ut_submit_crypto; 3229 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3230 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3231 g_seq_operations[ACCEL_OPC_FILL].count = 0; 3232 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3233 3234 g_drv_operations[ACCEL_OPC_FILL].supported = true; 3235 g_drv_operations[ACCEL_OPC_DECOMPRESS].supported = true; 3236 3237 /* First check a sequence that is fully executed using a driver, with the copy at the end 3238 * being removed */ 3239 seq = NULL; 3240 completed = 0; 3241 memset(buf, 0, sizeof(buf)); 3242 memset(tmp[0], 0, sizeof(tmp[0])); 3243 memset(tmp[1], 0, sizeof(tmp[1])); 3244 memset(&expected[0], 0xa5, 2048); 3245 memset(&expected[2048], 0xbe, 2048); 3246 3247 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], 2048, NULL, NULL, 0xa5, 0, 3248 ut_sequence_step_cb, &completed); 3249 CU_ASSERT_EQUAL(rc, 0); 3250 rc = spdk_accel_append_fill(&seq, ioch, &tmp[0][2048], 2048, NULL, NULL, 0xbe, 0, 3251 ut_sequence_step_cb, &completed); 3252 CU_ASSERT_EQUAL(rc, 0); 3253 3254 dst_iovs[0].iov_base = tmp[1]; 3255 dst_iovs[0].iov_len = sizeof(tmp[1]); 3256 src_iovs[0].iov_base = tmp[0]; 3257 src_iovs[0].iov_len = sizeof(tmp[0]); 3258 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3259 &src_iovs[0], 1, NULL, NULL, 0, 3260 ut_sequence_step_cb, &completed); 3261 CU_ASSERT_EQUAL(rc, 0); 3262 3263 dst_iovs[1].iov_base = buf; 3264 dst_iovs[1].iov_len = sizeof(buf); 3265 src_iovs[1].iov_base = tmp[1]; 3266 src_iovs[1].iov_len = sizeof(tmp[1]); 3267 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3268 &src_iovs[1], 1, NULL, NULL, 0, 3269 ut_sequence_step_cb, &completed); 3270 CU_ASSERT_EQUAL(rc, 0); 3271 3272 ut_seq.complete = false; 3273 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3274 CU_ASSERT_EQUAL(rc, 0); 3275 3276 poll_threads(); 3277 3278 CU_ASSERT_EQUAL(completed, 4); 3279 CU_ASSERT(ut_seq.complete); 3280 CU_ASSERT_EQUAL(ut_seq.status, 0); 3281 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3282 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3283 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 3284 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 2); 3285 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3286 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3287 3288 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3289 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3290 3291 /* Check a sequence when the first two operations are executed by a driver, while the rest 3292 * is executed via modules */ 3293 seq = NULL; 3294 completed = 0; 3295 memset(buf, 0, sizeof(buf)); 3296 memset(tmp[0], 0, sizeof(tmp[0])); 3297 memset(tmp[1], 0, sizeof(tmp[1])); 3298 memset(tmp[2], 0, sizeof(tmp[2])); 3299 memset(&expected[0], 0xfe, 4096); 3300 3301 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 0, 3302 ut_sequence_step_cb, &completed); 3303 CU_ASSERT_EQUAL(rc, 0); 3304 3305 dst_iovs[0].iov_base = tmp[1]; 3306 dst_iovs[0].iov_len = sizeof(tmp[1]); 3307 src_iovs[0].iov_base = tmp[0]; 3308 src_iovs[0].iov_len = sizeof(tmp[0]); 3309 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3310 &src_iovs[0], 1, NULL, NULL, 0, 3311 ut_sequence_step_cb, &completed); 3312 CU_ASSERT_EQUAL(rc, 0); 3313 3314 dst_iovs[1].iov_base = tmp[2]; 3315 dst_iovs[1].iov_len = sizeof(tmp[2]); 3316 src_iovs[1].iov_base = tmp[1]; 3317 src_iovs[1].iov_len = sizeof(tmp[1]); 3318 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3319 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3320 ut_sequence_step_cb, &completed); 3321 CU_ASSERT_EQUAL(rc, 0); 3322 3323 dst_iovs[2].iov_base = buf; 3324 dst_iovs[2].iov_len = sizeof(buf); 3325 src_iovs[2].iov_base = tmp[2]; 3326 src_iovs[2].iov_len = sizeof(tmp[2]); 3327 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3328 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3329 ut_sequence_step_cb, &completed); 3330 CU_ASSERT_EQUAL(rc, 0); 3331 3332 ut_seq.complete = false; 3333 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3334 CU_ASSERT_EQUAL(rc, 0); 3335 3336 poll_threads(); 3337 3338 CU_ASSERT_EQUAL(completed, 4); 3339 CU_ASSERT(ut_seq.complete); 3340 CU_ASSERT_EQUAL(ut_seq.status, 0); 3341 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3342 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3343 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3344 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3345 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3346 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3347 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3348 3349 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3350 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3351 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3352 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3353 3354 /* Check sequence when the first and last operations are executed through modules, while the 3355 * ones in the middle are executed by the driver */ 3356 seq = NULL; 3357 completed = 0; 3358 memset(buf, 0, sizeof(buf)); 3359 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3360 memset(tmp[1], 0, sizeof(tmp[1])); 3361 memset(tmp[2], 0, sizeof(tmp[2])); 3362 memset(&expected[0], 0xfe, 2048); 3363 memset(&expected[2048], 0xa5, 2048); 3364 3365 dst_iovs[0].iov_base = tmp[1]; 3366 dst_iovs[0].iov_len = sizeof(tmp[1]); 3367 src_iovs[0].iov_base = tmp[0]; 3368 src_iovs[0].iov_len = sizeof(tmp[0]); 3369 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3370 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3371 ut_sequence_step_cb, &completed); 3372 CU_ASSERT_EQUAL(rc, 0); 3373 3374 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xfe, 0, 3375 ut_sequence_step_cb, &completed); 3376 CU_ASSERT_EQUAL(rc, 0); 3377 3378 dst_iovs[1].iov_base = tmp[2]; 3379 dst_iovs[1].iov_len = sizeof(tmp[2]); 3380 src_iovs[1].iov_base = tmp[1]; 3381 src_iovs[1].iov_len = sizeof(tmp[1]); 3382 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3383 &src_iovs[1], 1, NULL, NULL, 0, 3384 ut_sequence_step_cb, &completed); 3385 CU_ASSERT_EQUAL(rc, 0); 3386 3387 dst_iovs[2].iov_base = buf; 3388 dst_iovs[2].iov_len = sizeof(buf); 3389 src_iovs[2].iov_base = tmp[2]; 3390 src_iovs[2].iov_len = sizeof(tmp[2]); 3391 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3392 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3393 ut_sequence_step_cb, &completed); 3394 CU_ASSERT_EQUAL(rc, 0); 3395 3396 ut_seq.complete = false; 3397 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3398 CU_ASSERT_EQUAL(rc, 0); 3399 3400 poll_threads(); 3401 3402 CU_ASSERT_EQUAL(completed, 4); 3403 CU_ASSERT(ut_seq.complete); 3404 CU_ASSERT_EQUAL(ut_seq.status, 0); 3405 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3406 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3407 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3408 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3409 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3410 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3411 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3412 3413 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3414 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3415 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3416 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3417 3418 /* Check a sequence with operations executed by: module, driver, module, driver */ 3419 seq = NULL; 3420 completed = 0; 3421 memset(buf, 0, sizeof(buf)); 3422 memset(tmp[0], 0x5a, sizeof(tmp[0])); 3423 memset(tmp[1], 0, sizeof(tmp[1])); 3424 memset(tmp[2], 0, sizeof(tmp[2])); 3425 memset(&expected[0], 0xef, 2048); 3426 memset(&expected[2048], 0x5a, 2048); 3427 3428 dst_iovs[0].iov_base = tmp[1]; 3429 dst_iovs[0].iov_len = sizeof(tmp[1]); 3430 src_iovs[0].iov_base = tmp[0]; 3431 src_iovs[0].iov_len = sizeof(tmp[0]); 3432 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3433 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3434 ut_sequence_step_cb, &completed); 3435 CU_ASSERT_EQUAL(rc, 0); 3436 3437 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3438 ut_sequence_step_cb, &completed); 3439 CU_ASSERT_EQUAL(rc, 0); 3440 3441 dst_iovs[1].iov_base = tmp[2]; 3442 dst_iovs[1].iov_len = sizeof(tmp[2]); 3443 src_iovs[1].iov_base = tmp[1]; 3444 src_iovs[1].iov_len = sizeof(tmp[1]); 3445 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3446 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3447 ut_sequence_step_cb, &completed); 3448 CU_ASSERT_EQUAL(rc, 0); 3449 3450 dst_iovs[2].iov_base = buf; 3451 dst_iovs[2].iov_len = sizeof(buf); 3452 src_iovs[2].iov_base = tmp[2]; 3453 src_iovs[2].iov_len = sizeof(tmp[2]); 3454 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3455 &src_iovs[2], 1, NULL, NULL, 0, 3456 ut_sequence_step_cb, &completed); 3457 CU_ASSERT_EQUAL(rc, 0); 3458 3459 ut_seq.complete = false; 3460 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3461 CU_ASSERT_EQUAL(rc, 0); 3462 3463 poll_threads(); 3464 3465 CU_ASSERT_EQUAL(completed, 4); 3466 CU_ASSERT(ut_seq.complete); 3467 CU_ASSERT_EQUAL(ut_seq.status, 0); 3468 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3469 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3470 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3471 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3472 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3473 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3474 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3475 3476 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3477 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3478 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3479 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3480 3481 /* Check that an error returned from driver's execute_sequence() will fail the whole 3482 * sequence and any subsequent operations won't be processed */ 3483 seq = NULL; 3484 completed = 0; 3485 memset(buf, 0, sizeof(buf)); 3486 memset(expected, 0, sizeof(expected)); 3487 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3488 g_drv_operations[ACCEL_OPC_FILL].submit_status = -EPERM; 3489 3490 dst_iovs[0].iov_base = tmp[1]; 3491 dst_iovs[0].iov_len = sizeof(tmp[1]); 3492 src_iovs[0].iov_base = tmp[0]; 3493 src_iovs[0].iov_len = sizeof(tmp[0]); 3494 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3495 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3496 ut_sequence_step_cb, &completed); 3497 CU_ASSERT_EQUAL(rc, 0); 3498 3499 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3500 ut_sequence_step_cb, &completed); 3501 CU_ASSERT_EQUAL(rc, 0); 3502 3503 dst_iovs[1].iov_base = buf; 3504 dst_iovs[1].iov_len = sizeof(buf); 3505 src_iovs[1].iov_base = tmp[1]; 3506 src_iovs[1].iov_len = sizeof(tmp[1]); 3507 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3508 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3509 ut_sequence_step_cb, &completed); 3510 CU_ASSERT_EQUAL(rc, 0); 3511 3512 ut_seq.complete = false; 3513 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3514 CU_ASSERT_EQUAL(rc, 0); 3515 3516 poll_threads(); 3517 3518 CU_ASSERT_EQUAL(completed, 3); 3519 CU_ASSERT(ut_seq.complete); 3520 CU_ASSERT_EQUAL(ut_seq.status, -EPERM); 3521 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3522 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3523 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 0); 3524 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3525 CU_ASSERT_EQUAL(memcmp(buf, expected, 4096), 0); 3526 3527 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3528 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3529 g_drv_operations[ACCEL_OPC_FILL].submit_status = 0; 3530 3531 /* Check that a failed task completed by a driver will cause the whole sequence to be failed 3532 * and any subsequent operations won't be processed */ 3533 seq = NULL; 3534 completed = 0; 3535 memset(buf, 0, sizeof(buf)); 3536 memset(expected, 0, sizeof(expected)); 3537 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3538 g_drv_operations[ACCEL_OPC_FILL].complete_status = -ENOENT; 3539 3540 dst_iovs[0].iov_base = tmp[1]; 3541 dst_iovs[0].iov_len = sizeof(tmp[1]); 3542 src_iovs[0].iov_base = tmp[0]; 3543 src_iovs[0].iov_len = sizeof(tmp[0]); 3544 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3545 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3546 ut_sequence_step_cb, &completed); 3547 CU_ASSERT_EQUAL(rc, 0); 3548 3549 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3550 ut_sequence_step_cb, &completed); 3551 CU_ASSERT_EQUAL(rc, 0); 3552 3553 dst_iovs[1].iov_base = buf; 3554 dst_iovs[1].iov_len = sizeof(buf); 3555 src_iovs[1].iov_base = tmp[1]; 3556 src_iovs[1].iov_len = sizeof(tmp[1]); 3557 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3558 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3559 ut_sequence_step_cb, &completed); 3560 CU_ASSERT_EQUAL(rc, 0); 3561 3562 ut_seq.complete = false; 3563 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3564 CU_ASSERT_EQUAL(rc, 0); 3565 3566 poll_threads(); 3567 3568 CU_ASSERT_EQUAL(completed, 3); 3569 CU_ASSERT(ut_seq.complete); 3570 CU_ASSERT_EQUAL(ut_seq.status, -ENOENT); 3571 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3572 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3573 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 0); 3574 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3575 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3576 3577 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3578 g_modules_opc[i] = modules[i]; 3579 } 3580 3581 /* Clear the driver so that other tests won't use it */ 3582 g_accel_driver = NULL; 3583 memset(&g_drv_operations, 0, sizeof(g_drv_operations)); 3584 3585 ut_clear_operations(); 3586 spdk_put_io_channel(ioch); 3587 poll_threads(); 3588 } 3589 3590 struct ut_saved_iovs { 3591 struct iovec src; 3592 struct iovec dst; 3593 }; 3594 3595 static struct ut_saved_iovs g_seq_saved_iovs[ACCEL_OPC_LAST]; 3596 3597 static int 3598 ut_submit_save_iovs(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3599 { 3600 SPDK_CU_ASSERT_FATAL(task->s.iovcnt == 1); 3601 SPDK_CU_ASSERT_FATAL(task->d.iovcnt == 1); 3602 3603 g_seq_saved_iovs[task->op_code].src = task->s.iovs[0]; 3604 g_seq_saved_iovs[task->op_code].dst = task->d.iovs[0]; 3605 3606 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3607 3608 spdk_accel_task_complete(task, 0); 3609 3610 return 0; 3611 } 3612 3613 static void 3614 test_sequence_same_iovs(void) 3615 { 3616 struct spdk_accel_sequence *seq = NULL; 3617 struct spdk_io_channel *ioch; 3618 struct spdk_accel_crypto_key key = {}; 3619 struct ut_sequence ut_seq; 3620 struct accel_module modules[ACCEL_OPC_LAST]; 3621 char buf[4096], tmp[4096], expected[4096]; 3622 struct iovec iovs[3], expected_siov, expected_diov; 3623 struct spdk_memory_domain *domain; 3624 void *accel_buf, *domain_ctx; 3625 int i, rc, completed = 0; 3626 3627 ioch = spdk_accel_get_io_channel(); 3628 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3629 3630 /* Override the submit_tasks function */ 3631 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3632 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3633 modules[i] = g_modules_opc[i]; 3634 g_modules_opc[i] = g_module; 3635 } 3636 /* Intercept crypto operations, as they should be executed by an accel module */ 3637 g_seq_operations[ACCEL_OPC_ENCRYPT].submit = ut_submit_save_iovs; 3638 g_seq_operations[ACCEL_OPC_DECRYPT].submit = ut_submit_save_iovs; 3639 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3640 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3641 3642 /* Check that it's possible to use the same iovec ptr for different operations */ 3643 seq = NULL; 3644 completed = 0; 3645 memset(buf, 0, sizeof(buf)); 3646 memset(expected, 0xa5, sizeof(expected)); 3647 3648 iovs[0].iov_base = expected; 3649 iovs[0].iov_len = sizeof(expected); 3650 iovs[1].iov_base = tmp; 3651 iovs[1].iov_len = sizeof(tmp); 3652 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, NULL, NULL, 3653 &iovs[0], 1, NULL, NULL, 0, 4096, 0, 3654 ut_sequence_step_cb, &completed); 3655 CU_ASSERT_EQUAL(rc, 0); 3656 /* Reuse iov[1] as src */ 3657 iovs[2].iov_base = buf; 3658 iovs[2].iov_len = sizeof(buf); 3659 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3660 &iovs[1], 1, NULL, NULL, 0, 4096, 0, 3661 ut_sequence_step_cb, &completed); 3662 CU_ASSERT_EQUAL(rc, 0); 3663 3664 ut_seq.complete = false; 3665 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3666 CU_ASSERT_EQUAL(rc, 0); 3667 3668 poll_threads(); 3669 3670 CU_ASSERT_EQUAL(completed, 2); 3671 CU_ASSERT(ut_seq.complete); 3672 CU_ASSERT_EQUAL(ut_seq.status, 0); 3673 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3674 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3675 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3676 expected_siov.iov_base = expected; 3677 expected_siov.iov_len = sizeof(expected); 3678 expected_diov.iov_base = tmp; 3679 expected_diov.iov_len = sizeof(tmp); 3680 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_ENCRYPT].src, 3681 &expected_siov, sizeof(expected_siov)), 0); 3682 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_ENCRYPT].dst, 3683 &expected_diov, sizeof(expected_diov)), 0); 3684 expected_siov.iov_base = tmp; 3685 expected_siov.iov_len = sizeof(tmp); 3686 expected_diov.iov_base = buf; 3687 expected_diov.iov_len = sizeof(buf); 3688 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_DECRYPT].src, 3689 &expected_siov, sizeof(expected_siov)), 0); 3690 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_DECRYPT].dst, 3691 &expected_diov, sizeof(expected_diov)), 0); 3692 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3693 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3694 3695 /* Check the same with an accel buffer */ 3696 seq = NULL; 3697 completed = 0; 3698 memset(buf, 0, sizeof(buf)); 3699 memset(expected, 0x5a, sizeof(expected)); 3700 3701 rc = spdk_accel_get_buf(ioch, sizeof(buf), &accel_buf, &domain, &domain_ctx); 3702 CU_ASSERT_EQUAL(rc, 0); 3703 3704 iovs[0].iov_base = expected; 3705 iovs[0].iov_len = sizeof(expected); 3706 iovs[1].iov_base = accel_buf; 3707 iovs[1].iov_len = sizeof(buf); 3708 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, domain, domain_ctx, 3709 &iovs[0], 1, NULL, NULL, 0, 4096, 0, 3710 ut_sequence_step_cb, &completed); 3711 CU_ASSERT_EQUAL(rc, 0); 3712 /* Reuse iov[1] as src */ 3713 iovs[2].iov_base = buf; 3714 iovs[2].iov_len = sizeof(buf); 3715 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3716 &iovs[1], 1, domain, domain_ctx, 0, 4096, 0, 3717 ut_sequence_step_cb, &completed); 3718 CU_ASSERT_EQUAL(rc, 0); 3719 3720 ut_seq.complete = false; 3721 rc = spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3722 CU_ASSERT_EQUAL(rc, 0); 3723 3724 poll_threads(); 3725 3726 CU_ASSERT_EQUAL(completed, 2); 3727 CU_ASSERT(ut_seq.complete); 3728 CU_ASSERT_EQUAL(ut_seq.status, 0); 3729 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3730 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3731 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3732 expected_siov.iov_base = expected; 3733 expected_siov.iov_len = sizeof(expected); 3734 expected_diov.iov_base = buf; 3735 expected_diov.iov_len = sizeof(buf); 3736 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_ENCRYPT].src, 3737 &expected_siov, sizeof(expected_siov)), 0); 3738 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_DECRYPT].dst, 3739 &expected_diov, sizeof(expected_diov)), 0); 3740 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_ENCRYPT].dst, 3741 &g_seq_saved_iovs[ACCEL_OPC_DECRYPT].src, 3742 sizeof(struct iovec)), 0); 3743 spdk_accel_put_buf(ioch, accel_buf, domain, domain_ctx); 3744 3745 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3746 g_modules_opc[i] = modules[i]; 3747 } 3748 3749 ut_clear_operations(); 3750 spdk_put_io_channel(ioch); 3751 poll_threads(); 3752 } 3753 3754 static int 3755 test_sequence_setup(void) 3756 { 3757 int rc; 3758 3759 allocate_cores(1); 3760 allocate_threads(1); 3761 set_thread(0); 3762 3763 rc = spdk_iobuf_initialize(); 3764 if (rc != 0) { 3765 CU_ASSERT(false); 3766 return -1; 3767 } 3768 3769 rc = spdk_accel_initialize(); 3770 if (rc != 0) { 3771 CU_ASSERT(false); 3772 return -1; 3773 } 3774 3775 return 0; 3776 } 3777 3778 static void 3779 finish_cb(void *cb_arg) 3780 { 3781 bool *done = cb_arg; 3782 3783 *done = true; 3784 } 3785 3786 static int 3787 test_sequence_cleanup(void) 3788 { 3789 bool done = false; 3790 3791 spdk_accel_finish(finish_cb, &done); 3792 3793 while (!done) { 3794 poll_threads(); 3795 } 3796 3797 done = false; 3798 spdk_iobuf_finish(finish_cb, &done); 3799 while (!done) { 3800 poll_threads(); 3801 } 3802 3803 free_threads(); 3804 free_cores(); 3805 3806 return 0; 3807 } 3808 3809 int 3810 main(int argc, char **argv) 3811 { 3812 CU_pSuite suite = NULL, seq_suite; 3813 unsigned int num_failures; 3814 3815 CU_set_error_action(CUEA_ABORT); 3816 CU_initialize_registry(); 3817 3818 /* Sequence tests require accel to be initialized normally, so run them before the other 3819 * tests which register accel modules which aren't fully implemented, causing accel 3820 * initialization to fail. 3821 */ 3822 seq_suite = CU_add_suite("accel_sequence", test_sequence_setup, test_sequence_cleanup); 3823 CU_ADD_TEST(seq_suite, test_sequence_fill_copy); 3824 CU_ADD_TEST(seq_suite, test_sequence_abort); 3825 CU_ADD_TEST(seq_suite, test_sequence_append_error); 3826 CU_ADD_TEST(seq_suite, test_sequence_completion_error); 3827 #ifdef SPDK_CONFIG_ISAL /* accel_sw requires isa-l for compression */ 3828 CU_ADD_TEST(seq_suite, test_sequence_decompress); 3829 CU_ADD_TEST(seq_suite, test_sequence_reverse); 3830 #endif 3831 CU_ADD_TEST(seq_suite, test_sequence_copy_elision); 3832 CU_ADD_TEST(seq_suite, test_sequence_accel_buffers); 3833 CU_ADD_TEST(seq_suite, test_sequence_memory_domain); 3834 CU_ADD_TEST(seq_suite, test_sequence_module_memory_domain); 3835 #ifdef SPDK_CONFIG_ISAL_CRYPTO /* accel_sw requires isa-l-crypto for crypto operations */ 3836 CU_ADD_TEST(seq_suite, test_sequence_crypto); 3837 #endif 3838 CU_ADD_TEST(seq_suite, test_sequence_driver); 3839 CU_ADD_TEST(seq_suite, test_sequence_same_iovs); 3840 3841 suite = CU_add_suite("accel", test_setup, test_cleanup); 3842 CU_ADD_TEST(suite, test_spdk_accel_task_complete); 3843 CU_ADD_TEST(suite, test_get_task); 3844 CU_ADD_TEST(suite, test_spdk_accel_submit_copy); 3845 CU_ADD_TEST(suite, test_spdk_accel_submit_dualcast); 3846 CU_ADD_TEST(suite, test_spdk_accel_submit_compare); 3847 CU_ADD_TEST(suite, test_spdk_accel_submit_fill); 3848 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c); 3849 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32cv); 3850 CU_ADD_TEST(suite, test_spdk_accel_submit_copy_crc32c); 3851 CU_ADD_TEST(suite, test_spdk_accel_submit_xor); 3852 CU_ADD_TEST(suite, test_spdk_accel_module_find_by_name); 3853 CU_ADD_TEST(suite, test_spdk_accel_module_register); 3854 3855 CU_basic_set_mode(CU_BRM_VERBOSE); 3856 CU_basic_run_tests(); 3857 num_failures = CU_get_number_of_failures(); 3858 CU_cleanup_registry(); 3859 3860 return num_failures; 3861 } 3862