1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2021 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk_internal/cunit.h" 8 #include "spdk_internal/mock.h" 9 #include "spdk/accel_module.h" 10 #include "thread/thread_internal.h" 11 #include "common/lib/ut_multithread.c" 12 #include "common/lib/test_iobuf.c" 13 #include "accel/accel.c" 14 #include "accel/accel_sw.c" 15 #include "unit/lib/json_mock.c" 16 17 DEFINE_STUB_V(spdk_memory_domain_destroy, (struct spdk_memory_domain *domain)); 18 DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, 19 (struct spdk_memory_domain *domain), "UT_DMA"); 20 21 int 22 spdk_memory_domain_create(struct spdk_memory_domain **domain, enum spdk_dma_device_type type, 23 struct spdk_memory_domain_ctx *ctx, const char *id) 24 { 25 *domain = (void *)0xdeadbeef; 26 27 return 0; 28 } 29 30 struct ut_domain_ctx { 31 struct iovec iov; 32 struct iovec expected; 33 int pull_submit_status; 34 int push_submit_status; 35 int pull_complete_status; 36 int push_complete_status; 37 }; 38 39 static struct spdk_memory_domain *g_ut_domain = (void *)0xa55e1; 40 41 int 42 spdk_memory_domain_pull_data(struct spdk_memory_domain *sd, void *sctx, struct iovec *siov, 43 uint32_t siovcnt, struct iovec *diov, uint32_t diovcnt, 44 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 45 { 46 struct ut_domain_ctx *ctx = sctx; 47 48 CU_ASSERT_EQUAL(sd, g_ut_domain); 49 CU_ASSERT_EQUAL(siovcnt, 1); 50 CU_ASSERT_EQUAL(memcmp(siov, &ctx->expected, sizeof(*siov)), 0); 51 52 if (ctx->pull_submit_status != 0) { 53 return ctx->pull_submit_status; 54 } 55 56 if (ctx->pull_complete_status != 0) { 57 cpl_cb(cpl_arg, ctx->pull_complete_status); 58 return 0; 59 } 60 61 spdk_iovcpy(&ctx->iov, 1, diov, diovcnt); 62 cpl_cb(cpl_arg, 0); 63 64 return 0; 65 } 66 67 int 68 spdk_memory_domain_push_data(struct spdk_memory_domain *dd, void *dctx, struct iovec *diov, 69 uint32_t diovcnt, struct iovec *siov, uint32_t siovcnt, 70 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 71 { 72 struct ut_domain_ctx *ctx = dctx; 73 74 CU_ASSERT_EQUAL(dd, g_ut_domain); 75 CU_ASSERT_EQUAL(diovcnt, 1); 76 CU_ASSERT_EQUAL(memcmp(diov, &ctx->expected, sizeof(*diov)), 0); 77 78 if (ctx->push_submit_status != 0) { 79 return ctx->push_submit_status; 80 } 81 82 if (ctx->push_complete_status != 0) { 83 cpl_cb(cpl_arg, ctx->push_complete_status); 84 return 0; 85 } 86 87 spdk_iovcpy(siov, siovcnt, &ctx->iov, 1); 88 cpl_cb(cpl_arg, 0); 89 90 return 0; 91 } 92 93 /* global vars and setup/cleanup functions used for all test functions */ 94 struct spdk_accel_module_if g_module_if = {}; 95 struct accel_module g_module = { .module = &g_module_if }; 96 struct spdk_io_channel *g_ch = NULL; 97 struct accel_io_channel *g_accel_ch = NULL; 98 struct sw_accel_io_channel *g_sw_ch = NULL; 99 struct spdk_io_channel *g_module_ch = NULL; 100 101 static uint64_t g_opc_mask = 0; 102 103 static uint64_t 104 _accel_op_to_bit(enum accel_opcode opc) 105 { 106 return (1 << opc); 107 } 108 109 static bool 110 _supports_opcode(enum accel_opcode opc) 111 { 112 if (_accel_op_to_bit(opc) & g_opc_mask) { 113 return true; 114 } 115 return false; 116 } 117 118 static int 119 test_setup(void) 120 { 121 int i; 122 123 g_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct accel_io_channel)); 124 if (g_ch == NULL) { 125 /* for some reason the assert fatal macro doesn't work in the setup function. */ 126 CU_ASSERT(false); 127 return -1; 128 } 129 g_accel_ch = (struct accel_io_channel *)((char *)g_ch + sizeof(struct spdk_io_channel)); 130 g_module_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct sw_accel_io_channel)); 131 if (g_module_ch == NULL) { 132 CU_ASSERT(false); 133 return -1; 134 } 135 136 g_module_if.submit_tasks = sw_accel_submit_tasks; 137 g_module_if.name = "software"; 138 for (i = 0; i < ACCEL_OPC_LAST; i++) { 139 g_accel_ch->module_ch[i] = g_module_ch; 140 g_modules_opc[i] = g_module; 141 } 142 g_sw_ch = (struct sw_accel_io_channel *)((char *)g_module_ch + sizeof( 143 struct spdk_io_channel)); 144 TAILQ_INIT(&g_sw_ch->tasks_to_complete); 145 g_module_if.supports_opcode = _supports_opcode; 146 return 0; 147 } 148 149 static int 150 test_cleanup(void) 151 { 152 free(g_ch); 153 free(g_module_ch); 154 155 return 0; 156 } 157 158 #define DUMMY_ARG 0xDEADBEEF 159 static bool g_dummy_cb_called = false; 160 static void 161 dummy_cb_fn(void *cb_arg, int status) 162 { 163 CU_ASSERT(*(uint32_t *)cb_arg == DUMMY_ARG); 164 CU_ASSERT(status == 0); 165 g_dummy_cb_called = true; 166 } 167 168 static void 169 test_spdk_accel_task_complete(void) 170 { 171 struct spdk_accel_task accel_task = {}; 172 struct spdk_accel_task *expected_accel_task = NULL; 173 uint32_t cb_arg = DUMMY_ARG; 174 int status = 0; 175 176 accel_task.accel_ch = g_accel_ch; 177 accel_task.cb_fn = dummy_cb_fn; 178 accel_task.cb_arg = &cb_arg; 179 TAILQ_INIT(&g_accel_ch->task_pool); 180 181 /* Confirm cb is called and task added to list. */ 182 spdk_accel_task_complete(&accel_task, status); 183 CU_ASSERT(g_dummy_cb_called == true); 184 expected_accel_task = TAILQ_FIRST(&g_accel_ch->task_pool); 185 TAILQ_REMOVE(&g_accel_ch->task_pool, expected_accel_task, link); 186 CU_ASSERT(expected_accel_task == &accel_task); 187 } 188 189 static void 190 test_get_task(void) 191 { 192 struct spdk_accel_task *task; 193 struct spdk_accel_task _task; 194 void *cb_arg = NULL; 195 196 TAILQ_INIT(&g_accel_ch->task_pool); 197 198 /* no tasks left, return NULL. */ 199 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 200 CU_ASSERT(task == NULL); 201 202 _task.cb_fn = dummy_cb_fn; 203 _task.cb_arg = cb_arg; 204 _task.accel_ch = g_accel_ch; 205 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &_task, link); 206 207 /* Get a valid task. */ 208 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 209 CU_ASSERT(task == &_task); 210 CU_ASSERT(_task.cb_fn == dummy_cb_fn); 211 CU_ASSERT(_task.cb_arg == cb_arg); 212 CU_ASSERT(_task.accel_ch == g_accel_ch); 213 } 214 215 #define TEST_SUBMIT_SIZE 64 216 static void 217 test_spdk_accel_submit_copy(void) 218 { 219 const uint64_t nbytes = TEST_SUBMIT_SIZE; 220 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 221 uint8_t src[TEST_SUBMIT_SIZE] = {0}; 222 void *cb_arg = NULL; 223 int rc; 224 struct spdk_accel_task task; 225 struct spdk_accel_task *expected_accel_task = NULL; 226 int flags = 0; 227 228 TAILQ_INIT(&g_accel_ch->task_pool); 229 230 /* Fail with no tasks on _get_task() */ 231 rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, flags, NULL, cb_arg); 232 CU_ASSERT(rc == -ENOMEM); 233 234 task.accel_ch = g_accel_ch; 235 task.flags = 1; 236 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 237 238 /* submission OK. */ 239 rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, flags, NULL, cb_arg); 240 CU_ASSERT(rc == 0); 241 CU_ASSERT(task.op_code == ACCEL_OPC_COPY); 242 CU_ASSERT(task.flags == 0); 243 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 244 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 245 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 246 CU_ASSERT(expected_accel_task == &task); 247 } 248 249 static void 250 test_spdk_accel_submit_dualcast(void) 251 { 252 void *dst1; 253 void *dst2; 254 void *src; 255 uint32_t align = ALIGN_4K; 256 uint64_t nbytes = TEST_SUBMIT_SIZE; 257 void *cb_arg = NULL; 258 int rc; 259 struct spdk_accel_task task; 260 struct spdk_accel_task *expected_accel_task = NULL; 261 int flags = 0; 262 263 TAILQ_INIT(&g_accel_ch->task_pool); 264 265 /* Dualcast requires 4K alignment on dst addresses, 266 * hence using the hard coded address to test the buffer alignment 267 */ 268 dst1 = (void *)0x5000; 269 dst2 = (void *)0x60f0; 270 src = calloc(1, TEST_SUBMIT_SIZE); 271 SPDK_CU_ASSERT_FATAL(src != NULL); 272 memset(src, 0x5A, TEST_SUBMIT_SIZE); 273 274 /* This should fail since dst2 is not 4k aligned */ 275 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 276 CU_ASSERT(rc == -EINVAL); 277 278 dst1 = (void *)0x7010; 279 dst2 = (void *)0x6000; 280 /* This should fail since dst1 is not 4k aligned */ 281 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 282 CU_ASSERT(rc == -EINVAL); 283 284 /* Dualcast requires 4K alignment on dst addresses */ 285 dst1 = (void *)0x7000; 286 dst2 = (void *)0x6000; 287 /* Fail with no tasks on _get_task() */ 288 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 289 CU_ASSERT(rc == -ENOMEM); 290 291 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 292 293 /* accel submission OK., since we test the SW path , need to use valid memory addresses 294 * cannot hardcode them anymore */ 295 dst1 = spdk_dma_zmalloc(nbytes, align, NULL); 296 SPDK_CU_ASSERT_FATAL(dst1 != NULL); 297 dst2 = spdk_dma_zmalloc(nbytes, align, NULL); 298 SPDK_CU_ASSERT_FATAL(dst2 != NULL); 299 /* SW module does the dualcast. */ 300 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 301 CU_ASSERT(rc == 0); 302 CU_ASSERT(task.op_code == ACCEL_OPC_DUALCAST); 303 CU_ASSERT(task.flags == 0); 304 CU_ASSERT(memcmp(dst1, src, TEST_SUBMIT_SIZE) == 0); 305 CU_ASSERT(memcmp(dst2, src, TEST_SUBMIT_SIZE) == 0); 306 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 307 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 308 CU_ASSERT(expected_accel_task == &task); 309 310 free(src); 311 spdk_free(dst1); 312 spdk_free(dst2); 313 } 314 315 static void 316 test_spdk_accel_submit_compare(void) 317 { 318 void *src1; 319 void *src2; 320 uint64_t nbytes = TEST_SUBMIT_SIZE; 321 void *cb_arg = NULL; 322 int rc; 323 struct spdk_accel_task task; 324 struct spdk_accel_task *expected_accel_task = NULL; 325 326 TAILQ_INIT(&g_accel_ch->task_pool); 327 328 src1 = calloc(1, TEST_SUBMIT_SIZE); 329 SPDK_CU_ASSERT_FATAL(src1 != NULL); 330 src2 = calloc(1, TEST_SUBMIT_SIZE); 331 SPDK_CU_ASSERT_FATAL(src2 != NULL); 332 333 /* Fail with no tasks on _get_task() */ 334 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 335 CU_ASSERT(rc == -ENOMEM); 336 337 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 338 339 /* accel submission OK. */ 340 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 341 CU_ASSERT(rc == 0); 342 CU_ASSERT(task.op_code == ACCEL_OPC_COMPARE); 343 CU_ASSERT(memcmp(src1, src2, TEST_SUBMIT_SIZE) == 0); 344 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 345 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 346 CU_ASSERT(expected_accel_task == &task); 347 348 free(src1); 349 free(src2); 350 } 351 352 static void 353 test_spdk_accel_submit_fill(void) 354 { 355 void *dst; 356 void *src; 357 uint8_t fill = 0xf; 358 uint64_t fill64; 359 uint64_t nbytes = TEST_SUBMIT_SIZE; 360 void *cb_arg = NULL; 361 int rc; 362 struct spdk_accel_task task; 363 struct spdk_accel_task *expected_accel_task = NULL; 364 int flags = 0; 365 366 TAILQ_INIT(&g_accel_ch->task_pool); 367 368 dst = calloc(1, TEST_SUBMIT_SIZE); 369 SPDK_CU_ASSERT_FATAL(dst != NULL); 370 src = calloc(1, TEST_SUBMIT_SIZE); 371 SPDK_CU_ASSERT_FATAL(src != NULL); 372 memset(src, fill, TEST_SUBMIT_SIZE); 373 memset(&fill64, fill, sizeof(uint64_t)); 374 375 /* Fail with no tasks on _get_task() */ 376 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 377 CU_ASSERT(rc == -ENOMEM); 378 379 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 380 381 /* accel submission OK. */ 382 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 383 CU_ASSERT(rc == 0); 384 CU_ASSERT(task.fill_pattern == fill64); 385 CU_ASSERT(task.op_code == ACCEL_OPC_FILL); 386 CU_ASSERT(task.flags == 0); 387 388 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 389 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 390 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 391 CU_ASSERT(expected_accel_task == &task); 392 393 free(dst); 394 free(src); 395 } 396 397 static void 398 test_spdk_accel_submit_crc32c(void) 399 { 400 const uint64_t nbytes = TEST_SUBMIT_SIZE; 401 uint32_t crc_dst; 402 uint8_t src[TEST_SUBMIT_SIZE]; 403 uint32_t seed = 1; 404 void *cb_arg = NULL; 405 int rc; 406 struct spdk_accel_task task; 407 struct spdk_accel_task *expected_accel_task = NULL; 408 409 TAILQ_INIT(&g_accel_ch->task_pool); 410 411 /* Fail with no tasks on _get_task() */ 412 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 413 CU_ASSERT(rc == -ENOMEM); 414 415 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 416 417 /* accel submission OK. */ 418 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 419 CU_ASSERT(rc == 0); 420 CU_ASSERT(task.crc_dst == &crc_dst); 421 CU_ASSERT(task.seed == seed); 422 CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); 423 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 424 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 425 CU_ASSERT(expected_accel_task == &task); 426 } 427 428 static void 429 test_spdk_accel_submit_crc32cv(void) 430 { 431 uint32_t crc_dst; 432 uint32_t seed = 0; 433 uint32_t iov_cnt = 32; 434 void *cb_arg = NULL; 435 int rc; 436 uint32_t i = 0; 437 struct spdk_accel_task task; 438 struct iovec iov[32]; 439 struct spdk_accel_task *expected_accel_task = NULL; 440 441 TAILQ_INIT(&g_accel_ch->task_pool); 442 443 for (i = 0; i < iov_cnt; i++) { 444 iov[i].iov_base = calloc(1, TEST_SUBMIT_SIZE); 445 SPDK_CU_ASSERT_FATAL(iov[i].iov_base != NULL); 446 iov[i].iov_len = TEST_SUBMIT_SIZE; 447 } 448 449 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 450 451 /* accel submission OK. */ 452 rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, NULL, cb_arg); 453 CU_ASSERT(rc == 0); 454 CU_ASSERT(task.s.iovs == iov); 455 CU_ASSERT(task.s.iovcnt == iov_cnt); 456 CU_ASSERT(task.crc_dst == &crc_dst); 457 CU_ASSERT(task.seed == seed); 458 CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); 459 CU_ASSERT(task.cb_arg == cb_arg); 460 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 461 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 462 CU_ASSERT(expected_accel_task == &task); 463 464 for (i = 0; i < iov_cnt; i++) { 465 free(iov[i].iov_base); 466 } 467 } 468 469 static void 470 test_spdk_accel_submit_copy_crc32c(void) 471 { 472 const uint64_t nbytes = TEST_SUBMIT_SIZE; 473 uint32_t crc_dst; 474 uint8_t dst[TEST_SUBMIT_SIZE]; 475 uint8_t src[TEST_SUBMIT_SIZE]; 476 uint32_t seed = 0; 477 void *cb_arg = NULL; 478 int rc; 479 struct spdk_accel_task task; 480 struct spdk_accel_task *expected_accel_task = NULL; 481 int flags = 0; 482 483 TAILQ_INIT(&g_accel_ch->task_pool); 484 485 /* Fail with no tasks on _get_task() */ 486 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 487 NULL, cb_arg); 488 CU_ASSERT(rc == -ENOMEM); 489 490 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 491 492 /* accel submission OK. */ 493 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 494 NULL, cb_arg); 495 CU_ASSERT(rc == 0); 496 CU_ASSERT(task.crc_dst == &crc_dst); 497 CU_ASSERT(task.seed == seed); 498 CU_ASSERT(task.flags == 0); 499 CU_ASSERT(task.op_code == ACCEL_OPC_COPY_CRC32C); 500 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 501 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 502 CU_ASSERT(expected_accel_task == &task); 503 } 504 505 static void 506 test_spdk_accel_submit_xor(void) 507 { 508 const uint64_t nbytes = TEST_SUBMIT_SIZE; 509 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 510 uint8_t src1[TEST_SUBMIT_SIZE] = {0}; 511 uint8_t src2[TEST_SUBMIT_SIZE] = {0}; 512 void *sources[] = { src1, src2 }; 513 uint32_t nsrcs = SPDK_COUNTOF(sources); 514 int rc; 515 struct spdk_accel_task task; 516 struct spdk_accel_task *expected_accel_task = NULL; 517 518 TAILQ_INIT(&g_accel_ch->task_pool); 519 520 /* Fail with no tasks on _get_task() */ 521 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 522 CU_ASSERT(rc == -ENOMEM); 523 524 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 525 526 /* submission OK. */ 527 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 528 CU_ASSERT(rc == 0); 529 CU_ASSERT(task.nsrcs.srcs == sources); 530 CU_ASSERT(task.nsrcs.cnt == nsrcs); 531 CU_ASSERT(task.d.iovcnt == 1); 532 CU_ASSERT(task.d.iovs[0].iov_base == dst); 533 CU_ASSERT(task.d.iovs[0].iov_len == nbytes); 534 CU_ASSERT(task.op_code == ACCEL_OPC_XOR); 535 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 536 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 537 CU_ASSERT(expected_accel_task == &task); 538 } 539 540 static void 541 test_spdk_accel_module_find_by_name(void) 542 { 543 struct spdk_accel_module_if mod1 = {}; 544 struct spdk_accel_module_if mod2 = {}; 545 struct spdk_accel_module_if mod3 = {}; 546 struct spdk_accel_module_if *accel_module = NULL; 547 548 mod1.name = "ioat"; 549 mod2.name = "idxd"; 550 mod3.name = "software"; 551 552 TAILQ_INIT(&spdk_accel_module_list); 553 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod1, tailq); 554 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod2, tailq); 555 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod3, tailq); 556 557 /* Now let's find a valid engine */ 558 accel_module = _module_find_by_name("ioat"); 559 CU_ASSERT(accel_module != NULL); 560 561 /* Try to find one that doesn't exist */ 562 accel_module = _module_find_by_name("XXX"); 563 CU_ASSERT(accel_module == NULL); 564 } 565 566 static void 567 test_spdk_accel_module_register(void) 568 { 569 struct spdk_accel_module_if mod1 = {}; 570 struct spdk_accel_module_if mod2 = {}; 571 struct spdk_accel_module_if mod3 = {}; 572 struct spdk_accel_module_if mod4 = {}; 573 struct spdk_accel_module_if *accel_module = NULL; 574 int i = 0; 575 576 mod1.name = "ioat"; 577 mod2.name = "idxd"; 578 mod3.name = "software"; 579 mod4.name = "nothing"; 580 581 TAILQ_INIT(&spdk_accel_module_list); 582 583 spdk_accel_module_list_add(&mod1); 584 spdk_accel_module_list_add(&mod2); 585 spdk_accel_module_list_add(&mod3); 586 spdk_accel_module_list_add(&mod4); 587 588 /* Now confirm they're in the right order. */ 589 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 590 switch (i++) { 591 case 0: 592 CU_ASSERT(strcmp(accel_module->name, "software") == 0); 593 break; 594 case 1: 595 CU_ASSERT(strcmp(accel_module->name, "ioat") == 0); 596 break; 597 case 2: 598 CU_ASSERT(strcmp(accel_module->name, "idxd") == 0); 599 break; 600 case 3: 601 CU_ASSERT(strcmp(accel_module->name, "nothing") == 0); 602 break; 603 default: 604 CU_ASSERT(false); 605 break; 606 } 607 } 608 CU_ASSERT(i == 4); 609 } 610 611 struct ut_sequence { 612 bool complete; 613 int status; 614 }; 615 616 static void 617 ut_sequence_step_cb(void *cb_arg) 618 { 619 int *completed = cb_arg; 620 621 (*completed)++; 622 } 623 624 static void 625 ut_sequence_complete_cb(void *cb_arg, int status) 626 { 627 struct ut_sequence *seq = cb_arg; 628 629 seq->complete = true; 630 seq->status = status; 631 } 632 633 static void 634 test_sequence_fill_copy(void) 635 { 636 struct spdk_accel_sequence *seq = NULL; 637 struct spdk_io_channel *ioch; 638 struct ut_sequence ut_seq; 639 char buf[4096], tmp[2][4096], expected[4096]; 640 struct iovec src_iovs[2], dst_iovs[2]; 641 int rc, completed; 642 643 ioch = spdk_accel_get_io_channel(); 644 SPDK_CU_ASSERT_FATAL(ioch != NULL); 645 646 /* First check the simplest case - single task in a sequence */ 647 memset(buf, 0, sizeof(buf)); 648 memset(expected, 0xa5, sizeof(expected)); 649 completed = 0; 650 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 651 ut_sequence_step_cb, &completed); 652 CU_ASSERT_EQUAL(rc, 0); 653 CU_ASSERT_EQUAL(completed, 0); 654 655 ut_seq.complete = false; 656 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 657 658 poll_threads(); 659 CU_ASSERT_EQUAL(completed, 1); 660 CU_ASSERT(ut_seq.complete); 661 CU_ASSERT_EQUAL(ut_seq.status, 0); 662 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 663 664 /* Check a single copy operation */ 665 memset(buf, 0, sizeof(buf)); 666 memset(tmp[0], 0xa5, sizeof(tmp[0])); 667 memset(expected, 0xa5, sizeof(expected)); 668 completed = 0; 669 seq = NULL; 670 671 dst_iovs[0].iov_base = buf; 672 dst_iovs[0].iov_len = sizeof(buf); 673 src_iovs[0].iov_base = tmp[0]; 674 src_iovs[0].iov_len = sizeof(tmp[0]); 675 676 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 677 &src_iovs[0], 1, NULL, NULL, 0, 678 ut_sequence_step_cb, &completed); 679 CU_ASSERT_EQUAL(rc, 0); 680 681 ut_seq.complete = false; 682 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 683 684 poll_threads(); 685 CU_ASSERT_EQUAL(completed, 1); 686 CU_ASSERT(ut_seq.complete); 687 CU_ASSERT_EQUAL(ut_seq.status, 0); 688 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 689 690 /* Check multiple fill operations */ 691 memset(buf, 0, sizeof(buf)); 692 memset(expected, 0xfe, 4096); 693 memset(expected, 0xde, 2048); 694 memset(expected, 0xa5, 1024); 695 seq = NULL; 696 completed = 0; 697 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, NULL, NULL, 0xfe, 0, 698 ut_sequence_step_cb, &completed); 699 CU_ASSERT_EQUAL(rc, 0); 700 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xde, 0, 701 ut_sequence_step_cb, &completed); 702 CU_ASSERT_EQUAL(rc, 0); 703 rc = spdk_accel_append_fill(&seq, ioch, buf, 1024, NULL, NULL, 0xa5, 0, 704 ut_sequence_step_cb, &completed); 705 CU_ASSERT_EQUAL(rc, 0); 706 707 ut_seq.complete = false; 708 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 709 710 poll_threads(); 711 CU_ASSERT_EQUAL(completed, 3); 712 CU_ASSERT(ut_seq.complete); 713 CU_ASSERT_EQUAL(ut_seq.status, 0); 714 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 715 716 /* Check multiple copy operations */ 717 memset(buf, 0, sizeof(buf)); 718 memset(tmp[0], 0, sizeof(tmp[0])); 719 memset(tmp[1], 0, sizeof(tmp[1])); 720 memset(expected, 0xa5, sizeof(expected)); 721 seq = NULL; 722 completed = 0; 723 724 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 725 ut_sequence_step_cb, &completed); 726 CU_ASSERT_EQUAL(rc, 0); 727 728 dst_iovs[0].iov_base = tmp[1]; 729 dst_iovs[0].iov_len = sizeof(tmp[1]); 730 src_iovs[0].iov_base = tmp[0]; 731 src_iovs[0].iov_len = sizeof(tmp[0]); 732 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 733 &src_iovs[0], 1, NULL, NULL, 0, 734 ut_sequence_step_cb, &completed); 735 CU_ASSERT_EQUAL(rc, 0); 736 737 dst_iovs[1].iov_base = buf; 738 dst_iovs[1].iov_len = sizeof(buf); 739 src_iovs[1].iov_base = tmp[1]; 740 src_iovs[1].iov_len = sizeof(tmp[1]); 741 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 742 &src_iovs[1], 1, NULL, NULL, 0, 743 ut_sequence_step_cb, &completed); 744 CU_ASSERT_EQUAL(rc, 0); 745 746 ut_seq.complete = false; 747 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 748 749 poll_threads(); 750 CU_ASSERT_EQUAL(completed, 3); 751 CU_ASSERT(ut_seq.complete); 752 CU_ASSERT_EQUAL(ut_seq.status, 0); 753 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 754 755 /* Check that adding a copy operation at the end will change destination buffer */ 756 memset(buf, 0, sizeof(buf)); 757 memset(tmp[0], 0, sizeof(tmp[0])); 758 memset(expected, 0xa5, sizeof(buf)); 759 seq = NULL; 760 completed = 0; 761 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 762 ut_sequence_step_cb, &completed); 763 CU_ASSERT_EQUAL(rc, 0); 764 765 dst_iovs[0].iov_base = buf; 766 dst_iovs[0].iov_len = sizeof(buf); 767 src_iovs[0].iov_base = tmp[0]; 768 src_iovs[0].iov_len = sizeof(tmp[0]); 769 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 770 &src_iovs[0], 1, NULL, NULL, 0, 771 ut_sequence_step_cb, &completed); 772 CU_ASSERT_EQUAL(rc, 0); 773 774 ut_seq.complete = false; 775 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 776 777 poll_threads(); 778 CU_ASSERT_EQUAL(completed, 2); 779 CU_ASSERT(ut_seq.complete); 780 CU_ASSERT_EQUAL(ut_seq.status, 0); 781 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 782 783 /* Check that it's also possible to add copy operation at the beginning */ 784 memset(buf, 0, sizeof(buf)); 785 memset(tmp[0], 0xde, sizeof(tmp[0])); 786 memset(tmp[1], 0, sizeof(tmp[1])); 787 memset(expected, 0xa5, sizeof(expected)); 788 seq = NULL; 789 completed = 0; 790 791 dst_iovs[0].iov_base = tmp[1]; 792 dst_iovs[0].iov_len = sizeof(tmp[1]); 793 src_iovs[0].iov_base = tmp[0]; 794 src_iovs[0].iov_len = sizeof(tmp[0]); 795 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 796 &src_iovs[0], 1, NULL, NULL, 0, 797 ut_sequence_step_cb, &completed); 798 CU_ASSERT_EQUAL(rc, 0); 799 800 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], sizeof(tmp[1]), NULL, NULL, 0xa5, 0, 801 ut_sequence_step_cb, &completed); 802 CU_ASSERT_EQUAL(rc, 0); 803 804 dst_iovs[1].iov_base = buf; 805 dst_iovs[1].iov_len = sizeof(buf); 806 src_iovs[1].iov_base = tmp[1]; 807 src_iovs[1].iov_len = sizeof(tmp[1]); 808 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 809 &src_iovs[1], 1, NULL, NULL, 0, 810 ut_sequence_step_cb, &completed); 811 CU_ASSERT_EQUAL(rc, 0); 812 813 ut_seq.complete = false; 814 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 815 816 poll_threads(); 817 CU_ASSERT_EQUAL(completed, 3); 818 CU_ASSERT(ut_seq.complete); 819 CU_ASSERT_EQUAL(ut_seq.status, 0); 820 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 821 822 spdk_put_io_channel(ioch); 823 poll_threads(); 824 } 825 826 static void 827 test_sequence_abort(void) 828 { 829 struct spdk_accel_sequence *seq = NULL; 830 struct spdk_io_channel *ioch; 831 char buf[4096], tmp[2][4096], expected[4096]; 832 struct iovec src_iovs[2], dst_iovs[2]; 833 int rc, completed; 834 835 ioch = spdk_accel_get_io_channel(); 836 SPDK_CU_ASSERT_FATAL(ioch != NULL); 837 838 /* Check that aborting a sequence calls operation's callback, the operation is not executed 839 * and the sequence is freed 840 */ 841 memset(buf, 0, sizeof(buf)); 842 memset(expected, 0, sizeof(buf)); 843 completed = 0; 844 seq = NULL; 845 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 846 ut_sequence_step_cb, &completed); 847 CU_ASSERT_EQUAL(rc, 0); 848 849 spdk_accel_sequence_abort(seq); 850 CU_ASSERT_EQUAL(completed, 1); 851 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 852 853 /* Check sequence with multiple operations */ 854 memset(buf, 0, sizeof(buf)); 855 memset(expected, 0, sizeof(buf)); 856 completed = 0; 857 seq = NULL; 858 859 dst_iovs[0].iov_base = tmp[1]; 860 dst_iovs[0].iov_len = sizeof(tmp[1]); 861 src_iovs[0].iov_base = tmp[0]; 862 src_iovs[0].iov_len = sizeof(tmp[0]); 863 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 864 &src_iovs[0], 1, NULL, NULL, 0, 865 ut_sequence_step_cb, &completed); 866 CU_ASSERT_EQUAL(rc, 0); 867 868 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 4096, NULL, NULL, 0xa5, 0, 869 ut_sequence_step_cb, &completed); 870 CU_ASSERT_EQUAL(rc, 0); 871 872 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xde, 0, 873 ut_sequence_step_cb, &completed); 874 CU_ASSERT_EQUAL(rc, 0); 875 876 dst_iovs[1].iov_base = buf; 877 dst_iovs[1].iov_len = sizeof(buf); 878 src_iovs[1].iov_base = tmp[1]; 879 src_iovs[1].iov_len = sizeof(tmp[1]); 880 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 881 &src_iovs[1], 1, NULL, NULL, 0, 882 ut_sequence_step_cb, &completed); 883 CU_ASSERT_EQUAL(rc, 0); 884 885 spdk_accel_sequence_abort(seq); 886 CU_ASSERT_EQUAL(completed, 4); 887 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 888 889 /* This should be a no-op */ 890 spdk_accel_sequence_abort(NULL); 891 892 spdk_put_io_channel(ioch); 893 poll_threads(); 894 } 895 896 static void 897 test_sequence_append_error(void) 898 { 899 struct spdk_accel_sequence *seq = NULL; 900 struct spdk_io_channel *ioch; 901 struct accel_io_channel *accel_ch; 902 struct iovec src_iovs, dst_iovs; 903 char buf[4096]; 904 TAILQ_HEAD(, spdk_accel_task) tasks = TAILQ_HEAD_INITIALIZER(tasks); 905 TAILQ_HEAD(, spdk_accel_sequence) seqs = TAILQ_HEAD_INITIALIZER(seqs); 906 int rc; 907 908 ioch = spdk_accel_get_io_channel(); 909 SPDK_CU_ASSERT_FATAL(ioch != NULL); 910 accel_ch = spdk_io_channel_get_ctx(ioch); 911 912 /* Check that append fails and no sequence object is allocated when there are no more free 913 * tasks */ 914 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 915 916 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 917 ut_sequence_step_cb, NULL); 918 CU_ASSERT_EQUAL(rc, -ENOMEM); 919 CU_ASSERT_PTR_NULL(seq); 920 921 dst_iovs.iov_base = buf; 922 dst_iovs.iov_len = 2048; 923 src_iovs.iov_base = &buf[2048]; 924 src_iovs.iov_len = 2048; 925 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 926 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 927 CU_ASSERT_EQUAL(rc, -ENOMEM); 928 CU_ASSERT_PTR_NULL(seq); 929 930 dst_iovs.iov_base = buf; 931 dst_iovs.iov_len = 2048; 932 src_iovs.iov_base = &buf[2048]; 933 src_iovs.iov_len = 2048; 934 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 935 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 936 CU_ASSERT_EQUAL(rc, -ENOMEM); 937 CU_ASSERT_PTR_NULL(seq); 938 939 /* Check that the same happens when the sequence queue is empty */ 940 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 941 TAILQ_SWAP(&seqs, &accel_ch->seq_pool, spdk_accel_sequence, link); 942 943 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 944 ut_sequence_step_cb, NULL); 945 CU_ASSERT_EQUAL(rc, -ENOMEM); 946 CU_ASSERT_PTR_NULL(seq); 947 948 dst_iovs.iov_base = buf; 949 dst_iovs.iov_len = 2048; 950 src_iovs.iov_base = &buf[2048]; 951 src_iovs.iov_len = 2048; 952 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 953 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 954 CU_ASSERT_EQUAL(rc, -ENOMEM); 955 CU_ASSERT_PTR_NULL(seq); 956 957 dst_iovs.iov_base = buf; 958 dst_iovs.iov_len = 2048; 959 src_iovs.iov_base = &buf[2048]; 960 src_iovs.iov_len = 2048; 961 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 962 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 963 CU_ASSERT_EQUAL(rc, -ENOMEM); 964 CU_ASSERT_PTR_NULL(seq); 965 966 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 967 968 spdk_put_io_channel(ioch); 969 poll_threads(); 970 } 971 972 struct ut_sequence_operation { 973 int complete_status; 974 int submit_status; 975 int count; 976 struct iovec *src_iovs; 977 uint32_t src_iovcnt; 978 struct spdk_memory_domain *src_domain; 979 void *src_domain_ctx; 980 struct iovec *dst_iovs; 981 uint32_t dst_iovcnt; 982 struct spdk_memory_domain *dst_domain; 983 void *dst_domain_ctx; 984 int (*submit)(struct spdk_io_channel *ch, struct spdk_accel_task *t); 985 }; 986 987 static struct ut_sequence_operation g_seq_operations[ACCEL_OPC_LAST]; 988 989 static int 990 ut_sequnce_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *task) 991 { 992 struct ut_sequence_operation *op = &g_seq_operations[task->op_code]; 993 994 op->count++; 995 if (op->submit != NULL) { 996 return op->submit(ch, task); 997 } 998 if (op->src_iovs != NULL) { 999 CU_ASSERT_EQUAL(task->s.iovcnt, op->src_iovcnt); 1000 CU_ASSERT_EQUAL(memcmp(task->s.iovs, op->src_iovs, 1001 sizeof(struct iovec) * op->src_iovcnt), 0); 1002 } 1003 if (op->dst_iovs != NULL) { 1004 CU_ASSERT_EQUAL(task->d.iovcnt, op->dst_iovcnt); 1005 CU_ASSERT_EQUAL(memcmp(task->d.iovs, op->dst_iovs, 1006 sizeof(struct iovec) * op->dst_iovcnt), 0); 1007 } 1008 1009 CU_ASSERT_EQUAL(task->src_domain, op->src_domain); 1010 CU_ASSERT_EQUAL(task->dst_domain, op->dst_domain); 1011 if (op->src_domain != NULL) { 1012 CU_ASSERT_EQUAL(task->src_domain_ctx, op->src_domain_ctx); 1013 } 1014 if (op->dst_domain != NULL) { 1015 CU_ASSERT_EQUAL(task->dst_domain_ctx, op->dst_domain_ctx); 1016 } 1017 1018 if (op->submit_status != 0) { 1019 return op->submit_status; 1020 } 1021 1022 spdk_accel_task_complete(task, op->complete_status); 1023 1024 return 0; 1025 } 1026 1027 static void 1028 ut_clear_operations(void) 1029 { 1030 memset(&g_seq_operations, 0, sizeof(g_seq_operations)); 1031 } 1032 1033 static void 1034 test_sequence_completion_error(void) 1035 { 1036 struct spdk_accel_sequence *seq = NULL; 1037 struct spdk_io_channel *ioch; 1038 struct ut_sequence ut_seq; 1039 struct iovec src_iovs, dst_iovs; 1040 char buf[4096], tmp[4096]; 1041 struct accel_module modules[ACCEL_OPC_LAST]; 1042 int i, rc, completed; 1043 1044 ioch = spdk_accel_get_io_channel(); 1045 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1046 1047 /* Override the submit_tasks function */ 1048 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1049 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1050 modules[i] = g_modules_opc[i]; 1051 g_modules_opc[i] = g_module; 1052 } 1053 1054 memset(buf, 0, sizeof(buf)); 1055 memset(tmp, 0, sizeof(tmp)); 1056 1057 /* Check that if the first operation completes with an error, the whole sequence is 1058 * completed with that error and that all operations' completion callbacks are executed 1059 */ 1060 g_seq_operations[ACCEL_OPC_FILL].complete_status = -E2BIG; 1061 completed = 0; 1062 seq = NULL; 1063 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1064 ut_sequence_step_cb, &completed); 1065 CU_ASSERT_EQUAL(rc, 0); 1066 1067 dst_iovs.iov_base = buf; 1068 dst_iovs.iov_len = sizeof(buf); 1069 src_iovs.iov_base = tmp; 1070 src_iovs.iov_len = sizeof(tmp); 1071 1072 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1073 &src_iovs, 1, NULL, NULL, 0, 1074 ut_sequence_step_cb, &completed); 1075 CU_ASSERT_EQUAL(rc, 0); 1076 1077 ut_seq.complete = false; 1078 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1079 1080 poll_threads(); 1081 CU_ASSERT_EQUAL(completed, 2); 1082 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 1083 1084 /* Check the same with a second operation in the sequence */ 1085 g_seq_operations[ACCEL_OPC_DECOMPRESS].complete_status = -EACCES; 1086 g_seq_operations[ACCEL_OPC_FILL].complete_status = 0; 1087 completed = 0; 1088 seq = NULL; 1089 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1090 ut_sequence_step_cb, &completed); 1091 CU_ASSERT_EQUAL(rc, 0); 1092 1093 dst_iovs.iov_base = buf; 1094 dst_iovs.iov_len = sizeof(buf); 1095 src_iovs.iov_base = tmp; 1096 src_iovs.iov_len = sizeof(tmp); 1097 1098 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1099 &src_iovs, 1, NULL, NULL, 0, 1100 ut_sequence_step_cb, &completed); 1101 CU_ASSERT_EQUAL(rc, 0); 1102 1103 ut_seq.complete = false; 1104 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1105 1106 poll_threads(); 1107 CU_ASSERT_EQUAL(completed, 2); 1108 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 1109 1110 g_seq_operations[ACCEL_OPC_DECOMPRESS].complete_status = 0; 1111 g_seq_operations[ACCEL_OPC_FILL].complete_status = 0; 1112 1113 /* Check submission failure of the first operation */ 1114 g_seq_operations[ACCEL_OPC_FILL].submit_status = -EADDRINUSE; 1115 completed = 0; 1116 seq = NULL; 1117 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1118 ut_sequence_step_cb, &completed); 1119 CU_ASSERT_EQUAL(rc, 0); 1120 1121 dst_iovs.iov_base = buf; 1122 dst_iovs.iov_len = sizeof(buf); 1123 src_iovs.iov_base = tmp; 1124 src_iovs.iov_len = sizeof(tmp); 1125 1126 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1127 &src_iovs, 1, NULL, NULL, 0, 1128 ut_sequence_step_cb, &completed); 1129 CU_ASSERT_EQUAL(rc, 0); 1130 1131 ut_seq.complete = false; 1132 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1133 1134 poll_threads(); 1135 CU_ASSERT_EQUAL(completed, 2); 1136 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 1137 1138 /* Check the same with a second operation */ 1139 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit_status = -EADDRNOTAVAIL; 1140 g_seq_operations[ACCEL_OPC_FILL].submit_status = 0; 1141 completed = 0; 1142 seq = NULL; 1143 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1144 ut_sequence_step_cb, &completed); 1145 CU_ASSERT_EQUAL(rc, 0); 1146 1147 dst_iovs.iov_base = buf; 1148 dst_iovs.iov_len = sizeof(buf); 1149 src_iovs.iov_base = tmp; 1150 src_iovs.iov_len = sizeof(tmp); 1151 1152 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1153 &src_iovs, 1, NULL, NULL, 0, 1154 ut_sequence_step_cb, &completed); 1155 CU_ASSERT_EQUAL(rc, 0); 1156 1157 ut_seq.complete = false; 1158 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1159 1160 poll_threads(); 1161 CU_ASSERT_EQUAL(completed, 2); 1162 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 1163 1164 /* Cleanup module pointers to make subsequent tests work correctly */ 1165 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1166 g_modules_opc[i] = modules[i]; 1167 } 1168 1169 ut_clear_operations(); 1170 spdk_put_io_channel(ioch); 1171 poll_threads(); 1172 } 1173 1174 #ifdef SPDK_CONFIG_ISAL 1175 static void 1176 ut_compress_cb(void *cb_arg, int status) 1177 { 1178 int *completed = cb_arg; 1179 1180 CU_ASSERT_EQUAL(status, 0); 1181 1182 *completed = 1; 1183 } 1184 1185 static void 1186 test_sequence_decompress(void) 1187 { 1188 struct spdk_accel_sequence *seq = NULL; 1189 struct spdk_io_channel *ioch; 1190 struct ut_sequence ut_seq; 1191 char buf[4096], tmp[2][4096], expected[4096]; 1192 struct iovec src_iovs[2], dst_iovs[2]; 1193 uint32_t compressed_size; 1194 int rc, completed = 0; 1195 1196 ioch = spdk_accel_get_io_channel(); 1197 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1198 1199 memset(expected, 0xa5, sizeof(expected)); 1200 src_iovs[0].iov_base = expected; 1201 src_iovs[0].iov_len = sizeof(expected); 1202 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1203 &compressed_size, 0, ut_compress_cb, &completed); 1204 CU_ASSERT_EQUAL(rc, 0); 1205 1206 while (!completed) { 1207 poll_threads(); 1208 } 1209 1210 /* Check a single decompress operation in a sequence */ 1211 seq = NULL; 1212 completed = 0; 1213 1214 dst_iovs[0].iov_base = buf; 1215 dst_iovs[0].iov_len = sizeof(buf); 1216 src_iovs[0].iov_base = tmp[0]; 1217 src_iovs[0].iov_len = compressed_size; 1218 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1219 &src_iovs[0], 1, NULL, NULL, 0, 1220 ut_sequence_step_cb, &completed); 1221 CU_ASSERT_EQUAL(rc, 0); 1222 1223 ut_seq.complete = false; 1224 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1225 1226 poll_threads(); 1227 1228 CU_ASSERT_EQUAL(completed, 1); 1229 CU_ASSERT(ut_seq.complete); 1230 CU_ASSERT_EQUAL(ut_seq.status, 0); 1231 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1232 1233 /* Put the decompress operation in the middle of a sequence with a copy operation at the 1234 * beginning and a fill at the end modifying the first 2048B of the buffer. 1235 */ 1236 memset(expected, 0xfe, 2048); 1237 memset(buf, 0, sizeof(buf)); 1238 seq = NULL; 1239 completed = 0; 1240 1241 dst_iovs[0].iov_base = tmp[1]; 1242 dst_iovs[0].iov_len = compressed_size; 1243 src_iovs[0].iov_base = tmp[0]; 1244 src_iovs[0].iov_len = compressed_size; 1245 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1246 &src_iovs[0], 1, NULL, NULL, 0, 1247 ut_sequence_step_cb, &completed); 1248 CU_ASSERT_EQUAL(rc, 0); 1249 1250 dst_iovs[1].iov_base = buf; 1251 dst_iovs[1].iov_len = sizeof(buf); 1252 src_iovs[1].iov_base = tmp[1]; 1253 src_iovs[1].iov_len = compressed_size; 1254 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1255 &src_iovs[1], 1, NULL, NULL, 0, 1256 ut_sequence_step_cb, &completed); 1257 CU_ASSERT_EQUAL(rc, 0); 1258 1259 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1260 ut_sequence_step_cb, &completed); 1261 CU_ASSERT_EQUAL(rc, 0); 1262 1263 ut_seq.complete = false; 1264 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1265 1266 poll_threads(); 1267 1268 CU_ASSERT_EQUAL(completed, 3); 1269 CU_ASSERT(ut_seq.complete); 1270 CU_ASSERT_EQUAL(ut_seq.status, 0); 1271 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1272 1273 /* Check sequence with decompress at the beginning: decompress -> copy */ 1274 memset(expected, 0xa5, sizeof(expected)); 1275 memset(buf, 0, sizeof(buf)); 1276 seq = NULL; 1277 completed = 0; 1278 1279 dst_iovs[0].iov_base = tmp[1]; 1280 dst_iovs[0].iov_len = sizeof(tmp[1]); 1281 src_iovs[0].iov_base = tmp[0]; 1282 src_iovs[0].iov_len = compressed_size; 1283 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1284 &src_iovs[0], 1, NULL, NULL, 0, 1285 ut_sequence_step_cb, &completed); 1286 CU_ASSERT_EQUAL(rc, 0); 1287 1288 dst_iovs[1].iov_base = buf; 1289 dst_iovs[1].iov_len = sizeof(buf); 1290 src_iovs[1].iov_base = tmp[1]; 1291 src_iovs[1].iov_len = sizeof(tmp[1]); 1292 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1293 &src_iovs[1], 1, NULL, NULL, 0, 1294 ut_sequence_step_cb, &completed); 1295 CU_ASSERT_EQUAL(rc, 0); 1296 1297 ut_seq.complete = false; 1298 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1299 1300 poll_threads(); 1301 1302 CU_ASSERT_EQUAL(completed, 2); 1303 CU_ASSERT(ut_seq.complete); 1304 CU_ASSERT_EQUAL(ut_seq.status, 0); 1305 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1306 1307 spdk_put_io_channel(ioch); 1308 poll_threads(); 1309 } 1310 1311 static void 1312 test_sequence_reverse(void) 1313 { 1314 struct spdk_accel_sequence *seq = NULL; 1315 struct spdk_io_channel *ioch; 1316 struct ut_sequence ut_seq; 1317 char buf[4096], tmp[2][4096], expected[4096]; 1318 struct iovec src_iovs[2], dst_iovs[2]; 1319 uint32_t compressed_size; 1320 int rc, completed = 0; 1321 1322 ioch = spdk_accel_get_io_channel(); 1323 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1324 1325 memset(expected, 0xa5, sizeof(expected)); 1326 src_iovs[0].iov_base = expected; 1327 src_iovs[0].iov_len = sizeof(expected); 1328 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1329 &compressed_size, 0, ut_compress_cb, &completed); 1330 CU_ASSERT_EQUAL(rc, 0); 1331 1332 while (!completed) { 1333 poll_threads(); 1334 } 1335 1336 /* First check that reversing a sequnce with a single operation is a no-op */ 1337 memset(buf, 0, sizeof(buf)); 1338 seq = NULL; 1339 completed = 0; 1340 1341 dst_iovs[0].iov_base = buf; 1342 dst_iovs[0].iov_len = sizeof(buf); 1343 src_iovs[0].iov_base = tmp[0]; 1344 src_iovs[0].iov_len = compressed_size; 1345 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1346 &src_iovs[0], 1, NULL, NULL, 0, 1347 ut_sequence_step_cb, &completed); 1348 CU_ASSERT_EQUAL(rc, 0); 1349 1350 spdk_accel_sequence_reverse(seq); 1351 1352 ut_seq.complete = false; 1353 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1354 1355 poll_threads(); 1356 1357 CU_ASSERT_EQUAL(completed, 1); 1358 CU_ASSERT(ut_seq.complete); 1359 CU_ASSERT_EQUAL(ut_seq.status, 0); 1360 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1361 1362 /* Add a copy operation at the end with src set to the compressed data. After reverse(), 1363 * that copy operation should be first, so decompress() should receive compressed data in 1364 * its src buffer. 1365 */ 1366 memset(buf, 0, sizeof(buf)); 1367 memset(tmp[1], 0, sizeof(tmp[1])); 1368 seq = NULL; 1369 completed = 0; 1370 1371 dst_iovs[0].iov_base = buf; 1372 dst_iovs[0].iov_len = sizeof(buf); 1373 src_iovs[0].iov_base = tmp[1]; 1374 src_iovs[0].iov_len = compressed_size; 1375 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1376 &src_iovs[0], 1, NULL, NULL, 0, 1377 ut_sequence_step_cb, &completed); 1378 CU_ASSERT_EQUAL(rc, 0); 1379 1380 dst_iovs[1].iov_base = tmp[1]; 1381 dst_iovs[1].iov_len = compressed_size; 1382 src_iovs[1].iov_base = tmp[0]; 1383 src_iovs[1].iov_len = compressed_size; 1384 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1385 &src_iovs[1], 1, NULL, NULL, 0, 1386 ut_sequence_step_cb, &completed); 1387 CU_ASSERT_EQUAL(rc, 0); 1388 1389 spdk_accel_sequence_reverse(seq); 1390 1391 ut_seq.complete = false; 1392 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1393 1394 poll_threads(); 1395 1396 CU_ASSERT_EQUAL(completed, 2); 1397 CU_ASSERT(ut_seq.complete); 1398 CU_ASSERT_EQUAL(ut_seq.status, 0); 1399 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1400 1401 /* Check the same, but add an extra fill operation at the beginning that should execute last 1402 * after reverse(). 1403 */ 1404 memset(buf, 0, sizeof(buf)); 1405 memset(tmp[1], 0, sizeof(tmp[1])); 1406 memset(expected, 0xfe, 2048); 1407 seq = NULL; 1408 completed = 0; 1409 1410 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1411 ut_sequence_step_cb, &completed); 1412 CU_ASSERT_EQUAL(rc, 0); 1413 1414 dst_iovs[0].iov_base = buf; 1415 dst_iovs[0].iov_len = sizeof(buf); 1416 src_iovs[0].iov_base = tmp[1]; 1417 src_iovs[0].iov_len = compressed_size; 1418 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1419 &src_iovs[0], 1, NULL, NULL, 0, 1420 ut_sequence_step_cb, &completed); 1421 CU_ASSERT_EQUAL(rc, 0); 1422 1423 dst_iovs[1].iov_base = tmp[1]; 1424 dst_iovs[1].iov_len = compressed_size; 1425 src_iovs[1].iov_base = tmp[0]; 1426 src_iovs[1].iov_len = compressed_size; 1427 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1428 &src_iovs[1], 1, NULL, NULL, 0, 1429 ut_sequence_step_cb, &completed); 1430 CU_ASSERT_EQUAL(rc, 0); 1431 1432 spdk_accel_sequence_reverse(seq); 1433 1434 ut_seq.complete = false; 1435 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1436 1437 poll_threads(); 1438 1439 CU_ASSERT_EQUAL(completed, 3); 1440 CU_ASSERT(ut_seq.complete); 1441 CU_ASSERT_EQUAL(ut_seq.status, 0); 1442 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1443 1444 /* Build the sequence in order and then reverse it twice */ 1445 memset(buf, 0, sizeof(buf)); 1446 memset(tmp[1], 0, sizeof(tmp[1])); 1447 seq = NULL; 1448 completed = 0; 1449 1450 dst_iovs[0].iov_base = tmp[1]; 1451 dst_iovs[0].iov_len = compressed_size; 1452 src_iovs[0].iov_base = tmp[0]; 1453 src_iovs[0].iov_len = compressed_size; 1454 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1455 &src_iovs[0], 1, NULL, NULL, 0, 1456 ut_sequence_step_cb, &completed); 1457 CU_ASSERT_EQUAL(rc, 0); 1458 1459 dst_iovs[1].iov_base = buf; 1460 dst_iovs[1].iov_len = sizeof(buf); 1461 src_iovs[1].iov_base = tmp[1]; 1462 src_iovs[1].iov_len = compressed_size; 1463 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1464 &src_iovs[1], 1, NULL, NULL, 0, 1465 ut_sequence_step_cb, &completed); 1466 CU_ASSERT_EQUAL(rc, 0); 1467 1468 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1469 ut_sequence_step_cb, &completed); 1470 CU_ASSERT_EQUAL(rc, 0); 1471 1472 spdk_accel_sequence_reverse(seq); 1473 spdk_accel_sequence_reverse(seq); 1474 1475 ut_seq.complete = false; 1476 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1477 1478 poll_threads(); 1479 1480 CU_ASSERT_EQUAL(completed, 3); 1481 CU_ASSERT(ut_seq.complete); 1482 CU_ASSERT_EQUAL(ut_seq.status, 0); 1483 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1484 1485 spdk_put_io_channel(ioch); 1486 poll_threads(); 1487 } 1488 #endif 1489 1490 static void 1491 test_sequence_copy_elision(void) 1492 { 1493 struct spdk_accel_sequence *seq = NULL; 1494 struct spdk_io_channel *ioch; 1495 struct ut_sequence ut_seq; 1496 struct iovec src_iovs[4], dst_iovs[4], exp_iovs[2]; 1497 char buf[4096], tmp[4][4096]; 1498 struct accel_module modules[ACCEL_OPC_LAST]; 1499 struct spdk_accel_crypto_key key = {}; 1500 int i, rc, completed; 1501 1502 ioch = spdk_accel_get_io_channel(); 1503 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1504 1505 /* Override the submit_tasks function */ 1506 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1507 g_module.supports_memory_domains = true; 1508 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1509 g_seq_operations[i].complete_status = 0; 1510 g_seq_operations[i].submit_status = 0; 1511 g_seq_operations[i].count = 0; 1512 1513 modules[i] = g_modules_opc[i]; 1514 g_modules_opc[i] = g_module; 1515 } 1516 1517 /* Check that a copy operation at the beginning is removed */ 1518 seq = NULL; 1519 completed = 0; 1520 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovcnt = 1; 1521 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovcnt = 1; 1522 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1523 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1524 exp_iovs[0].iov_base = tmp[0]; 1525 exp_iovs[0].iov_len = sizeof(tmp[0]); 1526 exp_iovs[1].iov_base = buf; 1527 exp_iovs[1].iov_len = 2048; 1528 1529 dst_iovs[0].iov_base = tmp[1]; 1530 dst_iovs[0].iov_len = sizeof(tmp[1]); 1531 src_iovs[0].iov_base = tmp[0]; 1532 src_iovs[0].iov_len = sizeof(tmp[0]); 1533 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1534 &src_iovs[0], 1, NULL, NULL, 0, 1535 ut_sequence_step_cb, &completed); 1536 CU_ASSERT_EQUAL(rc, 0); 1537 1538 dst_iovs[1].iov_base = buf; 1539 dst_iovs[1].iov_len = 2048; 1540 src_iovs[1].iov_base = tmp[1]; 1541 src_iovs[1].iov_len = sizeof(tmp[1]); 1542 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1543 &src_iovs[1], 1, NULL, NULL, 0, 1544 ut_sequence_step_cb, &completed); 1545 CU_ASSERT_EQUAL(rc, 0); 1546 1547 ut_seq.complete = false; 1548 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1549 1550 poll_threads(); 1551 1552 CU_ASSERT_EQUAL(completed, 2); 1553 CU_ASSERT(ut_seq.complete); 1554 CU_ASSERT_EQUAL(ut_seq.status, 0); 1555 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1556 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 1557 1558 /* Check that a copy operation at the end is removed too */ 1559 seq = NULL; 1560 completed = 0; 1561 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1562 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 1563 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1564 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1565 exp_iovs[0].iov_base = tmp[0]; 1566 exp_iovs[0].iov_len = sizeof(tmp[0]); 1567 exp_iovs[1].iov_base = buf; 1568 exp_iovs[1].iov_len = 2048; 1569 1570 dst_iovs[0].iov_base = tmp[1]; 1571 dst_iovs[0].iov_len = 2048; 1572 src_iovs[0].iov_base = tmp[0]; 1573 src_iovs[0].iov_len = sizeof(tmp[0]); 1574 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1575 &src_iovs[0], 1, NULL, NULL, 0, 1576 ut_sequence_step_cb, &completed); 1577 CU_ASSERT_EQUAL(rc, 0); 1578 1579 dst_iovs[1].iov_base = buf; 1580 dst_iovs[1].iov_len = 2048; 1581 src_iovs[1].iov_base = tmp[1]; 1582 src_iovs[1].iov_len = 2048; 1583 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1584 &src_iovs[1], 1, NULL, NULL, 0, 1585 ut_sequence_step_cb, &completed); 1586 CU_ASSERT_EQUAL(rc, 0); 1587 1588 ut_seq.complete = false; 1589 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1590 1591 poll_threads(); 1592 1593 CU_ASSERT_EQUAL(completed, 2); 1594 CU_ASSERT(ut_seq.complete); 1595 CU_ASSERT_EQUAL(ut_seq.status, 0); 1596 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1597 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 1598 1599 /* Check a copy operation both at the beginning and the end */ 1600 seq = NULL; 1601 completed = 0; 1602 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1603 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 1604 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1605 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1606 exp_iovs[0].iov_base = tmp[0]; 1607 exp_iovs[0].iov_len = sizeof(tmp[0]); 1608 exp_iovs[1].iov_base = buf; 1609 exp_iovs[1].iov_len = 2048; 1610 1611 dst_iovs[0].iov_base = tmp[1]; 1612 dst_iovs[0].iov_len = sizeof(tmp[1]); 1613 src_iovs[0].iov_base = tmp[0]; 1614 src_iovs[0].iov_len = sizeof(tmp[0]); 1615 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1616 &src_iovs[0], 1, NULL, NULL, 0, 1617 ut_sequence_step_cb, &completed); 1618 CU_ASSERT_EQUAL(rc, 0); 1619 1620 dst_iovs[1].iov_base = tmp[2]; 1621 dst_iovs[1].iov_len = 2048; 1622 src_iovs[1].iov_base = tmp[1]; 1623 src_iovs[1].iov_len = sizeof(tmp[1]); 1624 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1625 &src_iovs[1], 1, NULL, NULL, 0, 1626 ut_sequence_step_cb, &completed); 1627 CU_ASSERT_EQUAL(rc, 0); 1628 1629 dst_iovs[2].iov_base = buf; 1630 dst_iovs[2].iov_len = 2048; 1631 src_iovs[2].iov_base = tmp[2]; 1632 src_iovs[2].iov_len = 2048; 1633 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1634 &src_iovs[2], 1, NULL, NULL, 0, 1635 ut_sequence_step_cb, &completed); 1636 CU_ASSERT_EQUAL(rc, 0); 1637 1638 ut_seq.complete = false; 1639 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1640 1641 poll_threads(); 1642 1643 CU_ASSERT_EQUAL(completed, 3); 1644 CU_ASSERT(ut_seq.complete); 1645 CU_ASSERT_EQUAL(ut_seq.status, 0); 1646 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1647 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 1648 1649 /* Check decompress + copy + decompress + copy */ 1650 seq = NULL; 1651 completed = 0; 1652 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1653 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 1654 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = NULL; 1655 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = NULL; 1656 1657 dst_iovs[0].iov_base = tmp[1]; 1658 dst_iovs[0].iov_len = sizeof(tmp[1]); 1659 src_iovs[0].iov_base = tmp[0]; 1660 src_iovs[0].iov_len = sizeof(tmp[0]); 1661 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1662 &src_iovs[0], 1, NULL, NULL, 0, 1663 ut_sequence_step_cb, &completed); 1664 CU_ASSERT_EQUAL(rc, 0); 1665 1666 dst_iovs[1].iov_base = tmp[2]; 1667 dst_iovs[1].iov_len = 2048; 1668 src_iovs[1].iov_base = tmp[1]; 1669 src_iovs[1].iov_len = sizeof(tmp[1]); 1670 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1671 &src_iovs[1], 1, NULL, NULL, 0, 1672 ut_sequence_step_cb, &completed); 1673 CU_ASSERT_EQUAL(rc, 0); 1674 1675 dst_iovs[2].iov_base = tmp[3]; 1676 dst_iovs[2].iov_len = 1024; 1677 src_iovs[2].iov_base = tmp[2]; 1678 src_iovs[2].iov_len = 2048; 1679 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1680 &src_iovs[2], 1, NULL, NULL, 0, 1681 ut_sequence_step_cb, &completed); 1682 CU_ASSERT_EQUAL(rc, 0); 1683 1684 dst_iovs[3].iov_base = buf; 1685 dst_iovs[3].iov_len = 1024; 1686 src_iovs[3].iov_base = tmp[3]; 1687 src_iovs[3].iov_len = 1024; 1688 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 1689 &src_iovs[3], 1, NULL, NULL, 0, 1690 ut_sequence_step_cb, &completed); 1691 CU_ASSERT_EQUAL(rc, 0); 1692 1693 ut_seq.complete = false; 1694 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1695 1696 poll_threads(); 1697 1698 CU_ASSERT_EQUAL(completed, 4); 1699 CU_ASSERT(ut_seq.complete); 1700 CU_ASSERT_EQUAL(ut_seq.status, 0); 1701 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1702 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 2); 1703 1704 /* Check two copy operations - one of them should be removed, while the other should be 1705 * executed normally */ 1706 seq = NULL; 1707 completed = 0; 1708 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1709 g_seq_operations[ACCEL_OPC_COPY].dst_iovcnt = 1; 1710 g_seq_operations[ACCEL_OPC_COPY].src_iovcnt = 1; 1711 g_seq_operations[ACCEL_OPC_COPY].src_iovs = &exp_iovs[0]; 1712 g_seq_operations[ACCEL_OPC_COPY].dst_iovs = &exp_iovs[1]; 1713 exp_iovs[0].iov_base = tmp[0]; 1714 exp_iovs[0].iov_len = sizeof(tmp[0]); 1715 exp_iovs[1].iov_base = buf; 1716 exp_iovs[1].iov_len = sizeof(buf); 1717 1718 dst_iovs[0].iov_base = tmp[1]; 1719 dst_iovs[0].iov_len = sizeof(tmp[1]); 1720 src_iovs[0].iov_base = tmp[0]; 1721 src_iovs[0].iov_len = sizeof(tmp[0]); 1722 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1723 &src_iovs[0], 1, NULL, NULL, 0, 1724 ut_sequence_step_cb, &completed); 1725 CU_ASSERT_EQUAL(rc, 0); 1726 1727 dst_iovs[1].iov_base = buf; 1728 dst_iovs[1].iov_len = sizeof(buf); 1729 src_iovs[1].iov_base = tmp[1]; 1730 src_iovs[1].iov_len = sizeof(tmp[1]); 1731 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1732 &src_iovs[1], 1, NULL, NULL, 0, 1733 ut_sequence_step_cb, &completed); 1734 CU_ASSERT_EQUAL(rc, 0); 1735 1736 ut_seq.complete = false; 1737 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1738 1739 poll_threads(); 1740 1741 CU_ASSERT_EQUAL(completed, 2); 1742 CU_ASSERT(ut_seq.complete); 1743 CU_ASSERT_EQUAL(ut_seq.status, 0); 1744 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 1); 1745 1746 /* Check fill + copy */ 1747 seq = NULL; 1748 completed = 0; 1749 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1750 g_seq_operations[ACCEL_OPC_FILL].count = 0; 1751 g_seq_operations[ACCEL_OPC_COPY].src_iovs = NULL; 1752 g_seq_operations[ACCEL_OPC_COPY].dst_iovs = NULL; 1753 g_seq_operations[ACCEL_OPC_FILL].dst_iovcnt = 1; 1754 g_seq_operations[ACCEL_OPC_FILL].dst_iovs = &exp_iovs[0]; 1755 exp_iovs[0].iov_base = buf; 1756 exp_iovs[0].iov_len = sizeof(buf); 1757 1758 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 1759 ut_sequence_step_cb, &completed); 1760 CU_ASSERT_EQUAL(rc, 0); 1761 1762 dst_iovs[0].iov_base = buf; 1763 dst_iovs[0].iov_len = sizeof(buf); 1764 src_iovs[0].iov_base = tmp[0]; 1765 src_iovs[0].iov_len = sizeof(tmp[0]); 1766 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1767 &src_iovs[0], 1, NULL, NULL, 0, 1768 ut_sequence_step_cb, &completed); 1769 CU_ASSERT_EQUAL(rc, 0); 1770 1771 ut_seq.complete = false; 1772 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1773 1774 poll_threads(); 1775 1776 CU_ASSERT_EQUAL(completed, 2); 1777 CU_ASSERT(ut_seq.complete); 1778 CU_ASSERT_EQUAL(ut_seq.status, 0); 1779 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1780 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 1); 1781 1782 /* Check copy + encrypt + copy */ 1783 seq = NULL; 1784 completed = 0; 1785 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1786 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 1787 g_seq_operations[ACCEL_OPC_ENCRYPT].dst_iovcnt = 1; 1788 g_seq_operations[ACCEL_OPC_ENCRYPT].src_iovcnt = 1; 1789 g_seq_operations[ACCEL_OPC_ENCRYPT].src_iovs = &exp_iovs[0]; 1790 g_seq_operations[ACCEL_OPC_ENCRYPT].dst_iovs = &exp_iovs[1]; 1791 exp_iovs[0].iov_base = tmp[0]; 1792 exp_iovs[0].iov_len = sizeof(tmp[0]); 1793 exp_iovs[1].iov_base = buf; 1794 exp_iovs[1].iov_len = sizeof(buf); 1795 1796 dst_iovs[0].iov_base = tmp[1]; 1797 dst_iovs[0].iov_len = sizeof(tmp[1]); 1798 src_iovs[0].iov_base = tmp[0]; 1799 src_iovs[0].iov_len = sizeof(tmp[0]); 1800 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1801 &src_iovs[0], 1, NULL, NULL, 0, 1802 ut_sequence_step_cb, &completed); 1803 CU_ASSERT_EQUAL(rc, 0); 1804 1805 dst_iovs[1].iov_base = tmp[2]; 1806 dst_iovs[1].iov_len = sizeof(tmp[2]); 1807 src_iovs[1].iov_base = tmp[1]; 1808 src_iovs[1].iov_len = sizeof(tmp[1]); 1809 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1810 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1811 ut_sequence_step_cb, &completed); 1812 CU_ASSERT_EQUAL(rc, 0); 1813 1814 dst_iovs[2].iov_base = buf; 1815 dst_iovs[2].iov_len = sizeof(buf); 1816 src_iovs[2].iov_base = tmp[2]; 1817 src_iovs[2].iov_len = sizeof(tmp[2]); 1818 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1819 &src_iovs[2], 1, NULL, NULL, 0, 1820 ut_sequence_step_cb, &completed); 1821 CU_ASSERT_EQUAL(rc, 0); 1822 1823 ut_seq.complete = false; 1824 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1825 1826 poll_threads(); 1827 1828 CU_ASSERT_EQUAL(completed, 3); 1829 CU_ASSERT(ut_seq.complete); 1830 CU_ASSERT_EQUAL(ut_seq.status, 0); 1831 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1832 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 1833 1834 /* Check copy + decrypt + copy */ 1835 seq = NULL; 1836 completed = 0; 1837 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1838 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 1839 g_seq_operations[ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1840 g_seq_operations[ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1841 g_seq_operations[ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1842 g_seq_operations[ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1843 exp_iovs[0].iov_base = tmp[0]; 1844 exp_iovs[0].iov_len = sizeof(tmp[0]); 1845 exp_iovs[1].iov_base = buf; 1846 exp_iovs[1].iov_len = sizeof(buf); 1847 1848 dst_iovs[0].iov_base = tmp[1]; 1849 dst_iovs[0].iov_len = sizeof(tmp[1]); 1850 src_iovs[0].iov_base = tmp[0]; 1851 src_iovs[0].iov_len = sizeof(tmp[0]); 1852 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1853 &src_iovs[0], 1, NULL, NULL, 0, 1854 ut_sequence_step_cb, &completed); 1855 CU_ASSERT_EQUAL(rc, 0); 1856 1857 dst_iovs[1].iov_base = tmp[2]; 1858 dst_iovs[1].iov_len = sizeof(tmp[2]); 1859 src_iovs[1].iov_base = tmp[1]; 1860 src_iovs[1].iov_len = sizeof(tmp[1]); 1861 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1862 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1863 ut_sequence_step_cb, &completed); 1864 CU_ASSERT_EQUAL(rc, 0); 1865 1866 dst_iovs[2].iov_base = buf; 1867 dst_iovs[2].iov_len = sizeof(buf); 1868 src_iovs[2].iov_base = tmp[2]; 1869 src_iovs[2].iov_len = sizeof(tmp[2]); 1870 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1871 &src_iovs[2], 1, NULL, NULL, 0, 1872 ut_sequence_step_cb, &completed); 1873 CU_ASSERT_EQUAL(rc, 0); 1874 1875 ut_seq.complete = false; 1876 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1877 1878 poll_threads(); 1879 1880 CU_ASSERT_EQUAL(completed, 3); 1881 CU_ASSERT(ut_seq.complete); 1882 CU_ASSERT_EQUAL(ut_seq.status, 0); 1883 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1884 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 1885 1886 /* Check a sequence with memory domains and verify their pointers (and their contexts) are 1887 * correctly set on the next/prev task when a copy is removed */ 1888 seq = NULL; 1889 completed = 0; 1890 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1891 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 1892 g_seq_operations[ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1893 g_seq_operations[ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1894 g_seq_operations[ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1895 g_seq_operations[ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1896 g_seq_operations[ACCEL_OPC_DECRYPT].dst_domain = (void *)0x1; 1897 g_seq_operations[ACCEL_OPC_DECRYPT].dst_domain_ctx = (void *)0x2; 1898 g_seq_operations[ACCEL_OPC_DECRYPT].src_domain = (void *)0x3; 1899 g_seq_operations[ACCEL_OPC_DECRYPT].src_domain_ctx = (void *)0x4; 1900 exp_iovs[0].iov_base = tmp[0]; 1901 exp_iovs[0].iov_len = sizeof(tmp[0]); 1902 exp_iovs[1].iov_base = buf; 1903 exp_iovs[1].iov_len = sizeof(buf); 1904 1905 dst_iovs[0].iov_base = tmp[1]; 1906 dst_iovs[0].iov_len = sizeof(tmp[1]); 1907 src_iovs[0].iov_base = tmp[0]; 1908 src_iovs[0].iov_len = sizeof(tmp[0]); 1909 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, (void *)0xdead, (void *)0xbeef, 1910 &src_iovs[0], 1, (void *)0x3, (void *)0x4, 0, 1911 ut_sequence_step_cb, &completed); 1912 CU_ASSERT_EQUAL(rc, 0); 1913 1914 dst_iovs[1].iov_base = tmp[2]; 1915 dst_iovs[1].iov_len = sizeof(tmp[2]); 1916 src_iovs[1].iov_base = tmp[1]; 1917 src_iovs[1].iov_len = sizeof(tmp[1]); 1918 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 1919 &src_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 0, 1920 sizeof(tmp[2]), 0, ut_sequence_step_cb, &completed); 1921 CU_ASSERT_EQUAL(rc, 0); 1922 1923 dst_iovs[2].iov_base = buf; 1924 dst_iovs[2].iov_len = sizeof(buf); 1925 src_iovs[2].iov_base = tmp[2]; 1926 src_iovs[2].iov_len = sizeof(tmp[2]); 1927 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, (void *)0x1, (void *)0x2, 1928 &src_iovs[2], 1, (void *)0xdead, (void *)0xbeef, 0, 1929 ut_sequence_step_cb, &completed); 1930 CU_ASSERT_EQUAL(rc, 0); 1931 1932 ut_seq.complete = false; 1933 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1934 1935 poll_threads(); 1936 1937 CU_ASSERT_EQUAL(completed, 3); 1938 CU_ASSERT(ut_seq.complete); 1939 CU_ASSERT_EQUAL(ut_seq.status, 0); 1940 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1941 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 1942 1943 /* Cleanup module pointers to make subsequent tests work correctly */ 1944 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1945 g_modules_opc[i] = modules[i]; 1946 } 1947 1948 g_module.supports_memory_domains = false; 1949 ut_clear_operations(); 1950 spdk_put_io_channel(ioch); 1951 poll_threads(); 1952 } 1953 1954 static int 1955 ut_submit_decompress(struct spdk_io_channel *ch, struct spdk_accel_task *task) 1956 { 1957 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 1958 1959 spdk_accel_task_complete(task, 0); 1960 1961 return 0; 1962 } 1963 1964 static void 1965 test_sequence_accel_buffers(void) 1966 { 1967 struct spdk_accel_sequence *seq = NULL; 1968 struct spdk_io_channel *ioch; 1969 struct accel_io_channel *accel_ch; 1970 struct ut_sequence ut_seq; 1971 struct iovec src_iovs[3], dst_iovs[3]; 1972 char srcbuf[4096], dstbuf[4096], expected[4096]; 1973 struct accel_module modules[ACCEL_OPC_LAST]; 1974 void *buf[2], *domain_ctx[2], *iobuf_buf; 1975 struct spdk_memory_domain *domain[2]; 1976 struct spdk_iobuf_buffer *cache_entry; 1977 spdk_iobuf_buffer_stailq_t small_cache; 1978 uint32_t small_cache_count; 1979 int i, rc, completed; 1980 struct spdk_iobuf_opts opts_iobuf = {}; 1981 1982 /* Set up the iobuf to always use the "small" pool */ 1983 opts_iobuf.large_bufsize = 0x20000; 1984 opts_iobuf.large_pool_count = 0; 1985 opts_iobuf.small_bufsize = 0x10000; 1986 opts_iobuf.small_pool_count = 32; 1987 1988 rc = spdk_iobuf_set_opts(&opts_iobuf); 1989 CU_ASSERT(rc == 0); 1990 1991 ioch = spdk_accel_get_io_channel(); 1992 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1993 1994 /* Override the submit_tasks function */ 1995 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1996 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1997 modules[i] = g_modules_opc[i]; 1998 g_modules_opc[i] = g_module; 1999 } 2000 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2001 * decompress operations together in one sequence. 2002 */ 2003 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2004 g_seq_operations[ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2005 g_seq_operations[ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2006 2007 /* Check the simplest case: one operation using accel buffer as destination + copy operation 2008 * specifying the actual destination buffer 2009 */ 2010 memset(srcbuf, 0xa5, 4096); 2011 memset(dstbuf, 0x0, 4096); 2012 memset(expected, 0xa5, 4096); 2013 completed = 0; 2014 seq = NULL; 2015 2016 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2017 CU_ASSERT_EQUAL(rc, 0); 2018 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2019 2020 src_iovs[0].iov_base = srcbuf; 2021 src_iovs[0].iov_len = 4096; 2022 dst_iovs[0].iov_base = buf[0]; 2023 dst_iovs[0].iov_len = 4096; 2024 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2025 &src_iovs[0], 1, NULL, NULL, 0, 2026 ut_sequence_step_cb, &completed); 2027 CU_ASSERT_EQUAL(rc, 0); 2028 2029 src_iovs[1].iov_base = buf[0]; 2030 src_iovs[1].iov_len = 4096; 2031 dst_iovs[1].iov_base = dstbuf; 2032 dst_iovs[1].iov_len = 4096; 2033 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2034 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 2035 ut_sequence_step_cb, &completed); 2036 CU_ASSERT_EQUAL(rc, 0); 2037 2038 ut_seq.complete = false; 2039 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2040 2041 poll_threads(); 2042 2043 CU_ASSERT_EQUAL(completed, 2); 2044 CU_ASSERT(ut_seq.complete); 2045 CU_ASSERT_EQUAL(ut_seq.status, 0); 2046 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2047 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2048 2049 /* Start with a fill operation using accel buffer, followed by a decompress using another 2050 * accel buffer as dst, followed by a copy operation specifying dst buffer of the whole 2051 * sequence 2052 */ 2053 memset(srcbuf, 0x0, 4096); 2054 memset(dstbuf, 0x0, 4096); 2055 memset(expected, 0x5a, 4096); 2056 completed = 0; 2057 seq = NULL; 2058 2059 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2060 CU_ASSERT_EQUAL(rc, 0); 2061 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2062 2063 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2064 ut_sequence_step_cb, &completed); 2065 CU_ASSERT_EQUAL(rc, 0); 2066 2067 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2068 CU_ASSERT_EQUAL(rc, 0); 2069 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2070 2071 src_iovs[0].iov_base = buf[0]; 2072 src_iovs[0].iov_len = 4096; 2073 dst_iovs[0].iov_base = buf[1]; 2074 dst_iovs[0].iov_len = 4096; 2075 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2076 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2077 ut_sequence_step_cb, &completed); 2078 CU_ASSERT_EQUAL(rc, 0); 2079 2080 src_iovs[1].iov_base = buf[1]; 2081 src_iovs[1].iov_len = 4096; 2082 dst_iovs[1].iov_base = dstbuf; 2083 dst_iovs[1].iov_len = 4096; 2084 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2085 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2086 ut_sequence_step_cb, &completed); 2087 2088 ut_seq.complete = false; 2089 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2090 2091 poll_threads(); 2092 2093 CU_ASSERT_EQUAL(completed, 3); 2094 CU_ASSERT(ut_seq.complete); 2095 CU_ASSERT_EQUAL(ut_seq.status, 0); 2096 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2097 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2098 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2099 2100 /* Check the same, but with two decompress operations with the first one being in-place */ 2101 memset(srcbuf, 0x0, 4096); 2102 memset(dstbuf, 0x0, 4096); 2103 memset(expected, 0x5a, 4096); 2104 completed = 0; 2105 seq = NULL; 2106 2107 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2108 CU_ASSERT_EQUAL(rc, 0); 2109 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2110 2111 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2112 ut_sequence_step_cb, &completed); 2113 CU_ASSERT_EQUAL(rc, 0); 2114 2115 src_iovs[0].iov_base = buf[0]; 2116 src_iovs[0].iov_len = 4096; 2117 dst_iovs[0].iov_base = buf[0]; 2118 dst_iovs[0].iov_len = 4096; 2119 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2120 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2121 ut_sequence_step_cb, &completed); 2122 CU_ASSERT_EQUAL(rc, 0); 2123 2124 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2125 CU_ASSERT_EQUAL(rc, 0); 2126 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2127 2128 src_iovs[1].iov_base = buf[0]; 2129 src_iovs[1].iov_len = 4096; 2130 dst_iovs[1].iov_base = buf[1]; 2131 dst_iovs[1].iov_len = 4096; 2132 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, domain[1], domain_ctx[1], 2133 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 2134 ut_sequence_step_cb, &completed); 2135 CU_ASSERT_EQUAL(rc, 0); 2136 2137 src_iovs[2].iov_base = buf[1]; 2138 src_iovs[2].iov_len = 4096; 2139 dst_iovs[2].iov_base = dstbuf; 2140 dst_iovs[2].iov_len = 4096; 2141 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 2142 &src_iovs[2], 1, domain[1], domain_ctx[1], 0, 2143 ut_sequence_step_cb, &completed); 2144 2145 ut_seq.complete = false; 2146 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2147 2148 poll_threads(); 2149 2150 CU_ASSERT_EQUAL(completed, 4); 2151 CU_ASSERT(ut_seq.complete); 2152 CU_ASSERT_EQUAL(ut_seq.status, 0); 2153 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2154 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2155 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2156 2157 /* Check that specifying offsets within accel buffers works correctly */ 2158 memset(srcbuf, 0x0, 4096); 2159 memset(dstbuf, 0x0, 4096); 2160 completed = 0; 2161 seq = NULL; 2162 2163 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2164 CU_ASSERT_EQUAL(rc, 0); 2165 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2166 2167 /* Fill in each 1K of the buffer with different pattern */ 2168 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2169 ut_sequence_step_cb, &completed); 2170 CU_ASSERT_EQUAL(rc, 0); 2171 2172 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2173 0x5a, 0, ut_sequence_step_cb, &completed); 2174 CU_ASSERT_EQUAL(rc, 0); 2175 2176 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2177 0xfe, 0, ut_sequence_step_cb, &completed); 2178 CU_ASSERT_EQUAL(rc, 0); 2179 2180 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2181 0xed, 0, ut_sequence_step_cb, &completed); 2182 CU_ASSERT_EQUAL(rc, 0); 2183 2184 src_iovs[0].iov_base = buf[0]; 2185 src_iovs[0].iov_len = 4096; 2186 dst_iovs[0].iov_base = dstbuf; 2187 dst_iovs[0].iov_len = 4096; 2188 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2189 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2190 ut_sequence_step_cb, &completed); 2191 2192 ut_seq.complete = false; 2193 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2194 2195 poll_threads(); 2196 2197 memset(&expected[0], 0xa5, 1024); 2198 memset(&expected[1024], 0x5a, 1024); 2199 memset(&expected[2048], 0xfe, 1024); 2200 memset(&expected[3072], 0xed, 1024); 2201 CU_ASSERT_EQUAL(completed, 5); 2202 CU_ASSERT(ut_seq.complete); 2203 CU_ASSERT_EQUAL(ut_seq.status, 0); 2204 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2205 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2206 2207 /* Check the same but this time with a decompress operation on part of the buffer (512B 2208 * offset) */ 2209 memset(srcbuf, 0x0, 4096); 2210 memset(dstbuf, 0x0, 4096); 2211 completed = 0; 2212 seq = NULL; 2213 2214 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2215 CU_ASSERT_EQUAL(rc, 0); 2216 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2217 2218 /* Fill in each 1K of the buffer with different pattern */ 2219 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2220 ut_sequence_step_cb, &completed); 2221 CU_ASSERT_EQUAL(rc, 0); 2222 2223 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2224 0x5a, 0, ut_sequence_step_cb, &completed); 2225 CU_ASSERT_EQUAL(rc, 0); 2226 2227 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2228 0xfe, 0, ut_sequence_step_cb, &completed); 2229 CU_ASSERT_EQUAL(rc, 0); 2230 2231 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2232 0xed, 0, ut_sequence_step_cb, &completed); 2233 CU_ASSERT_EQUAL(rc, 0); 2234 2235 rc = spdk_accel_get_buf(ioch, 3072, &buf[1], &domain[1], &domain_ctx[1]); 2236 CU_ASSERT_EQUAL(rc, 0); 2237 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2238 2239 src_iovs[0].iov_base = (char *)buf[0] + 512; 2240 src_iovs[0].iov_len = 3072; 2241 dst_iovs[0].iov_base = (char *)buf[1] + 256; 2242 dst_iovs[0].iov_len = 3072; 2243 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2244 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2245 ut_sequence_step_cb, &completed); 2246 2247 src_iovs[1].iov_base = (char *)buf[1] + 256; 2248 src_iovs[1].iov_len = 3072; 2249 dst_iovs[1].iov_base = dstbuf; 2250 dst_iovs[1].iov_len = 3072; 2251 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2252 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2253 ut_sequence_step_cb, &completed); 2254 2255 ut_seq.complete = false; 2256 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2257 2258 poll_threads(); 2259 2260 memset(&expected[0], 0xa5, 512); 2261 memset(&expected[512], 0x5a, 1024); 2262 memset(&expected[1536], 0xfe, 1024); 2263 memset(&expected[2560], 0xed, 512); 2264 CU_ASSERT_EQUAL(completed, 6); 2265 CU_ASSERT(ut_seq.complete); 2266 CU_ASSERT_EQUAL(ut_seq.status, 0); 2267 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 3072), 0); 2268 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2269 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2270 2271 /* Check that if iobuf pool is empty, the sequence processing will wait until a buffer is 2272 * available 2273 */ 2274 accel_ch = spdk_io_channel_get_ctx(ioch); 2275 small_cache_count = accel_ch->iobuf.small.cache_count; 2276 STAILQ_INIT(&small_cache); 2277 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2278 accel_ch->iobuf.small.cache_count = 0; 2279 accel_ch->iobuf.small.cache_size = 0; 2280 g_iobuf.small_pool_count = 0; 2281 2282 /* First allocate a single buffer used by two operations */ 2283 memset(srcbuf, 0x0, 4096); 2284 memset(dstbuf, 0x0, 4096); 2285 memset(expected, 0xa5, 4096); 2286 completed = 0; 2287 seq = NULL; 2288 2289 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2290 CU_ASSERT_EQUAL(rc, 0); 2291 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2292 2293 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0xa5, 0, 2294 ut_sequence_step_cb, &completed); 2295 CU_ASSERT_EQUAL(rc, 0); 2296 2297 src_iovs[0].iov_base = buf[0]; 2298 src_iovs[0].iov_len = 4096; 2299 dst_iovs[0].iov_base = dstbuf; 2300 dst_iovs[0].iov_len = 4096; 2301 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2302 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2303 ut_sequence_step_cb, &completed); 2304 2305 ut_seq.complete = false; 2306 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2307 2308 poll_threads(); 2309 2310 CU_ASSERT_EQUAL(completed, 0); 2311 CU_ASSERT(!ut_seq.complete); 2312 2313 /* Get a buffer and return it to the pool to trigger the sequence to finish */ 2314 g_iobuf.small_pool_count = 1; 2315 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2316 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2317 CU_ASSERT(g_iobuf.small_pool_count == 0); 2318 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2319 2320 poll_threads(); 2321 2322 CU_ASSERT_EQUAL(completed, 2); 2323 CU_ASSERT(ut_seq.complete); 2324 CU_ASSERT_EQUAL(ut_seq.status, 0); 2325 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2326 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2327 2328 /* Return the buffers back to the cache */ 2329 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2330 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2331 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2332 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2333 small_cache_count++; 2334 } 2335 accel_ch->iobuf.small.cache_count = 0; 2336 g_iobuf.small_pool_count = 0; 2337 2338 /* Check a bit more complex scenario, with two buffers in the sequence */ 2339 memset(srcbuf, 0x0, 4096); 2340 memset(dstbuf, 0x0, 4096); 2341 memset(expected, 0x5a, 4096); 2342 completed = 0; 2343 seq = NULL; 2344 2345 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2346 CU_ASSERT_EQUAL(rc, 0); 2347 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2348 2349 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2350 ut_sequence_step_cb, &completed); 2351 CU_ASSERT_EQUAL(rc, 0); 2352 2353 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2354 CU_ASSERT_EQUAL(rc, 0); 2355 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2356 2357 src_iovs[0].iov_base = buf[0]; 2358 src_iovs[0].iov_len = 4096; 2359 dst_iovs[0].iov_base = buf[1]; 2360 dst_iovs[0].iov_len = 4096; 2361 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2362 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2363 ut_sequence_step_cb, &completed); 2364 CU_ASSERT_EQUAL(rc, 0); 2365 2366 src_iovs[1].iov_base = buf[1]; 2367 src_iovs[1].iov_len = 4096; 2368 dst_iovs[1].iov_base = dstbuf; 2369 dst_iovs[1].iov_len = 4096; 2370 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2371 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2372 ut_sequence_step_cb, &completed); 2373 CU_ASSERT_EQUAL(rc, 0); 2374 2375 ut_seq.complete = false; 2376 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2377 2378 poll_threads(); 2379 2380 CU_ASSERT_EQUAL(completed, 0); 2381 CU_ASSERT(!ut_seq.complete); 2382 2383 g_iobuf.small_pool_count = 1; 2384 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2385 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2386 g_iobuf.small_pool_count = 0; 2387 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2388 2389 /* One buffer is not enough to finish the whole sequence */ 2390 poll_threads(); 2391 CU_ASSERT(!ut_seq.complete); 2392 2393 g_iobuf.small_pool_count = 1; 2394 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2395 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2396 g_iobuf.small_pool_count = 0; 2397 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2398 2399 poll_threads(); 2400 2401 CU_ASSERT_EQUAL(completed, 3); 2402 CU_ASSERT(ut_seq.complete); 2403 CU_ASSERT_EQUAL(ut_seq.status, 0); 2404 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2405 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2406 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2407 2408 /* Return the buffers back to the cache */ 2409 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2410 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2411 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2412 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2413 small_cache_count++; 2414 } 2415 accel_ch->iobuf.small.cache_count = 0; 2416 2417 g_iobuf.small_pool_count = 32; 2418 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2419 accel_ch->iobuf.small.cache_count = small_cache_count; 2420 2421 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2422 g_modules_opc[i] = modules[i]; 2423 } 2424 2425 ut_clear_operations(); 2426 spdk_put_io_channel(ioch); 2427 poll_threads(); 2428 } 2429 2430 static void 2431 ut_domain_ctx_init(struct ut_domain_ctx *ctx, void *base, size_t len, struct iovec *expected) 2432 { 2433 ctx->iov.iov_base = base; 2434 ctx->iov.iov_len = len; 2435 ctx->expected = *expected; 2436 ctx->pull_submit_status = 0; 2437 ctx->push_submit_status = 0; 2438 ctx->pull_complete_status = 0; 2439 ctx->push_complete_status = 0; 2440 } 2441 2442 static void 2443 test_sequence_memory_domain(void) 2444 { 2445 struct spdk_accel_sequence *seq = NULL; 2446 struct spdk_io_channel *ioch; 2447 struct accel_io_channel *accel_ch; 2448 struct ut_sequence ut_seq; 2449 struct ut_domain_ctx domctx[4]; 2450 struct spdk_iobuf_buffer *cache_entry; 2451 struct accel_module modules[ACCEL_OPC_LAST]; 2452 struct spdk_memory_domain *accel_domain; 2453 spdk_iobuf_buffer_stailq_t small_cache; 2454 char srcbuf[4096], dstbuf[4096], expected[4096], tmp[4096]; 2455 struct iovec src_iovs[2], dst_iovs[2]; 2456 uint32_t small_cache_count; 2457 void *accel_buf, *accel_domain_ctx, *iobuf_buf; 2458 int i, rc, completed; 2459 2460 ioch = spdk_accel_get_io_channel(); 2461 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2462 2463 /* Override the submit_tasks function */ 2464 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2465 g_module.supports_memory_domains = false; 2466 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2467 modules[i] = g_modules_opc[i]; 2468 g_modules_opc[i] = g_module; 2469 } 2470 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2471 * decompress operations together in one sequence. 2472 */ 2473 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2474 g_seq_operations[ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2475 g_seq_operations[ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2476 2477 /* First check the simplest case - single fill operation with dstbuf in domain */ 2478 memset(expected, 0xa5, sizeof(expected)); 2479 memset(dstbuf, 0x0, sizeof(dstbuf)); 2480 completed = 0; 2481 seq = NULL; 2482 2483 /* Use some garbage pointer as dst and use domain ctx to get the actual dstbuf */ 2484 dst_iovs[0].iov_base = (void *)0xcafebabe; 2485 dst_iovs[0].iov_len = sizeof(dstbuf); 2486 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2487 2488 rc = spdk_accel_append_fill(&seq, ioch, dst_iovs[0].iov_base, dst_iovs[0].iov_len, 2489 g_ut_domain, &domctx[0], 0xa5, 0, 2490 ut_sequence_step_cb, &completed); 2491 CU_ASSERT_EQUAL(rc, 0); 2492 2493 ut_seq.complete = false; 2494 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2495 2496 poll_threads(); 2497 CU_ASSERT_EQUAL(completed, 1); 2498 CU_ASSERT(ut_seq.complete); 2499 CU_ASSERT_EQUAL(ut_seq.status, 0); 2500 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2501 2502 /* Check operation with both srcbuf and dstbuf in remote memory domain */ 2503 memset(expected, 0x5a, sizeof(expected)); 2504 memset(dstbuf, 0x0, sizeof(dstbuf)); 2505 memset(srcbuf, 0x5a, sizeof(dstbuf)); 2506 completed = 0; 2507 seq = NULL; 2508 2509 src_iovs[0].iov_base = (void *)0xcafebabe; 2510 src_iovs[0].iov_len = sizeof(srcbuf); 2511 dst_iovs[0].iov_base = (void *)0xbeefdead; 2512 dst_iovs[0].iov_len = sizeof(dstbuf); 2513 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2514 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2515 2516 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2517 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2518 ut_sequence_step_cb, &completed); 2519 CU_ASSERT_EQUAL(rc, 0); 2520 2521 ut_seq.complete = false; 2522 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2523 2524 poll_threads(); 2525 CU_ASSERT_EQUAL(completed, 1); 2526 CU_ASSERT(ut_seq.complete); 2527 CU_ASSERT_EQUAL(ut_seq.status, 0); 2528 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2529 2530 /* Two operations using a buffer in remote domain */ 2531 memset(expected, 0xa5, sizeof(expected)); 2532 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2533 memset(tmp, 0x0, sizeof(tmp)); 2534 memset(dstbuf, 0x0, sizeof(dstbuf)); 2535 completed = 0; 2536 seq = NULL; 2537 2538 src_iovs[0].iov_base = (void *)0xcafebabe; 2539 src_iovs[0].iov_len = sizeof(srcbuf); 2540 dst_iovs[0].iov_base = (void *)0xbeefdead; 2541 dst_iovs[0].iov_len = sizeof(tmp); 2542 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2543 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2544 2545 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2546 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2547 ut_sequence_step_cb, &completed); 2548 CU_ASSERT_EQUAL(rc, 0); 2549 2550 src_iovs[1].iov_base = (void *)0xbeefdead; 2551 src_iovs[1].iov_len = sizeof(tmp); 2552 dst_iovs[1].iov_base = (void *)0xa5a5a5a5; 2553 dst_iovs[1].iov_len = sizeof(dstbuf); 2554 ut_domain_ctx_init(&domctx[2], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2555 ut_domain_ctx_init(&domctx[3], tmp, sizeof(tmp), &src_iovs[1]); 2556 2557 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[2], 2558 &src_iovs[1], 1, g_ut_domain, &domctx[3], 0, 2559 ut_sequence_step_cb, &completed); 2560 CU_ASSERT_EQUAL(rc, 0); 2561 2562 ut_seq.complete = false; 2563 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2564 2565 poll_threads(); 2566 CU_ASSERT_EQUAL(completed, 2); 2567 CU_ASSERT(ut_seq.complete); 2568 CU_ASSERT_EQUAL(ut_seq.status, 0); 2569 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2570 2571 /* Use buffer in remote memory domain and accel buffer at the same time */ 2572 memset(expected, 0x5a, sizeof(expected)); 2573 memset(srcbuf, 0x5a, sizeof(expected)); 2574 memset(dstbuf, 0x0, sizeof(dstbuf)); 2575 completed = 0; 2576 seq = NULL; 2577 2578 rc = spdk_accel_get_buf(ioch, sizeof(dstbuf), &accel_buf, &accel_domain, &accel_domain_ctx); 2579 CU_ASSERT_EQUAL(rc, 0); 2580 2581 src_iovs[0].iov_base = (void *)0xfeedbeef; 2582 src_iovs[0].iov_len = sizeof(srcbuf); 2583 dst_iovs[0].iov_base = accel_buf; 2584 dst_iovs[0].iov_len = sizeof(dstbuf); 2585 ut_domain_ctx_init(&domctx[0], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2586 2587 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, accel_domain, accel_domain_ctx, 2588 &src_iovs[0], 1, g_ut_domain, &domctx[0], 0, 2589 ut_sequence_step_cb, &completed); 2590 CU_ASSERT_EQUAL(rc, 0); 2591 2592 src_iovs[1].iov_base = accel_buf; 2593 src_iovs[1].iov_len = sizeof(dstbuf); 2594 dst_iovs[1].iov_base = (void *)0xbeeffeed; 2595 dst_iovs[1].iov_len = sizeof(dstbuf); 2596 ut_domain_ctx_init(&domctx[1], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2597 2598 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[1], 2599 &src_iovs[1], 1, accel_domain, accel_domain_ctx, 0, 2600 ut_sequence_step_cb, &completed); 2601 CU_ASSERT_EQUAL(rc, 0); 2602 2603 ut_seq.complete = false; 2604 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2605 2606 poll_threads(); 2607 CU_ASSERT_EQUAL(completed, 2); 2608 CU_ASSERT(ut_seq.complete); 2609 CU_ASSERT_EQUAL(ut_seq.status, 0); 2610 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2611 spdk_accel_put_buf(ioch, accel_buf, accel_domain, accel_domain_ctx); 2612 2613 /* Check that a sequence with memory domains is correctly executed if buffers are not 2614 * immediately available */ 2615 memset(expected, 0xa5, sizeof(expected)); 2616 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2617 memset(dstbuf, 0x0, sizeof(dstbuf)); 2618 completed = 0; 2619 seq = NULL; 2620 /* Make sure the buffer pool is empty */ 2621 accel_ch = spdk_io_channel_get_ctx(ioch); 2622 small_cache_count = accel_ch->iobuf.small.cache_count; 2623 STAILQ_INIT(&small_cache); 2624 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2625 accel_ch->iobuf.small.cache_count = 0; 2626 g_iobuf.small_pool_count = 0; 2627 2628 src_iovs[0].iov_base = (void *)0xdeadbeef; 2629 src_iovs[0].iov_len = sizeof(srcbuf); 2630 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2631 dst_iovs[0].iov_len = sizeof(dstbuf); 2632 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2633 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2634 2635 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2636 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2637 ut_sequence_step_cb, &completed); 2638 CU_ASSERT_EQUAL(rc, 0); 2639 2640 ut_seq.complete = false; 2641 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2642 2643 poll_threads(); 2644 CU_ASSERT_EQUAL(completed, 0); 2645 CU_ASSERT(!ut_seq.complete); 2646 2647 /* Get a buffer and return it to the pool to trigger the sequence to resume. It shouldn't 2648 * be able to complete, as it needs two buffers */ 2649 g_iobuf.small_pool_count = 1; 2650 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2651 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2652 g_iobuf.small_pool_count = 0; 2653 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2654 2655 CU_ASSERT_EQUAL(completed, 0); 2656 CU_ASSERT(!ut_seq.complete); 2657 2658 /* Return another buffer, this time the sequence should finish */ 2659 g_iobuf.small_pool_count = 1; 2660 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2661 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2662 g_iobuf.small_pool_count = 0; 2663 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2664 2665 poll_threads(); 2666 CU_ASSERT_EQUAL(completed, 1); 2667 CU_ASSERT(ut_seq.complete); 2668 CU_ASSERT_EQUAL(ut_seq.status, 0); 2669 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2670 2671 /* Return the buffers back to the cache */ 2672 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2673 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2674 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2675 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2676 small_cache_count++; 2677 } 2678 accel_ch->iobuf.small.cache_count = 0; 2679 2680 g_iobuf.small_pool_count = 32; 2681 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2682 accel_ch->iobuf.small.cache_count = small_cache_count; 2683 2684 /* Check error cases, starting with an error from spdk_memory_domain_pull_data() */ 2685 completed = 0; 2686 seq = NULL; 2687 2688 src_iovs[0].iov_base = (void *)0xdeadbeef; 2689 src_iovs[0].iov_len = sizeof(srcbuf); 2690 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2691 dst_iovs[0].iov_len = sizeof(dstbuf); 2692 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2693 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2694 domctx[1].pull_submit_status = -E2BIG; 2695 2696 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2697 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2698 ut_sequence_step_cb, &completed); 2699 CU_ASSERT_EQUAL(rc, 0); 2700 2701 ut_seq.complete = false; 2702 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2703 2704 CU_ASSERT_EQUAL(completed, 1); 2705 CU_ASSERT(ut_seq.complete); 2706 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 2707 2708 /* Check completion error from spdk_memory_domain_pull_data() */ 2709 completed = 0; 2710 seq = NULL; 2711 2712 src_iovs[0].iov_base = (void *)0xdeadbeef; 2713 src_iovs[0].iov_len = sizeof(srcbuf); 2714 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2715 dst_iovs[0].iov_len = sizeof(dstbuf); 2716 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2717 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2718 domctx[1].pull_complete_status = -EACCES; 2719 2720 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2721 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2722 ut_sequence_step_cb, &completed); 2723 CU_ASSERT_EQUAL(rc, 0); 2724 2725 ut_seq.complete = false; 2726 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2727 2728 CU_ASSERT_EQUAL(completed, 1); 2729 CU_ASSERT(ut_seq.complete); 2730 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 2731 2732 /* Check submission error from spdk_memory_domain_push_data() */ 2733 completed = 0; 2734 seq = NULL; 2735 2736 src_iovs[0].iov_base = (void *)0xdeadbeef; 2737 src_iovs[0].iov_len = sizeof(srcbuf); 2738 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2739 dst_iovs[0].iov_len = sizeof(dstbuf); 2740 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2741 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2742 domctx[0].push_submit_status = -EADDRINUSE; 2743 2744 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2745 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2746 ut_sequence_step_cb, &completed); 2747 CU_ASSERT_EQUAL(rc, 0); 2748 2749 ut_seq.complete = false; 2750 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2751 2752 CU_ASSERT_EQUAL(completed, 1); 2753 CU_ASSERT(ut_seq.complete); 2754 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 2755 2756 /* Check completion error from spdk_memory_domain_push_data() */ 2757 completed = 0; 2758 seq = NULL; 2759 2760 src_iovs[0].iov_base = (void *)0xdeadbeef; 2761 src_iovs[0].iov_len = sizeof(srcbuf); 2762 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2763 dst_iovs[0].iov_len = sizeof(dstbuf); 2764 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2765 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2766 domctx[0].push_complete_status = -EADDRNOTAVAIL; 2767 2768 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2769 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2770 ut_sequence_step_cb, &completed); 2771 CU_ASSERT_EQUAL(rc, 0); 2772 2773 ut_seq.complete = false; 2774 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2775 2776 CU_ASSERT_EQUAL(completed, 1); 2777 CU_ASSERT(ut_seq.complete); 2778 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 2779 2780 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2781 g_modules_opc[i] = modules[i]; 2782 } 2783 2784 ut_clear_operations(); 2785 spdk_put_io_channel(ioch); 2786 poll_threads(); 2787 } 2788 2789 static int 2790 ut_submit_decompress_memory_domain(struct spdk_io_channel *ch, struct spdk_accel_task *task) 2791 { 2792 struct ut_domain_ctx *ctx; 2793 struct iovec *src_iovs, *dst_iovs; 2794 uint32_t src_iovcnt, dst_iovcnt; 2795 2796 src_iovs = task->s.iovs; 2797 dst_iovs = task->d.iovs; 2798 src_iovcnt = task->s.iovcnt; 2799 dst_iovcnt = task->d.iovcnt; 2800 2801 if (task->src_domain != NULL) { 2802 ctx = task->src_domain_ctx; 2803 CU_ASSERT_EQUAL(memcmp(task->s.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2804 src_iovs = &ctx->iov; 2805 src_iovcnt = 1; 2806 } 2807 2808 if (task->dst_domain != NULL) { 2809 ctx = task->dst_domain_ctx; 2810 CU_ASSERT_EQUAL(memcmp(task->d.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2811 dst_iovs = &ctx->iov; 2812 dst_iovcnt = 1; 2813 } 2814 2815 spdk_iovcpy(src_iovs, src_iovcnt, dst_iovs, dst_iovcnt); 2816 spdk_accel_task_complete(task, 0); 2817 2818 return 0; 2819 } 2820 2821 static void 2822 test_sequence_module_memory_domain(void) 2823 { 2824 struct spdk_accel_sequence *seq = NULL; 2825 struct spdk_io_channel *ioch; 2826 struct accel_module modules[ACCEL_OPC_LAST]; 2827 struct spdk_memory_domain *accel_domain; 2828 struct ut_sequence ut_seq; 2829 struct ut_domain_ctx domctx[2]; 2830 struct iovec src_iovs[2], dst_iovs[2]; 2831 void *buf, *accel_domain_ctx; 2832 char srcbuf[4096], dstbuf[4096], tmp[4096], expected[4096]; 2833 int i, rc, completed; 2834 2835 ioch = spdk_accel_get_io_channel(); 2836 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2837 2838 /* Override the submit_tasks function */ 2839 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2840 g_module.supports_memory_domains = true; 2841 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2842 modules[i] = g_modules_opc[i]; 2843 g_modules_opc[i] = g_module; 2844 } 2845 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress_memory_domain; 2846 g_seq_operations[ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2847 2848 /* Check a sequence with both buffers in memory domains */ 2849 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2850 memset(expected, 0xa5, sizeof(expected)); 2851 memset(dstbuf, 0, sizeof(dstbuf)); 2852 seq = NULL; 2853 completed = 0; 2854 2855 src_iovs[0].iov_base = (void *)0xcafebabe; 2856 src_iovs[0].iov_len = sizeof(srcbuf); 2857 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2858 dst_iovs[0].iov_len = sizeof(dstbuf); 2859 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2860 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2861 2862 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2863 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2864 ut_sequence_step_cb, &completed); 2865 CU_ASSERT_EQUAL(rc, 0); 2866 2867 ut_seq.complete = false; 2868 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2869 2870 poll_threads(); 2871 2872 CU_ASSERT_EQUAL(completed, 1); 2873 CU_ASSERT(ut_seq.complete); 2874 CU_ASSERT_EQUAL(ut_seq.status, 0); 2875 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2876 2877 /* Check two operations each with a single buffer in memory domain */ 2878 memset(srcbuf, 0x5a, sizeof(srcbuf)); 2879 memset(expected, 0x5a, sizeof(expected)); 2880 memset(dstbuf, 0, sizeof(dstbuf)); 2881 memset(tmp, 0, sizeof(tmp)); 2882 seq = NULL; 2883 completed = 0; 2884 2885 src_iovs[0].iov_base = srcbuf; 2886 src_iovs[0].iov_len = sizeof(srcbuf); 2887 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2888 dst_iovs[0].iov_len = sizeof(tmp); 2889 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2890 2891 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2892 &src_iovs[0], 1, NULL, NULL, 0, 2893 ut_sequence_step_cb, &completed); 2894 CU_ASSERT_EQUAL(rc, 0); 2895 2896 src_iovs[1].iov_base = (void *)0xfeedbeef; 2897 src_iovs[1].iov_len = sizeof(tmp); 2898 dst_iovs[1].iov_base = dstbuf; 2899 dst_iovs[1].iov_len = sizeof(dstbuf); 2900 ut_domain_ctx_init(&domctx[1], tmp, sizeof(tmp), &src_iovs[1]); 2901 2902 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2903 &src_iovs[1], 1, g_ut_domain, &domctx[1], 0, 2904 ut_sequence_step_cb, &completed); 2905 CU_ASSERT_EQUAL(rc, 0); 2906 2907 ut_seq.complete = false; 2908 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2909 2910 poll_threads(); 2911 2912 CU_ASSERT_EQUAL(completed, 2); 2913 CU_ASSERT(ut_seq.complete); 2914 CU_ASSERT_EQUAL(ut_seq.status, 0); 2915 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2916 2917 /* Check a sequence with an accel buffer and a buffer in a regular memory domain */ 2918 memset(expected, 0xa5, sizeof(expected)); 2919 memset(dstbuf, 0, sizeof(dstbuf)); 2920 memset(tmp, 0, sizeof(tmp)); 2921 seq = NULL; 2922 completed = 0; 2923 2924 rc = spdk_accel_get_buf(ioch, 4096, &buf, &accel_domain, &accel_domain_ctx); 2925 CU_ASSERT_EQUAL(rc, 0); 2926 2927 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, accel_domain, accel_domain_ctx, 2928 0xa5, 0, ut_sequence_step_cb, &completed); 2929 CU_ASSERT_EQUAL(rc, 0); 2930 2931 src_iovs[0].iov_base = buf; 2932 src_iovs[0].iov_len = 4096; 2933 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2934 dst_iovs[0].iov_len = sizeof(dstbuf); 2935 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2936 2937 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2938 &src_iovs[0], 1, accel_domain, accel_domain_ctx, 0, 2939 ut_sequence_step_cb, &completed); 2940 CU_ASSERT_EQUAL(rc, 0); 2941 2942 ut_seq.complete = false; 2943 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2944 2945 poll_threads(); 2946 2947 CU_ASSERT_EQUAL(completed, 2); 2948 CU_ASSERT(ut_seq.complete); 2949 CU_ASSERT_EQUAL(ut_seq.status, 0); 2950 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2951 2952 spdk_accel_put_buf(ioch, buf, accel_domain, accel_domain_ctx); 2953 2954 g_module.supports_memory_domains = false; 2955 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2956 g_modules_opc[i] = modules[i]; 2957 } 2958 2959 ut_clear_operations(); 2960 spdk_put_io_channel(ioch); 2961 poll_threads(); 2962 } 2963 2964 #ifdef SPDK_CONFIG_ISAL_CRYPTO 2965 static void 2966 ut_encrypt_cb(void *cb_arg, int status) 2967 { 2968 int *completed = cb_arg; 2969 2970 CU_ASSERT_EQUAL(status, 0); 2971 2972 *completed = 1; 2973 } 2974 2975 static void 2976 test_sequence_crypto(void) 2977 { 2978 struct spdk_accel_sequence *seq = NULL; 2979 struct spdk_io_channel *ioch; 2980 struct spdk_accel_crypto_key *key; 2981 struct spdk_accel_crypto_key_create_param key_params = { 2982 .cipher = "AES_XTS", 2983 .hex_key = "00112233445566778899aabbccddeeff", 2984 .hex_key2 = "ffeeddccbbaa99887766554433221100", 2985 .key_name = "ut_key", 2986 }; 2987 struct ut_sequence ut_seq; 2988 unsigned char buf[4096], encrypted[4096] = {}, data[4096], tmp[3][4096]; 2989 struct iovec src_iovs[4], dst_iovs[4]; 2990 int rc, completed = 0; 2991 size_t i; 2992 2993 ioch = spdk_accel_get_io_channel(); 2994 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2995 2996 rc = spdk_accel_crypto_key_create(&key_params); 2997 CU_ASSERT_EQUAL(rc, 0); 2998 key = spdk_accel_crypto_key_get(key_params.key_name); 2999 SPDK_CU_ASSERT_FATAL(key != NULL); 3000 3001 for (i = 0; i < sizeof(data); ++i) { 3002 data[i] = (uint8_t)i & 0xff; 3003 } 3004 3005 dst_iovs[0].iov_base = encrypted; 3006 dst_iovs[0].iov_len = sizeof(encrypted); 3007 src_iovs[0].iov_base = data; 3008 src_iovs[0].iov_len = sizeof(data); 3009 rc = spdk_accel_submit_encrypt(ioch, key, &dst_iovs[0], 1, &src_iovs[0], 1, 0, 4096, 0, 3010 ut_encrypt_cb, &completed); 3011 CU_ASSERT_EQUAL(rc, 0); 3012 3013 while (!completed) { 3014 poll_threads(); 3015 } 3016 3017 /* Verify that encryption operation in a sequence produces the same result */ 3018 seq = NULL; 3019 completed = 0; 3020 3021 dst_iovs[0].iov_base = tmp[0]; 3022 dst_iovs[0].iov_len = sizeof(tmp[0]); 3023 src_iovs[0].iov_base = data; 3024 src_iovs[0].iov_len = sizeof(data); 3025 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3026 &src_iovs[0], 1, NULL, NULL, 0, 3027 ut_sequence_step_cb, &completed); 3028 CU_ASSERT_EQUAL(rc, 0); 3029 3030 dst_iovs[1].iov_base = tmp[1]; 3031 dst_iovs[1].iov_len = sizeof(tmp[1]); 3032 src_iovs[1].iov_base = tmp[0]; 3033 src_iovs[1].iov_len = sizeof(tmp[0]); 3034 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3035 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3036 ut_sequence_step_cb, &completed); 3037 CU_ASSERT_EQUAL(rc, 0); 3038 3039 dst_iovs[2].iov_base = buf; 3040 dst_iovs[2].iov_len = sizeof(buf); 3041 src_iovs[2].iov_base = tmp[1]; 3042 src_iovs[2].iov_len = sizeof(tmp[1]); 3043 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3044 &src_iovs[2], 1, NULL, NULL, 0, 3045 ut_sequence_step_cb, &completed); 3046 CU_ASSERT_EQUAL(rc, 0); 3047 3048 ut_seq.complete = false; 3049 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3050 3051 poll_threads(); 3052 3053 CU_ASSERT_EQUAL(completed, 3); 3054 CU_ASSERT(ut_seq.complete); 3055 CU_ASSERT_EQUAL(ut_seq.status, 0); 3056 CU_ASSERT_EQUAL(memcmp(buf, encrypted, sizeof(buf)), 0); 3057 3058 /* Check that decryption produces the original buffer */ 3059 seq = NULL; 3060 completed = 0; 3061 memset(buf, 0, sizeof(buf)); 3062 3063 dst_iovs[0].iov_base = tmp[0]; 3064 dst_iovs[0].iov_len = sizeof(tmp[0]); 3065 src_iovs[0].iov_base = encrypted; 3066 src_iovs[0].iov_len = sizeof(encrypted); 3067 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3068 &src_iovs[0], 1, NULL, NULL, 0, 3069 ut_sequence_step_cb, &completed); 3070 CU_ASSERT_EQUAL(rc, 0); 3071 3072 dst_iovs[1].iov_base = tmp[1]; 3073 dst_iovs[1].iov_len = sizeof(tmp[1]); 3074 src_iovs[1].iov_base = tmp[0]; 3075 src_iovs[1].iov_len = sizeof(tmp[0]); 3076 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3077 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3078 ut_sequence_step_cb, &completed); 3079 CU_ASSERT_EQUAL(rc, 0); 3080 3081 dst_iovs[2].iov_base = buf; 3082 dst_iovs[2].iov_len = sizeof(buf); 3083 src_iovs[2].iov_base = tmp[1]; 3084 src_iovs[2].iov_len = sizeof(tmp[1]); 3085 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3086 &src_iovs[2], 1, NULL, NULL, 0, 3087 ut_sequence_step_cb, &completed); 3088 CU_ASSERT_EQUAL(rc, 0); 3089 3090 ut_seq.complete = false; 3091 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3092 3093 poll_threads(); 3094 3095 CU_ASSERT_EQUAL(completed, 3); 3096 CU_ASSERT(ut_seq.complete); 3097 CU_ASSERT_EQUAL(ut_seq.status, 0); 3098 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3099 3100 /* Check encrypt + decrypt in a single sequence */ 3101 seq = NULL; 3102 completed = 0; 3103 memset(buf, 0, sizeof(buf)); 3104 3105 dst_iovs[0].iov_base = tmp[0]; 3106 dst_iovs[0].iov_len = sizeof(tmp[0]); 3107 src_iovs[0].iov_base = data; 3108 src_iovs[0].iov_len = sizeof(data); 3109 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3110 &src_iovs[0], 1, NULL, NULL, 0, 3111 ut_sequence_step_cb, &completed); 3112 CU_ASSERT_EQUAL(rc, 0); 3113 3114 dst_iovs[1].iov_base = tmp[1]; 3115 dst_iovs[1].iov_len = sizeof(tmp[1]); 3116 src_iovs[1].iov_base = tmp[0]; 3117 src_iovs[1].iov_len = sizeof(tmp[0]); 3118 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3119 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3120 ut_sequence_step_cb, &completed); 3121 CU_ASSERT_EQUAL(rc, 0); 3122 3123 3124 dst_iovs[2].iov_base = tmp[2]; 3125 dst_iovs[2].iov_len = sizeof(tmp[2]); 3126 src_iovs[2].iov_base = tmp[1]; 3127 src_iovs[2].iov_len = sizeof(tmp[1]); 3128 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[2], 1, NULL, NULL, 3129 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3130 ut_sequence_step_cb, &completed); 3131 CU_ASSERT_EQUAL(rc, 0); 3132 3133 dst_iovs[3].iov_base = buf; 3134 dst_iovs[3].iov_len = sizeof(buf); 3135 src_iovs[3].iov_base = tmp[2]; 3136 src_iovs[3].iov_len = sizeof(tmp[2]); 3137 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 3138 &src_iovs[3], 1, NULL, NULL, 0, 3139 ut_sequence_step_cb, &completed); 3140 CU_ASSERT_EQUAL(rc, 0); 3141 3142 ut_seq.complete = false; 3143 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3144 3145 poll_threads(); 3146 3147 CU_ASSERT_EQUAL(completed, 4); 3148 CU_ASSERT(ut_seq.complete); 3149 CU_ASSERT_EQUAL(ut_seq.status, 0); 3150 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3151 3152 rc = spdk_accel_crypto_key_destroy(key); 3153 CU_ASSERT_EQUAL(rc, 0); 3154 spdk_put_io_channel(ioch); 3155 poll_threads(); 3156 } 3157 #endif /* SPDK_CONFIG_ISAL_CRYPTO */ 3158 3159 static int 3160 ut_submit_crypto(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3161 { 3162 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3163 3164 spdk_accel_task_complete(task, 0); 3165 3166 return 0; 3167 } 3168 3169 struct ut_driver_operation { 3170 int complete_status; 3171 int submit_status; 3172 int count; 3173 bool supported; 3174 }; 3175 3176 static struct ut_driver_operation g_drv_operations[ACCEL_OPC_LAST]; 3177 static bool g_ut_driver_async_continue; 3178 3179 static void 3180 ut_driver_async_continue(void *arg) 3181 { 3182 struct spdk_accel_sequence *seq = arg; 3183 3184 spdk_accel_sequence_continue(seq); 3185 } 3186 3187 static int 3188 ut_driver_execute_sequence(struct spdk_io_channel *ch, struct spdk_accel_sequence *seq) 3189 { 3190 struct spdk_accel_task *task; 3191 struct ut_driver_operation *drv_ops; 3192 3193 while ((task = spdk_accel_sequence_first_task(seq)) != NULL) { 3194 drv_ops = &g_drv_operations[task->op_code]; 3195 if (!drv_ops->supported) { 3196 break; 3197 } 3198 3199 drv_ops->count++; 3200 if (drv_ops->submit_status != 0) { 3201 return drv_ops->submit_status; 3202 } 3203 3204 if (drv_ops->complete_status != 0) { 3205 spdk_accel_task_complete(task, drv_ops->complete_status); 3206 break; 3207 } 3208 3209 switch (task->op_code) { 3210 case ACCEL_OPC_DECOMPRESS: 3211 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3212 break; 3213 case ACCEL_OPC_FILL: 3214 spdk_iov_memset(task->d.iovs, task->d.iovcnt, 3215 (int)(task->fill_pattern & 0xff)); 3216 break; 3217 default: 3218 CU_ASSERT(0 && "unexpected opcode"); 3219 break; 3220 } 3221 3222 spdk_accel_task_complete(task, 0); 3223 } 3224 3225 if (g_ut_driver_async_continue) { 3226 spdk_thread_send_msg(spdk_get_thread(), ut_driver_async_continue, seq); 3227 } else { 3228 spdk_accel_sequence_continue(seq); 3229 } 3230 3231 return 0; 3232 } 3233 3234 static struct spdk_io_channel * 3235 ut_driver_get_io_channel(void) 3236 { 3237 return (void *)0xdeadbeef; 3238 } 3239 3240 static struct spdk_accel_driver g_ut_driver = { 3241 .name = "ut", 3242 .execute_sequence = ut_driver_execute_sequence, 3243 .get_io_channel = ut_driver_get_io_channel, 3244 }; 3245 3246 SPDK_ACCEL_DRIVER_REGISTER(ut, &g_ut_driver); 3247 3248 static void 3249 test_sequence_driver(void) 3250 { 3251 struct spdk_accel_sequence *seq = NULL; 3252 struct spdk_io_channel *ioch; 3253 struct spdk_accel_crypto_key key = {}; 3254 struct ut_sequence ut_seq; 3255 struct accel_module modules[ACCEL_OPC_LAST]; 3256 char buf[4096], tmp[3][4096], expected[4096]; 3257 struct iovec src_iovs[3], dst_iovs[3]; 3258 int i, rc, completed = 0; 3259 3260 ioch = spdk_accel_get_io_channel(); 3261 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3262 rc = spdk_accel_set_driver("ut"); 3263 SPDK_CU_ASSERT_FATAL(rc == 0); 3264 3265 /* Override the submit_tasks function */ 3266 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3267 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3268 modules[i] = g_modules_opc[i]; 3269 g_modules_opc[i] = g_module; 3270 } 3271 /* Intercept crypto operations, as they should be executed by an accel module */ 3272 g_seq_operations[ACCEL_OPC_ENCRYPT].submit = ut_submit_crypto; 3273 g_seq_operations[ACCEL_OPC_DECRYPT].submit = ut_submit_crypto; 3274 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3275 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3276 g_seq_operations[ACCEL_OPC_FILL].count = 0; 3277 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3278 3279 g_drv_operations[ACCEL_OPC_FILL].supported = true; 3280 g_drv_operations[ACCEL_OPC_DECOMPRESS].supported = true; 3281 3282 /* First check a sequence that is fully executed using a driver, with the copy at the end 3283 * being removed */ 3284 seq = NULL; 3285 completed = 0; 3286 memset(buf, 0, sizeof(buf)); 3287 memset(tmp[0], 0, sizeof(tmp[0])); 3288 memset(tmp[1], 0, sizeof(tmp[1])); 3289 memset(&expected[0], 0xa5, 2048); 3290 memset(&expected[2048], 0xbe, 2048); 3291 3292 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], 2048, NULL, NULL, 0xa5, 0, 3293 ut_sequence_step_cb, &completed); 3294 CU_ASSERT_EQUAL(rc, 0); 3295 rc = spdk_accel_append_fill(&seq, ioch, &tmp[0][2048], 2048, NULL, NULL, 0xbe, 0, 3296 ut_sequence_step_cb, &completed); 3297 CU_ASSERT_EQUAL(rc, 0); 3298 3299 dst_iovs[0].iov_base = tmp[1]; 3300 dst_iovs[0].iov_len = sizeof(tmp[1]); 3301 src_iovs[0].iov_base = tmp[0]; 3302 src_iovs[0].iov_len = sizeof(tmp[0]); 3303 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3304 &src_iovs[0], 1, NULL, NULL, 0, 3305 ut_sequence_step_cb, &completed); 3306 CU_ASSERT_EQUAL(rc, 0); 3307 3308 dst_iovs[1].iov_base = buf; 3309 dst_iovs[1].iov_len = sizeof(buf); 3310 src_iovs[1].iov_base = tmp[1]; 3311 src_iovs[1].iov_len = sizeof(tmp[1]); 3312 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3313 &src_iovs[1], 1, NULL, NULL, 0, 3314 ut_sequence_step_cb, &completed); 3315 CU_ASSERT_EQUAL(rc, 0); 3316 3317 ut_seq.complete = false; 3318 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3319 3320 poll_threads(); 3321 3322 CU_ASSERT_EQUAL(completed, 4); 3323 CU_ASSERT(ut_seq.complete); 3324 CU_ASSERT_EQUAL(ut_seq.status, 0); 3325 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3326 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3327 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 3328 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 2); 3329 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3330 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3331 3332 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3333 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3334 3335 /* Check a sequence when the first two operations are executed by a driver, while the rest 3336 * is executed via modules */ 3337 seq = NULL; 3338 completed = 0; 3339 memset(buf, 0, sizeof(buf)); 3340 memset(tmp[0], 0, sizeof(tmp[0])); 3341 memset(tmp[1], 0, sizeof(tmp[1])); 3342 memset(tmp[2], 0, sizeof(tmp[2])); 3343 memset(&expected[0], 0xfe, 4096); 3344 3345 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 0, 3346 ut_sequence_step_cb, &completed); 3347 CU_ASSERT_EQUAL(rc, 0); 3348 3349 dst_iovs[0].iov_base = tmp[1]; 3350 dst_iovs[0].iov_len = sizeof(tmp[1]); 3351 src_iovs[0].iov_base = tmp[0]; 3352 src_iovs[0].iov_len = sizeof(tmp[0]); 3353 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3354 &src_iovs[0], 1, NULL, NULL, 0, 3355 ut_sequence_step_cb, &completed); 3356 CU_ASSERT_EQUAL(rc, 0); 3357 3358 dst_iovs[1].iov_base = tmp[2]; 3359 dst_iovs[1].iov_len = sizeof(tmp[2]); 3360 src_iovs[1].iov_base = tmp[1]; 3361 src_iovs[1].iov_len = sizeof(tmp[1]); 3362 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3363 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3364 ut_sequence_step_cb, &completed); 3365 CU_ASSERT_EQUAL(rc, 0); 3366 3367 dst_iovs[2].iov_base = buf; 3368 dst_iovs[2].iov_len = sizeof(buf); 3369 src_iovs[2].iov_base = tmp[2]; 3370 src_iovs[2].iov_len = sizeof(tmp[2]); 3371 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3372 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3373 ut_sequence_step_cb, &completed); 3374 CU_ASSERT_EQUAL(rc, 0); 3375 3376 ut_seq.complete = false; 3377 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3378 3379 poll_threads(); 3380 3381 CU_ASSERT_EQUAL(completed, 4); 3382 CU_ASSERT(ut_seq.complete); 3383 CU_ASSERT_EQUAL(ut_seq.status, 0); 3384 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3385 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3386 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3387 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3388 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3389 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3390 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3391 3392 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3393 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3394 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3395 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3396 3397 /* Check sequence when the first and last operations are executed through modules, while the 3398 * ones in the middle are executed by the driver */ 3399 seq = NULL; 3400 completed = 0; 3401 memset(buf, 0, sizeof(buf)); 3402 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3403 memset(tmp[1], 0, sizeof(tmp[1])); 3404 memset(tmp[2], 0, sizeof(tmp[2])); 3405 memset(&expected[0], 0xfe, 2048); 3406 memset(&expected[2048], 0xa5, 2048); 3407 3408 dst_iovs[0].iov_base = tmp[1]; 3409 dst_iovs[0].iov_len = sizeof(tmp[1]); 3410 src_iovs[0].iov_base = tmp[0]; 3411 src_iovs[0].iov_len = sizeof(tmp[0]); 3412 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3413 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3414 ut_sequence_step_cb, &completed); 3415 CU_ASSERT_EQUAL(rc, 0); 3416 3417 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xfe, 0, 3418 ut_sequence_step_cb, &completed); 3419 CU_ASSERT_EQUAL(rc, 0); 3420 3421 dst_iovs[1].iov_base = tmp[2]; 3422 dst_iovs[1].iov_len = sizeof(tmp[2]); 3423 src_iovs[1].iov_base = tmp[1]; 3424 src_iovs[1].iov_len = sizeof(tmp[1]); 3425 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3426 &src_iovs[1], 1, NULL, NULL, 0, 3427 ut_sequence_step_cb, &completed); 3428 CU_ASSERT_EQUAL(rc, 0); 3429 3430 dst_iovs[2].iov_base = buf; 3431 dst_iovs[2].iov_len = sizeof(buf); 3432 src_iovs[2].iov_base = tmp[2]; 3433 src_iovs[2].iov_len = sizeof(tmp[2]); 3434 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3435 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3436 ut_sequence_step_cb, &completed); 3437 CU_ASSERT_EQUAL(rc, 0); 3438 3439 ut_seq.complete = false; 3440 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3441 3442 poll_threads(); 3443 3444 CU_ASSERT_EQUAL(completed, 4); 3445 CU_ASSERT(ut_seq.complete); 3446 CU_ASSERT_EQUAL(ut_seq.status, 0); 3447 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3448 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3449 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3450 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3451 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3452 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3453 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3454 3455 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3456 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3457 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3458 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3459 3460 /* Check a sequence with operations executed by: module, driver, module, driver */ 3461 seq = NULL; 3462 completed = 0; 3463 memset(buf, 0, sizeof(buf)); 3464 memset(tmp[0], 0x5a, sizeof(tmp[0])); 3465 memset(tmp[1], 0, sizeof(tmp[1])); 3466 memset(tmp[2], 0, sizeof(tmp[2])); 3467 memset(&expected[0], 0xef, 2048); 3468 memset(&expected[2048], 0x5a, 2048); 3469 3470 dst_iovs[0].iov_base = tmp[1]; 3471 dst_iovs[0].iov_len = sizeof(tmp[1]); 3472 src_iovs[0].iov_base = tmp[0]; 3473 src_iovs[0].iov_len = sizeof(tmp[0]); 3474 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3475 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3476 ut_sequence_step_cb, &completed); 3477 CU_ASSERT_EQUAL(rc, 0); 3478 3479 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3480 ut_sequence_step_cb, &completed); 3481 CU_ASSERT_EQUAL(rc, 0); 3482 3483 dst_iovs[1].iov_base = tmp[2]; 3484 dst_iovs[1].iov_len = sizeof(tmp[2]); 3485 src_iovs[1].iov_base = tmp[1]; 3486 src_iovs[1].iov_len = sizeof(tmp[1]); 3487 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3488 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3489 ut_sequence_step_cb, &completed); 3490 CU_ASSERT_EQUAL(rc, 0); 3491 3492 dst_iovs[2].iov_base = buf; 3493 dst_iovs[2].iov_len = sizeof(buf); 3494 src_iovs[2].iov_base = tmp[2]; 3495 src_iovs[2].iov_len = sizeof(tmp[2]); 3496 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3497 &src_iovs[2], 1, NULL, NULL, 0, 3498 ut_sequence_step_cb, &completed); 3499 CU_ASSERT_EQUAL(rc, 0); 3500 3501 ut_seq.complete = false; 3502 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3503 3504 poll_threads(); 3505 3506 CU_ASSERT_EQUAL(completed, 4); 3507 CU_ASSERT(ut_seq.complete); 3508 CU_ASSERT_EQUAL(ut_seq.status, 0); 3509 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3510 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3511 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3512 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3513 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3514 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3515 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3516 3517 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3518 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3519 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3520 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3521 3522 /* Check that an error returned from driver's execute_sequence() will fail the whole 3523 * sequence and any subsequent operations won't be processed */ 3524 seq = NULL; 3525 completed = 0; 3526 memset(buf, 0, sizeof(buf)); 3527 memset(expected, 0, sizeof(expected)); 3528 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3529 g_drv_operations[ACCEL_OPC_FILL].submit_status = -EPERM; 3530 3531 dst_iovs[0].iov_base = tmp[1]; 3532 dst_iovs[0].iov_len = sizeof(tmp[1]); 3533 src_iovs[0].iov_base = tmp[0]; 3534 src_iovs[0].iov_len = sizeof(tmp[0]); 3535 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3536 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3537 ut_sequence_step_cb, &completed); 3538 CU_ASSERT_EQUAL(rc, 0); 3539 3540 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3541 ut_sequence_step_cb, &completed); 3542 CU_ASSERT_EQUAL(rc, 0); 3543 3544 dst_iovs[1].iov_base = buf; 3545 dst_iovs[1].iov_len = sizeof(buf); 3546 src_iovs[1].iov_base = tmp[1]; 3547 src_iovs[1].iov_len = sizeof(tmp[1]); 3548 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3549 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3550 ut_sequence_step_cb, &completed); 3551 CU_ASSERT_EQUAL(rc, 0); 3552 3553 ut_seq.complete = false; 3554 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3555 3556 poll_threads(); 3557 3558 CU_ASSERT_EQUAL(completed, 3); 3559 CU_ASSERT(ut_seq.complete); 3560 CU_ASSERT_EQUAL(ut_seq.status, -EPERM); 3561 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3562 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3563 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 0); 3564 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3565 CU_ASSERT_EQUAL(memcmp(buf, expected, 4096), 0); 3566 3567 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3568 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3569 g_drv_operations[ACCEL_OPC_FILL].submit_status = 0; 3570 3571 /* Check that a failed task completed by a driver will cause the whole sequence to be failed 3572 * and any subsequent operations won't be processed */ 3573 seq = NULL; 3574 completed = 0; 3575 memset(buf, 0, sizeof(buf)); 3576 memset(expected, 0, sizeof(expected)); 3577 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3578 g_drv_operations[ACCEL_OPC_FILL].complete_status = -ENOENT; 3579 3580 dst_iovs[0].iov_base = tmp[1]; 3581 dst_iovs[0].iov_len = sizeof(tmp[1]); 3582 src_iovs[0].iov_base = tmp[0]; 3583 src_iovs[0].iov_len = sizeof(tmp[0]); 3584 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3585 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3586 ut_sequence_step_cb, &completed); 3587 CU_ASSERT_EQUAL(rc, 0); 3588 3589 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3590 ut_sequence_step_cb, &completed); 3591 CU_ASSERT_EQUAL(rc, 0); 3592 3593 dst_iovs[1].iov_base = buf; 3594 dst_iovs[1].iov_len = sizeof(buf); 3595 src_iovs[1].iov_base = tmp[1]; 3596 src_iovs[1].iov_len = sizeof(tmp[1]); 3597 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3598 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3599 ut_sequence_step_cb, &completed); 3600 CU_ASSERT_EQUAL(rc, 0); 3601 3602 ut_seq.complete = false; 3603 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3604 3605 poll_threads(); 3606 3607 CU_ASSERT_EQUAL(completed, 3); 3608 CU_ASSERT(ut_seq.complete); 3609 CU_ASSERT_EQUAL(ut_seq.status, -ENOENT); 3610 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3611 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3612 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 0); 3613 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3614 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3615 3616 g_drv_operations[ACCEL_OPC_FILL].complete_status = 0; 3617 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3618 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3619 3620 /* Check asynchronous spdk_accel_sequence_continue() */ 3621 g_ut_driver_async_continue = true; 3622 seq = NULL; 3623 completed = 0; 3624 memset(buf, 0, sizeof(buf)); 3625 memset(tmp[0], 0, sizeof(tmp[0])); 3626 memset(&expected[0], 0xfe, 4096); 3627 3628 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 0, 3629 ut_sequence_step_cb, &completed); 3630 CU_ASSERT_EQUAL(rc, 0); 3631 3632 dst_iovs[0].iov_base = buf; 3633 dst_iovs[0].iov_len = sizeof(buf); 3634 src_iovs[0].iov_base = tmp[0]; 3635 src_iovs[0].iov_len = sizeof(tmp[0]); 3636 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3637 &src_iovs[0], 1, NULL, NULL, 0, 3638 ut_sequence_step_cb, &completed); 3639 CU_ASSERT_EQUAL(rc, 0); 3640 3641 ut_seq.complete = false; 3642 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3643 3644 poll_threads(); 3645 3646 CU_ASSERT_EQUAL(completed, 2); 3647 CU_ASSERT(ut_seq.complete); 3648 CU_ASSERT_EQUAL(ut_seq.status, 0); 3649 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3650 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3651 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3652 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3653 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3654 3655 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3656 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3657 3658 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3659 g_modules_opc[i] = modules[i]; 3660 } 3661 3662 /* Clear the driver so that other tests won't use it */ 3663 g_accel_driver = NULL; 3664 memset(&g_drv_operations, 0, sizeof(g_drv_operations)); 3665 3666 ut_clear_operations(); 3667 spdk_put_io_channel(ioch); 3668 poll_threads(); 3669 } 3670 3671 struct ut_saved_iovs { 3672 struct iovec src; 3673 struct iovec dst; 3674 }; 3675 3676 static struct ut_saved_iovs g_seq_saved_iovs[ACCEL_OPC_LAST]; 3677 3678 static int 3679 ut_submit_save_iovs(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3680 { 3681 SPDK_CU_ASSERT_FATAL(task->s.iovcnt == 1); 3682 SPDK_CU_ASSERT_FATAL(task->d.iovcnt == 1); 3683 3684 g_seq_saved_iovs[task->op_code].src = task->s.iovs[0]; 3685 g_seq_saved_iovs[task->op_code].dst = task->d.iovs[0]; 3686 3687 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3688 3689 spdk_accel_task_complete(task, 0); 3690 3691 return 0; 3692 } 3693 3694 static void 3695 test_sequence_same_iovs(void) 3696 { 3697 struct spdk_accel_sequence *seq = NULL; 3698 struct spdk_io_channel *ioch; 3699 struct spdk_accel_crypto_key key = {}; 3700 struct ut_sequence ut_seq; 3701 struct accel_module modules[ACCEL_OPC_LAST]; 3702 char buf[4096], tmp[4096], expected[4096]; 3703 struct iovec iovs[3], expected_siov, expected_diov; 3704 struct spdk_memory_domain *domain; 3705 void *accel_buf, *domain_ctx; 3706 int i, rc, completed = 0; 3707 3708 ioch = spdk_accel_get_io_channel(); 3709 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3710 3711 /* Override the submit_tasks function */ 3712 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3713 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3714 modules[i] = g_modules_opc[i]; 3715 g_modules_opc[i] = g_module; 3716 } 3717 /* Intercept crypto operations, as they should be executed by an accel module */ 3718 g_seq_operations[ACCEL_OPC_ENCRYPT].submit = ut_submit_save_iovs; 3719 g_seq_operations[ACCEL_OPC_DECRYPT].submit = ut_submit_save_iovs; 3720 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3721 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3722 3723 /* Check that it's possible to use the same iovec ptr for different operations */ 3724 seq = NULL; 3725 completed = 0; 3726 memset(buf, 0, sizeof(buf)); 3727 memset(expected, 0xa5, sizeof(expected)); 3728 3729 iovs[0].iov_base = expected; 3730 iovs[0].iov_len = sizeof(expected); 3731 iovs[1].iov_base = tmp; 3732 iovs[1].iov_len = sizeof(tmp); 3733 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, NULL, NULL, 3734 &iovs[0], 1, NULL, NULL, 0, 4096, 0, 3735 ut_sequence_step_cb, &completed); 3736 CU_ASSERT_EQUAL(rc, 0); 3737 /* Reuse iov[1] as src */ 3738 iovs[2].iov_base = buf; 3739 iovs[2].iov_len = sizeof(buf); 3740 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3741 &iovs[1], 1, NULL, NULL, 0, 4096, 0, 3742 ut_sequence_step_cb, &completed); 3743 CU_ASSERT_EQUAL(rc, 0); 3744 3745 ut_seq.complete = false; 3746 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3747 3748 poll_threads(); 3749 3750 CU_ASSERT_EQUAL(completed, 2); 3751 CU_ASSERT(ut_seq.complete); 3752 CU_ASSERT_EQUAL(ut_seq.status, 0); 3753 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3754 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3755 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3756 expected_siov.iov_base = expected; 3757 expected_siov.iov_len = sizeof(expected); 3758 expected_diov.iov_base = tmp; 3759 expected_diov.iov_len = sizeof(tmp); 3760 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_ENCRYPT].src, 3761 &expected_siov, sizeof(expected_siov)), 0); 3762 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_ENCRYPT].dst, 3763 &expected_diov, sizeof(expected_diov)), 0); 3764 expected_siov.iov_base = tmp; 3765 expected_siov.iov_len = sizeof(tmp); 3766 expected_diov.iov_base = buf; 3767 expected_diov.iov_len = sizeof(buf); 3768 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_DECRYPT].src, 3769 &expected_siov, sizeof(expected_siov)), 0); 3770 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_DECRYPT].dst, 3771 &expected_diov, sizeof(expected_diov)), 0); 3772 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3773 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3774 3775 /* Check the same with an accel buffer */ 3776 seq = NULL; 3777 completed = 0; 3778 memset(buf, 0, sizeof(buf)); 3779 memset(expected, 0x5a, sizeof(expected)); 3780 3781 rc = spdk_accel_get_buf(ioch, sizeof(buf), &accel_buf, &domain, &domain_ctx); 3782 CU_ASSERT_EQUAL(rc, 0); 3783 3784 iovs[0].iov_base = expected; 3785 iovs[0].iov_len = sizeof(expected); 3786 iovs[1].iov_base = accel_buf; 3787 iovs[1].iov_len = sizeof(buf); 3788 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, domain, domain_ctx, 3789 &iovs[0], 1, NULL, NULL, 0, 4096, 0, 3790 ut_sequence_step_cb, &completed); 3791 CU_ASSERT_EQUAL(rc, 0); 3792 /* Reuse iov[1] as src */ 3793 iovs[2].iov_base = buf; 3794 iovs[2].iov_len = sizeof(buf); 3795 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3796 &iovs[1], 1, domain, domain_ctx, 0, 4096, 0, 3797 ut_sequence_step_cb, &completed); 3798 CU_ASSERT_EQUAL(rc, 0); 3799 3800 ut_seq.complete = false; 3801 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3802 3803 poll_threads(); 3804 3805 CU_ASSERT_EQUAL(completed, 2); 3806 CU_ASSERT(ut_seq.complete); 3807 CU_ASSERT_EQUAL(ut_seq.status, 0); 3808 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3809 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3810 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3811 expected_siov.iov_base = expected; 3812 expected_siov.iov_len = sizeof(expected); 3813 expected_diov.iov_base = buf; 3814 expected_diov.iov_len = sizeof(buf); 3815 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_ENCRYPT].src, 3816 &expected_siov, sizeof(expected_siov)), 0); 3817 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_DECRYPT].dst, 3818 &expected_diov, sizeof(expected_diov)), 0); 3819 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_ENCRYPT].dst, 3820 &g_seq_saved_iovs[ACCEL_OPC_DECRYPT].src, 3821 sizeof(struct iovec)), 0); 3822 spdk_accel_put_buf(ioch, accel_buf, domain, domain_ctx); 3823 3824 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3825 g_modules_opc[i] = modules[i]; 3826 } 3827 3828 ut_clear_operations(); 3829 spdk_put_io_channel(ioch); 3830 poll_threads(); 3831 } 3832 3833 static void 3834 test_sequence_crc32(void) 3835 { 3836 struct spdk_accel_sequence *seq = NULL; 3837 struct spdk_io_channel *ioch; 3838 struct ut_sequence ut_seq; 3839 struct accel_module modules[ACCEL_OPC_LAST]; 3840 char buf[4096], tmp[3][4096]; 3841 struct iovec src_iovs[4], dst_iovs[4]; 3842 uint32_t crc, crc2; 3843 int i, rc, completed; 3844 3845 ioch = spdk_accel_get_io_channel(); 3846 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3847 3848 /* Override the submit_tasks function */ 3849 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3850 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3851 g_seq_operations[i].submit = sw_accel_submit_tasks; 3852 modules[i] = g_modules_opc[i]; 3853 g_modules_opc[i] = g_module; 3854 } 3855 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 3856 3857 /* First check the simplest case - single crc32c operation */ 3858 seq = NULL; 3859 completed = 0; 3860 crc = 0; 3861 memset(buf, 0xa5, sizeof(buf)); 3862 3863 src_iovs[0].iov_base = buf; 3864 src_iovs[0].iov_len = sizeof(buf); 3865 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3866 ut_sequence_step_cb, &completed); 3867 CU_ASSERT_EQUAL(rc, 0); 3868 3869 ut_seq.complete = false; 3870 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3871 3872 poll_threads(); 3873 CU_ASSERT_EQUAL(completed, 1); 3874 CU_ASSERT(ut_seq.complete); 3875 CU_ASSERT_EQUAL(ut_seq.status, 0); 3876 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_CRC32C].count, 1); 3877 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3878 g_seq_operations[ACCEL_OPC_CRC32C].count = 0; 3879 3880 /* Now check copy+crc - this should remove the copy operation and change the buffer for the 3881 * crc operation */ 3882 seq = NULL; 3883 completed = 0; 3884 crc = 0; 3885 memset(buf, 0x5a, sizeof(buf)); 3886 memset(&tmp[0], 0, sizeof(tmp[0])); 3887 3888 dst_iovs[0].iov_base = tmp[0]; 3889 dst_iovs[0].iov_len = sizeof(tmp[0]); 3890 src_iovs[0].iov_base = buf; 3891 src_iovs[0].iov_len = sizeof(buf); 3892 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3893 &src_iovs[0], 1, NULL, NULL, 0, 3894 ut_sequence_step_cb, &completed); 3895 CU_ASSERT_EQUAL(rc, 0); 3896 3897 src_iovs[1].iov_base = tmp[0]; 3898 src_iovs[1].iov_len = sizeof(tmp[0]); 3899 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 3900 ut_sequence_step_cb, &completed); 3901 CU_ASSERT_EQUAL(rc, 0); 3902 3903 ut_seq.complete = false; 3904 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3905 3906 poll_threads(); 3907 CU_ASSERT_EQUAL(completed, 2); 3908 CU_ASSERT(ut_seq.complete); 3909 CU_ASSERT_EQUAL(ut_seq.status, 0); 3910 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_CRC32C].count, 1); 3911 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 3912 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3913 g_seq_operations[ACCEL_OPC_CRC32C].count = 0; 3914 3915 /* Check crc+copy - this time the copy cannot be removed, because there's no operation 3916 * before crc to change the buffer */ 3917 seq = NULL; 3918 completed = 0; 3919 crc = 0; 3920 memset(buf, 0, sizeof(buf)); 3921 memset(&tmp[0], 0xa5, sizeof(tmp[0])); 3922 3923 src_iovs[0].iov_base = tmp[0]; 3924 src_iovs[0].iov_len = sizeof(tmp[0]); 3925 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3926 ut_sequence_step_cb, &completed); 3927 CU_ASSERT_EQUAL(rc, 0); 3928 3929 dst_iovs[1].iov_base = buf; 3930 dst_iovs[1].iov_len = sizeof(buf); 3931 src_iovs[1].iov_base = tmp[0]; 3932 src_iovs[1].iov_len = sizeof(tmp[0]); 3933 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3934 &src_iovs[1], 1, NULL, NULL, 0, 3935 ut_sequence_step_cb, &completed); 3936 CU_ASSERT_EQUAL(rc, 0); 3937 3938 ut_seq.complete = false; 3939 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3940 3941 poll_threads(); 3942 CU_ASSERT_EQUAL(completed, 2); 3943 CU_ASSERT(ut_seq.complete); 3944 CU_ASSERT_EQUAL(ut_seq.status, 0); 3945 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_CRC32C].count, 1); 3946 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 1); 3947 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 3948 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 3949 g_seq_operations[ACCEL_OPC_CRC32C].count = 0; 3950 g_seq_operations[ACCEL_OPC_COPY].count = 0; 3951 3952 /* Check a sequence with an operation at the beginning that can have its buffer changed, two 3953 * crc operations and a copy at the end. The copy should be removed and the dst buffer of 3954 * the first operation and the src buffer of the crc operations should be changed. 3955 */ 3956 seq = NULL; 3957 completed = 0; 3958 crc = crc2 = 0; 3959 memset(buf, 0, sizeof(buf)); 3960 memset(&tmp[0], 0x5a, sizeof(tmp[0])); 3961 dst_iovs[0].iov_base = tmp[1]; 3962 dst_iovs[0].iov_len = sizeof(tmp[1]); 3963 src_iovs[0].iov_base = tmp[0]; 3964 src_iovs[0].iov_len = sizeof(tmp[0]); 3965 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3966 &src_iovs[0], 1, NULL, NULL, 0, 3967 ut_sequence_step_cb, &completed); 3968 CU_ASSERT_EQUAL(rc, 0); 3969 3970 src_iovs[1].iov_base = tmp[1]; 3971 src_iovs[1].iov_len = sizeof(tmp[1]); 3972 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 3973 ut_sequence_step_cb, &completed); 3974 CU_ASSERT_EQUAL(rc, 0); 3975 3976 src_iovs[2].iov_base = tmp[1]; 3977 src_iovs[2].iov_len = sizeof(tmp[1]); 3978 rc = spdk_accel_append_crc32c(&seq, ioch, &crc2, &src_iovs[2], 1, NULL, NULL, 0, 3979 ut_sequence_step_cb, &completed); 3980 CU_ASSERT_EQUAL(rc, 0); 3981 3982 dst_iovs[3].iov_base = buf; 3983 dst_iovs[3].iov_len = sizeof(buf); 3984 src_iovs[3].iov_base = tmp[1]; 3985 src_iovs[3].iov_len = sizeof(tmp[1]); 3986 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 3987 &src_iovs[3], 1, NULL, NULL, 0, 3988 ut_sequence_step_cb, &completed); 3989 CU_ASSERT_EQUAL(rc, 0); 3990 3991 ut_seq.complete = false; 3992 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3993 3994 poll_threads(); 3995 CU_ASSERT_EQUAL(completed, 4); 3996 CU_ASSERT(ut_seq.complete); 3997 CU_ASSERT_EQUAL(ut_seq.status, 0); 3998 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3999 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_CRC32C].count, 2); 4000 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 4001 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 4002 CU_ASSERT_EQUAL(crc, crc2); 4003 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 4004 g_seq_operations[ACCEL_OPC_CRC32C].count = 0; 4005 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 4006 4007 /* Check that a copy won't be removed if the buffers don't match */ 4008 seq = NULL; 4009 completed = 0; 4010 crc = 0; 4011 memset(buf, 0, sizeof(buf)); 4012 memset(&tmp[0], 0xa5, 2048); 4013 memset(&tmp[1], 0xfe, sizeof(tmp[1])); 4014 memset(&tmp[2], 0xfe, sizeof(tmp[1])); 4015 dst_iovs[0].iov_base = &tmp[1][2048]; 4016 dst_iovs[0].iov_len = 2048; 4017 src_iovs[0].iov_base = tmp[0]; 4018 src_iovs[0].iov_len = 2048; 4019 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 4020 &src_iovs[0], 1, NULL, NULL, 0, 4021 ut_sequence_step_cb, &completed); 4022 CU_ASSERT_EQUAL(rc, 0); 4023 4024 src_iovs[1].iov_base = &tmp[1][2048]; 4025 src_iovs[1].iov_len = 2048; 4026 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 4027 ut_sequence_step_cb, &completed); 4028 CU_ASSERT_EQUAL(rc, 0); 4029 4030 dst_iovs[2].iov_base = buf; 4031 dst_iovs[2].iov_len = sizeof(buf); 4032 src_iovs[2].iov_base = tmp[1]; 4033 src_iovs[2].iov_len = sizeof(tmp[1]); 4034 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 4035 &src_iovs[2], 1, NULL, NULL, 0, 4036 ut_sequence_step_cb, &completed); 4037 CU_ASSERT_EQUAL(rc, 0); 4038 4039 ut_seq.complete = false; 4040 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4041 4042 poll_threads(); 4043 CU_ASSERT_EQUAL(completed, 3); 4044 CU_ASSERT(ut_seq.complete); 4045 CU_ASSERT_EQUAL(ut_seq.status, 0); 4046 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 4047 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_CRC32C].count, 1); 4048 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 1); 4049 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], 2048, ~0u)); 4050 CU_ASSERT_EQUAL(memcmp(buf, tmp[2], 2048), 0); 4051 CU_ASSERT_EQUAL(memcmp(&buf[2048], tmp[0], 2048), 0); 4052 g_seq_operations[ACCEL_OPC_CRC32C].count = 0; 4053 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 4054 g_seq_operations[ACCEL_OPC_COPY].count = 0; 4055 4056 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 4057 g_modules_opc[i] = modules[i]; 4058 } 4059 4060 ut_clear_operations(); 4061 spdk_put_io_channel(ioch); 4062 poll_threads(); 4063 } 4064 4065 static int 4066 test_sequence_setup(void) 4067 { 4068 int rc; 4069 4070 allocate_cores(1); 4071 allocate_threads(1); 4072 set_thread(0); 4073 4074 rc = spdk_iobuf_initialize(); 4075 if (rc != 0) { 4076 CU_ASSERT(false); 4077 return -1; 4078 } 4079 4080 rc = spdk_accel_initialize(); 4081 if (rc != 0) { 4082 CU_ASSERT(false); 4083 return -1; 4084 } 4085 4086 return 0; 4087 } 4088 4089 static void 4090 finish_cb(void *cb_arg) 4091 { 4092 bool *done = cb_arg; 4093 4094 *done = true; 4095 } 4096 4097 static int 4098 test_sequence_cleanup(void) 4099 { 4100 bool done = false; 4101 4102 spdk_accel_finish(finish_cb, &done); 4103 4104 while (!done) { 4105 poll_threads(); 4106 } 4107 4108 done = false; 4109 spdk_iobuf_finish(finish_cb, &done); 4110 while (!done) { 4111 poll_threads(); 4112 } 4113 4114 free_threads(); 4115 free_cores(); 4116 4117 return 0; 4118 } 4119 4120 int 4121 main(int argc, char **argv) 4122 { 4123 CU_pSuite suite = NULL, seq_suite; 4124 unsigned int num_failures; 4125 4126 CU_initialize_registry(); 4127 4128 /* Sequence tests require accel to be initialized normally, so run them before the other 4129 * tests which register accel modules which aren't fully implemented, causing accel 4130 * initialization to fail. 4131 */ 4132 seq_suite = CU_add_suite("accel_sequence", test_sequence_setup, test_sequence_cleanup); 4133 CU_ADD_TEST(seq_suite, test_sequence_fill_copy); 4134 CU_ADD_TEST(seq_suite, test_sequence_abort); 4135 CU_ADD_TEST(seq_suite, test_sequence_append_error); 4136 CU_ADD_TEST(seq_suite, test_sequence_completion_error); 4137 #ifdef SPDK_CONFIG_ISAL /* accel_sw requires isa-l for compression */ 4138 CU_ADD_TEST(seq_suite, test_sequence_decompress); 4139 CU_ADD_TEST(seq_suite, test_sequence_reverse); 4140 #endif 4141 CU_ADD_TEST(seq_suite, test_sequence_copy_elision); 4142 CU_ADD_TEST(seq_suite, test_sequence_accel_buffers); 4143 CU_ADD_TEST(seq_suite, test_sequence_memory_domain); 4144 CU_ADD_TEST(seq_suite, test_sequence_module_memory_domain); 4145 #ifdef SPDK_CONFIG_ISAL_CRYPTO /* accel_sw requires isa-l-crypto for crypto operations */ 4146 CU_ADD_TEST(seq_suite, test_sequence_crypto); 4147 #endif 4148 CU_ADD_TEST(seq_suite, test_sequence_driver); 4149 CU_ADD_TEST(seq_suite, test_sequence_same_iovs); 4150 CU_ADD_TEST(seq_suite, test_sequence_crc32); 4151 4152 suite = CU_add_suite("accel", test_setup, test_cleanup); 4153 CU_ADD_TEST(suite, test_spdk_accel_task_complete); 4154 CU_ADD_TEST(suite, test_get_task); 4155 CU_ADD_TEST(suite, test_spdk_accel_submit_copy); 4156 CU_ADD_TEST(suite, test_spdk_accel_submit_dualcast); 4157 CU_ADD_TEST(suite, test_spdk_accel_submit_compare); 4158 CU_ADD_TEST(suite, test_spdk_accel_submit_fill); 4159 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c); 4160 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32cv); 4161 CU_ADD_TEST(suite, test_spdk_accel_submit_copy_crc32c); 4162 CU_ADD_TEST(suite, test_spdk_accel_submit_xor); 4163 CU_ADD_TEST(suite, test_spdk_accel_module_find_by_name); 4164 CU_ADD_TEST(suite, test_spdk_accel_module_register); 4165 4166 num_failures = spdk_ut_run_tests(argc, argv, NULL); 4167 CU_cleanup_registry(); 4168 4169 return num_failures; 4170 } 4171