1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2021 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk_cunit.h" 8 #include "spdk_internal/mock.h" 9 #include "spdk/accel_module.h" 10 #include "thread/thread_internal.h" 11 #include "common/lib/ut_multithread.c" 12 #include "common/lib/test_iobuf.c" 13 #include "accel/accel.c" 14 #include "accel/accel_sw.c" 15 #include "unit/lib/json_mock.c" 16 17 DEFINE_STUB_V(spdk_memory_domain_destroy, (struct spdk_memory_domain *domain)); 18 DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, 19 (struct spdk_memory_domain *domain), "UT_DMA"); 20 21 int 22 spdk_memory_domain_create(struct spdk_memory_domain **domain, enum spdk_dma_device_type type, 23 struct spdk_memory_domain_ctx *ctx, const char *id) 24 { 25 *domain = (void *)0xdeadbeef; 26 27 return 0; 28 } 29 30 struct ut_domain_ctx { 31 struct iovec iov; 32 struct iovec expected; 33 int pull_submit_status; 34 int push_submit_status; 35 int pull_complete_status; 36 int push_complete_status; 37 }; 38 39 static struct spdk_memory_domain *g_ut_domain = (void *)0xa55e1; 40 41 int 42 spdk_memory_domain_pull_data(struct spdk_memory_domain *sd, void *sctx, struct iovec *siov, 43 uint32_t siovcnt, struct iovec *diov, uint32_t diovcnt, 44 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 45 { 46 struct ut_domain_ctx *ctx = sctx; 47 48 CU_ASSERT_EQUAL(sd, g_ut_domain); 49 CU_ASSERT_EQUAL(siovcnt, 1); 50 CU_ASSERT_EQUAL(memcmp(siov, &ctx->expected, sizeof(*siov)), 0); 51 52 if (ctx->pull_submit_status != 0) { 53 return ctx->pull_submit_status; 54 } 55 56 if (ctx->pull_complete_status != 0) { 57 cpl_cb(cpl_arg, ctx->pull_complete_status); 58 return 0; 59 } 60 61 spdk_iovcpy(&ctx->iov, 1, diov, diovcnt); 62 cpl_cb(cpl_arg, 0); 63 64 return 0; 65 } 66 67 int 68 spdk_memory_domain_push_data(struct spdk_memory_domain *dd, void *dctx, struct iovec *diov, 69 uint32_t diovcnt, struct iovec *siov, uint32_t siovcnt, 70 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 71 { 72 struct ut_domain_ctx *ctx = dctx; 73 74 CU_ASSERT_EQUAL(dd, g_ut_domain); 75 CU_ASSERT_EQUAL(diovcnt, 1); 76 CU_ASSERT_EQUAL(memcmp(diov, &ctx->expected, sizeof(*diov)), 0); 77 78 if (ctx->push_submit_status != 0) { 79 return ctx->push_submit_status; 80 } 81 82 if (ctx->push_complete_status != 0) { 83 cpl_cb(cpl_arg, ctx->push_complete_status); 84 return 0; 85 } 86 87 spdk_iovcpy(siov, siovcnt, &ctx->iov, 1); 88 cpl_cb(cpl_arg, 0); 89 90 return 0; 91 } 92 93 /* global vars and setup/cleanup functions used for all test functions */ 94 struct spdk_accel_module_if g_module_if = {}; 95 struct accel_module g_module = { .module = &g_module_if }; 96 struct spdk_io_channel *g_ch = NULL; 97 struct accel_io_channel *g_accel_ch = NULL; 98 struct sw_accel_io_channel *g_sw_ch = NULL; 99 struct spdk_io_channel *g_module_ch = NULL; 100 101 static uint64_t g_opc_mask = 0; 102 103 static uint64_t 104 _accel_op_to_bit(enum accel_opcode opc) 105 { 106 return (1 << opc); 107 } 108 109 static bool 110 _supports_opcode(enum accel_opcode opc) 111 { 112 if (_accel_op_to_bit(opc) & g_opc_mask) { 113 return true; 114 } 115 return false; 116 } 117 118 static int 119 test_setup(void) 120 { 121 int i; 122 123 g_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct accel_io_channel)); 124 if (g_ch == NULL) { 125 /* for some reason the assert fatal macro doesn't work in the setup function. */ 126 CU_ASSERT(false); 127 return -1; 128 } 129 g_accel_ch = (struct accel_io_channel *)((char *)g_ch + sizeof(struct spdk_io_channel)); 130 g_module_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct sw_accel_io_channel)); 131 if (g_module_ch == NULL) { 132 CU_ASSERT(false); 133 return -1; 134 } 135 136 g_module_if.submit_tasks = sw_accel_submit_tasks; 137 g_module_if.name = "software"; 138 for (i = 0; i < ACCEL_OPC_LAST; i++) { 139 g_accel_ch->module_ch[i] = g_module_ch; 140 g_modules_opc[i] = g_module; 141 } 142 g_sw_ch = (struct sw_accel_io_channel *)((char *)g_module_ch + sizeof( 143 struct spdk_io_channel)); 144 TAILQ_INIT(&g_sw_ch->tasks_to_complete); 145 g_module_if.supports_opcode = _supports_opcode; 146 return 0; 147 } 148 149 static int 150 test_cleanup(void) 151 { 152 free(g_ch); 153 free(g_module_ch); 154 155 return 0; 156 } 157 158 #define DUMMY_ARG 0xDEADBEEF 159 static bool g_dummy_cb_called = false; 160 static void 161 dummy_cb_fn(void *cb_arg, int status) 162 { 163 CU_ASSERT(*(uint32_t *)cb_arg == DUMMY_ARG); 164 CU_ASSERT(status == 0); 165 g_dummy_cb_called = true; 166 } 167 168 static void 169 test_spdk_accel_task_complete(void) 170 { 171 struct spdk_accel_task accel_task = {}; 172 struct spdk_accel_task *expected_accel_task = NULL; 173 uint32_t cb_arg = DUMMY_ARG; 174 int status = 0; 175 176 accel_task.accel_ch = g_accel_ch; 177 accel_task.cb_fn = dummy_cb_fn; 178 accel_task.cb_arg = &cb_arg; 179 TAILQ_INIT(&g_accel_ch->task_pool); 180 181 /* Confirm cb is called and task added to list. */ 182 spdk_accel_task_complete(&accel_task, status); 183 CU_ASSERT(g_dummy_cb_called == true); 184 expected_accel_task = TAILQ_FIRST(&g_accel_ch->task_pool); 185 TAILQ_REMOVE(&g_accel_ch->task_pool, expected_accel_task, link); 186 CU_ASSERT(expected_accel_task == &accel_task); 187 } 188 189 static void 190 test_get_task(void) 191 { 192 struct spdk_accel_task *task; 193 struct spdk_accel_task _task; 194 void *cb_arg = NULL; 195 196 TAILQ_INIT(&g_accel_ch->task_pool); 197 198 /* no tasks left, return NULL. */ 199 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 200 CU_ASSERT(task == NULL); 201 202 _task.cb_fn = dummy_cb_fn; 203 _task.cb_arg = cb_arg; 204 _task.accel_ch = g_accel_ch; 205 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &_task, link); 206 207 /* Get a valid task. */ 208 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 209 CU_ASSERT(task == &_task); 210 CU_ASSERT(_task.cb_fn == dummy_cb_fn); 211 CU_ASSERT(_task.cb_arg == cb_arg); 212 CU_ASSERT(_task.accel_ch == g_accel_ch); 213 } 214 215 #define TEST_SUBMIT_SIZE 64 216 static void 217 test_spdk_accel_submit_copy(void) 218 { 219 const uint64_t nbytes = TEST_SUBMIT_SIZE; 220 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 221 uint8_t src[TEST_SUBMIT_SIZE] = {0}; 222 void *cb_arg = NULL; 223 int rc; 224 struct spdk_accel_task task; 225 struct spdk_accel_task *expected_accel_task = NULL; 226 int flags = 0; 227 228 TAILQ_INIT(&g_accel_ch->task_pool); 229 230 /* Fail with no tasks on _get_task() */ 231 rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, flags, NULL, cb_arg); 232 CU_ASSERT(rc == -ENOMEM); 233 234 task.accel_ch = g_accel_ch; 235 task.flags = 1; 236 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 237 238 /* submission OK. */ 239 rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, flags, NULL, cb_arg); 240 CU_ASSERT(rc == 0); 241 CU_ASSERT(task.op_code == ACCEL_OPC_COPY); 242 CU_ASSERT(task.flags == 0); 243 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 244 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 245 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 246 CU_ASSERT(expected_accel_task == &task); 247 } 248 249 static void 250 test_spdk_accel_submit_dualcast(void) 251 { 252 void *dst1; 253 void *dst2; 254 void *src; 255 uint32_t align = ALIGN_4K; 256 uint64_t nbytes = TEST_SUBMIT_SIZE; 257 void *cb_arg = NULL; 258 int rc; 259 struct spdk_accel_task task; 260 struct spdk_accel_task *expected_accel_task = NULL; 261 int flags = 0; 262 263 TAILQ_INIT(&g_accel_ch->task_pool); 264 265 /* Dualcast requires 4K alignment on dst addresses, 266 * hence using the hard coded address to test the buffer alignment 267 */ 268 dst1 = (void *)0x5000; 269 dst2 = (void *)0x60f0; 270 src = calloc(1, TEST_SUBMIT_SIZE); 271 SPDK_CU_ASSERT_FATAL(src != NULL); 272 memset(src, 0x5A, TEST_SUBMIT_SIZE); 273 274 /* This should fail since dst2 is not 4k aligned */ 275 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 276 CU_ASSERT(rc == -EINVAL); 277 278 dst1 = (void *)0x7010; 279 dst2 = (void *)0x6000; 280 /* This should fail since dst1 is not 4k aligned */ 281 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 282 CU_ASSERT(rc == -EINVAL); 283 284 /* Dualcast requires 4K alignment on dst addresses */ 285 dst1 = (void *)0x7000; 286 dst2 = (void *)0x6000; 287 /* Fail with no tasks on _get_task() */ 288 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 289 CU_ASSERT(rc == -ENOMEM); 290 291 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 292 293 /* accel submission OK., since we test the SW path , need to use valid memory addresses 294 * cannot hardcode them anymore */ 295 dst1 = spdk_dma_zmalloc(nbytes, align, NULL); 296 SPDK_CU_ASSERT_FATAL(dst1 != NULL); 297 dst2 = spdk_dma_zmalloc(nbytes, align, NULL); 298 SPDK_CU_ASSERT_FATAL(dst2 != NULL); 299 /* SW module does the dualcast. */ 300 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 301 CU_ASSERT(rc == 0); 302 CU_ASSERT(task.op_code == ACCEL_OPC_DUALCAST); 303 CU_ASSERT(task.flags == 0); 304 CU_ASSERT(memcmp(dst1, src, TEST_SUBMIT_SIZE) == 0); 305 CU_ASSERT(memcmp(dst2, src, TEST_SUBMIT_SIZE) == 0); 306 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 307 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 308 CU_ASSERT(expected_accel_task == &task); 309 310 free(src); 311 spdk_free(dst1); 312 spdk_free(dst2); 313 } 314 315 static void 316 test_spdk_accel_submit_compare(void) 317 { 318 void *src1; 319 void *src2; 320 uint64_t nbytes = TEST_SUBMIT_SIZE; 321 void *cb_arg = NULL; 322 int rc; 323 struct spdk_accel_task task; 324 struct spdk_accel_task *expected_accel_task = NULL; 325 326 TAILQ_INIT(&g_accel_ch->task_pool); 327 328 src1 = calloc(1, TEST_SUBMIT_SIZE); 329 SPDK_CU_ASSERT_FATAL(src1 != NULL); 330 src2 = calloc(1, TEST_SUBMIT_SIZE); 331 SPDK_CU_ASSERT_FATAL(src2 != NULL); 332 333 /* Fail with no tasks on _get_task() */ 334 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 335 CU_ASSERT(rc == -ENOMEM); 336 337 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 338 339 /* accel submission OK. */ 340 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 341 CU_ASSERT(rc == 0); 342 CU_ASSERT(task.op_code == ACCEL_OPC_COMPARE); 343 CU_ASSERT(memcmp(src1, src2, TEST_SUBMIT_SIZE) == 0); 344 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 345 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 346 CU_ASSERT(expected_accel_task == &task); 347 348 free(src1); 349 free(src2); 350 } 351 352 static void 353 test_spdk_accel_submit_fill(void) 354 { 355 void *dst; 356 void *src; 357 uint8_t fill = 0xf; 358 uint64_t fill64; 359 uint64_t nbytes = TEST_SUBMIT_SIZE; 360 void *cb_arg = NULL; 361 int rc; 362 struct spdk_accel_task task; 363 struct spdk_accel_task *expected_accel_task = NULL; 364 int flags = 0; 365 366 TAILQ_INIT(&g_accel_ch->task_pool); 367 368 dst = calloc(1, TEST_SUBMIT_SIZE); 369 SPDK_CU_ASSERT_FATAL(dst != NULL); 370 src = calloc(1, TEST_SUBMIT_SIZE); 371 SPDK_CU_ASSERT_FATAL(src != NULL); 372 memset(src, fill, TEST_SUBMIT_SIZE); 373 memset(&fill64, fill, sizeof(uint64_t)); 374 375 /* Fail with no tasks on _get_task() */ 376 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 377 CU_ASSERT(rc == -ENOMEM); 378 379 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 380 381 /* accel submission OK. */ 382 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 383 CU_ASSERT(rc == 0); 384 CU_ASSERT(task.fill_pattern == fill64); 385 CU_ASSERT(task.op_code == ACCEL_OPC_FILL); 386 CU_ASSERT(task.flags == 0); 387 388 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 389 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 390 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 391 CU_ASSERT(expected_accel_task == &task); 392 393 free(dst); 394 free(src); 395 } 396 397 static void 398 test_spdk_accel_submit_crc32c(void) 399 { 400 const uint64_t nbytes = TEST_SUBMIT_SIZE; 401 uint32_t crc_dst; 402 uint8_t src[TEST_SUBMIT_SIZE]; 403 uint32_t seed = 1; 404 void *cb_arg = NULL; 405 int rc; 406 struct spdk_accel_task task; 407 struct spdk_accel_task *expected_accel_task = NULL; 408 409 TAILQ_INIT(&g_accel_ch->task_pool); 410 411 /* Fail with no tasks on _get_task() */ 412 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 413 CU_ASSERT(rc == -ENOMEM); 414 415 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 416 417 /* accel submission OK. */ 418 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 419 CU_ASSERT(rc == 0); 420 CU_ASSERT(task.crc_dst == &crc_dst); 421 CU_ASSERT(task.seed == seed); 422 CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); 423 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 424 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 425 CU_ASSERT(expected_accel_task == &task); 426 } 427 428 static void 429 test_spdk_accel_submit_crc32cv(void) 430 { 431 uint32_t crc_dst; 432 uint32_t seed = 0; 433 uint32_t iov_cnt = 32; 434 void *cb_arg = NULL; 435 int rc; 436 uint32_t i = 0; 437 struct spdk_accel_task task; 438 struct iovec iov[32]; 439 struct spdk_accel_task *expected_accel_task = NULL; 440 441 TAILQ_INIT(&g_accel_ch->task_pool); 442 443 for (i = 0; i < iov_cnt; i++) { 444 iov[i].iov_base = calloc(1, TEST_SUBMIT_SIZE); 445 SPDK_CU_ASSERT_FATAL(iov[i].iov_base != NULL); 446 iov[i].iov_len = TEST_SUBMIT_SIZE; 447 } 448 449 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 450 451 /* accel submission OK. */ 452 rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, NULL, cb_arg); 453 CU_ASSERT(rc == 0); 454 CU_ASSERT(task.s.iovs == iov); 455 CU_ASSERT(task.s.iovcnt == iov_cnt); 456 CU_ASSERT(task.crc_dst == &crc_dst); 457 CU_ASSERT(task.seed == seed); 458 CU_ASSERT(task.op_code == ACCEL_OPC_CRC32C); 459 CU_ASSERT(task.cb_arg == cb_arg); 460 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 461 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 462 CU_ASSERT(expected_accel_task == &task); 463 464 for (i = 0; i < iov_cnt; i++) { 465 free(iov[i].iov_base); 466 } 467 } 468 469 static void 470 test_spdk_accel_submit_copy_crc32c(void) 471 { 472 const uint64_t nbytes = TEST_SUBMIT_SIZE; 473 uint32_t crc_dst; 474 uint8_t dst[TEST_SUBMIT_SIZE]; 475 uint8_t src[TEST_SUBMIT_SIZE]; 476 uint32_t seed = 0; 477 void *cb_arg = NULL; 478 int rc; 479 struct spdk_accel_task task; 480 struct spdk_accel_task *expected_accel_task = NULL; 481 int flags = 0; 482 483 TAILQ_INIT(&g_accel_ch->task_pool); 484 485 /* Fail with no tasks on _get_task() */ 486 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 487 NULL, cb_arg); 488 CU_ASSERT(rc == -ENOMEM); 489 490 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 491 492 /* accel submission OK. */ 493 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 494 NULL, cb_arg); 495 CU_ASSERT(rc == 0); 496 CU_ASSERT(task.crc_dst == &crc_dst); 497 CU_ASSERT(task.seed == seed); 498 CU_ASSERT(task.flags == 0); 499 CU_ASSERT(task.op_code == ACCEL_OPC_COPY_CRC32C); 500 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 501 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 502 CU_ASSERT(expected_accel_task == &task); 503 } 504 505 static void 506 test_spdk_accel_submit_xor(void) 507 { 508 const uint64_t nbytes = TEST_SUBMIT_SIZE; 509 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 510 uint8_t src1[TEST_SUBMIT_SIZE] = {0}; 511 uint8_t src2[TEST_SUBMIT_SIZE] = {0}; 512 void *sources[] = { src1, src2 }; 513 uint32_t nsrcs = SPDK_COUNTOF(sources); 514 int rc; 515 struct spdk_accel_task task; 516 struct spdk_accel_task *expected_accel_task = NULL; 517 518 TAILQ_INIT(&g_accel_ch->task_pool); 519 520 /* Fail with no tasks on _get_task() */ 521 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 522 CU_ASSERT(rc == -ENOMEM); 523 524 TAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 525 526 /* submission OK. */ 527 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 528 CU_ASSERT(rc == 0); 529 CU_ASSERT(task.nsrcs.srcs == sources); 530 CU_ASSERT(task.nsrcs.cnt == nsrcs); 531 CU_ASSERT(task.d.iovcnt == 1); 532 CU_ASSERT(task.d.iovs[0].iov_base == dst); 533 CU_ASSERT(task.d.iovs[0].iov_len == nbytes); 534 CU_ASSERT(task.op_code == ACCEL_OPC_XOR); 535 expected_accel_task = TAILQ_FIRST(&g_sw_ch->tasks_to_complete); 536 TAILQ_REMOVE(&g_sw_ch->tasks_to_complete, expected_accel_task, link); 537 CU_ASSERT(expected_accel_task == &task); 538 } 539 540 static void 541 test_spdk_accel_module_find_by_name(void) 542 { 543 struct spdk_accel_module_if mod1 = {}; 544 struct spdk_accel_module_if mod2 = {}; 545 struct spdk_accel_module_if mod3 = {}; 546 struct spdk_accel_module_if *accel_module = NULL; 547 548 mod1.name = "ioat"; 549 mod2.name = "idxd"; 550 mod3.name = "software"; 551 552 TAILQ_INIT(&spdk_accel_module_list); 553 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod1, tailq); 554 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod2, tailq); 555 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod3, tailq); 556 557 /* Now let's find a valid engine */ 558 accel_module = _module_find_by_name("ioat"); 559 CU_ASSERT(accel_module != NULL); 560 561 /* Try to find one that doesn't exist */ 562 accel_module = _module_find_by_name("XXX"); 563 CU_ASSERT(accel_module == NULL); 564 } 565 566 static void 567 test_spdk_accel_module_register(void) 568 { 569 struct spdk_accel_module_if mod1 = {}; 570 struct spdk_accel_module_if mod2 = {}; 571 struct spdk_accel_module_if mod3 = {}; 572 struct spdk_accel_module_if mod4 = {}; 573 struct spdk_accel_module_if *accel_module = NULL; 574 int i = 0; 575 576 mod1.name = "ioat"; 577 mod2.name = "idxd"; 578 mod3.name = "software"; 579 mod4.name = "nothing"; 580 581 TAILQ_INIT(&spdk_accel_module_list); 582 583 spdk_accel_module_list_add(&mod1); 584 spdk_accel_module_list_add(&mod2); 585 spdk_accel_module_list_add(&mod3); 586 spdk_accel_module_list_add(&mod4); 587 588 /* Now confirm they're in the right order. */ 589 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 590 switch (i++) { 591 case 0: 592 CU_ASSERT(strcmp(accel_module->name, "software") == 0); 593 break; 594 case 1: 595 CU_ASSERT(strcmp(accel_module->name, "ioat") == 0); 596 break; 597 case 2: 598 CU_ASSERT(strcmp(accel_module->name, "idxd") == 0); 599 break; 600 case 3: 601 CU_ASSERT(strcmp(accel_module->name, "nothing") == 0); 602 break; 603 default: 604 CU_ASSERT(false); 605 break; 606 } 607 } 608 CU_ASSERT(i == 4); 609 } 610 611 struct ut_sequence { 612 bool complete; 613 int status; 614 }; 615 616 static void 617 ut_sequence_step_cb(void *cb_arg) 618 { 619 int *completed = cb_arg; 620 621 (*completed)++; 622 } 623 624 static void 625 ut_sequence_complete_cb(void *cb_arg, int status) 626 { 627 struct ut_sequence *seq = cb_arg; 628 629 seq->complete = true; 630 seq->status = status; 631 } 632 633 static void 634 test_sequence_fill_copy(void) 635 { 636 struct spdk_accel_sequence *seq = NULL; 637 struct spdk_io_channel *ioch; 638 struct ut_sequence ut_seq; 639 char buf[4096], tmp[2][4096], expected[4096]; 640 struct iovec src_iovs[2], dst_iovs[2]; 641 int rc, completed; 642 643 ioch = spdk_accel_get_io_channel(); 644 SPDK_CU_ASSERT_FATAL(ioch != NULL); 645 646 /* First check the simplest case - single task in a sequence */ 647 memset(buf, 0, sizeof(buf)); 648 memset(expected, 0xa5, sizeof(expected)); 649 completed = 0; 650 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 651 ut_sequence_step_cb, &completed); 652 CU_ASSERT_EQUAL(rc, 0); 653 CU_ASSERT_EQUAL(completed, 0); 654 655 ut_seq.complete = false; 656 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 657 658 poll_threads(); 659 CU_ASSERT_EQUAL(completed, 1); 660 CU_ASSERT(ut_seq.complete); 661 CU_ASSERT_EQUAL(ut_seq.status, 0); 662 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 663 664 /* Check a single copy operation */ 665 memset(buf, 0, sizeof(buf)); 666 memset(tmp[0], 0xa5, sizeof(tmp[0])); 667 memset(expected, 0xa5, sizeof(expected)); 668 completed = 0; 669 seq = NULL; 670 671 dst_iovs[0].iov_base = buf; 672 dst_iovs[0].iov_len = sizeof(buf); 673 src_iovs[0].iov_base = tmp[0]; 674 src_iovs[0].iov_len = sizeof(tmp[0]); 675 676 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 677 &src_iovs[0], 1, NULL, NULL, 0, 678 ut_sequence_step_cb, &completed); 679 CU_ASSERT_EQUAL(rc, 0); 680 681 ut_seq.complete = false; 682 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 683 684 poll_threads(); 685 CU_ASSERT_EQUAL(completed, 1); 686 CU_ASSERT(ut_seq.complete); 687 CU_ASSERT_EQUAL(ut_seq.status, 0); 688 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 689 690 /* Check multiple fill operations */ 691 memset(buf, 0, sizeof(buf)); 692 memset(expected, 0xfe, 4096); 693 memset(expected, 0xde, 2048); 694 memset(expected, 0xa5, 1024); 695 seq = NULL; 696 completed = 0; 697 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, NULL, NULL, 0xfe, 0, 698 ut_sequence_step_cb, &completed); 699 CU_ASSERT_EQUAL(rc, 0); 700 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xde, 0, 701 ut_sequence_step_cb, &completed); 702 CU_ASSERT_EQUAL(rc, 0); 703 rc = spdk_accel_append_fill(&seq, ioch, buf, 1024, NULL, NULL, 0xa5, 0, 704 ut_sequence_step_cb, &completed); 705 CU_ASSERT_EQUAL(rc, 0); 706 707 ut_seq.complete = false; 708 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 709 710 poll_threads(); 711 CU_ASSERT_EQUAL(completed, 3); 712 CU_ASSERT(ut_seq.complete); 713 CU_ASSERT_EQUAL(ut_seq.status, 0); 714 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 715 716 /* Check multiple copy operations */ 717 memset(buf, 0, sizeof(buf)); 718 memset(tmp[0], 0, sizeof(tmp[0])); 719 memset(tmp[1], 0, sizeof(tmp[1])); 720 memset(expected, 0xa5, sizeof(expected)); 721 seq = NULL; 722 completed = 0; 723 724 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 725 ut_sequence_step_cb, &completed); 726 CU_ASSERT_EQUAL(rc, 0); 727 728 dst_iovs[0].iov_base = tmp[1]; 729 dst_iovs[0].iov_len = sizeof(tmp[1]); 730 src_iovs[0].iov_base = tmp[0]; 731 src_iovs[0].iov_len = sizeof(tmp[0]); 732 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 733 &src_iovs[0], 1, NULL, NULL, 0, 734 ut_sequence_step_cb, &completed); 735 CU_ASSERT_EQUAL(rc, 0); 736 737 dst_iovs[1].iov_base = buf; 738 dst_iovs[1].iov_len = sizeof(buf); 739 src_iovs[1].iov_base = tmp[1]; 740 src_iovs[1].iov_len = sizeof(tmp[1]); 741 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 742 &src_iovs[1], 1, NULL, NULL, 0, 743 ut_sequence_step_cb, &completed); 744 CU_ASSERT_EQUAL(rc, 0); 745 746 ut_seq.complete = false; 747 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 748 749 poll_threads(); 750 CU_ASSERT_EQUAL(completed, 3); 751 CU_ASSERT(ut_seq.complete); 752 CU_ASSERT_EQUAL(ut_seq.status, 0); 753 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 754 755 /* Check that adding a copy operation at the end will change destination buffer */ 756 memset(buf, 0, sizeof(buf)); 757 memset(tmp[0], 0, sizeof(tmp[0])); 758 memset(expected, 0xa5, sizeof(buf)); 759 seq = NULL; 760 completed = 0; 761 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 762 ut_sequence_step_cb, &completed); 763 CU_ASSERT_EQUAL(rc, 0); 764 765 dst_iovs[0].iov_base = buf; 766 dst_iovs[0].iov_len = sizeof(buf); 767 src_iovs[0].iov_base = tmp[0]; 768 src_iovs[0].iov_len = sizeof(tmp[0]); 769 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 770 &src_iovs[0], 1, NULL, NULL, 0, 771 ut_sequence_step_cb, &completed); 772 CU_ASSERT_EQUAL(rc, 0); 773 774 ut_seq.complete = false; 775 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 776 777 poll_threads(); 778 CU_ASSERT_EQUAL(completed, 2); 779 CU_ASSERT(ut_seq.complete); 780 CU_ASSERT_EQUAL(ut_seq.status, 0); 781 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 782 783 /* Check that it's also possible to add copy operation at the beginning */ 784 memset(buf, 0, sizeof(buf)); 785 memset(tmp[0], 0xde, sizeof(tmp[0])); 786 memset(tmp[1], 0, sizeof(tmp[1])); 787 memset(expected, 0xa5, sizeof(expected)); 788 seq = NULL; 789 completed = 0; 790 791 dst_iovs[0].iov_base = tmp[1]; 792 dst_iovs[0].iov_len = sizeof(tmp[1]); 793 src_iovs[0].iov_base = tmp[0]; 794 src_iovs[0].iov_len = sizeof(tmp[0]); 795 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 796 &src_iovs[0], 1, NULL, NULL, 0, 797 ut_sequence_step_cb, &completed); 798 CU_ASSERT_EQUAL(rc, 0); 799 800 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], sizeof(tmp[1]), NULL, NULL, 0xa5, 0, 801 ut_sequence_step_cb, &completed); 802 CU_ASSERT_EQUAL(rc, 0); 803 804 dst_iovs[1].iov_base = buf; 805 dst_iovs[1].iov_len = sizeof(buf); 806 src_iovs[1].iov_base = tmp[1]; 807 src_iovs[1].iov_len = sizeof(tmp[1]); 808 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 809 &src_iovs[1], 1, NULL, NULL, 0, 810 ut_sequence_step_cb, &completed); 811 CU_ASSERT_EQUAL(rc, 0); 812 813 ut_seq.complete = false; 814 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 815 816 poll_threads(); 817 CU_ASSERT_EQUAL(completed, 3); 818 CU_ASSERT(ut_seq.complete); 819 CU_ASSERT_EQUAL(ut_seq.status, 0); 820 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 821 822 spdk_put_io_channel(ioch); 823 poll_threads(); 824 } 825 826 static void 827 test_sequence_abort(void) 828 { 829 struct spdk_accel_sequence *seq = NULL; 830 struct spdk_io_channel *ioch; 831 char buf[4096], tmp[2][4096], expected[4096]; 832 struct iovec src_iovs[2], dst_iovs[2]; 833 int rc, completed; 834 835 ioch = spdk_accel_get_io_channel(); 836 SPDK_CU_ASSERT_FATAL(ioch != NULL); 837 838 /* Check that aborting a sequence calls operation's callback, the operation is not executed 839 * and the sequence is freed 840 */ 841 memset(buf, 0, sizeof(buf)); 842 memset(expected, 0, sizeof(buf)); 843 completed = 0; 844 seq = NULL; 845 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 846 ut_sequence_step_cb, &completed); 847 CU_ASSERT_EQUAL(rc, 0); 848 849 spdk_accel_sequence_abort(seq); 850 CU_ASSERT_EQUAL(completed, 1); 851 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 852 853 /* Check sequence with multiple operations */ 854 memset(buf, 0, sizeof(buf)); 855 memset(expected, 0, sizeof(buf)); 856 completed = 0; 857 seq = NULL; 858 859 dst_iovs[0].iov_base = tmp[1]; 860 dst_iovs[0].iov_len = sizeof(tmp[1]); 861 src_iovs[0].iov_base = tmp[0]; 862 src_iovs[0].iov_len = sizeof(tmp[0]); 863 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 864 &src_iovs[0], 1, NULL, NULL, 0, 865 ut_sequence_step_cb, &completed); 866 CU_ASSERT_EQUAL(rc, 0); 867 868 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 4096, NULL, NULL, 0xa5, 0, 869 ut_sequence_step_cb, &completed); 870 CU_ASSERT_EQUAL(rc, 0); 871 872 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xde, 0, 873 ut_sequence_step_cb, &completed); 874 CU_ASSERT_EQUAL(rc, 0); 875 876 dst_iovs[1].iov_base = buf; 877 dst_iovs[1].iov_len = sizeof(buf); 878 src_iovs[1].iov_base = tmp[1]; 879 src_iovs[1].iov_len = sizeof(tmp[1]); 880 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 881 &src_iovs[1], 1, NULL, NULL, 0, 882 ut_sequence_step_cb, &completed); 883 CU_ASSERT_EQUAL(rc, 0); 884 885 spdk_accel_sequence_abort(seq); 886 CU_ASSERT_EQUAL(completed, 4); 887 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 888 889 /* This should be a no-op */ 890 spdk_accel_sequence_abort(NULL); 891 892 spdk_put_io_channel(ioch); 893 poll_threads(); 894 } 895 896 static void 897 test_sequence_append_error(void) 898 { 899 struct spdk_accel_sequence *seq = NULL; 900 struct spdk_io_channel *ioch; 901 struct accel_io_channel *accel_ch; 902 struct iovec src_iovs, dst_iovs; 903 char buf[4096]; 904 TAILQ_HEAD(, spdk_accel_task) tasks = TAILQ_HEAD_INITIALIZER(tasks); 905 TAILQ_HEAD(, spdk_accel_sequence) seqs = TAILQ_HEAD_INITIALIZER(seqs); 906 int rc; 907 908 ioch = spdk_accel_get_io_channel(); 909 SPDK_CU_ASSERT_FATAL(ioch != NULL); 910 accel_ch = spdk_io_channel_get_ctx(ioch); 911 912 /* Check that append fails and no sequence object is allocated when there are no more free 913 * tasks */ 914 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 915 916 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 917 ut_sequence_step_cb, NULL); 918 CU_ASSERT_EQUAL(rc, -ENOMEM); 919 CU_ASSERT_PTR_NULL(seq); 920 921 dst_iovs.iov_base = buf; 922 dst_iovs.iov_len = 2048; 923 src_iovs.iov_base = &buf[2048]; 924 src_iovs.iov_len = 2048; 925 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 926 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 927 CU_ASSERT_EQUAL(rc, -ENOMEM); 928 CU_ASSERT_PTR_NULL(seq); 929 930 dst_iovs.iov_base = buf; 931 dst_iovs.iov_len = 2048; 932 src_iovs.iov_base = &buf[2048]; 933 src_iovs.iov_len = 2048; 934 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 935 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 936 CU_ASSERT_EQUAL(rc, -ENOMEM); 937 CU_ASSERT_PTR_NULL(seq); 938 939 /* Check that the same happens when the sequence queue is empty */ 940 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 941 TAILQ_SWAP(&seqs, &accel_ch->seq_pool, spdk_accel_sequence, link); 942 943 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 944 ut_sequence_step_cb, NULL); 945 CU_ASSERT_EQUAL(rc, -ENOMEM); 946 CU_ASSERT_PTR_NULL(seq); 947 948 dst_iovs.iov_base = buf; 949 dst_iovs.iov_len = 2048; 950 src_iovs.iov_base = &buf[2048]; 951 src_iovs.iov_len = 2048; 952 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 953 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 954 CU_ASSERT_EQUAL(rc, -ENOMEM); 955 CU_ASSERT_PTR_NULL(seq); 956 957 dst_iovs.iov_base = buf; 958 dst_iovs.iov_len = 2048; 959 src_iovs.iov_base = &buf[2048]; 960 src_iovs.iov_len = 2048; 961 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 962 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 963 CU_ASSERT_EQUAL(rc, -ENOMEM); 964 CU_ASSERT_PTR_NULL(seq); 965 966 TAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task, link); 967 968 spdk_put_io_channel(ioch); 969 poll_threads(); 970 } 971 972 struct ut_sequence_operation { 973 int complete_status; 974 int submit_status; 975 int count; 976 struct iovec *src_iovs; 977 uint32_t src_iovcnt; 978 struct spdk_memory_domain *src_domain; 979 void *src_domain_ctx; 980 struct iovec *dst_iovs; 981 uint32_t dst_iovcnt; 982 struct spdk_memory_domain *dst_domain; 983 void *dst_domain_ctx; 984 int (*submit)(struct spdk_io_channel *ch, struct spdk_accel_task *t); 985 }; 986 987 static struct ut_sequence_operation g_seq_operations[ACCEL_OPC_LAST]; 988 989 static int 990 ut_sequnce_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *task) 991 { 992 struct ut_sequence_operation *op = &g_seq_operations[task->op_code]; 993 994 op->count++; 995 if (op->submit != NULL) { 996 return op->submit(ch, task); 997 } 998 if (op->src_iovs != NULL) { 999 CU_ASSERT_EQUAL(task->s.iovcnt, op->src_iovcnt); 1000 CU_ASSERT_EQUAL(memcmp(task->s.iovs, op->src_iovs, 1001 sizeof(struct iovec) * op->src_iovcnt), 0); 1002 } 1003 if (op->dst_iovs != NULL) { 1004 CU_ASSERT_EQUAL(task->d.iovcnt, op->dst_iovcnt); 1005 CU_ASSERT_EQUAL(memcmp(task->d.iovs, op->dst_iovs, 1006 sizeof(struct iovec) * op->dst_iovcnt), 0); 1007 } 1008 1009 CU_ASSERT_EQUAL(task->src_domain, op->src_domain); 1010 CU_ASSERT_EQUAL(task->dst_domain, op->dst_domain); 1011 if (op->src_domain != NULL) { 1012 CU_ASSERT_EQUAL(task->src_domain_ctx, op->src_domain_ctx); 1013 } 1014 if (op->dst_domain != NULL) { 1015 CU_ASSERT_EQUAL(task->dst_domain_ctx, op->dst_domain_ctx); 1016 } 1017 1018 if (op->submit_status != 0) { 1019 return op->submit_status; 1020 } 1021 1022 spdk_accel_task_complete(task, op->complete_status); 1023 1024 return 0; 1025 } 1026 1027 static void 1028 ut_clear_operations(void) 1029 { 1030 memset(&g_seq_operations, 0, sizeof(g_seq_operations)); 1031 } 1032 1033 static void 1034 test_sequence_completion_error(void) 1035 { 1036 struct spdk_accel_sequence *seq = NULL; 1037 struct spdk_io_channel *ioch; 1038 struct ut_sequence ut_seq; 1039 struct iovec src_iovs, dst_iovs; 1040 char buf[4096], tmp[4096]; 1041 struct accel_module modules[ACCEL_OPC_LAST]; 1042 int i, rc, completed; 1043 1044 ioch = spdk_accel_get_io_channel(); 1045 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1046 1047 /* Override the submit_tasks function */ 1048 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1049 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1050 modules[i] = g_modules_opc[i]; 1051 g_modules_opc[i] = g_module; 1052 } 1053 1054 memset(buf, 0, sizeof(buf)); 1055 memset(tmp, 0, sizeof(tmp)); 1056 1057 /* Check that if the first operation completes with an error, the whole sequence is 1058 * completed with that error and that all operations' completion callbacks are executed 1059 */ 1060 g_seq_operations[ACCEL_OPC_FILL].complete_status = -E2BIG; 1061 completed = 0; 1062 seq = NULL; 1063 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1064 ut_sequence_step_cb, &completed); 1065 CU_ASSERT_EQUAL(rc, 0); 1066 1067 dst_iovs.iov_base = buf; 1068 dst_iovs.iov_len = sizeof(buf); 1069 src_iovs.iov_base = tmp; 1070 src_iovs.iov_len = sizeof(tmp); 1071 1072 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1073 &src_iovs, 1, NULL, NULL, 0, 1074 ut_sequence_step_cb, &completed); 1075 CU_ASSERT_EQUAL(rc, 0); 1076 1077 ut_seq.complete = false; 1078 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1079 1080 poll_threads(); 1081 CU_ASSERT_EQUAL(completed, 2); 1082 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 1083 1084 /* Check the same with a second operation in the sequence */ 1085 g_seq_operations[ACCEL_OPC_DECOMPRESS].complete_status = -EACCES; 1086 g_seq_operations[ACCEL_OPC_FILL].complete_status = 0; 1087 completed = 0; 1088 seq = NULL; 1089 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1090 ut_sequence_step_cb, &completed); 1091 CU_ASSERT_EQUAL(rc, 0); 1092 1093 dst_iovs.iov_base = buf; 1094 dst_iovs.iov_len = sizeof(buf); 1095 src_iovs.iov_base = tmp; 1096 src_iovs.iov_len = sizeof(tmp); 1097 1098 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1099 &src_iovs, 1, NULL, NULL, 0, 1100 ut_sequence_step_cb, &completed); 1101 CU_ASSERT_EQUAL(rc, 0); 1102 1103 ut_seq.complete = false; 1104 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1105 1106 poll_threads(); 1107 CU_ASSERT_EQUAL(completed, 2); 1108 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 1109 1110 g_seq_operations[ACCEL_OPC_DECOMPRESS].complete_status = 0; 1111 g_seq_operations[ACCEL_OPC_FILL].complete_status = 0; 1112 1113 /* Check submission failure of the first operation */ 1114 g_seq_operations[ACCEL_OPC_FILL].submit_status = -EADDRINUSE; 1115 completed = 0; 1116 seq = NULL; 1117 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1118 ut_sequence_step_cb, &completed); 1119 CU_ASSERT_EQUAL(rc, 0); 1120 1121 dst_iovs.iov_base = buf; 1122 dst_iovs.iov_len = sizeof(buf); 1123 src_iovs.iov_base = tmp; 1124 src_iovs.iov_len = sizeof(tmp); 1125 1126 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1127 &src_iovs, 1, NULL, NULL, 0, 1128 ut_sequence_step_cb, &completed); 1129 CU_ASSERT_EQUAL(rc, 0); 1130 1131 ut_seq.complete = false; 1132 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1133 1134 poll_threads(); 1135 CU_ASSERT_EQUAL(completed, 2); 1136 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 1137 1138 /* Check the same with a second operation */ 1139 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit_status = -EADDRNOTAVAIL; 1140 g_seq_operations[ACCEL_OPC_FILL].submit_status = 0; 1141 completed = 0; 1142 seq = NULL; 1143 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1144 ut_sequence_step_cb, &completed); 1145 CU_ASSERT_EQUAL(rc, 0); 1146 1147 dst_iovs.iov_base = buf; 1148 dst_iovs.iov_len = sizeof(buf); 1149 src_iovs.iov_base = tmp; 1150 src_iovs.iov_len = sizeof(tmp); 1151 1152 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1153 &src_iovs, 1, NULL, NULL, 0, 1154 ut_sequence_step_cb, &completed); 1155 CU_ASSERT_EQUAL(rc, 0); 1156 1157 ut_seq.complete = false; 1158 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1159 1160 poll_threads(); 1161 CU_ASSERT_EQUAL(completed, 2); 1162 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 1163 1164 /* Cleanup module pointers to make subsequent tests work correctly */ 1165 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1166 g_modules_opc[i] = modules[i]; 1167 } 1168 1169 ut_clear_operations(); 1170 spdk_put_io_channel(ioch); 1171 poll_threads(); 1172 } 1173 1174 #ifdef SPDK_CONFIG_ISAL 1175 static void 1176 ut_compress_cb(void *cb_arg, int status) 1177 { 1178 int *completed = cb_arg; 1179 1180 CU_ASSERT_EQUAL(status, 0); 1181 1182 *completed = 1; 1183 } 1184 1185 static void 1186 test_sequence_decompress(void) 1187 { 1188 struct spdk_accel_sequence *seq = NULL; 1189 struct spdk_io_channel *ioch; 1190 struct ut_sequence ut_seq; 1191 char buf[4096], tmp[2][4096], expected[4096]; 1192 struct iovec src_iovs[2], dst_iovs[2]; 1193 uint32_t compressed_size; 1194 int rc, completed = 0; 1195 1196 ioch = spdk_accel_get_io_channel(); 1197 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1198 1199 memset(expected, 0xa5, sizeof(expected)); 1200 src_iovs[0].iov_base = expected; 1201 src_iovs[0].iov_len = sizeof(expected); 1202 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1203 &compressed_size, 0, ut_compress_cb, &completed); 1204 CU_ASSERT_EQUAL(rc, 0); 1205 1206 while (!completed) { 1207 poll_threads(); 1208 } 1209 1210 /* Check a single decompress operation in a sequence */ 1211 seq = NULL; 1212 completed = 0; 1213 1214 dst_iovs[0].iov_base = buf; 1215 dst_iovs[0].iov_len = sizeof(buf); 1216 src_iovs[0].iov_base = tmp[0]; 1217 src_iovs[0].iov_len = compressed_size; 1218 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1219 &src_iovs[0], 1, NULL, NULL, 0, 1220 ut_sequence_step_cb, &completed); 1221 CU_ASSERT_EQUAL(rc, 0); 1222 1223 ut_seq.complete = false; 1224 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1225 1226 poll_threads(); 1227 1228 CU_ASSERT_EQUAL(completed, 1); 1229 CU_ASSERT(ut_seq.complete); 1230 CU_ASSERT_EQUAL(ut_seq.status, 0); 1231 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1232 1233 /* Put the decompress operation in the middle of a sequence with a copy operation at the 1234 * beginning and a fill at the end modifying the first 2048B of the buffer. 1235 */ 1236 memset(expected, 0xfe, 2048); 1237 memset(buf, 0, sizeof(buf)); 1238 seq = NULL; 1239 completed = 0; 1240 1241 dst_iovs[0].iov_base = tmp[1]; 1242 dst_iovs[0].iov_len = compressed_size; 1243 src_iovs[0].iov_base = tmp[0]; 1244 src_iovs[0].iov_len = compressed_size; 1245 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1246 &src_iovs[0], 1, NULL, NULL, 0, 1247 ut_sequence_step_cb, &completed); 1248 CU_ASSERT_EQUAL(rc, 0); 1249 1250 dst_iovs[1].iov_base = buf; 1251 dst_iovs[1].iov_len = sizeof(buf); 1252 src_iovs[1].iov_base = tmp[1]; 1253 src_iovs[1].iov_len = compressed_size; 1254 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1255 &src_iovs[1], 1, NULL, NULL, 0, 1256 ut_sequence_step_cb, &completed); 1257 CU_ASSERT_EQUAL(rc, 0); 1258 1259 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1260 ut_sequence_step_cb, &completed); 1261 CU_ASSERT_EQUAL(rc, 0); 1262 1263 ut_seq.complete = false; 1264 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1265 1266 poll_threads(); 1267 1268 CU_ASSERT_EQUAL(completed, 3); 1269 CU_ASSERT(ut_seq.complete); 1270 CU_ASSERT_EQUAL(ut_seq.status, 0); 1271 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1272 1273 /* Check sequence with decompress at the beginning: decompress -> copy */ 1274 memset(expected, 0xa5, sizeof(expected)); 1275 memset(buf, 0, sizeof(buf)); 1276 seq = NULL; 1277 completed = 0; 1278 1279 dst_iovs[0].iov_base = tmp[1]; 1280 dst_iovs[0].iov_len = sizeof(tmp[1]); 1281 src_iovs[0].iov_base = tmp[0]; 1282 src_iovs[0].iov_len = compressed_size; 1283 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1284 &src_iovs[0], 1, NULL, NULL, 0, 1285 ut_sequence_step_cb, &completed); 1286 CU_ASSERT_EQUAL(rc, 0); 1287 1288 dst_iovs[1].iov_base = buf; 1289 dst_iovs[1].iov_len = sizeof(buf); 1290 src_iovs[1].iov_base = tmp[1]; 1291 src_iovs[1].iov_len = sizeof(tmp[1]); 1292 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1293 &src_iovs[1], 1, NULL, NULL, 0, 1294 ut_sequence_step_cb, &completed); 1295 CU_ASSERT_EQUAL(rc, 0); 1296 1297 ut_seq.complete = false; 1298 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1299 1300 poll_threads(); 1301 1302 CU_ASSERT_EQUAL(completed, 2); 1303 CU_ASSERT(ut_seq.complete); 1304 CU_ASSERT_EQUAL(ut_seq.status, 0); 1305 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1306 1307 spdk_put_io_channel(ioch); 1308 poll_threads(); 1309 } 1310 1311 static void 1312 test_sequence_reverse(void) 1313 { 1314 struct spdk_accel_sequence *seq = NULL; 1315 struct spdk_io_channel *ioch; 1316 struct ut_sequence ut_seq; 1317 char buf[4096], tmp[2][4096], expected[4096]; 1318 struct iovec src_iovs[2], dst_iovs[2]; 1319 uint32_t compressed_size; 1320 int rc, completed = 0; 1321 1322 ioch = spdk_accel_get_io_channel(); 1323 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1324 1325 memset(expected, 0xa5, sizeof(expected)); 1326 src_iovs[0].iov_base = expected; 1327 src_iovs[0].iov_len = sizeof(expected); 1328 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1329 &compressed_size, 0, ut_compress_cb, &completed); 1330 CU_ASSERT_EQUAL(rc, 0); 1331 1332 while (!completed) { 1333 poll_threads(); 1334 } 1335 1336 /* First check that reversing a sequnce with a single operation is a no-op */ 1337 memset(buf, 0, sizeof(buf)); 1338 seq = NULL; 1339 completed = 0; 1340 1341 dst_iovs[0].iov_base = buf; 1342 dst_iovs[0].iov_len = sizeof(buf); 1343 src_iovs[0].iov_base = tmp[0]; 1344 src_iovs[0].iov_len = compressed_size; 1345 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1346 &src_iovs[0], 1, NULL, NULL, 0, 1347 ut_sequence_step_cb, &completed); 1348 CU_ASSERT_EQUAL(rc, 0); 1349 1350 spdk_accel_sequence_reverse(seq); 1351 1352 ut_seq.complete = false; 1353 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1354 1355 poll_threads(); 1356 1357 CU_ASSERT_EQUAL(completed, 1); 1358 CU_ASSERT(ut_seq.complete); 1359 CU_ASSERT_EQUAL(ut_seq.status, 0); 1360 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1361 1362 /* Add a copy operation at the end with src set to the compressed data. After reverse(), 1363 * that copy operation should be first, so decompress() should receive compressed data in 1364 * its src buffer. 1365 */ 1366 memset(buf, 0, sizeof(buf)); 1367 memset(tmp[1], 0, sizeof(tmp[1])); 1368 seq = NULL; 1369 completed = 0; 1370 1371 dst_iovs[0].iov_base = buf; 1372 dst_iovs[0].iov_len = sizeof(buf); 1373 src_iovs[0].iov_base = tmp[1]; 1374 src_iovs[0].iov_len = compressed_size; 1375 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1376 &src_iovs[0], 1, NULL, NULL, 0, 1377 ut_sequence_step_cb, &completed); 1378 CU_ASSERT_EQUAL(rc, 0); 1379 1380 dst_iovs[1].iov_base = tmp[1]; 1381 dst_iovs[1].iov_len = compressed_size; 1382 src_iovs[1].iov_base = tmp[0]; 1383 src_iovs[1].iov_len = compressed_size; 1384 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1385 &src_iovs[1], 1, NULL, NULL, 0, 1386 ut_sequence_step_cb, &completed); 1387 CU_ASSERT_EQUAL(rc, 0); 1388 1389 spdk_accel_sequence_reverse(seq); 1390 1391 ut_seq.complete = false; 1392 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1393 1394 poll_threads(); 1395 1396 CU_ASSERT_EQUAL(completed, 2); 1397 CU_ASSERT(ut_seq.complete); 1398 CU_ASSERT_EQUAL(ut_seq.status, 0); 1399 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1400 1401 /* Check the same, but add an extra fill operation at the beginning that should execute last 1402 * after reverse(). 1403 */ 1404 memset(buf, 0, sizeof(buf)); 1405 memset(tmp[1], 0, sizeof(tmp[1])); 1406 memset(expected, 0xfe, 2048); 1407 seq = NULL; 1408 completed = 0; 1409 1410 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1411 ut_sequence_step_cb, &completed); 1412 CU_ASSERT_EQUAL(rc, 0); 1413 1414 dst_iovs[0].iov_base = buf; 1415 dst_iovs[0].iov_len = sizeof(buf); 1416 src_iovs[0].iov_base = tmp[1]; 1417 src_iovs[0].iov_len = compressed_size; 1418 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1419 &src_iovs[0], 1, NULL, NULL, 0, 1420 ut_sequence_step_cb, &completed); 1421 CU_ASSERT_EQUAL(rc, 0); 1422 1423 dst_iovs[1].iov_base = tmp[1]; 1424 dst_iovs[1].iov_len = compressed_size; 1425 src_iovs[1].iov_base = tmp[0]; 1426 src_iovs[1].iov_len = compressed_size; 1427 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1428 &src_iovs[1], 1, NULL, NULL, 0, 1429 ut_sequence_step_cb, &completed); 1430 CU_ASSERT_EQUAL(rc, 0); 1431 1432 spdk_accel_sequence_reverse(seq); 1433 1434 ut_seq.complete = false; 1435 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1436 1437 poll_threads(); 1438 1439 CU_ASSERT_EQUAL(completed, 3); 1440 CU_ASSERT(ut_seq.complete); 1441 CU_ASSERT_EQUAL(ut_seq.status, 0); 1442 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1443 1444 /* Build the sequence in order and then reverse it twice */ 1445 memset(buf, 0, sizeof(buf)); 1446 memset(tmp[1], 0, sizeof(tmp[1])); 1447 seq = NULL; 1448 completed = 0; 1449 1450 dst_iovs[0].iov_base = tmp[1]; 1451 dst_iovs[0].iov_len = compressed_size; 1452 src_iovs[0].iov_base = tmp[0]; 1453 src_iovs[0].iov_len = compressed_size; 1454 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1455 &src_iovs[0], 1, NULL, NULL, 0, 1456 ut_sequence_step_cb, &completed); 1457 CU_ASSERT_EQUAL(rc, 0); 1458 1459 dst_iovs[1].iov_base = buf; 1460 dst_iovs[1].iov_len = sizeof(buf); 1461 src_iovs[1].iov_base = tmp[1]; 1462 src_iovs[1].iov_len = compressed_size; 1463 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1464 &src_iovs[1], 1, NULL, NULL, 0, 1465 ut_sequence_step_cb, &completed); 1466 CU_ASSERT_EQUAL(rc, 0); 1467 1468 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1469 ut_sequence_step_cb, &completed); 1470 CU_ASSERT_EQUAL(rc, 0); 1471 1472 spdk_accel_sequence_reverse(seq); 1473 spdk_accel_sequence_reverse(seq); 1474 1475 ut_seq.complete = false; 1476 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1477 1478 poll_threads(); 1479 1480 CU_ASSERT_EQUAL(completed, 3); 1481 CU_ASSERT(ut_seq.complete); 1482 CU_ASSERT_EQUAL(ut_seq.status, 0); 1483 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1484 1485 spdk_put_io_channel(ioch); 1486 poll_threads(); 1487 } 1488 #endif 1489 1490 static void 1491 test_sequence_copy_elision(void) 1492 { 1493 struct spdk_accel_sequence *seq = NULL; 1494 struct spdk_io_channel *ioch; 1495 struct ut_sequence ut_seq; 1496 struct iovec src_iovs[4], dst_iovs[4], exp_iovs[2]; 1497 char buf[4096], tmp[4][4096]; 1498 struct accel_module modules[ACCEL_OPC_LAST]; 1499 struct spdk_accel_crypto_key key = {}; 1500 int i, rc, completed; 1501 1502 ioch = spdk_accel_get_io_channel(); 1503 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1504 1505 /* Override the submit_tasks function */ 1506 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1507 g_module.supports_memory_domains = true; 1508 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1509 g_seq_operations[i].complete_status = 0; 1510 g_seq_operations[i].submit_status = 0; 1511 g_seq_operations[i].count = 0; 1512 1513 modules[i] = g_modules_opc[i]; 1514 g_modules_opc[i] = g_module; 1515 } 1516 1517 /* Check that a copy operation at the beginning is removed */ 1518 seq = NULL; 1519 completed = 0; 1520 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovcnt = 1; 1521 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovcnt = 1; 1522 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1523 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1524 exp_iovs[0].iov_base = tmp[0]; 1525 exp_iovs[0].iov_len = sizeof(tmp[0]); 1526 exp_iovs[1].iov_base = buf; 1527 exp_iovs[1].iov_len = 2048; 1528 1529 dst_iovs[0].iov_base = tmp[1]; 1530 dst_iovs[0].iov_len = sizeof(tmp[1]); 1531 src_iovs[0].iov_base = tmp[0]; 1532 src_iovs[0].iov_len = sizeof(tmp[0]); 1533 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1534 &src_iovs[0], 1, NULL, NULL, 0, 1535 ut_sequence_step_cb, &completed); 1536 CU_ASSERT_EQUAL(rc, 0); 1537 1538 dst_iovs[1].iov_base = buf; 1539 dst_iovs[1].iov_len = 2048; 1540 src_iovs[1].iov_base = tmp[1]; 1541 src_iovs[1].iov_len = sizeof(tmp[1]); 1542 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1543 &src_iovs[1], 1, NULL, NULL, 0, 1544 ut_sequence_step_cb, &completed); 1545 CU_ASSERT_EQUAL(rc, 0); 1546 1547 ut_seq.complete = false; 1548 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1549 1550 poll_threads(); 1551 1552 CU_ASSERT_EQUAL(completed, 2); 1553 CU_ASSERT(ut_seq.complete); 1554 CU_ASSERT_EQUAL(ut_seq.status, 0); 1555 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1556 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 1557 1558 /* Check that a copy operation at the end is removed too */ 1559 seq = NULL; 1560 completed = 0; 1561 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1562 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 1563 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1564 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1565 exp_iovs[0].iov_base = tmp[0]; 1566 exp_iovs[0].iov_len = sizeof(tmp[0]); 1567 exp_iovs[1].iov_base = buf; 1568 exp_iovs[1].iov_len = 2048; 1569 1570 dst_iovs[0].iov_base = tmp[1]; 1571 dst_iovs[0].iov_len = 2048; 1572 src_iovs[0].iov_base = tmp[0]; 1573 src_iovs[0].iov_len = sizeof(tmp[0]); 1574 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1575 &src_iovs[0], 1, NULL, NULL, 0, 1576 ut_sequence_step_cb, &completed); 1577 CU_ASSERT_EQUAL(rc, 0); 1578 1579 dst_iovs[1].iov_base = buf; 1580 dst_iovs[1].iov_len = 2048; 1581 src_iovs[1].iov_base = tmp[1]; 1582 src_iovs[1].iov_len = 2048; 1583 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1584 &src_iovs[1], 1, NULL, NULL, 0, 1585 ut_sequence_step_cb, &completed); 1586 CU_ASSERT_EQUAL(rc, 0); 1587 1588 ut_seq.complete = false; 1589 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1590 1591 poll_threads(); 1592 1593 CU_ASSERT_EQUAL(completed, 2); 1594 CU_ASSERT(ut_seq.complete); 1595 CU_ASSERT_EQUAL(ut_seq.status, 0); 1596 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1597 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 1598 1599 /* Check a copy operation both at the beginning and the end */ 1600 seq = NULL; 1601 completed = 0; 1602 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1603 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 1604 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1605 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1606 exp_iovs[0].iov_base = tmp[0]; 1607 exp_iovs[0].iov_len = sizeof(tmp[0]); 1608 exp_iovs[1].iov_base = buf; 1609 exp_iovs[1].iov_len = 2048; 1610 1611 dst_iovs[0].iov_base = tmp[1]; 1612 dst_iovs[0].iov_len = sizeof(tmp[1]); 1613 src_iovs[0].iov_base = tmp[0]; 1614 src_iovs[0].iov_len = sizeof(tmp[0]); 1615 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1616 &src_iovs[0], 1, NULL, NULL, 0, 1617 ut_sequence_step_cb, &completed); 1618 CU_ASSERT_EQUAL(rc, 0); 1619 1620 dst_iovs[1].iov_base = tmp[2]; 1621 dst_iovs[1].iov_len = 2048; 1622 src_iovs[1].iov_base = tmp[1]; 1623 src_iovs[1].iov_len = sizeof(tmp[1]); 1624 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1625 &src_iovs[1], 1, NULL, NULL, 0, 1626 ut_sequence_step_cb, &completed); 1627 CU_ASSERT_EQUAL(rc, 0); 1628 1629 dst_iovs[2].iov_base = buf; 1630 dst_iovs[2].iov_len = 2048; 1631 src_iovs[2].iov_base = tmp[2]; 1632 src_iovs[2].iov_len = 2048; 1633 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1634 &src_iovs[2], 1, NULL, NULL, 0, 1635 ut_sequence_step_cb, &completed); 1636 CU_ASSERT_EQUAL(rc, 0); 1637 1638 ut_seq.complete = false; 1639 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1640 1641 poll_threads(); 1642 1643 CU_ASSERT_EQUAL(completed, 3); 1644 CU_ASSERT(ut_seq.complete); 1645 CU_ASSERT_EQUAL(ut_seq.status, 0); 1646 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1647 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 1648 1649 /* Check decompress + copy + decompress + copy */ 1650 seq = NULL; 1651 completed = 0; 1652 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1653 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 1654 g_seq_operations[ACCEL_OPC_DECOMPRESS].src_iovs = NULL; 1655 g_seq_operations[ACCEL_OPC_DECOMPRESS].dst_iovs = NULL; 1656 1657 dst_iovs[0].iov_base = tmp[1]; 1658 dst_iovs[0].iov_len = sizeof(tmp[1]); 1659 src_iovs[0].iov_base = tmp[0]; 1660 src_iovs[0].iov_len = sizeof(tmp[0]); 1661 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1662 &src_iovs[0], 1, NULL, NULL, 0, 1663 ut_sequence_step_cb, &completed); 1664 CU_ASSERT_EQUAL(rc, 0); 1665 1666 dst_iovs[1].iov_base = tmp[2]; 1667 dst_iovs[1].iov_len = 2048; 1668 src_iovs[1].iov_base = tmp[1]; 1669 src_iovs[1].iov_len = sizeof(tmp[1]); 1670 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1671 &src_iovs[1], 1, NULL, NULL, 0, 1672 ut_sequence_step_cb, &completed); 1673 CU_ASSERT_EQUAL(rc, 0); 1674 1675 dst_iovs[2].iov_base = tmp[3]; 1676 dst_iovs[2].iov_len = 1024; 1677 src_iovs[2].iov_base = tmp[2]; 1678 src_iovs[2].iov_len = 2048; 1679 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1680 &src_iovs[2], 1, NULL, NULL, 0, 1681 ut_sequence_step_cb, &completed); 1682 CU_ASSERT_EQUAL(rc, 0); 1683 1684 dst_iovs[3].iov_base = buf; 1685 dst_iovs[3].iov_len = 1024; 1686 src_iovs[3].iov_base = tmp[3]; 1687 src_iovs[3].iov_len = 1024; 1688 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 1689 &src_iovs[3], 1, NULL, NULL, 0, 1690 ut_sequence_step_cb, &completed); 1691 CU_ASSERT_EQUAL(rc, 0); 1692 1693 ut_seq.complete = false; 1694 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1695 1696 poll_threads(); 1697 1698 CU_ASSERT_EQUAL(completed, 4); 1699 CU_ASSERT(ut_seq.complete); 1700 CU_ASSERT_EQUAL(ut_seq.status, 0); 1701 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1702 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 2); 1703 1704 /* Check two copy operations - one of them should be removed, while the other should be 1705 * executed normally */ 1706 seq = NULL; 1707 completed = 0; 1708 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1709 g_seq_operations[ACCEL_OPC_COPY].dst_iovcnt = 1; 1710 g_seq_operations[ACCEL_OPC_COPY].src_iovcnt = 1; 1711 g_seq_operations[ACCEL_OPC_COPY].src_iovs = &exp_iovs[0]; 1712 g_seq_operations[ACCEL_OPC_COPY].dst_iovs = &exp_iovs[1]; 1713 exp_iovs[0].iov_base = tmp[0]; 1714 exp_iovs[0].iov_len = sizeof(tmp[0]); 1715 exp_iovs[1].iov_base = buf; 1716 exp_iovs[1].iov_len = sizeof(buf); 1717 1718 dst_iovs[0].iov_base = tmp[1]; 1719 dst_iovs[0].iov_len = sizeof(tmp[1]); 1720 src_iovs[0].iov_base = tmp[0]; 1721 src_iovs[0].iov_len = sizeof(tmp[0]); 1722 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1723 &src_iovs[0], 1, NULL, NULL, 0, 1724 ut_sequence_step_cb, &completed); 1725 CU_ASSERT_EQUAL(rc, 0); 1726 1727 dst_iovs[1].iov_base = buf; 1728 dst_iovs[1].iov_len = sizeof(buf); 1729 src_iovs[1].iov_base = tmp[1]; 1730 src_iovs[1].iov_len = sizeof(tmp[1]); 1731 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1732 &src_iovs[1], 1, NULL, NULL, 0, 1733 ut_sequence_step_cb, &completed); 1734 CU_ASSERT_EQUAL(rc, 0); 1735 1736 ut_seq.complete = false; 1737 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1738 1739 poll_threads(); 1740 1741 CU_ASSERT_EQUAL(completed, 2); 1742 CU_ASSERT(ut_seq.complete); 1743 CU_ASSERT_EQUAL(ut_seq.status, 0); 1744 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 1); 1745 1746 /* Check fill + copy */ 1747 seq = NULL; 1748 completed = 0; 1749 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1750 g_seq_operations[ACCEL_OPC_FILL].count = 0; 1751 g_seq_operations[ACCEL_OPC_COPY].src_iovs = NULL; 1752 g_seq_operations[ACCEL_OPC_COPY].dst_iovs = NULL; 1753 g_seq_operations[ACCEL_OPC_FILL].dst_iovcnt = 1; 1754 g_seq_operations[ACCEL_OPC_FILL].dst_iovs = &exp_iovs[0]; 1755 exp_iovs[0].iov_base = buf; 1756 exp_iovs[0].iov_len = sizeof(buf); 1757 1758 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 1759 ut_sequence_step_cb, &completed); 1760 CU_ASSERT_EQUAL(rc, 0); 1761 1762 dst_iovs[0].iov_base = buf; 1763 dst_iovs[0].iov_len = sizeof(buf); 1764 src_iovs[0].iov_base = tmp[0]; 1765 src_iovs[0].iov_len = sizeof(tmp[0]); 1766 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1767 &src_iovs[0], 1, NULL, NULL, 0, 1768 ut_sequence_step_cb, &completed); 1769 CU_ASSERT_EQUAL(rc, 0); 1770 1771 ut_seq.complete = false; 1772 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1773 1774 poll_threads(); 1775 1776 CU_ASSERT_EQUAL(completed, 2); 1777 CU_ASSERT(ut_seq.complete); 1778 CU_ASSERT_EQUAL(ut_seq.status, 0); 1779 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1780 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 1); 1781 1782 /* Check copy + encrypt + copy */ 1783 seq = NULL; 1784 completed = 0; 1785 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1786 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 1787 g_seq_operations[ACCEL_OPC_ENCRYPT].dst_iovcnt = 1; 1788 g_seq_operations[ACCEL_OPC_ENCRYPT].src_iovcnt = 1; 1789 g_seq_operations[ACCEL_OPC_ENCRYPT].src_iovs = &exp_iovs[0]; 1790 g_seq_operations[ACCEL_OPC_ENCRYPT].dst_iovs = &exp_iovs[1]; 1791 exp_iovs[0].iov_base = tmp[0]; 1792 exp_iovs[0].iov_len = sizeof(tmp[0]); 1793 exp_iovs[1].iov_base = buf; 1794 exp_iovs[1].iov_len = sizeof(buf); 1795 1796 dst_iovs[0].iov_base = tmp[1]; 1797 dst_iovs[0].iov_len = sizeof(tmp[1]); 1798 src_iovs[0].iov_base = tmp[0]; 1799 src_iovs[0].iov_len = sizeof(tmp[0]); 1800 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1801 &src_iovs[0], 1, NULL, NULL, 0, 1802 ut_sequence_step_cb, &completed); 1803 CU_ASSERT_EQUAL(rc, 0); 1804 1805 dst_iovs[1].iov_base = tmp[2]; 1806 dst_iovs[1].iov_len = sizeof(tmp[2]); 1807 src_iovs[1].iov_base = tmp[1]; 1808 src_iovs[1].iov_len = sizeof(tmp[1]); 1809 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1810 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1811 ut_sequence_step_cb, &completed); 1812 CU_ASSERT_EQUAL(rc, 0); 1813 1814 dst_iovs[2].iov_base = buf; 1815 dst_iovs[2].iov_len = sizeof(buf); 1816 src_iovs[2].iov_base = tmp[2]; 1817 src_iovs[2].iov_len = sizeof(tmp[2]); 1818 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1819 &src_iovs[2], 1, NULL, NULL, 0, 1820 ut_sequence_step_cb, &completed); 1821 CU_ASSERT_EQUAL(rc, 0); 1822 1823 ut_seq.complete = false; 1824 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1825 1826 poll_threads(); 1827 1828 CU_ASSERT_EQUAL(completed, 3); 1829 CU_ASSERT(ut_seq.complete); 1830 CU_ASSERT_EQUAL(ut_seq.status, 0); 1831 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1832 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 1833 1834 /* Check copy + decrypt + copy */ 1835 seq = NULL; 1836 completed = 0; 1837 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1838 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 1839 g_seq_operations[ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1840 g_seq_operations[ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1841 g_seq_operations[ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1842 g_seq_operations[ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1843 exp_iovs[0].iov_base = tmp[0]; 1844 exp_iovs[0].iov_len = sizeof(tmp[0]); 1845 exp_iovs[1].iov_base = buf; 1846 exp_iovs[1].iov_len = sizeof(buf); 1847 1848 dst_iovs[0].iov_base = tmp[1]; 1849 dst_iovs[0].iov_len = sizeof(tmp[1]); 1850 src_iovs[0].iov_base = tmp[0]; 1851 src_iovs[0].iov_len = sizeof(tmp[0]); 1852 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1853 &src_iovs[0], 1, NULL, NULL, 0, 1854 ut_sequence_step_cb, &completed); 1855 CU_ASSERT_EQUAL(rc, 0); 1856 1857 dst_iovs[1].iov_base = tmp[2]; 1858 dst_iovs[1].iov_len = sizeof(tmp[2]); 1859 src_iovs[1].iov_base = tmp[1]; 1860 src_iovs[1].iov_len = sizeof(tmp[1]); 1861 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1862 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1863 ut_sequence_step_cb, &completed); 1864 CU_ASSERT_EQUAL(rc, 0); 1865 1866 dst_iovs[2].iov_base = buf; 1867 dst_iovs[2].iov_len = sizeof(buf); 1868 src_iovs[2].iov_base = tmp[2]; 1869 src_iovs[2].iov_len = sizeof(tmp[2]); 1870 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1871 &src_iovs[2], 1, NULL, NULL, 0, 1872 ut_sequence_step_cb, &completed); 1873 CU_ASSERT_EQUAL(rc, 0); 1874 1875 ut_seq.complete = false; 1876 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1877 1878 poll_threads(); 1879 1880 CU_ASSERT_EQUAL(completed, 3); 1881 CU_ASSERT(ut_seq.complete); 1882 CU_ASSERT_EQUAL(ut_seq.status, 0); 1883 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1884 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 1885 1886 /* Check a sequence with memory domains and verify their pointers (and their contexts) are 1887 * correctly set on the next/prev task when a copy is removed */ 1888 seq = NULL; 1889 completed = 0; 1890 g_seq_operations[ACCEL_OPC_COPY].count = 0; 1891 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 1892 g_seq_operations[ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1893 g_seq_operations[ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1894 g_seq_operations[ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1895 g_seq_operations[ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1896 g_seq_operations[ACCEL_OPC_DECRYPT].dst_domain = (void *)0x1; 1897 g_seq_operations[ACCEL_OPC_DECRYPT].dst_domain_ctx = (void *)0x2; 1898 g_seq_operations[ACCEL_OPC_DECRYPT].src_domain = (void *)0x3; 1899 g_seq_operations[ACCEL_OPC_DECRYPT].src_domain_ctx = (void *)0x4; 1900 exp_iovs[0].iov_base = tmp[0]; 1901 exp_iovs[0].iov_len = sizeof(tmp[0]); 1902 exp_iovs[1].iov_base = buf; 1903 exp_iovs[1].iov_len = sizeof(buf); 1904 1905 dst_iovs[0].iov_base = tmp[1]; 1906 dst_iovs[0].iov_len = sizeof(tmp[1]); 1907 src_iovs[0].iov_base = tmp[0]; 1908 src_iovs[0].iov_len = sizeof(tmp[0]); 1909 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, (void *)0xdead, (void *)0xbeef, 1910 &src_iovs[0], 1, (void *)0x3, (void *)0x4, 0, 1911 ut_sequence_step_cb, &completed); 1912 CU_ASSERT_EQUAL(rc, 0); 1913 1914 dst_iovs[1].iov_base = tmp[2]; 1915 dst_iovs[1].iov_len = sizeof(tmp[2]); 1916 src_iovs[1].iov_base = tmp[1]; 1917 src_iovs[1].iov_len = sizeof(tmp[1]); 1918 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 1919 &src_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 0, 1920 sizeof(tmp[2]), 0, ut_sequence_step_cb, &completed); 1921 CU_ASSERT_EQUAL(rc, 0); 1922 1923 dst_iovs[2].iov_base = buf; 1924 dst_iovs[2].iov_len = sizeof(buf); 1925 src_iovs[2].iov_base = tmp[2]; 1926 src_iovs[2].iov_len = sizeof(tmp[2]); 1927 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, (void *)0x1, (void *)0x2, 1928 &src_iovs[2], 1, (void *)0xdead, (void *)0xbeef, 0, 1929 ut_sequence_step_cb, &completed); 1930 CU_ASSERT_EQUAL(rc, 0); 1931 1932 ut_seq.complete = false; 1933 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1934 1935 poll_threads(); 1936 1937 CU_ASSERT_EQUAL(completed, 3); 1938 CU_ASSERT(ut_seq.complete); 1939 CU_ASSERT_EQUAL(ut_seq.status, 0); 1940 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 1941 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 1942 1943 /* Cleanup module pointers to make subsequent tests work correctly */ 1944 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1945 g_modules_opc[i] = modules[i]; 1946 } 1947 1948 g_module.supports_memory_domains = false; 1949 ut_clear_operations(); 1950 spdk_put_io_channel(ioch); 1951 poll_threads(); 1952 } 1953 1954 static int 1955 ut_submit_decompress(struct spdk_io_channel *ch, struct spdk_accel_task *task) 1956 { 1957 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 1958 1959 spdk_accel_task_complete(task, 0); 1960 1961 return 0; 1962 } 1963 1964 static void 1965 test_sequence_accel_buffers(void) 1966 { 1967 struct spdk_accel_sequence *seq = NULL; 1968 struct spdk_io_channel *ioch; 1969 struct accel_io_channel *accel_ch; 1970 struct ut_sequence ut_seq; 1971 struct iovec src_iovs[3], dst_iovs[3]; 1972 char srcbuf[4096], dstbuf[4096], expected[4096]; 1973 struct accel_module modules[ACCEL_OPC_LAST]; 1974 void *buf[2], *domain_ctx[2], *iobuf_buf; 1975 struct spdk_memory_domain *domain[2]; 1976 struct spdk_iobuf_buffer *cache_entry; 1977 spdk_iobuf_buffer_stailq_t small_cache; 1978 uint32_t small_cache_count; 1979 int i, rc, completed; 1980 struct spdk_iobuf_opts opts_iobuf = {}; 1981 1982 /* Set up the iobuf to always use the "small" pool */ 1983 opts_iobuf.large_bufsize = 0x20000; 1984 opts_iobuf.large_pool_count = 0; 1985 opts_iobuf.small_bufsize = 0x10000; 1986 opts_iobuf.small_pool_count = 32; 1987 1988 rc = spdk_iobuf_set_opts(&opts_iobuf); 1989 CU_ASSERT(rc == 0); 1990 1991 ioch = spdk_accel_get_io_channel(); 1992 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1993 1994 /* Override the submit_tasks function */ 1995 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1996 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 1997 modules[i] = g_modules_opc[i]; 1998 g_modules_opc[i] = g_module; 1999 } 2000 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2001 * decompress operations together in one sequence. 2002 */ 2003 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2004 g_seq_operations[ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2005 g_seq_operations[ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2006 2007 /* Check the simplest case: one operation using accel buffer as destination + copy operation 2008 * specifying the actual destination buffer 2009 */ 2010 memset(srcbuf, 0xa5, 4096); 2011 memset(dstbuf, 0x0, 4096); 2012 memset(expected, 0xa5, 4096); 2013 completed = 0; 2014 seq = NULL; 2015 2016 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2017 CU_ASSERT_EQUAL(rc, 0); 2018 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2019 2020 src_iovs[0].iov_base = srcbuf; 2021 src_iovs[0].iov_len = 4096; 2022 dst_iovs[0].iov_base = buf[0]; 2023 dst_iovs[0].iov_len = 4096; 2024 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2025 &src_iovs[0], 1, NULL, NULL, 0, 2026 ut_sequence_step_cb, &completed); 2027 CU_ASSERT_EQUAL(rc, 0); 2028 2029 src_iovs[1].iov_base = buf[0]; 2030 src_iovs[1].iov_len = 4096; 2031 dst_iovs[1].iov_base = dstbuf; 2032 dst_iovs[1].iov_len = 4096; 2033 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2034 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 2035 ut_sequence_step_cb, &completed); 2036 CU_ASSERT_EQUAL(rc, 0); 2037 2038 ut_seq.complete = false; 2039 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2040 2041 poll_threads(); 2042 2043 CU_ASSERT_EQUAL(completed, 2); 2044 CU_ASSERT(ut_seq.complete); 2045 CU_ASSERT_EQUAL(ut_seq.status, 0); 2046 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2047 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2048 2049 /* Start with a fill operation using accel buffer, followed by a decompress using another 2050 * accel buffer as dst, followed by a copy operation specifying dst buffer of the whole 2051 * sequence 2052 */ 2053 memset(srcbuf, 0x0, 4096); 2054 memset(dstbuf, 0x0, 4096); 2055 memset(expected, 0x5a, 4096); 2056 completed = 0; 2057 seq = NULL; 2058 2059 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2060 CU_ASSERT_EQUAL(rc, 0); 2061 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2062 2063 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2064 ut_sequence_step_cb, &completed); 2065 CU_ASSERT_EQUAL(rc, 0); 2066 2067 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2068 CU_ASSERT_EQUAL(rc, 0); 2069 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2070 2071 src_iovs[0].iov_base = buf[0]; 2072 src_iovs[0].iov_len = 4096; 2073 dst_iovs[0].iov_base = buf[1]; 2074 dst_iovs[0].iov_len = 4096; 2075 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2076 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2077 ut_sequence_step_cb, &completed); 2078 CU_ASSERT_EQUAL(rc, 0); 2079 2080 src_iovs[1].iov_base = buf[1]; 2081 src_iovs[1].iov_len = 4096; 2082 dst_iovs[1].iov_base = dstbuf; 2083 dst_iovs[1].iov_len = 4096; 2084 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2085 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2086 ut_sequence_step_cb, &completed); 2087 2088 ut_seq.complete = false; 2089 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2090 2091 poll_threads(); 2092 2093 CU_ASSERT_EQUAL(completed, 3); 2094 CU_ASSERT(ut_seq.complete); 2095 CU_ASSERT_EQUAL(ut_seq.status, 0); 2096 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2097 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2098 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2099 2100 /* Check the same, but with two decompress operations with the first one being in-place */ 2101 memset(srcbuf, 0x0, 4096); 2102 memset(dstbuf, 0x0, 4096); 2103 memset(expected, 0x5a, 4096); 2104 completed = 0; 2105 seq = NULL; 2106 2107 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2108 CU_ASSERT_EQUAL(rc, 0); 2109 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2110 2111 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2112 ut_sequence_step_cb, &completed); 2113 CU_ASSERT_EQUAL(rc, 0); 2114 2115 src_iovs[0].iov_base = buf[0]; 2116 src_iovs[0].iov_len = 4096; 2117 dst_iovs[0].iov_base = buf[0]; 2118 dst_iovs[0].iov_len = 4096; 2119 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2120 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2121 ut_sequence_step_cb, &completed); 2122 CU_ASSERT_EQUAL(rc, 0); 2123 2124 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2125 CU_ASSERT_EQUAL(rc, 0); 2126 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2127 2128 src_iovs[1].iov_base = buf[0]; 2129 src_iovs[1].iov_len = 4096; 2130 dst_iovs[1].iov_base = buf[1]; 2131 dst_iovs[1].iov_len = 4096; 2132 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, domain[1], domain_ctx[1], 2133 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 2134 ut_sequence_step_cb, &completed); 2135 CU_ASSERT_EQUAL(rc, 0); 2136 2137 src_iovs[2].iov_base = buf[1]; 2138 src_iovs[2].iov_len = 4096; 2139 dst_iovs[2].iov_base = dstbuf; 2140 dst_iovs[2].iov_len = 4096; 2141 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 2142 &src_iovs[2], 1, domain[1], domain_ctx[1], 0, 2143 ut_sequence_step_cb, &completed); 2144 2145 ut_seq.complete = false; 2146 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2147 2148 poll_threads(); 2149 2150 CU_ASSERT_EQUAL(completed, 4); 2151 CU_ASSERT(ut_seq.complete); 2152 CU_ASSERT_EQUAL(ut_seq.status, 0); 2153 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2154 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2155 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2156 2157 /* Check that specifying offsets within accel buffers works correctly */ 2158 memset(srcbuf, 0x0, 4096); 2159 memset(dstbuf, 0x0, 4096); 2160 completed = 0; 2161 seq = NULL; 2162 2163 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2164 CU_ASSERT_EQUAL(rc, 0); 2165 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2166 2167 /* Fill in each 1K of the buffer with different pattern */ 2168 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2169 ut_sequence_step_cb, &completed); 2170 CU_ASSERT_EQUAL(rc, 0); 2171 2172 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2173 0x5a, 0, ut_sequence_step_cb, &completed); 2174 CU_ASSERT_EQUAL(rc, 0); 2175 2176 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2177 0xfe, 0, ut_sequence_step_cb, &completed); 2178 CU_ASSERT_EQUAL(rc, 0); 2179 2180 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2181 0xed, 0, ut_sequence_step_cb, &completed); 2182 CU_ASSERT_EQUAL(rc, 0); 2183 2184 src_iovs[0].iov_base = buf[0]; 2185 src_iovs[0].iov_len = 4096; 2186 dst_iovs[0].iov_base = dstbuf; 2187 dst_iovs[0].iov_len = 4096; 2188 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2189 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2190 ut_sequence_step_cb, &completed); 2191 2192 ut_seq.complete = false; 2193 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2194 2195 poll_threads(); 2196 2197 memset(&expected[0], 0xa5, 1024); 2198 memset(&expected[1024], 0x5a, 1024); 2199 memset(&expected[2048], 0xfe, 1024); 2200 memset(&expected[3072], 0xed, 1024); 2201 CU_ASSERT_EQUAL(completed, 5); 2202 CU_ASSERT(ut_seq.complete); 2203 CU_ASSERT_EQUAL(ut_seq.status, 0); 2204 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2205 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2206 2207 /* Check the same but this time with a decompress operation on part of the buffer (512B 2208 * offset) */ 2209 memset(srcbuf, 0x0, 4096); 2210 memset(dstbuf, 0x0, 4096); 2211 completed = 0; 2212 seq = NULL; 2213 2214 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2215 CU_ASSERT_EQUAL(rc, 0); 2216 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2217 2218 /* Fill in each 1K of the buffer with different pattern */ 2219 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2220 ut_sequence_step_cb, &completed); 2221 CU_ASSERT_EQUAL(rc, 0); 2222 2223 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2224 0x5a, 0, ut_sequence_step_cb, &completed); 2225 CU_ASSERT_EQUAL(rc, 0); 2226 2227 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2228 0xfe, 0, ut_sequence_step_cb, &completed); 2229 CU_ASSERT_EQUAL(rc, 0); 2230 2231 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2232 0xed, 0, ut_sequence_step_cb, &completed); 2233 CU_ASSERT_EQUAL(rc, 0); 2234 2235 rc = spdk_accel_get_buf(ioch, 3072, &buf[1], &domain[1], &domain_ctx[1]); 2236 CU_ASSERT_EQUAL(rc, 0); 2237 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2238 2239 src_iovs[0].iov_base = (char *)buf[0] + 512; 2240 src_iovs[0].iov_len = 3072; 2241 dst_iovs[0].iov_base = (char *)buf[1] + 256; 2242 dst_iovs[0].iov_len = 3072; 2243 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2244 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2245 ut_sequence_step_cb, &completed); 2246 2247 src_iovs[1].iov_base = (char *)buf[1] + 256; 2248 src_iovs[1].iov_len = 3072; 2249 dst_iovs[1].iov_base = dstbuf; 2250 dst_iovs[1].iov_len = 3072; 2251 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2252 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2253 ut_sequence_step_cb, &completed); 2254 2255 ut_seq.complete = false; 2256 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2257 2258 poll_threads(); 2259 2260 memset(&expected[0], 0xa5, 512); 2261 memset(&expected[512], 0x5a, 1024); 2262 memset(&expected[1536], 0xfe, 1024); 2263 memset(&expected[2560], 0xed, 512); 2264 CU_ASSERT_EQUAL(completed, 6); 2265 CU_ASSERT(ut_seq.complete); 2266 CU_ASSERT_EQUAL(ut_seq.status, 0); 2267 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 3072), 0); 2268 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2269 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2270 2271 /* Check that if iobuf pool is empty, the sequence processing will wait until a buffer is 2272 * available 2273 */ 2274 accel_ch = spdk_io_channel_get_ctx(ioch); 2275 small_cache_count = accel_ch->iobuf.small.cache_count; 2276 STAILQ_INIT(&small_cache); 2277 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2278 accel_ch->iobuf.small.cache_count = 0; 2279 accel_ch->iobuf.small.cache_size = 0; 2280 g_iobuf.small_pool_count = 0; 2281 2282 /* First allocate a single buffer used by two operations */ 2283 memset(srcbuf, 0x0, 4096); 2284 memset(dstbuf, 0x0, 4096); 2285 memset(expected, 0xa5, 4096); 2286 completed = 0; 2287 seq = NULL; 2288 2289 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2290 CU_ASSERT_EQUAL(rc, 0); 2291 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2292 2293 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0xa5, 0, 2294 ut_sequence_step_cb, &completed); 2295 CU_ASSERT_EQUAL(rc, 0); 2296 2297 src_iovs[0].iov_base = buf[0]; 2298 src_iovs[0].iov_len = 4096; 2299 dst_iovs[0].iov_base = dstbuf; 2300 dst_iovs[0].iov_len = 4096; 2301 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2302 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2303 ut_sequence_step_cb, &completed); 2304 2305 ut_seq.complete = false; 2306 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2307 2308 poll_threads(); 2309 2310 CU_ASSERT_EQUAL(completed, 0); 2311 CU_ASSERT(!ut_seq.complete); 2312 2313 /* Get a buffer and return it to the pool to trigger the sequence to finish */ 2314 g_iobuf.small_pool_count = 1; 2315 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2316 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2317 CU_ASSERT(g_iobuf.small_pool_count == 0); 2318 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2319 2320 poll_threads(); 2321 2322 CU_ASSERT_EQUAL(completed, 2); 2323 CU_ASSERT(ut_seq.complete); 2324 CU_ASSERT_EQUAL(ut_seq.status, 0); 2325 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2326 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2327 2328 /* Return the buffers back to the cache */ 2329 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2330 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2331 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2332 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2333 small_cache_count++; 2334 } 2335 accel_ch->iobuf.small.cache_count = 0; 2336 g_iobuf.small_pool_count = 0; 2337 2338 /* Check a bit more complex scenario, with two buffers in the sequence */ 2339 memset(srcbuf, 0x0, 4096); 2340 memset(dstbuf, 0x0, 4096); 2341 memset(expected, 0x5a, 4096); 2342 completed = 0; 2343 seq = NULL; 2344 2345 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2346 CU_ASSERT_EQUAL(rc, 0); 2347 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2348 2349 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2350 ut_sequence_step_cb, &completed); 2351 CU_ASSERT_EQUAL(rc, 0); 2352 2353 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2354 CU_ASSERT_EQUAL(rc, 0); 2355 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2356 2357 src_iovs[0].iov_base = buf[0]; 2358 src_iovs[0].iov_len = 4096; 2359 dst_iovs[0].iov_base = buf[1]; 2360 dst_iovs[0].iov_len = 4096; 2361 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2362 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2363 ut_sequence_step_cb, &completed); 2364 CU_ASSERT_EQUAL(rc, 0); 2365 2366 src_iovs[1].iov_base = buf[1]; 2367 src_iovs[1].iov_len = 4096; 2368 dst_iovs[1].iov_base = dstbuf; 2369 dst_iovs[1].iov_len = 4096; 2370 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2371 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2372 ut_sequence_step_cb, &completed); 2373 CU_ASSERT_EQUAL(rc, 0); 2374 2375 ut_seq.complete = false; 2376 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2377 2378 poll_threads(); 2379 2380 CU_ASSERT_EQUAL(completed, 0); 2381 CU_ASSERT(!ut_seq.complete); 2382 2383 g_iobuf.small_pool_count = 1; 2384 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2385 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2386 g_iobuf.small_pool_count = 0; 2387 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2388 2389 /* One buffer is not enough to finish the whole sequence */ 2390 poll_threads(); 2391 CU_ASSERT(!ut_seq.complete); 2392 2393 g_iobuf.small_pool_count = 1; 2394 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2395 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2396 g_iobuf.small_pool_count = 0; 2397 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2398 2399 poll_threads(); 2400 2401 CU_ASSERT_EQUAL(completed, 3); 2402 CU_ASSERT(ut_seq.complete); 2403 CU_ASSERT_EQUAL(ut_seq.status, 0); 2404 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2405 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2406 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2407 2408 /* Return the buffers back to the cache */ 2409 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2410 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2411 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2412 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2413 small_cache_count++; 2414 } 2415 accel_ch->iobuf.small.cache_count = 0; 2416 2417 g_iobuf.small_pool_count = 32; 2418 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2419 accel_ch->iobuf.small.cache_count = small_cache_count; 2420 2421 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2422 g_modules_opc[i] = modules[i]; 2423 } 2424 2425 ut_clear_operations(); 2426 spdk_put_io_channel(ioch); 2427 poll_threads(); 2428 } 2429 2430 static void 2431 ut_domain_ctx_init(struct ut_domain_ctx *ctx, void *base, size_t len, struct iovec *expected) 2432 { 2433 ctx->iov.iov_base = base; 2434 ctx->iov.iov_len = len; 2435 ctx->expected = *expected; 2436 ctx->pull_submit_status = 0; 2437 ctx->push_submit_status = 0; 2438 ctx->pull_complete_status = 0; 2439 ctx->push_complete_status = 0; 2440 } 2441 2442 static void 2443 test_sequence_memory_domain(void) 2444 { 2445 struct spdk_accel_sequence *seq = NULL; 2446 struct spdk_io_channel *ioch; 2447 struct accel_io_channel *accel_ch; 2448 struct ut_sequence ut_seq; 2449 struct ut_domain_ctx domctx[4]; 2450 struct spdk_iobuf_buffer *cache_entry; 2451 struct accel_module modules[ACCEL_OPC_LAST]; 2452 struct spdk_memory_domain *accel_domain; 2453 spdk_iobuf_buffer_stailq_t small_cache; 2454 char srcbuf[4096], dstbuf[4096], expected[4096], tmp[4096]; 2455 struct iovec src_iovs[2], dst_iovs[2]; 2456 uint32_t small_cache_count; 2457 void *accel_buf, *accel_domain_ctx, *iobuf_buf; 2458 int i, rc, completed; 2459 2460 ioch = spdk_accel_get_io_channel(); 2461 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2462 2463 /* Override the submit_tasks function */ 2464 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2465 g_module.supports_memory_domains = false; 2466 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2467 modules[i] = g_modules_opc[i]; 2468 g_modules_opc[i] = g_module; 2469 } 2470 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2471 * decompress operations together in one sequence. 2472 */ 2473 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2474 g_seq_operations[ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2475 g_seq_operations[ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2476 2477 /* First check the simplest case - single fill operation with dstbuf in domain */ 2478 memset(expected, 0xa5, sizeof(expected)); 2479 memset(dstbuf, 0x0, sizeof(dstbuf)); 2480 completed = 0; 2481 seq = NULL; 2482 2483 /* Use some garbage pointer as dst and use domain ctx to get the actual dstbuf */ 2484 dst_iovs[0].iov_base = (void *)0xcafebabe; 2485 dst_iovs[0].iov_len = sizeof(dstbuf); 2486 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2487 2488 rc = spdk_accel_append_fill(&seq, ioch, dst_iovs[0].iov_base, dst_iovs[0].iov_len, 2489 g_ut_domain, &domctx[0], 0xa5, 0, 2490 ut_sequence_step_cb, &completed); 2491 CU_ASSERT_EQUAL(rc, 0); 2492 2493 ut_seq.complete = false; 2494 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2495 2496 poll_threads(); 2497 CU_ASSERT_EQUAL(completed, 1); 2498 CU_ASSERT(ut_seq.complete); 2499 CU_ASSERT_EQUAL(ut_seq.status, 0); 2500 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2501 2502 /* Check operation with both srcbuf and dstbuf in remote memory domain */ 2503 memset(expected, 0x5a, sizeof(expected)); 2504 memset(dstbuf, 0x0, sizeof(dstbuf)); 2505 memset(srcbuf, 0x5a, sizeof(dstbuf)); 2506 completed = 0; 2507 seq = NULL; 2508 2509 src_iovs[0].iov_base = (void *)0xcafebabe; 2510 src_iovs[0].iov_len = sizeof(srcbuf); 2511 dst_iovs[0].iov_base = (void *)0xbeefdead; 2512 dst_iovs[0].iov_len = sizeof(dstbuf); 2513 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2514 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2515 2516 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2517 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2518 ut_sequence_step_cb, &completed); 2519 CU_ASSERT_EQUAL(rc, 0); 2520 2521 ut_seq.complete = false; 2522 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2523 2524 poll_threads(); 2525 CU_ASSERT_EQUAL(completed, 1); 2526 CU_ASSERT(ut_seq.complete); 2527 CU_ASSERT_EQUAL(ut_seq.status, 0); 2528 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2529 2530 /* Two operations using a buffer in remote domain */ 2531 memset(expected, 0xa5, sizeof(expected)); 2532 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2533 memset(tmp, 0x0, sizeof(tmp)); 2534 memset(dstbuf, 0x0, sizeof(dstbuf)); 2535 completed = 0; 2536 seq = NULL; 2537 2538 src_iovs[0].iov_base = (void *)0xcafebabe; 2539 src_iovs[0].iov_len = sizeof(srcbuf); 2540 dst_iovs[0].iov_base = (void *)0xbeefdead; 2541 dst_iovs[0].iov_len = sizeof(tmp); 2542 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2543 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2544 2545 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2546 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2547 ut_sequence_step_cb, &completed); 2548 CU_ASSERT_EQUAL(rc, 0); 2549 2550 src_iovs[1].iov_base = (void *)0xbeefdead; 2551 src_iovs[1].iov_len = sizeof(tmp); 2552 dst_iovs[1].iov_base = (void *)0xa5a5a5a5; 2553 dst_iovs[1].iov_len = sizeof(dstbuf); 2554 ut_domain_ctx_init(&domctx[2], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2555 ut_domain_ctx_init(&domctx[3], tmp, sizeof(tmp), &src_iovs[1]); 2556 2557 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[2], 2558 &src_iovs[1], 1, g_ut_domain, &domctx[3], 0, 2559 ut_sequence_step_cb, &completed); 2560 CU_ASSERT_EQUAL(rc, 0); 2561 2562 ut_seq.complete = false; 2563 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2564 2565 poll_threads(); 2566 CU_ASSERT_EQUAL(completed, 2); 2567 CU_ASSERT(ut_seq.complete); 2568 CU_ASSERT_EQUAL(ut_seq.status, 0); 2569 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2570 2571 /* Use buffer in remote memory domain and accel buffer at the same time */ 2572 memset(expected, 0x5a, sizeof(expected)); 2573 memset(srcbuf, 0x5a, sizeof(expected)); 2574 memset(dstbuf, 0x0, sizeof(dstbuf)); 2575 completed = 0; 2576 seq = NULL; 2577 2578 rc = spdk_accel_get_buf(ioch, sizeof(dstbuf), &accel_buf, &accel_domain, &accel_domain_ctx); 2579 CU_ASSERT_EQUAL(rc, 0); 2580 2581 src_iovs[0].iov_base = (void *)0xfeedbeef; 2582 src_iovs[0].iov_len = sizeof(srcbuf); 2583 dst_iovs[0].iov_base = accel_buf; 2584 dst_iovs[0].iov_len = sizeof(dstbuf); 2585 ut_domain_ctx_init(&domctx[0], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2586 2587 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, accel_domain, accel_domain_ctx, 2588 &src_iovs[0], 1, g_ut_domain, &domctx[0], 0, 2589 ut_sequence_step_cb, &completed); 2590 CU_ASSERT_EQUAL(rc, 0); 2591 2592 src_iovs[1].iov_base = accel_buf; 2593 src_iovs[1].iov_len = sizeof(dstbuf); 2594 dst_iovs[1].iov_base = (void *)0xbeeffeed; 2595 dst_iovs[1].iov_len = sizeof(dstbuf); 2596 ut_domain_ctx_init(&domctx[1], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2597 2598 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[1], 2599 &src_iovs[1], 1, accel_domain, accel_domain_ctx, 0, 2600 ut_sequence_step_cb, &completed); 2601 CU_ASSERT_EQUAL(rc, 0); 2602 2603 ut_seq.complete = false; 2604 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2605 2606 poll_threads(); 2607 CU_ASSERT_EQUAL(completed, 2); 2608 CU_ASSERT(ut_seq.complete); 2609 CU_ASSERT_EQUAL(ut_seq.status, 0); 2610 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2611 spdk_accel_put_buf(ioch, accel_buf, accel_domain, accel_domain_ctx); 2612 2613 /* Check that a sequence with memory domains is correctly executed if buffers are not 2614 * immediately available */ 2615 memset(expected, 0xa5, sizeof(expected)); 2616 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2617 memset(dstbuf, 0x0, sizeof(dstbuf)); 2618 completed = 0; 2619 seq = NULL; 2620 /* Make sure the buffer pool is empty */ 2621 accel_ch = spdk_io_channel_get_ctx(ioch); 2622 small_cache_count = accel_ch->iobuf.small.cache_count; 2623 STAILQ_INIT(&small_cache); 2624 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2625 accel_ch->iobuf.small.cache_count = 0; 2626 g_iobuf.small_pool_count = 0; 2627 2628 src_iovs[0].iov_base = (void *)0xdeadbeef; 2629 src_iovs[0].iov_len = sizeof(srcbuf); 2630 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2631 dst_iovs[0].iov_len = sizeof(dstbuf); 2632 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2633 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2634 2635 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2636 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2637 ut_sequence_step_cb, &completed); 2638 CU_ASSERT_EQUAL(rc, 0); 2639 2640 ut_seq.complete = false; 2641 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2642 2643 poll_threads(); 2644 CU_ASSERT_EQUAL(completed, 0); 2645 CU_ASSERT(!ut_seq.complete); 2646 2647 /* Get a buffer and return it to the pool to trigger the sequence to resume. It shouldn't 2648 * be able to complete, as it needs two buffers */ 2649 g_iobuf.small_pool_count = 1; 2650 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2651 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2652 g_iobuf.small_pool_count = 0; 2653 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2654 2655 CU_ASSERT_EQUAL(completed, 0); 2656 CU_ASSERT(!ut_seq.complete); 2657 2658 /* Return another buffer, this time the sequence should finish */ 2659 g_iobuf.small_pool_count = 1; 2660 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2661 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2662 g_iobuf.small_pool_count = 0; 2663 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2664 2665 poll_threads(); 2666 CU_ASSERT_EQUAL(completed, 1); 2667 CU_ASSERT(ut_seq.complete); 2668 CU_ASSERT_EQUAL(ut_seq.status, 0); 2669 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2670 2671 /* Return the buffers back to the cache */ 2672 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2673 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2674 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2675 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2676 small_cache_count++; 2677 } 2678 accel_ch->iobuf.small.cache_count = 0; 2679 2680 g_iobuf.small_pool_count = 32; 2681 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2682 accel_ch->iobuf.small.cache_count = small_cache_count; 2683 2684 /* Check error cases, starting with an error from spdk_memory_domain_pull_data() */ 2685 completed = 0; 2686 seq = NULL; 2687 2688 src_iovs[0].iov_base = (void *)0xdeadbeef; 2689 src_iovs[0].iov_len = sizeof(srcbuf); 2690 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2691 dst_iovs[0].iov_len = sizeof(dstbuf); 2692 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2693 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2694 domctx[1].pull_submit_status = -E2BIG; 2695 2696 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2697 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2698 ut_sequence_step_cb, &completed); 2699 CU_ASSERT_EQUAL(rc, 0); 2700 2701 ut_seq.complete = false; 2702 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2703 2704 CU_ASSERT_EQUAL(completed, 1); 2705 CU_ASSERT(ut_seq.complete); 2706 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 2707 2708 /* Check completion error from spdk_memory_domain_pull_data() */ 2709 completed = 0; 2710 seq = NULL; 2711 2712 src_iovs[0].iov_base = (void *)0xdeadbeef; 2713 src_iovs[0].iov_len = sizeof(srcbuf); 2714 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2715 dst_iovs[0].iov_len = sizeof(dstbuf); 2716 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2717 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2718 domctx[1].pull_complete_status = -EACCES; 2719 2720 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2721 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2722 ut_sequence_step_cb, &completed); 2723 CU_ASSERT_EQUAL(rc, 0); 2724 2725 ut_seq.complete = false; 2726 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2727 2728 CU_ASSERT_EQUAL(completed, 1); 2729 CU_ASSERT(ut_seq.complete); 2730 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 2731 2732 /* Check submission error from spdk_memory_domain_push_data() */ 2733 completed = 0; 2734 seq = NULL; 2735 2736 src_iovs[0].iov_base = (void *)0xdeadbeef; 2737 src_iovs[0].iov_len = sizeof(srcbuf); 2738 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2739 dst_iovs[0].iov_len = sizeof(dstbuf); 2740 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2741 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2742 domctx[0].push_submit_status = -EADDRINUSE; 2743 2744 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2745 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2746 ut_sequence_step_cb, &completed); 2747 CU_ASSERT_EQUAL(rc, 0); 2748 2749 ut_seq.complete = false; 2750 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2751 2752 CU_ASSERT_EQUAL(completed, 1); 2753 CU_ASSERT(ut_seq.complete); 2754 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 2755 2756 /* Check completion error from spdk_memory_domain_push_data() */ 2757 completed = 0; 2758 seq = NULL; 2759 2760 src_iovs[0].iov_base = (void *)0xdeadbeef; 2761 src_iovs[0].iov_len = sizeof(srcbuf); 2762 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2763 dst_iovs[0].iov_len = sizeof(dstbuf); 2764 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2765 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2766 domctx[0].push_complete_status = -EADDRNOTAVAIL; 2767 2768 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2769 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2770 ut_sequence_step_cb, &completed); 2771 CU_ASSERT_EQUAL(rc, 0); 2772 2773 ut_seq.complete = false; 2774 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2775 2776 CU_ASSERT_EQUAL(completed, 1); 2777 CU_ASSERT(ut_seq.complete); 2778 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 2779 2780 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2781 g_modules_opc[i] = modules[i]; 2782 } 2783 2784 ut_clear_operations(); 2785 spdk_put_io_channel(ioch); 2786 poll_threads(); 2787 } 2788 2789 static int 2790 ut_submit_decompress_memory_domain(struct spdk_io_channel *ch, struct spdk_accel_task *task) 2791 { 2792 struct ut_domain_ctx *ctx; 2793 struct iovec *src_iovs, *dst_iovs; 2794 uint32_t src_iovcnt, dst_iovcnt; 2795 2796 src_iovs = task->s.iovs; 2797 dst_iovs = task->d.iovs; 2798 src_iovcnt = task->s.iovcnt; 2799 dst_iovcnt = task->d.iovcnt; 2800 2801 if (task->src_domain != NULL) { 2802 ctx = task->src_domain_ctx; 2803 CU_ASSERT_EQUAL(memcmp(task->s.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2804 src_iovs = &ctx->iov; 2805 src_iovcnt = 1; 2806 } 2807 2808 if (task->dst_domain != NULL) { 2809 ctx = task->dst_domain_ctx; 2810 CU_ASSERT_EQUAL(memcmp(task->d.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2811 dst_iovs = &ctx->iov; 2812 dst_iovcnt = 1; 2813 } 2814 2815 spdk_iovcpy(src_iovs, src_iovcnt, dst_iovs, dst_iovcnt); 2816 spdk_accel_task_complete(task, 0); 2817 2818 return 0; 2819 } 2820 2821 static void 2822 test_sequence_module_memory_domain(void) 2823 { 2824 struct spdk_accel_sequence *seq = NULL; 2825 struct spdk_io_channel *ioch; 2826 struct accel_module modules[ACCEL_OPC_LAST]; 2827 struct spdk_memory_domain *accel_domain; 2828 struct ut_sequence ut_seq; 2829 struct ut_domain_ctx domctx[2]; 2830 struct iovec src_iovs[2], dst_iovs[2]; 2831 void *buf, *accel_domain_ctx; 2832 char srcbuf[4096], dstbuf[4096], tmp[4096], expected[4096]; 2833 int i, rc, completed; 2834 2835 ioch = spdk_accel_get_io_channel(); 2836 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2837 2838 /* Override the submit_tasks function */ 2839 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2840 g_module.supports_memory_domains = true; 2841 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2842 modules[i] = g_modules_opc[i]; 2843 g_modules_opc[i] = g_module; 2844 } 2845 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress_memory_domain; 2846 g_seq_operations[ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2847 2848 /* Check a sequence with both buffers in memory domains */ 2849 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2850 memset(expected, 0xa5, sizeof(expected)); 2851 memset(dstbuf, 0, sizeof(dstbuf)); 2852 seq = NULL; 2853 completed = 0; 2854 2855 src_iovs[0].iov_base = (void *)0xcafebabe; 2856 src_iovs[0].iov_len = sizeof(srcbuf); 2857 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2858 dst_iovs[0].iov_len = sizeof(dstbuf); 2859 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2860 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2861 2862 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2863 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2864 ut_sequence_step_cb, &completed); 2865 CU_ASSERT_EQUAL(rc, 0); 2866 2867 ut_seq.complete = false; 2868 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2869 2870 poll_threads(); 2871 2872 CU_ASSERT_EQUAL(completed, 1); 2873 CU_ASSERT(ut_seq.complete); 2874 CU_ASSERT_EQUAL(ut_seq.status, 0); 2875 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2876 2877 /* Check two operations each with a single buffer in memory domain */ 2878 memset(srcbuf, 0x5a, sizeof(srcbuf)); 2879 memset(expected, 0x5a, sizeof(expected)); 2880 memset(dstbuf, 0, sizeof(dstbuf)); 2881 memset(tmp, 0, sizeof(tmp)); 2882 seq = NULL; 2883 completed = 0; 2884 2885 src_iovs[0].iov_base = srcbuf; 2886 src_iovs[0].iov_len = sizeof(srcbuf); 2887 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2888 dst_iovs[0].iov_len = sizeof(tmp); 2889 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2890 2891 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2892 &src_iovs[0], 1, NULL, NULL, 0, 2893 ut_sequence_step_cb, &completed); 2894 CU_ASSERT_EQUAL(rc, 0); 2895 2896 src_iovs[1].iov_base = (void *)0xfeedbeef; 2897 src_iovs[1].iov_len = sizeof(tmp); 2898 dst_iovs[1].iov_base = dstbuf; 2899 dst_iovs[1].iov_len = sizeof(dstbuf); 2900 ut_domain_ctx_init(&domctx[1], tmp, sizeof(tmp), &src_iovs[1]); 2901 2902 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2903 &src_iovs[1], 1, g_ut_domain, &domctx[1], 0, 2904 ut_sequence_step_cb, &completed); 2905 CU_ASSERT_EQUAL(rc, 0); 2906 2907 ut_seq.complete = false; 2908 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2909 2910 poll_threads(); 2911 2912 CU_ASSERT_EQUAL(completed, 2); 2913 CU_ASSERT(ut_seq.complete); 2914 CU_ASSERT_EQUAL(ut_seq.status, 0); 2915 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2916 2917 /* Check a sequence with an accel buffer and a buffer in a regular memory domain */ 2918 memset(expected, 0xa5, sizeof(expected)); 2919 memset(dstbuf, 0, sizeof(dstbuf)); 2920 memset(tmp, 0, sizeof(tmp)); 2921 seq = NULL; 2922 completed = 0; 2923 2924 rc = spdk_accel_get_buf(ioch, 4096, &buf, &accel_domain, &accel_domain_ctx); 2925 CU_ASSERT_EQUAL(rc, 0); 2926 2927 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, accel_domain, accel_domain_ctx, 2928 0xa5, 0, ut_sequence_step_cb, &completed); 2929 CU_ASSERT_EQUAL(rc, 0); 2930 2931 src_iovs[0].iov_base = buf; 2932 src_iovs[0].iov_len = 4096; 2933 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2934 dst_iovs[0].iov_len = sizeof(dstbuf); 2935 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2936 2937 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2938 &src_iovs[0], 1, accel_domain, accel_domain_ctx, 0, 2939 ut_sequence_step_cb, &completed); 2940 CU_ASSERT_EQUAL(rc, 0); 2941 2942 ut_seq.complete = false; 2943 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2944 2945 poll_threads(); 2946 2947 CU_ASSERT_EQUAL(completed, 2); 2948 CU_ASSERT(ut_seq.complete); 2949 CU_ASSERT_EQUAL(ut_seq.status, 0); 2950 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2951 2952 spdk_accel_put_buf(ioch, buf, accel_domain, accel_domain_ctx); 2953 2954 g_module.supports_memory_domains = false; 2955 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2956 g_modules_opc[i] = modules[i]; 2957 } 2958 2959 ut_clear_operations(); 2960 spdk_put_io_channel(ioch); 2961 poll_threads(); 2962 } 2963 2964 #ifdef SPDK_CONFIG_ISAL_CRYPTO 2965 static void 2966 ut_encrypt_cb(void *cb_arg, int status) 2967 { 2968 int *completed = cb_arg; 2969 2970 CU_ASSERT_EQUAL(status, 0); 2971 2972 *completed = 1; 2973 } 2974 2975 static void 2976 test_sequence_crypto(void) 2977 { 2978 struct spdk_accel_sequence *seq = NULL; 2979 struct spdk_io_channel *ioch; 2980 struct spdk_accel_crypto_key *key; 2981 struct spdk_accel_crypto_key_create_param key_params = { 2982 .cipher = "AES_XTS", 2983 .hex_key = "00112233445566778899aabbccddeeff", 2984 .hex_key2 = "ffeeddccbbaa99887766554433221100", 2985 .key_name = "ut_key", 2986 }; 2987 struct ut_sequence ut_seq; 2988 unsigned char buf[4096], encrypted[4096] = {}, data[4096], tmp[3][4096]; 2989 struct iovec src_iovs[4], dst_iovs[4]; 2990 int rc, completed = 0; 2991 size_t i; 2992 2993 ioch = spdk_accel_get_io_channel(); 2994 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2995 2996 rc = spdk_accel_crypto_key_create(&key_params); 2997 CU_ASSERT_EQUAL(rc, 0); 2998 key = spdk_accel_crypto_key_get(key_params.key_name); 2999 SPDK_CU_ASSERT_FATAL(key != NULL); 3000 3001 for (i = 0; i < sizeof(data); ++i) { 3002 data[i] = (uint8_t)i & 0xff; 3003 } 3004 3005 dst_iovs[0].iov_base = encrypted; 3006 dst_iovs[0].iov_len = sizeof(encrypted); 3007 src_iovs[0].iov_base = data; 3008 src_iovs[0].iov_len = sizeof(data); 3009 rc = spdk_accel_submit_encrypt(ioch, key, &dst_iovs[0], 1, &src_iovs[0], 1, 0, 4096, 0, 3010 ut_encrypt_cb, &completed); 3011 CU_ASSERT_EQUAL(rc, 0); 3012 3013 while (!completed) { 3014 poll_threads(); 3015 } 3016 3017 /* Verify that encryption operation in a sequence produces the same result */ 3018 seq = NULL; 3019 completed = 0; 3020 3021 dst_iovs[0].iov_base = tmp[0]; 3022 dst_iovs[0].iov_len = sizeof(tmp[0]); 3023 src_iovs[0].iov_base = data; 3024 src_iovs[0].iov_len = sizeof(data); 3025 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3026 &src_iovs[0], 1, NULL, NULL, 0, 3027 ut_sequence_step_cb, &completed); 3028 CU_ASSERT_EQUAL(rc, 0); 3029 3030 dst_iovs[1].iov_base = tmp[1]; 3031 dst_iovs[1].iov_len = sizeof(tmp[1]); 3032 src_iovs[1].iov_base = tmp[0]; 3033 src_iovs[1].iov_len = sizeof(tmp[0]); 3034 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3035 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3036 ut_sequence_step_cb, &completed); 3037 CU_ASSERT_EQUAL(rc, 0); 3038 3039 dst_iovs[2].iov_base = buf; 3040 dst_iovs[2].iov_len = sizeof(buf); 3041 src_iovs[2].iov_base = tmp[1]; 3042 src_iovs[2].iov_len = sizeof(tmp[1]); 3043 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3044 &src_iovs[2], 1, NULL, NULL, 0, 3045 ut_sequence_step_cb, &completed); 3046 CU_ASSERT_EQUAL(rc, 0); 3047 3048 ut_seq.complete = false; 3049 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3050 3051 poll_threads(); 3052 3053 CU_ASSERT_EQUAL(completed, 3); 3054 CU_ASSERT(ut_seq.complete); 3055 CU_ASSERT_EQUAL(ut_seq.status, 0); 3056 CU_ASSERT_EQUAL(memcmp(buf, encrypted, sizeof(buf)), 0); 3057 3058 /* Check that decryption produces the original buffer */ 3059 seq = NULL; 3060 completed = 0; 3061 memset(buf, 0, sizeof(buf)); 3062 3063 dst_iovs[0].iov_base = tmp[0]; 3064 dst_iovs[0].iov_len = sizeof(tmp[0]); 3065 src_iovs[0].iov_base = encrypted; 3066 src_iovs[0].iov_len = sizeof(encrypted); 3067 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3068 &src_iovs[0], 1, NULL, NULL, 0, 3069 ut_sequence_step_cb, &completed); 3070 CU_ASSERT_EQUAL(rc, 0); 3071 3072 dst_iovs[1].iov_base = tmp[1]; 3073 dst_iovs[1].iov_len = sizeof(tmp[1]); 3074 src_iovs[1].iov_base = tmp[0]; 3075 src_iovs[1].iov_len = sizeof(tmp[0]); 3076 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3077 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3078 ut_sequence_step_cb, &completed); 3079 CU_ASSERT_EQUAL(rc, 0); 3080 3081 dst_iovs[2].iov_base = buf; 3082 dst_iovs[2].iov_len = sizeof(buf); 3083 src_iovs[2].iov_base = tmp[1]; 3084 src_iovs[2].iov_len = sizeof(tmp[1]); 3085 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3086 &src_iovs[2], 1, NULL, NULL, 0, 3087 ut_sequence_step_cb, &completed); 3088 CU_ASSERT_EQUAL(rc, 0); 3089 3090 ut_seq.complete = false; 3091 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3092 3093 poll_threads(); 3094 3095 CU_ASSERT_EQUAL(completed, 3); 3096 CU_ASSERT(ut_seq.complete); 3097 CU_ASSERT_EQUAL(ut_seq.status, 0); 3098 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3099 3100 /* Check encrypt + decrypt in a single sequence */ 3101 seq = NULL; 3102 completed = 0; 3103 memset(buf, 0, sizeof(buf)); 3104 3105 dst_iovs[0].iov_base = tmp[0]; 3106 dst_iovs[0].iov_len = sizeof(tmp[0]); 3107 src_iovs[0].iov_base = data; 3108 src_iovs[0].iov_len = sizeof(data); 3109 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3110 &src_iovs[0], 1, NULL, NULL, 0, 3111 ut_sequence_step_cb, &completed); 3112 CU_ASSERT_EQUAL(rc, 0); 3113 3114 dst_iovs[1].iov_base = tmp[1]; 3115 dst_iovs[1].iov_len = sizeof(tmp[1]); 3116 src_iovs[1].iov_base = tmp[0]; 3117 src_iovs[1].iov_len = sizeof(tmp[0]); 3118 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3119 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3120 ut_sequence_step_cb, &completed); 3121 CU_ASSERT_EQUAL(rc, 0); 3122 3123 3124 dst_iovs[2].iov_base = tmp[2]; 3125 dst_iovs[2].iov_len = sizeof(tmp[2]); 3126 src_iovs[2].iov_base = tmp[1]; 3127 src_iovs[2].iov_len = sizeof(tmp[1]); 3128 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[2], 1, NULL, NULL, 3129 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3130 ut_sequence_step_cb, &completed); 3131 CU_ASSERT_EQUAL(rc, 0); 3132 3133 dst_iovs[3].iov_base = buf; 3134 dst_iovs[3].iov_len = sizeof(buf); 3135 src_iovs[3].iov_base = tmp[2]; 3136 src_iovs[3].iov_len = sizeof(tmp[2]); 3137 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 3138 &src_iovs[3], 1, NULL, NULL, 0, 3139 ut_sequence_step_cb, &completed); 3140 CU_ASSERT_EQUAL(rc, 0); 3141 3142 ut_seq.complete = false; 3143 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3144 3145 poll_threads(); 3146 3147 CU_ASSERT_EQUAL(completed, 4); 3148 CU_ASSERT(ut_seq.complete); 3149 CU_ASSERT_EQUAL(ut_seq.status, 0); 3150 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3151 3152 rc = spdk_accel_crypto_key_destroy(key); 3153 CU_ASSERT_EQUAL(rc, 0); 3154 spdk_put_io_channel(ioch); 3155 poll_threads(); 3156 } 3157 #endif /* SPDK_CONFIG_ISAL_CRYPTO */ 3158 3159 static int 3160 ut_submit_crypto(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3161 { 3162 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3163 3164 spdk_accel_task_complete(task, 0); 3165 3166 return 0; 3167 } 3168 3169 struct ut_driver_operation { 3170 int complete_status; 3171 int submit_status; 3172 int count; 3173 bool supported; 3174 }; 3175 3176 static struct ut_driver_operation g_drv_operations[ACCEL_OPC_LAST]; 3177 3178 static int 3179 ut_driver_execute_sequence(struct spdk_accel_sequence *seq) 3180 { 3181 struct spdk_accel_task *task; 3182 struct ut_driver_operation *drv_ops; 3183 3184 while ((task = spdk_accel_sequence_first_task(seq)) != NULL) { 3185 drv_ops = &g_drv_operations[task->op_code]; 3186 if (!drv_ops->supported) { 3187 break; 3188 } 3189 3190 drv_ops->count++; 3191 if (drv_ops->submit_status != 0) { 3192 return drv_ops->submit_status; 3193 } 3194 3195 if (drv_ops->complete_status != 0) { 3196 spdk_accel_task_complete(task, drv_ops->complete_status); 3197 break; 3198 } 3199 3200 switch (task->op_code) { 3201 case ACCEL_OPC_DECOMPRESS: 3202 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3203 break; 3204 case ACCEL_OPC_FILL: 3205 spdk_iov_memset(task->d.iovs, task->d.iovcnt, 3206 (int)(task->fill_pattern & 0xff)); 3207 break; 3208 default: 3209 CU_ASSERT(0 && "unexpected opcode"); 3210 break; 3211 } 3212 3213 spdk_accel_task_complete(task, 0); 3214 } 3215 3216 spdk_accel_sequence_continue(seq); 3217 3218 return 0; 3219 } 3220 3221 static struct spdk_accel_driver g_ut_driver = { 3222 .name = "ut", 3223 .execute_sequence = ut_driver_execute_sequence, 3224 }; 3225 3226 SPDK_ACCEL_DRIVER_REGISTER(ut, &g_ut_driver); 3227 3228 static void 3229 test_sequence_driver(void) 3230 { 3231 struct spdk_accel_sequence *seq = NULL; 3232 struct spdk_io_channel *ioch; 3233 struct spdk_accel_crypto_key key = {}; 3234 struct ut_sequence ut_seq; 3235 struct accel_module modules[ACCEL_OPC_LAST]; 3236 char buf[4096], tmp[3][4096], expected[4096]; 3237 struct iovec src_iovs[3], dst_iovs[3]; 3238 int i, rc, completed = 0; 3239 3240 ioch = spdk_accel_get_io_channel(); 3241 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3242 rc = spdk_accel_set_driver("ut"); 3243 SPDK_CU_ASSERT_FATAL(rc == 0); 3244 3245 /* Override the submit_tasks function */ 3246 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3247 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3248 modules[i] = g_modules_opc[i]; 3249 g_modules_opc[i] = g_module; 3250 } 3251 /* Intercept crypto operations, as they should be executed by an accel module */ 3252 g_seq_operations[ACCEL_OPC_ENCRYPT].submit = ut_submit_crypto; 3253 g_seq_operations[ACCEL_OPC_DECRYPT].submit = ut_submit_crypto; 3254 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3255 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3256 g_seq_operations[ACCEL_OPC_FILL].count = 0; 3257 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3258 3259 g_drv_operations[ACCEL_OPC_FILL].supported = true; 3260 g_drv_operations[ACCEL_OPC_DECOMPRESS].supported = true; 3261 3262 /* First check a sequence that is fully executed using a driver, with the copy at the end 3263 * being removed */ 3264 seq = NULL; 3265 completed = 0; 3266 memset(buf, 0, sizeof(buf)); 3267 memset(tmp[0], 0, sizeof(tmp[0])); 3268 memset(tmp[1], 0, sizeof(tmp[1])); 3269 memset(&expected[0], 0xa5, 2048); 3270 memset(&expected[2048], 0xbe, 2048); 3271 3272 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], 2048, NULL, NULL, 0xa5, 0, 3273 ut_sequence_step_cb, &completed); 3274 CU_ASSERT_EQUAL(rc, 0); 3275 rc = spdk_accel_append_fill(&seq, ioch, &tmp[0][2048], 2048, NULL, NULL, 0xbe, 0, 3276 ut_sequence_step_cb, &completed); 3277 CU_ASSERT_EQUAL(rc, 0); 3278 3279 dst_iovs[0].iov_base = tmp[1]; 3280 dst_iovs[0].iov_len = sizeof(tmp[1]); 3281 src_iovs[0].iov_base = tmp[0]; 3282 src_iovs[0].iov_len = sizeof(tmp[0]); 3283 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3284 &src_iovs[0], 1, NULL, NULL, 0, 3285 ut_sequence_step_cb, &completed); 3286 CU_ASSERT_EQUAL(rc, 0); 3287 3288 dst_iovs[1].iov_base = buf; 3289 dst_iovs[1].iov_len = sizeof(buf); 3290 src_iovs[1].iov_base = tmp[1]; 3291 src_iovs[1].iov_len = sizeof(tmp[1]); 3292 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3293 &src_iovs[1], 1, NULL, NULL, 0, 3294 ut_sequence_step_cb, &completed); 3295 CU_ASSERT_EQUAL(rc, 0); 3296 3297 ut_seq.complete = false; 3298 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3299 3300 poll_threads(); 3301 3302 CU_ASSERT_EQUAL(completed, 4); 3303 CU_ASSERT(ut_seq.complete); 3304 CU_ASSERT_EQUAL(ut_seq.status, 0); 3305 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3306 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3307 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 3308 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 2); 3309 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3310 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3311 3312 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3313 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3314 3315 /* Check a sequence when the first two operations are executed by a driver, while the rest 3316 * is executed via modules */ 3317 seq = NULL; 3318 completed = 0; 3319 memset(buf, 0, sizeof(buf)); 3320 memset(tmp[0], 0, sizeof(tmp[0])); 3321 memset(tmp[1], 0, sizeof(tmp[1])); 3322 memset(tmp[2], 0, sizeof(tmp[2])); 3323 memset(&expected[0], 0xfe, 4096); 3324 3325 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 0, 3326 ut_sequence_step_cb, &completed); 3327 CU_ASSERT_EQUAL(rc, 0); 3328 3329 dst_iovs[0].iov_base = tmp[1]; 3330 dst_iovs[0].iov_len = sizeof(tmp[1]); 3331 src_iovs[0].iov_base = tmp[0]; 3332 src_iovs[0].iov_len = sizeof(tmp[0]); 3333 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3334 &src_iovs[0], 1, NULL, NULL, 0, 3335 ut_sequence_step_cb, &completed); 3336 CU_ASSERT_EQUAL(rc, 0); 3337 3338 dst_iovs[1].iov_base = tmp[2]; 3339 dst_iovs[1].iov_len = sizeof(tmp[2]); 3340 src_iovs[1].iov_base = tmp[1]; 3341 src_iovs[1].iov_len = sizeof(tmp[1]); 3342 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3343 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3344 ut_sequence_step_cb, &completed); 3345 CU_ASSERT_EQUAL(rc, 0); 3346 3347 dst_iovs[2].iov_base = buf; 3348 dst_iovs[2].iov_len = sizeof(buf); 3349 src_iovs[2].iov_base = tmp[2]; 3350 src_iovs[2].iov_len = sizeof(tmp[2]); 3351 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3352 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3353 ut_sequence_step_cb, &completed); 3354 CU_ASSERT_EQUAL(rc, 0); 3355 3356 ut_seq.complete = false; 3357 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3358 3359 poll_threads(); 3360 3361 CU_ASSERT_EQUAL(completed, 4); 3362 CU_ASSERT(ut_seq.complete); 3363 CU_ASSERT_EQUAL(ut_seq.status, 0); 3364 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3365 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3366 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3367 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3368 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3369 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3370 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3371 3372 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3373 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3374 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3375 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3376 3377 /* Check sequence when the first and last operations are executed through modules, while the 3378 * ones in the middle are executed by the driver */ 3379 seq = NULL; 3380 completed = 0; 3381 memset(buf, 0, sizeof(buf)); 3382 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3383 memset(tmp[1], 0, sizeof(tmp[1])); 3384 memset(tmp[2], 0, sizeof(tmp[2])); 3385 memset(&expected[0], 0xfe, 2048); 3386 memset(&expected[2048], 0xa5, 2048); 3387 3388 dst_iovs[0].iov_base = tmp[1]; 3389 dst_iovs[0].iov_len = sizeof(tmp[1]); 3390 src_iovs[0].iov_base = tmp[0]; 3391 src_iovs[0].iov_len = sizeof(tmp[0]); 3392 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3393 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3394 ut_sequence_step_cb, &completed); 3395 CU_ASSERT_EQUAL(rc, 0); 3396 3397 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xfe, 0, 3398 ut_sequence_step_cb, &completed); 3399 CU_ASSERT_EQUAL(rc, 0); 3400 3401 dst_iovs[1].iov_base = tmp[2]; 3402 dst_iovs[1].iov_len = sizeof(tmp[2]); 3403 src_iovs[1].iov_base = tmp[1]; 3404 src_iovs[1].iov_len = sizeof(tmp[1]); 3405 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3406 &src_iovs[1], 1, NULL, NULL, 0, 3407 ut_sequence_step_cb, &completed); 3408 CU_ASSERT_EQUAL(rc, 0); 3409 3410 dst_iovs[2].iov_base = buf; 3411 dst_iovs[2].iov_len = sizeof(buf); 3412 src_iovs[2].iov_base = tmp[2]; 3413 src_iovs[2].iov_len = sizeof(tmp[2]); 3414 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3415 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3416 ut_sequence_step_cb, &completed); 3417 CU_ASSERT_EQUAL(rc, 0); 3418 3419 ut_seq.complete = false; 3420 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3421 3422 poll_threads(); 3423 3424 CU_ASSERT_EQUAL(completed, 4); 3425 CU_ASSERT(ut_seq.complete); 3426 CU_ASSERT_EQUAL(ut_seq.status, 0); 3427 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3428 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3429 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3430 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3431 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3432 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3433 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3434 3435 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3436 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3437 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3438 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3439 3440 /* Check a sequence with operations executed by: module, driver, module, driver */ 3441 seq = NULL; 3442 completed = 0; 3443 memset(buf, 0, sizeof(buf)); 3444 memset(tmp[0], 0x5a, sizeof(tmp[0])); 3445 memset(tmp[1], 0, sizeof(tmp[1])); 3446 memset(tmp[2], 0, sizeof(tmp[2])); 3447 memset(&expected[0], 0xef, 2048); 3448 memset(&expected[2048], 0x5a, 2048); 3449 3450 dst_iovs[0].iov_base = tmp[1]; 3451 dst_iovs[0].iov_len = sizeof(tmp[1]); 3452 src_iovs[0].iov_base = tmp[0]; 3453 src_iovs[0].iov_len = sizeof(tmp[0]); 3454 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3455 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3456 ut_sequence_step_cb, &completed); 3457 CU_ASSERT_EQUAL(rc, 0); 3458 3459 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3460 ut_sequence_step_cb, &completed); 3461 CU_ASSERT_EQUAL(rc, 0); 3462 3463 dst_iovs[1].iov_base = tmp[2]; 3464 dst_iovs[1].iov_len = sizeof(tmp[2]); 3465 src_iovs[1].iov_base = tmp[1]; 3466 src_iovs[1].iov_len = sizeof(tmp[1]); 3467 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3468 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3469 ut_sequence_step_cb, &completed); 3470 CU_ASSERT_EQUAL(rc, 0); 3471 3472 dst_iovs[2].iov_base = buf; 3473 dst_iovs[2].iov_len = sizeof(buf); 3474 src_iovs[2].iov_base = tmp[2]; 3475 src_iovs[2].iov_len = sizeof(tmp[2]); 3476 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3477 &src_iovs[2], 1, NULL, NULL, 0, 3478 ut_sequence_step_cb, &completed); 3479 CU_ASSERT_EQUAL(rc, 0); 3480 3481 ut_seq.complete = false; 3482 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3483 3484 poll_threads(); 3485 3486 CU_ASSERT_EQUAL(completed, 4); 3487 CU_ASSERT(ut_seq.complete); 3488 CU_ASSERT_EQUAL(ut_seq.status, 0); 3489 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3490 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 0); 3491 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3492 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3493 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3494 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3495 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3496 3497 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3498 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3499 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3500 g_drv_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3501 3502 /* Check that an error returned from driver's execute_sequence() will fail the whole 3503 * sequence and any subsequent operations won't be processed */ 3504 seq = NULL; 3505 completed = 0; 3506 memset(buf, 0, sizeof(buf)); 3507 memset(expected, 0, sizeof(expected)); 3508 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3509 g_drv_operations[ACCEL_OPC_FILL].submit_status = -EPERM; 3510 3511 dst_iovs[0].iov_base = tmp[1]; 3512 dst_iovs[0].iov_len = sizeof(tmp[1]); 3513 src_iovs[0].iov_base = tmp[0]; 3514 src_iovs[0].iov_len = sizeof(tmp[0]); 3515 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3516 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3517 ut_sequence_step_cb, &completed); 3518 CU_ASSERT_EQUAL(rc, 0); 3519 3520 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3521 ut_sequence_step_cb, &completed); 3522 CU_ASSERT_EQUAL(rc, 0); 3523 3524 dst_iovs[1].iov_base = buf; 3525 dst_iovs[1].iov_len = sizeof(buf); 3526 src_iovs[1].iov_base = tmp[1]; 3527 src_iovs[1].iov_len = sizeof(tmp[1]); 3528 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3529 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3530 ut_sequence_step_cb, &completed); 3531 CU_ASSERT_EQUAL(rc, 0); 3532 3533 ut_seq.complete = false; 3534 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3535 3536 poll_threads(); 3537 3538 CU_ASSERT_EQUAL(completed, 3); 3539 CU_ASSERT(ut_seq.complete); 3540 CU_ASSERT_EQUAL(ut_seq.status, -EPERM); 3541 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3542 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3543 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 0); 3544 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3545 CU_ASSERT_EQUAL(memcmp(buf, expected, 4096), 0); 3546 3547 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3548 g_drv_operations[ACCEL_OPC_FILL].count = 0; 3549 g_drv_operations[ACCEL_OPC_FILL].submit_status = 0; 3550 3551 /* Check that a failed task completed by a driver will cause the whole sequence to be failed 3552 * and any subsequent operations won't be processed */ 3553 seq = NULL; 3554 completed = 0; 3555 memset(buf, 0, sizeof(buf)); 3556 memset(expected, 0, sizeof(expected)); 3557 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3558 g_drv_operations[ACCEL_OPC_FILL].complete_status = -ENOENT; 3559 3560 dst_iovs[0].iov_base = tmp[1]; 3561 dst_iovs[0].iov_len = sizeof(tmp[1]); 3562 src_iovs[0].iov_base = tmp[0]; 3563 src_iovs[0].iov_len = sizeof(tmp[0]); 3564 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3565 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3566 ut_sequence_step_cb, &completed); 3567 CU_ASSERT_EQUAL(rc, 0); 3568 3569 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3570 ut_sequence_step_cb, &completed); 3571 CU_ASSERT_EQUAL(rc, 0); 3572 3573 dst_iovs[1].iov_base = buf; 3574 dst_iovs[1].iov_len = sizeof(buf); 3575 src_iovs[1].iov_base = tmp[1]; 3576 src_iovs[1].iov_len = sizeof(tmp[1]); 3577 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3578 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3579 ut_sequence_step_cb, &completed); 3580 CU_ASSERT_EQUAL(rc, 0); 3581 3582 ut_seq.complete = false; 3583 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3584 3585 poll_threads(); 3586 3587 CU_ASSERT_EQUAL(completed, 3); 3588 CU_ASSERT(ut_seq.complete); 3589 CU_ASSERT_EQUAL(ut_seq.status, -ENOENT); 3590 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_FILL].count, 0); 3591 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3592 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 0); 3593 CU_ASSERT_EQUAL(g_drv_operations[ACCEL_OPC_FILL].count, 1); 3594 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3595 3596 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3597 g_modules_opc[i] = modules[i]; 3598 } 3599 3600 /* Clear the driver so that other tests won't use it */ 3601 g_accel_driver = NULL; 3602 memset(&g_drv_operations, 0, sizeof(g_drv_operations)); 3603 3604 ut_clear_operations(); 3605 spdk_put_io_channel(ioch); 3606 poll_threads(); 3607 } 3608 3609 struct ut_saved_iovs { 3610 struct iovec src; 3611 struct iovec dst; 3612 }; 3613 3614 static struct ut_saved_iovs g_seq_saved_iovs[ACCEL_OPC_LAST]; 3615 3616 static int 3617 ut_submit_save_iovs(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3618 { 3619 SPDK_CU_ASSERT_FATAL(task->s.iovcnt == 1); 3620 SPDK_CU_ASSERT_FATAL(task->d.iovcnt == 1); 3621 3622 g_seq_saved_iovs[task->op_code].src = task->s.iovs[0]; 3623 g_seq_saved_iovs[task->op_code].dst = task->d.iovs[0]; 3624 3625 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3626 3627 spdk_accel_task_complete(task, 0); 3628 3629 return 0; 3630 } 3631 3632 static void 3633 test_sequence_same_iovs(void) 3634 { 3635 struct spdk_accel_sequence *seq = NULL; 3636 struct spdk_io_channel *ioch; 3637 struct spdk_accel_crypto_key key = {}; 3638 struct ut_sequence ut_seq; 3639 struct accel_module modules[ACCEL_OPC_LAST]; 3640 char buf[4096], tmp[4096], expected[4096]; 3641 struct iovec iovs[3], expected_siov, expected_diov; 3642 struct spdk_memory_domain *domain; 3643 void *accel_buf, *domain_ctx; 3644 int i, rc, completed = 0; 3645 3646 ioch = spdk_accel_get_io_channel(); 3647 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3648 3649 /* Override the submit_tasks function */ 3650 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3651 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3652 modules[i] = g_modules_opc[i]; 3653 g_modules_opc[i] = g_module; 3654 } 3655 /* Intercept crypto operations, as they should be executed by an accel module */ 3656 g_seq_operations[ACCEL_OPC_ENCRYPT].submit = ut_submit_save_iovs; 3657 g_seq_operations[ACCEL_OPC_DECRYPT].submit = ut_submit_save_iovs; 3658 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3659 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3660 3661 /* Check that it's possible to use the same iovec ptr for different operations */ 3662 seq = NULL; 3663 completed = 0; 3664 memset(buf, 0, sizeof(buf)); 3665 memset(expected, 0xa5, sizeof(expected)); 3666 3667 iovs[0].iov_base = expected; 3668 iovs[0].iov_len = sizeof(expected); 3669 iovs[1].iov_base = tmp; 3670 iovs[1].iov_len = sizeof(tmp); 3671 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, NULL, NULL, 3672 &iovs[0], 1, NULL, NULL, 0, 4096, 0, 3673 ut_sequence_step_cb, &completed); 3674 CU_ASSERT_EQUAL(rc, 0); 3675 /* Reuse iov[1] as src */ 3676 iovs[2].iov_base = buf; 3677 iovs[2].iov_len = sizeof(buf); 3678 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3679 &iovs[1], 1, NULL, NULL, 0, 4096, 0, 3680 ut_sequence_step_cb, &completed); 3681 CU_ASSERT_EQUAL(rc, 0); 3682 3683 ut_seq.complete = false; 3684 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3685 3686 poll_threads(); 3687 3688 CU_ASSERT_EQUAL(completed, 2); 3689 CU_ASSERT(ut_seq.complete); 3690 CU_ASSERT_EQUAL(ut_seq.status, 0); 3691 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3692 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3693 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3694 expected_siov.iov_base = expected; 3695 expected_siov.iov_len = sizeof(expected); 3696 expected_diov.iov_base = tmp; 3697 expected_diov.iov_len = sizeof(tmp); 3698 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_ENCRYPT].src, 3699 &expected_siov, sizeof(expected_siov)), 0); 3700 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_ENCRYPT].dst, 3701 &expected_diov, sizeof(expected_diov)), 0); 3702 expected_siov.iov_base = tmp; 3703 expected_siov.iov_len = sizeof(tmp); 3704 expected_diov.iov_base = buf; 3705 expected_diov.iov_len = sizeof(buf); 3706 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_DECRYPT].src, 3707 &expected_siov, sizeof(expected_siov)), 0); 3708 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_DECRYPT].dst, 3709 &expected_diov, sizeof(expected_diov)), 0); 3710 g_seq_operations[ACCEL_OPC_ENCRYPT].count = 0; 3711 g_seq_operations[ACCEL_OPC_DECRYPT].count = 0; 3712 3713 /* Check the same with an accel buffer */ 3714 seq = NULL; 3715 completed = 0; 3716 memset(buf, 0, sizeof(buf)); 3717 memset(expected, 0x5a, sizeof(expected)); 3718 3719 rc = spdk_accel_get_buf(ioch, sizeof(buf), &accel_buf, &domain, &domain_ctx); 3720 CU_ASSERT_EQUAL(rc, 0); 3721 3722 iovs[0].iov_base = expected; 3723 iovs[0].iov_len = sizeof(expected); 3724 iovs[1].iov_base = accel_buf; 3725 iovs[1].iov_len = sizeof(buf); 3726 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, domain, domain_ctx, 3727 &iovs[0], 1, NULL, NULL, 0, 4096, 0, 3728 ut_sequence_step_cb, &completed); 3729 CU_ASSERT_EQUAL(rc, 0); 3730 /* Reuse iov[1] as src */ 3731 iovs[2].iov_base = buf; 3732 iovs[2].iov_len = sizeof(buf); 3733 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3734 &iovs[1], 1, domain, domain_ctx, 0, 4096, 0, 3735 ut_sequence_step_cb, &completed); 3736 CU_ASSERT_EQUAL(rc, 0); 3737 3738 ut_seq.complete = false; 3739 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3740 3741 poll_threads(); 3742 3743 CU_ASSERT_EQUAL(completed, 2); 3744 CU_ASSERT(ut_seq.complete); 3745 CU_ASSERT_EQUAL(ut_seq.status, 0); 3746 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_ENCRYPT].count, 1); 3747 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECRYPT].count, 1); 3748 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3749 expected_siov.iov_base = expected; 3750 expected_siov.iov_len = sizeof(expected); 3751 expected_diov.iov_base = buf; 3752 expected_diov.iov_len = sizeof(buf); 3753 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_ENCRYPT].src, 3754 &expected_siov, sizeof(expected_siov)), 0); 3755 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_DECRYPT].dst, 3756 &expected_diov, sizeof(expected_diov)), 0); 3757 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[ACCEL_OPC_ENCRYPT].dst, 3758 &g_seq_saved_iovs[ACCEL_OPC_DECRYPT].src, 3759 sizeof(struct iovec)), 0); 3760 spdk_accel_put_buf(ioch, accel_buf, domain, domain_ctx); 3761 3762 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3763 g_modules_opc[i] = modules[i]; 3764 } 3765 3766 ut_clear_operations(); 3767 spdk_put_io_channel(ioch); 3768 poll_threads(); 3769 } 3770 3771 static void 3772 test_sequence_crc32(void) 3773 { 3774 struct spdk_accel_sequence *seq = NULL; 3775 struct spdk_io_channel *ioch; 3776 struct ut_sequence ut_seq; 3777 struct accel_module modules[ACCEL_OPC_LAST]; 3778 char buf[4096], tmp[3][4096]; 3779 struct iovec src_iovs[4], dst_iovs[4]; 3780 uint32_t crc, crc2; 3781 int i, rc, completed; 3782 3783 ioch = spdk_accel_get_io_channel(); 3784 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3785 3786 /* Override the submit_tasks function */ 3787 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3788 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3789 g_seq_operations[i].submit = sw_accel_submit_tasks; 3790 modules[i] = g_modules_opc[i]; 3791 g_modules_opc[i] = g_module; 3792 } 3793 g_seq_operations[ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 3794 3795 /* First check the simplest case - single crc32c operation */ 3796 seq = NULL; 3797 completed = 0; 3798 crc = 0; 3799 memset(buf, 0xa5, sizeof(buf)); 3800 3801 src_iovs[0].iov_base = buf; 3802 src_iovs[0].iov_len = sizeof(buf); 3803 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3804 ut_sequence_step_cb, &completed); 3805 CU_ASSERT_EQUAL(rc, 0); 3806 3807 ut_seq.complete = false; 3808 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3809 3810 poll_threads(); 3811 CU_ASSERT_EQUAL(completed, 1); 3812 CU_ASSERT(ut_seq.complete); 3813 CU_ASSERT_EQUAL(ut_seq.status, 0); 3814 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_CRC32C].count, 1); 3815 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3816 g_seq_operations[ACCEL_OPC_CRC32C].count = 0; 3817 3818 /* Now check copy+crc - this should remove the copy operation and change the buffer for the 3819 * crc operation */ 3820 seq = NULL; 3821 completed = 0; 3822 crc = 0; 3823 memset(buf, 0x5a, sizeof(buf)); 3824 memset(&tmp[0], 0, sizeof(tmp[0])); 3825 3826 dst_iovs[0].iov_base = tmp[0]; 3827 dst_iovs[0].iov_len = sizeof(tmp[0]); 3828 src_iovs[0].iov_base = buf; 3829 src_iovs[0].iov_len = sizeof(buf); 3830 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3831 &src_iovs[0], 1, NULL, NULL, 0, 3832 ut_sequence_step_cb, &completed); 3833 CU_ASSERT_EQUAL(rc, 0); 3834 3835 src_iovs[1].iov_base = tmp[0]; 3836 src_iovs[1].iov_len = sizeof(tmp[0]); 3837 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 3838 ut_sequence_step_cb, &completed); 3839 CU_ASSERT_EQUAL(rc, 0); 3840 3841 ut_seq.complete = false; 3842 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3843 3844 poll_threads(); 3845 CU_ASSERT_EQUAL(completed, 2); 3846 CU_ASSERT(ut_seq.complete); 3847 CU_ASSERT_EQUAL(ut_seq.status, 0); 3848 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_CRC32C].count, 1); 3849 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 3850 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3851 g_seq_operations[ACCEL_OPC_CRC32C].count = 0; 3852 3853 /* Check crc+copy - this time the copy cannot be removed, because there's no operation 3854 * before crc to change the buffer */ 3855 seq = NULL; 3856 completed = 0; 3857 crc = 0; 3858 memset(buf, 0, sizeof(buf)); 3859 memset(&tmp[0], 0xa5, sizeof(tmp[0])); 3860 3861 src_iovs[0].iov_base = tmp[0]; 3862 src_iovs[0].iov_len = sizeof(tmp[0]); 3863 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3864 ut_sequence_step_cb, &completed); 3865 CU_ASSERT_EQUAL(rc, 0); 3866 3867 dst_iovs[1].iov_base = buf; 3868 dst_iovs[1].iov_len = sizeof(buf); 3869 src_iovs[1].iov_base = tmp[0]; 3870 src_iovs[1].iov_len = sizeof(tmp[0]); 3871 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3872 &src_iovs[1], 1, NULL, NULL, 0, 3873 ut_sequence_step_cb, &completed); 3874 CU_ASSERT_EQUAL(rc, 0); 3875 3876 ut_seq.complete = false; 3877 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3878 3879 poll_threads(); 3880 CU_ASSERT_EQUAL(completed, 2); 3881 CU_ASSERT(ut_seq.complete); 3882 CU_ASSERT_EQUAL(ut_seq.status, 0); 3883 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_CRC32C].count, 1); 3884 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 1); 3885 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 3886 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 3887 g_seq_operations[ACCEL_OPC_CRC32C].count = 0; 3888 g_seq_operations[ACCEL_OPC_COPY].count = 0; 3889 3890 /* Check a sequence with an operation at the beginning that can have its buffer changed, two 3891 * crc operations and a copy at the end. The copy should be removed and the dst buffer of 3892 * the first operation and the src buffer of the crc operations should be changed. 3893 */ 3894 seq = NULL; 3895 completed = 0; 3896 crc = crc2 = 0; 3897 memset(buf, 0, sizeof(buf)); 3898 memset(&tmp[0], 0x5a, sizeof(tmp[0])); 3899 dst_iovs[0].iov_base = tmp[1]; 3900 dst_iovs[0].iov_len = sizeof(tmp[1]); 3901 src_iovs[0].iov_base = tmp[0]; 3902 src_iovs[0].iov_len = sizeof(tmp[0]); 3903 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3904 &src_iovs[0], 1, NULL, NULL, 0, 3905 ut_sequence_step_cb, &completed); 3906 CU_ASSERT_EQUAL(rc, 0); 3907 3908 src_iovs[1].iov_base = tmp[1]; 3909 src_iovs[1].iov_len = sizeof(tmp[1]); 3910 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 3911 ut_sequence_step_cb, &completed); 3912 CU_ASSERT_EQUAL(rc, 0); 3913 3914 src_iovs[2].iov_base = tmp[1]; 3915 src_iovs[2].iov_len = sizeof(tmp[1]); 3916 rc = spdk_accel_append_crc32c(&seq, ioch, &crc2, &src_iovs[2], 1, NULL, NULL, 0, 3917 ut_sequence_step_cb, &completed); 3918 CU_ASSERT_EQUAL(rc, 0); 3919 3920 dst_iovs[3].iov_base = buf; 3921 dst_iovs[3].iov_len = sizeof(buf); 3922 src_iovs[3].iov_base = tmp[1]; 3923 src_iovs[3].iov_len = sizeof(tmp[1]); 3924 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 3925 &src_iovs[3], 1, NULL, NULL, 0, 3926 ut_sequence_step_cb, &completed); 3927 CU_ASSERT_EQUAL(rc, 0); 3928 3929 ut_seq.complete = false; 3930 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3931 3932 poll_threads(); 3933 CU_ASSERT_EQUAL(completed, 4); 3934 CU_ASSERT(ut_seq.complete); 3935 CU_ASSERT_EQUAL(ut_seq.status, 0); 3936 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3937 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_CRC32C].count, 2); 3938 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 0); 3939 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 3940 CU_ASSERT_EQUAL(crc, crc2); 3941 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 3942 g_seq_operations[ACCEL_OPC_CRC32C].count = 0; 3943 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3944 3945 /* Check that a copy won't be removed if the buffers don't match */ 3946 seq = NULL; 3947 completed = 0; 3948 crc = 0; 3949 memset(buf, 0, sizeof(buf)); 3950 memset(&tmp[0], 0xa5, 2048); 3951 memset(&tmp[1], 0xfe, sizeof(tmp[1])); 3952 memset(&tmp[2], 0xfe, sizeof(tmp[1])); 3953 dst_iovs[0].iov_base = &tmp[1][2048]; 3954 dst_iovs[0].iov_len = 2048; 3955 src_iovs[0].iov_base = tmp[0]; 3956 src_iovs[0].iov_len = 2048; 3957 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3958 &src_iovs[0], 1, NULL, NULL, 0, 3959 ut_sequence_step_cb, &completed); 3960 CU_ASSERT_EQUAL(rc, 0); 3961 3962 src_iovs[1].iov_base = &tmp[1][2048]; 3963 src_iovs[1].iov_len = 2048; 3964 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 3965 ut_sequence_step_cb, &completed); 3966 CU_ASSERT_EQUAL(rc, 0); 3967 3968 dst_iovs[2].iov_base = buf; 3969 dst_iovs[2].iov_len = sizeof(buf); 3970 src_iovs[2].iov_base = tmp[1]; 3971 src_iovs[2].iov_len = sizeof(tmp[1]); 3972 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3973 &src_iovs[2], 1, NULL, NULL, 0, 3974 ut_sequence_step_cb, &completed); 3975 CU_ASSERT_EQUAL(rc, 0); 3976 3977 ut_seq.complete = false; 3978 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3979 3980 poll_threads(); 3981 CU_ASSERT_EQUAL(completed, 3); 3982 CU_ASSERT(ut_seq.complete); 3983 CU_ASSERT_EQUAL(ut_seq.status, 0); 3984 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_DECOMPRESS].count, 1); 3985 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_CRC32C].count, 1); 3986 CU_ASSERT_EQUAL(g_seq_operations[ACCEL_OPC_COPY].count, 1); 3987 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], 2048, ~0u)); 3988 CU_ASSERT_EQUAL(memcmp(buf, tmp[2], 2048), 0); 3989 CU_ASSERT_EQUAL(memcmp(&buf[2048], tmp[0], 2048), 0); 3990 g_seq_operations[ACCEL_OPC_CRC32C].count = 0; 3991 g_seq_operations[ACCEL_OPC_DECOMPRESS].count = 0; 3992 g_seq_operations[ACCEL_OPC_COPY].count = 0; 3993 3994 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 3995 g_modules_opc[i] = modules[i]; 3996 } 3997 3998 ut_clear_operations(); 3999 spdk_put_io_channel(ioch); 4000 poll_threads(); 4001 } 4002 4003 static int 4004 test_sequence_setup(void) 4005 { 4006 int rc; 4007 4008 allocate_cores(1); 4009 allocate_threads(1); 4010 set_thread(0); 4011 4012 rc = spdk_iobuf_initialize(); 4013 if (rc != 0) { 4014 CU_ASSERT(false); 4015 return -1; 4016 } 4017 4018 rc = spdk_accel_initialize(); 4019 if (rc != 0) { 4020 CU_ASSERT(false); 4021 return -1; 4022 } 4023 4024 return 0; 4025 } 4026 4027 static void 4028 finish_cb(void *cb_arg) 4029 { 4030 bool *done = cb_arg; 4031 4032 *done = true; 4033 } 4034 4035 static int 4036 test_sequence_cleanup(void) 4037 { 4038 bool done = false; 4039 4040 spdk_accel_finish(finish_cb, &done); 4041 4042 while (!done) { 4043 poll_threads(); 4044 } 4045 4046 done = false; 4047 spdk_iobuf_finish(finish_cb, &done); 4048 while (!done) { 4049 poll_threads(); 4050 } 4051 4052 free_threads(); 4053 free_cores(); 4054 4055 return 0; 4056 } 4057 4058 int 4059 main(int argc, char **argv) 4060 { 4061 CU_pSuite suite = NULL, seq_suite; 4062 unsigned int num_failures; 4063 4064 CU_set_error_action(CUEA_ABORT); 4065 CU_initialize_registry(); 4066 4067 /* Sequence tests require accel to be initialized normally, so run them before the other 4068 * tests which register accel modules which aren't fully implemented, causing accel 4069 * initialization to fail. 4070 */ 4071 seq_suite = CU_add_suite("accel_sequence", test_sequence_setup, test_sequence_cleanup); 4072 CU_ADD_TEST(seq_suite, test_sequence_fill_copy); 4073 CU_ADD_TEST(seq_suite, test_sequence_abort); 4074 CU_ADD_TEST(seq_suite, test_sequence_append_error); 4075 CU_ADD_TEST(seq_suite, test_sequence_completion_error); 4076 #ifdef SPDK_CONFIG_ISAL /* accel_sw requires isa-l for compression */ 4077 CU_ADD_TEST(seq_suite, test_sequence_decompress); 4078 CU_ADD_TEST(seq_suite, test_sequence_reverse); 4079 #endif 4080 CU_ADD_TEST(seq_suite, test_sequence_copy_elision); 4081 CU_ADD_TEST(seq_suite, test_sequence_accel_buffers); 4082 CU_ADD_TEST(seq_suite, test_sequence_memory_domain); 4083 CU_ADD_TEST(seq_suite, test_sequence_module_memory_domain); 4084 #ifdef SPDK_CONFIG_ISAL_CRYPTO /* accel_sw requires isa-l-crypto for crypto operations */ 4085 CU_ADD_TEST(seq_suite, test_sequence_crypto); 4086 #endif 4087 CU_ADD_TEST(seq_suite, test_sequence_driver); 4088 CU_ADD_TEST(seq_suite, test_sequence_same_iovs); 4089 CU_ADD_TEST(seq_suite, test_sequence_crc32); 4090 4091 suite = CU_add_suite("accel", test_setup, test_cleanup); 4092 CU_ADD_TEST(suite, test_spdk_accel_task_complete); 4093 CU_ADD_TEST(suite, test_get_task); 4094 CU_ADD_TEST(suite, test_spdk_accel_submit_copy); 4095 CU_ADD_TEST(suite, test_spdk_accel_submit_dualcast); 4096 CU_ADD_TEST(suite, test_spdk_accel_submit_compare); 4097 CU_ADD_TEST(suite, test_spdk_accel_submit_fill); 4098 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c); 4099 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32cv); 4100 CU_ADD_TEST(suite, test_spdk_accel_submit_copy_crc32c); 4101 CU_ADD_TEST(suite, test_spdk_accel_submit_xor); 4102 CU_ADD_TEST(suite, test_spdk_accel_module_find_by_name); 4103 CU_ADD_TEST(suite, test_spdk_accel_module_register); 4104 4105 CU_basic_set_mode(CU_BRM_VERBOSE); 4106 CU_basic_run_tests(); 4107 num_failures = CU_get_number_of_failures(); 4108 CU_cleanup_registry(); 4109 4110 return num_failures; 4111 } 4112