1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2021 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk_internal/cunit.h" 8 #include "spdk_internal/mock.h" 9 #include "spdk/accel_module.h" 10 #include "thread/thread_internal.h" 11 #include "common/lib/ut_multithread.c" 12 #include "common/lib/test_iobuf.c" 13 #include "accel/accel.c" 14 #include "accel/accel_sw.c" 15 #include "unit/lib/json_mock.c" 16 17 DEFINE_STUB_V(spdk_memory_domain_destroy, (struct spdk_memory_domain *domain)); 18 DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, 19 (struct spdk_memory_domain *domain), "UT_DMA"); 20 21 int 22 spdk_memory_domain_create(struct spdk_memory_domain **domain, enum spdk_dma_device_type type, 23 struct spdk_memory_domain_ctx *ctx, const char *id) 24 { 25 *domain = (void *)0xdeadbeef; 26 27 return 0; 28 } 29 30 struct ut_domain_ctx { 31 struct iovec iov; 32 struct iovec expected; 33 int pull_submit_status; 34 int push_submit_status; 35 int pull_complete_status; 36 int push_complete_status; 37 }; 38 39 static struct spdk_memory_domain *g_ut_domain = (void *)0xa55e1; 40 41 int 42 spdk_memory_domain_pull_data(struct spdk_memory_domain *sd, void *sctx, struct iovec *siov, 43 uint32_t siovcnt, struct iovec *diov, uint32_t diovcnt, 44 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 45 { 46 struct ut_domain_ctx *ctx = sctx; 47 48 CU_ASSERT_EQUAL(sd, g_ut_domain); 49 CU_ASSERT_EQUAL(siovcnt, 1); 50 CU_ASSERT_EQUAL(memcmp(siov, &ctx->expected, sizeof(*siov)), 0); 51 52 if (ctx->pull_submit_status != 0) { 53 return ctx->pull_submit_status; 54 } 55 56 if (ctx->pull_complete_status != 0) { 57 cpl_cb(cpl_arg, ctx->pull_complete_status); 58 return 0; 59 } 60 61 spdk_iovcpy(&ctx->iov, 1, diov, diovcnt); 62 cpl_cb(cpl_arg, 0); 63 64 return 0; 65 } 66 67 int 68 spdk_memory_domain_push_data(struct spdk_memory_domain *dd, void *dctx, struct iovec *diov, 69 uint32_t diovcnt, struct iovec *siov, uint32_t siovcnt, 70 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 71 { 72 struct ut_domain_ctx *ctx = dctx; 73 74 CU_ASSERT_EQUAL(dd, g_ut_domain); 75 CU_ASSERT_EQUAL(diovcnt, 1); 76 CU_ASSERT_EQUAL(memcmp(diov, &ctx->expected, sizeof(*diov)), 0); 77 78 if (ctx->push_submit_status != 0) { 79 return ctx->push_submit_status; 80 } 81 82 if (ctx->push_complete_status != 0) { 83 cpl_cb(cpl_arg, ctx->push_complete_status); 84 return 0; 85 } 86 87 spdk_iovcpy(siov, siovcnt, &ctx->iov, 1); 88 cpl_cb(cpl_arg, 0); 89 90 return 0; 91 } 92 93 /* global vars and setup/cleanup functions used for all test functions */ 94 struct spdk_accel_module_if g_module_if = {}; 95 struct accel_module g_module = { .module = &g_module_if }; 96 struct spdk_io_channel *g_ch = NULL; 97 struct accel_io_channel *g_accel_ch = NULL; 98 struct sw_accel_io_channel *g_sw_ch = NULL; 99 struct spdk_io_channel *g_module_ch = NULL; 100 101 static uint64_t g_opc_mask = 0; 102 103 static uint64_t 104 _accel_op_to_bit(enum spdk_accel_opcode opc) 105 { 106 return (1 << opc); 107 } 108 109 static bool 110 _supports_opcode(enum spdk_accel_opcode opc) 111 { 112 if (_accel_op_to_bit(opc) & g_opc_mask) { 113 return true; 114 } 115 return false; 116 } 117 118 static int 119 test_setup(void) 120 { 121 int i; 122 123 g_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct accel_io_channel)); 124 if (g_ch == NULL) { 125 /* for some reason the assert fatal macro doesn't work in the setup function. */ 126 CU_ASSERT(false); 127 return -1; 128 } 129 g_accel_ch = (struct accel_io_channel *)((char *)g_ch + sizeof(struct spdk_io_channel)); 130 g_module_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct sw_accel_io_channel)); 131 if (g_module_ch == NULL) { 132 CU_ASSERT(false); 133 return -1; 134 } 135 136 g_module_if.submit_tasks = sw_accel_submit_tasks; 137 g_module_if.name = "software"; 138 for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) { 139 g_accel_ch->module_ch[i] = g_module_ch; 140 g_modules_opc[i] = g_module; 141 } 142 g_sw_ch = (struct sw_accel_io_channel *)((char *)g_module_ch + sizeof( 143 struct spdk_io_channel)); 144 STAILQ_INIT(&g_sw_ch->tasks_to_complete); 145 g_module_if.supports_opcode = _supports_opcode; 146 return 0; 147 } 148 149 static int 150 test_cleanup(void) 151 { 152 free(g_ch); 153 free(g_module_ch); 154 155 return 0; 156 } 157 158 #define DUMMY_ARG 0xDEADBEEF 159 static bool g_dummy_cb_called = false; 160 static void 161 dummy_cb_fn(void *cb_arg, int status) 162 { 163 CU_ASSERT(*(uint32_t *)cb_arg == DUMMY_ARG); 164 CU_ASSERT(status == 0); 165 g_dummy_cb_called = true; 166 } 167 168 static void 169 test_spdk_accel_task_complete(void) 170 { 171 struct spdk_accel_task accel_task = {}; 172 struct spdk_accel_task *expected_accel_task = NULL; 173 uint32_t cb_arg = DUMMY_ARG; 174 int status = 0; 175 176 accel_task.accel_ch = g_accel_ch; 177 accel_task.cb_fn = dummy_cb_fn; 178 accel_task.cb_arg = &cb_arg; 179 STAILQ_INIT(&g_accel_ch->task_pool); 180 181 /* Confirm cb is called and task added to list. */ 182 spdk_accel_task_complete(&accel_task, status); 183 CU_ASSERT(g_dummy_cb_called == true); 184 expected_accel_task = STAILQ_FIRST(&g_accel_ch->task_pool); 185 STAILQ_REMOVE_HEAD(&g_accel_ch->task_pool, link); 186 CU_ASSERT(expected_accel_task == &accel_task); 187 } 188 189 static void 190 test_get_task(void) 191 { 192 struct spdk_accel_task *task; 193 struct spdk_accel_task _task; 194 void *cb_arg = NULL; 195 196 STAILQ_INIT(&g_accel_ch->task_pool); 197 198 /* no tasks left, return NULL. */ 199 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 200 CU_ASSERT(task == NULL); 201 202 _task.cb_fn = dummy_cb_fn; 203 _task.cb_arg = cb_arg; 204 _task.accel_ch = g_accel_ch; 205 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &_task, link); 206 207 /* Get a valid task. */ 208 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 209 CU_ASSERT(task == &_task); 210 CU_ASSERT(_task.cb_fn == dummy_cb_fn); 211 CU_ASSERT(_task.cb_arg == cb_arg); 212 CU_ASSERT(_task.accel_ch == g_accel_ch); 213 } 214 215 #define TEST_SUBMIT_SIZE 64 216 static void 217 test_spdk_accel_submit_copy(void) 218 { 219 const uint64_t nbytes = TEST_SUBMIT_SIZE; 220 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 221 uint8_t src[TEST_SUBMIT_SIZE] = {0}; 222 void *cb_arg = NULL; 223 int rc; 224 struct spdk_accel_task task; 225 struct spdk_accel_task_aux_data task_aux; 226 struct spdk_accel_task *expected_accel_task = NULL; 227 int flags = 0; 228 229 STAILQ_INIT(&g_accel_ch->task_pool); 230 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 231 232 /* Fail with no tasks on _get_task() */ 233 rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, flags, NULL, cb_arg); 234 CU_ASSERT(rc == -ENOMEM); 235 236 task.accel_ch = g_accel_ch; 237 task.flags = 1; 238 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 239 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 240 241 /* submission OK. */ 242 rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, flags, NULL, cb_arg); 243 CU_ASSERT(rc == 0); 244 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COPY); 245 CU_ASSERT(task.flags == 0); 246 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 247 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 248 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 249 CU_ASSERT(expected_accel_task == &task); 250 } 251 252 static void 253 test_spdk_accel_submit_dualcast(void) 254 { 255 void *dst1; 256 void *dst2; 257 void *src; 258 uint32_t align = ALIGN_4K; 259 uint64_t nbytes = TEST_SUBMIT_SIZE; 260 void *cb_arg = NULL; 261 int rc; 262 struct spdk_accel_task task; 263 struct spdk_accel_task_aux_data task_aux; 264 struct spdk_accel_task *expected_accel_task = NULL; 265 int flags = 0; 266 267 STAILQ_INIT(&g_accel_ch->task_pool); 268 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 269 270 /* Dualcast requires 4K alignment on dst addresses, 271 * hence using the hard coded address to test the buffer alignment 272 */ 273 dst1 = (void *)0x5000; 274 dst2 = (void *)0x60f0; 275 src = calloc(1, TEST_SUBMIT_SIZE); 276 SPDK_CU_ASSERT_FATAL(src != NULL); 277 memset(src, 0x5A, TEST_SUBMIT_SIZE); 278 279 /* This should fail since dst2 is not 4k aligned */ 280 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 281 CU_ASSERT(rc == -EINVAL); 282 283 dst1 = (void *)0x7010; 284 dst2 = (void *)0x6000; 285 /* This should fail since dst1 is not 4k aligned */ 286 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 287 CU_ASSERT(rc == -EINVAL); 288 289 /* Dualcast requires 4K alignment on dst addresses */ 290 dst1 = (void *)0x7000; 291 dst2 = (void *)0x6000; 292 /* Fail with no tasks on _get_task() */ 293 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 294 CU_ASSERT(rc == -ENOMEM); 295 296 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 297 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 298 299 /* accel submission OK., since we test the SW path , need to use valid memory addresses 300 * cannot hardcode them anymore */ 301 dst1 = spdk_dma_zmalloc(nbytes, align, NULL); 302 SPDK_CU_ASSERT_FATAL(dst1 != NULL); 303 dst2 = spdk_dma_zmalloc(nbytes, align, NULL); 304 SPDK_CU_ASSERT_FATAL(dst2 != NULL); 305 /* SW module does the dualcast. */ 306 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, flags, NULL, cb_arg); 307 CU_ASSERT(rc == 0); 308 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_DUALCAST); 309 CU_ASSERT(task.flags == 0); 310 CU_ASSERT(memcmp(dst1, src, TEST_SUBMIT_SIZE) == 0); 311 CU_ASSERT(memcmp(dst2, src, TEST_SUBMIT_SIZE) == 0); 312 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 313 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 314 CU_ASSERT(expected_accel_task == &task); 315 316 free(src); 317 spdk_free(dst1); 318 spdk_free(dst2); 319 } 320 321 static void 322 test_spdk_accel_submit_compare(void) 323 { 324 void *src1; 325 void *src2; 326 uint64_t nbytes = TEST_SUBMIT_SIZE; 327 void *cb_arg = NULL; 328 int rc; 329 struct spdk_accel_task task; 330 struct spdk_accel_task_aux_data task_aux; 331 struct spdk_accel_task *expected_accel_task = NULL; 332 333 STAILQ_INIT(&g_accel_ch->task_pool); 334 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 335 336 src1 = calloc(1, TEST_SUBMIT_SIZE); 337 SPDK_CU_ASSERT_FATAL(src1 != NULL); 338 src2 = calloc(1, TEST_SUBMIT_SIZE); 339 SPDK_CU_ASSERT_FATAL(src2 != NULL); 340 341 /* Fail with no tasks on _get_task() */ 342 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 343 CU_ASSERT(rc == -ENOMEM); 344 345 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 346 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 347 348 /* accel submission OK. */ 349 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 350 CU_ASSERT(rc == 0); 351 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COMPARE); 352 CU_ASSERT(memcmp(src1, src2, TEST_SUBMIT_SIZE) == 0); 353 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 354 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 355 CU_ASSERT(expected_accel_task == &task); 356 357 free(src1); 358 free(src2); 359 } 360 361 static void 362 test_spdk_accel_submit_fill(void) 363 { 364 void *dst; 365 void *src; 366 uint8_t fill = 0xf; 367 uint64_t fill64; 368 uint64_t nbytes = TEST_SUBMIT_SIZE; 369 void *cb_arg = NULL; 370 int rc; 371 struct spdk_accel_task task; 372 struct spdk_accel_task_aux_data task_aux; 373 struct spdk_accel_task *expected_accel_task = NULL; 374 int flags = 0; 375 376 STAILQ_INIT(&g_accel_ch->task_pool); 377 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 378 379 dst = calloc(1, TEST_SUBMIT_SIZE); 380 SPDK_CU_ASSERT_FATAL(dst != NULL); 381 src = calloc(1, TEST_SUBMIT_SIZE); 382 SPDK_CU_ASSERT_FATAL(src != NULL); 383 memset(src, fill, TEST_SUBMIT_SIZE); 384 memset(&fill64, fill, sizeof(uint64_t)); 385 386 /* Fail with no tasks on _get_task() */ 387 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 388 CU_ASSERT(rc == -ENOMEM); 389 390 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 391 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 392 393 /* accel submission OK. */ 394 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, flags, NULL, cb_arg); 395 CU_ASSERT(rc == 0); 396 CU_ASSERT(task.fill_pattern == fill64); 397 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_FILL); 398 CU_ASSERT(task.flags == 0); 399 400 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 401 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 402 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 403 CU_ASSERT(expected_accel_task == &task); 404 405 free(dst); 406 free(src); 407 } 408 409 static void 410 test_spdk_accel_submit_crc32c(void) 411 { 412 const uint64_t nbytes = TEST_SUBMIT_SIZE; 413 uint32_t crc_dst; 414 uint8_t src[TEST_SUBMIT_SIZE]; 415 uint32_t seed = 1; 416 void *cb_arg = NULL; 417 int rc; 418 struct spdk_accel_task task; 419 struct spdk_accel_task_aux_data task_aux; 420 struct spdk_accel_task *expected_accel_task = NULL; 421 422 STAILQ_INIT(&g_accel_ch->task_pool); 423 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 424 425 /* Fail with no tasks on _get_task() */ 426 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 427 CU_ASSERT(rc == -ENOMEM); 428 429 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 430 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 431 432 /* accel submission OK. */ 433 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 434 CU_ASSERT(rc == 0); 435 CU_ASSERT(task.crc_dst == &crc_dst); 436 CU_ASSERT(task.seed == seed); 437 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_CRC32C); 438 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 439 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 440 CU_ASSERT(expected_accel_task == &task); 441 } 442 443 static void 444 test_spdk_accel_submit_crc32cv(void) 445 { 446 uint32_t crc_dst; 447 uint32_t seed = 0; 448 uint32_t iov_cnt = 32; 449 void *cb_arg = NULL; 450 int rc; 451 uint32_t i = 0; 452 struct spdk_accel_task task; 453 struct spdk_accel_task_aux_data task_aux; 454 struct iovec iov[32]; 455 struct spdk_accel_task *expected_accel_task = NULL; 456 457 STAILQ_INIT(&g_accel_ch->task_pool); 458 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 459 460 for (i = 0; i < iov_cnt; i++) { 461 iov[i].iov_base = calloc(1, TEST_SUBMIT_SIZE); 462 SPDK_CU_ASSERT_FATAL(iov[i].iov_base != NULL); 463 iov[i].iov_len = TEST_SUBMIT_SIZE; 464 } 465 466 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 467 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 468 469 /* accel submission OK. */ 470 rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, NULL, cb_arg); 471 CU_ASSERT(rc == 0); 472 CU_ASSERT(task.s.iovs == iov); 473 CU_ASSERT(task.s.iovcnt == iov_cnt); 474 CU_ASSERT(task.crc_dst == &crc_dst); 475 CU_ASSERT(task.seed == seed); 476 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_CRC32C); 477 CU_ASSERT(task.cb_arg == cb_arg); 478 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 479 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 480 CU_ASSERT(expected_accel_task == &task); 481 482 for (i = 0; i < iov_cnt; i++) { 483 free(iov[i].iov_base); 484 } 485 } 486 487 static void 488 test_spdk_accel_submit_copy_crc32c(void) 489 { 490 const uint64_t nbytes = TEST_SUBMIT_SIZE; 491 uint32_t crc_dst; 492 uint8_t dst[TEST_SUBMIT_SIZE]; 493 uint8_t src[TEST_SUBMIT_SIZE]; 494 uint32_t seed = 0; 495 void *cb_arg = NULL; 496 int rc; 497 struct spdk_accel_task task; 498 struct spdk_accel_task_aux_data task_aux; 499 struct spdk_accel_task *expected_accel_task = NULL; 500 int flags = 0; 501 502 STAILQ_INIT(&g_accel_ch->task_pool); 503 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 504 505 /* Fail with no tasks on _get_task() */ 506 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 507 NULL, cb_arg); 508 CU_ASSERT(rc == -ENOMEM); 509 510 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 511 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 512 513 /* accel submission OK. */ 514 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, flags, 515 NULL, cb_arg); 516 CU_ASSERT(rc == 0); 517 CU_ASSERT(task.crc_dst == &crc_dst); 518 CU_ASSERT(task.seed == seed); 519 CU_ASSERT(task.flags == 0); 520 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COPY_CRC32C); 521 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 522 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 523 CU_ASSERT(expected_accel_task == &task); 524 } 525 526 static void 527 test_spdk_accel_submit_xor(void) 528 { 529 const uint64_t nbytes = TEST_SUBMIT_SIZE; 530 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 531 uint8_t src1[TEST_SUBMIT_SIZE] = {0}; 532 uint8_t src2[TEST_SUBMIT_SIZE] = {0}; 533 void *sources[] = { src1, src2 }; 534 uint32_t nsrcs = SPDK_COUNTOF(sources); 535 int rc; 536 struct spdk_accel_task task; 537 struct spdk_accel_task_aux_data task_aux; 538 struct spdk_accel_task *expected_accel_task = NULL; 539 540 STAILQ_INIT(&g_accel_ch->task_pool); 541 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 542 543 /* Fail with no tasks on _get_task() */ 544 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 545 CU_ASSERT(rc == -ENOMEM); 546 547 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 548 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 549 550 /* submission OK. */ 551 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 552 CU_ASSERT(rc == 0); 553 CU_ASSERT(task.nsrcs.srcs == sources); 554 CU_ASSERT(task.nsrcs.cnt == nsrcs); 555 CU_ASSERT(task.d.iovcnt == 1); 556 CU_ASSERT(task.d.iovs[0].iov_base == dst); 557 CU_ASSERT(task.d.iovs[0].iov_len == nbytes); 558 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_XOR); 559 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 560 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 561 CU_ASSERT(expected_accel_task == &task); 562 } 563 564 static void 565 test_spdk_accel_module_find_by_name(void) 566 { 567 struct spdk_accel_module_if mod1 = {}; 568 struct spdk_accel_module_if mod2 = {}; 569 struct spdk_accel_module_if mod3 = {}; 570 struct spdk_accel_module_if *accel_module = NULL; 571 572 mod1.name = "ioat"; 573 mod2.name = "idxd"; 574 mod3.name = "software"; 575 576 TAILQ_INIT(&spdk_accel_module_list); 577 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod1, tailq); 578 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod2, tailq); 579 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod3, tailq); 580 581 /* Now let's find a valid engine */ 582 accel_module = _module_find_by_name("ioat"); 583 CU_ASSERT(accel_module != NULL); 584 585 /* Try to find one that doesn't exist */ 586 accel_module = _module_find_by_name("XXX"); 587 CU_ASSERT(accel_module == NULL); 588 } 589 590 static int 591 ut_module_init_nop(void) 592 { 593 return 0; 594 } 595 596 static bool 597 ut_supports_opcode_all(enum spdk_accel_opcode opcode) 598 { 599 return true; 600 } 601 602 static void 603 ut_accel_module_priority_finish_done(void *done) 604 { 605 *(int *)done = 1; 606 } 607 608 static void 609 test_spdk_accel_module_register(void) 610 { 611 struct spdk_accel_module_if mods[] = { 612 { .name = "mod1", .priority = 1, }, 613 { .name = "mod3", .priority = 3, }, 614 { .name = "mod0", .priority = 0, }, 615 { .name = "mod2", .priority = 2, }, 616 }; 617 int rc, done = 0; 618 const char *modname = NULL; 619 size_t i; 620 621 allocate_cores(1); 622 allocate_threads(1); 623 set_thread(0); 624 625 TAILQ_INIT(&spdk_accel_module_list); 626 for (i = 0; i < SPDK_COUNTOF(mods); ++i) { 627 mods[i].module_init = ut_module_init_nop; 628 mods[i].supports_opcode = ut_supports_opcode_all; 629 spdk_accel_module_list_add(&mods[i]); 630 } 631 632 rc = spdk_accel_initialize(); 633 CU_ASSERT_EQUAL(rc, 0); 634 635 /* All opcodes should be assigned to the module with highest prio - mod3 */ 636 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 637 rc = spdk_accel_get_opc_module_name((enum spdk_accel_opcode)i, &modname); 638 CU_ASSERT_EQUAL(rc, 0); 639 CU_ASSERT_STRING_EQUAL(modname, "mod3"); 640 } 641 642 spdk_accel_finish(ut_accel_module_priority_finish_done, &done); 643 while (!done) { 644 poll_threads(); 645 } 646 647 TAILQ_INIT(&spdk_accel_module_list); 648 free_threads(); 649 free_cores(); 650 } 651 652 struct ut_sequence { 653 bool complete; 654 int status; 655 }; 656 657 static void 658 ut_sequence_step_cb(void *cb_arg) 659 { 660 int *completed = cb_arg; 661 662 (*completed)++; 663 } 664 665 static void 666 ut_sequence_complete_cb(void *cb_arg, int status) 667 { 668 struct ut_sequence *seq = cb_arg; 669 670 seq->complete = true; 671 seq->status = status; 672 } 673 674 static void 675 test_sequence_fill_copy(void) 676 { 677 struct spdk_accel_sequence *seq = NULL; 678 struct spdk_io_channel *ioch; 679 struct ut_sequence ut_seq; 680 char buf[4096], tmp[2][4096], expected[4096]; 681 struct iovec src_iovs[2], dst_iovs[2]; 682 int rc, completed; 683 684 ioch = spdk_accel_get_io_channel(); 685 SPDK_CU_ASSERT_FATAL(ioch != NULL); 686 687 /* First check the simplest case - single task in a sequence */ 688 memset(buf, 0, sizeof(buf)); 689 memset(expected, 0xa5, sizeof(expected)); 690 completed = 0; 691 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 692 ut_sequence_step_cb, &completed); 693 CU_ASSERT_EQUAL(rc, 0); 694 CU_ASSERT_EQUAL(completed, 0); 695 696 ut_seq.complete = false; 697 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 698 699 poll_threads(); 700 CU_ASSERT_EQUAL(completed, 1); 701 CU_ASSERT(ut_seq.complete); 702 CU_ASSERT_EQUAL(ut_seq.status, 0); 703 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 704 705 /* Check a single copy operation */ 706 memset(buf, 0, sizeof(buf)); 707 memset(tmp[0], 0xa5, sizeof(tmp[0])); 708 memset(expected, 0xa5, sizeof(expected)); 709 completed = 0; 710 seq = NULL; 711 712 dst_iovs[0].iov_base = buf; 713 dst_iovs[0].iov_len = sizeof(buf); 714 src_iovs[0].iov_base = tmp[0]; 715 src_iovs[0].iov_len = sizeof(tmp[0]); 716 717 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 718 &src_iovs[0], 1, NULL, NULL, 0, 719 ut_sequence_step_cb, &completed); 720 CU_ASSERT_EQUAL(rc, 0); 721 722 ut_seq.complete = false; 723 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 724 725 poll_threads(); 726 CU_ASSERT_EQUAL(completed, 1); 727 CU_ASSERT(ut_seq.complete); 728 CU_ASSERT_EQUAL(ut_seq.status, 0); 729 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 730 731 /* Check multiple fill operations */ 732 memset(buf, 0, sizeof(buf)); 733 memset(expected, 0xfe, 4096); 734 memset(expected, 0xde, 2048); 735 memset(expected, 0xa5, 1024); 736 seq = NULL; 737 completed = 0; 738 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, NULL, NULL, 0xfe, 0, 739 ut_sequence_step_cb, &completed); 740 CU_ASSERT_EQUAL(rc, 0); 741 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xde, 0, 742 ut_sequence_step_cb, &completed); 743 CU_ASSERT_EQUAL(rc, 0); 744 rc = spdk_accel_append_fill(&seq, ioch, buf, 1024, NULL, NULL, 0xa5, 0, 745 ut_sequence_step_cb, &completed); 746 CU_ASSERT_EQUAL(rc, 0); 747 748 ut_seq.complete = false; 749 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 750 751 poll_threads(); 752 CU_ASSERT_EQUAL(completed, 3); 753 CU_ASSERT(ut_seq.complete); 754 CU_ASSERT_EQUAL(ut_seq.status, 0); 755 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 756 757 /* Check multiple copy operations */ 758 memset(buf, 0, sizeof(buf)); 759 memset(tmp[0], 0, sizeof(tmp[0])); 760 memset(tmp[1], 0, sizeof(tmp[1])); 761 memset(expected, 0xa5, sizeof(expected)); 762 seq = NULL; 763 completed = 0; 764 765 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 766 ut_sequence_step_cb, &completed); 767 CU_ASSERT_EQUAL(rc, 0); 768 769 dst_iovs[0].iov_base = tmp[1]; 770 dst_iovs[0].iov_len = sizeof(tmp[1]); 771 src_iovs[0].iov_base = tmp[0]; 772 src_iovs[0].iov_len = sizeof(tmp[0]); 773 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 774 &src_iovs[0], 1, NULL, NULL, 0, 775 ut_sequence_step_cb, &completed); 776 CU_ASSERT_EQUAL(rc, 0); 777 778 dst_iovs[1].iov_base = buf; 779 dst_iovs[1].iov_len = sizeof(buf); 780 src_iovs[1].iov_base = tmp[1]; 781 src_iovs[1].iov_len = sizeof(tmp[1]); 782 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 783 &src_iovs[1], 1, NULL, NULL, 0, 784 ut_sequence_step_cb, &completed); 785 CU_ASSERT_EQUAL(rc, 0); 786 787 ut_seq.complete = false; 788 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 789 790 poll_threads(); 791 CU_ASSERT_EQUAL(completed, 3); 792 CU_ASSERT(ut_seq.complete); 793 CU_ASSERT_EQUAL(ut_seq.status, 0); 794 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 795 796 /* Check that adding a copy operation at the end will change destination buffer */ 797 memset(buf, 0, sizeof(buf)); 798 memset(tmp[0], 0, sizeof(tmp[0])); 799 memset(expected, 0xa5, sizeof(buf)); 800 seq = NULL; 801 completed = 0; 802 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 803 ut_sequence_step_cb, &completed); 804 CU_ASSERT_EQUAL(rc, 0); 805 806 dst_iovs[0].iov_base = buf; 807 dst_iovs[0].iov_len = sizeof(buf); 808 src_iovs[0].iov_base = tmp[0]; 809 src_iovs[0].iov_len = sizeof(tmp[0]); 810 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 811 &src_iovs[0], 1, NULL, NULL, 0, 812 ut_sequence_step_cb, &completed); 813 CU_ASSERT_EQUAL(rc, 0); 814 815 ut_seq.complete = false; 816 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 817 818 poll_threads(); 819 CU_ASSERT_EQUAL(completed, 2); 820 CU_ASSERT(ut_seq.complete); 821 CU_ASSERT_EQUAL(ut_seq.status, 0); 822 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 823 824 /* Check that it's also possible to add copy operation at the beginning */ 825 memset(buf, 0, sizeof(buf)); 826 memset(tmp[0], 0xde, sizeof(tmp[0])); 827 memset(tmp[1], 0, sizeof(tmp[1])); 828 memset(expected, 0xa5, sizeof(expected)); 829 seq = NULL; 830 completed = 0; 831 832 dst_iovs[0].iov_base = tmp[1]; 833 dst_iovs[0].iov_len = sizeof(tmp[1]); 834 src_iovs[0].iov_base = tmp[0]; 835 src_iovs[0].iov_len = sizeof(tmp[0]); 836 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 837 &src_iovs[0], 1, NULL, NULL, 0, 838 ut_sequence_step_cb, &completed); 839 CU_ASSERT_EQUAL(rc, 0); 840 841 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], sizeof(tmp[1]), NULL, NULL, 0xa5, 0, 842 ut_sequence_step_cb, &completed); 843 CU_ASSERT_EQUAL(rc, 0); 844 845 dst_iovs[1].iov_base = buf; 846 dst_iovs[1].iov_len = sizeof(buf); 847 src_iovs[1].iov_base = tmp[1]; 848 src_iovs[1].iov_len = sizeof(tmp[1]); 849 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 850 &src_iovs[1], 1, NULL, NULL, 0, 851 ut_sequence_step_cb, &completed); 852 CU_ASSERT_EQUAL(rc, 0); 853 854 ut_seq.complete = false; 855 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 856 857 poll_threads(); 858 CU_ASSERT_EQUAL(completed, 3); 859 CU_ASSERT(ut_seq.complete); 860 CU_ASSERT_EQUAL(ut_seq.status, 0); 861 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 862 863 spdk_put_io_channel(ioch); 864 poll_threads(); 865 } 866 867 static void 868 test_sequence_abort(void) 869 { 870 struct spdk_accel_sequence *seq = NULL; 871 struct spdk_io_channel *ioch; 872 char buf[4096], tmp[2][4096], expected[4096]; 873 struct iovec src_iovs[2], dst_iovs[2]; 874 int rc, completed; 875 876 ioch = spdk_accel_get_io_channel(); 877 SPDK_CU_ASSERT_FATAL(ioch != NULL); 878 879 /* Check that aborting a sequence calls operation's callback, the operation is not executed 880 * and the sequence is freed 881 */ 882 memset(buf, 0, sizeof(buf)); 883 memset(expected, 0, sizeof(buf)); 884 completed = 0; 885 seq = NULL; 886 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 887 ut_sequence_step_cb, &completed); 888 CU_ASSERT_EQUAL(rc, 0); 889 890 spdk_accel_sequence_abort(seq); 891 CU_ASSERT_EQUAL(completed, 1); 892 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 893 894 /* Check sequence with multiple operations */ 895 memset(buf, 0, sizeof(buf)); 896 memset(expected, 0, sizeof(buf)); 897 completed = 0; 898 seq = NULL; 899 900 dst_iovs[0].iov_base = tmp[1]; 901 dst_iovs[0].iov_len = sizeof(tmp[1]); 902 src_iovs[0].iov_base = tmp[0]; 903 src_iovs[0].iov_len = sizeof(tmp[0]); 904 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 905 &src_iovs[0], 1, NULL, NULL, 0, 906 ut_sequence_step_cb, &completed); 907 CU_ASSERT_EQUAL(rc, 0); 908 909 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 4096, NULL, NULL, 0xa5, 0, 910 ut_sequence_step_cb, &completed); 911 CU_ASSERT_EQUAL(rc, 0); 912 913 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xde, 0, 914 ut_sequence_step_cb, &completed); 915 CU_ASSERT_EQUAL(rc, 0); 916 917 dst_iovs[1].iov_base = buf; 918 dst_iovs[1].iov_len = sizeof(buf); 919 src_iovs[1].iov_base = tmp[1]; 920 src_iovs[1].iov_len = sizeof(tmp[1]); 921 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 922 &src_iovs[1], 1, NULL, NULL, 0, 923 ut_sequence_step_cb, &completed); 924 CU_ASSERT_EQUAL(rc, 0); 925 926 spdk_accel_sequence_abort(seq); 927 CU_ASSERT_EQUAL(completed, 4); 928 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 929 930 /* This should be a no-op */ 931 spdk_accel_sequence_abort(NULL); 932 933 spdk_put_io_channel(ioch); 934 poll_threads(); 935 } 936 937 static void 938 test_sequence_append_error(void) 939 { 940 struct spdk_accel_sequence *seq = NULL; 941 struct spdk_io_channel *ioch; 942 struct accel_io_channel *accel_ch; 943 struct iovec src_iovs, dst_iovs; 944 char buf[4096]; 945 STAILQ_HEAD(, spdk_accel_task) tasks = STAILQ_HEAD_INITIALIZER(tasks); 946 SLIST_HEAD(, spdk_accel_sequence) seqs = SLIST_HEAD_INITIALIZER(seqs); 947 int rc; 948 949 ioch = spdk_accel_get_io_channel(); 950 SPDK_CU_ASSERT_FATAL(ioch != NULL); 951 accel_ch = spdk_io_channel_get_ctx(ioch); 952 953 /* Check that append fails and no sequence object is allocated when there are no more free 954 * tasks */ 955 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 956 957 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 958 ut_sequence_step_cb, NULL); 959 CU_ASSERT_EQUAL(rc, -ENOMEM); 960 CU_ASSERT_PTR_NULL(seq); 961 962 dst_iovs.iov_base = buf; 963 dst_iovs.iov_len = 2048; 964 src_iovs.iov_base = &buf[2048]; 965 src_iovs.iov_len = 2048; 966 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 967 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 968 CU_ASSERT_EQUAL(rc, -ENOMEM); 969 CU_ASSERT_PTR_NULL(seq); 970 971 dst_iovs.iov_base = buf; 972 dst_iovs.iov_len = 2048; 973 src_iovs.iov_base = &buf[2048]; 974 src_iovs.iov_len = 2048; 975 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 976 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 977 CU_ASSERT_EQUAL(rc, -ENOMEM); 978 CU_ASSERT_PTR_NULL(seq); 979 980 /* Check that the same happens when the sequence queue is empty */ 981 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 982 SLIST_SWAP(&seqs, &accel_ch->seq_pool, spdk_accel_sequence); 983 984 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 0, 985 ut_sequence_step_cb, NULL); 986 CU_ASSERT_EQUAL(rc, -ENOMEM); 987 CU_ASSERT_PTR_NULL(seq); 988 989 dst_iovs.iov_base = buf; 990 dst_iovs.iov_len = 2048; 991 src_iovs.iov_base = &buf[2048]; 992 src_iovs.iov_len = 2048; 993 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 994 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 995 CU_ASSERT_EQUAL(rc, -ENOMEM); 996 CU_ASSERT_PTR_NULL(seq); 997 998 dst_iovs.iov_base = buf; 999 dst_iovs.iov_len = 2048; 1000 src_iovs.iov_base = &buf[2048]; 1001 src_iovs.iov_len = 2048; 1002 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1003 &src_iovs, 1, NULL, NULL, 0, ut_sequence_step_cb, NULL); 1004 CU_ASSERT_EQUAL(rc, -ENOMEM); 1005 CU_ASSERT_PTR_NULL(seq); 1006 1007 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 1008 1009 spdk_put_io_channel(ioch); 1010 poll_threads(); 1011 } 1012 1013 struct ut_sequence_operation { 1014 int complete_status; 1015 int submit_status; 1016 int count; 1017 struct iovec *src_iovs; 1018 uint32_t src_iovcnt; 1019 struct spdk_memory_domain *src_domain; 1020 void *src_domain_ctx; 1021 struct iovec *dst_iovs; 1022 uint32_t dst_iovcnt; 1023 struct spdk_memory_domain *dst_domain; 1024 void *dst_domain_ctx; 1025 int (*submit)(struct spdk_io_channel *ch, struct spdk_accel_task *t); 1026 }; 1027 1028 static struct ut_sequence_operation g_seq_operations[SPDK_ACCEL_OPC_LAST]; 1029 1030 static int 1031 ut_sequnce_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *task) 1032 { 1033 struct ut_sequence_operation *op = &g_seq_operations[task->op_code]; 1034 1035 op->count++; 1036 if (op->submit != NULL) { 1037 return op->submit(ch, task); 1038 } 1039 if (op->src_iovs != NULL) { 1040 CU_ASSERT_EQUAL(task->s.iovcnt, op->src_iovcnt); 1041 CU_ASSERT_EQUAL(memcmp(task->s.iovs, op->src_iovs, 1042 sizeof(struct iovec) * op->src_iovcnt), 0); 1043 } 1044 if (op->dst_iovs != NULL) { 1045 CU_ASSERT_EQUAL(task->d.iovcnt, op->dst_iovcnt); 1046 CU_ASSERT_EQUAL(memcmp(task->d.iovs, op->dst_iovs, 1047 sizeof(struct iovec) * op->dst_iovcnt), 0); 1048 } 1049 1050 CU_ASSERT_EQUAL(task->src_domain, op->src_domain); 1051 CU_ASSERT_EQUAL(task->dst_domain, op->dst_domain); 1052 if (op->src_domain != NULL) { 1053 CU_ASSERT_EQUAL(task->src_domain_ctx, op->src_domain_ctx); 1054 } 1055 if (op->dst_domain != NULL) { 1056 CU_ASSERT_EQUAL(task->dst_domain_ctx, op->dst_domain_ctx); 1057 } 1058 1059 if (op->submit_status != 0) { 1060 return op->submit_status; 1061 } 1062 1063 spdk_accel_task_complete(task, op->complete_status); 1064 1065 return 0; 1066 } 1067 1068 static void 1069 ut_clear_operations(void) 1070 { 1071 memset(&g_seq_operations, 0, sizeof(g_seq_operations)); 1072 } 1073 1074 static void 1075 test_sequence_completion_error(void) 1076 { 1077 struct spdk_accel_sequence *seq = NULL; 1078 struct spdk_io_channel *ioch; 1079 struct ut_sequence ut_seq; 1080 struct iovec src_iovs, dst_iovs; 1081 char buf[4096], tmp[4096]; 1082 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 1083 int i, rc, completed; 1084 1085 ioch = spdk_accel_get_io_channel(); 1086 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1087 1088 /* Override the submit_tasks function */ 1089 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1090 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1091 modules[i] = g_modules_opc[i]; 1092 g_modules_opc[i] = g_module; 1093 } 1094 1095 memset(buf, 0, sizeof(buf)); 1096 memset(tmp, 0, sizeof(tmp)); 1097 1098 /* Check that if the first operation completes with an error, the whole sequence is 1099 * completed with that error and that all operations' completion callbacks are executed 1100 */ 1101 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = -E2BIG; 1102 completed = 0; 1103 seq = NULL; 1104 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1105 ut_sequence_step_cb, &completed); 1106 CU_ASSERT_EQUAL(rc, 0); 1107 1108 dst_iovs.iov_base = buf; 1109 dst_iovs.iov_len = sizeof(buf); 1110 src_iovs.iov_base = tmp; 1111 src_iovs.iov_len = sizeof(tmp); 1112 1113 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1114 &src_iovs, 1, NULL, NULL, 0, 1115 ut_sequence_step_cb, &completed); 1116 CU_ASSERT_EQUAL(rc, 0); 1117 1118 ut_seq.complete = false; 1119 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1120 1121 poll_threads(); 1122 CU_ASSERT_EQUAL(completed, 2); 1123 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 1124 1125 /* Check the same with a second operation in the sequence */ 1126 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].complete_status = -EACCES; 1127 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 1128 completed = 0; 1129 seq = NULL; 1130 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1131 ut_sequence_step_cb, &completed); 1132 CU_ASSERT_EQUAL(rc, 0); 1133 1134 dst_iovs.iov_base = buf; 1135 dst_iovs.iov_len = sizeof(buf); 1136 src_iovs.iov_base = tmp; 1137 src_iovs.iov_len = sizeof(tmp); 1138 1139 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1140 &src_iovs, 1, NULL, NULL, 0, 1141 ut_sequence_step_cb, &completed); 1142 CU_ASSERT_EQUAL(rc, 0); 1143 1144 ut_seq.complete = false; 1145 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1146 1147 poll_threads(); 1148 CU_ASSERT_EQUAL(completed, 2); 1149 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 1150 1151 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].complete_status = 0; 1152 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 1153 1154 /* Check submission failure of the first operation */ 1155 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit_status = -EADDRINUSE; 1156 completed = 0; 1157 seq = NULL; 1158 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1159 ut_sequence_step_cb, &completed); 1160 CU_ASSERT_EQUAL(rc, 0); 1161 1162 dst_iovs.iov_base = buf; 1163 dst_iovs.iov_len = sizeof(buf); 1164 src_iovs.iov_base = tmp; 1165 src_iovs.iov_len = sizeof(tmp); 1166 1167 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1168 &src_iovs, 1, NULL, NULL, 0, 1169 ut_sequence_step_cb, &completed); 1170 CU_ASSERT_EQUAL(rc, 0); 1171 1172 ut_seq.complete = false; 1173 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1174 1175 poll_threads(); 1176 CU_ASSERT_EQUAL(completed, 2); 1177 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 1178 1179 /* Check the same with a second operation */ 1180 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit_status = -EADDRNOTAVAIL; 1181 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit_status = 0; 1182 completed = 0; 1183 seq = NULL; 1184 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 0, 1185 ut_sequence_step_cb, &completed); 1186 CU_ASSERT_EQUAL(rc, 0); 1187 1188 dst_iovs.iov_base = buf; 1189 dst_iovs.iov_len = sizeof(buf); 1190 src_iovs.iov_base = tmp; 1191 src_iovs.iov_len = sizeof(tmp); 1192 1193 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1194 &src_iovs, 1, NULL, NULL, 0, 1195 ut_sequence_step_cb, &completed); 1196 CU_ASSERT_EQUAL(rc, 0); 1197 1198 ut_seq.complete = false; 1199 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1200 1201 poll_threads(); 1202 CU_ASSERT_EQUAL(completed, 2); 1203 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 1204 1205 /* Cleanup module pointers to make subsequent tests work correctly */ 1206 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1207 g_modules_opc[i] = modules[i]; 1208 } 1209 1210 ut_clear_operations(); 1211 spdk_put_io_channel(ioch); 1212 poll_threads(); 1213 } 1214 1215 #ifdef SPDK_CONFIG_ISAL 1216 static void 1217 ut_compress_cb(void *cb_arg, int status) 1218 { 1219 int *completed = cb_arg; 1220 1221 CU_ASSERT_EQUAL(status, 0); 1222 1223 *completed = 1; 1224 } 1225 1226 static void 1227 test_sequence_decompress(void) 1228 { 1229 struct spdk_accel_sequence *seq = NULL; 1230 struct spdk_io_channel *ioch; 1231 struct ut_sequence ut_seq; 1232 char buf[4096], tmp[2][4096], expected[4096]; 1233 struct iovec src_iovs[2], dst_iovs[2]; 1234 uint32_t compressed_size; 1235 int rc, completed = 0; 1236 1237 ioch = spdk_accel_get_io_channel(); 1238 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1239 1240 memset(expected, 0xa5, sizeof(expected)); 1241 src_iovs[0].iov_base = expected; 1242 src_iovs[0].iov_len = sizeof(expected); 1243 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1244 &compressed_size, 0, ut_compress_cb, &completed); 1245 CU_ASSERT_EQUAL(rc, 0); 1246 1247 while (!completed) { 1248 poll_threads(); 1249 } 1250 1251 /* Check a single decompress operation in a sequence */ 1252 seq = NULL; 1253 completed = 0; 1254 1255 dst_iovs[0].iov_base = buf; 1256 dst_iovs[0].iov_len = sizeof(buf); 1257 src_iovs[0].iov_base = tmp[0]; 1258 src_iovs[0].iov_len = compressed_size; 1259 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1260 &src_iovs[0], 1, NULL, NULL, 0, 1261 ut_sequence_step_cb, &completed); 1262 CU_ASSERT_EQUAL(rc, 0); 1263 1264 ut_seq.complete = false; 1265 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1266 1267 poll_threads(); 1268 1269 CU_ASSERT_EQUAL(completed, 1); 1270 CU_ASSERT(ut_seq.complete); 1271 CU_ASSERT_EQUAL(ut_seq.status, 0); 1272 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1273 1274 /* Put the decompress operation in the middle of a sequence with a copy operation at the 1275 * beginning and a fill at the end modifying the first 2048B of the buffer. 1276 */ 1277 memset(expected, 0xfe, 2048); 1278 memset(buf, 0, sizeof(buf)); 1279 seq = NULL; 1280 completed = 0; 1281 1282 dst_iovs[0].iov_base = tmp[1]; 1283 dst_iovs[0].iov_len = compressed_size; 1284 src_iovs[0].iov_base = tmp[0]; 1285 src_iovs[0].iov_len = compressed_size; 1286 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1287 &src_iovs[0], 1, NULL, NULL, 0, 1288 ut_sequence_step_cb, &completed); 1289 CU_ASSERT_EQUAL(rc, 0); 1290 1291 dst_iovs[1].iov_base = buf; 1292 dst_iovs[1].iov_len = sizeof(buf); 1293 src_iovs[1].iov_base = tmp[1]; 1294 src_iovs[1].iov_len = compressed_size; 1295 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1296 &src_iovs[1], 1, NULL, NULL, 0, 1297 ut_sequence_step_cb, &completed); 1298 CU_ASSERT_EQUAL(rc, 0); 1299 1300 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1301 ut_sequence_step_cb, &completed); 1302 CU_ASSERT_EQUAL(rc, 0); 1303 1304 ut_seq.complete = false; 1305 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1306 1307 poll_threads(); 1308 1309 CU_ASSERT_EQUAL(completed, 3); 1310 CU_ASSERT(ut_seq.complete); 1311 CU_ASSERT_EQUAL(ut_seq.status, 0); 1312 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1313 1314 /* Check sequence with decompress at the beginning: decompress -> copy */ 1315 memset(expected, 0xa5, sizeof(expected)); 1316 memset(buf, 0, sizeof(buf)); 1317 seq = NULL; 1318 completed = 0; 1319 1320 dst_iovs[0].iov_base = tmp[1]; 1321 dst_iovs[0].iov_len = sizeof(tmp[1]); 1322 src_iovs[0].iov_base = tmp[0]; 1323 src_iovs[0].iov_len = compressed_size; 1324 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1325 &src_iovs[0], 1, NULL, NULL, 0, 1326 ut_sequence_step_cb, &completed); 1327 CU_ASSERT_EQUAL(rc, 0); 1328 1329 dst_iovs[1].iov_base = buf; 1330 dst_iovs[1].iov_len = sizeof(buf); 1331 src_iovs[1].iov_base = tmp[1]; 1332 src_iovs[1].iov_len = sizeof(tmp[1]); 1333 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1334 &src_iovs[1], 1, NULL, NULL, 0, 1335 ut_sequence_step_cb, &completed); 1336 CU_ASSERT_EQUAL(rc, 0); 1337 1338 ut_seq.complete = false; 1339 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1340 1341 poll_threads(); 1342 1343 CU_ASSERT_EQUAL(completed, 2); 1344 CU_ASSERT(ut_seq.complete); 1345 CU_ASSERT_EQUAL(ut_seq.status, 0); 1346 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1347 1348 spdk_put_io_channel(ioch); 1349 poll_threads(); 1350 } 1351 1352 static void 1353 test_sequence_reverse(void) 1354 { 1355 struct spdk_accel_sequence *seq = NULL; 1356 struct spdk_io_channel *ioch; 1357 struct ut_sequence ut_seq; 1358 char buf[4096], tmp[2][4096], expected[4096]; 1359 struct iovec src_iovs[2], dst_iovs[2]; 1360 uint32_t compressed_size; 1361 int rc, completed = 0; 1362 1363 ioch = spdk_accel_get_io_channel(); 1364 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1365 1366 memset(expected, 0xa5, sizeof(expected)); 1367 src_iovs[0].iov_base = expected; 1368 src_iovs[0].iov_len = sizeof(expected); 1369 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1370 &compressed_size, 0, ut_compress_cb, &completed); 1371 CU_ASSERT_EQUAL(rc, 0); 1372 1373 while (!completed) { 1374 poll_threads(); 1375 } 1376 1377 /* First check that reversing a sequnce with a single operation is a no-op */ 1378 memset(buf, 0, sizeof(buf)); 1379 seq = NULL; 1380 completed = 0; 1381 1382 dst_iovs[0].iov_base = buf; 1383 dst_iovs[0].iov_len = sizeof(buf); 1384 src_iovs[0].iov_base = tmp[0]; 1385 src_iovs[0].iov_len = compressed_size; 1386 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1387 &src_iovs[0], 1, NULL, NULL, 0, 1388 ut_sequence_step_cb, &completed); 1389 CU_ASSERT_EQUAL(rc, 0); 1390 1391 spdk_accel_sequence_reverse(seq); 1392 1393 ut_seq.complete = false; 1394 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1395 1396 poll_threads(); 1397 1398 CU_ASSERT_EQUAL(completed, 1); 1399 CU_ASSERT(ut_seq.complete); 1400 CU_ASSERT_EQUAL(ut_seq.status, 0); 1401 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1402 1403 /* Add a copy operation at the end with src set to the compressed data. After reverse(), 1404 * that copy operation should be first, so decompress() should receive compressed data in 1405 * its src buffer. 1406 */ 1407 memset(buf, 0, sizeof(buf)); 1408 memset(tmp[1], 0, sizeof(tmp[1])); 1409 seq = NULL; 1410 completed = 0; 1411 1412 dst_iovs[0].iov_base = buf; 1413 dst_iovs[0].iov_len = sizeof(buf); 1414 src_iovs[0].iov_base = tmp[1]; 1415 src_iovs[0].iov_len = compressed_size; 1416 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1417 &src_iovs[0], 1, NULL, NULL, 0, 1418 ut_sequence_step_cb, &completed); 1419 CU_ASSERT_EQUAL(rc, 0); 1420 1421 dst_iovs[1].iov_base = tmp[1]; 1422 dst_iovs[1].iov_len = compressed_size; 1423 src_iovs[1].iov_base = tmp[0]; 1424 src_iovs[1].iov_len = compressed_size; 1425 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1426 &src_iovs[1], 1, NULL, NULL, 0, 1427 ut_sequence_step_cb, &completed); 1428 CU_ASSERT_EQUAL(rc, 0); 1429 1430 spdk_accel_sequence_reverse(seq); 1431 1432 ut_seq.complete = false; 1433 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1434 1435 poll_threads(); 1436 1437 CU_ASSERT_EQUAL(completed, 2); 1438 CU_ASSERT(ut_seq.complete); 1439 CU_ASSERT_EQUAL(ut_seq.status, 0); 1440 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1441 1442 /* Check the same, but add an extra fill operation at the beginning that should execute last 1443 * after reverse(). 1444 */ 1445 memset(buf, 0, sizeof(buf)); 1446 memset(tmp[1], 0, sizeof(tmp[1])); 1447 memset(expected, 0xfe, 2048); 1448 seq = NULL; 1449 completed = 0; 1450 1451 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1452 ut_sequence_step_cb, &completed); 1453 CU_ASSERT_EQUAL(rc, 0); 1454 1455 dst_iovs[0].iov_base = buf; 1456 dst_iovs[0].iov_len = sizeof(buf); 1457 src_iovs[0].iov_base = tmp[1]; 1458 src_iovs[0].iov_len = compressed_size; 1459 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1460 &src_iovs[0], 1, NULL, NULL, 0, 1461 ut_sequence_step_cb, &completed); 1462 CU_ASSERT_EQUAL(rc, 0); 1463 1464 dst_iovs[1].iov_base = tmp[1]; 1465 dst_iovs[1].iov_len = compressed_size; 1466 src_iovs[1].iov_base = tmp[0]; 1467 src_iovs[1].iov_len = compressed_size; 1468 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1469 &src_iovs[1], 1, NULL, NULL, 0, 1470 ut_sequence_step_cb, &completed); 1471 CU_ASSERT_EQUAL(rc, 0); 1472 1473 spdk_accel_sequence_reverse(seq); 1474 1475 ut_seq.complete = false; 1476 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1477 1478 poll_threads(); 1479 1480 CU_ASSERT_EQUAL(completed, 3); 1481 CU_ASSERT(ut_seq.complete); 1482 CU_ASSERT_EQUAL(ut_seq.status, 0); 1483 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1484 1485 /* Build the sequence in order and then reverse it twice */ 1486 memset(buf, 0, sizeof(buf)); 1487 memset(tmp[1], 0, sizeof(tmp[1])); 1488 seq = NULL; 1489 completed = 0; 1490 1491 dst_iovs[0].iov_base = tmp[1]; 1492 dst_iovs[0].iov_len = compressed_size; 1493 src_iovs[0].iov_base = tmp[0]; 1494 src_iovs[0].iov_len = compressed_size; 1495 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1496 &src_iovs[0], 1, NULL, NULL, 0, 1497 ut_sequence_step_cb, &completed); 1498 CU_ASSERT_EQUAL(rc, 0); 1499 1500 dst_iovs[1].iov_base = buf; 1501 dst_iovs[1].iov_len = sizeof(buf); 1502 src_iovs[1].iov_base = tmp[1]; 1503 src_iovs[1].iov_len = compressed_size; 1504 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1505 &src_iovs[1], 1, NULL, NULL, 0, 1506 ut_sequence_step_cb, &completed); 1507 CU_ASSERT_EQUAL(rc, 0); 1508 1509 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 0, 1510 ut_sequence_step_cb, &completed); 1511 CU_ASSERT_EQUAL(rc, 0); 1512 1513 spdk_accel_sequence_reverse(seq); 1514 spdk_accel_sequence_reverse(seq); 1515 1516 ut_seq.complete = false; 1517 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1518 1519 poll_threads(); 1520 1521 CU_ASSERT_EQUAL(completed, 3); 1522 CU_ASSERT(ut_seq.complete); 1523 CU_ASSERT_EQUAL(ut_seq.status, 0); 1524 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1525 1526 spdk_put_io_channel(ioch); 1527 poll_threads(); 1528 } 1529 #endif 1530 1531 static void 1532 test_sequence_copy_elision(void) 1533 { 1534 struct spdk_accel_sequence *seq = NULL; 1535 struct spdk_io_channel *ioch; 1536 struct ut_sequence ut_seq; 1537 struct iovec src_iovs[4], dst_iovs[4], exp_iovs[2]; 1538 char buf[4096], tmp[4][4096]; 1539 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 1540 struct spdk_accel_crypto_key key = {}; 1541 int i, rc, completed; 1542 1543 ioch = spdk_accel_get_io_channel(); 1544 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1545 1546 /* Override the submit_tasks function */ 1547 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 1548 g_module.supports_memory_domains = true; 1549 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1550 g_seq_operations[i].complete_status = 0; 1551 g_seq_operations[i].submit_status = 0; 1552 g_seq_operations[i].count = 0; 1553 1554 modules[i] = g_modules_opc[i]; 1555 g_modules_opc[i] = g_module; 1556 } 1557 1558 /* Check that a copy operation at the beginning is removed */ 1559 seq = NULL; 1560 completed = 0; 1561 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovcnt = 1; 1562 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovcnt = 1; 1563 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1564 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1565 exp_iovs[0].iov_base = tmp[0]; 1566 exp_iovs[0].iov_len = sizeof(tmp[0]); 1567 exp_iovs[1].iov_base = buf; 1568 exp_iovs[1].iov_len = 2048; 1569 1570 dst_iovs[0].iov_base = tmp[1]; 1571 dst_iovs[0].iov_len = sizeof(tmp[1]); 1572 src_iovs[0].iov_base = tmp[0]; 1573 src_iovs[0].iov_len = sizeof(tmp[0]); 1574 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1575 &src_iovs[0], 1, NULL, NULL, 0, 1576 ut_sequence_step_cb, &completed); 1577 CU_ASSERT_EQUAL(rc, 0); 1578 1579 dst_iovs[1].iov_base = buf; 1580 dst_iovs[1].iov_len = 2048; 1581 src_iovs[1].iov_base = tmp[1]; 1582 src_iovs[1].iov_len = sizeof(tmp[1]); 1583 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1584 &src_iovs[1], 1, NULL, NULL, 0, 1585 ut_sequence_step_cb, &completed); 1586 CU_ASSERT_EQUAL(rc, 0); 1587 1588 ut_seq.complete = false; 1589 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1590 1591 poll_threads(); 1592 1593 CU_ASSERT_EQUAL(completed, 2); 1594 CU_ASSERT(ut_seq.complete); 1595 CU_ASSERT_EQUAL(ut_seq.status, 0); 1596 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1597 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1598 1599 /* Check that a copy operation at the end is removed too */ 1600 seq = NULL; 1601 completed = 0; 1602 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1603 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1604 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1605 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1606 exp_iovs[0].iov_base = tmp[0]; 1607 exp_iovs[0].iov_len = sizeof(tmp[0]); 1608 exp_iovs[1].iov_base = buf; 1609 exp_iovs[1].iov_len = 2048; 1610 1611 dst_iovs[0].iov_base = tmp[1]; 1612 dst_iovs[0].iov_len = 2048; 1613 src_iovs[0].iov_base = tmp[0]; 1614 src_iovs[0].iov_len = sizeof(tmp[0]); 1615 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1616 &src_iovs[0], 1, NULL, NULL, 0, 1617 ut_sequence_step_cb, &completed); 1618 CU_ASSERT_EQUAL(rc, 0); 1619 1620 dst_iovs[1].iov_base = buf; 1621 dst_iovs[1].iov_len = 2048; 1622 src_iovs[1].iov_base = tmp[1]; 1623 src_iovs[1].iov_len = 2048; 1624 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1625 &src_iovs[1], 1, NULL, NULL, 0, 1626 ut_sequence_step_cb, &completed); 1627 CU_ASSERT_EQUAL(rc, 0); 1628 1629 ut_seq.complete = false; 1630 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1631 1632 poll_threads(); 1633 1634 CU_ASSERT_EQUAL(completed, 2); 1635 CU_ASSERT(ut_seq.complete); 1636 CU_ASSERT_EQUAL(ut_seq.status, 0); 1637 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1638 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1639 1640 /* Check a copy operation both at the beginning and the end */ 1641 seq = NULL; 1642 completed = 0; 1643 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1644 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1645 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1646 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1647 exp_iovs[0].iov_base = tmp[0]; 1648 exp_iovs[0].iov_len = sizeof(tmp[0]); 1649 exp_iovs[1].iov_base = buf; 1650 exp_iovs[1].iov_len = 2048; 1651 1652 dst_iovs[0].iov_base = tmp[1]; 1653 dst_iovs[0].iov_len = sizeof(tmp[1]); 1654 src_iovs[0].iov_base = tmp[0]; 1655 src_iovs[0].iov_len = sizeof(tmp[0]); 1656 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1657 &src_iovs[0], 1, NULL, NULL, 0, 1658 ut_sequence_step_cb, &completed); 1659 CU_ASSERT_EQUAL(rc, 0); 1660 1661 dst_iovs[1].iov_base = tmp[2]; 1662 dst_iovs[1].iov_len = 2048; 1663 src_iovs[1].iov_base = tmp[1]; 1664 src_iovs[1].iov_len = sizeof(tmp[1]); 1665 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1666 &src_iovs[1], 1, NULL, NULL, 0, 1667 ut_sequence_step_cb, &completed); 1668 CU_ASSERT_EQUAL(rc, 0); 1669 1670 dst_iovs[2].iov_base = buf; 1671 dst_iovs[2].iov_len = 2048; 1672 src_iovs[2].iov_base = tmp[2]; 1673 src_iovs[2].iov_len = 2048; 1674 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1675 &src_iovs[2], 1, NULL, NULL, 0, 1676 ut_sequence_step_cb, &completed); 1677 CU_ASSERT_EQUAL(rc, 0); 1678 1679 ut_seq.complete = false; 1680 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1681 1682 poll_threads(); 1683 1684 CU_ASSERT_EQUAL(completed, 3); 1685 CU_ASSERT(ut_seq.complete); 1686 CU_ASSERT_EQUAL(ut_seq.status, 0); 1687 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1688 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1689 1690 /* Check decompress + copy + decompress + copy */ 1691 seq = NULL; 1692 completed = 0; 1693 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1694 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1695 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = NULL; 1696 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = NULL; 1697 1698 dst_iovs[0].iov_base = tmp[1]; 1699 dst_iovs[0].iov_len = sizeof(tmp[1]); 1700 src_iovs[0].iov_base = tmp[0]; 1701 src_iovs[0].iov_len = sizeof(tmp[0]); 1702 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1703 &src_iovs[0], 1, NULL, NULL, 0, 1704 ut_sequence_step_cb, &completed); 1705 CU_ASSERT_EQUAL(rc, 0); 1706 1707 dst_iovs[1].iov_base = tmp[2]; 1708 dst_iovs[1].iov_len = 2048; 1709 src_iovs[1].iov_base = tmp[1]; 1710 src_iovs[1].iov_len = sizeof(tmp[1]); 1711 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1712 &src_iovs[1], 1, NULL, NULL, 0, 1713 ut_sequence_step_cb, &completed); 1714 CU_ASSERT_EQUAL(rc, 0); 1715 1716 dst_iovs[2].iov_base = tmp[3]; 1717 dst_iovs[2].iov_len = 1024; 1718 src_iovs[2].iov_base = tmp[2]; 1719 src_iovs[2].iov_len = 2048; 1720 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1721 &src_iovs[2], 1, NULL, NULL, 0, 1722 ut_sequence_step_cb, &completed); 1723 CU_ASSERT_EQUAL(rc, 0); 1724 1725 dst_iovs[3].iov_base = buf; 1726 dst_iovs[3].iov_len = 1024; 1727 src_iovs[3].iov_base = tmp[3]; 1728 src_iovs[3].iov_len = 1024; 1729 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 1730 &src_iovs[3], 1, NULL, NULL, 0, 1731 ut_sequence_step_cb, &completed); 1732 CU_ASSERT_EQUAL(rc, 0); 1733 1734 ut_seq.complete = false; 1735 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1736 1737 poll_threads(); 1738 1739 CU_ASSERT_EQUAL(completed, 4); 1740 CU_ASSERT(ut_seq.complete); 1741 CU_ASSERT_EQUAL(ut_seq.status, 0); 1742 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1743 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 2); 1744 1745 /* Check two copy operations - one of them should be removed, while the other should be 1746 * executed normally */ 1747 seq = NULL; 1748 completed = 0; 1749 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1750 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovcnt = 1; 1751 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovcnt = 1; 1752 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovs = &exp_iovs[0]; 1753 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovs = &exp_iovs[1]; 1754 exp_iovs[0].iov_base = tmp[0]; 1755 exp_iovs[0].iov_len = sizeof(tmp[0]); 1756 exp_iovs[1].iov_base = buf; 1757 exp_iovs[1].iov_len = sizeof(buf); 1758 1759 dst_iovs[0].iov_base = tmp[1]; 1760 dst_iovs[0].iov_len = sizeof(tmp[1]); 1761 src_iovs[0].iov_base = tmp[0]; 1762 src_iovs[0].iov_len = sizeof(tmp[0]); 1763 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1764 &src_iovs[0], 1, NULL, NULL, 0, 1765 ut_sequence_step_cb, &completed); 1766 CU_ASSERT_EQUAL(rc, 0); 1767 1768 dst_iovs[1].iov_base = buf; 1769 dst_iovs[1].iov_len = sizeof(buf); 1770 src_iovs[1].iov_base = tmp[1]; 1771 src_iovs[1].iov_len = sizeof(tmp[1]); 1772 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1773 &src_iovs[1], 1, NULL, NULL, 0, 1774 ut_sequence_step_cb, &completed); 1775 CU_ASSERT_EQUAL(rc, 0); 1776 1777 ut_seq.complete = false; 1778 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1779 1780 poll_threads(); 1781 1782 CU_ASSERT_EQUAL(completed, 2); 1783 CU_ASSERT(ut_seq.complete); 1784 CU_ASSERT_EQUAL(ut_seq.status, 0); 1785 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 1786 1787 /* Check fill + copy */ 1788 seq = NULL; 1789 completed = 0; 1790 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1791 g_seq_operations[SPDK_ACCEL_OPC_FILL].count = 0; 1792 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovs = NULL; 1793 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovs = NULL; 1794 g_seq_operations[SPDK_ACCEL_OPC_FILL].dst_iovcnt = 1; 1795 g_seq_operations[SPDK_ACCEL_OPC_FILL].dst_iovs = &exp_iovs[0]; 1796 exp_iovs[0].iov_base = buf; 1797 exp_iovs[0].iov_len = sizeof(buf); 1798 1799 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 0, 1800 ut_sequence_step_cb, &completed); 1801 CU_ASSERT_EQUAL(rc, 0); 1802 1803 dst_iovs[0].iov_base = buf; 1804 dst_iovs[0].iov_len = sizeof(buf); 1805 src_iovs[0].iov_base = tmp[0]; 1806 src_iovs[0].iov_len = sizeof(tmp[0]); 1807 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1808 &src_iovs[0], 1, NULL, NULL, 0, 1809 ut_sequence_step_cb, &completed); 1810 CU_ASSERT_EQUAL(rc, 0); 1811 1812 ut_seq.complete = false; 1813 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1814 1815 poll_threads(); 1816 1817 CU_ASSERT_EQUAL(completed, 2); 1818 CU_ASSERT(ut_seq.complete); 1819 CU_ASSERT_EQUAL(ut_seq.status, 0); 1820 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1821 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 1); 1822 1823 /* Check copy + encrypt + copy */ 1824 seq = NULL; 1825 completed = 0; 1826 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1827 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 1828 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovcnt = 1; 1829 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovcnt = 1; 1830 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovs = &exp_iovs[0]; 1831 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovs = &exp_iovs[1]; 1832 exp_iovs[0].iov_base = tmp[0]; 1833 exp_iovs[0].iov_len = sizeof(tmp[0]); 1834 exp_iovs[1].iov_base = buf; 1835 exp_iovs[1].iov_len = sizeof(buf); 1836 1837 dst_iovs[0].iov_base = tmp[1]; 1838 dst_iovs[0].iov_len = sizeof(tmp[1]); 1839 src_iovs[0].iov_base = tmp[0]; 1840 src_iovs[0].iov_len = sizeof(tmp[0]); 1841 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1842 &src_iovs[0], 1, NULL, NULL, 0, 1843 ut_sequence_step_cb, &completed); 1844 CU_ASSERT_EQUAL(rc, 0); 1845 1846 dst_iovs[1].iov_base = tmp[2]; 1847 dst_iovs[1].iov_len = sizeof(tmp[2]); 1848 src_iovs[1].iov_base = tmp[1]; 1849 src_iovs[1].iov_len = sizeof(tmp[1]); 1850 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1851 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1852 ut_sequence_step_cb, &completed); 1853 CU_ASSERT_EQUAL(rc, 0); 1854 1855 dst_iovs[2].iov_base = buf; 1856 dst_iovs[2].iov_len = sizeof(buf); 1857 src_iovs[2].iov_base = tmp[2]; 1858 src_iovs[2].iov_len = sizeof(tmp[2]); 1859 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1860 &src_iovs[2], 1, NULL, NULL, 0, 1861 ut_sequence_step_cb, &completed); 1862 CU_ASSERT_EQUAL(rc, 0); 1863 1864 ut_seq.complete = false; 1865 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1866 1867 poll_threads(); 1868 1869 CU_ASSERT_EQUAL(completed, 3); 1870 CU_ASSERT(ut_seq.complete); 1871 CU_ASSERT_EQUAL(ut_seq.status, 0); 1872 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1873 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 1874 1875 /* Check copy + decrypt + copy */ 1876 seq = NULL; 1877 completed = 0; 1878 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1879 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 1880 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1881 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1882 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1883 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1884 exp_iovs[0].iov_base = tmp[0]; 1885 exp_iovs[0].iov_len = sizeof(tmp[0]); 1886 exp_iovs[1].iov_base = buf; 1887 exp_iovs[1].iov_len = sizeof(buf); 1888 1889 dst_iovs[0].iov_base = tmp[1]; 1890 dst_iovs[0].iov_len = sizeof(tmp[1]); 1891 src_iovs[0].iov_base = tmp[0]; 1892 src_iovs[0].iov_len = sizeof(tmp[0]); 1893 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1894 &src_iovs[0], 1, NULL, NULL, 0, 1895 ut_sequence_step_cb, &completed); 1896 CU_ASSERT_EQUAL(rc, 0); 1897 1898 dst_iovs[1].iov_base = tmp[2]; 1899 dst_iovs[1].iov_len = sizeof(tmp[2]); 1900 src_iovs[1].iov_base = tmp[1]; 1901 src_iovs[1].iov_len = sizeof(tmp[1]); 1902 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1903 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 0, 1904 ut_sequence_step_cb, &completed); 1905 CU_ASSERT_EQUAL(rc, 0); 1906 1907 dst_iovs[2].iov_base = buf; 1908 dst_iovs[2].iov_len = sizeof(buf); 1909 src_iovs[2].iov_base = tmp[2]; 1910 src_iovs[2].iov_len = sizeof(tmp[2]); 1911 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1912 &src_iovs[2], 1, NULL, NULL, 0, 1913 ut_sequence_step_cb, &completed); 1914 CU_ASSERT_EQUAL(rc, 0); 1915 1916 ut_seq.complete = false; 1917 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1918 1919 poll_threads(); 1920 1921 CU_ASSERT_EQUAL(completed, 3); 1922 CU_ASSERT(ut_seq.complete); 1923 CU_ASSERT_EQUAL(ut_seq.status, 0); 1924 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1925 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 1926 1927 /* Check a sequence with memory domains and verify their pointers (and their contexts) are 1928 * correctly set on the next/prev task when a copy is removed */ 1929 seq = NULL; 1930 completed = 0; 1931 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1932 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 1933 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1934 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1935 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1936 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1937 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_domain = (void *)0x1; 1938 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_domain_ctx = (void *)0x2; 1939 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_domain = (void *)0x3; 1940 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_domain_ctx = (void *)0x4; 1941 exp_iovs[0].iov_base = tmp[0]; 1942 exp_iovs[0].iov_len = sizeof(tmp[0]); 1943 exp_iovs[1].iov_base = buf; 1944 exp_iovs[1].iov_len = sizeof(buf); 1945 1946 dst_iovs[0].iov_base = tmp[1]; 1947 dst_iovs[0].iov_len = sizeof(tmp[1]); 1948 src_iovs[0].iov_base = tmp[0]; 1949 src_iovs[0].iov_len = sizeof(tmp[0]); 1950 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, (void *)0xdead, (void *)0xbeef, 1951 &src_iovs[0], 1, (void *)0x3, (void *)0x4, 0, 1952 ut_sequence_step_cb, &completed); 1953 CU_ASSERT_EQUAL(rc, 0); 1954 1955 dst_iovs[1].iov_base = tmp[2]; 1956 dst_iovs[1].iov_len = sizeof(tmp[2]); 1957 src_iovs[1].iov_base = tmp[1]; 1958 src_iovs[1].iov_len = sizeof(tmp[1]); 1959 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 1960 &src_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 0, 1961 sizeof(tmp[2]), 0, ut_sequence_step_cb, &completed); 1962 CU_ASSERT_EQUAL(rc, 0); 1963 1964 dst_iovs[2].iov_base = buf; 1965 dst_iovs[2].iov_len = sizeof(buf); 1966 src_iovs[2].iov_base = tmp[2]; 1967 src_iovs[2].iov_len = sizeof(tmp[2]); 1968 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, (void *)0x1, (void *)0x2, 1969 &src_iovs[2], 1, (void *)0xdead, (void *)0xbeef, 0, 1970 ut_sequence_step_cb, &completed); 1971 CU_ASSERT_EQUAL(rc, 0); 1972 1973 ut_seq.complete = false; 1974 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1975 1976 poll_threads(); 1977 1978 CU_ASSERT_EQUAL(completed, 3); 1979 CU_ASSERT(ut_seq.complete); 1980 CU_ASSERT_EQUAL(ut_seq.status, 0); 1981 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1982 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 1983 1984 /* Cleanup module pointers to make subsequent tests work correctly */ 1985 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1986 g_modules_opc[i] = modules[i]; 1987 } 1988 1989 g_module.supports_memory_domains = false; 1990 ut_clear_operations(); 1991 spdk_put_io_channel(ioch); 1992 poll_threads(); 1993 } 1994 1995 static int 1996 ut_submit_decompress(struct spdk_io_channel *ch, struct spdk_accel_task *task) 1997 { 1998 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 1999 2000 spdk_accel_task_complete(task, 0); 2001 2002 return 0; 2003 } 2004 2005 static void 2006 test_sequence_accel_buffers(void) 2007 { 2008 struct spdk_accel_sequence *seq = NULL; 2009 struct spdk_io_channel *ioch; 2010 struct accel_io_channel *accel_ch; 2011 struct ut_sequence ut_seq; 2012 struct iovec src_iovs[3], dst_iovs[3]; 2013 char srcbuf[4096], dstbuf[4096], expected[4096]; 2014 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2015 void *buf[2], *domain_ctx[2], *iobuf_buf; 2016 struct spdk_memory_domain *domain[2]; 2017 struct spdk_iobuf_buffer *cache_entry; 2018 spdk_iobuf_buffer_stailq_t small_cache; 2019 uint32_t small_cache_count; 2020 int i, rc, completed; 2021 struct spdk_iobuf_opts opts_iobuf = {}; 2022 2023 /* Set up the iobuf to always use the "small" pool */ 2024 opts_iobuf.large_bufsize = 0x20000; 2025 opts_iobuf.large_pool_count = 0; 2026 opts_iobuf.small_bufsize = 0x10000; 2027 opts_iobuf.small_pool_count = 32; 2028 2029 rc = spdk_iobuf_set_opts(&opts_iobuf); 2030 CU_ASSERT(rc == 0); 2031 2032 ioch = spdk_accel_get_io_channel(); 2033 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2034 2035 /* Override the submit_tasks function */ 2036 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2037 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2038 modules[i] = g_modules_opc[i]; 2039 g_modules_opc[i] = g_module; 2040 } 2041 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2042 * decompress operations together in one sequence. 2043 */ 2044 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2045 g_seq_operations[SPDK_ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2046 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2047 2048 /* Check the simplest case: one operation using accel buffer as destination + copy operation 2049 * specifying the actual destination buffer 2050 */ 2051 memset(srcbuf, 0xa5, 4096); 2052 memset(dstbuf, 0x0, 4096); 2053 memset(expected, 0xa5, 4096); 2054 completed = 0; 2055 seq = NULL; 2056 2057 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2058 CU_ASSERT_EQUAL(rc, 0); 2059 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2060 2061 src_iovs[0].iov_base = srcbuf; 2062 src_iovs[0].iov_len = 4096; 2063 dst_iovs[0].iov_base = buf[0]; 2064 dst_iovs[0].iov_len = 4096; 2065 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2066 &src_iovs[0], 1, NULL, NULL, 0, 2067 ut_sequence_step_cb, &completed); 2068 CU_ASSERT_EQUAL(rc, 0); 2069 2070 src_iovs[1].iov_base = buf[0]; 2071 src_iovs[1].iov_len = 4096; 2072 dst_iovs[1].iov_base = dstbuf; 2073 dst_iovs[1].iov_len = 4096; 2074 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2075 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 2076 ut_sequence_step_cb, &completed); 2077 CU_ASSERT_EQUAL(rc, 0); 2078 2079 ut_seq.complete = false; 2080 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2081 2082 poll_threads(); 2083 2084 CU_ASSERT_EQUAL(completed, 2); 2085 CU_ASSERT(ut_seq.complete); 2086 CU_ASSERT_EQUAL(ut_seq.status, 0); 2087 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2088 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2089 2090 /* Start with a fill operation using accel buffer, followed by a decompress using another 2091 * accel buffer as dst, followed by a copy operation specifying dst buffer of the whole 2092 * sequence 2093 */ 2094 memset(srcbuf, 0x0, 4096); 2095 memset(dstbuf, 0x0, 4096); 2096 memset(expected, 0x5a, 4096); 2097 completed = 0; 2098 seq = NULL; 2099 2100 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2101 CU_ASSERT_EQUAL(rc, 0); 2102 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2103 2104 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2105 ut_sequence_step_cb, &completed); 2106 CU_ASSERT_EQUAL(rc, 0); 2107 2108 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2109 CU_ASSERT_EQUAL(rc, 0); 2110 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2111 2112 src_iovs[0].iov_base = buf[0]; 2113 src_iovs[0].iov_len = 4096; 2114 dst_iovs[0].iov_base = buf[1]; 2115 dst_iovs[0].iov_len = 4096; 2116 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2117 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2118 ut_sequence_step_cb, &completed); 2119 CU_ASSERT_EQUAL(rc, 0); 2120 2121 src_iovs[1].iov_base = buf[1]; 2122 src_iovs[1].iov_len = 4096; 2123 dst_iovs[1].iov_base = dstbuf; 2124 dst_iovs[1].iov_len = 4096; 2125 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2126 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2127 ut_sequence_step_cb, &completed); 2128 2129 ut_seq.complete = false; 2130 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2131 2132 poll_threads(); 2133 2134 CU_ASSERT_EQUAL(completed, 3); 2135 CU_ASSERT(ut_seq.complete); 2136 CU_ASSERT_EQUAL(ut_seq.status, 0); 2137 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2138 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2139 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2140 2141 /* Check the same, but with two decompress operations with the first one being in-place */ 2142 memset(srcbuf, 0x0, 4096); 2143 memset(dstbuf, 0x0, 4096); 2144 memset(expected, 0x5a, 4096); 2145 completed = 0; 2146 seq = NULL; 2147 2148 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2149 CU_ASSERT_EQUAL(rc, 0); 2150 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2151 2152 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2153 ut_sequence_step_cb, &completed); 2154 CU_ASSERT_EQUAL(rc, 0); 2155 2156 src_iovs[0].iov_base = buf[0]; 2157 src_iovs[0].iov_len = 4096; 2158 dst_iovs[0].iov_base = buf[0]; 2159 dst_iovs[0].iov_len = 4096; 2160 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2161 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2162 ut_sequence_step_cb, &completed); 2163 CU_ASSERT_EQUAL(rc, 0); 2164 2165 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2166 CU_ASSERT_EQUAL(rc, 0); 2167 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2168 2169 src_iovs[1].iov_base = buf[0]; 2170 src_iovs[1].iov_len = 4096; 2171 dst_iovs[1].iov_base = buf[1]; 2172 dst_iovs[1].iov_len = 4096; 2173 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, domain[1], domain_ctx[1], 2174 &src_iovs[1], 1, domain[0], domain_ctx[0], 0, 2175 ut_sequence_step_cb, &completed); 2176 CU_ASSERT_EQUAL(rc, 0); 2177 2178 src_iovs[2].iov_base = buf[1]; 2179 src_iovs[2].iov_len = 4096; 2180 dst_iovs[2].iov_base = dstbuf; 2181 dst_iovs[2].iov_len = 4096; 2182 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 2183 &src_iovs[2], 1, domain[1], domain_ctx[1], 0, 2184 ut_sequence_step_cb, &completed); 2185 2186 ut_seq.complete = false; 2187 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2188 2189 poll_threads(); 2190 2191 CU_ASSERT_EQUAL(completed, 4); 2192 CU_ASSERT(ut_seq.complete); 2193 CU_ASSERT_EQUAL(ut_seq.status, 0); 2194 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2195 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2196 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2197 2198 /* Check that specifying offsets within accel buffers works correctly */ 2199 memset(srcbuf, 0x0, 4096); 2200 memset(dstbuf, 0x0, 4096); 2201 completed = 0; 2202 seq = NULL; 2203 2204 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2205 CU_ASSERT_EQUAL(rc, 0); 2206 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2207 2208 /* Fill in each 1K of the buffer with different pattern */ 2209 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2210 ut_sequence_step_cb, &completed); 2211 CU_ASSERT_EQUAL(rc, 0); 2212 2213 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2214 0x5a, 0, ut_sequence_step_cb, &completed); 2215 CU_ASSERT_EQUAL(rc, 0); 2216 2217 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2218 0xfe, 0, ut_sequence_step_cb, &completed); 2219 CU_ASSERT_EQUAL(rc, 0); 2220 2221 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2222 0xed, 0, ut_sequence_step_cb, &completed); 2223 CU_ASSERT_EQUAL(rc, 0); 2224 2225 src_iovs[0].iov_base = buf[0]; 2226 src_iovs[0].iov_len = 4096; 2227 dst_iovs[0].iov_base = dstbuf; 2228 dst_iovs[0].iov_len = 4096; 2229 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2230 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2231 ut_sequence_step_cb, &completed); 2232 2233 ut_seq.complete = false; 2234 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2235 2236 poll_threads(); 2237 2238 memset(&expected[0], 0xa5, 1024); 2239 memset(&expected[1024], 0x5a, 1024); 2240 memset(&expected[2048], 0xfe, 1024); 2241 memset(&expected[3072], 0xed, 1024); 2242 CU_ASSERT_EQUAL(completed, 5); 2243 CU_ASSERT(ut_seq.complete); 2244 CU_ASSERT_EQUAL(ut_seq.status, 0); 2245 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2246 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2247 2248 /* Check the same but this time with a decompress operation on part of the buffer (512B 2249 * offset) */ 2250 memset(srcbuf, 0x0, 4096); 2251 memset(dstbuf, 0x0, 4096); 2252 completed = 0; 2253 seq = NULL; 2254 2255 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2256 CU_ASSERT_EQUAL(rc, 0); 2257 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2258 2259 /* Fill in each 1K of the buffer with different pattern */ 2260 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 0, 2261 ut_sequence_step_cb, &completed); 2262 CU_ASSERT_EQUAL(rc, 0); 2263 2264 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2265 0x5a, 0, ut_sequence_step_cb, &completed); 2266 CU_ASSERT_EQUAL(rc, 0); 2267 2268 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2269 0xfe, 0, ut_sequence_step_cb, &completed); 2270 CU_ASSERT_EQUAL(rc, 0); 2271 2272 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2273 0xed, 0, ut_sequence_step_cb, &completed); 2274 CU_ASSERT_EQUAL(rc, 0); 2275 2276 rc = spdk_accel_get_buf(ioch, 3072, &buf[1], &domain[1], &domain_ctx[1]); 2277 CU_ASSERT_EQUAL(rc, 0); 2278 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2279 2280 src_iovs[0].iov_base = (char *)buf[0] + 512; 2281 src_iovs[0].iov_len = 3072; 2282 dst_iovs[0].iov_base = (char *)buf[1] + 256; 2283 dst_iovs[0].iov_len = 3072; 2284 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2285 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2286 ut_sequence_step_cb, &completed); 2287 2288 src_iovs[1].iov_base = (char *)buf[1] + 256; 2289 src_iovs[1].iov_len = 3072; 2290 dst_iovs[1].iov_base = dstbuf; 2291 dst_iovs[1].iov_len = 3072; 2292 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2293 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2294 ut_sequence_step_cb, &completed); 2295 2296 ut_seq.complete = false; 2297 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2298 2299 poll_threads(); 2300 2301 memset(&expected[0], 0xa5, 512); 2302 memset(&expected[512], 0x5a, 1024); 2303 memset(&expected[1536], 0xfe, 1024); 2304 memset(&expected[2560], 0xed, 512); 2305 CU_ASSERT_EQUAL(completed, 6); 2306 CU_ASSERT(ut_seq.complete); 2307 CU_ASSERT_EQUAL(ut_seq.status, 0); 2308 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 3072), 0); 2309 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2310 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2311 2312 /* Check that if iobuf pool is empty, the sequence processing will wait until a buffer is 2313 * available 2314 */ 2315 accel_ch = spdk_io_channel_get_ctx(ioch); 2316 small_cache_count = accel_ch->iobuf.small.cache_count; 2317 STAILQ_INIT(&small_cache); 2318 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2319 accel_ch->iobuf.small.cache_count = 0; 2320 accel_ch->iobuf.small.cache_size = 0; 2321 g_iobuf.small_pool_count = 0; 2322 2323 /* First allocate a single buffer used by two operations */ 2324 memset(srcbuf, 0x0, 4096); 2325 memset(dstbuf, 0x0, 4096); 2326 memset(expected, 0xa5, 4096); 2327 completed = 0; 2328 seq = NULL; 2329 2330 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2331 CU_ASSERT_EQUAL(rc, 0); 2332 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2333 2334 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0xa5, 0, 2335 ut_sequence_step_cb, &completed); 2336 CU_ASSERT_EQUAL(rc, 0); 2337 2338 src_iovs[0].iov_base = buf[0]; 2339 src_iovs[0].iov_len = 4096; 2340 dst_iovs[0].iov_base = dstbuf; 2341 dst_iovs[0].iov_len = 4096; 2342 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2343 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2344 ut_sequence_step_cb, &completed); 2345 2346 ut_seq.complete = false; 2347 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2348 2349 poll_threads(); 2350 2351 CU_ASSERT_EQUAL(completed, 0); 2352 CU_ASSERT(!ut_seq.complete); 2353 2354 /* Get a buffer and return it to the pool to trigger the sequence to finish */ 2355 g_iobuf.small_pool_count = 1; 2356 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2357 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2358 CU_ASSERT(g_iobuf.small_pool_count == 0); 2359 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2360 2361 poll_threads(); 2362 2363 CU_ASSERT_EQUAL(completed, 2); 2364 CU_ASSERT(ut_seq.complete); 2365 CU_ASSERT_EQUAL(ut_seq.status, 0); 2366 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2367 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2368 2369 /* Return the buffers back to the cache */ 2370 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2371 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2372 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2373 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2374 small_cache_count++; 2375 } 2376 accel_ch->iobuf.small.cache_count = 0; 2377 g_iobuf.small_pool_count = 0; 2378 2379 /* Check a bit more complex scenario, with two buffers in the sequence */ 2380 memset(srcbuf, 0x0, 4096); 2381 memset(dstbuf, 0x0, 4096); 2382 memset(expected, 0x5a, 4096); 2383 completed = 0; 2384 seq = NULL; 2385 2386 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2387 CU_ASSERT_EQUAL(rc, 0); 2388 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2389 2390 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 0, 2391 ut_sequence_step_cb, &completed); 2392 CU_ASSERT_EQUAL(rc, 0); 2393 2394 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2395 CU_ASSERT_EQUAL(rc, 0); 2396 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2397 2398 src_iovs[0].iov_base = buf[0]; 2399 src_iovs[0].iov_len = 4096; 2400 dst_iovs[0].iov_base = buf[1]; 2401 dst_iovs[0].iov_len = 4096; 2402 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2403 &src_iovs[0], 1, domain[0], domain_ctx[0], 0, 2404 ut_sequence_step_cb, &completed); 2405 CU_ASSERT_EQUAL(rc, 0); 2406 2407 src_iovs[1].iov_base = buf[1]; 2408 src_iovs[1].iov_len = 4096; 2409 dst_iovs[1].iov_base = dstbuf; 2410 dst_iovs[1].iov_len = 4096; 2411 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2412 &src_iovs[1], 1, domain[1], domain_ctx[1], 0, 2413 ut_sequence_step_cb, &completed); 2414 CU_ASSERT_EQUAL(rc, 0); 2415 2416 ut_seq.complete = false; 2417 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2418 2419 poll_threads(); 2420 2421 CU_ASSERT_EQUAL(completed, 0); 2422 CU_ASSERT(!ut_seq.complete); 2423 2424 g_iobuf.small_pool_count = 1; 2425 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2426 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2427 g_iobuf.small_pool_count = 0; 2428 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2429 2430 /* One buffer is not enough to finish the whole sequence */ 2431 poll_threads(); 2432 CU_ASSERT(!ut_seq.complete); 2433 2434 g_iobuf.small_pool_count = 1; 2435 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2436 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2437 g_iobuf.small_pool_count = 0; 2438 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2439 2440 poll_threads(); 2441 2442 CU_ASSERT_EQUAL(completed, 3); 2443 CU_ASSERT(ut_seq.complete); 2444 CU_ASSERT_EQUAL(ut_seq.status, 0); 2445 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2446 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2447 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2448 2449 /* Return the buffers back to the cache */ 2450 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2451 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2452 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2453 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2454 small_cache_count++; 2455 } 2456 accel_ch->iobuf.small.cache_count = 0; 2457 2458 g_iobuf.small_pool_count = 32; 2459 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2460 accel_ch->iobuf.small.cache_count = small_cache_count; 2461 2462 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2463 g_modules_opc[i] = modules[i]; 2464 } 2465 2466 ut_clear_operations(); 2467 spdk_put_io_channel(ioch); 2468 poll_threads(); 2469 } 2470 2471 static void 2472 ut_domain_ctx_init(struct ut_domain_ctx *ctx, void *base, size_t len, struct iovec *expected) 2473 { 2474 ctx->iov.iov_base = base; 2475 ctx->iov.iov_len = len; 2476 ctx->expected = *expected; 2477 ctx->pull_submit_status = 0; 2478 ctx->push_submit_status = 0; 2479 ctx->pull_complete_status = 0; 2480 ctx->push_complete_status = 0; 2481 } 2482 2483 static void 2484 test_sequence_memory_domain(void) 2485 { 2486 struct spdk_accel_sequence *seq = NULL; 2487 struct spdk_io_channel *ioch; 2488 struct accel_io_channel *accel_ch; 2489 struct ut_sequence ut_seq; 2490 struct ut_domain_ctx domctx[4]; 2491 struct spdk_iobuf_buffer *cache_entry; 2492 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2493 struct spdk_memory_domain *accel_domain; 2494 spdk_iobuf_buffer_stailq_t small_cache; 2495 char srcbuf[4096], dstbuf[4096], expected[4096], tmp[4096]; 2496 struct iovec src_iovs[2], dst_iovs[2]; 2497 uint32_t small_cache_count; 2498 void *accel_buf, *accel_domain_ctx, *iobuf_buf; 2499 int i, rc, completed; 2500 2501 ioch = spdk_accel_get_io_channel(); 2502 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2503 2504 /* Override the submit_tasks function */ 2505 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2506 g_module.supports_memory_domains = false; 2507 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2508 modules[i] = g_modules_opc[i]; 2509 g_modules_opc[i] = g_module; 2510 } 2511 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2512 * decompress operations together in one sequence. 2513 */ 2514 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2515 g_seq_operations[SPDK_ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2516 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2517 2518 /* First check the simplest case - single fill operation with dstbuf in domain */ 2519 memset(expected, 0xa5, sizeof(expected)); 2520 memset(dstbuf, 0x0, sizeof(dstbuf)); 2521 completed = 0; 2522 seq = NULL; 2523 2524 /* Use some garbage pointer as dst and use domain ctx to get the actual dstbuf */ 2525 dst_iovs[0].iov_base = (void *)0xcafebabe; 2526 dst_iovs[0].iov_len = sizeof(dstbuf); 2527 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2528 2529 rc = spdk_accel_append_fill(&seq, ioch, dst_iovs[0].iov_base, dst_iovs[0].iov_len, 2530 g_ut_domain, &domctx[0], 0xa5, 0, 2531 ut_sequence_step_cb, &completed); 2532 CU_ASSERT_EQUAL(rc, 0); 2533 2534 ut_seq.complete = false; 2535 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2536 2537 poll_threads(); 2538 CU_ASSERT_EQUAL(completed, 1); 2539 CU_ASSERT(ut_seq.complete); 2540 CU_ASSERT_EQUAL(ut_seq.status, 0); 2541 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2542 2543 /* Check operation with both srcbuf and dstbuf in remote memory domain */ 2544 memset(expected, 0x5a, sizeof(expected)); 2545 memset(dstbuf, 0x0, sizeof(dstbuf)); 2546 memset(srcbuf, 0x5a, sizeof(dstbuf)); 2547 completed = 0; 2548 seq = NULL; 2549 2550 src_iovs[0].iov_base = (void *)0xcafebabe; 2551 src_iovs[0].iov_len = sizeof(srcbuf); 2552 dst_iovs[0].iov_base = (void *)0xbeefdead; 2553 dst_iovs[0].iov_len = sizeof(dstbuf); 2554 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2555 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2556 2557 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2558 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2559 ut_sequence_step_cb, &completed); 2560 CU_ASSERT_EQUAL(rc, 0); 2561 2562 ut_seq.complete = false; 2563 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2564 2565 poll_threads(); 2566 CU_ASSERT_EQUAL(completed, 1); 2567 CU_ASSERT(ut_seq.complete); 2568 CU_ASSERT_EQUAL(ut_seq.status, 0); 2569 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2570 2571 /* Two operations using a buffer in remote domain */ 2572 memset(expected, 0xa5, sizeof(expected)); 2573 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2574 memset(tmp, 0x0, sizeof(tmp)); 2575 memset(dstbuf, 0x0, sizeof(dstbuf)); 2576 completed = 0; 2577 seq = NULL; 2578 2579 src_iovs[0].iov_base = (void *)0xcafebabe; 2580 src_iovs[0].iov_len = sizeof(srcbuf); 2581 dst_iovs[0].iov_base = (void *)0xbeefdead; 2582 dst_iovs[0].iov_len = sizeof(tmp); 2583 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2584 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2585 2586 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2587 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2588 ut_sequence_step_cb, &completed); 2589 CU_ASSERT_EQUAL(rc, 0); 2590 2591 src_iovs[1].iov_base = (void *)0xbeefdead; 2592 src_iovs[1].iov_len = sizeof(tmp); 2593 dst_iovs[1].iov_base = (void *)0xa5a5a5a5; 2594 dst_iovs[1].iov_len = sizeof(dstbuf); 2595 ut_domain_ctx_init(&domctx[2], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2596 ut_domain_ctx_init(&domctx[3], tmp, sizeof(tmp), &src_iovs[1]); 2597 2598 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[2], 2599 &src_iovs[1], 1, g_ut_domain, &domctx[3], 0, 2600 ut_sequence_step_cb, &completed); 2601 CU_ASSERT_EQUAL(rc, 0); 2602 2603 ut_seq.complete = false; 2604 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2605 2606 poll_threads(); 2607 CU_ASSERT_EQUAL(completed, 2); 2608 CU_ASSERT(ut_seq.complete); 2609 CU_ASSERT_EQUAL(ut_seq.status, 0); 2610 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2611 2612 /* Use buffer in remote memory domain and accel buffer at the same time */ 2613 memset(expected, 0x5a, sizeof(expected)); 2614 memset(srcbuf, 0x5a, sizeof(expected)); 2615 memset(dstbuf, 0x0, sizeof(dstbuf)); 2616 completed = 0; 2617 seq = NULL; 2618 2619 rc = spdk_accel_get_buf(ioch, sizeof(dstbuf), &accel_buf, &accel_domain, &accel_domain_ctx); 2620 CU_ASSERT_EQUAL(rc, 0); 2621 2622 src_iovs[0].iov_base = (void *)0xfeedbeef; 2623 src_iovs[0].iov_len = sizeof(srcbuf); 2624 dst_iovs[0].iov_base = accel_buf; 2625 dst_iovs[0].iov_len = sizeof(dstbuf); 2626 ut_domain_ctx_init(&domctx[0], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2627 2628 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, accel_domain, accel_domain_ctx, 2629 &src_iovs[0], 1, g_ut_domain, &domctx[0], 0, 2630 ut_sequence_step_cb, &completed); 2631 CU_ASSERT_EQUAL(rc, 0); 2632 2633 src_iovs[1].iov_base = accel_buf; 2634 src_iovs[1].iov_len = sizeof(dstbuf); 2635 dst_iovs[1].iov_base = (void *)0xbeeffeed; 2636 dst_iovs[1].iov_len = sizeof(dstbuf); 2637 ut_domain_ctx_init(&domctx[1], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2638 2639 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[1], 2640 &src_iovs[1], 1, accel_domain, accel_domain_ctx, 0, 2641 ut_sequence_step_cb, &completed); 2642 CU_ASSERT_EQUAL(rc, 0); 2643 2644 ut_seq.complete = false; 2645 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2646 2647 poll_threads(); 2648 CU_ASSERT_EQUAL(completed, 2); 2649 CU_ASSERT(ut_seq.complete); 2650 CU_ASSERT_EQUAL(ut_seq.status, 0); 2651 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2652 spdk_accel_put_buf(ioch, accel_buf, accel_domain, accel_domain_ctx); 2653 2654 /* Check that a sequence with memory domains is correctly executed if buffers are not 2655 * immediately available */ 2656 memset(expected, 0xa5, sizeof(expected)); 2657 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2658 memset(dstbuf, 0x0, sizeof(dstbuf)); 2659 completed = 0; 2660 seq = NULL; 2661 /* Make sure the buffer pool is empty */ 2662 accel_ch = spdk_io_channel_get_ctx(ioch); 2663 small_cache_count = accel_ch->iobuf.small.cache_count; 2664 STAILQ_INIT(&small_cache); 2665 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2666 accel_ch->iobuf.small.cache_count = 0; 2667 g_iobuf.small_pool_count = 0; 2668 2669 src_iovs[0].iov_base = (void *)0xdeadbeef; 2670 src_iovs[0].iov_len = sizeof(srcbuf); 2671 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2672 dst_iovs[0].iov_len = sizeof(dstbuf); 2673 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2674 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2675 2676 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2677 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2678 ut_sequence_step_cb, &completed); 2679 CU_ASSERT_EQUAL(rc, 0); 2680 2681 ut_seq.complete = false; 2682 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2683 2684 poll_threads(); 2685 CU_ASSERT_EQUAL(completed, 0); 2686 CU_ASSERT(!ut_seq.complete); 2687 2688 /* Get a buffer and return it to the pool to trigger the sequence to resume. It shouldn't 2689 * be able to complete, as it needs two buffers */ 2690 g_iobuf.small_pool_count = 1; 2691 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2692 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2693 g_iobuf.small_pool_count = 0; 2694 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2695 2696 CU_ASSERT_EQUAL(completed, 0); 2697 CU_ASSERT(!ut_seq.complete); 2698 2699 /* Return another buffer, this time the sequence should finish */ 2700 g_iobuf.small_pool_count = 1; 2701 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2702 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2703 g_iobuf.small_pool_count = 0; 2704 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2705 2706 poll_threads(); 2707 CU_ASSERT_EQUAL(completed, 1); 2708 CU_ASSERT(ut_seq.complete); 2709 CU_ASSERT_EQUAL(ut_seq.status, 0); 2710 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2711 2712 /* Return the buffers back to the cache */ 2713 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2714 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2715 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2716 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2717 small_cache_count++; 2718 } 2719 accel_ch->iobuf.small.cache_count = 0; 2720 2721 g_iobuf.small_pool_count = 32; 2722 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2723 accel_ch->iobuf.small.cache_count = small_cache_count; 2724 2725 /* Check error cases, starting with an error from spdk_memory_domain_pull_data() */ 2726 completed = 0; 2727 seq = NULL; 2728 2729 src_iovs[0].iov_base = (void *)0xdeadbeef; 2730 src_iovs[0].iov_len = sizeof(srcbuf); 2731 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2732 dst_iovs[0].iov_len = sizeof(dstbuf); 2733 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2734 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2735 domctx[1].pull_submit_status = -E2BIG; 2736 2737 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2738 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2739 ut_sequence_step_cb, &completed); 2740 CU_ASSERT_EQUAL(rc, 0); 2741 2742 ut_seq.complete = false; 2743 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2744 2745 CU_ASSERT_EQUAL(completed, 1); 2746 CU_ASSERT(ut_seq.complete); 2747 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 2748 2749 /* Check completion error from spdk_memory_domain_pull_data() */ 2750 completed = 0; 2751 seq = NULL; 2752 2753 src_iovs[0].iov_base = (void *)0xdeadbeef; 2754 src_iovs[0].iov_len = sizeof(srcbuf); 2755 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2756 dst_iovs[0].iov_len = sizeof(dstbuf); 2757 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2758 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2759 domctx[1].pull_complete_status = -EACCES; 2760 2761 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2762 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2763 ut_sequence_step_cb, &completed); 2764 CU_ASSERT_EQUAL(rc, 0); 2765 2766 ut_seq.complete = false; 2767 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2768 2769 CU_ASSERT_EQUAL(completed, 1); 2770 CU_ASSERT(ut_seq.complete); 2771 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 2772 2773 /* Check submission error from spdk_memory_domain_push_data() */ 2774 completed = 0; 2775 seq = NULL; 2776 2777 src_iovs[0].iov_base = (void *)0xdeadbeef; 2778 src_iovs[0].iov_len = sizeof(srcbuf); 2779 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2780 dst_iovs[0].iov_len = sizeof(dstbuf); 2781 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2782 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2783 domctx[0].push_submit_status = -EADDRINUSE; 2784 2785 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2786 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2787 ut_sequence_step_cb, &completed); 2788 CU_ASSERT_EQUAL(rc, 0); 2789 2790 ut_seq.complete = false; 2791 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2792 2793 CU_ASSERT_EQUAL(completed, 1); 2794 CU_ASSERT(ut_seq.complete); 2795 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 2796 2797 /* Check completion error from spdk_memory_domain_push_data() */ 2798 completed = 0; 2799 seq = NULL; 2800 2801 src_iovs[0].iov_base = (void *)0xdeadbeef; 2802 src_iovs[0].iov_len = sizeof(srcbuf); 2803 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2804 dst_iovs[0].iov_len = sizeof(dstbuf); 2805 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2806 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2807 domctx[0].push_complete_status = -EADDRNOTAVAIL; 2808 2809 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2810 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2811 ut_sequence_step_cb, &completed); 2812 CU_ASSERT_EQUAL(rc, 0); 2813 2814 ut_seq.complete = false; 2815 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2816 2817 CU_ASSERT_EQUAL(completed, 1); 2818 CU_ASSERT(ut_seq.complete); 2819 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 2820 2821 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2822 g_modules_opc[i] = modules[i]; 2823 } 2824 2825 ut_clear_operations(); 2826 spdk_put_io_channel(ioch); 2827 poll_threads(); 2828 } 2829 2830 static int 2831 ut_submit_decompress_memory_domain(struct spdk_io_channel *ch, struct spdk_accel_task *task) 2832 { 2833 struct ut_domain_ctx *ctx; 2834 struct iovec *src_iovs, *dst_iovs; 2835 uint32_t src_iovcnt, dst_iovcnt; 2836 2837 src_iovs = task->s.iovs; 2838 dst_iovs = task->d.iovs; 2839 src_iovcnt = task->s.iovcnt; 2840 dst_iovcnt = task->d.iovcnt; 2841 2842 if (task->src_domain != NULL) { 2843 ctx = task->src_domain_ctx; 2844 CU_ASSERT_EQUAL(memcmp(task->s.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2845 src_iovs = &ctx->iov; 2846 src_iovcnt = 1; 2847 } 2848 2849 if (task->dst_domain != NULL) { 2850 ctx = task->dst_domain_ctx; 2851 CU_ASSERT_EQUAL(memcmp(task->d.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2852 dst_iovs = &ctx->iov; 2853 dst_iovcnt = 1; 2854 } 2855 2856 spdk_iovcpy(src_iovs, src_iovcnt, dst_iovs, dst_iovcnt); 2857 spdk_accel_task_complete(task, 0); 2858 2859 return 0; 2860 } 2861 2862 static void 2863 test_sequence_module_memory_domain(void) 2864 { 2865 struct spdk_accel_sequence *seq = NULL; 2866 struct spdk_io_channel *ioch; 2867 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2868 struct spdk_memory_domain *accel_domain; 2869 struct ut_sequence ut_seq; 2870 struct ut_domain_ctx domctx[2]; 2871 struct iovec src_iovs[2], dst_iovs[2]; 2872 void *buf, *accel_domain_ctx; 2873 char srcbuf[4096], dstbuf[4096], tmp[4096], expected[4096]; 2874 int i, rc, completed; 2875 2876 ioch = spdk_accel_get_io_channel(); 2877 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2878 2879 /* Override the submit_tasks function */ 2880 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 2881 g_module.supports_memory_domains = true; 2882 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2883 modules[i] = g_modules_opc[i]; 2884 g_modules_opc[i] = g_module; 2885 } 2886 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress_memory_domain; 2887 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2888 2889 /* Check a sequence with both buffers in memory domains */ 2890 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2891 memset(expected, 0xa5, sizeof(expected)); 2892 memset(dstbuf, 0, sizeof(dstbuf)); 2893 seq = NULL; 2894 completed = 0; 2895 2896 src_iovs[0].iov_base = (void *)0xcafebabe; 2897 src_iovs[0].iov_len = sizeof(srcbuf); 2898 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2899 dst_iovs[0].iov_len = sizeof(dstbuf); 2900 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2901 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2902 2903 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2904 &src_iovs[0], 1, g_ut_domain, &domctx[1], 0, 2905 ut_sequence_step_cb, &completed); 2906 CU_ASSERT_EQUAL(rc, 0); 2907 2908 ut_seq.complete = false; 2909 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2910 2911 poll_threads(); 2912 2913 CU_ASSERT_EQUAL(completed, 1); 2914 CU_ASSERT(ut_seq.complete); 2915 CU_ASSERT_EQUAL(ut_seq.status, 0); 2916 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2917 2918 /* Check two operations each with a single buffer in memory domain */ 2919 memset(srcbuf, 0x5a, sizeof(srcbuf)); 2920 memset(expected, 0x5a, sizeof(expected)); 2921 memset(dstbuf, 0, sizeof(dstbuf)); 2922 memset(tmp, 0, sizeof(tmp)); 2923 seq = NULL; 2924 completed = 0; 2925 2926 src_iovs[0].iov_base = srcbuf; 2927 src_iovs[0].iov_len = sizeof(srcbuf); 2928 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2929 dst_iovs[0].iov_len = sizeof(tmp); 2930 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2931 2932 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2933 &src_iovs[0], 1, NULL, NULL, 0, 2934 ut_sequence_step_cb, &completed); 2935 CU_ASSERT_EQUAL(rc, 0); 2936 2937 src_iovs[1].iov_base = (void *)0xfeedbeef; 2938 src_iovs[1].iov_len = sizeof(tmp); 2939 dst_iovs[1].iov_base = dstbuf; 2940 dst_iovs[1].iov_len = sizeof(dstbuf); 2941 ut_domain_ctx_init(&domctx[1], tmp, sizeof(tmp), &src_iovs[1]); 2942 2943 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2944 &src_iovs[1], 1, g_ut_domain, &domctx[1], 0, 2945 ut_sequence_step_cb, &completed); 2946 CU_ASSERT_EQUAL(rc, 0); 2947 2948 ut_seq.complete = false; 2949 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2950 2951 poll_threads(); 2952 2953 CU_ASSERT_EQUAL(completed, 2); 2954 CU_ASSERT(ut_seq.complete); 2955 CU_ASSERT_EQUAL(ut_seq.status, 0); 2956 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2957 2958 /* Check a sequence with an accel buffer and a buffer in a regular memory domain */ 2959 memset(expected, 0xa5, sizeof(expected)); 2960 memset(dstbuf, 0, sizeof(dstbuf)); 2961 memset(tmp, 0, sizeof(tmp)); 2962 seq = NULL; 2963 completed = 0; 2964 2965 rc = spdk_accel_get_buf(ioch, 4096, &buf, &accel_domain, &accel_domain_ctx); 2966 CU_ASSERT_EQUAL(rc, 0); 2967 2968 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, accel_domain, accel_domain_ctx, 2969 0xa5, 0, ut_sequence_step_cb, &completed); 2970 CU_ASSERT_EQUAL(rc, 0); 2971 2972 src_iovs[0].iov_base = buf; 2973 src_iovs[0].iov_len = 4096; 2974 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2975 dst_iovs[0].iov_len = sizeof(dstbuf); 2976 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2977 2978 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2979 &src_iovs[0], 1, accel_domain, accel_domain_ctx, 0, 2980 ut_sequence_step_cb, &completed); 2981 CU_ASSERT_EQUAL(rc, 0); 2982 2983 ut_seq.complete = false; 2984 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2985 2986 poll_threads(); 2987 2988 CU_ASSERT_EQUAL(completed, 2); 2989 CU_ASSERT(ut_seq.complete); 2990 CU_ASSERT_EQUAL(ut_seq.status, 0); 2991 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2992 2993 spdk_accel_put_buf(ioch, buf, accel_domain, accel_domain_ctx); 2994 2995 g_module.supports_memory_domains = false; 2996 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2997 g_modules_opc[i] = modules[i]; 2998 } 2999 3000 ut_clear_operations(); 3001 spdk_put_io_channel(ioch); 3002 poll_threads(); 3003 } 3004 3005 #ifdef SPDK_CONFIG_ISAL_CRYPTO 3006 static void 3007 ut_encrypt_cb(void *cb_arg, int status) 3008 { 3009 int *completed = cb_arg; 3010 3011 CU_ASSERT_EQUAL(status, 0); 3012 3013 *completed = 1; 3014 } 3015 3016 static void 3017 test_sequence_crypto(void) 3018 { 3019 struct spdk_accel_sequence *seq = NULL; 3020 struct spdk_io_channel *ioch; 3021 struct spdk_accel_crypto_key *key; 3022 struct spdk_accel_crypto_key_create_param key_params = { 3023 .cipher = "AES_XTS", 3024 .hex_key = "00112233445566778899aabbccddeeff", 3025 .hex_key2 = "ffeeddccbbaa99887766554433221100", 3026 .key_name = "ut_key", 3027 }; 3028 struct ut_sequence ut_seq; 3029 unsigned char buf[4096], encrypted[4096] = {}, data[4096], tmp[3][4096]; 3030 struct iovec src_iovs[4], dst_iovs[4]; 3031 int rc, completed = 0; 3032 size_t i; 3033 3034 ioch = spdk_accel_get_io_channel(); 3035 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3036 3037 rc = spdk_accel_crypto_key_create(&key_params); 3038 CU_ASSERT_EQUAL(rc, 0); 3039 key = spdk_accel_crypto_key_get(key_params.key_name); 3040 SPDK_CU_ASSERT_FATAL(key != NULL); 3041 3042 for (i = 0; i < sizeof(data); ++i) { 3043 data[i] = (uint8_t)i & 0xff; 3044 } 3045 3046 dst_iovs[0].iov_base = encrypted; 3047 dst_iovs[0].iov_len = sizeof(encrypted); 3048 src_iovs[0].iov_base = data; 3049 src_iovs[0].iov_len = sizeof(data); 3050 rc = spdk_accel_submit_encrypt(ioch, key, &dst_iovs[0], 1, &src_iovs[0], 1, 0, 4096, 0, 3051 ut_encrypt_cb, &completed); 3052 CU_ASSERT_EQUAL(rc, 0); 3053 3054 while (!completed) { 3055 poll_threads(); 3056 } 3057 3058 /* Verify that encryption operation in a sequence produces the same result */ 3059 seq = NULL; 3060 completed = 0; 3061 3062 dst_iovs[0].iov_base = tmp[0]; 3063 dst_iovs[0].iov_len = sizeof(tmp[0]); 3064 src_iovs[0].iov_base = data; 3065 src_iovs[0].iov_len = sizeof(data); 3066 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3067 &src_iovs[0], 1, NULL, NULL, 0, 3068 ut_sequence_step_cb, &completed); 3069 CU_ASSERT_EQUAL(rc, 0); 3070 3071 dst_iovs[1].iov_base = tmp[1]; 3072 dst_iovs[1].iov_len = sizeof(tmp[1]); 3073 src_iovs[1].iov_base = tmp[0]; 3074 src_iovs[1].iov_len = sizeof(tmp[0]); 3075 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3076 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3077 ut_sequence_step_cb, &completed); 3078 CU_ASSERT_EQUAL(rc, 0); 3079 3080 dst_iovs[2].iov_base = buf; 3081 dst_iovs[2].iov_len = sizeof(buf); 3082 src_iovs[2].iov_base = tmp[1]; 3083 src_iovs[2].iov_len = sizeof(tmp[1]); 3084 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3085 &src_iovs[2], 1, NULL, NULL, 0, 3086 ut_sequence_step_cb, &completed); 3087 CU_ASSERT_EQUAL(rc, 0); 3088 3089 ut_seq.complete = false; 3090 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3091 3092 poll_threads(); 3093 3094 CU_ASSERT_EQUAL(completed, 3); 3095 CU_ASSERT(ut_seq.complete); 3096 CU_ASSERT_EQUAL(ut_seq.status, 0); 3097 CU_ASSERT_EQUAL(memcmp(buf, encrypted, sizeof(buf)), 0); 3098 3099 /* Check that decryption produces the original buffer */ 3100 seq = NULL; 3101 completed = 0; 3102 memset(buf, 0, sizeof(buf)); 3103 3104 dst_iovs[0].iov_base = tmp[0]; 3105 dst_iovs[0].iov_len = sizeof(tmp[0]); 3106 src_iovs[0].iov_base = encrypted; 3107 src_iovs[0].iov_len = sizeof(encrypted); 3108 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3109 &src_iovs[0], 1, NULL, NULL, 0, 3110 ut_sequence_step_cb, &completed); 3111 CU_ASSERT_EQUAL(rc, 0); 3112 3113 dst_iovs[1].iov_base = tmp[1]; 3114 dst_iovs[1].iov_len = sizeof(tmp[1]); 3115 src_iovs[1].iov_base = tmp[0]; 3116 src_iovs[1].iov_len = sizeof(tmp[0]); 3117 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3118 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3119 ut_sequence_step_cb, &completed); 3120 CU_ASSERT_EQUAL(rc, 0); 3121 3122 dst_iovs[2].iov_base = buf; 3123 dst_iovs[2].iov_len = sizeof(buf); 3124 src_iovs[2].iov_base = tmp[1]; 3125 src_iovs[2].iov_len = sizeof(tmp[1]); 3126 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3127 &src_iovs[2], 1, NULL, NULL, 0, 3128 ut_sequence_step_cb, &completed); 3129 CU_ASSERT_EQUAL(rc, 0); 3130 3131 ut_seq.complete = false; 3132 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3133 3134 poll_threads(); 3135 3136 CU_ASSERT_EQUAL(completed, 3); 3137 CU_ASSERT(ut_seq.complete); 3138 CU_ASSERT_EQUAL(ut_seq.status, 0); 3139 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3140 3141 /* Check encrypt + decrypt in a single sequence */ 3142 seq = NULL; 3143 completed = 0; 3144 memset(buf, 0, sizeof(buf)); 3145 3146 dst_iovs[0].iov_base = tmp[0]; 3147 dst_iovs[0].iov_len = sizeof(tmp[0]); 3148 src_iovs[0].iov_base = data; 3149 src_iovs[0].iov_len = sizeof(data); 3150 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3151 &src_iovs[0], 1, NULL, NULL, 0, 3152 ut_sequence_step_cb, &completed); 3153 CU_ASSERT_EQUAL(rc, 0); 3154 3155 dst_iovs[1].iov_base = tmp[1]; 3156 dst_iovs[1].iov_len = sizeof(tmp[1]); 3157 src_iovs[1].iov_base = tmp[0]; 3158 src_iovs[1].iov_len = sizeof(tmp[0]); 3159 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3160 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3161 ut_sequence_step_cb, &completed); 3162 CU_ASSERT_EQUAL(rc, 0); 3163 3164 3165 dst_iovs[2].iov_base = tmp[2]; 3166 dst_iovs[2].iov_len = sizeof(tmp[2]); 3167 src_iovs[2].iov_base = tmp[1]; 3168 src_iovs[2].iov_len = sizeof(tmp[1]); 3169 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[2], 1, NULL, NULL, 3170 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3171 ut_sequence_step_cb, &completed); 3172 CU_ASSERT_EQUAL(rc, 0); 3173 3174 dst_iovs[3].iov_base = buf; 3175 dst_iovs[3].iov_len = sizeof(buf); 3176 src_iovs[3].iov_base = tmp[2]; 3177 src_iovs[3].iov_len = sizeof(tmp[2]); 3178 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 3179 &src_iovs[3], 1, NULL, NULL, 0, 3180 ut_sequence_step_cb, &completed); 3181 CU_ASSERT_EQUAL(rc, 0); 3182 3183 ut_seq.complete = false; 3184 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3185 3186 poll_threads(); 3187 3188 CU_ASSERT_EQUAL(completed, 4); 3189 CU_ASSERT(ut_seq.complete); 3190 CU_ASSERT_EQUAL(ut_seq.status, 0); 3191 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3192 3193 rc = spdk_accel_crypto_key_destroy(key); 3194 CU_ASSERT_EQUAL(rc, 0); 3195 spdk_put_io_channel(ioch); 3196 poll_threads(); 3197 } 3198 #endif /* SPDK_CONFIG_ISAL_CRYPTO */ 3199 3200 static int 3201 ut_submit_crypto(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3202 { 3203 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3204 3205 spdk_accel_task_complete(task, 0); 3206 3207 return 0; 3208 } 3209 3210 struct ut_driver_operation { 3211 int complete_status; 3212 int submit_status; 3213 int count; 3214 bool supported; 3215 }; 3216 3217 static struct ut_driver_operation g_drv_operations[SPDK_ACCEL_OPC_LAST]; 3218 static bool g_ut_driver_async_continue; 3219 3220 static void 3221 ut_driver_async_continue(void *arg) 3222 { 3223 struct spdk_accel_sequence *seq = arg; 3224 3225 spdk_accel_sequence_continue(seq); 3226 } 3227 3228 static int 3229 ut_driver_execute_sequence(struct spdk_io_channel *ch, struct spdk_accel_sequence *seq) 3230 { 3231 struct spdk_accel_task *task; 3232 struct ut_driver_operation *drv_ops; 3233 3234 while ((task = spdk_accel_sequence_first_task(seq)) != NULL) { 3235 drv_ops = &g_drv_operations[task->op_code]; 3236 if (!drv_ops->supported) { 3237 break; 3238 } 3239 3240 drv_ops->count++; 3241 if (drv_ops->submit_status != 0) { 3242 return drv_ops->submit_status; 3243 } 3244 3245 if (drv_ops->complete_status != 0) { 3246 spdk_accel_task_complete(task, drv_ops->complete_status); 3247 break; 3248 } 3249 3250 switch (task->op_code) { 3251 case SPDK_ACCEL_OPC_DECOMPRESS: 3252 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3253 break; 3254 case SPDK_ACCEL_OPC_FILL: 3255 spdk_iov_memset(task->d.iovs, task->d.iovcnt, 3256 (int)(task->fill_pattern & 0xff)); 3257 break; 3258 default: 3259 CU_ASSERT(0 && "unexpected opcode"); 3260 break; 3261 } 3262 3263 spdk_accel_task_complete(task, 0); 3264 } 3265 3266 if (g_ut_driver_async_continue) { 3267 spdk_thread_send_msg(spdk_get_thread(), ut_driver_async_continue, seq); 3268 } else { 3269 spdk_accel_sequence_continue(seq); 3270 } 3271 3272 return 0; 3273 } 3274 3275 static struct spdk_io_channel * 3276 ut_driver_get_io_channel(void) 3277 { 3278 return (void *)0xdeadbeef; 3279 } 3280 3281 static struct spdk_accel_driver g_ut_driver = { 3282 .name = "ut", 3283 .execute_sequence = ut_driver_execute_sequence, 3284 .get_io_channel = ut_driver_get_io_channel, 3285 }; 3286 3287 SPDK_ACCEL_DRIVER_REGISTER(ut, &g_ut_driver); 3288 3289 static void 3290 test_sequence_driver(void) 3291 { 3292 struct spdk_accel_sequence *seq = NULL; 3293 struct spdk_io_channel *ioch; 3294 struct spdk_accel_crypto_key key = {}; 3295 struct ut_sequence ut_seq; 3296 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3297 char buf[4096], tmp[3][4096], expected[4096]; 3298 struct iovec src_iovs[3], dst_iovs[3]; 3299 int i, rc, completed = 0; 3300 3301 ioch = spdk_accel_get_io_channel(); 3302 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3303 rc = spdk_accel_set_driver("ut"); 3304 SPDK_CU_ASSERT_FATAL(rc == 0); 3305 3306 /* Override the submit_tasks function */ 3307 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3308 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3309 modules[i] = g_modules_opc[i]; 3310 g_modules_opc[i] = g_module; 3311 } 3312 /* Intercept crypto operations, as they should be executed by an accel module */ 3313 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].submit = ut_submit_crypto; 3314 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].submit = ut_submit_crypto; 3315 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3316 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3317 g_seq_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3318 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3319 3320 g_drv_operations[SPDK_ACCEL_OPC_FILL].supported = true; 3321 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].supported = true; 3322 3323 /* First check a sequence that is fully executed using a driver, with the copy at the end 3324 * being removed */ 3325 seq = NULL; 3326 completed = 0; 3327 memset(buf, 0, sizeof(buf)); 3328 memset(tmp[0], 0, sizeof(tmp[0])); 3329 memset(tmp[1], 0, sizeof(tmp[1])); 3330 memset(&expected[0], 0xa5, 2048); 3331 memset(&expected[2048], 0xbe, 2048); 3332 3333 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], 2048, NULL, NULL, 0xa5, 0, 3334 ut_sequence_step_cb, &completed); 3335 CU_ASSERT_EQUAL(rc, 0); 3336 rc = spdk_accel_append_fill(&seq, ioch, &tmp[0][2048], 2048, NULL, NULL, 0xbe, 0, 3337 ut_sequence_step_cb, &completed); 3338 CU_ASSERT_EQUAL(rc, 0); 3339 3340 dst_iovs[0].iov_base = tmp[1]; 3341 dst_iovs[0].iov_len = sizeof(tmp[1]); 3342 src_iovs[0].iov_base = tmp[0]; 3343 src_iovs[0].iov_len = sizeof(tmp[0]); 3344 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3345 &src_iovs[0], 1, NULL, NULL, 0, 3346 ut_sequence_step_cb, &completed); 3347 CU_ASSERT_EQUAL(rc, 0); 3348 3349 dst_iovs[1].iov_base = buf; 3350 dst_iovs[1].iov_len = sizeof(buf); 3351 src_iovs[1].iov_base = tmp[1]; 3352 src_iovs[1].iov_len = sizeof(tmp[1]); 3353 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3354 &src_iovs[1], 1, NULL, NULL, 0, 3355 ut_sequence_step_cb, &completed); 3356 CU_ASSERT_EQUAL(rc, 0); 3357 3358 ut_seq.complete = false; 3359 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3360 3361 poll_threads(); 3362 3363 CU_ASSERT_EQUAL(completed, 4); 3364 CU_ASSERT(ut_seq.complete); 3365 CU_ASSERT_EQUAL(ut_seq.status, 0); 3366 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3367 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3368 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 3369 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 2); 3370 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3371 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3372 3373 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3374 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3375 3376 /* Check a sequence when the first two operations are executed by a driver, while the rest 3377 * is executed via modules */ 3378 seq = NULL; 3379 completed = 0; 3380 memset(buf, 0, sizeof(buf)); 3381 memset(tmp[0], 0, sizeof(tmp[0])); 3382 memset(tmp[1], 0, sizeof(tmp[1])); 3383 memset(tmp[2], 0, sizeof(tmp[2])); 3384 memset(&expected[0], 0xfe, 4096); 3385 3386 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 0, 3387 ut_sequence_step_cb, &completed); 3388 CU_ASSERT_EQUAL(rc, 0); 3389 3390 dst_iovs[0].iov_base = tmp[1]; 3391 dst_iovs[0].iov_len = sizeof(tmp[1]); 3392 src_iovs[0].iov_base = tmp[0]; 3393 src_iovs[0].iov_len = sizeof(tmp[0]); 3394 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3395 &src_iovs[0], 1, NULL, NULL, 0, 3396 ut_sequence_step_cb, &completed); 3397 CU_ASSERT_EQUAL(rc, 0); 3398 3399 dst_iovs[1].iov_base = tmp[2]; 3400 dst_iovs[1].iov_len = sizeof(tmp[2]); 3401 src_iovs[1].iov_base = tmp[1]; 3402 src_iovs[1].iov_len = sizeof(tmp[1]); 3403 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3404 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3405 ut_sequence_step_cb, &completed); 3406 CU_ASSERT_EQUAL(rc, 0); 3407 3408 dst_iovs[2].iov_base = buf; 3409 dst_iovs[2].iov_len = sizeof(buf); 3410 src_iovs[2].iov_base = tmp[2]; 3411 src_iovs[2].iov_len = sizeof(tmp[2]); 3412 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3413 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3414 ut_sequence_step_cb, &completed); 3415 CU_ASSERT_EQUAL(rc, 0); 3416 3417 ut_seq.complete = false; 3418 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3419 3420 poll_threads(); 3421 3422 CU_ASSERT_EQUAL(completed, 4); 3423 CU_ASSERT(ut_seq.complete); 3424 CU_ASSERT_EQUAL(ut_seq.status, 0); 3425 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3426 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3427 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3428 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3429 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3430 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3431 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3432 3433 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3434 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3435 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3436 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3437 3438 /* Check sequence when the first and last operations are executed through modules, while the 3439 * ones in the middle are executed by the driver */ 3440 seq = NULL; 3441 completed = 0; 3442 memset(buf, 0, sizeof(buf)); 3443 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3444 memset(tmp[1], 0, sizeof(tmp[1])); 3445 memset(tmp[2], 0, sizeof(tmp[2])); 3446 memset(&expected[0], 0xfe, 2048); 3447 memset(&expected[2048], 0xa5, 2048); 3448 3449 dst_iovs[0].iov_base = tmp[1]; 3450 dst_iovs[0].iov_len = sizeof(tmp[1]); 3451 src_iovs[0].iov_base = tmp[0]; 3452 src_iovs[0].iov_len = sizeof(tmp[0]); 3453 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3454 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3455 ut_sequence_step_cb, &completed); 3456 CU_ASSERT_EQUAL(rc, 0); 3457 3458 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xfe, 0, 3459 ut_sequence_step_cb, &completed); 3460 CU_ASSERT_EQUAL(rc, 0); 3461 3462 dst_iovs[1].iov_base = tmp[2]; 3463 dst_iovs[1].iov_len = sizeof(tmp[2]); 3464 src_iovs[1].iov_base = tmp[1]; 3465 src_iovs[1].iov_len = sizeof(tmp[1]); 3466 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3467 &src_iovs[1], 1, NULL, NULL, 0, 3468 ut_sequence_step_cb, &completed); 3469 CU_ASSERT_EQUAL(rc, 0); 3470 3471 dst_iovs[2].iov_base = buf; 3472 dst_iovs[2].iov_len = sizeof(buf); 3473 src_iovs[2].iov_base = tmp[2]; 3474 src_iovs[2].iov_len = sizeof(tmp[2]); 3475 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3476 &src_iovs[2], 1, NULL, NULL, 0, 4096, 0, 3477 ut_sequence_step_cb, &completed); 3478 CU_ASSERT_EQUAL(rc, 0); 3479 3480 ut_seq.complete = false; 3481 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3482 3483 poll_threads(); 3484 3485 CU_ASSERT_EQUAL(completed, 4); 3486 CU_ASSERT(ut_seq.complete); 3487 CU_ASSERT_EQUAL(ut_seq.status, 0); 3488 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3489 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3490 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3491 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3492 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3493 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3494 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3495 3496 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3497 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3498 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3499 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3500 3501 /* Check a sequence with operations executed by: module, driver, module, driver */ 3502 seq = NULL; 3503 completed = 0; 3504 memset(buf, 0, sizeof(buf)); 3505 memset(tmp[0], 0x5a, sizeof(tmp[0])); 3506 memset(tmp[1], 0, sizeof(tmp[1])); 3507 memset(tmp[2], 0, sizeof(tmp[2])); 3508 memset(&expected[0], 0xef, 2048); 3509 memset(&expected[2048], 0x5a, 2048); 3510 3511 dst_iovs[0].iov_base = tmp[1]; 3512 dst_iovs[0].iov_len = sizeof(tmp[1]); 3513 src_iovs[0].iov_base = tmp[0]; 3514 src_iovs[0].iov_len = sizeof(tmp[0]); 3515 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3516 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3517 ut_sequence_step_cb, &completed); 3518 CU_ASSERT_EQUAL(rc, 0); 3519 3520 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3521 ut_sequence_step_cb, &completed); 3522 CU_ASSERT_EQUAL(rc, 0); 3523 3524 dst_iovs[1].iov_base = tmp[2]; 3525 dst_iovs[1].iov_len = sizeof(tmp[2]); 3526 src_iovs[1].iov_base = tmp[1]; 3527 src_iovs[1].iov_len = sizeof(tmp[1]); 3528 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3529 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3530 ut_sequence_step_cb, &completed); 3531 CU_ASSERT_EQUAL(rc, 0); 3532 3533 dst_iovs[2].iov_base = buf; 3534 dst_iovs[2].iov_len = sizeof(buf); 3535 src_iovs[2].iov_base = tmp[2]; 3536 src_iovs[2].iov_len = sizeof(tmp[2]); 3537 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3538 &src_iovs[2], 1, NULL, NULL, 0, 3539 ut_sequence_step_cb, &completed); 3540 CU_ASSERT_EQUAL(rc, 0); 3541 3542 ut_seq.complete = false; 3543 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3544 3545 poll_threads(); 3546 3547 CU_ASSERT_EQUAL(completed, 4); 3548 CU_ASSERT(ut_seq.complete); 3549 CU_ASSERT_EQUAL(ut_seq.status, 0); 3550 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3551 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3552 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3553 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3554 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3555 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3556 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3557 3558 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3559 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3560 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3561 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3562 3563 /* Check that an error returned from driver's execute_sequence() will fail the whole 3564 * sequence and any subsequent operations won't be processed */ 3565 seq = NULL; 3566 completed = 0; 3567 memset(buf, 0, sizeof(buf)); 3568 memset(expected, 0, sizeof(expected)); 3569 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3570 g_drv_operations[SPDK_ACCEL_OPC_FILL].submit_status = -EPERM; 3571 3572 dst_iovs[0].iov_base = tmp[1]; 3573 dst_iovs[0].iov_len = sizeof(tmp[1]); 3574 src_iovs[0].iov_base = tmp[0]; 3575 src_iovs[0].iov_len = sizeof(tmp[0]); 3576 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3577 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3578 ut_sequence_step_cb, &completed); 3579 CU_ASSERT_EQUAL(rc, 0); 3580 3581 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3582 ut_sequence_step_cb, &completed); 3583 CU_ASSERT_EQUAL(rc, 0); 3584 3585 dst_iovs[1].iov_base = buf; 3586 dst_iovs[1].iov_len = sizeof(buf); 3587 src_iovs[1].iov_base = tmp[1]; 3588 src_iovs[1].iov_len = sizeof(tmp[1]); 3589 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3590 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3591 ut_sequence_step_cb, &completed); 3592 CU_ASSERT_EQUAL(rc, 0); 3593 3594 ut_seq.complete = false; 3595 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3596 3597 poll_threads(); 3598 3599 CU_ASSERT_EQUAL(completed, 3); 3600 CU_ASSERT(ut_seq.complete); 3601 CU_ASSERT_EQUAL(ut_seq.status, -EPERM); 3602 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3603 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3604 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 0); 3605 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3606 CU_ASSERT_EQUAL(memcmp(buf, expected, 4096), 0); 3607 3608 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3609 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3610 g_drv_operations[SPDK_ACCEL_OPC_FILL].submit_status = 0; 3611 3612 /* Check that a failed task completed by a driver will cause the whole sequence to be failed 3613 * and any subsequent operations won't be processed */ 3614 seq = NULL; 3615 completed = 0; 3616 memset(buf, 0, sizeof(buf)); 3617 memset(expected, 0, sizeof(expected)); 3618 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3619 g_drv_operations[SPDK_ACCEL_OPC_FILL].complete_status = -ENOENT; 3620 3621 dst_iovs[0].iov_base = tmp[1]; 3622 dst_iovs[0].iov_len = sizeof(tmp[1]); 3623 src_iovs[0].iov_base = tmp[0]; 3624 src_iovs[0].iov_len = sizeof(tmp[0]); 3625 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3626 &src_iovs[0], 1, NULL, NULL, 0, 4096, 0, 3627 ut_sequence_step_cb, &completed); 3628 CU_ASSERT_EQUAL(rc, 0); 3629 3630 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 0, 3631 ut_sequence_step_cb, &completed); 3632 CU_ASSERT_EQUAL(rc, 0); 3633 3634 dst_iovs[1].iov_base = buf; 3635 dst_iovs[1].iov_len = sizeof(buf); 3636 src_iovs[1].iov_base = tmp[1]; 3637 src_iovs[1].iov_len = sizeof(tmp[1]); 3638 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3639 &src_iovs[1], 1, NULL, NULL, 0, 4096, 0, 3640 ut_sequence_step_cb, &completed); 3641 CU_ASSERT_EQUAL(rc, 0); 3642 3643 ut_seq.complete = false; 3644 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3645 3646 poll_threads(); 3647 3648 CU_ASSERT_EQUAL(completed, 3); 3649 CU_ASSERT(ut_seq.complete); 3650 CU_ASSERT_EQUAL(ut_seq.status, -ENOENT); 3651 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3652 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3653 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 0); 3654 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3655 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3656 3657 g_drv_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 3658 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3659 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3660 3661 /* Check asynchronous spdk_accel_sequence_continue() */ 3662 g_ut_driver_async_continue = true; 3663 seq = NULL; 3664 completed = 0; 3665 memset(buf, 0, sizeof(buf)); 3666 memset(tmp[0], 0, sizeof(tmp[0])); 3667 memset(&expected[0], 0xfe, 4096); 3668 3669 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 0, 3670 ut_sequence_step_cb, &completed); 3671 CU_ASSERT_EQUAL(rc, 0); 3672 3673 dst_iovs[0].iov_base = buf; 3674 dst_iovs[0].iov_len = sizeof(buf); 3675 src_iovs[0].iov_base = tmp[0]; 3676 src_iovs[0].iov_len = sizeof(tmp[0]); 3677 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3678 &src_iovs[0], 1, NULL, NULL, 0, 3679 ut_sequence_step_cb, &completed); 3680 CU_ASSERT_EQUAL(rc, 0); 3681 3682 ut_seq.complete = false; 3683 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3684 3685 poll_threads(); 3686 3687 CU_ASSERT_EQUAL(completed, 2); 3688 CU_ASSERT(ut_seq.complete); 3689 CU_ASSERT_EQUAL(ut_seq.status, 0); 3690 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3691 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3692 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3693 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3694 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3695 3696 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3697 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3698 3699 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3700 g_modules_opc[i] = modules[i]; 3701 } 3702 3703 /* Clear the driver so that other tests won't use it */ 3704 g_accel_driver = NULL; 3705 memset(&g_drv_operations, 0, sizeof(g_drv_operations)); 3706 3707 ut_clear_operations(); 3708 spdk_put_io_channel(ioch); 3709 poll_threads(); 3710 } 3711 3712 struct ut_saved_iovs { 3713 struct iovec src; 3714 struct iovec dst; 3715 }; 3716 3717 static struct ut_saved_iovs g_seq_saved_iovs[SPDK_ACCEL_OPC_LAST]; 3718 3719 static int 3720 ut_submit_save_iovs(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3721 { 3722 SPDK_CU_ASSERT_FATAL(task->s.iovcnt == 1); 3723 SPDK_CU_ASSERT_FATAL(task->d.iovcnt == 1); 3724 3725 g_seq_saved_iovs[task->op_code].src = task->s.iovs[0]; 3726 g_seq_saved_iovs[task->op_code].dst = task->d.iovs[0]; 3727 3728 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3729 3730 spdk_accel_task_complete(task, 0); 3731 3732 return 0; 3733 } 3734 3735 static void 3736 test_sequence_same_iovs(void) 3737 { 3738 struct spdk_accel_sequence *seq = NULL; 3739 struct spdk_io_channel *ioch; 3740 struct spdk_accel_crypto_key key = {}; 3741 struct ut_sequence ut_seq; 3742 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3743 char buf[4096], tmp[4096], expected[4096]; 3744 struct iovec iovs[3], expected_siov, expected_diov; 3745 struct spdk_memory_domain *domain; 3746 void *accel_buf, *domain_ctx; 3747 int i, rc, completed = 0; 3748 3749 ioch = spdk_accel_get_io_channel(); 3750 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3751 3752 /* Override the submit_tasks function */ 3753 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3754 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3755 modules[i] = g_modules_opc[i]; 3756 g_modules_opc[i] = g_module; 3757 } 3758 /* Intercept crypto operations, as they should be executed by an accel module */ 3759 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].submit = ut_submit_save_iovs; 3760 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].submit = ut_submit_save_iovs; 3761 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3762 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3763 3764 /* Check that it's possible to use the same iovec ptr for different operations */ 3765 seq = NULL; 3766 completed = 0; 3767 memset(buf, 0, sizeof(buf)); 3768 memset(expected, 0xa5, sizeof(expected)); 3769 3770 iovs[0].iov_base = expected; 3771 iovs[0].iov_len = sizeof(expected); 3772 iovs[1].iov_base = tmp; 3773 iovs[1].iov_len = sizeof(tmp); 3774 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, NULL, NULL, 3775 &iovs[0], 1, NULL, NULL, 0, 4096, 0, 3776 ut_sequence_step_cb, &completed); 3777 CU_ASSERT_EQUAL(rc, 0); 3778 /* Reuse iov[1] as src */ 3779 iovs[2].iov_base = buf; 3780 iovs[2].iov_len = sizeof(buf); 3781 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3782 &iovs[1], 1, NULL, NULL, 0, 4096, 0, 3783 ut_sequence_step_cb, &completed); 3784 CU_ASSERT_EQUAL(rc, 0); 3785 3786 ut_seq.complete = false; 3787 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3788 3789 poll_threads(); 3790 3791 CU_ASSERT_EQUAL(completed, 2); 3792 CU_ASSERT(ut_seq.complete); 3793 CU_ASSERT_EQUAL(ut_seq.status, 0); 3794 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3795 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3796 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3797 expected_siov.iov_base = expected; 3798 expected_siov.iov_len = sizeof(expected); 3799 expected_diov.iov_base = tmp; 3800 expected_diov.iov_len = sizeof(tmp); 3801 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].src, 3802 &expected_siov, sizeof(expected_siov)), 0); 3803 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].dst, 3804 &expected_diov, sizeof(expected_diov)), 0); 3805 expected_siov.iov_base = tmp; 3806 expected_siov.iov_len = sizeof(tmp); 3807 expected_diov.iov_base = buf; 3808 expected_diov.iov_len = sizeof(buf); 3809 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].src, 3810 &expected_siov, sizeof(expected_siov)), 0); 3811 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].dst, 3812 &expected_diov, sizeof(expected_diov)), 0); 3813 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3814 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3815 3816 /* Check the same with an accel buffer */ 3817 seq = NULL; 3818 completed = 0; 3819 memset(buf, 0, sizeof(buf)); 3820 memset(expected, 0x5a, sizeof(expected)); 3821 3822 rc = spdk_accel_get_buf(ioch, sizeof(buf), &accel_buf, &domain, &domain_ctx); 3823 CU_ASSERT_EQUAL(rc, 0); 3824 3825 iovs[0].iov_base = expected; 3826 iovs[0].iov_len = sizeof(expected); 3827 iovs[1].iov_base = accel_buf; 3828 iovs[1].iov_len = sizeof(buf); 3829 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, domain, domain_ctx, 3830 &iovs[0], 1, NULL, NULL, 0, 4096, 0, 3831 ut_sequence_step_cb, &completed); 3832 CU_ASSERT_EQUAL(rc, 0); 3833 /* Reuse iov[1] as src */ 3834 iovs[2].iov_base = buf; 3835 iovs[2].iov_len = sizeof(buf); 3836 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3837 &iovs[1], 1, domain, domain_ctx, 0, 4096, 0, 3838 ut_sequence_step_cb, &completed); 3839 CU_ASSERT_EQUAL(rc, 0); 3840 3841 ut_seq.complete = false; 3842 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3843 3844 poll_threads(); 3845 3846 CU_ASSERT_EQUAL(completed, 2); 3847 CU_ASSERT(ut_seq.complete); 3848 CU_ASSERT_EQUAL(ut_seq.status, 0); 3849 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3850 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3851 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3852 expected_siov.iov_base = expected; 3853 expected_siov.iov_len = sizeof(expected); 3854 expected_diov.iov_base = buf; 3855 expected_diov.iov_len = sizeof(buf); 3856 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].src, 3857 &expected_siov, sizeof(expected_siov)), 0); 3858 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].dst, 3859 &expected_diov, sizeof(expected_diov)), 0); 3860 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].dst, 3861 &g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].src, 3862 sizeof(struct iovec)), 0); 3863 spdk_accel_put_buf(ioch, accel_buf, domain, domain_ctx); 3864 3865 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3866 g_modules_opc[i] = modules[i]; 3867 } 3868 3869 ut_clear_operations(); 3870 spdk_put_io_channel(ioch); 3871 poll_threads(); 3872 } 3873 3874 static void 3875 test_sequence_crc32(void) 3876 { 3877 struct spdk_accel_sequence *seq = NULL; 3878 struct spdk_io_channel *ioch; 3879 struct ut_sequence ut_seq; 3880 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3881 char buf[4096], tmp[3][4096]; 3882 struct iovec src_iovs[4], dst_iovs[4]; 3883 uint32_t crc, crc2; 3884 int i, rc, completed; 3885 3886 ioch = spdk_accel_get_io_channel(); 3887 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3888 3889 /* Override the submit_tasks function */ 3890 g_module_if.submit_tasks = ut_sequnce_submit_tasks; 3891 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3892 g_seq_operations[i].submit = sw_accel_submit_tasks; 3893 modules[i] = g_modules_opc[i]; 3894 g_modules_opc[i] = g_module; 3895 } 3896 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 3897 3898 /* First check the simplest case - single crc32c operation */ 3899 seq = NULL; 3900 completed = 0; 3901 crc = 0; 3902 memset(buf, 0xa5, sizeof(buf)); 3903 3904 src_iovs[0].iov_base = buf; 3905 src_iovs[0].iov_len = sizeof(buf); 3906 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3907 ut_sequence_step_cb, &completed); 3908 CU_ASSERT_EQUAL(rc, 0); 3909 3910 ut_seq.complete = false; 3911 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3912 3913 poll_threads(); 3914 CU_ASSERT_EQUAL(completed, 1); 3915 CU_ASSERT(ut_seq.complete); 3916 CU_ASSERT_EQUAL(ut_seq.status, 0); 3917 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3918 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3919 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3920 3921 /* Now check copy+crc - this should remove the copy operation and change the buffer for the 3922 * crc operation */ 3923 seq = NULL; 3924 completed = 0; 3925 crc = 0; 3926 memset(buf, 0x5a, sizeof(buf)); 3927 memset(&tmp[0], 0, sizeof(tmp[0])); 3928 3929 dst_iovs[0].iov_base = tmp[0]; 3930 dst_iovs[0].iov_len = sizeof(tmp[0]); 3931 src_iovs[0].iov_base = buf; 3932 src_iovs[0].iov_len = sizeof(buf); 3933 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3934 &src_iovs[0], 1, NULL, NULL, 0, 3935 ut_sequence_step_cb, &completed); 3936 CU_ASSERT_EQUAL(rc, 0); 3937 3938 src_iovs[1].iov_base = tmp[0]; 3939 src_iovs[1].iov_len = sizeof(tmp[0]); 3940 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 3941 ut_sequence_step_cb, &completed); 3942 CU_ASSERT_EQUAL(rc, 0); 3943 3944 ut_seq.complete = false; 3945 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3946 3947 poll_threads(); 3948 CU_ASSERT_EQUAL(completed, 2); 3949 CU_ASSERT(ut_seq.complete); 3950 CU_ASSERT_EQUAL(ut_seq.status, 0); 3951 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3952 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 3953 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3954 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3955 3956 /* Check crc+copy - this time the copy cannot be removed, because there's no operation 3957 * before crc to change the buffer */ 3958 seq = NULL; 3959 completed = 0; 3960 crc = 0; 3961 memset(buf, 0, sizeof(buf)); 3962 memset(&tmp[0], 0xa5, sizeof(tmp[0])); 3963 3964 src_iovs[0].iov_base = tmp[0]; 3965 src_iovs[0].iov_len = sizeof(tmp[0]); 3966 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3967 ut_sequence_step_cb, &completed); 3968 CU_ASSERT_EQUAL(rc, 0); 3969 3970 dst_iovs[1].iov_base = buf; 3971 dst_iovs[1].iov_len = sizeof(buf); 3972 src_iovs[1].iov_base = tmp[0]; 3973 src_iovs[1].iov_len = sizeof(tmp[0]); 3974 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3975 &src_iovs[1], 1, NULL, NULL, 0, 3976 ut_sequence_step_cb, &completed); 3977 CU_ASSERT_EQUAL(rc, 0); 3978 3979 ut_seq.complete = false; 3980 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3981 3982 poll_threads(); 3983 CU_ASSERT_EQUAL(completed, 2); 3984 CU_ASSERT(ut_seq.complete); 3985 CU_ASSERT_EQUAL(ut_seq.status, 0); 3986 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3987 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 3988 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 3989 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 3990 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3991 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 3992 3993 /* Check a sequence with an operation at the beginning that can have its buffer changed, two 3994 * crc operations and a copy at the end. The copy should be removed and the dst buffer of 3995 * the first operation and the src buffer of the crc operations should be changed. 3996 */ 3997 seq = NULL; 3998 completed = 0; 3999 crc = crc2 = 0; 4000 memset(buf, 0, sizeof(buf)); 4001 memset(&tmp[0], 0x5a, sizeof(tmp[0])); 4002 dst_iovs[0].iov_base = tmp[1]; 4003 dst_iovs[0].iov_len = sizeof(tmp[1]); 4004 src_iovs[0].iov_base = tmp[0]; 4005 src_iovs[0].iov_len = sizeof(tmp[0]); 4006 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 4007 &src_iovs[0], 1, NULL, NULL, 0, 4008 ut_sequence_step_cb, &completed); 4009 CU_ASSERT_EQUAL(rc, 0); 4010 4011 src_iovs[1].iov_base = tmp[1]; 4012 src_iovs[1].iov_len = sizeof(tmp[1]); 4013 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 4014 ut_sequence_step_cb, &completed); 4015 CU_ASSERT_EQUAL(rc, 0); 4016 4017 src_iovs[2].iov_base = tmp[1]; 4018 src_iovs[2].iov_len = sizeof(tmp[1]); 4019 rc = spdk_accel_append_crc32c(&seq, ioch, &crc2, &src_iovs[2], 1, NULL, NULL, 0, 4020 ut_sequence_step_cb, &completed); 4021 CU_ASSERT_EQUAL(rc, 0); 4022 4023 dst_iovs[3].iov_base = buf; 4024 dst_iovs[3].iov_len = sizeof(buf); 4025 src_iovs[3].iov_base = tmp[1]; 4026 src_iovs[3].iov_len = sizeof(tmp[1]); 4027 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 4028 &src_iovs[3], 1, NULL, NULL, 0, 4029 ut_sequence_step_cb, &completed); 4030 CU_ASSERT_EQUAL(rc, 0); 4031 4032 ut_seq.complete = false; 4033 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4034 4035 poll_threads(); 4036 CU_ASSERT_EQUAL(completed, 4); 4037 CU_ASSERT(ut_seq.complete); 4038 CU_ASSERT_EQUAL(ut_seq.status, 0); 4039 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 4040 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 2); 4041 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 4042 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 4043 CU_ASSERT_EQUAL(crc, crc2); 4044 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 4045 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 4046 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 4047 4048 /* Check that a copy won't be removed if the buffers don't match */ 4049 seq = NULL; 4050 completed = 0; 4051 crc = 0; 4052 memset(buf, 0, sizeof(buf)); 4053 memset(&tmp[0], 0xa5, 2048); 4054 memset(&tmp[1], 0xfe, sizeof(tmp[1])); 4055 memset(&tmp[2], 0xfe, sizeof(tmp[1])); 4056 dst_iovs[0].iov_base = &tmp[1][2048]; 4057 dst_iovs[0].iov_len = 2048; 4058 src_iovs[0].iov_base = tmp[0]; 4059 src_iovs[0].iov_len = 2048; 4060 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 4061 &src_iovs[0], 1, NULL, NULL, 0, 4062 ut_sequence_step_cb, &completed); 4063 CU_ASSERT_EQUAL(rc, 0); 4064 4065 src_iovs[1].iov_base = &tmp[1][2048]; 4066 src_iovs[1].iov_len = 2048; 4067 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 4068 ut_sequence_step_cb, &completed); 4069 CU_ASSERT_EQUAL(rc, 0); 4070 4071 dst_iovs[2].iov_base = buf; 4072 dst_iovs[2].iov_len = sizeof(buf); 4073 src_iovs[2].iov_base = tmp[1]; 4074 src_iovs[2].iov_len = sizeof(tmp[1]); 4075 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 4076 &src_iovs[2], 1, NULL, NULL, 0, 4077 ut_sequence_step_cb, &completed); 4078 CU_ASSERT_EQUAL(rc, 0); 4079 4080 ut_seq.complete = false; 4081 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4082 4083 poll_threads(); 4084 CU_ASSERT_EQUAL(completed, 3); 4085 CU_ASSERT(ut_seq.complete); 4086 CU_ASSERT_EQUAL(ut_seq.status, 0); 4087 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 4088 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 4089 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 4090 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], 2048, ~0u)); 4091 CU_ASSERT_EQUAL(memcmp(buf, tmp[2], 2048), 0); 4092 CU_ASSERT_EQUAL(memcmp(&buf[2048], tmp[0], 2048), 0); 4093 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 4094 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 4095 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 4096 4097 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 4098 g_modules_opc[i] = modules[i]; 4099 } 4100 4101 ut_clear_operations(); 4102 spdk_put_io_channel(ioch); 4103 poll_threads(); 4104 } 4105 4106 static int 4107 test_sequence_setup(void) 4108 { 4109 int rc; 4110 4111 allocate_cores(1); 4112 allocate_threads(1); 4113 set_thread(0); 4114 4115 rc = spdk_iobuf_initialize(); 4116 if (rc != 0) { 4117 CU_ASSERT(false); 4118 return -1; 4119 } 4120 4121 rc = spdk_accel_initialize(); 4122 if (rc != 0) { 4123 CU_ASSERT(false); 4124 return -1; 4125 } 4126 4127 return 0; 4128 } 4129 4130 static void 4131 finish_cb(void *cb_arg) 4132 { 4133 bool *done = cb_arg; 4134 4135 *done = true; 4136 } 4137 4138 static int 4139 test_sequence_cleanup(void) 4140 { 4141 bool done = false; 4142 4143 spdk_accel_finish(finish_cb, &done); 4144 4145 while (!done) { 4146 poll_threads(); 4147 } 4148 4149 done = false; 4150 spdk_iobuf_finish(finish_cb, &done); 4151 while (!done) { 4152 poll_threads(); 4153 } 4154 4155 free_threads(); 4156 free_cores(); 4157 4158 return 0; 4159 } 4160 4161 int 4162 main(int argc, char **argv) 4163 { 4164 CU_pSuite suite = NULL, seq_suite; 4165 unsigned int num_failures; 4166 4167 CU_initialize_registry(); 4168 4169 /* Sequence tests require accel to be initialized normally, so run them before the other 4170 * tests which register accel modules which aren't fully implemented, causing accel 4171 * initialization to fail. 4172 */ 4173 seq_suite = CU_add_suite("accel_sequence", test_sequence_setup, test_sequence_cleanup); 4174 CU_ADD_TEST(seq_suite, test_sequence_fill_copy); 4175 CU_ADD_TEST(seq_suite, test_sequence_abort); 4176 CU_ADD_TEST(seq_suite, test_sequence_append_error); 4177 CU_ADD_TEST(seq_suite, test_sequence_completion_error); 4178 #ifdef SPDK_CONFIG_ISAL /* accel_sw requires isa-l for compression */ 4179 CU_ADD_TEST(seq_suite, test_sequence_decompress); 4180 CU_ADD_TEST(seq_suite, test_sequence_reverse); 4181 #endif 4182 CU_ADD_TEST(seq_suite, test_sequence_copy_elision); 4183 CU_ADD_TEST(seq_suite, test_sequence_accel_buffers); 4184 CU_ADD_TEST(seq_suite, test_sequence_memory_domain); 4185 CU_ADD_TEST(seq_suite, test_sequence_module_memory_domain); 4186 #ifdef SPDK_CONFIG_ISAL_CRYPTO /* accel_sw requires isa-l-crypto for crypto operations */ 4187 CU_ADD_TEST(seq_suite, test_sequence_crypto); 4188 #endif 4189 CU_ADD_TEST(seq_suite, test_sequence_driver); 4190 CU_ADD_TEST(seq_suite, test_sequence_same_iovs); 4191 CU_ADD_TEST(seq_suite, test_sequence_crc32); 4192 4193 suite = CU_add_suite("accel", test_setup, test_cleanup); 4194 CU_ADD_TEST(suite, test_spdk_accel_task_complete); 4195 CU_ADD_TEST(suite, test_get_task); 4196 CU_ADD_TEST(suite, test_spdk_accel_submit_copy); 4197 CU_ADD_TEST(suite, test_spdk_accel_submit_dualcast); 4198 CU_ADD_TEST(suite, test_spdk_accel_submit_compare); 4199 CU_ADD_TEST(suite, test_spdk_accel_submit_fill); 4200 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c); 4201 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32cv); 4202 CU_ADD_TEST(suite, test_spdk_accel_submit_copy_crc32c); 4203 CU_ADD_TEST(suite, test_spdk_accel_submit_xor); 4204 CU_ADD_TEST(suite, test_spdk_accel_module_find_by_name); 4205 CU_ADD_TEST(suite, test_spdk_accel_module_register); 4206 4207 num_failures = spdk_ut_run_tests(argc, argv, NULL); 4208 CU_cleanup_registry(); 4209 4210 return num_failures; 4211 } 4212