1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2021 Intel Corporation. 3 * All rights reserved. 4 * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk_internal/cunit.h" 8 #include "spdk_internal/mock.h" 9 #include "spdk/accel_module.h" 10 #include "thread/thread_internal.h" 11 #include "common/lib/ut_multithread.c" 12 #include "common/lib/test_iobuf.c" 13 #include "accel/accel.c" 14 #include "accel/accel_sw.c" 15 #include "unit/lib/json_mock.c" 16 17 DEFINE_STUB_V(spdk_memory_domain_destroy, (struct spdk_memory_domain *domain)); 18 DEFINE_STUB(spdk_memory_domain_get_dma_device_id, const char *, 19 (struct spdk_memory_domain *domain), "UT_DMA"); 20 DEFINE_STUB(spdk_memory_domain_get_system_domain, struct spdk_memory_domain *, (void), 21 (void *)0xabc); 22 DEFINE_STUB_V(spdk_memory_domain_set_invalidate, (struct spdk_memory_domain *domain, 23 spdk_memory_domain_invalidate_data_cb invalidate_cb)); 24 DEFINE_STUB_V(spdk_memory_domain_set_translation, (struct spdk_memory_domain *domain, 25 spdk_memory_domain_translate_memory_cb translate_cb)); 26 27 int 28 spdk_memory_domain_create(struct spdk_memory_domain **domain, enum spdk_dma_device_type type, 29 struct spdk_memory_domain_ctx *ctx, const char *id) 30 { 31 *domain = (void *)0xdeadbeef; 32 33 return 0; 34 } 35 36 struct ut_domain_ctx { 37 struct iovec iov; 38 struct iovec expected; 39 int pull_submit_status; 40 int push_submit_status; 41 int pull_complete_status; 42 int push_complete_status; 43 }; 44 45 static struct spdk_memory_domain *g_ut_domain = (void *)0xa55e1; 46 47 int 48 spdk_memory_domain_pull_data(struct spdk_memory_domain *sd, void *sctx, struct iovec *siov, 49 uint32_t siovcnt, struct iovec *diov, uint32_t diovcnt, 50 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 51 { 52 struct ut_domain_ctx *ctx = sctx; 53 54 CU_ASSERT_EQUAL(sd, g_ut_domain); 55 CU_ASSERT_EQUAL(siovcnt, 1); 56 CU_ASSERT_EQUAL(memcmp(siov, &ctx->expected, sizeof(*siov)), 0); 57 58 if (ctx->pull_submit_status != 0) { 59 return ctx->pull_submit_status; 60 } 61 62 if (ctx->pull_complete_status != 0) { 63 cpl_cb(cpl_arg, ctx->pull_complete_status); 64 return 0; 65 } 66 67 spdk_iovcpy(&ctx->iov, 1, diov, diovcnt); 68 cpl_cb(cpl_arg, 0); 69 70 return 0; 71 } 72 73 int 74 spdk_memory_domain_push_data(struct spdk_memory_domain *dd, void *dctx, struct iovec *diov, 75 uint32_t diovcnt, struct iovec *siov, uint32_t siovcnt, 76 spdk_memory_domain_data_cpl_cb cpl_cb, void *cpl_arg) 77 { 78 struct ut_domain_ctx *ctx = dctx; 79 80 CU_ASSERT_EQUAL(dd, g_ut_domain); 81 CU_ASSERT_EQUAL(diovcnt, 1); 82 CU_ASSERT_EQUAL(memcmp(diov, &ctx->expected, sizeof(*diov)), 0); 83 84 if (ctx->push_submit_status != 0) { 85 return ctx->push_submit_status; 86 } 87 88 if (ctx->push_complete_status != 0) { 89 cpl_cb(cpl_arg, ctx->push_complete_status); 90 return 0; 91 } 92 93 spdk_iovcpy(siov, siovcnt, &ctx->iov, 1); 94 cpl_cb(cpl_arg, 0); 95 96 return 0; 97 } 98 99 /* global vars and setup/cleanup functions used for all test functions */ 100 struct spdk_accel_module_if g_module_if = {}; 101 struct accel_module g_module = { .module = &g_module_if }; 102 struct spdk_io_channel *g_ch = NULL; 103 struct accel_io_channel *g_accel_ch = NULL; 104 struct sw_accel_io_channel *g_sw_ch = NULL; 105 struct spdk_io_channel *g_module_ch = NULL; 106 107 static uint64_t g_opc_mask = 0; 108 109 static uint64_t 110 _accel_op_to_bit(enum spdk_accel_opcode opc) 111 { 112 return (1 << opc); 113 } 114 115 static bool 116 _supports_opcode(enum spdk_accel_opcode opc) 117 { 118 if (_accel_op_to_bit(opc) & g_opc_mask) { 119 return true; 120 } 121 return false; 122 } 123 124 static int 125 test_setup(void) 126 { 127 int i; 128 129 g_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct accel_io_channel)); 130 if (g_ch == NULL) { 131 /* for some reason the assert fatal macro doesn't work in the setup function. */ 132 CU_ASSERT(false); 133 return -1; 134 } 135 g_accel_ch = (struct accel_io_channel *)((char *)g_ch + sizeof(struct spdk_io_channel)); 136 g_module_ch = calloc(1, sizeof(struct spdk_io_channel) + sizeof(struct sw_accel_io_channel)); 137 if (g_module_ch == NULL) { 138 CU_ASSERT(false); 139 return -1; 140 } 141 142 g_module_if.submit_tasks = sw_accel_submit_tasks; 143 g_module_if.name = "software"; 144 for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) { 145 g_accel_ch->module_ch[i] = g_module_ch; 146 g_modules_opc[i] = g_module; 147 } 148 g_sw_ch = (struct sw_accel_io_channel *)((char *)g_module_ch + sizeof( 149 struct spdk_io_channel)); 150 /* Prevent lazy initialization of poller. */ 151 g_sw_ch->completion_poller = (void *)0xdeadbeef; 152 STAILQ_INIT(&g_sw_ch->tasks_to_complete); 153 g_module_if.supports_opcode = _supports_opcode; 154 return 0; 155 } 156 157 static int 158 test_cleanup(void) 159 { 160 free(g_ch); 161 free(g_module_ch); 162 163 return 0; 164 } 165 166 #define DUMMY_ARG 0xDEADBEEF 167 static bool g_dummy_cb_called = false; 168 static void 169 dummy_cb_fn(void *cb_arg, int status) 170 { 171 CU_ASSERT(*(uint32_t *)cb_arg == DUMMY_ARG); 172 CU_ASSERT(status == 0); 173 g_dummy_cb_called = true; 174 } 175 176 static void 177 test_spdk_accel_task_complete(void) 178 { 179 struct spdk_accel_task accel_task = {}; 180 struct spdk_accel_task *expected_accel_task = NULL; 181 uint32_t cb_arg = DUMMY_ARG; 182 int status = 0; 183 184 accel_task.accel_ch = g_accel_ch; 185 accel_task.cb_fn = dummy_cb_fn; 186 accel_task.cb_arg = &cb_arg; 187 STAILQ_INIT(&g_accel_ch->task_pool); 188 189 /* Confirm cb is called and task added to list. */ 190 spdk_accel_task_complete(&accel_task, status); 191 CU_ASSERT(g_dummy_cb_called == true); 192 expected_accel_task = STAILQ_FIRST(&g_accel_ch->task_pool); 193 STAILQ_REMOVE_HEAD(&g_accel_ch->task_pool, link); 194 CU_ASSERT(expected_accel_task == &accel_task); 195 } 196 197 static void 198 test_get_task(void) 199 { 200 struct spdk_accel_task *task; 201 struct spdk_accel_task _task; 202 void *cb_arg = NULL; 203 204 STAILQ_INIT(&g_accel_ch->task_pool); 205 206 /* no tasks left, return NULL. */ 207 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 208 CU_ASSERT(task == NULL); 209 210 _task.cb_fn = dummy_cb_fn; 211 _task.cb_arg = cb_arg; 212 _task.accel_ch = g_accel_ch; 213 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &_task, link); 214 215 /* Get a valid task. */ 216 task = _get_task(g_accel_ch, dummy_cb_fn, cb_arg); 217 CU_ASSERT(task == &_task); 218 CU_ASSERT(_task.cb_fn == dummy_cb_fn); 219 CU_ASSERT(_task.cb_arg == cb_arg); 220 CU_ASSERT(_task.accel_ch == g_accel_ch); 221 } 222 223 #define TEST_SUBMIT_SIZE 64 224 static void 225 test_spdk_accel_submit_copy(void) 226 { 227 const uint64_t nbytes = TEST_SUBMIT_SIZE; 228 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 229 uint8_t src[TEST_SUBMIT_SIZE] = {0}; 230 void *cb_arg = NULL; 231 int rc; 232 struct spdk_accel_task task; 233 struct spdk_accel_task_aux_data task_aux; 234 struct spdk_accel_task *expected_accel_task = NULL; 235 236 STAILQ_INIT(&g_accel_ch->task_pool); 237 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 238 239 /* Fail with no tasks on _get_task() */ 240 rc = spdk_accel_submit_copy(g_ch, src, dst, nbytes, NULL, cb_arg); 241 CU_ASSERT(rc == -ENOMEM); 242 243 task.accel_ch = g_accel_ch; 244 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 245 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 246 247 /* submission OK. */ 248 rc = spdk_accel_submit_copy(g_ch, dst, src, nbytes, NULL, cb_arg); 249 CU_ASSERT(rc == 0); 250 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COPY); 251 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 252 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 253 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 254 CU_ASSERT(expected_accel_task == &task); 255 } 256 257 static void 258 test_spdk_accel_submit_dualcast(void) 259 { 260 void *dst1; 261 void *dst2; 262 void *src; 263 uint32_t align = ALIGN_4K; 264 uint64_t nbytes = TEST_SUBMIT_SIZE; 265 void *cb_arg = NULL; 266 int rc; 267 struct spdk_accel_task task; 268 struct spdk_accel_task_aux_data task_aux; 269 struct spdk_accel_task *expected_accel_task = NULL; 270 271 STAILQ_INIT(&g_accel_ch->task_pool); 272 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 273 274 /* Dualcast requires 4K alignment on dst addresses, 275 * hence using the hard coded address to test the buffer alignment 276 */ 277 dst1 = (void *)0x5000; 278 dst2 = (void *)0x60f0; 279 src = calloc(1, TEST_SUBMIT_SIZE); 280 SPDK_CU_ASSERT_FATAL(src != NULL); 281 memset(src, 0x5A, TEST_SUBMIT_SIZE); 282 283 /* This should fail since dst2 is not 4k aligned */ 284 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, NULL, cb_arg); 285 CU_ASSERT(rc == -EINVAL); 286 287 dst1 = (void *)0x7010; 288 dst2 = (void *)0x6000; 289 /* This should fail since dst1 is not 4k aligned */ 290 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, NULL, cb_arg); 291 CU_ASSERT(rc == -EINVAL); 292 293 /* Dualcast requires 4K alignment on dst addresses */ 294 dst1 = (void *)0x7000; 295 dst2 = (void *)0x6000; 296 /* Fail with no tasks on _get_task() */ 297 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, NULL, cb_arg); 298 CU_ASSERT(rc == -ENOMEM); 299 300 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 301 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 302 303 /* accel submission OK., since we test the SW path , need to use valid memory addresses 304 * cannot hardcode them anymore */ 305 dst1 = spdk_dma_zmalloc(nbytes, align, NULL); 306 SPDK_CU_ASSERT_FATAL(dst1 != NULL); 307 dst2 = spdk_dma_zmalloc(nbytes, align, NULL); 308 SPDK_CU_ASSERT_FATAL(dst2 != NULL); 309 /* SW module does the dualcast. */ 310 rc = spdk_accel_submit_dualcast(g_ch, dst1, dst2, src, nbytes, NULL, cb_arg); 311 CU_ASSERT(rc == 0); 312 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_DUALCAST); 313 CU_ASSERT(memcmp(dst1, src, TEST_SUBMIT_SIZE) == 0); 314 CU_ASSERT(memcmp(dst2, src, TEST_SUBMIT_SIZE) == 0); 315 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 316 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 317 CU_ASSERT(expected_accel_task == &task); 318 319 free(src); 320 spdk_free(dst1); 321 spdk_free(dst2); 322 } 323 324 static void 325 test_spdk_accel_submit_compare(void) 326 { 327 void *src1; 328 void *src2; 329 uint64_t nbytes = TEST_SUBMIT_SIZE; 330 void *cb_arg = NULL; 331 int rc; 332 struct spdk_accel_task task; 333 struct spdk_accel_task_aux_data task_aux; 334 struct spdk_accel_task *expected_accel_task = NULL; 335 336 STAILQ_INIT(&g_accel_ch->task_pool); 337 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 338 339 src1 = calloc(1, TEST_SUBMIT_SIZE); 340 SPDK_CU_ASSERT_FATAL(src1 != NULL); 341 src2 = calloc(1, TEST_SUBMIT_SIZE); 342 SPDK_CU_ASSERT_FATAL(src2 != NULL); 343 344 /* Fail with no tasks on _get_task() */ 345 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 346 CU_ASSERT(rc == -ENOMEM); 347 348 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 349 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 350 351 /* accel submission OK. */ 352 rc = spdk_accel_submit_compare(g_ch, src1, src2, nbytes, NULL, cb_arg); 353 CU_ASSERT(rc == 0); 354 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COMPARE); 355 CU_ASSERT(memcmp(src1, src2, TEST_SUBMIT_SIZE) == 0); 356 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 357 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 358 CU_ASSERT(expected_accel_task == &task); 359 360 free(src1); 361 free(src2); 362 } 363 364 static void 365 test_spdk_accel_submit_fill(void) 366 { 367 void *dst; 368 void *src; 369 uint8_t fill = 0xf; 370 uint64_t fill64; 371 uint64_t nbytes = TEST_SUBMIT_SIZE; 372 void *cb_arg = NULL; 373 int rc; 374 struct spdk_accel_task task; 375 struct spdk_accel_task_aux_data task_aux; 376 struct spdk_accel_task *expected_accel_task = NULL; 377 378 STAILQ_INIT(&g_accel_ch->task_pool); 379 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 380 381 dst = calloc(1, TEST_SUBMIT_SIZE); 382 SPDK_CU_ASSERT_FATAL(dst != NULL); 383 src = calloc(1, TEST_SUBMIT_SIZE); 384 SPDK_CU_ASSERT_FATAL(src != NULL); 385 memset(src, fill, TEST_SUBMIT_SIZE); 386 memset(&fill64, fill, sizeof(uint64_t)); 387 388 /* Fail with no tasks on _get_task() */ 389 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, NULL, cb_arg); 390 CU_ASSERT(rc == -ENOMEM); 391 392 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 393 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 394 395 /* accel submission OK. */ 396 rc = spdk_accel_submit_fill(g_ch, dst, fill, nbytes, NULL, cb_arg); 397 CU_ASSERT(rc == 0); 398 CU_ASSERT(task.fill_pattern == fill64); 399 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_FILL); 400 401 CU_ASSERT(memcmp(dst, src, TEST_SUBMIT_SIZE) == 0); 402 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 403 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 404 CU_ASSERT(expected_accel_task == &task); 405 406 free(dst); 407 free(src); 408 } 409 410 static void 411 test_spdk_accel_submit_crc32c(void) 412 { 413 const uint64_t nbytes = TEST_SUBMIT_SIZE; 414 uint32_t crc_dst; 415 uint8_t src[TEST_SUBMIT_SIZE]; 416 uint32_t seed = 1; 417 void *cb_arg = NULL; 418 int rc; 419 struct spdk_accel_task task; 420 struct spdk_accel_task_aux_data task_aux; 421 struct spdk_accel_task *expected_accel_task = NULL; 422 423 STAILQ_INIT(&g_accel_ch->task_pool); 424 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 425 426 /* Fail with no tasks on _get_task() */ 427 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 428 CU_ASSERT(rc == -ENOMEM); 429 430 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 431 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 432 433 /* accel submission OK. */ 434 rc = spdk_accel_submit_crc32c(g_ch, &crc_dst, src, seed, nbytes, NULL, cb_arg); 435 CU_ASSERT(rc == 0); 436 CU_ASSERT(task.crc_dst == &crc_dst); 437 CU_ASSERT(task.seed == seed); 438 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_CRC32C); 439 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 440 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 441 CU_ASSERT(expected_accel_task == &task); 442 } 443 444 static void 445 test_spdk_accel_submit_crc32cv(void) 446 { 447 uint32_t crc_dst; 448 uint32_t seed = 0; 449 uint32_t iov_cnt = 32; 450 void *cb_arg = NULL; 451 int rc; 452 uint32_t i = 0; 453 struct spdk_accel_task task; 454 struct spdk_accel_task_aux_data task_aux; 455 struct iovec iov[32]; 456 struct spdk_accel_task *expected_accel_task = NULL; 457 458 STAILQ_INIT(&g_accel_ch->task_pool); 459 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 460 461 for (i = 0; i < iov_cnt; i++) { 462 iov[i].iov_base = calloc(1, TEST_SUBMIT_SIZE); 463 SPDK_CU_ASSERT_FATAL(iov[i].iov_base != NULL); 464 iov[i].iov_len = TEST_SUBMIT_SIZE; 465 } 466 467 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 468 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 469 470 /* accel submission OK. */ 471 rc = spdk_accel_submit_crc32cv(g_ch, &crc_dst, iov, iov_cnt, seed, NULL, cb_arg); 472 CU_ASSERT(rc == 0); 473 CU_ASSERT(task.s.iovs == iov); 474 CU_ASSERT(task.s.iovcnt == iov_cnt); 475 CU_ASSERT(task.crc_dst == &crc_dst); 476 CU_ASSERT(task.seed == seed); 477 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_CRC32C); 478 CU_ASSERT(task.cb_arg == cb_arg); 479 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 480 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 481 CU_ASSERT(expected_accel_task == &task); 482 483 for (i = 0; i < iov_cnt; i++) { 484 free(iov[i].iov_base); 485 } 486 } 487 488 static void 489 test_spdk_accel_submit_copy_crc32c(void) 490 { 491 const uint64_t nbytes = TEST_SUBMIT_SIZE; 492 uint32_t crc_dst; 493 uint8_t dst[TEST_SUBMIT_SIZE]; 494 uint8_t src[TEST_SUBMIT_SIZE]; 495 uint32_t seed = 0; 496 void *cb_arg = NULL; 497 int rc; 498 struct spdk_accel_task task; 499 struct spdk_accel_task_aux_data task_aux; 500 struct spdk_accel_task *expected_accel_task = NULL; 501 502 STAILQ_INIT(&g_accel_ch->task_pool); 503 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 504 505 /* Fail with no tasks on _get_task() */ 506 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, NULL, cb_arg); 507 CU_ASSERT(rc == -ENOMEM); 508 509 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 510 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 511 512 /* accel submission OK. */ 513 rc = spdk_accel_submit_copy_crc32c(g_ch, dst, src, &crc_dst, seed, nbytes, NULL, cb_arg); 514 CU_ASSERT(rc == 0); 515 CU_ASSERT(task.crc_dst == &crc_dst); 516 CU_ASSERT(task.seed == seed); 517 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_COPY_CRC32C); 518 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 519 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 520 CU_ASSERT(expected_accel_task == &task); 521 } 522 523 static void 524 test_spdk_accel_submit_xor(void) 525 { 526 const uint64_t nbytes = TEST_SUBMIT_SIZE; 527 uint8_t dst[TEST_SUBMIT_SIZE] = {0}; 528 uint8_t src1[TEST_SUBMIT_SIZE] = {0}; 529 uint8_t src2[TEST_SUBMIT_SIZE] = {0}; 530 void *sources[] = { src1, src2 }; 531 uint32_t nsrcs = SPDK_COUNTOF(sources); 532 int rc; 533 struct spdk_accel_task task; 534 struct spdk_accel_task_aux_data task_aux; 535 struct spdk_accel_task *expected_accel_task = NULL; 536 537 STAILQ_INIT(&g_accel_ch->task_pool); 538 SLIST_INIT(&g_accel_ch->task_aux_data_pool); 539 540 /* Fail with no tasks on _get_task() */ 541 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 542 CU_ASSERT(rc == -ENOMEM); 543 544 STAILQ_INSERT_TAIL(&g_accel_ch->task_pool, &task, link); 545 SLIST_INSERT_HEAD(&g_accel_ch->task_aux_data_pool, &task_aux, link); 546 547 /* submission OK. */ 548 rc = spdk_accel_submit_xor(g_ch, dst, sources, nsrcs, nbytes, NULL, NULL); 549 CU_ASSERT(rc == 0); 550 CU_ASSERT(task.nsrcs.srcs == sources); 551 CU_ASSERT(task.nsrcs.cnt == nsrcs); 552 CU_ASSERT(task.d.iovcnt == 1); 553 CU_ASSERT(task.d.iovs[0].iov_base == dst); 554 CU_ASSERT(task.d.iovs[0].iov_len == nbytes); 555 CU_ASSERT(task.op_code == SPDK_ACCEL_OPC_XOR); 556 expected_accel_task = STAILQ_FIRST(&g_sw_ch->tasks_to_complete); 557 STAILQ_REMOVE_HEAD(&g_sw_ch->tasks_to_complete, link); 558 CU_ASSERT(expected_accel_task == &task); 559 } 560 561 static void 562 test_spdk_accel_module_find_by_name(void) 563 { 564 struct spdk_accel_module_if mod1 = {}; 565 struct spdk_accel_module_if mod2 = {}; 566 struct spdk_accel_module_if mod3 = {}; 567 struct spdk_accel_module_if *accel_module = NULL; 568 569 mod1.name = "ioat"; 570 mod2.name = "idxd"; 571 mod3.name = "software"; 572 573 TAILQ_INIT(&spdk_accel_module_list); 574 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod1, tailq); 575 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod2, tailq); 576 TAILQ_INSERT_TAIL(&spdk_accel_module_list, &mod3, tailq); 577 578 /* Now let's find a valid engine */ 579 accel_module = _module_find_by_name("ioat"); 580 CU_ASSERT(accel_module != NULL); 581 582 /* Try to find one that doesn't exist */ 583 accel_module = _module_find_by_name("XXX"); 584 CU_ASSERT(accel_module == NULL); 585 } 586 587 static int 588 ut_module_init_nop(void) 589 { 590 return 0; 591 } 592 593 static bool 594 ut_supports_opcode_all(enum spdk_accel_opcode opcode) 595 { 596 return true; 597 } 598 599 static void 600 ut_accel_module_priority_finish_done(void *done) 601 { 602 *(int *)done = 1; 603 } 604 605 static void 606 test_spdk_accel_module_register(void) 607 { 608 struct spdk_accel_module_if mods[] = { 609 { .name = "mod1", .priority = 1, }, 610 { .name = "mod3", .priority = 3, }, 611 { .name = "mod0", .priority = 0, }, 612 { .name = "mod2", .priority = 2, }, 613 }; 614 int rc, done = 0; 615 const char *modname = NULL; 616 size_t i; 617 618 allocate_cores(1); 619 allocate_threads(1); 620 set_thread(0); 621 622 TAILQ_INIT(&spdk_accel_module_list); 623 for (i = 0; i < SPDK_COUNTOF(mods); ++i) { 624 mods[i].module_init = ut_module_init_nop; 625 mods[i].supports_opcode = ut_supports_opcode_all; 626 spdk_accel_module_list_add(&mods[i]); 627 } 628 629 rc = spdk_accel_initialize(); 630 CU_ASSERT_EQUAL(rc, 0); 631 632 /* All opcodes should be assigned to the module with highest prio - mod3 */ 633 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 634 rc = spdk_accel_get_opc_module_name((enum spdk_accel_opcode)i, &modname); 635 CU_ASSERT_EQUAL(rc, 0); 636 CU_ASSERT_STRING_EQUAL(modname, "mod3"); 637 } 638 639 spdk_accel_finish(ut_accel_module_priority_finish_done, &done); 640 while (!done) { 641 poll_threads(); 642 } 643 644 TAILQ_INIT(&spdk_accel_module_list); 645 free_threads(); 646 free_cores(); 647 } 648 649 struct ut_sequence { 650 bool complete; 651 int status; 652 }; 653 654 static void 655 ut_sequence_step_cb(void *cb_arg) 656 { 657 int *completed = cb_arg; 658 659 (*completed)++; 660 } 661 662 static void 663 ut_sequence_complete_cb(void *cb_arg, int status) 664 { 665 struct ut_sequence *seq = cb_arg; 666 667 seq->complete = true; 668 seq->status = status; 669 } 670 671 static void 672 test_sequence_fill_copy(void) 673 { 674 struct spdk_accel_sequence *seq = NULL; 675 struct spdk_io_channel *ioch; 676 struct ut_sequence ut_seq; 677 char buf[4096], tmp[2][4096], expected[4096]; 678 struct iovec src_iovs[2], dst_iovs[2]; 679 int rc, completed; 680 681 ioch = spdk_accel_get_io_channel(); 682 SPDK_CU_ASSERT_FATAL(ioch != NULL); 683 684 /* First check the simplest case - single task in a sequence */ 685 memset(buf, 0, sizeof(buf)); 686 memset(expected, 0xa5, sizeof(expected)); 687 completed = 0; 688 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 689 ut_sequence_step_cb, &completed); 690 CU_ASSERT_EQUAL(rc, 0); 691 CU_ASSERT_EQUAL(completed, 0); 692 693 ut_seq.complete = false; 694 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 695 696 poll_threads(); 697 CU_ASSERT_EQUAL(completed, 1); 698 CU_ASSERT(ut_seq.complete); 699 CU_ASSERT_EQUAL(ut_seq.status, 0); 700 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 701 702 /* Check a single copy operation */ 703 memset(buf, 0, sizeof(buf)); 704 memset(tmp[0], 0xa5, sizeof(tmp[0])); 705 memset(expected, 0xa5, sizeof(expected)); 706 completed = 0; 707 seq = NULL; 708 709 dst_iovs[0].iov_base = buf; 710 dst_iovs[0].iov_len = sizeof(buf); 711 src_iovs[0].iov_base = tmp[0]; 712 src_iovs[0].iov_len = sizeof(tmp[0]); 713 714 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 715 &src_iovs[0], 1, NULL, NULL, 716 ut_sequence_step_cb, &completed); 717 CU_ASSERT_EQUAL(rc, 0); 718 719 ut_seq.complete = false; 720 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 721 722 poll_threads(); 723 CU_ASSERT_EQUAL(completed, 1); 724 CU_ASSERT(ut_seq.complete); 725 CU_ASSERT_EQUAL(ut_seq.status, 0); 726 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 727 728 /* Check multiple fill operations */ 729 memset(buf, 0, sizeof(buf)); 730 memset(expected, 0xfe, 4096); 731 memset(expected, 0xde, 2048); 732 memset(expected, 0xa5, 1024); 733 seq = NULL; 734 completed = 0; 735 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, NULL, NULL, 0xfe, 736 ut_sequence_step_cb, &completed); 737 CU_ASSERT_EQUAL(rc, 0); 738 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xde, 739 ut_sequence_step_cb, &completed); 740 CU_ASSERT_EQUAL(rc, 0); 741 rc = spdk_accel_append_fill(&seq, ioch, buf, 1024, NULL, NULL, 0xa5, 742 ut_sequence_step_cb, &completed); 743 CU_ASSERT_EQUAL(rc, 0); 744 745 ut_seq.complete = false; 746 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 747 748 poll_threads(); 749 CU_ASSERT_EQUAL(completed, 3); 750 CU_ASSERT(ut_seq.complete); 751 CU_ASSERT_EQUAL(ut_seq.status, 0); 752 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 753 754 /* Check multiple copy operations */ 755 memset(buf, 0, sizeof(buf)); 756 memset(tmp[0], 0, sizeof(tmp[0])); 757 memset(tmp[1], 0, sizeof(tmp[1])); 758 memset(expected, 0xa5, sizeof(expected)); 759 seq = NULL; 760 completed = 0; 761 762 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 763 ut_sequence_step_cb, &completed); 764 CU_ASSERT_EQUAL(rc, 0); 765 766 dst_iovs[0].iov_base = tmp[1]; 767 dst_iovs[0].iov_len = sizeof(tmp[1]); 768 src_iovs[0].iov_base = tmp[0]; 769 src_iovs[0].iov_len = sizeof(tmp[0]); 770 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 771 &src_iovs[0], 1, NULL, NULL, 772 ut_sequence_step_cb, &completed); 773 CU_ASSERT_EQUAL(rc, 0); 774 775 dst_iovs[1].iov_base = buf; 776 dst_iovs[1].iov_len = sizeof(buf); 777 src_iovs[1].iov_base = tmp[1]; 778 src_iovs[1].iov_len = sizeof(tmp[1]); 779 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 780 &src_iovs[1], 1, NULL, NULL, 781 ut_sequence_step_cb, &completed); 782 CU_ASSERT_EQUAL(rc, 0); 783 784 ut_seq.complete = false; 785 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 786 787 poll_threads(); 788 CU_ASSERT_EQUAL(completed, 3); 789 CU_ASSERT(ut_seq.complete); 790 CU_ASSERT_EQUAL(ut_seq.status, 0); 791 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 792 793 /* Check that adding a copy operation at the end will change destination buffer */ 794 memset(buf, 0, sizeof(buf)); 795 memset(tmp[0], 0, sizeof(tmp[0])); 796 memset(expected, 0xa5, sizeof(buf)); 797 seq = NULL; 798 completed = 0; 799 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 800 ut_sequence_step_cb, &completed); 801 CU_ASSERT_EQUAL(rc, 0); 802 803 dst_iovs[0].iov_base = buf; 804 dst_iovs[0].iov_len = sizeof(buf); 805 src_iovs[0].iov_base = tmp[0]; 806 src_iovs[0].iov_len = sizeof(tmp[0]); 807 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 808 &src_iovs[0], 1, NULL, NULL, 809 ut_sequence_step_cb, &completed); 810 CU_ASSERT_EQUAL(rc, 0); 811 812 ut_seq.complete = false; 813 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 814 815 poll_threads(); 816 CU_ASSERT_EQUAL(completed, 2); 817 CU_ASSERT(ut_seq.complete); 818 CU_ASSERT_EQUAL(ut_seq.status, 0); 819 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 820 821 /* Check that it's also possible to add copy operation at the beginning */ 822 memset(buf, 0, sizeof(buf)); 823 memset(tmp[0], 0xde, sizeof(tmp[0])); 824 memset(tmp[1], 0, sizeof(tmp[1])); 825 memset(expected, 0xa5, sizeof(expected)); 826 seq = NULL; 827 completed = 0; 828 829 dst_iovs[0].iov_base = tmp[1]; 830 dst_iovs[0].iov_len = sizeof(tmp[1]); 831 src_iovs[0].iov_base = tmp[0]; 832 src_iovs[0].iov_len = sizeof(tmp[0]); 833 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 834 &src_iovs[0], 1, NULL, NULL, 835 ut_sequence_step_cb, &completed); 836 CU_ASSERT_EQUAL(rc, 0); 837 838 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], sizeof(tmp[1]), NULL, NULL, 0xa5, 839 ut_sequence_step_cb, &completed); 840 CU_ASSERT_EQUAL(rc, 0); 841 842 dst_iovs[1].iov_base = buf; 843 dst_iovs[1].iov_len = sizeof(buf); 844 src_iovs[1].iov_base = tmp[1]; 845 src_iovs[1].iov_len = sizeof(tmp[1]); 846 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 847 &src_iovs[1], 1, NULL, NULL, 848 ut_sequence_step_cb, &completed); 849 CU_ASSERT_EQUAL(rc, 0); 850 851 ut_seq.complete = false; 852 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 853 854 poll_threads(); 855 CU_ASSERT_EQUAL(completed, 3); 856 CU_ASSERT(ut_seq.complete); 857 CU_ASSERT_EQUAL(ut_seq.status, 0); 858 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 859 860 spdk_put_io_channel(ioch); 861 poll_threads(); 862 } 863 864 static void 865 test_sequence_abort(void) 866 { 867 struct spdk_accel_sequence *seq = NULL; 868 struct spdk_io_channel *ioch; 869 char buf[4096], tmp[2][4096], expected[4096]; 870 struct iovec src_iovs[2], dst_iovs[2]; 871 int rc, completed; 872 873 ioch = spdk_accel_get_io_channel(); 874 SPDK_CU_ASSERT_FATAL(ioch != NULL); 875 876 /* Check that aborting a sequence calls operation's callback, the operation is not executed 877 * and the sequence is freed 878 */ 879 memset(buf, 0, sizeof(buf)); 880 memset(expected, 0, sizeof(buf)); 881 completed = 0; 882 seq = NULL; 883 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 884 ut_sequence_step_cb, &completed); 885 CU_ASSERT_EQUAL(rc, 0); 886 887 spdk_accel_sequence_abort(seq); 888 CU_ASSERT_EQUAL(completed, 1); 889 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 890 891 /* Check sequence with multiple operations */ 892 memset(buf, 0, sizeof(buf)); 893 memset(expected, 0, sizeof(buf)); 894 completed = 0; 895 seq = NULL; 896 897 dst_iovs[0].iov_base = tmp[1]; 898 dst_iovs[0].iov_len = sizeof(tmp[1]); 899 src_iovs[0].iov_base = tmp[0]; 900 src_iovs[0].iov_len = sizeof(tmp[0]); 901 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 902 &src_iovs[0], 1, NULL, NULL, 903 ut_sequence_step_cb, &completed); 904 CU_ASSERT_EQUAL(rc, 0); 905 906 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 4096, NULL, NULL, 0xa5, 907 ut_sequence_step_cb, &completed); 908 CU_ASSERT_EQUAL(rc, 0); 909 910 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xde, 911 ut_sequence_step_cb, &completed); 912 CU_ASSERT_EQUAL(rc, 0); 913 914 dst_iovs[1].iov_base = buf; 915 dst_iovs[1].iov_len = sizeof(buf); 916 src_iovs[1].iov_base = tmp[1]; 917 src_iovs[1].iov_len = sizeof(tmp[1]); 918 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 919 &src_iovs[1], 1, NULL, NULL, 920 ut_sequence_step_cb, &completed); 921 CU_ASSERT_EQUAL(rc, 0); 922 923 spdk_accel_sequence_abort(seq); 924 CU_ASSERT_EQUAL(completed, 4); 925 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 926 927 /* This should be a no-op */ 928 spdk_accel_sequence_abort(NULL); 929 930 spdk_put_io_channel(ioch); 931 poll_threads(); 932 } 933 934 static void 935 test_sequence_append_error(void) 936 { 937 struct spdk_accel_sequence *seq = NULL; 938 struct spdk_io_channel *ioch; 939 struct accel_io_channel *accel_ch; 940 struct iovec src_iovs, dst_iovs; 941 char buf[4096]; 942 STAILQ_HEAD(, spdk_accel_task) tasks = STAILQ_HEAD_INITIALIZER(tasks); 943 SLIST_HEAD(, spdk_accel_sequence) seqs = SLIST_HEAD_INITIALIZER(seqs); 944 int rc; 945 946 ioch = spdk_accel_get_io_channel(); 947 SPDK_CU_ASSERT_FATAL(ioch != NULL); 948 accel_ch = spdk_io_channel_get_ctx(ioch); 949 950 /* Check that append fails and no sequence object is allocated when there are no more free 951 * tasks */ 952 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 953 954 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 955 ut_sequence_step_cb, NULL); 956 CU_ASSERT_EQUAL(rc, -ENOMEM); 957 CU_ASSERT_PTR_NULL(seq); 958 959 dst_iovs.iov_base = buf; 960 dst_iovs.iov_len = 2048; 961 src_iovs.iov_base = &buf[2048]; 962 src_iovs.iov_len = 2048; 963 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 964 &src_iovs, 1, NULL, NULL, ut_sequence_step_cb, NULL); 965 CU_ASSERT_EQUAL(rc, -ENOMEM); 966 CU_ASSERT_PTR_NULL(seq); 967 968 dst_iovs.iov_base = buf; 969 dst_iovs.iov_len = 2048; 970 src_iovs.iov_base = &buf[2048]; 971 src_iovs.iov_len = 2048; 972 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 973 &src_iovs, 1, NULL, NULL, ut_sequence_step_cb, NULL); 974 CU_ASSERT_EQUAL(rc, -ENOMEM); 975 CU_ASSERT_PTR_NULL(seq); 976 977 /* Check that the same happens when the sequence queue is empty */ 978 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 979 SLIST_SWAP(&seqs, &accel_ch->seq_pool, spdk_accel_sequence); 980 981 rc = spdk_accel_append_fill(&seq, ioch, buf, sizeof(buf), NULL, NULL, 0xa5, 982 ut_sequence_step_cb, NULL); 983 CU_ASSERT_EQUAL(rc, -ENOMEM); 984 CU_ASSERT_PTR_NULL(seq); 985 986 dst_iovs.iov_base = buf; 987 dst_iovs.iov_len = 2048; 988 src_iovs.iov_base = &buf[2048]; 989 src_iovs.iov_len = 2048; 990 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs, 1, NULL, NULL, 991 &src_iovs, 1, NULL, NULL, ut_sequence_step_cb, NULL); 992 CU_ASSERT_EQUAL(rc, -ENOMEM); 993 CU_ASSERT_PTR_NULL(seq); 994 995 dst_iovs.iov_base = buf; 996 dst_iovs.iov_len = 2048; 997 src_iovs.iov_base = &buf[2048]; 998 src_iovs.iov_len = 2048; 999 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1000 &src_iovs, 1, NULL, NULL, ut_sequence_step_cb, NULL); 1001 CU_ASSERT_EQUAL(rc, -ENOMEM); 1002 CU_ASSERT_PTR_NULL(seq); 1003 1004 STAILQ_SWAP(&tasks, &accel_ch->task_pool, spdk_accel_task); 1005 1006 spdk_put_io_channel(ioch); 1007 poll_threads(); 1008 } 1009 1010 struct ut_sequence_operation { 1011 int complete_status; 1012 int submit_status; 1013 int count; 1014 struct iovec *src_iovs; 1015 uint32_t src_iovcnt; 1016 struct spdk_memory_domain *src_domain; 1017 void *src_domain_ctx; 1018 struct iovec *dst_iovs; 1019 uint32_t dst_iovcnt; 1020 struct spdk_memory_domain *dst_domain; 1021 void *dst_domain_ctx; 1022 int (*submit)(struct spdk_io_channel *ch, struct spdk_accel_task *t); 1023 }; 1024 1025 static struct ut_sequence_operation g_seq_operations[SPDK_ACCEL_OPC_LAST]; 1026 1027 static int 1028 ut_sequence_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *task) 1029 { 1030 struct ut_sequence_operation *op = &g_seq_operations[task->op_code]; 1031 1032 op->count++; 1033 if (op->submit != NULL) { 1034 return op->submit(ch, task); 1035 } 1036 if (op->src_iovs != NULL) { 1037 CU_ASSERT_EQUAL(task->s.iovcnt, op->src_iovcnt); 1038 CU_ASSERT_EQUAL(memcmp(task->s.iovs, op->src_iovs, 1039 sizeof(struct iovec) * op->src_iovcnt), 0); 1040 } 1041 if (op->dst_iovs != NULL) { 1042 CU_ASSERT_EQUAL(task->d.iovcnt, op->dst_iovcnt); 1043 CU_ASSERT_EQUAL(memcmp(task->d.iovs, op->dst_iovs, 1044 sizeof(struct iovec) * op->dst_iovcnt), 0); 1045 } 1046 1047 CU_ASSERT_EQUAL(task->src_domain, op->src_domain); 1048 CU_ASSERT_EQUAL(task->dst_domain, op->dst_domain); 1049 if (op->src_domain != NULL) { 1050 CU_ASSERT_EQUAL(task->src_domain_ctx, op->src_domain_ctx); 1051 } 1052 if (op->dst_domain != NULL) { 1053 CU_ASSERT_EQUAL(task->dst_domain_ctx, op->dst_domain_ctx); 1054 } 1055 1056 if (op->submit_status != 0) { 1057 return op->submit_status; 1058 } 1059 1060 spdk_accel_task_complete(task, op->complete_status); 1061 1062 return 0; 1063 } 1064 1065 static void 1066 ut_clear_operations(void) 1067 { 1068 memset(&g_seq_operations, 0, sizeof(g_seq_operations)); 1069 } 1070 1071 static void 1072 test_sequence_completion_error(void) 1073 { 1074 struct spdk_accel_sequence *seq = NULL; 1075 struct spdk_io_channel *ioch; 1076 struct ut_sequence ut_seq; 1077 struct iovec src_iovs, dst_iovs; 1078 char buf[4096], tmp[4096]; 1079 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 1080 int i, rc, completed; 1081 1082 ioch = spdk_accel_get_io_channel(); 1083 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1084 1085 /* Override the submit_tasks function */ 1086 g_module_if.submit_tasks = ut_sequence_submit_tasks; 1087 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1088 modules[i] = g_modules_opc[i]; 1089 g_modules_opc[i] = g_module; 1090 } 1091 1092 memset(buf, 0, sizeof(buf)); 1093 memset(tmp, 0, sizeof(tmp)); 1094 1095 /* Check that if the first operation completes with an error, the whole sequence is 1096 * completed with that error and that all operations' completion callbacks are executed 1097 */ 1098 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = -E2BIG; 1099 completed = 0; 1100 seq = NULL; 1101 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 1102 ut_sequence_step_cb, &completed); 1103 CU_ASSERT_EQUAL(rc, 0); 1104 1105 dst_iovs.iov_base = buf; 1106 dst_iovs.iov_len = sizeof(buf); 1107 src_iovs.iov_base = tmp; 1108 src_iovs.iov_len = sizeof(tmp); 1109 1110 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1111 &src_iovs, 1, NULL, NULL, 1112 ut_sequence_step_cb, &completed); 1113 CU_ASSERT_EQUAL(rc, 0); 1114 1115 ut_seq.complete = false; 1116 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1117 1118 poll_threads(); 1119 CU_ASSERT_EQUAL(completed, 2); 1120 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 1121 1122 /* Check the same with a second operation in the sequence */ 1123 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].complete_status = -EACCES; 1124 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 1125 completed = 0; 1126 seq = NULL; 1127 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 1128 ut_sequence_step_cb, &completed); 1129 CU_ASSERT_EQUAL(rc, 0); 1130 1131 dst_iovs.iov_base = buf; 1132 dst_iovs.iov_len = sizeof(buf); 1133 src_iovs.iov_base = tmp; 1134 src_iovs.iov_len = sizeof(tmp); 1135 1136 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1137 &src_iovs, 1, NULL, NULL, 1138 ut_sequence_step_cb, &completed); 1139 CU_ASSERT_EQUAL(rc, 0); 1140 1141 ut_seq.complete = false; 1142 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1143 1144 poll_threads(); 1145 CU_ASSERT_EQUAL(completed, 2); 1146 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 1147 1148 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].complete_status = 0; 1149 g_seq_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 1150 1151 /* Check submission failure of the first operation */ 1152 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit_status = -EADDRINUSE; 1153 completed = 0; 1154 seq = NULL; 1155 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 1156 ut_sequence_step_cb, &completed); 1157 CU_ASSERT_EQUAL(rc, 0); 1158 1159 dst_iovs.iov_base = buf; 1160 dst_iovs.iov_len = sizeof(buf); 1161 src_iovs.iov_base = tmp; 1162 src_iovs.iov_len = sizeof(tmp); 1163 1164 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1165 &src_iovs, 1, NULL, NULL, 1166 ut_sequence_step_cb, &completed); 1167 CU_ASSERT_EQUAL(rc, 0); 1168 1169 ut_seq.complete = false; 1170 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1171 1172 poll_threads(); 1173 CU_ASSERT_EQUAL(completed, 2); 1174 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 1175 1176 /* Check the same with a second operation */ 1177 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit_status = -EADDRNOTAVAIL; 1178 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit_status = 0; 1179 completed = 0; 1180 seq = NULL; 1181 rc = spdk_accel_append_fill(&seq, ioch, tmp, sizeof(tmp), NULL, NULL, 0xa5, 1182 ut_sequence_step_cb, &completed); 1183 CU_ASSERT_EQUAL(rc, 0); 1184 1185 dst_iovs.iov_base = buf; 1186 dst_iovs.iov_len = sizeof(buf); 1187 src_iovs.iov_base = tmp; 1188 src_iovs.iov_len = sizeof(tmp); 1189 1190 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs, 1, NULL, NULL, 1191 &src_iovs, 1, NULL, NULL, 1192 ut_sequence_step_cb, &completed); 1193 CU_ASSERT_EQUAL(rc, 0); 1194 1195 ut_seq.complete = false; 1196 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1197 1198 poll_threads(); 1199 CU_ASSERT_EQUAL(completed, 2); 1200 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 1201 1202 /* Cleanup module pointers to make subsequent tests work correctly */ 1203 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1204 g_modules_opc[i] = modules[i]; 1205 } 1206 1207 ut_clear_operations(); 1208 spdk_put_io_channel(ioch); 1209 poll_threads(); 1210 } 1211 1212 #ifdef SPDK_CONFIG_ISAL 1213 static void 1214 ut_compress_cb(void *cb_arg, int status) 1215 { 1216 int *completed = cb_arg; 1217 1218 CU_ASSERT_EQUAL(status, 0); 1219 1220 *completed = 1; 1221 } 1222 1223 static void 1224 test_sequence_decompress(void) 1225 { 1226 struct spdk_accel_sequence *seq = NULL; 1227 struct spdk_io_channel *ioch; 1228 struct ut_sequence ut_seq; 1229 char buf[4096], tmp[2][4096], expected[4096]; 1230 struct iovec src_iovs[2], dst_iovs[2]; 1231 uint32_t compressed_size; 1232 int rc, completed = 0; 1233 1234 ioch = spdk_accel_get_io_channel(); 1235 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1236 1237 memset(expected, 0xa5, sizeof(expected)); 1238 src_iovs[0].iov_base = expected; 1239 src_iovs[0].iov_len = sizeof(expected); 1240 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1241 &compressed_size, ut_compress_cb, &completed); 1242 CU_ASSERT_EQUAL(rc, 0); 1243 1244 while (!completed) { 1245 poll_threads(); 1246 } 1247 1248 /* Check a single decompress operation in a sequence */ 1249 seq = NULL; 1250 completed = 0; 1251 1252 dst_iovs[0].iov_base = buf; 1253 dst_iovs[0].iov_len = sizeof(buf); 1254 src_iovs[0].iov_base = tmp[0]; 1255 src_iovs[0].iov_len = compressed_size; 1256 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1257 &src_iovs[0], 1, NULL, NULL, 1258 ut_sequence_step_cb, &completed); 1259 CU_ASSERT_EQUAL(rc, 0); 1260 1261 ut_seq.complete = false; 1262 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1263 1264 poll_threads(); 1265 1266 CU_ASSERT_EQUAL(completed, 1); 1267 CU_ASSERT(ut_seq.complete); 1268 CU_ASSERT_EQUAL(ut_seq.status, 0); 1269 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1270 1271 /* Put the decompress operation in the middle of a sequence with a copy operation at the 1272 * beginning and a fill at the end modifying the first 2048B of the buffer. 1273 */ 1274 memset(expected, 0xfe, 2048); 1275 memset(buf, 0, sizeof(buf)); 1276 seq = NULL; 1277 completed = 0; 1278 1279 dst_iovs[0].iov_base = tmp[1]; 1280 dst_iovs[0].iov_len = compressed_size; 1281 src_iovs[0].iov_base = tmp[0]; 1282 src_iovs[0].iov_len = compressed_size; 1283 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1284 &src_iovs[0], 1, NULL, NULL, 1285 ut_sequence_step_cb, &completed); 1286 CU_ASSERT_EQUAL(rc, 0); 1287 1288 dst_iovs[1].iov_base = buf; 1289 dst_iovs[1].iov_len = sizeof(buf); 1290 src_iovs[1].iov_base = tmp[1]; 1291 src_iovs[1].iov_len = compressed_size; 1292 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1293 &src_iovs[1], 1, NULL, NULL, 1294 ut_sequence_step_cb, &completed); 1295 CU_ASSERT_EQUAL(rc, 0); 1296 1297 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 1298 ut_sequence_step_cb, &completed); 1299 CU_ASSERT_EQUAL(rc, 0); 1300 1301 ut_seq.complete = false; 1302 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1303 1304 poll_threads(); 1305 1306 CU_ASSERT_EQUAL(completed, 3); 1307 CU_ASSERT(ut_seq.complete); 1308 CU_ASSERT_EQUAL(ut_seq.status, 0); 1309 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1310 1311 /* Check sequence with decompress at the beginning: decompress -> copy */ 1312 memset(expected, 0xa5, sizeof(expected)); 1313 memset(buf, 0, sizeof(buf)); 1314 seq = NULL; 1315 completed = 0; 1316 1317 dst_iovs[0].iov_base = tmp[1]; 1318 dst_iovs[0].iov_len = sizeof(tmp[1]); 1319 src_iovs[0].iov_base = tmp[0]; 1320 src_iovs[0].iov_len = compressed_size; 1321 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1322 &src_iovs[0], 1, NULL, NULL, 1323 ut_sequence_step_cb, &completed); 1324 CU_ASSERT_EQUAL(rc, 0); 1325 1326 dst_iovs[1].iov_base = buf; 1327 dst_iovs[1].iov_len = sizeof(buf); 1328 src_iovs[1].iov_base = tmp[1]; 1329 src_iovs[1].iov_len = sizeof(tmp[1]); 1330 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1331 &src_iovs[1], 1, NULL, NULL, 1332 ut_sequence_step_cb, &completed); 1333 CU_ASSERT_EQUAL(rc, 0); 1334 1335 ut_seq.complete = false; 1336 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1337 1338 poll_threads(); 1339 1340 CU_ASSERT_EQUAL(completed, 2); 1341 CU_ASSERT(ut_seq.complete); 1342 CU_ASSERT_EQUAL(ut_seq.status, 0); 1343 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1344 1345 spdk_put_io_channel(ioch); 1346 poll_threads(); 1347 } 1348 1349 static void 1350 test_sequence_reverse(void) 1351 { 1352 struct spdk_accel_sequence *seq = NULL; 1353 struct spdk_io_channel *ioch; 1354 struct ut_sequence ut_seq; 1355 char buf[4096], tmp[2][4096], expected[4096]; 1356 struct iovec src_iovs[2], dst_iovs[2]; 1357 uint32_t compressed_size; 1358 int rc, completed = 0; 1359 1360 ioch = spdk_accel_get_io_channel(); 1361 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1362 1363 memset(expected, 0xa5, sizeof(expected)); 1364 src_iovs[0].iov_base = expected; 1365 src_iovs[0].iov_len = sizeof(expected); 1366 rc = spdk_accel_submit_compress(ioch, tmp[0], sizeof(tmp[0]), &src_iovs[0], 1, 1367 &compressed_size, ut_compress_cb, &completed); 1368 CU_ASSERT_EQUAL(rc, 0); 1369 1370 while (!completed) { 1371 poll_threads(); 1372 } 1373 1374 /* First check that reversing a sequence with a single operation is a no-op */ 1375 memset(buf, 0, sizeof(buf)); 1376 seq = NULL; 1377 completed = 0; 1378 1379 dst_iovs[0].iov_base = buf; 1380 dst_iovs[0].iov_len = sizeof(buf); 1381 src_iovs[0].iov_base = tmp[0]; 1382 src_iovs[0].iov_len = compressed_size; 1383 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1384 &src_iovs[0], 1, NULL, NULL, 1385 ut_sequence_step_cb, &completed); 1386 CU_ASSERT_EQUAL(rc, 0); 1387 1388 spdk_accel_sequence_reverse(seq); 1389 1390 ut_seq.complete = false; 1391 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1392 1393 poll_threads(); 1394 1395 CU_ASSERT_EQUAL(completed, 1); 1396 CU_ASSERT(ut_seq.complete); 1397 CU_ASSERT_EQUAL(ut_seq.status, 0); 1398 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1399 1400 /* Add a copy operation at the end with src set to the compressed data. After reverse(), 1401 * that copy operation should be first, so decompress() should receive compressed data in 1402 * its src buffer. 1403 */ 1404 memset(buf, 0, sizeof(buf)); 1405 memset(tmp[1], 0, sizeof(tmp[1])); 1406 seq = NULL; 1407 completed = 0; 1408 1409 dst_iovs[0].iov_base = buf; 1410 dst_iovs[0].iov_len = sizeof(buf); 1411 src_iovs[0].iov_base = tmp[1]; 1412 src_iovs[0].iov_len = compressed_size; 1413 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1414 &src_iovs[0], 1, NULL, NULL, 1415 ut_sequence_step_cb, &completed); 1416 CU_ASSERT_EQUAL(rc, 0); 1417 1418 dst_iovs[1].iov_base = tmp[1]; 1419 dst_iovs[1].iov_len = compressed_size; 1420 src_iovs[1].iov_base = tmp[0]; 1421 src_iovs[1].iov_len = compressed_size; 1422 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1423 &src_iovs[1], 1, NULL, NULL, 1424 ut_sequence_step_cb, &completed); 1425 CU_ASSERT_EQUAL(rc, 0); 1426 1427 spdk_accel_sequence_reverse(seq); 1428 1429 ut_seq.complete = false; 1430 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1431 1432 poll_threads(); 1433 1434 CU_ASSERT_EQUAL(completed, 2); 1435 CU_ASSERT(ut_seq.complete); 1436 CU_ASSERT_EQUAL(ut_seq.status, 0); 1437 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1438 1439 /* Check the same, but add an extra fill operation at the beginning that should execute last 1440 * after reverse(). 1441 */ 1442 memset(buf, 0, sizeof(buf)); 1443 memset(tmp[1], 0, sizeof(tmp[1])); 1444 memset(expected, 0xfe, 2048); 1445 seq = NULL; 1446 completed = 0; 1447 1448 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 1449 ut_sequence_step_cb, &completed); 1450 CU_ASSERT_EQUAL(rc, 0); 1451 1452 dst_iovs[0].iov_base = buf; 1453 dst_iovs[0].iov_len = sizeof(buf); 1454 src_iovs[0].iov_base = tmp[1]; 1455 src_iovs[0].iov_len = compressed_size; 1456 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1457 &src_iovs[0], 1, NULL, NULL, 1458 ut_sequence_step_cb, &completed); 1459 CU_ASSERT_EQUAL(rc, 0); 1460 1461 dst_iovs[1].iov_base = tmp[1]; 1462 dst_iovs[1].iov_len = compressed_size; 1463 src_iovs[1].iov_base = tmp[0]; 1464 src_iovs[1].iov_len = compressed_size; 1465 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1466 &src_iovs[1], 1, NULL, NULL, 1467 ut_sequence_step_cb, &completed); 1468 CU_ASSERT_EQUAL(rc, 0); 1469 1470 spdk_accel_sequence_reverse(seq); 1471 1472 ut_seq.complete = false; 1473 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1474 1475 poll_threads(); 1476 1477 CU_ASSERT_EQUAL(completed, 3); 1478 CU_ASSERT(ut_seq.complete); 1479 CU_ASSERT_EQUAL(ut_seq.status, 0); 1480 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1481 1482 /* Build the sequence in order and then reverse it twice */ 1483 memset(buf, 0, sizeof(buf)); 1484 memset(tmp[1], 0, sizeof(tmp[1])); 1485 seq = NULL; 1486 completed = 0; 1487 1488 dst_iovs[0].iov_base = tmp[1]; 1489 dst_iovs[0].iov_len = compressed_size; 1490 src_iovs[0].iov_base = tmp[0]; 1491 src_iovs[0].iov_len = compressed_size; 1492 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1493 &src_iovs[0], 1, NULL, NULL, 1494 ut_sequence_step_cb, &completed); 1495 CU_ASSERT_EQUAL(rc, 0); 1496 1497 dst_iovs[1].iov_base = buf; 1498 dst_iovs[1].iov_len = sizeof(buf); 1499 src_iovs[1].iov_base = tmp[1]; 1500 src_iovs[1].iov_len = compressed_size; 1501 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1502 &src_iovs[1], 1, NULL, NULL, 1503 ut_sequence_step_cb, &completed); 1504 CU_ASSERT_EQUAL(rc, 0); 1505 1506 rc = spdk_accel_append_fill(&seq, ioch, buf, 2048, NULL, NULL, 0xfe, 1507 ut_sequence_step_cb, &completed); 1508 CU_ASSERT_EQUAL(rc, 0); 1509 1510 spdk_accel_sequence_reverse(seq); 1511 spdk_accel_sequence_reverse(seq); 1512 1513 ut_seq.complete = false; 1514 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1515 1516 poll_threads(); 1517 1518 CU_ASSERT_EQUAL(completed, 3); 1519 CU_ASSERT(ut_seq.complete); 1520 CU_ASSERT_EQUAL(ut_seq.status, 0); 1521 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 1522 1523 spdk_put_io_channel(ioch); 1524 poll_threads(); 1525 } 1526 #endif 1527 1528 static void 1529 test_sequence_copy_elision(void) 1530 { 1531 struct spdk_accel_sequence *seq = NULL; 1532 struct spdk_io_channel *ioch; 1533 struct ut_sequence ut_seq; 1534 struct iovec src_iovs[4], dst_iovs[4], exp_iovs[2]; 1535 char buf[4096], tmp[4][4096]; 1536 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 1537 struct spdk_accel_crypto_key key = {}; 1538 int i, rc, completed; 1539 1540 ioch = spdk_accel_get_io_channel(); 1541 SPDK_CU_ASSERT_FATAL(ioch != NULL); 1542 1543 /* Override the submit_tasks function */ 1544 g_module_if.submit_tasks = ut_sequence_submit_tasks; 1545 g_module.supports_memory_domains = true; 1546 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1547 g_seq_operations[i].complete_status = 0; 1548 g_seq_operations[i].submit_status = 0; 1549 g_seq_operations[i].count = 0; 1550 1551 modules[i] = g_modules_opc[i]; 1552 g_modules_opc[i] = g_module; 1553 } 1554 1555 /* Check that a copy operation at the beginning is removed */ 1556 seq = NULL; 1557 completed = 0; 1558 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovcnt = 1; 1559 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovcnt = 1; 1560 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1561 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1562 exp_iovs[0].iov_base = tmp[0]; 1563 exp_iovs[0].iov_len = sizeof(tmp[0]); 1564 exp_iovs[1].iov_base = buf; 1565 exp_iovs[1].iov_len = 2048; 1566 1567 dst_iovs[0].iov_base = tmp[1]; 1568 dst_iovs[0].iov_len = sizeof(tmp[1]); 1569 src_iovs[0].iov_base = tmp[0]; 1570 src_iovs[0].iov_len = sizeof(tmp[0]); 1571 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1572 &src_iovs[0], 1, NULL, NULL, 1573 ut_sequence_step_cb, &completed); 1574 CU_ASSERT_EQUAL(rc, 0); 1575 1576 dst_iovs[1].iov_base = buf; 1577 dst_iovs[1].iov_len = 2048; 1578 src_iovs[1].iov_base = tmp[1]; 1579 src_iovs[1].iov_len = sizeof(tmp[1]); 1580 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1581 &src_iovs[1], 1, NULL, NULL, 1582 ut_sequence_step_cb, &completed); 1583 CU_ASSERT_EQUAL(rc, 0); 1584 1585 ut_seq.complete = false; 1586 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1587 1588 poll_threads(); 1589 1590 CU_ASSERT_EQUAL(completed, 2); 1591 CU_ASSERT(ut_seq.complete); 1592 CU_ASSERT_EQUAL(ut_seq.status, 0); 1593 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1594 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1595 1596 /* Check that a copy operation at the end is removed too */ 1597 seq = NULL; 1598 completed = 0; 1599 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1600 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1601 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1602 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1603 exp_iovs[0].iov_base = tmp[0]; 1604 exp_iovs[0].iov_len = sizeof(tmp[0]); 1605 exp_iovs[1].iov_base = buf; 1606 exp_iovs[1].iov_len = 2048; 1607 1608 dst_iovs[0].iov_base = tmp[1]; 1609 dst_iovs[0].iov_len = 2048; 1610 src_iovs[0].iov_base = tmp[0]; 1611 src_iovs[0].iov_len = sizeof(tmp[0]); 1612 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1613 &src_iovs[0], 1, NULL, NULL, 1614 ut_sequence_step_cb, &completed); 1615 CU_ASSERT_EQUAL(rc, 0); 1616 1617 dst_iovs[1].iov_base = buf; 1618 dst_iovs[1].iov_len = 2048; 1619 src_iovs[1].iov_base = tmp[1]; 1620 src_iovs[1].iov_len = 2048; 1621 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1622 &src_iovs[1], 1, NULL, NULL, 1623 ut_sequence_step_cb, &completed); 1624 CU_ASSERT_EQUAL(rc, 0); 1625 1626 ut_seq.complete = false; 1627 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1628 1629 poll_threads(); 1630 1631 CU_ASSERT_EQUAL(completed, 2); 1632 CU_ASSERT(ut_seq.complete); 1633 CU_ASSERT_EQUAL(ut_seq.status, 0); 1634 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1635 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1636 1637 /* Check a copy operation both at the beginning and the end */ 1638 seq = NULL; 1639 completed = 0; 1640 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1641 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1642 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = &exp_iovs[0]; 1643 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = &exp_iovs[1]; 1644 exp_iovs[0].iov_base = tmp[0]; 1645 exp_iovs[0].iov_len = sizeof(tmp[0]); 1646 exp_iovs[1].iov_base = buf; 1647 exp_iovs[1].iov_len = 2048; 1648 1649 dst_iovs[0].iov_base = tmp[1]; 1650 dst_iovs[0].iov_len = sizeof(tmp[1]); 1651 src_iovs[0].iov_base = tmp[0]; 1652 src_iovs[0].iov_len = sizeof(tmp[0]); 1653 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1654 &src_iovs[0], 1, NULL, NULL, 1655 ut_sequence_step_cb, &completed); 1656 CU_ASSERT_EQUAL(rc, 0); 1657 1658 dst_iovs[1].iov_base = tmp[2]; 1659 dst_iovs[1].iov_len = 2048; 1660 src_iovs[1].iov_base = tmp[1]; 1661 src_iovs[1].iov_len = sizeof(tmp[1]); 1662 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1663 &src_iovs[1], 1, NULL, NULL, 1664 ut_sequence_step_cb, &completed); 1665 CU_ASSERT_EQUAL(rc, 0); 1666 1667 dst_iovs[2].iov_base = buf; 1668 dst_iovs[2].iov_len = 2048; 1669 src_iovs[2].iov_base = tmp[2]; 1670 src_iovs[2].iov_len = 2048; 1671 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1672 &src_iovs[2], 1, NULL, NULL, 1673 ut_sequence_step_cb, &completed); 1674 CU_ASSERT_EQUAL(rc, 0); 1675 1676 ut_seq.complete = false; 1677 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1678 1679 poll_threads(); 1680 1681 CU_ASSERT_EQUAL(completed, 3); 1682 CU_ASSERT(ut_seq.complete); 1683 CU_ASSERT_EQUAL(ut_seq.status, 0); 1684 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1685 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 1686 1687 /* Check decompress + copy + decompress + copy */ 1688 seq = NULL; 1689 completed = 0; 1690 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1691 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 1692 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].src_iovs = NULL; 1693 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].dst_iovs = NULL; 1694 1695 dst_iovs[0].iov_base = tmp[1]; 1696 dst_iovs[0].iov_len = sizeof(tmp[1]); 1697 src_iovs[0].iov_base = tmp[0]; 1698 src_iovs[0].iov_len = sizeof(tmp[0]); 1699 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1700 &src_iovs[0], 1, NULL, NULL, 1701 ut_sequence_step_cb, &completed); 1702 CU_ASSERT_EQUAL(rc, 0); 1703 1704 dst_iovs[1].iov_base = tmp[2]; 1705 dst_iovs[1].iov_len = 2048; 1706 src_iovs[1].iov_base = tmp[1]; 1707 src_iovs[1].iov_len = sizeof(tmp[1]); 1708 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1709 &src_iovs[1], 1, NULL, NULL, 1710 ut_sequence_step_cb, &completed); 1711 CU_ASSERT_EQUAL(rc, 0); 1712 1713 dst_iovs[2].iov_base = tmp[3]; 1714 dst_iovs[2].iov_len = 1024; 1715 src_iovs[2].iov_base = tmp[2]; 1716 src_iovs[2].iov_len = 2048; 1717 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1718 &src_iovs[2], 1, NULL, NULL, 1719 ut_sequence_step_cb, &completed); 1720 CU_ASSERT_EQUAL(rc, 0); 1721 1722 dst_iovs[3].iov_base = buf; 1723 dst_iovs[3].iov_len = 1024; 1724 src_iovs[3].iov_base = tmp[3]; 1725 src_iovs[3].iov_len = 1024; 1726 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 1727 &src_iovs[3], 1, NULL, NULL, 1728 ut_sequence_step_cb, &completed); 1729 CU_ASSERT_EQUAL(rc, 0); 1730 1731 ut_seq.complete = false; 1732 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1733 1734 poll_threads(); 1735 1736 CU_ASSERT_EQUAL(completed, 4); 1737 CU_ASSERT(ut_seq.complete); 1738 CU_ASSERT_EQUAL(ut_seq.status, 0); 1739 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1740 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 2); 1741 1742 /* Check two copy operations - one of them should be removed, while the other should be 1743 * executed normally */ 1744 seq = NULL; 1745 completed = 0; 1746 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1747 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovcnt = 1; 1748 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovcnt = 1; 1749 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovs = &exp_iovs[0]; 1750 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovs = &exp_iovs[1]; 1751 exp_iovs[0].iov_base = tmp[0]; 1752 exp_iovs[0].iov_len = sizeof(tmp[0]); 1753 exp_iovs[1].iov_base = buf; 1754 exp_iovs[1].iov_len = sizeof(buf); 1755 1756 dst_iovs[0].iov_base = tmp[1]; 1757 dst_iovs[0].iov_len = sizeof(tmp[1]); 1758 src_iovs[0].iov_base = tmp[0]; 1759 src_iovs[0].iov_len = sizeof(tmp[0]); 1760 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1761 &src_iovs[0], 1, NULL, NULL, 1762 ut_sequence_step_cb, &completed); 1763 CU_ASSERT_EQUAL(rc, 0); 1764 1765 dst_iovs[1].iov_base = buf; 1766 dst_iovs[1].iov_len = sizeof(buf); 1767 src_iovs[1].iov_base = tmp[1]; 1768 src_iovs[1].iov_len = sizeof(tmp[1]); 1769 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 1770 &src_iovs[1], 1, NULL, NULL, 1771 ut_sequence_step_cb, &completed); 1772 CU_ASSERT_EQUAL(rc, 0); 1773 1774 ut_seq.complete = false; 1775 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1776 1777 poll_threads(); 1778 1779 CU_ASSERT_EQUAL(completed, 2); 1780 CU_ASSERT(ut_seq.complete); 1781 CU_ASSERT_EQUAL(ut_seq.status, 0); 1782 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 1783 1784 /* Check fill + copy */ 1785 seq = NULL; 1786 completed = 0; 1787 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1788 g_seq_operations[SPDK_ACCEL_OPC_FILL].count = 0; 1789 g_seq_operations[SPDK_ACCEL_OPC_COPY].src_iovs = NULL; 1790 g_seq_operations[SPDK_ACCEL_OPC_COPY].dst_iovs = NULL; 1791 g_seq_operations[SPDK_ACCEL_OPC_FILL].dst_iovcnt = 1; 1792 g_seq_operations[SPDK_ACCEL_OPC_FILL].dst_iovs = &exp_iovs[0]; 1793 exp_iovs[0].iov_base = buf; 1794 exp_iovs[0].iov_len = sizeof(buf); 1795 1796 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xa5, 1797 ut_sequence_step_cb, &completed); 1798 CU_ASSERT_EQUAL(rc, 0); 1799 1800 dst_iovs[0].iov_base = buf; 1801 dst_iovs[0].iov_len = sizeof(buf); 1802 src_iovs[0].iov_base = tmp[0]; 1803 src_iovs[0].iov_len = sizeof(tmp[0]); 1804 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1805 &src_iovs[0], 1, NULL, NULL, 1806 ut_sequence_step_cb, &completed); 1807 CU_ASSERT_EQUAL(rc, 0); 1808 1809 ut_seq.complete = false; 1810 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1811 1812 poll_threads(); 1813 1814 CU_ASSERT_EQUAL(completed, 2); 1815 CU_ASSERT(ut_seq.complete); 1816 CU_ASSERT_EQUAL(ut_seq.status, 0); 1817 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1818 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 1); 1819 1820 /* Check copy + encrypt + copy */ 1821 seq = NULL; 1822 completed = 0; 1823 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1824 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 1825 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovcnt = 1; 1826 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovcnt = 1; 1827 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].src_iovs = &exp_iovs[0]; 1828 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].dst_iovs = &exp_iovs[1]; 1829 exp_iovs[0].iov_base = tmp[0]; 1830 exp_iovs[0].iov_len = sizeof(tmp[0]); 1831 exp_iovs[1].iov_base = buf; 1832 exp_iovs[1].iov_len = sizeof(buf); 1833 1834 dst_iovs[0].iov_base = tmp[1]; 1835 dst_iovs[0].iov_len = sizeof(tmp[1]); 1836 src_iovs[0].iov_base = tmp[0]; 1837 src_iovs[0].iov_len = sizeof(tmp[0]); 1838 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1839 &src_iovs[0], 1, NULL, NULL, 1840 ut_sequence_step_cb, &completed); 1841 CU_ASSERT_EQUAL(rc, 0); 1842 1843 dst_iovs[1].iov_base = tmp[2]; 1844 dst_iovs[1].iov_len = sizeof(tmp[2]); 1845 src_iovs[1].iov_base = tmp[1]; 1846 src_iovs[1].iov_len = sizeof(tmp[1]); 1847 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1848 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 1849 ut_sequence_step_cb, &completed); 1850 CU_ASSERT_EQUAL(rc, 0); 1851 1852 dst_iovs[2].iov_base = buf; 1853 dst_iovs[2].iov_len = sizeof(buf); 1854 src_iovs[2].iov_base = tmp[2]; 1855 src_iovs[2].iov_len = sizeof(tmp[2]); 1856 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1857 &src_iovs[2], 1, NULL, NULL, 1858 ut_sequence_step_cb, &completed); 1859 CU_ASSERT_EQUAL(rc, 0); 1860 1861 ut_seq.complete = false; 1862 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1863 1864 poll_threads(); 1865 1866 CU_ASSERT_EQUAL(completed, 3); 1867 CU_ASSERT(ut_seq.complete); 1868 CU_ASSERT_EQUAL(ut_seq.status, 0); 1869 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1870 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 1871 1872 /* Check copy + decrypt + copy */ 1873 seq = NULL; 1874 completed = 0; 1875 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1876 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 1877 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1878 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1879 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1880 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1881 exp_iovs[0].iov_base = tmp[0]; 1882 exp_iovs[0].iov_len = sizeof(tmp[0]); 1883 exp_iovs[1].iov_base = buf; 1884 exp_iovs[1].iov_len = sizeof(buf); 1885 1886 dst_iovs[0].iov_base = tmp[1]; 1887 dst_iovs[0].iov_len = sizeof(tmp[1]); 1888 src_iovs[0].iov_base = tmp[0]; 1889 src_iovs[0].iov_len = sizeof(tmp[0]); 1890 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 1891 &src_iovs[0], 1, NULL, NULL, 1892 ut_sequence_step_cb, &completed); 1893 CU_ASSERT_EQUAL(rc, 0); 1894 1895 dst_iovs[1].iov_base = tmp[2]; 1896 dst_iovs[1].iov_len = sizeof(tmp[2]); 1897 src_iovs[1].iov_base = tmp[1]; 1898 src_iovs[1].iov_len = sizeof(tmp[1]); 1899 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 1900 &src_iovs[1], 1, NULL, NULL, 0, sizeof(tmp[2]), 1901 ut_sequence_step_cb, &completed); 1902 CU_ASSERT_EQUAL(rc, 0); 1903 1904 dst_iovs[2].iov_base = buf; 1905 dst_iovs[2].iov_len = sizeof(buf); 1906 src_iovs[2].iov_base = tmp[2]; 1907 src_iovs[2].iov_len = sizeof(tmp[2]); 1908 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 1909 &src_iovs[2], 1, NULL, NULL, 1910 ut_sequence_step_cb, &completed); 1911 CU_ASSERT_EQUAL(rc, 0); 1912 1913 ut_seq.complete = false; 1914 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1915 1916 poll_threads(); 1917 1918 CU_ASSERT_EQUAL(completed, 3); 1919 CU_ASSERT(ut_seq.complete); 1920 CU_ASSERT_EQUAL(ut_seq.status, 0); 1921 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1922 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 1923 1924 /* Check a sequence with memory domains and verify their pointers (and their contexts) are 1925 * correctly set on the next/prev task when a copy is removed */ 1926 seq = NULL; 1927 completed = 0; 1928 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 1929 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 1930 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovcnt = 1; 1931 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovcnt = 1; 1932 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_iovs = &exp_iovs[0]; 1933 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_iovs = &exp_iovs[1]; 1934 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_domain = (void *)0x1; 1935 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].dst_domain_ctx = (void *)0x2; 1936 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_domain = (void *)0x3; 1937 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].src_domain_ctx = (void *)0x4; 1938 exp_iovs[0].iov_base = tmp[0]; 1939 exp_iovs[0].iov_len = sizeof(tmp[0]); 1940 exp_iovs[1].iov_base = buf; 1941 exp_iovs[1].iov_len = sizeof(buf); 1942 1943 dst_iovs[0].iov_base = tmp[1]; 1944 dst_iovs[0].iov_len = sizeof(tmp[1]); 1945 src_iovs[0].iov_base = tmp[0]; 1946 src_iovs[0].iov_len = sizeof(tmp[0]); 1947 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, (void *)0xdead, (void *)0xbeef, 1948 &src_iovs[0], 1, (void *)0x3, (void *)0x4, 1949 ut_sequence_step_cb, &completed); 1950 CU_ASSERT_EQUAL(rc, 0); 1951 1952 dst_iovs[1].iov_base = tmp[2]; 1953 dst_iovs[1].iov_len = sizeof(tmp[2]); 1954 src_iovs[1].iov_base = tmp[1]; 1955 src_iovs[1].iov_len = sizeof(tmp[1]); 1956 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 1957 &src_iovs[1], 1, (void *)0xdead, (void *)0xbeef, 0, 1958 sizeof(tmp[2]), ut_sequence_step_cb, &completed); 1959 CU_ASSERT_EQUAL(rc, 0); 1960 1961 dst_iovs[2].iov_base = buf; 1962 dst_iovs[2].iov_len = sizeof(buf); 1963 src_iovs[2].iov_base = tmp[2]; 1964 src_iovs[2].iov_len = sizeof(tmp[2]); 1965 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, (void *)0x1, (void *)0x2, 1966 &src_iovs[2], 1, (void *)0xdead, (void *)0xbeef, 1967 ut_sequence_step_cb, &completed); 1968 CU_ASSERT_EQUAL(rc, 0); 1969 1970 ut_seq.complete = false; 1971 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 1972 1973 poll_threads(); 1974 1975 CU_ASSERT_EQUAL(completed, 3); 1976 CU_ASSERT(ut_seq.complete); 1977 CU_ASSERT_EQUAL(ut_seq.status, 0); 1978 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 1979 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 1980 1981 /* Cleanup module pointers to make subsequent tests work correctly */ 1982 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 1983 g_modules_opc[i] = modules[i]; 1984 } 1985 1986 g_module.supports_memory_domains = false; 1987 ut_clear_operations(); 1988 spdk_put_io_channel(ioch); 1989 poll_threads(); 1990 } 1991 1992 static int 1993 ut_submit_decompress(struct spdk_io_channel *ch, struct spdk_accel_task *task) 1994 { 1995 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 1996 1997 spdk_accel_task_complete(task, 0); 1998 1999 return 0; 2000 } 2001 2002 static void 2003 test_sequence_accel_buffers(void) 2004 { 2005 struct spdk_accel_sequence *seq = NULL; 2006 struct spdk_io_channel *ioch; 2007 struct accel_io_channel *accel_ch; 2008 struct ut_sequence ut_seq; 2009 struct iovec src_iovs[3], dst_iovs[3]; 2010 char srcbuf[4096], dstbuf[4096], expected[4096]; 2011 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2012 void *buf[2], *domain_ctx[2], *iobuf_buf; 2013 struct spdk_memory_domain *domain[2]; 2014 struct spdk_iobuf_buffer *cache_entry; 2015 spdk_iobuf_buffer_stailq_t small_cache; 2016 uint32_t small_cache_count; 2017 int i, rc, completed; 2018 struct spdk_iobuf_opts opts_iobuf = {}; 2019 2020 /* Set up the iobuf to always use the "small" pool */ 2021 opts_iobuf.large_bufsize = 0x20000; 2022 opts_iobuf.large_pool_count = 0; 2023 opts_iobuf.small_bufsize = 0x10000; 2024 opts_iobuf.small_pool_count = 32; 2025 2026 rc = spdk_iobuf_set_opts(&opts_iobuf); 2027 CU_ASSERT(rc == 0); 2028 2029 ioch = spdk_accel_get_io_channel(); 2030 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2031 2032 /* Override the submit_tasks function */ 2033 g_module_if.submit_tasks = ut_sequence_submit_tasks; 2034 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2035 modules[i] = g_modules_opc[i]; 2036 g_modules_opc[i] = g_module; 2037 } 2038 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2039 * decompress operations together in one sequence. 2040 */ 2041 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2042 g_seq_operations[SPDK_ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2043 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2044 2045 /* Check the simplest case: one operation using accel buffer as destination + copy operation 2046 * specifying the actual destination buffer 2047 */ 2048 memset(srcbuf, 0xa5, 4096); 2049 memset(dstbuf, 0x0, 4096); 2050 memset(expected, 0xa5, 4096); 2051 completed = 0; 2052 seq = NULL; 2053 2054 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2055 CU_ASSERT_EQUAL(rc, 0); 2056 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2057 2058 src_iovs[0].iov_base = srcbuf; 2059 src_iovs[0].iov_len = 4096; 2060 dst_iovs[0].iov_base = buf[0]; 2061 dst_iovs[0].iov_len = 4096; 2062 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2063 &src_iovs[0], 1, NULL, NULL, 2064 ut_sequence_step_cb, &completed); 2065 CU_ASSERT_EQUAL(rc, 0); 2066 2067 src_iovs[1].iov_base = buf[0]; 2068 src_iovs[1].iov_len = 4096; 2069 dst_iovs[1].iov_base = dstbuf; 2070 dst_iovs[1].iov_len = 4096; 2071 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2072 &src_iovs[1], 1, domain[0], domain_ctx[0], 2073 ut_sequence_step_cb, &completed); 2074 CU_ASSERT_EQUAL(rc, 0); 2075 2076 ut_seq.complete = false; 2077 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2078 2079 poll_threads(); 2080 2081 CU_ASSERT_EQUAL(completed, 2); 2082 CU_ASSERT(ut_seq.complete); 2083 CU_ASSERT_EQUAL(ut_seq.status, 0); 2084 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2085 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2086 2087 /* Start with a fill operation using accel buffer, followed by a decompress using another 2088 * accel buffer as dst, followed by a copy operation specifying dst buffer of the whole 2089 * sequence 2090 */ 2091 memset(srcbuf, 0x0, 4096); 2092 memset(dstbuf, 0x0, 4096); 2093 memset(expected, 0x5a, 4096); 2094 completed = 0; 2095 seq = NULL; 2096 2097 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2098 CU_ASSERT_EQUAL(rc, 0); 2099 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2100 2101 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 2102 ut_sequence_step_cb, &completed); 2103 CU_ASSERT_EQUAL(rc, 0); 2104 2105 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2106 CU_ASSERT_EQUAL(rc, 0); 2107 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2108 2109 src_iovs[0].iov_base = buf[0]; 2110 src_iovs[0].iov_len = 4096; 2111 dst_iovs[0].iov_base = buf[1]; 2112 dst_iovs[0].iov_len = 4096; 2113 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2114 &src_iovs[0], 1, domain[0], domain_ctx[0], 2115 ut_sequence_step_cb, &completed); 2116 CU_ASSERT_EQUAL(rc, 0); 2117 2118 src_iovs[1].iov_base = buf[1]; 2119 src_iovs[1].iov_len = 4096; 2120 dst_iovs[1].iov_base = dstbuf; 2121 dst_iovs[1].iov_len = 4096; 2122 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2123 &src_iovs[1], 1, domain[1], domain_ctx[1], 2124 ut_sequence_step_cb, &completed); 2125 2126 ut_seq.complete = false; 2127 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2128 2129 poll_threads(); 2130 2131 CU_ASSERT_EQUAL(completed, 3); 2132 CU_ASSERT(ut_seq.complete); 2133 CU_ASSERT_EQUAL(ut_seq.status, 0); 2134 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2135 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2136 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2137 2138 /* Check the same, but with two decompress operations with the first one being in-place */ 2139 memset(srcbuf, 0x0, 4096); 2140 memset(dstbuf, 0x0, 4096); 2141 memset(expected, 0x5a, 4096); 2142 completed = 0; 2143 seq = NULL; 2144 2145 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2146 CU_ASSERT_EQUAL(rc, 0); 2147 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2148 2149 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 2150 ut_sequence_step_cb, &completed); 2151 CU_ASSERT_EQUAL(rc, 0); 2152 2153 src_iovs[0].iov_base = buf[0]; 2154 src_iovs[0].iov_len = 4096; 2155 dst_iovs[0].iov_base = buf[0]; 2156 dst_iovs[0].iov_len = 4096; 2157 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[0], domain_ctx[0], 2158 &src_iovs[0], 1, domain[0], domain_ctx[0], 2159 ut_sequence_step_cb, &completed); 2160 CU_ASSERT_EQUAL(rc, 0); 2161 2162 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2163 CU_ASSERT_EQUAL(rc, 0); 2164 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2165 2166 src_iovs[1].iov_base = buf[0]; 2167 src_iovs[1].iov_len = 4096; 2168 dst_iovs[1].iov_base = buf[1]; 2169 dst_iovs[1].iov_len = 4096; 2170 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, domain[1], domain_ctx[1], 2171 &src_iovs[1], 1, domain[0], domain_ctx[0], 2172 ut_sequence_step_cb, &completed); 2173 CU_ASSERT_EQUAL(rc, 0); 2174 2175 src_iovs[2].iov_base = buf[1]; 2176 src_iovs[2].iov_len = 4096; 2177 dst_iovs[2].iov_base = dstbuf; 2178 dst_iovs[2].iov_len = 4096; 2179 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 2180 &src_iovs[2], 1, domain[1], domain_ctx[1], 2181 ut_sequence_step_cb, &completed); 2182 2183 ut_seq.complete = false; 2184 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2185 2186 poll_threads(); 2187 2188 CU_ASSERT_EQUAL(completed, 4); 2189 CU_ASSERT(ut_seq.complete); 2190 CU_ASSERT_EQUAL(ut_seq.status, 0); 2191 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2192 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2193 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2194 2195 /* Check that specifying offsets within accel buffers works correctly */ 2196 memset(srcbuf, 0x0, 4096); 2197 memset(dstbuf, 0x0, 4096); 2198 completed = 0; 2199 seq = NULL; 2200 2201 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2202 CU_ASSERT_EQUAL(rc, 0); 2203 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2204 2205 /* Fill in each 1K of the buffer with different pattern */ 2206 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 2207 ut_sequence_step_cb, &completed); 2208 CU_ASSERT_EQUAL(rc, 0); 2209 2210 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2211 0x5a, ut_sequence_step_cb, &completed); 2212 CU_ASSERT_EQUAL(rc, 0); 2213 2214 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2215 0xfe, ut_sequence_step_cb, &completed); 2216 CU_ASSERT_EQUAL(rc, 0); 2217 2218 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2219 0xed, ut_sequence_step_cb, &completed); 2220 CU_ASSERT_EQUAL(rc, 0); 2221 2222 src_iovs[0].iov_base = buf[0]; 2223 src_iovs[0].iov_len = 4096; 2224 dst_iovs[0].iov_base = dstbuf; 2225 dst_iovs[0].iov_len = 4096; 2226 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2227 &src_iovs[0], 1, domain[0], domain_ctx[0], 2228 ut_sequence_step_cb, &completed); 2229 2230 ut_seq.complete = false; 2231 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2232 2233 poll_threads(); 2234 2235 memset(&expected[0], 0xa5, 1024); 2236 memset(&expected[1024], 0x5a, 1024); 2237 memset(&expected[2048], 0xfe, 1024); 2238 memset(&expected[3072], 0xed, 1024); 2239 CU_ASSERT_EQUAL(completed, 5); 2240 CU_ASSERT(ut_seq.complete); 2241 CU_ASSERT_EQUAL(ut_seq.status, 0); 2242 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2243 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2244 2245 /* Check the same but this time with a decompress operation on part of the buffer (512B 2246 * offset) */ 2247 memset(srcbuf, 0x0, 4096); 2248 memset(dstbuf, 0x0, 4096); 2249 completed = 0; 2250 seq = NULL; 2251 2252 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2253 CU_ASSERT_EQUAL(rc, 0); 2254 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2255 2256 /* Fill in each 1K of the buffer with different pattern */ 2257 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 1024, domain[0], domain_ctx[0], 0xa5, 2258 ut_sequence_step_cb, &completed); 2259 CU_ASSERT_EQUAL(rc, 0); 2260 2261 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 1024, 1024, domain[0], domain_ctx[0], 2262 0x5a, ut_sequence_step_cb, &completed); 2263 CU_ASSERT_EQUAL(rc, 0); 2264 2265 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 2048, 1024, domain[0], domain_ctx[0], 2266 0xfe, ut_sequence_step_cb, &completed); 2267 CU_ASSERT_EQUAL(rc, 0); 2268 2269 rc = spdk_accel_append_fill(&seq, ioch, (char *)buf[0] + 3072, 1024, domain[0], domain_ctx[0], 2270 0xed, ut_sequence_step_cb, &completed); 2271 CU_ASSERT_EQUAL(rc, 0); 2272 2273 rc = spdk_accel_get_buf(ioch, 3072, &buf[1], &domain[1], &domain_ctx[1]); 2274 CU_ASSERT_EQUAL(rc, 0); 2275 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2276 2277 src_iovs[0].iov_base = (char *)buf[0] + 512; 2278 src_iovs[0].iov_len = 3072; 2279 dst_iovs[0].iov_base = (char *)buf[1] + 256; 2280 dst_iovs[0].iov_len = 3072; 2281 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2282 &src_iovs[0], 1, domain[0], domain_ctx[0], 2283 ut_sequence_step_cb, &completed); 2284 2285 src_iovs[1].iov_base = (char *)buf[1] + 256; 2286 src_iovs[1].iov_len = 3072; 2287 dst_iovs[1].iov_base = dstbuf; 2288 dst_iovs[1].iov_len = 3072; 2289 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2290 &src_iovs[1], 1, domain[1], domain_ctx[1], 2291 ut_sequence_step_cb, &completed); 2292 2293 ut_seq.complete = false; 2294 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2295 2296 poll_threads(); 2297 2298 memset(&expected[0], 0xa5, 512); 2299 memset(&expected[512], 0x5a, 1024); 2300 memset(&expected[1536], 0xfe, 1024); 2301 memset(&expected[2560], 0xed, 512); 2302 CU_ASSERT_EQUAL(completed, 6); 2303 CU_ASSERT(ut_seq.complete); 2304 CU_ASSERT_EQUAL(ut_seq.status, 0); 2305 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 3072), 0); 2306 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2307 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2308 2309 /* Check that if iobuf pool is empty, the sequence processing will wait until a buffer is 2310 * available 2311 */ 2312 accel_ch = spdk_io_channel_get_ctx(ioch); 2313 small_cache_count = accel_ch->iobuf.small.cache_count; 2314 STAILQ_INIT(&small_cache); 2315 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2316 accel_ch->iobuf.small.cache_count = 0; 2317 accel_ch->iobuf.small.cache_size = 0; 2318 g_iobuf.small_pool_count = 0; 2319 2320 /* First allocate a single buffer used by two operations */ 2321 memset(srcbuf, 0x0, 4096); 2322 memset(dstbuf, 0x0, 4096); 2323 memset(expected, 0xa5, 4096); 2324 completed = 0; 2325 seq = NULL; 2326 2327 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2328 CU_ASSERT_EQUAL(rc, 0); 2329 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2330 2331 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0xa5, 2332 ut_sequence_step_cb, &completed); 2333 CU_ASSERT_EQUAL(rc, 0); 2334 2335 src_iovs[0].iov_base = buf[0]; 2336 src_iovs[0].iov_len = 4096; 2337 dst_iovs[0].iov_base = dstbuf; 2338 dst_iovs[0].iov_len = 4096; 2339 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 2340 &src_iovs[0], 1, domain[0], domain_ctx[0], 2341 ut_sequence_step_cb, &completed); 2342 2343 ut_seq.complete = false; 2344 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2345 2346 poll_threads(); 2347 2348 CU_ASSERT_EQUAL(completed, 0); 2349 CU_ASSERT(!ut_seq.complete); 2350 2351 /* Get a buffer and return it to the pool to trigger the sequence to finish */ 2352 g_iobuf.small_pool_count = 1; 2353 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2354 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2355 CU_ASSERT(g_iobuf.small_pool_count == 0); 2356 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2357 2358 poll_threads(); 2359 2360 CU_ASSERT_EQUAL(completed, 2); 2361 CU_ASSERT(ut_seq.complete); 2362 CU_ASSERT_EQUAL(ut_seq.status, 0); 2363 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2364 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2365 2366 /* Return the buffers back to the cache */ 2367 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2368 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2369 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2370 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2371 small_cache_count++; 2372 } 2373 accel_ch->iobuf.small.cache_count = 0; 2374 g_iobuf.small_pool_count = 0; 2375 2376 /* Check a bit more complex scenario, with two buffers in the sequence */ 2377 memset(srcbuf, 0x0, 4096); 2378 memset(dstbuf, 0x0, 4096); 2379 memset(expected, 0x5a, 4096); 2380 completed = 0; 2381 seq = NULL; 2382 2383 rc = spdk_accel_get_buf(ioch, 4096, &buf[0], &domain[0], &domain_ctx[0]); 2384 CU_ASSERT_EQUAL(rc, 0); 2385 CU_ASSERT_PTR_NOT_NULL(buf[0]); 2386 2387 rc = spdk_accel_append_fill(&seq, ioch, buf[0], 4096, domain[0], domain_ctx[0], 0x5a, 2388 ut_sequence_step_cb, &completed); 2389 CU_ASSERT_EQUAL(rc, 0); 2390 2391 rc = spdk_accel_get_buf(ioch, 4096, &buf[1], &domain[1], &domain_ctx[1]); 2392 CU_ASSERT_EQUAL(rc, 0); 2393 CU_ASSERT_PTR_NOT_NULL(buf[1]); 2394 2395 src_iovs[0].iov_base = buf[0]; 2396 src_iovs[0].iov_len = 4096; 2397 dst_iovs[0].iov_base = buf[1]; 2398 dst_iovs[0].iov_len = 4096; 2399 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, domain[1], domain_ctx[1], 2400 &src_iovs[0], 1, domain[0], domain_ctx[0], 2401 ut_sequence_step_cb, &completed); 2402 CU_ASSERT_EQUAL(rc, 0); 2403 2404 src_iovs[1].iov_base = buf[1]; 2405 src_iovs[1].iov_len = 4096; 2406 dst_iovs[1].iov_base = dstbuf; 2407 dst_iovs[1].iov_len = 4096; 2408 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2409 &src_iovs[1], 1, domain[1], domain_ctx[1], 2410 ut_sequence_step_cb, &completed); 2411 CU_ASSERT_EQUAL(rc, 0); 2412 2413 ut_seq.complete = false; 2414 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2415 2416 poll_threads(); 2417 2418 CU_ASSERT_EQUAL(completed, 0); 2419 CU_ASSERT(!ut_seq.complete); 2420 2421 g_iobuf.small_pool_count = 1; 2422 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2423 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2424 g_iobuf.small_pool_count = 0; 2425 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2426 2427 /* One buffer is not enough to finish the whole sequence */ 2428 poll_threads(); 2429 CU_ASSERT(!ut_seq.complete); 2430 2431 g_iobuf.small_pool_count = 1; 2432 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, 4096, NULL, NULL); 2433 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2434 g_iobuf.small_pool_count = 0; 2435 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, 4096); 2436 2437 poll_threads(); 2438 2439 CU_ASSERT_EQUAL(completed, 3); 2440 CU_ASSERT(ut_seq.complete); 2441 CU_ASSERT_EQUAL(ut_seq.status, 0); 2442 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2443 spdk_accel_put_buf(ioch, buf[0], domain[0], domain_ctx[0]); 2444 spdk_accel_put_buf(ioch, buf[1], domain[1], domain_ctx[1]); 2445 2446 /* Return the buffers back to the cache */ 2447 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2448 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2449 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2450 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2451 small_cache_count++; 2452 } 2453 accel_ch->iobuf.small.cache_count = 0; 2454 2455 g_iobuf.small_pool_count = 32; 2456 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2457 accel_ch->iobuf.small.cache_count = small_cache_count; 2458 2459 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2460 g_modules_opc[i] = modules[i]; 2461 } 2462 2463 ut_clear_operations(); 2464 spdk_put_io_channel(ioch); 2465 poll_threads(); 2466 } 2467 2468 static void 2469 ut_domain_ctx_init(struct ut_domain_ctx *ctx, void *base, size_t len, struct iovec *expected) 2470 { 2471 ctx->iov.iov_base = base; 2472 ctx->iov.iov_len = len; 2473 ctx->expected = *expected; 2474 ctx->pull_submit_status = 0; 2475 ctx->push_submit_status = 0; 2476 ctx->pull_complete_status = 0; 2477 ctx->push_complete_status = 0; 2478 } 2479 2480 static void 2481 test_sequence_memory_domain(void) 2482 { 2483 struct spdk_accel_sequence *seq = NULL; 2484 struct spdk_io_channel *ioch; 2485 struct accel_io_channel *accel_ch; 2486 struct ut_sequence ut_seq; 2487 struct ut_domain_ctx domctx[4]; 2488 struct spdk_iobuf_buffer *cache_entry; 2489 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2490 struct spdk_memory_domain *accel_domain; 2491 spdk_iobuf_buffer_stailq_t small_cache; 2492 char srcbuf[4096], dstbuf[4096], expected[4096], tmp[4096]; 2493 struct iovec src_iovs[2], dst_iovs[2]; 2494 uint32_t small_cache_count; 2495 void *accel_buf, *accel_domain_ctx, *iobuf_buf; 2496 int i, rc, completed; 2497 2498 ioch = spdk_accel_get_io_channel(); 2499 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2500 2501 /* Override the submit_tasks function */ 2502 g_module_if.submit_tasks = ut_sequence_submit_tasks; 2503 g_module.supports_memory_domains = false; 2504 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2505 modules[i] = g_modules_opc[i]; 2506 g_modules_opc[i] = g_module; 2507 } 2508 /* Intercept decompress to make it simply copy the data, so that we can chain multiple 2509 * decompress operations together in one sequence. 2510 */ 2511 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 2512 g_seq_operations[SPDK_ACCEL_OPC_COPY].submit = sw_accel_submit_tasks; 2513 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2514 2515 /* First check the simplest case - single fill operation with dstbuf in domain */ 2516 memset(expected, 0xa5, sizeof(expected)); 2517 memset(dstbuf, 0x0, sizeof(dstbuf)); 2518 completed = 0; 2519 seq = NULL; 2520 2521 /* Use some garbage pointer as dst and use domain ctx to get the actual dstbuf */ 2522 dst_iovs[0].iov_base = (void *)0xcafebabe; 2523 dst_iovs[0].iov_len = sizeof(dstbuf); 2524 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2525 2526 rc = spdk_accel_append_fill(&seq, ioch, dst_iovs[0].iov_base, dst_iovs[0].iov_len, 2527 g_ut_domain, &domctx[0], 0xa5, 2528 ut_sequence_step_cb, &completed); 2529 CU_ASSERT_EQUAL(rc, 0); 2530 2531 ut_seq.complete = false; 2532 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2533 2534 poll_threads(); 2535 CU_ASSERT_EQUAL(completed, 1); 2536 CU_ASSERT(ut_seq.complete); 2537 CU_ASSERT_EQUAL(ut_seq.status, 0); 2538 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2539 2540 /* Check operation with both srcbuf and dstbuf in remote memory domain */ 2541 memset(expected, 0x5a, sizeof(expected)); 2542 memset(dstbuf, 0x0, sizeof(dstbuf)); 2543 memset(srcbuf, 0x5a, sizeof(dstbuf)); 2544 completed = 0; 2545 seq = NULL; 2546 2547 src_iovs[0].iov_base = (void *)0xcafebabe; 2548 src_iovs[0].iov_len = sizeof(srcbuf); 2549 dst_iovs[0].iov_base = (void *)0xbeefdead; 2550 dst_iovs[0].iov_len = sizeof(dstbuf); 2551 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2552 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2553 2554 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2555 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2556 ut_sequence_step_cb, &completed); 2557 CU_ASSERT_EQUAL(rc, 0); 2558 2559 ut_seq.complete = false; 2560 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2561 2562 poll_threads(); 2563 CU_ASSERT_EQUAL(completed, 1); 2564 CU_ASSERT(ut_seq.complete); 2565 CU_ASSERT_EQUAL(ut_seq.status, 0); 2566 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2567 2568 /* Two operations using a buffer in remote domain */ 2569 memset(expected, 0xa5, sizeof(expected)); 2570 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2571 memset(tmp, 0x0, sizeof(tmp)); 2572 memset(dstbuf, 0x0, sizeof(dstbuf)); 2573 completed = 0; 2574 seq = NULL; 2575 2576 src_iovs[0].iov_base = (void *)0xcafebabe; 2577 src_iovs[0].iov_len = sizeof(srcbuf); 2578 dst_iovs[0].iov_base = (void *)0xbeefdead; 2579 dst_iovs[0].iov_len = sizeof(tmp); 2580 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2581 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2582 2583 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2584 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2585 ut_sequence_step_cb, &completed); 2586 CU_ASSERT_EQUAL(rc, 0); 2587 2588 src_iovs[1].iov_base = (void *)0xbeefdead; 2589 src_iovs[1].iov_len = sizeof(tmp); 2590 dst_iovs[1].iov_base = (void *)0xa5a5a5a5; 2591 dst_iovs[1].iov_len = sizeof(dstbuf); 2592 ut_domain_ctx_init(&domctx[2], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2593 ut_domain_ctx_init(&domctx[3], tmp, sizeof(tmp), &src_iovs[1]); 2594 2595 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[2], 2596 &src_iovs[1], 1, g_ut_domain, &domctx[3], 2597 ut_sequence_step_cb, &completed); 2598 CU_ASSERT_EQUAL(rc, 0); 2599 2600 ut_seq.complete = false; 2601 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2602 2603 poll_threads(); 2604 CU_ASSERT_EQUAL(completed, 2); 2605 CU_ASSERT(ut_seq.complete); 2606 CU_ASSERT_EQUAL(ut_seq.status, 0); 2607 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2608 2609 /* Use buffer in remote memory domain and accel buffer at the same time */ 2610 memset(expected, 0x5a, sizeof(expected)); 2611 memset(srcbuf, 0x5a, sizeof(expected)); 2612 memset(dstbuf, 0x0, sizeof(dstbuf)); 2613 completed = 0; 2614 seq = NULL; 2615 2616 rc = spdk_accel_get_buf(ioch, sizeof(dstbuf), &accel_buf, &accel_domain, &accel_domain_ctx); 2617 CU_ASSERT_EQUAL(rc, 0); 2618 2619 src_iovs[0].iov_base = (void *)0xfeedbeef; 2620 src_iovs[0].iov_len = sizeof(srcbuf); 2621 dst_iovs[0].iov_base = accel_buf; 2622 dst_iovs[0].iov_len = sizeof(dstbuf); 2623 ut_domain_ctx_init(&domctx[0], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2624 2625 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, accel_domain, accel_domain_ctx, 2626 &src_iovs[0], 1, g_ut_domain, &domctx[0], 2627 ut_sequence_step_cb, &completed); 2628 CU_ASSERT_EQUAL(rc, 0); 2629 2630 src_iovs[1].iov_base = accel_buf; 2631 src_iovs[1].iov_len = sizeof(dstbuf); 2632 dst_iovs[1].iov_base = (void *)0xbeeffeed; 2633 dst_iovs[1].iov_len = sizeof(dstbuf); 2634 ut_domain_ctx_init(&domctx[1], dstbuf, sizeof(dstbuf), &dst_iovs[1]); 2635 2636 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, g_ut_domain, &domctx[1], 2637 &src_iovs[1], 1, accel_domain, accel_domain_ctx, 2638 ut_sequence_step_cb, &completed); 2639 CU_ASSERT_EQUAL(rc, 0); 2640 2641 ut_seq.complete = false; 2642 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2643 2644 poll_threads(); 2645 CU_ASSERT_EQUAL(completed, 2); 2646 CU_ASSERT(ut_seq.complete); 2647 CU_ASSERT_EQUAL(ut_seq.status, 0); 2648 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2649 spdk_accel_put_buf(ioch, accel_buf, accel_domain, accel_domain_ctx); 2650 2651 /* Check that a sequence with memory domains is correctly executed if buffers are not 2652 * immediately available */ 2653 memset(expected, 0xa5, sizeof(expected)); 2654 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2655 memset(dstbuf, 0x0, sizeof(dstbuf)); 2656 completed = 0; 2657 seq = NULL; 2658 /* Make sure the buffer pool is empty */ 2659 accel_ch = spdk_io_channel_get_ctx(ioch); 2660 small_cache_count = accel_ch->iobuf.small.cache_count; 2661 STAILQ_INIT(&small_cache); 2662 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2663 accel_ch->iobuf.small.cache_count = 0; 2664 g_iobuf.small_pool_count = 0; 2665 2666 src_iovs[0].iov_base = (void *)0xdeadbeef; 2667 src_iovs[0].iov_len = sizeof(srcbuf); 2668 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2669 dst_iovs[0].iov_len = sizeof(dstbuf); 2670 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2671 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2672 2673 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2674 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2675 ut_sequence_step_cb, &completed); 2676 CU_ASSERT_EQUAL(rc, 0); 2677 2678 ut_seq.complete = false; 2679 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2680 2681 poll_threads(); 2682 CU_ASSERT_EQUAL(completed, 0); 2683 CU_ASSERT(!ut_seq.complete); 2684 2685 /* Get a buffer and return it to the pool to trigger the sequence to resume. It shouldn't 2686 * be able to complete, as it needs two buffers */ 2687 g_iobuf.small_pool_count = 1; 2688 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2689 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2690 g_iobuf.small_pool_count = 0; 2691 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2692 2693 CU_ASSERT_EQUAL(completed, 0); 2694 CU_ASSERT(!ut_seq.complete); 2695 2696 /* Return another buffer, this time the sequence should finish */ 2697 g_iobuf.small_pool_count = 1; 2698 iobuf_buf = spdk_iobuf_get(&accel_ch->iobuf, sizeof(dstbuf), NULL, NULL); 2699 CU_ASSERT_PTR_NOT_NULL(iobuf_buf); 2700 g_iobuf.small_pool_count = 0; 2701 spdk_iobuf_put(&accel_ch->iobuf, iobuf_buf, sizeof(dstbuf)); 2702 2703 poll_threads(); 2704 CU_ASSERT_EQUAL(completed, 1); 2705 CU_ASSERT(ut_seq.complete); 2706 CU_ASSERT_EQUAL(ut_seq.status, 0); 2707 CU_ASSERT_EQUAL(memcmp(expected, dstbuf, sizeof(dstbuf)), 0); 2708 2709 /* Return the buffers back to the cache */ 2710 while (!STAILQ_EMPTY(&accel_ch->iobuf.small.cache)) { 2711 cache_entry = STAILQ_FIRST(&accel_ch->iobuf.small.cache); 2712 STAILQ_REMOVE_HEAD(&accel_ch->iobuf.small.cache, stailq); 2713 STAILQ_INSERT_HEAD(&small_cache, cache_entry, stailq); 2714 small_cache_count++; 2715 } 2716 accel_ch->iobuf.small.cache_count = 0; 2717 2718 g_iobuf.small_pool_count = 32; 2719 STAILQ_SWAP(&accel_ch->iobuf.small.cache, &small_cache, spdk_iobuf_buffer); 2720 accel_ch->iobuf.small.cache_count = small_cache_count; 2721 2722 /* Check error cases, starting with an error from spdk_memory_domain_pull_data() */ 2723 completed = 0; 2724 seq = NULL; 2725 2726 src_iovs[0].iov_base = (void *)0xdeadbeef; 2727 src_iovs[0].iov_len = sizeof(srcbuf); 2728 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2729 dst_iovs[0].iov_len = sizeof(dstbuf); 2730 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2731 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2732 domctx[1].pull_submit_status = -E2BIG; 2733 2734 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2735 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2736 ut_sequence_step_cb, &completed); 2737 CU_ASSERT_EQUAL(rc, 0); 2738 2739 ut_seq.complete = false; 2740 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2741 2742 CU_ASSERT_EQUAL(completed, 1); 2743 CU_ASSERT(ut_seq.complete); 2744 CU_ASSERT_EQUAL(ut_seq.status, -E2BIG); 2745 2746 /* Check completion error from spdk_memory_domain_pull_data() */ 2747 completed = 0; 2748 seq = NULL; 2749 2750 src_iovs[0].iov_base = (void *)0xdeadbeef; 2751 src_iovs[0].iov_len = sizeof(srcbuf); 2752 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2753 dst_iovs[0].iov_len = sizeof(dstbuf); 2754 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2755 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2756 domctx[1].pull_complete_status = -EACCES; 2757 2758 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2759 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2760 ut_sequence_step_cb, &completed); 2761 CU_ASSERT_EQUAL(rc, 0); 2762 2763 ut_seq.complete = false; 2764 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2765 2766 CU_ASSERT_EQUAL(completed, 1); 2767 CU_ASSERT(ut_seq.complete); 2768 CU_ASSERT_EQUAL(ut_seq.status, -EACCES); 2769 2770 /* Check submission error from spdk_memory_domain_push_data() */ 2771 completed = 0; 2772 seq = NULL; 2773 2774 src_iovs[0].iov_base = (void *)0xdeadbeef; 2775 src_iovs[0].iov_len = sizeof(srcbuf); 2776 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2777 dst_iovs[0].iov_len = sizeof(dstbuf); 2778 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2779 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2780 domctx[0].push_submit_status = -EADDRINUSE; 2781 2782 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2783 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2784 ut_sequence_step_cb, &completed); 2785 CU_ASSERT_EQUAL(rc, 0); 2786 2787 ut_seq.complete = false; 2788 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2789 2790 CU_ASSERT_EQUAL(completed, 1); 2791 CU_ASSERT(ut_seq.complete); 2792 CU_ASSERT_EQUAL(ut_seq.status, -EADDRINUSE); 2793 2794 /* Check completion error from spdk_memory_domain_push_data() */ 2795 completed = 0; 2796 seq = NULL; 2797 2798 src_iovs[0].iov_base = (void *)0xdeadbeef; 2799 src_iovs[0].iov_len = sizeof(srcbuf); 2800 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2801 dst_iovs[0].iov_len = sizeof(dstbuf); 2802 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2803 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2804 domctx[0].push_complete_status = -EADDRNOTAVAIL; 2805 2806 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2807 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2808 ut_sequence_step_cb, &completed); 2809 CU_ASSERT_EQUAL(rc, 0); 2810 2811 ut_seq.complete = false; 2812 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2813 2814 CU_ASSERT_EQUAL(completed, 1); 2815 CU_ASSERT(ut_seq.complete); 2816 CU_ASSERT_EQUAL(ut_seq.status, -EADDRNOTAVAIL); 2817 2818 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2819 g_modules_opc[i] = modules[i]; 2820 } 2821 2822 ut_clear_operations(); 2823 spdk_put_io_channel(ioch); 2824 poll_threads(); 2825 } 2826 2827 static int 2828 ut_submit_decompress_memory_domain(struct spdk_io_channel *ch, struct spdk_accel_task *task) 2829 { 2830 struct ut_domain_ctx *ctx; 2831 struct iovec *src_iovs, *dst_iovs; 2832 uint32_t src_iovcnt, dst_iovcnt; 2833 2834 src_iovs = task->s.iovs; 2835 dst_iovs = task->d.iovs; 2836 src_iovcnt = task->s.iovcnt; 2837 dst_iovcnt = task->d.iovcnt; 2838 2839 if (task->src_domain != NULL) { 2840 ctx = task->src_domain_ctx; 2841 CU_ASSERT_EQUAL(memcmp(task->s.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2842 src_iovs = &ctx->iov; 2843 src_iovcnt = 1; 2844 } 2845 2846 if (task->dst_domain != NULL) { 2847 ctx = task->dst_domain_ctx; 2848 CU_ASSERT_EQUAL(memcmp(task->d.iovs, &ctx->expected, sizeof(struct iovec)), 0); 2849 dst_iovs = &ctx->iov; 2850 dst_iovcnt = 1; 2851 } 2852 2853 spdk_iovcpy(src_iovs, src_iovcnt, dst_iovs, dst_iovcnt); 2854 spdk_accel_task_complete(task, 0); 2855 2856 return 0; 2857 } 2858 2859 static void 2860 test_sequence_module_memory_domain(void) 2861 { 2862 struct spdk_accel_sequence *seq = NULL; 2863 struct spdk_io_channel *ioch; 2864 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 2865 struct spdk_memory_domain *accel_domain; 2866 struct ut_sequence ut_seq; 2867 struct ut_domain_ctx domctx[2]; 2868 struct iovec src_iovs[2], dst_iovs[2]; 2869 void *buf, *accel_domain_ctx; 2870 char srcbuf[4096], dstbuf[4096], tmp[4096], expected[4096]; 2871 int i, rc, completed; 2872 2873 ioch = spdk_accel_get_io_channel(); 2874 SPDK_CU_ASSERT_FATAL(ioch != NULL); 2875 2876 /* Override the submit_tasks function */ 2877 g_module_if.submit_tasks = ut_sequence_submit_tasks; 2878 g_module.supports_memory_domains = true; 2879 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2880 modules[i] = g_modules_opc[i]; 2881 g_modules_opc[i] = g_module; 2882 } 2883 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress_memory_domain; 2884 g_seq_operations[SPDK_ACCEL_OPC_FILL].submit = sw_accel_submit_tasks; 2885 2886 /* Check a sequence with both buffers in memory domains */ 2887 memset(srcbuf, 0xa5, sizeof(srcbuf)); 2888 memset(expected, 0xa5, sizeof(expected)); 2889 memset(dstbuf, 0, sizeof(dstbuf)); 2890 seq = NULL; 2891 completed = 0; 2892 2893 src_iovs[0].iov_base = (void *)0xcafebabe; 2894 src_iovs[0].iov_len = sizeof(srcbuf); 2895 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2896 dst_iovs[0].iov_len = sizeof(dstbuf); 2897 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2898 ut_domain_ctx_init(&domctx[1], srcbuf, sizeof(srcbuf), &src_iovs[0]); 2899 2900 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2901 &src_iovs[0], 1, g_ut_domain, &domctx[1], 2902 ut_sequence_step_cb, &completed); 2903 CU_ASSERT_EQUAL(rc, 0); 2904 2905 ut_seq.complete = false; 2906 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2907 2908 poll_threads(); 2909 2910 CU_ASSERT_EQUAL(completed, 1); 2911 CU_ASSERT(ut_seq.complete); 2912 CU_ASSERT_EQUAL(ut_seq.status, 0); 2913 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2914 2915 /* Check two operations each with a single buffer in memory domain */ 2916 memset(srcbuf, 0x5a, sizeof(srcbuf)); 2917 memset(expected, 0x5a, sizeof(expected)); 2918 memset(dstbuf, 0, sizeof(dstbuf)); 2919 memset(tmp, 0, sizeof(tmp)); 2920 seq = NULL; 2921 completed = 0; 2922 2923 src_iovs[0].iov_base = srcbuf; 2924 src_iovs[0].iov_len = sizeof(srcbuf); 2925 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2926 dst_iovs[0].iov_len = sizeof(tmp); 2927 ut_domain_ctx_init(&domctx[0], tmp, sizeof(tmp), &dst_iovs[0]); 2928 2929 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2930 &src_iovs[0], 1, NULL, NULL, 2931 ut_sequence_step_cb, &completed); 2932 CU_ASSERT_EQUAL(rc, 0); 2933 2934 src_iovs[1].iov_base = (void *)0xfeedbeef; 2935 src_iovs[1].iov_len = sizeof(tmp); 2936 dst_iovs[1].iov_base = dstbuf; 2937 dst_iovs[1].iov_len = sizeof(dstbuf); 2938 ut_domain_ctx_init(&domctx[1], tmp, sizeof(tmp), &src_iovs[1]); 2939 2940 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 2941 &src_iovs[1], 1, g_ut_domain, &domctx[1], 2942 ut_sequence_step_cb, &completed); 2943 CU_ASSERT_EQUAL(rc, 0); 2944 2945 ut_seq.complete = false; 2946 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2947 2948 poll_threads(); 2949 2950 CU_ASSERT_EQUAL(completed, 2); 2951 CU_ASSERT(ut_seq.complete); 2952 CU_ASSERT_EQUAL(ut_seq.status, 0); 2953 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2954 2955 /* Check a sequence with an accel buffer and a buffer in a regular memory domain */ 2956 memset(expected, 0xa5, sizeof(expected)); 2957 memset(dstbuf, 0, sizeof(dstbuf)); 2958 memset(tmp, 0, sizeof(tmp)); 2959 seq = NULL; 2960 completed = 0; 2961 2962 rc = spdk_accel_get_buf(ioch, 4096, &buf, &accel_domain, &accel_domain_ctx); 2963 CU_ASSERT_EQUAL(rc, 0); 2964 2965 rc = spdk_accel_append_fill(&seq, ioch, buf, 4096, accel_domain, accel_domain_ctx, 2966 0xa5, ut_sequence_step_cb, &completed); 2967 CU_ASSERT_EQUAL(rc, 0); 2968 2969 src_iovs[0].iov_base = buf; 2970 src_iovs[0].iov_len = 4096; 2971 dst_iovs[0].iov_base = (void *)0xfeedbeef; 2972 dst_iovs[0].iov_len = sizeof(dstbuf); 2973 ut_domain_ctx_init(&domctx[0], dstbuf, sizeof(dstbuf), &dst_iovs[0]); 2974 2975 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, g_ut_domain, &domctx[0], 2976 &src_iovs[0], 1, accel_domain, accel_domain_ctx, 2977 ut_sequence_step_cb, &completed); 2978 CU_ASSERT_EQUAL(rc, 0); 2979 2980 ut_seq.complete = false; 2981 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 2982 2983 poll_threads(); 2984 2985 CU_ASSERT_EQUAL(completed, 2); 2986 CU_ASSERT(ut_seq.complete); 2987 CU_ASSERT_EQUAL(ut_seq.status, 0); 2988 CU_ASSERT_EQUAL(memcmp(dstbuf, expected, 4096), 0); 2989 2990 spdk_accel_put_buf(ioch, buf, accel_domain, accel_domain_ctx); 2991 2992 g_module.supports_memory_domains = false; 2993 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2994 g_modules_opc[i] = modules[i]; 2995 } 2996 2997 ut_clear_operations(); 2998 spdk_put_io_channel(ioch); 2999 poll_threads(); 3000 } 3001 3002 #ifdef SPDK_CONFIG_ISAL_CRYPTO 3003 static void 3004 ut_encrypt_cb(void *cb_arg, int status) 3005 { 3006 int *completed = cb_arg; 3007 3008 CU_ASSERT_EQUAL(status, 0); 3009 3010 *completed = 1; 3011 } 3012 3013 static void 3014 test_sequence_crypto(void) 3015 { 3016 struct spdk_accel_sequence *seq = NULL; 3017 struct spdk_io_channel *ioch; 3018 struct spdk_accel_crypto_key *key; 3019 struct spdk_accel_crypto_key_create_param key_params = { 3020 .cipher = "AES_XTS", 3021 .hex_key = "00112233445566778899aabbccddeeff", 3022 .hex_key2 = "ffeeddccbbaa99887766554433221100", 3023 .key_name = "ut_key", 3024 }; 3025 struct ut_sequence ut_seq; 3026 unsigned char buf[4096], encrypted[4096] = {}, data[4096], tmp[3][4096]; 3027 struct iovec src_iovs[4], dst_iovs[4]; 3028 int rc, completed = 0; 3029 size_t i; 3030 3031 ioch = spdk_accel_get_io_channel(); 3032 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3033 3034 rc = spdk_accel_crypto_key_create(&key_params); 3035 CU_ASSERT_EQUAL(rc, 0); 3036 key = spdk_accel_crypto_key_get(key_params.key_name); 3037 SPDK_CU_ASSERT_FATAL(key != NULL); 3038 3039 for (i = 0; i < sizeof(data); ++i) { 3040 data[i] = (uint8_t)i & 0xff; 3041 } 3042 3043 dst_iovs[0].iov_base = encrypted; 3044 dst_iovs[0].iov_len = sizeof(encrypted); 3045 src_iovs[0].iov_base = data; 3046 src_iovs[0].iov_len = sizeof(data); 3047 rc = spdk_accel_submit_encrypt(ioch, key, &dst_iovs[0], 1, &src_iovs[0], 1, 0, 4096, 3048 ut_encrypt_cb, &completed); 3049 CU_ASSERT_EQUAL(rc, 0); 3050 3051 while (!completed) { 3052 poll_threads(); 3053 } 3054 3055 /* Verify that encryption operation in a sequence produces the same result */ 3056 seq = NULL; 3057 completed = 0; 3058 3059 dst_iovs[0].iov_base = tmp[0]; 3060 dst_iovs[0].iov_len = sizeof(tmp[0]); 3061 src_iovs[0].iov_base = data; 3062 src_iovs[0].iov_len = sizeof(data); 3063 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3064 &src_iovs[0], 1, NULL, NULL, 3065 ut_sequence_step_cb, &completed); 3066 CU_ASSERT_EQUAL(rc, 0); 3067 3068 dst_iovs[1].iov_base = tmp[1]; 3069 dst_iovs[1].iov_len = sizeof(tmp[1]); 3070 src_iovs[1].iov_base = tmp[0]; 3071 src_iovs[1].iov_len = sizeof(tmp[0]); 3072 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3073 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3074 ut_sequence_step_cb, &completed); 3075 CU_ASSERT_EQUAL(rc, 0); 3076 3077 dst_iovs[2].iov_base = buf; 3078 dst_iovs[2].iov_len = sizeof(buf); 3079 src_iovs[2].iov_base = tmp[1]; 3080 src_iovs[2].iov_len = sizeof(tmp[1]); 3081 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3082 &src_iovs[2], 1, NULL, NULL, 3083 ut_sequence_step_cb, &completed); 3084 CU_ASSERT_EQUAL(rc, 0); 3085 3086 ut_seq.complete = false; 3087 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3088 3089 poll_threads(); 3090 3091 CU_ASSERT_EQUAL(completed, 3); 3092 CU_ASSERT(ut_seq.complete); 3093 CU_ASSERT_EQUAL(ut_seq.status, 0); 3094 CU_ASSERT_EQUAL(memcmp(buf, encrypted, sizeof(buf)), 0); 3095 3096 /* Check that decryption produces the original buffer */ 3097 seq = NULL; 3098 completed = 0; 3099 memset(buf, 0, sizeof(buf)); 3100 3101 dst_iovs[0].iov_base = tmp[0]; 3102 dst_iovs[0].iov_len = sizeof(tmp[0]); 3103 src_iovs[0].iov_base = encrypted; 3104 src_iovs[0].iov_len = sizeof(encrypted); 3105 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3106 &src_iovs[0], 1, NULL, NULL, 3107 ut_sequence_step_cb, &completed); 3108 CU_ASSERT_EQUAL(rc, 0); 3109 3110 dst_iovs[1].iov_base = tmp[1]; 3111 dst_iovs[1].iov_len = sizeof(tmp[1]); 3112 src_iovs[1].iov_base = tmp[0]; 3113 src_iovs[1].iov_len = sizeof(tmp[0]); 3114 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3115 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3116 ut_sequence_step_cb, &completed); 3117 CU_ASSERT_EQUAL(rc, 0); 3118 3119 dst_iovs[2].iov_base = buf; 3120 dst_iovs[2].iov_len = sizeof(buf); 3121 src_iovs[2].iov_base = tmp[1]; 3122 src_iovs[2].iov_len = sizeof(tmp[1]); 3123 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3124 &src_iovs[2], 1, NULL, NULL, 3125 ut_sequence_step_cb, &completed); 3126 CU_ASSERT_EQUAL(rc, 0); 3127 3128 ut_seq.complete = false; 3129 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3130 3131 poll_threads(); 3132 3133 CU_ASSERT_EQUAL(completed, 3); 3134 CU_ASSERT(ut_seq.complete); 3135 CU_ASSERT_EQUAL(ut_seq.status, 0); 3136 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3137 3138 /* Check encrypt + decrypt in a single sequence */ 3139 seq = NULL; 3140 completed = 0; 3141 memset(buf, 0, sizeof(buf)); 3142 3143 dst_iovs[0].iov_base = tmp[0]; 3144 dst_iovs[0].iov_len = sizeof(tmp[0]); 3145 src_iovs[0].iov_base = data; 3146 src_iovs[0].iov_len = sizeof(data); 3147 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3148 &src_iovs[0], 1, NULL, NULL, 3149 ut_sequence_step_cb, &completed); 3150 CU_ASSERT_EQUAL(rc, 0); 3151 3152 dst_iovs[1].iov_base = tmp[1]; 3153 dst_iovs[1].iov_len = sizeof(tmp[1]); 3154 src_iovs[1].iov_base = tmp[0]; 3155 src_iovs[1].iov_len = sizeof(tmp[0]); 3156 rc = spdk_accel_append_encrypt(&seq, ioch, key, &dst_iovs[1], 1, NULL, NULL, 3157 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3158 ut_sequence_step_cb, &completed); 3159 CU_ASSERT_EQUAL(rc, 0); 3160 3161 3162 dst_iovs[2].iov_base = tmp[2]; 3163 dst_iovs[2].iov_len = sizeof(tmp[2]); 3164 src_iovs[2].iov_base = tmp[1]; 3165 src_iovs[2].iov_len = sizeof(tmp[1]); 3166 rc = spdk_accel_append_decrypt(&seq, ioch, key, &dst_iovs[2], 1, NULL, NULL, 3167 &src_iovs[2], 1, NULL, NULL, 0, 4096, 3168 ut_sequence_step_cb, &completed); 3169 CU_ASSERT_EQUAL(rc, 0); 3170 3171 dst_iovs[3].iov_base = buf; 3172 dst_iovs[3].iov_len = sizeof(buf); 3173 src_iovs[3].iov_base = tmp[2]; 3174 src_iovs[3].iov_len = sizeof(tmp[2]); 3175 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 3176 &src_iovs[3], 1, NULL, NULL, 3177 ut_sequence_step_cb, &completed); 3178 CU_ASSERT_EQUAL(rc, 0); 3179 3180 ut_seq.complete = false; 3181 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3182 3183 poll_threads(); 3184 3185 CU_ASSERT_EQUAL(completed, 4); 3186 CU_ASSERT(ut_seq.complete); 3187 CU_ASSERT_EQUAL(ut_seq.status, 0); 3188 CU_ASSERT_EQUAL(memcmp(buf, data, sizeof(buf)), 0); 3189 3190 rc = spdk_accel_crypto_key_destroy(key); 3191 CU_ASSERT_EQUAL(rc, 0); 3192 spdk_put_io_channel(ioch); 3193 poll_threads(); 3194 } 3195 #endif /* SPDK_CONFIG_ISAL_CRYPTO */ 3196 3197 static int 3198 ut_submit_crypto(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3199 { 3200 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3201 3202 spdk_accel_task_complete(task, 0); 3203 3204 return 0; 3205 } 3206 3207 struct ut_driver_operation { 3208 int complete_status; 3209 int submit_status; 3210 int count; 3211 bool supported; 3212 }; 3213 3214 static struct ut_driver_operation g_drv_operations[SPDK_ACCEL_OPC_LAST]; 3215 static bool g_ut_driver_async_continue; 3216 3217 static void 3218 ut_driver_async_continue(void *arg) 3219 { 3220 struct spdk_accel_sequence *seq = arg; 3221 3222 spdk_accel_sequence_continue(seq); 3223 } 3224 3225 static int 3226 ut_driver_execute_sequence(struct spdk_io_channel *ch, struct spdk_accel_sequence *seq) 3227 { 3228 struct spdk_accel_task *task; 3229 struct ut_driver_operation *drv_ops; 3230 3231 while ((task = spdk_accel_sequence_first_task(seq)) != NULL) { 3232 drv_ops = &g_drv_operations[task->op_code]; 3233 if (!drv_ops->supported) { 3234 break; 3235 } 3236 3237 drv_ops->count++; 3238 if (drv_ops->submit_status != 0) { 3239 return drv_ops->submit_status; 3240 } 3241 3242 if (drv_ops->complete_status != 0) { 3243 spdk_accel_task_complete(task, drv_ops->complete_status); 3244 break; 3245 } 3246 3247 switch (task->op_code) { 3248 case SPDK_ACCEL_OPC_DECOMPRESS: 3249 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3250 break; 3251 case SPDK_ACCEL_OPC_FILL: 3252 spdk_iov_memset(task->d.iovs, task->d.iovcnt, 3253 (int)(task->fill_pattern & 0xff)); 3254 break; 3255 default: 3256 CU_ASSERT(0 && "unexpected opcode"); 3257 break; 3258 } 3259 3260 spdk_accel_task_complete(task, 0); 3261 } 3262 3263 if (g_ut_driver_async_continue) { 3264 spdk_thread_send_msg(spdk_get_thread(), ut_driver_async_continue, seq); 3265 } else { 3266 spdk_accel_sequence_continue(seq); 3267 } 3268 3269 return 0; 3270 } 3271 3272 static struct spdk_io_channel * 3273 ut_driver_get_io_channel(void) 3274 { 3275 return (void *)0xdeadbeef; 3276 } 3277 3278 static struct spdk_accel_driver g_ut_driver = { 3279 .name = "ut", 3280 .execute_sequence = ut_driver_execute_sequence, 3281 .get_io_channel = ut_driver_get_io_channel, 3282 }; 3283 3284 SPDK_ACCEL_DRIVER_REGISTER(ut, &g_ut_driver); 3285 3286 static void 3287 test_sequence_driver(void) 3288 { 3289 struct spdk_accel_sequence *seq = NULL; 3290 struct spdk_io_channel *ioch; 3291 struct spdk_accel_crypto_key key = {}; 3292 struct ut_sequence ut_seq; 3293 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3294 char buf[4096], tmp[3][4096], expected[4096]; 3295 struct iovec src_iovs[3], dst_iovs[3]; 3296 int i, rc, completed = 0; 3297 3298 ioch = spdk_accel_get_io_channel(); 3299 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3300 rc = spdk_accel_set_driver("ut"); 3301 SPDK_CU_ASSERT_FATAL(rc == 0); 3302 3303 /* Override the submit_tasks function */ 3304 g_module_if.submit_tasks = ut_sequence_submit_tasks; 3305 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3306 modules[i] = g_modules_opc[i]; 3307 g_modules_opc[i] = g_module; 3308 } 3309 /* Intercept crypto operations, as they should be executed by an accel module */ 3310 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].submit = ut_submit_crypto; 3311 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].submit = ut_submit_crypto; 3312 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3313 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3314 g_seq_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3315 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3316 3317 g_drv_operations[SPDK_ACCEL_OPC_FILL].supported = true; 3318 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].supported = true; 3319 3320 /* First check a sequence that is fully executed using a driver, with the copy at the end 3321 * being removed */ 3322 seq = NULL; 3323 completed = 0; 3324 memset(buf, 0, sizeof(buf)); 3325 memset(tmp[0], 0, sizeof(tmp[0])); 3326 memset(tmp[1], 0, sizeof(tmp[1])); 3327 memset(&expected[0], 0xa5, 2048); 3328 memset(&expected[2048], 0xbe, 2048); 3329 3330 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], 2048, NULL, NULL, 0xa5, 3331 ut_sequence_step_cb, &completed); 3332 CU_ASSERT_EQUAL(rc, 0); 3333 rc = spdk_accel_append_fill(&seq, ioch, &tmp[0][2048], 2048, NULL, NULL, 0xbe, 3334 ut_sequence_step_cb, &completed); 3335 CU_ASSERT_EQUAL(rc, 0); 3336 3337 dst_iovs[0].iov_base = tmp[1]; 3338 dst_iovs[0].iov_len = sizeof(tmp[1]); 3339 src_iovs[0].iov_base = tmp[0]; 3340 src_iovs[0].iov_len = sizeof(tmp[0]); 3341 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3342 &src_iovs[0], 1, NULL, NULL, 3343 ut_sequence_step_cb, &completed); 3344 CU_ASSERT_EQUAL(rc, 0); 3345 3346 dst_iovs[1].iov_base = buf; 3347 dst_iovs[1].iov_len = sizeof(buf); 3348 src_iovs[1].iov_base = tmp[1]; 3349 src_iovs[1].iov_len = sizeof(tmp[1]); 3350 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3351 &src_iovs[1], 1, NULL, NULL, 3352 ut_sequence_step_cb, &completed); 3353 CU_ASSERT_EQUAL(rc, 0); 3354 3355 ut_seq.complete = false; 3356 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3357 3358 poll_threads(); 3359 3360 CU_ASSERT_EQUAL(completed, 4); 3361 CU_ASSERT(ut_seq.complete); 3362 CU_ASSERT_EQUAL(ut_seq.status, 0); 3363 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3364 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3365 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 3366 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 2); 3367 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3368 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3369 3370 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3371 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3372 3373 /* Check a sequence when the first two operations are executed by a driver, while the rest 3374 * is executed via modules */ 3375 seq = NULL; 3376 completed = 0; 3377 memset(buf, 0, sizeof(buf)); 3378 memset(tmp[0], 0, sizeof(tmp[0])); 3379 memset(tmp[1], 0, sizeof(tmp[1])); 3380 memset(tmp[2], 0, sizeof(tmp[2])); 3381 memset(&expected[0], 0xfe, 4096); 3382 3383 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 3384 ut_sequence_step_cb, &completed); 3385 CU_ASSERT_EQUAL(rc, 0); 3386 3387 dst_iovs[0].iov_base = tmp[1]; 3388 dst_iovs[0].iov_len = sizeof(tmp[1]); 3389 src_iovs[0].iov_base = tmp[0]; 3390 src_iovs[0].iov_len = sizeof(tmp[0]); 3391 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3392 &src_iovs[0], 1, NULL, NULL, 3393 ut_sequence_step_cb, &completed); 3394 CU_ASSERT_EQUAL(rc, 0); 3395 3396 dst_iovs[1].iov_base = tmp[2]; 3397 dst_iovs[1].iov_len = sizeof(tmp[2]); 3398 src_iovs[1].iov_base = tmp[1]; 3399 src_iovs[1].iov_len = sizeof(tmp[1]); 3400 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3401 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3402 ut_sequence_step_cb, &completed); 3403 CU_ASSERT_EQUAL(rc, 0); 3404 3405 dst_iovs[2].iov_base = buf; 3406 dst_iovs[2].iov_len = sizeof(buf); 3407 src_iovs[2].iov_base = tmp[2]; 3408 src_iovs[2].iov_len = sizeof(tmp[2]); 3409 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3410 &src_iovs[2], 1, NULL, NULL, 0, 4096, 3411 ut_sequence_step_cb, &completed); 3412 CU_ASSERT_EQUAL(rc, 0); 3413 3414 ut_seq.complete = false; 3415 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3416 3417 poll_threads(); 3418 3419 CU_ASSERT_EQUAL(completed, 4); 3420 CU_ASSERT(ut_seq.complete); 3421 CU_ASSERT_EQUAL(ut_seq.status, 0); 3422 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3423 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3424 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3425 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3426 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3427 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3428 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3429 3430 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3431 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3432 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3433 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3434 3435 /* Check sequence when the first and last operations are executed through modules, while the 3436 * ones in the middle are executed by the driver */ 3437 seq = NULL; 3438 completed = 0; 3439 memset(buf, 0, sizeof(buf)); 3440 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3441 memset(tmp[1], 0, sizeof(tmp[1])); 3442 memset(tmp[2], 0, sizeof(tmp[2])); 3443 memset(&expected[0], 0xfe, 2048); 3444 memset(&expected[2048], 0xa5, 2048); 3445 3446 dst_iovs[0].iov_base = tmp[1]; 3447 dst_iovs[0].iov_len = sizeof(tmp[1]); 3448 src_iovs[0].iov_base = tmp[0]; 3449 src_iovs[0].iov_len = sizeof(tmp[0]); 3450 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3451 &src_iovs[0], 1, NULL, NULL, 0, 4096, 3452 ut_sequence_step_cb, &completed); 3453 CU_ASSERT_EQUAL(rc, 0); 3454 3455 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xfe, 3456 ut_sequence_step_cb, &completed); 3457 CU_ASSERT_EQUAL(rc, 0); 3458 3459 dst_iovs[1].iov_base = tmp[2]; 3460 dst_iovs[1].iov_len = sizeof(tmp[2]); 3461 src_iovs[1].iov_base = tmp[1]; 3462 src_iovs[1].iov_len = sizeof(tmp[1]); 3463 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3464 &src_iovs[1], 1, NULL, NULL, 3465 ut_sequence_step_cb, &completed); 3466 CU_ASSERT_EQUAL(rc, 0); 3467 3468 dst_iovs[2].iov_base = buf; 3469 dst_iovs[2].iov_len = sizeof(buf); 3470 src_iovs[2].iov_base = tmp[2]; 3471 src_iovs[2].iov_len = sizeof(tmp[2]); 3472 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[2], 1, NULL, NULL, 3473 &src_iovs[2], 1, NULL, NULL, 0, 4096, 3474 ut_sequence_step_cb, &completed); 3475 CU_ASSERT_EQUAL(rc, 0); 3476 3477 ut_seq.complete = false; 3478 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3479 3480 poll_threads(); 3481 3482 CU_ASSERT_EQUAL(completed, 4); 3483 CU_ASSERT(ut_seq.complete); 3484 CU_ASSERT_EQUAL(ut_seq.status, 0); 3485 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3486 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3487 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3488 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3489 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3490 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3491 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3492 3493 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3494 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3495 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3496 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3497 3498 /* Check a sequence with operations executed by: module, driver, module, driver */ 3499 seq = NULL; 3500 completed = 0; 3501 memset(buf, 0, sizeof(buf)); 3502 memset(tmp[0], 0x5a, sizeof(tmp[0])); 3503 memset(tmp[1], 0, sizeof(tmp[1])); 3504 memset(tmp[2], 0, sizeof(tmp[2])); 3505 memset(&expected[0], 0xef, 2048); 3506 memset(&expected[2048], 0x5a, 2048); 3507 3508 dst_iovs[0].iov_base = tmp[1]; 3509 dst_iovs[0].iov_len = sizeof(tmp[1]); 3510 src_iovs[0].iov_base = tmp[0]; 3511 src_iovs[0].iov_len = sizeof(tmp[0]); 3512 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3513 &src_iovs[0], 1, NULL, NULL, 0, 4096, 3514 ut_sequence_step_cb, &completed); 3515 CU_ASSERT_EQUAL(rc, 0); 3516 3517 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 3518 ut_sequence_step_cb, &completed); 3519 CU_ASSERT_EQUAL(rc, 0); 3520 3521 dst_iovs[1].iov_base = tmp[2]; 3522 dst_iovs[1].iov_len = sizeof(tmp[2]); 3523 src_iovs[1].iov_base = tmp[1]; 3524 src_iovs[1].iov_len = sizeof(tmp[1]); 3525 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3526 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3527 ut_sequence_step_cb, &completed); 3528 CU_ASSERT_EQUAL(rc, 0); 3529 3530 dst_iovs[2].iov_base = buf; 3531 dst_iovs[2].iov_len = sizeof(buf); 3532 src_iovs[2].iov_base = tmp[2]; 3533 src_iovs[2].iov_len = sizeof(tmp[2]); 3534 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 3535 &src_iovs[2], 1, NULL, NULL, 3536 ut_sequence_step_cb, &completed); 3537 CU_ASSERT_EQUAL(rc, 0); 3538 3539 ut_seq.complete = false; 3540 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3541 3542 poll_threads(); 3543 3544 CU_ASSERT_EQUAL(completed, 4); 3545 CU_ASSERT(ut_seq.complete); 3546 CU_ASSERT_EQUAL(ut_seq.status, 0); 3547 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3548 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3549 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3550 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3551 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3552 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3553 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3554 3555 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3556 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3557 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3558 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3559 3560 /* Check that an error returned from driver's execute_sequence() will fail the whole 3561 * sequence and any subsequent operations won't be processed */ 3562 seq = NULL; 3563 completed = 0; 3564 memset(buf, 0, sizeof(buf)); 3565 memset(expected, 0, sizeof(expected)); 3566 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3567 g_drv_operations[SPDK_ACCEL_OPC_FILL].submit_status = -EPERM; 3568 3569 dst_iovs[0].iov_base = tmp[1]; 3570 dst_iovs[0].iov_len = sizeof(tmp[1]); 3571 src_iovs[0].iov_base = tmp[0]; 3572 src_iovs[0].iov_len = sizeof(tmp[0]); 3573 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3574 &src_iovs[0], 1, NULL, NULL, 0, 4096, 3575 ut_sequence_step_cb, &completed); 3576 CU_ASSERT_EQUAL(rc, 0); 3577 3578 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 3579 ut_sequence_step_cb, &completed); 3580 CU_ASSERT_EQUAL(rc, 0); 3581 3582 dst_iovs[1].iov_base = buf; 3583 dst_iovs[1].iov_len = sizeof(buf); 3584 src_iovs[1].iov_base = tmp[1]; 3585 src_iovs[1].iov_len = sizeof(tmp[1]); 3586 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3587 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3588 ut_sequence_step_cb, &completed); 3589 CU_ASSERT_EQUAL(rc, 0); 3590 3591 ut_seq.complete = false; 3592 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3593 3594 poll_threads(); 3595 3596 CU_ASSERT_EQUAL(completed, 3); 3597 CU_ASSERT(ut_seq.complete); 3598 CU_ASSERT_EQUAL(ut_seq.status, -EPERM); 3599 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3600 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3601 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 0); 3602 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3603 CU_ASSERT_EQUAL(memcmp(buf, expected, 4096), 0); 3604 3605 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3606 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3607 g_drv_operations[SPDK_ACCEL_OPC_FILL].submit_status = 0; 3608 3609 /* Check that a failed task completed by a driver will cause the whole sequence to be failed 3610 * and any subsequent operations won't be processed */ 3611 seq = NULL; 3612 completed = 0; 3613 memset(buf, 0, sizeof(buf)); 3614 memset(expected, 0, sizeof(expected)); 3615 memset(tmp[0], 0xa5, sizeof(tmp[0])); 3616 g_drv_operations[SPDK_ACCEL_OPC_FILL].complete_status = -ENOENT; 3617 3618 dst_iovs[0].iov_base = tmp[1]; 3619 dst_iovs[0].iov_len = sizeof(tmp[1]); 3620 src_iovs[0].iov_base = tmp[0]; 3621 src_iovs[0].iov_len = sizeof(tmp[0]); 3622 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &dst_iovs[0], 1, NULL, NULL, 3623 &src_iovs[0], 1, NULL, NULL, 0, 4096, 3624 ut_sequence_step_cb, &completed); 3625 CU_ASSERT_EQUAL(rc, 0); 3626 3627 rc = spdk_accel_append_fill(&seq, ioch, tmp[1], 2048, NULL, NULL, 0xef, 3628 ut_sequence_step_cb, &completed); 3629 CU_ASSERT_EQUAL(rc, 0); 3630 3631 dst_iovs[1].iov_base = buf; 3632 dst_iovs[1].iov_len = sizeof(buf); 3633 src_iovs[1].iov_base = tmp[1]; 3634 src_iovs[1].iov_len = sizeof(tmp[1]); 3635 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &dst_iovs[1], 1, NULL, NULL, 3636 &src_iovs[1], 1, NULL, NULL, 0, 4096, 3637 ut_sequence_step_cb, &completed); 3638 CU_ASSERT_EQUAL(rc, 0); 3639 3640 ut_seq.complete = false; 3641 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3642 3643 poll_threads(); 3644 3645 CU_ASSERT_EQUAL(completed, 3); 3646 CU_ASSERT(ut_seq.complete); 3647 CU_ASSERT_EQUAL(ut_seq.status, -ENOENT); 3648 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3649 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3650 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 0); 3651 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3652 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3653 3654 g_drv_operations[SPDK_ACCEL_OPC_FILL].complete_status = 0; 3655 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3656 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3657 3658 /* Check asynchronous spdk_accel_sequence_continue() */ 3659 g_ut_driver_async_continue = true; 3660 seq = NULL; 3661 completed = 0; 3662 memset(buf, 0, sizeof(buf)); 3663 memset(tmp[0], 0, sizeof(tmp[0])); 3664 memset(&expected[0], 0xfe, 4096); 3665 3666 rc = spdk_accel_append_fill(&seq, ioch, tmp[0], sizeof(tmp[0]), NULL, NULL, 0xfe, 3667 ut_sequence_step_cb, &completed); 3668 CU_ASSERT_EQUAL(rc, 0); 3669 3670 dst_iovs[0].iov_base = buf; 3671 dst_iovs[0].iov_len = sizeof(buf); 3672 src_iovs[0].iov_base = tmp[0]; 3673 src_iovs[0].iov_len = sizeof(tmp[0]); 3674 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3675 &src_iovs[0], 1, NULL, NULL, 3676 ut_sequence_step_cb, &completed); 3677 CU_ASSERT_EQUAL(rc, 0); 3678 3679 ut_seq.complete = false; 3680 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3681 3682 poll_threads(); 3683 3684 CU_ASSERT_EQUAL(completed, 2); 3685 CU_ASSERT(ut_seq.complete); 3686 CU_ASSERT_EQUAL(ut_seq.status, 0); 3687 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_FILL].count, 0); 3688 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 0); 3689 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_FILL].count, 1); 3690 CU_ASSERT_EQUAL(g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 3691 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3692 3693 g_drv_operations[SPDK_ACCEL_OPC_FILL].count = 0; 3694 g_drv_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 3695 3696 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3697 g_modules_opc[i] = modules[i]; 3698 } 3699 3700 /* Clear the driver so that other tests won't use it */ 3701 g_accel_driver = NULL; 3702 memset(&g_drv_operations, 0, sizeof(g_drv_operations)); 3703 3704 ut_clear_operations(); 3705 spdk_put_io_channel(ioch); 3706 poll_threads(); 3707 } 3708 3709 struct ut_saved_iovs { 3710 struct iovec src; 3711 struct iovec dst; 3712 }; 3713 3714 static struct ut_saved_iovs g_seq_saved_iovs[SPDK_ACCEL_OPC_LAST]; 3715 3716 static int 3717 ut_submit_save_iovs(struct spdk_io_channel *ch, struct spdk_accel_task *task) 3718 { 3719 SPDK_CU_ASSERT_FATAL(task->s.iovcnt == 1); 3720 SPDK_CU_ASSERT_FATAL(task->d.iovcnt == 1); 3721 3722 g_seq_saved_iovs[task->op_code].src = task->s.iovs[0]; 3723 g_seq_saved_iovs[task->op_code].dst = task->d.iovs[0]; 3724 3725 spdk_iovmove(task->s.iovs, task->s.iovcnt, task->d.iovs, task->d.iovcnt); 3726 3727 spdk_accel_task_complete(task, 0); 3728 3729 return 0; 3730 } 3731 3732 static void 3733 test_sequence_same_iovs(void) 3734 { 3735 struct spdk_accel_sequence *seq = NULL; 3736 struct spdk_io_channel *ioch; 3737 struct spdk_accel_crypto_key key = {}; 3738 struct ut_sequence ut_seq; 3739 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3740 char buf[4096], tmp[4096], expected[4096]; 3741 struct iovec iovs[3], expected_siov, expected_diov; 3742 struct spdk_memory_domain *domain; 3743 void *accel_buf, *domain_ctx; 3744 int i, rc, completed = 0; 3745 3746 ioch = spdk_accel_get_io_channel(); 3747 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3748 3749 /* Override the submit_tasks function */ 3750 g_module_if.submit_tasks = ut_sequence_submit_tasks; 3751 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3752 modules[i] = g_modules_opc[i]; 3753 g_modules_opc[i] = g_module; 3754 } 3755 /* Intercept crypto operations, as they should be executed by an accel module */ 3756 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].submit = ut_submit_save_iovs; 3757 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].submit = ut_submit_save_iovs; 3758 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3759 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3760 3761 /* Check that it's possible to use the same iovec ptr for different operations */ 3762 seq = NULL; 3763 completed = 0; 3764 memset(buf, 0, sizeof(buf)); 3765 memset(expected, 0xa5, sizeof(expected)); 3766 3767 iovs[0].iov_base = expected; 3768 iovs[0].iov_len = sizeof(expected); 3769 iovs[1].iov_base = tmp; 3770 iovs[1].iov_len = sizeof(tmp); 3771 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, NULL, NULL, 3772 &iovs[0], 1, NULL, NULL, 0, 4096, 3773 ut_sequence_step_cb, &completed); 3774 CU_ASSERT_EQUAL(rc, 0); 3775 /* Reuse iov[1] as src */ 3776 iovs[2].iov_base = buf; 3777 iovs[2].iov_len = sizeof(buf); 3778 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3779 &iovs[1], 1, NULL, NULL, 0, 4096, 3780 ut_sequence_step_cb, &completed); 3781 CU_ASSERT_EQUAL(rc, 0); 3782 3783 ut_seq.complete = false; 3784 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3785 3786 poll_threads(); 3787 3788 CU_ASSERT_EQUAL(completed, 2); 3789 CU_ASSERT(ut_seq.complete); 3790 CU_ASSERT_EQUAL(ut_seq.status, 0); 3791 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3792 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3793 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3794 expected_siov.iov_base = expected; 3795 expected_siov.iov_len = sizeof(expected); 3796 expected_diov.iov_base = tmp; 3797 expected_diov.iov_len = sizeof(tmp); 3798 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].src, 3799 &expected_siov, sizeof(expected_siov)), 0); 3800 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].dst, 3801 &expected_diov, sizeof(expected_diov)), 0); 3802 expected_siov.iov_base = tmp; 3803 expected_siov.iov_len = sizeof(tmp); 3804 expected_diov.iov_base = buf; 3805 expected_diov.iov_len = sizeof(buf); 3806 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].src, 3807 &expected_siov, sizeof(expected_siov)), 0); 3808 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].dst, 3809 &expected_diov, sizeof(expected_diov)), 0); 3810 g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count = 0; 3811 g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count = 0; 3812 3813 /* Check the same with an accel buffer */ 3814 seq = NULL; 3815 completed = 0; 3816 memset(buf, 0, sizeof(buf)); 3817 memset(expected, 0x5a, sizeof(expected)); 3818 3819 rc = spdk_accel_get_buf(ioch, sizeof(buf), &accel_buf, &domain, &domain_ctx); 3820 CU_ASSERT_EQUAL(rc, 0); 3821 3822 iovs[0].iov_base = expected; 3823 iovs[0].iov_len = sizeof(expected); 3824 iovs[1].iov_base = accel_buf; 3825 iovs[1].iov_len = sizeof(buf); 3826 rc = spdk_accel_append_encrypt(&seq, ioch, &key, &iovs[1], 1, domain, domain_ctx, 3827 &iovs[0], 1, NULL, NULL, 0, 4096, 3828 ut_sequence_step_cb, &completed); 3829 CU_ASSERT_EQUAL(rc, 0); 3830 /* Reuse iov[1] as src */ 3831 iovs[2].iov_base = buf; 3832 iovs[2].iov_len = sizeof(buf); 3833 rc = spdk_accel_append_decrypt(&seq, ioch, &key, &iovs[2], 1, NULL, NULL, 3834 &iovs[1], 1, domain, domain_ctx, 0, 4096, 3835 ut_sequence_step_cb, &completed); 3836 CU_ASSERT_EQUAL(rc, 0); 3837 3838 ut_seq.complete = false; 3839 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3840 3841 poll_threads(); 3842 3843 CU_ASSERT_EQUAL(completed, 2); 3844 CU_ASSERT(ut_seq.complete); 3845 CU_ASSERT_EQUAL(ut_seq.status, 0); 3846 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_ENCRYPT].count, 1); 3847 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECRYPT].count, 1); 3848 CU_ASSERT_EQUAL(memcmp(buf, expected, sizeof(buf)), 0); 3849 expected_siov.iov_base = expected; 3850 expected_siov.iov_len = sizeof(expected); 3851 expected_diov.iov_base = buf; 3852 expected_diov.iov_len = sizeof(buf); 3853 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].src, 3854 &expected_siov, sizeof(expected_siov)), 0); 3855 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].dst, 3856 &expected_diov, sizeof(expected_diov)), 0); 3857 CU_ASSERT_EQUAL(memcmp(&g_seq_saved_iovs[SPDK_ACCEL_OPC_ENCRYPT].dst, 3858 &g_seq_saved_iovs[SPDK_ACCEL_OPC_DECRYPT].src, 3859 sizeof(struct iovec)), 0); 3860 spdk_accel_put_buf(ioch, accel_buf, domain, domain_ctx); 3861 3862 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3863 g_modules_opc[i] = modules[i]; 3864 } 3865 3866 ut_clear_operations(); 3867 spdk_put_io_channel(ioch); 3868 poll_threads(); 3869 } 3870 3871 static void 3872 test_sequence_crc32(void) 3873 { 3874 struct spdk_accel_sequence *seq = NULL; 3875 struct spdk_io_channel *ioch; 3876 struct ut_sequence ut_seq; 3877 struct accel_module modules[SPDK_ACCEL_OPC_LAST]; 3878 char buf[4096], tmp[3][4096]; 3879 struct iovec src_iovs[4], dst_iovs[4]; 3880 uint32_t crc, crc2; 3881 int i, rc, completed; 3882 3883 ioch = spdk_accel_get_io_channel(); 3884 SPDK_CU_ASSERT_FATAL(ioch != NULL); 3885 3886 /* Override the submit_tasks function */ 3887 g_module_if.submit_tasks = ut_sequence_submit_tasks; 3888 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 3889 g_seq_operations[i].submit = sw_accel_submit_tasks; 3890 modules[i] = g_modules_opc[i]; 3891 g_modules_opc[i] = g_module; 3892 } 3893 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].submit = ut_submit_decompress; 3894 3895 /* First check the simplest case - single crc32c operation */ 3896 seq = NULL; 3897 completed = 0; 3898 crc = 0; 3899 memset(buf, 0xa5, sizeof(buf)); 3900 3901 src_iovs[0].iov_base = buf; 3902 src_iovs[0].iov_len = sizeof(buf); 3903 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3904 ut_sequence_step_cb, &completed); 3905 CU_ASSERT_EQUAL(rc, 0); 3906 3907 ut_seq.complete = false; 3908 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3909 3910 poll_threads(); 3911 CU_ASSERT_EQUAL(completed, 1); 3912 CU_ASSERT(ut_seq.complete); 3913 CU_ASSERT_EQUAL(ut_seq.status, 0); 3914 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3915 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3916 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3917 3918 /* Now check copy+crc - This should not remove the copy. Otherwise the data does not 3919 * end up where the user expected it to be. */ 3920 seq = NULL; 3921 completed = 0; 3922 crc = 0; 3923 memset(buf, 0x5a, sizeof(buf)); 3924 memset(&tmp[0], 0, sizeof(tmp[0])); 3925 3926 dst_iovs[0].iov_base = tmp[0]; 3927 dst_iovs[0].iov_len = sizeof(tmp[0]); 3928 src_iovs[0].iov_base = buf; 3929 src_iovs[0].iov_len = sizeof(buf); 3930 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 3931 &src_iovs[0], 1, NULL, NULL, 3932 ut_sequence_step_cb, &completed); 3933 CU_ASSERT_EQUAL(rc, 0); 3934 3935 src_iovs[1].iov_base = tmp[0]; 3936 src_iovs[1].iov_len = sizeof(tmp[0]); 3937 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 3938 ut_sequence_step_cb, &completed); 3939 CU_ASSERT_EQUAL(rc, 0); 3940 3941 ut_seq.complete = false; 3942 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3943 3944 poll_threads(); 3945 CU_ASSERT_EQUAL(completed, 2); 3946 CU_ASSERT(ut_seq.complete); 3947 CU_ASSERT_EQUAL(ut_seq.status, 0); 3948 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3949 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 3950 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 3951 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(buf, sizeof(buf), ~0u)); 3952 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3953 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 3954 3955 /* Check crc+copy - Again, the copy cannot be removed. */ 3956 seq = NULL; 3957 completed = 0; 3958 crc = 0; 3959 memset(buf, 0, sizeof(buf)); 3960 memset(&tmp[0], 0xa5, sizeof(tmp[0])); 3961 3962 src_iovs[0].iov_base = tmp[0]; 3963 src_iovs[0].iov_len = sizeof(tmp[0]); 3964 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[0], 1, NULL, NULL, 0, 3965 ut_sequence_step_cb, &completed); 3966 CU_ASSERT_EQUAL(rc, 0); 3967 3968 dst_iovs[1].iov_base = buf; 3969 dst_iovs[1].iov_len = sizeof(buf); 3970 src_iovs[1].iov_base = tmp[0]; 3971 src_iovs[1].iov_len = sizeof(tmp[0]); 3972 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[1], 1, NULL, NULL, 3973 &src_iovs[1], 1, NULL, NULL, 3974 ut_sequence_step_cb, &completed); 3975 CU_ASSERT_EQUAL(rc, 0); 3976 3977 ut_seq.complete = false; 3978 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 3979 3980 poll_threads(); 3981 CU_ASSERT_EQUAL(completed, 2); 3982 CU_ASSERT(ut_seq.complete); 3983 CU_ASSERT_EQUAL(ut_seq.status, 0); 3984 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 3985 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 3986 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 3987 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 3988 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 3989 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 3990 3991 /* Check a sequence with an operation at the beginning that can have its buffer changed, two 3992 * crc operations and a copy at the end. The copy should be removed and the dst buffer of 3993 * the first operation and the src buffer of the crc operations should be changed. 3994 */ 3995 seq = NULL; 3996 completed = 0; 3997 crc = crc2 = 0; 3998 memset(buf, 0, sizeof(buf)); 3999 memset(&tmp[0], 0x5a, sizeof(tmp[0])); 4000 dst_iovs[0].iov_base = tmp[1]; 4001 dst_iovs[0].iov_len = sizeof(tmp[1]); 4002 src_iovs[0].iov_base = tmp[0]; 4003 src_iovs[0].iov_len = sizeof(tmp[0]); 4004 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 4005 &src_iovs[0], 1, NULL, NULL, 4006 ut_sequence_step_cb, &completed); 4007 CU_ASSERT_EQUAL(rc, 0); 4008 4009 src_iovs[1].iov_base = tmp[1]; 4010 src_iovs[1].iov_len = sizeof(tmp[1]); 4011 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 4012 ut_sequence_step_cb, &completed); 4013 CU_ASSERT_EQUAL(rc, 0); 4014 4015 src_iovs[2].iov_base = tmp[1]; 4016 src_iovs[2].iov_len = sizeof(tmp[1]); 4017 rc = spdk_accel_append_crc32c(&seq, ioch, &crc2, &src_iovs[2], 1, NULL, NULL, 0, 4018 ut_sequence_step_cb, &completed); 4019 CU_ASSERT_EQUAL(rc, 0); 4020 4021 dst_iovs[3].iov_base = buf; 4022 dst_iovs[3].iov_len = sizeof(buf); 4023 src_iovs[3].iov_base = tmp[1]; 4024 src_iovs[3].iov_len = sizeof(tmp[1]); 4025 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[3], 1, NULL, NULL, 4026 &src_iovs[3], 1, NULL, NULL, 4027 ut_sequence_step_cb, &completed); 4028 CU_ASSERT_EQUAL(rc, 0); 4029 4030 ut_seq.complete = false; 4031 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4032 4033 poll_threads(); 4034 CU_ASSERT_EQUAL(completed, 4); 4035 CU_ASSERT(ut_seq.complete); 4036 CU_ASSERT_EQUAL(ut_seq.status, 0); 4037 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 4038 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 2); 4039 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 0); 4040 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], sizeof(tmp[0]), ~0u)); 4041 CU_ASSERT_EQUAL(crc, crc2); 4042 CU_ASSERT_EQUAL(memcmp(buf, tmp[0], sizeof(buf)), 0); 4043 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 4044 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 4045 4046 /* Check that a copy won't be removed if the buffers don't match */ 4047 seq = NULL; 4048 completed = 0; 4049 crc = 0; 4050 memset(buf, 0, sizeof(buf)); 4051 memset(&tmp[0], 0xa5, 2048); 4052 memset(&tmp[1], 0xfe, sizeof(tmp[1])); 4053 memset(&tmp[2], 0xfe, sizeof(tmp[1])); 4054 dst_iovs[0].iov_base = &tmp[1][2048]; 4055 dst_iovs[0].iov_len = 2048; 4056 src_iovs[0].iov_base = tmp[0]; 4057 src_iovs[0].iov_len = 2048; 4058 rc = spdk_accel_append_decompress(&seq, ioch, &dst_iovs[0], 1, NULL, NULL, 4059 &src_iovs[0], 1, NULL, NULL, 4060 ut_sequence_step_cb, &completed); 4061 CU_ASSERT_EQUAL(rc, 0); 4062 4063 src_iovs[1].iov_base = &tmp[1][2048]; 4064 src_iovs[1].iov_len = 2048; 4065 rc = spdk_accel_append_crc32c(&seq, ioch, &crc, &src_iovs[1], 1, NULL, NULL, 0, 4066 ut_sequence_step_cb, &completed); 4067 CU_ASSERT_EQUAL(rc, 0); 4068 4069 dst_iovs[2].iov_base = buf; 4070 dst_iovs[2].iov_len = sizeof(buf); 4071 src_iovs[2].iov_base = tmp[1]; 4072 src_iovs[2].iov_len = sizeof(tmp[1]); 4073 rc = spdk_accel_append_copy(&seq, ioch, &dst_iovs[2], 1, NULL, NULL, 4074 &src_iovs[2], 1, NULL, NULL, 4075 ut_sequence_step_cb, &completed); 4076 CU_ASSERT_EQUAL(rc, 0); 4077 4078 ut_seq.complete = false; 4079 spdk_accel_sequence_finish(seq, ut_sequence_complete_cb, &ut_seq); 4080 4081 poll_threads(); 4082 CU_ASSERT_EQUAL(completed, 3); 4083 CU_ASSERT(ut_seq.complete); 4084 CU_ASSERT_EQUAL(ut_seq.status, 0); 4085 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count, 1); 4086 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count, 1); 4087 CU_ASSERT_EQUAL(g_seq_operations[SPDK_ACCEL_OPC_COPY].count, 1); 4088 CU_ASSERT_EQUAL(crc, spdk_crc32c_update(tmp[0], 2048, ~0u)); 4089 CU_ASSERT_EQUAL(memcmp(buf, tmp[2], 2048), 0); 4090 CU_ASSERT_EQUAL(memcmp(&buf[2048], tmp[0], 2048), 0); 4091 g_seq_operations[SPDK_ACCEL_OPC_CRC32C].count = 0; 4092 g_seq_operations[SPDK_ACCEL_OPC_DECOMPRESS].count = 0; 4093 g_seq_operations[SPDK_ACCEL_OPC_COPY].count = 0; 4094 4095 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 4096 g_modules_opc[i] = modules[i]; 4097 } 4098 4099 ut_clear_operations(); 4100 spdk_put_io_channel(ioch); 4101 poll_threads(); 4102 } 4103 4104 static int 4105 test_sequence_setup(void) 4106 { 4107 int rc; 4108 4109 allocate_cores(1); 4110 allocate_threads(1); 4111 set_thread(0); 4112 4113 rc = spdk_iobuf_initialize(); 4114 if (rc != 0) { 4115 CU_ASSERT(false); 4116 return -1; 4117 } 4118 4119 rc = spdk_accel_initialize(); 4120 if (rc != 0) { 4121 CU_ASSERT(false); 4122 return -1; 4123 } 4124 4125 return 0; 4126 } 4127 4128 static void 4129 finish_cb(void *cb_arg) 4130 { 4131 bool *done = cb_arg; 4132 4133 *done = true; 4134 } 4135 4136 static int 4137 test_sequence_cleanup(void) 4138 { 4139 bool done = false; 4140 4141 spdk_accel_finish(finish_cb, &done); 4142 4143 while (!done) { 4144 poll_threads(); 4145 } 4146 4147 done = false; 4148 spdk_iobuf_finish(finish_cb, &done); 4149 while (!done) { 4150 poll_threads(); 4151 } 4152 4153 free_threads(); 4154 free_cores(); 4155 4156 return 0; 4157 } 4158 4159 int 4160 main(int argc, char **argv) 4161 { 4162 CU_pSuite suite = NULL, seq_suite; 4163 unsigned int num_failures; 4164 4165 CU_initialize_registry(); 4166 4167 /* Sequence tests require accel to be initialized normally, so run them before the other 4168 * tests which register accel modules which aren't fully implemented, causing accel 4169 * initialization to fail. 4170 */ 4171 seq_suite = CU_add_suite("accel_sequence", test_sequence_setup, test_sequence_cleanup); 4172 CU_ADD_TEST(seq_suite, test_sequence_fill_copy); 4173 CU_ADD_TEST(seq_suite, test_sequence_abort); 4174 CU_ADD_TEST(seq_suite, test_sequence_append_error); 4175 CU_ADD_TEST(seq_suite, test_sequence_completion_error); 4176 #ifdef SPDK_CONFIG_ISAL /* accel_sw requires isa-l for compression */ 4177 CU_ADD_TEST(seq_suite, test_sequence_decompress); 4178 CU_ADD_TEST(seq_suite, test_sequence_reverse); 4179 #endif 4180 CU_ADD_TEST(seq_suite, test_sequence_copy_elision); 4181 CU_ADD_TEST(seq_suite, test_sequence_accel_buffers); 4182 CU_ADD_TEST(seq_suite, test_sequence_memory_domain); 4183 CU_ADD_TEST(seq_suite, test_sequence_module_memory_domain); 4184 #ifdef SPDK_CONFIG_ISAL_CRYPTO /* accel_sw requires isa-l-crypto for crypto operations */ 4185 CU_ADD_TEST(seq_suite, test_sequence_crypto); 4186 #endif 4187 CU_ADD_TEST(seq_suite, test_sequence_driver); 4188 CU_ADD_TEST(seq_suite, test_sequence_same_iovs); 4189 CU_ADD_TEST(seq_suite, test_sequence_crc32); 4190 4191 suite = CU_add_suite("accel", test_setup, test_cleanup); 4192 CU_ADD_TEST(suite, test_spdk_accel_task_complete); 4193 CU_ADD_TEST(suite, test_get_task); 4194 CU_ADD_TEST(suite, test_spdk_accel_submit_copy); 4195 CU_ADD_TEST(suite, test_spdk_accel_submit_dualcast); 4196 CU_ADD_TEST(suite, test_spdk_accel_submit_compare); 4197 CU_ADD_TEST(suite, test_spdk_accel_submit_fill); 4198 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32c); 4199 CU_ADD_TEST(suite, test_spdk_accel_submit_crc32cv); 4200 CU_ADD_TEST(suite, test_spdk_accel_submit_copy_crc32c); 4201 CU_ADD_TEST(suite, test_spdk_accel_submit_xor); 4202 CU_ADD_TEST(suite, test_spdk_accel_module_find_by_name); 4203 CU_ADD_TEST(suite, test_spdk_accel_module_register); 4204 4205 num_failures = spdk_ut_run_tests(argc, argv, NULL); 4206 CU_cleanup_registry(); 4207 4208 return num_failures; 4209 } 4210