1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES 4 * All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk_internal/accel_module.h" 10 #include "accel_internal.h" 11 12 #include "spdk/env.h" 13 #include "spdk/likely.h" 14 #include "spdk/log.h" 15 #include "spdk/thread.h" 16 #include "spdk/json.h" 17 #include "spdk/crc32.h" 18 #include "spdk/util.h" 19 #include "spdk/xor.h" 20 21 #ifdef SPDK_CONFIG_PMDK 22 #include "libpmem.h" 23 #endif 24 25 #ifdef SPDK_CONFIG_ISAL 26 #include "../isa-l/include/igzip_lib.h" 27 #ifdef SPDK_CONFIG_ISAL_CRYPTO 28 #include "../isa-l-crypto/include/aes_xts.h" 29 #endif 30 #endif 31 32 #define ACCEL_AES_XTS_128_KEY_SIZE 16 33 #define ACCEL_AES_XTS_256_KEY_SIZE 32 34 #define ACCEL_AES_XTS "AES_XTS" 35 /* Per the AES-XTS spec, the size of data unit cannot be bigger than 2^20 blocks, 128b each block */ 36 #define ACCEL_AES_XTS_MAX_BLOCK_SIZE (1 << 24) 37 38 struct sw_accel_io_channel { 39 /* for ISAL */ 40 #ifdef SPDK_CONFIG_ISAL 41 struct isal_zstream stream; 42 struct inflate_state state; 43 #endif 44 struct spdk_poller *completion_poller; 45 TAILQ_HEAD(, spdk_accel_task) tasks_to_complete; 46 }; 47 48 typedef void (*sw_accel_crypto_op)(uint8_t *k2, uint8_t *k1, uint8_t *tweak, uint64_t lba_size, 49 const uint8_t *src, uint8_t *dst); 50 51 struct sw_accel_crypto_key_data { 52 sw_accel_crypto_op encrypt; 53 sw_accel_crypto_op decrypt; 54 }; 55 56 static struct spdk_accel_module_if g_sw_module; 57 58 static void sw_accel_crypto_key_deinit(struct spdk_accel_crypto_key *_key); 59 static int sw_accel_crypto_key_init(struct spdk_accel_crypto_key *key); 60 61 /* Post SW completions to a list and complete in a poller as we don't want to 62 * complete them on the caller's stack as they'll likely submit another. */ 63 inline static void 64 _add_to_comp_list(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task, int status) 65 { 66 accel_task->status = status; 67 TAILQ_INSERT_TAIL(&sw_ch->tasks_to_complete, accel_task, link); 68 } 69 70 SPDK_LOG_DEPRECATION_REGISTER(accel_flag_persistent, 71 "PMDK libpmem accel_sw integration", "SPDK 23.05", 10); 72 73 /* Used when the SW engine is selected and the durable flag is set. */ 74 inline static int 75 _check_flags(int flags) 76 { 77 if (flags & ACCEL_FLAG_PERSISTENT) { 78 SPDK_LOG_DEPRECATED(accel_flag_persistent); 79 #ifndef SPDK_CONFIG_PMDK 80 /* PMDK is required to use this flag. */ 81 SPDK_ERRLOG("ACCEL_FLAG_PERSISTENT set but PMDK not configured. Configure PMDK or do not use this flag.\n"); 82 return -EINVAL; 83 #endif 84 } 85 return 0; 86 } 87 88 static bool 89 sw_accel_supports_opcode(enum accel_opcode opc) 90 { 91 switch (opc) { 92 case ACCEL_OPC_COPY: 93 case ACCEL_OPC_FILL: 94 case ACCEL_OPC_DUALCAST: 95 case ACCEL_OPC_COMPARE: 96 case ACCEL_OPC_CRC32C: 97 case ACCEL_OPC_COPY_CRC32C: 98 case ACCEL_OPC_COMPRESS: 99 case ACCEL_OPC_DECOMPRESS: 100 case ACCEL_OPC_ENCRYPT: 101 case ACCEL_OPC_DECRYPT: 102 case ACCEL_OPC_XOR: 103 return true; 104 default: 105 return false; 106 } 107 } 108 109 static inline void 110 _pmem_memcpy(void *dst, const void *src, size_t len) 111 { 112 #ifdef SPDK_CONFIG_PMDK 113 int is_pmem = pmem_is_pmem(dst, len); 114 115 if (is_pmem) { 116 pmem_memcpy_persist(dst, src, len); 117 } else { 118 memcpy(dst, src, len); 119 pmem_msync(dst, len); 120 } 121 #else 122 SPDK_ERRLOG("Function not defined without SPDK_CONFIG_PMDK enabled.\n"); 123 assert(0); 124 #endif 125 } 126 127 static void 128 _sw_accel_dualcast(void *dst1, void *dst2, void *src, size_t nbytes, int flags) 129 { 130 if (flags & ACCEL_FLAG_PERSISTENT) { 131 _pmem_memcpy(dst1, src, nbytes); 132 _pmem_memcpy(dst2, src, nbytes); 133 } else { 134 memcpy(dst1, src, nbytes); 135 memcpy(dst2, src, nbytes); 136 } 137 } 138 139 static int 140 _sw_accel_dualcast_iovs(struct iovec *dst_iovs, uint32_t dst_iovcnt, 141 struct iovec *dst2_iovs, uint32_t dst2_iovcnt, 142 struct iovec *src_iovs, uint32_t src_iovcnt, int flags) 143 { 144 if (spdk_unlikely(dst_iovcnt != 1 || dst2_iovcnt != 1 || src_iovcnt != 1)) { 145 return -EINVAL; 146 } 147 148 if (spdk_unlikely(dst_iovs[0].iov_len != src_iovs[0].iov_len || 149 dst_iovs[0].iov_len != dst2_iovs[0].iov_len)) { 150 return -EINVAL; 151 } 152 153 _sw_accel_dualcast(dst_iovs[0].iov_base, dst2_iovs[0].iov_base, src_iovs[0].iov_base, 154 dst_iovs[0].iov_len, flags); 155 156 return 0; 157 } 158 159 static void 160 _sw_accel_copy(void *dst, void *src, size_t nbytes, int flags) 161 { 162 163 if (flags & ACCEL_FLAG_PERSISTENT) { 164 _pmem_memcpy(dst, src, nbytes); 165 } else { 166 memcpy(dst, src, nbytes); 167 } 168 } 169 170 static void 171 _sw_accel_copy_iovs(struct iovec *dst_iovs, uint32_t dst_iovcnt, 172 struct iovec *src_iovs, uint32_t src_iovcnt, int flags) 173 { 174 struct spdk_ioviter iter; 175 void *src, *dst; 176 size_t len; 177 178 for (len = spdk_ioviter_first(&iter, src_iovs, src_iovcnt, 179 dst_iovs, dst_iovcnt, &src, &dst); 180 len != 0; 181 len = spdk_ioviter_next(&iter, &src, &dst)) { 182 _sw_accel_copy(dst, src, len, flags); 183 } 184 } 185 186 static int 187 _sw_accel_compare(struct iovec *src_iovs, uint32_t src_iovcnt, 188 struct iovec *src2_iovs, uint32_t src2_iovcnt) 189 { 190 if (spdk_unlikely(src_iovcnt != 1 || src2_iovcnt != 1)) { 191 return -EINVAL; 192 } 193 194 if (spdk_unlikely(src_iovs[0].iov_len != src2_iovs[0].iov_len)) { 195 return -EINVAL; 196 } 197 198 return memcmp(src_iovs[0].iov_base, src2_iovs[0].iov_base, src_iovs[0].iov_len); 199 } 200 201 static int 202 _sw_accel_fill(struct iovec *iovs, uint32_t iovcnt, uint8_t fill, int flags) 203 { 204 void *dst; 205 size_t nbytes; 206 207 if (spdk_unlikely(iovcnt != 1)) { 208 return -EINVAL; 209 } 210 211 dst = iovs[0].iov_base; 212 nbytes = iovs[0].iov_len; 213 214 if (flags & ACCEL_FLAG_PERSISTENT) { 215 #ifdef SPDK_CONFIG_PMDK 216 int is_pmem = pmem_is_pmem(dst, nbytes); 217 218 if (is_pmem) { 219 pmem_memset_persist(dst, fill, nbytes); 220 } else { 221 memset(dst, fill, nbytes); 222 pmem_msync(dst, nbytes); 223 } 224 #else 225 SPDK_ERRLOG("Function not defined without SPDK_CONFIG_PMDK enabled.\n"); 226 assert(0); 227 #endif 228 } else { 229 memset(dst, fill, nbytes); 230 } 231 232 return 0; 233 } 234 235 static void 236 _sw_accel_crc32cv(uint32_t *crc_dst, struct iovec *iov, uint32_t iovcnt, uint32_t seed) 237 { 238 *crc_dst = spdk_crc32c_iov_update(iov, iovcnt, ~seed); 239 } 240 241 static int 242 _sw_accel_compress(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 243 { 244 #ifdef SPDK_CONFIG_ISAL 245 size_t last_seglen = accel_task->s.iovs[accel_task->s.iovcnt - 1].iov_len; 246 struct iovec *siov = accel_task->s.iovs; 247 struct iovec *diov = accel_task->d.iovs; 248 size_t remaining; 249 uint32_t i, s = 0, d = 0; 250 int rc = 0; 251 252 remaining = 0; 253 for (i = 0; i < accel_task->s.iovcnt; ++i) { 254 remaining += accel_task->s.iovs[i].iov_len; 255 } 256 257 isal_deflate_reset(&sw_ch->stream); 258 sw_ch->stream.end_of_stream = 0; 259 sw_ch->stream.next_out = diov[d].iov_base; 260 sw_ch->stream.avail_out = diov[d].iov_len; 261 sw_ch->stream.next_in = siov[s].iov_base; 262 sw_ch->stream.avail_in = siov[s].iov_len; 263 264 do { 265 /* if isal has exhausted the current dst iovec, move to the next 266 * one if there is one */ 267 if (sw_ch->stream.avail_out == 0) { 268 if (++d < accel_task->d.iovcnt) { 269 sw_ch->stream.next_out = diov[d].iov_base; 270 sw_ch->stream.avail_out = diov[d].iov_len; 271 assert(sw_ch->stream.avail_out > 0); 272 } else { 273 /* we have no avail_out but also no more iovecs left so this is 274 * the case where either the output buffer was a perfect fit 275 * or not enough was provided. Check the ISAL state to determine 276 * which. */ 277 if (sw_ch->stream.internal_state.state != ZSTATE_END) { 278 SPDK_ERRLOG("Not enough destination buffer provided.\n"); 279 rc = -ENOMEM; 280 } 281 break; 282 } 283 } 284 285 /* if isal has exhausted the current src iovec, move to the next 286 * one if there is one */ 287 if (sw_ch->stream.avail_in == 0 && ((s + 1) < accel_task->s.iovcnt)) { 288 s++; 289 sw_ch->stream.next_in = siov[s].iov_base; 290 sw_ch->stream.avail_in = siov[s].iov_len; 291 assert(sw_ch->stream.avail_in > 0); 292 } 293 294 if (remaining <= last_seglen) { 295 /* Need to set end of stream on last block */ 296 sw_ch->stream.end_of_stream = 1; 297 } 298 299 rc = isal_deflate(&sw_ch->stream); 300 if (rc) { 301 SPDK_ERRLOG("isal_deflate returned error %d.\n", rc); 302 } 303 304 if (remaining > 0) { 305 assert(siov[s].iov_len > sw_ch->stream.avail_in); 306 remaining -= (siov[s].iov_len - sw_ch->stream.avail_in); 307 } 308 309 } while (remaining > 0 || sw_ch->stream.avail_out == 0); 310 assert(sw_ch->stream.avail_in == 0); 311 312 /* Get our total output size */ 313 if (accel_task->output_size != NULL) { 314 assert(sw_ch->stream.total_out > 0); 315 *accel_task->output_size = sw_ch->stream.total_out; 316 } 317 318 return rc; 319 #else 320 SPDK_ERRLOG("ISAL option is required to use software compression.\n"); 321 return -EINVAL; 322 #endif 323 } 324 325 static int 326 _sw_accel_decompress(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 327 { 328 #ifdef SPDK_CONFIG_ISAL 329 struct iovec *siov = accel_task->s.iovs; 330 struct iovec *diov = accel_task->d.iovs; 331 uint32_t s = 0, d = 0; 332 int rc = 0; 333 334 isal_inflate_reset(&sw_ch->state); 335 sw_ch->state.next_out = diov[d].iov_base; 336 sw_ch->state.avail_out = diov[d].iov_len; 337 sw_ch->state.next_in = siov[s].iov_base; 338 sw_ch->state.avail_in = siov[s].iov_len; 339 340 do { 341 /* if isal has exhausted the current dst iovec, move to the next 342 * one if there is one */ 343 if (sw_ch->state.avail_out == 0 && ((d + 1) < accel_task->d.iovcnt)) { 344 d++; 345 sw_ch->state.next_out = diov[d].iov_base; 346 sw_ch->state.avail_out = diov[d].iov_len; 347 assert(sw_ch->state.avail_out > 0); 348 } 349 350 /* if isal has exhausted the current src iovec, move to the next 351 * one if there is one */ 352 if (sw_ch->state.avail_in == 0 && ((s + 1) < accel_task->s.iovcnt)) { 353 s++; 354 sw_ch->state.next_in = siov[s].iov_base; 355 sw_ch->state.avail_in = siov[s].iov_len; 356 assert(sw_ch->state.avail_in > 0); 357 } 358 359 rc = isal_inflate(&sw_ch->state); 360 if (rc) { 361 SPDK_ERRLOG("isal_inflate returned error %d.\n", rc); 362 } 363 364 } while (sw_ch->state.block_state < ISAL_BLOCK_FINISH); 365 assert(sw_ch->state.avail_in == 0); 366 367 /* Get our total output size */ 368 if (accel_task->output_size != NULL) { 369 assert(sw_ch->state.total_out > 0); 370 *accel_task->output_size = sw_ch->state.total_out; 371 } 372 373 return rc; 374 #else 375 SPDK_ERRLOG("ISAL option is required to use software decompression.\n"); 376 return -EINVAL; 377 #endif 378 } 379 380 static int 381 _sw_accel_crypto_operation(struct spdk_accel_task *accel_task, struct spdk_accel_crypto_key *key, 382 sw_accel_crypto_op op) 383 { 384 #ifdef SPDK_CONFIG_ISAL_CRYPTO 385 uint64_t iv[2]; 386 size_t remaining_len, dst_len; 387 uint64_t src_offset = 0, dst_offset = 0; 388 uint32_t src_iovpos = 0, dst_iovpos = 0, src_iovcnt, dst_iovcnt; 389 uint32_t i, block_size, crypto_len, crypto_accum_len = 0; 390 struct iovec *src_iov, *dst_iov; 391 uint8_t *src, *dst; 392 393 /* iv is 128 bits, since we are using logical block address (64 bits) as iv, fill first 8 bytes with zeroes */ 394 iv[0] = 0; 395 iv[1] = accel_task->iv; 396 src_iov = accel_task->s.iovs; 397 src_iovcnt = accel_task->s.iovcnt; 398 if (accel_task->d.iovcnt) { 399 dst_iov = accel_task->d.iovs; 400 dst_iovcnt = accel_task->d.iovcnt; 401 } else { 402 /* inplace operation */ 403 dst_iov = accel_task->s.iovs; 404 dst_iovcnt = accel_task->s.iovcnt; 405 } 406 block_size = accel_task->block_size; 407 408 if (!src_iovcnt || !dst_iovcnt || !block_size || !op) { 409 SPDK_ERRLOG("src_iovcnt %d, dst_iovcnt %d, block_size %d, op %p\n", src_iovcnt, dst_iovcnt, 410 block_size, op); 411 return -EINVAL; 412 } 413 414 remaining_len = 0; 415 for (i = 0; i < src_iovcnt; i++) { 416 remaining_len += src_iov[i].iov_len; 417 } 418 dst_len = 0; 419 for (i = 0; i < dst_iovcnt; i++) { 420 dst_len += dst_iov[i].iov_len; 421 } 422 423 if (spdk_unlikely(remaining_len != dst_len || !remaining_len)) { 424 return -ERANGE; 425 } 426 if (spdk_unlikely(remaining_len % accel_task->block_size != 0)) { 427 return -EINVAL; 428 } 429 430 while (remaining_len) { 431 crypto_len = spdk_min(block_size - crypto_accum_len, src_iov->iov_len - src_offset); 432 crypto_len = spdk_min(crypto_len, dst_iov->iov_len - dst_offset); 433 src = (uint8_t *)src_iov->iov_base + src_offset; 434 dst = (uint8_t *)dst_iov->iov_base + dst_offset; 435 436 op((uint8_t *)key->key2, (uint8_t *)key->key, (uint8_t *)iv, crypto_len, src, dst); 437 438 src_offset += crypto_len; 439 dst_offset += crypto_len; 440 crypto_accum_len += crypto_len; 441 remaining_len -= crypto_len; 442 443 if (crypto_accum_len == block_size) { 444 /* we can process part of logical block. Once the whole block is processed, increment iv */ 445 crypto_accum_len = 0; 446 iv[1]++; 447 } 448 if (src_offset == src_iov->iov_len) { 449 src_iov++; 450 src_iovpos++; 451 src_offset = 0; 452 } 453 if (src_iovpos == src_iovcnt) { 454 break; 455 } 456 if (dst_offset == dst_iov->iov_len) { 457 dst_iov++; 458 dst_iovpos++; 459 dst_offset = 0; 460 } 461 if (dst_iovpos == dst_iovcnt) { 462 break; 463 } 464 } 465 466 if (remaining_len) { 467 SPDK_ERRLOG("remaining len %zu\n", remaining_len); 468 return -EINVAL; 469 } 470 471 return 0; 472 #else 473 return -ENOTSUP; 474 #endif 475 } 476 477 static int 478 _sw_accel_encrypt(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 479 { 480 struct spdk_accel_crypto_key *key; 481 struct sw_accel_crypto_key_data *key_data; 482 483 key = accel_task->crypto_key; 484 if (spdk_unlikely(key->module_if != &g_sw_module || !key->priv)) { 485 return -EINVAL; 486 } 487 if (spdk_unlikely(accel_task->block_size > ACCEL_AES_XTS_MAX_BLOCK_SIZE)) { 488 SPDK_WARNLOG("Max block size for AES_XTS is limited to %u, current size %u\n", 489 ACCEL_AES_XTS_MAX_BLOCK_SIZE, accel_task->block_size); 490 return -ERANGE; 491 } 492 key_data = key->priv; 493 return _sw_accel_crypto_operation(accel_task, key, key_data->encrypt); 494 } 495 496 static int 497 _sw_accel_decrypt(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 498 { 499 struct spdk_accel_crypto_key *key; 500 struct sw_accel_crypto_key_data *key_data; 501 502 key = accel_task->crypto_key; 503 if (spdk_unlikely(key->module_if != &g_sw_module || !key->priv)) { 504 return -EINVAL; 505 } 506 if (spdk_unlikely(accel_task->block_size > ACCEL_AES_XTS_MAX_BLOCK_SIZE)) { 507 SPDK_WARNLOG("Max block size for AES_XTS is limited to %u, current size %u\n", 508 ACCEL_AES_XTS_MAX_BLOCK_SIZE, accel_task->block_size); 509 return -ERANGE; 510 } 511 key_data = key->priv; 512 return _sw_accel_crypto_operation(accel_task, key, key_data->decrypt); 513 } 514 515 static int 516 _sw_accel_xor(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 517 { 518 return spdk_xor_gen(accel_task->d.iovs[0].iov_base, 519 accel_task->nsrcs.srcs, 520 accel_task->nsrcs.cnt, 521 accel_task->d.iovs[0].iov_len); 522 } 523 524 static int 525 sw_accel_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task) 526 { 527 struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch); 528 struct spdk_accel_task *tmp; 529 int rc = 0; 530 531 do { 532 switch (accel_task->op_code) { 533 case ACCEL_OPC_COPY: 534 rc = _check_flags(accel_task->flags); 535 if (rc == 0) { 536 _sw_accel_copy_iovs(accel_task->d.iovs, accel_task->d.iovcnt, 537 accel_task->s.iovs, accel_task->s.iovcnt, 538 accel_task->flags); 539 } 540 break; 541 case ACCEL_OPC_FILL: 542 rc = _check_flags(accel_task->flags); 543 if (rc == 0) { 544 rc = _sw_accel_fill(accel_task->d.iovs, accel_task->d.iovcnt, 545 accel_task->fill_pattern, accel_task->flags); 546 } 547 break; 548 case ACCEL_OPC_DUALCAST: 549 rc = _check_flags(accel_task->flags); 550 if (rc == 0) { 551 rc = _sw_accel_dualcast_iovs(accel_task->d.iovs, accel_task->d.iovcnt, 552 accel_task->d2.iovs, accel_task->d2.iovcnt, 553 accel_task->s.iovs, accel_task->s.iovcnt, 554 accel_task->flags); 555 } 556 break; 557 case ACCEL_OPC_COMPARE: 558 rc = _sw_accel_compare(accel_task->s.iovs, accel_task->s.iovcnt, 559 accel_task->s2.iovs, accel_task->s2.iovcnt); 560 break; 561 case ACCEL_OPC_CRC32C: 562 _sw_accel_crc32cv(accel_task->crc_dst, accel_task->s.iovs, accel_task->s.iovcnt, accel_task->seed); 563 break; 564 case ACCEL_OPC_COPY_CRC32C: 565 rc = _check_flags(accel_task->flags); 566 if (rc == 0) { 567 _sw_accel_copy_iovs(accel_task->d.iovs, accel_task->d.iovcnt, 568 accel_task->s.iovs, accel_task->s.iovcnt, 569 accel_task->flags); 570 _sw_accel_crc32cv(accel_task->crc_dst, accel_task->s.iovs, 571 accel_task->s.iovcnt, accel_task->seed); 572 } 573 break; 574 case ACCEL_OPC_COMPRESS: 575 rc = _sw_accel_compress(sw_ch, accel_task); 576 break; 577 case ACCEL_OPC_DECOMPRESS: 578 rc = _sw_accel_decompress(sw_ch, accel_task); 579 break; 580 case ACCEL_OPC_XOR: 581 rc = _sw_accel_xor(sw_ch, accel_task); 582 break; 583 case ACCEL_OPC_ENCRYPT: 584 rc = _sw_accel_encrypt(sw_ch, accel_task); 585 break; 586 case ACCEL_OPC_DECRYPT: 587 rc = _sw_accel_decrypt(sw_ch, accel_task); 588 break; 589 default: 590 assert(false); 591 break; 592 } 593 594 tmp = TAILQ_NEXT(accel_task, link); 595 596 _add_to_comp_list(sw_ch, accel_task, rc); 597 598 accel_task = tmp; 599 } while (accel_task); 600 601 return 0; 602 } 603 604 static struct spdk_io_channel *sw_accel_get_io_channel(void); 605 static int sw_accel_module_init(void); 606 static void sw_accel_module_fini(void *ctxt); 607 static size_t sw_accel_module_get_ctx_size(void); 608 609 static struct spdk_accel_module_if g_sw_module = { 610 .module_init = sw_accel_module_init, 611 .module_fini = sw_accel_module_fini, 612 .write_config_json = NULL, 613 .get_ctx_size = sw_accel_module_get_ctx_size, 614 .name = "software", 615 .supports_opcode = sw_accel_supports_opcode, 616 .get_io_channel = sw_accel_get_io_channel, 617 .submit_tasks = sw_accel_submit_tasks, 618 .crypto_key_init = sw_accel_crypto_key_init, 619 .crypto_key_deinit = sw_accel_crypto_key_deinit, 620 }; 621 622 static int 623 accel_comp_poll(void *arg) 624 { 625 struct sw_accel_io_channel *sw_ch = arg; 626 TAILQ_HEAD(, spdk_accel_task) tasks_to_complete; 627 struct spdk_accel_task *accel_task; 628 629 if (TAILQ_EMPTY(&sw_ch->tasks_to_complete)) { 630 return SPDK_POLLER_IDLE; 631 } 632 633 TAILQ_INIT(&tasks_to_complete); 634 TAILQ_SWAP(&tasks_to_complete, &sw_ch->tasks_to_complete, spdk_accel_task, link); 635 636 while ((accel_task = TAILQ_FIRST(&tasks_to_complete))) { 637 TAILQ_REMOVE(&tasks_to_complete, accel_task, link); 638 spdk_accel_task_complete(accel_task, accel_task->status); 639 } 640 641 return SPDK_POLLER_BUSY; 642 } 643 644 static int 645 sw_accel_create_cb(void *io_device, void *ctx_buf) 646 { 647 struct sw_accel_io_channel *sw_ch = ctx_buf; 648 649 TAILQ_INIT(&sw_ch->tasks_to_complete); 650 sw_ch->completion_poller = SPDK_POLLER_REGISTER(accel_comp_poll, sw_ch, 0); 651 652 #ifdef SPDK_CONFIG_ISAL 653 isal_deflate_init(&sw_ch->stream); 654 sw_ch->stream.flush = NO_FLUSH; 655 sw_ch->stream.level = 1; 656 sw_ch->stream.level_buf = calloc(1, ISAL_DEF_LVL1_DEFAULT); 657 if (sw_ch->stream.level_buf == NULL) { 658 SPDK_ERRLOG("Could not allocate isal internal buffer\n"); 659 return -ENOMEM; 660 } 661 sw_ch->stream.level_buf_size = ISAL_DEF_LVL1_DEFAULT; 662 isal_inflate_init(&sw_ch->state); 663 #endif 664 665 return 0; 666 } 667 668 static void 669 sw_accel_destroy_cb(void *io_device, void *ctx_buf) 670 { 671 struct sw_accel_io_channel *sw_ch = ctx_buf; 672 673 #ifdef SPDK_CONFIG_ISAL 674 free(sw_ch->stream.level_buf); 675 #endif 676 677 spdk_poller_unregister(&sw_ch->completion_poller); 678 } 679 680 static struct spdk_io_channel * 681 sw_accel_get_io_channel(void) 682 { 683 return spdk_get_io_channel(&g_sw_module); 684 } 685 686 static size_t 687 sw_accel_module_get_ctx_size(void) 688 { 689 return sizeof(struct spdk_accel_task); 690 } 691 692 static int 693 sw_accel_module_init(void) 694 { 695 SPDK_NOTICELOG("Accel framework software module initialized.\n"); 696 spdk_io_device_register(&g_sw_module, sw_accel_create_cb, sw_accel_destroy_cb, 697 sizeof(struct sw_accel_io_channel), "sw_accel_module"); 698 699 return 0; 700 } 701 702 static void 703 sw_accel_module_fini(void *ctxt) 704 { 705 spdk_io_device_unregister(&g_sw_module, NULL); 706 spdk_accel_module_finish(); 707 } 708 709 static int 710 sw_accel_create_aes_xts(struct spdk_accel_crypto_key *key) 711 { 712 #ifdef SPDK_CONFIG_ISAL_CRYPTO 713 struct sw_accel_crypto_key_data *key_data; 714 715 if (!key->key || !key->key2) { 716 SPDK_ERRLOG("key or key2 are missing\n"); 717 return -EINVAL; 718 } 719 720 if (!key->key_size || key->key_size != key->key2_size) { 721 SPDK_ERRLOG("key size %zu is not equal to key2 size %zu or is 0\n", key->key_size, 722 key->key2_size); 723 return -EINVAL; 724 } 725 726 key_data = calloc(1, sizeof(*key_data)); 727 if (!key_data) { 728 return -ENOMEM; 729 } 730 731 switch (key->key_size) { 732 case ACCEL_AES_XTS_128_KEY_SIZE: 733 key_data->encrypt = XTS_AES_128_enc; 734 key_data->decrypt = XTS_AES_128_dec; 735 break; 736 case ACCEL_AES_XTS_256_KEY_SIZE: 737 key_data->encrypt = XTS_AES_256_enc; 738 key_data->decrypt = XTS_AES_256_dec; 739 break; 740 default: 741 SPDK_ERRLOG("Incorrect key size %zu, should be %d for AEX_XTS_128 or %d for AES_XTS_256\n", 742 key->key_size, ACCEL_AES_XTS_128_KEY_SIZE, ACCEL_AES_XTS_256_KEY_SIZE); 743 free(key_data); 744 return -EINVAL; 745 } 746 747 key->priv = key_data; 748 749 return 0; 750 #else 751 return -ENOTSUP; 752 #endif 753 } 754 755 static int 756 sw_accel_crypto_key_init(struct spdk_accel_crypto_key *key) 757 { 758 if (!key || !key->param.cipher) { 759 return -EINVAL; 760 } 761 if (strcmp(key->param.cipher, ACCEL_AES_XTS) == 0) { 762 return sw_accel_create_aes_xts(key); 763 } else { 764 SPDK_ERRLOG("Only %s cipher is supported\n", ACCEL_AES_XTS); 765 return -EINVAL; 766 } 767 } 768 769 static void 770 sw_accel_crypto_key_deinit(struct spdk_accel_crypto_key *key) 771 { 772 if (!key || key->module_if != &g_sw_module || !key->priv) { 773 return; 774 } 775 776 free(key->priv); 777 } 778 779 SPDK_ACCEL_MODULE_REGISTER(sw, &g_sw_module) 780