1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES 4 * All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/accel_module.h" 10 #include "accel_internal.h" 11 12 #include "spdk/env.h" 13 #include "spdk/likely.h" 14 #include "spdk/log.h" 15 #include "spdk/thread.h" 16 #include "spdk/json.h" 17 #include "spdk/crc32.h" 18 #include "spdk/util.h" 19 #include "spdk/xor.h" 20 #include "spdk/dif.h" 21 22 #ifdef SPDK_CONFIG_HAVE_LZ4 23 #include <lz4.h> 24 #endif 25 26 #ifdef SPDK_CONFIG_ISAL 27 #include "../isa-l/include/igzip_lib.h" 28 #ifdef SPDK_CONFIG_ISAL_CRYPTO 29 #include "../isa-l-crypto/include/aes_xts.h" 30 #include "../isa-l-crypto/include/isal_crypto_api.h" 31 #endif 32 #endif 33 34 /* Per the AES-XTS spec, the size of data unit cannot be bigger than 2^20 blocks, 128b each block */ 35 #define ACCEL_AES_XTS_MAX_BLOCK_SIZE (1 << 24) 36 37 #ifdef SPDK_CONFIG_ISAL 38 #define COMP_DEFLATE_MIN_LEVEL ISAL_DEF_MIN_LEVEL 39 #define COMP_DEFLATE_MAX_LEVEL ISAL_DEF_MAX_LEVEL 40 #else 41 #define COMP_DEFLATE_MIN_LEVEL 0 42 #define COMP_DEFLATE_MAX_LEVEL 0 43 #endif 44 45 #define COMP_DEFLATE_LEVEL_NUM (COMP_DEFLATE_MAX_LEVEL + 1) 46 47 struct comp_deflate_level_buf { 48 uint32_t size; 49 uint8_t *buf; 50 }; 51 52 struct sw_accel_io_channel { 53 /* for ISAL */ 54 #ifdef SPDK_CONFIG_ISAL 55 struct isal_zstream stream; 56 struct inflate_state state; 57 /* The array index corresponds to the algorithm level */ 58 struct comp_deflate_level_buf deflate_level_bufs[COMP_DEFLATE_LEVEL_NUM]; 59 uint8_t level_buf_mem[ISAL_DEF_LVL0_DEFAULT + ISAL_DEF_LVL1_DEFAULT + 60 ISAL_DEF_LVL2_DEFAULT + ISAL_DEF_LVL3_DEFAULT]; 61 #endif 62 #ifdef SPDK_CONFIG_HAVE_LZ4 63 /* for lz4 */ 64 LZ4_stream_t *lz4_stream; 65 LZ4_streamDecode_t *lz4_stream_decode; 66 #endif 67 struct spdk_poller *completion_poller; 68 STAILQ_HEAD(, spdk_accel_task) tasks_to_complete; 69 }; 70 71 typedef int (*sw_accel_crypto_op)(const uint8_t *k2, const uint8_t *k1, 72 const uint8_t *initial_tweak, const uint64_t len_bytes, 73 const void *in, void *out); 74 75 struct sw_accel_crypto_key_data { 76 sw_accel_crypto_op encrypt; 77 sw_accel_crypto_op decrypt; 78 }; 79 80 static struct spdk_accel_module_if g_sw_module; 81 82 static void sw_accel_crypto_key_deinit(struct spdk_accel_crypto_key *_key); 83 static int sw_accel_crypto_key_init(struct spdk_accel_crypto_key *key); 84 static bool sw_accel_crypto_supports_tweak_mode(enum spdk_accel_crypto_tweak_mode tweak_mode); 85 static bool sw_accel_crypto_supports_cipher(enum spdk_accel_cipher cipher, size_t key_size); 86 87 /* Post SW completions to a list; processed by ->completion_poller. */ 88 inline static void 89 _add_to_comp_list(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task, int status) 90 { 91 accel_task->status = status; 92 STAILQ_INSERT_TAIL(&sw_ch->tasks_to_complete, accel_task, link); 93 } 94 95 static bool 96 sw_accel_supports_opcode(enum spdk_accel_opcode opc) 97 { 98 switch (opc) { 99 case SPDK_ACCEL_OPC_COPY: 100 case SPDK_ACCEL_OPC_FILL: 101 case SPDK_ACCEL_OPC_DUALCAST: 102 case SPDK_ACCEL_OPC_COMPARE: 103 case SPDK_ACCEL_OPC_CRC32C: 104 case SPDK_ACCEL_OPC_COPY_CRC32C: 105 case SPDK_ACCEL_OPC_COMPRESS: 106 case SPDK_ACCEL_OPC_DECOMPRESS: 107 case SPDK_ACCEL_OPC_ENCRYPT: 108 case SPDK_ACCEL_OPC_DECRYPT: 109 case SPDK_ACCEL_OPC_XOR: 110 case SPDK_ACCEL_OPC_DIF_VERIFY: 111 case SPDK_ACCEL_OPC_DIF_GENERATE: 112 case SPDK_ACCEL_OPC_DIF_GENERATE_COPY: 113 case SPDK_ACCEL_OPC_DIF_VERIFY_COPY: 114 case SPDK_ACCEL_OPC_DIX_GENERATE: 115 case SPDK_ACCEL_OPC_DIX_VERIFY: 116 return true; 117 default: 118 return false; 119 } 120 } 121 122 static int 123 _sw_accel_dualcast_iovs(struct iovec *dst_iovs, uint32_t dst_iovcnt, 124 struct iovec *dst2_iovs, uint32_t dst2_iovcnt, 125 struct iovec *src_iovs, uint32_t src_iovcnt) 126 { 127 if (spdk_unlikely(dst_iovcnt != 1 || dst2_iovcnt != 1 || src_iovcnt != 1)) { 128 return -EINVAL; 129 } 130 131 if (spdk_unlikely(dst_iovs[0].iov_len != src_iovs[0].iov_len || 132 dst_iovs[0].iov_len != dst2_iovs[0].iov_len)) { 133 return -EINVAL; 134 } 135 136 memcpy(dst_iovs[0].iov_base, src_iovs[0].iov_base, dst_iovs[0].iov_len); 137 memcpy(dst2_iovs[0].iov_base, src_iovs[0].iov_base, dst_iovs[0].iov_len); 138 139 return 0; 140 } 141 142 static void 143 _sw_accel_copy_iovs(struct iovec *dst_iovs, uint32_t dst_iovcnt, 144 struct iovec *src_iovs, uint32_t src_iovcnt) 145 { 146 struct spdk_ioviter iter; 147 void *src, *dst; 148 size_t len; 149 150 for (len = spdk_ioviter_first(&iter, src_iovs, src_iovcnt, 151 dst_iovs, dst_iovcnt, &src, &dst); 152 len != 0; 153 len = spdk_ioviter_next(&iter, &src, &dst)) { 154 memcpy(dst, src, len); 155 } 156 } 157 158 static int 159 _sw_accel_compare(struct iovec *src_iovs, uint32_t src_iovcnt, 160 struct iovec *src2_iovs, uint32_t src2_iovcnt) 161 { 162 if (spdk_unlikely(src_iovcnt != 1 || src2_iovcnt != 1)) { 163 return -EINVAL; 164 } 165 166 if (spdk_unlikely(src_iovs[0].iov_len != src2_iovs[0].iov_len)) { 167 return -EINVAL; 168 } 169 170 return memcmp(src_iovs[0].iov_base, src2_iovs[0].iov_base, src_iovs[0].iov_len); 171 } 172 173 static int 174 _sw_accel_fill(struct iovec *iovs, uint32_t iovcnt, uint8_t fill) 175 { 176 void *dst; 177 size_t nbytes; 178 179 if (spdk_unlikely(iovcnt != 1)) { 180 return -EINVAL; 181 } 182 183 dst = iovs[0].iov_base; 184 nbytes = iovs[0].iov_len; 185 186 memset(dst, fill, nbytes); 187 188 return 0; 189 } 190 191 static void 192 _sw_accel_crc32cv(uint32_t *crc_dst, struct iovec *iov, uint32_t iovcnt, uint32_t seed) 193 { 194 *crc_dst = spdk_crc32c_iov_update(iov, iovcnt, ~seed); 195 } 196 197 static int 198 _sw_accel_compress_lz4(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 199 { 200 #ifdef SPDK_CONFIG_HAVE_LZ4 201 LZ4_stream_t *stream = sw_ch->lz4_stream; 202 struct iovec *siov = accel_task->s.iovs; 203 struct iovec *diov = accel_task->d.iovs; 204 size_t dst_segoffset = 0; 205 int32_t comp_size = 0; 206 uint32_t output_size = 0; 207 uint32_t i, d = 0; 208 int rc = 0; 209 210 LZ4_resetStream(stream); 211 for (i = 0; i < accel_task->s.iovcnt; i++) { 212 if ((diov[d].iov_len - dst_segoffset) == 0) { 213 if (++d < accel_task->d.iovcnt) { 214 dst_segoffset = 0; 215 } else { 216 SPDK_ERRLOG("Not enough destination buffer provided.\n"); 217 rc = -ENOMEM; 218 break; 219 } 220 } 221 222 comp_size = LZ4_compress_fast_continue(stream, siov[i].iov_base, diov[d].iov_base + dst_segoffset, 223 siov[i].iov_len, diov[d].iov_len - dst_segoffset, 224 accel_task->comp.level); 225 if (comp_size <= 0) { 226 SPDK_ERRLOG("LZ4_compress_fast_continue was incorrectly executed.\n"); 227 rc = -EIO; 228 break; 229 } 230 231 dst_segoffset += comp_size; 232 output_size += comp_size; 233 } 234 235 /* Get our total output size */ 236 if (accel_task->output_size != NULL) { 237 *accel_task->output_size = output_size; 238 } 239 240 return rc; 241 #else 242 SPDK_ERRLOG("LZ4 library is required to use software compression.\n"); 243 return -EINVAL; 244 #endif 245 } 246 247 static int 248 _sw_accel_decompress_lz4(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 249 { 250 #ifdef SPDK_CONFIG_HAVE_LZ4 251 LZ4_streamDecode_t *stream = sw_ch->lz4_stream_decode; 252 struct iovec *siov = accel_task->s.iovs; 253 struct iovec *diov = accel_task->d.iovs; 254 size_t dst_segoffset = 0; 255 int32_t decomp_size = 0; 256 uint32_t output_size = 0; 257 uint32_t i, d = 0; 258 int rc = 0; 259 260 LZ4_setStreamDecode(stream, NULL, 0); 261 for (i = 0; i < accel_task->s.iovcnt; ++i) { 262 if ((diov[d].iov_len - dst_segoffset) == 0) { 263 if (++d < accel_task->d.iovcnt) { 264 dst_segoffset = 0; 265 } else { 266 SPDK_ERRLOG("Not enough destination buffer provided.\n"); 267 rc = -ENOMEM; 268 break; 269 } 270 } 271 decomp_size = LZ4_decompress_safe_continue(stream, siov[i].iov_base, 272 diov[d].iov_base + dst_segoffset, siov[i].iov_len, 273 diov[d].iov_len - dst_segoffset); 274 if (decomp_size < 0) { 275 SPDK_ERRLOG("LZ4_compress_fast_continue was incorrectly executed.\n"); 276 rc = -EIO; 277 break; 278 } 279 dst_segoffset += decomp_size; 280 output_size += decomp_size; 281 } 282 283 /* Get our total output size */ 284 if (accel_task->output_size != NULL) { 285 *accel_task->output_size = output_size; 286 } 287 288 return rc; 289 #else 290 SPDK_ERRLOG("LZ4 library is required to use software compression.\n"); 291 return -EINVAL; 292 #endif 293 } 294 295 static int 296 _sw_accel_compress_deflate(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 297 { 298 #ifdef SPDK_CONFIG_ISAL 299 size_t last_seglen = accel_task->s.iovs[accel_task->s.iovcnt - 1].iov_len; 300 struct iovec *siov = accel_task->s.iovs; 301 struct iovec *diov = accel_task->d.iovs; 302 size_t remaining; 303 uint32_t i, s = 0, d = 0; 304 int rc = 0; 305 306 if (accel_task->comp.level > COMP_DEFLATE_MAX_LEVEL) { 307 SPDK_ERRLOG("isal_deflate doesn't support this algorithm level(%u)\n", accel_task->comp.level); 308 return -EINVAL; 309 } 310 311 remaining = 0; 312 for (i = 0; i < accel_task->s.iovcnt; ++i) { 313 remaining += accel_task->s.iovs[i].iov_len; 314 } 315 316 isal_deflate_reset(&sw_ch->stream); 317 sw_ch->stream.end_of_stream = 0; 318 sw_ch->stream.next_out = diov[d].iov_base; 319 sw_ch->stream.avail_out = diov[d].iov_len; 320 sw_ch->stream.next_in = siov[s].iov_base; 321 sw_ch->stream.avail_in = siov[s].iov_len; 322 sw_ch->stream.level = accel_task->comp.level; 323 sw_ch->stream.level_buf = sw_ch->deflate_level_bufs[accel_task->comp.level].buf; 324 sw_ch->stream.level_buf_size = sw_ch->deflate_level_bufs[accel_task->comp.level].size; 325 326 do { 327 /* if isal has exhausted the current dst iovec, move to the next 328 * one if there is one */ 329 if (sw_ch->stream.avail_out == 0) { 330 if (++d < accel_task->d.iovcnt) { 331 sw_ch->stream.next_out = diov[d].iov_base; 332 sw_ch->stream.avail_out = diov[d].iov_len; 333 assert(sw_ch->stream.avail_out > 0); 334 } else { 335 /* we have no avail_out but also no more iovecs left so this is 336 * the case where either the output buffer was a perfect fit 337 * or not enough was provided. Check the ISAL state to determine 338 * which. */ 339 if (sw_ch->stream.internal_state.state != ZSTATE_END) { 340 SPDK_ERRLOG("Not enough destination buffer provided.\n"); 341 rc = -ENOMEM; 342 } 343 break; 344 } 345 } 346 347 /* if isal has exhausted the current src iovec, move to the next 348 * one if there is one */ 349 if (sw_ch->stream.avail_in == 0 && ((s + 1) < accel_task->s.iovcnt)) { 350 s++; 351 sw_ch->stream.next_in = siov[s].iov_base; 352 sw_ch->stream.avail_in = siov[s].iov_len; 353 assert(sw_ch->stream.avail_in > 0); 354 } 355 356 if (remaining <= last_seglen) { 357 /* Need to set end of stream on last block */ 358 sw_ch->stream.end_of_stream = 1; 359 } 360 361 rc = isal_deflate(&sw_ch->stream); 362 if (rc) { 363 SPDK_ERRLOG("isal_deflate returned error %d.\n", rc); 364 } 365 366 if (remaining > 0) { 367 assert(siov[s].iov_len > sw_ch->stream.avail_in); 368 remaining -= (siov[s].iov_len - sw_ch->stream.avail_in); 369 } 370 371 } while (remaining > 0 || sw_ch->stream.avail_out == 0); 372 assert(sw_ch->stream.avail_in == 0); 373 374 /* Get our total output size */ 375 if (accel_task->output_size != NULL) { 376 assert(sw_ch->stream.total_out > 0); 377 *accel_task->output_size = sw_ch->stream.total_out; 378 } 379 380 return rc; 381 #else 382 SPDK_ERRLOG("ISAL option is required to use software compression.\n"); 383 return -EINVAL; 384 #endif 385 } 386 387 static int 388 _sw_accel_decompress_deflate(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 389 { 390 #ifdef SPDK_CONFIG_ISAL 391 struct iovec *siov = accel_task->s.iovs; 392 struct iovec *diov = accel_task->d.iovs; 393 uint32_t s = 0, d = 0; 394 int rc = 0; 395 396 isal_inflate_reset(&sw_ch->state); 397 sw_ch->state.next_out = diov[d].iov_base; 398 sw_ch->state.avail_out = diov[d].iov_len; 399 sw_ch->state.next_in = siov[s].iov_base; 400 sw_ch->state.avail_in = siov[s].iov_len; 401 402 do { 403 /* if isal has exhausted the current dst iovec, move to the next 404 * one if there is one */ 405 if (sw_ch->state.avail_out == 0 && ((d + 1) < accel_task->d.iovcnt)) { 406 d++; 407 sw_ch->state.next_out = diov[d].iov_base; 408 sw_ch->state.avail_out = diov[d].iov_len; 409 assert(sw_ch->state.avail_out > 0); 410 } 411 412 /* if isal has exhausted the current src iovec, move to the next 413 * one if there is one */ 414 if (sw_ch->state.avail_in == 0 && ((s + 1) < accel_task->s.iovcnt)) { 415 s++; 416 sw_ch->state.next_in = siov[s].iov_base; 417 sw_ch->state.avail_in = siov[s].iov_len; 418 assert(sw_ch->state.avail_in > 0); 419 } 420 421 rc = isal_inflate(&sw_ch->state); 422 if (rc) { 423 SPDK_ERRLOG("isal_inflate returned error %d.\n", rc); 424 } 425 426 } while (sw_ch->state.block_state < ISAL_BLOCK_FINISH); 427 assert(sw_ch->state.avail_in == 0); 428 429 /* Get our total output size */ 430 if (accel_task->output_size != NULL) { 431 assert(sw_ch->state.total_out > 0); 432 *accel_task->output_size = sw_ch->state.total_out; 433 } 434 435 return rc; 436 #else 437 SPDK_ERRLOG("ISAL option is required to use software decompression.\n"); 438 return -EINVAL; 439 #endif 440 } 441 442 static int 443 _sw_accel_compress(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 444 { 445 switch (accel_task->comp.algo) { 446 case SPDK_ACCEL_COMP_ALGO_DEFLATE: 447 return _sw_accel_compress_deflate(sw_ch, accel_task); 448 case SPDK_ACCEL_COMP_ALGO_LZ4: 449 return _sw_accel_compress_lz4(sw_ch, accel_task); 450 default: 451 assert(0); 452 return -EINVAL; 453 } 454 } 455 456 static int 457 _sw_accel_decompress(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 458 { 459 switch (accel_task->comp.algo) { 460 case SPDK_ACCEL_COMP_ALGO_DEFLATE: 461 return _sw_accel_decompress_deflate(sw_ch, accel_task); 462 case SPDK_ACCEL_COMP_ALGO_LZ4: 463 return _sw_accel_decompress_lz4(sw_ch, accel_task); 464 default: 465 assert(0); 466 return -EINVAL; 467 } 468 } 469 470 static int 471 _sw_accel_crypto_operation(struct spdk_accel_task *accel_task, struct spdk_accel_crypto_key *key, 472 sw_accel_crypto_op op) 473 { 474 #ifdef SPDK_CONFIG_ISAL_CRYPTO 475 uint64_t iv[2]; 476 size_t remaining_len, dst_len; 477 uint64_t src_offset = 0, dst_offset = 0; 478 uint32_t src_iovpos = 0, dst_iovpos = 0, src_iovcnt, dst_iovcnt; 479 uint32_t i, block_size, crypto_len, crypto_accum_len = 0; 480 struct iovec *src_iov, *dst_iov; 481 uint8_t *src, *dst; 482 int rc; 483 484 /* iv is 128 bits, since we are using logical block address (64 bits) as iv, fill first 8 bytes with zeroes */ 485 iv[0] = 0; 486 iv[1] = accel_task->iv; 487 src_iov = accel_task->s.iovs; 488 src_iovcnt = accel_task->s.iovcnt; 489 if (accel_task->d.iovcnt) { 490 dst_iov = accel_task->d.iovs; 491 dst_iovcnt = accel_task->d.iovcnt; 492 } else { 493 /* inplace operation */ 494 dst_iov = accel_task->s.iovs; 495 dst_iovcnt = accel_task->s.iovcnt; 496 } 497 block_size = accel_task->block_size; 498 499 if (!src_iovcnt || !dst_iovcnt || !block_size || !op) { 500 SPDK_ERRLOG("src_iovcnt %d, dst_iovcnt %d, block_size %d, op %p\n", src_iovcnt, dst_iovcnt, 501 block_size, op); 502 return -EINVAL; 503 } 504 505 remaining_len = 0; 506 for (i = 0; i < src_iovcnt; i++) { 507 remaining_len += src_iov[i].iov_len; 508 } 509 dst_len = 0; 510 for (i = 0; i < dst_iovcnt; i++) { 511 dst_len += dst_iov[i].iov_len; 512 } 513 514 if (spdk_unlikely(remaining_len != dst_len || !remaining_len)) { 515 return -ERANGE; 516 } 517 if (spdk_unlikely(remaining_len % accel_task->block_size != 0)) { 518 return -EINVAL; 519 } 520 521 while (remaining_len) { 522 crypto_len = spdk_min(block_size - crypto_accum_len, src_iov->iov_len - src_offset); 523 crypto_len = spdk_min(crypto_len, dst_iov->iov_len - dst_offset); 524 src = (uint8_t *)src_iov->iov_base + src_offset; 525 dst = (uint8_t *)dst_iov->iov_base + dst_offset; 526 527 rc = op((uint8_t *)key->key2, (uint8_t *)key->key, (uint8_t *)iv, crypto_len, src, dst); 528 if (rc != ISAL_CRYPTO_ERR_NONE) { 529 break; 530 } 531 532 src_offset += crypto_len; 533 dst_offset += crypto_len; 534 crypto_accum_len += crypto_len; 535 remaining_len -= crypto_len; 536 537 if (crypto_accum_len == block_size) { 538 /* we can process part of logical block. Once the whole block is processed, increment iv */ 539 crypto_accum_len = 0; 540 iv[1]++; 541 } 542 if (src_offset == src_iov->iov_len) { 543 src_iov++; 544 src_iovpos++; 545 src_offset = 0; 546 } 547 if (src_iovpos == src_iovcnt) { 548 break; 549 } 550 if (dst_offset == dst_iov->iov_len) { 551 dst_iov++; 552 dst_iovpos++; 553 dst_offset = 0; 554 } 555 if (dst_iovpos == dst_iovcnt) { 556 break; 557 } 558 } 559 560 if (remaining_len) { 561 SPDK_ERRLOG("remaining len %zu\n", remaining_len); 562 return -EINVAL; 563 } 564 565 return 0; 566 #else 567 return -ENOTSUP; 568 #endif 569 } 570 571 static int 572 _sw_accel_encrypt(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 573 { 574 struct spdk_accel_crypto_key *key; 575 struct sw_accel_crypto_key_data *key_data; 576 577 key = accel_task->crypto_key; 578 if (spdk_unlikely(key->module_if != &g_sw_module || !key->priv)) { 579 return -EINVAL; 580 } 581 if (spdk_unlikely(accel_task->block_size > ACCEL_AES_XTS_MAX_BLOCK_SIZE)) { 582 SPDK_WARNLOG("Max block size for AES_XTS is limited to %u, current size %u\n", 583 ACCEL_AES_XTS_MAX_BLOCK_SIZE, accel_task->block_size); 584 return -ERANGE; 585 } 586 key_data = key->priv; 587 return _sw_accel_crypto_operation(accel_task, key, key_data->encrypt); 588 } 589 590 static int 591 _sw_accel_decrypt(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 592 { 593 struct spdk_accel_crypto_key *key; 594 struct sw_accel_crypto_key_data *key_data; 595 596 key = accel_task->crypto_key; 597 if (spdk_unlikely(key->module_if != &g_sw_module || !key->priv)) { 598 return -EINVAL; 599 } 600 if (spdk_unlikely(accel_task->block_size > ACCEL_AES_XTS_MAX_BLOCK_SIZE)) { 601 SPDK_WARNLOG("Max block size for AES_XTS is limited to %u, current size %u\n", 602 ACCEL_AES_XTS_MAX_BLOCK_SIZE, accel_task->block_size); 603 return -ERANGE; 604 } 605 key_data = key->priv; 606 return _sw_accel_crypto_operation(accel_task, key, key_data->decrypt); 607 } 608 609 static int 610 _sw_accel_xor(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 611 { 612 return spdk_xor_gen(accel_task->d.iovs[0].iov_base, 613 accel_task->nsrcs.srcs, 614 accel_task->nsrcs.cnt, 615 accel_task->d.iovs[0].iov_len); 616 } 617 618 static int 619 _sw_accel_dif_verify(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 620 { 621 return spdk_dif_verify(accel_task->s.iovs, 622 accel_task->s.iovcnt, 623 accel_task->dif.num_blocks, 624 accel_task->dif.ctx, 625 accel_task->dif.err); 626 } 627 628 static int 629 _sw_accel_dif_verify_copy(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 630 { 631 return spdk_dif_verify_copy(accel_task->d.iovs, 632 accel_task->d.iovcnt, 633 accel_task->s.iovs, 634 accel_task->s.iovcnt, 635 accel_task->dif.num_blocks, 636 accel_task->dif.ctx, 637 accel_task->dif.err); 638 } 639 640 static int 641 _sw_accel_dif_generate(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 642 { 643 return spdk_dif_generate(accel_task->s.iovs, 644 accel_task->s.iovcnt, 645 accel_task->dif.num_blocks, 646 accel_task->dif.ctx); 647 } 648 649 static int 650 _sw_accel_dif_generate_copy(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 651 { 652 return spdk_dif_generate_copy(accel_task->s.iovs, 653 accel_task->s.iovcnt, 654 accel_task->d.iovs, 655 accel_task->d.iovcnt, 656 accel_task->dif.num_blocks, 657 accel_task->dif.ctx); 658 } 659 660 static int 661 _sw_accel_dix_generate(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 662 { 663 return spdk_dix_generate(accel_task->s.iovs, 664 accel_task->s.iovcnt, 665 accel_task->d.iovs, 666 accel_task->dif.num_blocks, 667 accel_task->dif.ctx); 668 } 669 670 static int 671 _sw_accel_dix_verify(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 672 { 673 return spdk_dix_verify(accel_task->s.iovs, 674 accel_task->s.iovcnt, 675 accel_task->d.iovs, 676 accel_task->dif.num_blocks, 677 accel_task->dif.ctx, 678 accel_task->dif.err); 679 } 680 681 static int 682 accel_comp_poll(void *arg) 683 { 684 struct sw_accel_io_channel *sw_ch = arg; 685 STAILQ_HEAD(, spdk_accel_task) tasks_to_complete; 686 struct spdk_accel_task *accel_task; 687 688 if (STAILQ_EMPTY(&sw_ch->tasks_to_complete)) { 689 return SPDK_POLLER_IDLE; 690 } 691 692 STAILQ_INIT(&tasks_to_complete); 693 STAILQ_SWAP(&tasks_to_complete, &sw_ch->tasks_to_complete, spdk_accel_task); 694 695 while ((accel_task = STAILQ_FIRST(&tasks_to_complete))) { 696 STAILQ_REMOVE_HEAD(&tasks_to_complete, link); 697 spdk_accel_task_complete(accel_task, accel_task->status); 698 } 699 700 return SPDK_POLLER_BUSY; 701 } 702 703 static int 704 sw_accel_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task) 705 { 706 struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch); 707 struct spdk_accel_task *tmp; 708 int rc = 0; 709 710 /* 711 * Lazily initialize our completion poller. We don't want to complete 712 * them inline as they'll likely submit another. 713 */ 714 if (spdk_unlikely(sw_ch->completion_poller == NULL)) { 715 sw_ch->completion_poller = SPDK_POLLER_REGISTER(accel_comp_poll, sw_ch, 0); 716 } 717 718 do { 719 switch (accel_task->op_code) { 720 case SPDK_ACCEL_OPC_COPY: 721 _sw_accel_copy_iovs(accel_task->d.iovs, accel_task->d.iovcnt, 722 accel_task->s.iovs, accel_task->s.iovcnt); 723 break; 724 case SPDK_ACCEL_OPC_FILL: 725 rc = _sw_accel_fill(accel_task->d.iovs, accel_task->d.iovcnt, 726 accel_task->fill_pattern); 727 break; 728 case SPDK_ACCEL_OPC_DUALCAST: 729 rc = _sw_accel_dualcast_iovs(accel_task->d.iovs, accel_task->d.iovcnt, 730 accel_task->d2.iovs, accel_task->d2.iovcnt, 731 accel_task->s.iovs, accel_task->s.iovcnt); 732 break; 733 case SPDK_ACCEL_OPC_COMPARE: 734 rc = _sw_accel_compare(accel_task->s.iovs, accel_task->s.iovcnt, 735 accel_task->s2.iovs, accel_task->s2.iovcnt); 736 break; 737 case SPDK_ACCEL_OPC_CRC32C: 738 _sw_accel_crc32cv(accel_task->crc_dst, accel_task->s.iovs, accel_task->s.iovcnt, accel_task->seed); 739 break; 740 case SPDK_ACCEL_OPC_COPY_CRC32C: 741 _sw_accel_copy_iovs(accel_task->d.iovs, accel_task->d.iovcnt, 742 accel_task->s.iovs, accel_task->s.iovcnt); 743 _sw_accel_crc32cv(accel_task->crc_dst, accel_task->s.iovs, 744 accel_task->s.iovcnt, accel_task->seed); 745 break; 746 case SPDK_ACCEL_OPC_COMPRESS: 747 rc = _sw_accel_compress(sw_ch, accel_task); 748 break; 749 case SPDK_ACCEL_OPC_DECOMPRESS: 750 rc = _sw_accel_decompress(sw_ch, accel_task); 751 break; 752 case SPDK_ACCEL_OPC_XOR: 753 rc = _sw_accel_xor(sw_ch, accel_task); 754 break; 755 case SPDK_ACCEL_OPC_ENCRYPT: 756 rc = _sw_accel_encrypt(sw_ch, accel_task); 757 break; 758 case SPDK_ACCEL_OPC_DECRYPT: 759 rc = _sw_accel_decrypt(sw_ch, accel_task); 760 break; 761 case SPDK_ACCEL_OPC_DIF_VERIFY: 762 rc = _sw_accel_dif_verify(sw_ch, accel_task); 763 break; 764 case SPDK_ACCEL_OPC_DIF_VERIFY_COPY: 765 rc = _sw_accel_dif_verify_copy(sw_ch, accel_task); 766 break; 767 case SPDK_ACCEL_OPC_DIF_GENERATE: 768 rc = _sw_accel_dif_generate(sw_ch, accel_task); 769 break; 770 case SPDK_ACCEL_OPC_DIF_GENERATE_COPY: 771 rc = _sw_accel_dif_generate_copy(sw_ch, accel_task); 772 break; 773 case SPDK_ACCEL_OPC_DIX_GENERATE: 774 rc = _sw_accel_dix_generate(sw_ch, accel_task); 775 break; 776 case SPDK_ACCEL_OPC_DIX_VERIFY: 777 rc = _sw_accel_dix_verify(sw_ch, accel_task); 778 break; 779 default: 780 assert(false); 781 break; 782 } 783 784 tmp = STAILQ_NEXT(accel_task, link); 785 786 _add_to_comp_list(sw_ch, accel_task, rc); 787 788 accel_task = tmp; 789 } while (accel_task); 790 791 return 0; 792 } 793 794 static int 795 sw_accel_create_cb(void *io_device, void *ctx_buf) 796 { 797 struct sw_accel_io_channel *sw_ch = ctx_buf; 798 #ifdef SPDK_CONFIG_ISAL 799 struct comp_deflate_level_buf *deflate_level_bufs; 800 #endif 801 int i; 802 803 STAILQ_INIT(&sw_ch->tasks_to_complete); 804 sw_ch->completion_poller = NULL; 805 806 #ifdef SPDK_CONFIG_HAVE_LZ4 807 sw_ch->lz4_stream = LZ4_createStream(); 808 if (sw_ch->lz4_stream == NULL) { 809 SPDK_ERRLOG("Failed to create the lz4 stream for compression\n"); 810 return -ENOMEM; 811 } 812 sw_ch->lz4_stream_decode = LZ4_createStreamDecode(); 813 if (sw_ch->lz4_stream_decode == NULL) { 814 SPDK_ERRLOG("Failed to create the lz4 stream for decompression\n"); 815 LZ4_freeStream(sw_ch->lz4_stream); 816 return -ENOMEM; 817 } 818 #endif 819 #ifdef SPDK_CONFIG_ISAL 820 sw_ch->deflate_level_bufs[0].buf = sw_ch->level_buf_mem; 821 deflate_level_bufs = sw_ch->deflate_level_bufs; 822 deflate_level_bufs[0].size = ISAL_DEF_LVL0_DEFAULT; 823 for (i = 1; i < COMP_DEFLATE_LEVEL_NUM; i++) { 824 deflate_level_bufs[i].buf = deflate_level_bufs[i - 1].buf + 825 deflate_level_bufs[i - 1].size; 826 switch (i) { 827 case 1: 828 deflate_level_bufs[i].size = ISAL_DEF_LVL1_DEFAULT; 829 break; 830 case 2: 831 deflate_level_bufs[i].size = ISAL_DEF_LVL2_DEFAULT; 832 break; 833 case 3: 834 deflate_level_bufs[i].size = ISAL_DEF_LVL3_DEFAULT; 835 break; 836 default: 837 assert(false); 838 } 839 } 840 841 isal_deflate_init(&sw_ch->stream); 842 sw_ch->stream.flush = NO_FLUSH; 843 isal_inflate_init(&sw_ch->state); 844 #endif 845 846 return 0; 847 } 848 849 static void 850 sw_accel_destroy_cb(void *io_device, void *ctx_buf) 851 { 852 struct sw_accel_io_channel *sw_ch = ctx_buf; 853 854 #ifdef SPDK_CONFIG_HAVE_LZ4 855 LZ4_freeStream(sw_ch->lz4_stream); 856 LZ4_freeStreamDecode(sw_ch->lz4_stream_decode); 857 #endif 858 spdk_poller_unregister(&sw_ch->completion_poller); 859 } 860 861 static struct spdk_io_channel * 862 sw_accel_get_io_channel(void) 863 { 864 return spdk_get_io_channel(&g_sw_module); 865 } 866 867 static size_t 868 sw_accel_module_get_ctx_size(void) 869 { 870 return sizeof(struct spdk_accel_task); 871 } 872 873 static int 874 sw_accel_module_init(void) 875 { 876 spdk_io_device_register(&g_sw_module, sw_accel_create_cb, sw_accel_destroy_cb, 877 sizeof(struct sw_accel_io_channel), "sw_accel_module"); 878 879 return 0; 880 } 881 882 static void 883 sw_accel_module_fini(void *ctxt) 884 { 885 spdk_io_device_unregister(&g_sw_module, NULL); 886 spdk_accel_module_finish(); 887 } 888 889 static int 890 sw_accel_create_aes_xts(struct spdk_accel_crypto_key *key) 891 { 892 #ifdef SPDK_CONFIG_ISAL_CRYPTO 893 struct sw_accel_crypto_key_data *key_data; 894 895 key_data = calloc(1, sizeof(*key_data)); 896 if (!key_data) { 897 return -ENOMEM; 898 } 899 900 switch (key->key_size) { 901 case SPDK_ACCEL_AES_XTS_128_KEY_SIZE: 902 key_data->encrypt = isal_aes_xts_enc_128; 903 key_data->decrypt = isal_aes_xts_dec_128; 904 break; 905 case SPDK_ACCEL_AES_XTS_256_KEY_SIZE: 906 key_data->encrypt = isal_aes_xts_enc_256; 907 key_data->decrypt = isal_aes_xts_dec_256; 908 break; 909 default: 910 assert(0); 911 free(key_data); 912 return -EINVAL; 913 } 914 915 key->priv = key_data; 916 917 return 0; 918 #else 919 return -ENOTSUP; 920 #endif 921 } 922 923 static int 924 sw_accel_crypto_key_init(struct spdk_accel_crypto_key *key) 925 { 926 return sw_accel_create_aes_xts(key); 927 } 928 929 static void 930 sw_accel_crypto_key_deinit(struct spdk_accel_crypto_key *key) 931 { 932 if (!key || key->module_if != &g_sw_module || !key->priv) { 933 return; 934 } 935 936 free(key->priv); 937 } 938 939 static bool 940 sw_accel_crypto_supports_tweak_mode(enum spdk_accel_crypto_tweak_mode tweak_mode) 941 { 942 return tweak_mode == SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA; 943 } 944 945 static bool 946 sw_accel_crypto_supports_cipher(enum spdk_accel_cipher cipher, size_t key_size) 947 { 948 switch (cipher) { 949 case SPDK_ACCEL_CIPHER_AES_XTS: 950 return key_size == SPDK_ACCEL_AES_XTS_128_KEY_SIZE || key_size == SPDK_ACCEL_AES_XTS_256_KEY_SIZE; 951 default: 952 return false; 953 } 954 } 955 956 static bool 957 sw_accel_compress_supports_algo(enum spdk_accel_comp_algo algo) 958 { 959 switch (algo) { 960 case SPDK_ACCEL_COMP_ALGO_DEFLATE: 961 #ifdef SPDK_CONFIG_HAVE_LZ4 962 case SPDK_ACCEL_COMP_ALGO_LZ4: 963 #endif 964 return true; 965 default: 966 return false; 967 } 968 } 969 970 static int 971 sw_accel_get_compress_level_range(enum spdk_accel_comp_algo algo, 972 uint32_t *min_level, uint32_t *max_level) 973 { 974 switch (algo) { 975 case SPDK_ACCEL_COMP_ALGO_DEFLATE: 976 #ifdef SPDK_CONFIG_ISAL 977 *min_level = COMP_DEFLATE_MIN_LEVEL; 978 *max_level = COMP_DEFLATE_MAX_LEVEL; 979 return 0; 980 #else 981 SPDK_ERRLOG("ISAL option is required to use software compression.\n"); 982 return -EINVAL; 983 #endif 984 case SPDK_ACCEL_COMP_ALGO_LZ4: 985 #ifdef SPDK_CONFIG_HAVE_LZ4 986 *min_level = 1; 987 *max_level = 65537; 988 return 0; 989 #else 990 SPDK_ERRLOG("LZ4 library is required to use software compression.\n"); 991 return -EINVAL; 992 #endif 993 default: 994 return -EINVAL; 995 } 996 } 997 998 static int 999 sw_accel_get_operation_info(enum spdk_accel_opcode opcode, 1000 const struct spdk_accel_operation_exec_ctx *ctx, 1001 struct spdk_accel_opcode_info *info) 1002 { 1003 info->required_alignment = 0; 1004 1005 return 0; 1006 } 1007 1008 static struct spdk_accel_module_if g_sw_module = { 1009 .module_init = sw_accel_module_init, 1010 .module_fini = sw_accel_module_fini, 1011 .write_config_json = NULL, 1012 .get_ctx_size = sw_accel_module_get_ctx_size, 1013 .name = "software", 1014 .priority = SPDK_ACCEL_SW_PRIORITY, 1015 .supports_opcode = sw_accel_supports_opcode, 1016 .get_io_channel = sw_accel_get_io_channel, 1017 .submit_tasks = sw_accel_submit_tasks, 1018 .crypto_key_init = sw_accel_crypto_key_init, 1019 .crypto_key_deinit = sw_accel_crypto_key_deinit, 1020 .crypto_supports_tweak_mode = sw_accel_crypto_supports_tweak_mode, 1021 .crypto_supports_cipher = sw_accel_crypto_supports_cipher, 1022 .compress_supports_algo = sw_accel_compress_supports_algo, 1023 .get_compress_level_range = sw_accel_get_compress_level_range, 1024 .get_operation_info = sw_accel_get_operation_info, 1025 }; 1026 1027 SPDK_ACCEL_MODULE_REGISTER(sw, &g_sw_module) 1028