1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES 4 * All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/accel_module.h" 10 #include "accel_internal.h" 11 12 #include "spdk/env.h" 13 #include "spdk/likely.h" 14 #include "spdk/log.h" 15 #include "spdk/thread.h" 16 #include "spdk/json.h" 17 #include "spdk/crc32.h" 18 #include "spdk/util.h" 19 #include "spdk/xor.h" 20 #include "spdk/dif.h" 21 22 #ifdef SPDK_CONFIG_ISAL 23 #include "../isa-l/include/igzip_lib.h" 24 #ifdef SPDK_CONFIG_ISAL_CRYPTO 25 #include "../isa-l-crypto/include/aes_xts.h" 26 #include "../isa-l-crypto/include/isal_crypto_api.h" 27 #endif 28 #endif 29 30 /* Per the AES-XTS spec, the size of data unit cannot be bigger than 2^20 blocks, 128b each block */ 31 #define ACCEL_AES_XTS_MAX_BLOCK_SIZE (1 << 24) 32 33 #ifdef SPDK_CONFIG_ISAL 34 #define COMP_DEFLATE_MIN_LEVEL ISAL_DEF_MIN_LEVEL 35 #define COMP_DEFLATE_MAX_LEVEL ISAL_DEF_MAX_LEVEL 36 #else 37 #define COMP_DEFLATE_MIN_LEVEL 0 38 #define COMP_DEFLATE_MAX_LEVEL 0 39 #endif 40 41 #define COMP_DEFLATE_LEVEL_NUM (COMP_DEFLATE_MAX_LEVEL + 1) 42 43 struct comp_deflate_level_buf { 44 uint32_t size; 45 uint8_t *buf; 46 }; 47 48 struct sw_accel_io_channel { 49 /* for ISAL */ 50 #ifdef SPDK_CONFIG_ISAL 51 struct isal_zstream stream; 52 struct inflate_state state; 53 /* The array index corresponds to the algorithm level */ 54 struct comp_deflate_level_buf deflate_level_bufs[COMP_DEFLATE_LEVEL_NUM]; 55 uint8_t level_buf_mem[ISAL_DEF_LVL0_DEFAULT + ISAL_DEF_LVL1_DEFAULT + 56 ISAL_DEF_LVL2_DEFAULT + ISAL_DEF_LVL3_DEFAULT]; 57 #endif 58 struct spdk_poller *completion_poller; 59 STAILQ_HEAD(, spdk_accel_task) tasks_to_complete; 60 }; 61 62 typedef int (*sw_accel_crypto_op)(const uint8_t *k2, const uint8_t *k1, 63 const uint8_t *initial_tweak, const uint64_t len_bytes, 64 const void *in, void *out); 65 66 struct sw_accel_crypto_key_data { 67 sw_accel_crypto_op encrypt; 68 sw_accel_crypto_op decrypt; 69 }; 70 71 static struct spdk_accel_module_if g_sw_module; 72 73 static void sw_accel_crypto_key_deinit(struct spdk_accel_crypto_key *_key); 74 static int sw_accel_crypto_key_init(struct spdk_accel_crypto_key *key); 75 static bool sw_accel_crypto_supports_tweak_mode(enum spdk_accel_crypto_tweak_mode tweak_mode); 76 static bool sw_accel_crypto_supports_cipher(enum spdk_accel_cipher cipher, size_t key_size); 77 78 /* Post SW completions to a list; processed by ->completion_poller. */ 79 inline static void 80 _add_to_comp_list(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task, int status) 81 { 82 accel_task->status = status; 83 STAILQ_INSERT_TAIL(&sw_ch->tasks_to_complete, accel_task, link); 84 } 85 86 static bool 87 sw_accel_supports_opcode(enum spdk_accel_opcode opc) 88 { 89 switch (opc) { 90 case SPDK_ACCEL_OPC_COPY: 91 case SPDK_ACCEL_OPC_FILL: 92 case SPDK_ACCEL_OPC_DUALCAST: 93 case SPDK_ACCEL_OPC_COMPARE: 94 case SPDK_ACCEL_OPC_CRC32C: 95 case SPDK_ACCEL_OPC_COPY_CRC32C: 96 case SPDK_ACCEL_OPC_COMPRESS: 97 case SPDK_ACCEL_OPC_DECOMPRESS: 98 case SPDK_ACCEL_OPC_ENCRYPT: 99 case SPDK_ACCEL_OPC_DECRYPT: 100 case SPDK_ACCEL_OPC_XOR: 101 case SPDK_ACCEL_OPC_DIF_VERIFY: 102 case SPDK_ACCEL_OPC_DIF_GENERATE: 103 case SPDK_ACCEL_OPC_DIF_GENERATE_COPY: 104 case SPDK_ACCEL_OPC_DIF_VERIFY_COPY: 105 case SPDK_ACCEL_OPC_DIX_GENERATE: 106 case SPDK_ACCEL_OPC_DIX_VERIFY: 107 return true; 108 default: 109 return false; 110 } 111 } 112 113 static int 114 _sw_accel_dualcast_iovs(struct iovec *dst_iovs, uint32_t dst_iovcnt, 115 struct iovec *dst2_iovs, uint32_t dst2_iovcnt, 116 struct iovec *src_iovs, uint32_t src_iovcnt) 117 { 118 if (spdk_unlikely(dst_iovcnt != 1 || dst2_iovcnt != 1 || src_iovcnt != 1)) { 119 return -EINVAL; 120 } 121 122 if (spdk_unlikely(dst_iovs[0].iov_len != src_iovs[0].iov_len || 123 dst_iovs[0].iov_len != dst2_iovs[0].iov_len)) { 124 return -EINVAL; 125 } 126 127 memcpy(dst_iovs[0].iov_base, src_iovs[0].iov_base, dst_iovs[0].iov_len); 128 memcpy(dst2_iovs[0].iov_base, src_iovs[0].iov_base, dst_iovs[0].iov_len); 129 130 return 0; 131 } 132 133 static void 134 _sw_accel_copy_iovs(struct iovec *dst_iovs, uint32_t dst_iovcnt, 135 struct iovec *src_iovs, uint32_t src_iovcnt) 136 { 137 struct spdk_ioviter iter; 138 void *src, *dst; 139 size_t len; 140 141 for (len = spdk_ioviter_first(&iter, src_iovs, src_iovcnt, 142 dst_iovs, dst_iovcnt, &src, &dst); 143 len != 0; 144 len = spdk_ioviter_next(&iter, &src, &dst)) { 145 memcpy(dst, src, len); 146 } 147 } 148 149 static int 150 _sw_accel_compare(struct iovec *src_iovs, uint32_t src_iovcnt, 151 struct iovec *src2_iovs, uint32_t src2_iovcnt) 152 { 153 if (spdk_unlikely(src_iovcnt != 1 || src2_iovcnt != 1)) { 154 return -EINVAL; 155 } 156 157 if (spdk_unlikely(src_iovs[0].iov_len != src2_iovs[0].iov_len)) { 158 return -EINVAL; 159 } 160 161 return memcmp(src_iovs[0].iov_base, src2_iovs[0].iov_base, src_iovs[0].iov_len); 162 } 163 164 static int 165 _sw_accel_fill(struct iovec *iovs, uint32_t iovcnt, uint8_t fill) 166 { 167 void *dst; 168 size_t nbytes; 169 170 if (spdk_unlikely(iovcnt != 1)) { 171 return -EINVAL; 172 } 173 174 dst = iovs[0].iov_base; 175 nbytes = iovs[0].iov_len; 176 177 memset(dst, fill, nbytes); 178 179 return 0; 180 } 181 182 static void 183 _sw_accel_crc32cv(uint32_t *crc_dst, struct iovec *iov, uint32_t iovcnt, uint32_t seed) 184 { 185 *crc_dst = spdk_crc32c_iov_update(iov, iovcnt, ~seed); 186 } 187 188 static int 189 _sw_accel_compress_deflate(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 190 { 191 #ifdef SPDK_CONFIG_ISAL 192 size_t last_seglen = accel_task->s.iovs[accel_task->s.iovcnt - 1].iov_len; 193 struct iovec *siov = accel_task->s.iovs; 194 struct iovec *diov = accel_task->d.iovs; 195 size_t remaining; 196 uint32_t i, s = 0, d = 0; 197 int rc = 0; 198 199 if (accel_task->comp.level > COMP_DEFLATE_MAX_LEVEL) { 200 SPDK_ERRLOG("isal_deflate doesn't support this algorithm level(%u)\n", accel_task->comp.level); 201 return -EINVAL; 202 } 203 204 remaining = 0; 205 for (i = 0; i < accel_task->s.iovcnt; ++i) { 206 remaining += accel_task->s.iovs[i].iov_len; 207 } 208 209 isal_deflate_reset(&sw_ch->stream); 210 sw_ch->stream.end_of_stream = 0; 211 sw_ch->stream.next_out = diov[d].iov_base; 212 sw_ch->stream.avail_out = diov[d].iov_len; 213 sw_ch->stream.next_in = siov[s].iov_base; 214 sw_ch->stream.avail_in = siov[s].iov_len; 215 sw_ch->stream.level = accel_task->comp.level; 216 sw_ch->stream.level_buf = sw_ch->deflate_level_bufs[accel_task->comp.level].buf; 217 sw_ch->stream.level_buf_size = sw_ch->deflate_level_bufs[accel_task->comp.level].size; 218 219 do { 220 /* if isal has exhausted the current dst iovec, move to the next 221 * one if there is one */ 222 if (sw_ch->stream.avail_out == 0) { 223 if (++d < accel_task->d.iovcnt) { 224 sw_ch->stream.next_out = diov[d].iov_base; 225 sw_ch->stream.avail_out = diov[d].iov_len; 226 assert(sw_ch->stream.avail_out > 0); 227 } else { 228 /* we have no avail_out but also no more iovecs left so this is 229 * the case where either the output buffer was a perfect fit 230 * or not enough was provided. Check the ISAL state to determine 231 * which. */ 232 if (sw_ch->stream.internal_state.state != ZSTATE_END) { 233 SPDK_ERRLOG("Not enough destination buffer provided.\n"); 234 rc = -ENOMEM; 235 } 236 break; 237 } 238 } 239 240 /* if isal has exhausted the current src iovec, move to the next 241 * one if there is one */ 242 if (sw_ch->stream.avail_in == 0 && ((s + 1) < accel_task->s.iovcnt)) { 243 s++; 244 sw_ch->stream.next_in = siov[s].iov_base; 245 sw_ch->stream.avail_in = siov[s].iov_len; 246 assert(sw_ch->stream.avail_in > 0); 247 } 248 249 if (remaining <= last_seglen) { 250 /* Need to set end of stream on last block */ 251 sw_ch->stream.end_of_stream = 1; 252 } 253 254 rc = isal_deflate(&sw_ch->stream); 255 if (rc) { 256 SPDK_ERRLOG("isal_deflate returned error %d.\n", rc); 257 } 258 259 if (remaining > 0) { 260 assert(siov[s].iov_len > sw_ch->stream.avail_in); 261 remaining -= (siov[s].iov_len - sw_ch->stream.avail_in); 262 } 263 264 } while (remaining > 0 || sw_ch->stream.avail_out == 0); 265 assert(sw_ch->stream.avail_in == 0); 266 267 /* Get our total output size */ 268 if (accel_task->output_size != NULL) { 269 assert(sw_ch->stream.total_out > 0); 270 *accel_task->output_size = sw_ch->stream.total_out; 271 } 272 273 return rc; 274 #else 275 SPDK_ERRLOG("ISAL option is required to use software compression.\n"); 276 return -EINVAL; 277 #endif 278 } 279 280 static int 281 _sw_accel_decompress_deflate(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 282 { 283 #ifdef SPDK_CONFIG_ISAL 284 struct iovec *siov = accel_task->s.iovs; 285 struct iovec *diov = accel_task->d.iovs; 286 uint32_t s = 0, d = 0; 287 int rc = 0; 288 289 isal_inflate_reset(&sw_ch->state); 290 sw_ch->state.next_out = diov[d].iov_base; 291 sw_ch->state.avail_out = diov[d].iov_len; 292 sw_ch->state.next_in = siov[s].iov_base; 293 sw_ch->state.avail_in = siov[s].iov_len; 294 295 do { 296 /* if isal has exhausted the current dst iovec, move to the next 297 * one if there is one */ 298 if (sw_ch->state.avail_out == 0 && ((d + 1) < accel_task->d.iovcnt)) { 299 d++; 300 sw_ch->state.next_out = diov[d].iov_base; 301 sw_ch->state.avail_out = diov[d].iov_len; 302 assert(sw_ch->state.avail_out > 0); 303 } 304 305 /* if isal has exhausted the current src iovec, move to the next 306 * one if there is one */ 307 if (sw_ch->state.avail_in == 0 && ((s + 1) < accel_task->s.iovcnt)) { 308 s++; 309 sw_ch->state.next_in = siov[s].iov_base; 310 sw_ch->state.avail_in = siov[s].iov_len; 311 assert(sw_ch->state.avail_in > 0); 312 } 313 314 rc = isal_inflate(&sw_ch->state); 315 if (rc) { 316 SPDK_ERRLOG("isal_inflate returned error %d.\n", rc); 317 } 318 319 } while (sw_ch->state.block_state < ISAL_BLOCK_FINISH); 320 assert(sw_ch->state.avail_in == 0); 321 322 /* Get our total output size */ 323 if (accel_task->output_size != NULL) { 324 assert(sw_ch->state.total_out > 0); 325 *accel_task->output_size = sw_ch->state.total_out; 326 } 327 328 return rc; 329 #else 330 SPDK_ERRLOG("ISAL option is required to use software decompression.\n"); 331 return -EINVAL; 332 #endif 333 } 334 335 static int 336 _sw_accel_compress(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 337 { 338 switch (accel_task->comp.algo) { 339 case SPDK_ACCEL_COMP_ALGO_DEFLATE: 340 return _sw_accel_compress_deflate(sw_ch, accel_task); 341 default: 342 assert(0); 343 return -EINVAL; 344 } 345 } 346 347 static int 348 _sw_accel_decompress(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 349 { 350 switch (accel_task->comp.algo) { 351 case SPDK_ACCEL_COMP_ALGO_DEFLATE: 352 return _sw_accel_decompress_deflate(sw_ch, accel_task); 353 default: 354 assert(0); 355 return -EINVAL; 356 } 357 } 358 359 static int 360 _sw_accel_crypto_operation(struct spdk_accel_task *accel_task, struct spdk_accel_crypto_key *key, 361 sw_accel_crypto_op op) 362 { 363 #ifdef SPDK_CONFIG_ISAL_CRYPTO 364 uint64_t iv[2]; 365 size_t remaining_len, dst_len; 366 uint64_t src_offset = 0, dst_offset = 0; 367 uint32_t src_iovpos = 0, dst_iovpos = 0, src_iovcnt, dst_iovcnt; 368 uint32_t i, block_size, crypto_len, crypto_accum_len = 0; 369 struct iovec *src_iov, *dst_iov; 370 uint8_t *src, *dst; 371 int rc; 372 373 /* iv is 128 bits, since we are using logical block address (64 bits) as iv, fill first 8 bytes with zeroes */ 374 iv[0] = 0; 375 iv[1] = accel_task->iv; 376 src_iov = accel_task->s.iovs; 377 src_iovcnt = accel_task->s.iovcnt; 378 if (accel_task->d.iovcnt) { 379 dst_iov = accel_task->d.iovs; 380 dst_iovcnt = accel_task->d.iovcnt; 381 } else { 382 /* inplace operation */ 383 dst_iov = accel_task->s.iovs; 384 dst_iovcnt = accel_task->s.iovcnt; 385 } 386 block_size = accel_task->block_size; 387 388 if (!src_iovcnt || !dst_iovcnt || !block_size || !op) { 389 SPDK_ERRLOG("src_iovcnt %d, dst_iovcnt %d, block_size %d, op %p\n", src_iovcnt, dst_iovcnt, 390 block_size, op); 391 return -EINVAL; 392 } 393 394 remaining_len = 0; 395 for (i = 0; i < src_iovcnt; i++) { 396 remaining_len += src_iov[i].iov_len; 397 } 398 dst_len = 0; 399 for (i = 0; i < dst_iovcnt; i++) { 400 dst_len += dst_iov[i].iov_len; 401 } 402 403 if (spdk_unlikely(remaining_len != dst_len || !remaining_len)) { 404 return -ERANGE; 405 } 406 if (spdk_unlikely(remaining_len % accel_task->block_size != 0)) { 407 return -EINVAL; 408 } 409 410 while (remaining_len) { 411 crypto_len = spdk_min(block_size - crypto_accum_len, src_iov->iov_len - src_offset); 412 crypto_len = spdk_min(crypto_len, dst_iov->iov_len - dst_offset); 413 src = (uint8_t *)src_iov->iov_base + src_offset; 414 dst = (uint8_t *)dst_iov->iov_base + dst_offset; 415 416 rc = op((uint8_t *)key->key2, (uint8_t *)key->key, (uint8_t *)iv, crypto_len, src, dst); 417 if (rc != ISAL_CRYPTO_ERR_NONE) { 418 break; 419 } 420 421 src_offset += crypto_len; 422 dst_offset += crypto_len; 423 crypto_accum_len += crypto_len; 424 remaining_len -= crypto_len; 425 426 if (crypto_accum_len == block_size) { 427 /* we can process part of logical block. Once the whole block is processed, increment iv */ 428 crypto_accum_len = 0; 429 iv[1]++; 430 } 431 if (src_offset == src_iov->iov_len) { 432 src_iov++; 433 src_iovpos++; 434 src_offset = 0; 435 } 436 if (src_iovpos == src_iovcnt) { 437 break; 438 } 439 if (dst_offset == dst_iov->iov_len) { 440 dst_iov++; 441 dst_iovpos++; 442 dst_offset = 0; 443 } 444 if (dst_iovpos == dst_iovcnt) { 445 break; 446 } 447 } 448 449 if (remaining_len) { 450 SPDK_ERRLOG("remaining len %zu\n", remaining_len); 451 return -EINVAL; 452 } 453 454 return 0; 455 #else 456 return -ENOTSUP; 457 #endif 458 } 459 460 static int 461 _sw_accel_encrypt(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 462 { 463 struct spdk_accel_crypto_key *key; 464 struct sw_accel_crypto_key_data *key_data; 465 466 key = accel_task->crypto_key; 467 if (spdk_unlikely(key->module_if != &g_sw_module || !key->priv)) { 468 return -EINVAL; 469 } 470 if (spdk_unlikely(accel_task->block_size > ACCEL_AES_XTS_MAX_BLOCK_SIZE)) { 471 SPDK_WARNLOG("Max block size for AES_XTS is limited to %u, current size %u\n", 472 ACCEL_AES_XTS_MAX_BLOCK_SIZE, accel_task->block_size); 473 return -ERANGE; 474 } 475 key_data = key->priv; 476 return _sw_accel_crypto_operation(accel_task, key, key_data->encrypt); 477 } 478 479 static int 480 _sw_accel_decrypt(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 481 { 482 struct spdk_accel_crypto_key *key; 483 struct sw_accel_crypto_key_data *key_data; 484 485 key = accel_task->crypto_key; 486 if (spdk_unlikely(key->module_if != &g_sw_module || !key->priv)) { 487 return -EINVAL; 488 } 489 if (spdk_unlikely(accel_task->block_size > ACCEL_AES_XTS_MAX_BLOCK_SIZE)) { 490 SPDK_WARNLOG("Max block size for AES_XTS is limited to %u, current size %u\n", 491 ACCEL_AES_XTS_MAX_BLOCK_SIZE, accel_task->block_size); 492 return -ERANGE; 493 } 494 key_data = key->priv; 495 return _sw_accel_crypto_operation(accel_task, key, key_data->decrypt); 496 } 497 498 static int 499 _sw_accel_xor(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 500 { 501 return spdk_xor_gen(accel_task->d.iovs[0].iov_base, 502 accel_task->nsrcs.srcs, 503 accel_task->nsrcs.cnt, 504 accel_task->d.iovs[0].iov_len); 505 } 506 507 static int 508 _sw_accel_dif_verify(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 509 { 510 return spdk_dif_verify(accel_task->s.iovs, 511 accel_task->s.iovcnt, 512 accel_task->dif.num_blocks, 513 accel_task->dif.ctx, 514 accel_task->dif.err); 515 } 516 517 static int 518 _sw_accel_dif_verify_copy(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 519 { 520 return spdk_dif_verify_copy(accel_task->d.iovs, 521 accel_task->d.iovcnt, 522 accel_task->s.iovs, 523 accel_task->s.iovcnt, 524 accel_task->dif.num_blocks, 525 accel_task->dif.ctx, 526 accel_task->dif.err); 527 } 528 529 static int 530 _sw_accel_dif_generate(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 531 { 532 return spdk_dif_generate(accel_task->s.iovs, 533 accel_task->s.iovcnt, 534 accel_task->dif.num_blocks, 535 accel_task->dif.ctx); 536 } 537 538 static int 539 _sw_accel_dif_generate_copy(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 540 { 541 return spdk_dif_generate_copy(accel_task->s.iovs, 542 accel_task->s.iovcnt, 543 accel_task->d.iovs, 544 accel_task->d.iovcnt, 545 accel_task->dif.num_blocks, 546 accel_task->dif.ctx); 547 } 548 549 static int 550 _sw_accel_dix_generate(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 551 { 552 return spdk_dix_generate(accel_task->s.iovs, 553 accel_task->s.iovcnt, 554 accel_task->d.iovs, 555 accel_task->dif.num_blocks, 556 accel_task->dif.ctx); 557 } 558 559 static int 560 _sw_accel_dix_verify(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 561 { 562 return spdk_dix_verify(accel_task->s.iovs, 563 accel_task->s.iovcnt, 564 accel_task->d.iovs, 565 accel_task->dif.num_blocks, 566 accel_task->dif.ctx, 567 accel_task->dif.err); 568 } 569 570 static int 571 accel_comp_poll(void *arg) 572 { 573 struct sw_accel_io_channel *sw_ch = arg; 574 STAILQ_HEAD(, spdk_accel_task) tasks_to_complete; 575 struct spdk_accel_task *accel_task; 576 577 if (STAILQ_EMPTY(&sw_ch->tasks_to_complete)) { 578 return SPDK_POLLER_IDLE; 579 } 580 581 STAILQ_INIT(&tasks_to_complete); 582 STAILQ_SWAP(&tasks_to_complete, &sw_ch->tasks_to_complete, spdk_accel_task); 583 584 while ((accel_task = STAILQ_FIRST(&tasks_to_complete))) { 585 STAILQ_REMOVE_HEAD(&tasks_to_complete, link); 586 spdk_accel_task_complete(accel_task, accel_task->status); 587 } 588 589 return SPDK_POLLER_BUSY; 590 } 591 592 static int 593 sw_accel_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task) 594 { 595 struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch); 596 struct spdk_accel_task *tmp; 597 int rc = 0; 598 599 /* 600 * Lazily initialize our completion poller. We don't want to complete 601 * them inline as they'll likely submit another. 602 */ 603 if (spdk_unlikely(sw_ch->completion_poller == NULL)) { 604 sw_ch->completion_poller = SPDK_POLLER_REGISTER(accel_comp_poll, sw_ch, 0); 605 } 606 607 do { 608 switch (accel_task->op_code) { 609 case SPDK_ACCEL_OPC_COPY: 610 _sw_accel_copy_iovs(accel_task->d.iovs, accel_task->d.iovcnt, 611 accel_task->s.iovs, accel_task->s.iovcnt); 612 break; 613 case SPDK_ACCEL_OPC_FILL: 614 rc = _sw_accel_fill(accel_task->d.iovs, accel_task->d.iovcnt, 615 accel_task->fill_pattern); 616 break; 617 case SPDK_ACCEL_OPC_DUALCAST: 618 rc = _sw_accel_dualcast_iovs(accel_task->d.iovs, accel_task->d.iovcnt, 619 accel_task->d2.iovs, accel_task->d2.iovcnt, 620 accel_task->s.iovs, accel_task->s.iovcnt); 621 break; 622 case SPDK_ACCEL_OPC_COMPARE: 623 rc = _sw_accel_compare(accel_task->s.iovs, accel_task->s.iovcnt, 624 accel_task->s2.iovs, accel_task->s2.iovcnt); 625 break; 626 case SPDK_ACCEL_OPC_CRC32C: 627 _sw_accel_crc32cv(accel_task->crc_dst, accel_task->s.iovs, accel_task->s.iovcnt, accel_task->seed); 628 break; 629 case SPDK_ACCEL_OPC_COPY_CRC32C: 630 _sw_accel_copy_iovs(accel_task->d.iovs, accel_task->d.iovcnt, 631 accel_task->s.iovs, accel_task->s.iovcnt); 632 _sw_accel_crc32cv(accel_task->crc_dst, accel_task->s.iovs, 633 accel_task->s.iovcnt, accel_task->seed); 634 break; 635 case SPDK_ACCEL_OPC_COMPRESS: 636 rc = _sw_accel_compress(sw_ch, accel_task); 637 break; 638 case SPDK_ACCEL_OPC_DECOMPRESS: 639 rc = _sw_accel_decompress(sw_ch, accel_task); 640 break; 641 case SPDK_ACCEL_OPC_XOR: 642 rc = _sw_accel_xor(sw_ch, accel_task); 643 break; 644 case SPDK_ACCEL_OPC_ENCRYPT: 645 rc = _sw_accel_encrypt(sw_ch, accel_task); 646 break; 647 case SPDK_ACCEL_OPC_DECRYPT: 648 rc = _sw_accel_decrypt(sw_ch, accel_task); 649 break; 650 case SPDK_ACCEL_OPC_DIF_VERIFY: 651 rc = _sw_accel_dif_verify(sw_ch, accel_task); 652 break; 653 case SPDK_ACCEL_OPC_DIF_VERIFY_COPY: 654 rc = _sw_accel_dif_verify_copy(sw_ch, accel_task); 655 break; 656 case SPDK_ACCEL_OPC_DIF_GENERATE: 657 rc = _sw_accel_dif_generate(sw_ch, accel_task); 658 break; 659 case SPDK_ACCEL_OPC_DIF_GENERATE_COPY: 660 rc = _sw_accel_dif_generate_copy(sw_ch, accel_task); 661 break; 662 case SPDK_ACCEL_OPC_DIX_GENERATE: 663 rc = _sw_accel_dix_generate(sw_ch, accel_task); 664 break; 665 case SPDK_ACCEL_OPC_DIX_VERIFY: 666 rc = _sw_accel_dix_verify(sw_ch, accel_task); 667 break; 668 default: 669 assert(false); 670 break; 671 } 672 673 tmp = STAILQ_NEXT(accel_task, link); 674 675 _add_to_comp_list(sw_ch, accel_task, rc); 676 677 accel_task = tmp; 678 } while (accel_task); 679 680 return 0; 681 } 682 683 static int 684 sw_accel_create_cb(void *io_device, void *ctx_buf) 685 { 686 struct sw_accel_io_channel *sw_ch = ctx_buf; 687 #ifdef SPDK_CONFIG_ISAL 688 struct comp_deflate_level_buf *deflate_level_bufs; 689 #endif 690 int i; 691 692 STAILQ_INIT(&sw_ch->tasks_to_complete); 693 sw_ch->completion_poller = NULL; 694 695 #ifdef SPDK_CONFIG_ISAL 696 sw_ch->deflate_level_bufs[0].buf = sw_ch->level_buf_mem; 697 deflate_level_bufs = sw_ch->deflate_level_bufs; 698 deflate_level_bufs[0].size = ISAL_DEF_LVL0_DEFAULT; 699 for (i = 1; i < COMP_DEFLATE_LEVEL_NUM; i++) { 700 deflate_level_bufs[i].buf = deflate_level_bufs[i - 1].buf + 701 deflate_level_bufs[i - 1].size; 702 switch (i) { 703 case 1: 704 deflate_level_bufs[i].size = ISAL_DEF_LVL1_DEFAULT; 705 break; 706 case 2: 707 deflate_level_bufs[i].size = ISAL_DEF_LVL2_DEFAULT; 708 break; 709 case 3: 710 deflate_level_bufs[i].size = ISAL_DEF_LVL3_DEFAULT; 711 break; 712 default: 713 assert(false); 714 } 715 } 716 717 isal_deflate_init(&sw_ch->stream); 718 sw_ch->stream.flush = NO_FLUSH; 719 isal_inflate_init(&sw_ch->state); 720 #endif 721 722 return 0; 723 } 724 725 static void 726 sw_accel_destroy_cb(void *io_device, void *ctx_buf) 727 { 728 struct sw_accel_io_channel *sw_ch = ctx_buf; 729 730 spdk_poller_unregister(&sw_ch->completion_poller); 731 } 732 733 static struct spdk_io_channel * 734 sw_accel_get_io_channel(void) 735 { 736 return spdk_get_io_channel(&g_sw_module); 737 } 738 739 static size_t 740 sw_accel_module_get_ctx_size(void) 741 { 742 return sizeof(struct spdk_accel_task); 743 } 744 745 static int 746 sw_accel_module_init(void) 747 { 748 spdk_io_device_register(&g_sw_module, sw_accel_create_cb, sw_accel_destroy_cb, 749 sizeof(struct sw_accel_io_channel), "sw_accel_module"); 750 751 return 0; 752 } 753 754 static void 755 sw_accel_module_fini(void *ctxt) 756 { 757 spdk_io_device_unregister(&g_sw_module, NULL); 758 spdk_accel_module_finish(); 759 } 760 761 static int 762 sw_accel_create_aes_xts(struct spdk_accel_crypto_key *key) 763 { 764 #ifdef SPDK_CONFIG_ISAL_CRYPTO 765 struct sw_accel_crypto_key_data *key_data; 766 767 key_data = calloc(1, sizeof(*key_data)); 768 if (!key_data) { 769 return -ENOMEM; 770 } 771 772 switch (key->key_size) { 773 case SPDK_ACCEL_AES_XTS_128_KEY_SIZE: 774 key_data->encrypt = isal_aes_xts_enc_128; 775 key_data->decrypt = isal_aes_xts_dec_128; 776 break; 777 case SPDK_ACCEL_AES_XTS_256_KEY_SIZE: 778 key_data->encrypt = isal_aes_xts_enc_256; 779 key_data->decrypt = isal_aes_xts_dec_256; 780 break; 781 default: 782 assert(0); 783 free(key_data); 784 return -EINVAL; 785 } 786 787 key->priv = key_data; 788 789 return 0; 790 #else 791 return -ENOTSUP; 792 #endif 793 } 794 795 static int 796 sw_accel_crypto_key_init(struct spdk_accel_crypto_key *key) 797 { 798 return sw_accel_create_aes_xts(key); 799 } 800 801 static void 802 sw_accel_crypto_key_deinit(struct spdk_accel_crypto_key *key) 803 { 804 if (!key || key->module_if != &g_sw_module || !key->priv) { 805 return; 806 } 807 808 free(key->priv); 809 } 810 811 static bool 812 sw_accel_crypto_supports_tweak_mode(enum spdk_accel_crypto_tweak_mode tweak_mode) 813 { 814 return tweak_mode == SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA; 815 } 816 817 static bool 818 sw_accel_crypto_supports_cipher(enum spdk_accel_cipher cipher, size_t key_size) 819 { 820 switch (cipher) { 821 case SPDK_ACCEL_CIPHER_AES_XTS: 822 return key_size == SPDK_ACCEL_AES_XTS_128_KEY_SIZE || key_size == SPDK_ACCEL_AES_XTS_256_KEY_SIZE; 823 default: 824 return false; 825 } 826 } 827 828 static bool 829 sw_accel_compress_supports_algo(enum spdk_accel_comp_algo algo) 830 { 831 if (algo == SPDK_ACCEL_COMP_ALGO_DEFLATE) { 832 return true; 833 } 834 835 return false; 836 } 837 838 static int 839 sw_accel_get_compress_level_range(enum spdk_accel_comp_algo algo, 840 uint32_t *min_level, uint32_t *max_level) 841 { 842 switch (algo) { 843 case SPDK_ACCEL_COMP_ALGO_DEFLATE: 844 #ifdef SPDK_CONFIG_ISAL 845 *min_level = COMP_DEFLATE_MIN_LEVEL; 846 *max_level = COMP_DEFLATE_MAX_LEVEL; 847 return 0; 848 #else 849 SPDK_ERRLOG("ISAL option is required to use software compression.\n"); 850 return -EINVAL; 851 #endif 852 default: 853 return -EINVAL; 854 } 855 } 856 857 static int 858 sw_accel_get_operation_info(enum spdk_accel_opcode opcode, 859 const struct spdk_accel_operation_exec_ctx *ctx, 860 struct spdk_accel_opcode_info *info) 861 { 862 info->required_alignment = 0; 863 864 return 0; 865 } 866 867 static struct spdk_accel_module_if g_sw_module = { 868 .module_init = sw_accel_module_init, 869 .module_fini = sw_accel_module_fini, 870 .write_config_json = NULL, 871 .get_ctx_size = sw_accel_module_get_ctx_size, 872 .name = "software", 873 .priority = SPDK_ACCEL_SW_PRIORITY, 874 .supports_opcode = sw_accel_supports_opcode, 875 .get_io_channel = sw_accel_get_io_channel, 876 .submit_tasks = sw_accel_submit_tasks, 877 .crypto_key_init = sw_accel_crypto_key_init, 878 .crypto_key_deinit = sw_accel_crypto_key_deinit, 879 .crypto_supports_tweak_mode = sw_accel_crypto_supports_tweak_mode, 880 .crypto_supports_cipher = sw_accel_crypto_supports_cipher, 881 .compress_supports_algo = sw_accel_compress_supports_algo, 882 .get_compress_level_range = sw_accel_get_compress_level_range, 883 .get_operation_info = sw_accel_get_operation_info, 884 }; 885 886 SPDK_ACCEL_MODULE_REGISTER(sw, &g_sw_module) 887