1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_common.h> 6 #include <rte_hexdump.h> 7 #include <rte_cryptodev.h> 8 #include <cryptodev_pmd.h> 9 #include <bus_vdev_driver.h> 10 #include <rte_malloc.h> 11 #include <rte_cpuflags.h> 12 13 #include <openssl/cmac.h> 14 #include <openssl/hmac.h> 15 #include <openssl/evp.h> 16 #include <openssl/ec.h> 17 18 #include "openssl_pmd_private.h" 19 #include "compat.h" 20 21 #define DES_BLOCK_SIZE 8 22 23 static uint8_t cryptodev_driver_id; 24 25 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 26 static HMAC_CTX *HMAC_CTX_new(void) 27 { 28 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 29 30 if (ctx != NULL) 31 HMAC_CTX_init(ctx); 32 return ctx; 33 } 34 35 static void HMAC_CTX_free(HMAC_CTX *ctx) 36 { 37 if (ctx != NULL) { 38 HMAC_CTX_cleanup(ctx); 39 OPENSSL_free(ctx); 40 } 41 } 42 #endif 43 44 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 45 46 #include <openssl/provider.h> 47 #include <openssl/core_names.h> 48 #include <openssl/param_build.h> 49 50 #define MAX_OSSL_ALGO_NAME_SIZE 16 51 52 OSSL_PROVIDER *legacy; 53 OSSL_PROVIDER *deflt; 54 55 static void ossl_legacy_provider_load(void) 56 { 57 /* Load Multiple providers into the default (NULL) library context */ 58 legacy = OSSL_PROVIDER_load(NULL, "legacy"); 59 if (legacy == NULL) { 60 OPENSSL_LOG(ERR, "Failed to load Legacy provider"); 61 return; 62 } 63 64 deflt = OSSL_PROVIDER_load(NULL, "default"); 65 if (deflt == NULL) { 66 OPENSSL_LOG(ERR, "Failed to load Default provider"); 67 OSSL_PROVIDER_unload(legacy); 68 return; 69 } 70 } 71 72 static void ossl_legacy_provider_unload(void) 73 { 74 OSSL_PROVIDER_unload(legacy); 75 OSSL_PROVIDER_unload(deflt); 76 } 77 78 static __rte_always_inline const char * 79 digest_name_get(enum rte_crypto_auth_algorithm algo) 80 { 81 switch (algo) { 82 case RTE_CRYPTO_AUTH_MD5_HMAC: 83 return OSSL_DIGEST_NAME_MD5; 84 case RTE_CRYPTO_AUTH_SHA1_HMAC: 85 return OSSL_DIGEST_NAME_SHA1; 86 case RTE_CRYPTO_AUTH_SHA224_HMAC: 87 return OSSL_DIGEST_NAME_SHA2_224; 88 case RTE_CRYPTO_AUTH_SHA256_HMAC: 89 return OSSL_DIGEST_NAME_SHA2_256; 90 case RTE_CRYPTO_AUTH_SHA384_HMAC: 91 return OSSL_DIGEST_NAME_SHA2_384; 92 case RTE_CRYPTO_AUTH_SHA512_HMAC: 93 return OSSL_DIGEST_NAME_SHA2_512; 94 default: 95 return NULL; 96 } 97 } 98 #endif 99 100 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 101 102 /*----------------------------------------------------------------------------*/ 103 104 /** 105 * Increment counter by 1 106 * Counter is 64 bit array, big-endian 107 */ 108 static void 109 ctr_inc(uint8_t *ctr) 110 { 111 uint64_t *ctr64 = (uint64_t *)ctr; 112 113 *ctr64 = __builtin_bswap64(*ctr64); 114 (*ctr64)++; 115 *ctr64 = __builtin_bswap64(*ctr64); 116 } 117 118 /* 119 *------------------------------------------------------------------------------ 120 * Session Prepare 121 *------------------------------------------------------------------------------ 122 */ 123 124 /** Get xform chain order */ 125 static enum openssl_chain_order 126 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 127 { 128 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 129 130 if (xform != NULL) { 131 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 132 if (xform->next == NULL) 133 res = OPENSSL_CHAIN_ONLY_AUTH; 134 else if (xform->next->type == 135 RTE_CRYPTO_SYM_XFORM_CIPHER) 136 res = OPENSSL_CHAIN_AUTH_CIPHER; 137 } 138 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 139 if (xform->next == NULL) 140 res = OPENSSL_CHAIN_ONLY_CIPHER; 141 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 142 res = OPENSSL_CHAIN_CIPHER_AUTH; 143 } 144 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 145 res = OPENSSL_CHAIN_COMBINED; 146 } 147 148 return res; 149 } 150 151 /** Get session cipher key from input cipher key */ 152 static void 153 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key) 154 { 155 memcpy(session_key, input_key, keylen); 156 } 157 158 /** Get key ede 24 bytes standard from input key */ 159 static int 160 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede) 161 { 162 int res = 0; 163 164 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 165 switch (keylen) { 166 case 24: 167 memcpy(key_ede, key, 24); 168 break; 169 case 16: 170 /* K3 = K1 */ 171 memcpy(key_ede, key, 16); 172 memcpy(key_ede + 16, key, 8); 173 break; 174 case 8: 175 /* K1 = K2 = K3 (DES compatibility) */ 176 memcpy(key_ede, key, 8); 177 memcpy(key_ede + 8, key, 8); 178 memcpy(key_ede + 16, key, 8); 179 break; 180 default: 181 OPENSSL_LOG(ERR, "Unsupported key size"); 182 res = -EINVAL; 183 } 184 185 return res; 186 } 187 188 /** Get adequate openssl function for input cipher algorithm */ 189 static uint8_t 190 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 191 const EVP_CIPHER **algo) 192 { 193 int res = 0; 194 195 if (algo != NULL) { 196 switch (sess_algo) { 197 case RTE_CRYPTO_CIPHER_3DES_CBC: 198 switch (keylen) { 199 case 8: 200 *algo = EVP_des_cbc(); 201 break; 202 case 16: 203 *algo = EVP_des_ede_cbc(); 204 break; 205 case 24: 206 *algo = EVP_des_ede3_cbc(); 207 break; 208 default: 209 res = -EINVAL; 210 } 211 break; 212 case RTE_CRYPTO_CIPHER_3DES_CTR: 213 break; 214 case RTE_CRYPTO_CIPHER_AES_CBC: 215 switch (keylen) { 216 case 16: 217 *algo = EVP_aes_128_cbc(); 218 break; 219 case 24: 220 *algo = EVP_aes_192_cbc(); 221 break; 222 case 32: 223 *algo = EVP_aes_256_cbc(); 224 break; 225 default: 226 res = -EINVAL; 227 } 228 break; 229 case RTE_CRYPTO_CIPHER_AES_CTR: 230 switch (keylen) { 231 case 16: 232 *algo = EVP_aes_128_ctr(); 233 break; 234 case 24: 235 *algo = EVP_aes_192_ctr(); 236 break; 237 case 32: 238 *algo = EVP_aes_256_ctr(); 239 break; 240 default: 241 res = -EINVAL; 242 } 243 break; 244 default: 245 res = -EINVAL; 246 break; 247 } 248 } else { 249 res = -EINVAL; 250 } 251 252 return res; 253 } 254 255 /** Get adequate openssl function for input auth algorithm */ 256 static uint8_t 257 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 258 const EVP_MD **algo) 259 { 260 int res = 0; 261 262 if (algo != NULL) { 263 switch (sessalgo) { 264 case RTE_CRYPTO_AUTH_MD5: 265 case RTE_CRYPTO_AUTH_MD5_HMAC: 266 *algo = EVP_md5(); 267 break; 268 case RTE_CRYPTO_AUTH_SHA1: 269 case RTE_CRYPTO_AUTH_SHA1_HMAC: 270 *algo = EVP_sha1(); 271 break; 272 case RTE_CRYPTO_AUTH_SHA224: 273 case RTE_CRYPTO_AUTH_SHA224_HMAC: 274 *algo = EVP_sha224(); 275 break; 276 case RTE_CRYPTO_AUTH_SHA256: 277 case RTE_CRYPTO_AUTH_SHA256_HMAC: 278 *algo = EVP_sha256(); 279 break; 280 case RTE_CRYPTO_AUTH_SHA384: 281 case RTE_CRYPTO_AUTH_SHA384_HMAC: 282 *algo = EVP_sha384(); 283 break; 284 case RTE_CRYPTO_AUTH_SHA512: 285 case RTE_CRYPTO_AUTH_SHA512_HMAC: 286 *algo = EVP_sha512(); 287 break; 288 default: 289 res = -EINVAL; 290 break; 291 } 292 } else { 293 res = -EINVAL; 294 } 295 296 return res; 297 } 298 299 /** Get adequate openssl function for input cipher algorithm */ 300 static uint8_t 301 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 302 const EVP_CIPHER **algo) 303 { 304 int res = 0; 305 306 if (algo != NULL) { 307 switch (sess_algo) { 308 case RTE_CRYPTO_AEAD_AES_GCM: 309 switch (keylen) { 310 case 16: 311 *algo = EVP_aes_128_gcm(); 312 break; 313 case 24: 314 *algo = EVP_aes_192_gcm(); 315 break; 316 case 32: 317 *algo = EVP_aes_256_gcm(); 318 break; 319 default: 320 res = -EINVAL; 321 } 322 break; 323 case RTE_CRYPTO_AEAD_AES_CCM: 324 switch (keylen) { 325 case 16: 326 *algo = EVP_aes_128_ccm(); 327 break; 328 case 24: 329 *algo = EVP_aes_192_ccm(); 330 break; 331 case 32: 332 *algo = EVP_aes_256_ccm(); 333 break; 334 default: 335 res = -EINVAL; 336 } 337 break; 338 default: 339 res = -EINVAL; 340 break; 341 } 342 } else { 343 res = -EINVAL; 344 } 345 346 return res; 347 } 348 349 /* Set session AEAD encryption parameters */ 350 static int 351 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 352 enum rte_crypto_aead_algorithm algo, 353 uint8_t tag_len, const uint8_t *key, 354 EVP_CIPHER_CTX **ctx) 355 { 356 int iv_type = 0; 357 unsigned int do_ccm; 358 359 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 360 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 361 362 /* Select AEAD algo */ 363 switch (algo) { 364 case RTE_CRYPTO_AEAD_AES_GCM: 365 iv_type = EVP_CTRL_GCM_SET_IVLEN; 366 if (tag_len != 16) 367 return -EINVAL; 368 do_ccm = 0; 369 break; 370 case RTE_CRYPTO_AEAD_AES_CCM: 371 iv_type = EVP_CTRL_CCM_SET_IVLEN; 372 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 373 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 374 return -EINVAL; 375 do_ccm = 1; 376 break; 377 default: 378 return -ENOTSUP; 379 } 380 381 sess->cipher.mode = OPENSSL_CIPHER_LIB; 382 *ctx = EVP_CIPHER_CTX_new(); 383 384 if (get_aead_algo(algo, sess->cipher.key.length, 385 &sess->cipher.evp_algo) != 0) 386 return -EINVAL; 387 388 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 389 390 sess->chain_order = OPENSSL_CHAIN_COMBINED; 391 392 if (EVP_EncryptInit_ex(*ctx, sess->cipher.evp_algo, 393 NULL, NULL, NULL) <= 0) 394 return -EINVAL; 395 396 if (EVP_CIPHER_CTX_ctrl(*ctx, iv_type, sess->iv.length, 397 NULL) <= 0) 398 return -EINVAL; 399 400 if (do_ccm) 401 EVP_CIPHER_CTX_ctrl(*ctx, EVP_CTRL_CCM_SET_TAG, 402 tag_len, NULL); 403 404 if (EVP_EncryptInit_ex(*ctx, NULL, NULL, key, NULL) <= 0) 405 return -EINVAL; 406 407 return 0; 408 } 409 410 /* Set session AEAD decryption parameters */ 411 static int 412 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 413 enum rte_crypto_aead_algorithm algo, 414 uint8_t tag_len, const uint8_t *key, 415 EVP_CIPHER_CTX **ctx) 416 { 417 int iv_type = 0; 418 unsigned int do_ccm = 0; 419 420 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 421 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 422 423 /* Select AEAD algo */ 424 switch (algo) { 425 case RTE_CRYPTO_AEAD_AES_GCM: 426 iv_type = EVP_CTRL_GCM_SET_IVLEN; 427 if (tag_len != 16) 428 return -EINVAL; 429 break; 430 case RTE_CRYPTO_AEAD_AES_CCM: 431 iv_type = EVP_CTRL_CCM_SET_IVLEN; 432 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 433 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 434 return -EINVAL; 435 do_ccm = 1; 436 break; 437 default: 438 return -ENOTSUP; 439 } 440 441 sess->cipher.mode = OPENSSL_CIPHER_LIB; 442 *ctx = EVP_CIPHER_CTX_new(); 443 444 if (get_aead_algo(algo, sess->cipher.key.length, 445 &sess->cipher.evp_algo) != 0) 446 return -EINVAL; 447 448 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 449 450 sess->chain_order = OPENSSL_CHAIN_COMBINED; 451 452 if (EVP_DecryptInit_ex(*ctx, sess->cipher.evp_algo, 453 NULL, NULL, NULL) <= 0) 454 return -EINVAL; 455 456 if (EVP_CIPHER_CTX_ctrl(*ctx, iv_type, 457 sess->iv.length, NULL) <= 0) 458 return -EINVAL; 459 460 if (do_ccm) 461 EVP_CIPHER_CTX_ctrl(*ctx, EVP_CTRL_CCM_SET_TAG, 462 tag_len, NULL); 463 464 if (EVP_DecryptInit_ex(*ctx, NULL, NULL, key, NULL) <= 0) 465 return -EINVAL; 466 467 return 0; 468 } 469 470 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30200000L) 471 static int openssl_aesni_ctx_clone(EVP_CIPHER_CTX **dest, 472 struct openssl_session *sess) 473 { 474 /* OpenSSL versions 3.0.0 <= V < 3.2.0 have no dupctx() implementation 475 * for AES-GCM and AES-CCM. In this case, we have to create new empty 476 * contexts and initialise, as we did the original context. 477 */ 478 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) 479 sess->aead_algo = RTE_CRYPTO_AEAD_AES_GCM; 480 481 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 482 return openssl_set_sess_aead_enc_param(sess, sess->aead_algo, 483 sess->auth.digest_length, sess->cipher.key.data, 484 dest); 485 else 486 return openssl_set_sess_aead_dec_param(sess, sess->aead_algo, 487 sess->auth.digest_length, sess->cipher.key.data, 488 dest); 489 } 490 #endif 491 492 /** Set session cipher parameters */ 493 static int 494 openssl_set_session_cipher_parameters(struct openssl_session *sess, 495 const struct rte_crypto_sym_xform *xform) 496 { 497 /* Select cipher direction */ 498 sess->cipher.direction = xform->cipher.op; 499 /* Select cipher key */ 500 sess->cipher.key.length = xform->cipher.key.length; 501 502 /* Set IV parameters */ 503 sess->iv.offset = xform->cipher.iv.offset; 504 sess->iv.length = xform->cipher.iv.length; 505 506 /* Select cipher algo */ 507 switch (xform->cipher.algo) { 508 case RTE_CRYPTO_CIPHER_3DES_CBC: 509 case RTE_CRYPTO_CIPHER_AES_CBC: 510 case RTE_CRYPTO_CIPHER_AES_CTR: 511 sess->cipher.mode = OPENSSL_CIPHER_LIB; 512 sess->cipher.algo = xform->cipher.algo; 513 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 514 515 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 516 &sess->cipher.evp_algo) != 0) 517 return -EINVAL; 518 519 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 520 sess->cipher.key.data); 521 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 522 if (EVP_EncryptInit_ex(sess->cipher.ctx, 523 sess->cipher.evp_algo, 524 NULL, xform->cipher.key.data, 525 NULL) != 1) { 526 return -EINVAL; 527 } 528 } else if (sess->cipher.direction == 529 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 530 if (EVP_DecryptInit_ex(sess->cipher.ctx, 531 sess->cipher.evp_algo, 532 NULL, xform->cipher.key.data, 533 NULL) != 1) { 534 return -EINVAL; 535 } 536 } 537 538 break; 539 540 case RTE_CRYPTO_CIPHER_3DES_CTR: 541 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 542 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 543 544 if (get_cipher_key_ede(xform->cipher.key.data, 545 sess->cipher.key.length, 546 sess->cipher.key.data) != 0) 547 return -EINVAL; 548 549 550 /* We use 3DES encryption also for decryption. 551 * IV is not important for 3DES ECB. 552 */ 553 if (EVP_EncryptInit_ex(sess->cipher.ctx, EVP_des_ede3_ecb(), 554 NULL, sess->cipher.key.data, NULL) != 1) 555 return -EINVAL; 556 557 break; 558 559 case RTE_CRYPTO_CIPHER_DES_CBC: 560 sess->cipher.algo = xform->cipher.algo; 561 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 562 sess->cipher.evp_algo = EVP_des_cbc(); 563 564 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 565 sess->cipher.key.data); 566 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 567 if (EVP_EncryptInit_ex(sess->cipher.ctx, 568 sess->cipher.evp_algo, 569 NULL, xform->cipher.key.data, 570 NULL) != 1) { 571 return -EINVAL; 572 } 573 } else if (sess->cipher.direction == 574 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 575 if (EVP_DecryptInit_ex(sess->cipher.ctx, 576 sess->cipher.evp_algo, 577 NULL, xform->cipher.key.data, 578 NULL) != 1) { 579 return -EINVAL; 580 } 581 } 582 583 break; 584 585 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 586 sess->cipher.algo = xform->cipher.algo; 587 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 588 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 589 sess->cipher.evp_algo = EVP_des_cbc(); 590 591 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 592 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 593 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 594 NULL, xform->cipher.key.data, 0) != 1) 595 return -EINVAL; 596 597 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 598 sess->cipher.key.data); 599 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 600 if (EVP_EncryptInit_ex(sess->cipher.ctx, 601 sess->cipher.evp_algo, 602 NULL, xform->cipher.key.data, 603 NULL) != 1) { 604 return -EINVAL; 605 } 606 } else if (sess->cipher.direction == 607 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 608 if (EVP_DecryptInit_ex(sess->cipher.ctx, 609 sess->cipher.evp_algo, 610 NULL, xform->cipher.key.data, 611 NULL) != 1) { 612 return -EINVAL; 613 } 614 } 615 616 break; 617 default: 618 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 619 return -ENOTSUP; 620 } 621 622 EVP_CIPHER_CTX_set_padding(sess->cipher.ctx, 0); 623 624 return 0; 625 } 626 627 /* Set session auth parameters */ 628 static int 629 openssl_set_session_auth_parameters(struct openssl_session *sess, 630 const struct rte_crypto_sym_xform *xform) 631 { 632 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 633 char algo_name[MAX_OSSL_ALGO_NAME_SIZE]; 634 OSSL_PARAM params[2]; 635 const char *algo; 636 EVP_MAC *mac; 637 # endif 638 /* Select auth generate/verify */ 639 sess->auth.operation = xform->auth.op; 640 sess->auth.algo = xform->auth.algo; 641 642 sess->auth.digest_length = xform->auth.digest_length; 643 644 /* Select auth algo */ 645 switch (xform->auth.algo) { 646 case RTE_CRYPTO_AUTH_AES_GMAC: 647 /* 648 * OpenSSL requires GMAC to be a GCM operation 649 * with no cipher data length 650 */ 651 sess->cipher.key.length = xform->auth.key.length; 652 653 /* Set IV parameters */ 654 sess->iv.offset = xform->auth.iv.offset; 655 sess->iv.length = xform->auth.iv.length; 656 657 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 658 return openssl_set_sess_aead_enc_param(sess, 659 RTE_CRYPTO_AEAD_AES_GCM, 660 xform->auth.digest_length, 661 xform->auth.key.data, 662 &sess->cipher.ctx); 663 else 664 return openssl_set_sess_aead_dec_param(sess, 665 RTE_CRYPTO_AEAD_AES_GCM, 666 xform->auth.digest_length, 667 xform->auth.key.data, 668 &sess->cipher.ctx); 669 break; 670 671 case RTE_CRYPTO_AUTH_MD5: 672 case RTE_CRYPTO_AUTH_SHA1: 673 case RTE_CRYPTO_AUTH_SHA224: 674 case RTE_CRYPTO_AUTH_SHA256: 675 case RTE_CRYPTO_AUTH_SHA384: 676 case RTE_CRYPTO_AUTH_SHA512: 677 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 678 if (get_auth_algo(xform->auth.algo, 679 &sess->auth.auth.evp_algo) != 0) 680 return -EINVAL; 681 sess->auth.auth.ctx = EVP_MD_CTX_create(); 682 break; 683 684 case RTE_CRYPTO_AUTH_AES_CMAC: 685 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 686 if (xform->auth.key.length == 16) 687 algo = SN_aes_128_cbc; 688 else if (xform->auth.key.length == 24) 689 algo = SN_aes_192_cbc; 690 else if (xform->auth.key.length == 32) 691 algo = SN_aes_256_cbc; 692 else 693 return -EINVAL; 694 695 rte_memcpy(algo_name, algo, strlen(algo) + 1); 696 params[0] = OSSL_PARAM_construct_utf8_string( 697 OSSL_MAC_PARAM_CIPHER, algo_name, 0); 698 params[1] = OSSL_PARAM_construct_end(); 699 700 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 701 mac = EVP_MAC_fetch(NULL, OSSL_MAC_NAME_CMAC, NULL); 702 sess->auth.cmac.ctx = EVP_MAC_CTX_new(mac); 703 EVP_MAC_free(mac); 704 705 if (EVP_MAC_init(sess->auth.cmac.ctx, 706 xform->auth.key.data, 707 xform->auth.key.length, 708 params) != 1) 709 return -EINVAL; 710 # else 711 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 712 sess->auth.cmac.ctx = CMAC_CTX_new(); 713 if (get_cipher_algo(RTE_CRYPTO_CIPHER_AES_CBC, 714 xform->auth.key.length, 715 &sess->auth.cmac.evp_algo) != 0) 716 return -EINVAL; 717 if (CMAC_Init(sess->auth.cmac.ctx, 718 xform->auth.key.data, 719 xform->auth.key.length, 720 sess->auth.cmac.evp_algo, NULL) != 1) 721 return -EINVAL; 722 # endif 723 break; 724 725 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 726 case RTE_CRYPTO_AUTH_MD5_HMAC: 727 case RTE_CRYPTO_AUTH_SHA1_HMAC: 728 case RTE_CRYPTO_AUTH_SHA224_HMAC: 729 case RTE_CRYPTO_AUTH_SHA256_HMAC: 730 case RTE_CRYPTO_AUTH_SHA384_HMAC: 731 case RTE_CRYPTO_AUTH_SHA512_HMAC: 732 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 733 734 algo = digest_name_get(xform->auth.algo); 735 if (!algo) 736 return -EINVAL; 737 strlcpy(algo_name, algo, sizeof(algo_name)); 738 739 mac = EVP_MAC_fetch(NULL, "HMAC", NULL); 740 sess->auth.hmac.ctx = EVP_MAC_CTX_new(mac); 741 EVP_MAC_free(mac); 742 if (get_auth_algo(xform->auth.algo, 743 &sess->auth.hmac.evp_algo) != 0) 744 return -EINVAL; 745 746 params[0] = OSSL_PARAM_construct_utf8_string("digest", 747 algo_name, 0); 748 params[1] = OSSL_PARAM_construct_end(); 749 if (EVP_MAC_init(sess->auth.hmac.ctx, 750 xform->auth.key.data, 751 xform->auth.key.length, 752 params) != 1) 753 return -EINVAL; 754 break; 755 # else 756 case RTE_CRYPTO_AUTH_MD5_HMAC: 757 case RTE_CRYPTO_AUTH_SHA1_HMAC: 758 case RTE_CRYPTO_AUTH_SHA224_HMAC: 759 case RTE_CRYPTO_AUTH_SHA256_HMAC: 760 case RTE_CRYPTO_AUTH_SHA384_HMAC: 761 case RTE_CRYPTO_AUTH_SHA512_HMAC: 762 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 763 sess->auth.hmac.ctx = HMAC_CTX_new(); 764 if (get_auth_algo(xform->auth.algo, 765 &sess->auth.hmac.evp_algo) != 0) 766 return -EINVAL; 767 768 if (HMAC_Init_ex(sess->auth.hmac.ctx, 769 xform->auth.key.data, 770 xform->auth.key.length, 771 sess->auth.hmac.evp_algo, NULL) != 1) 772 return -EINVAL; 773 break; 774 # endif 775 default: 776 return -ENOTSUP; 777 } 778 779 return 0; 780 } 781 782 /* Set session AEAD parameters */ 783 static int 784 openssl_set_session_aead_parameters(struct openssl_session *sess, 785 const struct rte_crypto_sym_xform *xform) 786 { 787 /* Select cipher key */ 788 sess->cipher.key.length = xform->aead.key.length; 789 790 /* Set IV parameters */ 791 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 792 /* 793 * For AES-CCM, the actual IV is placed 794 * one byte after the start of the IV field, 795 * according to the API. 796 */ 797 sess->iv.offset = xform->aead.iv.offset + 1; 798 else 799 sess->iv.offset = xform->aead.iv.offset; 800 801 sess->iv.length = xform->aead.iv.length; 802 803 sess->auth.aad_length = xform->aead.aad_length; 804 sess->auth.digest_length = xform->aead.digest_length; 805 806 sess->aead_algo = xform->aead.algo; 807 /* Select cipher direction */ 808 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 809 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 810 xform->aead.digest_length, xform->aead.key.data, 811 &sess->cipher.ctx); 812 else 813 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 814 xform->aead.digest_length, xform->aead.key.data, 815 &sess->cipher.ctx); 816 } 817 818 /** Parse crypto xform chain and set private session parameters */ 819 int 820 openssl_set_session_parameters(struct openssl_session *sess, 821 const struct rte_crypto_sym_xform *xform, 822 uint16_t nb_queue_pairs) 823 { 824 const struct rte_crypto_sym_xform *cipher_xform = NULL; 825 const struct rte_crypto_sym_xform *auth_xform = NULL; 826 const struct rte_crypto_sym_xform *aead_xform = NULL; 827 int ret; 828 829 sess->chain_order = openssl_get_chain_order(xform); 830 switch (sess->chain_order) { 831 case OPENSSL_CHAIN_ONLY_CIPHER: 832 cipher_xform = xform; 833 break; 834 case OPENSSL_CHAIN_ONLY_AUTH: 835 auth_xform = xform; 836 break; 837 case OPENSSL_CHAIN_CIPHER_AUTH: 838 cipher_xform = xform; 839 auth_xform = xform->next; 840 break; 841 case OPENSSL_CHAIN_AUTH_CIPHER: 842 auth_xform = xform; 843 cipher_xform = xform->next; 844 break; 845 case OPENSSL_CHAIN_COMBINED: 846 aead_xform = xform; 847 break; 848 default: 849 return -EINVAL; 850 } 851 852 /* Default IV length = 0 */ 853 sess->iv.length = 0; 854 855 /* cipher_xform must be check before auth_xform */ 856 if (cipher_xform) { 857 ret = openssl_set_session_cipher_parameters( 858 sess, cipher_xform); 859 if (ret != 0) { 860 OPENSSL_LOG(ERR, 861 "Invalid/unsupported cipher parameters"); 862 return ret; 863 } 864 } 865 866 if (auth_xform) { 867 ret = openssl_set_session_auth_parameters(sess, auth_xform); 868 if (ret != 0) { 869 OPENSSL_LOG(ERR, 870 "Invalid/unsupported auth parameters"); 871 return ret; 872 } 873 } 874 875 if (aead_xform) { 876 ret = openssl_set_session_aead_parameters(sess, aead_xform); 877 if (ret != 0) { 878 OPENSSL_LOG(ERR, 879 "Invalid/unsupported AEAD parameters"); 880 return ret; 881 } 882 } 883 884 /* 885 * With only one queue pair, the array of copies is not needed. 886 * Otherwise, one entry per queue pair is required. 887 */ 888 sess->ctx_copies_len = nb_queue_pairs > 1 ? nb_queue_pairs : 0; 889 890 return 0; 891 } 892 893 /** Reset private session parameters */ 894 void 895 openssl_reset_session(struct openssl_session *sess) 896 { 897 /* Free all the qp_ctx entries. */ 898 for (uint16_t i = 0; i < sess->ctx_copies_len; i++) { 899 if (sess->qp_ctx[i].cipher != NULL) { 900 EVP_CIPHER_CTX_free(sess->qp_ctx[i].cipher); 901 sess->qp_ctx[i].cipher = NULL; 902 } 903 904 switch (sess->auth.mode) { 905 case OPENSSL_AUTH_AS_AUTH: 906 EVP_MD_CTX_destroy(sess->qp_ctx[i].auth); 907 sess->qp_ctx[i].auth = NULL; 908 break; 909 case OPENSSL_AUTH_AS_HMAC: 910 free_hmac_ctx(sess->qp_ctx[i].hmac); 911 sess->qp_ctx[i].hmac = NULL; 912 break; 913 case OPENSSL_AUTH_AS_CMAC: 914 free_cmac_ctx(sess->qp_ctx[i].cmac); 915 sess->qp_ctx[i].cmac = NULL; 916 break; 917 } 918 } 919 920 EVP_CIPHER_CTX_free(sess->cipher.ctx); 921 922 switch (sess->auth.mode) { 923 case OPENSSL_AUTH_AS_AUTH: 924 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 925 break; 926 case OPENSSL_AUTH_AS_HMAC: 927 free_hmac_ctx(sess->auth.hmac.ctx); 928 break; 929 case OPENSSL_AUTH_AS_CMAC: 930 free_cmac_ctx(sess->auth.cmac.ctx); 931 break; 932 } 933 934 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 935 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 936 } 937 938 /** Provide session for operation */ 939 static void * 940 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 941 { 942 struct openssl_session *sess = NULL; 943 struct openssl_asym_session *asym_sess = NULL; 944 945 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 946 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 947 /* get existing session */ 948 if (likely(op->sym->session != NULL)) 949 sess = CRYPTODEV_GET_SYM_SESS_PRIV( 950 op->sym->session); 951 } else { 952 if (likely(op->asym->session != NULL)) 953 asym_sess = (struct openssl_asym_session *) 954 op->asym->session->sess_private_data; 955 if (asym_sess == NULL) 956 op->status = 957 RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 958 return asym_sess; 959 } 960 } else { 961 struct rte_cryptodev_sym_session *_sess; 962 /* sessionless asymmetric not supported */ 963 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) 964 return NULL; 965 966 /* provide internal session */ 967 rte_mempool_get(qp->sess_mp, (void **)&_sess); 968 969 if (_sess == NULL) 970 return NULL; 971 972 sess = (struct openssl_session *)_sess->driver_priv_data; 973 974 if (unlikely(openssl_set_session_parameters(sess, 975 op->sym->xform, 1) != 0)) { 976 rte_mempool_put(qp->sess_mp, _sess); 977 sess = NULL; 978 } 979 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 980 981 } 982 983 if (sess == NULL) 984 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 985 986 return sess; 987 } 988 989 /* 990 *------------------------------------------------------------------------------ 991 * Process Operations 992 *------------------------------------------------------------------------------ 993 */ 994 static inline int 995 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 996 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 997 { 998 struct rte_mbuf *m; 999 int dstlen; 1000 int l, n = srclen; 1001 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 1002 1003 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1004 m = m->next) 1005 offset -= rte_pktmbuf_data_len(m); 1006 1007 if (m == 0) 1008 return -1; 1009 1010 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1011 if (inplace) 1012 *dst = src; 1013 1014 l = rte_pktmbuf_data_len(m) - offset; 1015 if (srclen <= l) { 1016 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 1017 return -1; 1018 *dst += l; 1019 return 0; 1020 } 1021 1022 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1023 return -1; 1024 1025 *dst += dstlen; 1026 n -= l; 1027 1028 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1029 uint8_t diff = l - dstlen, rem; 1030 1031 src = rte_pktmbuf_mtod(m, uint8_t *); 1032 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 1033 if (diff && inplace) { 1034 rem = RTE_MIN(l, 1035 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 1036 if (EVP_EncryptUpdate(ctx, temp, 1037 &dstlen, src, rem) <= 0) 1038 return -1; 1039 n -= rem; 1040 rte_memcpy(*dst, temp, diff); 1041 rte_memcpy(src, temp + diff, rem); 1042 src += rem; 1043 l -= rem; 1044 } 1045 if (inplace) 1046 *dst = src; 1047 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1048 return -1; 1049 *dst += dstlen; 1050 n -= l; 1051 } 1052 1053 return 0; 1054 } 1055 1056 static inline int 1057 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 1058 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 1059 { 1060 struct rte_mbuf *m; 1061 int dstlen; 1062 int l, n = srclen; 1063 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 1064 1065 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1066 m = m->next) 1067 offset -= rte_pktmbuf_data_len(m); 1068 1069 if (m == 0) 1070 return -1; 1071 1072 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1073 if (inplace) 1074 *dst = src; 1075 1076 l = rte_pktmbuf_data_len(m) - offset; 1077 if (srclen <= l) { 1078 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 1079 return -1; 1080 *dst += l; 1081 return 0; 1082 } 1083 1084 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1085 return -1; 1086 1087 *dst += dstlen; 1088 n -= l; 1089 1090 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1091 uint8_t diff = l - dstlen, rem; 1092 1093 src = rte_pktmbuf_mtod(m, uint8_t *); 1094 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 1095 if (diff && inplace) { 1096 rem = RTE_MIN(l, 1097 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 1098 if (EVP_DecryptUpdate(ctx, temp, 1099 &dstlen, src, rem) <= 0) 1100 return -1; 1101 n -= rem; 1102 rte_memcpy(*dst, temp, diff); 1103 rte_memcpy(src, temp + diff, rem); 1104 src += rem; 1105 l -= rem; 1106 } 1107 if (inplace) 1108 *dst = src; 1109 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1110 return -1; 1111 *dst += dstlen; 1112 n -= l; 1113 } 1114 1115 return 0; 1116 } 1117 1118 /** Process standard openssl cipher encryption */ 1119 static int 1120 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1121 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1122 uint8_t inplace) 1123 { 1124 int totlen; 1125 1126 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1127 goto process_cipher_encrypt_err; 1128 1129 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1130 srclen, ctx, inplace)) 1131 goto process_cipher_encrypt_err; 1132 1133 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 1134 goto process_cipher_encrypt_err; 1135 1136 return 0; 1137 1138 process_cipher_encrypt_err: 1139 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 1140 return -EINVAL; 1141 } 1142 1143 /** Process standard openssl cipher encryption */ 1144 static int 1145 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 1146 uint8_t *iv, int srclen, 1147 EVP_CIPHER_CTX *ctx) 1148 { 1149 uint8_t i; 1150 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 1151 int encrypted_ivlen; 1152 1153 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 1154 iv, DES_BLOCK_SIZE) <= 0) 1155 goto process_cipher_encrypt_err; 1156 1157 for (i = 0; i < srclen; i++) 1158 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 1159 1160 return 0; 1161 1162 process_cipher_encrypt_err: 1163 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 1164 return -EINVAL; 1165 } 1166 /** Process standard openssl cipher decryption */ 1167 static int 1168 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1169 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1170 uint8_t inplace) 1171 { 1172 int totlen; 1173 1174 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1175 goto process_cipher_decrypt_err; 1176 1177 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1178 srclen, ctx, inplace)) 1179 goto process_cipher_decrypt_err; 1180 1181 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 1182 goto process_cipher_decrypt_err; 1183 return 0; 1184 1185 process_cipher_decrypt_err: 1186 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 1187 return -EINVAL; 1188 } 1189 1190 /** Process cipher des 3 ctr encryption, decryption algorithm */ 1191 static int 1192 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 1193 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) 1194 { 1195 uint8_t ebuf[8], ctr[8]; 1196 int unused, n; 1197 struct rte_mbuf *m; 1198 uint8_t *src; 1199 int l; 1200 1201 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1202 m = m->next) 1203 offset -= rte_pktmbuf_data_len(m); 1204 1205 if (m == 0) 1206 goto process_cipher_des3ctr_err; 1207 1208 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1209 l = rte_pktmbuf_data_len(m) - offset; 1210 1211 memcpy(ctr, iv, 8); 1212 1213 for (n = 0; n < srclen; n++) { 1214 if (n % 8 == 0) { 1215 if (EVP_EncryptUpdate(ctx, 1216 (unsigned char *)&ebuf, &unused, 1217 (const unsigned char *)&ctr, 8) <= 0) 1218 goto process_cipher_des3ctr_err; 1219 ctr_inc(ctr); 1220 } 1221 dst[n] = *(src++) ^ ebuf[n % 8]; 1222 1223 l--; 1224 if (!l) { 1225 m = m->next; 1226 if (m) { 1227 src = rte_pktmbuf_mtod(m, uint8_t *); 1228 l = rte_pktmbuf_data_len(m); 1229 } 1230 } 1231 } 1232 1233 return 0; 1234 1235 process_cipher_des3ctr_err: 1236 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 1237 return -EINVAL; 1238 } 1239 1240 /** Process AES-GCM encrypt algorithm */ 1241 static int 1242 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1243 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1244 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1245 { 1246 int len = 0; 1247 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1248 int unused = 0; 1249 uint8_t empty[] = {}; 1250 #endif 1251 1252 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1253 goto process_auth_encryption_gcm_err; 1254 1255 if (aadlen > 0) 1256 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1257 goto process_auth_encryption_gcm_err; 1258 1259 if (srclen > 0) 1260 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1261 srclen, ctx, 0)) 1262 goto process_auth_encryption_gcm_err; 1263 1264 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1265 /* Workaround open ssl bug in version less then 1.0.1f */ 1266 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1267 goto process_auth_encryption_gcm_err; 1268 #endif 1269 1270 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1271 goto process_auth_encryption_gcm_err; 1272 1273 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1274 goto process_auth_encryption_gcm_err; 1275 1276 return 0; 1277 1278 process_auth_encryption_gcm_err: 1279 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1280 return -EINVAL; 1281 } 1282 1283 /** Process AES-CCM encrypt algorithm */ 1284 static int 1285 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1286 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1287 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1288 { 1289 int len = 0; 1290 1291 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1292 goto process_auth_encryption_ccm_err; 1293 1294 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1295 goto process_auth_encryption_ccm_err; 1296 1297 if (aadlen > 0) 1298 /* 1299 * For AES-CCM, the actual AAD is placed 1300 * 18 bytes after the start of the AAD field, 1301 * according to the API. 1302 */ 1303 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1304 goto process_auth_encryption_ccm_err; 1305 1306 if (srclen >= 0) 1307 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1308 srclen, ctx, 0)) 1309 goto process_auth_encryption_ccm_err; 1310 1311 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1312 goto process_auth_encryption_ccm_err; 1313 1314 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1315 goto process_auth_encryption_ccm_err; 1316 1317 return 0; 1318 1319 process_auth_encryption_ccm_err: 1320 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1321 return -EINVAL; 1322 } 1323 1324 /** Process AES-GCM decrypt algorithm */ 1325 static int 1326 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1327 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1328 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1329 { 1330 int len = 0; 1331 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1332 int unused = 0; 1333 uint8_t empty[] = {}; 1334 #endif 1335 1336 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1337 goto process_auth_decryption_gcm_err; 1338 1339 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1340 goto process_auth_decryption_gcm_err; 1341 1342 if (aadlen > 0) 1343 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1344 goto process_auth_decryption_gcm_err; 1345 1346 if (srclen > 0) 1347 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1348 srclen, ctx, 0)) 1349 goto process_auth_decryption_gcm_err; 1350 1351 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1352 /* Workaround open ssl bug in version less then 1.0.1f */ 1353 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1354 goto process_auth_decryption_gcm_err; 1355 #endif 1356 1357 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1358 return -EFAULT; 1359 1360 return 0; 1361 1362 process_auth_decryption_gcm_err: 1363 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1364 return -EINVAL; 1365 } 1366 1367 /** Process AES-CCM decrypt algorithm */ 1368 static int 1369 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1370 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1371 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1372 EVP_CIPHER_CTX *ctx) 1373 { 1374 int len = 0; 1375 1376 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1377 goto process_auth_decryption_ccm_err; 1378 1379 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1380 goto process_auth_decryption_ccm_err; 1381 1382 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1383 goto process_auth_decryption_ccm_err; 1384 1385 if (aadlen > 0) 1386 /* 1387 * For AES-CCM, the actual AAD is placed 1388 * 18 bytes after the start of the AAD field, 1389 * according to the API. 1390 */ 1391 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1392 goto process_auth_decryption_ccm_err; 1393 1394 if (srclen >= 0) 1395 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1396 srclen, ctx, 0)) 1397 return -EFAULT; 1398 1399 return 0; 1400 1401 process_auth_decryption_ccm_err: 1402 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1403 return -EINVAL; 1404 } 1405 1406 /** Process standard openssl auth algorithms */ 1407 static int 1408 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1409 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1410 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1411 { 1412 size_t dstlen; 1413 struct rte_mbuf *m; 1414 int l, n = srclen; 1415 uint8_t *src; 1416 1417 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1418 m = m->next) 1419 offset -= rte_pktmbuf_data_len(m); 1420 1421 if (m == 0) 1422 goto process_auth_err; 1423 1424 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1425 goto process_auth_err; 1426 1427 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1428 1429 l = rte_pktmbuf_data_len(m) - offset; 1430 if (srclen <= l) { 1431 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1432 goto process_auth_err; 1433 goto process_auth_final; 1434 } 1435 1436 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1437 goto process_auth_err; 1438 1439 n -= l; 1440 1441 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1442 src = rte_pktmbuf_mtod(m, uint8_t *); 1443 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1444 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1445 goto process_auth_err; 1446 n -= l; 1447 } 1448 1449 process_auth_final: 1450 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1451 goto process_auth_err; 1452 return 0; 1453 1454 process_auth_err: 1455 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1456 return -EINVAL; 1457 } 1458 1459 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1460 /** Process standard openssl auth algorithms with hmac/cmac */ 1461 static int 1462 process_openssl_auth_mac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1463 int srclen, EVP_MAC_CTX *ctx) 1464 { 1465 size_t dstlen; 1466 struct rte_mbuf *m; 1467 int l, n = srclen; 1468 uint8_t *src; 1469 1470 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1471 m = m->next) 1472 offset -= rte_pktmbuf_data_len(m); 1473 1474 if (m == 0) 1475 goto process_auth_err; 1476 1477 if (EVP_MAC_init(ctx, NULL, 0, NULL) <= 0) 1478 goto process_auth_err; 1479 1480 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1481 1482 l = rte_pktmbuf_data_len(m) - offset; 1483 if (srclen <= l) { 1484 if (EVP_MAC_update(ctx, (unsigned char *)src, srclen) != 1) 1485 goto process_auth_err; 1486 goto process_auth_final; 1487 } 1488 1489 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1490 goto process_auth_err; 1491 1492 n -= l; 1493 1494 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1495 src = rte_pktmbuf_mtod(m, uint8_t *); 1496 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1497 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1498 goto process_auth_err; 1499 n -= l; 1500 } 1501 1502 process_auth_final: 1503 if (EVP_MAC_final(ctx, dst, &dstlen, DIGEST_LENGTH_MAX) != 1) 1504 goto process_auth_err; 1505 1506 return 0; 1507 1508 process_auth_err: 1509 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1510 return -EINVAL; 1511 } 1512 # else 1513 /** Process standard openssl auth algorithms with hmac */ 1514 static int 1515 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1516 int srclen, HMAC_CTX *ctx) 1517 { 1518 unsigned int dstlen; 1519 struct rte_mbuf *m; 1520 int l, n = srclen; 1521 uint8_t *src; 1522 1523 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1524 m = m->next) 1525 offset -= rte_pktmbuf_data_len(m); 1526 1527 if (m == 0) 1528 goto process_auth_err; 1529 1530 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1531 1532 l = rte_pktmbuf_data_len(m) - offset; 1533 if (srclen <= l) { 1534 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1535 goto process_auth_err; 1536 goto process_auth_final; 1537 } 1538 1539 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1540 goto process_auth_err; 1541 1542 n -= l; 1543 1544 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1545 src = rte_pktmbuf_mtod(m, uint8_t *); 1546 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1547 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1548 goto process_auth_err; 1549 n -= l; 1550 } 1551 1552 process_auth_final: 1553 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1554 goto process_auth_err; 1555 1556 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1557 goto process_auth_err; 1558 1559 return 0; 1560 1561 process_auth_err: 1562 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1563 return -EINVAL; 1564 } 1565 1566 /** Process standard openssl auth algorithms with cmac */ 1567 static int 1568 process_openssl_auth_cmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1569 int srclen, CMAC_CTX *ctx) 1570 { 1571 unsigned int dstlen; 1572 struct rte_mbuf *m; 1573 int l, n = srclen; 1574 uint8_t *src; 1575 1576 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1577 m = m->next) 1578 offset -= rte_pktmbuf_data_len(m); 1579 1580 if (m == 0) 1581 goto process_auth_err; 1582 1583 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1584 1585 l = rte_pktmbuf_data_len(m) - offset; 1586 if (srclen <= l) { 1587 if (CMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1588 goto process_auth_err; 1589 goto process_auth_final; 1590 } 1591 1592 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1593 goto process_auth_err; 1594 1595 n -= l; 1596 1597 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1598 src = rte_pktmbuf_mtod(m, uint8_t *); 1599 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1600 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1601 goto process_auth_err; 1602 n -= l; 1603 } 1604 1605 process_auth_final: 1606 if (CMAC_Final(ctx, dst, (size_t *)&dstlen) != 1) 1607 goto process_auth_err; 1608 1609 CMAC_CTX_cleanup(ctx); 1610 1611 return 0; 1612 1613 process_auth_err: 1614 OPENSSL_LOG(ERR, "Process openssl cmac auth failed"); 1615 return -EINVAL; 1616 } 1617 # endif 1618 /*----------------------------------------------------------------------------*/ 1619 1620 static inline EVP_CIPHER_CTX * 1621 get_local_cipher_ctx(struct openssl_session *sess, struct openssl_qp *qp) 1622 { 1623 /* If the array is not being used, just return the main context. */ 1624 if (sess->ctx_copies_len == 0) 1625 return sess->cipher.ctx; 1626 1627 EVP_CIPHER_CTX **lctx = &sess->qp_ctx[qp->id].cipher; 1628 1629 if (unlikely(*lctx == NULL)) { 1630 #if OPENSSL_VERSION_NUMBER >= 0x30200000L 1631 /* EVP_CIPHER_CTX_dup() added in OSSL 3.2 */ 1632 *lctx = EVP_CIPHER_CTX_dup(sess->cipher.ctx); 1633 return *lctx; 1634 #elif OPENSSL_VERSION_NUMBER >= 0x30000000L 1635 if (sess->chain_order == OPENSSL_CHAIN_COMBINED) { 1636 /* AESNI special-cased to use openssl_aesni_ctx_clone() 1637 * to allow for working around lack of 1638 * EVP_CIPHER_CTX_copy support for 3.0.0 <= OSSL Version 1639 * < 3.2.0. 1640 */ 1641 if (openssl_aesni_ctx_clone(lctx, sess) != 0) 1642 *lctx = NULL; 1643 return *lctx; 1644 } 1645 #endif 1646 1647 *lctx = EVP_CIPHER_CTX_new(); 1648 EVP_CIPHER_CTX_copy(*lctx, sess->cipher.ctx); 1649 } 1650 1651 return *lctx; 1652 } 1653 1654 static inline EVP_MD_CTX * 1655 get_local_auth_ctx(struct openssl_session *sess, struct openssl_qp *qp) 1656 { 1657 /* If the array is not being used, just return the main context. */ 1658 if (sess->ctx_copies_len == 0) 1659 return sess->auth.auth.ctx; 1660 1661 EVP_MD_CTX **lctx = &sess->qp_ctx[qp->id].auth; 1662 1663 if (unlikely(*lctx == NULL)) { 1664 #if OPENSSL_VERSION_NUMBER >= 0x30100000L 1665 /* EVP_MD_CTX_dup() added in OSSL 3.1 */ 1666 *lctx = EVP_MD_CTX_dup(sess->auth.auth.ctx); 1667 #else 1668 *lctx = EVP_MD_CTX_new(); 1669 EVP_MD_CTX_copy(*lctx, sess->auth.auth.ctx); 1670 #endif 1671 } 1672 1673 return *lctx; 1674 } 1675 1676 #if OPENSSL_VERSION_NUMBER >= 0x30000000L 1677 static inline EVP_MAC_CTX * 1678 #else 1679 static inline HMAC_CTX * 1680 #endif 1681 get_local_hmac_ctx(struct openssl_session *sess, struct openssl_qp *qp) 1682 { 1683 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30003000L) 1684 /* For OpenSSL versions 3.0.0 <= v < 3.0.3, re-initing of 1685 * EVP_MAC_CTXs is broken, and doesn't actually reset their 1686 * state. This was fixed in OSSL commit c9ddc5af5199 ("Avoid 1687 * undefined behavior of provided macs on EVP_MAC 1688 * reinitialization"). In cases where the fix is not present, 1689 * fall back to duplicating the context every buffer as a 1690 * workaround, at the cost of performance. 1691 */ 1692 RTE_SET_USED(qp); 1693 return EVP_MAC_CTX_dup(sess->auth.hmac.ctx); 1694 #else 1695 if (sess->ctx_copies_len == 0) 1696 return sess->auth.hmac.ctx; 1697 1698 #if OPENSSL_VERSION_NUMBER >= 0x30000000L 1699 EVP_MAC_CTX **lctx = 1700 #else 1701 HMAC_CTX **lctx = 1702 #endif 1703 &sess->qp_ctx[qp->id].hmac; 1704 1705 if (unlikely(*lctx == NULL)) { 1706 #if OPENSSL_VERSION_NUMBER >= 0x30000000L 1707 *lctx = EVP_MAC_CTX_dup(sess->auth.hmac.ctx); 1708 #else 1709 *lctx = HMAC_CTX_new(); 1710 HMAC_CTX_copy(*lctx, sess->auth.hmac.ctx); 1711 #endif 1712 } 1713 1714 return *lctx; 1715 #endif 1716 } 1717 1718 #if OPENSSL_VERSION_NUMBER >= 0x30000000L 1719 static inline EVP_MAC_CTX * 1720 #else 1721 static inline CMAC_CTX * 1722 #endif 1723 get_local_cmac_ctx(struct openssl_session *sess, struct openssl_qp *qp) 1724 { 1725 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30003000L) 1726 /* For OpenSSL versions 3.0.0 <= v < 3.0.3, re-initing of 1727 * EVP_MAC_CTXs is broken, and doesn't actually reset their 1728 * state. This was fixed in OSSL commit c9ddc5af5199 ("Avoid 1729 * undefined behavior of provided macs on EVP_MAC 1730 * reinitialization"). In cases where the fix is not present, 1731 * fall back to duplicating the context every buffer as a 1732 * workaround, at the cost of performance. 1733 */ 1734 RTE_SET_USED(qp); 1735 return EVP_MAC_CTX_dup(sess->auth.cmac.ctx); 1736 #else 1737 if (sess->ctx_copies_len == 0) 1738 return sess->auth.cmac.ctx; 1739 1740 #if OPENSSL_VERSION_NUMBER >= 0x30000000L 1741 EVP_MAC_CTX **lctx = 1742 #else 1743 CMAC_CTX **lctx = 1744 #endif 1745 &sess->qp_ctx[qp->id].cmac; 1746 1747 if (unlikely(*lctx == NULL)) { 1748 #if OPENSSL_VERSION_NUMBER >= 0x30000000L 1749 *lctx = EVP_MAC_CTX_dup(sess->auth.cmac.ctx); 1750 #else 1751 *lctx = CMAC_CTX_new(); 1752 CMAC_CTX_copy(*lctx, sess->auth.cmac.ctx); 1753 #endif 1754 } 1755 1756 return *lctx; 1757 #endif 1758 } 1759 1760 /** Process auth/cipher combined operation */ 1761 static void 1762 process_openssl_combined_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1763 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1764 struct rte_mbuf *mbuf_dst) 1765 { 1766 /* cipher */ 1767 uint8_t *dst = NULL, *iv, *tag, *aad; 1768 int srclen, aadlen, status = -1; 1769 uint32_t offset; 1770 uint8_t taglen; 1771 1772 /* 1773 * Segmented destination buffer is not supported for 1774 * encryption/decryption 1775 */ 1776 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1777 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1778 return; 1779 } 1780 1781 EVP_CIPHER_CTX *ctx = get_local_cipher_ctx(sess, qp); 1782 1783 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1784 sess->iv.offset); 1785 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1786 srclen = 0; 1787 offset = op->sym->auth.data.offset; 1788 aadlen = op->sym->auth.data.length; 1789 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1790 op->sym->auth.data.offset); 1791 tag = op->sym->auth.digest.data; 1792 if (tag == NULL) 1793 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1794 offset + aadlen); 1795 } else { 1796 srclen = op->sym->aead.data.length; 1797 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1798 op->sym->aead.data.offset); 1799 offset = op->sym->aead.data.offset; 1800 aad = op->sym->aead.aad.data; 1801 aadlen = sess->auth.aad_length; 1802 tag = op->sym->aead.digest.data; 1803 if (tag == NULL) 1804 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1805 offset + srclen); 1806 } 1807 1808 taglen = sess->auth.digest_length; 1809 1810 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1811 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1812 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1813 status = process_openssl_auth_encryption_gcm( 1814 mbuf_src, offset, srclen, 1815 aad, aadlen, iv, 1816 dst, tag, ctx); 1817 else 1818 status = process_openssl_auth_encryption_ccm( 1819 mbuf_src, offset, srclen, 1820 aad, aadlen, iv, 1821 dst, tag, taglen, ctx); 1822 1823 } else { 1824 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1825 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1826 status = process_openssl_auth_decryption_gcm( 1827 mbuf_src, offset, srclen, 1828 aad, aadlen, iv, 1829 dst, tag, ctx); 1830 else 1831 status = process_openssl_auth_decryption_ccm( 1832 mbuf_src, offset, srclen, 1833 aad, aadlen, iv, 1834 dst, tag, taglen, ctx); 1835 } 1836 1837 if (status != 0) { 1838 if (status == (-EFAULT) && 1839 sess->auth.operation == 1840 RTE_CRYPTO_AUTH_OP_VERIFY) 1841 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1842 else 1843 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1844 } 1845 } 1846 1847 /** Process cipher operation */ 1848 static void 1849 process_openssl_cipher_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1850 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1851 struct rte_mbuf *mbuf_dst) 1852 { 1853 uint8_t *dst, *iv; 1854 int srclen, status; 1855 uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0; 1856 1857 /* 1858 * Segmented OOP destination buffer is not supported for encryption/ 1859 * decryption. In case of des3ctr, even inplace segmented buffers are 1860 * not supported. 1861 */ 1862 if (!rte_pktmbuf_is_contiguous(mbuf_dst) && 1863 (!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) { 1864 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1865 return; 1866 } 1867 1868 srclen = op->sym->cipher.data.length; 1869 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1870 op->sym->cipher.data.offset); 1871 1872 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1873 sess->iv.offset); 1874 1875 EVP_CIPHER_CTX *ctx = get_local_cipher_ctx(sess, qp); 1876 1877 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1878 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1879 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1880 op->sym->cipher.data.offset, iv, 1881 srclen, ctx, inplace); 1882 else 1883 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1884 op->sym->cipher.data.offset, iv, 1885 srclen, ctx, inplace); 1886 else 1887 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1888 op->sym->cipher.data.offset, iv, srclen, ctx); 1889 1890 if (status != 0) 1891 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1892 } 1893 1894 /** Process cipher operation */ 1895 static void 1896 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1897 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1898 struct rte_mbuf *mbuf_dst) 1899 { 1900 uint8_t *src, *dst, *iv; 1901 uint8_t block_size, last_block_len; 1902 int srclen, status = 0; 1903 1904 srclen = op->sym->cipher.data.length; 1905 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1906 op->sym->cipher.data.offset); 1907 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1908 op->sym->cipher.data.offset); 1909 1910 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1911 sess->iv.offset); 1912 1913 block_size = DES_BLOCK_SIZE; 1914 1915 last_block_len = srclen % block_size; 1916 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1917 /* Encrypt only with ECB mode XOR IV */ 1918 if (srclen < block_size) { 1919 status = process_openssl_cipher_bpi_encrypt(src, dst, 1920 iv, srclen, 1921 sess->cipher.bpi_ctx); 1922 } else { 1923 srclen -= last_block_len; 1924 /* Encrypt with the block aligned stream with CBC mode */ 1925 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1926 op->sym->cipher.data.offset, iv, 1927 srclen, sess->cipher.ctx, 0); 1928 if (last_block_len) { 1929 /* Point at last block */ 1930 dst += srclen; 1931 /* 1932 * IV is the last encrypted block from 1933 * the previous operation 1934 */ 1935 iv = dst - block_size; 1936 src += srclen; 1937 srclen = last_block_len; 1938 /* Encrypt the last frame with ECB mode */ 1939 status |= process_openssl_cipher_bpi_encrypt(src, 1940 dst, iv, 1941 srclen, sess->cipher.bpi_ctx); 1942 } 1943 } 1944 } else { 1945 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1946 if (srclen < block_size) { 1947 status = process_openssl_cipher_bpi_encrypt(src, dst, 1948 iv, 1949 srclen, 1950 sess->cipher.bpi_ctx); 1951 } else { 1952 if (last_block_len) { 1953 /* Point at last block */ 1954 dst += srclen - last_block_len; 1955 src += srclen - last_block_len; 1956 /* 1957 * IV is the last full block 1958 */ 1959 iv = src - block_size; 1960 /* 1961 * Decrypt the last frame with ECB mode 1962 * (encrypt, as it is the same operation) 1963 */ 1964 status = process_openssl_cipher_bpi_encrypt(src, 1965 dst, iv, 1966 last_block_len, sess->cipher.bpi_ctx); 1967 /* Prepare parameters for CBC mode op */ 1968 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1969 sess->iv.offset); 1970 dst += last_block_len - srclen; 1971 srclen -= last_block_len; 1972 } 1973 1974 /* Decrypt with CBC mode */ 1975 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1976 op->sym->cipher.data.offset, iv, 1977 srclen, sess->cipher.ctx, 0); 1978 } 1979 } 1980 1981 if (status != 0) 1982 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1983 } 1984 1985 /** Process auth operation */ 1986 static void 1987 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1988 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1989 struct rte_mbuf *mbuf_dst) 1990 { 1991 uint8_t *dst; 1992 int srclen, status; 1993 EVP_MD_CTX *ctx_a; 1994 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1995 EVP_MAC_CTX *ctx_h; 1996 EVP_MAC_CTX *ctx_c; 1997 # else 1998 HMAC_CTX *ctx_h; 1999 CMAC_CTX *ctx_c; 2000 # endif 2001 2002 srclen = op->sym->auth.data.length; 2003 2004 dst = qp->temp_digest; 2005 2006 switch (sess->auth.mode) { 2007 case OPENSSL_AUTH_AS_AUTH: 2008 ctx_a = get_local_auth_ctx(sess, qp); 2009 status = process_openssl_auth(mbuf_src, dst, 2010 op->sym->auth.data.offset, NULL, NULL, srclen, 2011 ctx_a, sess->auth.auth.evp_algo); 2012 break; 2013 case OPENSSL_AUTH_AS_HMAC: 2014 ctx_h = get_local_hmac_ctx(sess, qp); 2015 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 2016 status = process_openssl_auth_mac(mbuf_src, dst, 2017 op->sym->auth.data.offset, srclen, 2018 ctx_h); 2019 # else 2020 status = process_openssl_auth_hmac(mbuf_src, dst, 2021 op->sym->auth.data.offset, srclen, 2022 ctx_h); 2023 # endif 2024 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30003000L) 2025 EVP_MAC_CTX_free(ctx_h); 2026 #endif 2027 break; 2028 case OPENSSL_AUTH_AS_CMAC: 2029 ctx_c = get_local_cmac_ctx(sess, qp); 2030 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 2031 status = process_openssl_auth_mac(mbuf_src, dst, 2032 op->sym->auth.data.offset, srclen, 2033 ctx_c); 2034 # else 2035 status = process_openssl_auth_cmac(mbuf_src, dst, 2036 op->sym->auth.data.offset, srclen, 2037 ctx_c); 2038 # endif 2039 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30003000L) 2040 EVP_MAC_CTX_free(ctx_c); 2041 #endif 2042 break; 2043 default: 2044 status = -1; 2045 break; 2046 } 2047 2048 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 2049 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data, 2050 sess->auth.digest_length) != 0) { 2051 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 2052 } 2053 } else { 2054 uint8_t *auth_dst; 2055 2056 auth_dst = op->sym->auth.digest.data; 2057 if (auth_dst == NULL) 2058 auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 2059 op->sym->auth.data.offset + 2060 op->sym->auth.data.length); 2061 memcpy(auth_dst, dst, sess->auth.digest_length); 2062 } 2063 2064 if (status != 0) 2065 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 2066 } 2067 2068 /* process dsa sign operation */ 2069 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2070 static int 2071 process_openssl_dsa_sign_op_evp(struct rte_crypto_op *cop, 2072 struct openssl_asym_session *sess) 2073 { 2074 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2075 EVP_PKEY_CTX *dsa_ctx = NULL; 2076 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 2077 EVP_PKEY *pkey = NULL; 2078 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 2079 OSSL_PARAM *params = NULL; 2080 2081 size_t outlen; 2082 unsigned char *dsa_sign_data; 2083 const unsigned char *dsa_sign_data_p; 2084 int ret = -1; 2085 2086 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2087 params = OSSL_PARAM_BLD_to_param(param_bld); 2088 if (!params) { 2089 OSSL_PARAM_BLD_free(param_bld); 2090 return -1; 2091 } 2092 2093 if (key_ctx == NULL 2094 || EVP_PKEY_fromdata_init(key_ctx) <= 0 2095 || EVP_PKEY_fromdata(key_ctx, &pkey, 2096 EVP_PKEY_KEYPAIR, params) <= 0) 2097 goto err_dsa_sign; 2098 2099 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 2100 if (!dsa_ctx) 2101 goto err_dsa_sign; 2102 2103 if (EVP_PKEY_sign_init(dsa_ctx) <= 0) 2104 goto err_dsa_sign; 2105 2106 if (EVP_PKEY_sign(dsa_ctx, NULL, &outlen, op->message.data, 2107 op->message.length) <= 0) 2108 goto err_dsa_sign; 2109 2110 if (outlen <= 0) 2111 goto err_dsa_sign; 2112 2113 dsa_sign_data = OPENSSL_malloc(outlen); 2114 if (!dsa_sign_data) 2115 goto err_dsa_sign; 2116 2117 if (EVP_PKEY_sign(dsa_ctx, dsa_sign_data, &outlen, op->message.data, 2118 op->message.length) <= 0) { 2119 OPENSSL_free(dsa_sign_data); 2120 goto err_dsa_sign; 2121 } 2122 2123 dsa_sign_data_p = (const unsigned char *)dsa_sign_data; 2124 DSA_SIG *sign = d2i_DSA_SIG(NULL, &dsa_sign_data_p, outlen); 2125 if (!sign) { 2126 OPENSSL_LOG(ERR, "%s:%d", __func__, __LINE__); 2127 OPENSSL_free(dsa_sign_data); 2128 goto err_dsa_sign; 2129 } else { 2130 const BIGNUM *r = NULL, *s = NULL; 2131 get_dsa_sign(sign, &r, &s); 2132 2133 op->r.length = BN_bn2bin(r, op->r.data); 2134 op->s.length = BN_bn2bin(s, op->s.data); 2135 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2136 } 2137 2138 ret = 0; 2139 DSA_SIG_free(sign); 2140 OPENSSL_free(dsa_sign_data); 2141 2142 err_dsa_sign: 2143 if (params) 2144 OSSL_PARAM_free(params); 2145 EVP_PKEY_CTX_free(key_ctx); 2146 EVP_PKEY_CTX_free(dsa_ctx); 2147 EVP_PKEY_free(pkey); 2148 return ret; 2149 } 2150 2151 /* process dsa verify operation */ 2152 static int 2153 process_openssl_dsa_verify_op_evp(struct rte_crypto_op *cop, 2154 struct openssl_asym_session *sess) 2155 { 2156 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2157 DSA_SIG *sign = DSA_SIG_new(); 2158 BIGNUM *r = NULL, *s = NULL; 2159 BIGNUM *pub_key = NULL; 2160 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 2161 OSSL_PARAM *params = NULL; 2162 EVP_PKEY *pkey = NULL; 2163 EVP_PKEY_CTX *dsa_ctx = NULL; 2164 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 2165 unsigned char *dsa_sig = NULL; 2166 size_t sig_len; 2167 int ret = -1; 2168 2169 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2170 if (!param_bld) { 2171 OPENSSL_LOG(ERR, " %s:%d", __func__, __LINE__); 2172 return -1; 2173 } 2174 2175 r = BN_bin2bn(op->r.data, op->r.length, r); 2176 s = BN_bin2bn(op->s.data, op->s.length, s); 2177 pub_key = BN_bin2bn(op->y.data, op->y.length, pub_key); 2178 if (!r || !s || !pub_key) { 2179 BN_free(r); 2180 BN_free(s); 2181 BN_free(pub_key); 2182 OSSL_PARAM_BLD_free(param_bld); 2183 goto err_dsa_verify; 2184 } 2185 2186 set_dsa_sign(sign, r, s); 2187 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PUB_KEY, pub_key)) { 2188 OSSL_PARAM_BLD_free(param_bld); 2189 goto err_dsa_verify; 2190 } 2191 2192 params = OSSL_PARAM_BLD_to_param(param_bld); 2193 if (!params) { 2194 OSSL_PARAM_BLD_free(param_bld); 2195 goto err_dsa_verify; 2196 } 2197 2198 if (key_ctx == NULL 2199 || EVP_PKEY_fromdata_init(key_ctx) <= 0 2200 || EVP_PKEY_fromdata(key_ctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2201 goto err_dsa_verify; 2202 2203 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 2204 if (!dsa_ctx) 2205 goto err_dsa_verify; 2206 2207 if (!sign) 2208 goto err_dsa_verify; 2209 2210 sig_len = i2d_DSA_SIG(sign, &dsa_sig); 2211 if (EVP_PKEY_verify_init(dsa_ctx) <= 0) 2212 goto err_dsa_verify; 2213 2214 ret = EVP_PKEY_verify(dsa_ctx, dsa_sig, sig_len, 2215 op->message.data, op->message.length); 2216 if (ret == 1) { 2217 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2218 ret = 0; 2219 } 2220 2221 OPENSSL_free(dsa_sig); 2222 err_dsa_verify: 2223 if (sign) 2224 DSA_SIG_free(sign); 2225 if (params) 2226 OSSL_PARAM_free(params); 2227 EVP_PKEY_CTX_free(key_ctx); 2228 EVP_PKEY_CTX_free(dsa_ctx); 2229 2230 BN_free(pub_key); 2231 EVP_PKEY_free(pkey); 2232 2233 return ret; 2234 } 2235 #else 2236 static int 2237 process_openssl_dsa_sign_op(struct rte_crypto_op *cop, 2238 struct openssl_asym_session *sess) 2239 { 2240 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2241 DSA *dsa = sess->u.s.dsa; 2242 DSA_SIG *sign = NULL; 2243 2244 sign = DSA_do_sign(op->message.data, 2245 op->message.length, 2246 dsa); 2247 2248 if (sign == NULL) { 2249 OPENSSL_LOG(ERR, "%s:%d", __func__, __LINE__); 2250 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2251 } else { 2252 const BIGNUM *r = NULL, *s = NULL; 2253 get_dsa_sign(sign, &r, &s); 2254 2255 op->r.length = BN_bn2bin(r, op->r.data); 2256 op->s.length = BN_bn2bin(s, op->s.data); 2257 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2258 } 2259 2260 DSA_SIG_free(sign); 2261 2262 return 0; 2263 } 2264 2265 /* process dsa verify operation */ 2266 static int 2267 process_openssl_dsa_verify_op(struct rte_crypto_op *cop, 2268 struct openssl_asym_session *sess) 2269 { 2270 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2271 DSA *dsa = sess->u.s.dsa; 2272 int ret; 2273 DSA_SIG *sign = DSA_SIG_new(); 2274 BIGNUM *r = NULL, *s = NULL; 2275 BIGNUM *pub_key = NULL; 2276 2277 if (sign == NULL) { 2278 OPENSSL_LOG(ERR, " %s:%d", __func__, __LINE__); 2279 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2280 return -1; 2281 } 2282 2283 r = BN_bin2bn(op->r.data, 2284 op->r.length, 2285 r); 2286 s = BN_bin2bn(op->s.data, 2287 op->s.length, 2288 s); 2289 pub_key = BN_bin2bn(op->y.data, 2290 op->y.length, 2291 pub_key); 2292 if (!r || !s || !pub_key) { 2293 BN_free(r); 2294 BN_free(s); 2295 BN_free(pub_key); 2296 2297 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2298 return -1; 2299 } 2300 set_dsa_sign(sign, r, s); 2301 set_dsa_pub_key(dsa, pub_key); 2302 2303 ret = DSA_do_verify(op->message.data, 2304 op->message.length, 2305 sign, 2306 dsa); 2307 2308 if (ret != 1) 2309 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2310 else 2311 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2312 2313 DSA_SIG_free(sign); 2314 2315 return 0; 2316 } 2317 #endif 2318 2319 /* process dh operation */ 2320 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2321 static int 2322 process_openssl_dh_op_evp(struct rte_crypto_op *cop, 2323 struct openssl_asym_session *sess) 2324 { 2325 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2326 OSSL_PARAM_BLD *param_bld = sess->u.dh.param_bld; 2327 OSSL_PARAM_BLD *param_bld_peer = sess->u.dh.param_bld_peer; 2328 OSSL_PARAM *params = NULL; 2329 EVP_PKEY *dhpkey = NULL; 2330 EVP_PKEY *peerkey = NULL; 2331 BIGNUM *priv_key = NULL; 2332 BIGNUM *pub_key = NULL; 2333 int ret = -1; 2334 2335 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2336 EVP_PKEY_CTX *dh_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2337 if (dh_ctx == NULL || param_bld == NULL) 2338 return ret; 2339 2340 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2341 OSSL_PARAM *params_peer = NULL; 2342 2343 if (!param_bld_peer) 2344 return ret; 2345 2346 pub_key = BN_bin2bn(op->pub_key.data, op->pub_key.length, 2347 pub_key); 2348 if (pub_key == NULL) { 2349 OSSL_PARAM_BLD_free(param_bld_peer); 2350 return ret; 2351 } 2352 2353 if (!OSSL_PARAM_BLD_push_BN(param_bld_peer, OSSL_PKEY_PARAM_PUB_KEY, 2354 pub_key)) { 2355 OPENSSL_LOG(ERR, "Failed to set public key"); 2356 OSSL_PARAM_BLD_free(param_bld_peer); 2357 BN_free(pub_key); 2358 return ret; 2359 } 2360 2361 params_peer = OSSL_PARAM_BLD_to_param(param_bld_peer); 2362 if (!params_peer) { 2363 OSSL_PARAM_BLD_free(param_bld_peer); 2364 BN_free(pub_key); 2365 return ret; 2366 } 2367 2368 EVP_PKEY_CTX *peer_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2369 if (EVP_PKEY_keygen_init(peer_ctx) != 1) { 2370 OSSL_PARAM_free(params_peer); 2371 BN_free(pub_key); 2372 return ret; 2373 } 2374 2375 if (EVP_PKEY_CTX_set_params(peer_ctx, params_peer) != 1) { 2376 EVP_PKEY_CTX_free(peer_ctx); 2377 OSSL_PARAM_free(params_peer); 2378 BN_free(pub_key); 2379 return ret; 2380 } 2381 2382 if (EVP_PKEY_keygen(peer_ctx, &peerkey) != 1) { 2383 EVP_PKEY_CTX_free(peer_ctx); 2384 OSSL_PARAM_free(params_peer); 2385 BN_free(pub_key); 2386 return ret; 2387 } 2388 2389 priv_key = BN_bin2bn(op->priv_key.data, op->priv_key.length, 2390 priv_key); 2391 if (priv_key == NULL) { 2392 EVP_PKEY_CTX_free(peer_ctx); 2393 OSSL_PARAM_free(params_peer); 2394 BN_free(pub_key); 2395 return ret; 2396 } 2397 2398 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY, 2399 priv_key)) { 2400 OPENSSL_LOG(ERR, "Failed to set private key"); 2401 EVP_PKEY_CTX_free(peer_ctx); 2402 OSSL_PARAM_free(params_peer); 2403 BN_free(pub_key); 2404 BN_free(priv_key); 2405 return ret; 2406 } 2407 2408 OSSL_PARAM_free(params_peer); 2409 EVP_PKEY_CTX_free(peer_ctx); 2410 } 2411 2412 params = OSSL_PARAM_BLD_to_param(param_bld); 2413 if (!params) 2414 goto err_dh; 2415 2416 if (EVP_PKEY_keygen_init(dh_ctx) != 1) 2417 goto err_dh; 2418 2419 if (EVP_PKEY_CTX_set_params(dh_ctx, params) != 1) 2420 goto err_dh; 2421 2422 if (EVP_PKEY_keygen(dh_ctx, &dhpkey) != 1) 2423 goto err_dh; 2424 2425 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2426 OPENSSL_LOG(DEBUG, "%s:%d updated pub key", __func__, __LINE__); 2427 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PUB_KEY, &pub_key)) 2428 goto err_dh; 2429 /* output public key */ 2430 op->pub_key.length = BN_bn2bin(pub_key, op->pub_key.data); 2431 } 2432 2433 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2434 2435 OPENSSL_LOG(DEBUG, "%s:%d updated priv key", __func__, __LINE__); 2436 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PRIV_KEY, &priv_key)) 2437 goto err_dh; 2438 2439 /* provide generated private key back to user */ 2440 op->priv_key.length = BN_bn2bin(priv_key, op->priv_key.data); 2441 } 2442 2443 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2444 size_t skey_len; 2445 EVP_PKEY_CTX *sc_ctx = EVP_PKEY_CTX_new(dhpkey, NULL); 2446 if (!sc_ctx) 2447 goto err_dh; 2448 2449 if (EVP_PKEY_derive_init(sc_ctx) <= 0) { 2450 EVP_PKEY_CTX_free(sc_ctx); 2451 goto err_dh; 2452 } 2453 2454 if (!peerkey) { 2455 EVP_PKEY_CTX_free(sc_ctx); 2456 goto err_dh; 2457 } 2458 2459 if (EVP_PKEY_derive_set_peer(sc_ctx, peerkey) <= 0) { 2460 EVP_PKEY_CTX_free(sc_ctx); 2461 goto err_dh; 2462 } 2463 2464 /* Determine buffer length */ 2465 if (EVP_PKEY_derive(sc_ctx, NULL, &skey_len) <= 0) { 2466 EVP_PKEY_CTX_free(sc_ctx); 2467 goto err_dh; 2468 } 2469 2470 if (EVP_PKEY_derive(sc_ctx, op->shared_secret.data, &skey_len) <= 0) { 2471 EVP_PKEY_CTX_free(sc_ctx); 2472 goto err_dh; 2473 } 2474 2475 op->shared_secret.length = skey_len; 2476 EVP_PKEY_CTX_free(sc_ctx); 2477 } 2478 2479 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2480 ret = 0; 2481 2482 err_dh: 2483 BN_free(pub_key); 2484 BN_free(priv_key); 2485 if (params) 2486 OSSL_PARAM_free(params); 2487 EVP_PKEY_free(dhpkey); 2488 EVP_PKEY_free(peerkey); 2489 2490 EVP_PKEY_CTX_free(dh_ctx); 2491 2492 return ret; 2493 } 2494 #else 2495 static int 2496 process_openssl_dh_op(struct rte_crypto_op *cop, 2497 struct openssl_asym_session *sess) 2498 { 2499 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2500 struct rte_crypto_asym_op *asym_op = cop->asym; 2501 DH *dh_key = sess->u.dh.dh_key; 2502 BIGNUM *priv_key = NULL; 2503 int ret = 0; 2504 2505 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2506 /* compute shared secret using peer public key 2507 * and current private key 2508 * shared secret = peer_key ^ priv_key mod p 2509 */ 2510 BIGNUM *peer_key = NULL; 2511 2512 /* copy private key and peer key and compute shared secret */ 2513 peer_key = BN_bin2bn(op->pub_key.data, 2514 op->pub_key.length, 2515 peer_key); 2516 if (peer_key == NULL) { 2517 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2518 return -1; 2519 } 2520 priv_key = BN_bin2bn(op->priv_key.data, 2521 op->priv_key.length, 2522 priv_key); 2523 if (priv_key == NULL) { 2524 BN_free(peer_key); 2525 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2526 return -1; 2527 } 2528 ret = set_dh_priv_key(dh_key, priv_key); 2529 if (ret) { 2530 OPENSSL_LOG(ERR, "Failed to set private key"); 2531 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2532 BN_free(peer_key); 2533 BN_free(priv_key); 2534 return 0; 2535 } 2536 2537 ret = DH_compute_key( 2538 op->shared_secret.data, 2539 peer_key, dh_key); 2540 if (ret < 0) { 2541 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2542 BN_free(peer_key); 2543 /* priv key is already loaded into dh, 2544 * let's not free that directly here. 2545 * DH_free() will auto free it later. 2546 */ 2547 return 0; 2548 } 2549 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2550 op->shared_secret.length = ret; 2551 BN_free(peer_key); 2552 return 0; 2553 } 2554 2555 /* 2556 * other options are public and private key generations. 2557 * 2558 * if user provides private key, 2559 * then first set DH with user provided private key 2560 */ 2561 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE && 2562 op->priv_key.length) { 2563 /* generate public key using user-provided private key 2564 * pub_key = g ^ priv_key mod p 2565 */ 2566 2567 /* load private key into DH */ 2568 priv_key = BN_bin2bn(op->priv_key.data, 2569 op->priv_key.length, 2570 priv_key); 2571 if (priv_key == NULL) { 2572 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2573 return -1; 2574 } 2575 ret = set_dh_priv_key(dh_key, priv_key); 2576 if (ret) { 2577 OPENSSL_LOG(ERR, "Failed to set private key"); 2578 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2579 BN_free(priv_key); 2580 return 0; 2581 } 2582 } 2583 2584 /* generate public and private key pair. 2585 * 2586 * if private key already set, generates only public key. 2587 * 2588 * if private key is not already set, then set it to random value 2589 * and update internal private key. 2590 */ 2591 if (!DH_generate_key(dh_key)) { 2592 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2593 return 0; 2594 } 2595 2596 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2597 const BIGNUM *pub_key = NULL; 2598 2599 OPENSSL_LOG(DEBUG, "%s:%d update public key", 2600 __func__, __LINE__); 2601 2602 /* get the generated keys */ 2603 get_dh_pub_key(dh_key, &pub_key); 2604 2605 /* output public key */ 2606 op->pub_key.length = BN_bn2bin(pub_key, 2607 op->pub_key.data); 2608 } 2609 2610 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2611 const BIGNUM *priv_key = NULL; 2612 2613 OPENSSL_LOG(DEBUG, "%s:%d updated priv key", 2614 __func__, __LINE__); 2615 2616 /* get the generated keys */ 2617 get_dh_priv_key(dh_key, &priv_key); 2618 2619 /* provide generated private key back to user */ 2620 op->priv_key.length = BN_bn2bin(priv_key, 2621 op->priv_key.data); 2622 } 2623 2624 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2625 2626 return 0; 2627 } 2628 #endif 2629 2630 /* process modinv operation */ 2631 static int 2632 process_openssl_modinv_op(struct rte_crypto_op *cop, 2633 struct openssl_asym_session *sess) 2634 { 2635 struct rte_crypto_asym_op *op = cop->asym; 2636 BIGNUM *base = BN_CTX_get(sess->u.m.ctx); 2637 BIGNUM *res = BN_CTX_get(sess->u.m.ctx); 2638 2639 if (unlikely(base == NULL || res == NULL)) { 2640 BN_free(base); 2641 BN_free(res); 2642 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2643 return -1; 2644 } 2645 2646 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 2647 op->modinv.base.length, base); 2648 2649 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) { 2650 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2651 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data); 2652 } else { 2653 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2654 } 2655 2656 BN_clear(res); 2657 BN_clear(base); 2658 2659 return 0; 2660 } 2661 2662 /* process modexp operation */ 2663 static int 2664 process_openssl_modexp_op(struct rte_crypto_op *cop, 2665 struct openssl_asym_session *sess) 2666 { 2667 struct rte_crypto_asym_op *op = cop->asym; 2668 BIGNUM *base = BN_CTX_get(sess->u.e.ctx); 2669 BIGNUM *res = BN_CTX_get(sess->u.e.ctx); 2670 2671 if (unlikely(base == NULL || res == NULL)) { 2672 BN_free(base); 2673 BN_free(res); 2674 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2675 return -1; 2676 } 2677 2678 base = BN_bin2bn((const unsigned char *)op->modex.base.data, 2679 op->modex.base.length, base); 2680 2681 if (BN_mod_exp(res, base, sess->u.e.exp, 2682 sess->u.e.mod, sess->u.e.ctx)) { 2683 op->modex.result.length = BN_bn2bin(res, op->modex.result.data); 2684 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2685 } else { 2686 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2687 } 2688 2689 BN_clear(res); 2690 BN_clear(base); 2691 2692 return 0; 2693 } 2694 2695 /* process rsa operations */ 2696 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2697 static int 2698 process_openssl_rsa_op_evp(struct rte_crypto_op *cop, 2699 struct openssl_asym_session *sess) 2700 { 2701 struct rte_crypto_asym_op *op = cop->asym; 2702 uint32_t pad = (op->rsa.padding.type); 2703 uint8_t *tmp; 2704 size_t outlen = 0; 2705 int ret = -1; 2706 2707 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2708 EVP_PKEY_CTX *rsa_ctx = sess->u.r.ctx; 2709 if (!rsa_ctx) 2710 return ret; 2711 2712 switch (pad) { 2713 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2714 pad = RSA_PKCS1_PADDING; 2715 break; 2716 case RTE_CRYPTO_RSA_PADDING_NONE: 2717 pad = RSA_NO_PADDING; 2718 break; 2719 default: 2720 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2721 OPENSSL_LOG(ERR, 2722 "rsa pad type not supported %d", pad); 2723 return ret; 2724 } 2725 2726 switch (op->rsa.op_type) { 2727 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2728 if (EVP_PKEY_encrypt_init(rsa_ctx) != 1) 2729 goto err_rsa; 2730 2731 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2732 goto err_rsa; 2733 2734 if (EVP_PKEY_encrypt(rsa_ctx, NULL, &outlen, 2735 op->rsa.message.data, 2736 op->rsa.message.length) <= 0) 2737 goto err_rsa; 2738 2739 if (outlen <= 0) 2740 goto err_rsa; 2741 2742 if (EVP_PKEY_encrypt(rsa_ctx, op->rsa.cipher.data, &outlen, 2743 op->rsa.message.data, 2744 op->rsa.message.length) <= 0) 2745 goto err_rsa; 2746 op->rsa.cipher.length = outlen; 2747 2748 OPENSSL_LOG(DEBUG, 2749 "length of encrypted text %zu", outlen); 2750 break; 2751 2752 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2753 if (EVP_PKEY_decrypt_init(rsa_ctx) != 1) 2754 goto err_rsa; 2755 2756 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2757 goto err_rsa; 2758 2759 if (EVP_PKEY_decrypt(rsa_ctx, NULL, &outlen, 2760 op->rsa.cipher.data, 2761 op->rsa.cipher.length) <= 0) 2762 goto err_rsa; 2763 2764 if (outlen <= 0) 2765 goto err_rsa; 2766 2767 if (EVP_PKEY_decrypt(rsa_ctx, op->rsa.message.data, &outlen, 2768 op->rsa.cipher.data, 2769 op->rsa.cipher.length) <= 0) 2770 goto err_rsa; 2771 op->rsa.message.length = outlen; 2772 2773 OPENSSL_LOG(DEBUG, "length of decrypted text %zu", outlen); 2774 break; 2775 2776 case RTE_CRYPTO_ASYM_OP_SIGN: 2777 if (EVP_PKEY_sign_init(rsa_ctx) <= 0) 2778 goto err_rsa; 2779 2780 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2781 goto err_rsa; 2782 2783 if (EVP_PKEY_sign(rsa_ctx, NULL, &outlen, 2784 op->rsa.message.data, 2785 op->rsa.message.length) <= 0) 2786 goto err_rsa; 2787 2788 if (outlen <= 0) 2789 goto err_rsa; 2790 2791 if (EVP_PKEY_sign(rsa_ctx, op->rsa.sign.data, &outlen, 2792 op->rsa.message.data, 2793 op->rsa.message.length) <= 0) 2794 goto err_rsa; 2795 op->rsa.sign.length = outlen; 2796 break; 2797 2798 case RTE_CRYPTO_ASYM_OP_VERIFY: 2799 if (EVP_PKEY_verify_recover_init(rsa_ctx) <= 0) 2800 goto err_rsa; 2801 2802 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2803 goto err_rsa; 2804 2805 if (EVP_PKEY_verify_recover(rsa_ctx, NULL, &outlen, 2806 op->rsa.sign.data, 2807 op->rsa.sign.length) <= 0) 2808 goto err_rsa; 2809 2810 if ((outlen <= 0) || (outlen != op->rsa.sign.length)) 2811 goto err_rsa; 2812 2813 tmp = OPENSSL_malloc(outlen); 2814 if (tmp == NULL) { 2815 OPENSSL_LOG(ERR, "Memory allocation failed"); 2816 goto err_rsa; 2817 } 2818 2819 if (EVP_PKEY_verify_recover(rsa_ctx, tmp, &outlen, 2820 op->rsa.sign.data, 2821 op->rsa.sign.length) <= 0) { 2822 OPENSSL_free(tmp); 2823 goto err_rsa; 2824 } 2825 2826 OPENSSL_LOG(DEBUG, 2827 "Length of public_decrypt %zu " 2828 "length of message %zd", 2829 outlen, op->rsa.message.length); 2830 if (CRYPTO_memcmp(tmp, op->rsa.message.data, 2831 op->rsa.message.length)) { 2832 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 2833 } 2834 OPENSSL_free(tmp); 2835 break; 2836 2837 default: 2838 /* allow ops with invalid args to be pushed to 2839 * completion queue 2840 */ 2841 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2842 goto err_rsa; 2843 } 2844 2845 ret = 0; 2846 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2847 err_rsa: 2848 return ret; 2849 2850 } 2851 2852 static int 2853 process_openssl_sm2_op_evp(struct rte_crypto_op *cop, 2854 struct openssl_asym_session *sess) 2855 { 2856 EVP_PKEY_CTX *kctx = NULL, *sctx = NULL, *cctx = NULL; 2857 struct rte_crypto_asym_op *op = cop->asym; 2858 OSSL_PARAM *params = sess->u.sm2.params; 2859 EVP_MD_CTX *md_ctx = NULL; 2860 ECDSA_SIG *ec_sign = NULL; 2861 EVP_MD *check_md = NULL; 2862 EVP_PKEY *pkey = NULL; 2863 int ret = -1; 2864 2865 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2866 2867 if (cop->asym->sm2.k.data != NULL) 2868 goto err_sm2; 2869 2870 switch (op->sm2.op_type) { 2871 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2872 { 2873 OSSL_PARAM *eparams = sess->u.sm2.params; 2874 size_t output_len = 0; 2875 2876 kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_SM2, NULL); 2877 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2878 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2879 goto err_sm2; 2880 2881 cctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2882 if (!cctx) 2883 goto err_sm2; 2884 2885 if (!EVP_PKEY_encrypt_init(cctx)) 2886 goto err_sm2; 2887 2888 if (!EVP_PKEY_CTX_set_params(cctx, eparams)) 2889 goto err_sm2; 2890 2891 if (!EVP_PKEY_encrypt(cctx, op->sm2.cipher.data, &output_len, 2892 op->sm2.message.data, 2893 op->sm2.message.length)) 2894 goto err_sm2; 2895 op->sm2.cipher.length = output_len; 2896 } 2897 break; 2898 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2899 { 2900 OSSL_PARAM *eparams = sess->u.sm2.params; 2901 2902 kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_SM2, NULL); 2903 if (kctx == NULL 2904 || EVP_PKEY_fromdata_init(kctx) <= 0 2905 || EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2906 goto err_sm2; 2907 2908 cctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2909 if (!cctx) 2910 goto err_sm2; 2911 2912 if (!EVP_PKEY_decrypt_init(cctx)) 2913 goto err_sm2; 2914 2915 if (!EVP_PKEY_CTX_set_params(cctx, eparams)) 2916 goto err_sm2; 2917 2918 if (!EVP_PKEY_decrypt(cctx, op->sm2.message.data, &op->sm2.message.length, 2919 op->sm2.cipher.data, op->sm2.cipher.length)) 2920 goto err_sm2; 2921 } 2922 break; 2923 case RTE_CRYPTO_ASYM_OP_SIGN: 2924 { 2925 unsigned char signbuf[128] = {0}; 2926 const unsigned char *signptr; 2927 const BIGNUM *r, *s; 2928 size_t signlen; 2929 2930 kctx = EVP_PKEY_CTX_new_from_name(NULL, "SM2", NULL); 2931 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2932 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2933 goto err_sm2; 2934 2935 md_ctx = EVP_MD_CTX_new(); 2936 if (!md_ctx) 2937 goto err_sm2; 2938 2939 sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2940 if (!sctx) 2941 goto err_sm2; 2942 2943 EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx); 2944 2945 check_md = EVP_MD_fetch(NULL, "sm3", NULL); 2946 if (!check_md) 2947 goto err_sm2; 2948 2949 if (!EVP_DigestSignInit(md_ctx, NULL, check_md, NULL, pkey)) 2950 goto err_sm2; 2951 2952 if (EVP_PKEY_CTX_set1_id(sctx, op->sm2.id.data, op->sm2.id.length) <= 0) 2953 goto err_sm2; 2954 2955 if (!EVP_DigestSignUpdate(md_ctx, op->sm2.message.data, 2956 op->sm2.message.length)) 2957 goto err_sm2; 2958 2959 if (!EVP_DigestSignFinal(md_ctx, NULL, &signlen)) 2960 goto err_sm2; 2961 2962 if (!EVP_DigestSignFinal(md_ctx, signbuf, &signlen)) 2963 goto err_sm2; 2964 2965 signptr = signbuf; 2966 ec_sign = d2i_ECDSA_SIG(NULL, &signptr, signlen); 2967 if (!ec_sign) 2968 goto err_sm2; 2969 2970 r = ECDSA_SIG_get0_r(ec_sign); 2971 s = ECDSA_SIG_get0_s(ec_sign); 2972 if (!r || !s) 2973 goto err_sm2; 2974 2975 op->sm2.r.length = BN_num_bytes(r); 2976 op->sm2.s.length = BN_num_bytes(s); 2977 BN_bn2bin(r, op->sm2.r.data); 2978 BN_bn2bin(s, op->sm2.s.data); 2979 2980 ECDSA_SIG_free(ec_sign); 2981 } 2982 break; 2983 case RTE_CRYPTO_ASYM_OP_VERIFY: 2984 { 2985 unsigned char signbuf[128] = {0}, *signbuf_new = NULL; 2986 BIGNUM *r = NULL, *s = NULL; 2987 size_t signlen; 2988 2989 kctx = EVP_PKEY_CTX_new_from_name(NULL, "SM2", NULL); 2990 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2991 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_PUBLIC_KEY, params) <= 0) 2992 goto err_sm2; 2993 2994 if (!EVP_PKEY_is_a(pkey, "SM2")) 2995 goto err_sm2; 2996 2997 md_ctx = EVP_MD_CTX_new(); 2998 if (!md_ctx) 2999 goto err_sm2; 3000 3001 sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 3002 if (!sctx) 3003 goto err_sm2; 3004 3005 EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx); 3006 3007 check_md = EVP_MD_fetch(NULL, "sm3", NULL); 3008 if (!check_md) 3009 goto err_sm2; 3010 3011 if (!EVP_DigestVerifyInit(md_ctx, NULL, check_md, NULL, pkey)) 3012 goto err_sm2; 3013 3014 if (EVP_PKEY_CTX_set1_id(sctx, op->sm2.id.data, op->sm2.id.length) <= 0) 3015 goto err_sm2; 3016 3017 if (!EVP_DigestVerifyUpdate(md_ctx, op->sm2.message.data, 3018 op->sm2.message.length)) 3019 goto err_sm2; 3020 3021 ec_sign = ECDSA_SIG_new(); 3022 if (!ec_sign) 3023 goto err_sm2; 3024 3025 r = BN_bin2bn(op->sm2.r.data, op->sm2.r.length, r); 3026 s = BN_bin2bn(op->sm2.s.data, op->sm2.s.length, s); 3027 if (!r || !s) 3028 goto err_sm2; 3029 3030 if (!ECDSA_SIG_set0(ec_sign, r, s)) { 3031 BN_free(r); 3032 BN_free(s); 3033 goto err_sm2; 3034 } 3035 3036 r = NULL; 3037 s = NULL; 3038 3039 signbuf_new = signbuf; 3040 signlen = i2d_ECDSA_SIG(ec_sign, (unsigned char **)&signbuf_new); 3041 if (signlen <= 0) 3042 goto err_sm2; 3043 3044 if (!EVP_DigestVerifyFinal(md_ctx, signbuf_new, signlen)) 3045 goto err_sm2; 3046 3047 BN_free(r); 3048 BN_free(s); 3049 ECDSA_SIG_free(ec_sign); 3050 } 3051 break; 3052 default: 3053 /* allow ops with invalid args to be pushed to 3054 * completion queue 3055 */ 3056 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3057 goto err_sm2; 3058 } 3059 3060 ret = 0; 3061 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3062 err_sm2: 3063 EVP_MD_free(check_md); 3064 EVP_MD_CTX_free(md_ctx); 3065 3066 EVP_PKEY_CTX_free(kctx); 3067 3068 EVP_PKEY_CTX_free(sctx); 3069 3070 EVP_PKEY_CTX_free(cctx); 3071 3072 EVP_PKEY_free(pkey); 3073 3074 return ret; 3075 } 3076 3077 #else 3078 static int 3079 process_openssl_rsa_op(struct rte_crypto_op *cop, 3080 struct openssl_asym_session *sess) 3081 { 3082 int ret = 0; 3083 struct rte_crypto_asym_op *op = cop->asym; 3084 RSA *rsa = sess->u.r.rsa; 3085 uint32_t pad = (op->rsa.padding.type); 3086 uint8_t *tmp; 3087 3088 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3089 3090 switch (pad) { 3091 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 3092 pad = RSA_PKCS1_PADDING; 3093 break; 3094 case RTE_CRYPTO_RSA_PADDING_NONE: 3095 pad = RSA_NO_PADDING; 3096 break; 3097 default: 3098 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3099 OPENSSL_LOG(ERR, 3100 "rsa pad type not supported %d", pad); 3101 return 0; 3102 } 3103 3104 switch (op->rsa.op_type) { 3105 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 3106 ret = RSA_public_encrypt(op->rsa.message.length, 3107 op->rsa.message.data, 3108 op->rsa.cipher.data, 3109 rsa, 3110 pad); 3111 3112 if (ret > 0) 3113 op->rsa.cipher.length = ret; 3114 OPENSSL_LOG(DEBUG, 3115 "length of encrypted text %d", ret); 3116 break; 3117 3118 case RTE_CRYPTO_ASYM_OP_DECRYPT: 3119 ret = RSA_private_decrypt(op->rsa.cipher.length, 3120 op->rsa.cipher.data, 3121 op->rsa.message.data, 3122 rsa, 3123 pad); 3124 if (ret > 0) 3125 op->rsa.message.length = ret; 3126 break; 3127 3128 case RTE_CRYPTO_ASYM_OP_SIGN: 3129 ret = RSA_private_encrypt(op->rsa.message.length, 3130 op->rsa.message.data, 3131 op->rsa.sign.data, 3132 rsa, 3133 pad); 3134 if (ret > 0) 3135 op->rsa.sign.length = ret; 3136 break; 3137 3138 case RTE_CRYPTO_ASYM_OP_VERIFY: 3139 tmp = rte_malloc(NULL, op->rsa.sign.length, 0); 3140 if (tmp == NULL) { 3141 OPENSSL_LOG(ERR, "Memory allocation failed"); 3142 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 3143 break; 3144 } 3145 ret = RSA_public_decrypt(op->rsa.sign.length, 3146 op->rsa.sign.data, 3147 tmp, 3148 rsa, 3149 pad); 3150 3151 OPENSSL_LOG(DEBUG, 3152 "Length of public_decrypt %d " 3153 "length of message %zd", 3154 ret, op->rsa.message.length); 3155 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data, 3156 op->rsa.message.length))) { 3157 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 3158 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 3159 } 3160 rte_free(tmp); 3161 break; 3162 3163 default: 3164 /* allow ops with invalid args to be pushed to 3165 * completion queue 3166 */ 3167 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3168 break; 3169 } 3170 3171 if (ret < 0) 3172 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 3173 3174 return 0; 3175 } 3176 3177 static int 3178 process_openssl_sm2_op(struct rte_crypto_op *cop, 3179 struct openssl_asym_session *sess) 3180 { 3181 RTE_SET_USED(cop); 3182 RTE_SET_USED(sess); 3183 return -ENOTSUP; 3184 } 3185 #endif 3186 3187 static int 3188 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op, 3189 struct openssl_asym_session *sess) 3190 { 3191 int retval = 0; 3192 3193 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 3194 3195 switch (sess->xfrm_type) { 3196 case RTE_CRYPTO_ASYM_XFORM_RSA: 3197 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3198 retval = process_openssl_rsa_op_evp(op, sess); 3199 # else 3200 retval = process_openssl_rsa_op(op, sess); 3201 #endif 3202 break; 3203 case RTE_CRYPTO_ASYM_XFORM_MODEX: 3204 retval = process_openssl_modexp_op(op, sess); 3205 break; 3206 case RTE_CRYPTO_ASYM_XFORM_MODINV: 3207 retval = process_openssl_modinv_op(op, sess); 3208 break; 3209 case RTE_CRYPTO_ASYM_XFORM_DH: 3210 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3211 retval = process_openssl_dh_op_evp(op, sess); 3212 # else 3213 retval = process_openssl_dh_op(op, sess); 3214 #endif 3215 break; 3216 case RTE_CRYPTO_ASYM_XFORM_DSA: 3217 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3218 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 3219 retval = process_openssl_dsa_sign_op_evp(op, sess); 3220 else if (op->asym->dsa.op_type == 3221 RTE_CRYPTO_ASYM_OP_VERIFY) 3222 retval = 3223 process_openssl_dsa_verify_op_evp(op, sess); 3224 #else 3225 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 3226 retval = process_openssl_dsa_sign_op(op, sess); 3227 else if (op->asym->dsa.op_type == 3228 RTE_CRYPTO_ASYM_OP_VERIFY) 3229 retval = 3230 process_openssl_dsa_verify_op(op, sess); 3231 else 3232 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3233 #endif 3234 break; 3235 case RTE_CRYPTO_ASYM_XFORM_SM2: 3236 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3237 retval = process_openssl_sm2_op_evp(op, sess); 3238 #else 3239 retval = process_openssl_sm2_op(op, sess); 3240 #endif 3241 break; 3242 default: 3243 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3244 break; 3245 } 3246 if (!retval) { 3247 /* op processed so push to completion queue as processed */ 3248 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 3249 if (retval) 3250 /* return error if failed to put in completion queue */ 3251 retval = -1; 3252 } 3253 3254 return retval; 3255 } 3256 3257 static void 3258 copy_plaintext(struct rte_mbuf *m_src, struct rte_mbuf *m_dst, 3259 struct rte_crypto_op *op) 3260 { 3261 uint8_t *p_src, *p_dst; 3262 3263 p_src = rte_pktmbuf_mtod(m_src, uint8_t *); 3264 p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *); 3265 3266 /** 3267 * Copy the content between cipher offset and auth offset 3268 * for generating correct digest. 3269 */ 3270 if (op->sym->cipher.data.offset > op->sym->auth.data.offset) 3271 memcpy(p_dst + op->sym->auth.data.offset, 3272 p_src + op->sym->auth.data.offset, 3273 op->sym->cipher.data.offset - 3274 op->sym->auth.data.offset); 3275 } 3276 3277 /** Process crypto operation for mbuf */ 3278 static int 3279 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 3280 struct openssl_session *sess) 3281 { 3282 struct rte_mbuf *msrc, *mdst; 3283 int retval; 3284 3285 msrc = op->sym->m_src; 3286 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 3287 3288 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 3289 3290 switch (sess->chain_order) { 3291 case OPENSSL_CHAIN_ONLY_CIPHER: 3292 process_openssl_cipher_op(qp, op, sess, msrc, mdst); 3293 break; 3294 case OPENSSL_CHAIN_ONLY_AUTH: 3295 process_openssl_auth_op(qp, op, sess, msrc, mdst); 3296 break; 3297 case OPENSSL_CHAIN_CIPHER_AUTH: 3298 process_openssl_cipher_op(qp, op, sess, msrc, mdst); 3299 /* OOP */ 3300 if (msrc != mdst) 3301 copy_plaintext(msrc, mdst, op); 3302 process_openssl_auth_op(qp, op, sess, mdst, mdst); 3303 break; 3304 case OPENSSL_CHAIN_AUTH_CIPHER: 3305 process_openssl_auth_op(qp, op, sess, msrc, mdst); 3306 process_openssl_cipher_op(qp, op, sess, msrc, mdst); 3307 break; 3308 case OPENSSL_CHAIN_COMBINED: 3309 process_openssl_combined_op(qp, op, sess, msrc, mdst); 3310 break; 3311 case OPENSSL_CHAIN_CIPHER_BPI: 3312 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 3313 break; 3314 default: 3315 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3316 break; 3317 } 3318 3319 /* Free session if a session-less crypto op */ 3320 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 3321 openssl_reset_session(sess); 3322 memset(sess, 0, sizeof(struct openssl_session)); 3323 rte_mempool_put(qp->sess_mp, op->sym->session); 3324 op->sym->session = NULL; 3325 } 3326 3327 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 3328 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3329 3330 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 3331 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 3332 else 3333 retval = -1; 3334 3335 return retval; 3336 } 3337 3338 /* 3339 *------------------------------------------------------------------------------ 3340 * PMD Framework 3341 *------------------------------------------------------------------------------ 3342 */ 3343 3344 /** Enqueue burst */ 3345 static uint16_t 3346 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 3347 uint16_t nb_ops) 3348 { 3349 void *sess; 3350 struct openssl_qp *qp = queue_pair; 3351 int i, retval; 3352 3353 for (i = 0; i < nb_ops; i++) { 3354 sess = get_session(qp, ops[i]); 3355 if (unlikely(sess == NULL)) 3356 goto enqueue_err; 3357 3358 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) 3359 retval = process_op(qp, ops[i], 3360 (struct openssl_session *) sess); 3361 else 3362 retval = process_asym_op(qp, ops[i], 3363 (struct openssl_asym_session *) sess); 3364 if (unlikely(retval < 0)) 3365 goto enqueue_err; 3366 } 3367 3368 qp->stats.enqueued_count += i; 3369 return i; 3370 3371 enqueue_err: 3372 qp->stats.enqueue_err_count++; 3373 return i; 3374 } 3375 3376 /** Dequeue burst */ 3377 static uint16_t 3378 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 3379 uint16_t nb_ops) 3380 { 3381 struct openssl_qp *qp = queue_pair; 3382 3383 unsigned int nb_dequeued = 0; 3384 3385 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 3386 (void **)ops, nb_ops, NULL); 3387 qp->stats.dequeued_count += nb_dequeued; 3388 3389 return nb_dequeued; 3390 } 3391 3392 /** Create OPENSSL crypto device */ 3393 static int 3394 cryptodev_openssl_create(const char *name, 3395 struct rte_vdev_device *vdev, 3396 struct rte_cryptodev_pmd_init_params *init_params) 3397 { 3398 struct rte_cryptodev *dev; 3399 struct openssl_private *internals; 3400 3401 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 3402 if (dev == NULL) { 3403 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 3404 goto init_error; 3405 } 3406 3407 dev->driver_id = cryptodev_driver_id; 3408 dev->dev_ops = rte_openssl_pmd_ops; 3409 3410 /* register rx/tx burst functions for data path */ 3411 dev->dequeue_burst = openssl_pmd_dequeue_burst; 3412 dev->enqueue_burst = openssl_pmd_enqueue_burst; 3413 3414 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 3415 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 3416 RTE_CRYPTODEV_FF_CPU_AESNI | 3417 RTE_CRYPTODEV_FF_IN_PLACE_SGL | 3418 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 3419 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | 3420 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO | 3421 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP | 3422 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT | 3423 RTE_CRYPTODEV_FF_SYM_SESSIONLESS; 3424 3425 internals = dev->data->dev_private; 3426 3427 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 3428 3429 rte_cryptodev_pmd_probing_finish(dev); 3430 3431 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3432 /* Load legacy provider 3433 * Some algorithms are no longer available in earlier version of openssl, 3434 * unless the legacy provider explicitly loaded. e.g. DES 3435 */ 3436 ossl_legacy_provider_load(); 3437 # endif 3438 return 0; 3439 3440 init_error: 3441 OPENSSL_LOG(ERR, "driver %s: create failed", 3442 init_params->name); 3443 3444 cryptodev_openssl_remove(vdev); 3445 return -EFAULT; 3446 } 3447 3448 /** Initialise OPENSSL crypto device */ 3449 static int 3450 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 3451 { 3452 struct rte_cryptodev_pmd_init_params init_params = { 3453 "", 3454 sizeof(struct openssl_private), 3455 rte_socket_id(), 3456 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 3457 }; 3458 const char *name; 3459 const char *input_args; 3460 3461 name = rte_vdev_device_name(vdev); 3462 if (name == NULL) 3463 return -EINVAL; 3464 input_args = rte_vdev_device_args(vdev); 3465 3466 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 3467 3468 return cryptodev_openssl_create(name, vdev, &init_params); 3469 } 3470 3471 /** Uninitialise OPENSSL crypto device */ 3472 static int 3473 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 3474 { 3475 struct rte_cryptodev *cryptodev; 3476 const char *name; 3477 3478 name = rte_vdev_device_name(vdev); 3479 if (name == NULL) 3480 return -EINVAL; 3481 3482 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 3483 if (cryptodev == NULL) 3484 return -ENODEV; 3485 3486 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3487 ossl_legacy_provider_unload(); 3488 # endif 3489 return rte_cryptodev_pmd_destroy(cryptodev); 3490 } 3491 3492 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 3493 .probe = cryptodev_openssl_probe, 3494 .remove = cryptodev_openssl_remove 3495 }; 3496 3497 static struct cryptodev_driver openssl_crypto_drv; 3498 3499 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 3500 cryptodev_openssl_pmd_drv); 3501 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 3502 "max_nb_queue_pairs=<int> " 3503 "socket_id=<int>"); 3504 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 3505 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 3506 RTE_LOG_REGISTER_DEFAULT(openssl_logtype_driver, INFO); 3507