1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_byteorder.h> 6 #include <rte_common.h> 7 #include <rte_hexdump.h> 8 #include <rte_cryptodev.h> 9 #include <cryptodev_pmd.h> 10 #include <bus_vdev_driver.h> 11 #include <rte_malloc.h> 12 #include <rte_cpuflags.h> 13 14 #include <openssl/cmac.h> 15 #include <openssl/hmac.h> 16 #include <openssl/evp.h> 17 #include <openssl/ec.h> 18 19 #include "openssl_pmd_private.h" 20 #include "compat.h" 21 22 #define DES_BLOCK_SIZE 8 23 24 static uint8_t cryptodev_driver_id; 25 26 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 27 static HMAC_CTX *HMAC_CTX_new(void) 28 { 29 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 30 31 if (ctx != NULL) 32 HMAC_CTX_init(ctx); 33 return ctx; 34 } 35 36 static void HMAC_CTX_free(HMAC_CTX *ctx) 37 { 38 if (ctx != NULL) { 39 HMAC_CTX_cleanup(ctx); 40 OPENSSL_free(ctx); 41 } 42 } 43 #endif 44 45 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 46 47 #include <openssl/provider.h> 48 #include <openssl/core_names.h> 49 #include <openssl/param_build.h> 50 51 #define MAX_OSSL_ALGO_NAME_SIZE 16 52 53 OSSL_PROVIDER *legacy; 54 OSSL_PROVIDER *deflt; 55 56 static void ossl_legacy_provider_load(void) 57 { 58 /* Load Multiple providers into the default (NULL) library context */ 59 legacy = OSSL_PROVIDER_load(NULL, "legacy"); 60 if (legacy == NULL) { 61 OPENSSL_LOG(ERR, "Failed to load Legacy provider"); 62 return; 63 } 64 65 deflt = OSSL_PROVIDER_load(NULL, "default"); 66 if (deflt == NULL) { 67 OPENSSL_LOG(ERR, "Failed to load Default provider"); 68 OSSL_PROVIDER_unload(legacy); 69 return; 70 } 71 } 72 73 static void ossl_legacy_provider_unload(void) 74 { 75 OSSL_PROVIDER_unload(legacy); 76 OSSL_PROVIDER_unload(deflt); 77 } 78 79 static __rte_always_inline const char * 80 digest_name_get(enum rte_crypto_auth_algorithm algo) 81 { 82 switch (algo) { 83 case RTE_CRYPTO_AUTH_MD5_HMAC: 84 return OSSL_DIGEST_NAME_MD5; 85 case RTE_CRYPTO_AUTH_SHA1_HMAC: 86 return OSSL_DIGEST_NAME_SHA1; 87 case RTE_CRYPTO_AUTH_SHA224_HMAC: 88 return OSSL_DIGEST_NAME_SHA2_224; 89 case RTE_CRYPTO_AUTH_SHA256_HMAC: 90 return OSSL_DIGEST_NAME_SHA2_256; 91 case RTE_CRYPTO_AUTH_SHA384_HMAC: 92 return OSSL_DIGEST_NAME_SHA2_384; 93 case RTE_CRYPTO_AUTH_SHA512_HMAC: 94 return OSSL_DIGEST_NAME_SHA2_512; 95 default: 96 return NULL; 97 } 98 } 99 #endif 100 101 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 102 103 /* 104 *------------------------------------------------------------------------------ 105 * Session Prepare 106 *------------------------------------------------------------------------------ 107 */ 108 109 /** Get xform chain order */ 110 static enum openssl_chain_order 111 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 112 { 113 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 114 115 if (xform != NULL) { 116 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 117 if (xform->next == NULL) 118 res = OPENSSL_CHAIN_ONLY_AUTH; 119 else if (xform->next->type == 120 RTE_CRYPTO_SYM_XFORM_CIPHER) 121 res = OPENSSL_CHAIN_AUTH_CIPHER; 122 } 123 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 124 if (xform->next == NULL) 125 res = OPENSSL_CHAIN_ONLY_CIPHER; 126 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 127 res = OPENSSL_CHAIN_CIPHER_AUTH; 128 } 129 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 130 res = OPENSSL_CHAIN_COMBINED; 131 } 132 133 return res; 134 } 135 136 /** Get session cipher key from input cipher key */ 137 static void 138 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key) 139 { 140 memcpy(session_key, input_key, keylen); 141 } 142 143 /** Get key ede 24 bytes standard from input key */ 144 static int 145 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede) 146 { 147 int res = 0; 148 149 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 150 switch (keylen) { 151 case 24: 152 memcpy(key_ede, key, 24); 153 break; 154 case 16: 155 /* K3 = K1 */ 156 memcpy(key_ede, key, 16); 157 memcpy(key_ede + 16, key, 8); 158 break; 159 case 8: 160 /* K1 = K2 = K3 (DES compatibility) */ 161 memcpy(key_ede, key, 8); 162 memcpy(key_ede + 8, key, 8); 163 memcpy(key_ede + 16, key, 8); 164 break; 165 default: 166 OPENSSL_LOG(ERR, "Unsupported key size"); 167 res = -EINVAL; 168 } 169 170 return res; 171 } 172 173 /** Get adequate openssl function for input cipher algorithm */ 174 static uint8_t 175 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 176 const EVP_CIPHER **algo) 177 { 178 int res = 0; 179 180 if (algo != NULL) { 181 switch (sess_algo) { 182 case RTE_CRYPTO_CIPHER_3DES_CBC: 183 switch (keylen) { 184 case 8: 185 *algo = EVP_des_cbc(); 186 break; 187 case 16: 188 *algo = EVP_des_ede_cbc(); 189 break; 190 case 24: 191 *algo = EVP_des_ede3_cbc(); 192 break; 193 default: 194 res = -EINVAL; 195 } 196 break; 197 case RTE_CRYPTO_CIPHER_3DES_CTR: 198 break; 199 case RTE_CRYPTO_CIPHER_AES_CBC: 200 switch (keylen) { 201 case 16: 202 *algo = EVP_aes_128_cbc(); 203 break; 204 case 24: 205 *algo = EVP_aes_192_cbc(); 206 break; 207 case 32: 208 *algo = EVP_aes_256_cbc(); 209 break; 210 default: 211 res = -EINVAL; 212 } 213 break; 214 case RTE_CRYPTO_CIPHER_AES_CTR: 215 switch (keylen) { 216 case 16: 217 *algo = EVP_aes_128_ctr(); 218 break; 219 case 24: 220 *algo = EVP_aes_192_ctr(); 221 break; 222 case 32: 223 *algo = EVP_aes_256_ctr(); 224 break; 225 default: 226 res = -EINVAL; 227 } 228 break; 229 default: 230 res = -EINVAL; 231 break; 232 } 233 } else { 234 res = -EINVAL; 235 } 236 237 return res; 238 } 239 240 /** Get adequate openssl function for input auth algorithm */ 241 static uint8_t 242 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 243 const EVP_MD **algo) 244 { 245 int res = 0; 246 247 if (algo != NULL) { 248 switch (sessalgo) { 249 case RTE_CRYPTO_AUTH_MD5: 250 case RTE_CRYPTO_AUTH_MD5_HMAC: 251 *algo = EVP_md5(); 252 break; 253 case RTE_CRYPTO_AUTH_SHA1: 254 case RTE_CRYPTO_AUTH_SHA1_HMAC: 255 *algo = EVP_sha1(); 256 break; 257 case RTE_CRYPTO_AUTH_SHA224: 258 case RTE_CRYPTO_AUTH_SHA224_HMAC: 259 *algo = EVP_sha224(); 260 break; 261 case RTE_CRYPTO_AUTH_SHA256: 262 case RTE_CRYPTO_AUTH_SHA256_HMAC: 263 *algo = EVP_sha256(); 264 break; 265 case RTE_CRYPTO_AUTH_SHA384: 266 case RTE_CRYPTO_AUTH_SHA384_HMAC: 267 *algo = EVP_sha384(); 268 break; 269 case RTE_CRYPTO_AUTH_SHA512: 270 case RTE_CRYPTO_AUTH_SHA512_HMAC: 271 *algo = EVP_sha512(); 272 break; 273 default: 274 res = -EINVAL; 275 break; 276 } 277 } else { 278 res = -EINVAL; 279 } 280 281 return res; 282 } 283 284 /** Get adequate openssl function for input cipher algorithm */ 285 static uint8_t 286 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 287 const EVP_CIPHER **algo) 288 { 289 int res = 0; 290 291 if (algo != NULL) { 292 switch (sess_algo) { 293 case RTE_CRYPTO_AEAD_AES_GCM: 294 switch (keylen) { 295 case 16: 296 *algo = EVP_aes_128_gcm(); 297 break; 298 case 24: 299 *algo = EVP_aes_192_gcm(); 300 break; 301 case 32: 302 *algo = EVP_aes_256_gcm(); 303 break; 304 default: 305 res = -EINVAL; 306 } 307 break; 308 case RTE_CRYPTO_AEAD_AES_CCM: 309 switch (keylen) { 310 case 16: 311 *algo = EVP_aes_128_ccm(); 312 break; 313 case 24: 314 *algo = EVP_aes_192_ccm(); 315 break; 316 case 32: 317 *algo = EVP_aes_256_ccm(); 318 break; 319 default: 320 res = -EINVAL; 321 } 322 break; 323 default: 324 res = -EINVAL; 325 break; 326 } 327 } else { 328 res = -EINVAL; 329 } 330 331 return res; 332 } 333 334 /* Set session AEAD encryption parameters */ 335 static int 336 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 337 enum rte_crypto_aead_algorithm algo, 338 uint8_t tag_len, const uint8_t *key, 339 EVP_CIPHER_CTX **ctx) 340 { 341 int iv_type = 0; 342 unsigned int do_ccm; 343 344 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 345 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 346 347 /* Select AEAD algo */ 348 switch (algo) { 349 case RTE_CRYPTO_AEAD_AES_GCM: 350 iv_type = EVP_CTRL_GCM_SET_IVLEN; 351 if (tag_len != 16) 352 return -EINVAL; 353 do_ccm = 0; 354 break; 355 case RTE_CRYPTO_AEAD_AES_CCM: 356 iv_type = EVP_CTRL_CCM_SET_IVLEN; 357 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 358 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 359 return -EINVAL; 360 do_ccm = 1; 361 break; 362 default: 363 return -ENOTSUP; 364 } 365 366 sess->cipher.mode = OPENSSL_CIPHER_LIB; 367 *ctx = EVP_CIPHER_CTX_new(); 368 369 if (get_aead_algo(algo, sess->cipher.key.length, 370 &sess->cipher.evp_algo) != 0) 371 return -EINVAL; 372 373 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 374 375 sess->chain_order = OPENSSL_CHAIN_COMBINED; 376 377 if (EVP_EncryptInit_ex(*ctx, sess->cipher.evp_algo, 378 NULL, NULL, NULL) <= 0) 379 return -EINVAL; 380 381 if (EVP_CIPHER_CTX_ctrl(*ctx, iv_type, sess->iv.length, 382 NULL) <= 0) 383 return -EINVAL; 384 385 if (do_ccm) 386 EVP_CIPHER_CTX_ctrl(*ctx, EVP_CTRL_CCM_SET_TAG, 387 tag_len, NULL); 388 389 if (EVP_EncryptInit_ex(*ctx, NULL, NULL, key, NULL) <= 0) 390 return -EINVAL; 391 392 return 0; 393 } 394 395 /* Set session AEAD decryption parameters */ 396 static int 397 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 398 enum rte_crypto_aead_algorithm algo, 399 uint8_t tag_len, const uint8_t *key, 400 EVP_CIPHER_CTX **ctx) 401 { 402 int iv_type = 0; 403 unsigned int do_ccm = 0; 404 405 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 406 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 407 408 /* Select AEAD algo */ 409 switch (algo) { 410 case RTE_CRYPTO_AEAD_AES_GCM: 411 iv_type = EVP_CTRL_GCM_SET_IVLEN; 412 if (tag_len != 16) 413 return -EINVAL; 414 break; 415 case RTE_CRYPTO_AEAD_AES_CCM: 416 iv_type = EVP_CTRL_CCM_SET_IVLEN; 417 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 418 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 419 return -EINVAL; 420 do_ccm = 1; 421 break; 422 default: 423 return -ENOTSUP; 424 } 425 426 sess->cipher.mode = OPENSSL_CIPHER_LIB; 427 *ctx = EVP_CIPHER_CTX_new(); 428 429 if (get_aead_algo(algo, sess->cipher.key.length, 430 &sess->cipher.evp_algo) != 0) 431 return -EINVAL; 432 433 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 434 435 sess->chain_order = OPENSSL_CHAIN_COMBINED; 436 437 if (EVP_DecryptInit_ex(*ctx, sess->cipher.evp_algo, 438 NULL, NULL, NULL) <= 0) 439 return -EINVAL; 440 441 if (EVP_CIPHER_CTX_ctrl(*ctx, iv_type, 442 sess->iv.length, NULL) <= 0) 443 return -EINVAL; 444 445 if (do_ccm) 446 EVP_CIPHER_CTX_ctrl(*ctx, EVP_CTRL_CCM_SET_TAG, 447 tag_len, NULL); 448 449 if (EVP_DecryptInit_ex(*ctx, NULL, NULL, key, NULL) <= 0) 450 return -EINVAL; 451 452 return 0; 453 } 454 455 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30200000L) 456 static int openssl_aesni_ctx_clone(EVP_CIPHER_CTX **dest, 457 struct openssl_session *sess) 458 { 459 /* OpenSSL versions 3.0.0 <= V < 3.2.0 have no dupctx() implementation 460 * for AES-GCM and AES-CCM. In this case, we have to create new empty 461 * contexts and initialise, as we did the original context. 462 */ 463 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) 464 sess->aead_algo = RTE_CRYPTO_AEAD_AES_GCM; 465 466 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 467 return openssl_set_sess_aead_enc_param(sess, sess->aead_algo, 468 sess->auth.digest_length, sess->cipher.key.data, 469 dest); 470 else 471 return openssl_set_sess_aead_dec_param(sess, sess->aead_algo, 472 sess->auth.digest_length, sess->cipher.key.data, 473 dest); 474 } 475 #endif 476 477 /** Set session cipher parameters */ 478 static int 479 openssl_set_session_cipher_parameters(struct openssl_session *sess, 480 const struct rte_crypto_sym_xform *xform) 481 { 482 /* Select cipher direction */ 483 sess->cipher.direction = xform->cipher.op; 484 /* Select cipher key */ 485 sess->cipher.key.length = xform->cipher.key.length; 486 487 /* Set IV parameters */ 488 sess->iv.offset = xform->cipher.iv.offset; 489 sess->iv.length = xform->cipher.iv.length; 490 491 /* Select cipher algo */ 492 switch (xform->cipher.algo) { 493 case RTE_CRYPTO_CIPHER_3DES_CBC: 494 case RTE_CRYPTO_CIPHER_AES_CBC: 495 case RTE_CRYPTO_CIPHER_AES_CTR: 496 sess->cipher.mode = OPENSSL_CIPHER_LIB; 497 sess->cipher.algo = xform->cipher.algo; 498 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 499 500 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 501 &sess->cipher.evp_algo) != 0) 502 return -EINVAL; 503 504 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 505 sess->cipher.key.data); 506 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 507 if (EVP_EncryptInit_ex(sess->cipher.ctx, 508 sess->cipher.evp_algo, 509 NULL, xform->cipher.key.data, 510 NULL) != 1) { 511 return -EINVAL; 512 } 513 } else if (sess->cipher.direction == 514 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 515 if (EVP_DecryptInit_ex(sess->cipher.ctx, 516 sess->cipher.evp_algo, 517 NULL, xform->cipher.key.data, 518 NULL) != 1) { 519 return -EINVAL; 520 } 521 } 522 523 break; 524 525 case RTE_CRYPTO_CIPHER_3DES_CTR: 526 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 527 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 528 529 if (get_cipher_key_ede(xform->cipher.key.data, 530 sess->cipher.key.length, 531 sess->cipher.key.data) != 0) 532 return -EINVAL; 533 534 535 /* We use 3DES encryption also for decryption. 536 * IV is not important for 3DES ECB. 537 */ 538 if (EVP_EncryptInit_ex(sess->cipher.ctx, EVP_des_ede3_ecb(), 539 NULL, sess->cipher.key.data, NULL) != 1) 540 return -EINVAL; 541 542 break; 543 544 case RTE_CRYPTO_CIPHER_DES_CBC: 545 sess->cipher.algo = xform->cipher.algo; 546 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 547 sess->cipher.evp_algo = EVP_des_cbc(); 548 549 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 550 sess->cipher.key.data); 551 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 552 if (EVP_EncryptInit_ex(sess->cipher.ctx, 553 sess->cipher.evp_algo, 554 NULL, xform->cipher.key.data, 555 NULL) != 1) { 556 return -EINVAL; 557 } 558 } else if (sess->cipher.direction == 559 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 560 if (EVP_DecryptInit_ex(sess->cipher.ctx, 561 sess->cipher.evp_algo, 562 NULL, xform->cipher.key.data, 563 NULL) != 1) { 564 return -EINVAL; 565 } 566 } 567 568 break; 569 570 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 571 sess->cipher.algo = xform->cipher.algo; 572 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 573 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 574 sess->cipher.evp_algo = EVP_des_cbc(); 575 576 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 577 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 578 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 579 NULL, xform->cipher.key.data, 0) != 1) 580 return -EINVAL; 581 582 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 583 sess->cipher.key.data); 584 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 585 if (EVP_EncryptInit_ex(sess->cipher.ctx, 586 sess->cipher.evp_algo, 587 NULL, xform->cipher.key.data, 588 NULL) != 1) { 589 return -EINVAL; 590 } 591 } else if (sess->cipher.direction == 592 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 593 if (EVP_DecryptInit_ex(sess->cipher.ctx, 594 sess->cipher.evp_algo, 595 NULL, xform->cipher.key.data, 596 NULL) != 1) { 597 return -EINVAL; 598 } 599 } 600 601 break; 602 default: 603 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 604 return -ENOTSUP; 605 } 606 607 EVP_CIPHER_CTX_set_padding(sess->cipher.ctx, 0); 608 609 return 0; 610 } 611 612 /* Set session auth parameters */ 613 static int 614 openssl_set_session_auth_parameters(struct openssl_session *sess, 615 const struct rte_crypto_sym_xform *xform) 616 { 617 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 618 char algo_name[MAX_OSSL_ALGO_NAME_SIZE]; 619 OSSL_PARAM params[2]; 620 const char *algo; 621 EVP_MAC *mac; 622 # endif 623 /* Select auth generate/verify */ 624 sess->auth.operation = xform->auth.op; 625 sess->auth.algo = xform->auth.algo; 626 627 sess->auth.digest_length = xform->auth.digest_length; 628 629 /* Select auth algo */ 630 switch (xform->auth.algo) { 631 case RTE_CRYPTO_AUTH_AES_GMAC: 632 /* 633 * OpenSSL requires GMAC to be a GCM operation 634 * with no cipher data length 635 */ 636 sess->cipher.key.length = xform->auth.key.length; 637 638 /* Set IV parameters */ 639 sess->iv.offset = xform->auth.iv.offset; 640 sess->iv.length = xform->auth.iv.length; 641 642 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 643 return openssl_set_sess_aead_enc_param(sess, 644 RTE_CRYPTO_AEAD_AES_GCM, 645 xform->auth.digest_length, 646 xform->auth.key.data, 647 &sess->cipher.ctx); 648 else 649 return openssl_set_sess_aead_dec_param(sess, 650 RTE_CRYPTO_AEAD_AES_GCM, 651 xform->auth.digest_length, 652 xform->auth.key.data, 653 &sess->cipher.ctx); 654 break; 655 656 case RTE_CRYPTO_AUTH_MD5: 657 case RTE_CRYPTO_AUTH_SHA1: 658 case RTE_CRYPTO_AUTH_SHA224: 659 case RTE_CRYPTO_AUTH_SHA256: 660 case RTE_CRYPTO_AUTH_SHA384: 661 case RTE_CRYPTO_AUTH_SHA512: 662 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 663 if (get_auth_algo(xform->auth.algo, 664 &sess->auth.auth.evp_algo) != 0) 665 return -EINVAL; 666 sess->auth.auth.ctx = EVP_MD_CTX_create(); 667 break; 668 669 case RTE_CRYPTO_AUTH_AES_CMAC: 670 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 671 if (xform->auth.key.length == 16) 672 algo = SN_aes_128_cbc; 673 else if (xform->auth.key.length == 24) 674 algo = SN_aes_192_cbc; 675 else if (xform->auth.key.length == 32) 676 algo = SN_aes_256_cbc; 677 else 678 return -EINVAL; 679 680 strlcpy(algo_name, algo, sizeof(algo_name)); 681 params[0] = OSSL_PARAM_construct_utf8_string( 682 OSSL_MAC_PARAM_CIPHER, algo_name, 0); 683 params[1] = OSSL_PARAM_construct_end(); 684 685 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 686 mac = EVP_MAC_fetch(NULL, OSSL_MAC_NAME_CMAC, NULL); 687 sess->auth.cmac.ctx = EVP_MAC_CTX_new(mac); 688 EVP_MAC_free(mac); 689 690 if (EVP_MAC_init(sess->auth.cmac.ctx, 691 xform->auth.key.data, 692 xform->auth.key.length, 693 params) != 1) 694 return -EINVAL; 695 # else 696 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 697 sess->auth.cmac.ctx = CMAC_CTX_new(); 698 if (get_cipher_algo(RTE_CRYPTO_CIPHER_AES_CBC, 699 xform->auth.key.length, 700 &sess->auth.cmac.evp_algo) != 0) 701 return -EINVAL; 702 if (CMAC_Init(sess->auth.cmac.ctx, 703 xform->auth.key.data, 704 xform->auth.key.length, 705 sess->auth.cmac.evp_algo, NULL) != 1) 706 return -EINVAL; 707 # endif 708 break; 709 710 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 711 case RTE_CRYPTO_AUTH_MD5_HMAC: 712 case RTE_CRYPTO_AUTH_SHA1_HMAC: 713 case RTE_CRYPTO_AUTH_SHA224_HMAC: 714 case RTE_CRYPTO_AUTH_SHA256_HMAC: 715 case RTE_CRYPTO_AUTH_SHA384_HMAC: 716 case RTE_CRYPTO_AUTH_SHA512_HMAC: 717 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 718 719 algo = digest_name_get(xform->auth.algo); 720 if (!algo) 721 return -EINVAL; 722 strlcpy(algo_name, algo, sizeof(algo_name)); 723 724 mac = EVP_MAC_fetch(NULL, "HMAC", NULL); 725 sess->auth.hmac.ctx = EVP_MAC_CTX_new(mac); 726 EVP_MAC_free(mac); 727 if (get_auth_algo(xform->auth.algo, 728 &sess->auth.hmac.evp_algo) != 0) 729 return -EINVAL; 730 731 params[0] = OSSL_PARAM_construct_utf8_string("digest", 732 algo_name, 0); 733 params[1] = OSSL_PARAM_construct_end(); 734 if (EVP_MAC_init(sess->auth.hmac.ctx, 735 xform->auth.key.data, 736 xform->auth.key.length, 737 params) != 1) 738 return -EINVAL; 739 break; 740 # else 741 case RTE_CRYPTO_AUTH_MD5_HMAC: 742 case RTE_CRYPTO_AUTH_SHA1_HMAC: 743 case RTE_CRYPTO_AUTH_SHA224_HMAC: 744 case RTE_CRYPTO_AUTH_SHA256_HMAC: 745 case RTE_CRYPTO_AUTH_SHA384_HMAC: 746 case RTE_CRYPTO_AUTH_SHA512_HMAC: 747 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 748 sess->auth.hmac.ctx = HMAC_CTX_new(); 749 if (get_auth_algo(xform->auth.algo, 750 &sess->auth.hmac.evp_algo) != 0) 751 return -EINVAL; 752 753 if (HMAC_Init_ex(sess->auth.hmac.ctx, 754 xform->auth.key.data, 755 xform->auth.key.length, 756 sess->auth.hmac.evp_algo, NULL) != 1) 757 return -EINVAL; 758 break; 759 # endif 760 default: 761 return -ENOTSUP; 762 } 763 764 return 0; 765 } 766 767 /* Set session AEAD parameters */ 768 static int 769 openssl_set_session_aead_parameters(struct openssl_session *sess, 770 const struct rte_crypto_sym_xform *xform) 771 { 772 /* Select cipher key */ 773 sess->cipher.key.length = xform->aead.key.length; 774 775 /* Set IV parameters */ 776 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 777 /* 778 * For AES-CCM, the actual IV is placed 779 * one byte after the start of the IV field, 780 * according to the API. 781 */ 782 sess->iv.offset = xform->aead.iv.offset + 1; 783 else 784 sess->iv.offset = xform->aead.iv.offset; 785 786 sess->iv.length = xform->aead.iv.length; 787 788 sess->auth.aad_length = xform->aead.aad_length; 789 sess->auth.digest_length = xform->aead.digest_length; 790 791 sess->aead_algo = xform->aead.algo; 792 /* Select cipher direction */ 793 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 794 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 795 xform->aead.digest_length, xform->aead.key.data, 796 &sess->cipher.ctx); 797 else 798 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 799 xform->aead.digest_length, xform->aead.key.data, 800 &sess->cipher.ctx); 801 } 802 803 /** Parse crypto xform chain and set private session parameters */ 804 int 805 openssl_set_session_parameters(struct openssl_session *sess, 806 const struct rte_crypto_sym_xform *xform, 807 uint16_t nb_queue_pairs) 808 { 809 const struct rte_crypto_sym_xform *cipher_xform = NULL; 810 const struct rte_crypto_sym_xform *auth_xform = NULL; 811 const struct rte_crypto_sym_xform *aead_xform = NULL; 812 int ret; 813 814 sess->chain_order = openssl_get_chain_order(xform); 815 switch (sess->chain_order) { 816 case OPENSSL_CHAIN_ONLY_CIPHER: 817 cipher_xform = xform; 818 break; 819 case OPENSSL_CHAIN_ONLY_AUTH: 820 auth_xform = xform; 821 break; 822 case OPENSSL_CHAIN_CIPHER_AUTH: 823 cipher_xform = xform; 824 auth_xform = xform->next; 825 break; 826 case OPENSSL_CHAIN_AUTH_CIPHER: 827 auth_xform = xform; 828 cipher_xform = xform->next; 829 break; 830 case OPENSSL_CHAIN_COMBINED: 831 aead_xform = xform; 832 break; 833 default: 834 return -EINVAL; 835 } 836 837 /* Default IV length = 0 */ 838 sess->iv.length = 0; 839 840 /* cipher_xform must be check before auth_xform */ 841 if (cipher_xform) { 842 ret = openssl_set_session_cipher_parameters( 843 sess, cipher_xform); 844 if (ret != 0) { 845 OPENSSL_LOG(ERR, 846 "Invalid/unsupported cipher parameters"); 847 return ret; 848 } 849 } 850 851 if (auth_xform) { 852 ret = openssl_set_session_auth_parameters(sess, auth_xform); 853 if (ret != 0) { 854 OPENSSL_LOG(ERR, 855 "Invalid/unsupported auth parameters"); 856 return ret; 857 } 858 } 859 860 if (aead_xform) { 861 ret = openssl_set_session_aead_parameters(sess, aead_xform); 862 if (ret != 0) { 863 OPENSSL_LOG(ERR, 864 "Invalid/unsupported AEAD parameters"); 865 return ret; 866 } 867 } 868 869 /* 870 * With only one queue pair, the array of copies is not needed. 871 * Otherwise, one entry per queue pair is required. 872 */ 873 sess->ctx_copies_len = nb_queue_pairs > 1 ? nb_queue_pairs : 0; 874 875 return 0; 876 } 877 878 /** Reset private session parameters */ 879 void 880 openssl_reset_session(struct openssl_session *sess) 881 { 882 /* Free all the qp_ctx entries. */ 883 for (uint16_t i = 0; i < sess->ctx_copies_len; i++) { 884 if (sess->qp_ctx[i].cipher != NULL) { 885 EVP_CIPHER_CTX_free(sess->qp_ctx[i].cipher); 886 sess->qp_ctx[i].cipher = NULL; 887 } 888 889 switch (sess->auth.mode) { 890 case OPENSSL_AUTH_AS_AUTH: 891 EVP_MD_CTX_destroy(sess->qp_ctx[i].auth); 892 sess->qp_ctx[i].auth = NULL; 893 break; 894 case OPENSSL_AUTH_AS_HMAC: 895 free_hmac_ctx(sess->qp_ctx[i].hmac); 896 sess->qp_ctx[i].hmac = NULL; 897 break; 898 case OPENSSL_AUTH_AS_CMAC: 899 free_cmac_ctx(sess->qp_ctx[i].cmac); 900 sess->qp_ctx[i].cmac = NULL; 901 break; 902 } 903 } 904 905 EVP_CIPHER_CTX_free(sess->cipher.ctx); 906 907 switch (sess->auth.mode) { 908 case OPENSSL_AUTH_AS_AUTH: 909 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 910 break; 911 case OPENSSL_AUTH_AS_HMAC: 912 free_hmac_ctx(sess->auth.hmac.ctx); 913 break; 914 case OPENSSL_AUTH_AS_CMAC: 915 free_cmac_ctx(sess->auth.cmac.ctx); 916 break; 917 } 918 919 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 920 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 921 } 922 923 /** Provide session for operation */ 924 static void * 925 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 926 { 927 struct openssl_session *sess = NULL; 928 struct openssl_asym_session *asym_sess = NULL; 929 930 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 931 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 932 /* get existing session */ 933 if (likely(op->sym->session != NULL)) 934 sess = CRYPTODEV_GET_SYM_SESS_PRIV( 935 op->sym->session); 936 } else { 937 if (likely(op->asym->session != NULL)) 938 asym_sess = (struct openssl_asym_session *) 939 op->asym->session->sess_private_data; 940 if (asym_sess == NULL) 941 op->status = 942 RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 943 return asym_sess; 944 } 945 } else { 946 struct rte_cryptodev_sym_session *_sess; 947 /* sessionless asymmetric not supported */ 948 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) 949 return NULL; 950 951 /* provide internal session */ 952 rte_mempool_get(qp->sess_mp, (void **)&_sess); 953 954 if (_sess == NULL) 955 return NULL; 956 957 sess = (struct openssl_session *)_sess->driver_priv_data; 958 959 if (unlikely(openssl_set_session_parameters(sess, 960 op->sym->xform, 1) != 0)) { 961 rte_mempool_put(qp->sess_mp, _sess); 962 sess = NULL; 963 } 964 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 965 966 } 967 968 if (sess == NULL) 969 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 970 971 return sess; 972 } 973 974 /* 975 *------------------------------------------------------------------------------ 976 * Process Operations 977 *------------------------------------------------------------------------------ 978 */ 979 static inline int 980 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 981 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 982 { 983 struct rte_mbuf *m; 984 int dstlen; 985 int l, n = srclen; 986 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 987 988 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 989 m = m->next) 990 offset -= rte_pktmbuf_data_len(m); 991 992 if (m == 0) 993 return -1; 994 995 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 996 if (inplace) 997 *dst = src; 998 999 l = rte_pktmbuf_data_len(m) - offset; 1000 if (srclen <= l) { 1001 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 1002 return -1; 1003 *dst += l; 1004 return 0; 1005 } 1006 1007 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1008 return -1; 1009 1010 *dst += dstlen; 1011 n -= l; 1012 1013 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1014 uint8_t diff = l - dstlen, rem; 1015 1016 src = rte_pktmbuf_mtod(m, uint8_t *); 1017 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 1018 if (diff && inplace) { 1019 rem = RTE_MIN(l, 1020 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 1021 if (EVP_EncryptUpdate(ctx, temp, 1022 &dstlen, src, rem) <= 0) 1023 return -1; 1024 n -= rem; 1025 rte_memcpy(*dst, temp, diff); 1026 rte_memcpy(src, temp + diff, rem); 1027 src += rem; 1028 l -= rem; 1029 } 1030 if (inplace) 1031 *dst = src; 1032 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1033 return -1; 1034 *dst += dstlen; 1035 n -= l; 1036 } 1037 1038 return 0; 1039 } 1040 1041 static inline int 1042 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 1043 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 1044 { 1045 struct rte_mbuf *m; 1046 int dstlen; 1047 int l, n = srclen; 1048 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 1049 1050 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1051 m = m->next) 1052 offset -= rte_pktmbuf_data_len(m); 1053 1054 if (m == 0) 1055 return -1; 1056 1057 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1058 if (inplace) 1059 *dst = src; 1060 1061 l = rte_pktmbuf_data_len(m) - offset; 1062 if (srclen <= l) { 1063 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 1064 return -1; 1065 *dst += l; 1066 return 0; 1067 } 1068 1069 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1070 return -1; 1071 1072 *dst += dstlen; 1073 n -= l; 1074 1075 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1076 uint8_t diff = l - dstlen, rem; 1077 1078 src = rte_pktmbuf_mtod(m, uint8_t *); 1079 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 1080 if (diff && inplace) { 1081 rem = RTE_MIN(l, 1082 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 1083 if (EVP_DecryptUpdate(ctx, temp, 1084 &dstlen, src, rem) <= 0) 1085 return -1; 1086 n -= rem; 1087 rte_memcpy(*dst, temp, diff); 1088 rte_memcpy(src, temp + diff, rem); 1089 src += rem; 1090 l -= rem; 1091 } 1092 if (inplace) 1093 *dst = src; 1094 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1095 return -1; 1096 *dst += dstlen; 1097 n -= l; 1098 } 1099 1100 return 0; 1101 } 1102 1103 /** Process standard openssl cipher encryption */ 1104 static int 1105 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1106 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1107 uint8_t inplace) 1108 { 1109 int totlen; 1110 1111 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1112 goto process_cipher_encrypt_err; 1113 1114 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1115 srclen, ctx, inplace)) 1116 goto process_cipher_encrypt_err; 1117 1118 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 1119 goto process_cipher_encrypt_err; 1120 1121 return 0; 1122 1123 process_cipher_encrypt_err: 1124 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 1125 return -EINVAL; 1126 } 1127 1128 /** Process standard openssl cipher encryption */ 1129 static int 1130 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 1131 uint8_t *iv, int srclen, 1132 EVP_CIPHER_CTX *ctx) 1133 { 1134 uint8_t i; 1135 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 1136 int encrypted_ivlen; 1137 1138 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 1139 iv, DES_BLOCK_SIZE) <= 0) 1140 goto process_cipher_encrypt_err; 1141 1142 for (i = 0; i < srclen; i++) 1143 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 1144 1145 return 0; 1146 1147 process_cipher_encrypt_err: 1148 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 1149 return -EINVAL; 1150 } 1151 /** Process standard openssl cipher decryption */ 1152 static int 1153 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1154 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1155 uint8_t inplace) 1156 { 1157 int totlen; 1158 1159 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1160 goto process_cipher_decrypt_err; 1161 1162 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1163 srclen, ctx, inplace)) 1164 goto process_cipher_decrypt_err; 1165 1166 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 1167 goto process_cipher_decrypt_err; 1168 return 0; 1169 1170 process_cipher_decrypt_err: 1171 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 1172 return -EINVAL; 1173 } 1174 1175 /** Process cipher des 3 ctr encryption, decryption algorithm */ 1176 static int 1177 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 1178 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) 1179 { 1180 uint8_t ebuf[8]; 1181 uint64_t ctr; 1182 int unused, n; 1183 struct rte_mbuf *m; 1184 uint8_t *src; 1185 int l; 1186 1187 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1188 m = m->next) 1189 offset -= rte_pktmbuf_data_len(m); 1190 1191 if (m == 0) 1192 goto process_cipher_des3ctr_err; 1193 1194 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1195 l = rte_pktmbuf_data_len(m) - offset; 1196 1197 memcpy(&ctr, iv, 8); 1198 1199 for (n = 0; n < srclen; n++) { 1200 if (n % 8 == 0) { 1201 uint64_t cpu_ctr; 1202 1203 if (EVP_EncryptUpdate(ctx, 1204 (unsigned char *)&ebuf, &unused, 1205 (const unsigned char *)&ctr, 8) <= 0) 1206 goto process_cipher_des3ctr_err; 1207 cpu_ctr = rte_be_to_cpu_64(ctr); 1208 cpu_ctr++; 1209 ctr = rte_cpu_to_be_64(cpu_ctr); 1210 } 1211 dst[n] = *(src++) ^ ebuf[n % 8]; 1212 1213 l--; 1214 if (!l) { 1215 m = m->next; 1216 if (m) { 1217 src = rte_pktmbuf_mtod(m, uint8_t *); 1218 l = rte_pktmbuf_data_len(m); 1219 } 1220 } 1221 } 1222 1223 return 0; 1224 1225 process_cipher_des3ctr_err: 1226 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 1227 return -EINVAL; 1228 } 1229 1230 /** Process AES-GCM encrypt algorithm */ 1231 static int 1232 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1233 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1234 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1235 { 1236 int len = 0; 1237 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1238 int unused = 0; 1239 uint8_t empty[] = {}; 1240 #endif 1241 1242 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1243 goto process_auth_encryption_gcm_err; 1244 1245 if (aadlen > 0) 1246 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1247 goto process_auth_encryption_gcm_err; 1248 1249 if (srclen > 0) 1250 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1251 srclen, ctx, 0)) 1252 goto process_auth_encryption_gcm_err; 1253 1254 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1255 /* Workaround open ssl bug in version less then 1.0.1f */ 1256 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1257 goto process_auth_encryption_gcm_err; 1258 #endif 1259 1260 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1261 goto process_auth_encryption_gcm_err; 1262 1263 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1264 goto process_auth_encryption_gcm_err; 1265 1266 return 0; 1267 1268 process_auth_encryption_gcm_err: 1269 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1270 return -EINVAL; 1271 } 1272 1273 /** Process AES-CCM encrypt algorithm */ 1274 static int 1275 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1276 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1277 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1278 { 1279 int len = 0; 1280 1281 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1282 goto process_auth_encryption_ccm_err; 1283 1284 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1285 goto process_auth_encryption_ccm_err; 1286 1287 if (aadlen > 0) 1288 /* 1289 * For AES-CCM, the actual AAD is placed 1290 * 18 bytes after the start of the AAD field, 1291 * according to the API. 1292 */ 1293 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1294 goto process_auth_encryption_ccm_err; 1295 1296 if (srclen >= 0) 1297 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1298 srclen, ctx, 0)) 1299 goto process_auth_encryption_ccm_err; 1300 1301 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1302 goto process_auth_encryption_ccm_err; 1303 1304 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1305 goto process_auth_encryption_ccm_err; 1306 1307 return 0; 1308 1309 process_auth_encryption_ccm_err: 1310 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1311 return -EINVAL; 1312 } 1313 1314 /** Process AES-GCM decrypt algorithm */ 1315 static int 1316 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1317 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1318 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1319 { 1320 int len = 0; 1321 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1322 int unused = 0; 1323 uint8_t empty[] = {}; 1324 #endif 1325 1326 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1327 goto process_auth_decryption_gcm_err; 1328 1329 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1330 goto process_auth_decryption_gcm_err; 1331 1332 if (aadlen > 0) 1333 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1334 goto process_auth_decryption_gcm_err; 1335 1336 if (srclen > 0) 1337 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1338 srclen, ctx, 0)) 1339 goto process_auth_decryption_gcm_err; 1340 1341 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1342 /* Workaround open ssl bug in version less then 1.0.1f */ 1343 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1344 goto process_auth_decryption_gcm_err; 1345 #endif 1346 1347 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1348 return -EFAULT; 1349 1350 return 0; 1351 1352 process_auth_decryption_gcm_err: 1353 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1354 return -EINVAL; 1355 } 1356 1357 /** Process AES-CCM decrypt algorithm */ 1358 static int 1359 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1360 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1361 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1362 EVP_CIPHER_CTX *ctx) 1363 { 1364 int len = 0; 1365 1366 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1367 goto process_auth_decryption_ccm_err; 1368 1369 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1370 goto process_auth_decryption_ccm_err; 1371 1372 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1373 goto process_auth_decryption_ccm_err; 1374 1375 if (aadlen > 0) 1376 /* 1377 * For AES-CCM, the actual AAD is placed 1378 * 18 bytes after the start of the AAD field, 1379 * according to the API. 1380 */ 1381 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1382 goto process_auth_decryption_ccm_err; 1383 1384 if (srclen >= 0) 1385 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1386 srclen, ctx, 0)) 1387 return -EFAULT; 1388 1389 return 0; 1390 1391 process_auth_decryption_ccm_err: 1392 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1393 return -EINVAL; 1394 } 1395 1396 /** Process standard openssl auth algorithms */ 1397 static int 1398 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1399 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1400 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1401 { 1402 size_t dstlen; 1403 struct rte_mbuf *m; 1404 int l, n = srclen; 1405 uint8_t *src; 1406 1407 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1408 m = m->next) 1409 offset -= rte_pktmbuf_data_len(m); 1410 1411 if (m == 0) 1412 goto process_auth_err; 1413 1414 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1415 goto process_auth_err; 1416 1417 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1418 1419 l = rte_pktmbuf_data_len(m) - offset; 1420 if (srclen <= l) { 1421 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1422 goto process_auth_err; 1423 goto process_auth_final; 1424 } 1425 1426 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1427 goto process_auth_err; 1428 1429 n -= l; 1430 1431 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1432 src = rte_pktmbuf_mtod(m, uint8_t *); 1433 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1434 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1435 goto process_auth_err; 1436 n -= l; 1437 } 1438 1439 process_auth_final: 1440 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1441 goto process_auth_err; 1442 return 0; 1443 1444 process_auth_err: 1445 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1446 return -EINVAL; 1447 } 1448 1449 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1450 /** Process standard openssl auth algorithms with hmac/cmac */ 1451 static int 1452 process_openssl_auth_mac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1453 int srclen, EVP_MAC_CTX *ctx) 1454 { 1455 size_t dstlen; 1456 struct rte_mbuf *m; 1457 int l, n = srclen; 1458 uint8_t *src; 1459 1460 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1461 m = m->next) 1462 offset -= rte_pktmbuf_data_len(m); 1463 1464 if (m == 0) 1465 goto process_auth_err; 1466 1467 if (EVP_MAC_init(ctx, NULL, 0, NULL) <= 0) 1468 goto process_auth_err; 1469 1470 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1471 1472 l = rte_pktmbuf_data_len(m) - offset; 1473 if (srclen <= l) { 1474 if (EVP_MAC_update(ctx, (unsigned char *)src, srclen) != 1) 1475 goto process_auth_err; 1476 goto process_auth_final; 1477 } 1478 1479 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1480 goto process_auth_err; 1481 1482 n -= l; 1483 1484 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1485 src = rte_pktmbuf_mtod(m, uint8_t *); 1486 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1487 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1488 goto process_auth_err; 1489 n -= l; 1490 } 1491 1492 process_auth_final: 1493 if (EVP_MAC_final(ctx, dst, &dstlen, DIGEST_LENGTH_MAX) != 1) 1494 goto process_auth_err; 1495 1496 return 0; 1497 1498 process_auth_err: 1499 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1500 return -EINVAL; 1501 } 1502 # else 1503 /** Process standard openssl auth algorithms with hmac */ 1504 static int 1505 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1506 int srclen, HMAC_CTX *ctx) 1507 { 1508 unsigned int dstlen; 1509 struct rte_mbuf *m; 1510 int l, n = srclen; 1511 uint8_t *src; 1512 1513 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1514 m = m->next) 1515 offset -= rte_pktmbuf_data_len(m); 1516 1517 if (m == 0) 1518 goto process_auth_err; 1519 1520 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1521 1522 l = rte_pktmbuf_data_len(m) - offset; 1523 if (srclen <= l) { 1524 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1525 goto process_auth_err; 1526 goto process_auth_final; 1527 } 1528 1529 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1530 goto process_auth_err; 1531 1532 n -= l; 1533 1534 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1535 src = rte_pktmbuf_mtod(m, uint8_t *); 1536 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1537 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1538 goto process_auth_err; 1539 n -= l; 1540 } 1541 1542 process_auth_final: 1543 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1544 goto process_auth_err; 1545 1546 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1547 goto process_auth_err; 1548 1549 return 0; 1550 1551 process_auth_err: 1552 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1553 return -EINVAL; 1554 } 1555 1556 /** Process standard openssl auth algorithms with cmac */ 1557 static int 1558 process_openssl_auth_cmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1559 int srclen, CMAC_CTX *ctx) 1560 { 1561 unsigned int dstlen; 1562 struct rte_mbuf *m; 1563 int l, n = srclen; 1564 uint8_t *src; 1565 1566 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1567 m = m->next) 1568 offset -= rte_pktmbuf_data_len(m); 1569 1570 if (m == 0) 1571 goto process_auth_err; 1572 1573 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1574 1575 l = rte_pktmbuf_data_len(m) - offset; 1576 if (srclen <= l) { 1577 if (CMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1578 goto process_auth_err; 1579 goto process_auth_final; 1580 } 1581 1582 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1583 goto process_auth_err; 1584 1585 n -= l; 1586 1587 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1588 src = rte_pktmbuf_mtod(m, uint8_t *); 1589 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1590 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1591 goto process_auth_err; 1592 n -= l; 1593 } 1594 1595 process_auth_final: 1596 if (CMAC_Final(ctx, dst, (size_t *)&dstlen) != 1) 1597 goto process_auth_err; 1598 1599 CMAC_CTX_cleanup(ctx); 1600 1601 return 0; 1602 1603 process_auth_err: 1604 OPENSSL_LOG(ERR, "Process openssl cmac auth failed"); 1605 return -EINVAL; 1606 } 1607 # endif 1608 /*----------------------------------------------------------------------------*/ 1609 1610 static inline EVP_CIPHER_CTX * 1611 get_local_cipher_ctx(struct openssl_session *sess, struct openssl_qp *qp) 1612 { 1613 /* If the array is not being used, just return the main context. */ 1614 if (sess->ctx_copies_len == 0) 1615 return sess->cipher.ctx; 1616 1617 EVP_CIPHER_CTX **lctx = &sess->qp_ctx[qp->id].cipher; 1618 1619 if (unlikely(*lctx == NULL)) { 1620 #if OPENSSL_VERSION_NUMBER >= 0x30200000L 1621 /* EVP_CIPHER_CTX_dup() added in OSSL 3.2 */ 1622 *lctx = EVP_CIPHER_CTX_dup(sess->cipher.ctx); 1623 return *lctx; 1624 #elif OPENSSL_VERSION_NUMBER >= 0x30000000L 1625 if (sess->chain_order == OPENSSL_CHAIN_COMBINED) { 1626 /* AESNI special-cased to use openssl_aesni_ctx_clone() 1627 * to allow for working around lack of 1628 * EVP_CIPHER_CTX_copy support for 3.0.0 <= OSSL Version 1629 * < 3.2.0. 1630 */ 1631 if (openssl_aesni_ctx_clone(lctx, sess) != 0) 1632 *lctx = NULL; 1633 return *lctx; 1634 } 1635 #endif 1636 1637 *lctx = EVP_CIPHER_CTX_new(); 1638 EVP_CIPHER_CTX_copy(*lctx, sess->cipher.ctx); 1639 } 1640 1641 return *lctx; 1642 } 1643 1644 static inline EVP_MD_CTX * 1645 get_local_auth_ctx(struct openssl_session *sess, struct openssl_qp *qp) 1646 { 1647 /* If the array is not being used, just return the main context. */ 1648 if (sess->ctx_copies_len == 0) 1649 return sess->auth.auth.ctx; 1650 1651 EVP_MD_CTX **lctx = &sess->qp_ctx[qp->id].auth; 1652 1653 if (unlikely(*lctx == NULL)) { 1654 #if OPENSSL_VERSION_NUMBER >= 0x30100000L 1655 /* EVP_MD_CTX_dup() added in OSSL 3.1 */ 1656 *lctx = EVP_MD_CTX_dup(sess->auth.auth.ctx); 1657 #else 1658 *lctx = EVP_MD_CTX_new(); 1659 EVP_MD_CTX_copy(*lctx, sess->auth.auth.ctx); 1660 #endif 1661 } 1662 1663 return *lctx; 1664 } 1665 1666 #if OPENSSL_VERSION_NUMBER >= 0x30000000L 1667 static inline EVP_MAC_CTX * 1668 #else 1669 static inline HMAC_CTX * 1670 #endif 1671 get_local_hmac_ctx(struct openssl_session *sess, struct openssl_qp *qp) 1672 { 1673 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30003000L) 1674 /* For OpenSSL versions 3.0.0 <= v < 3.0.3, re-initing of 1675 * EVP_MAC_CTXs is broken, and doesn't actually reset their 1676 * state. This was fixed in OSSL commit c9ddc5af5199 ("Avoid 1677 * undefined behavior of provided macs on EVP_MAC 1678 * reinitialization"). In cases where the fix is not present, 1679 * fall back to duplicating the context every buffer as a 1680 * workaround, at the cost of performance. 1681 */ 1682 RTE_SET_USED(qp); 1683 return EVP_MAC_CTX_dup(sess->auth.hmac.ctx); 1684 #else 1685 if (sess->ctx_copies_len == 0) 1686 return sess->auth.hmac.ctx; 1687 1688 #if OPENSSL_VERSION_NUMBER >= 0x30000000L 1689 EVP_MAC_CTX **lctx = 1690 #else 1691 HMAC_CTX **lctx = 1692 #endif 1693 &sess->qp_ctx[qp->id].hmac; 1694 1695 if (unlikely(*lctx == NULL)) { 1696 #if OPENSSL_VERSION_NUMBER >= 0x30000000L 1697 *lctx = EVP_MAC_CTX_dup(sess->auth.hmac.ctx); 1698 #else 1699 *lctx = HMAC_CTX_new(); 1700 HMAC_CTX_copy(*lctx, sess->auth.hmac.ctx); 1701 #endif 1702 } 1703 1704 return *lctx; 1705 #endif 1706 } 1707 1708 #if OPENSSL_VERSION_NUMBER >= 0x30000000L 1709 static inline EVP_MAC_CTX * 1710 #else 1711 static inline CMAC_CTX * 1712 #endif 1713 get_local_cmac_ctx(struct openssl_session *sess, struct openssl_qp *qp) 1714 { 1715 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30003000L) 1716 /* For OpenSSL versions 3.0.0 <= v < 3.0.3, re-initing of 1717 * EVP_MAC_CTXs is broken, and doesn't actually reset their 1718 * state. This was fixed in OSSL commit c9ddc5af5199 ("Avoid 1719 * undefined behavior of provided macs on EVP_MAC 1720 * reinitialization"). In cases where the fix is not present, 1721 * fall back to duplicating the context every buffer as a 1722 * workaround, at the cost of performance. 1723 */ 1724 RTE_SET_USED(qp); 1725 return EVP_MAC_CTX_dup(sess->auth.cmac.ctx); 1726 #else 1727 if (sess->ctx_copies_len == 0) 1728 return sess->auth.cmac.ctx; 1729 1730 #if OPENSSL_VERSION_NUMBER >= 0x30000000L 1731 EVP_MAC_CTX **lctx = 1732 #else 1733 CMAC_CTX **lctx = 1734 #endif 1735 &sess->qp_ctx[qp->id].cmac; 1736 1737 if (unlikely(*lctx == NULL)) { 1738 #if OPENSSL_VERSION_NUMBER >= 0x30000000L 1739 *lctx = EVP_MAC_CTX_dup(sess->auth.cmac.ctx); 1740 #else 1741 *lctx = CMAC_CTX_new(); 1742 CMAC_CTX_copy(*lctx, sess->auth.cmac.ctx); 1743 #endif 1744 } 1745 1746 return *lctx; 1747 #endif 1748 } 1749 1750 /** Process auth/cipher combined operation */ 1751 static void 1752 process_openssl_combined_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1753 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1754 struct rte_mbuf *mbuf_dst) 1755 { 1756 /* cipher */ 1757 uint8_t *dst = NULL, *iv, *tag, *aad; 1758 int srclen, aadlen, status = -1; 1759 uint32_t offset; 1760 uint8_t taglen; 1761 1762 /* 1763 * Segmented destination buffer is not supported for 1764 * encryption/decryption 1765 */ 1766 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1767 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1768 return; 1769 } 1770 1771 EVP_CIPHER_CTX *ctx = get_local_cipher_ctx(sess, qp); 1772 1773 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1774 sess->iv.offset); 1775 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1776 srclen = 0; 1777 offset = op->sym->auth.data.offset; 1778 aadlen = op->sym->auth.data.length; 1779 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1780 op->sym->auth.data.offset); 1781 tag = op->sym->auth.digest.data; 1782 if (tag == NULL) 1783 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1784 offset + aadlen); 1785 } else { 1786 srclen = op->sym->aead.data.length; 1787 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1788 op->sym->aead.data.offset); 1789 offset = op->sym->aead.data.offset; 1790 aad = op->sym->aead.aad.data; 1791 aadlen = sess->auth.aad_length; 1792 tag = op->sym->aead.digest.data; 1793 if (tag == NULL) 1794 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1795 offset + srclen); 1796 } 1797 1798 taglen = sess->auth.digest_length; 1799 1800 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1801 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1802 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1803 status = process_openssl_auth_encryption_gcm( 1804 mbuf_src, offset, srclen, 1805 aad, aadlen, iv, 1806 dst, tag, ctx); 1807 else 1808 status = process_openssl_auth_encryption_ccm( 1809 mbuf_src, offset, srclen, 1810 aad, aadlen, iv, 1811 dst, tag, taglen, ctx); 1812 1813 } else { 1814 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1815 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1816 status = process_openssl_auth_decryption_gcm( 1817 mbuf_src, offset, srclen, 1818 aad, aadlen, iv, 1819 dst, tag, ctx); 1820 else 1821 status = process_openssl_auth_decryption_ccm( 1822 mbuf_src, offset, srclen, 1823 aad, aadlen, iv, 1824 dst, tag, taglen, ctx); 1825 } 1826 1827 if (status != 0) { 1828 if (status == (-EFAULT) && 1829 sess->auth.operation == 1830 RTE_CRYPTO_AUTH_OP_VERIFY) 1831 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1832 else 1833 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1834 } 1835 } 1836 1837 /** Process cipher operation */ 1838 static void 1839 process_openssl_cipher_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1840 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1841 struct rte_mbuf *mbuf_dst) 1842 { 1843 uint8_t *dst, *iv; 1844 int srclen, status; 1845 uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0; 1846 1847 /* 1848 * Segmented OOP destination buffer is not supported for encryption/ 1849 * decryption. In case of des3ctr, even inplace segmented buffers are 1850 * not supported. 1851 */ 1852 if (!rte_pktmbuf_is_contiguous(mbuf_dst) && 1853 (!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) { 1854 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1855 return; 1856 } 1857 1858 srclen = op->sym->cipher.data.length; 1859 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1860 op->sym->cipher.data.offset); 1861 1862 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1863 sess->iv.offset); 1864 1865 EVP_CIPHER_CTX *ctx = get_local_cipher_ctx(sess, qp); 1866 1867 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1868 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1869 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1870 op->sym->cipher.data.offset, iv, 1871 srclen, ctx, inplace); 1872 else 1873 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1874 op->sym->cipher.data.offset, iv, 1875 srclen, ctx, inplace); 1876 else 1877 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1878 op->sym->cipher.data.offset, iv, srclen, ctx); 1879 1880 if (status != 0) 1881 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1882 } 1883 1884 /** Process cipher operation */ 1885 static void 1886 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1887 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1888 struct rte_mbuf *mbuf_dst) 1889 { 1890 uint8_t *src, *dst, *iv; 1891 uint8_t block_size, last_block_len; 1892 int srclen, status = 0; 1893 1894 srclen = op->sym->cipher.data.length; 1895 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1896 op->sym->cipher.data.offset); 1897 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1898 op->sym->cipher.data.offset); 1899 1900 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1901 sess->iv.offset); 1902 1903 block_size = DES_BLOCK_SIZE; 1904 1905 last_block_len = srclen % block_size; 1906 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1907 /* Encrypt only with ECB mode XOR IV */ 1908 if (srclen < block_size) { 1909 status = process_openssl_cipher_bpi_encrypt(src, dst, 1910 iv, srclen, 1911 sess->cipher.bpi_ctx); 1912 } else { 1913 srclen -= last_block_len; 1914 /* Encrypt with the block aligned stream with CBC mode */ 1915 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1916 op->sym->cipher.data.offset, iv, 1917 srclen, sess->cipher.ctx, 0); 1918 if (last_block_len) { 1919 /* Point at last block */ 1920 dst += srclen; 1921 /* 1922 * IV is the last encrypted block from 1923 * the previous operation 1924 */ 1925 iv = dst - block_size; 1926 src += srclen; 1927 srclen = last_block_len; 1928 /* Encrypt the last frame with ECB mode */ 1929 status |= process_openssl_cipher_bpi_encrypt(src, 1930 dst, iv, 1931 srclen, sess->cipher.bpi_ctx); 1932 } 1933 } 1934 } else { 1935 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1936 if (srclen < block_size) { 1937 status = process_openssl_cipher_bpi_encrypt(src, dst, 1938 iv, 1939 srclen, 1940 sess->cipher.bpi_ctx); 1941 } else { 1942 if (last_block_len) { 1943 /* Point at last block */ 1944 dst += srclen - last_block_len; 1945 src += srclen - last_block_len; 1946 /* 1947 * IV is the last full block 1948 */ 1949 iv = src - block_size; 1950 /* 1951 * Decrypt the last frame with ECB mode 1952 * (encrypt, as it is the same operation) 1953 */ 1954 status = process_openssl_cipher_bpi_encrypt(src, 1955 dst, iv, 1956 last_block_len, sess->cipher.bpi_ctx); 1957 /* Prepare parameters for CBC mode op */ 1958 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1959 sess->iv.offset); 1960 dst += last_block_len - srclen; 1961 srclen -= last_block_len; 1962 } 1963 1964 /* Decrypt with CBC mode */ 1965 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1966 op->sym->cipher.data.offset, iv, 1967 srclen, sess->cipher.ctx, 0); 1968 } 1969 } 1970 1971 if (status != 0) 1972 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1973 } 1974 1975 /** Process auth operation */ 1976 static void 1977 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1978 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1979 struct rte_mbuf *mbuf_dst) 1980 { 1981 uint8_t *dst; 1982 int srclen, status; 1983 EVP_MD_CTX *ctx_a; 1984 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1985 EVP_MAC_CTX *ctx_h; 1986 EVP_MAC_CTX *ctx_c; 1987 # else 1988 HMAC_CTX *ctx_h; 1989 CMAC_CTX *ctx_c; 1990 # endif 1991 1992 srclen = op->sym->auth.data.length; 1993 1994 dst = qp->temp_digest; 1995 1996 switch (sess->auth.mode) { 1997 case OPENSSL_AUTH_AS_AUTH: 1998 ctx_a = get_local_auth_ctx(sess, qp); 1999 status = process_openssl_auth(mbuf_src, dst, 2000 op->sym->auth.data.offset, NULL, NULL, srclen, 2001 ctx_a, sess->auth.auth.evp_algo); 2002 break; 2003 case OPENSSL_AUTH_AS_HMAC: 2004 ctx_h = get_local_hmac_ctx(sess, qp); 2005 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 2006 status = process_openssl_auth_mac(mbuf_src, dst, 2007 op->sym->auth.data.offset, srclen, 2008 ctx_h); 2009 # else 2010 status = process_openssl_auth_hmac(mbuf_src, dst, 2011 op->sym->auth.data.offset, srclen, 2012 ctx_h); 2013 # endif 2014 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30003000L) 2015 EVP_MAC_CTX_free(ctx_h); 2016 #endif 2017 break; 2018 case OPENSSL_AUTH_AS_CMAC: 2019 ctx_c = get_local_cmac_ctx(sess, qp); 2020 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 2021 status = process_openssl_auth_mac(mbuf_src, dst, 2022 op->sym->auth.data.offset, srclen, 2023 ctx_c); 2024 # else 2025 status = process_openssl_auth_cmac(mbuf_src, dst, 2026 op->sym->auth.data.offset, srclen, 2027 ctx_c); 2028 # endif 2029 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L && OPENSSL_VERSION_NUMBER < 0x30003000L) 2030 EVP_MAC_CTX_free(ctx_c); 2031 #endif 2032 break; 2033 default: 2034 status = -1; 2035 break; 2036 } 2037 2038 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 2039 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data, 2040 sess->auth.digest_length) != 0) { 2041 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 2042 } 2043 } else { 2044 uint8_t *auth_dst; 2045 2046 auth_dst = op->sym->auth.digest.data; 2047 if (auth_dst == NULL) 2048 auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 2049 op->sym->auth.data.offset + 2050 op->sym->auth.data.length); 2051 memcpy(auth_dst, dst, sess->auth.digest_length); 2052 } 2053 2054 if (status != 0) 2055 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 2056 } 2057 2058 /* process dsa sign operation */ 2059 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2060 static int 2061 process_openssl_dsa_sign_op_evp(struct rte_crypto_op *cop, 2062 struct openssl_asym_session *sess) 2063 { 2064 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2065 EVP_PKEY_CTX *dsa_ctx = NULL; 2066 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 2067 EVP_PKEY *pkey = NULL; 2068 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 2069 OSSL_PARAM *params = NULL; 2070 2071 size_t outlen; 2072 unsigned char *dsa_sign_data; 2073 const unsigned char *dsa_sign_data_p; 2074 int ret = -1; 2075 2076 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2077 params = OSSL_PARAM_BLD_to_param(param_bld); 2078 if (!params) { 2079 OSSL_PARAM_BLD_free(param_bld); 2080 return -1; 2081 } 2082 2083 if (key_ctx == NULL 2084 || EVP_PKEY_fromdata_init(key_ctx) <= 0 2085 || EVP_PKEY_fromdata(key_ctx, &pkey, 2086 EVP_PKEY_KEYPAIR, params) <= 0) 2087 goto err_dsa_sign; 2088 2089 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 2090 if (!dsa_ctx) 2091 goto err_dsa_sign; 2092 2093 if (EVP_PKEY_sign_init(dsa_ctx) <= 0) 2094 goto err_dsa_sign; 2095 2096 if (EVP_PKEY_sign(dsa_ctx, NULL, &outlen, op->message.data, 2097 op->message.length) <= 0) 2098 goto err_dsa_sign; 2099 2100 if (outlen <= 0) 2101 goto err_dsa_sign; 2102 2103 dsa_sign_data = OPENSSL_malloc(outlen); 2104 if (!dsa_sign_data) 2105 goto err_dsa_sign; 2106 2107 if (EVP_PKEY_sign(dsa_ctx, dsa_sign_data, &outlen, op->message.data, 2108 op->message.length) <= 0) { 2109 OPENSSL_free(dsa_sign_data); 2110 goto err_dsa_sign; 2111 } 2112 2113 dsa_sign_data_p = (const unsigned char *)dsa_sign_data; 2114 DSA_SIG *sign = d2i_DSA_SIG(NULL, &dsa_sign_data_p, outlen); 2115 if (!sign) { 2116 OPENSSL_LOG(ERR, "%s:%d", __func__, __LINE__); 2117 OPENSSL_free(dsa_sign_data); 2118 goto err_dsa_sign; 2119 } else { 2120 const BIGNUM *r = NULL, *s = NULL; 2121 get_dsa_sign(sign, &r, &s); 2122 2123 op->r.length = BN_bn2bin(r, op->r.data); 2124 op->s.length = BN_bn2bin(s, op->s.data); 2125 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2126 } 2127 2128 ret = 0; 2129 DSA_SIG_free(sign); 2130 OPENSSL_free(dsa_sign_data); 2131 2132 err_dsa_sign: 2133 if (params) 2134 OSSL_PARAM_free(params); 2135 EVP_PKEY_CTX_free(key_ctx); 2136 EVP_PKEY_CTX_free(dsa_ctx); 2137 EVP_PKEY_free(pkey); 2138 return ret; 2139 } 2140 2141 /* process dsa verify operation */ 2142 static int 2143 process_openssl_dsa_verify_op_evp(struct rte_crypto_op *cop, 2144 struct openssl_asym_session *sess) 2145 { 2146 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2147 DSA_SIG *sign = DSA_SIG_new(); 2148 BIGNUM *r = NULL, *s = NULL; 2149 BIGNUM *pub_key = NULL; 2150 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 2151 OSSL_PARAM *params = NULL; 2152 EVP_PKEY *pkey = NULL; 2153 EVP_PKEY_CTX *dsa_ctx = NULL; 2154 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 2155 unsigned char *dsa_sig = NULL; 2156 size_t sig_len; 2157 int ret = -1; 2158 2159 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2160 if (!param_bld) { 2161 OPENSSL_LOG(ERR, " %s:%d", __func__, __LINE__); 2162 return -1; 2163 } 2164 2165 r = BN_bin2bn(op->r.data, op->r.length, r); 2166 s = BN_bin2bn(op->s.data, op->s.length, s); 2167 pub_key = BN_bin2bn(op->y.data, op->y.length, pub_key); 2168 if (!r || !s || !pub_key) { 2169 BN_free(r); 2170 BN_free(s); 2171 BN_free(pub_key); 2172 OSSL_PARAM_BLD_free(param_bld); 2173 goto err_dsa_verify; 2174 } 2175 2176 set_dsa_sign(sign, r, s); 2177 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PUB_KEY, pub_key)) { 2178 OSSL_PARAM_BLD_free(param_bld); 2179 goto err_dsa_verify; 2180 } 2181 2182 params = OSSL_PARAM_BLD_to_param(param_bld); 2183 if (!params) { 2184 OSSL_PARAM_BLD_free(param_bld); 2185 goto err_dsa_verify; 2186 } 2187 2188 if (key_ctx == NULL 2189 || EVP_PKEY_fromdata_init(key_ctx) <= 0 2190 || EVP_PKEY_fromdata(key_ctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2191 goto err_dsa_verify; 2192 2193 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 2194 if (!dsa_ctx) 2195 goto err_dsa_verify; 2196 2197 if (!sign) 2198 goto err_dsa_verify; 2199 2200 sig_len = i2d_DSA_SIG(sign, &dsa_sig); 2201 if (EVP_PKEY_verify_init(dsa_ctx) <= 0) 2202 goto err_dsa_verify; 2203 2204 ret = EVP_PKEY_verify(dsa_ctx, dsa_sig, sig_len, 2205 op->message.data, op->message.length); 2206 if (ret == 1) { 2207 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2208 ret = 0; 2209 } 2210 2211 OPENSSL_free(dsa_sig); 2212 err_dsa_verify: 2213 if (sign) 2214 DSA_SIG_free(sign); 2215 if (params) 2216 OSSL_PARAM_free(params); 2217 EVP_PKEY_CTX_free(key_ctx); 2218 EVP_PKEY_CTX_free(dsa_ctx); 2219 2220 BN_free(pub_key); 2221 EVP_PKEY_free(pkey); 2222 2223 return ret; 2224 } 2225 #else 2226 static int 2227 process_openssl_dsa_sign_op(struct rte_crypto_op *cop, 2228 struct openssl_asym_session *sess) 2229 { 2230 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2231 DSA *dsa = sess->u.s.dsa; 2232 DSA_SIG *sign = NULL; 2233 2234 sign = DSA_do_sign(op->message.data, 2235 op->message.length, 2236 dsa); 2237 2238 if (sign == NULL) { 2239 OPENSSL_LOG(ERR, "%s:%d", __func__, __LINE__); 2240 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2241 } else { 2242 const BIGNUM *r = NULL, *s = NULL; 2243 get_dsa_sign(sign, &r, &s); 2244 2245 op->r.length = BN_bn2bin(r, op->r.data); 2246 op->s.length = BN_bn2bin(s, op->s.data); 2247 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2248 } 2249 2250 DSA_SIG_free(sign); 2251 2252 return 0; 2253 } 2254 2255 /* process dsa verify operation */ 2256 static int 2257 process_openssl_dsa_verify_op(struct rte_crypto_op *cop, 2258 struct openssl_asym_session *sess) 2259 { 2260 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2261 DSA *dsa = sess->u.s.dsa; 2262 int ret; 2263 DSA_SIG *sign = DSA_SIG_new(); 2264 BIGNUM *r = NULL, *s = NULL; 2265 BIGNUM *pub_key = NULL; 2266 2267 if (sign == NULL) { 2268 OPENSSL_LOG(ERR, " %s:%d", __func__, __LINE__); 2269 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2270 return -1; 2271 } 2272 2273 r = BN_bin2bn(op->r.data, 2274 op->r.length, 2275 r); 2276 s = BN_bin2bn(op->s.data, 2277 op->s.length, 2278 s); 2279 pub_key = BN_bin2bn(op->y.data, 2280 op->y.length, 2281 pub_key); 2282 if (!r || !s || !pub_key) { 2283 BN_free(r); 2284 BN_free(s); 2285 BN_free(pub_key); 2286 2287 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2288 return -1; 2289 } 2290 set_dsa_sign(sign, r, s); 2291 set_dsa_pub_key(dsa, pub_key); 2292 2293 ret = DSA_do_verify(op->message.data, 2294 op->message.length, 2295 sign, 2296 dsa); 2297 2298 if (ret != 1) 2299 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2300 else 2301 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2302 2303 DSA_SIG_free(sign); 2304 2305 return 0; 2306 } 2307 #endif 2308 2309 /* process dh operation */ 2310 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2311 static int 2312 process_openssl_dh_op_evp(struct rte_crypto_op *cop, 2313 struct openssl_asym_session *sess) 2314 { 2315 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2316 OSSL_PARAM_BLD *param_bld = sess->u.dh.param_bld; 2317 OSSL_PARAM_BLD *param_bld_peer = sess->u.dh.param_bld_peer; 2318 OSSL_PARAM *params = NULL; 2319 EVP_PKEY *dhpkey = NULL; 2320 EVP_PKEY *peerkey = NULL; 2321 BIGNUM *priv_key = NULL; 2322 BIGNUM *pub_key = NULL; 2323 int ret = -1; 2324 2325 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2326 EVP_PKEY_CTX *dh_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2327 if (dh_ctx == NULL || param_bld == NULL) 2328 return ret; 2329 2330 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2331 OSSL_PARAM *params_peer = NULL; 2332 2333 if (!param_bld_peer) 2334 return ret; 2335 2336 pub_key = BN_bin2bn(op->pub_key.data, op->pub_key.length, 2337 pub_key); 2338 if (pub_key == NULL) { 2339 OSSL_PARAM_BLD_free(param_bld_peer); 2340 return ret; 2341 } 2342 2343 if (!OSSL_PARAM_BLD_push_BN(param_bld_peer, OSSL_PKEY_PARAM_PUB_KEY, 2344 pub_key)) { 2345 OPENSSL_LOG(ERR, "Failed to set public key"); 2346 OSSL_PARAM_BLD_free(param_bld_peer); 2347 BN_free(pub_key); 2348 return ret; 2349 } 2350 2351 params_peer = OSSL_PARAM_BLD_to_param(param_bld_peer); 2352 if (!params_peer) { 2353 OSSL_PARAM_BLD_free(param_bld_peer); 2354 BN_free(pub_key); 2355 return ret; 2356 } 2357 2358 EVP_PKEY_CTX *peer_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2359 if (EVP_PKEY_keygen_init(peer_ctx) != 1) { 2360 OSSL_PARAM_free(params_peer); 2361 BN_free(pub_key); 2362 return ret; 2363 } 2364 2365 if (EVP_PKEY_CTX_set_params(peer_ctx, params_peer) != 1) { 2366 EVP_PKEY_CTX_free(peer_ctx); 2367 OSSL_PARAM_free(params_peer); 2368 BN_free(pub_key); 2369 return ret; 2370 } 2371 2372 if (EVP_PKEY_keygen(peer_ctx, &peerkey) != 1) { 2373 EVP_PKEY_CTX_free(peer_ctx); 2374 OSSL_PARAM_free(params_peer); 2375 BN_free(pub_key); 2376 return ret; 2377 } 2378 2379 priv_key = BN_bin2bn(op->priv_key.data, op->priv_key.length, 2380 priv_key); 2381 if (priv_key == NULL) { 2382 EVP_PKEY_CTX_free(peer_ctx); 2383 OSSL_PARAM_free(params_peer); 2384 BN_free(pub_key); 2385 return ret; 2386 } 2387 2388 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY, 2389 priv_key)) { 2390 OPENSSL_LOG(ERR, "Failed to set private key"); 2391 EVP_PKEY_CTX_free(peer_ctx); 2392 OSSL_PARAM_free(params_peer); 2393 BN_free(pub_key); 2394 BN_free(priv_key); 2395 return ret; 2396 } 2397 2398 OSSL_PARAM_free(params_peer); 2399 EVP_PKEY_CTX_free(peer_ctx); 2400 } 2401 2402 params = OSSL_PARAM_BLD_to_param(param_bld); 2403 if (!params) 2404 goto err_dh; 2405 2406 if (EVP_PKEY_keygen_init(dh_ctx) != 1) 2407 goto err_dh; 2408 2409 if (EVP_PKEY_CTX_set_params(dh_ctx, params) != 1) 2410 goto err_dh; 2411 2412 if (EVP_PKEY_keygen(dh_ctx, &dhpkey) != 1) 2413 goto err_dh; 2414 2415 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2416 OPENSSL_LOG(DEBUG, "%s:%d updated pub key", __func__, __LINE__); 2417 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PUB_KEY, &pub_key)) 2418 goto err_dh; 2419 /* output public key */ 2420 op->pub_key.length = BN_bn2bin(pub_key, op->pub_key.data); 2421 } 2422 2423 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2424 2425 OPENSSL_LOG(DEBUG, "%s:%d updated priv key", __func__, __LINE__); 2426 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PRIV_KEY, &priv_key)) 2427 goto err_dh; 2428 2429 /* provide generated private key back to user */ 2430 op->priv_key.length = BN_bn2bin(priv_key, op->priv_key.data); 2431 } 2432 2433 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2434 size_t skey_len; 2435 EVP_PKEY_CTX *sc_ctx = EVP_PKEY_CTX_new(dhpkey, NULL); 2436 if (!sc_ctx) 2437 goto err_dh; 2438 2439 if (EVP_PKEY_derive_init(sc_ctx) <= 0) { 2440 EVP_PKEY_CTX_free(sc_ctx); 2441 goto err_dh; 2442 } 2443 2444 if (!peerkey) { 2445 EVP_PKEY_CTX_free(sc_ctx); 2446 goto err_dh; 2447 } 2448 2449 if (EVP_PKEY_derive_set_peer(sc_ctx, peerkey) <= 0) { 2450 EVP_PKEY_CTX_free(sc_ctx); 2451 goto err_dh; 2452 } 2453 2454 /* Determine buffer length */ 2455 if (EVP_PKEY_derive(sc_ctx, NULL, &skey_len) <= 0) { 2456 EVP_PKEY_CTX_free(sc_ctx); 2457 goto err_dh; 2458 } 2459 2460 if (EVP_PKEY_derive(sc_ctx, op->shared_secret.data, &skey_len) <= 0) { 2461 EVP_PKEY_CTX_free(sc_ctx); 2462 goto err_dh; 2463 } 2464 2465 op->shared_secret.length = skey_len; 2466 EVP_PKEY_CTX_free(sc_ctx); 2467 } 2468 2469 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2470 ret = 0; 2471 2472 err_dh: 2473 BN_free(pub_key); 2474 BN_free(priv_key); 2475 if (params) 2476 OSSL_PARAM_free(params); 2477 EVP_PKEY_free(dhpkey); 2478 EVP_PKEY_free(peerkey); 2479 2480 EVP_PKEY_CTX_free(dh_ctx); 2481 2482 return ret; 2483 } 2484 #else 2485 static int 2486 process_openssl_dh_op(struct rte_crypto_op *cop, 2487 struct openssl_asym_session *sess) 2488 { 2489 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2490 struct rte_crypto_asym_op *asym_op = cop->asym; 2491 DH *dh_key = sess->u.dh.dh_key; 2492 BIGNUM *priv_key = NULL; 2493 int ret = 0; 2494 2495 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2496 /* compute shared secret using peer public key 2497 * and current private key 2498 * shared secret = peer_key ^ priv_key mod p 2499 */ 2500 BIGNUM *peer_key = NULL; 2501 2502 /* copy private key and peer key and compute shared secret */ 2503 peer_key = BN_bin2bn(op->pub_key.data, 2504 op->pub_key.length, 2505 peer_key); 2506 if (peer_key == NULL) { 2507 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2508 return -1; 2509 } 2510 priv_key = BN_bin2bn(op->priv_key.data, 2511 op->priv_key.length, 2512 priv_key); 2513 if (priv_key == NULL) { 2514 BN_free(peer_key); 2515 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2516 return -1; 2517 } 2518 ret = set_dh_priv_key(dh_key, priv_key); 2519 if (ret) { 2520 OPENSSL_LOG(ERR, "Failed to set private key"); 2521 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2522 BN_free(peer_key); 2523 BN_free(priv_key); 2524 return 0; 2525 } 2526 2527 ret = DH_compute_key( 2528 op->shared_secret.data, 2529 peer_key, dh_key); 2530 if (ret < 0) { 2531 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2532 BN_free(peer_key); 2533 /* priv key is already loaded into dh, 2534 * let's not free that directly here. 2535 * DH_free() will auto free it later. 2536 */ 2537 return 0; 2538 } 2539 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2540 op->shared_secret.length = ret; 2541 BN_free(peer_key); 2542 return 0; 2543 } 2544 2545 /* 2546 * other options are public and private key generations. 2547 * 2548 * if user provides private key, 2549 * then first set DH with user provided private key 2550 */ 2551 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE && 2552 op->priv_key.length) { 2553 /* generate public key using user-provided private key 2554 * pub_key = g ^ priv_key mod p 2555 */ 2556 2557 /* load private key into DH */ 2558 priv_key = BN_bin2bn(op->priv_key.data, 2559 op->priv_key.length, 2560 priv_key); 2561 if (priv_key == NULL) { 2562 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2563 return -1; 2564 } 2565 ret = set_dh_priv_key(dh_key, priv_key); 2566 if (ret) { 2567 OPENSSL_LOG(ERR, "Failed to set private key"); 2568 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2569 BN_free(priv_key); 2570 return 0; 2571 } 2572 } 2573 2574 /* generate public and private key pair. 2575 * 2576 * if private key already set, generates only public key. 2577 * 2578 * if private key is not already set, then set it to random value 2579 * and update internal private key. 2580 */ 2581 if (!DH_generate_key(dh_key)) { 2582 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2583 return 0; 2584 } 2585 2586 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2587 const BIGNUM *pub_key = NULL; 2588 2589 OPENSSL_LOG(DEBUG, "%s:%d update public key", 2590 __func__, __LINE__); 2591 2592 /* get the generated keys */ 2593 get_dh_pub_key(dh_key, &pub_key); 2594 2595 /* output public key */ 2596 op->pub_key.length = BN_bn2bin(pub_key, 2597 op->pub_key.data); 2598 } 2599 2600 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2601 const BIGNUM *priv_key = NULL; 2602 2603 OPENSSL_LOG(DEBUG, "%s:%d updated priv key", 2604 __func__, __LINE__); 2605 2606 /* get the generated keys */ 2607 get_dh_priv_key(dh_key, &priv_key); 2608 2609 /* provide generated private key back to user */ 2610 op->priv_key.length = BN_bn2bin(priv_key, 2611 op->priv_key.data); 2612 } 2613 2614 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2615 2616 return 0; 2617 } 2618 #endif 2619 2620 /* process modinv operation */ 2621 static int 2622 process_openssl_modinv_op(struct rte_crypto_op *cop, 2623 struct openssl_asym_session *sess) 2624 { 2625 struct rte_crypto_asym_op *op = cop->asym; 2626 BIGNUM *base = BN_CTX_get(sess->u.m.ctx); 2627 BIGNUM *res = BN_CTX_get(sess->u.m.ctx); 2628 2629 if (unlikely(base == NULL || res == NULL)) { 2630 BN_free(base); 2631 BN_free(res); 2632 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2633 return -1; 2634 } 2635 2636 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 2637 op->modinv.base.length, base); 2638 2639 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) { 2640 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2641 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data); 2642 } else { 2643 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2644 } 2645 2646 BN_clear(res); 2647 BN_clear(base); 2648 2649 return 0; 2650 } 2651 2652 /* process modexp operation */ 2653 static int 2654 process_openssl_modexp_op(struct rte_crypto_op *cop, 2655 struct openssl_asym_session *sess) 2656 { 2657 struct rte_crypto_asym_op *op = cop->asym; 2658 BIGNUM *base = BN_CTX_get(sess->u.e.ctx); 2659 BIGNUM *res = BN_CTX_get(sess->u.e.ctx); 2660 2661 if (unlikely(base == NULL || res == NULL)) { 2662 BN_free(base); 2663 BN_free(res); 2664 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2665 return -1; 2666 } 2667 2668 base = BN_bin2bn((const unsigned char *)op->modex.base.data, 2669 op->modex.base.length, base); 2670 2671 if (BN_mod_exp(res, base, sess->u.e.exp, 2672 sess->u.e.mod, sess->u.e.ctx)) { 2673 op->modex.result.length = BN_bn2bin(res, op->modex.result.data); 2674 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2675 } else { 2676 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2677 } 2678 2679 BN_clear(res); 2680 BN_clear(base); 2681 2682 return 0; 2683 } 2684 2685 /* process rsa operations */ 2686 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2687 static int 2688 process_openssl_rsa_op_evp(struct rte_crypto_op *cop, 2689 struct openssl_asym_session *sess) 2690 { 2691 struct rte_crypto_asym_op *op = cop->asym; 2692 uint32_t pad = sess->u.r.pad; 2693 uint8_t *tmp; 2694 size_t outlen = 0; 2695 int ret = -1; 2696 2697 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2698 EVP_PKEY_CTX *rsa_ctx = sess->u.r.ctx; 2699 if (!rsa_ctx) 2700 return ret; 2701 2702 switch (pad) { 2703 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2704 pad = RSA_PKCS1_PADDING; 2705 break; 2706 case RTE_CRYPTO_RSA_PADDING_NONE: 2707 pad = RSA_NO_PADDING; 2708 break; 2709 default: 2710 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2711 OPENSSL_LOG(ERR, 2712 "rsa pad type not supported %d", pad); 2713 return ret; 2714 } 2715 2716 switch (op->rsa.op_type) { 2717 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2718 if (EVP_PKEY_encrypt_init(rsa_ctx) != 1) 2719 goto err_rsa; 2720 2721 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2722 goto err_rsa; 2723 2724 if (EVP_PKEY_encrypt(rsa_ctx, NULL, &outlen, 2725 op->rsa.message.data, 2726 op->rsa.message.length) <= 0) 2727 goto err_rsa; 2728 2729 if (outlen <= 0) 2730 goto err_rsa; 2731 2732 if (EVP_PKEY_encrypt(rsa_ctx, op->rsa.cipher.data, &outlen, 2733 op->rsa.message.data, 2734 op->rsa.message.length) <= 0) 2735 goto err_rsa; 2736 op->rsa.cipher.length = outlen; 2737 2738 OPENSSL_LOG(DEBUG, 2739 "length of encrypted text %zu", outlen); 2740 break; 2741 2742 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2743 if (EVP_PKEY_decrypt_init(rsa_ctx) != 1) 2744 goto err_rsa; 2745 2746 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2747 goto err_rsa; 2748 2749 if (EVP_PKEY_decrypt(rsa_ctx, NULL, &outlen, 2750 op->rsa.cipher.data, 2751 op->rsa.cipher.length) <= 0) 2752 goto err_rsa; 2753 2754 if (outlen <= 0) 2755 goto err_rsa; 2756 2757 if (EVP_PKEY_decrypt(rsa_ctx, op->rsa.message.data, &outlen, 2758 op->rsa.cipher.data, 2759 op->rsa.cipher.length) <= 0) 2760 goto err_rsa; 2761 op->rsa.message.length = outlen; 2762 2763 OPENSSL_LOG(DEBUG, "length of decrypted text %zu", outlen); 2764 break; 2765 2766 case RTE_CRYPTO_ASYM_OP_SIGN: 2767 if (EVP_PKEY_sign_init(rsa_ctx) <= 0) 2768 goto err_rsa; 2769 2770 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2771 goto err_rsa; 2772 2773 if (EVP_PKEY_sign(rsa_ctx, NULL, &outlen, 2774 op->rsa.message.data, 2775 op->rsa.message.length) <= 0) 2776 goto err_rsa; 2777 2778 if (outlen <= 0) 2779 goto err_rsa; 2780 2781 if (EVP_PKEY_sign(rsa_ctx, op->rsa.sign.data, &outlen, 2782 op->rsa.message.data, 2783 op->rsa.message.length) <= 0) 2784 goto err_rsa; 2785 op->rsa.sign.length = outlen; 2786 break; 2787 2788 case RTE_CRYPTO_ASYM_OP_VERIFY: 2789 if (EVP_PKEY_verify_recover_init(rsa_ctx) <= 0) 2790 goto err_rsa; 2791 2792 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2793 goto err_rsa; 2794 2795 if (EVP_PKEY_verify_recover(rsa_ctx, NULL, &outlen, 2796 op->rsa.sign.data, 2797 op->rsa.sign.length) <= 0) 2798 goto err_rsa; 2799 2800 if ((outlen <= 0) || (outlen != op->rsa.sign.length)) 2801 goto err_rsa; 2802 2803 tmp = OPENSSL_malloc(outlen); 2804 if (tmp == NULL) { 2805 OPENSSL_LOG(ERR, "Memory allocation failed"); 2806 goto err_rsa; 2807 } 2808 2809 if (EVP_PKEY_verify_recover(rsa_ctx, tmp, &outlen, 2810 op->rsa.sign.data, 2811 op->rsa.sign.length) <= 0) { 2812 OPENSSL_free(tmp); 2813 goto err_rsa; 2814 } 2815 2816 OPENSSL_LOG(DEBUG, 2817 "Length of public_decrypt %zu " 2818 "length of message %zd", 2819 outlen, op->rsa.message.length); 2820 if (CRYPTO_memcmp(tmp, op->rsa.message.data, 2821 op->rsa.message.length)) { 2822 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 2823 } 2824 OPENSSL_free(tmp); 2825 break; 2826 2827 default: 2828 /* allow ops with invalid args to be pushed to 2829 * completion queue 2830 */ 2831 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2832 goto err_rsa; 2833 } 2834 2835 ret = 0; 2836 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2837 err_rsa: 2838 return ret; 2839 2840 } 2841 2842 static int 2843 process_openssl_ecfpm_op_evp(struct rte_crypto_op *cop, 2844 struct openssl_asym_session *sess) 2845 { 2846 const EC_GROUP *ecgrp = sess->u.ec.group; 2847 EC_POINT *ecpt = NULL; 2848 BN_CTX *ctx = NULL; 2849 BIGNUM *n = NULL; 2850 int ret = -1; 2851 2852 n = BN_bin2bn((const unsigned char *) 2853 cop->asym->ecpm.scalar.data, 2854 cop->asym->ecpm.scalar.length, 2855 BN_new()); 2856 2857 ctx = BN_CTX_new(); 2858 if (!ctx) 2859 goto err_ecfpm; 2860 2861 if (!EC_POINT_mul(ecgrp, ecpt, n, NULL, NULL, ctx)) 2862 goto err_ecfpm; 2863 2864 if (cop->asym->flags & RTE_CRYPTO_ASYM_FLAG_PUB_KEY_COMPRESSED) { 2865 unsigned char *buf = cop->asym->ecpm.r.x.data; 2866 size_t sz; 2867 2868 sz = EC_POINT_point2oct(ecgrp, ecpt, POINT_CONVERSION_COMPRESSED, buf, 0, ctx); 2869 if (!sz) 2870 goto err_ecfpm; 2871 2872 cop->asym->ecpm.r.x.length = sz; 2873 } 2874 2875 err_ecfpm: 2876 BN_CTX_free(ctx); 2877 BN_free(n); 2878 return ret; 2879 } 2880 2881 static int 2882 process_openssl_sm2_op_evp(struct rte_crypto_op *cop, 2883 struct openssl_asym_session *sess) 2884 { 2885 EVP_PKEY_CTX *kctx = NULL, *sctx = NULL, *cctx = NULL; 2886 struct rte_crypto_asym_op *op = cop->asym; 2887 OSSL_PARAM *params = sess->u.sm2.params; 2888 EVP_MD_CTX *md_ctx = NULL; 2889 ECDSA_SIG *ec_sign = NULL; 2890 EVP_MD *check_md = NULL; 2891 EVP_PKEY *pkey = NULL; 2892 int ret = -1; 2893 2894 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2895 2896 if (cop->asym->sm2.k.data != NULL) 2897 goto err_sm2; 2898 2899 switch (op->sm2.op_type) { 2900 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2901 { 2902 OSSL_PARAM *eparams = sess->u.sm2.params; 2903 size_t output_len = 0; 2904 2905 kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_SM2, NULL); 2906 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2907 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2908 goto err_sm2; 2909 2910 cctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2911 if (!cctx) 2912 goto err_sm2; 2913 2914 if (!EVP_PKEY_encrypt_init(cctx)) 2915 goto err_sm2; 2916 2917 if (!EVP_PKEY_CTX_set_params(cctx, eparams)) 2918 goto err_sm2; 2919 2920 if (!EVP_PKEY_encrypt(cctx, op->sm2.cipher.data, &output_len, 2921 op->sm2.message.data, 2922 op->sm2.message.length)) 2923 goto err_sm2; 2924 op->sm2.cipher.length = output_len; 2925 } 2926 break; 2927 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2928 { 2929 OSSL_PARAM *eparams = sess->u.sm2.params; 2930 2931 kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_SM2, NULL); 2932 if (kctx == NULL 2933 || EVP_PKEY_fromdata_init(kctx) <= 0 2934 || EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2935 goto err_sm2; 2936 2937 cctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2938 if (!cctx) 2939 goto err_sm2; 2940 2941 if (!EVP_PKEY_decrypt_init(cctx)) 2942 goto err_sm2; 2943 2944 if (!EVP_PKEY_CTX_set_params(cctx, eparams)) 2945 goto err_sm2; 2946 2947 if (!EVP_PKEY_decrypt(cctx, op->sm2.message.data, &op->sm2.message.length, 2948 op->sm2.cipher.data, op->sm2.cipher.length)) 2949 goto err_sm2; 2950 } 2951 break; 2952 case RTE_CRYPTO_ASYM_OP_SIGN: 2953 { 2954 unsigned char signbuf[128] = {0}; 2955 const unsigned char *signptr; 2956 const BIGNUM *r, *s; 2957 size_t signlen; 2958 2959 kctx = EVP_PKEY_CTX_new_from_name(NULL, "SM2", NULL); 2960 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2961 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2962 goto err_sm2; 2963 2964 md_ctx = EVP_MD_CTX_new(); 2965 if (!md_ctx) 2966 goto err_sm2; 2967 2968 sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2969 if (!sctx) 2970 goto err_sm2; 2971 2972 EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx); 2973 2974 check_md = EVP_MD_fetch(NULL, "sm3", NULL); 2975 if (!check_md) 2976 goto err_sm2; 2977 2978 if (!EVP_DigestSignInit(md_ctx, NULL, check_md, NULL, pkey)) 2979 goto err_sm2; 2980 2981 if (EVP_PKEY_CTX_set1_id(sctx, op->sm2.id.data, op->sm2.id.length) <= 0) 2982 goto err_sm2; 2983 2984 if (!EVP_DigestSignUpdate(md_ctx, op->sm2.message.data, 2985 op->sm2.message.length)) 2986 goto err_sm2; 2987 2988 if (!EVP_DigestSignFinal(md_ctx, NULL, &signlen)) 2989 goto err_sm2; 2990 2991 if (!EVP_DigestSignFinal(md_ctx, signbuf, &signlen)) 2992 goto err_sm2; 2993 2994 signptr = signbuf; 2995 ec_sign = d2i_ECDSA_SIG(NULL, &signptr, signlen); 2996 if (!ec_sign) 2997 goto err_sm2; 2998 2999 r = ECDSA_SIG_get0_r(ec_sign); 3000 s = ECDSA_SIG_get0_s(ec_sign); 3001 if (!r || !s) 3002 goto err_sm2; 3003 3004 op->sm2.r.length = BN_num_bytes(r); 3005 op->sm2.s.length = BN_num_bytes(s); 3006 BN_bn2bin(r, op->sm2.r.data); 3007 BN_bn2bin(s, op->sm2.s.data); 3008 3009 ECDSA_SIG_free(ec_sign); 3010 } 3011 break; 3012 case RTE_CRYPTO_ASYM_OP_VERIFY: 3013 { 3014 unsigned char signbuf[128] = {0}, *signbuf_new = NULL; 3015 BIGNUM *r = NULL, *s = NULL; 3016 size_t signlen; 3017 3018 kctx = EVP_PKEY_CTX_new_from_name(NULL, "SM2", NULL); 3019 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 3020 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_PUBLIC_KEY, params) <= 0) 3021 goto err_sm2; 3022 3023 if (!EVP_PKEY_is_a(pkey, "SM2")) 3024 goto err_sm2; 3025 3026 md_ctx = EVP_MD_CTX_new(); 3027 if (!md_ctx) 3028 goto err_sm2; 3029 3030 sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 3031 if (!sctx) 3032 goto err_sm2; 3033 3034 EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx); 3035 3036 check_md = EVP_MD_fetch(NULL, "sm3", NULL); 3037 if (!check_md) 3038 goto err_sm2; 3039 3040 if (!EVP_DigestVerifyInit(md_ctx, NULL, check_md, NULL, pkey)) 3041 goto err_sm2; 3042 3043 if (EVP_PKEY_CTX_set1_id(sctx, op->sm2.id.data, op->sm2.id.length) <= 0) 3044 goto err_sm2; 3045 3046 if (!EVP_DigestVerifyUpdate(md_ctx, op->sm2.message.data, 3047 op->sm2.message.length)) 3048 goto err_sm2; 3049 3050 ec_sign = ECDSA_SIG_new(); 3051 if (!ec_sign) 3052 goto err_sm2; 3053 3054 r = BN_bin2bn(op->sm2.r.data, op->sm2.r.length, r); 3055 s = BN_bin2bn(op->sm2.s.data, op->sm2.s.length, s); 3056 if (!r || !s) 3057 goto err_sm2; 3058 3059 if (!ECDSA_SIG_set0(ec_sign, r, s)) { 3060 BN_free(r); 3061 BN_free(s); 3062 goto err_sm2; 3063 } 3064 3065 r = NULL; 3066 s = NULL; 3067 3068 signbuf_new = signbuf; 3069 signlen = i2d_ECDSA_SIG(ec_sign, (unsigned char **)&signbuf_new); 3070 if (signlen <= 0) 3071 goto err_sm2; 3072 3073 if (!EVP_DigestVerifyFinal(md_ctx, signbuf_new, signlen)) 3074 goto err_sm2; 3075 3076 BN_free(r); 3077 BN_free(s); 3078 ECDSA_SIG_free(ec_sign); 3079 } 3080 break; 3081 default: 3082 /* allow ops with invalid args to be pushed to 3083 * completion queue 3084 */ 3085 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3086 goto err_sm2; 3087 } 3088 3089 ret = 0; 3090 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3091 err_sm2: 3092 EVP_MD_free(check_md); 3093 EVP_MD_CTX_free(md_ctx); 3094 3095 EVP_PKEY_CTX_free(kctx); 3096 3097 EVP_PKEY_CTX_free(sctx); 3098 3099 EVP_PKEY_CTX_free(cctx); 3100 3101 EVP_PKEY_free(pkey); 3102 3103 return ret; 3104 } 3105 3106 static int 3107 process_openssl_eddsa_op_evp(struct rte_crypto_op *cop, 3108 struct openssl_asym_session *sess) 3109 { 3110 static const char * const instance[] = {"Ed25519", "Ed25519ctx", "Ed25519ph", 3111 "Ed448", "Ed448ph"}; 3112 EVP_PKEY_CTX *kctx = NULL, *sctx = NULL, *cctx = NULL; 3113 const uint8_t curve_id = sess->u.eddsa.curve_id; 3114 struct rte_crypto_asym_op *op = cop->asym; 3115 OSSL_PARAM *params = sess->u.eddsa.params; 3116 OSSL_PARAM_BLD *iparam_bld = NULL; 3117 OSSL_PARAM *iparams = NULL; 3118 uint8_t signbuf[128] = {0}; 3119 EVP_MD_CTX *md_ctx = NULL; 3120 EVP_PKEY *pkey = NULL; 3121 size_t signlen; 3122 int ret = -1; 3123 3124 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 3125 3126 iparam_bld = OSSL_PARAM_BLD_new(); 3127 if (!iparam_bld) 3128 goto err_eddsa; 3129 3130 if (op->eddsa.instance == RTE_CRYPTO_EDCURVE_25519CTX) { 3131 OSSL_PARAM_BLD_push_octet_string(iparam_bld, "context-string", 3132 op->eddsa.context.data, op->eddsa.context.length); 3133 3134 } 3135 3136 OSSL_PARAM_BLD_push_utf8_string(iparam_bld, "instance", 3137 instance[op->eddsa.instance], strlen(instance[op->eddsa.instance])); 3138 3139 iparams = OSSL_PARAM_BLD_to_param(iparam_bld); 3140 if (!iparams) 3141 goto err_eddsa; 3142 3143 switch (op->eddsa.op_type) { 3144 case RTE_CRYPTO_ASYM_OP_SIGN: 3145 { 3146 if (curve_id == RTE_CRYPTO_EC_GROUP_ED25519) 3147 kctx = EVP_PKEY_CTX_new_from_name(NULL, "ED25519", NULL); 3148 else 3149 kctx = EVP_PKEY_CTX_new_from_name(NULL, "ED448", NULL); 3150 3151 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 3152 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 3153 goto err_eddsa; 3154 3155 md_ctx = EVP_MD_CTX_new(); 3156 if (!md_ctx) 3157 goto err_eddsa; 3158 3159 sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 3160 if (!sctx) 3161 goto err_eddsa; 3162 3163 EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx); 3164 3165 #if (OPENSSL_VERSION_NUMBER >= 0x30300000L) 3166 if (!EVP_DigestSignInit_ex(md_ctx, NULL, NULL, NULL, NULL, pkey, iparams)) 3167 goto err_eddsa; 3168 #else 3169 if (op->eddsa.instance == RTE_CRYPTO_EDCURVE_25519 || 3170 op->eddsa.instance == RTE_CRYPTO_EDCURVE_448) { 3171 if (!EVP_DigestSignInit(md_ctx, NULL, NULL, NULL, pkey)) 3172 goto err_eddsa; 3173 } else 3174 goto err_eddsa; 3175 #endif 3176 3177 if (!EVP_DigestSign(md_ctx, NULL, &signlen, op->eddsa.message.data, 3178 op->eddsa.message.length)) 3179 goto err_eddsa; 3180 3181 if (signlen > RTE_DIM(signbuf)) 3182 goto err_eddsa; 3183 3184 if (!EVP_DigestSign(md_ctx, signbuf, &signlen, op->eddsa.message.data, 3185 op->eddsa.message.length)) 3186 goto err_eddsa; 3187 3188 memcpy(op->eddsa.sign.data, &signbuf[0], signlen); 3189 op->eddsa.sign.length = signlen; 3190 } 3191 break; 3192 case RTE_CRYPTO_ASYM_OP_VERIFY: 3193 { 3194 if (curve_id == RTE_CRYPTO_EC_GROUP_ED25519) 3195 kctx = EVP_PKEY_CTX_new_from_name(NULL, "ED25519", NULL); 3196 else 3197 kctx = EVP_PKEY_CTX_new_from_name(NULL, "ED448", NULL); 3198 3199 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 3200 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_PUBLIC_KEY, params) <= 0) 3201 goto err_eddsa; 3202 3203 md_ctx = EVP_MD_CTX_new(); 3204 if (!md_ctx) 3205 goto err_eddsa; 3206 3207 sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 3208 if (!sctx) 3209 goto err_eddsa; 3210 3211 EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx); 3212 3213 #if (OPENSSL_VERSION_NUMBER >= 0x30300000L) 3214 if (!EVP_DigestVerifyInit_ex(md_ctx, NULL, NULL, NULL, NULL, pkey, iparams)) 3215 goto err_eddsa; 3216 #else 3217 if (op->eddsa.instance == RTE_CRYPTO_EDCURVE_25519 || 3218 op->eddsa.instance == RTE_CRYPTO_EDCURVE_448) { 3219 if (!EVP_DigestVerifyInit(md_ctx, NULL, NULL, NULL, pkey)) 3220 goto err_eddsa; 3221 } else 3222 goto err_eddsa; 3223 #endif 3224 3225 signlen = op->eddsa.sign.length; 3226 memcpy(&signbuf[0], op->eddsa.sign.data, op->eddsa.sign.length); 3227 3228 ret = EVP_DigestVerify(md_ctx, signbuf, signlen, op->eddsa.message.data, 3229 op->eddsa.message.length); 3230 if (ret == 0) 3231 goto err_eddsa; 3232 } 3233 break; 3234 default: 3235 /* allow ops with invalid args to be pushed to 3236 * completion queue 3237 */ 3238 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3239 goto err_eddsa; 3240 } 3241 3242 ret = 0; 3243 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3244 err_eddsa: 3245 OSSL_PARAM_BLD_free(iparam_bld); 3246 3247 EVP_PKEY_CTX_free(sctx); 3248 3249 EVP_PKEY_CTX_free(cctx); 3250 3251 EVP_PKEY_free(pkey); 3252 3253 return ret; 3254 } 3255 #else 3256 static int 3257 process_openssl_rsa_op(struct rte_crypto_op *cop, 3258 struct openssl_asym_session *sess) 3259 { 3260 int ret = 0; 3261 struct rte_crypto_asym_op *op = cop->asym; 3262 RSA *rsa = sess->u.r.rsa; 3263 uint32_t pad = sess->u.r.pad; 3264 uint8_t *tmp; 3265 3266 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3267 3268 switch (pad) { 3269 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 3270 pad = RSA_PKCS1_PADDING; 3271 break; 3272 case RTE_CRYPTO_RSA_PADDING_NONE: 3273 pad = RSA_NO_PADDING; 3274 break; 3275 default: 3276 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3277 OPENSSL_LOG(ERR, 3278 "rsa pad type not supported %d", pad); 3279 return 0; 3280 } 3281 3282 switch (op->rsa.op_type) { 3283 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 3284 ret = RSA_public_encrypt(op->rsa.message.length, 3285 op->rsa.message.data, 3286 op->rsa.cipher.data, 3287 rsa, 3288 pad); 3289 3290 if (ret > 0) 3291 op->rsa.cipher.length = ret; 3292 OPENSSL_LOG(DEBUG, 3293 "length of encrypted text %d", ret); 3294 break; 3295 3296 case RTE_CRYPTO_ASYM_OP_DECRYPT: 3297 ret = RSA_private_decrypt(op->rsa.cipher.length, 3298 op->rsa.cipher.data, 3299 op->rsa.message.data, 3300 rsa, 3301 pad); 3302 if (ret > 0) 3303 op->rsa.message.length = ret; 3304 break; 3305 3306 case RTE_CRYPTO_ASYM_OP_SIGN: 3307 ret = RSA_private_encrypt(op->rsa.message.length, 3308 op->rsa.message.data, 3309 op->rsa.sign.data, 3310 rsa, 3311 pad); 3312 if (ret > 0) 3313 op->rsa.sign.length = ret; 3314 break; 3315 3316 case RTE_CRYPTO_ASYM_OP_VERIFY: 3317 tmp = rte_malloc(NULL, op->rsa.sign.length, 0); 3318 if (tmp == NULL) { 3319 OPENSSL_LOG(ERR, "Memory allocation failed"); 3320 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 3321 break; 3322 } 3323 ret = RSA_public_decrypt(op->rsa.sign.length, 3324 op->rsa.sign.data, 3325 tmp, 3326 rsa, 3327 pad); 3328 3329 OPENSSL_LOG(DEBUG, 3330 "Length of public_decrypt %d " 3331 "length of message %zd", 3332 ret, op->rsa.message.length); 3333 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data, 3334 op->rsa.message.length))) { 3335 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 3336 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 3337 } 3338 rte_free(tmp); 3339 break; 3340 3341 default: 3342 /* allow ops with invalid args to be pushed to 3343 * completion queue 3344 */ 3345 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3346 break; 3347 } 3348 3349 if (ret < 0) 3350 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 3351 3352 return 0; 3353 } 3354 3355 static int 3356 process_openssl_ecfpm_op(struct rte_crypto_op *cop, 3357 struct openssl_asym_session *sess) 3358 { 3359 RTE_SET_USED(cop); 3360 RTE_SET_USED(sess); 3361 return -ENOTSUP; 3362 } 3363 3364 static int 3365 process_openssl_sm2_op(struct rte_crypto_op *cop, 3366 struct openssl_asym_session *sess) 3367 { 3368 RTE_SET_USED(cop); 3369 RTE_SET_USED(sess); 3370 return -ENOTSUP; 3371 } 3372 3373 static int 3374 process_openssl_eddsa_op(struct rte_crypto_op *cop, 3375 struct openssl_asym_session *sess) 3376 { 3377 RTE_SET_USED(cop); 3378 RTE_SET_USED(sess); 3379 return -ENOTSUP; 3380 } 3381 #endif 3382 3383 static int 3384 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op, 3385 struct openssl_asym_session *sess) 3386 { 3387 int retval = 0; 3388 3389 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 3390 3391 switch (sess->xfrm_type) { 3392 case RTE_CRYPTO_ASYM_XFORM_RSA: 3393 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3394 retval = process_openssl_rsa_op_evp(op, sess); 3395 # else 3396 retval = process_openssl_rsa_op(op, sess); 3397 #endif 3398 break; 3399 case RTE_CRYPTO_ASYM_XFORM_MODEX: 3400 retval = process_openssl_modexp_op(op, sess); 3401 break; 3402 case RTE_CRYPTO_ASYM_XFORM_MODINV: 3403 retval = process_openssl_modinv_op(op, sess); 3404 break; 3405 case RTE_CRYPTO_ASYM_XFORM_DH: 3406 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3407 retval = process_openssl_dh_op_evp(op, sess); 3408 # else 3409 retval = process_openssl_dh_op(op, sess); 3410 #endif 3411 break; 3412 case RTE_CRYPTO_ASYM_XFORM_DSA: 3413 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3414 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 3415 retval = process_openssl_dsa_sign_op_evp(op, sess); 3416 else if (op->asym->dsa.op_type == 3417 RTE_CRYPTO_ASYM_OP_VERIFY) 3418 retval = 3419 process_openssl_dsa_verify_op_evp(op, sess); 3420 #else 3421 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 3422 retval = process_openssl_dsa_sign_op(op, sess); 3423 else if (op->asym->dsa.op_type == 3424 RTE_CRYPTO_ASYM_OP_VERIFY) 3425 retval = 3426 process_openssl_dsa_verify_op(op, sess); 3427 else 3428 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3429 #endif 3430 break; 3431 case RTE_CRYPTO_ASYM_XFORM_ECFPM: 3432 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3433 retval = process_openssl_ecfpm_op_evp(op, sess); 3434 #else 3435 retval = process_openssl_ecfpm_op(op, sess); 3436 #endif 3437 break; 3438 case RTE_CRYPTO_ASYM_XFORM_SM2: 3439 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3440 retval = process_openssl_sm2_op_evp(op, sess); 3441 #else 3442 retval = process_openssl_sm2_op(op, sess); 3443 #endif 3444 break; 3445 case RTE_CRYPTO_ASYM_XFORM_EDDSA: 3446 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3447 retval = process_openssl_eddsa_op_evp(op, sess); 3448 #else 3449 retval = process_openssl_eddsa_op(op, sess); 3450 #endif 3451 break; 3452 default: 3453 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3454 break; 3455 } 3456 if (!retval) { 3457 /* op processed so push to completion queue as processed */ 3458 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 3459 if (retval) 3460 /* return error if failed to put in completion queue */ 3461 retval = -1; 3462 } 3463 3464 return retval; 3465 } 3466 3467 static void 3468 copy_plaintext(struct rte_mbuf *m_src, struct rte_mbuf *m_dst, 3469 struct rte_crypto_op *op) 3470 { 3471 uint8_t *p_src, *p_dst; 3472 3473 p_src = rte_pktmbuf_mtod(m_src, uint8_t *); 3474 p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *); 3475 3476 /** 3477 * Copy the content between cipher offset and auth offset 3478 * for generating correct digest. 3479 */ 3480 if (op->sym->cipher.data.offset > op->sym->auth.data.offset) 3481 memcpy(p_dst + op->sym->auth.data.offset, 3482 p_src + op->sym->auth.data.offset, 3483 op->sym->cipher.data.offset - 3484 op->sym->auth.data.offset); 3485 } 3486 3487 /** Process crypto operation for mbuf */ 3488 static int 3489 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 3490 struct openssl_session *sess) 3491 { 3492 struct rte_mbuf *msrc, *mdst; 3493 int retval; 3494 3495 msrc = op->sym->m_src; 3496 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 3497 3498 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 3499 3500 switch (sess->chain_order) { 3501 case OPENSSL_CHAIN_ONLY_CIPHER: 3502 process_openssl_cipher_op(qp, op, sess, msrc, mdst); 3503 break; 3504 case OPENSSL_CHAIN_ONLY_AUTH: 3505 process_openssl_auth_op(qp, op, sess, msrc, mdst); 3506 break; 3507 case OPENSSL_CHAIN_CIPHER_AUTH: 3508 process_openssl_cipher_op(qp, op, sess, msrc, mdst); 3509 /* OOP */ 3510 if (msrc != mdst) 3511 copy_plaintext(msrc, mdst, op); 3512 process_openssl_auth_op(qp, op, sess, mdst, mdst); 3513 break; 3514 case OPENSSL_CHAIN_AUTH_CIPHER: 3515 process_openssl_auth_op(qp, op, sess, msrc, mdst); 3516 process_openssl_cipher_op(qp, op, sess, msrc, mdst); 3517 break; 3518 case OPENSSL_CHAIN_COMBINED: 3519 process_openssl_combined_op(qp, op, sess, msrc, mdst); 3520 break; 3521 case OPENSSL_CHAIN_CIPHER_BPI: 3522 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 3523 break; 3524 default: 3525 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3526 break; 3527 } 3528 3529 /* Free session if a session-less crypto op */ 3530 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 3531 openssl_reset_session(sess); 3532 memset(sess, 0, sizeof(struct openssl_session)); 3533 rte_mempool_put(qp->sess_mp, op->sym->session); 3534 op->sym->session = NULL; 3535 } 3536 3537 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 3538 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3539 3540 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 3541 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 3542 else 3543 retval = -1; 3544 3545 return retval; 3546 } 3547 3548 /* 3549 *------------------------------------------------------------------------------ 3550 * PMD Framework 3551 *------------------------------------------------------------------------------ 3552 */ 3553 3554 /** Enqueue burst */ 3555 static uint16_t 3556 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 3557 uint16_t nb_ops) 3558 { 3559 void *sess; 3560 struct openssl_qp *qp = queue_pair; 3561 int i, retval; 3562 3563 for (i = 0; i < nb_ops; i++) { 3564 sess = get_session(qp, ops[i]); 3565 if (unlikely(sess == NULL)) 3566 goto enqueue_err; 3567 3568 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) 3569 retval = process_op(qp, ops[i], 3570 (struct openssl_session *) sess); 3571 else 3572 retval = process_asym_op(qp, ops[i], 3573 (struct openssl_asym_session *) sess); 3574 if (unlikely(retval < 0)) 3575 goto enqueue_err; 3576 } 3577 3578 qp->stats.enqueued_count += i; 3579 return i; 3580 3581 enqueue_err: 3582 qp->stats.enqueue_err_count++; 3583 return i; 3584 } 3585 3586 /** Dequeue burst */ 3587 static uint16_t 3588 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 3589 uint16_t nb_ops) 3590 { 3591 struct openssl_qp *qp = queue_pair; 3592 3593 unsigned int nb_dequeued = 0; 3594 3595 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 3596 (void **)ops, nb_ops, NULL); 3597 qp->stats.dequeued_count += nb_dequeued; 3598 3599 return nb_dequeued; 3600 } 3601 3602 /** Create OPENSSL crypto device */ 3603 static int 3604 cryptodev_openssl_create(const char *name, 3605 struct rte_vdev_device *vdev, 3606 struct rte_cryptodev_pmd_init_params *init_params) 3607 { 3608 struct rte_cryptodev *dev; 3609 struct openssl_private *internals; 3610 3611 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 3612 if (dev == NULL) { 3613 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 3614 goto init_error; 3615 } 3616 3617 dev->driver_id = cryptodev_driver_id; 3618 dev->dev_ops = rte_openssl_pmd_ops; 3619 3620 /* register rx/tx burst functions for data path */ 3621 dev->dequeue_burst = openssl_pmd_dequeue_burst; 3622 dev->enqueue_burst = openssl_pmd_enqueue_burst; 3623 3624 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 3625 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 3626 RTE_CRYPTODEV_FF_CPU_AESNI | 3627 RTE_CRYPTODEV_FF_IN_PLACE_SGL | 3628 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 3629 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | 3630 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO | 3631 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP | 3632 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT | 3633 RTE_CRYPTODEV_FF_SYM_SESSIONLESS; 3634 3635 internals = dev->data->dev_private; 3636 3637 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 3638 3639 rte_cryptodev_pmd_probing_finish(dev); 3640 3641 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3642 /* Load legacy provider 3643 * Some algorithms are no longer available in earlier version of openssl, 3644 * unless the legacy provider explicitly loaded. e.g. DES 3645 */ 3646 ossl_legacy_provider_load(); 3647 # endif 3648 return 0; 3649 3650 init_error: 3651 OPENSSL_LOG(ERR, "driver %s: create failed", 3652 init_params->name); 3653 3654 cryptodev_openssl_remove(vdev); 3655 return -EFAULT; 3656 } 3657 3658 /** Initialise OPENSSL crypto device */ 3659 static int 3660 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 3661 { 3662 struct rte_cryptodev_pmd_init_params init_params = { 3663 "", 3664 sizeof(struct openssl_private), 3665 rte_socket_id(), 3666 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 3667 }; 3668 const char *name; 3669 const char *input_args; 3670 3671 name = rte_vdev_device_name(vdev); 3672 if (name == NULL) 3673 return -EINVAL; 3674 input_args = rte_vdev_device_args(vdev); 3675 3676 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 3677 3678 return cryptodev_openssl_create(name, vdev, &init_params); 3679 } 3680 3681 /** Uninitialise OPENSSL crypto device */ 3682 static int 3683 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 3684 { 3685 struct rte_cryptodev *cryptodev; 3686 const char *name; 3687 3688 name = rte_vdev_device_name(vdev); 3689 if (name == NULL) 3690 return -EINVAL; 3691 3692 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 3693 if (cryptodev == NULL) 3694 return -ENODEV; 3695 3696 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3697 ossl_legacy_provider_unload(); 3698 # endif 3699 return rte_cryptodev_pmd_destroy(cryptodev); 3700 } 3701 3702 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 3703 .probe = cryptodev_openssl_probe, 3704 .remove = cryptodev_openssl_remove 3705 }; 3706 3707 static struct cryptodev_driver openssl_crypto_drv; 3708 3709 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 3710 cryptodev_openssl_pmd_drv); 3711 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 3712 "max_nb_queue_pairs=<int> " 3713 "socket_id=<int>"); 3714 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 3715 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 3716 RTE_LOG_REGISTER_DEFAULT(openssl_logtype_driver, INFO); 3717