1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_common.h> 6 #include <rte_hexdump.h> 7 #include <rte_cryptodev.h> 8 #include <cryptodev_pmd.h> 9 #include <bus_vdev_driver.h> 10 #include <rte_malloc.h> 11 #include <rte_cpuflags.h> 12 13 #include <openssl/cmac.h> 14 #include <openssl/hmac.h> 15 #include <openssl/evp.h> 16 #include <openssl/ec.h> 17 18 #include "openssl_pmd_private.h" 19 #include "compat.h" 20 21 #define DES_BLOCK_SIZE 8 22 23 static uint8_t cryptodev_driver_id; 24 25 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 26 static HMAC_CTX *HMAC_CTX_new(void) 27 { 28 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 29 30 if (ctx != NULL) 31 HMAC_CTX_init(ctx); 32 return ctx; 33 } 34 35 static void HMAC_CTX_free(HMAC_CTX *ctx) 36 { 37 if (ctx != NULL) { 38 HMAC_CTX_cleanup(ctx); 39 OPENSSL_free(ctx); 40 } 41 } 42 #endif 43 44 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 45 46 #include <openssl/provider.h> 47 #include <openssl/core_names.h> 48 #include <openssl/param_build.h> 49 50 #define MAX_OSSL_ALGO_NAME_SIZE 16 51 52 OSSL_PROVIDER *legacy; 53 OSSL_PROVIDER *deflt; 54 55 static void ossl_legacy_provider_load(void) 56 { 57 /* Load Multiple providers into the default (NULL) library context */ 58 legacy = OSSL_PROVIDER_load(NULL, "legacy"); 59 if (legacy == NULL) { 60 OPENSSL_LOG(ERR, "Failed to load Legacy provider\n"); 61 return; 62 } 63 64 deflt = OSSL_PROVIDER_load(NULL, "default"); 65 if (deflt == NULL) { 66 OPENSSL_LOG(ERR, "Failed to load Default provider\n"); 67 OSSL_PROVIDER_unload(legacy); 68 return; 69 } 70 } 71 72 static void ossl_legacy_provider_unload(void) 73 { 74 OSSL_PROVIDER_unload(legacy); 75 OSSL_PROVIDER_unload(deflt); 76 } 77 78 static __rte_always_inline const char * 79 digest_name_get(enum rte_crypto_auth_algorithm algo) 80 { 81 switch (algo) { 82 case RTE_CRYPTO_AUTH_MD5_HMAC: 83 return OSSL_DIGEST_NAME_MD5; 84 case RTE_CRYPTO_AUTH_SHA1_HMAC: 85 return OSSL_DIGEST_NAME_SHA1; 86 case RTE_CRYPTO_AUTH_SHA224_HMAC: 87 return OSSL_DIGEST_NAME_SHA2_224; 88 case RTE_CRYPTO_AUTH_SHA256_HMAC: 89 return OSSL_DIGEST_NAME_SHA2_256; 90 case RTE_CRYPTO_AUTH_SHA384_HMAC: 91 return OSSL_DIGEST_NAME_SHA2_384; 92 case RTE_CRYPTO_AUTH_SHA512_HMAC: 93 return OSSL_DIGEST_NAME_SHA2_512; 94 default: 95 return NULL; 96 } 97 } 98 #endif 99 100 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 101 102 /*----------------------------------------------------------------------------*/ 103 104 /** 105 * Increment counter by 1 106 * Counter is 64 bit array, big-endian 107 */ 108 static void 109 ctr_inc(uint8_t *ctr) 110 { 111 uint64_t *ctr64 = (uint64_t *)ctr; 112 113 *ctr64 = __builtin_bswap64(*ctr64); 114 (*ctr64)++; 115 *ctr64 = __builtin_bswap64(*ctr64); 116 } 117 118 /* 119 *------------------------------------------------------------------------------ 120 * Session Prepare 121 *------------------------------------------------------------------------------ 122 */ 123 124 /** Get xform chain order */ 125 static enum openssl_chain_order 126 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 127 { 128 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 129 130 if (xform != NULL) { 131 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 132 if (xform->next == NULL) 133 res = OPENSSL_CHAIN_ONLY_AUTH; 134 else if (xform->next->type == 135 RTE_CRYPTO_SYM_XFORM_CIPHER) 136 res = OPENSSL_CHAIN_AUTH_CIPHER; 137 } 138 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 139 if (xform->next == NULL) 140 res = OPENSSL_CHAIN_ONLY_CIPHER; 141 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 142 res = OPENSSL_CHAIN_CIPHER_AUTH; 143 } 144 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 145 res = OPENSSL_CHAIN_COMBINED; 146 } 147 148 return res; 149 } 150 151 /** Get session cipher key from input cipher key */ 152 static void 153 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key) 154 { 155 memcpy(session_key, input_key, keylen); 156 } 157 158 /** Get key ede 24 bytes standard from input key */ 159 static int 160 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede) 161 { 162 int res = 0; 163 164 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 165 switch (keylen) { 166 case 24: 167 memcpy(key_ede, key, 24); 168 break; 169 case 16: 170 /* K3 = K1 */ 171 memcpy(key_ede, key, 16); 172 memcpy(key_ede + 16, key, 8); 173 break; 174 case 8: 175 /* K1 = K2 = K3 (DES compatibility) */ 176 memcpy(key_ede, key, 8); 177 memcpy(key_ede + 8, key, 8); 178 memcpy(key_ede + 16, key, 8); 179 break; 180 default: 181 OPENSSL_LOG(ERR, "Unsupported key size"); 182 res = -EINVAL; 183 } 184 185 return res; 186 } 187 188 /** Get adequate openssl function for input cipher algorithm */ 189 static uint8_t 190 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 191 const EVP_CIPHER **algo) 192 { 193 int res = 0; 194 195 if (algo != NULL) { 196 switch (sess_algo) { 197 case RTE_CRYPTO_CIPHER_3DES_CBC: 198 switch (keylen) { 199 case 8: 200 *algo = EVP_des_cbc(); 201 break; 202 case 16: 203 *algo = EVP_des_ede_cbc(); 204 break; 205 case 24: 206 *algo = EVP_des_ede3_cbc(); 207 break; 208 default: 209 res = -EINVAL; 210 } 211 break; 212 case RTE_CRYPTO_CIPHER_3DES_CTR: 213 break; 214 case RTE_CRYPTO_CIPHER_AES_CBC: 215 switch (keylen) { 216 case 16: 217 *algo = EVP_aes_128_cbc(); 218 break; 219 case 24: 220 *algo = EVP_aes_192_cbc(); 221 break; 222 case 32: 223 *algo = EVP_aes_256_cbc(); 224 break; 225 default: 226 res = -EINVAL; 227 } 228 break; 229 case RTE_CRYPTO_CIPHER_AES_CTR: 230 switch (keylen) { 231 case 16: 232 *algo = EVP_aes_128_ctr(); 233 break; 234 case 24: 235 *algo = EVP_aes_192_ctr(); 236 break; 237 case 32: 238 *algo = EVP_aes_256_ctr(); 239 break; 240 default: 241 res = -EINVAL; 242 } 243 break; 244 default: 245 res = -EINVAL; 246 break; 247 } 248 } else { 249 res = -EINVAL; 250 } 251 252 return res; 253 } 254 255 /** Get adequate openssl function for input auth algorithm */ 256 static uint8_t 257 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 258 const EVP_MD **algo) 259 { 260 int res = 0; 261 262 if (algo != NULL) { 263 switch (sessalgo) { 264 case RTE_CRYPTO_AUTH_MD5: 265 case RTE_CRYPTO_AUTH_MD5_HMAC: 266 *algo = EVP_md5(); 267 break; 268 case RTE_CRYPTO_AUTH_SHA1: 269 case RTE_CRYPTO_AUTH_SHA1_HMAC: 270 *algo = EVP_sha1(); 271 break; 272 case RTE_CRYPTO_AUTH_SHA224: 273 case RTE_CRYPTO_AUTH_SHA224_HMAC: 274 *algo = EVP_sha224(); 275 break; 276 case RTE_CRYPTO_AUTH_SHA256: 277 case RTE_CRYPTO_AUTH_SHA256_HMAC: 278 *algo = EVP_sha256(); 279 break; 280 case RTE_CRYPTO_AUTH_SHA384: 281 case RTE_CRYPTO_AUTH_SHA384_HMAC: 282 *algo = EVP_sha384(); 283 break; 284 case RTE_CRYPTO_AUTH_SHA512: 285 case RTE_CRYPTO_AUTH_SHA512_HMAC: 286 *algo = EVP_sha512(); 287 break; 288 default: 289 res = -EINVAL; 290 break; 291 } 292 } else { 293 res = -EINVAL; 294 } 295 296 return res; 297 } 298 299 /** Get adequate openssl function for input cipher algorithm */ 300 static uint8_t 301 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 302 const EVP_CIPHER **algo) 303 { 304 int res = 0; 305 306 if (algo != NULL) { 307 switch (sess_algo) { 308 case RTE_CRYPTO_AEAD_AES_GCM: 309 switch (keylen) { 310 case 16: 311 *algo = EVP_aes_128_gcm(); 312 break; 313 case 24: 314 *algo = EVP_aes_192_gcm(); 315 break; 316 case 32: 317 *algo = EVP_aes_256_gcm(); 318 break; 319 default: 320 res = -EINVAL; 321 } 322 break; 323 case RTE_CRYPTO_AEAD_AES_CCM: 324 switch (keylen) { 325 case 16: 326 *algo = EVP_aes_128_ccm(); 327 break; 328 case 24: 329 *algo = EVP_aes_192_ccm(); 330 break; 331 case 32: 332 *algo = EVP_aes_256_ccm(); 333 break; 334 default: 335 res = -EINVAL; 336 } 337 break; 338 default: 339 res = -EINVAL; 340 break; 341 } 342 } else { 343 res = -EINVAL; 344 } 345 346 return res; 347 } 348 349 /* Set session AEAD encryption parameters */ 350 static int 351 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 352 enum rte_crypto_aead_algorithm algo, 353 uint8_t tag_len, const uint8_t *key) 354 { 355 int iv_type = 0; 356 unsigned int do_ccm; 357 358 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 359 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 360 361 /* Select AEAD algo */ 362 switch (algo) { 363 case RTE_CRYPTO_AEAD_AES_GCM: 364 iv_type = EVP_CTRL_GCM_SET_IVLEN; 365 if (tag_len != 16) 366 return -EINVAL; 367 do_ccm = 0; 368 break; 369 case RTE_CRYPTO_AEAD_AES_CCM: 370 iv_type = EVP_CTRL_CCM_SET_IVLEN; 371 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 372 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 373 return -EINVAL; 374 do_ccm = 1; 375 break; 376 default: 377 return -ENOTSUP; 378 } 379 380 sess->cipher.mode = OPENSSL_CIPHER_LIB; 381 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 382 383 if (get_aead_algo(algo, sess->cipher.key.length, 384 &sess->cipher.evp_algo) != 0) 385 return -EINVAL; 386 387 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 388 389 sess->chain_order = OPENSSL_CHAIN_COMBINED; 390 391 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 392 NULL, NULL, NULL) <= 0) 393 return -EINVAL; 394 395 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, 396 NULL) <= 0) 397 return -EINVAL; 398 399 if (do_ccm) 400 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 401 tag_len, NULL); 402 403 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 404 return -EINVAL; 405 406 return 0; 407 } 408 409 /* Set session AEAD decryption parameters */ 410 static int 411 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 412 enum rte_crypto_aead_algorithm algo, 413 uint8_t tag_len, const uint8_t *key) 414 { 415 int iv_type = 0; 416 unsigned int do_ccm = 0; 417 418 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 419 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 420 421 /* Select AEAD algo */ 422 switch (algo) { 423 case RTE_CRYPTO_AEAD_AES_GCM: 424 iv_type = EVP_CTRL_GCM_SET_IVLEN; 425 if (tag_len != 16) 426 return -EINVAL; 427 break; 428 case RTE_CRYPTO_AEAD_AES_CCM: 429 iv_type = EVP_CTRL_CCM_SET_IVLEN; 430 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 431 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 432 return -EINVAL; 433 do_ccm = 1; 434 break; 435 default: 436 return -ENOTSUP; 437 } 438 439 sess->cipher.mode = OPENSSL_CIPHER_LIB; 440 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 441 442 if (get_aead_algo(algo, sess->cipher.key.length, 443 &sess->cipher.evp_algo) != 0) 444 return -EINVAL; 445 446 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 447 448 sess->chain_order = OPENSSL_CHAIN_COMBINED; 449 450 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 451 NULL, NULL, NULL) <= 0) 452 return -EINVAL; 453 454 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, 455 sess->iv.length, NULL) <= 0) 456 return -EINVAL; 457 458 if (do_ccm) 459 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 460 tag_len, NULL); 461 462 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 463 return -EINVAL; 464 465 return 0; 466 } 467 468 /** Set session cipher parameters */ 469 static int 470 openssl_set_session_cipher_parameters(struct openssl_session *sess, 471 const struct rte_crypto_sym_xform *xform) 472 { 473 /* Select cipher direction */ 474 sess->cipher.direction = xform->cipher.op; 475 /* Select cipher key */ 476 sess->cipher.key.length = xform->cipher.key.length; 477 478 /* Set IV parameters */ 479 sess->iv.offset = xform->cipher.iv.offset; 480 sess->iv.length = xform->cipher.iv.length; 481 482 /* Select cipher algo */ 483 switch (xform->cipher.algo) { 484 case RTE_CRYPTO_CIPHER_3DES_CBC: 485 case RTE_CRYPTO_CIPHER_AES_CBC: 486 case RTE_CRYPTO_CIPHER_AES_CTR: 487 sess->cipher.mode = OPENSSL_CIPHER_LIB; 488 sess->cipher.algo = xform->cipher.algo; 489 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 490 491 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 492 &sess->cipher.evp_algo) != 0) 493 return -EINVAL; 494 495 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 496 sess->cipher.key.data); 497 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 498 if (EVP_EncryptInit_ex(sess->cipher.ctx, 499 sess->cipher.evp_algo, 500 NULL, xform->cipher.key.data, 501 NULL) != 1) { 502 return -EINVAL; 503 } 504 } else if (sess->cipher.direction == 505 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 506 if (EVP_DecryptInit_ex(sess->cipher.ctx, 507 sess->cipher.evp_algo, 508 NULL, xform->cipher.key.data, 509 NULL) != 1) { 510 return -EINVAL; 511 } 512 } 513 514 break; 515 516 case RTE_CRYPTO_CIPHER_3DES_CTR: 517 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 518 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 519 520 if (get_cipher_key_ede(xform->cipher.key.data, 521 sess->cipher.key.length, 522 sess->cipher.key.data) != 0) 523 return -EINVAL; 524 break; 525 526 case RTE_CRYPTO_CIPHER_DES_CBC: 527 sess->cipher.algo = xform->cipher.algo; 528 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 529 sess->cipher.evp_algo = EVP_des_cbc(); 530 531 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 532 sess->cipher.key.data); 533 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 534 if (EVP_EncryptInit_ex(sess->cipher.ctx, 535 sess->cipher.evp_algo, 536 NULL, xform->cipher.key.data, 537 NULL) != 1) { 538 return -EINVAL; 539 } 540 } else if (sess->cipher.direction == 541 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 542 if (EVP_DecryptInit_ex(sess->cipher.ctx, 543 sess->cipher.evp_algo, 544 NULL, xform->cipher.key.data, 545 NULL) != 1) { 546 return -EINVAL; 547 } 548 } 549 550 break; 551 552 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 553 sess->cipher.algo = xform->cipher.algo; 554 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 555 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 556 sess->cipher.evp_algo = EVP_des_cbc(); 557 558 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 559 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 560 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 561 NULL, xform->cipher.key.data, 0) != 1) 562 return -EINVAL; 563 564 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 565 sess->cipher.key.data); 566 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 567 if (EVP_EncryptInit_ex(sess->cipher.ctx, 568 sess->cipher.evp_algo, 569 NULL, xform->cipher.key.data, 570 NULL) != 1) { 571 return -EINVAL; 572 } 573 } else if (sess->cipher.direction == 574 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 575 if (EVP_DecryptInit_ex(sess->cipher.ctx, 576 sess->cipher.evp_algo, 577 NULL, xform->cipher.key.data, 578 NULL) != 1) { 579 return -EINVAL; 580 } 581 } 582 583 break; 584 default: 585 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 586 return -ENOTSUP; 587 } 588 589 return 0; 590 } 591 592 /* Set session auth parameters */ 593 static int 594 openssl_set_session_auth_parameters(struct openssl_session *sess, 595 const struct rte_crypto_sym_xform *xform) 596 { 597 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 598 char algo_name[MAX_OSSL_ALGO_NAME_SIZE]; 599 OSSL_PARAM params[2]; 600 const char *algo; 601 EVP_MAC *mac; 602 # endif 603 /* Select auth generate/verify */ 604 sess->auth.operation = xform->auth.op; 605 sess->auth.algo = xform->auth.algo; 606 607 sess->auth.digest_length = xform->auth.digest_length; 608 609 /* Select auth algo */ 610 switch (xform->auth.algo) { 611 case RTE_CRYPTO_AUTH_AES_GMAC: 612 /* 613 * OpenSSL requires GMAC to be a GCM operation 614 * with no cipher data length 615 */ 616 sess->cipher.key.length = xform->auth.key.length; 617 618 /* Set IV parameters */ 619 sess->iv.offset = xform->auth.iv.offset; 620 sess->iv.length = xform->auth.iv.length; 621 622 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 623 return openssl_set_sess_aead_enc_param(sess, 624 RTE_CRYPTO_AEAD_AES_GCM, 625 xform->auth.digest_length, 626 xform->auth.key.data); 627 else 628 return openssl_set_sess_aead_dec_param(sess, 629 RTE_CRYPTO_AEAD_AES_GCM, 630 xform->auth.digest_length, 631 xform->auth.key.data); 632 break; 633 634 case RTE_CRYPTO_AUTH_MD5: 635 case RTE_CRYPTO_AUTH_SHA1: 636 case RTE_CRYPTO_AUTH_SHA224: 637 case RTE_CRYPTO_AUTH_SHA256: 638 case RTE_CRYPTO_AUTH_SHA384: 639 case RTE_CRYPTO_AUTH_SHA512: 640 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 641 if (get_auth_algo(xform->auth.algo, 642 &sess->auth.auth.evp_algo) != 0) 643 return -EINVAL; 644 sess->auth.auth.ctx = EVP_MD_CTX_create(); 645 break; 646 647 case RTE_CRYPTO_AUTH_AES_CMAC: 648 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 649 if (xform->auth.key.length == 16) 650 algo = SN_aes_128_cbc; 651 else if (xform->auth.key.length == 24) 652 algo = SN_aes_192_cbc; 653 else if (xform->auth.key.length == 32) 654 algo = SN_aes_256_cbc; 655 else 656 return -EINVAL; 657 658 rte_memcpy(algo_name, algo, strlen(algo) + 1); 659 params[0] = OSSL_PARAM_construct_utf8_string( 660 OSSL_MAC_PARAM_CIPHER, algo_name, 0); 661 params[1] = OSSL_PARAM_construct_end(); 662 663 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 664 mac = EVP_MAC_fetch(NULL, OSSL_MAC_NAME_CMAC, NULL); 665 sess->auth.cmac.ctx = EVP_MAC_CTX_new(mac); 666 EVP_MAC_free(mac); 667 668 if (EVP_MAC_init(sess->auth.cmac.ctx, 669 xform->auth.key.data, 670 xform->auth.key.length, 671 params) != 1) 672 return -EINVAL; 673 # else 674 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 675 sess->auth.cmac.ctx = CMAC_CTX_new(); 676 if (get_cipher_algo(RTE_CRYPTO_CIPHER_AES_CBC, 677 xform->auth.key.length, 678 &sess->auth.cmac.evp_algo) != 0) 679 return -EINVAL; 680 if (CMAC_Init(sess->auth.cmac.ctx, 681 xform->auth.key.data, 682 xform->auth.key.length, 683 sess->auth.cmac.evp_algo, NULL) != 1) 684 return -EINVAL; 685 # endif 686 break; 687 688 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 689 case RTE_CRYPTO_AUTH_MD5_HMAC: 690 case RTE_CRYPTO_AUTH_SHA1_HMAC: 691 case RTE_CRYPTO_AUTH_SHA224_HMAC: 692 case RTE_CRYPTO_AUTH_SHA256_HMAC: 693 case RTE_CRYPTO_AUTH_SHA384_HMAC: 694 case RTE_CRYPTO_AUTH_SHA512_HMAC: 695 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 696 697 algo = digest_name_get(xform->auth.algo); 698 if (!algo) 699 return -EINVAL; 700 strlcpy(algo_name, algo, sizeof(algo_name)); 701 702 mac = EVP_MAC_fetch(NULL, "HMAC", NULL); 703 sess->auth.hmac.ctx = EVP_MAC_CTX_new(mac); 704 EVP_MAC_free(mac); 705 if (get_auth_algo(xform->auth.algo, 706 &sess->auth.hmac.evp_algo) != 0) 707 return -EINVAL; 708 709 params[0] = OSSL_PARAM_construct_utf8_string("digest", 710 algo_name, 0); 711 params[1] = OSSL_PARAM_construct_end(); 712 if (EVP_MAC_init(sess->auth.hmac.ctx, 713 xform->auth.key.data, 714 xform->auth.key.length, 715 params) != 1) 716 return -EINVAL; 717 break; 718 # else 719 case RTE_CRYPTO_AUTH_MD5_HMAC: 720 case RTE_CRYPTO_AUTH_SHA1_HMAC: 721 case RTE_CRYPTO_AUTH_SHA224_HMAC: 722 case RTE_CRYPTO_AUTH_SHA256_HMAC: 723 case RTE_CRYPTO_AUTH_SHA384_HMAC: 724 case RTE_CRYPTO_AUTH_SHA512_HMAC: 725 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 726 sess->auth.hmac.ctx = HMAC_CTX_new(); 727 if (get_auth_algo(xform->auth.algo, 728 &sess->auth.hmac.evp_algo) != 0) 729 return -EINVAL; 730 731 if (HMAC_Init_ex(sess->auth.hmac.ctx, 732 xform->auth.key.data, 733 xform->auth.key.length, 734 sess->auth.hmac.evp_algo, NULL) != 1) 735 return -EINVAL; 736 break; 737 # endif 738 default: 739 return -ENOTSUP; 740 } 741 742 return 0; 743 } 744 745 /* Set session AEAD parameters */ 746 static int 747 openssl_set_session_aead_parameters(struct openssl_session *sess, 748 const struct rte_crypto_sym_xform *xform) 749 { 750 /* Select cipher key */ 751 sess->cipher.key.length = xform->aead.key.length; 752 753 /* Set IV parameters */ 754 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 755 /* 756 * For AES-CCM, the actual IV is placed 757 * one byte after the start of the IV field, 758 * according to the API. 759 */ 760 sess->iv.offset = xform->aead.iv.offset + 1; 761 else 762 sess->iv.offset = xform->aead.iv.offset; 763 764 sess->iv.length = xform->aead.iv.length; 765 766 sess->auth.aad_length = xform->aead.aad_length; 767 sess->auth.digest_length = xform->aead.digest_length; 768 769 sess->aead_algo = xform->aead.algo; 770 /* Select cipher direction */ 771 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 772 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 773 xform->aead.digest_length, xform->aead.key.data); 774 else 775 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 776 xform->aead.digest_length, xform->aead.key.data); 777 } 778 779 /** Parse crypto xform chain and set private session parameters */ 780 int 781 openssl_set_session_parameters(struct openssl_session *sess, 782 const struct rte_crypto_sym_xform *xform) 783 { 784 const struct rte_crypto_sym_xform *cipher_xform = NULL; 785 const struct rte_crypto_sym_xform *auth_xform = NULL; 786 const struct rte_crypto_sym_xform *aead_xform = NULL; 787 int ret; 788 789 sess->chain_order = openssl_get_chain_order(xform); 790 switch (sess->chain_order) { 791 case OPENSSL_CHAIN_ONLY_CIPHER: 792 cipher_xform = xform; 793 break; 794 case OPENSSL_CHAIN_ONLY_AUTH: 795 auth_xform = xform; 796 break; 797 case OPENSSL_CHAIN_CIPHER_AUTH: 798 cipher_xform = xform; 799 auth_xform = xform->next; 800 break; 801 case OPENSSL_CHAIN_AUTH_CIPHER: 802 auth_xform = xform; 803 cipher_xform = xform->next; 804 break; 805 case OPENSSL_CHAIN_COMBINED: 806 aead_xform = xform; 807 break; 808 default: 809 return -EINVAL; 810 } 811 812 /* Default IV length = 0 */ 813 sess->iv.length = 0; 814 815 /* cipher_xform must be check before auth_xform */ 816 if (cipher_xform) { 817 ret = openssl_set_session_cipher_parameters( 818 sess, cipher_xform); 819 if (ret != 0) { 820 OPENSSL_LOG(ERR, 821 "Invalid/unsupported cipher parameters"); 822 return ret; 823 } 824 } 825 826 if (auth_xform) { 827 ret = openssl_set_session_auth_parameters(sess, auth_xform); 828 if (ret != 0) { 829 OPENSSL_LOG(ERR, 830 "Invalid/unsupported auth parameters"); 831 return ret; 832 } 833 } 834 835 if (aead_xform) { 836 ret = openssl_set_session_aead_parameters(sess, aead_xform); 837 if (ret != 0) { 838 OPENSSL_LOG(ERR, 839 "Invalid/unsupported AEAD parameters"); 840 return ret; 841 } 842 } 843 844 return 0; 845 } 846 847 /** Reset private session parameters */ 848 void 849 openssl_reset_session(struct openssl_session *sess) 850 { 851 EVP_CIPHER_CTX_free(sess->cipher.ctx); 852 853 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 854 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 855 856 switch (sess->auth.mode) { 857 case OPENSSL_AUTH_AS_AUTH: 858 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 859 break; 860 case OPENSSL_AUTH_AS_HMAC: 861 EVP_PKEY_free(sess->auth.hmac.pkey); 862 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 863 EVP_MAC_CTX_free(sess->auth.hmac.ctx); 864 # else 865 HMAC_CTX_free(sess->auth.hmac.ctx); 866 # endif 867 break; 868 case OPENSSL_AUTH_AS_CMAC: 869 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 870 EVP_MAC_CTX_free(sess->auth.cmac.ctx); 871 # else 872 CMAC_CTX_free(sess->auth.cmac.ctx); 873 # endif 874 break; 875 default: 876 break; 877 } 878 } 879 880 /** Provide session for operation */ 881 static void * 882 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 883 { 884 struct openssl_session *sess = NULL; 885 struct openssl_asym_session *asym_sess = NULL; 886 887 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 888 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 889 /* get existing session */ 890 if (likely(op->sym->session != NULL)) 891 sess = CRYPTODEV_GET_SYM_SESS_PRIV( 892 op->sym->session); 893 } else { 894 if (likely(op->asym->session != NULL)) 895 asym_sess = (struct openssl_asym_session *) 896 op->asym->session->sess_private_data; 897 if (asym_sess == NULL) 898 op->status = 899 RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 900 return asym_sess; 901 } 902 } else { 903 struct rte_cryptodev_sym_session *_sess; 904 /* sessionless asymmetric not supported */ 905 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) 906 return NULL; 907 908 /* provide internal session */ 909 rte_mempool_get(qp->sess_mp, (void **)&_sess); 910 911 if (_sess == NULL) 912 return NULL; 913 914 sess = (struct openssl_session *)_sess->driver_priv_data; 915 916 if (unlikely(openssl_set_session_parameters(sess, 917 op->sym->xform) != 0)) { 918 rte_mempool_put(qp->sess_mp, _sess); 919 sess = NULL; 920 } 921 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 922 923 } 924 925 if (sess == NULL) 926 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 927 928 return sess; 929 } 930 931 /* 932 *------------------------------------------------------------------------------ 933 * Process Operations 934 *------------------------------------------------------------------------------ 935 */ 936 static inline int 937 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 938 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 939 { 940 struct rte_mbuf *m; 941 int dstlen; 942 int l, n = srclen; 943 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 944 945 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 946 m = m->next) 947 offset -= rte_pktmbuf_data_len(m); 948 949 if (m == 0) 950 return -1; 951 952 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 953 if (inplace) 954 *dst = src; 955 956 l = rte_pktmbuf_data_len(m) - offset; 957 if (srclen <= l) { 958 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 959 return -1; 960 *dst += l; 961 return 0; 962 } 963 964 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 965 return -1; 966 967 *dst += dstlen; 968 n -= l; 969 970 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 971 uint8_t diff = l - dstlen, rem; 972 973 src = rte_pktmbuf_mtod(m, uint8_t *); 974 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 975 if (diff && inplace) { 976 rem = RTE_MIN(l, 977 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 978 if (EVP_EncryptUpdate(ctx, temp, 979 &dstlen, src, rem) <= 0) 980 return -1; 981 n -= rem; 982 rte_memcpy(*dst, temp, diff); 983 rte_memcpy(src, temp + diff, rem); 984 src += rem; 985 l -= rem; 986 } 987 if (inplace) 988 *dst = src; 989 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 990 return -1; 991 *dst += dstlen; 992 n -= l; 993 } 994 995 return 0; 996 } 997 998 static inline int 999 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 1000 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 1001 { 1002 struct rte_mbuf *m; 1003 int dstlen; 1004 int l, n = srclen; 1005 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 1006 1007 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1008 m = m->next) 1009 offset -= rte_pktmbuf_data_len(m); 1010 1011 if (m == 0) 1012 return -1; 1013 1014 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1015 if (inplace) 1016 *dst = src; 1017 1018 l = rte_pktmbuf_data_len(m) - offset; 1019 if (srclen <= l) { 1020 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 1021 return -1; 1022 *dst += l; 1023 return 0; 1024 } 1025 1026 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1027 return -1; 1028 1029 *dst += dstlen; 1030 n -= l; 1031 1032 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1033 uint8_t diff = l - dstlen, rem; 1034 1035 src = rte_pktmbuf_mtod(m, uint8_t *); 1036 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 1037 if (diff && inplace) { 1038 rem = RTE_MIN(l, 1039 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 1040 if (EVP_DecryptUpdate(ctx, temp, 1041 &dstlen, src, rem) <= 0) 1042 return -1; 1043 n -= rem; 1044 rte_memcpy(*dst, temp, diff); 1045 rte_memcpy(src, temp + diff, rem); 1046 src += rem; 1047 l -= rem; 1048 } 1049 if (inplace) 1050 *dst = src; 1051 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1052 return -1; 1053 *dst += dstlen; 1054 n -= l; 1055 } 1056 1057 return 0; 1058 } 1059 1060 /** Process standard openssl cipher encryption */ 1061 static int 1062 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1063 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1064 uint8_t inplace) 1065 { 1066 int totlen; 1067 1068 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1069 goto process_cipher_encrypt_err; 1070 1071 EVP_CIPHER_CTX_set_padding(ctx, 0); 1072 1073 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1074 srclen, ctx, inplace)) 1075 goto process_cipher_encrypt_err; 1076 1077 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 1078 goto process_cipher_encrypt_err; 1079 1080 return 0; 1081 1082 process_cipher_encrypt_err: 1083 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 1084 return -EINVAL; 1085 } 1086 1087 /** Process standard openssl cipher encryption */ 1088 static int 1089 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 1090 uint8_t *iv, int srclen, 1091 EVP_CIPHER_CTX *ctx) 1092 { 1093 uint8_t i; 1094 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 1095 int encrypted_ivlen; 1096 1097 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 1098 iv, DES_BLOCK_SIZE) <= 0) 1099 goto process_cipher_encrypt_err; 1100 1101 for (i = 0; i < srclen; i++) 1102 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 1103 1104 return 0; 1105 1106 process_cipher_encrypt_err: 1107 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 1108 return -EINVAL; 1109 } 1110 /** Process standard openssl cipher decryption */ 1111 static int 1112 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1113 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1114 uint8_t inplace) 1115 { 1116 int totlen; 1117 1118 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1119 goto process_cipher_decrypt_err; 1120 1121 EVP_CIPHER_CTX_set_padding(ctx, 0); 1122 1123 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1124 srclen, ctx, inplace)) 1125 goto process_cipher_decrypt_err; 1126 1127 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 1128 goto process_cipher_decrypt_err; 1129 return 0; 1130 1131 process_cipher_decrypt_err: 1132 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 1133 return -EINVAL; 1134 } 1135 1136 /** Process cipher des 3 ctr encryption, decryption algorithm */ 1137 static int 1138 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 1139 int offset, uint8_t *iv, uint8_t *key, int srclen, 1140 EVP_CIPHER_CTX *ctx) 1141 { 1142 uint8_t ebuf[8], ctr[8]; 1143 int unused, n; 1144 struct rte_mbuf *m; 1145 uint8_t *src; 1146 int l; 1147 1148 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1149 m = m->next) 1150 offset -= rte_pktmbuf_data_len(m); 1151 1152 if (m == 0) 1153 goto process_cipher_des3ctr_err; 1154 1155 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1156 l = rte_pktmbuf_data_len(m) - offset; 1157 1158 /* We use 3DES encryption also for decryption. 1159 * IV is not important for 3DES ecb 1160 */ 1161 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 1162 goto process_cipher_des3ctr_err; 1163 1164 memcpy(ctr, iv, 8); 1165 1166 for (n = 0; n < srclen; n++) { 1167 if (n % 8 == 0) { 1168 if (EVP_EncryptUpdate(ctx, 1169 (unsigned char *)&ebuf, &unused, 1170 (const unsigned char *)&ctr, 8) <= 0) 1171 goto process_cipher_des3ctr_err; 1172 ctr_inc(ctr); 1173 } 1174 dst[n] = *(src++) ^ ebuf[n % 8]; 1175 1176 l--; 1177 if (!l) { 1178 m = m->next; 1179 if (m) { 1180 src = rte_pktmbuf_mtod(m, uint8_t *); 1181 l = rte_pktmbuf_data_len(m); 1182 } 1183 } 1184 } 1185 1186 return 0; 1187 1188 process_cipher_des3ctr_err: 1189 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 1190 return -EINVAL; 1191 } 1192 1193 /** Process AES-GCM encrypt algorithm */ 1194 static int 1195 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1196 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1197 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1198 { 1199 int len = 0, unused = 0; 1200 uint8_t empty[] = {}; 1201 1202 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1203 goto process_auth_encryption_gcm_err; 1204 1205 if (aadlen > 0) 1206 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1207 goto process_auth_encryption_gcm_err; 1208 1209 if (srclen > 0) 1210 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1211 srclen, ctx, 0)) 1212 goto process_auth_encryption_gcm_err; 1213 1214 /* Workaround open ssl bug in version less then 1.0.1f */ 1215 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1216 goto process_auth_encryption_gcm_err; 1217 1218 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1219 goto process_auth_encryption_gcm_err; 1220 1221 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1222 goto process_auth_encryption_gcm_err; 1223 1224 return 0; 1225 1226 process_auth_encryption_gcm_err: 1227 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1228 return -EINVAL; 1229 } 1230 1231 /** Process AES-CCM encrypt algorithm */ 1232 static int 1233 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1234 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1235 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1236 { 1237 int len = 0; 1238 1239 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1240 goto process_auth_encryption_ccm_err; 1241 1242 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1243 goto process_auth_encryption_ccm_err; 1244 1245 if (aadlen > 0) 1246 /* 1247 * For AES-CCM, the actual AAD is placed 1248 * 18 bytes after the start of the AAD field, 1249 * according to the API. 1250 */ 1251 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1252 goto process_auth_encryption_ccm_err; 1253 1254 if (srclen >= 0) 1255 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1256 srclen, ctx, 0)) 1257 goto process_auth_encryption_ccm_err; 1258 1259 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1260 goto process_auth_encryption_ccm_err; 1261 1262 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1263 goto process_auth_encryption_ccm_err; 1264 1265 return 0; 1266 1267 process_auth_encryption_ccm_err: 1268 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1269 return -EINVAL; 1270 } 1271 1272 /** Process AES-GCM decrypt algorithm */ 1273 static int 1274 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1275 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1276 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1277 { 1278 int len = 0, unused = 0; 1279 uint8_t empty[] = {}; 1280 1281 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1282 goto process_auth_decryption_gcm_err; 1283 1284 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1285 goto process_auth_decryption_gcm_err; 1286 1287 if (aadlen > 0) 1288 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1289 goto process_auth_decryption_gcm_err; 1290 1291 if (srclen > 0) 1292 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1293 srclen, ctx, 0)) 1294 goto process_auth_decryption_gcm_err; 1295 1296 /* Workaround open ssl bug in version less then 1.0.1f */ 1297 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1298 goto process_auth_decryption_gcm_err; 1299 1300 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1301 return -EFAULT; 1302 1303 return 0; 1304 1305 process_auth_decryption_gcm_err: 1306 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1307 return -EINVAL; 1308 } 1309 1310 /** Process AES-CCM decrypt algorithm */ 1311 static int 1312 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1313 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1314 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1315 EVP_CIPHER_CTX *ctx) 1316 { 1317 int len = 0; 1318 1319 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1320 goto process_auth_decryption_ccm_err; 1321 1322 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1323 goto process_auth_decryption_ccm_err; 1324 1325 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1326 goto process_auth_decryption_ccm_err; 1327 1328 if (aadlen > 0) 1329 /* 1330 * For AES-CCM, the actual AAD is placed 1331 * 18 bytes after the start of the AAD field, 1332 * according to the API. 1333 */ 1334 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1335 goto process_auth_decryption_ccm_err; 1336 1337 if (srclen >= 0) 1338 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1339 srclen, ctx, 0)) 1340 return -EFAULT; 1341 1342 return 0; 1343 1344 process_auth_decryption_ccm_err: 1345 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1346 return -EINVAL; 1347 } 1348 1349 /** Process standard openssl auth algorithms */ 1350 static int 1351 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1352 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1353 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1354 { 1355 size_t dstlen; 1356 struct rte_mbuf *m; 1357 int l, n = srclen; 1358 uint8_t *src; 1359 1360 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1361 m = m->next) 1362 offset -= rte_pktmbuf_data_len(m); 1363 1364 if (m == 0) 1365 goto process_auth_err; 1366 1367 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1368 goto process_auth_err; 1369 1370 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1371 1372 l = rte_pktmbuf_data_len(m) - offset; 1373 if (srclen <= l) { 1374 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1375 goto process_auth_err; 1376 goto process_auth_final; 1377 } 1378 1379 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1380 goto process_auth_err; 1381 1382 n -= l; 1383 1384 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1385 src = rte_pktmbuf_mtod(m, uint8_t *); 1386 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1387 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1388 goto process_auth_err; 1389 n -= l; 1390 } 1391 1392 process_auth_final: 1393 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1394 goto process_auth_err; 1395 return 0; 1396 1397 process_auth_err: 1398 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1399 return -EINVAL; 1400 } 1401 1402 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1403 /** Process standard openssl auth algorithms with hmac/cmac */ 1404 static int 1405 process_openssl_auth_mac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1406 int srclen, EVP_MAC_CTX *ctx) 1407 { 1408 size_t dstlen; 1409 struct rte_mbuf *m; 1410 int l, n = srclen; 1411 uint8_t *src; 1412 1413 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1414 m = m->next) 1415 offset -= rte_pktmbuf_data_len(m); 1416 1417 if (m == 0) 1418 goto process_auth_err; 1419 1420 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1421 1422 l = rte_pktmbuf_data_len(m) - offset; 1423 if (srclen <= l) { 1424 if (EVP_MAC_update(ctx, (unsigned char *)src, srclen) != 1) 1425 goto process_auth_err; 1426 goto process_auth_final; 1427 } 1428 1429 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1430 goto process_auth_err; 1431 1432 n -= l; 1433 1434 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1435 src = rte_pktmbuf_mtod(m, uint8_t *); 1436 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1437 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1438 goto process_auth_err; 1439 n -= l; 1440 } 1441 1442 process_auth_final: 1443 if (EVP_MAC_final(ctx, dst, &dstlen, DIGEST_LENGTH_MAX) != 1) 1444 goto process_auth_err; 1445 1446 EVP_MAC_CTX_free(ctx); 1447 return 0; 1448 1449 process_auth_err: 1450 EVP_MAC_CTX_free(ctx); 1451 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1452 return -EINVAL; 1453 } 1454 # else 1455 /** Process standard openssl auth algorithms with hmac */ 1456 static int 1457 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1458 int srclen, HMAC_CTX *ctx) 1459 { 1460 unsigned int dstlen; 1461 struct rte_mbuf *m; 1462 int l, n = srclen; 1463 uint8_t *src; 1464 1465 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1466 m = m->next) 1467 offset -= rte_pktmbuf_data_len(m); 1468 1469 if (m == 0) 1470 goto process_auth_err; 1471 1472 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1473 1474 l = rte_pktmbuf_data_len(m) - offset; 1475 if (srclen <= l) { 1476 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1477 goto process_auth_err; 1478 goto process_auth_final; 1479 } 1480 1481 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1482 goto process_auth_err; 1483 1484 n -= l; 1485 1486 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1487 src = rte_pktmbuf_mtod(m, uint8_t *); 1488 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1489 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1490 goto process_auth_err; 1491 n -= l; 1492 } 1493 1494 process_auth_final: 1495 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1496 goto process_auth_err; 1497 1498 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1499 goto process_auth_err; 1500 1501 return 0; 1502 1503 process_auth_err: 1504 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1505 return -EINVAL; 1506 } 1507 1508 /** Process standard openssl auth algorithms with cmac */ 1509 static int 1510 process_openssl_auth_cmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1511 int srclen, CMAC_CTX *ctx) 1512 { 1513 unsigned int dstlen; 1514 struct rte_mbuf *m; 1515 int l, n = srclen; 1516 uint8_t *src; 1517 1518 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1519 m = m->next) 1520 offset -= rte_pktmbuf_data_len(m); 1521 1522 if (m == 0) 1523 goto process_auth_err; 1524 1525 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1526 1527 l = rte_pktmbuf_data_len(m) - offset; 1528 if (srclen <= l) { 1529 if (CMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1530 goto process_auth_err; 1531 goto process_auth_final; 1532 } 1533 1534 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1535 goto process_auth_err; 1536 1537 n -= l; 1538 1539 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1540 src = rte_pktmbuf_mtod(m, uint8_t *); 1541 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1542 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1543 goto process_auth_err; 1544 n -= l; 1545 } 1546 1547 process_auth_final: 1548 if (CMAC_Final(ctx, dst, (size_t *)&dstlen) != 1) 1549 goto process_auth_err; 1550 1551 CMAC_CTX_cleanup(ctx); 1552 1553 return 0; 1554 1555 process_auth_err: 1556 OPENSSL_LOG(ERR, "Process openssl cmac auth failed"); 1557 return -EINVAL; 1558 } 1559 # endif 1560 /*----------------------------------------------------------------------------*/ 1561 1562 /** Process auth/cipher combined operation */ 1563 static void 1564 process_openssl_combined_op 1565 (struct rte_crypto_op *op, struct openssl_session *sess, 1566 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1567 { 1568 /* cipher */ 1569 uint8_t *dst = NULL, *iv, *tag, *aad; 1570 int srclen, aadlen, status = -1; 1571 uint32_t offset; 1572 uint8_t taglen; 1573 1574 /* 1575 * Segmented destination buffer is not supported for 1576 * encryption/decryption 1577 */ 1578 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1579 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1580 return; 1581 } 1582 1583 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1584 sess->iv.offset); 1585 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1586 srclen = 0; 1587 offset = op->sym->auth.data.offset; 1588 aadlen = op->sym->auth.data.length; 1589 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1590 op->sym->auth.data.offset); 1591 tag = op->sym->auth.digest.data; 1592 if (tag == NULL) 1593 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1594 offset + aadlen); 1595 } else { 1596 srclen = op->sym->aead.data.length; 1597 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1598 op->sym->aead.data.offset); 1599 offset = op->sym->aead.data.offset; 1600 aad = op->sym->aead.aad.data; 1601 aadlen = sess->auth.aad_length; 1602 tag = op->sym->aead.digest.data; 1603 if (tag == NULL) 1604 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1605 offset + srclen); 1606 } 1607 1608 taglen = sess->auth.digest_length; 1609 1610 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1611 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1612 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1613 status = process_openssl_auth_encryption_gcm( 1614 mbuf_src, offset, srclen, 1615 aad, aadlen, iv, 1616 dst, tag, sess->cipher.ctx); 1617 else 1618 status = process_openssl_auth_encryption_ccm( 1619 mbuf_src, offset, srclen, 1620 aad, aadlen, iv, 1621 dst, tag, taglen, sess->cipher.ctx); 1622 1623 } else { 1624 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1625 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1626 status = process_openssl_auth_decryption_gcm( 1627 mbuf_src, offset, srclen, 1628 aad, aadlen, iv, 1629 dst, tag, sess->cipher.ctx); 1630 else 1631 status = process_openssl_auth_decryption_ccm( 1632 mbuf_src, offset, srclen, 1633 aad, aadlen, iv, 1634 dst, tag, taglen, sess->cipher.ctx); 1635 } 1636 1637 if (status != 0) { 1638 if (status == (-EFAULT) && 1639 sess->auth.operation == 1640 RTE_CRYPTO_AUTH_OP_VERIFY) 1641 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1642 else 1643 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1644 } 1645 } 1646 1647 /** Process cipher operation */ 1648 static void 1649 process_openssl_cipher_op 1650 (struct rte_crypto_op *op, struct openssl_session *sess, 1651 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1652 { 1653 uint8_t *dst, *iv; 1654 int srclen, status; 1655 uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0; 1656 EVP_CIPHER_CTX *ctx_copy; 1657 1658 /* 1659 * Segmented OOP destination buffer is not supported for encryption/ 1660 * decryption. In case of des3ctr, even inplace segmented buffers are 1661 * not supported. 1662 */ 1663 if (!rte_pktmbuf_is_contiguous(mbuf_dst) && 1664 (!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) { 1665 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1666 return; 1667 } 1668 1669 srclen = op->sym->cipher.data.length; 1670 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1671 op->sym->cipher.data.offset); 1672 1673 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1674 sess->iv.offset); 1675 ctx_copy = EVP_CIPHER_CTX_new(); 1676 EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx); 1677 1678 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1679 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1680 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1681 op->sym->cipher.data.offset, iv, 1682 srclen, ctx_copy, inplace); 1683 else 1684 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1685 op->sym->cipher.data.offset, iv, 1686 srclen, ctx_copy, inplace); 1687 else 1688 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1689 op->sym->cipher.data.offset, iv, 1690 sess->cipher.key.data, srclen, 1691 ctx_copy); 1692 1693 EVP_CIPHER_CTX_free(ctx_copy); 1694 if (status != 0) 1695 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1696 } 1697 1698 /** Process cipher operation */ 1699 static void 1700 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1701 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1702 struct rte_mbuf *mbuf_dst) 1703 { 1704 uint8_t *src, *dst, *iv; 1705 uint8_t block_size, last_block_len; 1706 int srclen, status = 0; 1707 1708 srclen = op->sym->cipher.data.length; 1709 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1710 op->sym->cipher.data.offset); 1711 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1712 op->sym->cipher.data.offset); 1713 1714 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1715 sess->iv.offset); 1716 1717 block_size = DES_BLOCK_SIZE; 1718 1719 last_block_len = srclen % block_size; 1720 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1721 /* Encrypt only with ECB mode XOR IV */ 1722 if (srclen < block_size) { 1723 status = process_openssl_cipher_bpi_encrypt(src, dst, 1724 iv, srclen, 1725 sess->cipher.bpi_ctx); 1726 } else { 1727 srclen -= last_block_len; 1728 /* Encrypt with the block aligned stream with CBC mode */ 1729 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1730 op->sym->cipher.data.offset, iv, 1731 srclen, sess->cipher.ctx, 0); 1732 if (last_block_len) { 1733 /* Point at last block */ 1734 dst += srclen; 1735 /* 1736 * IV is the last encrypted block from 1737 * the previous operation 1738 */ 1739 iv = dst - block_size; 1740 src += srclen; 1741 srclen = last_block_len; 1742 /* Encrypt the last frame with ECB mode */ 1743 status |= process_openssl_cipher_bpi_encrypt(src, 1744 dst, iv, 1745 srclen, sess->cipher.bpi_ctx); 1746 } 1747 } 1748 } else { 1749 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1750 if (srclen < block_size) { 1751 status = process_openssl_cipher_bpi_encrypt(src, dst, 1752 iv, 1753 srclen, 1754 sess->cipher.bpi_ctx); 1755 } else { 1756 if (last_block_len) { 1757 /* Point at last block */ 1758 dst += srclen - last_block_len; 1759 src += srclen - last_block_len; 1760 /* 1761 * IV is the last full block 1762 */ 1763 iv = src - block_size; 1764 /* 1765 * Decrypt the last frame with ECB mode 1766 * (encrypt, as it is the same operation) 1767 */ 1768 status = process_openssl_cipher_bpi_encrypt(src, 1769 dst, iv, 1770 last_block_len, sess->cipher.bpi_ctx); 1771 /* Prepare parameters for CBC mode op */ 1772 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1773 sess->iv.offset); 1774 dst += last_block_len - srclen; 1775 srclen -= last_block_len; 1776 } 1777 1778 /* Decrypt with CBC mode */ 1779 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1780 op->sym->cipher.data.offset, iv, 1781 srclen, sess->cipher.ctx, 0); 1782 } 1783 } 1784 1785 if (status != 0) 1786 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1787 } 1788 1789 /** Process auth operation */ 1790 static void 1791 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1792 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1793 struct rte_mbuf *mbuf_dst) 1794 { 1795 uint8_t *dst; 1796 int srclen, status; 1797 EVP_MD_CTX *ctx_a; 1798 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1799 EVP_MAC_CTX *ctx_h; 1800 EVP_MAC_CTX *ctx_c; 1801 EVP_MAC *mac; 1802 # else 1803 HMAC_CTX *ctx_h; 1804 CMAC_CTX *ctx_c; 1805 # endif 1806 1807 srclen = op->sym->auth.data.length; 1808 1809 dst = qp->temp_digest; 1810 1811 switch (sess->auth.mode) { 1812 case OPENSSL_AUTH_AS_AUTH: 1813 ctx_a = EVP_MD_CTX_create(); 1814 EVP_MD_CTX_copy_ex(ctx_a, sess->auth.auth.ctx); 1815 status = process_openssl_auth(mbuf_src, dst, 1816 op->sym->auth.data.offset, NULL, NULL, srclen, 1817 ctx_a, sess->auth.auth.evp_algo); 1818 EVP_MD_CTX_destroy(ctx_a); 1819 break; 1820 case OPENSSL_AUTH_AS_HMAC: 1821 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1822 mac = EVP_MAC_fetch(NULL, "HMAC", NULL); 1823 ctx_h = EVP_MAC_CTX_new(mac); 1824 ctx_h = EVP_MAC_CTX_dup(sess->auth.hmac.ctx); 1825 EVP_MAC_free(mac); 1826 status = process_openssl_auth_mac(mbuf_src, dst, 1827 op->sym->auth.data.offset, srclen, 1828 ctx_h); 1829 # else 1830 ctx_h = HMAC_CTX_new(); 1831 HMAC_CTX_copy(ctx_h, sess->auth.hmac.ctx); 1832 status = process_openssl_auth_hmac(mbuf_src, dst, 1833 op->sym->auth.data.offset, srclen, 1834 ctx_h); 1835 HMAC_CTX_free(ctx_h); 1836 # endif 1837 break; 1838 case OPENSSL_AUTH_AS_CMAC: 1839 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1840 mac = EVP_MAC_fetch(NULL, OSSL_MAC_NAME_CMAC, NULL); 1841 ctx_c = EVP_MAC_CTX_new(mac); 1842 ctx_c = EVP_MAC_CTX_dup(sess->auth.cmac.ctx); 1843 EVP_MAC_free(mac); 1844 status = process_openssl_auth_mac(mbuf_src, dst, 1845 op->sym->auth.data.offset, srclen, 1846 ctx_c); 1847 # else 1848 ctx_c = CMAC_CTX_new(); 1849 CMAC_CTX_copy(ctx_c, sess->auth.cmac.ctx); 1850 status = process_openssl_auth_cmac(mbuf_src, dst, 1851 op->sym->auth.data.offset, srclen, 1852 ctx_c); 1853 CMAC_CTX_free(ctx_c); 1854 # endif 1855 break; 1856 default: 1857 status = -1; 1858 break; 1859 } 1860 1861 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1862 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data, 1863 sess->auth.digest_length) != 0) { 1864 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1865 } 1866 } else { 1867 uint8_t *auth_dst; 1868 1869 auth_dst = op->sym->auth.digest.data; 1870 if (auth_dst == NULL) 1871 auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1872 op->sym->auth.data.offset + 1873 op->sym->auth.data.length); 1874 memcpy(auth_dst, dst, sess->auth.digest_length); 1875 } 1876 1877 if (status != 0) 1878 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1879 } 1880 1881 /* process dsa sign operation */ 1882 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 1883 static int 1884 process_openssl_dsa_sign_op_evp(struct rte_crypto_op *cop, 1885 struct openssl_asym_session *sess) 1886 { 1887 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1888 EVP_PKEY_CTX *dsa_ctx = NULL; 1889 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 1890 EVP_PKEY *pkey = NULL; 1891 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 1892 OSSL_PARAM *params = NULL; 1893 1894 size_t outlen; 1895 unsigned char *dsa_sign_data; 1896 const unsigned char *dsa_sign_data_p; 1897 1898 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1899 params = OSSL_PARAM_BLD_to_param(param_bld); 1900 if (!params) { 1901 OSSL_PARAM_BLD_free(param_bld); 1902 return -1; 1903 } 1904 1905 if (key_ctx == NULL 1906 || EVP_PKEY_fromdata_init(key_ctx) <= 0 1907 || EVP_PKEY_fromdata(key_ctx, &pkey, 1908 EVP_PKEY_KEYPAIR, params) <= 0) 1909 goto err_dsa_sign; 1910 1911 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 1912 if (!dsa_ctx) 1913 goto err_dsa_sign; 1914 1915 if (EVP_PKEY_sign_init(dsa_ctx) <= 0) 1916 goto err_dsa_sign; 1917 1918 if (EVP_PKEY_sign(dsa_ctx, NULL, &outlen, op->message.data, 1919 op->message.length) <= 0) 1920 goto err_dsa_sign; 1921 1922 if (outlen <= 0) 1923 goto err_dsa_sign; 1924 1925 dsa_sign_data = OPENSSL_malloc(outlen); 1926 if (!dsa_sign_data) 1927 goto err_dsa_sign; 1928 1929 if (EVP_PKEY_sign(dsa_ctx, dsa_sign_data, &outlen, op->message.data, 1930 op->message.length) <= 0) { 1931 OPENSSL_free(dsa_sign_data); 1932 goto err_dsa_sign; 1933 } 1934 1935 dsa_sign_data_p = (const unsigned char *)dsa_sign_data; 1936 DSA_SIG *sign = d2i_DSA_SIG(NULL, &dsa_sign_data_p, outlen); 1937 if (!sign) { 1938 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 1939 OPENSSL_free(dsa_sign_data); 1940 goto err_dsa_sign; 1941 } else { 1942 const BIGNUM *r = NULL, *s = NULL; 1943 get_dsa_sign(sign, &r, &s); 1944 1945 op->r.length = BN_bn2bin(r, op->r.data); 1946 op->s.length = BN_bn2bin(s, op->s.data); 1947 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1948 } 1949 1950 DSA_SIG_free(sign); 1951 OPENSSL_free(dsa_sign_data); 1952 return 0; 1953 1954 err_dsa_sign: 1955 if (params) 1956 OSSL_PARAM_free(params); 1957 if (key_ctx) 1958 EVP_PKEY_CTX_free(key_ctx); 1959 if (dsa_ctx) 1960 EVP_PKEY_CTX_free(dsa_ctx); 1961 return -1; 1962 } 1963 1964 /* process dsa verify operation */ 1965 static int 1966 process_openssl_dsa_verify_op_evp(struct rte_crypto_op *cop, 1967 struct openssl_asym_session *sess) 1968 { 1969 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1970 DSA_SIG *sign = DSA_SIG_new(); 1971 BIGNUM *r = NULL, *s = NULL; 1972 BIGNUM *pub_key = NULL; 1973 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 1974 OSSL_PARAM *params = NULL; 1975 EVP_PKEY *pkey = NULL; 1976 EVP_PKEY_CTX *dsa_ctx = NULL; 1977 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 1978 unsigned char *dsa_sig = NULL; 1979 size_t sig_len; 1980 int ret = -1; 1981 1982 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1983 if (!param_bld) { 1984 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 1985 return -1; 1986 } 1987 1988 r = BN_bin2bn(op->r.data, op->r.length, r); 1989 s = BN_bin2bn(op->s.data, op->s.length, s); 1990 pub_key = BN_bin2bn(op->y.data, op->y.length, pub_key); 1991 if (!r || !s || !pub_key) { 1992 BN_free(r); 1993 BN_free(s); 1994 BN_free(pub_key); 1995 OSSL_PARAM_BLD_free(param_bld); 1996 goto err_dsa_verify; 1997 } 1998 1999 set_dsa_sign(sign, r, s); 2000 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PUB_KEY, pub_key)) { 2001 OSSL_PARAM_BLD_free(param_bld); 2002 goto err_dsa_verify; 2003 } 2004 2005 params = OSSL_PARAM_BLD_to_param(param_bld); 2006 if (!params) { 2007 OSSL_PARAM_BLD_free(param_bld); 2008 goto err_dsa_verify; 2009 } 2010 2011 if (key_ctx == NULL 2012 || EVP_PKEY_fromdata_init(key_ctx) <= 0 2013 || EVP_PKEY_fromdata(key_ctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2014 goto err_dsa_verify; 2015 2016 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 2017 if (!dsa_ctx) 2018 goto err_dsa_verify; 2019 2020 if (!sign) 2021 goto err_dsa_verify; 2022 2023 sig_len = i2d_DSA_SIG(sign, &dsa_sig); 2024 if (EVP_PKEY_verify_init(dsa_ctx) <= 0) 2025 goto err_dsa_verify; 2026 2027 ret = EVP_PKEY_verify(dsa_ctx, dsa_sig, sig_len, 2028 op->message.data, op->message.length); 2029 if (ret == 1) { 2030 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2031 ret = 0; 2032 } 2033 2034 err_dsa_verify: 2035 if (sign) 2036 DSA_SIG_free(sign); 2037 if (params) 2038 OSSL_PARAM_free(params); 2039 if (key_ctx) 2040 EVP_PKEY_CTX_free(key_ctx); 2041 if (dsa_ctx) 2042 EVP_PKEY_CTX_free(dsa_ctx); 2043 2044 return ret; 2045 } 2046 #else 2047 static int 2048 process_openssl_dsa_sign_op(struct rte_crypto_op *cop, 2049 struct openssl_asym_session *sess) 2050 { 2051 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2052 DSA *dsa = sess->u.s.dsa; 2053 DSA_SIG *sign = NULL; 2054 2055 sign = DSA_do_sign(op->message.data, 2056 op->message.length, 2057 dsa); 2058 2059 if (sign == NULL) { 2060 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 2061 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2062 } else { 2063 const BIGNUM *r = NULL, *s = NULL; 2064 get_dsa_sign(sign, &r, &s); 2065 2066 op->r.length = BN_bn2bin(r, op->r.data); 2067 op->s.length = BN_bn2bin(s, op->s.data); 2068 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2069 } 2070 2071 DSA_SIG_free(sign); 2072 2073 return 0; 2074 } 2075 2076 /* process dsa verify operation */ 2077 static int 2078 process_openssl_dsa_verify_op(struct rte_crypto_op *cop, 2079 struct openssl_asym_session *sess) 2080 { 2081 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2082 DSA *dsa = sess->u.s.dsa; 2083 int ret; 2084 DSA_SIG *sign = DSA_SIG_new(); 2085 BIGNUM *r = NULL, *s = NULL; 2086 BIGNUM *pub_key = NULL; 2087 2088 if (sign == NULL) { 2089 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 2090 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2091 return -1; 2092 } 2093 2094 r = BN_bin2bn(op->r.data, 2095 op->r.length, 2096 r); 2097 s = BN_bin2bn(op->s.data, 2098 op->s.length, 2099 s); 2100 pub_key = BN_bin2bn(op->y.data, 2101 op->y.length, 2102 pub_key); 2103 if (!r || !s || !pub_key) { 2104 BN_free(r); 2105 BN_free(s); 2106 BN_free(pub_key); 2107 2108 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2109 return -1; 2110 } 2111 set_dsa_sign(sign, r, s); 2112 set_dsa_pub_key(dsa, pub_key); 2113 2114 ret = DSA_do_verify(op->message.data, 2115 op->message.length, 2116 sign, 2117 dsa); 2118 2119 if (ret != 1) 2120 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2121 else 2122 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2123 2124 DSA_SIG_free(sign); 2125 2126 return 0; 2127 } 2128 #endif 2129 2130 /* process dh operation */ 2131 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2132 static int 2133 process_openssl_dh_op_evp(struct rte_crypto_op *cop, 2134 struct openssl_asym_session *sess) 2135 { 2136 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2137 OSSL_PARAM_BLD *param_bld = sess->u.dh.param_bld; 2138 OSSL_PARAM_BLD *param_bld_peer = sess->u.dh.param_bld_peer; 2139 OSSL_PARAM *params = NULL; 2140 EVP_PKEY *dhpkey = NULL; 2141 EVP_PKEY *peerkey = NULL; 2142 BIGNUM *priv_key = NULL; 2143 BIGNUM *pub_key = NULL; 2144 int ret = -1; 2145 2146 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2147 EVP_PKEY_CTX *dh_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2148 if (dh_ctx == NULL || param_bld == NULL) 2149 return ret; 2150 2151 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2152 OSSL_PARAM *params_peer = NULL; 2153 2154 if (!param_bld_peer) 2155 return ret; 2156 2157 pub_key = BN_bin2bn(op->pub_key.data, op->pub_key.length, 2158 pub_key); 2159 if (pub_key == NULL) { 2160 OSSL_PARAM_BLD_free(param_bld_peer); 2161 return ret; 2162 } 2163 2164 if (!OSSL_PARAM_BLD_push_BN(param_bld_peer, OSSL_PKEY_PARAM_PUB_KEY, 2165 pub_key)) { 2166 OPENSSL_LOG(ERR, "Failed to set public key\n"); 2167 OSSL_PARAM_BLD_free(param_bld_peer); 2168 BN_free(pub_key); 2169 return ret; 2170 } 2171 2172 params_peer = OSSL_PARAM_BLD_to_param(param_bld_peer); 2173 if (!params_peer) { 2174 OSSL_PARAM_BLD_free(param_bld_peer); 2175 BN_free(pub_key); 2176 return ret; 2177 } 2178 2179 EVP_PKEY_CTX *peer_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2180 if (EVP_PKEY_keygen_init(peer_ctx) != 1) { 2181 OSSL_PARAM_free(params_peer); 2182 BN_free(pub_key); 2183 return ret; 2184 } 2185 2186 if (EVP_PKEY_CTX_set_params(peer_ctx, params_peer) != 1) { 2187 EVP_PKEY_CTX_free(peer_ctx); 2188 OSSL_PARAM_free(params_peer); 2189 BN_free(pub_key); 2190 return ret; 2191 } 2192 2193 if (EVP_PKEY_keygen(peer_ctx, &peerkey) != 1) { 2194 EVP_PKEY_CTX_free(peer_ctx); 2195 OSSL_PARAM_free(params_peer); 2196 BN_free(pub_key); 2197 return ret; 2198 } 2199 2200 priv_key = BN_bin2bn(op->priv_key.data, op->priv_key.length, 2201 priv_key); 2202 if (priv_key == NULL) { 2203 EVP_PKEY_CTX_free(peer_ctx); 2204 OSSL_PARAM_free(params_peer); 2205 BN_free(pub_key); 2206 return ret; 2207 } 2208 2209 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY, 2210 priv_key)) { 2211 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2212 EVP_PKEY_CTX_free(peer_ctx); 2213 OSSL_PARAM_free(params_peer); 2214 BN_free(pub_key); 2215 BN_free(priv_key); 2216 return ret; 2217 } 2218 2219 OSSL_PARAM_free(params_peer); 2220 EVP_PKEY_CTX_free(peer_ctx); 2221 } 2222 2223 params = OSSL_PARAM_BLD_to_param(param_bld); 2224 if (!params) 2225 goto err_dh; 2226 2227 if (EVP_PKEY_keygen_init(dh_ctx) != 1) 2228 goto err_dh; 2229 2230 if (EVP_PKEY_CTX_set_params(dh_ctx, params) != 1) 2231 goto err_dh; 2232 2233 if (EVP_PKEY_keygen(dh_ctx, &dhpkey) != 1) 2234 goto err_dh; 2235 2236 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2237 OPENSSL_LOG(DEBUG, "%s:%d updated pub key\n", __func__, __LINE__); 2238 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PUB_KEY, &pub_key)) 2239 goto err_dh; 2240 /* output public key */ 2241 op->pub_key.length = BN_bn2bin(pub_key, op->pub_key.data); 2242 } 2243 2244 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2245 2246 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", __func__, __LINE__); 2247 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PRIV_KEY, &priv_key)) 2248 goto err_dh; 2249 2250 /* provide generated private key back to user */ 2251 op->priv_key.length = BN_bn2bin(priv_key, op->priv_key.data); 2252 } 2253 2254 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2255 size_t skey_len; 2256 EVP_PKEY_CTX *sc_ctx = EVP_PKEY_CTX_new(dhpkey, NULL); 2257 if (!sc_ctx) 2258 goto err_dh; 2259 2260 if (EVP_PKEY_derive_init(sc_ctx) <= 0) { 2261 EVP_PKEY_CTX_free(sc_ctx); 2262 goto err_dh; 2263 } 2264 2265 if (!peerkey) { 2266 EVP_PKEY_CTX_free(sc_ctx); 2267 goto err_dh; 2268 } 2269 2270 if (EVP_PKEY_derive_set_peer(sc_ctx, peerkey) <= 0) { 2271 EVP_PKEY_CTX_free(sc_ctx); 2272 goto err_dh; 2273 } 2274 2275 /* Determine buffer length */ 2276 if (EVP_PKEY_derive(sc_ctx, NULL, &skey_len) <= 0) { 2277 EVP_PKEY_CTX_free(sc_ctx); 2278 goto err_dh; 2279 } 2280 2281 if (EVP_PKEY_derive(sc_ctx, op->shared_secret.data, &skey_len) <= 0) { 2282 EVP_PKEY_CTX_free(sc_ctx); 2283 goto err_dh; 2284 } 2285 2286 op->shared_secret.length = skey_len; 2287 EVP_PKEY_CTX_free(sc_ctx); 2288 } 2289 2290 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2291 ret = 0; 2292 2293 err_dh: 2294 if (pub_key) 2295 BN_free(pub_key); 2296 if (priv_key) 2297 BN_free(priv_key); 2298 if (params) 2299 OSSL_PARAM_free(params); 2300 if (dhpkey) 2301 EVP_PKEY_free(dhpkey); 2302 if (peerkey) 2303 EVP_PKEY_free(peerkey); 2304 2305 EVP_PKEY_CTX_free(dh_ctx); 2306 2307 return ret; 2308 } 2309 #else 2310 static int 2311 process_openssl_dh_op(struct rte_crypto_op *cop, 2312 struct openssl_asym_session *sess) 2313 { 2314 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2315 struct rte_crypto_asym_op *asym_op = cop->asym; 2316 DH *dh_key = sess->u.dh.dh_key; 2317 BIGNUM *priv_key = NULL; 2318 int ret = 0; 2319 2320 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2321 /* compute shared secret using peer public key 2322 * and current private key 2323 * shared secret = peer_key ^ priv_key mod p 2324 */ 2325 BIGNUM *peer_key = NULL; 2326 2327 /* copy private key and peer key and compute shared secret */ 2328 peer_key = BN_bin2bn(op->pub_key.data, 2329 op->pub_key.length, 2330 peer_key); 2331 if (peer_key == NULL) { 2332 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2333 return -1; 2334 } 2335 priv_key = BN_bin2bn(op->priv_key.data, 2336 op->priv_key.length, 2337 priv_key); 2338 if (priv_key == NULL) { 2339 BN_free(peer_key); 2340 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2341 return -1; 2342 } 2343 ret = set_dh_priv_key(dh_key, priv_key); 2344 if (ret) { 2345 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2346 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2347 BN_free(peer_key); 2348 BN_free(priv_key); 2349 return 0; 2350 } 2351 2352 ret = DH_compute_key( 2353 op->shared_secret.data, 2354 peer_key, dh_key); 2355 if (ret < 0) { 2356 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2357 BN_free(peer_key); 2358 /* priv key is already loaded into dh, 2359 * let's not free that directly here. 2360 * DH_free() will auto free it later. 2361 */ 2362 return 0; 2363 } 2364 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2365 op->shared_secret.length = ret; 2366 BN_free(peer_key); 2367 return 0; 2368 } 2369 2370 /* 2371 * other options are public and private key generations. 2372 * 2373 * if user provides private key, 2374 * then first set DH with user provided private key 2375 */ 2376 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE && 2377 op->priv_key.length) { 2378 /* generate public key using user-provided private key 2379 * pub_key = g ^ priv_key mod p 2380 */ 2381 2382 /* load private key into DH */ 2383 priv_key = BN_bin2bn(op->priv_key.data, 2384 op->priv_key.length, 2385 priv_key); 2386 if (priv_key == NULL) { 2387 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2388 return -1; 2389 } 2390 ret = set_dh_priv_key(dh_key, priv_key); 2391 if (ret) { 2392 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2393 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2394 BN_free(priv_key); 2395 return 0; 2396 } 2397 } 2398 2399 /* generate public and private key pair. 2400 * 2401 * if private key already set, generates only public key. 2402 * 2403 * if private key is not already set, then set it to random value 2404 * and update internal private key. 2405 */ 2406 if (!DH_generate_key(dh_key)) { 2407 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2408 return 0; 2409 } 2410 2411 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2412 const BIGNUM *pub_key = NULL; 2413 2414 OPENSSL_LOG(DEBUG, "%s:%d update public key\n", 2415 __func__, __LINE__); 2416 2417 /* get the generated keys */ 2418 get_dh_pub_key(dh_key, &pub_key); 2419 2420 /* output public key */ 2421 op->pub_key.length = BN_bn2bin(pub_key, 2422 op->pub_key.data); 2423 } 2424 2425 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2426 const BIGNUM *priv_key = NULL; 2427 2428 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", 2429 __func__, __LINE__); 2430 2431 /* get the generated keys */ 2432 get_dh_priv_key(dh_key, &priv_key); 2433 2434 /* provide generated private key back to user */ 2435 op->priv_key.length = BN_bn2bin(priv_key, 2436 op->priv_key.data); 2437 } 2438 2439 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2440 2441 return 0; 2442 } 2443 #endif 2444 2445 /* process modinv operation */ 2446 static int 2447 process_openssl_modinv_op(struct rte_crypto_op *cop, 2448 struct openssl_asym_session *sess) 2449 { 2450 struct rte_crypto_asym_op *op = cop->asym; 2451 BIGNUM *base = BN_CTX_get(sess->u.m.ctx); 2452 BIGNUM *res = BN_CTX_get(sess->u.m.ctx); 2453 2454 if (unlikely(base == NULL || res == NULL)) { 2455 BN_free(base); 2456 BN_free(res); 2457 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2458 return -1; 2459 } 2460 2461 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 2462 op->modinv.base.length, base); 2463 2464 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) { 2465 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2466 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data); 2467 } else { 2468 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2469 } 2470 2471 BN_clear(res); 2472 BN_clear(base); 2473 2474 return 0; 2475 } 2476 2477 /* process modexp operation */ 2478 static int 2479 process_openssl_modexp_op(struct rte_crypto_op *cop, 2480 struct openssl_asym_session *sess) 2481 { 2482 struct rte_crypto_asym_op *op = cop->asym; 2483 BIGNUM *base = BN_CTX_get(sess->u.e.ctx); 2484 BIGNUM *res = BN_CTX_get(sess->u.e.ctx); 2485 2486 if (unlikely(base == NULL || res == NULL)) { 2487 BN_free(base); 2488 BN_free(res); 2489 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2490 return -1; 2491 } 2492 2493 base = BN_bin2bn((const unsigned char *)op->modex.base.data, 2494 op->modex.base.length, base); 2495 2496 if (BN_mod_exp(res, base, sess->u.e.exp, 2497 sess->u.e.mod, sess->u.e.ctx)) { 2498 op->modex.result.length = BN_bn2bin(res, op->modex.result.data); 2499 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2500 } else { 2501 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2502 } 2503 2504 BN_clear(res); 2505 BN_clear(base); 2506 2507 return 0; 2508 } 2509 2510 /* process rsa operations */ 2511 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2512 static int 2513 process_openssl_rsa_op_evp(struct rte_crypto_op *cop, 2514 struct openssl_asym_session *sess) 2515 { 2516 struct rte_crypto_asym_op *op = cop->asym; 2517 uint32_t pad = (op->rsa.padding.type); 2518 uint8_t *tmp; 2519 size_t outlen = 0; 2520 int ret = -1; 2521 2522 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2523 EVP_PKEY_CTX *rsa_ctx = sess->u.r.ctx; 2524 if (!rsa_ctx) 2525 return ret; 2526 2527 switch (pad) { 2528 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2529 pad = RSA_PKCS1_PADDING; 2530 break; 2531 case RTE_CRYPTO_RSA_PADDING_NONE: 2532 pad = RSA_NO_PADDING; 2533 break; 2534 default: 2535 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2536 OPENSSL_LOG(ERR, 2537 "rsa pad type not supported %d\n", pad); 2538 return ret; 2539 } 2540 2541 switch (op->rsa.op_type) { 2542 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2543 if (EVP_PKEY_encrypt_init(rsa_ctx) != 1) 2544 goto err_rsa; 2545 2546 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2547 goto err_rsa; 2548 2549 if (EVP_PKEY_encrypt(rsa_ctx, NULL, &outlen, 2550 op->rsa.message.data, 2551 op->rsa.message.length) <= 0) 2552 goto err_rsa; 2553 2554 if (outlen <= 0) 2555 goto err_rsa; 2556 2557 if (EVP_PKEY_encrypt(rsa_ctx, op->rsa.cipher.data, &outlen, 2558 op->rsa.message.data, 2559 op->rsa.message.length) <= 0) 2560 goto err_rsa; 2561 op->rsa.cipher.length = outlen; 2562 2563 OPENSSL_LOG(DEBUG, 2564 "length of encrypted text %zu\n", outlen); 2565 break; 2566 2567 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2568 if (EVP_PKEY_decrypt_init(rsa_ctx) != 1) 2569 goto err_rsa; 2570 2571 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2572 goto err_rsa; 2573 2574 if (EVP_PKEY_decrypt(rsa_ctx, NULL, &outlen, 2575 op->rsa.cipher.data, 2576 op->rsa.cipher.length) <= 0) 2577 goto err_rsa; 2578 2579 if (outlen <= 0) 2580 goto err_rsa; 2581 2582 if (EVP_PKEY_decrypt(rsa_ctx, op->rsa.message.data, &outlen, 2583 op->rsa.cipher.data, 2584 op->rsa.cipher.length) <= 0) 2585 goto err_rsa; 2586 op->rsa.message.length = outlen; 2587 2588 OPENSSL_LOG(DEBUG, "length of decrypted text %zu\n", outlen); 2589 break; 2590 2591 case RTE_CRYPTO_ASYM_OP_SIGN: 2592 if (EVP_PKEY_sign_init(rsa_ctx) <= 0) 2593 goto err_rsa; 2594 2595 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2596 goto err_rsa; 2597 2598 if (EVP_PKEY_sign(rsa_ctx, NULL, &outlen, 2599 op->rsa.message.data, 2600 op->rsa.message.length) <= 0) 2601 goto err_rsa; 2602 2603 if (outlen <= 0) 2604 goto err_rsa; 2605 2606 if (EVP_PKEY_sign(rsa_ctx, op->rsa.sign.data, &outlen, 2607 op->rsa.message.data, 2608 op->rsa.message.length) <= 0) 2609 goto err_rsa; 2610 op->rsa.sign.length = outlen; 2611 break; 2612 2613 case RTE_CRYPTO_ASYM_OP_VERIFY: 2614 if (EVP_PKEY_verify_recover_init(rsa_ctx) <= 0) 2615 goto err_rsa; 2616 2617 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2618 goto err_rsa; 2619 2620 if (EVP_PKEY_verify_recover(rsa_ctx, NULL, &outlen, 2621 op->rsa.sign.data, 2622 op->rsa.sign.length) <= 0) 2623 goto err_rsa; 2624 2625 if ((outlen <= 0) || (outlen != op->rsa.sign.length)) 2626 goto err_rsa; 2627 2628 tmp = OPENSSL_malloc(outlen); 2629 if (tmp == NULL) { 2630 OPENSSL_LOG(ERR, "Memory allocation failed"); 2631 goto err_rsa; 2632 } 2633 2634 if (EVP_PKEY_verify_recover(rsa_ctx, tmp, &outlen, 2635 op->rsa.sign.data, 2636 op->rsa.sign.length) <= 0) { 2637 OPENSSL_free(tmp); 2638 goto err_rsa; 2639 } 2640 2641 OPENSSL_LOG(DEBUG, 2642 "Length of public_decrypt %zu " 2643 "length of message %zd\n", 2644 outlen, op->rsa.message.length); 2645 if (CRYPTO_memcmp(tmp, op->rsa.message.data, 2646 op->rsa.message.length)) { 2647 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 2648 } 2649 OPENSSL_free(tmp); 2650 break; 2651 2652 default: 2653 /* allow ops with invalid args to be pushed to 2654 * completion queue 2655 */ 2656 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2657 goto err_rsa; 2658 } 2659 2660 ret = 0; 2661 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2662 err_rsa: 2663 return ret; 2664 2665 } 2666 2667 static int 2668 process_openssl_sm2_op_evp(struct rte_crypto_op *cop, 2669 struct openssl_asym_session *sess) 2670 { 2671 EVP_PKEY_CTX *kctx = NULL, *sctx = NULL, *cctx = NULL; 2672 struct rte_crypto_asym_op *op = cop->asym; 2673 OSSL_PARAM_BLD *param_bld = NULL; 2674 OSSL_PARAM *params = NULL; 2675 EVP_PKEY *pkey = NULL; 2676 BIGNUM *pkey_bn = NULL; 2677 uint8_t pubkey[64]; 2678 size_t len = 0; 2679 int ret = -1; 2680 2681 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2682 2683 if (cop->asym->sm2.k.data != NULL) 2684 goto err_sm2; 2685 2686 param_bld = OSSL_PARAM_BLD_new(); 2687 if (!param_bld) { 2688 OPENSSL_LOG(ERR, "failed to allocate params\n"); 2689 goto err_sm2; 2690 } 2691 2692 ret = OSSL_PARAM_BLD_push_utf8_string(param_bld, 2693 OSSL_PKEY_PARAM_GROUP_NAME, "SM2", 0); 2694 if (!ret) { 2695 OPENSSL_LOG(ERR, "failed to push params\n"); 2696 goto err_sm2; 2697 } 2698 2699 pkey_bn = BN_bin2bn((const unsigned char *)op->sm2.pkey.data, 2700 op->sm2.pkey.length, pkey_bn); 2701 2702 memset(pubkey, 0, RTE_DIM(pubkey)); 2703 pubkey[0] = 0x04; 2704 len += 1; 2705 memcpy(&pubkey[len], op->sm2.q.x.data, op->sm2.q.x.length); 2706 len += op->sm2.q.x.length; 2707 memcpy(&pubkey[len], op->sm2.q.y.data, op->sm2.q.y.length); 2708 len += op->sm2.q.y.length; 2709 2710 ret = OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY, 2711 pkey_bn); 2712 if (!ret) { 2713 OPENSSL_LOG(ERR, "failed to push params\n"); 2714 goto err_sm2; 2715 } 2716 2717 ret = OSSL_PARAM_BLD_push_octet_string(param_bld, 2718 OSSL_PKEY_PARAM_PUB_KEY, pubkey, len); 2719 if (!ret) { 2720 OPENSSL_LOG(ERR, "failed to push params\n"); 2721 goto err_sm2; 2722 } 2723 2724 params = OSSL_PARAM_BLD_to_param(param_bld); 2725 if (!params) { 2726 OPENSSL_LOG(ERR, "failed to push params\n"); 2727 goto err_sm2; 2728 } 2729 2730 switch (op->sm2.op_type) { 2731 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2732 { 2733 OSSL_PARAM *eparams = sess->u.sm2.params; 2734 size_t output_len; 2735 2736 kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_SM2, NULL); 2737 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2738 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2739 goto err_sm2; 2740 2741 cctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2742 if (!cctx) 2743 goto err_sm2; 2744 2745 if (!EVP_PKEY_encrypt_init(cctx)) 2746 goto err_sm2; 2747 2748 if (!EVP_PKEY_CTX_set_params(cctx, eparams)) 2749 goto err_sm2; 2750 2751 if (!EVP_PKEY_encrypt(cctx, op->sm2.cipher.data, &output_len, 2752 op->sm2.message.data, 2753 op->sm2.message.length)) 2754 goto err_sm2; 2755 op->sm2.cipher.length = output_len; 2756 } 2757 break; 2758 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2759 { 2760 OSSL_PARAM *eparams = sess->u.sm2.params; 2761 2762 kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_SM2, NULL); 2763 if (kctx == NULL 2764 || EVP_PKEY_fromdata_init(kctx) <= 0 2765 || EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2766 goto err_sm2; 2767 2768 cctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2769 if (!cctx) 2770 goto err_sm2; 2771 2772 if (!EVP_PKEY_decrypt_init(cctx)) 2773 goto err_sm2; 2774 2775 if (!EVP_PKEY_CTX_set_params(cctx, eparams)) 2776 goto err_sm2; 2777 2778 if (!EVP_PKEY_decrypt(cctx, op->sm2.message.data, &op->sm2.message.length, 2779 op->sm2.cipher.data, op->sm2.cipher.length)) 2780 goto err_sm2; 2781 } 2782 break; 2783 case RTE_CRYPTO_ASYM_OP_SIGN: 2784 { 2785 unsigned char signbuf[128] = {0}; 2786 const unsigned char *signptr; 2787 EVP_MD_CTX *md_ctx = NULL; 2788 const BIGNUM *r, *s; 2789 ECDSA_SIG *ec_sign; 2790 EVP_MD *check_md; 2791 size_t signlen; 2792 2793 kctx = EVP_PKEY_CTX_new_from_name(NULL, "SM2", NULL); 2794 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2795 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2796 goto err_sm2; 2797 2798 md_ctx = EVP_MD_CTX_new(); 2799 if (!md_ctx) 2800 goto err_sm2; 2801 2802 sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2803 if (!sctx) 2804 goto err_sm2; 2805 2806 EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx); 2807 2808 check_md = EVP_MD_fetch(NULL, "sm3", NULL); 2809 if (!check_md) 2810 goto err_sm2; 2811 2812 if (!EVP_DigestSignInit(md_ctx, NULL, check_md, NULL, pkey)) 2813 goto err_sm2; 2814 2815 if (EVP_PKEY_CTX_set1_id(sctx, op->sm2.id.data, op->sm2.id.length) <= 0) 2816 goto err_sm2; 2817 2818 if (!EVP_DigestSignUpdate(md_ctx, op->sm2.message.data, 2819 op->sm2.message.length)) 2820 goto err_sm2; 2821 2822 if (!EVP_DigestSignFinal(md_ctx, NULL, &signlen)) 2823 goto err_sm2; 2824 2825 if (!EVP_DigestSignFinal(md_ctx, signbuf, &signlen)) 2826 goto err_sm2; 2827 2828 signptr = signbuf; 2829 ec_sign = d2i_ECDSA_SIG(NULL, &signptr, signlen); 2830 if (!ec_sign) 2831 goto err_sm2; 2832 2833 r = ECDSA_SIG_get0_r(ec_sign); 2834 s = ECDSA_SIG_get0_s(ec_sign); 2835 if (!r || !s) 2836 goto err_sm2; 2837 2838 op->sm2.r.length = BN_num_bytes(r); 2839 op->sm2.s.length = BN_num_bytes(s); 2840 BN_bn2bin(r, op->sm2.r.data); 2841 BN_bn2bin(s, op->sm2.s.data); 2842 2843 ECDSA_SIG_free(ec_sign); 2844 } 2845 break; 2846 case RTE_CRYPTO_ASYM_OP_VERIFY: 2847 { 2848 unsigned char signbuf[128] = {0}; 2849 BIGNUM *r = NULL, *s = NULL; 2850 EVP_MD_CTX *md_ctx = NULL; 2851 ECDSA_SIG *ec_sign; 2852 EVP_MD *check_md; 2853 size_t signlen; 2854 2855 kctx = EVP_PKEY_CTX_new_from_name(NULL, "SM2", NULL); 2856 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2857 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_PUBLIC_KEY, params) <= 0) 2858 goto err_sm2; 2859 2860 if (!EVP_PKEY_is_a(pkey, "SM2")) 2861 goto err_sm2; 2862 2863 md_ctx = EVP_MD_CTX_new(); 2864 if (!md_ctx) 2865 goto err_sm2; 2866 2867 sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2868 if (!sctx) 2869 goto err_sm2; 2870 2871 EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx); 2872 2873 check_md = EVP_MD_fetch(NULL, "sm3", NULL); 2874 if (!check_md) 2875 goto err_sm2; 2876 2877 if (!EVP_DigestVerifyInit(md_ctx, NULL, check_md, NULL, pkey)) 2878 goto err_sm2; 2879 2880 if (EVP_PKEY_CTX_set1_id(sctx, op->sm2.id.data, op->sm2.id.length) <= 0) 2881 goto err_sm2; 2882 2883 if (!EVP_DigestVerifyUpdate(md_ctx, op->sm2.message.data, 2884 op->sm2.message.length)) 2885 goto err_sm2; 2886 2887 ec_sign = ECDSA_SIG_new(); 2888 if (!ec_sign) 2889 goto err_sm2; 2890 2891 r = BN_bin2bn(op->sm2.r.data, op->sm2.r.length, r); 2892 s = BN_bin2bn(op->sm2.s.data, op->sm2.s.length, s); 2893 if (!r || !s) 2894 goto err_sm2; 2895 2896 if (!ECDSA_SIG_set0(ec_sign, r, s)) { 2897 BN_free(r); 2898 BN_free(s); 2899 goto err_sm2; 2900 } 2901 2902 r = NULL; 2903 s = NULL; 2904 2905 signlen = i2d_ECDSA_SIG(ec_sign, (unsigned char **)&signbuf); 2906 if (signlen <= 0) 2907 goto err_sm2; 2908 2909 if (!EVP_DigestVerifyFinal(md_ctx, signbuf, signlen)) 2910 goto err_sm2; 2911 2912 BN_free(r); 2913 BN_free(s); 2914 ECDSA_SIG_free(ec_sign); 2915 } 2916 break; 2917 default: 2918 /* allow ops with invalid args to be pushed to 2919 * completion queue 2920 */ 2921 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2922 goto err_sm2; 2923 } 2924 2925 ret = 0; 2926 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2927 err_sm2: 2928 if (kctx) 2929 EVP_PKEY_CTX_free(kctx); 2930 2931 if (sctx) 2932 EVP_PKEY_CTX_free(sctx); 2933 2934 if (cctx) 2935 EVP_PKEY_CTX_free(cctx); 2936 2937 if (pkey) 2938 EVP_PKEY_free(pkey); 2939 2940 if (param_bld) 2941 OSSL_PARAM_BLD_free(param_bld); 2942 2943 return ret; 2944 } 2945 2946 #else 2947 static int 2948 process_openssl_rsa_op(struct rte_crypto_op *cop, 2949 struct openssl_asym_session *sess) 2950 { 2951 int ret = 0; 2952 struct rte_crypto_asym_op *op = cop->asym; 2953 RSA *rsa = sess->u.r.rsa; 2954 uint32_t pad = (op->rsa.padding.type); 2955 uint8_t *tmp; 2956 2957 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2958 2959 switch (pad) { 2960 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2961 pad = RSA_PKCS1_PADDING; 2962 break; 2963 case RTE_CRYPTO_RSA_PADDING_NONE: 2964 pad = RSA_NO_PADDING; 2965 break; 2966 default: 2967 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2968 OPENSSL_LOG(ERR, 2969 "rsa pad type not supported %d\n", pad); 2970 return 0; 2971 } 2972 2973 switch (op->rsa.op_type) { 2974 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2975 ret = RSA_public_encrypt(op->rsa.message.length, 2976 op->rsa.message.data, 2977 op->rsa.cipher.data, 2978 rsa, 2979 pad); 2980 2981 if (ret > 0) 2982 op->rsa.cipher.length = ret; 2983 OPENSSL_LOG(DEBUG, 2984 "length of encrypted text %d\n", ret); 2985 break; 2986 2987 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2988 ret = RSA_private_decrypt(op->rsa.cipher.length, 2989 op->rsa.cipher.data, 2990 op->rsa.message.data, 2991 rsa, 2992 pad); 2993 if (ret > 0) 2994 op->rsa.message.length = ret; 2995 break; 2996 2997 case RTE_CRYPTO_ASYM_OP_SIGN: 2998 ret = RSA_private_encrypt(op->rsa.message.length, 2999 op->rsa.message.data, 3000 op->rsa.sign.data, 3001 rsa, 3002 pad); 3003 if (ret > 0) 3004 op->rsa.sign.length = ret; 3005 break; 3006 3007 case RTE_CRYPTO_ASYM_OP_VERIFY: 3008 tmp = rte_malloc(NULL, op->rsa.sign.length, 0); 3009 if (tmp == NULL) { 3010 OPENSSL_LOG(ERR, "Memory allocation failed"); 3011 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 3012 break; 3013 } 3014 ret = RSA_public_decrypt(op->rsa.sign.length, 3015 op->rsa.sign.data, 3016 tmp, 3017 rsa, 3018 pad); 3019 3020 OPENSSL_LOG(DEBUG, 3021 "Length of public_decrypt %d " 3022 "length of message %zd\n", 3023 ret, op->rsa.message.length); 3024 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data, 3025 op->rsa.message.length))) { 3026 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 3027 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 3028 } 3029 rte_free(tmp); 3030 break; 3031 3032 default: 3033 /* allow ops with invalid args to be pushed to 3034 * completion queue 3035 */ 3036 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3037 break; 3038 } 3039 3040 if (ret < 0) 3041 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 3042 3043 return 0; 3044 } 3045 3046 static int 3047 process_openssl_sm2_op(struct rte_crypto_op *cop, 3048 struct openssl_asym_session *sess) 3049 { 3050 RTE_SET_USED(cop); 3051 RTE_SET_USED(sess); 3052 return -ENOTSUP; 3053 } 3054 #endif 3055 3056 static int 3057 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op, 3058 struct openssl_asym_session *sess) 3059 { 3060 int retval = 0; 3061 3062 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 3063 3064 switch (sess->xfrm_type) { 3065 case RTE_CRYPTO_ASYM_XFORM_RSA: 3066 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3067 retval = process_openssl_rsa_op_evp(op, sess); 3068 # else 3069 retval = process_openssl_rsa_op(op, sess); 3070 #endif 3071 break; 3072 case RTE_CRYPTO_ASYM_XFORM_MODEX: 3073 retval = process_openssl_modexp_op(op, sess); 3074 break; 3075 case RTE_CRYPTO_ASYM_XFORM_MODINV: 3076 retval = process_openssl_modinv_op(op, sess); 3077 break; 3078 case RTE_CRYPTO_ASYM_XFORM_DH: 3079 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3080 retval = process_openssl_dh_op_evp(op, sess); 3081 # else 3082 retval = process_openssl_dh_op(op, sess); 3083 #endif 3084 break; 3085 case RTE_CRYPTO_ASYM_XFORM_DSA: 3086 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3087 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 3088 retval = process_openssl_dsa_sign_op_evp(op, sess); 3089 else if (op->asym->dsa.op_type == 3090 RTE_CRYPTO_ASYM_OP_VERIFY) 3091 retval = 3092 process_openssl_dsa_verify_op_evp(op, sess); 3093 #else 3094 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 3095 retval = process_openssl_dsa_sign_op(op, sess); 3096 else if (op->asym->dsa.op_type == 3097 RTE_CRYPTO_ASYM_OP_VERIFY) 3098 retval = 3099 process_openssl_dsa_verify_op(op, sess); 3100 else 3101 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3102 #endif 3103 break; 3104 case RTE_CRYPTO_ASYM_XFORM_SM2: 3105 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3106 retval = process_openssl_sm2_op_evp(op, sess); 3107 #else 3108 retval = process_openssl_sm2_op(op, sess); 3109 #endif 3110 break; 3111 default: 3112 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3113 break; 3114 } 3115 if (!retval) { 3116 /* op processed so push to completion queue as processed */ 3117 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 3118 if (retval) 3119 /* return error if failed to put in completion queue */ 3120 retval = -1; 3121 } 3122 3123 return retval; 3124 } 3125 3126 static void 3127 copy_plaintext(struct rte_mbuf *m_src, struct rte_mbuf *m_dst, 3128 struct rte_crypto_op *op) 3129 { 3130 uint8_t *p_src, *p_dst; 3131 3132 p_src = rte_pktmbuf_mtod(m_src, uint8_t *); 3133 p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *); 3134 3135 /** 3136 * Copy the content between cipher offset and auth offset 3137 * for generating correct digest. 3138 */ 3139 if (op->sym->cipher.data.offset > op->sym->auth.data.offset) 3140 memcpy(p_dst + op->sym->auth.data.offset, 3141 p_src + op->sym->auth.data.offset, 3142 op->sym->cipher.data.offset - 3143 op->sym->auth.data.offset); 3144 } 3145 3146 /** Process crypto operation for mbuf */ 3147 static int 3148 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 3149 struct openssl_session *sess) 3150 { 3151 struct rte_mbuf *msrc, *mdst; 3152 int retval; 3153 3154 msrc = op->sym->m_src; 3155 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 3156 3157 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 3158 3159 switch (sess->chain_order) { 3160 case OPENSSL_CHAIN_ONLY_CIPHER: 3161 process_openssl_cipher_op(op, sess, msrc, mdst); 3162 break; 3163 case OPENSSL_CHAIN_ONLY_AUTH: 3164 process_openssl_auth_op(qp, op, sess, msrc, mdst); 3165 break; 3166 case OPENSSL_CHAIN_CIPHER_AUTH: 3167 process_openssl_cipher_op(op, sess, msrc, mdst); 3168 /* OOP */ 3169 if (msrc != mdst) 3170 copy_plaintext(msrc, mdst, op); 3171 process_openssl_auth_op(qp, op, sess, mdst, mdst); 3172 break; 3173 case OPENSSL_CHAIN_AUTH_CIPHER: 3174 process_openssl_auth_op(qp, op, sess, msrc, mdst); 3175 process_openssl_cipher_op(op, sess, msrc, mdst); 3176 break; 3177 case OPENSSL_CHAIN_COMBINED: 3178 process_openssl_combined_op(op, sess, msrc, mdst); 3179 break; 3180 case OPENSSL_CHAIN_CIPHER_BPI: 3181 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 3182 break; 3183 default: 3184 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3185 break; 3186 } 3187 3188 /* Free session if a session-less crypto op */ 3189 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 3190 openssl_reset_session(sess); 3191 memset(sess, 0, sizeof(struct openssl_session)); 3192 rte_mempool_put(qp->sess_mp, op->sym->session); 3193 op->sym->session = NULL; 3194 } 3195 3196 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 3197 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3198 3199 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 3200 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 3201 else 3202 retval = -1; 3203 3204 return retval; 3205 } 3206 3207 /* 3208 *------------------------------------------------------------------------------ 3209 * PMD Framework 3210 *------------------------------------------------------------------------------ 3211 */ 3212 3213 /** Enqueue burst */ 3214 static uint16_t 3215 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 3216 uint16_t nb_ops) 3217 { 3218 void *sess; 3219 struct openssl_qp *qp = queue_pair; 3220 int i, retval; 3221 3222 for (i = 0; i < nb_ops; i++) { 3223 sess = get_session(qp, ops[i]); 3224 if (unlikely(sess == NULL)) 3225 goto enqueue_err; 3226 3227 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) 3228 retval = process_op(qp, ops[i], 3229 (struct openssl_session *) sess); 3230 else 3231 retval = process_asym_op(qp, ops[i], 3232 (struct openssl_asym_session *) sess); 3233 if (unlikely(retval < 0)) 3234 goto enqueue_err; 3235 } 3236 3237 qp->stats.enqueued_count += i; 3238 return i; 3239 3240 enqueue_err: 3241 qp->stats.enqueue_err_count++; 3242 return i; 3243 } 3244 3245 /** Dequeue burst */ 3246 static uint16_t 3247 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 3248 uint16_t nb_ops) 3249 { 3250 struct openssl_qp *qp = queue_pair; 3251 3252 unsigned int nb_dequeued = 0; 3253 3254 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 3255 (void **)ops, nb_ops, NULL); 3256 qp->stats.dequeued_count += nb_dequeued; 3257 3258 return nb_dequeued; 3259 } 3260 3261 /** Create OPENSSL crypto device */ 3262 static int 3263 cryptodev_openssl_create(const char *name, 3264 struct rte_vdev_device *vdev, 3265 struct rte_cryptodev_pmd_init_params *init_params) 3266 { 3267 struct rte_cryptodev *dev; 3268 struct openssl_private *internals; 3269 3270 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 3271 if (dev == NULL) { 3272 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 3273 goto init_error; 3274 } 3275 3276 dev->driver_id = cryptodev_driver_id; 3277 dev->dev_ops = rte_openssl_pmd_ops; 3278 3279 /* register rx/tx burst functions for data path */ 3280 dev->dequeue_burst = openssl_pmd_dequeue_burst; 3281 dev->enqueue_burst = openssl_pmd_enqueue_burst; 3282 3283 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 3284 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 3285 RTE_CRYPTODEV_FF_CPU_AESNI | 3286 RTE_CRYPTODEV_FF_IN_PLACE_SGL | 3287 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 3288 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | 3289 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO | 3290 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP | 3291 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT | 3292 RTE_CRYPTODEV_FF_SYM_SESSIONLESS; 3293 3294 internals = dev->data->dev_private; 3295 3296 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 3297 3298 rte_cryptodev_pmd_probing_finish(dev); 3299 3300 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3301 /* Load legacy provider 3302 * Some algorithms are no longer available in earlier version of openssl, 3303 * unless the legacy provider explicitly loaded. e.g. DES 3304 */ 3305 ossl_legacy_provider_load(); 3306 # endif 3307 return 0; 3308 3309 init_error: 3310 OPENSSL_LOG(ERR, "driver %s: create failed", 3311 init_params->name); 3312 3313 cryptodev_openssl_remove(vdev); 3314 return -EFAULT; 3315 } 3316 3317 /** Initialise OPENSSL crypto device */ 3318 static int 3319 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 3320 { 3321 struct rte_cryptodev_pmd_init_params init_params = { 3322 "", 3323 sizeof(struct openssl_private), 3324 rte_socket_id(), 3325 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 3326 }; 3327 const char *name; 3328 const char *input_args; 3329 3330 name = rte_vdev_device_name(vdev); 3331 if (name == NULL) 3332 return -EINVAL; 3333 input_args = rte_vdev_device_args(vdev); 3334 3335 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 3336 3337 return cryptodev_openssl_create(name, vdev, &init_params); 3338 } 3339 3340 /** Uninitialise OPENSSL crypto device */ 3341 static int 3342 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 3343 { 3344 struct rte_cryptodev *cryptodev; 3345 const char *name; 3346 3347 name = rte_vdev_device_name(vdev); 3348 if (name == NULL) 3349 return -EINVAL; 3350 3351 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 3352 if (cryptodev == NULL) 3353 return -ENODEV; 3354 3355 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3356 ossl_legacy_provider_unload(); 3357 # endif 3358 return rte_cryptodev_pmd_destroy(cryptodev); 3359 } 3360 3361 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 3362 .probe = cryptodev_openssl_probe, 3363 .remove = cryptodev_openssl_remove 3364 }; 3365 3366 static struct cryptodev_driver openssl_crypto_drv; 3367 3368 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 3369 cryptodev_openssl_pmd_drv); 3370 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 3371 "max_nb_queue_pairs=<int> " 3372 "socket_id=<int>"); 3373 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 3374 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 3375 RTE_LOG_REGISTER_DEFAULT(openssl_logtype_driver, INFO); 3376