1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_common.h> 6 #include <rte_hexdump.h> 7 #include <rte_cryptodev.h> 8 #include <cryptodev_pmd.h> 9 #include <bus_vdev_driver.h> 10 #include <rte_malloc.h> 11 #include <rte_cpuflags.h> 12 13 #include <openssl/cmac.h> 14 #include <openssl/hmac.h> 15 #include <openssl/evp.h> 16 #include <openssl/ec.h> 17 18 #include "openssl_pmd_private.h" 19 #include "compat.h" 20 21 #define DES_BLOCK_SIZE 8 22 23 static uint8_t cryptodev_driver_id; 24 25 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 26 static HMAC_CTX *HMAC_CTX_new(void) 27 { 28 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 29 30 if (ctx != NULL) 31 HMAC_CTX_init(ctx); 32 return ctx; 33 } 34 35 static void HMAC_CTX_free(HMAC_CTX *ctx) 36 { 37 if (ctx != NULL) { 38 HMAC_CTX_cleanup(ctx); 39 OPENSSL_free(ctx); 40 } 41 } 42 #endif 43 44 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 45 46 #include <openssl/provider.h> 47 #include <openssl/core_names.h> 48 #include <openssl/param_build.h> 49 50 #define MAX_OSSL_ALGO_NAME_SIZE 16 51 52 OSSL_PROVIDER *legacy; 53 OSSL_PROVIDER *deflt; 54 55 static void ossl_legacy_provider_load(void) 56 { 57 /* Load Multiple providers into the default (NULL) library context */ 58 legacy = OSSL_PROVIDER_load(NULL, "legacy"); 59 if (legacy == NULL) { 60 OPENSSL_LOG(ERR, "Failed to load Legacy provider\n"); 61 return; 62 } 63 64 deflt = OSSL_PROVIDER_load(NULL, "default"); 65 if (deflt == NULL) { 66 OPENSSL_LOG(ERR, "Failed to load Default provider\n"); 67 OSSL_PROVIDER_unload(legacy); 68 return; 69 } 70 } 71 72 static void ossl_legacy_provider_unload(void) 73 { 74 OSSL_PROVIDER_unload(legacy); 75 OSSL_PROVIDER_unload(deflt); 76 } 77 78 static __rte_always_inline const char * 79 digest_name_get(enum rte_crypto_auth_algorithm algo) 80 { 81 switch (algo) { 82 case RTE_CRYPTO_AUTH_MD5_HMAC: 83 return OSSL_DIGEST_NAME_MD5; 84 case RTE_CRYPTO_AUTH_SHA1_HMAC: 85 return OSSL_DIGEST_NAME_SHA1; 86 case RTE_CRYPTO_AUTH_SHA224_HMAC: 87 return OSSL_DIGEST_NAME_SHA2_224; 88 case RTE_CRYPTO_AUTH_SHA256_HMAC: 89 return OSSL_DIGEST_NAME_SHA2_256; 90 case RTE_CRYPTO_AUTH_SHA384_HMAC: 91 return OSSL_DIGEST_NAME_SHA2_384; 92 case RTE_CRYPTO_AUTH_SHA512_HMAC: 93 return OSSL_DIGEST_NAME_SHA2_512; 94 default: 95 return NULL; 96 } 97 } 98 #endif 99 100 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 101 102 /*----------------------------------------------------------------------------*/ 103 104 /** 105 * Increment counter by 1 106 * Counter is 64 bit array, big-endian 107 */ 108 static void 109 ctr_inc(uint8_t *ctr) 110 { 111 uint64_t *ctr64 = (uint64_t *)ctr; 112 113 *ctr64 = __builtin_bswap64(*ctr64); 114 (*ctr64)++; 115 *ctr64 = __builtin_bswap64(*ctr64); 116 } 117 118 /* 119 *------------------------------------------------------------------------------ 120 * Session Prepare 121 *------------------------------------------------------------------------------ 122 */ 123 124 /** Get xform chain order */ 125 static enum openssl_chain_order 126 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 127 { 128 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 129 130 if (xform != NULL) { 131 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 132 if (xform->next == NULL) 133 res = OPENSSL_CHAIN_ONLY_AUTH; 134 else if (xform->next->type == 135 RTE_CRYPTO_SYM_XFORM_CIPHER) 136 res = OPENSSL_CHAIN_AUTH_CIPHER; 137 } 138 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 139 if (xform->next == NULL) 140 res = OPENSSL_CHAIN_ONLY_CIPHER; 141 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 142 res = OPENSSL_CHAIN_CIPHER_AUTH; 143 } 144 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 145 res = OPENSSL_CHAIN_COMBINED; 146 } 147 148 return res; 149 } 150 151 /** Get session cipher key from input cipher key */ 152 static void 153 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key) 154 { 155 memcpy(session_key, input_key, keylen); 156 } 157 158 /** Get key ede 24 bytes standard from input key */ 159 static int 160 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede) 161 { 162 int res = 0; 163 164 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 165 switch (keylen) { 166 case 24: 167 memcpy(key_ede, key, 24); 168 break; 169 case 16: 170 /* K3 = K1 */ 171 memcpy(key_ede, key, 16); 172 memcpy(key_ede + 16, key, 8); 173 break; 174 case 8: 175 /* K1 = K2 = K3 (DES compatibility) */ 176 memcpy(key_ede, key, 8); 177 memcpy(key_ede + 8, key, 8); 178 memcpy(key_ede + 16, key, 8); 179 break; 180 default: 181 OPENSSL_LOG(ERR, "Unsupported key size"); 182 res = -EINVAL; 183 } 184 185 return res; 186 } 187 188 /** Get adequate openssl function for input cipher algorithm */ 189 static uint8_t 190 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 191 const EVP_CIPHER **algo) 192 { 193 int res = 0; 194 195 if (algo != NULL) { 196 switch (sess_algo) { 197 case RTE_CRYPTO_CIPHER_3DES_CBC: 198 switch (keylen) { 199 case 8: 200 *algo = EVP_des_cbc(); 201 break; 202 case 16: 203 *algo = EVP_des_ede_cbc(); 204 break; 205 case 24: 206 *algo = EVP_des_ede3_cbc(); 207 break; 208 default: 209 res = -EINVAL; 210 } 211 break; 212 case RTE_CRYPTO_CIPHER_3DES_CTR: 213 break; 214 case RTE_CRYPTO_CIPHER_AES_CBC: 215 switch (keylen) { 216 case 16: 217 *algo = EVP_aes_128_cbc(); 218 break; 219 case 24: 220 *algo = EVP_aes_192_cbc(); 221 break; 222 case 32: 223 *algo = EVP_aes_256_cbc(); 224 break; 225 default: 226 res = -EINVAL; 227 } 228 break; 229 case RTE_CRYPTO_CIPHER_AES_CTR: 230 switch (keylen) { 231 case 16: 232 *algo = EVP_aes_128_ctr(); 233 break; 234 case 24: 235 *algo = EVP_aes_192_ctr(); 236 break; 237 case 32: 238 *algo = EVP_aes_256_ctr(); 239 break; 240 default: 241 res = -EINVAL; 242 } 243 break; 244 default: 245 res = -EINVAL; 246 break; 247 } 248 } else { 249 res = -EINVAL; 250 } 251 252 return res; 253 } 254 255 /** Get adequate openssl function for input auth algorithm */ 256 static uint8_t 257 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 258 const EVP_MD **algo) 259 { 260 int res = 0; 261 262 if (algo != NULL) { 263 switch (sessalgo) { 264 case RTE_CRYPTO_AUTH_MD5: 265 case RTE_CRYPTO_AUTH_MD5_HMAC: 266 *algo = EVP_md5(); 267 break; 268 case RTE_CRYPTO_AUTH_SHA1: 269 case RTE_CRYPTO_AUTH_SHA1_HMAC: 270 *algo = EVP_sha1(); 271 break; 272 case RTE_CRYPTO_AUTH_SHA224: 273 case RTE_CRYPTO_AUTH_SHA224_HMAC: 274 *algo = EVP_sha224(); 275 break; 276 case RTE_CRYPTO_AUTH_SHA256: 277 case RTE_CRYPTO_AUTH_SHA256_HMAC: 278 *algo = EVP_sha256(); 279 break; 280 case RTE_CRYPTO_AUTH_SHA384: 281 case RTE_CRYPTO_AUTH_SHA384_HMAC: 282 *algo = EVP_sha384(); 283 break; 284 case RTE_CRYPTO_AUTH_SHA512: 285 case RTE_CRYPTO_AUTH_SHA512_HMAC: 286 *algo = EVP_sha512(); 287 break; 288 default: 289 res = -EINVAL; 290 break; 291 } 292 } else { 293 res = -EINVAL; 294 } 295 296 return res; 297 } 298 299 /** Get adequate openssl function for input cipher algorithm */ 300 static uint8_t 301 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 302 const EVP_CIPHER **algo) 303 { 304 int res = 0; 305 306 if (algo != NULL) { 307 switch (sess_algo) { 308 case RTE_CRYPTO_AEAD_AES_GCM: 309 switch (keylen) { 310 case 16: 311 *algo = EVP_aes_128_gcm(); 312 break; 313 case 24: 314 *algo = EVP_aes_192_gcm(); 315 break; 316 case 32: 317 *algo = EVP_aes_256_gcm(); 318 break; 319 default: 320 res = -EINVAL; 321 } 322 break; 323 case RTE_CRYPTO_AEAD_AES_CCM: 324 switch (keylen) { 325 case 16: 326 *algo = EVP_aes_128_ccm(); 327 break; 328 case 24: 329 *algo = EVP_aes_192_ccm(); 330 break; 331 case 32: 332 *algo = EVP_aes_256_ccm(); 333 break; 334 default: 335 res = -EINVAL; 336 } 337 break; 338 default: 339 res = -EINVAL; 340 break; 341 } 342 } else { 343 res = -EINVAL; 344 } 345 346 return res; 347 } 348 349 /* Set session AEAD encryption parameters */ 350 static int 351 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 352 enum rte_crypto_aead_algorithm algo, 353 uint8_t tag_len, const uint8_t *key) 354 { 355 int iv_type = 0; 356 unsigned int do_ccm; 357 358 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 359 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 360 361 /* Select AEAD algo */ 362 switch (algo) { 363 case RTE_CRYPTO_AEAD_AES_GCM: 364 iv_type = EVP_CTRL_GCM_SET_IVLEN; 365 if (tag_len != 16) 366 return -EINVAL; 367 do_ccm = 0; 368 break; 369 case RTE_CRYPTO_AEAD_AES_CCM: 370 iv_type = EVP_CTRL_CCM_SET_IVLEN; 371 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 372 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 373 return -EINVAL; 374 do_ccm = 1; 375 break; 376 default: 377 return -ENOTSUP; 378 } 379 380 sess->cipher.mode = OPENSSL_CIPHER_LIB; 381 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 382 383 if (get_aead_algo(algo, sess->cipher.key.length, 384 &sess->cipher.evp_algo) != 0) 385 return -EINVAL; 386 387 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 388 389 sess->chain_order = OPENSSL_CHAIN_COMBINED; 390 391 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 392 NULL, NULL, NULL) <= 0) 393 return -EINVAL; 394 395 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, 396 NULL) <= 0) 397 return -EINVAL; 398 399 if (do_ccm) 400 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 401 tag_len, NULL); 402 403 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 404 return -EINVAL; 405 406 return 0; 407 } 408 409 /* Set session AEAD decryption parameters */ 410 static int 411 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 412 enum rte_crypto_aead_algorithm algo, 413 uint8_t tag_len, const uint8_t *key) 414 { 415 int iv_type = 0; 416 unsigned int do_ccm = 0; 417 418 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 419 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 420 421 /* Select AEAD algo */ 422 switch (algo) { 423 case RTE_CRYPTO_AEAD_AES_GCM: 424 iv_type = EVP_CTRL_GCM_SET_IVLEN; 425 if (tag_len != 16) 426 return -EINVAL; 427 break; 428 case RTE_CRYPTO_AEAD_AES_CCM: 429 iv_type = EVP_CTRL_CCM_SET_IVLEN; 430 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 431 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 432 return -EINVAL; 433 do_ccm = 1; 434 break; 435 default: 436 return -ENOTSUP; 437 } 438 439 sess->cipher.mode = OPENSSL_CIPHER_LIB; 440 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 441 442 if (get_aead_algo(algo, sess->cipher.key.length, 443 &sess->cipher.evp_algo) != 0) 444 return -EINVAL; 445 446 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 447 448 sess->chain_order = OPENSSL_CHAIN_COMBINED; 449 450 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 451 NULL, NULL, NULL) <= 0) 452 return -EINVAL; 453 454 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, 455 sess->iv.length, NULL) <= 0) 456 return -EINVAL; 457 458 if (do_ccm) 459 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 460 tag_len, NULL); 461 462 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 463 return -EINVAL; 464 465 return 0; 466 } 467 468 /** Set session cipher parameters */ 469 static int 470 openssl_set_session_cipher_parameters(struct openssl_session *sess, 471 const struct rte_crypto_sym_xform *xform) 472 { 473 /* Select cipher direction */ 474 sess->cipher.direction = xform->cipher.op; 475 /* Select cipher key */ 476 sess->cipher.key.length = xform->cipher.key.length; 477 478 /* Set IV parameters */ 479 sess->iv.offset = xform->cipher.iv.offset; 480 sess->iv.length = xform->cipher.iv.length; 481 482 /* Select cipher algo */ 483 switch (xform->cipher.algo) { 484 case RTE_CRYPTO_CIPHER_3DES_CBC: 485 case RTE_CRYPTO_CIPHER_AES_CBC: 486 case RTE_CRYPTO_CIPHER_AES_CTR: 487 sess->cipher.mode = OPENSSL_CIPHER_LIB; 488 sess->cipher.algo = xform->cipher.algo; 489 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 490 491 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 492 &sess->cipher.evp_algo) != 0) 493 return -EINVAL; 494 495 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 496 sess->cipher.key.data); 497 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 498 if (EVP_EncryptInit_ex(sess->cipher.ctx, 499 sess->cipher.evp_algo, 500 NULL, xform->cipher.key.data, 501 NULL) != 1) { 502 return -EINVAL; 503 } 504 } else if (sess->cipher.direction == 505 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 506 if (EVP_DecryptInit_ex(sess->cipher.ctx, 507 sess->cipher.evp_algo, 508 NULL, xform->cipher.key.data, 509 NULL) != 1) { 510 return -EINVAL; 511 } 512 } 513 514 break; 515 516 case RTE_CRYPTO_CIPHER_3DES_CTR: 517 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 518 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 519 520 if (get_cipher_key_ede(xform->cipher.key.data, 521 sess->cipher.key.length, 522 sess->cipher.key.data) != 0) 523 return -EINVAL; 524 break; 525 526 case RTE_CRYPTO_CIPHER_DES_CBC: 527 sess->cipher.algo = xform->cipher.algo; 528 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 529 sess->cipher.evp_algo = EVP_des_cbc(); 530 531 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 532 sess->cipher.key.data); 533 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 534 if (EVP_EncryptInit_ex(sess->cipher.ctx, 535 sess->cipher.evp_algo, 536 NULL, xform->cipher.key.data, 537 NULL) != 1) { 538 return -EINVAL; 539 } 540 } else if (sess->cipher.direction == 541 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 542 if (EVP_DecryptInit_ex(sess->cipher.ctx, 543 sess->cipher.evp_algo, 544 NULL, xform->cipher.key.data, 545 NULL) != 1) { 546 return -EINVAL; 547 } 548 } 549 550 break; 551 552 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 553 sess->cipher.algo = xform->cipher.algo; 554 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 555 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 556 sess->cipher.evp_algo = EVP_des_cbc(); 557 558 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 559 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 560 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 561 NULL, xform->cipher.key.data, 0) != 1) 562 return -EINVAL; 563 564 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 565 sess->cipher.key.data); 566 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 567 if (EVP_EncryptInit_ex(sess->cipher.ctx, 568 sess->cipher.evp_algo, 569 NULL, xform->cipher.key.data, 570 NULL) != 1) { 571 return -EINVAL; 572 } 573 } else if (sess->cipher.direction == 574 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 575 if (EVP_DecryptInit_ex(sess->cipher.ctx, 576 sess->cipher.evp_algo, 577 NULL, xform->cipher.key.data, 578 NULL) != 1) { 579 return -EINVAL; 580 } 581 } 582 583 break; 584 default: 585 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 586 return -ENOTSUP; 587 } 588 589 return 0; 590 } 591 592 /* Set session auth parameters */ 593 static int 594 openssl_set_session_auth_parameters(struct openssl_session *sess, 595 const struct rte_crypto_sym_xform *xform) 596 { 597 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 598 char algo_name[MAX_OSSL_ALGO_NAME_SIZE]; 599 OSSL_PARAM params[2]; 600 const char *algo; 601 EVP_MAC *mac; 602 # endif 603 /* Select auth generate/verify */ 604 sess->auth.operation = xform->auth.op; 605 sess->auth.algo = xform->auth.algo; 606 607 sess->auth.digest_length = xform->auth.digest_length; 608 609 /* Select auth algo */ 610 switch (xform->auth.algo) { 611 case RTE_CRYPTO_AUTH_AES_GMAC: 612 /* 613 * OpenSSL requires GMAC to be a GCM operation 614 * with no cipher data length 615 */ 616 sess->cipher.key.length = xform->auth.key.length; 617 618 /* Set IV parameters */ 619 sess->iv.offset = xform->auth.iv.offset; 620 sess->iv.length = xform->auth.iv.length; 621 622 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 623 return openssl_set_sess_aead_enc_param(sess, 624 RTE_CRYPTO_AEAD_AES_GCM, 625 xform->auth.digest_length, 626 xform->auth.key.data); 627 else 628 return openssl_set_sess_aead_dec_param(sess, 629 RTE_CRYPTO_AEAD_AES_GCM, 630 xform->auth.digest_length, 631 xform->auth.key.data); 632 break; 633 634 case RTE_CRYPTO_AUTH_MD5: 635 case RTE_CRYPTO_AUTH_SHA1: 636 case RTE_CRYPTO_AUTH_SHA224: 637 case RTE_CRYPTO_AUTH_SHA256: 638 case RTE_CRYPTO_AUTH_SHA384: 639 case RTE_CRYPTO_AUTH_SHA512: 640 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 641 if (get_auth_algo(xform->auth.algo, 642 &sess->auth.auth.evp_algo) != 0) 643 return -EINVAL; 644 sess->auth.auth.ctx = EVP_MD_CTX_create(); 645 break; 646 647 case RTE_CRYPTO_AUTH_AES_CMAC: 648 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 649 if (xform->auth.key.length == 16) 650 algo = SN_aes_128_cbc; 651 else if (xform->auth.key.length == 24) 652 algo = SN_aes_192_cbc; 653 else if (xform->auth.key.length == 32) 654 algo = SN_aes_256_cbc; 655 else 656 return -EINVAL; 657 658 rte_memcpy(algo_name, algo, strlen(algo) + 1); 659 params[0] = OSSL_PARAM_construct_utf8_string( 660 OSSL_MAC_PARAM_CIPHER, algo_name, 0); 661 params[1] = OSSL_PARAM_construct_end(); 662 663 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 664 mac = EVP_MAC_fetch(NULL, OSSL_MAC_NAME_CMAC, NULL); 665 sess->auth.cmac.ctx = EVP_MAC_CTX_new(mac); 666 EVP_MAC_free(mac); 667 668 if (EVP_MAC_init(sess->auth.cmac.ctx, 669 xform->auth.key.data, 670 xform->auth.key.length, 671 params) != 1) 672 return -EINVAL; 673 # else 674 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 675 sess->auth.cmac.ctx = CMAC_CTX_new(); 676 if (get_cipher_algo(RTE_CRYPTO_CIPHER_AES_CBC, 677 xform->auth.key.length, 678 &sess->auth.cmac.evp_algo) != 0) 679 return -EINVAL; 680 if (CMAC_Init(sess->auth.cmac.ctx, 681 xform->auth.key.data, 682 xform->auth.key.length, 683 sess->auth.cmac.evp_algo, NULL) != 1) 684 return -EINVAL; 685 # endif 686 break; 687 688 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 689 case RTE_CRYPTO_AUTH_MD5_HMAC: 690 case RTE_CRYPTO_AUTH_SHA1_HMAC: 691 case RTE_CRYPTO_AUTH_SHA224_HMAC: 692 case RTE_CRYPTO_AUTH_SHA256_HMAC: 693 case RTE_CRYPTO_AUTH_SHA384_HMAC: 694 case RTE_CRYPTO_AUTH_SHA512_HMAC: 695 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 696 697 algo = digest_name_get(xform->auth.algo); 698 if (!algo) 699 return -EINVAL; 700 strlcpy(algo_name, algo, sizeof(algo_name)); 701 702 mac = EVP_MAC_fetch(NULL, "HMAC", NULL); 703 sess->auth.hmac.ctx = EVP_MAC_CTX_new(mac); 704 EVP_MAC_free(mac); 705 if (get_auth_algo(xform->auth.algo, 706 &sess->auth.hmac.evp_algo) != 0) 707 return -EINVAL; 708 709 params[0] = OSSL_PARAM_construct_utf8_string("digest", 710 algo_name, 0); 711 params[1] = OSSL_PARAM_construct_end(); 712 if (EVP_MAC_init(sess->auth.hmac.ctx, 713 xform->auth.key.data, 714 xform->auth.key.length, 715 params) != 1) 716 return -EINVAL; 717 break; 718 # else 719 case RTE_CRYPTO_AUTH_MD5_HMAC: 720 case RTE_CRYPTO_AUTH_SHA1_HMAC: 721 case RTE_CRYPTO_AUTH_SHA224_HMAC: 722 case RTE_CRYPTO_AUTH_SHA256_HMAC: 723 case RTE_CRYPTO_AUTH_SHA384_HMAC: 724 case RTE_CRYPTO_AUTH_SHA512_HMAC: 725 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 726 sess->auth.hmac.ctx = HMAC_CTX_new(); 727 if (get_auth_algo(xform->auth.algo, 728 &sess->auth.hmac.evp_algo) != 0) 729 return -EINVAL; 730 731 if (HMAC_Init_ex(sess->auth.hmac.ctx, 732 xform->auth.key.data, 733 xform->auth.key.length, 734 sess->auth.hmac.evp_algo, NULL) != 1) 735 return -EINVAL; 736 break; 737 # endif 738 default: 739 return -ENOTSUP; 740 } 741 742 return 0; 743 } 744 745 /* Set session AEAD parameters */ 746 static int 747 openssl_set_session_aead_parameters(struct openssl_session *sess, 748 const struct rte_crypto_sym_xform *xform) 749 { 750 /* Select cipher key */ 751 sess->cipher.key.length = xform->aead.key.length; 752 753 /* Set IV parameters */ 754 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 755 /* 756 * For AES-CCM, the actual IV is placed 757 * one byte after the start of the IV field, 758 * according to the API. 759 */ 760 sess->iv.offset = xform->aead.iv.offset + 1; 761 else 762 sess->iv.offset = xform->aead.iv.offset; 763 764 sess->iv.length = xform->aead.iv.length; 765 766 sess->auth.aad_length = xform->aead.aad_length; 767 sess->auth.digest_length = xform->aead.digest_length; 768 769 sess->aead_algo = xform->aead.algo; 770 /* Select cipher direction */ 771 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 772 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 773 xform->aead.digest_length, xform->aead.key.data); 774 else 775 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 776 xform->aead.digest_length, xform->aead.key.data); 777 } 778 779 /** Parse crypto xform chain and set private session parameters */ 780 int 781 openssl_set_session_parameters(struct openssl_session *sess, 782 const struct rte_crypto_sym_xform *xform) 783 { 784 const struct rte_crypto_sym_xform *cipher_xform = NULL; 785 const struct rte_crypto_sym_xform *auth_xform = NULL; 786 const struct rte_crypto_sym_xform *aead_xform = NULL; 787 int ret; 788 789 sess->chain_order = openssl_get_chain_order(xform); 790 switch (sess->chain_order) { 791 case OPENSSL_CHAIN_ONLY_CIPHER: 792 cipher_xform = xform; 793 break; 794 case OPENSSL_CHAIN_ONLY_AUTH: 795 auth_xform = xform; 796 break; 797 case OPENSSL_CHAIN_CIPHER_AUTH: 798 cipher_xform = xform; 799 auth_xform = xform->next; 800 break; 801 case OPENSSL_CHAIN_AUTH_CIPHER: 802 auth_xform = xform; 803 cipher_xform = xform->next; 804 break; 805 case OPENSSL_CHAIN_COMBINED: 806 aead_xform = xform; 807 break; 808 default: 809 return -EINVAL; 810 } 811 812 /* Default IV length = 0 */ 813 sess->iv.length = 0; 814 815 /* cipher_xform must be check before auth_xform */ 816 if (cipher_xform) { 817 ret = openssl_set_session_cipher_parameters( 818 sess, cipher_xform); 819 if (ret != 0) { 820 OPENSSL_LOG(ERR, 821 "Invalid/unsupported cipher parameters"); 822 return ret; 823 } 824 } 825 826 if (auth_xform) { 827 ret = openssl_set_session_auth_parameters(sess, auth_xform); 828 if (ret != 0) { 829 OPENSSL_LOG(ERR, 830 "Invalid/unsupported auth parameters"); 831 return ret; 832 } 833 } 834 835 if (aead_xform) { 836 ret = openssl_set_session_aead_parameters(sess, aead_xform); 837 if (ret != 0) { 838 OPENSSL_LOG(ERR, 839 "Invalid/unsupported AEAD parameters"); 840 return ret; 841 } 842 } 843 844 return 0; 845 } 846 847 /** Reset private session parameters */ 848 void 849 openssl_reset_session(struct openssl_session *sess) 850 { 851 EVP_CIPHER_CTX_free(sess->cipher.ctx); 852 853 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 854 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 855 856 switch (sess->auth.mode) { 857 case OPENSSL_AUTH_AS_AUTH: 858 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 859 break; 860 case OPENSSL_AUTH_AS_HMAC: 861 EVP_PKEY_free(sess->auth.hmac.pkey); 862 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 863 EVP_MAC_CTX_free(sess->auth.hmac.ctx); 864 # else 865 HMAC_CTX_free(sess->auth.hmac.ctx); 866 # endif 867 break; 868 case OPENSSL_AUTH_AS_CMAC: 869 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 870 EVP_MAC_CTX_free(sess->auth.cmac.ctx); 871 # else 872 CMAC_CTX_free(sess->auth.cmac.ctx); 873 # endif 874 break; 875 default: 876 break; 877 } 878 } 879 880 /** Provide session for operation */ 881 static void * 882 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 883 { 884 struct openssl_session *sess = NULL; 885 struct openssl_asym_session *asym_sess = NULL; 886 887 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 888 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 889 /* get existing session */ 890 if (likely(op->sym->session != NULL)) 891 sess = CRYPTODEV_GET_SYM_SESS_PRIV( 892 op->sym->session); 893 } else { 894 if (likely(op->asym->session != NULL)) 895 asym_sess = (struct openssl_asym_session *) 896 op->asym->session->sess_private_data; 897 if (asym_sess == NULL) 898 op->status = 899 RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 900 return asym_sess; 901 } 902 } else { 903 struct rte_cryptodev_sym_session *_sess; 904 /* sessionless asymmetric not supported */ 905 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) 906 return NULL; 907 908 /* provide internal session */ 909 rte_mempool_get(qp->sess_mp, (void **)&_sess); 910 911 if (_sess == NULL) 912 return NULL; 913 914 sess = (struct openssl_session *)_sess->driver_priv_data; 915 916 if (unlikely(openssl_set_session_parameters(sess, 917 op->sym->xform) != 0)) { 918 rte_mempool_put(qp->sess_mp, _sess); 919 sess = NULL; 920 } 921 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 922 923 } 924 925 if (sess == NULL) 926 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 927 928 return sess; 929 } 930 931 /* 932 *------------------------------------------------------------------------------ 933 * Process Operations 934 *------------------------------------------------------------------------------ 935 */ 936 static inline int 937 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 938 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 939 { 940 struct rte_mbuf *m; 941 int dstlen; 942 int l, n = srclen; 943 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 944 945 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 946 m = m->next) 947 offset -= rte_pktmbuf_data_len(m); 948 949 if (m == 0) 950 return -1; 951 952 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 953 if (inplace) 954 *dst = src; 955 956 l = rte_pktmbuf_data_len(m) - offset; 957 if (srclen <= l) { 958 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 959 return -1; 960 *dst += l; 961 return 0; 962 } 963 964 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 965 return -1; 966 967 *dst += dstlen; 968 n -= l; 969 970 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 971 uint8_t diff = l - dstlen, rem; 972 973 src = rte_pktmbuf_mtod(m, uint8_t *); 974 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 975 if (diff && inplace) { 976 rem = RTE_MIN(l, 977 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 978 if (EVP_EncryptUpdate(ctx, temp, 979 &dstlen, src, rem) <= 0) 980 return -1; 981 n -= rem; 982 rte_memcpy(*dst, temp, diff); 983 rte_memcpy(src, temp + diff, rem); 984 src += rem; 985 l -= rem; 986 } 987 if (inplace) 988 *dst = src; 989 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 990 return -1; 991 *dst += dstlen; 992 n -= l; 993 } 994 995 return 0; 996 } 997 998 static inline int 999 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 1000 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 1001 { 1002 struct rte_mbuf *m; 1003 int dstlen; 1004 int l, n = srclen; 1005 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 1006 1007 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1008 m = m->next) 1009 offset -= rte_pktmbuf_data_len(m); 1010 1011 if (m == 0) 1012 return -1; 1013 1014 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1015 if (inplace) 1016 *dst = src; 1017 1018 l = rte_pktmbuf_data_len(m) - offset; 1019 if (srclen <= l) { 1020 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 1021 return -1; 1022 *dst += l; 1023 return 0; 1024 } 1025 1026 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1027 return -1; 1028 1029 *dst += dstlen; 1030 n -= l; 1031 1032 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1033 uint8_t diff = l - dstlen, rem; 1034 1035 src = rte_pktmbuf_mtod(m, uint8_t *); 1036 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 1037 if (diff && inplace) { 1038 rem = RTE_MIN(l, 1039 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 1040 if (EVP_DecryptUpdate(ctx, temp, 1041 &dstlen, src, rem) <= 0) 1042 return -1; 1043 n -= rem; 1044 rte_memcpy(*dst, temp, diff); 1045 rte_memcpy(src, temp + diff, rem); 1046 src += rem; 1047 l -= rem; 1048 } 1049 if (inplace) 1050 *dst = src; 1051 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1052 return -1; 1053 *dst += dstlen; 1054 n -= l; 1055 } 1056 1057 return 0; 1058 } 1059 1060 /** Process standard openssl cipher encryption */ 1061 static int 1062 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1063 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1064 uint8_t inplace) 1065 { 1066 int totlen; 1067 1068 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1069 goto process_cipher_encrypt_err; 1070 1071 EVP_CIPHER_CTX_set_padding(ctx, 0); 1072 1073 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1074 srclen, ctx, inplace)) 1075 goto process_cipher_encrypt_err; 1076 1077 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 1078 goto process_cipher_encrypt_err; 1079 1080 return 0; 1081 1082 process_cipher_encrypt_err: 1083 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 1084 return -EINVAL; 1085 } 1086 1087 /** Process standard openssl cipher encryption */ 1088 static int 1089 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 1090 uint8_t *iv, int srclen, 1091 EVP_CIPHER_CTX *ctx) 1092 { 1093 uint8_t i; 1094 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 1095 int encrypted_ivlen; 1096 1097 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 1098 iv, DES_BLOCK_SIZE) <= 0) 1099 goto process_cipher_encrypt_err; 1100 1101 for (i = 0; i < srclen; i++) 1102 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 1103 1104 return 0; 1105 1106 process_cipher_encrypt_err: 1107 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 1108 return -EINVAL; 1109 } 1110 /** Process standard openssl cipher decryption */ 1111 static int 1112 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1113 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1114 uint8_t inplace) 1115 { 1116 int totlen; 1117 1118 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1119 goto process_cipher_decrypt_err; 1120 1121 EVP_CIPHER_CTX_set_padding(ctx, 0); 1122 1123 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1124 srclen, ctx, inplace)) 1125 goto process_cipher_decrypt_err; 1126 1127 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 1128 goto process_cipher_decrypt_err; 1129 return 0; 1130 1131 process_cipher_decrypt_err: 1132 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 1133 return -EINVAL; 1134 } 1135 1136 /** Process cipher des 3 ctr encryption, decryption algorithm */ 1137 static int 1138 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 1139 int offset, uint8_t *iv, uint8_t *key, int srclen, 1140 EVP_CIPHER_CTX *ctx) 1141 { 1142 uint8_t ebuf[8], ctr[8]; 1143 int unused, n; 1144 struct rte_mbuf *m; 1145 uint8_t *src; 1146 int l; 1147 1148 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1149 m = m->next) 1150 offset -= rte_pktmbuf_data_len(m); 1151 1152 if (m == 0) 1153 goto process_cipher_des3ctr_err; 1154 1155 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1156 l = rte_pktmbuf_data_len(m) - offset; 1157 1158 /* We use 3DES encryption also for decryption. 1159 * IV is not important for 3DES ecb 1160 */ 1161 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 1162 goto process_cipher_des3ctr_err; 1163 1164 memcpy(ctr, iv, 8); 1165 1166 for (n = 0; n < srclen; n++) { 1167 if (n % 8 == 0) { 1168 if (EVP_EncryptUpdate(ctx, 1169 (unsigned char *)&ebuf, &unused, 1170 (const unsigned char *)&ctr, 8) <= 0) 1171 goto process_cipher_des3ctr_err; 1172 ctr_inc(ctr); 1173 } 1174 dst[n] = *(src++) ^ ebuf[n % 8]; 1175 1176 l--; 1177 if (!l) { 1178 m = m->next; 1179 if (m) { 1180 src = rte_pktmbuf_mtod(m, uint8_t *); 1181 l = rte_pktmbuf_data_len(m); 1182 } 1183 } 1184 } 1185 1186 return 0; 1187 1188 process_cipher_des3ctr_err: 1189 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 1190 return -EINVAL; 1191 } 1192 1193 /** Process AES-GCM encrypt algorithm */ 1194 static int 1195 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1196 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1197 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1198 { 1199 int len = 0; 1200 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1201 int unused = 0; 1202 uint8_t empty[] = {}; 1203 #endif 1204 1205 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1206 goto process_auth_encryption_gcm_err; 1207 1208 if (aadlen > 0) 1209 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1210 goto process_auth_encryption_gcm_err; 1211 1212 if (srclen > 0) 1213 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1214 srclen, ctx, 0)) 1215 goto process_auth_encryption_gcm_err; 1216 1217 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1218 /* Workaround open ssl bug in version less then 1.0.1f */ 1219 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1220 goto process_auth_encryption_gcm_err; 1221 #endif 1222 1223 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1224 goto process_auth_encryption_gcm_err; 1225 1226 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1227 goto process_auth_encryption_gcm_err; 1228 1229 return 0; 1230 1231 process_auth_encryption_gcm_err: 1232 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1233 return -EINVAL; 1234 } 1235 1236 /** Process AES-CCM encrypt algorithm */ 1237 static int 1238 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1239 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1240 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1241 { 1242 int len = 0; 1243 1244 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1245 goto process_auth_encryption_ccm_err; 1246 1247 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1248 goto process_auth_encryption_ccm_err; 1249 1250 if (aadlen > 0) 1251 /* 1252 * For AES-CCM, the actual AAD is placed 1253 * 18 bytes after the start of the AAD field, 1254 * according to the API. 1255 */ 1256 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1257 goto process_auth_encryption_ccm_err; 1258 1259 if (srclen >= 0) 1260 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1261 srclen, ctx, 0)) 1262 goto process_auth_encryption_ccm_err; 1263 1264 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1265 goto process_auth_encryption_ccm_err; 1266 1267 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1268 goto process_auth_encryption_ccm_err; 1269 1270 return 0; 1271 1272 process_auth_encryption_ccm_err: 1273 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1274 return -EINVAL; 1275 } 1276 1277 /** Process AES-GCM decrypt algorithm */ 1278 static int 1279 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1280 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1281 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1282 { 1283 int len = 0; 1284 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1285 int unused = 0; 1286 uint8_t empty[] = {}; 1287 #endif 1288 1289 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1290 goto process_auth_decryption_gcm_err; 1291 1292 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1293 goto process_auth_decryption_gcm_err; 1294 1295 if (aadlen > 0) 1296 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1297 goto process_auth_decryption_gcm_err; 1298 1299 if (srclen > 0) 1300 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1301 srclen, ctx, 0)) 1302 goto process_auth_decryption_gcm_err; 1303 1304 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1305 /* Workaround open ssl bug in version less then 1.0.1f */ 1306 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1307 goto process_auth_decryption_gcm_err; 1308 #endif 1309 1310 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1311 return -EFAULT; 1312 1313 return 0; 1314 1315 process_auth_decryption_gcm_err: 1316 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1317 return -EINVAL; 1318 } 1319 1320 /** Process AES-CCM decrypt algorithm */ 1321 static int 1322 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1323 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1324 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1325 EVP_CIPHER_CTX *ctx) 1326 { 1327 int len = 0; 1328 1329 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1330 goto process_auth_decryption_ccm_err; 1331 1332 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1333 goto process_auth_decryption_ccm_err; 1334 1335 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1336 goto process_auth_decryption_ccm_err; 1337 1338 if (aadlen > 0) 1339 /* 1340 * For AES-CCM, the actual AAD is placed 1341 * 18 bytes after the start of the AAD field, 1342 * according to the API. 1343 */ 1344 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1345 goto process_auth_decryption_ccm_err; 1346 1347 if (srclen >= 0) 1348 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1349 srclen, ctx, 0)) 1350 return -EFAULT; 1351 1352 return 0; 1353 1354 process_auth_decryption_ccm_err: 1355 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1356 return -EINVAL; 1357 } 1358 1359 /** Process standard openssl auth algorithms */ 1360 static int 1361 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1362 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1363 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1364 { 1365 size_t dstlen; 1366 struct rte_mbuf *m; 1367 int l, n = srclen; 1368 uint8_t *src; 1369 1370 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1371 m = m->next) 1372 offset -= rte_pktmbuf_data_len(m); 1373 1374 if (m == 0) 1375 goto process_auth_err; 1376 1377 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1378 goto process_auth_err; 1379 1380 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1381 1382 l = rte_pktmbuf_data_len(m) - offset; 1383 if (srclen <= l) { 1384 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1385 goto process_auth_err; 1386 goto process_auth_final; 1387 } 1388 1389 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1390 goto process_auth_err; 1391 1392 n -= l; 1393 1394 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1395 src = rte_pktmbuf_mtod(m, uint8_t *); 1396 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1397 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1398 goto process_auth_err; 1399 n -= l; 1400 } 1401 1402 process_auth_final: 1403 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1404 goto process_auth_err; 1405 return 0; 1406 1407 process_auth_err: 1408 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1409 return -EINVAL; 1410 } 1411 1412 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1413 /** Process standard openssl auth algorithms with hmac/cmac */ 1414 static int 1415 process_openssl_auth_mac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1416 int srclen, EVP_MAC_CTX *ctx) 1417 { 1418 size_t dstlen; 1419 struct rte_mbuf *m; 1420 int l, n = srclen; 1421 uint8_t *src; 1422 1423 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1424 m = m->next) 1425 offset -= rte_pktmbuf_data_len(m); 1426 1427 if (m == 0) 1428 goto process_auth_err; 1429 1430 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1431 1432 l = rte_pktmbuf_data_len(m) - offset; 1433 if (srclen <= l) { 1434 if (EVP_MAC_update(ctx, (unsigned char *)src, srclen) != 1) 1435 goto process_auth_err; 1436 goto process_auth_final; 1437 } 1438 1439 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1440 goto process_auth_err; 1441 1442 n -= l; 1443 1444 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1445 src = rte_pktmbuf_mtod(m, uint8_t *); 1446 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1447 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1448 goto process_auth_err; 1449 n -= l; 1450 } 1451 1452 process_auth_final: 1453 if (EVP_MAC_final(ctx, dst, &dstlen, DIGEST_LENGTH_MAX) != 1) 1454 goto process_auth_err; 1455 1456 EVP_MAC_CTX_free(ctx); 1457 return 0; 1458 1459 process_auth_err: 1460 EVP_MAC_CTX_free(ctx); 1461 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1462 return -EINVAL; 1463 } 1464 # else 1465 /** Process standard openssl auth algorithms with hmac */ 1466 static int 1467 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1468 int srclen, HMAC_CTX *ctx) 1469 { 1470 unsigned int dstlen; 1471 struct rte_mbuf *m; 1472 int l, n = srclen; 1473 uint8_t *src; 1474 1475 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1476 m = m->next) 1477 offset -= rte_pktmbuf_data_len(m); 1478 1479 if (m == 0) 1480 goto process_auth_err; 1481 1482 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1483 1484 l = rte_pktmbuf_data_len(m) - offset; 1485 if (srclen <= l) { 1486 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1487 goto process_auth_err; 1488 goto process_auth_final; 1489 } 1490 1491 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1492 goto process_auth_err; 1493 1494 n -= l; 1495 1496 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1497 src = rte_pktmbuf_mtod(m, uint8_t *); 1498 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1499 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1500 goto process_auth_err; 1501 n -= l; 1502 } 1503 1504 process_auth_final: 1505 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1506 goto process_auth_err; 1507 1508 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1509 goto process_auth_err; 1510 1511 return 0; 1512 1513 process_auth_err: 1514 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1515 return -EINVAL; 1516 } 1517 1518 /** Process standard openssl auth algorithms with cmac */ 1519 static int 1520 process_openssl_auth_cmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1521 int srclen, CMAC_CTX *ctx) 1522 { 1523 unsigned int dstlen; 1524 struct rte_mbuf *m; 1525 int l, n = srclen; 1526 uint8_t *src; 1527 1528 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1529 m = m->next) 1530 offset -= rte_pktmbuf_data_len(m); 1531 1532 if (m == 0) 1533 goto process_auth_err; 1534 1535 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1536 1537 l = rte_pktmbuf_data_len(m) - offset; 1538 if (srclen <= l) { 1539 if (CMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1540 goto process_auth_err; 1541 goto process_auth_final; 1542 } 1543 1544 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1545 goto process_auth_err; 1546 1547 n -= l; 1548 1549 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1550 src = rte_pktmbuf_mtod(m, uint8_t *); 1551 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1552 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1553 goto process_auth_err; 1554 n -= l; 1555 } 1556 1557 process_auth_final: 1558 if (CMAC_Final(ctx, dst, (size_t *)&dstlen) != 1) 1559 goto process_auth_err; 1560 1561 CMAC_CTX_cleanup(ctx); 1562 1563 return 0; 1564 1565 process_auth_err: 1566 OPENSSL_LOG(ERR, "Process openssl cmac auth failed"); 1567 return -EINVAL; 1568 } 1569 # endif 1570 /*----------------------------------------------------------------------------*/ 1571 1572 /** Process auth/cipher combined operation */ 1573 static void 1574 process_openssl_combined_op 1575 (struct rte_crypto_op *op, struct openssl_session *sess, 1576 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1577 { 1578 /* cipher */ 1579 uint8_t *dst = NULL, *iv, *tag, *aad; 1580 int srclen, aadlen, status = -1; 1581 uint32_t offset; 1582 uint8_t taglen; 1583 1584 /* 1585 * Segmented destination buffer is not supported for 1586 * encryption/decryption 1587 */ 1588 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1589 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1590 return; 1591 } 1592 1593 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1594 sess->iv.offset); 1595 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1596 srclen = 0; 1597 offset = op->sym->auth.data.offset; 1598 aadlen = op->sym->auth.data.length; 1599 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1600 op->sym->auth.data.offset); 1601 tag = op->sym->auth.digest.data; 1602 if (tag == NULL) 1603 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1604 offset + aadlen); 1605 } else { 1606 srclen = op->sym->aead.data.length; 1607 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1608 op->sym->aead.data.offset); 1609 offset = op->sym->aead.data.offset; 1610 aad = op->sym->aead.aad.data; 1611 aadlen = sess->auth.aad_length; 1612 tag = op->sym->aead.digest.data; 1613 if (tag == NULL) 1614 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1615 offset + srclen); 1616 } 1617 1618 taglen = sess->auth.digest_length; 1619 1620 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1621 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1622 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1623 status = process_openssl_auth_encryption_gcm( 1624 mbuf_src, offset, srclen, 1625 aad, aadlen, iv, 1626 dst, tag, sess->cipher.ctx); 1627 else 1628 status = process_openssl_auth_encryption_ccm( 1629 mbuf_src, offset, srclen, 1630 aad, aadlen, iv, 1631 dst, tag, taglen, sess->cipher.ctx); 1632 1633 } else { 1634 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1635 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1636 status = process_openssl_auth_decryption_gcm( 1637 mbuf_src, offset, srclen, 1638 aad, aadlen, iv, 1639 dst, tag, sess->cipher.ctx); 1640 else 1641 status = process_openssl_auth_decryption_ccm( 1642 mbuf_src, offset, srclen, 1643 aad, aadlen, iv, 1644 dst, tag, taglen, sess->cipher.ctx); 1645 } 1646 1647 if (status != 0) { 1648 if (status == (-EFAULT) && 1649 sess->auth.operation == 1650 RTE_CRYPTO_AUTH_OP_VERIFY) 1651 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1652 else 1653 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1654 } 1655 } 1656 1657 /** Process cipher operation */ 1658 static void 1659 process_openssl_cipher_op 1660 (struct rte_crypto_op *op, struct openssl_session *sess, 1661 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1662 { 1663 uint8_t *dst, *iv; 1664 int srclen, status; 1665 uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0; 1666 EVP_CIPHER_CTX *ctx_copy; 1667 1668 /* 1669 * Segmented OOP destination buffer is not supported for encryption/ 1670 * decryption. In case of des3ctr, even inplace segmented buffers are 1671 * not supported. 1672 */ 1673 if (!rte_pktmbuf_is_contiguous(mbuf_dst) && 1674 (!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) { 1675 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1676 return; 1677 } 1678 1679 srclen = op->sym->cipher.data.length; 1680 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1681 op->sym->cipher.data.offset); 1682 1683 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1684 sess->iv.offset); 1685 ctx_copy = EVP_CIPHER_CTX_new(); 1686 EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx); 1687 1688 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1689 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1690 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1691 op->sym->cipher.data.offset, iv, 1692 srclen, ctx_copy, inplace); 1693 else 1694 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1695 op->sym->cipher.data.offset, iv, 1696 srclen, ctx_copy, inplace); 1697 else 1698 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1699 op->sym->cipher.data.offset, iv, 1700 sess->cipher.key.data, srclen, 1701 ctx_copy); 1702 1703 EVP_CIPHER_CTX_free(ctx_copy); 1704 if (status != 0) 1705 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1706 } 1707 1708 /** Process cipher operation */ 1709 static void 1710 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1711 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1712 struct rte_mbuf *mbuf_dst) 1713 { 1714 uint8_t *src, *dst, *iv; 1715 uint8_t block_size, last_block_len; 1716 int srclen, status = 0; 1717 1718 srclen = op->sym->cipher.data.length; 1719 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1720 op->sym->cipher.data.offset); 1721 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1722 op->sym->cipher.data.offset); 1723 1724 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1725 sess->iv.offset); 1726 1727 block_size = DES_BLOCK_SIZE; 1728 1729 last_block_len = srclen % block_size; 1730 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1731 /* Encrypt only with ECB mode XOR IV */ 1732 if (srclen < block_size) { 1733 status = process_openssl_cipher_bpi_encrypt(src, dst, 1734 iv, srclen, 1735 sess->cipher.bpi_ctx); 1736 } else { 1737 srclen -= last_block_len; 1738 /* Encrypt with the block aligned stream with CBC mode */ 1739 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1740 op->sym->cipher.data.offset, iv, 1741 srclen, sess->cipher.ctx, 0); 1742 if (last_block_len) { 1743 /* Point at last block */ 1744 dst += srclen; 1745 /* 1746 * IV is the last encrypted block from 1747 * the previous operation 1748 */ 1749 iv = dst - block_size; 1750 src += srclen; 1751 srclen = last_block_len; 1752 /* Encrypt the last frame with ECB mode */ 1753 status |= process_openssl_cipher_bpi_encrypt(src, 1754 dst, iv, 1755 srclen, sess->cipher.bpi_ctx); 1756 } 1757 } 1758 } else { 1759 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1760 if (srclen < block_size) { 1761 status = process_openssl_cipher_bpi_encrypt(src, dst, 1762 iv, 1763 srclen, 1764 sess->cipher.bpi_ctx); 1765 } else { 1766 if (last_block_len) { 1767 /* Point at last block */ 1768 dst += srclen - last_block_len; 1769 src += srclen - last_block_len; 1770 /* 1771 * IV is the last full block 1772 */ 1773 iv = src - block_size; 1774 /* 1775 * Decrypt the last frame with ECB mode 1776 * (encrypt, as it is the same operation) 1777 */ 1778 status = process_openssl_cipher_bpi_encrypt(src, 1779 dst, iv, 1780 last_block_len, sess->cipher.bpi_ctx); 1781 /* Prepare parameters for CBC mode op */ 1782 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1783 sess->iv.offset); 1784 dst += last_block_len - srclen; 1785 srclen -= last_block_len; 1786 } 1787 1788 /* Decrypt with CBC mode */ 1789 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1790 op->sym->cipher.data.offset, iv, 1791 srclen, sess->cipher.ctx, 0); 1792 } 1793 } 1794 1795 if (status != 0) 1796 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1797 } 1798 1799 /** Process auth operation */ 1800 static void 1801 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1802 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1803 struct rte_mbuf *mbuf_dst) 1804 { 1805 uint8_t *dst; 1806 int srclen, status; 1807 EVP_MD_CTX *ctx_a; 1808 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1809 EVP_MAC_CTX *ctx_h; 1810 EVP_MAC_CTX *ctx_c; 1811 # else 1812 HMAC_CTX *ctx_h; 1813 CMAC_CTX *ctx_c; 1814 # endif 1815 1816 srclen = op->sym->auth.data.length; 1817 1818 dst = qp->temp_digest; 1819 1820 switch (sess->auth.mode) { 1821 case OPENSSL_AUTH_AS_AUTH: 1822 ctx_a = EVP_MD_CTX_create(); 1823 EVP_MD_CTX_copy_ex(ctx_a, sess->auth.auth.ctx); 1824 status = process_openssl_auth(mbuf_src, dst, 1825 op->sym->auth.data.offset, NULL, NULL, srclen, 1826 ctx_a, sess->auth.auth.evp_algo); 1827 EVP_MD_CTX_destroy(ctx_a); 1828 break; 1829 case OPENSSL_AUTH_AS_HMAC: 1830 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1831 ctx_h = EVP_MAC_CTX_dup(sess->auth.hmac.ctx); 1832 status = process_openssl_auth_mac(mbuf_src, dst, 1833 op->sym->auth.data.offset, srclen, 1834 ctx_h); 1835 # else 1836 ctx_h = HMAC_CTX_new(); 1837 HMAC_CTX_copy(ctx_h, sess->auth.hmac.ctx); 1838 status = process_openssl_auth_hmac(mbuf_src, dst, 1839 op->sym->auth.data.offset, srclen, 1840 ctx_h); 1841 HMAC_CTX_free(ctx_h); 1842 # endif 1843 break; 1844 case OPENSSL_AUTH_AS_CMAC: 1845 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1846 ctx_c = EVP_MAC_CTX_dup(sess->auth.cmac.ctx); 1847 status = process_openssl_auth_mac(mbuf_src, dst, 1848 op->sym->auth.data.offset, srclen, 1849 ctx_c); 1850 # else 1851 ctx_c = CMAC_CTX_new(); 1852 CMAC_CTX_copy(ctx_c, sess->auth.cmac.ctx); 1853 status = process_openssl_auth_cmac(mbuf_src, dst, 1854 op->sym->auth.data.offset, srclen, 1855 ctx_c); 1856 CMAC_CTX_free(ctx_c); 1857 # endif 1858 break; 1859 default: 1860 status = -1; 1861 break; 1862 } 1863 1864 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1865 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data, 1866 sess->auth.digest_length) != 0) { 1867 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1868 } 1869 } else { 1870 uint8_t *auth_dst; 1871 1872 auth_dst = op->sym->auth.digest.data; 1873 if (auth_dst == NULL) 1874 auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1875 op->sym->auth.data.offset + 1876 op->sym->auth.data.length); 1877 memcpy(auth_dst, dst, sess->auth.digest_length); 1878 } 1879 1880 if (status != 0) 1881 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1882 } 1883 1884 /* process dsa sign operation */ 1885 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 1886 static int 1887 process_openssl_dsa_sign_op_evp(struct rte_crypto_op *cop, 1888 struct openssl_asym_session *sess) 1889 { 1890 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1891 EVP_PKEY_CTX *dsa_ctx = NULL; 1892 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 1893 EVP_PKEY *pkey = NULL; 1894 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 1895 OSSL_PARAM *params = NULL; 1896 1897 size_t outlen; 1898 unsigned char *dsa_sign_data; 1899 const unsigned char *dsa_sign_data_p; 1900 1901 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1902 params = OSSL_PARAM_BLD_to_param(param_bld); 1903 if (!params) { 1904 OSSL_PARAM_BLD_free(param_bld); 1905 return -1; 1906 } 1907 1908 if (key_ctx == NULL 1909 || EVP_PKEY_fromdata_init(key_ctx) <= 0 1910 || EVP_PKEY_fromdata(key_ctx, &pkey, 1911 EVP_PKEY_KEYPAIR, params) <= 0) 1912 goto err_dsa_sign; 1913 1914 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 1915 if (!dsa_ctx) 1916 goto err_dsa_sign; 1917 1918 if (EVP_PKEY_sign_init(dsa_ctx) <= 0) 1919 goto err_dsa_sign; 1920 1921 if (EVP_PKEY_sign(dsa_ctx, NULL, &outlen, op->message.data, 1922 op->message.length) <= 0) 1923 goto err_dsa_sign; 1924 1925 if (outlen <= 0) 1926 goto err_dsa_sign; 1927 1928 dsa_sign_data = OPENSSL_malloc(outlen); 1929 if (!dsa_sign_data) 1930 goto err_dsa_sign; 1931 1932 if (EVP_PKEY_sign(dsa_ctx, dsa_sign_data, &outlen, op->message.data, 1933 op->message.length) <= 0) { 1934 OPENSSL_free(dsa_sign_data); 1935 goto err_dsa_sign; 1936 } 1937 1938 dsa_sign_data_p = (const unsigned char *)dsa_sign_data; 1939 DSA_SIG *sign = d2i_DSA_SIG(NULL, &dsa_sign_data_p, outlen); 1940 if (!sign) { 1941 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 1942 OPENSSL_free(dsa_sign_data); 1943 goto err_dsa_sign; 1944 } else { 1945 const BIGNUM *r = NULL, *s = NULL; 1946 get_dsa_sign(sign, &r, &s); 1947 1948 op->r.length = BN_bn2bin(r, op->r.data); 1949 op->s.length = BN_bn2bin(s, op->s.data); 1950 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1951 } 1952 1953 DSA_SIG_free(sign); 1954 OPENSSL_free(dsa_sign_data); 1955 return 0; 1956 1957 err_dsa_sign: 1958 if (params) 1959 OSSL_PARAM_free(params); 1960 if (key_ctx) 1961 EVP_PKEY_CTX_free(key_ctx); 1962 if (dsa_ctx) 1963 EVP_PKEY_CTX_free(dsa_ctx); 1964 return -1; 1965 } 1966 1967 /* process dsa verify operation */ 1968 static int 1969 process_openssl_dsa_verify_op_evp(struct rte_crypto_op *cop, 1970 struct openssl_asym_session *sess) 1971 { 1972 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1973 DSA_SIG *sign = DSA_SIG_new(); 1974 BIGNUM *r = NULL, *s = NULL; 1975 BIGNUM *pub_key = NULL; 1976 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 1977 OSSL_PARAM *params = NULL; 1978 EVP_PKEY *pkey = NULL; 1979 EVP_PKEY_CTX *dsa_ctx = NULL; 1980 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 1981 unsigned char *dsa_sig = NULL; 1982 size_t sig_len; 1983 int ret = -1; 1984 1985 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1986 if (!param_bld) { 1987 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 1988 return -1; 1989 } 1990 1991 r = BN_bin2bn(op->r.data, op->r.length, r); 1992 s = BN_bin2bn(op->s.data, op->s.length, s); 1993 pub_key = BN_bin2bn(op->y.data, op->y.length, pub_key); 1994 if (!r || !s || !pub_key) { 1995 BN_free(r); 1996 BN_free(s); 1997 BN_free(pub_key); 1998 OSSL_PARAM_BLD_free(param_bld); 1999 goto err_dsa_verify; 2000 } 2001 2002 set_dsa_sign(sign, r, s); 2003 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PUB_KEY, pub_key)) { 2004 OSSL_PARAM_BLD_free(param_bld); 2005 goto err_dsa_verify; 2006 } 2007 2008 params = OSSL_PARAM_BLD_to_param(param_bld); 2009 if (!params) { 2010 OSSL_PARAM_BLD_free(param_bld); 2011 goto err_dsa_verify; 2012 } 2013 2014 if (key_ctx == NULL 2015 || EVP_PKEY_fromdata_init(key_ctx) <= 0 2016 || EVP_PKEY_fromdata(key_ctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2017 goto err_dsa_verify; 2018 2019 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 2020 if (!dsa_ctx) 2021 goto err_dsa_verify; 2022 2023 if (!sign) 2024 goto err_dsa_verify; 2025 2026 sig_len = i2d_DSA_SIG(sign, &dsa_sig); 2027 if (EVP_PKEY_verify_init(dsa_ctx) <= 0) 2028 goto err_dsa_verify; 2029 2030 ret = EVP_PKEY_verify(dsa_ctx, dsa_sig, sig_len, 2031 op->message.data, op->message.length); 2032 if (ret == 1) { 2033 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2034 ret = 0; 2035 } 2036 2037 err_dsa_verify: 2038 if (sign) 2039 DSA_SIG_free(sign); 2040 if (params) 2041 OSSL_PARAM_free(params); 2042 if (key_ctx) 2043 EVP_PKEY_CTX_free(key_ctx); 2044 if (dsa_ctx) 2045 EVP_PKEY_CTX_free(dsa_ctx); 2046 2047 return ret; 2048 } 2049 #else 2050 static int 2051 process_openssl_dsa_sign_op(struct rte_crypto_op *cop, 2052 struct openssl_asym_session *sess) 2053 { 2054 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2055 DSA *dsa = sess->u.s.dsa; 2056 DSA_SIG *sign = NULL; 2057 2058 sign = DSA_do_sign(op->message.data, 2059 op->message.length, 2060 dsa); 2061 2062 if (sign == NULL) { 2063 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 2064 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2065 } else { 2066 const BIGNUM *r = NULL, *s = NULL; 2067 get_dsa_sign(sign, &r, &s); 2068 2069 op->r.length = BN_bn2bin(r, op->r.data); 2070 op->s.length = BN_bn2bin(s, op->s.data); 2071 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2072 } 2073 2074 DSA_SIG_free(sign); 2075 2076 return 0; 2077 } 2078 2079 /* process dsa verify operation */ 2080 static int 2081 process_openssl_dsa_verify_op(struct rte_crypto_op *cop, 2082 struct openssl_asym_session *sess) 2083 { 2084 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2085 DSA *dsa = sess->u.s.dsa; 2086 int ret; 2087 DSA_SIG *sign = DSA_SIG_new(); 2088 BIGNUM *r = NULL, *s = NULL; 2089 BIGNUM *pub_key = NULL; 2090 2091 if (sign == NULL) { 2092 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 2093 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2094 return -1; 2095 } 2096 2097 r = BN_bin2bn(op->r.data, 2098 op->r.length, 2099 r); 2100 s = BN_bin2bn(op->s.data, 2101 op->s.length, 2102 s); 2103 pub_key = BN_bin2bn(op->y.data, 2104 op->y.length, 2105 pub_key); 2106 if (!r || !s || !pub_key) { 2107 BN_free(r); 2108 BN_free(s); 2109 BN_free(pub_key); 2110 2111 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2112 return -1; 2113 } 2114 set_dsa_sign(sign, r, s); 2115 set_dsa_pub_key(dsa, pub_key); 2116 2117 ret = DSA_do_verify(op->message.data, 2118 op->message.length, 2119 sign, 2120 dsa); 2121 2122 if (ret != 1) 2123 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2124 else 2125 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2126 2127 DSA_SIG_free(sign); 2128 2129 return 0; 2130 } 2131 #endif 2132 2133 /* process dh operation */ 2134 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2135 static int 2136 process_openssl_dh_op_evp(struct rte_crypto_op *cop, 2137 struct openssl_asym_session *sess) 2138 { 2139 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2140 OSSL_PARAM_BLD *param_bld = sess->u.dh.param_bld; 2141 OSSL_PARAM_BLD *param_bld_peer = sess->u.dh.param_bld_peer; 2142 OSSL_PARAM *params = NULL; 2143 EVP_PKEY *dhpkey = NULL; 2144 EVP_PKEY *peerkey = NULL; 2145 BIGNUM *priv_key = NULL; 2146 BIGNUM *pub_key = NULL; 2147 int ret = -1; 2148 2149 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2150 EVP_PKEY_CTX *dh_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2151 if (dh_ctx == NULL || param_bld == NULL) 2152 return ret; 2153 2154 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2155 OSSL_PARAM *params_peer = NULL; 2156 2157 if (!param_bld_peer) 2158 return ret; 2159 2160 pub_key = BN_bin2bn(op->pub_key.data, op->pub_key.length, 2161 pub_key); 2162 if (pub_key == NULL) { 2163 OSSL_PARAM_BLD_free(param_bld_peer); 2164 return ret; 2165 } 2166 2167 if (!OSSL_PARAM_BLD_push_BN(param_bld_peer, OSSL_PKEY_PARAM_PUB_KEY, 2168 pub_key)) { 2169 OPENSSL_LOG(ERR, "Failed to set public key\n"); 2170 OSSL_PARAM_BLD_free(param_bld_peer); 2171 BN_free(pub_key); 2172 return ret; 2173 } 2174 2175 params_peer = OSSL_PARAM_BLD_to_param(param_bld_peer); 2176 if (!params_peer) { 2177 OSSL_PARAM_BLD_free(param_bld_peer); 2178 BN_free(pub_key); 2179 return ret; 2180 } 2181 2182 EVP_PKEY_CTX *peer_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2183 if (EVP_PKEY_keygen_init(peer_ctx) != 1) { 2184 OSSL_PARAM_free(params_peer); 2185 BN_free(pub_key); 2186 return ret; 2187 } 2188 2189 if (EVP_PKEY_CTX_set_params(peer_ctx, params_peer) != 1) { 2190 EVP_PKEY_CTX_free(peer_ctx); 2191 OSSL_PARAM_free(params_peer); 2192 BN_free(pub_key); 2193 return ret; 2194 } 2195 2196 if (EVP_PKEY_keygen(peer_ctx, &peerkey) != 1) { 2197 EVP_PKEY_CTX_free(peer_ctx); 2198 OSSL_PARAM_free(params_peer); 2199 BN_free(pub_key); 2200 return ret; 2201 } 2202 2203 priv_key = BN_bin2bn(op->priv_key.data, op->priv_key.length, 2204 priv_key); 2205 if (priv_key == NULL) { 2206 EVP_PKEY_CTX_free(peer_ctx); 2207 OSSL_PARAM_free(params_peer); 2208 BN_free(pub_key); 2209 return ret; 2210 } 2211 2212 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY, 2213 priv_key)) { 2214 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2215 EVP_PKEY_CTX_free(peer_ctx); 2216 OSSL_PARAM_free(params_peer); 2217 BN_free(pub_key); 2218 BN_free(priv_key); 2219 return ret; 2220 } 2221 2222 OSSL_PARAM_free(params_peer); 2223 EVP_PKEY_CTX_free(peer_ctx); 2224 } 2225 2226 params = OSSL_PARAM_BLD_to_param(param_bld); 2227 if (!params) 2228 goto err_dh; 2229 2230 if (EVP_PKEY_keygen_init(dh_ctx) != 1) 2231 goto err_dh; 2232 2233 if (EVP_PKEY_CTX_set_params(dh_ctx, params) != 1) 2234 goto err_dh; 2235 2236 if (EVP_PKEY_keygen(dh_ctx, &dhpkey) != 1) 2237 goto err_dh; 2238 2239 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2240 OPENSSL_LOG(DEBUG, "%s:%d updated pub key\n", __func__, __LINE__); 2241 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PUB_KEY, &pub_key)) 2242 goto err_dh; 2243 /* output public key */ 2244 op->pub_key.length = BN_bn2bin(pub_key, op->pub_key.data); 2245 } 2246 2247 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2248 2249 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", __func__, __LINE__); 2250 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PRIV_KEY, &priv_key)) 2251 goto err_dh; 2252 2253 /* provide generated private key back to user */ 2254 op->priv_key.length = BN_bn2bin(priv_key, op->priv_key.data); 2255 } 2256 2257 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2258 size_t skey_len; 2259 EVP_PKEY_CTX *sc_ctx = EVP_PKEY_CTX_new(dhpkey, NULL); 2260 if (!sc_ctx) 2261 goto err_dh; 2262 2263 if (EVP_PKEY_derive_init(sc_ctx) <= 0) { 2264 EVP_PKEY_CTX_free(sc_ctx); 2265 goto err_dh; 2266 } 2267 2268 if (!peerkey) { 2269 EVP_PKEY_CTX_free(sc_ctx); 2270 goto err_dh; 2271 } 2272 2273 if (EVP_PKEY_derive_set_peer(sc_ctx, peerkey) <= 0) { 2274 EVP_PKEY_CTX_free(sc_ctx); 2275 goto err_dh; 2276 } 2277 2278 /* Determine buffer length */ 2279 if (EVP_PKEY_derive(sc_ctx, NULL, &skey_len) <= 0) { 2280 EVP_PKEY_CTX_free(sc_ctx); 2281 goto err_dh; 2282 } 2283 2284 if (EVP_PKEY_derive(sc_ctx, op->shared_secret.data, &skey_len) <= 0) { 2285 EVP_PKEY_CTX_free(sc_ctx); 2286 goto err_dh; 2287 } 2288 2289 op->shared_secret.length = skey_len; 2290 EVP_PKEY_CTX_free(sc_ctx); 2291 } 2292 2293 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2294 ret = 0; 2295 2296 err_dh: 2297 if (pub_key) 2298 BN_free(pub_key); 2299 if (priv_key) 2300 BN_free(priv_key); 2301 if (params) 2302 OSSL_PARAM_free(params); 2303 if (dhpkey) 2304 EVP_PKEY_free(dhpkey); 2305 if (peerkey) 2306 EVP_PKEY_free(peerkey); 2307 2308 EVP_PKEY_CTX_free(dh_ctx); 2309 2310 return ret; 2311 } 2312 #else 2313 static int 2314 process_openssl_dh_op(struct rte_crypto_op *cop, 2315 struct openssl_asym_session *sess) 2316 { 2317 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2318 struct rte_crypto_asym_op *asym_op = cop->asym; 2319 DH *dh_key = sess->u.dh.dh_key; 2320 BIGNUM *priv_key = NULL; 2321 int ret = 0; 2322 2323 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2324 /* compute shared secret using peer public key 2325 * and current private key 2326 * shared secret = peer_key ^ priv_key mod p 2327 */ 2328 BIGNUM *peer_key = NULL; 2329 2330 /* copy private key and peer key and compute shared secret */ 2331 peer_key = BN_bin2bn(op->pub_key.data, 2332 op->pub_key.length, 2333 peer_key); 2334 if (peer_key == NULL) { 2335 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2336 return -1; 2337 } 2338 priv_key = BN_bin2bn(op->priv_key.data, 2339 op->priv_key.length, 2340 priv_key); 2341 if (priv_key == NULL) { 2342 BN_free(peer_key); 2343 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2344 return -1; 2345 } 2346 ret = set_dh_priv_key(dh_key, priv_key); 2347 if (ret) { 2348 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2349 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2350 BN_free(peer_key); 2351 BN_free(priv_key); 2352 return 0; 2353 } 2354 2355 ret = DH_compute_key( 2356 op->shared_secret.data, 2357 peer_key, dh_key); 2358 if (ret < 0) { 2359 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2360 BN_free(peer_key); 2361 /* priv key is already loaded into dh, 2362 * let's not free that directly here. 2363 * DH_free() will auto free it later. 2364 */ 2365 return 0; 2366 } 2367 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2368 op->shared_secret.length = ret; 2369 BN_free(peer_key); 2370 return 0; 2371 } 2372 2373 /* 2374 * other options are public and private key generations. 2375 * 2376 * if user provides private key, 2377 * then first set DH with user provided private key 2378 */ 2379 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE && 2380 op->priv_key.length) { 2381 /* generate public key using user-provided private key 2382 * pub_key = g ^ priv_key mod p 2383 */ 2384 2385 /* load private key into DH */ 2386 priv_key = BN_bin2bn(op->priv_key.data, 2387 op->priv_key.length, 2388 priv_key); 2389 if (priv_key == NULL) { 2390 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2391 return -1; 2392 } 2393 ret = set_dh_priv_key(dh_key, priv_key); 2394 if (ret) { 2395 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2396 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2397 BN_free(priv_key); 2398 return 0; 2399 } 2400 } 2401 2402 /* generate public and private key pair. 2403 * 2404 * if private key already set, generates only public key. 2405 * 2406 * if private key is not already set, then set it to random value 2407 * and update internal private key. 2408 */ 2409 if (!DH_generate_key(dh_key)) { 2410 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2411 return 0; 2412 } 2413 2414 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2415 const BIGNUM *pub_key = NULL; 2416 2417 OPENSSL_LOG(DEBUG, "%s:%d update public key\n", 2418 __func__, __LINE__); 2419 2420 /* get the generated keys */ 2421 get_dh_pub_key(dh_key, &pub_key); 2422 2423 /* output public key */ 2424 op->pub_key.length = BN_bn2bin(pub_key, 2425 op->pub_key.data); 2426 } 2427 2428 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2429 const BIGNUM *priv_key = NULL; 2430 2431 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", 2432 __func__, __LINE__); 2433 2434 /* get the generated keys */ 2435 get_dh_priv_key(dh_key, &priv_key); 2436 2437 /* provide generated private key back to user */ 2438 op->priv_key.length = BN_bn2bin(priv_key, 2439 op->priv_key.data); 2440 } 2441 2442 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2443 2444 return 0; 2445 } 2446 #endif 2447 2448 /* process modinv operation */ 2449 static int 2450 process_openssl_modinv_op(struct rte_crypto_op *cop, 2451 struct openssl_asym_session *sess) 2452 { 2453 struct rte_crypto_asym_op *op = cop->asym; 2454 BIGNUM *base = BN_CTX_get(sess->u.m.ctx); 2455 BIGNUM *res = BN_CTX_get(sess->u.m.ctx); 2456 2457 if (unlikely(base == NULL || res == NULL)) { 2458 BN_free(base); 2459 BN_free(res); 2460 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2461 return -1; 2462 } 2463 2464 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 2465 op->modinv.base.length, base); 2466 2467 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) { 2468 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2469 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data); 2470 } else { 2471 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2472 } 2473 2474 BN_clear(res); 2475 BN_clear(base); 2476 2477 return 0; 2478 } 2479 2480 /* process modexp operation */ 2481 static int 2482 process_openssl_modexp_op(struct rte_crypto_op *cop, 2483 struct openssl_asym_session *sess) 2484 { 2485 struct rte_crypto_asym_op *op = cop->asym; 2486 BIGNUM *base = BN_CTX_get(sess->u.e.ctx); 2487 BIGNUM *res = BN_CTX_get(sess->u.e.ctx); 2488 2489 if (unlikely(base == NULL || res == NULL)) { 2490 BN_free(base); 2491 BN_free(res); 2492 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2493 return -1; 2494 } 2495 2496 base = BN_bin2bn((const unsigned char *)op->modex.base.data, 2497 op->modex.base.length, base); 2498 2499 if (BN_mod_exp(res, base, sess->u.e.exp, 2500 sess->u.e.mod, sess->u.e.ctx)) { 2501 op->modex.result.length = BN_bn2bin(res, op->modex.result.data); 2502 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2503 } else { 2504 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2505 } 2506 2507 BN_clear(res); 2508 BN_clear(base); 2509 2510 return 0; 2511 } 2512 2513 /* process rsa operations */ 2514 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2515 static int 2516 process_openssl_rsa_op_evp(struct rte_crypto_op *cop, 2517 struct openssl_asym_session *sess) 2518 { 2519 struct rte_crypto_asym_op *op = cop->asym; 2520 uint32_t pad = (op->rsa.padding.type); 2521 uint8_t *tmp; 2522 size_t outlen = 0; 2523 int ret = -1; 2524 2525 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2526 EVP_PKEY_CTX *rsa_ctx = sess->u.r.ctx; 2527 if (!rsa_ctx) 2528 return ret; 2529 2530 switch (pad) { 2531 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2532 pad = RSA_PKCS1_PADDING; 2533 break; 2534 case RTE_CRYPTO_RSA_PADDING_NONE: 2535 pad = RSA_NO_PADDING; 2536 break; 2537 default: 2538 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2539 OPENSSL_LOG(ERR, 2540 "rsa pad type not supported %d\n", pad); 2541 return ret; 2542 } 2543 2544 switch (op->rsa.op_type) { 2545 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2546 if (EVP_PKEY_encrypt_init(rsa_ctx) != 1) 2547 goto err_rsa; 2548 2549 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2550 goto err_rsa; 2551 2552 if (EVP_PKEY_encrypt(rsa_ctx, NULL, &outlen, 2553 op->rsa.message.data, 2554 op->rsa.message.length) <= 0) 2555 goto err_rsa; 2556 2557 if (outlen <= 0) 2558 goto err_rsa; 2559 2560 if (EVP_PKEY_encrypt(rsa_ctx, op->rsa.cipher.data, &outlen, 2561 op->rsa.message.data, 2562 op->rsa.message.length) <= 0) 2563 goto err_rsa; 2564 op->rsa.cipher.length = outlen; 2565 2566 OPENSSL_LOG(DEBUG, 2567 "length of encrypted text %zu\n", outlen); 2568 break; 2569 2570 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2571 if (EVP_PKEY_decrypt_init(rsa_ctx) != 1) 2572 goto err_rsa; 2573 2574 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2575 goto err_rsa; 2576 2577 if (EVP_PKEY_decrypt(rsa_ctx, NULL, &outlen, 2578 op->rsa.cipher.data, 2579 op->rsa.cipher.length) <= 0) 2580 goto err_rsa; 2581 2582 if (outlen <= 0) 2583 goto err_rsa; 2584 2585 if (EVP_PKEY_decrypt(rsa_ctx, op->rsa.message.data, &outlen, 2586 op->rsa.cipher.data, 2587 op->rsa.cipher.length) <= 0) 2588 goto err_rsa; 2589 op->rsa.message.length = outlen; 2590 2591 OPENSSL_LOG(DEBUG, "length of decrypted text %zu\n", outlen); 2592 break; 2593 2594 case RTE_CRYPTO_ASYM_OP_SIGN: 2595 if (EVP_PKEY_sign_init(rsa_ctx) <= 0) 2596 goto err_rsa; 2597 2598 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2599 goto err_rsa; 2600 2601 if (EVP_PKEY_sign(rsa_ctx, NULL, &outlen, 2602 op->rsa.message.data, 2603 op->rsa.message.length) <= 0) 2604 goto err_rsa; 2605 2606 if (outlen <= 0) 2607 goto err_rsa; 2608 2609 if (EVP_PKEY_sign(rsa_ctx, op->rsa.sign.data, &outlen, 2610 op->rsa.message.data, 2611 op->rsa.message.length) <= 0) 2612 goto err_rsa; 2613 op->rsa.sign.length = outlen; 2614 break; 2615 2616 case RTE_CRYPTO_ASYM_OP_VERIFY: 2617 if (EVP_PKEY_verify_recover_init(rsa_ctx) <= 0) 2618 goto err_rsa; 2619 2620 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2621 goto err_rsa; 2622 2623 if (EVP_PKEY_verify_recover(rsa_ctx, NULL, &outlen, 2624 op->rsa.sign.data, 2625 op->rsa.sign.length) <= 0) 2626 goto err_rsa; 2627 2628 if ((outlen <= 0) || (outlen != op->rsa.sign.length)) 2629 goto err_rsa; 2630 2631 tmp = OPENSSL_malloc(outlen); 2632 if (tmp == NULL) { 2633 OPENSSL_LOG(ERR, "Memory allocation failed"); 2634 goto err_rsa; 2635 } 2636 2637 if (EVP_PKEY_verify_recover(rsa_ctx, tmp, &outlen, 2638 op->rsa.sign.data, 2639 op->rsa.sign.length) <= 0) { 2640 OPENSSL_free(tmp); 2641 goto err_rsa; 2642 } 2643 2644 OPENSSL_LOG(DEBUG, 2645 "Length of public_decrypt %zu " 2646 "length of message %zd\n", 2647 outlen, op->rsa.message.length); 2648 if (CRYPTO_memcmp(tmp, op->rsa.message.data, 2649 op->rsa.message.length)) { 2650 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 2651 } 2652 OPENSSL_free(tmp); 2653 break; 2654 2655 default: 2656 /* allow ops with invalid args to be pushed to 2657 * completion queue 2658 */ 2659 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2660 goto err_rsa; 2661 } 2662 2663 ret = 0; 2664 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2665 err_rsa: 2666 return ret; 2667 2668 } 2669 2670 static int 2671 process_openssl_sm2_op_evp(struct rte_crypto_op *cop, 2672 struct openssl_asym_session *sess) 2673 { 2674 EVP_PKEY_CTX *kctx = NULL, *sctx = NULL, *cctx = NULL; 2675 struct rte_crypto_asym_op *op = cop->asym; 2676 OSSL_PARAM_BLD *param_bld = NULL; 2677 OSSL_PARAM *params = NULL; 2678 EVP_PKEY *pkey = NULL; 2679 BIGNUM *pkey_bn = NULL; 2680 uint8_t pubkey[64]; 2681 size_t len = 0; 2682 int ret = -1; 2683 2684 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2685 2686 if (cop->asym->sm2.k.data != NULL) 2687 goto err_sm2; 2688 2689 param_bld = OSSL_PARAM_BLD_new(); 2690 if (!param_bld) { 2691 OPENSSL_LOG(ERR, "failed to allocate params\n"); 2692 goto err_sm2; 2693 } 2694 2695 ret = OSSL_PARAM_BLD_push_utf8_string(param_bld, 2696 OSSL_PKEY_PARAM_GROUP_NAME, "SM2", 0); 2697 if (!ret) { 2698 OPENSSL_LOG(ERR, "failed to push params\n"); 2699 goto err_sm2; 2700 } 2701 2702 pkey_bn = BN_bin2bn((const unsigned char *)op->sm2.pkey.data, 2703 op->sm2.pkey.length, pkey_bn); 2704 2705 memset(pubkey, 0, RTE_DIM(pubkey)); 2706 pubkey[0] = 0x04; 2707 len += 1; 2708 memcpy(&pubkey[len], op->sm2.q.x.data, op->sm2.q.x.length); 2709 len += op->sm2.q.x.length; 2710 memcpy(&pubkey[len], op->sm2.q.y.data, op->sm2.q.y.length); 2711 len += op->sm2.q.y.length; 2712 2713 ret = OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY, 2714 pkey_bn); 2715 if (!ret) { 2716 OPENSSL_LOG(ERR, "failed to push params\n"); 2717 goto err_sm2; 2718 } 2719 2720 ret = OSSL_PARAM_BLD_push_octet_string(param_bld, 2721 OSSL_PKEY_PARAM_PUB_KEY, pubkey, len); 2722 if (!ret) { 2723 OPENSSL_LOG(ERR, "failed to push params\n"); 2724 goto err_sm2; 2725 } 2726 2727 params = OSSL_PARAM_BLD_to_param(param_bld); 2728 if (!params) { 2729 OPENSSL_LOG(ERR, "failed to push params\n"); 2730 goto err_sm2; 2731 } 2732 2733 switch (op->sm2.op_type) { 2734 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2735 { 2736 OSSL_PARAM *eparams = sess->u.sm2.params; 2737 size_t output_len = 0; 2738 2739 kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_SM2, NULL); 2740 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2741 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2742 goto err_sm2; 2743 2744 cctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2745 if (!cctx) 2746 goto err_sm2; 2747 2748 if (!EVP_PKEY_encrypt_init(cctx)) 2749 goto err_sm2; 2750 2751 if (!EVP_PKEY_CTX_set_params(cctx, eparams)) 2752 goto err_sm2; 2753 2754 if (!EVP_PKEY_encrypt(cctx, op->sm2.cipher.data, &output_len, 2755 op->sm2.message.data, 2756 op->sm2.message.length)) 2757 goto err_sm2; 2758 op->sm2.cipher.length = output_len; 2759 } 2760 break; 2761 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2762 { 2763 OSSL_PARAM *eparams = sess->u.sm2.params; 2764 2765 kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_SM2, NULL); 2766 if (kctx == NULL 2767 || EVP_PKEY_fromdata_init(kctx) <= 0 2768 || EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2769 goto err_sm2; 2770 2771 cctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2772 if (!cctx) 2773 goto err_sm2; 2774 2775 if (!EVP_PKEY_decrypt_init(cctx)) 2776 goto err_sm2; 2777 2778 if (!EVP_PKEY_CTX_set_params(cctx, eparams)) 2779 goto err_sm2; 2780 2781 if (!EVP_PKEY_decrypt(cctx, op->sm2.message.data, &op->sm2.message.length, 2782 op->sm2.cipher.data, op->sm2.cipher.length)) 2783 goto err_sm2; 2784 } 2785 break; 2786 case RTE_CRYPTO_ASYM_OP_SIGN: 2787 { 2788 unsigned char signbuf[128] = {0}; 2789 const unsigned char *signptr; 2790 EVP_MD_CTX *md_ctx = NULL; 2791 const BIGNUM *r, *s; 2792 ECDSA_SIG *ec_sign; 2793 EVP_MD *check_md; 2794 size_t signlen; 2795 2796 kctx = EVP_PKEY_CTX_new_from_name(NULL, "SM2", NULL); 2797 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2798 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2799 goto err_sm2; 2800 2801 md_ctx = EVP_MD_CTX_new(); 2802 if (!md_ctx) 2803 goto err_sm2; 2804 2805 sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2806 if (!sctx) 2807 goto err_sm2; 2808 2809 EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx); 2810 2811 check_md = EVP_MD_fetch(NULL, "sm3", NULL); 2812 if (!check_md) 2813 goto err_sm2; 2814 2815 if (!EVP_DigestSignInit(md_ctx, NULL, check_md, NULL, pkey)) 2816 goto err_sm2; 2817 2818 if (EVP_PKEY_CTX_set1_id(sctx, op->sm2.id.data, op->sm2.id.length) <= 0) 2819 goto err_sm2; 2820 2821 if (!EVP_DigestSignUpdate(md_ctx, op->sm2.message.data, 2822 op->sm2.message.length)) 2823 goto err_sm2; 2824 2825 if (!EVP_DigestSignFinal(md_ctx, NULL, &signlen)) 2826 goto err_sm2; 2827 2828 if (!EVP_DigestSignFinal(md_ctx, signbuf, &signlen)) 2829 goto err_sm2; 2830 2831 signptr = signbuf; 2832 ec_sign = d2i_ECDSA_SIG(NULL, &signptr, signlen); 2833 if (!ec_sign) 2834 goto err_sm2; 2835 2836 r = ECDSA_SIG_get0_r(ec_sign); 2837 s = ECDSA_SIG_get0_s(ec_sign); 2838 if (!r || !s) 2839 goto err_sm2; 2840 2841 op->sm2.r.length = BN_num_bytes(r); 2842 op->sm2.s.length = BN_num_bytes(s); 2843 BN_bn2bin(r, op->sm2.r.data); 2844 BN_bn2bin(s, op->sm2.s.data); 2845 2846 ECDSA_SIG_free(ec_sign); 2847 } 2848 break; 2849 case RTE_CRYPTO_ASYM_OP_VERIFY: 2850 { 2851 unsigned char signbuf[128] = {0}; 2852 BIGNUM *r = NULL, *s = NULL; 2853 EVP_MD_CTX *md_ctx = NULL; 2854 ECDSA_SIG *ec_sign; 2855 EVP_MD *check_md; 2856 size_t signlen; 2857 2858 kctx = EVP_PKEY_CTX_new_from_name(NULL, "SM2", NULL); 2859 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2860 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_PUBLIC_KEY, params) <= 0) 2861 goto err_sm2; 2862 2863 if (!EVP_PKEY_is_a(pkey, "SM2")) 2864 goto err_sm2; 2865 2866 md_ctx = EVP_MD_CTX_new(); 2867 if (!md_ctx) 2868 goto err_sm2; 2869 2870 sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2871 if (!sctx) 2872 goto err_sm2; 2873 2874 EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx); 2875 2876 check_md = EVP_MD_fetch(NULL, "sm3", NULL); 2877 if (!check_md) 2878 goto err_sm2; 2879 2880 if (!EVP_DigestVerifyInit(md_ctx, NULL, check_md, NULL, pkey)) 2881 goto err_sm2; 2882 2883 if (EVP_PKEY_CTX_set1_id(sctx, op->sm2.id.data, op->sm2.id.length) <= 0) 2884 goto err_sm2; 2885 2886 if (!EVP_DigestVerifyUpdate(md_ctx, op->sm2.message.data, 2887 op->sm2.message.length)) 2888 goto err_sm2; 2889 2890 ec_sign = ECDSA_SIG_new(); 2891 if (!ec_sign) 2892 goto err_sm2; 2893 2894 r = BN_bin2bn(op->sm2.r.data, op->sm2.r.length, r); 2895 s = BN_bin2bn(op->sm2.s.data, op->sm2.s.length, s); 2896 if (!r || !s) 2897 goto err_sm2; 2898 2899 if (!ECDSA_SIG_set0(ec_sign, r, s)) { 2900 BN_free(r); 2901 BN_free(s); 2902 goto err_sm2; 2903 } 2904 2905 r = NULL; 2906 s = NULL; 2907 2908 signlen = i2d_ECDSA_SIG(ec_sign, (unsigned char **)&signbuf); 2909 if (signlen <= 0) 2910 goto err_sm2; 2911 2912 if (!EVP_DigestVerifyFinal(md_ctx, signbuf, signlen)) 2913 goto err_sm2; 2914 2915 BN_free(r); 2916 BN_free(s); 2917 ECDSA_SIG_free(ec_sign); 2918 } 2919 break; 2920 default: 2921 /* allow ops with invalid args to be pushed to 2922 * completion queue 2923 */ 2924 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2925 goto err_sm2; 2926 } 2927 2928 ret = 0; 2929 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2930 err_sm2: 2931 if (kctx) 2932 EVP_PKEY_CTX_free(kctx); 2933 2934 if (sctx) 2935 EVP_PKEY_CTX_free(sctx); 2936 2937 if (cctx) 2938 EVP_PKEY_CTX_free(cctx); 2939 2940 if (pkey) 2941 EVP_PKEY_free(pkey); 2942 2943 if (param_bld) 2944 OSSL_PARAM_BLD_free(param_bld); 2945 2946 return ret; 2947 } 2948 2949 #else 2950 static int 2951 process_openssl_rsa_op(struct rte_crypto_op *cop, 2952 struct openssl_asym_session *sess) 2953 { 2954 int ret = 0; 2955 struct rte_crypto_asym_op *op = cop->asym; 2956 RSA *rsa = sess->u.r.rsa; 2957 uint32_t pad = (op->rsa.padding.type); 2958 uint8_t *tmp; 2959 2960 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2961 2962 switch (pad) { 2963 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2964 pad = RSA_PKCS1_PADDING; 2965 break; 2966 case RTE_CRYPTO_RSA_PADDING_NONE: 2967 pad = RSA_NO_PADDING; 2968 break; 2969 default: 2970 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2971 OPENSSL_LOG(ERR, 2972 "rsa pad type not supported %d\n", pad); 2973 return 0; 2974 } 2975 2976 switch (op->rsa.op_type) { 2977 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2978 ret = RSA_public_encrypt(op->rsa.message.length, 2979 op->rsa.message.data, 2980 op->rsa.cipher.data, 2981 rsa, 2982 pad); 2983 2984 if (ret > 0) 2985 op->rsa.cipher.length = ret; 2986 OPENSSL_LOG(DEBUG, 2987 "length of encrypted text %d\n", ret); 2988 break; 2989 2990 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2991 ret = RSA_private_decrypt(op->rsa.cipher.length, 2992 op->rsa.cipher.data, 2993 op->rsa.message.data, 2994 rsa, 2995 pad); 2996 if (ret > 0) 2997 op->rsa.message.length = ret; 2998 break; 2999 3000 case RTE_CRYPTO_ASYM_OP_SIGN: 3001 ret = RSA_private_encrypt(op->rsa.message.length, 3002 op->rsa.message.data, 3003 op->rsa.sign.data, 3004 rsa, 3005 pad); 3006 if (ret > 0) 3007 op->rsa.sign.length = ret; 3008 break; 3009 3010 case RTE_CRYPTO_ASYM_OP_VERIFY: 3011 tmp = rte_malloc(NULL, op->rsa.sign.length, 0); 3012 if (tmp == NULL) { 3013 OPENSSL_LOG(ERR, "Memory allocation failed"); 3014 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 3015 break; 3016 } 3017 ret = RSA_public_decrypt(op->rsa.sign.length, 3018 op->rsa.sign.data, 3019 tmp, 3020 rsa, 3021 pad); 3022 3023 OPENSSL_LOG(DEBUG, 3024 "Length of public_decrypt %d " 3025 "length of message %zd\n", 3026 ret, op->rsa.message.length); 3027 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data, 3028 op->rsa.message.length))) { 3029 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 3030 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 3031 } 3032 rte_free(tmp); 3033 break; 3034 3035 default: 3036 /* allow ops with invalid args to be pushed to 3037 * completion queue 3038 */ 3039 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3040 break; 3041 } 3042 3043 if (ret < 0) 3044 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 3045 3046 return 0; 3047 } 3048 3049 static int 3050 process_openssl_sm2_op(struct rte_crypto_op *cop, 3051 struct openssl_asym_session *sess) 3052 { 3053 RTE_SET_USED(cop); 3054 RTE_SET_USED(sess); 3055 return -ENOTSUP; 3056 } 3057 #endif 3058 3059 static int 3060 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op, 3061 struct openssl_asym_session *sess) 3062 { 3063 int retval = 0; 3064 3065 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 3066 3067 switch (sess->xfrm_type) { 3068 case RTE_CRYPTO_ASYM_XFORM_RSA: 3069 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3070 retval = process_openssl_rsa_op_evp(op, sess); 3071 # else 3072 retval = process_openssl_rsa_op(op, sess); 3073 #endif 3074 break; 3075 case RTE_CRYPTO_ASYM_XFORM_MODEX: 3076 retval = process_openssl_modexp_op(op, sess); 3077 break; 3078 case RTE_CRYPTO_ASYM_XFORM_MODINV: 3079 retval = process_openssl_modinv_op(op, sess); 3080 break; 3081 case RTE_CRYPTO_ASYM_XFORM_DH: 3082 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3083 retval = process_openssl_dh_op_evp(op, sess); 3084 # else 3085 retval = process_openssl_dh_op(op, sess); 3086 #endif 3087 break; 3088 case RTE_CRYPTO_ASYM_XFORM_DSA: 3089 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3090 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 3091 retval = process_openssl_dsa_sign_op_evp(op, sess); 3092 else if (op->asym->dsa.op_type == 3093 RTE_CRYPTO_ASYM_OP_VERIFY) 3094 retval = 3095 process_openssl_dsa_verify_op_evp(op, sess); 3096 #else 3097 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 3098 retval = process_openssl_dsa_sign_op(op, sess); 3099 else if (op->asym->dsa.op_type == 3100 RTE_CRYPTO_ASYM_OP_VERIFY) 3101 retval = 3102 process_openssl_dsa_verify_op(op, sess); 3103 else 3104 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3105 #endif 3106 break; 3107 case RTE_CRYPTO_ASYM_XFORM_SM2: 3108 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3109 retval = process_openssl_sm2_op_evp(op, sess); 3110 #else 3111 retval = process_openssl_sm2_op(op, sess); 3112 #endif 3113 break; 3114 default: 3115 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3116 break; 3117 } 3118 if (!retval) { 3119 /* op processed so push to completion queue as processed */ 3120 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 3121 if (retval) 3122 /* return error if failed to put in completion queue */ 3123 retval = -1; 3124 } 3125 3126 return retval; 3127 } 3128 3129 static void 3130 copy_plaintext(struct rte_mbuf *m_src, struct rte_mbuf *m_dst, 3131 struct rte_crypto_op *op) 3132 { 3133 uint8_t *p_src, *p_dst; 3134 3135 p_src = rte_pktmbuf_mtod(m_src, uint8_t *); 3136 p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *); 3137 3138 /** 3139 * Copy the content between cipher offset and auth offset 3140 * for generating correct digest. 3141 */ 3142 if (op->sym->cipher.data.offset > op->sym->auth.data.offset) 3143 memcpy(p_dst + op->sym->auth.data.offset, 3144 p_src + op->sym->auth.data.offset, 3145 op->sym->cipher.data.offset - 3146 op->sym->auth.data.offset); 3147 } 3148 3149 /** Process crypto operation for mbuf */ 3150 static int 3151 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 3152 struct openssl_session *sess) 3153 { 3154 struct rte_mbuf *msrc, *mdst; 3155 int retval; 3156 3157 msrc = op->sym->m_src; 3158 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 3159 3160 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 3161 3162 switch (sess->chain_order) { 3163 case OPENSSL_CHAIN_ONLY_CIPHER: 3164 process_openssl_cipher_op(op, sess, msrc, mdst); 3165 break; 3166 case OPENSSL_CHAIN_ONLY_AUTH: 3167 process_openssl_auth_op(qp, op, sess, msrc, mdst); 3168 break; 3169 case OPENSSL_CHAIN_CIPHER_AUTH: 3170 process_openssl_cipher_op(op, sess, msrc, mdst); 3171 /* OOP */ 3172 if (msrc != mdst) 3173 copy_plaintext(msrc, mdst, op); 3174 process_openssl_auth_op(qp, op, sess, mdst, mdst); 3175 break; 3176 case OPENSSL_CHAIN_AUTH_CIPHER: 3177 process_openssl_auth_op(qp, op, sess, msrc, mdst); 3178 process_openssl_cipher_op(op, sess, msrc, mdst); 3179 break; 3180 case OPENSSL_CHAIN_COMBINED: 3181 process_openssl_combined_op(op, sess, msrc, mdst); 3182 break; 3183 case OPENSSL_CHAIN_CIPHER_BPI: 3184 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 3185 break; 3186 default: 3187 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3188 break; 3189 } 3190 3191 /* Free session if a session-less crypto op */ 3192 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 3193 openssl_reset_session(sess); 3194 memset(sess, 0, sizeof(struct openssl_session)); 3195 rte_mempool_put(qp->sess_mp, op->sym->session); 3196 op->sym->session = NULL; 3197 } 3198 3199 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 3200 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3201 3202 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 3203 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 3204 else 3205 retval = -1; 3206 3207 return retval; 3208 } 3209 3210 /* 3211 *------------------------------------------------------------------------------ 3212 * PMD Framework 3213 *------------------------------------------------------------------------------ 3214 */ 3215 3216 /** Enqueue burst */ 3217 static uint16_t 3218 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 3219 uint16_t nb_ops) 3220 { 3221 void *sess; 3222 struct openssl_qp *qp = queue_pair; 3223 int i, retval; 3224 3225 for (i = 0; i < nb_ops; i++) { 3226 sess = get_session(qp, ops[i]); 3227 if (unlikely(sess == NULL)) 3228 goto enqueue_err; 3229 3230 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) 3231 retval = process_op(qp, ops[i], 3232 (struct openssl_session *) sess); 3233 else 3234 retval = process_asym_op(qp, ops[i], 3235 (struct openssl_asym_session *) sess); 3236 if (unlikely(retval < 0)) 3237 goto enqueue_err; 3238 } 3239 3240 qp->stats.enqueued_count += i; 3241 return i; 3242 3243 enqueue_err: 3244 qp->stats.enqueue_err_count++; 3245 return i; 3246 } 3247 3248 /** Dequeue burst */ 3249 static uint16_t 3250 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 3251 uint16_t nb_ops) 3252 { 3253 struct openssl_qp *qp = queue_pair; 3254 3255 unsigned int nb_dequeued = 0; 3256 3257 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 3258 (void **)ops, nb_ops, NULL); 3259 qp->stats.dequeued_count += nb_dequeued; 3260 3261 return nb_dequeued; 3262 } 3263 3264 /** Create OPENSSL crypto device */ 3265 static int 3266 cryptodev_openssl_create(const char *name, 3267 struct rte_vdev_device *vdev, 3268 struct rte_cryptodev_pmd_init_params *init_params) 3269 { 3270 struct rte_cryptodev *dev; 3271 struct openssl_private *internals; 3272 3273 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 3274 if (dev == NULL) { 3275 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 3276 goto init_error; 3277 } 3278 3279 dev->driver_id = cryptodev_driver_id; 3280 dev->dev_ops = rte_openssl_pmd_ops; 3281 3282 /* register rx/tx burst functions for data path */ 3283 dev->dequeue_burst = openssl_pmd_dequeue_burst; 3284 dev->enqueue_burst = openssl_pmd_enqueue_burst; 3285 3286 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 3287 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 3288 RTE_CRYPTODEV_FF_CPU_AESNI | 3289 RTE_CRYPTODEV_FF_IN_PLACE_SGL | 3290 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 3291 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | 3292 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO | 3293 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP | 3294 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT | 3295 RTE_CRYPTODEV_FF_SYM_SESSIONLESS; 3296 3297 internals = dev->data->dev_private; 3298 3299 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 3300 3301 rte_cryptodev_pmd_probing_finish(dev); 3302 3303 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3304 /* Load legacy provider 3305 * Some algorithms are no longer available in earlier version of openssl, 3306 * unless the legacy provider explicitly loaded. e.g. DES 3307 */ 3308 ossl_legacy_provider_load(); 3309 # endif 3310 return 0; 3311 3312 init_error: 3313 OPENSSL_LOG(ERR, "driver %s: create failed", 3314 init_params->name); 3315 3316 cryptodev_openssl_remove(vdev); 3317 return -EFAULT; 3318 } 3319 3320 /** Initialise OPENSSL crypto device */ 3321 static int 3322 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 3323 { 3324 struct rte_cryptodev_pmd_init_params init_params = { 3325 "", 3326 sizeof(struct openssl_private), 3327 rte_socket_id(), 3328 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 3329 }; 3330 const char *name; 3331 const char *input_args; 3332 3333 name = rte_vdev_device_name(vdev); 3334 if (name == NULL) 3335 return -EINVAL; 3336 input_args = rte_vdev_device_args(vdev); 3337 3338 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 3339 3340 return cryptodev_openssl_create(name, vdev, &init_params); 3341 } 3342 3343 /** Uninitialise OPENSSL crypto device */ 3344 static int 3345 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 3346 { 3347 struct rte_cryptodev *cryptodev; 3348 const char *name; 3349 3350 name = rte_vdev_device_name(vdev); 3351 if (name == NULL) 3352 return -EINVAL; 3353 3354 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 3355 if (cryptodev == NULL) 3356 return -ENODEV; 3357 3358 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3359 ossl_legacy_provider_unload(); 3360 # endif 3361 return rte_cryptodev_pmd_destroy(cryptodev); 3362 } 3363 3364 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 3365 .probe = cryptodev_openssl_probe, 3366 .remove = cryptodev_openssl_remove 3367 }; 3368 3369 static struct cryptodev_driver openssl_crypto_drv; 3370 3371 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 3372 cryptodev_openssl_pmd_drv); 3373 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 3374 "max_nb_queue_pairs=<int> " 3375 "socket_id=<int>"); 3376 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 3377 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 3378 RTE_LOG_REGISTER_DEFAULT(openssl_logtype_driver, INFO); 3379