1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_common.h> 6 #include <rte_hexdump.h> 7 #include <rte_cryptodev.h> 8 #include <cryptodev_pmd.h> 9 #include <bus_vdev_driver.h> 10 #include <rte_malloc.h> 11 #include <rte_cpuflags.h> 12 13 #include <openssl/cmac.h> 14 #include <openssl/hmac.h> 15 #include <openssl/evp.h> 16 17 #include "openssl_pmd_private.h" 18 #include "compat.h" 19 20 #define DES_BLOCK_SIZE 8 21 22 static uint8_t cryptodev_driver_id; 23 24 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 25 static HMAC_CTX *HMAC_CTX_new(void) 26 { 27 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 28 29 if (ctx != NULL) 30 HMAC_CTX_init(ctx); 31 return ctx; 32 } 33 34 static void HMAC_CTX_free(HMAC_CTX *ctx) 35 { 36 if (ctx != NULL) { 37 HMAC_CTX_cleanup(ctx); 38 OPENSSL_free(ctx); 39 } 40 } 41 #endif 42 43 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 44 45 #include <openssl/provider.h> 46 #include <openssl/core_names.h> 47 #include <openssl/param_build.h> 48 49 #define MAX_OSSL_ALGO_NAME_SIZE 16 50 51 OSSL_PROVIDER *legacy; 52 OSSL_PROVIDER *deflt; 53 54 static void ossl_legacy_provider_load(void) 55 { 56 /* Load Multiple providers into the default (NULL) library context */ 57 legacy = OSSL_PROVIDER_load(NULL, "legacy"); 58 if (legacy == NULL) { 59 OPENSSL_LOG(ERR, "Failed to load Legacy provider\n"); 60 return; 61 } 62 63 deflt = OSSL_PROVIDER_load(NULL, "default"); 64 if (deflt == NULL) { 65 OPENSSL_LOG(ERR, "Failed to load Default provider\n"); 66 OSSL_PROVIDER_unload(legacy); 67 return; 68 } 69 } 70 71 static void ossl_legacy_provider_unload(void) 72 { 73 OSSL_PROVIDER_unload(legacy); 74 OSSL_PROVIDER_unload(deflt); 75 } 76 77 static __rte_always_inline const char * 78 digest_name_get(enum rte_crypto_auth_algorithm algo) 79 { 80 switch (algo) { 81 case RTE_CRYPTO_AUTH_MD5_HMAC: 82 return OSSL_DIGEST_NAME_MD5; 83 case RTE_CRYPTO_AUTH_SHA1_HMAC: 84 return OSSL_DIGEST_NAME_SHA1; 85 case RTE_CRYPTO_AUTH_SHA224_HMAC: 86 return OSSL_DIGEST_NAME_SHA2_224; 87 case RTE_CRYPTO_AUTH_SHA256_HMAC: 88 return OSSL_DIGEST_NAME_SHA2_256; 89 case RTE_CRYPTO_AUTH_SHA384_HMAC: 90 return OSSL_DIGEST_NAME_SHA2_384; 91 case RTE_CRYPTO_AUTH_SHA512_HMAC: 92 return OSSL_DIGEST_NAME_SHA2_512; 93 default: 94 return NULL; 95 } 96 } 97 #endif 98 99 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 100 101 /*----------------------------------------------------------------------------*/ 102 103 /** 104 * Increment counter by 1 105 * Counter is 64 bit array, big-endian 106 */ 107 static void 108 ctr_inc(uint8_t *ctr) 109 { 110 uint64_t *ctr64 = (uint64_t *)ctr; 111 112 *ctr64 = __builtin_bswap64(*ctr64); 113 (*ctr64)++; 114 *ctr64 = __builtin_bswap64(*ctr64); 115 } 116 117 /* 118 *------------------------------------------------------------------------------ 119 * Session Prepare 120 *------------------------------------------------------------------------------ 121 */ 122 123 /** Get xform chain order */ 124 static enum openssl_chain_order 125 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 126 { 127 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 128 129 if (xform != NULL) { 130 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 131 if (xform->next == NULL) 132 res = OPENSSL_CHAIN_ONLY_AUTH; 133 else if (xform->next->type == 134 RTE_CRYPTO_SYM_XFORM_CIPHER) 135 res = OPENSSL_CHAIN_AUTH_CIPHER; 136 } 137 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 138 if (xform->next == NULL) 139 res = OPENSSL_CHAIN_ONLY_CIPHER; 140 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 141 res = OPENSSL_CHAIN_CIPHER_AUTH; 142 } 143 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 144 res = OPENSSL_CHAIN_COMBINED; 145 } 146 147 return res; 148 } 149 150 /** Get session cipher key from input cipher key */ 151 static void 152 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key) 153 { 154 memcpy(session_key, input_key, keylen); 155 } 156 157 /** Get key ede 24 bytes standard from input key */ 158 static int 159 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede) 160 { 161 int res = 0; 162 163 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 164 switch (keylen) { 165 case 24: 166 memcpy(key_ede, key, 24); 167 break; 168 case 16: 169 /* K3 = K1 */ 170 memcpy(key_ede, key, 16); 171 memcpy(key_ede + 16, key, 8); 172 break; 173 case 8: 174 /* K1 = K2 = K3 (DES compatibility) */ 175 memcpy(key_ede, key, 8); 176 memcpy(key_ede + 8, key, 8); 177 memcpy(key_ede + 16, key, 8); 178 break; 179 default: 180 OPENSSL_LOG(ERR, "Unsupported key size"); 181 res = -EINVAL; 182 } 183 184 return res; 185 } 186 187 /** Get adequate openssl function for input cipher algorithm */ 188 static uint8_t 189 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 190 const EVP_CIPHER **algo) 191 { 192 int res = 0; 193 194 if (algo != NULL) { 195 switch (sess_algo) { 196 case RTE_CRYPTO_CIPHER_3DES_CBC: 197 switch (keylen) { 198 case 8: 199 *algo = EVP_des_cbc(); 200 break; 201 case 16: 202 *algo = EVP_des_ede_cbc(); 203 break; 204 case 24: 205 *algo = EVP_des_ede3_cbc(); 206 break; 207 default: 208 res = -EINVAL; 209 } 210 break; 211 case RTE_CRYPTO_CIPHER_3DES_CTR: 212 break; 213 case RTE_CRYPTO_CIPHER_AES_CBC: 214 switch (keylen) { 215 case 16: 216 *algo = EVP_aes_128_cbc(); 217 break; 218 case 24: 219 *algo = EVP_aes_192_cbc(); 220 break; 221 case 32: 222 *algo = EVP_aes_256_cbc(); 223 break; 224 default: 225 res = -EINVAL; 226 } 227 break; 228 case RTE_CRYPTO_CIPHER_AES_CTR: 229 switch (keylen) { 230 case 16: 231 *algo = EVP_aes_128_ctr(); 232 break; 233 case 24: 234 *algo = EVP_aes_192_ctr(); 235 break; 236 case 32: 237 *algo = EVP_aes_256_ctr(); 238 break; 239 default: 240 res = -EINVAL; 241 } 242 break; 243 default: 244 res = -EINVAL; 245 break; 246 } 247 } else { 248 res = -EINVAL; 249 } 250 251 return res; 252 } 253 254 /** Get adequate openssl function for input auth algorithm */ 255 static uint8_t 256 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 257 const EVP_MD **algo) 258 { 259 int res = 0; 260 261 if (algo != NULL) { 262 switch (sessalgo) { 263 case RTE_CRYPTO_AUTH_MD5: 264 case RTE_CRYPTO_AUTH_MD5_HMAC: 265 *algo = EVP_md5(); 266 break; 267 case RTE_CRYPTO_AUTH_SHA1: 268 case RTE_CRYPTO_AUTH_SHA1_HMAC: 269 *algo = EVP_sha1(); 270 break; 271 case RTE_CRYPTO_AUTH_SHA224: 272 case RTE_CRYPTO_AUTH_SHA224_HMAC: 273 *algo = EVP_sha224(); 274 break; 275 case RTE_CRYPTO_AUTH_SHA256: 276 case RTE_CRYPTO_AUTH_SHA256_HMAC: 277 *algo = EVP_sha256(); 278 break; 279 case RTE_CRYPTO_AUTH_SHA384: 280 case RTE_CRYPTO_AUTH_SHA384_HMAC: 281 *algo = EVP_sha384(); 282 break; 283 case RTE_CRYPTO_AUTH_SHA512: 284 case RTE_CRYPTO_AUTH_SHA512_HMAC: 285 *algo = EVP_sha512(); 286 break; 287 default: 288 res = -EINVAL; 289 break; 290 } 291 } else { 292 res = -EINVAL; 293 } 294 295 return res; 296 } 297 298 /** Get adequate openssl function for input cipher algorithm */ 299 static uint8_t 300 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 301 const EVP_CIPHER **algo) 302 { 303 int res = 0; 304 305 if (algo != NULL) { 306 switch (sess_algo) { 307 case RTE_CRYPTO_AEAD_AES_GCM: 308 switch (keylen) { 309 case 16: 310 *algo = EVP_aes_128_gcm(); 311 break; 312 case 24: 313 *algo = EVP_aes_192_gcm(); 314 break; 315 case 32: 316 *algo = EVP_aes_256_gcm(); 317 break; 318 default: 319 res = -EINVAL; 320 } 321 break; 322 case RTE_CRYPTO_AEAD_AES_CCM: 323 switch (keylen) { 324 case 16: 325 *algo = EVP_aes_128_ccm(); 326 break; 327 case 24: 328 *algo = EVP_aes_192_ccm(); 329 break; 330 case 32: 331 *algo = EVP_aes_256_ccm(); 332 break; 333 default: 334 res = -EINVAL; 335 } 336 break; 337 default: 338 res = -EINVAL; 339 break; 340 } 341 } else { 342 res = -EINVAL; 343 } 344 345 return res; 346 } 347 348 /* Set session AEAD encryption parameters */ 349 static int 350 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 351 enum rte_crypto_aead_algorithm algo, 352 uint8_t tag_len, const uint8_t *key) 353 { 354 int iv_type = 0; 355 unsigned int do_ccm; 356 357 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 358 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 359 360 /* Select AEAD algo */ 361 switch (algo) { 362 case RTE_CRYPTO_AEAD_AES_GCM: 363 iv_type = EVP_CTRL_GCM_SET_IVLEN; 364 if (tag_len != 16) 365 return -EINVAL; 366 do_ccm = 0; 367 break; 368 case RTE_CRYPTO_AEAD_AES_CCM: 369 iv_type = EVP_CTRL_CCM_SET_IVLEN; 370 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 371 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 372 return -EINVAL; 373 do_ccm = 1; 374 break; 375 default: 376 return -ENOTSUP; 377 } 378 379 sess->cipher.mode = OPENSSL_CIPHER_LIB; 380 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 381 382 if (get_aead_algo(algo, sess->cipher.key.length, 383 &sess->cipher.evp_algo) != 0) 384 return -EINVAL; 385 386 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 387 388 sess->chain_order = OPENSSL_CHAIN_COMBINED; 389 390 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 391 NULL, NULL, NULL) <= 0) 392 return -EINVAL; 393 394 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, 395 NULL) <= 0) 396 return -EINVAL; 397 398 if (do_ccm) 399 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 400 tag_len, NULL); 401 402 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 403 return -EINVAL; 404 405 return 0; 406 } 407 408 /* Set session AEAD decryption parameters */ 409 static int 410 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 411 enum rte_crypto_aead_algorithm algo, 412 uint8_t tag_len, const uint8_t *key) 413 { 414 int iv_type = 0; 415 unsigned int do_ccm = 0; 416 417 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 418 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 419 420 /* Select AEAD algo */ 421 switch (algo) { 422 case RTE_CRYPTO_AEAD_AES_GCM: 423 iv_type = EVP_CTRL_GCM_SET_IVLEN; 424 if (tag_len != 16) 425 return -EINVAL; 426 break; 427 case RTE_CRYPTO_AEAD_AES_CCM: 428 iv_type = EVP_CTRL_CCM_SET_IVLEN; 429 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 430 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 431 return -EINVAL; 432 do_ccm = 1; 433 break; 434 default: 435 return -ENOTSUP; 436 } 437 438 sess->cipher.mode = OPENSSL_CIPHER_LIB; 439 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 440 441 if (get_aead_algo(algo, sess->cipher.key.length, 442 &sess->cipher.evp_algo) != 0) 443 return -EINVAL; 444 445 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 446 447 sess->chain_order = OPENSSL_CHAIN_COMBINED; 448 449 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 450 NULL, NULL, NULL) <= 0) 451 return -EINVAL; 452 453 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, 454 sess->iv.length, NULL) <= 0) 455 return -EINVAL; 456 457 if (do_ccm) 458 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 459 tag_len, NULL); 460 461 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 462 return -EINVAL; 463 464 return 0; 465 } 466 467 /** Set session cipher parameters */ 468 static int 469 openssl_set_session_cipher_parameters(struct openssl_session *sess, 470 const struct rte_crypto_sym_xform *xform) 471 { 472 /* Select cipher direction */ 473 sess->cipher.direction = xform->cipher.op; 474 /* Select cipher key */ 475 sess->cipher.key.length = xform->cipher.key.length; 476 477 /* Set IV parameters */ 478 sess->iv.offset = xform->cipher.iv.offset; 479 sess->iv.length = xform->cipher.iv.length; 480 481 /* Select cipher algo */ 482 switch (xform->cipher.algo) { 483 case RTE_CRYPTO_CIPHER_3DES_CBC: 484 case RTE_CRYPTO_CIPHER_AES_CBC: 485 case RTE_CRYPTO_CIPHER_AES_CTR: 486 sess->cipher.mode = OPENSSL_CIPHER_LIB; 487 sess->cipher.algo = xform->cipher.algo; 488 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 489 490 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 491 &sess->cipher.evp_algo) != 0) 492 return -EINVAL; 493 494 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 495 sess->cipher.key.data); 496 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 497 if (EVP_EncryptInit_ex(sess->cipher.ctx, 498 sess->cipher.evp_algo, 499 NULL, xform->cipher.key.data, 500 NULL) != 1) { 501 return -EINVAL; 502 } 503 } else if (sess->cipher.direction == 504 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 505 if (EVP_DecryptInit_ex(sess->cipher.ctx, 506 sess->cipher.evp_algo, 507 NULL, xform->cipher.key.data, 508 NULL) != 1) { 509 return -EINVAL; 510 } 511 } 512 513 break; 514 515 case RTE_CRYPTO_CIPHER_3DES_CTR: 516 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 517 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 518 519 if (get_cipher_key_ede(xform->cipher.key.data, 520 sess->cipher.key.length, 521 sess->cipher.key.data) != 0) 522 return -EINVAL; 523 break; 524 525 case RTE_CRYPTO_CIPHER_DES_CBC: 526 sess->cipher.algo = xform->cipher.algo; 527 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 528 sess->cipher.evp_algo = EVP_des_cbc(); 529 530 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 531 sess->cipher.key.data); 532 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 533 if (EVP_EncryptInit_ex(sess->cipher.ctx, 534 sess->cipher.evp_algo, 535 NULL, xform->cipher.key.data, 536 NULL) != 1) { 537 return -EINVAL; 538 } 539 } else if (sess->cipher.direction == 540 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 541 if (EVP_DecryptInit_ex(sess->cipher.ctx, 542 sess->cipher.evp_algo, 543 NULL, xform->cipher.key.data, 544 NULL) != 1) { 545 return -EINVAL; 546 } 547 } 548 549 break; 550 551 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 552 sess->cipher.algo = xform->cipher.algo; 553 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 554 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 555 sess->cipher.evp_algo = EVP_des_cbc(); 556 557 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 558 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 559 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 560 NULL, xform->cipher.key.data, 0) != 1) 561 return -EINVAL; 562 563 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 564 sess->cipher.key.data); 565 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 566 if (EVP_EncryptInit_ex(sess->cipher.ctx, 567 sess->cipher.evp_algo, 568 NULL, xform->cipher.key.data, 569 NULL) != 1) { 570 return -EINVAL; 571 } 572 } else if (sess->cipher.direction == 573 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 574 if (EVP_DecryptInit_ex(sess->cipher.ctx, 575 sess->cipher.evp_algo, 576 NULL, xform->cipher.key.data, 577 NULL) != 1) { 578 return -EINVAL; 579 } 580 } 581 582 break; 583 default: 584 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 585 return -ENOTSUP; 586 } 587 588 return 0; 589 } 590 591 /* Set session auth parameters */ 592 static int 593 openssl_set_session_auth_parameters(struct openssl_session *sess, 594 const struct rte_crypto_sym_xform *xform) 595 { 596 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 597 char algo_name[MAX_OSSL_ALGO_NAME_SIZE]; 598 OSSL_PARAM params[2]; 599 const char *algo; 600 EVP_MAC *mac; 601 # endif 602 /* Select auth generate/verify */ 603 sess->auth.operation = xform->auth.op; 604 sess->auth.algo = xform->auth.algo; 605 606 sess->auth.digest_length = xform->auth.digest_length; 607 608 /* Select auth algo */ 609 switch (xform->auth.algo) { 610 case RTE_CRYPTO_AUTH_AES_GMAC: 611 /* 612 * OpenSSL requires GMAC to be a GCM operation 613 * with no cipher data length 614 */ 615 sess->cipher.key.length = xform->auth.key.length; 616 617 /* Set IV parameters */ 618 sess->iv.offset = xform->auth.iv.offset; 619 sess->iv.length = xform->auth.iv.length; 620 621 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 622 return openssl_set_sess_aead_enc_param(sess, 623 RTE_CRYPTO_AEAD_AES_GCM, 624 xform->auth.digest_length, 625 xform->auth.key.data); 626 else 627 return openssl_set_sess_aead_dec_param(sess, 628 RTE_CRYPTO_AEAD_AES_GCM, 629 xform->auth.digest_length, 630 xform->auth.key.data); 631 break; 632 633 case RTE_CRYPTO_AUTH_MD5: 634 case RTE_CRYPTO_AUTH_SHA1: 635 case RTE_CRYPTO_AUTH_SHA224: 636 case RTE_CRYPTO_AUTH_SHA256: 637 case RTE_CRYPTO_AUTH_SHA384: 638 case RTE_CRYPTO_AUTH_SHA512: 639 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 640 if (get_auth_algo(xform->auth.algo, 641 &sess->auth.auth.evp_algo) != 0) 642 return -EINVAL; 643 sess->auth.auth.ctx = EVP_MD_CTX_create(); 644 break; 645 646 case RTE_CRYPTO_AUTH_AES_CMAC: 647 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 648 if (xform->auth.key.length == 16) 649 algo = SN_aes_128_cbc; 650 else if (xform->auth.key.length == 24) 651 algo = SN_aes_192_cbc; 652 else if (xform->auth.key.length == 32) 653 algo = SN_aes_256_cbc; 654 else 655 return -EINVAL; 656 657 rte_memcpy(algo_name, algo, strlen(algo) + 1); 658 params[0] = OSSL_PARAM_construct_utf8_string( 659 OSSL_MAC_PARAM_CIPHER, algo_name, 0); 660 params[1] = OSSL_PARAM_construct_end(); 661 662 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 663 mac = EVP_MAC_fetch(NULL, OSSL_MAC_NAME_CMAC, NULL); 664 sess->auth.cmac.ctx = EVP_MAC_CTX_new(mac); 665 EVP_MAC_free(mac); 666 667 if (EVP_MAC_init(sess->auth.cmac.ctx, 668 xform->auth.key.data, 669 xform->auth.key.length, 670 params) != 1) 671 return -EINVAL; 672 # else 673 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 674 sess->auth.cmac.ctx = CMAC_CTX_new(); 675 if (get_cipher_algo(RTE_CRYPTO_CIPHER_AES_CBC, 676 xform->auth.key.length, 677 &sess->auth.cmac.evp_algo) != 0) 678 return -EINVAL; 679 if (CMAC_Init(sess->auth.cmac.ctx, 680 xform->auth.key.data, 681 xform->auth.key.length, 682 sess->auth.cmac.evp_algo, NULL) != 1) 683 return -EINVAL; 684 # endif 685 break; 686 687 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 688 case RTE_CRYPTO_AUTH_MD5_HMAC: 689 case RTE_CRYPTO_AUTH_SHA1_HMAC: 690 case RTE_CRYPTO_AUTH_SHA224_HMAC: 691 case RTE_CRYPTO_AUTH_SHA256_HMAC: 692 case RTE_CRYPTO_AUTH_SHA384_HMAC: 693 case RTE_CRYPTO_AUTH_SHA512_HMAC: 694 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 695 696 algo = digest_name_get(xform->auth.algo); 697 if (!algo) 698 return -EINVAL; 699 rte_memcpy(algo_name, algo, (sizeof(algo)+1)); 700 701 mac = EVP_MAC_fetch(NULL, "HMAC", NULL); 702 sess->auth.hmac.ctx = EVP_MAC_CTX_new(mac); 703 EVP_MAC_free(mac); 704 if (get_auth_algo(xform->auth.algo, 705 &sess->auth.hmac.evp_algo) != 0) 706 return -EINVAL; 707 708 params[0] = OSSL_PARAM_construct_utf8_string("digest", 709 algo_name, 0); 710 params[1] = OSSL_PARAM_construct_end(); 711 if (EVP_MAC_init(sess->auth.hmac.ctx, 712 xform->auth.key.data, 713 xform->auth.key.length, 714 params) != 1) 715 return -EINVAL; 716 break; 717 # else 718 case RTE_CRYPTO_AUTH_MD5_HMAC: 719 case RTE_CRYPTO_AUTH_SHA1_HMAC: 720 case RTE_CRYPTO_AUTH_SHA224_HMAC: 721 case RTE_CRYPTO_AUTH_SHA256_HMAC: 722 case RTE_CRYPTO_AUTH_SHA384_HMAC: 723 case RTE_CRYPTO_AUTH_SHA512_HMAC: 724 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 725 sess->auth.hmac.ctx = HMAC_CTX_new(); 726 if (get_auth_algo(xform->auth.algo, 727 &sess->auth.hmac.evp_algo) != 0) 728 return -EINVAL; 729 730 if (HMAC_Init_ex(sess->auth.hmac.ctx, 731 xform->auth.key.data, 732 xform->auth.key.length, 733 sess->auth.hmac.evp_algo, NULL) != 1) 734 return -EINVAL; 735 break; 736 # endif 737 default: 738 return -ENOTSUP; 739 } 740 741 return 0; 742 } 743 744 /* Set session AEAD parameters */ 745 static int 746 openssl_set_session_aead_parameters(struct openssl_session *sess, 747 const struct rte_crypto_sym_xform *xform) 748 { 749 /* Select cipher key */ 750 sess->cipher.key.length = xform->aead.key.length; 751 752 /* Set IV parameters */ 753 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 754 /* 755 * For AES-CCM, the actual IV is placed 756 * one byte after the start of the IV field, 757 * according to the API. 758 */ 759 sess->iv.offset = xform->aead.iv.offset + 1; 760 else 761 sess->iv.offset = xform->aead.iv.offset; 762 763 sess->iv.length = xform->aead.iv.length; 764 765 sess->auth.aad_length = xform->aead.aad_length; 766 sess->auth.digest_length = xform->aead.digest_length; 767 768 sess->aead_algo = xform->aead.algo; 769 /* Select cipher direction */ 770 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 771 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 772 xform->aead.digest_length, xform->aead.key.data); 773 else 774 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 775 xform->aead.digest_length, xform->aead.key.data); 776 } 777 778 /** Parse crypto xform chain and set private session parameters */ 779 int 780 openssl_set_session_parameters(struct openssl_session *sess, 781 const struct rte_crypto_sym_xform *xform) 782 { 783 const struct rte_crypto_sym_xform *cipher_xform = NULL; 784 const struct rte_crypto_sym_xform *auth_xform = NULL; 785 const struct rte_crypto_sym_xform *aead_xform = NULL; 786 int ret; 787 788 sess->chain_order = openssl_get_chain_order(xform); 789 switch (sess->chain_order) { 790 case OPENSSL_CHAIN_ONLY_CIPHER: 791 cipher_xform = xform; 792 break; 793 case OPENSSL_CHAIN_ONLY_AUTH: 794 auth_xform = xform; 795 break; 796 case OPENSSL_CHAIN_CIPHER_AUTH: 797 cipher_xform = xform; 798 auth_xform = xform->next; 799 break; 800 case OPENSSL_CHAIN_AUTH_CIPHER: 801 auth_xform = xform; 802 cipher_xform = xform->next; 803 break; 804 case OPENSSL_CHAIN_COMBINED: 805 aead_xform = xform; 806 break; 807 default: 808 return -EINVAL; 809 } 810 811 /* Default IV length = 0 */ 812 sess->iv.length = 0; 813 814 /* cipher_xform must be check before auth_xform */ 815 if (cipher_xform) { 816 ret = openssl_set_session_cipher_parameters( 817 sess, cipher_xform); 818 if (ret != 0) { 819 OPENSSL_LOG(ERR, 820 "Invalid/unsupported cipher parameters"); 821 return ret; 822 } 823 } 824 825 if (auth_xform) { 826 ret = openssl_set_session_auth_parameters(sess, auth_xform); 827 if (ret != 0) { 828 OPENSSL_LOG(ERR, 829 "Invalid/unsupported auth parameters"); 830 return ret; 831 } 832 } 833 834 if (aead_xform) { 835 ret = openssl_set_session_aead_parameters(sess, aead_xform); 836 if (ret != 0) { 837 OPENSSL_LOG(ERR, 838 "Invalid/unsupported AEAD parameters"); 839 return ret; 840 } 841 } 842 843 return 0; 844 } 845 846 /** Reset private session parameters */ 847 void 848 openssl_reset_session(struct openssl_session *sess) 849 { 850 EVP_CIPHER_CTX_free(sess->cipher.ctx); 851 852 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 853 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 854 855 switch (sess->auth.mode) { 856 case OPENSSL_AUTH_AS_AUTH: 857 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 858 break; 859 case OPENSSL_AUTH_AS_HMAC: 860 EVP_PKEY_free(sess->auth.hmac.pkey); 861 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 862 EVP_MAC_CTX_free(sess->auth.hmac.ctx); 863 # else 864 HMAC_CTX_free(sess->auth.hmac.ctx); 865 # endif 866 break; 867 case OPENSSL_AUTH_AS_CMAC: 868 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 869 EVP_MAC_CTX_free(sess->auth.cmac.ctx); 870 # else 871 CMAC_CTX_free(sess->auth.cmac.ctx); 872 # endif 873 break; 874 default: 875 break; 876 } 877 } 878 879 /** Provide session for operation */ 880 static void * 881 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 882 { 883 struct openssl_session *sess = NULL; 884 struct openssl_asym_session *asym_sess = NULL; 885 886 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 887 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 888 /* get existing session */ 889 if (likely(op->sym->session != NULL)) 890 sess = (struct openssl_session *) 891 get_sym_session_private_data( 892 op->sym->session, 893 cryptodev_driver_id); 894 } else { 895 if (likely(op->asym->session != NULL)) 896 asym_sess = (struct openssl_asym_session *) 897 op->asym->session->sess_private_data; 898 if (asym_sess == NULL) 899 op->status = 900 RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 901 return asym_sess; 902 } 903 } else { 904 /* sessionless asymmetric not supported */ 905 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) 906 return NULL; 907 908 /* provide internal session */ 909 void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp); 910 void *_sess_private_data = NULL; 911 912 if (_sess == NULL) 913 return NULL; 914 915 if (rte_mempool_get(qp->sess_mp_priv, 916 (void **)&_sess_private_data)) 917 return NULL; 918 919 sess = (struct openssl_session *)_sess_private_data; 920 921 if (unlikely(openssl_set_session_parameters(sess, 922 op->sym->xform) != 0)) { 923 rte_mempool_put(qp->sess_mp, _sess); 924 rte_mempool_put(qp->sess_mp_priv, _sess_private_data); 925 sess = NULL; 926 } 927 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 928 set_sym_session_private_data(op->sym->session, 929 cryptodev_driver_id, _sess_private_data); 930 } 931 932 if (sess == NULL) 933 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 934 935 return sess; 936 } 937 938 /* 939 *------------------------------------------------------------------------------ 940 * Process Operations 941 *------------------------------------------------------------------------------ 942 */ 943 static inline int 944 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 945 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 946 { 947 struct rte_mbuf *m; 948 int dstlen; 949 int l, n = srclen; 950 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 951 952 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 953 m = m->next) 954 offset -= rte_pktmbuf_data_len(m); 955 956 if (m == 0) 957 return -1; 958 959 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 960 if (inplace) 961 *dst = src; 962 963 l = rte_pktmbuf_data_len(m) - offset; 964 if (srclen <= l) { 965 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 966 return -1; 967 *dst += l; 968 return 0; 969 } 970 971 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 972 return -1; 973 974 *dst += dstlen; 975 n -= l; 976 977 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 978 uint8_t diff = l - dstlen, rem; 979 980 src = rte_pktmbuf_mtod(m, uint8_t *); 981 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 982 if (diff && inplace) { 983 rem = RTE_MIN(l, 984 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 985 if (EVP_EncryptUpdate(ctx, temp, 986 &dstlen, src, rem) <= 0) 987 return -1; 988 n -= rem; 989 rte_memcpy(*dst, temp, diff); 990 rte_memcpy(src, temp + diff, rem); 991 src += rem; 992 l -= rem; 993 } 994 if (inplace) 995 *dst = src; 996 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 997 return -1; 998 *dst += dstlen; 999 n -= l; 1000 } 1001 1002 return 0; 1003 } 1004 1005 static inline int 1006 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 1007 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 1008 { 1009 struct rte_mbuf *m; 1010 int dstlen; 1011 int l, n = srclen; 1012 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 1013 1014 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1015 m = m->next) 1016 offset -= rte_pktmbuf_data_len(m); 1017 1018 if (m == 0) 1019 return -1; 1020 1021 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1022 if (inplace) 1023 *dst = src; 1024 1025 l = rte_pktmbuf_data_len(m) - offset; 1026 if (srclen <= l) { 1027 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 1028 return -1; 1029 *dst += l; 1030 return 0; 1031 } 1032 1033 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1034 return -1; 1035 1036 *dst += dstlen; 1037 n -= l; 1038 1039 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1040 uint8_t diff = l - dstlen, rem; 1041 1042 src = rte_pktmbuf_mtod(m, uint8_t *); 1043 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 1044 if (diff && inplace) { 1045 rem = RTE_MIN(l, 1046 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 1047 if (EVP_DecryptUpdate(ctx, temp, 1048 &dstlen, src, rem) <= 0) 1049 return -1; 1050 n -= rem; 1051 rte_memcpy(*dst, temp, diff); 1052 rte_memcpy(src, temp + diff, rem); 1053 src += rem; 1054 l -= rem; 1055 } 1056 if (inplace) 1057 *dst = src; 1058 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1059 return -1; 1060 *dst += dstlen; 1061 n -= l; 1062 } 1063 1064 return 0; 1065 } 1066 1067 /** Process standard openssl cipher encryption */ 1068 static int 1069 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1070 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1071 uint8_t inplace) 1072 { 1073 int totlen; 1074 1075 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1076 goto process_cipher_encrypt_err; 1077 1078 EVP_CIPHER_CTX_set_padding(ctx, 0); 1079 1080 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1081 srclen, ctx, inplace)) 1082 goto process_cipher_encrypt_err; 1083 1084 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 1085 goto process_cipher_encrypt_err; 1086 1087 return 0; 1088 1089 process_cipher_encrypt_err: 1090 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 1091 return -EINVAL; 1092 } 1093 1094 /** Process standard openssl cipher encryption */ 1095 static int 1096 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 1097 uint8_t *iv, int srclen, 1098 EVP_CIPHER_CTX *ctx) 1099 { 1100 uint8_t i; 1101 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 1102 int encrypted_ivlen; 1103 1104 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 1105 iv, DES_BLOCK_SIZE) <= 0) 1106 goto process_cipher_encrypt_err; 1107 1108 for (i = 0; i < srclen; i++) 1109 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 1110 1111 return 0; 1112 1113 process_cipher_encrypt_err: 1114 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 1115 return -EINVAL; 1116 } 1117 /** Process standard openssl cipher decryption */ 1118 static int 1119 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1120 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1121 uint8_t inplace) 1122 { 1123 int totlen; 1124 1125 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1126 goto process_cipher_decrypt_err; 1127 1128 EVP_CIPHER_CTX_set_padding(ctx, 0); 1129 1130 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1131 srclen, ctx, inplace)) 1132 goto process_cipher_decrypt_err; 1133 1134 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 1135 goto process_cipher_decrypt_err; 1136 return 0; 1137 1138 process_cipher_decrypt_err: 1139 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 1140 return -EINVAL; 1141 } 1142 1143 /** Process cipher des 3 ctr encryption, decryption algorithm */ 1144 static int 1145 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 1146 int offset, uint8_t *iv, uint8_t *key, int srclen, 1147 EVP_CIPHER_CTX *ctx) 1148 { 1149 uint8_t ebuf[8], ctr[8]; 1150 int unused, n; 1151 struct rte_mbuf *m; 1152 uint8_t *src; 1153 int l; 1154 1155 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1156 m = m->next) 1157 offset -= rte_pktmbuf_data_len(m); 1158 1159 if (m == 0) 1160 goto process_cipher_des3ctr_err; 1161 1162 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1163 l = rte_pktmbuf_data_len(m) - offset; 1164 1165 /* We use 3DES encryption also for decryption. 1166 * IV is not important for 3DES ecb 1167 */ 1168 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 1169 goto process_cipher_des3ctr_err; 1170 1171 memcpy(ctr, iv, 8); 1172 1173 for (n = 0; n < srclen; n++) { 1174 if (n % 8 == 0) { 1175 if (EVP_EncryptUpdate(ctx, 1176 (unsigned char *)&ebuf, &unused, 1177 (const unsigned char *)&ctr, 8) <= 0) 1178 goto process_cipher_des3ctr_err; 1179 ctr_inc(ctr); 1180 } 1181 dst[n] = *(src++) ^ ebuf[n % 8]; 1182 1183 l--; 1184 if (!l) { 1185 m = m->next; 1186 if (m) { 1187 src = rte_pktmbuf_mtod(m, uint8_t *); 1188 l = rte_pktmbuf_data_len(m); 1189 } 1190 } 1191 } 1192 1193 return 0; 1194 1195 process_cipher_des3ctr_err: 1196 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 1197 return -EINVAL; 1198 } 1199 1200 /** Process AES-GCM encrypt algorithm */ 1201 static int 1202 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1203 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1204 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1205 { 1206 int len = 0, unused = 0; 1207 uint8_t empty[] = {}; 1208 1209 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1210 goto process_auth_encryption_gcm_err; 1211 1212 if (aadlen > 0) 1213 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1214 goto process_auth_encryption_gcm_err; 1215 1216 if (srclen > 0) 1217 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1218 srclen, ctx, 0)) 1219 goto process_auth_encryption_gcm_err; 1220 1221 /* Workaround open ssl bug in version less then 1.0.1f */ 1222 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1223 goto process_auth_encryption_gcm_err; 1224 1225 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1226 goto process_auth_encryption_gcm_err; 1227 1228 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1229 goto process_auth_encryption_gcm_err; 1230 1231 return 0; 1232 1233 process_auth_encryption_gcm_err: 1234 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1235 return -EINVAL; 1236 } 1237 1238 /** Process AES-CCM encrypt algorithm */ 1239 static int 1240 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1241 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1242 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1243 { 1244 int len = 0; 1245 1246 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1247 goto process_auth_encryption_ccm_err; 1248 1249 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1250 goto process_auth_encryption_ccm_err; 1251 1252 if (aadlen > 0) 1253 /* 1254 * For AES-CCM, the actual AAD is placed 1255 * 18 bytes after the start of the AAD field, 1256 * according to the API. 1257 */ 1258 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1259 goto process_auth_encryption_ccm_err; 1260 1261 if (srclen >= 0) 1262 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1263 srclen, ctx, 0)) 1264 goto process_auth_encryption_ccm_err; 1265 1266 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1267 goto process_auth_encryption_ccm_err; 1268 1269 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1270 goto process_auth_encryption_ccm_err; 1271 1272 return 0; 1273 1274 process_auth_encryption_ccm_err: 1275 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1276 return -EINVAL; 1277 } 1278 1279 /** Process AES-GCM decrypt algorithm */ 1280 static int 1281 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1282 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1283 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1284 { 1285 int len = 0, unused = 0; 1286 uint8_t empty[] = {}; 1287 1288 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1289 goto process_auth_decryption_gcm_err; 1290 1291 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1292 goto process_auth_decryption_gcm_err; 1293 1294 if (aadlen > 0) 1295 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1296 goto process_auth_decryption_gcm_err; 1297 1298 if (srclen > 0) 1299 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1300 srclen, ctx, 0)) 1301 goto process_auth_decryption_gcm_err; 1302 1303 /* Workaround open ssl bug in version less then 1.0.1f */ 1304 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1305 goto process_auth_decryption_gcm_err; 1306 1307 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1308 return -EFAULT; 1309 1310 return 0; 1311 1312 process_auth_decryption_gcm_err: 1313 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1314 return -EINVAL; 1315 } 1316 1317 /** Process AES-CCM decrypt algorithm */ 1318 static int 1319 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1320 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1321 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1322 EVP_CIPHER_CTX *ctx) 1323 { 1324 int len = 0; 1325 1326 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1327 goto process_auth_decryption_ccm_err; 1328 1329 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1330 goto process_auth_decryption_ccm_err; 1331 1332 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1333 goto process_auth_decryption_ccm_err; 1334 1335 if (aadlen > 0) 1336 /* 1337 * For AES-CCM, the actual AAD is placed 1338 * 18 bytes after the start of the AAD field, 1339 * according to the API. 1340 */ 1341 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1342 goto process_auth_decryption_ccm_err; 1343 1344 if (srclen >= 0) 1345 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1346 srclen, ctx, 0)) 1347 return -EFAULT; 1348 1349 return 0; 1350 1351 process_auth_decryption_ccm_err: 1352 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1353 return -EINVAL; 1354 } 1355 1356 /** Process standard openssl auth algorithms */ 1357 static int 1358 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1359 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1360 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1361 { 1362 size_t dstlen; 1363 struct rte_mbuf *m; 1364 int l, n = srclen; 1365 uint8_t *src; 1366 1367 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1368 m = m->next) 1369 offset -= rte_pktmbuf_data_len(m); 1370 1371 if (m == 0) 1372 goto process_auth_err; 1373 1374 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1375 goto process_auth_err; 1376 1377 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1378 1379 l = rte_pktmbuf_data_len(m) - offset; 1380 if (srclen <= l) { 1381 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1382 goto process_auth_err; 1383 goto process_auth_final; 1384 } 1385 1386 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1387 goto process_auth_err; 1388 1389 n -= l; 1390 1391 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1392 src = rte_pktmbuf_mtod(m, uint8_t *); 1393 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1394 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1395 goto process_auth_err; 1396 n -= l; 1397 } 1398 1399 process_auth_final: 1400 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1401 goto process_auth_err; 1402 return 0; 1403 1404 process_auth_err: 1405 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1406 return -EINVAL; 1407 } 1408 1409 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1410 /** Process standard openssl auth algorithms with hmac/cmac */ 1411 static int 1412 process_openssl_auth_mac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1413 int srclen, EVP_MAC_CTX *ctx) 1414 { 1415 size_t dstlen; 1416 struct rte_mbuf *m; 1417 int l, n = srclen; 1418 uint8_t *src; 1419 1420 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1421 m = m->next) 1422 offset -= rte_pktmbuf_data_len(m); 1423 1424 if (m == 0) 1425 goto process_auth_err; 1426 1427 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1428 1429 l = rte_pktmbuf_data_len(m) - offset; 1430 if (srclen <= l) { 1431 if (EVP_MAC_update(ctx, (unsigned char *)src, srclen) != 1) 1432 goto process_auth_err; 1433 goto process_auth_final; 1434 } 1435 1436 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1437 goto process_auth_err; 1438 1439 n -= l; 1440 1441 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1442 src = rte_pktmbuf_mtod(m, uint8_t *); 1443 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1444 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1445 goto process_auth_err; 1446 n -= l; 1447 } 1448 1449 process_auth_final: 1450 if (EVP_MAC_final(ctx, dst, &dstlen, sizeof(dst)) != 1) 1451 goto process_auth_err; 1452 1453 EVP_MAC_CTX_free(ctx); 1454 return 0; 1455 1456 process_auth_err: 1457 EVP_MAC_CTX_free(ctx); 1458 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1459 return -EINVAL; 1460 } 1461 # else 1462 /** Process standard openssl auth algorithms with hmac */ 1463 static int 1464 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1465 int srclen, HMAC_CTX *ctx) 1466 { 1467 unsigned int dstlen; 1468 struct rte_mbuf *m; 1469 int l, n = srclen; 1470 uint8_t *src; 1471 1472 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1473 m = m->next) 1474 offset -= rte_pktmbuf_data_len(m); 1475 1476 if (m == 0) 1477 goto process_auth_err; 1478 1479 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1480 1481 l = rte_pktmbuf_data_len(m) - offset; 1482 if (srclen <= l) { 1483 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1484 goto process_auth_err; 1485 goto process_auth_final; 1486 } 1487 1488 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1489 goto process_auth_err; 1490 1491 n -= l; 1492 1493 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1494 src = rte_pktmbuf_mtod(m, uint8_t *); 1495 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1496 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1497 goto process_auth_err; 1498 n -= l; 1499 } 1500 1501 process_auth_final: 1502 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1503 goto process_auth_err; 1504 1505 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1506 goto process_auth_err; 1507 1508 return 0; 1509 1510 process_auth_err: 1511 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1512 return -EINVAL; 1513 } 1514 1515 /** Process standard openssl auth algorithms with cmac */ 1516 static int 1517 process_openssl_auth_cmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1518 int srclen, CMAC_CTX *ctx) 1519 { 1520 unsigned int dstlen; 1521 struct rte_mbuf *m; 1522 int l, n = srclen; 1523 uint8_t *src; 1524 1525 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1526 m = m->next) 1527 offset -= rte_pktmbuf_data_len(m); 1528 1529 if (m == 0) 1530 goto process_auth_err; 1531 1532 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1533 1534 l = rte_pktmbuf_data_len(m) - offset; 1535 if (srclen <= l) { 1536 if (CMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1537 goto process_auth_err; 1538 goto process_auth_final; 1539 } 1540 1541 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1542 goto process_auth_err; 1543 1544 n -= l; 1545 1546 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1547 src = rte_pktmbuf_mtod(m, uint8_t *); 1548 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1549 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1550 goto process_auth_err; 1551 n -= l; 1552 } 1553 1554 process_auth_final: 1555 if (CMAC_Final(ctx, dst, (size_t *)&dstlen) != 1) 1556 goto process_auth_err; 1557 1558 CMAC_CTX_cleanup(ctx); 1559 1560 return 0; 1561 1562 process_auth_err: 1563 OPENSSL_LOG(ERR, "Process openssl cmac auth failed"); 1564 return -EINVAL; 1565 } 1566 # endif 1567 /*----------------------------------------------------------------------------*/ 1568 1569 /** Process auth/cipher combined operation */ 1570 static void 1571 process_openssl_combined_op 1572 (struct rte_crypto_op *op, struct openssl_session *sess, 1573 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1574 { 1575 /* cipher */ 1576 uint8_t *dst = NULL, *iv, *tag, *aad; 1577 int srclen, aadlen, status = -1; 1578 uint32_t offset; 1579 uint8_t taglen; 1580 1581 /* 1582 * Segmented destination buffer is not supported for 1583 * encryption/decryption 1584 */ 1585 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1586 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1587 return; 1588 } 1589 1590 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1591 sess->iv.offset); 1592 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1593 srclen = 0; 1594 offset = op->sym->auth.data.offset; 1595 aadlen = op->sym->auth.data.length; 1596 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1597 op->sym->auth.data.offset); 1598 tag = op->sym->auth.digest.data; 1599 if (tag == NULL) 1600 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1601 offset + aadlen); 1602 } else { 1603 srclen = op->sym->aead.data.length; 1604 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1605 op->sym->aead.data.offset); 1606 offset = op->sym->aead.data.offset; 1607 aad = op->sym->aead.aad.data; 1608 aadlen = sess->auth.aad_length; 1609 tag = op->sym->aead.digest.data; 1610 if (tag == NULL) 1611 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1612 offset + srclen); 1613 } 1614 1615 taglen = sess->auth.digest_length; 1616 1617 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1618 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1619 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1620 status = process_openssl_auth_encryption_gcm( 1621 mbuf_src, offset, srclen, 1622 aad, aadlen, iv, 1623 dst, tag, sess->cipher.ctx); 1624 else 1625 status = process_openssl_auth_encryption_ccm( 1626 mbuf_src, offset, srclen, 1627 aad, aadlen, iv, 1628 dst, tag, taglen, sess->cipher.ctx); 1629 1630 } else { 1631 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1632 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1633 status = process_openssl_auth_decryption_gcm( 1634 mbuf_src, offset, srclen, 1635 aad, aadlen, iv, 1636 dst, tag, sess->cipher.ctx); 1637 else 1638 status = process_openssl_auth_decryption_ccm( 1639 mbuf_src, offset, srclen, 1640 aad, aadlen, iv, 1641 dst, tag, taglen, sess->cipher.ctx); 1642 } 1643 1644 if (status != 0) { 1645 if (status == (-EFAULT) && 1646 sess->auth.operation == 1647 RTE_CRYPTO_AUTH_OP_VERIFY) 1648 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1649 else 1650 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1651 } 1652 } 1653 1654 /** Process cipher operation */ 1655 static void 1656 process_openssl_cipher_op 1657 (struct rte_crypto_op *op, struct openssl_session *sess, 1658 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1659 { 1660 uint8_t *dst, *iv; 1661 int srclen, status; 1662 uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0; 1663 EVP_CIPHER_CTX *ctx_copy; 1664 1665 /* 1666 * Segmented OOP destination buffer is not supported for encryption/ 1667 * decryption. In case of des3ctr, even inplace segmented buffers are 1668 * not supported. 1669 */ 1670 if (!rte_pktmbuf_is_contiguous(mbuf_dst) && 1671 (!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) { 1672 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1673 return; 1674 } 1675 1676 srclen = op->sym->cipher.data.length; 1677 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1678 op->sym->cipher.data.offset); 1679 1680 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1681 sess->iv.offset); 1682 ctx_copy = EVP_CIPHER_CTX_new(); 1683 EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx); 1684 1685 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1686 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1687 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1688 op->sym->cipher.data.offset, iv, 1689 srclen, ctx_copy, inplace); 1690 else 1691 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1692 op->sym->cipher.data.offset, iv, 1693 srclen, ctx_copy, inplace); 1694 else 1695 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1696 op->sym->cipher.data.offset, iv, 1697 sess->cipher.key.data, srclen, 1698 ctx_copy); 1699 1700 EVP_CIPHER_CTX_free(ctx_copy); 1701 if (status != 0) 1702 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1703 } 1704 1705 /** Process cipher operation */ 1706 static void 1707 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1708 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1709 struct rte_mbuf *mbuf_dst) 1710 { 1711 uint8_t *src, *dst, *iv; 1712 uint8_t block_size, last_block_len; 1713 int srclen, status = 0; 1714 1715 srclen = op->sym->cipher.data.length; 1716 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1717 op->sym->cipher.data.offset); 1718 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1719 op->sym->cipher.data.offset); 1720 1721 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1722 sess->iv.offset); 1723 1724 block_size = DES_BLOCK_SIZE; 1725 1726 last_block_len = srclen % block_size; 1727 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1728 /* Encrypt only with ECB mode XOR IV */ 1729 if (srclen < block_size) { 1730 status = process_openssl_cipher_bpi_encrypt(src, dst, 1731 iv, srclen, 1732 sess->cipher.bpi_ctx); 1733 } else { 1734 srclen -= last_block_len; 1735 /* Encrypt with the block aligned stream with CBC mode */ 1736 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1737 op->sym->cipher.data.offset, iv, 1738 srclen, sess->cipher.ctx, 0); 1739 if (last_block_len) { 1740 /* Point at last block */ 1741 dst += srclen; 1742 /* 1743 * IV is the last encrypted block from 1744 * the previous operation 1745 */ 1746 iv = dst - block_size; 1747 src += srclen; 1748 srclen = last_block_len; 1749 /* Encrypt the last frame with ECB mode */ 1750 status |= process_openssl_cipher_bpi_encrypt(src, 1751 dst, iv, 1752 srclen, sess->cipher.bpi_ctx); 1753 } 1754 } 1755 } else { 1756 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1757 if (srclen < block_size) { 1758 status = process_openssl_cipher_bpi_encrypt(src, dst, 1759 iv, 1760 srclen, 1761 sess->cipher.bpi_ctx); 1762 } else { 1763 if (last_block_len) { 1764 /* Point at last block */ 1765 dst += srclen - last_block_len; 1766 src += srclen - last_block_len; 1767 /* 1768 * IV is the last full block 1769 */ 1770 iv = src - block_size; 1771 /* 1772 * Decrypt the last frame with ECB mode 1773 * (encrypt, as it is the same operation) 1774 */ 1775 status = process_openssl_cipher_bpi_encrypt(src, 1776 dst, iv, 1777 last_block_len, sess->cipher.bpi_ctx); 1778 /* Prepare parameters for CBC mode op */ 1779 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1780 sess->iv.offset); 1781 dst += last_block_len - srclen; 1782 srclen -= last_block_len; 1783 } 1784 1785 /* Decrypt with CBC mode */ 1786 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1787 op->sym->cipher.data.offset, iv, 1788 srclen, sess->cipher.ctx, 0); 1789 } 1790 } 1791 1792 if (status != 0) 1793 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1794 } 1795 1796 /** Process auth operation */ 1797 static void 1798 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1799 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1800 struct rte_mbuf *mbuf_dst) 1801 { 1802 uint8_t *dst; 1803 int srclen, status; 1804 EVP_MD_CTX *ctx_a; 1805 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1806 EVP_MAC_CTX *ctx_h; 1807 EVP_MAC_CTX *ctx_c; 1808 EVP_MAC *mac; 1809 # else 1810 HMAC_CTX *ctx_h; 1811 CMAC_CTX *ctx_c; 1812 # endif 1813 1814 srclen = op->sym->auth.data.length; 1815 1816 dst = qp->temp_digest; 1817 1818 switch (sess->auth.mode) { 1819 case OPENSSL_AUTH_AS_AUTH: 1820 ctx_a = EVP_MD_CTX_create(); 1821 EVP_MD_CTX_copy_ex(ctx_a, sess->auth.auth.ctx); 1822 status = process_openssl_auth(mbuf_src, dst, 1823 op->sym->auth.data.offset, NULL, NULL, srclen, 1824 ctx_a, sess->auth.auth.evp_algo); 1825 EVP_MD_CTX_destroy(ctx_a); 1826 break; 1827 case OPENSSL_AUTH_AS_HMAC: 1828 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1829 mac = EVP_MAC_fetch(NULL, "HMAC", NULL); 1830 ctx_h = EVP_MAC_CTX_new(mac); 1831 ctx_h = EVP_MAC_CTX_dup(sess->auth.hmac.ctx); 1832 EVP_MAC_free(mac); 1833 status = process_openssl_auth_mac(mbuf_src, dst, 1834 op->sym->auth.data.offset, srclen, 1835 ctx_h); 1836 # else 1837 ctx_h = HMAC_CTX_new(); 1838 HMAC_CTX_copy(ctx_h, sess->auth.hmac.ctx); 1839 status = process_openssl_auth_hmac(mbuf_src, dst, 1840 op->sym->auth.data.offset, srclen, 1841 ctx_h); 1842 HMAC_CTX_free(ctx_h); 1843 # endif 1844 break; 1845 case OPENSSL_AUTH_AS_CMAC: 1846 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1847 mac = EVP_MAC_fetch(NULL, OSSL_MAC_NAME_CMAC, NULL); 1848 ctx_c = EVP_MAC_CTX_new(mac); 1849 ctx_c = EVP_MAC_CTX_dup(sess->auth.cmac.ctx); 1850 EVP_MAC_free(mac); 1851 status = process_openssl_auth_mac(mbuf_src, dst, 1852 op->sym->auth.data.offset, srclen, 1853 ctx_c); 1854 # else 1855 ctx_c = CMAC_CTX_new(); 1856 CMAC_CTX_copy(ctx_c, sess->auth.cmac.ctx); 1857 status = process_openssl_auth_cmac(mbuf_src, dst, 1858 op->sym->auth.data.offset, srclen, 1859 ctx_c); 1860 CMAC_CTX_free(ctx_c); 1861 # endif 1862 break; 1863 default: 1864 status = -1; 1865 break; 1866 } 1867 1868 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1869 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data, 1870 sess->auth.digest_length) != 0) { 1871 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1872 } 1873 } else { 1874 uint8_t *auth_dst; 1875 1876 auth_dst = op->sym->auth.digest.data; 1877 if (auth_dst == NULL) 1878 auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1879 op->sym->auth.data.offset + 1880 op->sym->auth.data.length); 1881 memcpy(auth_dst, dst, sess->auth.digest_length); 1882 } 1883 1884 if (status != 0) 1885 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1886 } 1887 1888 /* process dsa sign operation */ 1889 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 1890 static int 1891 process_openssl_dsa_sign_op_evp(struct rte_crypto_op *cop, 1892 struct openssl_asym_session *sess) 1893 { 1894 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1895 EVP_PKEY_CTX *dsa_ctx = NULL; 1896 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 1897 EVP_PKEY *pkey = NULL; 1898 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 1899 OSSL_PARAM *params = NULL; 1900 1901 size_t outlen; 1902 unsigned char *dsa_sign_data; 1903 const unsigned char *dsa_sign_data_p; 1904 1905 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1906 params = OSSL_PARAM_BLD_to_param(param_bld); 1907 if (!params) { 1908 OSSL_PARAM_BLD_free(param_bld); 1909 return -1; 1910 } 1911 1912 if (key_ctx == NULL 1913 || EVP_PKEY_fromdata_init(key_ctx) <= 0 1914 || EVP_PKEY_fromdata(key_ctx, &pkey, 1915 EVP_PKEY_KEYPAIR, params) <= 0) 1916 goto err_dsa_sign; 1917 1918 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 1919 if (!dsa_ctx) 1920 goto err_dsa_sign; 1921 1922 if (EVP_PKEY_sign_init(dsa_ctx) <= 0) 1923 goto err_dsa_sign; 1924 1925 if (EVP_PKEY_sign(dsa_ctx, NULL, &outlen, op->message.data, 1926 op->message.length) <= 0) 1927 goto err_dsa_sign; 1928 1929 if (outlen <= 0) 1930 goto err_dsa_sign; 1931 1932 dsa_sign_data = OPENSSL_malloc(outlen); 1933 if (!dsa_sign_data) 1934 goto err_dsa_sign; 1935 1936 if (EVP_PKEY_sign(dsa_ctx, dsa_sign_data, &outlen, op->message.data, 1937 op->message.length) <= 0) { 1938 free(dsa_sign_data); 1939 goto err_dsa_sign; 1940 } 1941 1942 dsa_sign_data_p = (const unsigned char *)dsa_sign_data; 1943 DSA_SIG *sign = d2i_DSA_SIG(NULL, &dsa_sign_data_p, outlen); 1944 if (!sign) { 1945 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 1946 free(dsa_sign_data); 1947 goto err_dsa_sign; 1948 } else { 1949 const BIGNUM *r = NULL, *s = NULL; 1950 get_dsa_sign(sign, &r, &s); 1951 1952 op->r.length = BN_bn2bin(r, op->r.data); 1953 op->s.length = BN_bn2bin(s, op->s.data); 1954 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1955 } 1956 1957 DSA_SIG_free(sign); 1958 free(dsa_sign_data); 1959 return 0; 1960 1961 err_dsa_sign: 1962 if (params) 1963 OSSL_PARAM_free(params); 1964 if (key_ctx) 1965 EVP_PKEY_CTX_free(key_ctx); 1966 if (dsa_ctx) 1967 EVP_PKEY_CTX_free(dsa_ctx); 1968 return -1; 1969 } 1970 1971 /* process dsa verify operation */ 1972 static int 1973 process_openssl_dsa_verify_op_evp(struct rte_crypto_op *cop, 1974 struct openssl_asym_session *sess) 1975 { 1976 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1977 DSA_SIG *sign = DSA_SIG_new(); 1978 BIGNUM *r = NULL, *s = NULL; 1979 BIGNUM *pub_key = NULL; 1980 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 1981 OSSL_PARAM *params = NULL; 1982 EVP_PKEY *pkey = NULL; 1983 EVP_PKEY_CTX *dsa_ctx = NULL; 1984 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 1985 unsigned char *dsa_sig = NULL; 1986 size_t sig_len; 1987 int ret = -1; 1988 1989 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1990 if (!param_bld) { 1991 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 1992 return -1; 1993 } 1994 1995 r = BN_bin2bn(op->r.data, op->r.length, r); 1996 s = BN_bin2bn(op->s.data, op->s.length, s); 1997 pub_key = BN_bin2bn(op->y.data, op->y.length, pub_key); 1998 if (!r || !s || !pub_key) { 1999 BN_free(r); 2000 BN_free(s); 2001 BN_free(pub_key); 2002 OSSL_PARAM_BLD_free(param_bld); 2003 goto err_dsa_verify; 2004 } 2005 2006 set_dsa_sign(sign, r, s); 2007 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PUB_KEY, pub_key)) { 2008 OSSL_PARAM_BLD_free(param_bld); 2009 goto err_dsa_verify; 2010 } 2011 2012 params = OSSL_PARAM_BLD_to_param(param_bld); 2013 if (!params) { 2014 OSSL_PARAM_BLD_free(param_bld); 2015 goto err_dsa_verify; 2016 } 2017 2018 if (key_ctx == NULL 2019 || EVP_PKEY_fromdata_init(key_ctx) <= 0 2020 || EVP_PKEY_fromdata(key_ctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2021 goto err_dsa_verify; 2022 2023 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 2024 if (!dsa_ctx) 2025 goto err_dsa_verify; 2026 2027 if (!sign) 2028 goto err_dsa_verify; 2029 2030 sig_len = i2d_DSA_SIG(sign, &dsa_sig); 2031 if (EVP_PKEY_verify_init(dsa_ctx) <= 0) 2032 goto err_dsa_verify; 2033 2034 ret = EVP_PKEY_verify(dsa_ctx, dsa_sig, sig_len, 2035 op->message.data, op->message.length); 2036 if (ret == 1) { 2037 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2038 ret = 0; 2039 } 2040 2041 err_dsa_verify: 2042 if (sign) 2043 DSA_SIG_free(sign); 2044 if (params) 2045 OSSL_PARAM_free(params); 2046 if (key_ctx) 2047 EVP_PKEY_CTX_free(key_ctx); 2048 if (dsa_ctx) 2049 EVP_PKEY_CTX_free(dsa_ctx); 2050 2051 return ret; 2052 } 2053 #else 2054 static int 2055 process_openssl_dsa_sign_op(struct rte_crypto_op *cop, 2056 struct openssl_asym_session *sess) 2057 { 2058 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2059 DSA *dsa = sess->u.s.dsa; 2060 DSA_SIG *sign = NULL; 2061 2062 sign = DSA_do_sign(op->message.data, 2063 op->message.length, 2064 dsa); 2065 2066 if (sign == NULL) { 2067 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 2068 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2069 } else { 2070 const BIGNUM *r = NULL, *s = NULL; 2071 get_dsa_sign(sign, &r, &s); 2072 2073 op->r.length = BN_bn2bin(r, op->r.data); 2074 op->s.length = BN_bn2bin(s, op->s.data); 2075 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2076 } 2077 2078 DSA_SIG_free(sign); 2079 2080 return 0; 2081 } 2082 2083 /* process dsa verify operation */ 2084 static int 2085 process_openssl_dsa_verify_op(struct rte_crypto_op *cop, 2086 struct openssl_asym_session *sess) 2087 { 2088 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2089 DSA *dsa = sess->u.s.dsa; 2090 int ret; 2091 DSA_SIG *sign = DSA_SIG_new(); 2092 BIGNUM *r = NULL, *s = NULL; 2093 BIGNUM *pub_key = NULL; 2094 2095 if (sign == NULL) { 2096 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 2097 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2098 return -1; 2099 } 2100 2101 r = BN_bin2bn(op->r.data, 2102 op->r.length, 2103 r); 2104 s = BN_bin2bn(op->s.data, 2105 op->s.length, 2106 s); 2107 pub_key = BN_bin2bn(op->y.data, 2108 op->y.length, 2109 pub_key); 2110 if (!r || !s || !pub_key) { 2111 BN_free(r); 2112 BN_free(s); 2113 BN_free(pub_key); 2114 2115 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2116 return -1; 2117 } 2118 set_dsa_sign(sign, r, s); 2119 set_dsa_pub_key(dsa, pub_key); 2120 2121 ret = DSA_do_verify(op->message.data, 2122 op->message.length, 2123 sign, 2124 dsa); 2125 2126 if (ret != 1) 2127 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2128 else 2129 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2130 2131 DSA_SIG_free(sign); 2132 2133 return 0; 2134 } 2135 #endif 2136 2137 /* process dh operation */ 2138 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2139 static int 2140 process_openssl_dh_op_evp(struct rte_crypto_op *cop, 2141 struct openssl_asym_session *sess) 2142 { 2143 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2144 OSSL_PARAM_BLD *param_bld = sess->u.dh.param_bld; 2145 OSSL_PARAM_BLD *param_bld_peer = sess->u.dh.param_bld_peer; 2146 OSSL_PARAM *params = NULL; 2147 EVP_PKEY *dhpkey = NULL; 2148 EVP_PKEY *peerkey = NULL; 2149 BIGNUM *priv_key = NULL; 2150 BIGNUM *pub_key = NULL; 2151 int ret = -1; 2152 2153 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2154 EVP_PKEY_CTX *dh_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2155 if (dh_ctx == NULL || param_bld == NULL) 2156 return ret; 2157 2158 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2159 OSSL_PARAM *params_peer = NULL; 2160 2161 if (!param_bld_peer) 2162 return ret; 2163 2164 pub_key = BN_bin2bn(op->pub_key.data, op->pub_key.length, 2165 pub_key); 2166 if (pub_key == NULL) { 2167 OSSL_PARAM_BLD_free(param_bld_peer); 2168 return ret; 2169 } 2170 2171 if (!OSSL_PARAM_BLD_push_BN(param_bld_peer, OSSL_PKEY_PARAM_PUB_KEY, 2172 pub_key)) { 2173 OPENSSL_LOG(ERR, "Failed to set public key\n"); 2174 OSSL_PARAM_BLD_free(param_bld_peer); 2175 BN_free(pub_key); 2176 return ret; 2177 } 2178 2179 params_peer = OSSL_PARAM_BLD_to_param(param_bld_peer); 2180 if (!params_peer) { 2181 OSSL_PARAM_BLD_free(param_bld_peer); 2182 BN_free(pub_key); 2183 return ret; 2184 } 2185 2186 EVP_PKEY_CTX *peer_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2187 if (EVP_PKEY_keygen_init(peer_ctx) != 1) { 2188 OSSL_PARAM_free(params_peer); 2189 BN_free(pub_key); 2190 return ret; 2191 } 2192 2193 if (EVP_PKEY_CTX_set_params(peer_ctx, params_peer) != 1) { 2194 EVP_PKEY_CTX_free(peer_ctx); 2195 OSSL_PARAM_free(params_peer); 2196 BN_free(pub_key); 2197 return ret; 2198 } 2199 2200 if (EVP_PKEY_keygen(peer_ctx, &peerkey) != 1) { 2201 EVP_PKEY_CTX_free(peer_ctx); 2202 OSSL_PARAM_free(params_peer); 2203 BN_free(pub_key); 2204 return ret; 2205 } 2206 2207 priv_key = BN_bin2bn(op->priv_key.data, op->priv_key.length, 2208 priv_key); 2209 if (priv_key == NULL) { 2210 EVP_PKEY_CTX_free(peer_ctx); 2211 OSSL_PARAM_free(params_peer); 2212 BN_free(pub_key); 2213 return ret; 2214 } 2215 2216 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY, 2217 priv_key)) { 2218 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2219 EVP_PKEY_CTX_free(peer_ctx); 2220 OSSL_PARAM_free(params_peer); 2221 BN_free(pub_key); 2222 BN_free(priv_key); 2223 return ret; 2224 } 2225 2226 OSSL_PARAM_free(params_peer); 2227 EVP_PKEY_CTX_free(peer_ctx); 2228 } 2229 2230 params = OSSL_PARAM_BLD_to_param(param_bld); 2231 if (!params) 2232 goto err_dh; 2233 2234 if (EVP_PKEY_keygen_init(dh_ctx) != 1) 2235 goto err_dh; 2236 2237 if (EVP_PKEY_CTX_set_params(dh_ctx, params) != 1) 2238 goto err_dh; 2239 2240 if (EVP_PKEY_keygen(dh_ctx, &dhpkey) != 1) 2241 goto err_dh; 2242 2243 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2244 OPENSSL_LOG(DEBUG, "%s:%d updated pub key\n", __func__, __LINE__); 2245 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PUB_KEY, &pub_key)) 2246 goto err_dh; 2247 /* output public key */ 2248 op->pub_key.length = BN_bn2bin(pub_key, op->pub_key.data); 2249 } 2250 2251 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2252 2253 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", __func__, __LINE__); 2254 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PRIV_KEY, &priv_key)) 2255 goto err_dh; 2256 2257 /* provide generated private key back to user */ 2258 op->priv_key.length = BN_bn2bin(priv_key, op->priv_key.data); 2259 } 2260 2261 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2262 size_t skey_len; 2263 EVP_PKEY_CTX *sc_ctx = EVP_PKEY_CTX_new(dhpkey, NULL); 2264 if (!sc_ctx) 2265 goto err_dh; 2266 2267 if (EVP_PKEY_derive_init(sc_ctx) <= 0) { 2268 EVP_PKEY_CTX_free(sc_ctx); 2269 goto err_dh; 2270 } 2271 2272 if (!peerkey) { 2273 EVP_PKEY_CTX_free(sc_ctx); 2274 goto err_dh; 2275 } 2276 2277 if (EVP_PKEY_derive_set_peer(sc_ctx, peerkey) <= 0) { 2278 EVP_PKEY_CTX_free(sc_ctx); 2279 goto err_dh; 2280 } 2281 2282 /* Determine buffer length */ 2283 if (EVP_PKEY_derive(sc_ctx, NULL, &skey_len) <= 0) { 2284 EVP_PKEY_CTX_free(sc_ctx); 2285 goto err_dh; 2286 } 2287 2288 if (EVP_PKEY_derive(sc_ctx, op->shared_secret.data, &skey_len) <= 0) { 2289 EVP_PKEY_CTX_free(sc_ctx); 2290 goto err_dh; 2291 } 2292 2293 op->shared_secret.length = skey_len; 2294 EVP_PKEY_CTX_free(sc_ctx); 2295 } 2296 2297 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2298 ret = 0; 2299 2300 err_dh: 2301 if (pub_key) 2302 BN_free(pub_key); 2303 if (priv_key) 2304 BN_free(priv_key); 2305 if (params) 2306 OSSL_PARAM_free(params); 2307 if (dhpkey) 2308 EVP_PKEY_free(dhpkey); 2309 if (peerkey) 2310 EVP_PKEY_free(peerkey); 2311 2312 EVP_PKEY_CTX_free(dh_ctx); 2313 2314 return ret; 2315 } 2316 #else 2317 static int 2318 process_openssl_dh_op(struct rte_crypto_op *cop, 2319 struct openssl_asym_session *sess) 2320 { 2321 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2322 struct rte_crypto_asym_op *asym_op = cop->asym; 2323 DH *dh_key = sess->u.dh.dh_key; 2324 BIGNUM *priv_key = NULL; 2325 int ret = 0; 2326 2327 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2328 /* compute shared secret using peer public key 2329 * and current private key 2330 * shared secret = peer_key ^ priv_key mod p 2331 */ 2332 BIGNUM *peer_key = NULL; 2333 2334 /* copy private key and peer key and compute shared secret */ 2335 peer_key = BN_bin2bn(op->pub_key.data, 2336 op->pub_key.length, 2337 peer_key); 2338 if (peer_key == NULL) { 2339 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2340 return -1; 2341 } 2342 priv_key = BN_bin2bn(op->priv_key.data, 2343 op->priv_key.length, 2344 priv_key); 2345 if (priv_key == NULL) { 2346 BN_free(peer_key); 2347 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2348 return -1; 2349 } 2350 ret = set_dh_priv_key(dh_key, priv_key); 2351 if (ret) { 2352 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2353 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2354 BN_free(peer_key); 2355 BN_free(priv_key); 2356 return 0; 2357 } 2358 2359 ret = DH_compute_key( 2360 op->shared_secret.data, 2361 peer_key, dh_key); 2362 if (ret < 0) { 2363 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2364 BN_free(peer_key); 2365 /* priv key is already loaded into dh, 2366 * let's not free that directly here. 2367 * DH_free() will auto free it later. 2368 */ 2369 return 0; 2370 } 2371 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2372 op->shared_secret.length = ret; 2373 BN_free(peer_key); 2374 return 0; 2375 } 2376 2377 /* 2378 * other options are public and private key generations. 2379 * 2380 * if user provides private key, 2381 * then first set DH with user provided private key 2382 */ 2383 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE && 2384 op->priv_key.length) { 2385 /* generate public key using user-provided private key 2386 * pub_key = g ^ priv_key mod p 2387 */ 2388 2389 /* load private key into DH */ 2390 priv_key = BN_bin2bn(op->priv_key.data, 2391 op->priv_key.length, 2392 priv_key); 2393 if (priv_key == NULL) { 2394 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2395 return -1; 2396 } 2397 ret = set_dh_priv_key(dh_key, priv_key); 2398 if (ret) { 2399 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2400 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2401 BN_free(priv_key); 2402 return 0; 2403 } 2404 } 2405 2406 /* generate public and private key pair. 2407 * 2408 * if private key already set, generates only public key. 2409 * 2410 * if private key is not already set, then set it to random value 2411 * and update internal private key. 2412 */ 2413 if (!DH_generate_key(dh_key)) { 2414 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2415 return 0; 2416 } 2417 2418 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2419 const BIGNUM *pub_key = NULL; 2420 2421 OPENSSL_LOG(DEBUG, "%s:%d update public key\n", 2422 __func__, __LINE__); 2423 2424 /* get the generated keys */ 2425 get_dh_pub_key(dh_key, &pub_key); 2426 2427 /* output public key */ 2428 op->pub_key.length = BN_bn2bin(pub_key, 2429 op->pub_key.data); 2430 } 2431 2432 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2433 const BIGNUM *priv_key = NULL; 2434 2435 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", 2436 __func__, __LINE__); 2437 2438 /* get the generated keys */ 2439 get_dh_priv_key(dh_key, &priv_key); 2440 2441 /* provide generated private key back to user */ 2442 op->priv_key.length = BN_bn2bin(priv_key, 2443 op->priv_key.data); 2444 } 2445 2446 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2447 2448 return 0; 2449 } 2450 #endif 2451 2452 /* process modinv operation */ 2453 static int 2454 process_openssl_modinv_op(struct rte_crypto_op *cop, 2455 struct openssl_asym_session *sess) 2456 { 2457 struct rte_crypto_asym_op *op = cop->asym; 2458 BIGNUM *base = BN_CTX_get(sess->u.m.ctx); 2459 BIGNUM *res = BN_CTX_get(sess->u.m.ctx); 2460 2461 if (unlikely(base == NULL || res == NULL)) { 2462 BN_free(base); 2463 BN_free(res); 2464 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2465 return -1; 2466 } 2467 2468 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 2469 op->modinv.base.length, base); 2470 2471 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) { 2472 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2473 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data); 2474 } else { 2475 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2476 } 2477 2478 BN_clear(res); 2479 BN_clear(base); 2480 2481 return 0; 2482 } 2483 2484 /* process modexp operation */ 2485 static int 2486 process_openssl_modexp_op(struct rte_crypto_op *cop, 2487 struct openssl_asym_session *sess) 2488 { 2489 struct rte_crypto_asym_op *op = cop->asym; 2490 BIGNUM *base = BN_CTX_get(sess->u.e.ctx); 2491 BIGNUM *res = BN_CTX_get(sess->u.e.ctx); 2492 2493 if (unlikely(base == NULL || res == NULL)) { 2494 BN_free(base); 2495 BN_free(res); 2496 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2497 return -1; 2498 } 2499 2500 base = BN_bin2bn((const unsigned char *)op->modex.base.data, 2501 op->modex.base.length, base); 2502 2503 if (BN_mod_exp(res, base, sess->u.e.exp, 2504 sess->u.e.mod, sess->u.e.ctx)) { 2505 op->modex.result.length = BN_bn2bin(res, op->modex.result.data); 2506 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2507 } else { 2508 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2509 } 2510 2511 BN_clear(res); 2512 BN_clear(base); 2513 2514 return 0; 2515 } 2516 2517 /* process rsa operations */ 2518 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2519 static int 2520 process_openssl_rsa_op_evp(struct rte_crypto_op *cop, 2521 struct openssl_asym_session *sess) 2522 { 2523 struct rte_crypto_asym_op *op = cop->asym; 2524 uint32_t pad = (op->rsa.padding.type); 2525 uint8_t *tmp; 2526 size_t outlen = 0; 2527 int ret = -1; 2528 2529 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2530 EVP_PKEY_CTX *rsa_ctx = sess->u.r.ctx; 2531 if (!rsa_ctx) 2532 return ret; 2533 2534 switch (pad) { 2535 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2536 pad = RSA_PKCS1_PADDING; 2537 break; 2538 case RTE_CRYPTO_RSA_PADDING_NONE: 2539 pad = RSA_NO_PADDING; 2540 break; 2541 default: 2542 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2543 OPENSSL_LOG(ERR, 2544 "rsa pad type not supported %d\n", pad); 2545 return ret; 2546 } 2547 2548 switch (op->rsa.op_type) { 2549 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2550 if (EVP_PKEY_encrypt_init(rsa_ctx) != 1) 2551 goto err_rsa; 2552 2553 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2554 goto err_rsa; 2555 2556 if (EVP_PKEY_encrypt(rsa_ctx, NULL, &outlen, 2557 op->rsa.message.data, 2558 op->rsa.message.length) <= 0) 2559 goto err_rsa; 2560 2561 if (outlen <= 0) 2562 goto err_rsa; 2563 2564 if (EVP_PKEY_encrypt(rsa_ctx, op->rsa.cipher.data, &outlen, 2565 op->rsa.message.data, 2566 op->rsa.message.length) <= 0) 2567 goto err_rsa; 2568 op->rsa.cipher.length = outlen; 2569 2570 OPENSSL_LOG(DEBUG, 2571 "length of encrypted text %zu\n", outlen); 2572 break; 2573 2574 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2575 if (EVP_PKEY_decrypt_init(rsa_ctx) != 1) 2576 goto err_rsa; 2577 2578 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2579 goto err_rsa; 2580 2581 if (EVP_PKEY_decrypt(rsa_ctx, NULL, &outlen, 2582 op->rsa.cipher.data, 2583 op->rsa.cipher.length) <= 0) 2584 goto err_rsa; 2585 2586 if (outlen <= 0) 2587 goto err_rsa; 2588 2589 if (EVP_PKEY_decrypt(rsa_ctx, op->rsa.message.data, &outlen, 2590 op->rsa.cipher.data, 2591 op->rsa.cipher.length) <= 0) 2592 goto err_rsa; 2593 op->rsa.message.length = outlen; 2594 2595 OPENSSL_LOG(DEBUG, "length of decrypted text %zu\n", outlen); 2596 break; 2597 2598 case RTE_CRYPTO_ASYM_OP_SIGN: 2599 if (EVP_PKEY_sign_init(rsa_ctx) <= 0) 2600 goto err_rsa; 2601 2602 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2603 goto err_rsa; 2604 2605 if (EVP_PKEY_sign(rsa_ctx, NULL, &outlen, 2606 op->rsa.message.data, 2607 op->rsa.message.length) <= 0) 2608 goto err_rsa; 2609 2610 if (outlen <= 0) 2611 goto err_rsa; 2612 2613 if (EVP_PKEY_sign(rsa_ctx, op->rsa.sign.data, &outlen, 2614 op->rsa.message.data, 2615 op->rsa.message.length) <= 0) 2616 goto err_rsa; 2617 op->rsa.sign.length = outlen; 2618 break; 2619 2620 case RTE_CRYPTO_ASYM_OP_VERIFY: 2621 if (EVP_PKEY_verify_recover_init(rsa_ctx) <= 0) 2622 goto err_rsa; 2623 2624 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2625 goto err_rsa; 2626 2627 if (EVP_PKEY_verify_recover(rsa_ctx, NULL, &outlen, 2628 op->rsa.sign.data, 2629 op->rsa.sign.length) <= 0) 2630 goto err_rsa; 2631 2632 if ((outlen <= 0) || (outlen != op->rsa.sign.length)) 2633 goto err_rsa; 2634 2635 tmp = OPENSSL_malloc(outlen); 2636 if (tmp == NULL) { 2637 OPENSSL_LOG(ERR, "Memory allocation failed"); 2638 goto err_rsa; 2639 } 2640 2641 if (EVP_PKEY_verify_recover(rsa_ctx, tmp, &outlen, 2642 op->rsa.sign.data, 2643 op->rsa.sign.length) <= 0) { 2644 rte_free(tmp); 2645 goto err_rsa; 2646 } 2647 2648 OPENSSL_LOG(DEBUG, 2649 "Length of public_decrypt %zu " 2650 "length of message %zd\n", 2651 outlen, op->rsa.message.length); 2652 if (CRYPTO_memcmp(tmp, op->rsa.message.data, 2653 op->rsa.message.length)) { 2654 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 2655 } 2656 rte_free(tmp); 2657 break; 2658 2659 default: 2660 /* allow ops with invalid args to be pushed to 2661 * completion queue 2662 */ 2663 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2664 goto err_rsa; 2665 } 2666 2667 ret = 0; 2668 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2669 err_rsa: 2670 return ret; 2671 2672 } 2673 #else 2674 static int 2675 process_openssl_rsa_op(struct rte_crypto_op *cop, 2676 struct openssl_asym_session *sess) 2677 { 2678 int ret = 0; 2679 struct rte_crypto_asym_op *op = cop->asym; 2680 RSA *rsa = sess->u.r.rsa; 2681 uint32_t pad = (op->rsa.padding.type); 2682 uint8_t *tmp; 2683 2684 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2685 2686 switch (pad) { 2687 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2688 pad = RSA_PKCS1_PADDING; 2689 break; 2690 case RTE_CRYPTO_RSA_PADDING_NONE: 2691 pad = RSA_NO_PADDING; 2692 break; 2693 default: 2694 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2695 OPENSSL_LOG(ERR, 2696 "rsa pad type not supported %d\n", pad); 2697 return 0; 2698 } 2699 2700 switch (op->rsa.op_type) { 2701 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2702 ret = RSA_public_encrypt(op->rsa.message.length, 2703 op->rsa.message.data, 2704 op->rsa.cipher.data, 2705 rsa, 2706 pad); 2707 2708 if (ret > 0) 2709 op->rsa.cipher.length = ret; 2710 OPENSSL_LOG(DEBUG, 2711 "length of encrypted text %d\n", ret); 2712 break; 2713 2714 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2715 ret = RSA_private_decrypt(op->rsa.cipher.length, 2716 op->rsa.cipher.data, 2717 op->rsa.message.data, 2718 rsa, 2719 pad); 2720 if (ret > 0) 2721 op->rsa.message.length = ret; 2722 break; 2723 2724 case RTE_CRYPTO_ASYM_OP_SIGN: 2725 ret = RSA_private_encrypt(op->rsa.message.length, 2726 op->rsa.message.data, 2727 op->rsa.sign.data, 2728 rsa, 2729 pad); 2730 if (ret > 0) 2731 op->rsa.sign.length = ret; 2732 break; 2733 2734 case RTE_CRYPTO_ASYM_OP_VERIFY: 2735 tmp = rte_malloc(NULL, op->rsa.sign.length, 0); 2736 if (tmp == NULL) { 2737 OPENSSL_LOG(ERR, "Memory allocation failed"); 2738 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2739 break; 2740 } 2741 ret = RSA_public_decrypt(op->rsa.sign.length, 2742 op->rsa.sign.data, 2743 tmp, 2744 rsa, 2745 pad); 2746 2747 OPENSSL_LOG(DEBUG, 2748 "Length of public_decrypt %d " 2749 "length of message %zd\n", 2750 ret, op->rsa.message.length); 2751 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data, 2752 op->rsa.message.length))) { 2753 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 2754 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2755 } 2756 rte_free(tmp); 2757 break; 2758 2759 default: 2760 /* allow ops with invalid args to be pushed to 2761 * completion queue 2762 */ 2763 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2764 break; 2765 } 2766 2767 if (ret < 0) 2768 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2769 2770 return 0; 2771 } 2772 #endif 2773 2774 static int 2775 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op, 2776 struct openssl_asym_session *sess) 2777 { 2778 int retval = 0; 2779 2780 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2781 2782 switch (sess->xfrm_type) { 2783 case RTE_CRYPTO_ASYM_XFORM_RSA: 2784 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2785 retval = process_openssl_rsa_op_evp(op, sess); 2786 # else 2787 retval = process_openssl_rsa_op(op, sess); 2788 #endif 2789 break; 2790 case RTE_CRYPTO_ASYM_XFORM_MODEX: 2791 retval = process_openssl_modexp_op(op, sess); 2792 break; 2793 case RTE_CRYPTO_ASYM_XFORM_MODINV: 2794 retval = process_openssl_modinv_op(op, sess); 2795 break; 2796 case RTE_CRYPTO_ASYM_XFORM_DH: 2797 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2798 retval = process_openssl_dh_op_evp(op, sess); 2799 # else 2800 retval = process_openssl_dh_op(op, sess); 2801 #endif 2802 break; 2803 case RTE_CRYPTO_ASYM_XFORM_DSA: 2804 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2805 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 2806 retval = process_openssl_dsa_sign_op_evp(op, sess); 2807 else if (op->asym->dsa.op_type == 2808 RTE_CRYPTO_ASYM_OP_VERIFY) 2809 retval = 2810 process_openssl_dsa_verify_op_evp(op, sess); 2811 #else 2812 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 2813 retval = process_openssl_dsa_sign_op(op, sess); 2814 else if (op->asym->dsa.op_type == 2815 RTE_CRYPTO_ASYM_OP_VERIFY) 2816 retval = 2817 process_openssl_dsa_verify_op(op, sess); 2818 else 2819 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2820 #endif 2821 break; 2822 default: 2823 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2824 break; 2825 } 2826 if (!retval) { 2827 /* op processed so push to completion queue as processed */ 2828 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 2829 if (retval) 2830 /* return error if failed to put in completion queue */ 2831 retval = -1; 2832 } 2833 2834 return retval; 2835 } 2836 2837 static void 2838 copy_plaintext(struct rte_mbuf *m_src, struct rte_mbuf *m_dst, 2839 struct rte_crypto_op *op) 2840 { 2841 uint8_t *p_src, *p_dst; 2842 2843 p_src = rte_pktmbuf_mtod(m_src, uint8_t *); 2844 p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *); 2845 2846 /** 2847 * Copy the content between cipher offset and auth offset 2848 * for generating correct digest. 2849 */ 2850 if (op->sym->cipher.data.offset > op->sym->auth.data.offset) 2851 memcpy(p_dst + op->sym->auth.data.offset, 2852 p_src + op->sym->auth.data.offset, 2853 op->sym->cipher.data.offset - 2854 op->sym->auth.data.offset); 2855 } 2856 2857 /** Process crypto operation for mbuf */ 2858 static int 2859 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 2860 struct openssl_session *sess) 2861 { 2862 struct rte_mbuf *msrc, *mdst; 2863 int retval; 2864 2865 msrc = op->sym->m_src; 2866 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 2867 2868 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2869 2870 switch (sess->chain_order) { 2871 case OPENSSL_CHAIN_ONLY_CIPHER: 2872 process_openssl_cipher_op(op, sess, msrc, mdst); 2873 break; 2874 case OPENSSL_CHAIN_ONLY_AUTH: 2875 process_openssl_auth_op(qp, op, sess, msrc, mdst); 2876 break; 2877 case OPENSSL_CHAIN_CIPHER_AUTH: 2878 process_openssl_cipher_op(op, sess, msrc, mdst); 2879 /* OOP */ 2880 if (msrc != mdst) 2881 copy_plaintext(msrc, mdst, op); 2882 process_openssl_auth_op(qp, op, sess, mdst, mdst); 2883 break; 2884 case OPENSSL_CHAIN_AUTH_CIPHER: 2885 process_openssl_auth_op(qp, op, sess, msrc, mdst); 2886 process_openssl_cipher_op(op, sess, msrc, mdst); 2887 break; 2888 case OPENSSL_CHAIN_COMBINED: 2889 process_openssl_combined_op(op, sess, msrc, mdst); 2890 break; 2891 case OPENSSL_CHAIN_CIPHER_BPI: 2892 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 2893 break; 2894 default: 2895 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 2896 break; 2897 } 2898 2899 /* Free session if a session-less crypto op */ 2900 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 2901 openssl_reset_session(sess); 2902 memset(sess, 0, sizeof(struct openssl_session)); 2903 memset(op->sym->session, 0, 2904 rte_cryptodev_sym_get_existing_header_session_size( 2905 op->sym->session)); 2906 rte_mempool_put(qp->sess_mp_priv, sess); 2907 rte_mempool_put(qp->sess_mp, op->sym->session); 2908 op->sym->session = NULL; 2909 } 2910 2911 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 2912 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2913 2914 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 2915 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 2916 else 2917 retval = -1; 2918 2919 return retval; 2920 } 2921 2922 /* 2923 *------------------------------------------------------------------------------ 2924 * PMD Framework 2925 *------------------------------------------------------------------------------ 2926 */ 2927 2928 /** Enqueue burst */ 2929 static uint16_t 2930 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 2931 uint16_t nb_ops) 2932 { 2933 void *sess; 2934 struct openssl_qp *qp = queue_pair; 2935 int i, retval; 2936 2937 for (i = 0; i < nb_ops; i++) { 2938 sess = get_session(qp, ops[i]); 2939 if (unlikely(sess == NULL)) 2940 goto enqueue_err; 2941 2942 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) 2943 retval = process_op(qp, ops[i], 2944 (struct openssl_session *) sess); 2945 else 2946 retval = process_asym_op(qp, ops[i], 2947 (struct openssl_asym_session *) sess); 2948 if (unlikely(retval < 0)) 2949 goto enqueue_err; 2950 } 2951 2952 qp->stats.enqueued_count += i; 2953 return i; 2954 2955 enqueue_err: 2956 qp->stats.enqueue_err_count++; 2957 return i; 2958 } 2959 2960 /** Dequeue burst */ 2961 static uint16_t 2962 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 2963 uint16_t nb_ops) 2964 { 2965 struct openssl_qp *qp = queue_pair; 2966 2967 unsigned int nb_dequeued = 0; 2968 2969 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 2970 (void **)ops, nb_ops, NULL); 2971 qp->stats.dequeued_count += nb_dequeued; 2972 2973 return nb_dequeued; 2974 } 2975 2976 /** Create OPENSSL crypto device */ 2977 static int 2978 cryptodev_openssl_create(const char *name, 2979 struct rte_vdev_device *vdev, 2980 struct rte_cryptodev_pmd_init_params *init_params) 2981 { 2982 struct rte_cryptodev *dev; 2983 struct openssl_private *internals; 2984 2985 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 2986 if (dev == NULL) { 2987 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 2988 goto init_error; 2989 } 2990 2991 dev->driver_id = cryptodev_driver_id; 2992 dev->dev_ops = rte_openssl_pmd_ops; 2993 2994 /* register rx/tx burst functions for data path */ 2995 dev->dequeue_burst = openssl_pmd_dequeue_burst; 2996 dev->enqueue_burst = openssl_pmd_enqueue_burst; 2997 2998 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 2999 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 3000 RTE_CRYPTODEV_FF_CPU_AESNI | 3001 RTE_CRYPTODEV_FF_IN_PLACE_SGL | 3002 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 3003 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | 3004 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO | 3005 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP | 3006 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT | 3007 RTE_CRYPTODEV_FF_SYM_SESSIONLESS; 3008 3009 internals = dev->data->dev_private; 3010 3011 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 3012 3013 rte_cryptodev_pmd_probing_finish(dev); 3014 3015 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3016 /* Load legacy provider 3017 * Some algorithms are no longer available in earlier version of openssl, 3018 * unless the legacy provider explicitly loaded. e.g. DES 3019 */ 3020 ossl_legacy_provider_load(); 3021 # endif 3022 return 0; 3023 3024 init_error: 3025 OPENSSL_LOG(ERR, "driver %s: create failed", 3026 init_params->name); 3027 3028 cryptodev_openssl_remove(vdev); 3029 return -EFAULT; 3030 } 3031 3032 /** Initialise OPENSSL crypto device */ 3033 static int 3034 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 3035 { 3036 struct rte_cryptodev_pmd_init_params init_params = { 3037 "", 3038 sizeof(struct openssl_private), 3039 rte_socket_id(), 3040 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 3041 }; 3042 const char *name; 3043 const char *input_args; 3044 3045 name = rte_vdev_device_name(vdev); 3046 if (name == NULL) 3047 return -EINVAL; 3048 input_args = rte_vdev_device_args(vdev); 3049 3050 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 3051 3052 return cryptodev_openssl_create(name, vdev, &init_params); 3053 } 3054 3055 /** Uninitialise OPENSSL crypto device */ 3056 static int 3057 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 3058 { 3059 struct rte_cryptodev *cryptodev; 3060 const char *name; 3061 3062 name = rte_vdev_device_name(vdev); 3063 if (name == NULL) 3064 return -EINVAL; 3065 3066 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 3067 if (cryptodev == NULL) 3068 return -ENODEV; 3069 3070 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3071 ossl_legacy_provider_unload(); 3072 # endif 3073 return rte_cryptodev_pmd_destroy(cryptodev); 3074 } 3075 3076 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 3077 .probe = cryptodev_openssl_probe, 3078 .remove = cryptodev_openssl_remove 3079 }; 3080 3081 static struct cryptodev_driver openssl_crypto_drv; 3082 3083 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 3084 cryptodev_openssl_pmd_drv); 3085 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 3086 "max_nb_queue_pairs=<int> " 3087 "socket_id=<int>"); 3088 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 3089 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 3090 RTE_LOG_REGISTER_DEFAULT(openssl_logtype_driver, INFO); 3091