1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_common.h> 6 #include <rte_hexdump.h> 7 #include <rte_cryptodev.h> 8 #include <cryptodev_pmd.h> 9 #include <bus_vdev_driver.h> 10 #include <rte_malloc.h> 11 #include <rte_cpuflags.h> 12 13 #include <openssl/cmac.h> 14 #include <openssl/hmac.h> 15 #include <openssl/evp.h> 16 17 #include "openssl_pmd_private.h" 18 #include "compat.h" 19 20 #define DES_BLOCK_SIZE 8 21 22 static uint8_t cryptodev_driver_id; 23 24 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 25 static HMAC_CTX *HMAC_CTX_new(void) 26 { 27 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 28 29 if (ctx != NULL) 30 HMAC_CTX_init(ctx); 31 return ctx; 32 } 33 34 static void HMAC_CTX_free(HMAC_CTX *ctx) 35 { 36 if (ctx != NULL) { 37 HMAC_CTX_cleanup(ctx); 38 OPENSSL_free(ctx); 39 } 40 } 41 #endif 42 43 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 44 45 #include <openssl/provider.h> 46 #include <openssl/core_names.h> 47 #include <openssl/param_build.h> 48 49 #define MAX_OSSL_ALGO_NAME_SIZE 16 50 51 OSSL_PROVIDER *legacy; 52 OSSL_PROVIDER *deflt; 53 54 static void ossl_legacy_provider_load(void) 55 { 56 /* Load Multiple providers into the default (NULL) library context */ 57 legacy = OSSL_PROVIDER_load(NULL, "legacy"); 58 if (legacy == NULL) { 59 OPENSSL_LOG(ERR, "Failed to load Legacy provider\n"); 60 return; 61 } 62 63 deflt = OSSL_PROVIDER_load(NULL, "default"); 64 if (deflt == NULL) { 65 OPENSSL_LOG(ERR, "Failed to load Default provider\n"); 66 OSSL_PROVIDER_unload(legacy); 67 return; 68 } 69 } 70 71 static void ossl_legacy_provider_unload(void) 72 { 73 OSSL_PROVIDER_unload(legacy); 74 OSSL_PROVIDER_unload(deflt); 75 } 76 77 static __rte_always_inline const char * 78 digest_name_get(enum rte_crypto_auth_algorithm algo) 79 { 80 switch (algo) { 81 case RTE_CRYPTO_AUTH_MD5_HMAC: 82 return OSSL_DIGEST_NAME_MD5; 83 case RTE_CRYPTO_AUTH_SHA1_HMAC: 84 return OSSL_DIGEST_NAME_SHA1; 85 case RTE_CRYPTO_AUTH_SHA224_HMAC: 86 return OSSL_DIGEST_NAME_SHA2_224; 87 case RTE_CRYPTO_AUTH_SHA256_HMAC: 88 return OSSL_DIGEST_NAME_SHA2_256; 89 case RTE_CRYPTO_AUTH_SHA384_HMAC: 90 return OSSL_DIGEST_NAME_SHA2_384; 91 case RTE_CRYPTO_AUTH_SHA512_HMAC: 92 return OSSL_DIGEST_NAME_SHA2_512; 93 default: 94 return NULL; 95 } 96 } 97 #endif 98 99 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 100 101 /*----------------------------------------------------------------------------*/ 102 103 /** 104 * Increment counter by 1 105 * Counter is 64 bit array, big-endian 106 */ 107 static void 108 ctr_inc(uint8_t *ctr) 109 { 110 uint64_t *ctr64 = (uint64_t *)ctr; 111 112 *ctr64 = __builtin_bswap64(*ctr64); 113 (*ctr64)++; 114 *ctr64 = __builtin_bswap64(*ctr64); 115 } 116 117 /* 118 *------------------------------------------------------------------------------ 119 * Session Prepare 120 *------------------------------------------------------------------------------ 121 */ 122 123 /** Get xform chain order */ 124 static enum openssl_chain_order 125 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 126 { 127 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 128 129 if (xform != NULL) { 130 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 131 if (xform->next == NULL) 132 res = OPENSSL_CHAIN_ONLY_AUTH; 133 else if (xform->next->type == 134 RTE_CRYPTO_SYM_XFORM_CIPHER) 135 res = OPENSSL_CHAIN_AUTH_CIPHER; 136 } 137 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 138 if (xform->next == NULL) 139 res = OPENSSL_CHAIN_ONLY_CIPHER; 140 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 141 res = OPENSSL_CHAIN_CIPHER_AUTH; 142 } 143 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 144 res = OPENSSL_CHAIN_COMBINED; 145 } 146 147 return res; 148 } 149 150 /** Get session cipher key from input cipher key */ 151 static void 152 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key) 153 { 154 memcpy(session_key, input_key, keylen); 155 } 156 157 /** Get key ede 24 bytes standard from input key */ 158 static int 159 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede) 160 { 161 int res = 0; 162 163 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 164 switch (keylen) { 165 case 24: 166 memcpy(key_ede, key, 24); 167 break; 168 case 16: 169 /* K3 = K1 */ 170 memcpy(key_ede, key, 16); 171 memcpy(key_ede + 16, key, 8); 172 break; 173 case 8: 174 /* K1 = K2 = K3 (DES compatibility) */ 175 memcpy(key_ede, key, 8); 176 memcpy(key_ede + 8, key, 8); 177 memcpy(key_ede + 16, key, 8); 178 break; 179 default: 180 OPENSSL_LOG(ERR, "Unsupported key size"); 181 res = -EINVAL; 182 } 183 184 return res; 185 } 186 187 /** Get adequate openssl function for input cipher algorithm */ 188 static uint8_t 189 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 190 const EVP_CIPHER **algo) 191 { 192 int res = 0; 193 194 if (algo != NULL) { 195 switch (sess_algo) { 196 case RTE_CRYPTO_CIPHER_3DES_CBC: 197 switch (keylen) { 198 case 8: 199 *algo = EVP_des_cbc(); 200 break; 201 case 16: 202 *algo = EVP_des_ede_cbc(); 203 break; 204 case 24: 205 *algo = EVP_des_ede3_cbc(); 206 break; 207 default: 208 res = -EINVAL; 209 } 210 break; 211 case RTE_CRYPTO_CIPHER_3DES_CTR: 212 break; 213 case RTE_CRYPTO_CIPHER_AES_CBC: 214 switch (keylen) { 215 case 16: 216 *algo = EVP_aes_128_cbc(); 217 break; 218 case 24: 219 *algo = EVP_aes_192_cbc(); 220 break; 221 case 32: 222 *algo = EVP_aes_256_cbc(); 223 break; 224 default: 225 res = -EINVAL; 226 } 227 break; 228 case RTE_CRYPTO_CIPHER_AES_CTR: 229 switch (keylen) { 230 case 16: 231 *algo = EVP_aes_128_ctr(); 232 break; 233 case 24: 234 *algo = EVP_aes_192_ctr(); 235 break; 236 case 32: 237 *algo = EVP_aes_256_ctr(); 238 break; 239 default: 240 res = -EINVAL; 241 } 242 break; 243 default: 244 res = -EINVAL; 245 break; 246 } 247 } else { 248 res = -EINVAL; 249 } 250 251 return res; 252 } 253 254 /** Get adequate openssl function for input auth algorithm */ 255 static uint8_t 256 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 257 const EVP_MD **algo) 258 { 259 int res = 0; 260 261 if (algo != NULL) { 262 switch (sessalgo) { 263 case RTE_CRYPTO_AUTH_MD5: 264 case RTE_CRYPTO_AUTH_MD5_HMAC: 265 *algo = EVP_md5(); 266 break; 267 case RTE_CRYPTO_AUTH_SHA1: 268 case RTE_CRYPTO_AUTH_SHA1_HMAC: 269 *algo = EVP_sha1(); 270 break; 271 case RTE_CRYPTO_AUTH_SHA224: 272 case RTE_CRYPTO_AUTH_SHA224_HMAC: 273 *algo = EVP_sha224(); 274 break; 275 case RTE_CRYPTO_AUTH_SHA256: 276 case RTE_CRYPTO_AUTH_SHA256_HMAC: 277 *algo = EVP_sha256(); 278 break; 279 case RTE_CRYPTO_AUTH_SHA384: 280 case RTE_CRYPTO_AUTH_SHA384_HMAC: 281 *algo = EVP_sha384(); 282 break; 283 case RTE_CRYPTO_AUTH_SHA512: 284 case RTE_CRYPTO_AUTH_SHA512_HMAC: 285 *algo = EVP_sha512(); 286 break; 287 default: 288 res = -EINVAL; 289 break; 290 } 291 } else { 292 res = -EINVAL; 293 } 294 295 return res; 296 } 297 298 /** Get adequate openssl function for input cipher algorithm */ 299 static uint8_t 300 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 301 const EVP_CIPHER **algo) 302 { 303 int res = 0; 304 305 if (algo != NULL) { 306 switch (sess_algo) { 307 case RTE_CRYPTO_AEAD_AES_GCM: 308 switch (keylen) { 309 case 16: 310 *algo = EVP_aes_128_gcm(); 311 break; 312 case 24: 313 *algo = EVP_aes_192_gcm(); 314 break; 315 case 32: 316 *algo = EVP_aes_256_gcm(); 317 break; 318 default: 319 res = -EINVAL; 320 } 321 break; 322 case RTE_CRYPTO_AEAD_AES_CCM: 323 switch (keylen) { 324 case 16: 325 *algo = EVP_aes_128_ccm(); 326 break; 327 case 24: 328 *algo = EVP_aes_192_ccm(); 329 break; 330 case 32: 331 *algo = EVP_aes_256_ccm(); 332 break; 333 default: 334 res = -EINVAL; 335 } 336 break; 337 default: 338 res = -EINVAL; 339 break; 340 } 341 } else { 342 res = -EINVAL; 343 } 344 345 return res; 346 } 347 348 /* Set session AEAD encryption parameters */ 349 static int 350 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 351 enum rte_crypto_aead_algorithm algo, 352 uint8_t tag_len, const uint8_t *key) 353 { 354 int iv_type = 0; 355 unsigned int do_ccm; 356 357 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 358 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 359 360 /* Select AEAD algo */ 361 switch (algo) { 362 case RTE_CRYPTO_AEAD_AES_GCM: 363 iv_type = EVP_CTRL_GCM_SET_IVLEN; 364 if (tag_len != 16) 365 return -EINVAL; 366 do_ccm = 0; 367 break; 368 case RTE_CRYPTO_AEAD_AES_CCM: 369 iv_type = EVP_CTRL_CCM_SET_IVLEN; 370 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 371 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 372 return -EINVAL; 373 do_ccm = 1; 374 break; 375 default: 376 return -ENOTSUP; 377 } 378 379 sess->cipher.mode = OPENSSL_CIPHER_LIB; 380 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 381 382 if (get_aead_algo(algo, sess->cipher.key.length, 383 &sess->cipher.evp_algo) != 0) 384 return -EINVAL; 385 386 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 387 388 sess->chain_order = OPENSSL_CHAIN_COMBINED; 389 390 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 391 NULL, NULL, NULL) <= 0) 392 return -EINVAL; 393 394 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, 395 NULL) <= 0) 396 return -EINVAL; 397 398 if (do_ccm) 399 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 400 tag_len, NULL); 401 402 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 403 return -EINVAL; 404 405 return 0; 406 } 407 408 /* Set session AEAD decryption parameters */ 409 static int 410 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 411 enum rte_crypto_aead_algorithm algo, 412 uint8_t tag_len, const uint8_t *key) 413 { 414 int iv_type = 0; 415 unsigned int do_ccm = 0; 416 417 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 418 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 419 420 /* Select AEAD algo */ 421 switch (algo) { 422 case RTE_CRYPTO_AEAD_AES_GCM: 423 iv_type = EVP_CTRL_GCM_SET_IVLEN; 424 if (tag_len != 16) 425 return -EINVAL; 426 break; 427 case RTE_CRYPTO_AEAD_AES_CCM: 428 iv_type = EVP_CTRL_CCM_SET_IVLEN; 429 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 430 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 431 return -EINVAL; 432 do_ccm = 1; 433 break; 434 default: 435 return -ENOTSUP; 436 } 437 438 sess->cipher.mode = OPENSSL_CIPHER_LIB; 439 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 440 441 if (get_aead_algo(algo, sess->cipher.key.length, 442 &sess->cipher.evp_algo) != 0) 443 return -EINVAL; 444 445 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 446 447 sess->chain_order = OPENSSL_CHAIN_COMBINED; 448 449 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 450 NULL, NULL, NULL) <= 0) 451 return -EINVAL; 452 453 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, 454 sess->iv.length, NULL) <= 0) 455 return -EINVAL; 456 457 if (do_ccm) 458 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 459 tag_len, NULL); 460 461 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 462 return -EINVAL; 463 464 return 0; 465 } 466 467 /** Set session cipher parameters */ 468 static int 469 openssl_set_session_cipher_parameters(struct openssl_session *sess, 470 const struct rte_crypto_sym_xform *xform) 471 { 472 /* Select cipher direction */ 473 sess->cipher.direction = xform->cipher.op; 474 /* Select cipher key */ 475 sess->cipher.key.length = xform->cipher.key.length; 476 477 /* Set IV parameters */ 478 sess->iv.offset = xform->cipher.iv.offset; 479 sess->iv.length = xform->cipher.iv.length; 480 481 /* Select cipher algo */ 482 switch (xform->cipher.algo) { 483 case RTE_CRYPTO_CIPHER_3DES_CBC: 484 case RTE_CRYPTO_CIPHER_AES_CBC: 485 case RTE_CRYPTO_CIPHER_AES_CTR: 486 sess->cipher.mode = OPENSSL_CIPHER_LIB; 487 sess->cipher.algo = xform->cipher.algo; 488 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 489 490 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 491 &sess->cipher.evp_algo) != 0) 492 return -EINVAL; 493 494 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 495 sess->cipher.key.data); 496 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 497 if (EVP_EncryptInit_ex(sess->cipher.ctx, 498 sess->cipher.evp_algo, 499 NULL, xform->cipher.key.data, 500 NULL) != 1) { 501 return -EINVAL; 502 } 503 } else if (sess->cipher.direction == 504 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 505 if (EVP_DecryptInit_ex(sess->cipher.ctx, 506 sess->cipher.evp_algo, 507 NULL, xform->cipher.key.data, 508 NULL) != 1) { 509 return -EINVAL; 510 } 511 } 512 513 break; 514 515 case RTE_CRYPTO_CIPHER_3DES_CTR: 516 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 517 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 518 519 if (get_cipher_key_ede(xform->cipher.key.data, 520 sess->cipher.key.length, 521 sess->cipher.key.data) != 0) 522 return -EINVAL; 523 break; 524 525 case RTE_CRYPTO_CIPHER_DES_CBC: 526 sess->cipher.algo = xform->cipher.algo; 527 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 528 sess->cipher.evp_algo = EVP_des_cbc(); 529 530 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 531 sess->cipher.key.data); 532 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 533 if (EVP_EncryptInit_ex(sess->cipher.ctx, 534 sess->cipher.evp_algo, 535 NULL, xform->cipher.key.data, 536 NULL) != 1) { 537 return -EINVAL; 538 } 539 } else if (sess->cipher.direction == 540 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 541 if (EVP_DecryptInit_ex(sess->cipher.ctx, 542 sess->cipher.evp_algo, 543 NULL, xform->cipher.key.data, 544 NULL) != 1) { 545 return -EINVAL; 546 } 547 } 548 549 break; 550 551 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 552 sess->cipher.algo = xform->cipher.algo; 553 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 554 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 555 sess->cipher.evp_algo = EVP_des_cbc(); 556 557 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 558 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 559 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 560 NULL, xform->cipher.key.data, 0) != 1) 561 return -EINVAL; 562 563 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 564 sess->cipher.key.data); 565 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 566 if (EVP_EncryptInit_ex(sess->cipher.ctx, 567 sess->cipher.evp_algo, 568 NULL, xform->cipher.key.data, 569 NULL) != 1) { 570 return -EINVAL; 571 } 572 } else if (sess->cipher.direction == 573 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 574 if (EVP_DecryptInit_ex(sess->cipher.ctx, 575 sess->cipher.evp_algo, 576 NULL, xform->cipher.key.data, 577 NULL) != 1) { 578 return -EINVAL; 579 } 580 } 581 582 break; 583 default: 584 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 585 return -ENOTSUP; 586 } 587 588 return 0; 589 } 590 591 /* Set session auth parameters */ 592 static int 593 openssl_set_session_auth_parameters(struct openssl_session *sess, 594 const struct rte_crypto_sym_xform *xform) 595 { 596 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 597 char algo_name[MAX_OSSL_ALGO_NAME_SIZE]; 598 OSSL_PARAM params[2]; 599 const char *algo; 600 EVP_MAC *mac; 601 # endif 602 /* Select auth generate/verify */ 603 sess->auth.operation = xform->auth.op; 604 sess->auth.algo = xform->auth.algo; 605 606 sess->auth.digest_length = xform->auth.digest_length; 607 608 /* Select auth algo */ 609 switch (xform->auth.algo) { 610 case RTE_CRYPTO_AUTH_AES_GMAC: 611 /* 612 * OpenSSL requires GMAC to be a GCM operation 613 * with no cipher data length 614 */ 615 sess->cipher.key.length = xform->auth.key.length; 616 617 /* Set IV parameters */ 618 sess->iv.offset = xform->auth.iv.offset; 619 sess->iv.length = xform->auth.iv.length; 620 621 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 622 return openssl_set_sess_aead_enc_param(sess, 623 RTE_CRYPTO_AEAD_AES_GCM, 624 xform->auth.digest_length, 625 xform->auth.key.data); 626 else 627 return openssl_set_sess_aead_dec_param(sess, 628 RTE_CRYPTO_AEAD_AES_GCM, 629 xform->auth.digest_length, 630 xform->auth.key.data); 631 break; 632 633 case RTE_CRYPTO_AUTH_MD5: 634 case RTE_CRYPTO_AUTH_SHA1: 635 case RTE_CRYPTO_AUTH_SHA224: 636 case RTE_CRYPTO_AUTH_SHA256: 637 case RTE_CRYPTO_AUTH_SHA384: 638 case RTE_CRYPTO_AUTH_SHA512: 639 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 640 if (get_auth_algo(xform->auth.algo, 641 &sess->auth.auth.evp_algo) != 0) 642 return -EINVAL; 643 sess->auth.auth.ctx = EVP_MD_CTX_create(); 644 break; 645 646 case RTE_CRYPTO_AUTH_AES_CMAC: 647 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 648 if (xform->auth.key.length == 16) 649 algo = SN_aes_128_cbc; 650 else if (xform->auth.key.length == 24) 651 algo = SN_aes_192_cbc; 652 else if (xform->auth.key.length == 32) 653 algo = SN_aes_256_cbc; 654 else 655 return -EINVAL; 656 657 rte_memcpy(algo_name, algo, strlen(algo) + 1); 658 params[0] = OSSL_PARAM_construct_utf8_string( 659 OSSL_MAC_PARAM_CIPHER, algo_name, 0); 660 params[1] = OSSL_PARAM_construct_end(); 661 662 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 663 mac = EVP_MAC_fetch(NULL, OSSL_MAC_NAME_CMAC, NULL); 664 sess->auth.cmac.ctx = EVP_MAC_CTX_new(mac); 665 EVP_MAC_free(mac); 666 667 if (EVP_MAC_init(sess->auth.cmac.ctx, 668 xform->auth.key.data, 669 xform->auth.key.length, 670 params) != 1) 671 return -EINVAL; 672 # else 673 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 674 sess->auth.cmac.ctx = CMAC_CTX_new(); 675 if (get_cipher_algo(RTE_CRYPTO_CIPHER_AES_CBC, 676 xform->auth.key.length, 677 &sess->auth.cmac.evp_algo) != 0) 678 return -EINVAL; 679 if (CMAC_Init(sess->auth.cmac.ctx, 680 xform->auth.key.data, 681 xform->auth.key.length, 682 sess->auth.cmac.evp_algo, NULL) != 1) 683 return -EINVAL; 684 # endif 685 break; 686 687 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 688 case RTE_CRYPTO_AUTH_MD5_HMAC: 689 case RTE_CRYPTO_AUTH_SHA1_HMAC: 690 case RTE_CRYPTO_AUTH_SHA224_HMAC: 691 case RTE_CRYPTO_AUTH_SHA256_HMAC: 692 case RTE_CRYPTO_AUTH_SHA384_HMAC: 693 case RTE_CRYPTO_AUTH_SHA512_HMAC: 694 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 695 696 algo = digest_name_get(xform->auth.algo); 697 if (!algo) 698 return -EINVAL; 699 strlcpy(algo_name, algo, sizeof(algo_name)); 700 701 mac = EVP_MAC_fetch(NULL, "HMAC", NULL); 702 sess->auth.hmac.ctx = EVP_MAC_CTX_new(mac); 703 EVP_MAC_free(mac); 704 if (get_auth_algo(xform->auth.algo, 705 &sess->auth.hmac.evp_algo) != 0) 706 return -EINVAL; 707 708 params[0] = OSSL_PARAM_construct_utf8_string("digest", 709 algo_name, 0); 710 params[1] = OSSL_PARAM_construct_end(); 711 if (EVP_MAC_init(sess->auth.hmac.ctx, 712 xform->auth.key.data, 713 xform->auth.key.length, 714 params) != 1) 715 return -EINVAL; 716 break; 717 # else 718 case RTE_CRYPTO_AUTH_MD5_HMAC: 719 case RTE_CRYPTO_AUTH_SHA1_HMAC: 720 case RTE_CRYPTO_AUTH_SHA224_HMAC: 721 case RTE_CRYPTO_AUTH_SHA256_HMAC: 722 case RTE_CRYPTO_AUTH_SHA384_HMAC: 723 case RTE_CRYPTO_AUTH_SHA512_HMAC: 724 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 725 sess->auth.hmac.ctx = HMAC_CTX_new(); 726 if (get_auth_algo(xform->auth.algo, 727 &sess->auth.hmac.evp_algo) != 0) 728 return -EINVAL; 729 730 if (HMAC_Init_ex(sess->auth.hmac.ctx, 731 xform->auth.key.data, 732 xform->auth.key.length, 733 sess->auth.hmac.evp_algo, NULL) != 1) 734 return -EINVAL; 735 break; 736 # endif 737 default: 738 return -ENOTSUP; 739 } 740 741 return 0; 742 } 743 744 /* Set session AEAD parameters */ 745 static int 746 openssl_set_session_aead_parameters(struct openssl_session *sess, 747 const struct rte_crypto_sym_xform *xform) 748 { 749 /* Select cipher key */ 750 sess->cipher.key.length = xform->aead.key.length; 751 752 /* Set IV parameters */ 753 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 754 /* 755 * For AES-CCM, the actual IV is placed 756 * one byte after the start of the IV field, 757 * according to the API. 758 */ 759 sess->iv.offset = xform->aead.iv.offset + 1; 760 else 761 sess->iv.offset = xform->aead.iv.offset; 762 763 sess->iv.length = xform->aead.iv.length; 764 765 sess->auth.aad_length = xform->aead.aad_length; 766 sess->auth.digest_length = xform->aead.digest_length; 767 768 sess->aead_algo = xform->aead.algo; 769 /* Select cipher direction */ 770 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 771 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 772 xform->aead.digest_length, xform->aead.key.data); 773 else 774 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 775 xform->aead.digest_length, xform->aead.key.data); 776 } 777 778 /** Parse crypto xform chain and set private session parameters */ 779 int 780 openssl_set_session_parameters(struct openssl_session *sess, 781 const struct rte_crypto_sym_xform *xform) 782 { 783 const struct rte_crypto_sym_xform *cipher_xform = NULL; 784 const struct rte_crypto_sym_xform *auth_xform = NULL; 785 const struct rte_crypto_sym_xform *aead_xform = NULL; 786 int ret; 787 788 sess->chain_order = openssl_get_chain_order(xform); 789 switch (sess->chain_order) { 790 case OPENSSL_CHAIN_ONLY_CIPHER: 791 cipher_xform = xform; 792 break; 793 case OPENSSL_CHAIN_ONLY_AUTH: 794 auth_xform = xform; 795 break; 796 case OPENSSL_CHAIN_CIPHER_AUTH: 797 cipher_xform = xform; 798 auth_xform = xform->next; 799 break; 800 case OPENSSL_CHAIN_AUTH_CIPHER: 801 auth_xform = xform; 802 cipher_xform = xform->next; 803 break; 804 case OPENSSL_CHAIN_COMBINED: 805 aead_xform = xform; 806 break; 807 default: 808 return -EINVAL; 809 } 810 811 /* Default IV length = 0 */ 812 sess->iv.length = 0; 813 814 /* cipher_xform must be check before auth_xform */ 815 if (cipher_xform) { 816 ret = openssl_set_session_cipher_parameters( 817 sess, cipher_xform); 818 if (ret != 0) { 819 OPENSSL_LOG(ERR, 820 "Invalid/unsupported cipher parameters"); 821 return ret; 822 } 823 } 824 825 if (auth_xform) { 826 ret = openssl_set_session_auth_parameters(sess, auth_xform); 827 if (ret != 0) { 828 OPENSSL_LOG(ERR, 829 "Invalid/unsupported auth parameters"); 830 return ret; 831 } 832 } 833 834 if (aead_xform) { 835 ret = openssl_set_session_aead_parameters(sess, aead_xform); 836 if (ret != 0) { 837 OPENSSL_LOG(ERR, 838 "Invalid/unsupported AEAD parameters"); 839 return ret; 840 } 841 } 842 843 return 0; 844 } 845 846 /** Reset private session parameters */ 847 void 848 openssl_reset_session(struct openssl_session *sess) 849 { 850 EVP_CIPHER_CTX_free(sess->cipher.ctx); 851 852 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 853 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 854 855 switch (sess->auth.mode) { 856 case OPENSSL_AUTH_AS_AUTH: 857 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 858 break; 859 case OPENSSL_AUTH_AS_HMAC: 860 EVP_PKEY_free(sess->auth.hmac.pkey); 861 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 862 EVP_MAC_CTX_free(sess->auth.hmac.ctx); 863 # else 864 HMAC_CTX_free(sess->auth.hmac.ctx); 865 # endif 866 break; 867 case OPENSSL_AUTH_AS_CMAC: 868 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 869 EVP_MAC_CTX_free(sess->auth.cmac.ctx); 870 # else 871 CMAC_CTX_free(sess->auth.cmac.ctx); 872 # endif 873 break; 874 default: 875 break; 876 } 877 } 878 879 /** Provide session for operation */ 880 static void * 881 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 882 { 883 struct openssl_session *sess = NULL; 884 struct openssl_asym_session *asym_sess = NULL; 885 886 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 887 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 888 /* get existing session */ 889 if (likely(op->sym->session != NULL)) 890 sess = CRYPTODEV_GET_SYM_SESS_PRIV( 891 op->sym->session); 892 } else { 893 if (likely(op->asym->session != NULL)) 894 asym_sess = (struct openssl_asym_session *) 895 op->asym->session->sess_private_data; 896 if (asym_sess == NULL) 897 op->status = 898 RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 899 return asym_sess; 900 } 901 } else { 902 struct rte_cryptodev_sym_session *_sess; 903 /* sessionless asymmetric not supported */ 904 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) 905 return NULL; 906 907 /* provide internal session */ 908 rte_mempool_get(qp->sess_mp, (void **)&_sess); 909 910 if (_sess == NULL) 911 return NULL; 912 913 sess = (struct openssl_session *)_sess->driver_priv_data; 914 915 if (unlikely(openssl_set_session_parameters(sess, 916 op->sym->xform) != 0)) { 917 rte_mempool_put(qp->sess_mp, _sess); 918 sess = NULL; 919 } 920 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 921 922 } 923 924 if (sess == NULL) 925 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 926 927 return sess; 928 } 929 930 /* 931 *------------------------------------------------------------------------------ 932 * Process Operations 933 *------------------------------------------------------------------------------ 934 */ 935 static inline int 936 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 937 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 938 { 939 struct rte_mbuf *m; 940 int dstlen; 941 int l, n = srclen; 942 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 943 944 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 945 m = m->next) 946 offset -= rte_pktmbuf_data_len(m); 947 948 if (m == 0) 949 return -1; 950 951 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 952 if (inplace) 953 *dst = src; 954 955 l = rte_pktmbuf_data_len(m) - offset; 956 if (srclen <= l) { 957 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 958 return -1; 959 *dst += l; 960 return 0; 961 } 962 963 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 964 return -1; 965 966 *dst += dstlen; 967 n -= l; 968 969 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 970 uint8_t diff = l - dstlen, rem; 971 972 src = rte_pktmbuf_mtod(m, uint8_t *); 973 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 974 if (diff && inplace) { 975 rem = RTE_MIN(l, 976 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 977 if (EVP_EncryptUpdate(ctx, temp, 978 &dstlen, src, rem) <= 0) 979 return -1; 980 n -= rem; 981 rte_memcpy(*dst, temp, diff); 982 rte_memcpy(src, temp + diff, rem); 983 src += rem; 984 l -= rem; 985 } 986 if (inplace) 987 *dst = src; 988 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 989 return -1; 990 *dst += dstlen; 991 n -= l; 992 } 993 994 return 0; 995 } 996 997 static inline int 998 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 999 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 1000 { 1001 struct rte_mbuf *m; 1002 int dstlen; 1003 int l, n = srclen; 1004 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 1005 1006 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1007 m = m->next) 1008 offset -= rte_pktmbuf_data_len(m); 1009 1010 if (m == 0) 1011 return -1; 1012 1013 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1014 if (inplace) 1015 *dst = src; 1016 1017 l = rte_pktmbuf_data_len(m) - offset; 1018 if (srclen <= l) { 1019 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 1020 return -1; 1021 *dst += l; 1022 return 0; 1023 } 1024 1025 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1026 return -1; 1027 1028 *dst += dstlen; 1029 n -= l; 1030 1031 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1032 uint8_t diff = l - dstlen, rem; 1033 1034 src = rte_pktmbuf_mtod(m, uint8_t *); 1035 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 1036 if (diff && inplace) { 1037 rem = RTE_MIN(l, 1038 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 1039 if (EVP_DecryptUpdate(ctx, temp, 1040 &dstlen, src, rem) <= 0) 1041 return -1; 1042 n -= rem; 1043 rte_memcpy(*dst, temp, diff); 1044 rte_memcpy(src, temp + diff, rem); 1045 src += rem; 1046 l -= rem; 1047 } 1048 if (inplace) 1049 *dst = src; 1050 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1051 return -1; 1052 *dst += dstlen; 1053 n -= l; 1054 } 1055 1056 return 0; 1057 } 1058 1059 /** Process standard openssl cipher encryption */ 1060 static int 1061 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1062 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1063 uint8_t inplace) 1064 { 1065 int totlen; 1066 1067 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1068 goto process_cipher_encrypt_err; 1069 1070 EVP_CIPHER_CTX_set_padding(ctx, 0); 1071 1072 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1073 srclen, ctx, inplace)) 1074 goto process_cipher_encrypt_err; 1075 1076 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 1077 goto process_cipher_encrypt_err; 1078 1079 return 0; 1080 1081 process_cipher_encrypt_err: 1082 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 1083 return -EINVAL; 1084 } 1085 1086 /** Process standard openssl cipher encryption */ 1087 static int 1088 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 1089 uint8_t *iv, int srclen, 1090 EVP_CIPHER_CTX *ctx) 1091 { 1092 uint8_t i; 1093 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 1094 int encrypted_ivlen; 1095 1096 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 1097 iv, DES_BLOCK_SIZE) <= 0) 1098 goto process_cipher_encrypt_err; 1099 1100 for (i = 0; i < srclen; i++) 1101 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 1102 1103 return 0; 1104 1105 process_cipher_encrypt_err: 1106 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 1107 return -EINVAL; 1108 } 1109 /** Process standard openssl cipher decryption */ 1110 static int 1111 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1112 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1113 uint8_t inplace) 1114 { 1115 int totlen; 1116 1117 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1118 goto process_cipher_decrypt_err; 1119 1120 EVP_CIPHER_CTX_set_padding(ctx, 0); 1121 1122 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1123 srclen, ctx, inplace)) 1124 goto process_cipher_decrypt_err; 1125 1126 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 1127 goto process_cipher_decrypt_err; 1128 return 0; 1129 1130 process_cipher_decrypt_err: 1131 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 1132 return -EINVAL; 1133 } 1134 1135 /** Process cipher des 3 ctr encryption, decryption algorithm */ 1136 static int 1137 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 1138 int offset, uint8_t *iv, uint8_t *key, int srclen, 1139 EVP_CIPHER_CTX *ctx) 1140 { 1141 uint8_t ebuf[8], ctr[8]; 1142 int unused, n; 1143 struct rte_mbuf *m; 1144 uint8_t *src; 1145 int l; 1146 1147 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1148 m = m->next) 1149 offset -= rte_pktmbuf_data_len(m); 1150 1151 if (m == 0) 1152 goto process_cipher_des3ctr_err; 1153 1154 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1155 l = rte_pktmbuf_data_len(m) - offset; 1156 1157 /* We use 3DES encryption also for decryption. 1158 * IV is not important for 3DES ecb 1159 */ 1160 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 1161 goto process_cipher_des3ctr_err; 1162 1163 memcpy(ctr, iv, 8); 1164 1165 for (n = 0; n < srclen; n++) { 1166 if (n % 8 == 0) { 1167 if (EVP_EncryptUpdate(ctx, 1168 (unsigned char *)&ebuf, &unused, 1169 (const unsigned char *)&ctr, 8) <= 0) 1170 goto process_cipher_des3ctr_err; 1171 ctr_inc(ctr); 1172 } 1173 dst[n] = *(src++) ^ ebuf[n % 8]; 1174 1175 l--; 1176 if (!l) { 1177 m = m->next; 1178 if (m) { 1179 src = rte_pktmbuf_mtod(m, uint8_t *); 1180 l = rte_pktmbuf_data_len(m); 1181 } 1182 } 1183 } 1184 1185 return 0; 1186 1187 process_cipher_des3ctr_err: 1188 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 1189 return -EINVAL; 1190 } 1191 1192 /** Process AES-GCM encrypt algorithm */ 1193 static int 1194 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1195 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1196 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1197 { 1198 int len = 0, unused = 0; 1199 uint8_t empty[] = {}; 1200 1201 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1202 goto process_auth_encryption_gcm_err; 1203 1204 if (aadlen > 0) 1205 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1206 goto process_auth_encryption_gcm_err; 1207 1208 if (srclen > 0) 1209 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1210 srclen, ctx, 0)) 1211 goto process_auth_encryption_gcm_err; 1212 1213 /* Workaround open ssl bug in version less then 1.0.1f */ 1214 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1215 goto process_auth_encryption_gcm_err; 1216 1217 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1218 goto process_auth_encryption_gcm_err; 1219 1220 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1221 goto process_auth_encryption_gcm_err; 1222 1223 return 0; 1224 1225 process_auth_encryption_gcm_err: 1226 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1227 return -EINVAL; 1228 } 1229 1230 /** Process AES-CCM encrypt algorithm */ 1231 static int 1232 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1233 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1234 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1235 { 1236 int len = 0; 1237 1238 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1239 goto process_auth_encryption_ccm_err; 1240 1241 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1242 goto process_auth_encryption_ccm_err; 1243 1244 if (aadlen > 0) 1245 /* 1246 * For AES-CCM, the actual AAD is placed 1247 * 18 bytes after the start of the AAD field, 1248 * according to the API. 1249 */ 1250 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1251 goto process_auth_encryption_ccm_err; 1252 1253 if (srclen >= 0) 1254 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1255 srclen, ctx, 0)) 1256 goto process_auth_encryption_ccm_err; 1257 1258 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1259 goto process_auth_encryption_ccm_err; 1260 1261 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1262 goto process_auth_encryption_ccm_err; 1263 1264 return 0; 1265 1266 process_auth_encryption_ccm_err: 1267 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1268 return -EINVAL; 1269 } 1270 1271 /** Process AES-GCM decrypt algorithm */ 1272 static int 1273 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1274 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1275 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1276 { 1277 int len = 0, unused = 0; 1278 uint8_t empty[] = {}; 1279 1280 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1281 goto process_auth_decryption_gcm_err; 1282 1283 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1284 goto process_auth_decryption_gcm_err; 1285 1286 if (aadlen > 0) 1287 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1288 goto process_auth_decryption_gcm_err; 1289 1290 if (srclen > 0) 1291 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1292 srclen, ctx, 0)) 1293 goto process_auth_decryption_gcm_err; 1294 1295 /* Workaround open ssl bug in version less then 1.0.1f */ 1296 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1297 goto process_auth_decryption_gcm_err; 1298 1299 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1300 return -EFAULT; 1301 1302 return 0; 1303 1304 process_auth_decryption_gcm_err: 1305 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1306 return -EINVAL; 1307 } 1308 1309 /** Process AES-CCM decrypt algorithm */ 1310 static int 1311 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1312 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1313 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1314 EVP_CIPHER_CTX *ctx) 1315 { 1316 int len = 0; 1317 1318 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1319 goto process_auth_decryption_ccm_err; 1320 1321 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1322 goto process_auth_decryption_ccm_err; 1323 1324 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1325 goto process_auth_decryption_ccm_err; 1326 1327 if (aadlen > 0) 1328 /* 1329 * For AES-CCM, the actual AAD is placed 1330 * 18 bytes after the start of the AAD field, 1331 * according to the API. 1332 */ 1333 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1334 goto process_auth_decryption_ccm_err; 1335 1336 if (srclen >= 0) 1337 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1338 srclen, ctx, 0)) 1339 return -EFAULT; 1340 1341 return 0; 1342 1343 process_auth_decryption_ccm_err: 1344 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1345 return -EINVAL; 1346 } 1347 1348 /** Process standard openssl auth algorithms */ 1349 static int 1350 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1351 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1352 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1353 { 1354 size_t dstlen; 1355 struct rte_mbuf *m; 1356 int l, n = srclen; 1357 uint8_t *src; 1358 1359 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1360 m = m->next) 1361 offset -= rte_pktmbuf_data_len(m); 1362 1363 if (m == 0) 1364 goto process_auth_err; 1365 1366 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1367 goto process_auth_err; 1368 1369 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1370 1371 l = rte_pktmbuf_data_len(m) - offset; 1372 if (srclen <= l) { 1373 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1374 goto process_auth_err; 1375 goto process_auth_final; 1376 } 1377 1378 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1379 goto process_auth_err; 1380 1381 n -= l; 1382 1383 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1384 src = rte_pktmbuf_mtod(m, uint8_t *); 1385 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1386 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1387 goto process_auth_err; 1388 n -= l; 1389 } 1390 1391 process_auth_final: 1392 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1393 goto process_auth_err; 1394 return 0; 1395 1396 process_auth_err: 1397 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1398 return -EINVAL; 1399 } 1400 1401 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1402 /** Process standard openssl auth algorithms with hmac/cmac */ 1403 static int 1404 process_openssl_auth_mac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1405 int srclen, EVP_MAC_CTX *ctx) 1406 { 1407 size_t dstlen; 1408 struct rte_mbuf *m; 1409 int l, n = srclen; 1410 uint8_t *src; 1411 1412 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1413 m = m->next) 1414 offset -= rte_pktmbuf_data_len(m); 1415 1416 if (m == 0) 1417 goto process_auth_err; 1418 1419 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1420 1421 l = rte_pktmbuf_data_len(m) - offset; 1422 if (srclen <= l) { 1423 if (EVP_MAC_update(ctx, (unsigned char *)src, srclen) != 1) 1424 goto process_auth_err; 1425 goto process_auth_final; 1426 } 1427 1428 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1429 goto process_auth_err; 1430 1431 n -= l; 1432 1433 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1434 src = rte_pktmbuf_mtod(m, uint8_t *); 1435 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1436 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1437 goto process_auth_err; 1438 n -= l; 1439 } 1440 1441 process_auth_final: 1442 if (EVP_MAC_final(ctx, dst, &dstlen, DIGEST_LENGTH_MAX) != 1) 1443 goto process_auth_err; 1444 1445 EVP_MAC_CTX_free(ctx); 1446 return 0; 1447 1448 process_auth_err: 1449 EVP_MAC_CTX_free(ctx); 1450 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1451 return -EINVAL; 1452 } 1453 # else 1454 /** Process standard openssl auth algorithms with hmac */ 1455 static int 1456 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1457 int srclen, HMAC_CTX *ctx) 1458 { 1459 unsigned int dstlen; 1460 struct rte_mbuf *m; 1461 int l, n = srclen; 1462 uint8_t *src; 1463 1464 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1465 m = m->next) 1466 offset -= rte_pktmbuf_data_len(m); 1467 1468 if (m == 0) 1469 goto process_auth_err; 1470 1471 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1472 1473 l = rte_pktmbuf_data_len(m) - offset; 1474 if (srclen <= l) { 1475 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1476 goto process_auth_err; 1477 goto process_auth_final; 1478 } 1479 1480 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1481 goto process_auth_err; 1482 1483 n -= l; 1484 1485 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1486 src = rte_pktmbuf_mtod(m, uint8_t *); 1487 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1488 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1489 goto process_auth_err; 1490 n -= l; 1491 } 1492 1493 process_auth_final: 1494 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1495 goto process_auth_err; 1496 1497 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1498 goto process_auth_err; 1499 1500 return 0; 1501 1502 process_auth_err: 1503 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1504 return -EINVAL; 1505 } 1506 1507 /** Process standard openssl auth algorithms with cmac */ 1508 static int 1509 process_openssl_auth_cmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1510 int srclen, CMAC_CTX *ctx) 1511 { 1512 unsigned int dstlen; 1513 struct rte_mbuf *m; 1514 int l, n = srclen; 1515 uint8_t *src; 1516 1517 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1518 m = m->next) 1519 offset -= rte_pktmbuf_data_len(m); 1520 1521 if (m == 0) 1522 goto process_auth_err; 1523 1524 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1525 1526 l = rte_pktmbuf_data_len(m) - offset; 1527 if (srclen <= l) { 1528 if (CMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1529 goto process_auth_err; 1530 goto process_auth_final; 1531 } 1532 1533 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1534 goto process_auth_err; 1535 1536 n -= l; 1537 1538 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1539 src = rte_pktmbuf_mtod(m, uint8_t *); 1540 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1541 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1542 goto process_auth_err; 1543 n -= l; 1544 } 1545 1546 process_auth_final: 1547 if (CMAC_Final(ctx, dst, (size_t *)&dstlen) != 1) 1548 goto process_auth_err; 1549 1550 CMAC_CTX_cleanup(ctx); 1551 1552 return 0; 1553 1554 process_auth_err: 1555 OPENSSL_LOG(ERR, "Process openssl cmac auth failed"); 1556 return -EINVAL; 1557 } 1558 # endif 1559 /*----------------------------------------------------------------------------*/ 1560 1561 /** Process auth/cipher combined operation */ 1562 static void 1563 process_openssl_combined_op 1564 (struct rte_crypto_op *op, struct openssl_session *sess, 1565 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1566 { 1567 /* cipher */ 1568 uint8_t *dst = NULL, *iv, *tag, *aad; 1569 int srclen, aadlen, status = -1; 1570 uint32_t offset; 1571 uint8_t taglen; 1572 1573 /* 1574 * Segmented destination buffer is not supported for 1575 * encryption/decryption 1576 */ 1577 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1578 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1579 return; 1580 } 1581 1582 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1583 sess->iv.offset); 1584 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1585 srclen = 0; 1586 offset = op->sym->auth.data.offset; 1587 aadlen = op->sym->auth.data.length; 1588 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1589 op->sym->auth.data.offset); 1590 tag = op->sym->auth.digest.data; 1591 if (tag == NULL) 1592 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1593 offset + aadlen); 1594 } else { 1595 srclen = op->sym->aead.data.length; 1596 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1597 op->sym->aead.data.offset); 1598 offset = op->sym->aead.data.offset; 1599 aad = op->sym->aead.aad.data; 1600 aadlen = sess->auth.aad_length; 1601 tag = op->sym->aead.digest.data; 1602 if (tag == NULL) 1603 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1604 offset + srclen); 1605 } 1606 1607 taglen = sess->auth.digest_length; 1608 1609 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1610 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1611 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1612 status = process_openssl_auth_encryption_gcm( 1613 mbuf_src, offset, srclen, 1614 aad, aadlen, iv, 1615 dst, tag, sess->cipher.ctx); 1616 else 1617 status = process_openssl_auth_encryption_ccm( 1618 mbuf_src, offset, srclen, 1619 aad, aadlen, iv, 1620 dst, tag, taglen, sess->cipher.ctx); 1621 1622 } else { 1623 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1624 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1625 status = process_openssl_auth_decryption_gcm( 1626 mbuf_src, offset, srclen, 1627 aad, aadlen, iv, 1628 dst, tag, sess->cipher.ctx); 1629 else 1630 status = process_openssl_auth_decryption_ccm( 1631 mbuf_src, offset, srclen, 1632 aad, aadlen, iv, 1633 dst, tag, taglen, sess->cipher.ctx); 1634 } 1635 1636 if (status != 0) { 1637 if (status == (-EFAULT) && 1638 sess->auth.operation == 1639 RTE_CRYPTO_AUTH_OP_VERIFY) 1640 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1641 else 1642 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1643 } 1644 } 1645 1646 /** Process cipher operation */ 1647 static void 1648 process_openssl_cipher_op 1649 (struct rte_crypto_op *op, struct openssl_session *sess, 1650 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1651 { 1652 uint8_t *dst, *iv; 1653 int srclen, status; 1654 uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0; 1655 EVP_CIPHER_CTX *ctx_copy; 1656 1657 /* 1658 * Segmented OOP destination buffer is not supported for encryption/ 1659 * decryption. In case of des3ctr, even inplace segmented buffers are 1660 * not supported. 1661 */ 1662 if (!rte_pktmbuf_is_contiguous(mbuf_dst) && 1663 (!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) { 1664 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1665 return; 1666 } 1667 1668 srclen = op->sym->cipher.data.length; 1669 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1670 op->sym->cipher.data.offset); 1671 1672 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1673 sess->iv.offset); 1674 ctx_copy = EVP_CIPHER_CTX_new(); 1675 EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx); 1676 1677 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1678 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1679 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1680 op->sym->cipher.data.offset, iv, 1681 srclen, ctx_copy, inplace); 1682 else 1683 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1684 op->sym->cipher.data.offset, iv, 1685 srclen, ctx_copy, inplace); 1686 else 1687 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1688 op->sym->cipher.data.offset, iv, 1689 sess->cipher.key.data, srclen, 1690 ctx_copy); 1691 1692 EVP_CIPHER_CTX_free(ctx_copy); 1693 if (status != 0) 1694 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1695 } 1696 1697 /** Process cipher operation */ 1698 static void 1699 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1700 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1701 struct rte_mbuf *mbuf_dst) 1702 { 1703 uint8_t *src, *dst, *iv; 1704 uint8_t block_size, last_block_len; 1705 int srclen, status = 0; 1706 1707 srclen = op->sym->cipher.data.length; 1708 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1709 op->sym->cipher.data.offset); 1710 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1711 op->sym->cipher.data.offset); 1712 1713 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1714 sess->iv.offset); 1715 1716 block_size = DES_BLOCK_SIZE; 1717 1718 last_block_len = srclen % block_size; 1719 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1720 /* Encrypt only with ECB mode XOR IV */ 1721 if (srclen < block_size) { 1722 status = process_openssl_cipher_bpi_encrypt(src, dst, 1723 iv, srclen, 1724 sess->cipher.bpi_ctx); 1725 } else { 1726 srclen -= last_block_len; 1727 /* Encrypt with the block aligned stream with CBC mode */ 1728 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1729 op->sym->cipher.data.offset, iv, 1730 srclen, sess->cipher.ctx, 0); 1731 if (last_block_len) { 1732 /* Point at last block */ 1733 dst += srclen; 1734 /* 1735 * IV is the last encrypted block from 1736 * the previous operation 1737 */ 1738 iv = dst - block_size; 1739 src += srclen; 1740 srclen = last_block_len; 1741 /* Encrypt the last frame with ECB mode */ 1742 status |= process_openssl_cipher_bpi_encrypt(src, 1743 dst, iv, 1744 srclen, sess->cipher.bpi_ctx); 1745 } 1746 } 1747 } else { 1748 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1749 if (srclen < block_size) { 1750 status = process_openssl_cipher_bpi_encrypt(src, dst, 1751 iv, 1752 srclen, 1753 sess->cipher.bpi_ctx); 1754 } else { 1755 if (last_block_len) { 1756 /* Point at last block */ 1757 dst += srclen - last_block_len; 1758 src += srclen - last_block_len; 1759 /* 1760 * IV is the last full block 1761 */ 1762 iv = src - block_size; 1763 /* 1764 * Decrypt the last frame with ECB mode 1765 * (encrypt, as it is the same operation) 1766 */ 1767 status = process_openssl_cipher_bpi_encrypt(src, 1768 dst, iv, 1769 last_block_len, sess->cipher.bpi_ctx); 1770 /* Prepare parameters for CBC mode op */ 1771 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1772 sess->iv.offset); 1773 dst += last_block_len - srclen; 1774 srclen -= last_block_len; 1775 } 1776 1777 /* Decrypt with CBC mode */ 1778 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1779 op->sym->cipher.data.offset, iv, 1780 srclen, sess->cipher.ctx, 0); 1781 } 1782 } 1783 1784 if (status != 0) 1785 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1786 } 1787 1788 /** Process auth operation */ 1789 static void 1790 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1791 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1792 struct rte_mbuf *mbuf_dst) 1793 { 1794 uint8_t *dst; 1795 int srclen, status; 1796 EVP_MD_CTX *ctx_a; 1797 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1798 EVP_MAC_CTX *ctx_h; 1799 EVP_MAC_CTX *ctx_c; 1800 EVP_MAC *mac; 1801 # else 1802 HMAC_CTX *ctx_h; 1803 CMAC_CTX *ctx_c; 1804 # endif 1805 1806 srclen = op->sym->auth.data.length; 1807 1808 dst = qp->temp_digest; 1809 1810 switch (sess->auth.mode) { 1811 case OPENSSL_AUTH_AS_AUTH: 1812 ctx_a = EVP_MD_CTX_create(); 1813 EVP_MD_CTX_copy_ex(ctx_a, sess->auth.auth.ctx); 1814 status = process_openssl_auth(mbuf_src, dst, 1815 op->sym->auth.data.offset, NULL, NULL, srclen, 1816 ctx_a, sess->auth.auth.evp_algo); 1817 EVP_MD_CTX_destroy(ctx_a); 1818 break; 1819 case OPENSSL_AUTH_AS_HMAC: 1820 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1821 mac = EVP_MAC_fetch(NULL, "HMAC", NULL); 1822 ctx_h = EVP_MAC_CTX_new(mac); 1823 ctx_h = EVP_MAC_CTX_dup(sess->auth.hmac.ctx); 1824 EVP_MAC_free(mac); 1825 status = process_openssl_auth_mac(mbuf_src, dst, 1826 op->sym->auth.data.offset, srclen, 1827 ctx_h); 1828 # else 1829 ctx_h = HMAC_CTX_new(); 1830 HMAC_CTX_copy(ctx_h, sess->auth.hmac.ctx); 1831 status = process_openssl_auth_hmac(mbuf_src, dst, 1832 op->sym->auth.data.offset, srclen, 1833 ctx_h); 1834 HMAC_CTX_free(ctx_h); 1835 # endif 1836 break; 1837 case OPENSSL_AUTH_AS_CMAC: 1838 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1839 mac = EVP_MAC_fetch(NULL, OSSL_MAC_NAME_CMAC, NULL); 1840 ctx_c = EVP_MAC_CTX_new(mac); 1841 ctx_c = EVP_MAC_CTX_dup(sess->auth.cmac.ctx); 1842 EVP_MAC_free(mac); 1843 status = process_openssl_auth_mac(mbuf_src, dst, 1844 op->sym->auth.data.offset, srclen, 1845 ctx_c); 1846 # else 1847 ctx_c = CMAC_CTX_new(); 1848 CMAC_CTX_copy(ctx_c, sess->auth.cmac.ctx); 1849 status = process_openssl_auth_cmac(mbuf_src, dst, 1850 op->sym->auth.data.offset, srclen, 1851 ctx_c); 1852 CMAC_CTX_free(ctx_c); 1853 # endif 1854 break; 1855 default: 1856 status = -1; 1857 break; 1858 } 1859 1860 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1861 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data, 1862 sess->auth.digest_length) != 0) { 1863 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1864 } 1865 } else { 1866 uint8_t *auth_dst; 1867 1868 auth_dst = op->sym->auth.digest.data; 1869 if (auth_dst == NULL) 1870 auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1871 op->sym->auth.data.offset + 1872 op->sym->auth.data.length); 1873 memcpy(auth_dst, dst, sess->auth.digest_length); 1874 } 1875 1876 if (status != 0) 1877 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1878 } 1879 1880 /* process dsa sign operation */ 1881 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 1882 static int 1883 process_openssl_dsa_sign_op_evp(struct rte_crypto_op *cop, 1884 struct openssl_asym_session *sess) 1885 { 1886 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1887 EVP_PKEY_CTX *dsa_ctx = NULL; 1888 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 1889 EVP_PKEY *pkey = NULL; 1890 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 1891 OSSL_PARAM *params = NULL; 1892 1893 size_t outlen; 1894 unsigned char *dsa_sign_data; 1895 const unsigned char *dsa_sign_data_p; 1896 1897 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1898 params = OSSL_PARAM_BLD_to_param(param_bld); 1899 if (!params) { 1900 OSSL_PARAM_BLD_free(param_bld); 1901 return -1; 1902 } 1903 1904 if (key_ctx == NULL 1905 || EVP_PKEY_fromdata_init(key_ctx) <= 0 1906 || EVP_PKEY_fromdata(key_ctx, &pkey, 1907 EVP_PKEY_KEYPAIR, params) <= 0) 1908 goto err_dsa_sign; 1909 1910 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 1911 if (!dsa_ctx) 1912 goto err_dsa_sign; 1913 1914 if (EVP_PKEY_sign_init(dsa_ctx) <= 0) 1915 goto err_dsa_sign; 1916 1917 if (EVP_PKEY_sign(dsa_ctx, NULL, &outlen, op->message.data, 1918 op->message.length) <= 0) 1919 goto err_dsa_sign; 1920 1921 if (outlen <= 0) 1922 goto err_dsa_sign; 1923 1924 dsa_sign_data = OPENSSL_malloc(outlen); 1925 if (!dsa_sign_data) 1926 goto err_dsa_sign; 1927 1928 if (EVP_PKEY_sign(dsa_ctx, dsa_sign_data, &outlen, op->message.data, 1929 op->message.length) <= 0) { 1930 free(dsa_sign_data); 1931 goto err_dsa_sign; 1932 } 1933 1934 dsa_sign_data_p = (const unsigned char *)dsa_sign_data; 1935 DSA_SIG *sign = d2i_DSA_SIG(NULL, &dsa_sign_data_p, outlen); 1936 if (!sign) { 1937 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 1938 free(dsa_sign_data); 1939 goto err_dsa_sign; 1940 } else { 1941 const BIGNUM *r = NULL, *s = NULL; 1942 get_dsa_sign(sign, &r, &s); 1943 1944 op->r.length = BN_bn2bin(r, op->r.data); 1945 op->s.length = BN_bn2bin(s, op->s.data); 1946 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1947 } 1948 1949 DSA_SIG_free(sign); 1950 free(dsa_sign_data); 1951 return 0; 1952 1953 err_dsa_sign: 1954 if (params) 1955 OSSL_PARAM_free(params); 1956 if (key_ctx) 1957 EVP_PKEY_CTX_free(key_ctx); 1958 if (dsa_ctx) 1959 EVP_PKEY_CTX_free(dsa_ctx); 1960 return -1; 1961 } 1962 1963 /* process dsa verify operation */ 1964 static int 1965 process_openssl_dsa_verify_op_evp(struct rte_crypto_op *cop, 1966 struct openssl_asym_session *sess) 1967 { 1968 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1969 DSA_SIG *sign = DSA_SIG_new(); 1970 BIGNUM *r = NULL, *s = NULL; 1971 BIGNUM *pub_key = NULL; 1972 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 1973 OSSL_PARAM *params = NULL; 1974 EVP_PKEY *pkey = NULL; 1975 EVP_PKEY_CTX *dsa_ctx = NULL; 1976 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 1977 unsigned char *dsa_sig = NULL; 1978 size_t sig_len; 1979 int ret = -1; 1980 1981 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1982 if (!param_bld) { 1983 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 1984 return -1; 1985 } 1986 1987 r = BN_bin2bn(op->r.data, op->r.length, r); 1988 s = BN_bin2bn(op->s.data, op->s.length, s); 1989 pub_key = BN_bin2bn(op->y.data, op->y.length, pub_key); 1990 if (!r || !s || !pub_key) { 1991 BN_free(r); 1992 BN_free(s); 1993 BN_free(pub_key); 1994 OSSL_PARAM_BLD_free(param_bld); 1995 goto err_dsa_verify; 1996 } 1997 1998 set_dsa_sign(sign, r, s); 1999 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PUB_KEY, pub_key)) { 2000 OSSL_PARAM_BLD_free(param_bld); 2001 goto err_dsa_verify; 2002 } 2003 2004 params = OSSL_PARAM_BLD_to_param(param_bld); 2005 if (!params) { 2006 OSSL_PARAM_BLD_free(param_bld); 2007 goto err_dsa_verify; 2008 } 2009 2010 if (key_ctx == NULL 2011 || EVP_PKEY_fromdata_init(key_ctx) <= 0 2012 || EVP_PKEY_fromdata(key_ctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2013 goto err_dsa_verify; 2014 2015 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 2016 if (!dsa_ctx) 2017 goto err_dsa_verify; 2018 2019 if (!sign) 2020 goto err_dsa_verify; 2021 2022 sig_len = i2d_DSA_SIG(sign, &dsa_sig); 2023 if (EVP_PKEY_verify_init(dsa_ctx) <= 0) 2024 goto err_dsa_verify; 2025 2026 ret = EVP_PKEY_verify(dsa_ctx, dsa_sig, sig_len, 2027 op->message.data, op->message.length); 2028 if (ret == 1) { 2029 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2030 ret = 0; 2031 } 2032 2033 err_dsa_verify: 2034 if (sign) 2035 DSA_SIG_free(sign); 2036 if (params) 2037 OSSL_PARAM_free(params); 2038 if (key_ctx) 2039 EVP_PKEY_CTX_free(key_ctx); 2040 if (dsa_ctx) 2041 EVP_PKEY_CTX_free(dsa_ctx); 2042 2043 return ret; 2044 } 2045 #else 2046 static int 2047 process_openssl_dsa_sign_op(struct rte_crypto_op *cop, 2048 struct openssl_asym_session *sess) 2049 { 2050 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2051 DSA *dsa = sess->u.s.dsa; 2052 DSA_SIG *sign = NULL; 2053 2054 sign = DSA_do_sign(op->message.data, 2055 op->message.length, 2056 dsa); 2057 2058 if (sign == NULL) { 2059 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 2060 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2061 } else { 2062 const BIGNUM *r = NULL, *s = NULL; 2063 get_dsa_sign(sign, &r, &s); 2064 2065 op->r.length = BN_bn2bin(r, op->r.data); 2066 op->s.length = BN_bn2bin(s, op->s.data); 2067 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2068 } 2069 2070 DSA_SIG_free(sign); 2071 2072 return 0; 2073 } 2074 2075 /* process dsa verify operation */ 2076 static int 2077 process_openssl_dsa_verify_op(struct rte_crypto_op *cop, 2078 struct openssl_asym_session *sess) 2079 { 2080 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2081 DSA *dsa = sess->u.s.dsa; 2082 int ret; 2083 DSA_SIG *sign = DSA_SIG_new(); 2084 BIGNUM *r = NULL, *s = NULL; 2085 BIGNUM *pub_key = NULL; 2086 2087 if (sign == NULL) { 2088 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 2089 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2090 return -1; 2091 } 2092 2093 r = BN_bin2bn(op->r.data, 2094 op->r.length, 2095 r); 2096 s = BN_bin2bn(op->s.data, 2097 op->s.length, 2098 s); 2099 pub_key = BN_bin2bn(op->y.data, 2100 op->y.length, 2101 pub_key); 2102 if (!r || !s || !pub_key) { 2103 BN_free(r); 2104 BN_free(s); 2105 BN_free(pub_key); 2106 2107 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2108 return -1; 2109 } 2110 set_dsa_sign(sign, r, s); 2111 set_dsa_pub_key(dsa, pub_key); 2112 2113 ret = DSA_do_verify(op->message.data, 2114 op->message.length, 2115 sign, 2116 dsa); 2117 2118 if (ret != 1) 2119 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2120 else 2121 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2122 2123 DSA_SIG_free(sign); 2124 2125 return 0; 2126 } 2127 #endif 2128 2129 /* process dh operation */ 2130 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2131 static int 2132 process_openssl_dh_op_evp(struct rte_crypto_op *cop, 2133 struct openssl_asym_session *sess) 2134 { 2135 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2136 OSSL_PARAM_BLD *param_bld = sess->u.dh.param_bld; 2137 OSSL_PARAM_BLD *param_bld_peer = sess->u.dh.param_bld_peer; 2138 OSSL_PARAM *params = NULL; 2139 EVP_PKEY *dhpkey = NULL; 2140 EVP_PKEY *peerkey = NULL; 2141 BIGNUM *priv_key = NULL; 2142 BIGNUM *pub_key = NULL; 2143 int ret = -1; 2144 2145 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2146 EVP_PKEY_CTX *dh_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2147 if (dh_ctx == NULL || param_bld == NULL) 2148 return ret; 2149 2150 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2151 OSSL_PARAM *params_peer = NULL; 2152 2153 if (!param_bld_peer) 2154 return ret; 2155 2156 pub_key = BN_bin2bn(op->pub_key.data, op->pub_key.length, 2157 pub_key); 2158 if (pub_key == NULL) { 2159 OSSL_PARAM_BLD_free(param_bld_peer); 2160 return ret; 2161 } 2162 2163 if (!OSSL_PARAM_BLD_push_BN(param_bld_peer, OSSL_PKEY_PARAM_PUB_KEY, 2164 pub_key)) { 2165 OPENSSL_LOG(ERR, "Failed to set public key\n"); 2166 OSSL_PARAM_BLD_free(param_bld_peer); 2167 BN_free(pub_key); 2168 return ret; 2169 } 2170 2171 params_peer = OSSL_PARAM_BLD_to_param(param_bld_peer); 2172 if (!params_peer) { 2173 OSSL_PARAM_BLD_free(param_bld_peer); 2174 BN_free(pub_key); 2175 return ret; 2176 } 2177 2178 EVP_PKEY_CTX *peer_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2179 if (EVP_PKEY_keygen_init(peer_ctx) != 1) { 2180 OSSL_PARAM_free(params_peer); 2181 BN_free(pub_key); 2182 return ret; 2183 } 2184 2185 if (EVP_PKEY_CTX_set_params(peer_ctx, params_peer) != 1) { 2186 EVP_PKEY_CTX_free(peer_ctx); 2187 OSSL_PARAM_free(params_peer); 2188 BN_free(pub_key); 2189 return ret; 2190 } 2191 2192 if (EVP_PKEY_keygen(peer_ctx, &peerkey) != 1) { 2193 EVP_PKEY_CTX_free(peer_ctx); 2194 OSSL_PARAM_free(params_peer); 2195 BN_free(pub_key); 2196 return ret; 2197 } 2198 2199 priv_key = BN_bin2bn(op->priv_key.data, op->priv_key.length, 2200 priv_key); 2201 if (priv_key == NULL) { 2202 EVP_PKEY_CTX_free(peer_ctx); 2203 OSSL_PARAM_free(params_peer); 2204 BN_free(pub_key); 2205 return ret; 2206 } 2207 2208 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY, 2209 priv_key)) { 2210 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2211 EVP_PKEY_CTX_free(peer_ctx); 2212 OSSL_PARAM_free(params_peer); 2213 BN_free(pub_key); 2214 BN_free(priv_key); 2215 return ret; 2216 } 2217 2218 OSSL_PARAM_free(params_peer); 2219 EVP_PKEY_CTX_free(peer_ctx); 2220 } 2221 2222 params = OSSL_PARAM_BLD_to_param(param_bld); 2223 if (!params) 2224 goto err_dh; 2225 2226 if (EVP_PKEY_keygen_init(dh_ctx) != 1) 2227 goto err_dh; 2228 2229 if (EVP_PKEY_CTX_set_params(dh_ctx, params) != 1) 2230 goto err_dh; 2231 2232 if (EVP_PKEY_keygen(dh_ctx, &dhpkey) != 1) 2233 goto err_dh; 2234 2235 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2236 OPENSSL_LOG(DEBUG, "%s:%d updated pub key\n", __func__, __LINE__); 2237 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PUB_KEY, &pub_key)) 2238 goto err_dh; 2239 /* output public key */ 2240 op->pub_key.length = BN_bn2bin(pub_key, op->pub_key.data); 2241 } 2242 2243 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2244 2245 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", __func__, __LINE__); 2246 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PRIV_KEY, &priv_key)) 2247 goto err_dh; 2248 2249 /* provide generated private key back to user */ 2250 op->priv_key.length = BN_bn2bin(priv_key, op->priv_key.data); 2251 } 2252 2253 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2254 size_t skey_len; 2255 EVP_PKEY_CTX *sc_ctx = EVP_PKEY_CTX_new(dhpkey, NULL); 2256 if (!sc_ctx) 2257 goto err_dh; 2258 2259 if (EVP_PKEY_derive_init(sc_ctx) <= 0) { 2260 EVP_PKEY_CTX_free(sc_ctx); 2261 goto err_dh; 2262 } 2263 2264 if (!peerkey) { 2265 EVP_PKEY_CTX_free(sc_ctx); 2266 goto err_dh; 2267 } 2268 2269 if (EVP_PKEY_derive_set_peer(sc_ctx, peerkey) <= 0) { 2270 EVP_PKEY_CTX_free(sc_ctx); 2271 goto err_dh; 2272 } 2273 2274 /* Determine buffer length */ 2275 if (EVP_PKEY_derive(sc_ctx, NULL, &skey_len) <= 0) { 2276 EVP_PKEY_CTX_free(sc_ctx); 2277 goto err_dh; 2278 } 2279 2280 if (EVP_PKEY_derive(sc_ctx, op->shared_secret.data, &skey_len) <= 0) { 2281 EVP_PKEY_CTX_free(sc_ctx); 2282 goto err_dh; 2283 } 2284 2285 op->shared_secret.length = skey_len; 2286 EVP_PKEY_CTX_free(sc_ctx); 2287 } 2288 2289 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2290 ret = 0; 2291 2292 err_dh: 2293 if (pub_key) 2294 BN_free(pub_key); 2295 if (priv_key) 2296 BN_free(priv_key); 2297 if (params) 2298 OSSL_PARAM_free(params); 2299 if (dhpkey) 2300 EVP_PKEY_free(dhpkey); 2301 if (peerkey) 2302 EVP_PKEY_free(peerkey); 2303 2304 EVP_PKEY_CTX_free(dh_ctx); 2305 2306 return ret; 2307 } 2308 #else 2309 static int 2310 process_openssl_dh_op(struct rte_crypto_op *cop, 2311 struct openssl_asym_session *sess) 2312 { 2313 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2314 struct rte_crypto_asym_op *asym_op = cop->asym; 2315 DH *dh_key = sess->u.dh.dh_key; 2316 BIGNUM *priv_key = NULL; 2317 int ret = 0; 2318 2319 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2320 /* compute shared secret using peer public key 2321 * and current private key 2322 * shared secret = peer_key ^ priv_key mod p 2323 */ 2324 BIGNUM *peer_key = NULL; 2325 2326 /* copy private key and peer key and compute shared secret */ 2327 peer_key = BN_bin2bn(op->pub_key.data, 2328 op->pub_key.length, 2329 peer_key); 2330 if (peer_key == NULL) { 2331 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2332 return -1; 2333 } 2334 priv_key = BN_bin2bn(op->priv_key.data, 2335 op->priv_key.length, 2336 priv_key); 2337 if (priv_key == NULL) { 2338 BN_free(peer_key); 2339 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2340 return -1; 2341 } 2342 ret = set_dh_priv_key(dh_key, priv_key); 2343 if (ret) { 2344 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2345 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2346 BN_free(peer_key); 2347 BN_free(priv_key); 2348 return 0; 2349 } 2350 2351 ret = DH_compute_key( 2352 op->shared_secret.data, 2353 peer_key, dh_key); 2354 if (ret < 0) { 2355 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2356 BN_free(peer_key); 2357 /* priv key is already loaded into dh, 2358 * let's not free that directly here. 2359 * DH_free() will auto free it later. 2360 */ 2361 return 0; 2362 } 2363 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2364 op->shared_secret.length = ret; 2365 BN_free(peer_key); 2366 return 0; 2367 } 2368 2369 /* 2370 * other options are public and private key generations. 2371 * 2372 * if user provides private key, 2373 * then first set DH with user provided private key 2374 */ 2375 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE && 2376 op->priv_key.length) { 2377 /* generate public key using user-provided private key 2378 * pub_key = g ^ priv_key mod p 2379 */ 2380 2381 /* load private key into DH */ 2382 priv_key = BN_bin2bn(op->priv_key.data, 2383 op->priv_key.length, 2384 priv_key); 2385 if (priv_key == NULL) { 2386 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2387 return -1; 2388 } 2389 ret = set_dh_priv_key(dh_key, priv_key); 2390 if (ret) { 2391 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2392 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2393 BN_free(priv_key); 2394 return 0; 2395 } 2396 } 2397 2398 /* generate public and private key pair. 2399 * 2400 * if private key already set, generates only public key. 2401 * 2402 * if private key is not already set, then set it to random value 2403 * and update internal private key. 2404 */ 2405 if (!DH_generate_key(dh_key)) { 2406 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2407 return 0; 2408 } 2409 2410 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2411 const BIGNUM *pub_key = NULL; 2412 2413 OPENSSL_LOG(DEBUG, "%s:%d update public key\n", 2414 __func__, __LINE__); 2415 2416 /* get the generated keys */ 2417 get_dh_pub_key(dh_key, &pub_key); 2418 2419 /* output public key */ 2420 op->pub_key.length = BN_bn2bin(pub_key, 2421 op->pub_key.data); 2422 } 2423 2424 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2425 const BIGNUM *priv_key = NULL; 2426 2427 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", 2428 __func__, __LINE__); 2429 2430 /* get the generated keys */ 2431 get_dh_priv_key(dh_key, &priv_key); 2432 2433 /* provide generated private key back to user */ 2434 op->priv_key.length = BN_bn2bin(priv_key, 2435 op->priv_key.data); 2436 } 2437 2438 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2439 2440 return 0; 2441 } 2442 #endif 2443 2444 /* process modinv operation */ 2445 static int 2446 process_openssl_modinv_op(struct rte_crypto_op *cop, 2447 struct openssl_asym_session *sess) 2448 { 2449 struct rte_crypto_asym_op *op = cop->asym; 2450 BIGNUM *base = BN_CTX_get(sess->u.m.ctx); 2451 BIGNUM *res = BN_CTX_get(sess->u.m.ctx); 2452 2453 if (unlikely(base == NULL || res == NULL)) { 2454 BN_free(base); 2455 BN_free(res); 2456 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2457 return -1; 2458 } 2459 2460 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 2461 op->modinv.base.length, base); 2462 2463 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) { 2464 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2465 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data); 2466 } else { 2467 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2468 } 2469 2470 BN_clear(res); 2471 BN_clear(base); 2472 2473 return 0; 2474 } 2475 2476 /* process modexp operation */ 2477 static int 2478 process_openssl_modexp_op(struct rte_crypto_op *cop, 2479 struct openssl_asym_session *sess) 2480 { 2481 struct rte_crypto_asym_op *op = cop->asym; 2482 BIGNUM *base = BN_CTX_get(sess->u.e.ctx); 2483 BIGNUM *res = BN_CTX_get(sess->u.e.ctx); 2484 2485 if (unlikely(base == NULL || res == NULL)) { 2486 BN_free(base); 2487 BN_free(res); 2488 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2489 return -1; 2490 } 2491 2492 base = BN_bin2bn((const unsigned char *)op->modex.base.data, 2493 op->modex.base.length, base); 2494 2495 if (BN_mod_exp(res, base, sess->u.e.exp, 2496 sess->u.e.mod, sess->u.e.ctx)) { 2497 op->modex.result.length = BN_bn2bin(res, op->modex.result.data); 2498 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2499 } else { 2500 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2501 } 2502 2503 BN_clear(res); 2504 BN_clear(base); 2505 2506 return 0; 2507 } 2508 2509 /* process rsa operations */ 2510 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2511 static int 2512 process_openssl_rsa_op_evp(struct rte_crypto_op *cop, 2513 struct openssl_asym_session *sess) 2514 { 2515 struct rte_crypto_asym_op *op = cop->asym; 2516 uint32_t pad = (op->rsa.padding.type); 2517 uint8_t *tmp; 2518 size_t outlen = 0; 2519 int ret = -1; 2520 2521 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2522 EVP_PKEY_CTX *rsa_ctx = sess->u.r.ctx; 2523 if (!rsa_ctx) 2524 return ret; 2525 2526 switch (pad) { 2527 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2528 pad = RSA_PKCS1_PADDING; 2529 break; 2530 case RTE_CRYPTO_RSA_PADDING_NONE: 2531 pad = RSA_NO_PADDING; 2532 break; 2533 default: 2534 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2535 OPENSSL_LOG(ERR, 2536 "rsa pad type not supported %d\n", pad); 2537 return ret; 2538 } 2539 2540 switch (op->rsa.op_type) { 2541 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2542 if (EVP_PKEY_encrypt_init(rsa_ctx) != 1) 2543 goto err_rsa; 2544 2545 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2546 goto err_rsa; 2547 2548 if (EVP_PKEY_encrypt(rsa_ctx, NULL, &outlen, 2549 op->rsa.message.data, 2550 op->rsa.message.length) <= 0) 2551 goto err_rsa; 2552 2553 if (outlen <= 0) 2554 goto err_rsa; 2555 2556 if (EVP_PKEY_encrypt(rsa_ctx, op->rsa.cipher.data, &outlen, 2557 op->rsa.message.data, 2558 op->rsa.message.length) <= 0) 2559 goto err_rsa; 2560 op->rsa.cipher.length = outlen; 2561 2562 OPENSSL_LOG(DEBUG, 2563 "length of encrypted text %zu\n", outlen); 2564 break; 2565 2566 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2567 if (EVP_PKEY_decrypt_init(rsa_ctx) != 1) 2568 goto err_rsa; 2569 2570 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2571 goto err_rsa; 2572 2573 if (EVP_PKEY_decrypt(rsa_ctx, NULL, &outlen, 2574 op->rsa.cipher.data, 2575 op->rsa.cipher.length) <= 0) 2576 goto err_rsa; 2577 2578 if (outlen <= 0) 2579 goto err_rsa; 2580 2581 if (EVP_PKEY_decrypt(rsa_ctx, op->rsa.message.data, &outlen, 2582 op->rsa.cipher.data, 2583 op->rsa.cipher.length) <= 0) 2584 goto err_rsa; 2585 op->rsa.message.length = outlen; 2586 2587 OPENSSL_LOG(DEBUG, "length of decrypted text %zu\n", outlen); 2588 break; 2589 2590 case RTE_CRYPTO_ASYM_OP_SIGN: 2591 if (EVP_PKEY_sign_init(rsa_ctx) <= 0) 2592 goto err_rsa; 2593 2594 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2595 goto err_rsa; 2596 2597 if (EVP_PKEY_sign(rsa_ctx, NULL, &outlen, 2598 op->rsa.message.data, 2599 op->rsa.message.length) <= 0) 2600 goto err_rsa; 2601 2602 if (outlen <= 0) 2603 goto err_rsa; 2604 2605 if (EVP_PKEY_sign(rsa_ctx, op->rsa.sign.data, &outlen, 2606 op->rsa.message.data, 2607 op->rsa.message.length) <= 0) 2608 goto err_rsa; 2609 op->rsa.sign.length = outlen; 2610 break; 2611 2612 case RTE_CRYPTO_ASYM_OP_VERIFY: 2613 if (EVP_PKEY_verify_recover_init(rsa_ctx) <= 0) 2614 goto err_rsa; 2615 2616 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2617 goto err_rsa; 2618 2619 if (EVP_PKEY_verify_recover(rsa_ctx, NULL, &outlen, 2620 op->rsa.sign.data, 2621 op->rsa.sign.length) <= 0) 2622 goto err_rsa; 2623 2624 if ((outlen <= 0) || (outlen != op->rsa.sign.length)) 2625 goto err_rsa; 2626 2627 tmp = OPENSSL_malloc(outlen); 2628 if (tmp == NULL) { 2629 OPENSSL_LOG(ERR, "Memory allocation failed"); 2630 goto err_rsa; 2631 } 2632 2633 if (EVP_PKEY_verify_recover(rsa_ctx, tmp, &outlen, 2634 op->rsa.sign.data, 2635 op->rsa.sign.length) <= 0) { 2636 rte_free(tmp); 2637 goto err_rsa; 2638 } 2639 2640 OPENSSL_LOG(DEBUG, 2641 "Length of public_decrypt %zu " 2642 "length of message %zd\n", 2643 outlen, op->rsa.message.length); 2644 if (CRYPTO_memcmp(tmp, op->rsa.message.data, 2645 op->rsa.message.length)) { 2646 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 2647 } 2648 rte_free(tmp); 2649 break; 2650 2651 default: 2652 /* allow ops with invalid args to be pushed to 2653 * completion queue 2654 */ 2655 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2656 goto err_rsa; 2657 } 2658 2659 ret = 0; 2660 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2661 err_rsa: 2662 return ret; 2663 2664 } 2665 #else 2666 static int 2667 process_openssl_rsa_op(struct rte_crypto_op *cop, 2668 struct openssl_asym_session *sess) 2669 { 2670 int ret = 0; 2671 struct rte_crypto_asym_op *op = cop->asym; 2672 RSA *rsa = sess->u.r.rsa; 2673 uint32_t pad = (op->rsa.padding.type); 2674 uint8_t *tmp; 2675 2676 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2677 2678 switch (pad) { 2679 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2680 pad = RSA_PKCS1_PADDING; 2681 break; 2682 case RTE_CRYPTO_RSA_PADDING_NONE: 2683 pad = RSA_NO_PADDING; 2684 break; 2685 default: 2686 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2687 OPENSSL_LOG(ERR, 2688 "rsa pad type not supported %d\n", pad); 2689 return 0; 2690 } 2691 2692 switch (op->rsa.op_type) { 2693 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2694 ret = RSA_public_encrypt(op->rsa.message.length, 2695 op->rsa.message.data, 2696 op->rsa.cipher.data, 2697 rsa, 2698 pad); 2699 2700 if (ret > 0) 2701 op->rsa.cipher.length = ret; 2702 OPENSSL_LOG(DEBUG, 2703 "length of encrypted text %d\n", ret); 2704 break; 2705 2706 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2707 ret = RSA_private_decrypt(op->rsa.cipher.length, 2708 op->rsa.cipher.data, 2709 op->rsa.message.data, 2710 rsa, 2711 pad); 2712 if (ret > 0) 2713 op->rsa.message.length = ret; 2714 break; 2715 2716 case RTE_CRYPTO_ASYM_OP_SIGN: 2717 ret = RSA_private_encrypt(op->rsa.message.length, 2718 op->rsa.message.data, 2719 op->rsa.sign.data, 2720 rsa, 2721 pad); 2722 if (ret > 0) 2723 op->rsa.sign.length = ret; 2724 break; 2725 2726 case RTE_CRYPTO_ASYM_OP_VERIFY: 2727 tmp = rte_malloc(NULL, op->rsa.sign.length, 0); 2728 if (tmp == NULL) { 2729 OPENSSL_LOG(ERR, "Memory allocation failed"); 2730 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2731 break; 2732 } 2733 ret = RSA_public_decrypt(op->rsa.sign.length, 2734 op->rsa.sign.data, 2735 tmp, 2736 rsa, 2737 pad); 2738 2739 OPENSSL_LOG(DEBUG, 2740 "Length of public_decrypt %d " 2741 "length of message %zd\n", 2742 ret, op->rsa.message.length); 2743 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data, 2744 op->rsa.message.length))) { 2745 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 2746 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2747 } 2748 rte_free(tmp); 2749 break; 2750 2751 default: 2752 /* allow ops with invalid args to be pushed to 2753 * completion queue 2754 */ 2755 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2756 break; 2757 } 2758 2759 if (ret < 0) 2760 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2761 2762 return 0; 2763 } 2764 #endif 2765 2766 static int 2767 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op, 2768 struct openssl_asym_session *sess) 2769 { 2770 int retval = 0; 2771 2772 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2773 2774 switch (sess->xfrm_type) { 2775 case RTE_CRYPTO_ASYM_XFORM_RSA: 2776 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2777 retval = process_openssl_rsa_op_evp(op, sess); 2778 # else 2779 retval = process_openssl_rsa_op(op, sess); 2780 #endif 2781 break; 2782 case RTE_CRYPTO_ASYM_XFORM_MODEX: 2783 retval = process_openssl_modexp_op(op, sess); 2784 break; 2785 case RTE_CRYPTO_ASYM_XFORM_MODINV: 2786 retval = process_openssl_modinv_op(op, sess); 2787 break; 2788 case RTE_CRYPTO_ASYM_XFORM_DH: 2789 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2790 retval = process_openssl_dh_op_evp(op, sess); 2791 # else 2792 retval = process_openssl_dh_op(op, sess); 2793 #endif 2794 break; 2795 case RTE_CRYPTO_ASYM_XFORM_DSA: 2796 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2797 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 2798 retval = process_openssl_dsa_sign_op_evp(op, sess); 2799 else if (op->asym->dsa.op_type == 2800 RTE_CRYPTO_ASYM_OP_VERIFY) 2801 retval = 2802 process_openssl_dsa_verify_op_evp(op, sess); 2803 #else 2804 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 2805 retval = process_openssl_dsa_sign_op(op, sess); 2806 else if (op->asym->dsa.op_type == 2807 RTE_CRYPTO_ASYM_OP_VERIFY) 2808 retval = 2809 process_openssl_dsa_verify_op(op, sess); 2810 else 2811 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2812 #endif 2813 break; 2814 default: 2815 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2816 break; 2817 } 2818 if (!retval) { 2819 /* op processed so push to completion queue as processed */ 2820 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 2821 if (retval) 2822 /* return error if failed to put in completion queue */ 2823 retval = -1; 2824 } 2825 2826 return retval; 2827 } 2828 2829 static void 2830 copy_plaintext(struct rte_mbuf *m_src, struct rte_mbuf *m_dst, 2831 struct rte_crypto_op *op) 2832 { 2833 uint8_t *p_src, *p_dst; 2834 2835 p_src = rte_pktmbuf_mtod(m_src, uint8_t *); 2836 p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *); 2837 2838 /** 2839 * Copy the content between cipher offset and auth offset 2840 * for generating correct digest. 2841 */ 2842 if (op->sym->cipher.data.offset > op->sym->auth.data.offset) 2843 memcpy(p_dst + op->sym->auth.data.offset, 2844 p_src + op->sym->auth.data.offset, 2845 op->sym->cipher.data.offset - 2846 op->sym->auth.data.offset); 2847 } 2848 2849 /** Process crypto operation for mbuf */ 2850 static int 2851 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 2852 struct openssl_session *sess) 2853 { 2854 struct rte_mbuf *msrc, *mdst; 2855 int retval; 2856 2857 msrc = op->sym->m_src; 2858 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 2859 2860 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2861 2862 switch (sess->chain_order) { 2863 case OPENSSL_CHAIN_ONLY_CIPHER: 2864 process_openssl_cipher_op(op, sess, msrc, mdst); 2865 break; 2866 case OPENSSL_CHAIN_ONLY_AUTH: 2867 process_openssl_auth_op(qp, op, sess, msrc, mdst); 2868 break; 2869 case OPENSSL_CHAIN_CIPHER_AUTH: 2870 process_openssl_cipher_op(op, sess, msrc, mdst); 2871 /* OOP */ 2872 if (msrc != mdst) 2873 copy_plaintext(msrc, mdst, op); 2874 process_openssl_auth_op(qp, op, sess, mdst, mdst); 2875 break; 2876 case OPENSSL_CHAIN_AUTH_CIPHER: 2877 process_openssl_auth_op(qp, op, sess, msrc, mdst); 2878 process_openssl_cipher_op(op, sess, msrc, mdst); 2879 break; 2880 case OPENSSL_CHAIN_COMBINED: 2881 process_openssl_combined_op(op, sess, msrc, mdst); 2882 break; 2883 case OPENSSL_CHAIN_CIPHER_BPI: 2884 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 2885 break; 2886 default: 2887 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 2888 break; 2889 } 2890 2891 /* Free session if a session-less crypto op */ 2892 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 2893 openssl_reset_session(sess); 2894 memset(sess, 0, sizeof(struct openssl_session)); 2895 rte_mempool_put(qp->sess_mp, op->sym->session); 2896 op->sym->session = NULL; 2897 } 2898 2899 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 2900 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2901 2902 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 2903 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 2904 else 2905 retval = -1; 2906 2907 return retval; 2908 } 2909 2910 /* 2911 *------------------------------------------------------------------------------ 2912 * PMD Framework 2913 *------------------------------------------------------------------------------ 2914 */ 2915 2916 /** Enqueue burst */ 2917 static uint16_t 2918 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 2919 uint16_t nb_ops) 2920 { 2921 void *sess; 2922 struct openssl_qp *qp = queue_pair; 2923 int i, retval; 2924 2925 for (i = 0; i < nb_ops; i++) { 2926 sess = get_session(qp, ops[i]); 2927 if (unlikely(sess == NULL)) 2928 goto enqueue_err; 2929 2930 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) 2931 retval = process_op(qp, ops[i], 2932 (struct openssl_session *) sess); 2933 else 2934 retval = process_asym_op(qp, ops[i], 2935 (struct openssl_asym_session *) sess); 2936 if (unlikely(retval < 0)) 2937 goto enqueue_err; 2938 } 2939 2940 qp->stats.enqueued_count += i; 2941 return i; 2942 2943 enqueue_err: 2944 qp->stats.enqueue_err_count++; 2945 return i; 2946 } 2947 2948 /** Dequeue burst */ 2949 static uint16_t 2950 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 2951 uint16_t nb_ops) 2952 { 2953 struct openssl_qp *qp = queue_pair; 2954 2955 unsigned int nb_dequeued = 0; 2956 2957 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 2958 (void **)ops, nb_ops, NULL); 2959 qp->stats.dequeued_count += nb_dequeued; 2960 2961 return nb_dequeued; 2962 } 2963 2964 /** Create OPENSSL crypto device */ 2965 static int 2966 cryptodev_openssl_create(const char *name, 2967 struct rte_vdev_device *vdev, 2968 struct rte_cryptodev_pmd_init_params *init_params) 2969 { 2970 struct rte_cryptodev *dev; 2971 struct openssl_private *internals; 2972 2973 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 2974 if (dev == NULL) { 2975 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 2976 goto init_error; 2977 } 2978 2979 dev->driver_id = cryptodev_driver_id; 2980 dev->dev_ops = rte_openssl_pmd_ops; 2981 2982 /* register rx/tx burst functions for data path */ 2983 dev->dequeue_burst = openssl_pmd_dequeue_burst; 2984 dev->enqueue_burst = openssl_pmd_enqueue_burst; 2985 2986 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 2987 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 2988 RTE_CRYPTODEV_FF_CPU_AESNI | 2989 RTE_CRYPTODEV_FF_IN_PLACE_SGL | 2990 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 2991 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | 2992 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO | 2993 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP | 2994 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT | 2995 RTE_CRYPTODEV_FF_SYM_SESSIONLESS; 2996 2997 internals = dev->data->dev_private; 2998 2999 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 3000 3001 rte_cryptodev_pmd_probing_finish(dev); 3002 3003 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3004 /* Load legacy provider 3005 * Some algorithms are no longer available in earlier version of openssl, 3006 * unless the legacy provider explicitly loaded. e.g. DES 3007 */ 3008 ossl_legacy_provider_load(); 3009 # endif 3010 return 0; 3011 3012 init_error: 3013 OPENSSL_LOG(ERR, "driver %s: create failed", 3014 init_params->name); 3015 3016 cryptodev_openssl_remove(vdev); 3017 return -EFAULT; 3018 } 3019 3020 /** Initialise OPENSSL crypto device */ 3021 static int 3022 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 3023 { 3024 struct rte_cryptodev_pmd_init_params init_params = { 3025 "", 3026 sizeof(struct openssl_private), 3027 rte_socket_id(), 3028 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 3029 }; 3030 const char *name; 3031 const char *input_args; 3032 3033 name = rte_vdev_device_name(vdev); 3034 if (name == NULL) 3035 return -EINVAL; 3036 input_args = rte_vdev_device_args(vdev); 3037 3038 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 3039 3040 return cryptodev_openssl_create(name, vdev, &init_params); 3041 } 3042 3043 /** Uninitialise OPENSSL crypto device */ 3044 static int 3045 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 3046 { 3047 struct rte_cryptodev *cryptodev; 3048 const char *name; 3049 3050 name = rte_vdev_device_name(vdev); 3051 if (name == NULL) 3052 return -EINVAL; 3053 3054 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 3055 if (cryptodev == NULL) 3056 return -ENODEV; 3057 3058 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3059 ossl_legacy_provider_unload(); 3060 # endif 3061 return rte_cryptodev_pmd_destroy(cryptodev); 3062 } 3063 3064 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 3065 .probe = cryptodev_openssl_probe, 3066 .remove = cryptodev_openssl_remove 3067 }; 3068 3069 static struct cryptodev_driver openssl_crypto_drv; 3070 3071 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 3072 cryptodev_openssl_pmd_drv); 3073 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 3074 "max_nb_queue_pairs=<int> " 3075 "socket_id=<int>"); 3076 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 3077 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 3078 RTE_LOG_REGISTER_DEFAULT(openssl_logtype_driver, INFO); 3079