1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_common.h> 6 #include <rte_hexdump.h> 7 #include <rte_cryptodev.h> 8 #include <cryptodev_pmd.h> 9 #include <bus_vdev_driver.h> 10 #include <rte_malloc.h> 11 #include <rte_cpuflags.h> 12 13 #include <openssl/cmac.h> 14 #include <openssl/hmac.h> 15 #include <openssl/evp.h> 16 #include <openssl/ec.h> 17 18 #include "openssl_pmd_private.h" 19 #include "compat.h" 20 21 #define DES_BLOCK_SIZE 8 22 23 static uint8_t cryptodev_driver_id; 24 25 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 26 static HMAC_CTX *HMAC_CTX_new(void) 27 { 28 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 29 30 if (ctx != NULL) 31 HMAC_CTX_init(ctx); 32 return ctx; 33 } 34 35 static void HMAC_CTX_free(HMAC_CTX *ctx) 36 { 37 if (ctx != NULL) { 38 HMAC_CTX_cleanup(ctx); 39 OPENSSL_free(ctx); 40 } 41 } 42 #endif 43 44 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 45 46 #include <openssl/provider.h> 47 #include <openssl/core_names.h> 48 #include <openssl/param_build.h> 49 50 #define MAX_OSSL_ALGO_NAME_SIZE 16 51 52 OSSL_PROVIDER *legacy; 53 OSSL_PROVIDER *deflt; 54 55 static void ossl_legacy_provider_load(void) 56 { 57 /* Load Multiple providers into the default (NULL) library context */ 58 legacy = OSSL_PROVIDER_load(NULL, "legacy"); 59 if (legacy == NULL) { 60 OPENSSL_LOG(ERR, "Failed to load Legacy provider\n"); 61 return; 62 } 63 64 deflt = OSSL_PROVIDER_load(NULL, "default"); 65 if (deflt == NULL) { 66 OPENSSL_LOG(ERR, "Failed to load Default provider\n"); 67 OSSL_PROVIDER_unload(legacy); 68 return; 69 } 70 } 71 72 static void ossl_legacy_provider_unload(void) 73 { 74 OSSL_PROVIDER_unload(legacy); 75 OSSL_PROVIDER_unload(deflt); 76 } 77 78 static __rte_always_inline const char * 79 digest_name_get(enum rte_crypto_auth_algorithm algo) 80 { 81 switch (algo) { 82 case RTE_CRYPTO_AUTH_MD5_HMAC: 83 return OSSL_DIGEST_NAME_MD5; 84 case RTE_CRYPTO_AUTH_SHA1_HMAC: 85 return OSSL_DIGEST_NAME_SHA1; 86 case RTE_CRYPTO_AUTH_SHA224_HMAC: 87 return OSSL_DIGEST_NAME_SHA2_224; 88 case RTE_CRYPTO_AUTH_SHA256_HMAC: 89 return OSSL_DIGEST_NAME_SHA2_256; 90 case RTE_CRYPTO_AUTH_SHA384_HMAC: 91 return OSSL_DIGEST_NAME_SHA2_384; 92 case RTE_CRYPTO_AUTH_SHA512_HMAC: 93 return OSSL_DIGEST_NAME_SHA2_512; 94 default: 95 return NULL; 96 } 97 } 98 #endif 99 100 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 101 102 /*----------------------------------------------------------------------------*/ 103 104 /** 105 * Increment counter by 1 106 * Counter is 64 bit array, big-endian 107 */ 108 static void 109 ctr_inc(uint8_t *ctr) 110 { 111 uint64_t *ctr64 = (uint64_t *)ctr; 112 113 *ctr64 = __builtin_bswap64(*ctr64); 114 (*ctr64)++; 115 *ctr64 = __builtin_bswap64(*ctr64); 116 } 117 118 /* 119 *------------------------------------------------------------------------------ 120 * Session Prepare 121 *------------------------------------------------------------------------------ 122 */ 123 124 /** Get xform chain order */ 125 static enum openssl_chain_order 126 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 127 { 128 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 129 130 if (xform != NULL) { 131 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 132 if (xform->next == NULL) 133 res = OPENSSL_CHAIN_ONLY_AUTH; 134 else if (xform->next->type == 135 RTE_CRYPTO_SYM_XFORM_CIPHER) 136 res = OPENSSL_CHAIN_AUTH_CIPHER; 137 } 138 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 139 if (xform->next == NULL) 140 res = OPENSSL_CHAIN_ONLY_CIPHER; 141 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 142 res = OPENSSL_CHAIN_CIPHER_AUTH; 143 } 144 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 145 res = OPENSSL_CHAIN_COMBINED; 146 } 147 148 return res; 149 } 150 151 /** Get session cipher key from input cipher key */ 152 static void 153 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key) 154 { 155 memcpy(session_key, input_key, keylen); 156 } 157 158 /** Get key ede 24 bytes standard from input key */ 159 static int 160 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede) 161 { 162 int res = 0; 163 164 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 165 switch (keylen) { 166 case 24: 167 memcpy(key_ede, key, 24); 168 break; 169 case 16: 170 /* K3 = K1 */ 171 memcpy(key_ede, key, 16); 172 memcpy(key_ede + 16, key, 8); 173 break; 174 case 8: 175 /* K1 = K2 = K3 (DES compatibility) */ 176 memcpy(key_ede, key, 8); 177 memcpy(key_ede + 8, key, 8); 178 memcpy(key_ede + 16, key, 8); 179 break; 180 default: 181 OPENSSL_LOG(ERR, "Unsupported key size"); 182 res = -EINVAL; 183 } 184 185 return res; 186 } 187 188 /** Get adequate openssl function for input cipher algorithm */ 189 static uint8_t 190 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 191 const EVP_CIPHER **algo) 192 { 193 int res = 0; 194 195 if (algo != NULL) { 196 switch (sess_algo) { 197 case RTE_CRYPTO_CIPHER_3DES_CBC: 198 switch (keylen) { 199 case 8: 200 *algo = EVP_des_cbc(); 201 break; 202 case 16: 203 *algo = EVP_des_ede_cbc(); 204 break; 205 case 24: 206 *algo = EVP_des_ede3_cbc(); 207 break; 208 default: 209 res = -EINVAL; 210 } 211 break; 212 case RTE_CRYPTO_CIPHER_3DES_CTR: 213 break; 214 case RTE_CRYPTO_CIPHER_AES_CBC: 215 switch (keylen) { 216 case 16: 217 *algo = EVP_aes_128_cbc(); 218 break; 219 case 24: 220 *algo = EVP_aes_192_cbc(); 221 break; 222 case 32: 223 *algo = EVP_aes_256_cbc(); 224 break; 225 default: 226 res = -EINVAL; 227 } 228 break; 229 case RTE_CRYPTO_CIPHER_AES_CTR: 230 switch (keylen) { 231 case 16: 232 *algo = EVP_aes_128_ctr(); 233 break; 234 case 24: 235 *algo = EVP_aes_192_ctr(); 236 break; 237 case 32: 238 *algo = EVP_aes_256_ctr(); 239 break; 240 default: 241 res = -EINVAL; 242 } 243 break; 244 default: 245 res = -EINVAL; 246 break; 247 } 248 } else { 249 res = -EINVAL; 250 } 251 252 return res; 253 } 254 255 /** Get adequate openssl function for input auth algorithm */ 256 static uint8_t 257 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 258 const EVP_MD **algo) 259 { 260 int res = 0; 261 262 if (algo != NULL) { 263 switch (sessalgo) { 264 case RTE_CRYPTO_AUTH_MD5: 265 case RTE_CRYPTO_AUTH_MD5_HMAC: 266 *algo = EVP_md5(); 267 break; 268 case RTE_CRYPTO_AUTH_SHA1: 269 case RTE_CRYPTO_AUTH_SHA1_HMAC: 270 *algo = EVP_sha1(); 271 break; 272 case RTE_CRYPTO_AUTH_SHA224: 273 case RTE_CRYPTO_AUTH_SHA224_HMAC: 274 *algo = EVP_sha224(); 275 break; 276 case RTE_CRYPTO_AUTH_SHA256: 277 case RTE_CRYPTO_AUTH_SHA256_HMAC: 278 *algo = EVP_sha256(); 279 break; 280 case RTE_CRYPTO_AUTH_SHA384: 281 case RTE_CRYPTO_AUTH_SHA384_HMAC: 282 *algo = EVP_sha384(); 283 break; 284 case RTE_CRYPTO_AUTH_SHA512: 285 case RTE_CRYPTO_AUTH_SHA512_HMAC: 286 *algo = EVP_sha512(); 287 break; 288 default: 289 res = -EINVAL; 290 break; 291 } 292 } else { 293 res = -EINVAL; 294 } 295 296 return res; 297 } 298 299 /** Get adequate openssl function for input cipher algorithm */ 300 static uint8_t 301 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 302 const EVP_CIPHER **algo) 303 { 304 int res = 0; 305 306 if (algo != NULL) { 307 switch (sess_algo) { 308 case RTE_CRYPTO_AEAD_AES_GCM: 309 switch (keylen) { 310 case 16: 311 *algo = EVP_aes_128_gcm(); 312 break; 313 case 24: 314 *algo = EVP_aes_192_gcm(); 315 break; 316 case 32: 317 *algo = EVP_aes_256_gcm(); 318 break; 319 default: 320 res = -EINVAL; 321 } 322 break; 323 case RTE_CRYPTO_AEAD_AES_CCM: 324 switch (keylen) { 325 case 16: 326 *algo = EVP_aes_128_ccm(); 327 break; 328 case 24: 329 *algo = EVP_aes_192_ccm(); 330 break; 331 case 32: 332 *algo = EVP_aes_256_ccm(); 333 break; 334 default: 335 res = -EINVAL; 336 } 337 break; 338 default: 339 res = -EINVAL; 340 break; 341 } 342 } else { 343 res = -EINVAL; 344 } 345 346 return res; 347 } 348 349 /* Set session AEAD encryption parameters */ 350 static int 351 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 352 enum rte_crypto_aead_algorithm algo, 353 uint8_t tag_len, const uint8_t *key) 354 { 355 int iv_type = 0; 356 unsigned int do_ccm; 357 358 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 359 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 360 361 /* Select AEAD algo */ 362 switch (algo) { 363 case RTE_CRYPTO_AEAD_AES_GCM: 364 iv_type = EVP_CTRL_GCM_SET_IVLEN; 365 if (tag_len != 16) 366 return -EINVAL; 367 do_ccm = 0; 368 break; 369 case RTE_CRYPTO_AEAD_AES_CCM: 370 iv_type = EVP_CTRL_CCM_SET_IVLEN; 371 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 372 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 373 return -EINVAL; 374 do_ccm = 1; 375 break; 376 default: 377 return -ENOTSUP; 378 } 379 380 sess->cipher.mode = OPENSSL_CIPHER_LIB; 381 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 382 383 if (get_aead_algo(algo, sess->cipher.key.length, 384 &sess->cipher.evp_algo) != 0) 385 return -EINVAL; 386 387 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 388 389 sess->chain_order = OPENSSL_CHAIN_COMBINED; 390 391 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 392 NULL, NULL, NULL) <= 0) 393 return -EINVAL; 394 395 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, 396 NULL) <= 0) 397 return -EINVAL; 398 399 if (do_ccm) 400 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 401 tag_len, NULL); 402 403 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 404 return -EINVAL; 405 406 return 0; 407 } 408 409 /* Set session AEAD decryption parameters */ 410 static int 411 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 412 enum rte_crypto_aead_algorithm algo, 413 uint8_t tag_len, const uint8_t *key) 414 { 415 int iv_type = 0; 416 unsigned int do_ccm = 0; 417 418 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 419 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 420 421 /* Select AEAD algo */ 422 switch (algo) { 423 case RTE_CRYPTO_AEAD_AES_GCM: 424 iv_type = EVP_CTRL_GCM_SET_IVLEN; 425 if (tag_len != 16) 426 return -EINVAL; 427 break; 428 case RTE_CRYPTO_AEAD_AES_CCM: 429 iv_type = EVP_CTRL_CCM_SET_IVLEN; 430 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 431 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 432 return -EINVAL; 433 do_ccm = 1; 434 break; 435 default: 436 return -ENOTSUP; 437 } 438 439 sess->cipher.mode = OPENSSL_CIPHER_LIB; 440 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 441 442 if (get_aead_algo(algo, sess->cipher.key.length, 443 &sess->cipher.evp_algo) != 0) 444 return -EINVAL; 445 446 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 447 448 sess->chain_order = OPENSSL_CHAIN_COMBINED; 449 450 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 451 NULL, NULL, NULL) <= 0) 452 return -EINVAL; 453 454 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, 455 sess->iv.length, NULL) <= 0) 456 return -EINVAL; 457 458 if (do_ccm) 459 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 460 tag_len, NULL); 461 462 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 463 return -EINVAL; 464 465 return 0; 466 } 467 468 /** Set session cipher parameters */ 469 static int 470 openssl_set_session_cipher_parameters(struct openssl_session *sess, 471 const struct rte_crypto_sym_xform *xform) 472 { 473 /* Select cipher direction */ 474 sess->cipher.direction = xform->cipher.op; 475 /* Select cipher key */ 476 sess->cipher.key.length = xform->cipher.key.length; 477 478 /* Set IV parameters */ 479 sess->iv.offset = xform->cipher.iv.offset; 480 sess->iv.length = xform->cipher.iv.length; 481 482 /* Select cipher algo */ 483 switch (xform->cipher.algo) { 484 case RTE_CRYPTO_CIPHER_3DES_CBC: 485 case RTE_CRYPTO_CIPHER_AES_CBC: 486 case RTE_CRYPTO_CIPHER_AES_CTR: 487 sess->cipher.mode = OPENSSL_CIPHER_LIB; 488 sess->cipher.algo = xform->cipher.algo; 489 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 490 491 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 492 &sess->cipher.evp_algo) != 0) 493 return -EINVAL; 494 495 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 496 sess->cipher.key.data); 497 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 498 if (EVP_EncryptInit_ex(sess->cipher.ctx, 499 sess->cipher.evp_algo, 500 NULL, xform->cipher.key.data, 501 NULL) != 1) { 502 return -EINVAL; 503 } 504 } else if (sess->cipher.direction == 505 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 506 if (EVP_DecryptInit_ex(sess->cipher.ctx, 507 sess->cipher.evp_algo, 508 NULL, xform->cipher.key.data, 509 NULL) != 1) { 510 return -EINVAL; 511 } 512 } 513 514 break; 515 516 case RTE_CRYPTO_CIPHER_3DES_CTR: 517 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 518 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 519 520 if (get_cipher_key_ede(xform->cipher.key.data, 521 sess->cipher.key.length, 522 sess->cipher.key.data) != 0) 523 return -EINVAL; 524 break; 525 526 case RTE_CRYPTO_CIPHER_DES_CBC: 527 sess->cipher.algo = xform->cipher.algo; 528 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 529 sess->cipher.evp_algo = EVP_des_cbc(); 530 531 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 532 sess->cipher.key.data); 533 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 534 if (EVP_EncryptInit_ex(sess->cipher.ctx, 535 sess->cipher.evp_algo, 536 NULL, xform->cipher.key.data, 537 NULL) != 1) { 538 return -EINVAL; 539 } 540 } else if (sess->cipher.direction == 541 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 542 if (EVP_DecryptInit_ex(sess->cipher.ctx, 543 sess->cipher.evp_algo, 544 NULL, xform->cipher.key.data, 545 NULL) != 1) { 546 return -EINVAL; 547 } 548 } 549 550 break; 551 552 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 553 sess->cipher.algo = xform->cipher.algo; 554 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 555 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 556 sess->cipher.evp_algo = EVP_des_cbc(); 557 558 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 559 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 560 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 561 NULL, xform->cipher.key.data, 0) != 1) 562 return -EINVAL; 563 564 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 565 sess->cipher.key.data); 566 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 567 if (EVP_EncryptInit_ex(sess->cipher.ctx, 568 sess->cipher.evp_algo, 569 NULL, xform->cipher.key.data, 570 NULL) != 1) { 571 return -EINVAL; 572 } 573 } else if (sess->cipher.direction == 574 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 575 if (EVP_DecryptInit_ex(sess->cipher.ctx, 576 sess->cipher.evp_algo, 577 NULL, xform->cipher.key.data, 578 NULL) != 1) { 579 return -EINVAL; 580 } 581 } 582 583 break; 584 default: 585 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 586 return -ENOTSUP; 587 } 588 589 return 0; 590 } 591 592 /* Set session auth parameters */ 593 static int 594 openssl_set_session_auth_parameters(struct openssl_session *sess, 595 const struct rte_crypto_sym_xform *xform) 596 { 597 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 598 char algo_name[MAX_OSSL_ALGO_NAME_SIZE]; 599 OSSL_PARAM params[2]; 600 const char *algo; 601 EVP_MAC *mac; 602 # endif 603 /* Select auth generate/verify */ 604 sess->auth.operation = xform->auth.op; 605 sess->auth.algo = xform->auth.algo; 606 607 sess->auth.digest_length = xform->auth.digest_length; 608 609 /* Select auth algo */ 610 switch (xform->auth.algo) { 611 case RTE_CRYPTO_AUTH_AES_GMAC: 612 /* 613 * OpenSSL requires GMAC to be a GCM operation 614 * with no cipher data length 615 */ 616 sess->cipher.key.length = xform->auth.key.length; 617 618 /* Set IV parameters */ 619 sess->iv.offset = xform->auth.iv.offset; 620 sess->iv.length = xform->auth.iv.length; 621 622 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 623 return openssl_set_sess_aead_enc_param(sess, 624 RTE_CRYPTO_AEAD_AES_GCM, 625 xform->auth.digest_length, 626 xform->auth.key.data); 627 else 628 return openssl_set_sess_aead_dec_param(sess, 629 RTE_CRYPTO_AEAD_AES_GCM, 630 xform->auth.digest_length, 631 xform->auth.key.data); 632 break; 633 634 case RTE_CRYPTO_AUTH_MD5: 635 case RTE_CRYPTO_AUTH_SHA1: 636 case RTE_CRYPTO_AUTH_SHA224: 637 case RTE_CRYPTO_AUTH_SHA256: 638 case RTE_CRYPTO_AUTH_SHA384: 639 case RTE_CRYPTO_AUTH_SHA512: 640 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 641 if (get_auth_algo(xform->auth.algo, 642 &sess->auth.auth.evp_algo) != 0) 643 return -EINVAL; 644 sess->auth.auth.ctx = EVP_MD_CTX_create(); 645 break; 646 647 case RTE_CRYPTO_AUTH_AES_CMAC: 648 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 649 if (xform->auth.key.length == 16) 650 algo = SN_aes_128_cbc; 651 else if (xform->auth.key.length == 24) 652 algo = SN_aes_192_cbc; 653 else if (xform->auth.key.length == 32) 654 algo = SN_aes_256_cbc; 655 else 656 return -EINVAL; 657 658 rte_memcpy(algo_name, algo, strlen(algo) + 1); 659 params[0] = OSSL_PARAM_construct_utf8_string( 660 OSSL_MAC_PARAM_CIPHER, algo_name, 0); 661 params[1] = OSSL_PARAM_construct_end(); 662 663 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 664 mac = EVP_MAC_fetch(NULL, OSSL_MAC_NAME_CMAC, NULL); 665 sess->auth.cmac.ctx = EVP_MAC_CTX_new(mac); 666 EVP_MAC_free(mac); 667 668 if (EVP_MAC_init(sess->auth.cmac.ctx, 669 xform->auth.key.data, 670 xform->auth.key.length, 671 params) != 1) 672 return -EINVAL; 673 # else 674 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 675 sess->auth.cmac.ctx = CMAC_CTX_new(); 676 if (get_cipher_algo(RTE_CRYPTO_CIPHER_AES_CBC, 677 xform->auth.key.length, 678 &sess->auth.cmac.evp_algo) != 0) 679 return -EINVAL; 680 if (CMAC_Init(sess->auth.cmac.ctx, 681 xform->auth.key.data, 682 xform->auth.key.length, 683 sess->auth.cmac.evp_algo, NULL) != 1) 684 return -EINVAL; 685 # endif 686 break; 687 688 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 689 case RTE_CRYPTO_AUTH_MD5_HMAC: 690 case RTE_CRYPTO_AUTH_SHA1_HMAC: 691 case RTE_CRYPTO_AUTH_SHA224_HMAC: 692 case RTE_CRYPTO_AUTH_SHA256_HMAC: 693 case RTE_CRYPTO_AUTH_SHA384_HMAC: 694 case RTE_CRYPTO_AUTH_SHA512_HMAC: 695 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 696 697 algo = digest_name_get(xform->auth.algo); 698 if (!algo) 699 return -EINVAL; 700 strlcpy(algo_name, algo, sizeof(algo_name)); 701 702 mac = EVP_MAC_fetch(NULL, "HMAC", NULL); 703 sess->auth.hmac.ctx = EVP_MAC_CTX_new(mac); 704 EVP_MAC_free(mac); 705 if (get_auth_algo(xform->auth.algo, 706 &sess->auth.hmac.evp_algo) != 0) 707 return -EINVAL; 708 709 params[0] = OSSL_PARAM_construct_utf8_string("digest", 710 algo_name, 0); 711 params[1] = OSSL_PARAM_construct_end(); 712 if (EVP_MAC_init(sess->auth.hmac.ctx, 713 xform->auth.key.data, 714 xform->auth.key.length, 715 params) != 1) 716 return -EINVAL; 717 break; 718 # else 719 case RTE_CRYPTO_AUTH_MD5_HMAC: 720 case RTE_CRYPTO_AUTH_SHA1_HMAC: 721 case RTE_CRYPTO_AUTH_SHA224_HMAC: 722 case RTE_CRYPTO_AUTH_SHA256_HMAC: 723 case RTE_CRYPTO_AUTH_SHA384_HMAC: 724 case RTE_CRYPTO_AUTH_SHA512_HMAC: 725 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 726 sess->auth.hmac.ctx = HMAC_CTX_new(); 727 if (get_auth_algo(xform->auth.algo, 728 &sess->auth.hmac.evp_algo) != 0) 729 return -EINVAL; 730 731 if (HMAC_Init_ex(sess->auth.hmac.ctx, 732 xform->auth.key.data, 733 xform->auth.key.length, 734 sess->auth.hmac.evp_algo, NULL) != 1) 735 return -EINVAL; 736 break; 737 # endif 738 default: 739 return -ENOTSUP; 740 } 741 742 return 0; 743 } 744 745 /* Set session AEAD parameters */ 746 static int 747 openssl_set_session_aead_parameters(struct openssl_session *sess, 748 const struct rte_crypto_sym_xform *xform) 749 { 750 /* Select cipher key */ 751 sess->cipher.key.length = xform->aead.key.length; 752 753 /* Set IV parameters */ 754 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 755 /* 756 * For AES-CCM, the actual IV is placed 757 * one byte after the start of the IV field, 758 * according to the API. 759 */ 760 sess->iv.offset = xform->aead.iv.offset + 1; 761 else 762 sess->iv.offset = xform->aead.iv.offset; 763 764 sess->iv.length = xform->aead.iv.length; 765 766 sess->auth.aad_length = xform->aead.aad_length; 767 sess->auth.digest_length = xform->aead.digest_length; 768 769 sess->aead_algo = xform->aead.algo; 770 /* Select cipher direction */ 771 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 772 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 773 xform->aead.digest_length, xform->aead.key.data); 774 else 775 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 776 xform->aead.digest_length, xform->aead.key.data); 777 } 778 779 /** Parse crypto xform chain and set private session parameters */ 780 int 781 openssl_set_session_parameters(struct openssl_session *sess, 782 const struct rte_crypto_sym_xform *xform) 783 { 784 const struct rte_crypto_sym_xform *cipher_xform = NULL; 785 const struct rte_crypto_sym_xform *auth_xform = NULL; 786 const struct rte_crypto_sym_xform *aead_xform = NULL; 787 int ret; 788 789 sess->chain_order = openssl_get_chain_order(xform); 790 switch (sess->chain_order) { 791 case OPENSSL_CHAIN_ONLY_CIPHER: 792 cipher_xform = xform; 793 break; 794 case OPENSSL_CHAIN_ONLY_AUTH: 795 auth_xform = xform; 796 break; 797 case OPENSSL_CHAIN_CIPHER_AUTH: 798 cipher_xform = xform; 799 auth_xform = xform->next; 800 break; 801 case OPENSSL_CHAIN_AUTH_CIPHER: 802 auth_xform = xform; 803 cipher_xform = xform->next; 804 break; 805 case OPENSSL_CHAIN_COMBINED: 806 aead_xform = xform; 807 break; 808 default: 809 return -EINVAL; 810 } 811 812 /* Default IV length = 0 */ 813 sess->iv.length = 0; 814 815 /* cipher_xform must be check before auth_xform */ 816 if (cipher_xform) { 817 ret = openssl_set_session_cipher_parameters( 818 sess, cipher_xform); 819 if (ret != 0) { 820 OPENSSL_LOG(ERR, 821 "Invalid/unsupported cipher parameters"); 822 return ret; 823 } 824 } 825 826 if (auth_xform) { 827 ret = openssl_set_session_auth_parameters(sess, auth_xform); 828 if (ret != 0) { 829 OPENSSL_LOG(ERR, 830 "Invalid/unsupported auth parameters"); 831 return ret; 832 } 833 } 834 835 if (aead_xform) { 836 ret = openssl_set_session_aead_parameters(sess, aead_xform); 837 if (ret != 0) { 838 OPENSSL_LOG(ERR, 839 "Invalid/unsupported AEAD parameters"); 840 return ret; 841 } 842 } 843 844 return 0; 845 } 846 847 /** Reset private session parameters */ 848 void 849 openssl_reset_session(struct openssl_session *sess) 850 { 851 EVP_CIPHER_CTX_free(sess->cipher.ctx); 852 853 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 854 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 855 856 switch (sess->auth.mode) { 857 case OPENSSL_AUTH_AS_AUTH: 858 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 859 break; 860 case OPENSSL_AUTH_AS_HMAC: 861 EVP_PKEY_free(sess->auth.hmac.pkey); 862 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 863 EVP_MAC_CTX_free(sess->auth.hmac.ctx); 864 # else 865 HMAC_CTX_free(sess->auth.hmac.ctx); 866 # endif 867 break; 868 case OPENSSL_AUTH_AS_CMAC: 869 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 870 EVP_MAC_CTX_free(sess->auth.cmac.ctx); 871 # else 872 CMAC_CTX_free(sess->auth.cmac.ctx); 873 # endif 874 break; 875 default: 876 break; 877 } 878 } 879 880 /** Provide session for operation */ 881 static void * 882 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 883 { 884 struct openssl_session *sess = NULL; 885 struct openssl_asym_session *asym_sess = NULL; 886 887 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 888 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 889 /* get existing session */ 890 if (likely(op->sym->session != NULL)) 891 sess = CRYPTODEV_GET_SYM_SESS_PRIV( 892 op->sym->session); 893 } else { 894 if (likely(op->asym->session != NULL)) 895 asym_sess = (struct openssl_asym_session *) 896 op->asym->session->sess_private_data; 897 if (asym_sess == NULL) 898 op->status = 899 RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 900 return asym_sess; 901 } 902 } else { 903 struct rte_cryptodev_sym_session *_sess; 904 /* sessionless asymmetric not supported */ 905 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) 906 return NULL; 907 908 /* provide internal session */ 909 rte_mempool_get(qp->sess_mp, (void **)&_sess); 910 911 if (_sess == NULL) 912 return NULL; 913 914 sess = (struct openssl_session *)_sess->driver_priv_data; 915 916 if (unlikely(openssl_set_session_parameters(sess, 917 op->sym->xform) != 0)) { 918 rte_mempool_put(qp->sess_mp, _sess); 919 sess = NULL; 920 } 921 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 922 923 } 924 925 if (sess == NULL) 926 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 927 928 return sess; 929 } 930 931 /* 932 *------------------------------------------------------------------------------ 933 * Process Operations 934 *------------------------------------------------------------------------------ 935 */ 936 static inline int 937 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 938 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 939 { 940 struct rte_mbuf *m; 941 int dstlen; 942 int l, n = srclen; 943 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 944 945 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 946 m = m->next) 947 offset -= rte_pktmbuf_data_len(m); 948 949 if (m == 0) 950 return -1; 951 952 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 953 if (inplace) 954 *dst = src; 955 956 l = rte_pktmbuf_data_len(m) - offset; 957 if (srclen <= l) { 958 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 959 return -1; 960 *dst += l; 961 return 0; 962 } 963 964 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 965 return -1; 966 967 *dst += dstlen; 968 n -= l; 969 970 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 971 uint8_t diff = l - dstlen, rem; 972 973 src = rte_pktmbuf_mtod(m, uint8_t *); 974 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 975 if (diff && inplace) { 976 rem = RTE_MIN(l, 977 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 978 if (EVP_EncryptUpdate(ctx, temp, 979 &dstlen, src, rem) <= 0) 980 return -1; 981 n -= rem; 982 rte_memcpy(*dst, temp, diff); 983 rte_memcpy(src, temp + diff, rem); 984 src += rem; 985 l -= rem; 986 } 987 if (inplace) 988 *dst = src; 989 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 990 return -1; 991 *dst += dstlen; 992 n -= l; 993 } 994 995 return 0; 996 } 997 998 static inline int 999 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 1000 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 1001 { 1002 struct rte_mbuf *m; 1003 int dstlen; 1004 int l, n = srclen; 1005 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 1006 1007 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1008 m = m->next) 1009 offset -= rte_pktmbuf_data_len(m); 1010 1011 if (m == 0) 1012 return -1; 1013 1014 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1015 if (inplace) 1016 *dst = src; 1017 1018 l = rte_pktmbuf_data_len(m) - offset; 1019 if (srclen <= l) { 1020 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 1021 return -1; 1022 *dst += l; 1023 return 0; 1024 } 1025 1026 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1027 return -1; 1028 1029 *dst += dstlen; 1030 n -= l; 1031 1032 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1033 uint8_t diff = l - dstlen, rem; 1034 1035 src = rte_pktmbuf_mtod(m, uint8_t *); 1036 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 1037 if (diff && inplace) { 1038 rem = RTE_MIN(l, 1039 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 1040 if (EVP_DecryptUpdate(ctx, temp, 1041 &dstlen, src, rem) <= 0) 1042 return -1; 1043 n -= rem; 1044 rte_memcpy(*dst, temp, diff); 1045 rte_memcpy(src, temp + diff, rem); 1046 src += rem; 1047 l -= rem; 1048 } 1049 if (inplace) 1050 *dst = src; 1051 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1052 return -1; 1053 *dst += dstlen; 1054 n -= l; 1055 } 1056 1057 return 0; 1058 } 1059 1060 /** Process standard openssl cipher encryption */ 1061 static int 1062 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1063 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1064 uint8_t inplace) 1065 { 1066 int totlen; 1067 1068 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1069 goto process_cipher_encrypt_err; 1070 1071 EVP_CIPHER_CTX_set_padding(ctx, 0); 1072 1073 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1074 srclen, ctx, inplace)) 1075 goto process_cipher_encrypt_err; 1076 1077 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 1078 goto process_cipher_encrypt_err; 1079 1080 return 0; 1081 1082 process_cipher_encrypt_err: 1083 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 1084 return -EINVAL; 1085 } 1086 1087 /** Process standard openssl cipher encryption */ 1088 static int 1089 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 1090 uint8_t *iv, int srclen, 1091 EVP_CIPHER_CTX *ctx) 1092 { 1093 uint8_t i; 1094 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 1095 int encrypted_ivlen; 1096 1097 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 1098 iv, DES_BLOCK_SIZE) <= 0) 1099 goto process_cipher_encrypt_err; 1100 1101 for (i = 0; i < srclen; i++) 1102 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 1103 1104 return 0; 1105 1106 process_cipher_encrypt_err: 1107 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 1108 return -EINVAL; 1109 } 1110 /** Process standard openssl cipher decryption */ 1111 static int 1112 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1113 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1114 uint8_t inplace) 1115 { 1116 int totlen; 1117 1118 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1119 goto process_cipher_decrypt_err; 1120 1121 EVP_CIPHER_CTX_set_padding(ctx, 0); 1122 1123 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1124 srclen, ctx, inplace)) 1125 goto process_cipher_decrypt_err; 1126 1127 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 1128 goto process_cipher_decrypt_err; 1129 return 0; 1130 1131 process_cipher_decrypt_err: 1132 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 1133 return -EINVAL; 1134 } 1135 1136 /** Process cipher des 3 ctr encryption, decryption algorithm */ 1137 static int 1138 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 1139 int offset, uint8_t *iv, uint8_t *key, int srclen, 1140 EVP_CIPHER_CTX *ctx) 1141 { 1142 uint8_t ebuf[8], ctr[8]; 1143 int unused, n; 1144 struct rte_mbuf *m; 1145 uint8_t *src; 1146 int l; 1147 1148 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1149 m = m->next) 1150 offset -= rte_pktmbuf_data_len(m); 1151 1152 if (m == 0) 1153 goto process_cipher_des3ctr_err; 1154 1155 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1156 l = rte_pktmbuf_data_len(m) - offset; 1157 1158 /* We use 3DES encryption also for decryption. 1159 * IV is not important for 3DES ecb 1160 */ 1161 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 1162 goto process_cipher_des3ctr_err; 1163 1164 memcpy(ctr, iv, 8); 1165 1166 for (n = 0; n < srclen; n++) { 1167 if (n % 8 == 0) { 1168 if (EVP_EncryptUpdate(ctx, 1169 (unsigned char *)&ebuf, &unused, 1170 (const unsigned char *)&ctr, 8) <= 0) 1171 goto process_cipher_des3ctr_err; 1172 ctr_inc(ctr); 1173 } 1174 dst[n] = *(src++) ^ ebuf[n % 8]; 1175 1176 l--; 1177 if (!l) { 1178 m = m->next; 1179 if (m) { 1180 src = rte_pktmbuf_mtod(m, uint8_t *); 1181 l = rte_pktmbuf_data_len(m); 1182 } 1183 } 1184 } 1185 1186 return 0; 1187 1188 process_cipher_des3ctr_err: 1189 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 1190 return -EINVAL; 1191 } 1192 1193 /** Process AES-GCM encrypt algorithm */ 1194 static int 1195 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1196 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1197 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1198 { 1199 int len = 0; 1200 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1201 int unused = 0; 1202 uint8_t empty[] = {}; 1203 #endif 1204 1205 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1206 goto process_auth_encryption_gcm_err; 1207 1208 if (aadlen > 0) 1209 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1210 goto process_auth_encryption_gcm_err; 1211 1212 if (srclen > 0) 1213 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1214 srclen, ctx, 0)) 1215 goto process_auth_encryption_gcm_err; 1216 1217 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1218 /* Workaround open ssl bug in version less then 1.0.1f */ 1219 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1220 goto process_auth_encryption_gcm_err; 1221 #endif 1222 1223 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1224 goto process_auth_encryption_gcm_err; 1225 1226 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1227 goto process_auth_encryption_gcm_err; 1228 1229 return 0; 1230 1231 process_auth_encryption_gcm_err: 1232 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1233 return -EINVAL; 1234 } 1235 1236 /** Process AES-CCM encrypt algorithm */ 1237 static int 1238 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1239 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1240 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1241 { 1242 int len = 0; 1243 1244 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1245 goto process_auth_encryption_ccm_err; 1246 1247 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1248 goto process_auth_encryption_ccm_err; 1249 1250 if (aadlen > 0) 1251 /* 1252 * For AES-CCM, the actual AAD is placed 1253 * 18 bytes after the start of the AAD field, 1254 * according to the API. 1255 */ 1256 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1257 goto process_auth_encryption_ccm_err; 1258 1259 if (srclen >= 0) 1260 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1261 srclen, ctx, 0)) 1262 goto process_auth_encryption_ccm_err; 1263 1264 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1265 goto process_auth_encryption_ccm_err; 1266 1267 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1268 goto process_auth_encryption_ccm_err; 1269 1270 return 0; 1271 1272 process_auth_encryption_ccm_err: 1273 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1274 return -EINVAL; 1275 } 1276 1277 /** Process AES-GCM decrypt algorithm */ 1278 static int 1279 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1280 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1281 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1282 { 1283 int len = 0; 1284 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1285 int unused = 0; 1286 uint8_t empty[] = {}; 1287 #endif 1288 1289 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1290 goto process_auth_decryption_gcm_err; 1291 1292 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1293 goto process_auth_decryption_gcm_err; 1294 1295 if (aadlen > 0) 1296 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1297 goto process_auth_decryption_gcm_err; 1298 1299 if (srclen > 0) 1300 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1301 srclen, ctx, 0)) 1302 goto process_auth_decryption_gcm_err; 1303 1304 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1305 /* Workaround open ssl bug in version less then 1.0.1f */ 1306 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1307 goto process_auth_decryption_gcm_err; 1308 #endif 1309 1310 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1311 return -EFAULT; 1312 1313 return 0; 1314 1315 process_auth_decryption_gcm_err: 1316 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1317 return -EINVAL; 1318 } 1319 1320 /** Process AES-CCM decrypt algorithm */ 1321 static int 1322 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1323 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1324 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1325 EVP_CIPHER_CTX *ctx) 1326 { 1327 int len = 0; 1328 1329 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1330 goto process_auth_decryption_ccm_err; 1331 1332 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1333 goto process_auth_decryption_ccm_err; 1334 1335 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1336 goto process_auth_decryption_ccm_err; 1337 1338 if (aadlen > 0) 1339 /* 1340 * For AES-CCM, the actual AAD is placed 1341 * 18 bytes after the start of the AAD field, 1342 * according to the API. 1343 */ 1344 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1345 goto process_auth_decryption_ccm_err; 1346 1347 if (srclen >= 0) 1348 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1349 srclen, ctx, 0)) 1350 return -EFAULT; 1351 1352 return 0; 1353 1354 process_auth_decryption_ccm_err: 1355 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1356 return -EINVAL; 1357 } 1358 1359 /** Process standard openssl auth algorithms */ 1360 static int 1361 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1362 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1363 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1364 { 1365 size_t dstlen; 1366 struct rte_mbuf *m; 1367 int l, n = srclen; 1368 uint8_t *src; 1369 1370 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1371 m = m->next) 1372 offset -= rte_pktmbuf_data_len(m); 1373 1374 if (m == 0) 1375 goto process_auth_err; 1376 1377 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1378 goto process_auth_err; 1379 1380 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1381 1382 l = rte_pktmbuf_data_len(m) - offset; 1383 if (srclen <= l) { 1384 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1385 goto process_auth_err; 1386 goto process_auth_final; 1387 } 1388 1389 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1390 goto process_auth_err; 1391 1392 n -= l; 1393 1394 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1395 src = rte_pktmbuf_mtod(m, uint8_t *); 1396 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1397 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1398 goto process_auth_err; 1399 n -= l; 1400 } 1401 1402 process_auth_final: 1403 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1404 goto process_auth_err; 1405 return 0; 1406 1407 process_auth_err: 1408 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1409 return -EINVAL; 1410 } 1411 1412 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1413 /** Process standard openssl auth algorithms with hmac/cmac */ 1414 static int 1415 process_openssl_auth_mac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1416 int srclen, EVP_MAC_CTX *ctx) 1417 { 1418 size_t dstlen; 1419 struct rte_mbuf *m; 1420 int l, n = srclen; 1421 uint8_t *src; 1422 1423 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1424 m = m->next) 1425 offset -= rte_pktmbuf_data_len(m); 1426 1427 if (m == 0) 1428 goto process_auth_err; 1429 1430 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1431 1432 l = rte_pktmbuf_data_len(m) - offset; 1433 if (srclen <= l) { 1434 if (EVP_MAC_update(ctx, (unsigned char *)src, srclen) != 1) 1435 goto process_auth_err; 1436 goto process_auth_final; 1437 } 1438 1439 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1440 goto process_auth_err; 1441 1442 n -= l; 1443 1444 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1445 src = rte_pktmbuf_mtod(m, uint8_t *); 1446 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1447 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1448 goto process_auth_err; 1449 n -= l; 1450 } 1451 1452 process_auth_final: 1453 if (EVP_MAC_final(ctx, dst, &dstlen, DIGEST_LENGTH_MAX) != 1) 1454 goto process_auth_err; 1455 1456 EVP_MAC_CTX_free(ctx); 1457 return 0; 1458 1459 process_auth_err: 1460 EVP_MAC_CTX_free(ctx); 1461 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1462 return -EINVAL; 1463 } 1464 # else 1465 /** Process standard openssl auth algorithms with hmac */ 1466 static int 1467 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1468 int srclen, HMAC_CTX *ctx) 1469 { 1470 unsigned int dstlen; 1471 struct rte_mbuf *m; 1472 int l, n = srclen; 1473 uint8_t *src; 1474 1475 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1476 m = m->next) 1477 offset -= rte_pktmbuf_data_len(m); 1478 1479 if (m == 0) 1480 goto process_auth_err; 1481 1482 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1483 1484 l = rte_pktmbuf_data_len(m) - offset; 1485 if (srclen <= l) { 1486 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1487 goto process_auth_err; 1488 goto process_auth_final; 1489 } 1490 1491 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1492 goto process_auth_err; 1493 1494 n -= l; 1495 1496 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1497 src = rte_pktmbuf_mtod(m, uint8_t *); 1498 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1499 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1500 goto process_auth_err; 1501 n -= l; 1502 } 1503 1504 process_auth_final: 1505 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1506 goto process_auth_err; 1507 1508 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1509 goto process_auth_err; 1510 1511 return 0; 1512 1513 process_auth_err: 1514 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1515 return -EINVAL; 1516 } 1517 1518 /** Process standard openssl auth algorithms with cmac */ 1519 static int 1520 process_openssl_auth_cmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1521 int srclen, CMAC_CTX *ctx) 1522 { 1523 unsigned int dstlen; 1524 struct rte_mbuf *m; 1525 int l, n = srclen; 1526 uint8_t *src; 1527 1528 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1529 m = m->next) 1530 offset -= rte_pktmbuf_data_len(m); 1531 1532 if (m == 0) 1533 goto process_auth_err; 1534 1535 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1536 1537 l = rte_pktmbuf_data_len(m) - offset; 1538 if (srclen <= l) { 1539 if (CMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1540 goto process_auth_err; 1541 goto process_auth_final; 1542 } 1543 1544 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1545 goto process_auth_err; 1546 1547 n -= l; 1548 1549 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1550 src = rte_pktmbuf_mtod(m, uint8_t *); 1551 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1552 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1553 goto process_auth_err; 1554 n -= l; 1555 } 1556 1557 process_auth_final: 1558 if (CMAC_Final(ctx, dst, (size_t *)&dstlen) != 1) 1559 goto process_auth_err; 1560 1561 CMAC_CTX_cleanup(ctx); 1562 1563 return 0; 1564 1565 process_auth_err: 1566 OPENSSL_LOG(ERR, "Process openssl cmac auth failed"); 1567 return -EINVAL; 1568 } 1569 # endif 1570 /*----------------------------------------------------------------------------*/ 1571 1572 /** Process auth/cipher combined operation */ 1573 static void 1574 process_openssl_combined_op 1575 (struct rte_crypto_op *op, struct openssl_session *sess, 1576 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1577 { 1578 /* cipher */ 1579 uint8_t *dst = NULL, *iv, *tag, *aad; 1580 int srclen, aadlen, status = -1; 1581 uint32_t offset; 1582 uint8_t taglen; 1583 1584 /* 1585 * Segmented destination buffer is not supported for 1586 * encryption/decryption 1587 */ 1588 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1589 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1590 return; 1591 } 1592 1593 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1594 sess->iv.offset); 1595 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1596 srclen = 0; 1597 offset = op->sym->auth.data.offset; 1598 aadlen = op->sym->auth.data.length; 1599 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1600 op->sym->auth.data.offset); 1601 tag = op->sym->auth.digest.data; 1602 if (tag == NULL) 1603 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1604 offset + aadlen); 1605 } else { 1606 srclen = op->sym->aead.data.length; 1607 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1608 op->sym->aead.data.offset); 1609 offset = op->sym->aead.data.offset; 1610 aad = op->sym->aead.aad.data; 1611 aadlen = sess->auth.aad_length; 1612 tag = op->sym->aead.digest.data; 1613 if (tag == NULL) 1614 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1615 offset + srclen); 1616 } 1617 1618 taglen = sess->auth.digest_length; 1619 1620 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1621 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1622 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1623 status = process_openssl_auth_encryption_gcm( 1624 mbuf_src, offset, srclen, 1625 aad, aadlen, iv, 1626 dst, tag, sess->cipher.ctx); 1627 else 1628 status = process_openssl_auth_encryption_ccm( 1629 mbuf_src, offset, srclen, 1630 aad, aadlen, iv, 1631 dst, tag, taglen, sess->cipher.ctx); 1632 1633 } else { 1634 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1635 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1636 status = process_openssl_auth_decryption_gcm( 1637 mbuf_src, offset, srclen, 1638 aad, aadlen, iv, 1639 dst, tag, sess->cipher.ctx); 1640 else 1641 status = process_openssl_auth_decryption_ccm( 1642 mbuf_src, offset, srclen, 1643 aad, aadlen, iv, 1644 dst, tag, taglen, sess->cipher.ctx); 1645 } 1646 1647 if (status != 0) { 1648 if (status == (-EFAULT) && 1649 sess->auth.operation == 1650 RTE_CRYPTO_AUTH_OP_VERIFY) 1651 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1652 else 1653 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1654 } 1655 } 1656 1657 /** Process cipher operation */ 1658 static void 1659 process_openssl_cipher_op 1660 (struct rte_crypto_op *op, struct openssl_session *sess, 1661 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1662 { 1663 uint8_t *dst, *iv; 1664 int srclen, status; 1665 uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0; 1666 EVP_CIPHER_CTX *ctx_copy; 1667 1668 /* 1669 * Segmented OOP destination buffer is not supported for encryption/ 1670 * decryption. In case of des3ctr, even inplace segmented buffers are 1671 * not supported. 1672 */ 1673 if (!rte_pktmbuf_is_contiguous(mbuf_dst) && 1674 (!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) { 1675 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1676 return; 1677 } 1678 1679 srclen = op->sym->cipher.data.length; 1680 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1681 op->sym->cipher.data.offset); 1682 1683 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1684 sess->iv.offset); 1685 ctx_copy = EVP_CIPHER_CTX_new(); 1686 EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx); 1687 1688 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1689 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1690 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1691 op->sym->cipher.data.offset, iv, 1692 srclen, ctx_copy, inplace); 1693 else 1694 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1695 op->sym->cipher.data.offset, iv, 1696 srclen, ctx_copy, inplace); 1697 else 1698 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1699 op->sym->cipher.data.offset, iv, 1700 sess->cipher.key.data, srclen, 1701 ctx_copy); 1702 1703 EVP_CIPHER_CTX_free(ctx_copy); 1704 if (status != 0) 1705 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1706 } 1707 1708 /** Process cipher operation */ 1709 static void 1710 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1711 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1712 struct rte_mbuf *mbuf_dst) 1713 { 1714 uint8_t *src, *dst, *iv; 1715 uint8_t block_size, last_block_len; 1716 int srclen, status = 0; 1717 1718 srclen = op->sym->cipher.data.length; 1719 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1720 op->sym->cipher.data.offset); 1721 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1722 op->sym->cipher.data.offset); 1723 1724 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1725 sess->iv.offset); 1726 1727 block_size = DES_BLOCK_SIZE; 1728 1729 last_block_len = srclen % block_size; 1730 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1731 /* Encrypt only with ECB mode XOR IV */ 1732 if (srclen < block_size) { 1733 status = process_openssl_cipher_bpi_encrypt(src, dst, 1734 iv, srclen, 1735 sess->cipher.bpi_ctx); 1736 } else { 1737 srclen -= last_block_len; 1738 /* Encrypt with the block aligned stream with CBC mode */ 1739 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1740 op->sym->cipher.data.offset, iv, 1741 srclen, sess->cipher.ctx, 0); 1742 if (last_block_len) { 1743 /* Point at last block */ 1744 dst += srclen; 1745 /* 1746 * IV is the last encrypted block from 1747 * the previous operation 1748 */ 1749 iv = dst - block_size; 1750 src += srclen; 1751 srclen = last_block_len; 1752 /* Encrypt the last frame with ECB mode */ 1753 status |= process_openssl_cipher_bpi_encrypt(src, 1754 dst, iv, 1755 srclen, sess->cipher.bpi_ctx); 1756 } 1757 } 1758 } else { 1759 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1760 if (srclen < block_size) { 1761 status = process_openssl_cipher_bpi_encrypt(src, dst, 1762 iv, 1763 srclen, 1764 sess->cipher.bpi_ctx); 1765 } else { 1766 if (last_block_len) { 1767 /* Point at last block */ 1768 dst += srclen - last_block_len; 1769 src += srclen - last_block_len; 1770 /* 1771 * IV is the last full block 1772 */ 1773 iv = src - block_size; 1774 /* 1775 * Decrypt the last frame with ECB mode 1776 * (encrypt, as it is the same operation) 1777 */ 1778 status = process_openssl_cipher_bpi_encrypt(src, 1779 dst, iv, 1780 last_block_len, sess->cipher.bpi_ctx); 1781 /* Prepare parameters for CBC mode op */ 1782 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1783 sess->iv.offset); 1784 dst += last_block_len - srclen; 1785 srclen -= last_block_len; 1786 } 1787 1788 /* Decrypt with CBC mode */ 1789 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1790 op->sym->cipher.data.offset, iv, 1791 srclen, sess->cipher.ctx, 0); 1792 } 1793 } 1794 1795 if (status != 0) 1796 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1797 } 1798 1799 /** Process auth operation */ 1800 static void 1801 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1802 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1803 struct rte_mbuf *mbuf_dst) 1804 { 1805 uint8_t *dst; 1806 int srclen, status; 1807 EVP_MD_CTX *ctx_a; 1808 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1809 EVP_MAC_CTX *ctx_h; 1810 EVP_MAC_CTX *ctx_c; 1811 # else 1812 HMAC_CTX *ctx_h; 1813 CMAC_CTX *ctx_c; 1814 # endif 1815 1816 srclen = op->sym->auth.data.length; 1817 1818 dst = qp->temp_digest; 1819 1820 switch (sess->auth.mode) { 1821 case OPENSSL_AUTH_AS_AUTH: 1822 ctx_a = EVP_MD_CTX_create(); 1823 EVP_MD_CTX_copy_ex(ctx_a, sess->auth.auth.ctx); 1824 status = process_openssl_auth(mbuf_src, dst, 1825 op->sym->auth.data.offset, NULL, NULL, srclen, 1826 ctx_a, sess->auth.auth.evp_algo); 1827 EVP_MD_CTX_destroy(ctx_a); 1828 break; 1829 case OPENSSL_AUTH_AS_HMAC: 1830 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1831 ctx_h = EVP_MAC_CTX_dup(sess->auth.hmac.ctx); 1832 status = process_openssl_auth_mac(mbuf_src, dst, 1833 op->sym->auth.data.offset, srclen, 1834 ctx_h); 1835 # else 1836 ctx_h = HMAC_CTX_new(); 1837 HMAC_CTX_copy(ctx_h, sess->auth.hmac.ctx); 1838 status = process_openssl_auth_hmac(mbuf_src, dst, 1839 op->sym->auth.data.offset, srclen, 1840 ctx_h); 1841 HMAC_CTX_free(ctx_h); 1842 # endif 1843 break; 1844 case OPENSSL_AUTH_AS_CMAC: 1845 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1846 ctx_c = EVP_MAC_CTX_dup(sess->auth.cmac.ctx); 1847 status = process_openssl_auth_mac(mbuf_src, dst, 1848 op->sym->auth.data.offset, srclen, 1849 ctx_c); 1850 # else 1851 ctx_c = CMAC_CTX_new(); 1852 CMAC_CTX_copy(ctx_c, sess->auth.cmac.ctx); 1853 status = process_openssl_auth_cmac(mbuf_src, dst, 1854 op->sym->auth.data.offset, srclen, 1855 ctx_c); 1856 CMAC_CTX_free(ctx_c); 1857 # endif 1858 break; 1859 default: 1860 status = -1; 1861 break; 1862 } 1863 1864 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1865 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data, 1866 sess->auth.digest_length) != 0) { 1867 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1868 } 1869 } else { 1870 uint8_t *auth_dst; 1871 1872 auth_dst = op->sym->auth.digest.data; 1873 if (auth_dst == NULL) 1874 auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1875 op->sym->auth.data.offset + 1876 op->sym->auth.data.length); 1877 memcpy(auth_dst, dst, sess->auth.digest_length); 1878 } 1879 1880 if (status != 0) 1881 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1882 } 1883 1884 /* process dsa sign operation */ 1885 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 1886 static int 1887 process_openssl_dsa_sign_op_evp(struct rte_crypto_op *cop, 1888 struct openssl_asym_session *sess) 1889 { 1890 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1891 EVP_PKEY_CTX *dsa_ctx = NULL; 1892 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 1893 EVP_PKEY *pkey = NULL; 1894 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 1895 OSSL_PARAM *params = NULL; 1896 1897 size_t outlen; 1898 unsigned char *dsa_sign_data; 1899 const unsigned char *dsa_sign_data_p; 1900 int ret = -1; 1901 1902 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1903 params = OSSL_PARAM_BLD_to_param(param_bld); 1904 if (!params) { 1905 OSSL_PARAM_BLD_free(param_bld); 1906 return -1; 1907 } 1908 1909 if (key_ctx == NULL 1910 || EVP_PKEY_fromdata_init(key_ctx) <= 0 1911 || EVP_PKEY_fromdata(key_ctx, &pkey, 1912 EVP_PKEY_KEYPAIR, params) <= 0) 1913 goto err_dsa_sign; 1914 1915 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 1916 if (!dsa_ctx) 1917 goto err_dsa_sign; 1918 1919 if (EVP_PKEY_sign_init(dsa_ctx) <= 0) 1920 goto err_dsa_sign; 1921 1922 if (EVP_PKEY_sign(dsa_ctx, NULL, &outlen, op->message.data, 1923 op->message.length) <= 0) 1924 goto err_dsa_sign; 1925 1926 if (outlen <= 0) 1927 goto err_dsa_sign; 1928 1929 dsa_sign_data = OPENSSL_malloc(outlen); 1930 if (!dsa_sign_data) 1931 goto err_dsa_sign; 1932 1933 if (EVP_PKEY_sign(dsa_ctx, dsa_sign_data, &outlen, op->message.data, 1934 op->message.length) <= 0) { 1935 OPENSSL_free(dsa_sign_data); 1936 goto err_dsa_sign; 1937 } 1938 1939 dsa_sign_data_p = (const unsigned char *)dsa_sign_data; 1940 DSA_SIG *sign = d2i_DSA_SIG(NULL, &dsa_sign_data_p, outlen); 1941 if (!sign) { 1942 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 1943 OPENSSL_free(dsa_sign_data); 1944 goto err_dsa_sign; 1945 } else { 1946 const BIGNUM *r = NULL, *s = NULL; 1947 get_dsa_sign(sign, &r, &s); 1948 1949 op->r.length = BN_bn2bin(r, op->r.data); 1950 op->s.length = BN_bn2bin(s, op->s.data); 1951 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1952 } 1953 1954 ret = 0; 1955 DSA_SIG_free(sign); 1956 OPENSSL_free(dsa_sign_data); 1957 1958 err_dsa_sign: 1959 if (params) 1960 OSSL_PARAM_free(params); 1961 EVP_PKEY_CTX_free(key_ctx); 1962 EVP_PKEY_CTX_free(dsa_ctx); 1963 return ret; 1964 } 1965 1966 /* process dsa verify operation */ 1967 static int 1968 process_openssl_dsa_verify_op_evp(struct rte_crypto_op *cop, 1969 struct openssl_asym_session *sess) 1970 { 1971 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1972 DSA_SIG *sign = DSA_SIG_new(); 1973 BIGNUM *r = NULL, *s = NULL; 1974 BIGNUM *pub_key = NULL; 1975 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 1976 OSSL_PARAM *params = NULL; 1977 EVP_PKEY *pkey = NULL; 1978 EVP_PKEY_CTX *dsa_ctx = NULL; 1979 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 1980 unsigned char *dsa_sig = NULL; 1981 size_t sig_len; 1982 int ret = -1; 1983 1984 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1985 if (!param_bld) { 1986 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 1987 return -1; 1988 } 1989 1990 r = BN_bin2bn(op->r.data, op->r.length, r); 1991 s = BN_bin2bn(op->s.data, op->s.length, s); 1992 pub_key = BN_bin2bn(op->y.data, op->y.length, pub_key); 1993 if (!r || !s || !pub_key) { 1994 BN_free(r); 1995 BN_free(s); 1996 BN_free(pub_key); 1997 OSSL_PARAM_BLD_free(param_bld); 1998 goto err_dsa_verify; 1999 } 2000 2001 set_dsa_sign(sign, r, s); 2002 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PUB_KEY, pub_key)) { 2003 OSSL_PARAM_BLD_free(param_bld); 2004 goto err_dsa_verify; 2005 } 2006 2007 params = OSSL_PARAM_BLD_to_param(param_bld); 2008 if (!params) { 2009 OSSL_PARAM_BLD_free(param_bld); 2010 goto err_dsa_verify; 2011 } 2012 2013 if (key_ctx == NULL 2014 || EVP_PKEY_fromdata_init(key_ctx) <= 0 2015 || EVP_PKEY_fromdata(key_ctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2016 goto err_dsa_verify; 2017 2018 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 2019 if (!dsa_ctx) 2020 goto err_dsa_verify; 2021 2022 if (!sign) 2023 goto err_dsa_verify; 2024 2025 sig_len = i2d_DSA_SIG(sign, &dsa_sig); 2026 if (EVP_PKEY_verify_init(dsa_ctx) <= 0) 2027 goto err_dsa_verify; 2028 2029 ret = EVP_PKEY_verify(dsa_ctx, dsa_sig, sig_len, 2030 op->message.data, op->message.length); 2031 if (ret == 1) { 2032 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2033 ret = 0; 2034 } 2035 2036 OPENSSL_free(dsa_sig); 2037 err_dsa_verify: 2038 if (sign) 2039 DSA_SIG_free(sign); 2040 if (params) 2041 OSSL_PARAM_free(params); 2042 EVP_PKEY_CTX_free(key_ctx); 2043 EVP_PKEY_CTX_free(dsa_ctx); 2044 2045 BN_free(pub_key); 2046 EVP_PKEY_free(pkey); 2047 2048 return ret; 2049 } 2050 #else 2051 static int 2052 process_openssl_dsa_sign_op(struct rte_crypto_op *cop, 2053 struct openssl_asym_session *sess) 2054 { 2055 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2056 DSA *dsa = sess->u.s.dsa; 2057 DSA_SIG *sign = NULL; 2058 2059 sign = DSA_do_sign(op->message.data, 2060 op->message.length, 2061 dsa); 2062 2063 if (sign == NULL) { 2064 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 2065 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2066 } else { 2067 const BIGNUM *r = NULL, *s = NULL; 2068 get_dsa_sign(sign, &r, &s); 2069 2070 op->r.length = BN_bn2bin(r, op->r.data); 2071 op->s.length = BN_bn2bin(s, op->s.data); 2072 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2073 } 2074 2075 DSA_SIG_free(sign); 2076 2077 return 0; 2078 } 2079 2080 /* process dsa verify operation */ 2081 static int 2082 process_openssl_dsa_verify_op(struct rte_crypto_op *cop, 2083 struct openssl_asym_session *sess) 2084 { 2085 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2086 DSA *dsa = sess->u.s.dsa; 2087 int ret; 2088 DSA_SIG *sign = DSA_SIG_new(); 2089 BIGNUM *r = NULL, *s = NULL; 2090 BIGNUM *pub_key = NULL; 2091 2092 if (sign == NULL) { 2093 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 2094 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2095 return -1; 2096 } 2097 2098 r = BN_bin2bn(op->r.data, 2099 op->r.length, 2100 r); 2101 s = BN_bin2bn(op->s.data, 2102 op->s.length, 2103 s); 2104 pub_key = BN_bin2bn(op->y.data, 2105 op->y.length, 2106 pub_key); 2107 if (!r || !s || !pub_key) { 2108 BN_free(r); 2109 BN_free(s); 2110 BN_free(pub_key); 2111 2112 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2113 return -1; 2114 } 2115 set_dsa_sign(sign, r, s); 2116 set_dsa_pub_key(dsa, pub_key); 2117 2118 ret = DSA_do_verify(op->message.data, 2119 op->message.length, 2120 sign, 2121 dsa); 2122 2123 if (ret != 1) 2124 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2125 else 2126 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2127 2128 DSA_SIG_free(sign); 2129 2130 return 0; 2131 } 2132 #endif 2133 2134 /* process dh operation */ 2135 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2136 static int 2137 process_openssl_dh_op_evp(struct rte_crypto_op *cop, 2138 struct openssl_asym_session *sess) 2139 { 2140 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2141 OSSL_PARAM_BLD *param_bld = sess->u.dh.param_bld; 2142 OSSL_PARAM_BLD *param_bld_peer = sess->u.dh.param_bld_peer; 2143 OSSL_PARAM *params = NULL; 2144 EVP_PKEY *dhpkey = NULL; 2145 EVP_PKEY *peerkey = NULL; 2146 BIGNUM *priv_key = NULL; 2147 BIGNUM *pub_key = NULL; 2148 int ret = -1; 2149 2150 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2151 EVP_PKEY_CTX *dh_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2152 if (dh_ctx == NULL || param_bld == NULL) 2153 return ret; 2154 2155 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2156 OSSL_PARAM *params_peer = NULL; 2157 2158 if (!param_bld_peer) 2159 return ret; 2160 2161 pub_key = BN_bin2bn(op->pub_key.data, op->pub_key.length, 2162 pub_key); 2163 if (pub_key == NULL) { 2164 OSSL_PARAM_BLD_free(param_bld_peer); 2165 return ret; 2166 } 2167 2168 if (!OSSL_PARAM_BLD_push_BN(param_bld_peer, OSSL_PKEY_PARAM_PUB_KEY, 2169 pub_key)) { 2170 OPENSSL_LOG(ERR, "Failed to set public key\n"); 2171 OSSL_PARAM_BLD_free(param_bld_peer); 2172 BN_free(pub_key); 2173 return ret; 2174 } 2175 2176 params_peer = OSSL_PARAM_BLD_to_param(param_bld_peer); 2177 if (!params_peer) { 2178 OSSL_PARAM_BLD_free(param_bld_peer); 2179 BN_free(pub_key); 2180 return ret; 2181 } 2182 2183 EVP_PKEY_CTX *peer_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2184 if (EVP_PKEY_keygen_init(peer_ctx) != 1) { 2185 OSSL_PARAM_free(params_peer); 2186 BN_free(pub_key); 2187 return ret; 2188 } 2189 2190 if (EVP_PKEY_CTX_set_params(peer_ctx, params_peer) != 1) { 2191 EVP_PKEY_CTX_free(peer_ctx); 2192 OSSL_PARAM_free(params_peer); 2193 BN_free(pub_key); 2194 return ret; 2195 } 2196 2197 if (EVP_PKEY_keygen(peer_ctx, &peerkey) != 1) { 2198 EVP_PKEY_CTX_free(peer_ctx); 2199 OSSL_PARAM_free(params_peer); 2200 BN_free(pub_key); 2201 return ret; 2202 } 2203 2204 priv_key = BN_bin2bn(op->priv_key.data, op->priv_key.length, 2205 priv_key); 2206 if (priv_key == NULL) { 2207 EVP_PKEY_CTX_free(peer_ctx); 2208 OSSL_PARAM_free(params_peer); 2209 BN_free(pub_key); 2210 return ret; 2211 } 2212 2213 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY, 2214 priv_key)) { 2215 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2216 EVP_PKEY_CTX_free(peer_ctx); 2217 OSSL_PARAM_free(params_peer); 2218 BN_free(pub_key); 2219 BN_free(priv_key); 2220 return ret; 2221 } 2222 2223 OSSL_PARAM_free(params_peer); 2224 EVP_PKEY_CTX_free(peer_ctx); 2225 } 2226 2227 params = OSSL_PARAM_BLD_to_param(param_bld); 2228 if (!params) 2229 goto err_dh; 2230 2231 if (EVP_PKEY_keygen_init(dh_ctx) != 1) 2232 goto err_dh; 2233 2234 if (EVP_PKEY_CTX_set_params(dh_ctx, params) != 1) 2235 goto err_dh; 2236 2237 if (EVP_PKEY_keygen(dh_ctx, &dhpkey) != 1) 2238 goto err_dh; 2239 2240 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2241 OPENSSL_LOG(DEBUG, "%s:%d updated pub key\n", __func__, __LINE__); 2242 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PUB_KEY, &pub_key)) 2243 goto err_dh; 2244 /* output public key */ 2245 op->pub_key.length = BN_bn2bin(pub_key, op->pub_key.data); 2246 } 2247 2248 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2249 2250 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", __func__, __LINE__); 2251 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PRIV_KEY, &priv_key)) 2252 goto err_dh; 2253 2254 /* provide generated private key back to user */ 2255 op->priv_key.length = BN_bn2bin(priv_key, op->priv_key.data); 2256 } 2257 2258 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2259 size_t skey_len; 2260 EVP_PKEY_CTX *sc_ctx = EVP_PKEY_CTX_new(dhpkey, NULL); 2261 if (!sc_ctx) 2262 goto err_dh; 2263 2264 if (EVP_PKEY_derive_init(sc_ctx) <= 0) { 2265 EVP_PKEY_CTX_free(sc_ctx); 2266 goto err_dh; 2267 } 2268 2269 if (!peerkey) { 2270 EVP_PKEY_CTX_free(sc_ctx); 2271 goto err_dh; 2272 } 2273 2274 if (EVP_PKEY_derive_set_peer(sc_ctx, peerkey) <= 0) { 2275 EVP_PKEY_CTX_free(sc_ctx); 2276 goto err_dh; 2277 } 2278 2279 /* Determine buffer length */ 2280 if (EVP_PKEY_derive(sc_ctx, NULL, &skey_len) <= 0) { 2281 EVP_PKEY_CTX_free(sc_ctx); 2282 goto err_dh; 2283 } 2284 2285 if (EVP_PKEY_derive(sc_ctx, op->shared_secret.data, &skey_len) <= 0) { 2286 EVP_PKEY_CTX_free(sc_ctx); 2287 goto err_dh; 2288 } 2289 2290 op->shared_secret.length = skey_len; 2291 EVP_PKEY_CTX_free(sc_ctx); 2292 } 2293 2294 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2295 ret = 0; 2296 2297 err_dh: 2298 BN_free(pub_key); 2299 BN_free(priv_key); 2300 if (params) 2301 OSSL_PARAM_free(params); 2302 EVP_PKEY_free(dhpkey); 2303 EVP_PKEY_free(peerkey); 2304 2305 EVP_PKEY_CTX_free(dh_ctx); 2306 2307 return ret; 2308 } 2309 #else 2310 static int 2311 process_openssl_dh_op(struct rte_crypto_op *cop, 2312 struct openssl_asym_session *sess) 2313 { 2314 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2315 struct rte_crypto_asym_op *asym_op = cop->asym; 2316 DH *dh_key = sess->u.dh.dh_key; 2317 BIGNUM *priv_key = NULL; 2318 int ret = 0; 2319 2320 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2321 /* compute shared secret using peer public key 2322 * and current private key 2323 * shared secret = peer_key ^ priv_key mod p 2324 */ 2325 BIGNUM *peer_key = NULL; 2326 2327 /* copy private key and peer key and compute shared secret */ 2328 peer_key = BN_bin2bn(op->pub_key.data, 2329 op->pub_key.length, 2330 peer_key); 2331 if (peer_key == NULL) { 2332 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2333 return -1; 2334 } 2335 priv_key = BN_bin2bn(op->priv_key.data, 2336 op->priv_key.length, 2337 priv_key); 2338 if (priv_key == NULL) { 2339 BN_free(peer_key); 2340 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2341 return -1; 2342 } 2343 ret = set_dh_priv_key(dh_key, priv_key); 2344 if (ret) { 2345 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2346 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2347 BN_free(peer_key); 2348 BN_free(priv_key); 2349 return 0; 2350 } 2351 2352 ret = DH_compute_key( 2353 op->shared_secret.data, 2354 peer_key, dh_key); 2355 if (ret < 0) { 2356 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2357 BN_free(peer_key); 2358 /* priv key is already loaded into dh, 2359 * let's not free that directly here. 2360 * DH_free() will auto free it later. 2361 */ 2362 return 0; 2363 } 2364 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2365 op->shared_secret.length = ret; 2366 BN_free(peer_key); 2367 return 0; 2368 } 2369 2370 /* 2371 * other options are public and private key generations. 2372 * 2373 * if user provides private key, 2374 * then first set DH with user provided private key 2375 */ 2376 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE && 2377 op->priv_key.length) { 2378 /* generate public key using user-provided private key 2379 * pub_key = g ^ priv_key mod p 2380 */ 2381 2382 /* load private key into DH */ 2383 priv_key = BN_bin2bn(op->priv_key.data, 2384 op->priv_key.length, 2385 priv_key); 2386 if (priv_key == NULL) { 2387 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2388 return -1; 2389 } 2390 ret = set_dh_priv_key(dh_key, priv_key); 2391 if (ret) { 2392 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2393 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2394 BN_free(priv_key); 2395 return 0; 2396 } 2397 } 2398 2399 /* generate public and private key pair. 2400 * 2401 * if private key already set, generates only public key. 2402 * 2403 * if private key is not already set, then set it to random value 2404 * and update internal private key. 2405 */ 2406 if (!DH_generate_key(dh_key)) { 2407 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2408 return 0; 2409 } 2410 2411 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2412 const BIGNUM *pub_key = NULL; 2413 2414 OPENSSL_LOG(DEBUG, "%s:%d update public key\n", 2415 __func__, __LINE__); 2416 2417 /* get the generated keys */ 2418 get_dh_pub_key(dh_key, &pub_key); 2419 2420 /* output public key */ 2421 op->pub_key.length = BN_bn2bin(pub_key, 2422 op->pub_key.data); 2423 } 2424 2425 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2426 const BIGNUM *priv_key = NULL; 2427 2428 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", 2429 __func__, __LINE__); 2430 2431 /* get the generated keys */ 2432 get_dh_priv_key(dh_key, &priv_key); 2433 2434 /* provide generated private key back to user */ 2435 op->priv_key.length = BN_bn2bin(priv_key, 2436 op->priv_key.data); 2437 } 2438 2439 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2440 2441 return 0; 2442 } 2443 #endif 2444 2445 /* process modinv operation */ 2446 static int 2447 process_openssl_modinv_op(struct rte_crypto_op *cop, 2448 struct openssl_asym_session *sess) 2449 { 2450 struct rte_crypto_asym_op *op = cop->asym; 2451 BIGNUM *base = BN_CTX_get(sess->u.m.ctx); 2452 BIGNUM *res = BN_CTX_get(sess->u.m.ctx); 2453 2454 if (unlikely(base == NULL || res == NULL)) { 2455 BN_free(base); 2456 BN_free(res); 2457 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2458 return -1; 2459 } 2460 2461 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 2462 op->modinv.base.length, base); 2463 2464 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) { 2465 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2466 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data); 2467 } else { 2468 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2469 } 2470 2471 BN_clear(res); 2472 BN_clear(base); 2473 2474 return 0; 2475 } 2476 2477 /* process modexp operation */ 2478 static int 2479 process_openssl_modexp_op(struct rte_crypto_op *cop, 2480 struct openssl_asym_session *sess) 2481 { 2482 struct rte_crypto_asym_op *op = cop->asym; 2483 BIGNUM *base = BN_CTX_get(sess->u.e.ctx); 2484 BIGNUM *res = BN_CTX_get(sess->u.e.ctx); 2485 2486 if (unlikely(base == NULL || res == NULL)) { 2487 BN_free(base); 2488 BN_free(res); 2489 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2490 return -1; 2491 } 2492 2493 base = BN_bin2bn((const unsigned char *)op->modex.base.data, 2494 op->modex.base.length, base); 2495 2496 if (BN_mod_exp(res, base, sess->u.e.exp, 2497 sess->u.e.mod, sess->u.e.ctx)) { 2498 op->modex.result.length = BN_bn2bin(res, op->modex.result.data); 2499 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2500 } else { 2501 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2502 } 2503 2504 BN_clear(res); 2505 BN_clear(base); 2506 2507 return 0; 2508 } 2509 2510 /* process rsa operations */ 2511 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2512 static int 2513 process_openssl_rsa_op_evp(struct rte_crypto_op *cop, 2514 struct openssl_asym_session *sess) 2515 { 2516 struct rte_crypto_asym_op *op = cop->asym; 2517 uint32_t pad = (op->rsa.padding.type); 2518 uint8_t *tmp; 2519 size_t outlen = 0; 2520 int ret = -1; 2521 2522 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2523 EVP_PKEY_CTX *rsa_ctx = sess->u.r.ctx; 2524 if (!rsa_ctx) 2525 return ret; 2526 2527 switch (pad) { 2528 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2529 pad = RSA_PKCS1_PADDING; 2530 break; 2531 case RTE_CRYPTO_RSA_PADDING_NONE: 2532 pad = RSA_NO_PADDING; 2533 break; 2534 default: 2535 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2536 OPENSSL_LOG(ERR, 2537 "rsa pad type not supported %d\n", pad); 2538 return ret; 2539 } 2540 2541 switch (op->rsa.op_type) { 2542 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2543 if (EVP_PKEY_encrypt_init(rsa_ctx) != 1) 2544 goto err_rsa; 2545 2546 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2547 goto err_rsa; 2548 2549 if (EVP_PKEY_encrypt(rsa_ctx, NULL, &outlen, 2550 op->rsa.message.data, 2551 op->rsa.message.length) <= 0) 2552 goto err_rsa; 2553 2554 if (outlen <= 0) 2555 goto err_rsa; 2556 2557 if (EVP_PKEY_encrypt(rsa_ctx, op->rsa.cipher.data, &outlen, 2558 op->rsa.message.data, 2559 op->rsa.message.length) <= 0) 2560 goto err_rsa; 2561 op->rsa.cipher.length = outlen; 2562 2563 OPENSSL_LOG(DEBUG, 2564 "length of encrypted text %zu\n", outlen); 2565 break; 2566 2567 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2568 if (EVP_PKEY_decrypt_init(rsa_ctx) != 1) 2569 goto err_rsa; 2570 2571 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2572 goto err_rsa; 2573 2574 if (EVP_PKEY_decrypt(rsa_ctx, NULL, &outlen, 2575 op->rsa.cipher.data, 2576 op->rsa.cipher.length) <= 0) 2577 goto err_rsa; 2578 2579 if (outlen <= 0) 2580 goto err_rsa; 2581 2582 if (EVP_PKEY_decrypt(rsa_ctx, op->rsa.message.data, &outlen, 2583 op->rsa.cipher.data, 2584 op->rsa.cipher.length) <= 0) 2585 goto err_rsa; 2586 op->rsa.message.length = outlen; 2587 2588 OPENSSL_LOG(DEBUG, "length of decrypted text %zu\n", outlen); 2589 break; 2590 2591 case RTE_CRYPTO_ASYM_OP_SIGN: 2592 if (EVP_PKEY_sign_init(rsa_ctx) <= 0) 2593 goto err_rsa; 2594 2595 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2596 goto err_rsa; 2597 2598 if (EVP_PKEY_sign(rsa_ctx, NULL, &outlen, 2599 op->rsa.message.data, 2600 op->rsa.message.length) <= 0) 2601 goto err_rsa; 2602 2603 if (outlen <= 0) 2604 goto err_rsa; 2605 2606 if (EVP_PKEY_sign(rsa_ctx, op->rsa.sign.data, &outlen, 2607 op->rsa.message.data, 2608 op->rsa.message.length) <= 0) 2609 goto err_rsa; 2610 op->rsa.sign.length = outlen; 2611 break; 2612 2613 case RTE_CRYPTO_ASYM_OP_VERIFY: 2614 if (EVP_PKEY_verify_recover_init(rsa_ctx) <= 0) 2615 goto err_rsa; 2616 2617 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2618 goto err_rsa; 2619 2620 if (EVP_PKEY_verify_recover(rsa_ctx, NULL, &outlen, 2621 op->rsa.sign.data, 2622 op->rsa.sign.length) <= 0) 2623 goto err_rsa; 2624 2625 if ((outlen <= 0) || (outlen != op->rsa.sign.length)) 2626 goto err_rsa; 2627 2628 tmp = OPENSSL_malloc(outlen); 2629 if (tmp == NULL) { 2630 OPENSSL_LOG(ERR, "Memory allocation failed"); 2631 goto err_rsa; 2632 } 2633 2634 if (EVP_PKEY_verify_recover(rsa_ctx, tmp, &outlen, 2635 op->rsa.sign.data, 2636 op->rsa.sign.length) <= 0) { 2637 OPENSSL_free(tmp); 2638 goto err_rsa; 2639 } 2640 2641 OPENSSL_LOG(DEBUG, 2642 "Length of public_decrypt %zu " 2643 "length of message %zd\n", 2644 outlen, op->rsa.message.length); 2645 if (CRYPTO_memcmp(tmp, op->rsa.message.data, 2646 op->rsa.message.length)) { 2647 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 2648 } 2649 OPENSSL_free(tmp); 2650 break; 2651 2652 default: 2653 /* allow ops with invalid args to be pushed to 2654 * completion queue 2655 */ 2656 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2657 goto err_rsa; 2658 } 2659 2660 ret = 0; 2661 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2662 err_rsa: 2663 return ret; 2664 2665 } 2666 2667 static int 2668 process_openssl_sm2_op_evp(struct rte_crypto_op *cop, 2669 struct openssl_asym_session *sess) 2670 { 2671 EVP_PKEY_CTX *kctx = NULL, *sctx = NULL, *cctx = NULL; 2672 struct rte_crypto_asym_op *op = cop->asym; 2673 OSSL_PARAM *params = sess->u.sm2.params; 2674 EVP_MD_CTX *md_ctx = NULL; 2675 ECDSA_SIG *ec_sign = NULL; 2676 EVP_MD *check_md = NULL; 2677 EVP_PKEY *pkey = NULL; 2678 int ret = -1; 2679 2680 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2681 2682 if (cop->asym->sm2.k.data != NULL) 2683 goto err_sm2; 2684 2685 switch (op->sm2.op_type) { 2686 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2687 { 2688 OSSL_PARAM *eparams = sess->u.sm2.params; 2689 size_t output_len = 0; 2690 2691 kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_SM2, NULL); 2692 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2693 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2694 goto err_sm2; 2695 2696 cctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2697 if (!cctx) 2698 goto err_sm2; 2699 2700 if (!EVP_PKEY_encrypt_init(cctx)) 2701 goto err_sm2; 2702 2703 if (!EVP_PKEY_CTX_set_params(cctx, eparams)) 2704 goto err_sm2; 2705 2706 if (!EVP_PKEY_encrypt(cctx, op->sm2.cipher.data, &output_len, 2707 op->sm2.message.data, 2708 op->sm2.message.length)) 2709 goto err_sm2; 2710 op->sm2.cipher.length = output_len; 2711 } 2712 break; 2713 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2714 { 2715 OSSL_PARAM *eparams = sess->u.sm2.params; 2716 2717 kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_SM2, NULL); 2718 if (kctx == NULL 2719 || EVP_PKEY_fromdata_init(kctx) <= 0 2720 || EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2721 goto err_sm2; 2722 2723 cctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2724 if (!cctx) 2725 goto err_sm2; 2726 2727 if (!EVP_PKEY_decrypt_init(cctx)) 2728 goto err_sm2; 2729 2730 if (!EVP_PKEY_CTX_set_params(cctx, eparams)) 2731 goto err_sm2; 2732 2733 if (!EVP_PKEY_decrypt(cctx, op->sm2.message.data, &op->sm2.message.length, 2734 op->sm2.cipher.data, op->sm2.cipher.length)) 2735 goto err_sm2; 2736 } 2737 break; 2738 case RTE_CRYPTO_ASYM_OP_SIGN: 2739 { 2740 unsigned char signbuf[128] = {0}; 2741 const unsigned char *signptr; 2742 const BIGNUM *r, *s; 2743 size_t signlen; 2744 2745 kctx = EVP_PKEY_CTX_new_from_name(NULL, "SM2", NULL); 2746 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2747 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2748 goto err_sm2; 2749 2750 md_ctx = EVP_MD_CTX_new(); 2751 if (!md_ctx) 2752 goto err_sm2; 2753 2754 sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2755 if (!sctx) 2756 goto err_sm2; 2757 2758 EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx); 2759 2760 check_md = EVP_MD_fetch(NULL, "sm3", NULL); 2761 if (!check_md) 2762 goto err_sm2; 2763 2764 if (!EVP_DigestSignInit(md_ctx, NULL, check_md, NULL, pkey)) 2765 goto err_sm2; 2766 2767 if (EVP_PKEY_CTX_set1_id(sctx, op->sm2.id.data, op->sm2.id.length) <= 0) 2768 goto err_sm2; 2769 2770 if (!EVP_DigestSignUpdate(md_ctx, op->sm2.message.data, 2771 op->sm2.message.length)) 2772 goto err_sm2; 2773 2774 if (!EVP_DigestSignFinal(md_ctx, NULL, &signlen)) 2775 goto err_sm2; 2776 2777 if (!EVP_DigestSignFinal(md_ctx, signbuf, &signlen)) 2778 goto err_sm2; 2779 2780 signptr = signbuf; 2781 ec_sign = d2i_ECDSA_SIG(NULL, &signptr, signlen); 2782 if (!ec_sign) 2783 goto err_sm2; 2784 2785 r = ECDSA_SIG_get0_r(ec_sign); 2786 s = ECDSA_SIG_get0_s(ec_sign); 2787 if (!r || !s) 2788 goto err_sm2; 2789 2790 op->sm2.r.length = BN_num_bytes(r); 2791 op->sm2.s.length = BN_num_bytes(s); 2792 BN_bn2bin(r, op->sm2.r.data); 2793 BN_bn2bin(s, op->sm2.s.data); 2794 2795 ECDSA_SIG_free(ec_sign); 2796 } 2797 break; 2798 case RTE_CRYPTO_ASYM_OP_VERIFY: 2799 { 2800 unsigned char signbuf[128] = {0}, *signbuf_new = NULL; 2801 BIGNUM *r = NULL, *s = NULL; 2802 size_t signlen; 2803 2804 kctx = EVP_PKEY_CTX_new_from_name(NULL, "SM2", NULL); 2805 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2806 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_PUBLIC_KEY, params) <= 0) 2807 goto err_sm2; 2808 2809 if (!EVP_PKEY_is_a(pkey, "SM2")) 2810 goto err_sm2; 2811 2812 md_ctx = EVP_MD_CTX_new(); 2813 if (!md_ctx) 2814 goto err_sm2; 2815 2816 sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2817 if (!sctx) 2818 goto err_sm2; 2819 2820 EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx); 2821 2822 check_md = EVP_MD_fetch(NULL, "sm3", NULL); 2823 if (!check_md) 2824 goto err_sm2; 2825 2826 if (!EVP_DigestVerifyInit(md_ctx, NULL, check_md, NULL, pkey)) 2827 goto err_sm2; 2828 2829 if (EVP_PKEY_CTX_set1_id(sctx, op->sm2.id.data, op->sm2.id.length) <= 0) 2830 goto err_sm2; 2831 2832 if (!EVP_DigestVerifyUpdate(md_ctx, op->sm2.message.data, 2833 op->sm2.message.length)) 2834 goto err_sm2; 2835 2836 ec_sign = ECDSA_SIG_new(); 2837 if (!ec_sign) 2838 goto err_sm2; 2839 2840 r = BN_bin2bn(op->sm2.r.data, op->sm2.r.length, r); 2841 s = BN_bin2bn(op->sm2.s.data, op->sm2.s.length, s); 2842 if (!r || !s) 2843 goto err_sm2; 2844 2845 if (!ECDSA_SIG_set0(ec_sign, r, s)) { 2846 BN_free(r); 2847 BN_free(s); 2848 goto err_sm2; 2849 } 2850 2851 r = NULL; 2852 s = NULL; 2853 2854 signbuf_new = signbuf; 2855 signlen = i2d_ECDSA_SIG(ec_sign, (unsigned char **)&signbuf_new); 2856 if (signlen <= 0) 2857 goto err_sm2; 2858 2859 if (!EVP_DigestVerifyFinal(md_ctx, signbuf_new, signlen)) 2860 goto err_sm2; 2861 2862 BN_free(r); 2863 BN_free(s); 2864 ECDSA_SIG_free(ec_sign); 2865 } 2866 break; 2867 default: 2868 /* allow ops with invalid args to be pushed to 2869 * completion queue 2870 */ 2871 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2872 goto err_sm2; 2873 } 2874 2875 ret = 0; 2876 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2877 err_sm2: 2878 EVP_MD_free(check_md); 2879 EVP_MD_CTX_free(md_ctx); 2880 2881 EVP_PKEY_CTX_free(kctx); 2882 2883 EVP_PKEY_CTX_free(sctx); 2884 2885 EVP_PKEY_CTX_free(cctx); 2886 2887 EVP_PKEY_free(pkey); 2888 2889 return ret; 2890 } 2891 2892 #else 2893 static int 2894 process_openssl_rsa_op(struct rte_crypto_op *cop, 2895 struct openssl_asym_session *sess) 2896 { 2897 int ret = 0; 2898 struct rte_crypto_asym_op *op = cop->asym; 2899 RSA *rsa = sess->u.r.rsa; 2900 uint32_t pad = (op->rsa.padding.type); 2901 uint8_t *tmp; 2902 2903 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2904 2905 switch (pad) { 2906 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2907 pad = RSA_PKCS1_PADDING; 2908 break; 2909 case RTE_CRYPTO_RSA_PADDING_NONE: 2910 pad = RSA_NO_PADDING; 2911 break; 2912 default: 2913 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2914 OPENSSL_LOG(ERR, 2915 "rsa pad type not supported %d\n", pad); 2916 return 0; 2917 } 2918 2919 switch (op->rsa.op_type) { 2920 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2921 ret = RSA_public_encrypt(op->rsa.message.length, 2922 op->rsa.message.data, 2923 op->rsa.cipher.data, 2924 rsa, 2925 pad); 2926 2927 if (ret > 0) 2928 op->rsa.cipher.length = ret; 2929 OPENSSL_LOG(DEBUG, 2930 "length of encrypted text %d\n", ret); 2931 break; 2932 2933 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2934 ret = RSA_private_decrypt(op->rsa.cipher.length, 2935 op->rsa.cipher.data, 2936 op->rsa.message.data, 2937 rsa, 2938 pad); 2939 if (ret > 0) 2940 op->rsa.message.length = ret; 2941 break; 2942 2943 case RTE_CRYPTO_ASYM_OP_SIGN: 2944 ret = RSA_private_encrypt(op->rsa.message.length, 2945 op->rsa.message.data, 2946 op->rsa.sign.data, 2947 rsa, 2948 pad); 2949 if (ret > 0) 2950 op->rsa.sign.length = ret; 2951 break; 2952 2953 case RTE_CRYPTO_ASYM_OP_VERIFY: 2954 tmp = rte_malloc(NULL, op->rsa.sign.length, 0); 2955 if (tmp == NULL) { 2956 OPENSSL_LOG(ERR, "Memory allocation failed"); 2957 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2958 break; 2959 } 2960 ret = RSA_public_decrypt(op->rsa.sign.length, 2961 op->rsa.sign.data, 2962 tmp, 2963 rsa, 2964 pad); 2965 2966 OPENSSL_LOG(DEBUG, 2967 "Length of public_decrypt %d " 2968 "length of message %zd\n", 2969 ret, op->rsa.message.length); 2970 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data, 2971 op->rsa.message.length))) { 2972 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 2973 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2974 } 2975 rte_free(tmp); 2976 break; 2977 2978 default: 2979 /* allow ops with invalid args to be pushed to 2980 * completion queue 2981 */ 2982 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2983 break; 2984 } 2985 2986 if (ret < 0) 2987 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2988 2989 return 0; 2990 } 2991 2992 static int 2993 process_openssl_sm2_op(struct rte_crypto_op *cop, 2994 struct openssl_asym_session *sess) 2995 { 2996 RTE_SET_USED(cop); 2997 RTE_SET_USED(sess); 2998 return -ENOTSUP; 2999 } 3000 #endif 3001 3002 static int 3003 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op, 3004 struct openssl_asym_session *sess) 3005 { 3006 int retval = 0; 3007 3008 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 3009 3010 switch (sess->xfrm_type) { 3011 case RTE_CRYPTO_ASYM_XFORM_RSA: 3012 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3013 retval = process_openssl_rsa_op_evp(op, sess); 3014 # else 3015 retval = process_openssl_rsa_op(op, sess); 3016 #endif 3017 break; 3018 case RTE_CRYPTO_ASYM_XFORM_MODEX: 3019 retval = process_openssl_modexp_op(op, sess); 3020 break; 3021 case RTE_CRYPTO_ASYM_XFORM_MODINV: 3022 retval = process_openssl_modinv_op(op, sess); 3023 break; 3024 case RTE_CRYPTO_ASYM_XFORM_DH: 3025 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3026 retval = process_openssl_dh_op_evp(op, sess); 3027 # else 3028 retval = process_openssl_dh_op(op, sess); 3029 #endif 3030 break; 3031 case RTE_CRYPTO_ASYM_XFORM_DSA: 3032 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3033 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 3034 retval = process_openssl_dsa_sign_op_evp(op, sess); 3035 else if (op->asym->dsa.op_type == 3036 RTE_CRYPTO_ASYM_OP_VERIFY) 3037 retval = 3038 process_openssl_dsa_verify_op_evp(op, sess); 3039 #else 3040 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 3041 retval = process_openssl_dsa_sign_op(op, sess); 3042 else if (op->asym->dsa.op_type == 3043 RTE_CRYPTO_ASYM_OP_VERIFY) 3044 retval = 3045 process_openssl_dsa_verify_op(op, sess); 3046 else 3047 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3048 #endif 3049 break; 3050 case RTE_CRYPTO_ASYM_XFORM_SM2: 3051 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3052 retval = process_openssl_sm2_op_evp(op, sess); 3053 #else 3054 retval = process_openssl_sm2_op(op, sess); 3055 #endif 3056 break; 3057 default: 3058 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3059 break; 3060 } 3061 if (!retval) { 3062 /* op processed so push to completion queue as processed */ 3063 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 3064 if (retval) 3065 /* return error if failed to put in completion queue */ 3066 retval = -1; 3067 } 3068 3069 return retval; 3070 } 3071 3072 static void 3073 copy_plaintext(struct rte_mbuf *m_src, struct rte_mbuf *m_dst, 3074 struct rte_crypto_op *op) 3075 { 3076 uint8_t *p_src, *p_dst; 3077 3078 p_src = rte_pktmbuf_mtod(m_src, uint8_t *); 3079 p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *); 3080 3081 /** 3082 * Copy the content between cipher offset and auth offset 3083 * for generating correct digest. 3084 */ 3085 if (op->sym->cipher.data.offset > op->sym->auth.data.offset) 3086 memcpy(p_dst + op->sym->auth.data.offset, 3087 p_src + op->sym->auth.data.offset, 3088 op->sym->cipher.data.offset - 3089 op->sym->auth.data.offset); 3090 } 3091 3092 /** Process crypto operation for mbuf */ 3093 static int 3094 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 3095 struct openssl_session *sess) 3096 { 3097 struct rte_mbuf *msrc, *mdst; 3098 int retval; 3099 3100 msrc = op->sym->m_src; 3101 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 3102 3103 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 3104 3105 switch (sess->chain_order) { 3106 case OPENSSL_CHAIN_ONLY_CIPHER: 3107 process_openssl_cipher_op(op, sess, msrc, mdst); 3108 break; 3109 case OPENSSL_CHAIN_ONLY_AUTH: 3110 process_openssl_auth_op(qp, op, sess, msrc, mdst); 3111 break; 3112 case OPENSSL_CHAIN_CIPHER_AUTH: 3113 process_openssl_cipher_op(op, sess, msrc, mdst); 3114 /* OOP */ 3115 if (msrc != mdst) 3116 copy_plaintext(msrc, mdst, op); 3117 process_openssl_auth_op(qp, op, sess, mdst, mdst); 3118 break; 3119 case OPENSSL_CHAIN_AUTH_CIPHER: 3120 process_openssl_auth_op(qp, op, sess, msrc, mdst); 3121 process_openssl_cipher_op(op, sess, msrc, mdst); 3122 break; 3123 case OPENSSL_CHAIN_COMBINED: 3124 process_openssl_combined_op(op, sess, msrc, mdst); 3125 break; 3126 case OPENSSL_CHAIN_CIPHER_BPI: 3127 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 3128 break; 3129 default: 3130 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3131 break; 3132 } 3133 3134 /* Free session if a session-less crypto op */ 3135 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 3136 openssl_reset_session(sess); 3137 memset(sess, 0, sizeof(struct openssl_session)); 3138 rte_mempool_put(qp->sess_mp, op->sym->session); 3139 op->sym->session = NULL; 3140 } 3141 3142 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 3143 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3144 3145 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 3146 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 3147 else 3148 retval = -1; 3149 3150 return retval; 3151 } 3152 3153 /* 3154 *------------------------------------------------------------------------------ 3155 * PMD Framework 3156 *------------------------------------------------------------------------------ 3157 */ 3158 3159 /** Enqueue burst */ 3160 static uint16_t 3161 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 3162 uint16_t nb_ops) 3163 { 3164 void *sess; 3165 struct openssl_qp *qp = queue_pair; 3166 int i, retval; 3167 3168 for (i = 0; i < nb_ops; i++) { 3169 sess = get_session(qp, ops[i]); 3170 if (unlikely(sess == NULL)) 3171 goto enqueue_err; 3172 3173 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) 3174 retval = process_op(qp, ops[i], 3175 (struct openssl_session *) sess); 3176 else 3177 retval = process_asym_op(qp, ops[i], 3178 (struct openssl_asym_session *) sess); 3179 if (unlikely(retval < 0)) 3180 goto enqueue_err; 3181 } 3182 3183 qp->stats.enqueued_count += i; 3184 return i; 3185 3186 enqueue_err: 3187 qp->stats.enqueue_err_count++; 3188 return i; 3189 } 3190 3191 /** Dequeue burst */ 3192 static uint16_t 3193 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 3194 uint16_t nb_ops) 3195 { 3196 struct openssl_qp *qp = queue_pair; 3197 3198 unsigned int nb_dequeued = 0; 3199 3200 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 3201 (void **)ops, nb_ops, NULL); 3202 qp->stats.dequeued_count += nb_dequeued; 3203 3204 return nb_dequeued; 3205 } 3206 3207 /** Create OPENSSL crypto device */ 3208 static int 3209 cryptodev_openssl_create(const char *name, 3210 struct rte_vdev_device *vdev, 3211 struct rte_cryptodev_pmd_init_params *init_params) 3212 { 3213 struct rte_cryptodev *dev; 3214 struct openssl_private *internals; 3215 3216 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 3217 if (dev == NULL) { 3218 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 3219 goto init_error; 3220 } 3221 3222 dev->driver_id = cryptodev_driver_id; 3223 dev->dev_ops = rte_openssl_pmd_ops; 3224 3225 /* register rx/tx burst functions for data path */ 3226 dev->dequeue_burst = openssl_pmd_dequeue_burst; 3227 dev->enqueue_burst = openssl_pmd_enqueue_burst; 3228 3229 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 3230 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 3231 RTE_CRYPTODEV_FF_CPU_AESNI | 3232 RTE_CRYPTODEV_FF_IN_PLACE_SGL | 3233 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 3234 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | 3235 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO | 3236 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP | 3237 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT | 3238 RTE_CRYPTODEV_FF_SYM_SESSIONLESS; 3239 3240 internals = dev->data->dev_private; 3241 3242 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 3243 3244 rte_cryptodev_pmd_probing_finish(dev); 3245 3246 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3247 /* Load legacy provider 3248 * Some algorithms are no longer available in earlier version of openssl, 3249 * unless the legacy provider explicitly loaded. e.g. DES 3250 */ 3251 ossl_legacy_provider_load(); 3252 # endif 3253 return 0; 3254 3255 init_error: 3256 OPENSSL_LOG(ERR, "driver %s: create failed", 3257 init_params->name); 3258 3259 cryptodev_openssl_remove(vdev); 3260 return -EFAULT; 3261 } 3262 3263 /** Initialise OPENSSL crypto device */ 3264 static int 3265 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 3266 { 3267 struct rte_cryptodev_pmd_init_params init_params = { 3268 "", 3269 sizeof(struct openssl_private), 3270 rte_socket_id(), 3271 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 3272 }; 3273 const char *name; 3274 const char *input_args; 3275 3276 name = rte_vdev_device_name(vdev); 3277 if (name == NULL) 3278 return -EINVAL; 3279 input_args = rte_vdev_device_args(vdev); 3280 3281 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 3282 3283 return cryptodev_openssl_create(name, vdev, &init_params); 3284 } 3285 3286 /** Uninitialise OPENSSL crypto device */ 3287 static int 3288 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 3289 { 3290 struct rte_cryptodev *cryptodev; 3291 const char *name; 3292 3293 name = rte_vdev_device_name(vdev); 3294 if (name == NULL) 3295 return -EINVAL; 3296 3297 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 3298 if (cryptodev == NULL) 3299 return -ENODEV; 3300 3301 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3302 ossl_legacy_provider_unload(); 3303 # endif 3304 return rte_cryptodev_pmd_destroy(cryptodev); 3305 } 3306 3307 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 3308 .probe = cryptodev_openssl_probe, 3309 .remove = cryptodev_openssl_remove 3310 }; 3311 3312 static struct cryptodev_driver openssl_crypto_drv; 3313 3314 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 3315 cryptodev_openssl_pmd_drv); 3316 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 3317 "max_nb_queue_pairs=<int> " 3318 "socket_id=<int>"); 3319 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 3320 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 3321 RTE_LOG_REGISTER_DEFAULT(openssl_logtype_driver, INFO); 3322