1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_common.h> 6 #include <rte_hexdump.h> 7 #include <rte_cryptodev.h> 8 #include <cryptodev_pmd.h> 9 #include <bus_vdev_driver.h> 10 #include <rte_malloc.h> 11 #include <rte_cpuflags.h> 12 13 #include <openssl/cmac.h> 14 #include <openssl/hmac.h> 15 #include <openssl/evp.h> 16 #include <openssl/ec.h> 17 18 #include "openssl_pmd_private.h" 19 #include "compat.h" 20 21 #define DES_BLOCK_SIZE 8 22 23 static uint8_t cryptodev_driver_id; 24 25 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 26 static HMAC_CTX *HMAC_CTX_new(void) 27 { 28 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 29 30 if (ctx != NULL) 31 HMAC_CTX_init(ctx); 32 return ctx; 33 } 34 35 static void HMAC_CTX_free(HMAC_CTX *ctx) 36 { 37 if (ctx != NULL) { 38 HMAC_CTX_cleanup(ctx); 39 OPENSSL_free(ctx); 40 } 41 } 42 #endif 43 44 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 45 46 #include <openssl/provider.h> 47 #include <openssl/core_names.h> 48 #include <openssl/param_build.h> 49 50 #define MAX_OSSL_ALGO_NAME_SIZE 16 51 52 OSSL_PROVIDER *legacy; 53 OSSL_PROVIDER *deflt; 54 55 static void ossl_legacy_provider_load(void) 56 { 57 /* Load Multiple providers into the default (NULL) library context */ 58 legacy = OSSL_PROVIDER_load(NULL, "legacy"); 59 if (legacy == NULL) { 60 OPENSSL_LOG(ERR, "Failed to load Legacy provider\n"); 61 return; 62 } 63 64 deflt = OSSL_PROVIDER_load(NULL, "default"); 65 if (deflt == NULL) { 66 OPENSSL_LOG(ERR, "Failed to load Default provider\n"); 67 OSSL_PROVIDER_unload(legacy); 68 return; 69 } 70 } 71 72 static void ossl_legacy_provider_unload(void) 73 { 74 OSSL_PROVIDER_unload(legacy); 75 OSSL_PROVIDER_unload(deflt); 76 } 77 78 static __rte_always_inline const char * 79 digest_name_get(enum rte_crypto_auth_algorithm algo) 80 { 81 switch (algo) { 82 case RTE_CRYPTO_AUTH_MD5_HMAC: 83 return OSSL_DIGEST_NAME_MD5; 84 case RTE_CRYPTO_AUTH_SHA1_HMAC: 85 return OSSL_DIGEST_NAME_SHA1; 86 case RTE_CRYPTO_AUTH_SHA224_HMAC: 87 return OSSL_DIGEST_NAME_SHA2_224; 88 case RTE_CRYPTO_AUTH_SHA256_HMAC: 89 return OSSL_DIGEST_NAME_SHA2_256; 90 case RTE_CRYPTO_AUTH_SHA384_HMAC: 91 return OSSL_DIGEST_NAME_SHA2_384; 92 case RTE_CRYPTO_AUTH_SHA512_HMAC: 93 return OSSL_DIGEST_NAME_SHA2_512; 94 default: 95 return NULL; 96 } 97 } 98 #endif 99 100 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 101 102 /*----------------------------------------------------------------------------*/ 103 104 /** 105 * Increment counter by 1 106 * Counter is 64 bit array, big-endian 107 */ 108 static void 109 ctr_inc(uint8_t *ctr) 110 { 111 uint64_t *ctr64 = (uint64_t *)ctr; 112 113 *ctr64 = __builtin_bswap64(*ctr64); 114 (*ctr64)++; 115 *ctr64 = __builtin_bswap64(*ctr64); 116 } 117 118 /* 119 *------------------------------------------------------------------------------ 120 * Session Prepare 121 *------------------------------------------------------------------------------ 122 */ 123 124 /** Get xform chain order */ 125 static enum openssl_chain_order 126 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 127 { 128 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 129 130 if (xform != NULL) { 131 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 132 if (xform->next == NULL) 133 res = OPENSSL_CHAIN_ONLY_AUTH; 134 else if (xform->next->type == 135 RTE_CRYPTO_SYM_XFORM_CIPHER) 136 res = OPENSSL_CHAIN_AUTH_CIPHER; 137 } 138 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 139 if (xform->next == NULL) 140 res = OPENSSL_CHAIN_ONLY_CIPHER; 141 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 142 res = OPENSSL_CHAIN_CIPHER_AUTH; 143 } 144 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 145 res = OPENSSL_CHAIN_COMBINED; 146 } 147 148 return res; 149 } 150 151 /** Get session cipher key from input cipher key */ 152 static void 153 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key) 154 { 155 memcpy(session_key, input_key, keylen); 156 } 157 158 /** Get key ede 24 bytes standard from input key */ 159 static int 160 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede) 161 { 162 int res = 0; 163 164 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 165 switch (keylen) { 166 case 24: 167 memcpy(key_ede, key, 24); 168 break; 169 case 16: 170 /* K3 = K1 */ 171 memcpy(key_ede, key, 16); 172 memcpy(key_ede + 16, key, 8); 173 break; 174 case 8: 175 /* K1 = K2 = K3 (DES compatibility) */ 176 memcpy(key_ede, key, 8); 177 memcpy(key_ede + 8, key, 8); 178 memcpy(key_ede + 16, key, 8); 179 break; 180 default: 181 OPENSSL_LOG(ERR, "Unsupported key size"); 182 res = -EINVAL; 183 } 184 185 return res; 186 } 187 188 /** Get adequate openssl function for input cipher algorithm */ 189 static uint8_t 190 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 191 const EVP_CIPHER **algo) 192 { 193 int res = 0; 194 195 if (algo != NULL) { 196 switch (sess_algo) { 197 case RTE_CRYPTO_CIPHER_3DES_CBC: 198 switch (keylen) { 199 case 8: 200 *algo = EVP_des_cbc(); 201 break; 202 case 16: 203 *algo = EVP_des_ede_cbc(); 204 break; 205 case 24: 206 *algo = EVP_des_ede3_cbc(); 207 break; 208 default: 209 res = -EINVAL; 210 } 211 break; 212 case RTE_CRYPTO_CIPHER_3DES_CTR: 213 break; 214 case RTE_CRYPTO_CIPHER_AES_CBC: 215 switch (keylen) { 216 case 16: 217 *algo = EVP_aes_128_cbc(); 218 break; 219 case 24: 220 *algo = EVP_aes_192_cbc(); 221 break; 222 case 32: 223 *algo = EVP_aes_256_cbc(); 224 break; 225 default: 226 res = -EINVAL; 227 } 228 break; 229 case RTE_CRYPTO_CIPHER_AES_CTR: 230 switch (keylen) { 231 case 16: 232 *algo = EVP_aes_128_ctr(); 233 break; 234 case 24: 235 *algo = EVP_aes_192_ctr(); 236 break; 237 case 32: 238 *algo = EVP_aes_256_ctr(); 239 break; 240 default: 241 res = -EINVAL; 242 } 243 break; 244 default: 245 res = -EINVAL; 246 break; 247 } 248 } else { 249 res = -EINVAL; 250 } 251 252 return res; 253 } 254 255 /** Get adequate openssl function for input auth algorithm */ 256 static uint8_t 257 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 258 const EVP_MD **algo) 259 { 260 int res = 0; 261 262 if (algo != NULL) { 263 switch (sessalgo) { 264 case RTE_CRYPTO_AUTH_MD5: 265 case RTE_CRYPTO_AUTH_MD5_HMAC: 266 *algo = EVP_md5(); 267 break; 268 case RTE_CRYPTO_AUTH_SHA1: 269 case RTE_CRYPTO_AUTH_SHA1_HMAC: 270 *algo = EVP_sha1(); 271 break; 272 case RTE_CRYPTO_AUTH_SHA224: 273 case RTE_CRYPTO_AUTH_SHA224_HMAC: 274 *algo = EVP_sha224(); 275 break; 276 case RTE_CRYPTO_AUTH_SHA256: 277 case RTE_CRYPTO_AUTH_SHA256_HMAC: 278 *algo = EVP_sha256(); 279 break; 280 case RTE_CRYPTO_AUTH_SHA384: 281 case RTE_CRYPTO_AUTH_SHA384_HMAC: 282 *algo = EVP_sha384(); 283 break; 284 case RTE_CRYPTO_AUTH_SHA512: 285 case RTE_CRYPTO_AUTH_SHA512_HMAC: 286 *algo = EVP_sha512(); 287 break; 288 default: 289 res = -EINVAL; 290 break; 291 } 292 } else { 293 res = -EINVAL; 294 } 295 296 return res; 297 } 298 299 /** Get adequate openssl function for input cipher algorithm */ 300 static uint8_t 301 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 302 const EVP_CIPHER **algo) 303 { 304 int res = 0; 305 306 if (algo != NULL) { 307 switch (sess_algo) { 308 case RTE_CRYPTO_AEAD_AES_GCM: 309 switch (keylen) { 310 case 16: 311 *algo = EVP_aes_128_gcm(); 312 break; 313 case 24: 314 *algo = EVP_aes_192_gcm(); 315 break; 316 case 32: 317 *algo = EVP_aes_256_gcm(); 318 break; 319 default: 320 res = -EINVAL; 321 } 322 break; 323 case RTE_CRYPTO_AEAD_AES_CCM: 324 switch (keylen) { 325 case 16: 326 *algo = EVP_aes_128_ccm(); 327 break; 328 case 24: 329 *algo = EVP_aes_192_ccm(); 330 break; 331 case 32: 332 *algo = EVP_aes_256_ccm(); 333 break; 334 default: 335 res = -EINVAL; 336 } 337 break; 338 default: 339 res = -EINVAL; 340 break; 341 } 342 } else { 343 res = -EINVAL; 344 } 345 346 return res; 347 } 348 349 /* Set session AEAD encryption parameters */ 350 static int 351 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 352 enum rte_crypto_aead_algorithm algo, 353 uint8_t tag_len, const uint8_t *key) 354 { 355 int iv_type = 0; 356 unsigned int do_ccm; 357 358 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 359 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 360 361 /* Select AEAD algo */ 362 switch (algo) { 363 case RTE_CRYPTO_AEAD_AES_GCM: 364 iv_type = EVP_CTRL_GCM_SET_IVLEN; 365 if (tag_len != 16) 366 return -EINVAL; 367 do_ccm = 0; 368 break; 369 case RTE_CRYPTO_AEAD_AES_CCM: 370 iv_type = EVP_CTRL_CCM_SET_IVLEN; 371 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 372 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 373 return -EINVAL; 374 do_ccm = 1; 375 break; 376 default: 377 return -ENOTSUP; 378 } 379 380 sess->cipher.mode = OPENSSL_CIPHER_LIB; 381 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 382 383 if (get_aead_algo(algo, sess->cipher.key.length, 384 &sess->cipher.evp_algo) != 0) 385 return -EINVAL; 386 387 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 388 389 sess->chain_order = OPENSSL_CHAIN_COMBINED; 390 391 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 392 NULL, NULL, NULL) <= 0) 393 return -EINVAL; 394 395 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, 396 NULL) <= 0) 397 return -EINVAL; 398 399 if (do_ccm) 400 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 401 tag_len, NULL); 402 403 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 404 return -EINVAL; 405 406 return 0; 407 } 408 409 /* Set session AEAD decryption parameters */ 410 static int 411 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 412 enum rte_crypto_aead_algorithm algo, 413 uint8_t tag_len, const uint8_t *key) 414 { 415 int iv_type = 0; 416 unsigned int do_ccm = 0; 417 418 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 419 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 420 421 /* Select AEAD algo */ 422 switch (algo) { 423 case RTE_CRYPTO_AEAD_AES_GCM: 424 iv_type = EVP_CTRL_GCM_SET_IVLEN; 425 if (tag_len != 16) 426 return -EINVAL; 427 break; 428 case RTE_CRYPTO_AEAD_AES_CCM: 429 iv_type = EVP_CTRL_CCM_SET_IVLEN; 430 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 431 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 432 return -EINVAL; 433 do_ccm = 1; 434 break; 435 default: 436 return -ENOTSUP; 437 } 438 439 sess->cipher.mode = OPENSSL_CIPHER_LIB; 440 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 441 442 if (get_aead_algo(algo, sess->cipher.key.length, 443 &sess->cipher.evp_algo) != 0) 444 return -EINVAL; 445 446 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 447 448 sess->chain_order = OPENSSL_CHAIN_COMBINED; 449 450 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 451 NULL, NULL, NULL) <= 0) 452 return -EINVAL; 453 454 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, 455 sess->iv.length, NULL) <= 0) 456 return -EINVAL; 457 458 if (do_ccm) 459 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 460 tag_len, NULL); 461 462 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 463 return -EINVAL; 464 465 return 0; 466 } 467 468 /** Set session cipher parameters */ 469 static int 470 openssl_set_session_cipher_parameters(struct openssl_session *sess, 471 const struct rte_crypto_sym_xform *xform) 472 { 473 /* Select cipher direction */ 474 sess->cipher.direction = xform->cipher.op; 475 /* Select cipher key */ 476 sess->cipher.key.length = xform->cipher.key.length; 477 478 /* Set IV parameters */ 479 sess->iv.offset = xform->cipher.iv.offset; 480 sess->iv.length = xform->cipher.iv.length; 481 482 /* Select cipher algo */ 483 switch (xform->cipher.algo) { 484 case RTE_CRYPTO_CIPHER_3DES_CBC: 485 case RTE_CRYPTO_CIPHER_AES_CBC: 486 case RTE_CRYPTO_CIPHER_AES_CTR: 487 sess->cipher.mode = OPENSSL_CIPHER_LIB; 488 sess->cipher.algo = xform->cipher.algo; 489 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 490 491 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 492 &sess->cipher.evp_algo) != 0) 493 return -EINVAL; 494 495 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 496 sess->cipher.key.data); 497 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 498 if (EVP_EncryptInit_ex(sess->cipher.ctx, 499 sess->cipher.evp_algo, 500 NULL, xform->cipher.key.data, 501 NULL) != 1) { 502 return -EINVAL; 503 } 504 } else if (sess->cipher.direction == 505 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 506 if (EVP_DecryptInit_ex(sess->cipher.ctx, 507 sess->cipher.evp_algo, 508 NULL, xform->cipher.key.data, 509 NULL) != 1) { 510 return -EINVAL; 511 } 512 } 513 514 break; 515 516 case RTE_CRYPTO_CIPHER_3DES_CTR: 517 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 518 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 519 520 if (get_cipher_key_ede(xform->cipher.key.data, 521 sess->cipher.key.length, 522 sess->cipher.key.data) != 0) 523 return -EINVAL; 524 break; 525 526 case RTE_CRYPTO_CIPHER_DES_CBC: 527 sess->cipher.algo = xform->cipher.algo; 528 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 529 sess->cipher.evp_algo = EVP_des_cbc(); 530 531 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 532 sess->cipher.key.data); 533 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 534 if (EVP_EncryptInit_ex(sess->cipher.ctx, 535 sess->cipher.evp_algo, 536 NULL, xform->cipher.key.data, 537 NULL) != 1) { 538 return -EINVAL; 539 } 540 } else if (sess->cipher.direction == 541 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 542 if (EVP_DecryptInit_ex(sess->cipher.ctx, 543 sess->cipher.evp_algo, 544 NULL, xform->cipher.key.data, 545 NULL) != 1) { 546 return -EINVAL; 547 } 548 } 549 550 break; 551 552 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 553 sess->cipher.algo = xform->cipher.algo; 554 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 555 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 556 sess->cipher.evp_algo = EVP_des_cbc(); 557 558 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 559 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 560 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 561 NULL, xform->cipher.key.data, 0) != 1) 562 return -EINVAL; 563 564 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 565 sess->cipher.key.data); 566 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 567 if (EVP_EncryptInit_ex(sess->cipher.ctx, 568 sess->cipher.evp_algo, 569 NULL, xform->cipher.key.data, 570 NULL) != 1) { 571 return -EINVAL; 572 } 573 } else if (sess->cipher.direction == 574 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 575 if (EVP_DecryptInit_ex(sess->cipher.ctx, 576 sess->cipher.evp_algo, 577 NULL, xform->cipher.key.data, 578 NULL) != 1) { 579 return -EINVAL; 580 } 581 } 582 583 break; 584 default: 585 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 586 return -ENOTSUP; 587 } 588 589 return 0; 590 } 591 592 /* Set session auth parameters */ 593 static int 594 openssl_set_session_auth_parameters(struct openssl_session *sess, 595 const struct rte_crypto_sym_xform *xform) 596 { 597 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 598 char algo_name[MAX_OSSL_ALGO_NAME_SIZE]; 599 OSSL_PARAM params[2]; 600 const char *algo; 601 EVP_MAC *mac; 602 # endif 603 /* Select auth generate/verify */ 604 sess->auth.operation = xform->auth.op; 605 sess->auth.algo = xform->auth.algo; 606 607 sess->auth.digest_length = xform->auth.digest_length; 608 609 /* Select auth algo */ 610 switch (xform->auth.algo) { 611 case RTE_CRYPTO_AUTH_AES_GMAC: 612 /* 613 * OpenSSL requires GMAC to be a GCM operation 614 * with no cipher data length 615 */ 616 sess->cipher.key.length = xform->auth.key.length; 617 618 /* Set IV parameters */ 619 sess->iv.offset = xform->auth.iv.offset; 620 sess->iv.length = xform->auth.iv.length; 621 622 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 623 return openssl_set_sess_aead_enc_param(sess, 624 RTE_CRYPTO_AEAD_AES_GCM, 625 xform->auth.digest_length, 626 xform->auth.key.data); 627 else 628 return openssl_set_sess_aead_dec_param(sess, 629 RTE_CRYPTO_AEAD_AES_GCM, 630 xform->auth.digest_length, 631 xform->auth.key.data); 632 break; 633 634 case RTE_CRYPTO_AUTH_MD5: 635 case RTE_CRYPTO_AUTH_SHA1: 636 case RTE_CRYPTO_AUTH_SHA224: 637 case RTE_CRYPTO_AUTH_SHA256: 638 case RTE_CRYPTO_AUTH_SHA384: 639 case RTE_CRYPTO_AUTH_SHA512: 640 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 641 if (get_auth_algo(xform->auth.algo, 642 &sess->auth.auth.evp_algo) != 0) 643 return -EINVAL; 644 sess->auth.auth.ctx = EVP_MD_CTX_create(); 645 break; 646 647 case RTE_CRYPTO_AUTH_AES_CMAC: 648 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 649 if (xform->auth.key.length == 16) 650 algo = SN_aes_128_cbc; 651 else if (xform->auth.key.length == 24) 652 algo = SN_aes_192_cbc; 653 else if (xform->auth.key.length == 32) 654 algo = SN_aes_256_cbc; 655 else 656 return -EINVAL; 657 658 rte_memcpy(algo_name, algo, strlen(algo) + 1); 659 params[0] = OSSL_PARAM_construct_utf8_string( 660 OSSL_MAC_PARAM_CIPHER, algo_name, 0); 661 params[1] = OSSL_PARAM_construct_end(); 662 663 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 664 mac = EVP_MAC_fetch(NULL, OSSL_MAC_NAME_CMAC, NULL); 665 sess->auth.cmac.ctx = EVP_MAC_CTX_new(mac); 666 EVP_MAC_free(mac); 667 668 if (EVP_MAC_init(sess->auth.cmac.ctx, 669 xform->auth.key.data, 670 xform->auth.key.length, 671 params) != 1) 672 return -EINVAL; 673 # else 674 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 675 sess->auth.cmac.ctx = CMAC_CTX_new(); 676 if (get_cipher_algo(RTE_CRYPTO_CIPHER_AES_CBC, 677 xform->auth.key.length, 678 &sess->auth.cmac.evp_algo) != 0) 679 return -EINVAL; 680 if (CMAC_Init(sess->auth.cmac.ctx, 681 xform->auth.key.data, 682 xform->auth.key.length, 683 sess->auth.cmac.evp_algo, NULL) != 1) 684 return -EINVAL; 685 # endif 686 break; 687 688 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 689 case RTE_CRYPTO_AUTH_MD5_HMAC: 690 case RTE_CRYPTO_AUTH_SHA1_HMAC: 691 case RTE_CRYPTO_AUTH_SHA224_HMAC: 692 case RTE_CRYPTO_AUTH_SHA256_HMAC: 693 case RTE_CRYPTO_AUTH_SHA384_HMAC: 694 case RTE_CRYPTO_AUTH_SHA512_HMAC: 695 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 696 697 algo = digest_name_get(xform->auth.algo); 698 if (!algo) 699 return -EINVAL; 700 strlcpy(algo_name, algo, sizeof(algo_name)); 701 702 mac = EVP_MAC_fetch(NULL, "HMAC", NULL); 703 sess->auth.hmac.ctx = EVP_MAC_CTX_new(mac); 704 EVP_MAC_free(mac); 705 if (get_auth_algo(xform->auth.algo, 706 &sess->auth.hmac.evp_algo) != 0) 707 return -EINVAL; 708 709 params[0] = OSSL_PARAM_construct_utf8_string("digest", 710 algo_name, 0); 711 params[1] = OSSL_PARAM_construct_end(); 712 if (EVP_MAC_init(sess->auth.hmac.ctx, 713 xform->auth.key.data, 714 xform->auth.key.length, 715 params) != 1) 716 return -EINVAL; 717 break; 718 # else 719 case RTE_CRYPTO_AUTH_MD5_HMAC: 720 case RTE_CRYPTO_AUTH_SHA1_HMAC: 721 case RTE_CRYPTO_AUTH_SHA224_HMAC: 722 case RTE_CRYPTO_AUTH_SHA256_HMAC: 723 case RTE_CRYPTO_AUTH_SHA384_HMAC: 724 case RTE_CRYPTO_AUTH_SHA512_HMAC: 725 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 726 sess->auth.hmac.ctx = HMAC_CTX_new(); 727 if (get_auth_algo(xform->auth.algo, 728 &sess->auth.hmac.evp_algo) != 0) 729 return -EINVAL; 730 731 if (HMAC_Init_ex(sess->auth.hmac.ctx, 732 xform->auth.key.data, 733 xform->auth.key.length, 734 sess->auth.hmac.evp_algo, NULL) != 1) 735 return -EINVAL; 736 break; 737 # endif 738 default: 739 return -ENOTSUP; 740 } 741 742 return 0; 743 } 744 745 /* Set session AEAD parameters */ 746 static int 747 openssl_set_session_aead_parameters(struct openssl_session *sess, 748 const struct rte_crypto_sym_xform *xform) 749 { 750 /* Select cipher key */ 751 sess->cipher.key.length = xform->aead.key.length; 752 753 /* Set IV parameters */ 754 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 755 /* 756 * For AES-CCM, the actual IV is placed 757 * one byte after the start of the IV field, 758 * according to the API. 759 */ 760 sess->iv.offset = xform->aead.iv.offset + 1; 761 else 762 sess->iv.offset = xform->aead.iv.offset; 763 764 sess->iv.length = xform->aead.iv.length; 765 766 sess->auth.aad_length = xform->aead.aad_length; 767 sess->auth.digest_length = xform->aead.digest_length; 768 769 sess->aead_algo = xform->aead.algo; 770 /* Select cipher direction */ 771 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 772 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 773 xform->aead.digest_length, xform->aead.key.data); 774 else 775 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 776 xform->aead.digest_length, xform->aead.key.data); 777 } 778 779 /** Parse crypto xform chain and set private session parameters */ 780 int 781 openssl_set_session_parameters(struct openssl_session *sess, 782 const struct rte_crypto_sym_xform *xform) 783 { 784 const struct rte_crypto_sym_xform *cipher_xform = NULL; 785 const struct rte_crypto_sym_xform *auth_xform = NULL; 786 const struct rte_crypto_sym_xform *aead_xform = NULL; 787 int ret; 788 789 sess->chain_order = openssl_get_chain_order(xform); 790 switch (sess->chain_order) { 791 case OPENSSL_CHAIN_ONLY_CIPHER: 792 cipher_xform = xform; 793 break; 794 case OPENSSL_CHAIN_ONLY_AUTH: 795 auth_xform = xform; 796 break; 797 case OPENSSL_CHAIN_CIPHER_AUTH: 798 cipher_xform = xform; 799 auth_xform = xform->next; 800 break; 801 case OPENSSL_CHAIN_AUTH_CIPHER: 802 auth_xform = xform; 803 cipher_xform = xform->next; 804 break; 805 case OPENSSL_CHAIN_COMBINED: 806 aead_xform = xform; 807 break; 808 default: 809 return -EINVAL; 810 } 811 812 /* Default IV length = 0 */ 813 sess->iv.length = 0; 814 815 /* cipher_xform must be check before auth_xform */ 816 if (cipher_xform) { 817 ret = openssl_set_session_cipher_parameters( 818 sess, cipher_xform); 819 if (ret != 0) { 820 OPENSSL_LOG(ERR, 821 "Invalid/unsupported cipher parameters"); 822 return ret; 823 } 824 } 825 826 if (auth_xform) { 827 ret = openssl_set_session_auth_parameters(sess, auth_xform); 828 if (ret != 0) { 829 OPENSSL_LOG(ERR, 830 "Invalid/unsupported auth parameters"); 831 return ret; 832 } 833 } 834 835 if (aead_xform) { 836 ret = openssl_set_session_aead_parameters(sess, aead_xform); 837 if (ret != 0) { 838 OPENSSL_LOG(ERR, 839 "Invalid/unsupported AEAD parameters"); 840 return ret; 841 } 842 } 843 844 return 0; 845 } 846 847 /** Reset private session parameters */ 848 void 849 openssl_reset_session(struct openssl_session *sess) 850 { 851 EVP_CIPHER_CTX_free(sess->cipher.ctx); 852 853 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 854 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 855 856 switch (sess->auth.mode) { 857 case OPENSSL_AUTH_AS_AUTH: 858 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 859 break; 860 case OPENSSL_AUTH_AS_HMAC: 861 EVP_PKEY_free(sess->auth.hmac.pkey); 862 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 863 EVP_MAC_CTX_free(sess->auth.hmac.ctx); 864 # else 865 HMAC_CTX_free(sess->auth.hmac.ctx); 866 # endif 867 break; 868 case OPENSSL_AUTH_AS_CMAC: 869 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 870 EVP_MAC_CTX_free(sess->auth.cmac.ctx); 871 # else 872 CMAC_CTX_free(sess->auth.cmac.ctx); 873 # endif 874 break; 875 default: 876 break; 877 } 878 } 879 880 /** Provide session for operation */ 881 static void * 882 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 883 { 884 struct openssl_session *sess = NULL; 885 struct openssl_asym_session *asym_sess = NULL; 886 887 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 888 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 889 /* get existing session */ 890 if (likely(op->sym->session != NULL)) 891 sess = CRYPTODEV_GET_SYM_SESS_PRIV( 892 op->sym->session); 893 } else { 894 if (likely(op->asym->session != NULL)) 895 asym_sess = (struct openssl_asym_session *) 896 op->asym->session->sess_private_data; 897 if (asym_sess == NULL) 898 op->status = 899 RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 900 return asym_sess; 901 } 902 } else { 903 struct rte_cryptodev_sym_session *_sess; 904 /* sessionless asymmetric not supported */ 905 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) 906 return NULL; 907 908 /* provide internal session */ 909 rte_mempool_get(qp->sess_mp, (void **)&_sess); 910 911 if (_sess == NULL) 912 return NULL; 913 914 sess = (struct openssl_session *)_sess->driver_priv_data; 915 916 if (unlikely(openssl_set_session_parameters(sess, 917 op->sym->xform) != 0)) { 918 rte_mempool_put(qp->sess_mp, _sess); 919 sess = NULL; 920 } 921 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 922 923 } 924 925 if (sess == NULL) 926 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 927 928 return sess; 929 } 930 931 /* 932 *------------------------------------------------------------------------------ 933 * Process Operations 934 *------------------------------------------------------------------------------ 935 */ 936 static inline int 937 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 938 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 939 { 940 struct rte_mbuf *m; 941 int dstlen; 942 int l, n = srclen; 943 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 944 945 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 946 m = m->next) 947 offset -= rte_pktmbuf_data_len(m); 948 949 if (m == 0) 950 return -1; 951 952 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 953 if (inplace) 954 *dst = src; 955 956 l = rte_pktmbuf_data_len(m) - offset; 957 if (srclen <= l) { 958 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 959 return -1; 960 *dst += l; 961 return 0; 962 } 963 964 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 965 return -1; 966 967 *dst += dstlen; 968 n -= l; 969 970 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 971 uint8_t diff = l - dstlen, rem; 972 973 src = rte_pktmbuf_mtod(m, uint8_t *); 974 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 975 if (diff && inplace) { 976 rem = RTE_MIN(l, 977 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 978 if (EVP_EncryptUpdate(ctx, temp, 979 &dstlen, src, rem) <= 0) 980 return -1; 981 n -= rem; 982 rte_memcpy(*dst, temp, diff); 983 rte_memcpy(src, temp + diff, rem); 984 src += rem; 985 l -= rem; 986 } 987 if (inplace) 988 *dst = src; 989 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 990 return -1; 991 *dst += dstlen; 992 n -= l; 993 } 994 995 return 0; 996 } 997 998 static inline int 999 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 1000 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 1001 { 1002 struct rte_mbuf *m; 1003 int dstlen; 1004 int l, n = srclen; 1005 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 1006 1007 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1008 m = m->next) 1009 offset -= rte_pktmbuf_data_len(m); 1010 1011 if (m == 0) 1012 return -1; 1013 1014 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1015 if (inplace) 1016 *dst = src; 1017 1018 l = rte_pktmbuf_data_len(m) - offset; 1019 if (srclen <= l) { 1020 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 1021 return -1; 1022 *dst += l; 1023 return 0; 1024 } 1025 1026 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1027 return -1; 1028 1029 *dst += dstlen; 1030 n -= l; 1031 1032 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1033 uint8_t diff = l - dstlen, rem; 1034 1035 src = rte_pktmbuf_mtod(m, uint8_t *); 1036 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 1037 if (diff && inplace) { 1038 rem = RTE_MIN(l, 1039 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 1040 if (EVP_DecryptUpdate(ctx, temp, 1041 &dstlen, src, rem) <= 0) 1042 return -1; 1043 n -= rem; 1044 rte_memcpy(*dst, temp, diff); 1045 rte_memcpy(src, temp + diff, rem); 1046 src += rem; 1047 l -= rem; 1048 } 1049 if (inplace) 1050 *dst = src; 1051 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1052 return -1; 1053 *dst += dstlen; 1054 n -= l; 1055 } 1056 1057 return 0; 1058 } 1059 1060 /** Process standard openssl cipher encryption */ 1061 static int 1062 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1063 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1064 uint8_t inplace) 1065 { 1066 int totlen; 1067 1068 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1069 goto process_cipher_encrypt_err; 1070 1071 EVP_CIPHER_CTX_set_padding(ctx, 0); 1072 1073 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1074 srclen, ctx, inplace)) 1075 goto process_cipher_encrypt_err; 1076 1077 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 1078 goto process_cipher_encrypt_err; 1079 1080 return 0; 1081 1082 process_cipher_encrypt_err: 1083 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 1084 return -EINVAL; 1085 } 1086 1087 /** Process standard openssl cipher encryption */ 1088 static int 1089 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 1090 uint8_t *iv, int srclen, 1091 EVP_CIPHER_CTX *ctx) 1092 { 1093 uint8_t i; 1094 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 1095 int encrypted_ivlen; 1096 1097 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 1098 iv, DES_BLOCK_SIZE) <= 0) 1099 goto process_cipher_encrypt_err; 1100 1101 for (i = 0; i < srclen; i++) 1102 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 1103 1104 return 0; 1105 1106 process_cipher_encrypt_err: 1107 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 1108 return -EINVAL; 1109 } 1110 /** Process standard openssl cipher decryption */ 1111 static int 1112 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1113 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1114 uint8_t inplace) 1115 { 1116 int totlen; 1117 1118 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1119 goto process_cipher_decrypt_err; 1120 1121 EVP_CIPHER_CTX_set_padding(ctx, 0); 1122 1123 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1124 srclen, ctx, inplace)) 1125 goto process_cipher_decrypt_err; 1126 1127 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 1128 goto process_cipher_decrypt_err; 1129 return 0; 1130 1131 process_cipher_decrypt_err: 1132 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 1133 return -EINVAL; 1134 } 1135 1136 /** Process cipher des 3 ctr encryption, decryption algorithm */ 1137 static int 1138 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 1139 int offset, uint8_t *iv, uint8_t *key, int srclen, 1140 EVP_CIPHER_CTX *ctx) 1141 { 1142 uint8_t ebuf[8], ctr[8]; 1143 int unused, n; 1144 struct rte_mbuf *m; 1145 uint8_t *src; 1146 int l; 1147 1148 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1149 m = m->next) 1150 offset -= rte_pktmbuf_data_len(m); 1151 1152 if (m == 0) 1153 goto process_cipher_des3ctr_err; 1154 1155 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1156 l = rte_pktmbuf_data_len(m) - offset; 1157 1158 /* We use 3DES encryption also for decryption. 1159 * IV is not important for 3DES ecb 1160 */ 1161 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 1162 goto process_cipher_des3ctr_err; 1163 1164 memcpy(ctr, iv, 8); 1165 1166 for (n = 0; n < srclen; n++) { 1167 if (n % 8 == 0) { 1168 if (EVP_EncryptUpdate(ctx, 1169 (unsigned char *)&ebuf, &unused, 1170 (const unsigned char *)&ctr, 8) <= 0) 1171 goto process_cipher_des3ctr_err; 1172 ctr_inc(ctr); 1173 } 1174 dst[n] = *(src++) ^ ebuf[n % 8]; 1175 1176 l--; 1177 if (!l) { 1178 m = m->next; 1179 if (m) { 1180 src = rte_pktmbuf_mtod(m, uint8_t *); 1181 l = rte_pktmbuf_data_len(m); 1182 } 1183 } 1184 } 1185 1186 return 0; 1187 1188 process_cipher_des3ctr_err: 1189 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 1190 return -EINVAL; 1191 } 1192 1193 /** Process AES-GCM encrypt algorithm */ 1194 static int 1195 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1196 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1197 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1198 { 1199 int len = 0; 1200 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1201 int unused = 0; 1202 uint8_t empty[] = {}; 1203 #endif 1204 1205 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1206 goto process_auth_encryption_gcm_err; 1207 1208 if (aadlen > 0) 1209 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1210 goto process_auth_encryption_gcm_err; 1211 1212 if (srclen > 0) 1213 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1214 srclen, ctx, 0)) 1215 goto process_auth_encryption_gcm_err; 1216 1217 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1218 /* Workaround open ssl bug in version less then 1.0.1f */ 1219 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1220 goto process_auth_encryption_gcm_err; 1221 #endif 1222 1223 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1224 goto process_auth_encryption_gcm_err; 1225 1226 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1227 goto process_auth_encryption_gcm_err; 1228 1229 return 0; 1230 1231 process_auth_encryption_gcm_err: 1232 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1233 return -EINVAL; 1234 } 1235 1236 /** Process AES-CCM encrypt algorithm */ 1237 static int 1238 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1239 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1240 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1241 { 1242 int len = 0; 1243 1244 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1245 goto process_auth_encryption_ccm_err; 1246 1247 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1248 goto process_auth_encryption_ccm_err; 1249 1250 if (aadlen > 0) 1251 /* 1252 * For AES-CCM, the actual AAD is placed 1253 * 18 bytes after the start of the AAD field, 1254 * according to the API. 1255 */ 1256 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1257 goto process_auth_encryption_ccm_err; 1258 1259 if (srclen >= 0) 1260 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1261 srclen, ctx, 0)) 1262 goto process_auth_encryption_ccm_err; 1263 1264 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1265 goto process_auth_encryption_ccm_err; 1266 1267 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1268 goto process_auth_encryption_ccm_err; 1269 1270 return 0; 1271 1272 process_auth_encryption_ccm_err: 1273 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1274 return -EINVAL; 1275 } 1276 1277 /** Process AES-GCM decrypt algorithm */ 1278 static int 1279 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1280 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1281 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1282 { 1283 int len = 0; 1284 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1285 int unused = 0; 1286 uint8_t empty[] = {}; 1287 #endif 1288 1289 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1290 goto process_auth_decryption_gcm_err; 1291 1292 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1293 goto process_auth_decryption_gcm_err; 1294 1295 if (aadlen > 0) 1296 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1297 goto process_auth_decryption_gcm_err; 1298 1299 if (srclen > 0) 1300 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1301 srclen, ctx, 0)) 1302 goto process_auth_decryption_gcm_err; 1303 1304 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1305 /* Workaround open ssl bug in version less then 1.0.1f */ 1306 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1307 goto process_auth_decryption_gcm_err; 1308 #endif 1309 1310 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1311 return -EFAULT; 1312 1313 return 0; 1314 1315 process_auth_decryption_gcm_err: 1316 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1317 return -EINVAL; 1318 } 1319 1320 /** Process AES-CCM decrypt algorithm */ 1321 static int 1322 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1323 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1324 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1325 EVP_CIPHER_CTX *ctx) 1326 { 1327 int len = 0; 1328 1329 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1330 goto process_auth_decryption_ccm_err; 1331 1332 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1333 goto process_auth_decryption_ccm_err; 1334 1335 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1336 goto process_auth_decryption_ccm_err; 1337 1338 if (aadlen > 0) 1339 /* 1340 * For AES-CCM, the actual AAD is placed 1341 * 18 bytes after the start of the AAD field, 1342 * according to the API. 1343 */ 1344 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1345 goto process_auth_decryption_ccm_err; 1346 1347 if (srclen >= 0) 1348 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1349 srclen, ctx, 0)) 1350 return -EFAULT; 1351 1352 return 0; 1353 1354 process_auth_decryption_ccm_err: 1355 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1356 return -EINVAL; 1357 } 1358 1359 /** Process standard openssl auth algorithms */ 1360 static int 1361 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1362 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1363 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1364 { 1365 size_t dstlen; 1366 struct rte_mbuf *m; 1367 int l, n = srclen; 1368 uint8_t *src; 1369 1370 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1371 m = m->next) 1372 offset -= rte_pktmbuf_data_len(m); 1373 1374 if (m == 0) 1375 goto process_auth_err; 1376 1377 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1378 goto process_auth_err; 1379 1380 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1381 1382 l = rte_pktmbuf_data_len(m) - offset; 1383 if (srclen <= l) { 1384 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1385 goto process_auth_err; 1386 goto process_auth_final; 1387 } 1388 1389 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1390 goto process_auth_err; 1391 1392 n -= l; 1393 1394 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1395 src = rte_pktmbuf_mtod(m, uint8_t *); 1396 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1397 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1398 goto process_auth_err; 1399 n -= l; 1400 } 1401 1402 process_auth_final: 1403 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1404 goto process_auth_err; 1405 return 0; 1406 1407 process_auth_err: 1408 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1409 return -EINVAL; 1410 } 1411 1412 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1413 /** Process standard openssl auth algorithms with hmac/cmac */ 1414 static int 1415 process_openssl_auth_mac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1416 int srclen, EVP_MAC_CTX *ctx) 1417 { 1418 size_t dstlen; 1419 struct rte_mbuf *m; 1420 int l, n = srclen; 1421 uint8_t *src; 1422 1423 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1424 m = m->next) 1425 offset -= rte_pktmbuf_data_len(m); 1426 1427 if (m == 0) 1428 goto process_auth_err; 1429 1430 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1431 1432 l = rte_pktmbuf_data_len(m) - offset; 1433 if (srclen <= l) { 1434 if (EVP_MAC_update(ctx, (unsigned char *)src, srclen) != 1) 1435 goto process_auth_err; 1436 goto process_auth_final; 1437 } 1438 1439 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1440 goto process_auth_err; 1441 1442 n -= l; 1443 1444 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1445 src = rte_pktmbuf_mtod(m, uint8_t *); 1446 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1447 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1448 goto process_auth_err; 1449 n -= l; 1450 } 1451 1452 process_auth_final: 1453 if (EVP_MAC_final(ctx, dst, &dstlen, DIGEST_LENGTH_MAX) != 1) 1454 goto process_auth_err; 1455 1456 EVP_MAC_CTX_free(ctx); 1457 return 0; 1458 1459 process_auth_err: 1460 EVP_MAC_CTX_free(ctx); 1461 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1462 return -EINVAL; 1463 } 1464 # else 1465 /** Process standard openssl auth algorithms with hmac */ 1466 static int 1467 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1468 int srclen, HMAC_CTX *ctx) 1469 { 1470 unsigned int dstlen; 1471 struct rte_mbuf *m; 1472 int l, n = srclen; 1473 uint8_t *src; 1474 1475 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1476 m = m->next) 1477 offset -= rte_pktmbuf_data_len(m); 1478 1479 if (m == 0) 1480 goto process_auth_err; 1481 1482 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1483 1484 l = rte_pktmbuf_data_len(m) - offset; 1485 if (srclen <= l) { 1486 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1487 goto process_auth_err; 1488 goto process_auth_final; 1489 } 1490 1491 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1492 goto process_auth_err; 1493 1494 n -= l; 1495 1496 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1497 src = rte_pktmbuf_mtod(m, uint8_t *); 1498 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1499 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1500 goto process_auth_err; 1501 n -= l; 1502 } 1503 1504 process_auth_final: 1505 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1506 goto process_auth_err; 1507 1508 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1509 goto process_auth_err; 1510 1511 return 0; 1512 1513 process_auth_err: 1514 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1515 return -EINVAL; 1516 } 1517 1518 /** Process standard openssl auth algorithms with cmac */ 1519 static int 1520 process_openssl_auth_cmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1521 int srclen, CMAC_CTX *ctx) 1522 { 1523 unsigned int dstlen; 1524 struct rte_mbuf *m; 1525 int l, n = srclen; 1526 uint8_t *src; 1527 1528 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1529 m = m->next) 1530 offset -= rte_pktmbuf_data_len(m); 1531 1532 if (m == 0) 1533 goto process_auth_err; 1534 1535 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1536 1537 l = rte_pktmbuf_data_len(m) - offset; 1538 if (srclen <= l) { 1539 if (CMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1540 goto process_auth_err; 1541 goto process_auth_final; 1542 } 1543 1544 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1545 goto process_auth_err; 1546 1547 n -= l; 1548 1549 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1550 src = rte_pktmbuf_mtod(m, uint8_t *); 1551 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1552 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1553 goto process_auth_err; 1554 n -= l; 1555 } 1556 1557 process_auth_final: 1558 if (CMAC_Final(ctx, dst, (size_t *)&dstlen) != 1) 1559 goto process_auth_err; 1560 1561 CMAC_CTX_cleanup(ctx); 1562 1563 return 0; 1564 1565 process_auth_err: 1566 OPENSSL_LOG(ERR, "Process openssl cmac auth failed"); 1567 return -EINVAL; 1568 } 1569 # endif 1570 /*----------------------------------------------------------------------------*/ 1571 1572 /** Process auth/cipher combined operation */ 1573 static void 1574 process_openssl_combined_op 1575 (struct rte_crypto_op *op, struct openssl_session *sess, 1576 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1577 { 1578 /* cipher */ 1579 uint8_t *dst = NULL, *iv, *tag, *aad; 1580 int srclen, aadlen, status = -1; 1581 uint32_t offset; 1582 uint8_t taglen; 1583 1584 /* 1585 * Segmented destination buffer is not supported for 1586 * encryption/decryption 1587 */ 1588 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1589 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1590 return; 1591 } 1592 1593 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1594 sess->iv.offset); 1595 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1596 srclen = 0; 1597 offset = op->sym->auth.data.offset; 1598 aadlen = op->sym->auth.data.length; 1599 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1600 op->sym->auth.data.offset); 1601 tag = op->sym->auth.digest.data; 1602 if (tag == NULL) 1603 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1604 offset + aadlen); 1605 } else { 1606 srclen = op->sym->aead.data.length; 1607 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1608 op->sym->aead.data.offset); 1609 offset = op->sym->aead.data.offset; 1610 aad = op->sym->aead.aad.data; 1611 aadlen = sess->auth.aad_length; 1612 tag = op->sym->aead.digest.data; 1613 if (tag == NULL) 1614 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1615 offset + srclen); 1616 } 1617 1618 taglen = sess->auth.digest_length; 1619 1620 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1621 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1622 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1623 status = process_openssl_auth_encryption_gcm( 1624 mbuf_src, offset, srclen, 1625 aad, aadlen, iv, 1626 dst, tag, sess->cipher.ctx); 1627 else 1628 status = process_openssl_auth_encryption_ccm( 1629 mbuf_src, offset, srclen, 1630 aad, aadlen, iv, 1631 dst, tag, taglen, sess->cipher.ctx); 1632 1633 } else { 1634 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1635 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1636 status = process_openssl_auth_decryption_gcm( 1637 mbuf_src, offset, srclen, 1638 aad, aadlen, iv, 1639 dst, tag, sess->cipher.ctx); 1640 else 1641 status = process_openssl_auth_decryption_ccm( 1642 mbuf_src, offset, srclen, 1643 aad, aadlen, iv, 1644 dst, tag, taglen, sess->cipher.ctx); 1645 } 1646 1647 if (status != 0) { 1648 if (status == (-EFAULT) && 1649 sess->auth.operation == 1650 RTE_CRYPTO_AUTH_OP_VERIFY) 1651 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1652 else 1653 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1654 } 1655 } 1656 1657 /** Process cipher operation */ 1658 static void 1659 process_openssl_cipher_op 1660 (struct rte_crypto_op *op, struct openssl_session *sess, 1661 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1662 { 1663 uint8_t *dst, *iv; 1664 int srclen, status; 1665 uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0; 1666 EVP_CIPHER_CTX *ctx_copy; 1667 1668 /* 1669 * Segmented OOP destination buffer is not supported for encryption/ 1670 * decryption. In case of des3ctr, even inplace segmented buffers are 1671 * not supported. 1672 */ 1673 if (!rte_pktmbuf_is_contiguous(mbuf_dst) && 1674 (!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) { 1675 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1676 return; 1677 } 1678 1679 srclen = op->sym->cipher.data.length; 1680 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1681 op->sym->cipher.data.offset); 1682 1683 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1684 sess->iv.offset); 1685 ctx_copy = EVP_CIPHER_CTX_new(); 1686 EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx); 1687 1688 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1689 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1690 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1691 op->sym->cipher.data.offset, iv, 1692 srclen, ctx_copy, inplace); 1693 else 1694 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1695 op->sym->cipher.data.offset, iv, 1696 srclen, ctx_copy, inplace); 1697 else 1698 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1699 op->sym->cipher.data.offset, iv, 1700 sess->cipher.key.data, srclen, 1701 ctx_copy); 1702 1703 EVP_CIPHER_CTX_free(ctx_copy); 1704 if (status != 0) 1705 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1706 } 1707 1708 /** Process cipher operation */ 1709 static void 1710 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1711 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1712 struct rte_mbuf *mbuf_dst) 1713 { 1714 uint8_t *src, *dst, *iv; 1715 uint8_t block_size, last_block_len; 1716 int srclen, status = 0; 1717 1718 srclen = op->sym->cipher.data.length; 1719 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1720 op->sym->cipher.data.offset); 1721 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1722 op->sym->cipher.data.offset); 1723 1724 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1725 sess->iv.offset); 1726 1727 block_size = DES_BLOCK_SIZE; 1728 1729 last_block_len = srclen % block_size; 1730 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1731 /* Encrypt only with ECB mode XOR IV */ 1732 if (srclen < block_size) { 1733 status = process_openssl_cipher_bpi_encrypt(src, dst, 1734 iv, srclen, 1735 sess->cipher.bpi_ctx); 1736 } else { 1737 srclen -= last_block_len; 1738 /* Encrypt with the block aligned stream with CBC mode */ 1739 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1740 op->sym->cipher.data.offset, iv, 1741 srclen, sess->cipher.ctx, 0); 1742 if (last_block_len) { 1743 /* Point at last block */ 1744 dst += srclen; 1745 /* 1746 * IV is the last encrypted block from 1747 * the previous operation 1748 */ 1749 iv = dst - block_size; 1750 src += srclen; 1751 srclen = last_block_len; 1752 /* Encrypt the last frame with ECB mode */ 1753 status |= process_openssl_cipher_bpi_encrypt(src, 1754 dst, iv, 1755 srclen, sess->cipher.bpi_ctx); 1756 } 1757 } 1758 } else { 1759 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1760 if (srclen < block_size) { 1761 status = process_openssl_cipher_bpi_encrypt(src, dst, 1762 iv, 1763 srclen, 1764 sess->cipher.bpi_ctx); 1765 } else { 1766 if (last_block_len) { 1767 /* Point at last block */ 1768 dst += srclen - last_block_len; 1769 src += srclen - last_block_len; 1770 /* 1771 * IV is the last full block 1772 */ 1773 iv = src - block_size; 1774 /* 1775 * Decrypt the last frame with ECB mode 1776 * (encrypt, as it is the same operation) 1777 */ 1778 status = process_openssl_cipher_bpi_encrypt(src, 1779 dst, iv, 1780 last_block_len, sess->cipher.bpi_ctx); 1781 /* Prepare parameters for CBC mode op */ 1782 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1783 sess->iv.offset); 1784 dst += last_block_len - srclen; 1785 srclen -= last_block_len; 1786 } 1787 1788 /* Decrypt with CBC mode */ 1789 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1790 op->sym->cipher.data.offset, iv, 1791 srclen, sess->cipher.ctx, 0); 1792 } 1793 } 1794 1795 if (status != 0) 1796 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1797 } 1798 1799 /** Process auth operation */ 1800 static void 1801 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1802 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1803 struct rte_mbuf *mbuf_dst) 1804 { 1805 uint8_t *dst; 1806 int srclen, status; 1807 EVP_MD_CTX *ctx_a; 1808 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1809 EVP_MAC_CTX *ctx_h; 1810 EVP_MAC_CTX *ctx_c; 1811 # else 1812 HMAC_CTX *ctx_h; 1813 CMAC_CTX *ctx_c; 1814 # endif 1815 1816 srclen = op->sym->auth.data.length; 1817 1818 dst = qp->temp_digest; 1819 1820 switch (sess->auth.mode) { 1821 case OPENSSL_AUTH_AS_AUTH: 1822 ctx_a = EVP_MD_CTX_create(); 1823 EVP_MD_CTX_copy_ex(ctx_a, sess->auth.auth.ctx); 1824 status = process_openssl_auth(mbuf_src, dst, 1825 op->sym->auth.data.offset, NULL, NULL, srclen, 1826 ctx_a, sess->auth.auth.evp_algo); 1827 EVP_MD_CTX_destroy(ctx_a); 1828 break; 1829 case OPENSSL_AUTH_AS_HMAC: 1830 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1831 ctx_h = EVP_MAC_CTX_dup(sess->auth.hmac.ctx); 1832 status = process_openssl_auth_mac(mbuf_src, dst, 1833 op->sym->auth.data.offset, srclen, 1834 ctx_h); 1835 # else 1836 ctx_h = HMAC_CTX_new(); 1837 HMAC_CTX_copy(ctx_h, sess->auth.hmac.ctx); 1838 status = process_openssl_auth_hmac(mbuf_src, dst, 1839 op->sym->auth.data.offset, srclen, 1840 ctx_h); 1841 HMAC_CTX_free(ctx_h); 1842 # endif 1843 break; 1844 case OPENSSL_AUTH_AS_CMAC: 1845 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1846 ctx_c = EVP_MAC_CTX_dup(sess->auth.cmac.ctx); 1847 status = process_openssl_auth_mac(mbuf_src, dst, 1848 op->sym->auth.data.offset, srclen, 1849 ctx_c); 1850 # else 1851 ctx_c = CMAC_CTX_new(); 1852 CMAC_CTX_copy(ctx_c, sess->auth.cmac.ctx); 1853 status = process_openssl_auth_cmac(mbuf_src, dst, 1854 op->sym->auth.data.offset, srclen, 1855 ctx_c); 1856 CMAC_CTX_free(ctx_c); 1857 # endif 1858 break; 1859 default: 1860 status = -1; 1861 break; 1862 } 1863 1864 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1865 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data, 1866 sess->auth.digest_length) != 0) { 1867 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1868 } 1869 } else { 1870 uint8_t *auth_dst; 1871 1872 auth_dst = op->sym->auth.digest.data; 1873 if (auth_dst == NULL) 1874 auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1875 op->sym->auth.data.offset + 1876 op->sym->auth.data.length); 1877 memcpy(auth_dst, dst, sess->auth.digest_length); 1878 } 1879 1880 if (status != 0) 1881 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1882 } 1883 1884 /* process dsa sign operation */ 1885 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 1886 static int 1887 process_openssl_dsa_sign_op_evp(struct rte_crypto_op *cop, 1888 struct openssl_asym_session *sess) 1889 { 1890 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1891 EVP_PKEY_CTX *dsa_ctx = NULL; 1892 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 1893 EVP_PKEY *pkey = NULL; 1894 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 1895 OSSL_PARAM *params = NULL; 1896 1897 size_t outlen; 1898 unsigned char *dsa_sign_data; 1899 const unsigned char *dsa_sign_data_p; 1900 int ret = -1; 1901 1902 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1903 params = OSSL_PARAM_BLD_to_param(param_bld); 1904 if (!params) { 1905 OSSL_PARAM_BLD_free(param_bld); 1906 return -1; 1907 } 1908 1909 if (key_ctx == NULL 1910 || EVP_PKEY_fromdata_init(key_ctx) <= 0 1911 || EVP_PKEY_fromdata(key_ctx, &pkey, 1912 EVP_PKEY_KEYPAIR, params) <= 0) 1913 goto err_dsa_sign; 1914 1915 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 1916 if (!dsa_ctx) 1917 goto err_dsa_sign; 1918 1919 if (EVP_PKEY_sign_init(dsa_ctx) <= 0) 1920 goto err_dsa_sign; 1921 1922 if (EVP_PKEY_sign(dsa_ctx, NULL, &outlen, op->message.data, 1923 op->message.length) <= 0) 1924 goto err_dsa_sign; 1925 1926 if (outlen <= 0) 1927 goto err_dsa_sign; 1928 1929 dsa_sign_data = OPENSSL_malloc(outlen); 1930 if (!dsa_sign_data) 1931 goto err_dsa_sign; 1932 1933 if (EVP_PKEY_sign(dsa_ctx, dsa_sign_data, &outlen, op->message.data, 1934 op->message.length) <= 0) { 1935 OPENSSL_free(dsa_sign_data); 1936 goto err_dsa_sign; 1937 } 1938 1939 dsa_sign_data_p = (const unsigned char *)dsa_sign_data; 1940 DSA_SIG *sign = d2i_DSA_SIG(NULL, &dsa_sign_data_p, outlen); 1941 if (!sign) { 1942 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 1943 OPENSSL_free(dsa_sign_data); 1944 goto err_dsa_sign; 1945 } else { 1946 const BIGNUM *r = NULL, *s = NULL; 1947 get_dsa_sign(sign, &r, &s); 1948 1949 op->r.length = BN_bn2bin(r, op->r.data); 1950 op->s.length = BN_bn2bin(s, op->s.data); 1951 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1952 } 1953 1954 ret = 0; 1955 DSA_SIG_free(sign); 1956 OPENSSL_free(dsa_sign_data); 1957 1958 err_dsa_sign: 1959 if (params) 1960 OSSL_PARAM_free(params); 1961 EVP_PKEY_CTX_free(key_ctx); 1962 EVP_PKEY_CTX_free(dsa_ctx); 1963 EVP_PKEY_free(pkey); 1964 return ret; 1965 } 1966 1967 /* process dsa verify operation */ 1968 static int 1969 process_openssl_dsa_verify_op_evp(struct rte_crypto_op *cop, 1970 struct openssl_asym_session *sess) 1971 { 1972 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1973 DSA_SIG *sign = DSA_SIG_new(); 1974 BIGNUM *r = NULL, *s = NULL; 1975 BIGNUM *pub_key = NULL; 1976 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 1977 OSSL_PARAM *params = NULL; 1978 EVP_PKEY *pkey = NULL; 1979 EVP_PKEY_CTX *dsa_ctx = NULL; 1980 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 1981 unsigned char *dsa_sig = NULL; 1982 size_t sig_len; 1983 int ret = -1; 1984 1985 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1986 if (!param_bld) { 1987 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 1988 return -1; 1989 } 1990 1991 r = BN_bin2bn(op->r.data, op->r.length, r); 1992 s = BN_bin2bn(op->s.data, op->s.length, s); 1993 pub_key = BN_bin2bn(op->y.data, op->y.length, pub_key); 1994 if (!r || !s || !pub_key) { 1995 BN_free(r); 1996 BN_free(s); 1997 BN_free(pub_key); 1998 OSSL_PARAM_BLD_free(param_bld); 1999 goto err_dsa_verify; 2000 } 2001 2002 set_dsa_sign(sign, r, s); 2003 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PUB_KEY, pub_key)) { 2004 OSSL_PARAM_BLD_free(param_bld); 2005 goto err_dsa_verify; 2006 } 2007 2008 params = OSSL_PARAM_BLD_to_param(param_bld); 2009 if (!params) { 2010 OSSL_PARAM_BLD_free(param_bld); 2011 goto err_dsa_verify; 2012 } 2013 2014 if (key_ctx == NULL 2015 || EVP_PKEY_fromdata_init(key_ctx) <= 0 2016 || EVP_PKEY_fromdata(key_ctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2017 goto err_dsa_verify; 2018 2019 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 2020 if (!dsa_ctx) 2021 goto err_dsa_verify; 2022 2023 if (!sign) 2024 goto err_dsa_verify; 2025 2026 sig_len = i2d_DSA_SIG(sign, &dsa_sig); 2027 if (EVP_PKEY_verify_init(dsa_ctx) <= 0) 2028 goto err_dsa_verify; 2029 2030 ret = EVP_PKEY_verify(dsa_ctx, dsa_sig, sig_len, 2031 op->message.data, op->message.length); 2032 if (ret == 1) { 2033 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2034 ret = 0; 2035 } 2036 2037 OPENSSL_free(dsa_sig); 2038 err_dsa_verify: 2039 if (sign) 2040 DSA_SIG_free(sign); 2041 if (params) 2042 OSSL_PARAM_free(params); 2043 EVP_PKEY_CTX_free(key_ctx); 2044 EVP_PKEY_CTX_free(dsa_ctx); 2045 2046 BN_free(pub_key); 2047 EVP_PKEY_free(pkey); 2048 2049 return ret; 2050 } 2051 #else 2052 static int 2053 process_openssl_dsa_sign_op(struct rte_crypto_op *cop, 2054 struct openssl_asym_session *sess) 2055 { 2056 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2057 DSA *dsa = sess->u.s.dsa; 2058 DSA_SIG *sign = NULL; 2059 2060 sign = DSA_do_sign(op->message.data, 2061 op->message.length, 2062 dsa); 2063 2064 if (sign == NULL) { 2065 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 2066 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2067 } else { 2068 const BIGNUM *r = NULL, *s = NULL; 2069 get_dsa_sign(sign, &r, &s); 2070 2071 op->r.length = BN_bn2bin(r, op->r.data); 2072 op->s.length = BN_bn2bin(s, op->s.data); 2073 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2074 } 2075 2076 DSA_SIG_free(sign); 2077 2078 return 0; 2079 } 2080 2081 /* process dsa verify operation */ 2082 static int 2083 process_openssl_dsa_verify_op(struct rte_crypto_op *cop, 2084 struct openssl_asym_session *sess) 2085 { 2086 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2087 DSA *dsa = sess->u.s.dsa; 2088 int ret; 2089 DSA_SIG *sign = DSA_SIG_new(); 2090 BIGNUM *r = NULL, *s = NULL; 2091 BIGNUM *pub_key = NULL; 2092 2093 if (sign == NULL) { 2094 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 2095 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2096 return -1; 2097 } 2098 2099 r = BN_bin2bn(op->r.data, 2100 op->r.length, 2101 r); 2102 s = BN_bin2bn(op->s.data, 2103 op->s.length, 2104 s); 2105 pub_key = BN_bin2bn(op->y.data, 2106 op->y.length, 2107 pub_key); 2108 if (!r || !s || !pub_key) { 2109 BN_free(r); 2110 BN_free(s); 2111 BN_free(pub_key); 2112 2113 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2114 return -1; 2115 } 2116 set_dsa_sign(sign, r, s); 2117 set_dsa_pub_key(dsa, pub_key); 2118 2119 ret = DSA_do_verify(op->message.data, 2120 op->message.length, 2121 sign, 2122 dsa); 2123 2124 if (ret != 1) 2125 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2126 else 2127 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2128 2129 DSA_SIG_free(sign); 2130 2131 return 0; 2132 } 2133 #endif 2134 2135 /* process dh operation */ 2136 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2137 static int 2138 process_openssl_dh_op_evp(struct rte_crypto_op *cop, 2139 struct openssl_asym_session *sess) 2140 { 2141 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2142 OSSL_PARAM_BLD *param_bld = sess->u.dh.param_bld; 2143 OSSL_PARAM_BLD *param_bld_peer = sess->u.dh.param_bld_peer; 2144 OSSL_PARAM *params = NULL; 2145 EVP_PKEY *dhpkey = NULL; 2146 EVP_PKEY *peerkey = NULL; 2147 BIGNUM *priv_key = NULL; 2148 BIGNUM *pub_key = NULL; 2149 int ret = -1; 2150 2151 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2152 EVP_PKEY_CTX *dh_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2153 if (dh_ctx == NULL || param_bld == NULL) 2154 return ret; 2155 2156 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2157 OSSL_PARAM *params_peer = NULL; 2158 2159 if (!param_bld_peer) 2160 return ret; 2161 2162 pub_key = BN_bin2bn(op->pub_key.data, op->pub_key.length, 2163 pub_key); 2164 if (pub_key == NULL) { 2165 OSSL_PARAM_BLD_free(param_bld_peer); 2166 return ret; 2167 } 2168 2169 if (!OSSL_PARAM_BLD_push_BN(param_bld_peer, OSSL_PKEY_PARAM_PUB_KEY, 2170 pub_key)) { 2171 OPENSSL_LOG(ERR, "Failed to set public key\n"); 2172 OSSL_PARAM_BLD_free(param_bld_peer); 2173 BN_free(pub_key); 2174 return ret; 2175 } 2176 2177 params_peer = OSSL_PARAM_BLD_to_param(param_bld_peer); 2178 if (!params_peer) { 2179 OSSL_PARAM_BLD_free(param_bld_peer); 2180 BN_free(pub_key); 2181 return ret; 2182 } 2183 2184 EVP_PKEY_CTX *peer_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2185 if (EVP_PKEY_keygen_init(peer_ctx) != 1) { 2186 OSSL_PARAM_free(params_peer); 2187 BN_free(pub_key); 2188 return ret; 2189 } 2190 2191 if (EVP_PKEY_CTX_set_params(peer_ctx, params_peer) != 1) { 2192 EVP_PKEY_CTX_free(peer_ctx); 2193 OSSL_PARAM_free(params_peer); 2194 BN_free(pub_key); 2195 return ret; 2196 } 2197 2198 if (EVP_PKEY_keygen(peer_ctx, &peerkey) != 1) { 2199 EVP_PKEY_CTX_free(peer_ctx); 2200 OSSL_PARAM_free(params_peer); 2201 BN_free(pub_key); 2202 return ret; 2203 } 2204 2205 priv_key = BN_bin2bn(op->priv_key.data, op->priv_key.length, 2206 priv_key); 2207 if (priv_key == NULL) { 2208 EVP_PKEY_CTX_free(peer_ctx); 2209 OSSL_PARAM_free(params_peer); 2210 BN_free(pub_key); 2211 return ret; 2212 } 2213 2214 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY, 2215 priv_key)) { 2216 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2217 EVP_PKEY_CTX_free(peer_ctx); 2218 OSSL_PARAM_free(params_peer); 2219 BN_free(pub_key); 2220 BN_free(priv_key); 2221 return ret; 2222 } 2223 2224 OSSL_PARAM_free(params_peer); 2225 EVP_PKEY_CTX_free(peer_ctx); 2226 } 2227 2228 params = OSSL_PARAM_BLD_to_param(param_bld); 2229 if (!params) 2230 goto err_dh; 2231 2232 if (EVP_PKEY_keygen_init(dh_ctx) != 1) 2233 goto err_dh; 2234 2235 if (EVP_PKEY_CTX_set_params(dh_ctx, params) != 1) 2236 goto err_dh; 2237 2238 if (EVP_PKEY_keygen(dh_ctx, &dhpkey) != 1) 2239 goto err_dh; 2240 2241 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2242 OPENSSL_LOG(DEBUG, "%s:%d updated pub key\n", __func__, __LINE__); 2243 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PUB_KEY, &pub_key)) 2244 goto err_dh; 2245 /* output public key */ 2246 op->pub_key.length = BN_bn2bin(pub_key, op->pub_key.data); 2247 } 2248 2249 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2250 2251 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", __func__, __LINE__); 2252 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PRIV_KEY, &priv_key)) 2253 goto err_dh; 2254 2255 /* provide generated private key back to user */ 2256 op->priv_key.length = BN_bn2bin(priv_key, op->priv_key.data); 2257 } 2258 2259 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2260 size_t skey_len; 2261 EVP_PKEY_CTX *sc_ctx = EVP_PKEY_CTX_new(dhpkey, NULL); 2262 if (!sc_ctx) 2263 goto err_dh; 2264 2265 if (EVP_PKEY_derive_init(sc_ctx) <= 0) { 2266 EVP_PKEY_CTX_free(sc_ctx); 2267 goto err_dh; 2268 } 2269 2270 if (!peerkey) { 2271 EVP_PKEY_CTX_free(sc_ctx); 2272 goto err_dh; 2273 } 2274 2275 if (EVP_PKEY_derive_set_peer(sc_ctx, peerkey) <= 0) { 2276 EVP_PKEY_CTX_free(sc_ctx); 2277 goto err_dh; 2278 } 2279 2280 /* Determine buffer length */ 2281 if (EVP_PKEY_derive(sc_ctx, NULL, &skey_len) <= 0) { 2282 EVP_PKEY_CTX_free(sc_ctx); 2283 goto err_dh; 2284 } 2285 2286 if (EVP_PKEY_derive(sc_ctx, op->shared_secret.data, &skey_len) <= 0) { 2287 EVP_PKEY_CTX_free(sc_ctx); 2288 goto err_dh; 2289 } 2290 2291 op->shared_secret.length = skey_len; 2292 EVP_PKEY_CTX_free(sc_ctx); 2293 } 2294 2295 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2296 ret = 0; 2297 2298 err_dh: 2299 BN_free(pub_key); 2300 BN_free(priv_key); 2301 if (params) 2302 OSSL_PARAM_free(params); 2303 EVP_PKEY_free(dhpkey); 2304 EVP_PKEY_free(peerkey); 2305 2306 EVP_PKEY_CTX_free(dh_ctx); 2307 2308 return ret; 2309 } 2310 #else 2311 static int 2312 process_openssl_dh_op(struct rte_crypto_op *cop, 2313 struct openssl_asym_session *sess) 2314 { 2315 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2316 struct rte_crypto_asym_op *asym_op = cop->asym; 2317 DH *dh_key = sess->u.dh.dh_key; 2318 BIGNUM *priv_key = NULL; 2319 int ret = 0; 2320 2321 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2322 /* compute shared secret using peer public key 2323 * and current private key 2324 * shared secret = peer_key ^ priv_key mod p 2325 */ 2326 BIGNUM *peer_key = NULL; 2327 2328 /* copy private key and peer key and compute shared secret */ 2329 peer_key = BN_bin2bn(op->pub_key.data, 2330 op->pub_key.length, 2331 peer_key); 2332 if (peer_key == NULL) { 2333 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2334 return -1; 2335 } 2336 priv_key = BN_bin2bn(op->priv_key.data, 2337 op->priv_key.length, 2338 priv_key); 2339 if (priv_key == NULL) { 2340 BN_free(peer_key); 2341 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2342 return -1; 2343 } 2344 ret = set_dh_priv_key(dh_key, priv_key); 2345 if (ret) { 2346 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2347 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2348 BN_free(peer_key); 2349 BN_free(priv_key); 2350 return 0; 2351 } 2352 2353 ret = DH_compute_key( 2354 op->shared_secret.data, 2355 peer_key, dh_key); 2356 if (ret < 0) { 2357 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2358 BN_free(peer_key); 2359 /* priv key is already loaded into dh, 2360 * let's not free that directly here. 2361 * DH_free() will auto free it later. 2362 */ 2363 return 0; 2364 } 2365 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2366 op->shared_secret.length = ret; 2367 BN_free(peer_key); 2368 return 0; 2369 } 2370 2371 /* 2372 * other options are public and private key generations. 2373 * 2374 * if user provides private key, 2375 * then first set DH with user provided private key 2376 */ 2377 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE && 2378 op->priv_key.length) { 2379 /* generate public key using user-provided private key 2380 * pub_key = g ^ priv_key mod p 2381 */ 2382 2383 /* load private key into DH */ 2384 priv_key = BN_bin2bn(op->priv_key.data, 2385 op->priv_key.length, 2386 priv_key); 2387 if (priv_key == NULL) { 2388 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2389 return -1; 2390 } 2391 ret = set_dh_priv_key(dh_key, priv_key); 2392 if (ret) { 2393 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2394 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2395 BN_free(priv_key); 2396 return 0; 2397 } 2398 } 2399 2400 /* generate public and private key pair. 2401 * 2402 * if private key already set, generates only public key. 2403 * 2404 * if private key is not already set, then set it to random value 2405 * and update internal private key. 2406 */ 2407 if (!DH_generate_key(dh_key)) { 2408 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2409 return 0; 2410 } 2411 2412 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2413 const BIGNUM *pub_key = NULL; 2414 2415 OPENSSL_LOG(DEBUG, "%s:%d update public key\n", 2416 __func__, __LINE__); 2417 2418 /* get the generated keys */ 2419 get_dh_pub_key(dh_key, &pub_key); 2420 2421 /* output public key */ 2422 op->pub_key.length = BN_bn2bin(pub_key, 2423 op->pub_key.data); 2424 } 2425 2426 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2427 const BIGNUM *priv_key = NULL; 2428 2429 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", 2430 __func__, __LINE__); 2431 2432 /* get the generated keys */ 2433 get_dh_priv_key(dh_key, &priv_key); 2434 2435 /* provide generated private key back to user */ 2436 op->priv_key.length = BN_bn2bin(priv_key, 2437 op->priv_key.data); 2438 } 2439 2440 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2441 2442 return 0; 2443 } 2444 #endif 2445 2446 /* process modinv operation */ 2447 static int 2448 process_openssl_modinv_op(struct rte_crypto_op *cop, 2449 struct openssl_asym_session *sess) 2450 { 2451 struct rte_crypto_asym_op *op = cop->asym; 2452 BIGNUM *base = BN_CTX_get(sess->u.m.ctx); 2453 BIGNUM *res = BN_CTX_get(sess->u.m.ctx); 2454 2455 if (unlikely(base == NULL || res == NULL)) { 2456 BN_free(base); 2457 BN_free(res); 2458 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2459 return -1; 2460 } 2461 2462 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 2463 op->modinv.base.length, base); 2464 2465 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) { 2466 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2467 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data); 2468 } else { 2469 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2470 } 2471 2472 BN_clear(res); 2473 BN_clear(base); 2474 2475 return 0; 2476 } 2477 2478 /* process modexp operation */ 2479 static int 2480 process_openssl_modexp_op(struct rte_crypto_op *cop, 2481 struct openssl_asym_session *sess) 2482 { 2483 struct rte_crypto_asym_op *op = cop->asym; 2484 BIGNUM *base = BN_CTX_get(sess->u.e.ctx); 2485 BIGNUM *res = BN_CTX_get(sess->u.e.ctx); 2486 2487 if (unlikely(base == NULL || res == NULL)) { 2488 BN_free(base); 2489 BN_free(res); 2490 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2491 return -1; 2492 } 2493 2494 base = BN_bin2bn((const unsigned char *)op->modex.base.data, 2495 op->modex.base.length, base); 2496 2497 if (BN_mod_exp(res, base, sess->u.e.exp, 2498 sess->u.e.mod, sess->u.e.ctx)) { 2499 op->modex.result.length = BN_bn2bin(res, op->modex.result.data); 2500 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2501 } else { 2502 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2503 } 2504 2505 BN_clear(res); 2506 BN_clear(base); 2507 2508 return 0; 2509 } 2510 2511 /* process rsa operations */ 2512 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2513 static int 2514 process_openssl_rsa_op_evp(struct rte_crypto_op *cop, 2515 struct openssl_asym_session *sess) 2516 { 2517 struct rte_crypto_asym_op *op = cop->asym; 2518 uint32_t pad = (op->rsa.padding.type); 2519 uint8_t *tmp; 2520 size_t outlen = 0; 2521 int ret = -1; 2522 2523 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2524 EVP_PKEY_CTX *rsa_ctx = sess->u.r.ctx; 2525 if (!rsa_ctx) 2526 return ret; 2527 2528 switch (pad) { 2529 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2530 pad = RSA_PKCS1_PADDING; 2531 break; 2532 case RTE_CRYPTO_RSA_PADDING_NONE: 2533 pad = RSA_NO_PADDING; 2534 break; 2535 default: 2536 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2537 OPENSSL_LOG(ERR, 2538 "rsa pad type not supported %d\n", pad); 2539 return ret; 2540 } 2541 2542 switch (op->rsa.op_type) { 2543 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2544 if (EVP_PKEY_encrypt_init(rsa_ctx) != 1) 2545 goto err_rsa; 2546 2547 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2548 goto err_rsa; 2549 2550 if (EVP_PKEY_encrypt(rsa_ctx, NULL, &outlen, 2551 op->rsa.message.data, 2552 op->rsa.message.length) <= 0) 2553 goto err_rsa; 2554 2555 if (outlen <= 0) 2556 goto err_rsa; 2557 2558 if (EVP_PKEY_encrypt(rsa_ctx, op->rsa.cipher.data, &outlen, 2559 op->rsa.message.data, 2560 op->rsa.message.length) <= 0) 2561 goto err_rsa; 2562 op->rsa.cipher.length = outlen; 2563 2564 OPENSSL_LOG(DEBUG, 2565 "length of encrypted text %zu\n", outlen); 2566 break; 2567 2568 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2569 if (EVP_PKEY_decrypt_init(rsa_ctx) != 1) 2570 goto err_rsa; 2571 2572 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2573 goto err_rsa; 2574 2575 if (EVP_PKEY_decrypt(rsa_ctx, NULL, &outlen, 2576 op->rsa.cipher.data, 2577 op->rsa.cipher.length) <= 0) 2578 goto err_rsa; 2579 2580 if (outlen <= 0) 2581 goto err_rsa; 2582 2583 if (EVP_PKEY_decrypt(rsa_ctx, op->rsa.message.data, &outlen, 2584 op->rsa.cipher.data, 2585 op->rsa.cipher.length) <= 0) 2586 goto err_rsa; 2587 op->rsa.message.length = outlen; 2588 2589 OPENSSL_LOG(DEBUG, "length of decrypted text %zu\n", outlen); 2590 break; 2591 2592 case RTE_CRYPTO_ASYM_OP_SIGN: 2593 if (EVP_PKEY_sign_init(rsa_ctx) <= 0) 2594 goto err_rsa; 2595 2596 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2597 goto err_rsa; 2598 2599 if (EVP_PKEY_sign(rsa_ctx, NULL, &outlen, 2600 op->rsa.message.data, 2601 op->rsa.message.length) <= 0) 2602 goto err_rsa; 2603 2604 if (outlen <= 0) 2605 goto err_rsa; 2606 2607 if (EVP_PKEY_sign(rsa_ctx, op->rsa.sign.data, &outlen, 2608 op->rsa.message.data, 2609 op->rsa.message.length) <= 0) 2610 goto err_rsa; 2611 op->rsa.sign.length = outlen; 2612 break; 2613 2614 case RTE_CRYPTO_ASYM_OP_VERIFY: 2615 if (EVP_PKEY_verify_recover_init(rsa_ctx) <= 0) 2616 goto err_rsa; 2617 2618 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2619 goto err_rsa; 2620 2621 if (EVP_PKEY_verify_recover(rsa_ctx, NULL, &outlen, 2622 op->rsa.sign.data, 2623 op->rsa.sign.length) <= 0) 2624 goto err_rsa; 2625 2626 if ((outlen <= 0) || (outlen != op->rsa.sign.length)) 2627 goto err_rsa; 2628 2629 tmp = OPENSSL_malloc(outlen); 2630 if (tmp == NULL) { 2631 OPENSSL_LOG(ERR, "Memory allocation failed"); 2632 goto err_rsa; 2633 } 2634 2635 if (EVP_PKEY_verify_recover(rsa_ctx, tmp, &outlen, 2636 op->rsa.sign.data, 2637 op->rsa.sign.length) <= 0) { 2638 OPENSSL_free(tmp); 2639 goto err_rsa; 2640 } 2641 2642 OPENSSL_LOG(DEBUG, 2643 "Length of public_decrypt %zu " 2644 "length of message %zd\n", 2645 outlen, op->rsa.message.length); 2646 if (CRYPTO_memcmp(tmp, op->rsa.message.data, 2647 op->rsa.message.length)) { 2648 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 2649 } 2650 OPENSSL_free(tmp); 2651 break; 2652 2653 default: 2654 /* allow ops with invalid args to be pushed to 2655 * completion queue 2656 */ 2657 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2658 goto err_rsa; 2659 } 2660 2661 ret = 0; 2662 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2663 err_rsa: 2664 return ret; 2665 2666 } 2667 2668 static int 2669 process_openssl_sm2_op_evp(struct rte_crypto_op *cop, 2670 struct openssl_asym_session *sess) 2671 { 2672 EVP_PKEY_CTX *kctx = NULL, *sctx = NULL, *cctx = NULL; 2673 struct rte_crypto_asym_op *op = cop->asym; 2674 OSSL_PARAM *params = sess->u.sm2.params; 2675 EVP_MD_CTX *md_ctx = NULL; 2676 ECDSA_SIG *ec_sign = NULL; 2677 EVP_MD *check_md = NULL; 2678 EVP_PKEY *pkey = NULL; 2679 int ret = -1; 2680 2681 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2682 2683 if (cop->asym->sm2.k.data != NULL) 2684 goto err_sm2; 2685 2686 switch (op->sm2.op_type) { 2687 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2688 { 2689 OSSL_PARAM *eparams = sess->u.sm2.params; 2690 size_t output_len = 0; 2691 2692 kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_SM2, NULL); 2693 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2694 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2695 goto err_sm2; 2696 2697 cctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2698 if (!cctx) 2699 goto err_sm2; 2700 2701 if (!EVP_PKEY_encrypt_init(cctx)) 2702 goto err_sm2; 2703 2704 if (!EVP_PKEY_CTX_set_params(cctx, eparams)) 2705 goto err_sm2; 2706 2707 if (!EVP_PKEY_encrypt(cctx, op->sm2.cipher.data, &output_len, 2708 op->sm2.message.data, 2709 op->sm2.message.length)) 2710 goto err_sm2; 2711 op->sm2.cipher.length = output_len; 2712 } 2713 break; 2714 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2715 { 2716 OSSL_PARAM *eparams = sess->u.sm2.params; 2717 2718 kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_SM2, NULL); 2719 if (kctx == NULL 2720 || EVP_PKEY_fromdata_init(kctx) <= 0 2721 || EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2722 goto err_sm2; 2723 2724 cctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2725 if (!cctx) 2726 goto err_sm2; 2727 2728 if (!EVP_PKEY_decrypt_init(cctx)) 2729 goto err_sm2; 2730 2731 if (!EVP_PKEY_CTX_set_params(cctx, eparams)) 2732 goto err_sm2; 2733 2734 if (!EVP_PKEY_decrypt(cctx, op->sm2.message.data, &op->sm2.message.length, 2735 op->sm2.cipher.data, op->sm2.cipher.length)) 2736 goto err_sm2; 2737 } 2738 break; 2739 case RTE_CRYPTO_ASYM_OP_SIGN: 2740 { 2741 unsigned char signbuf[128] = {0}; 2742 const unsigned char *signptr; 2743 const BIGNUM *r, *s; 2744 size_t signlen; 2745 2746 kctx = EVP_PKEY_CTX_new_from_name(NULL, "SM2", NULL); 2747 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2748 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2749 goto err_sm2; 2750 2751 md_ctx = EVP_MD_CTX_new(); 2752 if (!md_ctx) 2753 goto err_sm2; 2754 2755 sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2756 if (!sctx) 2757 goto err_sm2; 2758 2759 EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx); 2760 2761 check_md = EVP_MD_fetch(NULL, "sm3", NULL); 2762 if (!check_md) 2763 goto err_sm2; 2764 2765 if (!EVP_DigestSignInit(md_ctx, NULL, check_md, NULL, pkey)) 2766 goto err_sm2; 2767 2768 if (EVP_PKEY_CTX_set1_id(sctx, op->sm2.id.data, op->sm2.id.length) <= 0) 2769 goto err_sm2; 2770 2771 if (!EVP_DigestSignUpdate(md_ctx, op->sm2.message.data, 2772 op->sm2.message.length)) 2773 goto err_sm2; 2774 2775 if (!EVP_DigestSignFinal(md_ctx, NULL, &signlen)) 2776 goto err_sm2; 2777 2778 if (!EVP_DigestSignFinal(md_ctx, signbuf, &signlen)) 2779 goto err_sm2; 2780 2781 signptr = signbuf; 2782 ec_sign = d2i_ECDSA_SIG(NULL, &signptr, signlen); 2783 if (!ec_sign) 2784 goto err_sm2; 2785 2786 r = ECDSA_SIG_get0_r(ec_sign); 2787 s = ECDSA_SIG_get0_s(ec_sign); 2788 if (!r || !s) 2789 goto err_sm2; 2790 2791 op->sm2.r.length = BN_num_bytes(r); 2792 op->sm2.s.length = BN_num_bytes(s); 2793 BN_bn2bin(r, op->sm2.r.data); 2794 BN_bn2bin(s, op->sm2.s.data); 2795 2796 ECDSA_SIG_free(ec_sign); 2797 } 2798 break; 2799 case RTE_CRYPTO_ASYM_OP_VERIFY: 2800 { 2801 unsigned char signbuf[128] = {0}, *signbuf_new = NULL; 2802 BIGNUM *r = NULL, *s = NULL; 2803 size_t signlen; 2804 2805 kctx = EVP_PKEY_CTX_new_from_name(NULL, "SM2", NULL); 2806 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2807 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_PUBLIC_KEY, params) <= 0) 2808 goto err_sm2; 2809 2810 if (!EVP_PKEY_is_a(pkey, "SM2")) 2811 goto err_sm2; 2812 2813 md_ctx = EVP_MD_CTX_new(); 2814 if (!md_ctx) 2815 goto err_sm2; 2816 2817 sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2818 if (!sctx) 2819 goto err_sm2; 2820 2821 EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx); 2822 2823 check_md = EVP_MD_fetch(NULL, "sm3", NULL); 2824 if (!check_md) 2825 goto err_sm2; 2826 2827 if (!EVP_DigestVerifyInit(md_ctx, NULL, check_md, NULL, pkey)) 2828 goto err_sm2; 2829 2830 if (EVP_PKEY_CTX_set1_id(sctx, op->sm2.id.data, op->sm2.id.length) <= 0) 2831 goto err_sm2; 2832 2833 if (!EVP_DigestVerifyUpdate(md_ctx, op->sm2.message.data, 2834 op->sm2.message.length)) 2835 goto err_sm2; 2836 2837 ec_sign = ECDSA_SIG_new(); 2838 if (!ec_sign) 2839 goto err_sm2; 2840 2841 r = BN_bin2bn(op->sm2.r.data, op->sm2.r.length, r); 2842 s = BN_bin2bn(op->sm2.s.data, op->sm2.s.length, s); 2843 if (!r || !s) 2844 goto err_sm2; 2845 2846 if (!ECDSA_SIG_set0(ec_sign, r, s)) { 2847 BN_free(r); 2848 BN_free(s); 2849 goto err_sm2; 2850 } 2851 2852 r = NULL; 2853 s = NULL; 2854 2855 signbuf_new = signbuf; 2856 signlen = i2d_ECDSA_SIG(ec_sign, (unsigned char **)&signbuf_new); 2857 if (signlen <= 0) 2858 goto err_sm2; 2859 2860 if (!EVP_DigestVerifyFinal(md_ctx, signbuf_new, signlen)) 2861 goto err_sm2; 2862 2863 BN_free(r); 2864 BN_free(s); 2865 ECDSA_SIG_free(ec_sign); 2866 } 2867 break; 2868 default: 2869 /* allow ops with invalid args to be pushed to 2870 * completion queue 2871 */ 2872 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2873 goto err_sm2; 2874 } 2875 2876 ret = 0; 2877 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2878 err_sm2: 2879 EVP_MD_free(check_md); 2880 EVP_MD_CTX_free(md_ctx); 2881 2882 EVP_PKEY_CTX_free(kctx); 2883 2884 EVP_PKEY_CTX_free(sctx); 2885 2886 EVP_PKEY_CTX_free(cctx); 2887 2888 EVP_PKEY_free(pkey); 2889 2890 return ret; 2891 } 2892 2893 #else 2894 static int 2895 process_openssl_rsa_op(struct rte_crypto_op *cop, 2896 struct openssl_asym_session *sess) 2897 { 2898 int ret = 0; 2899 struct rte_crypto_asym_op *op = cop->asym; 2900 RSA *rsa = sess->u.r.rsa; 2901 uint32_t pad = (op->rsa.padding.type); 2902 uint8_t *tmp; 2903 2904 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2905 2906 switch (pad) { 2907 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2908 pad = RSA_PKCS1_PADDING; 2909 break; 2910 case RTE_CRYPTO_RSA_PADDING_NONE: 2911 pad = RSA_NO_PADDING; 2912 break; 2913 default: 2914 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2915 OPENSSL_LOG(ERR, 2916 "rsa pad type not supported %d\n", pad); 2917 return 0; 2918 } 2919 2920 switch (op->rsa.op_type) { 2921 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2922 ret = RSA_public_encrypt(op->rsa.message.length, 2923 op->rsa.message.data, 2924 op->rsa.cipher.data, 2925 rsa, 2926 pad); 2927 2928 if (ret > 0) 2929 op->rsa.cipher.length = ret; 2930 OPENSSL_LOG(DEBUG, 2931 "length of encrypted text %d\n", ret); 2932 break; 2933 2934 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2935 ret = RSA_private_decrypt(op->rsa.cipher.length, 2936 op->rsa.cipher.data, 2937 op->rsa.message.data, 2938 rsa, 2939 pad); 2940 if (ret > 0) 2941 op->rsa.message.length = ret; 2942 break; 2943 2944 case RTE_CRYPTO_ASYM_OP_SIGN: 2945 ret = RSA_private_encrypt(op->rsa.message.length, 2946 op->rsa.message.data, 2947 op->rsa.sign.data, 2948 rsa, 2949 pad); 2950 if (ret > 0) 2951 op->rsa.sign.length = ret; 2952 break; 2953 2954 case RTE_CRYPTO_ASYM_OP_VERIFY: 2955 tmp = rte_malloc(NULL, op->rsa.sign.length, 0); 2956 if (tmp == NULL) { 2957 OPENSSL_LOG(ERR, "Memory allocation failed"); 2958 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2959 break; 2960 } 2961 ret = RSA_public_decrypt(op->rsa.sign.length, 2962 op->rsa.sign.data, 2963 tmp, 2964 rsa, 2965 pad); 2966 2967 OPENSSL_LOG(DEBUG, 2968 "Length of public_decrypt %d " 2969 "length of message %zd\n", 2970 ret, op->rsa.message.length); 2971 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data, 2972 op->rsa.message.length))) { 2973 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 2974 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2975 } 2976 rte_free(tmp); 2977 break; 2978 2979 default: 2980 /* allow ops with invalid args to be pushed to 2981 * completion queue 2982 */ 2983 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2984 break; 2985 } 2986 2987 if (ret < 0) 2988 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2989 2990 return 0; 2991 } 2992 2993 static int 2994 process_openssl_sm2_op(struct rte_crypto_op *cop, 2995 struct openssl_asym_session *sess) 2996 { 2997 RTE_SET_USED(cop); 2998 RTE_SET_USED(sess); 2999 return -ENOTSUP; 3000 } 3001 #endif 3002 3003 static int 3004 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op, 3005 struct openssl_asym_session *sess) 3006 { 3007 int retval = 0; 3008 3009 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 3010 3011 switch (sess->xfrm_type) { 3012 case RTE_CRYPTO_ASYM_XFORM_RSA: 3013 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3014 retval = process_openssl_rsa_op_evp(op, sess); 3015 # else 3016 retval = process_openssl_rsa_op(op, sess); 3017 #endif 3018 break; 3019 case RTE_CRYPTO_ASYM_XFORM_MODEX: 3020 retval = process_openssl_modexp_op(op, sess); 3021 break; 3022 case RTE_CRYPTO_ASYM_XFORM_MODINV: 3023 retval = process_openssl_modinv_op(op, sess); 3024 break; 3025 case RTE_CRYPTO_ASYM_XFORM_DH: 3026 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3027 retval = process_openssl_dh_op_evp(op, sess); 3028 # else 3029 retval = process_openssl_dh_op(op, sess); 3030 #endif 3031 break; 3032 case RTE_CRYPTO_ASYM_XFORM_DSA: 3033 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3034 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 3035 retval = process_openssl_dsa_sign_op_evp(op, sess); 3036 else if (op->asym->dsa.op_type == 3037 RTE_CRYPTO_ASYM_OP_VERIFY) 3038 retval = 3039 process_openssl_dsa_verify_op_evp(op, sess); 3040 #else 3041 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 3042 retval = process_openssl_dsa_sign_op(op, sess); 3043 else if (op->asym->dsa.op_type == 3044 RTE_CRYPTO_ASYM_OP_VERIFY) 3045 retval = 3046 process_openssl_dsa_verify_op(op, sess); 3047 else 3048 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3049 #endif 3050 break; 3051 case RTE_CRYPTO_ASYM_XFORM_SM2: 3052 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3053 retval = process_openssl_sm2_op_evp(op, sess); 3054 #else 3055 retval = process_openssl_sm2_op(op, sess); 3056 #endif 3057 break; 3058 default: 3059 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3060 break; 3061 } 3062 if (!retval) { 3063 /* op processed so push to completion queue as processed */ 3064 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 3065 if (retval) 3066 /* return error if failed to put in completion queue */ 3067 retval = -1; 3068 } 3069 3070 return retval; 3071 } 3072 3073 static void 3074 copy_plaintext(struct rte_mbuf *m_src, struct rte_mbuf *m_dst, 3075 struct rte_crypto_op *op) 3076 { 3077 uint8_t *p_src, *p_dst; 3078 3079 p_src = rte_pktmbuf_mtod(m_src, uint8_t *); 3080 p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *); 3081 3082 /** 3083 * Copy the content between cipher offset and auth offset 3084 * for generating correct digest. 3085 */ 3086 if (op->sym->cipher.data.offset > op->sym->auth.data.offset) 3087 memcpy(p_dst + op->sym->auth.data.offset, 3088 p_src + op->sym->auth.data.offset, 3089 op->sym->cipher.data.offset - 3090 op->sym->auth.data.offset); 3091 } 3092 3093 /** Process crypto operation for mbuf */ 3094 static int 3095 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 3096 struct openssl_session *sess) 3097 { 3098 struct rte_mbuf *msrc, *mdst; 3099 int retval; 3100 3101 msrc = op->sym->m_src; 3102 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 3103 3104 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 3105 3106 switch (sess->chain_order) { 3107 case OPENSSL_CHAIN_ONLY_CIPHER: 3108 process_openssl_cipher_op(op, sess, msrc, mdst); 3109 break; 3110 case OPENSSL_CHAIN_ONLY_AUTH: 3111 process_openssl_auth_op(qp, op, sess, msrc, mdst); 3112 break; 3113 case OPENSSL_CHAIN_CIPHER_AUTH: 3114 process_openssl_cipher_op(op, sess, msrc, mdst); 3115 /* OOP */ 3116 if (msrc != mdst) 3117 copy_plaintext(msrc, mdst, op); 3118 process_openssl_auth_op(qp, op, sess, mdst, mdst); 3119 break; 3120 case OPENSSL_CHAIN_AUTH_CIPHER: 3121 process_openssl_auth_op(qp, op, sess, msrc, mdst); 3122 process_openssl_cipher_op(op, sess, msrc, mdst); 3123 break; 3124 case OPENSSL_CHAIN_COMBINED: 3125 process_openssl_combined_op(op, sess, msrc, mdst); 3126 break; 3127 case OPENSSL_CHAIN_CIPHER_BPI: 3128 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 3129 break; 3130 default: 3131 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3132 break; 3133 } 3134 3135 /* Free session if a session-less crypto op */ 3136 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 3137 openssl_reset_session(sess); 3138 memset(sess, 0, sizeof(struct openssl_session)); 3139 rte_mempool_put(qp->sess_mp, op->sym->session); 3140 op->sym->session = NULL; 3141 } 3142 3143 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 3144 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3145 3146 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 3147 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 3148 else 3149 retval = -1; 3150 3151 return retval; 3152 } 3153 3154 /* 3155 *------------------------------------------------------------------------------ 3156 * PMD Framework 3157 *------------------------------------------------------------------------------ 3158 */ 3159 3160 /** Enqueue burst */ 3161 static uint16_t 3162 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 3163 uint16_t nb_ops) 3164 { 3165 void *sess; 3166 struct openssl_qp *qp = queue_pair; 3167 int i, retval; 3168 3169 for (i = 0; i < nb_ops; i++) { 3170 sess = get_session(qp, ops[i]); 3171 if (unlikely(sess == NULL)) 3172 goto enqueue_err; 3173 3174 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) 3175 retval = process_op(qp, ops[i], 3176 (struct openssl_session *) sess); 3177 else 3178 retval = process_asym_op(qp, ops[i], 3179 (struct openssl_asym_session *) sess); 3180 if (unlikely(retval < 0)) 3181 goto enqueue_err; 3182 } 3183 3184 qp->stats.enqueued_count += i; 3185 return i; 3186 3187 enqueue_err: 3188 qp->stats.enqueue_err_count++; 3189 return i; 3190 } 3191 3192 /** Dequeue burst */ 3193 static uint16_t 3194 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 3195 uint16_t nb_ops) 3196 { 3197 struct openssl_qp *qp = queue_pair; 3198 3199 unsigned int nb_dequeued = 0; 3200 3201 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 3202 (void **)ops, nb_ops, NULL); 3203 qp->stats.dequeued_count += nb_dequeued; 3204 3205 return nb_dequeued; 3206 } 3207 3208 /** Create OPENSSL crypto device */ 3209 static int 3210 cryptodev_openssl_create(const char *name, 3211 struct rte_vdev_device *vdev, 3212 struct rte_cryptodev_pmd_init_params *init_params) 3213 { 3214 struct rte_cryptodev *dev; 3215 struct openssl_private *internals; 3216 3217 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 3218 if (dev == NULL) { 3219 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 3220 goto init_error; 3221 } 3222 3223 dev->driver_id = cryptodev_driver_id; 3224 dev->dev_ops = rte_openssl_pmd_ops; 3225 3226 /* register rx/tx burst functions for data path */ 3227 dev->dequeue_burst = openssl_pmd_dequeue_burst; 3228 dev->enqueue_burst = openssl_pmd_enqueue_burst; 3229 3230 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 3231 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 3232 RTE_CRYPTODEV_FF_CPU_AESNI | 3233 RTE_CRYPTODEV_FF_IN_PLACE_SGL | 3234 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 3235 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | 3236 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO | 3237 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP | 3238 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT | 3239 RTE_CRYPTODEV_FF_SYM_SESSIONLESS; 3240 3241 internals = dev->data->dev_private; 3242 3243 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 3244 3245 rte_cryptodev_pmd_probing_finish(dev); 3246 3247 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3248 /* Load legacy provider 3249 * Some algorithms are no longer available in earlier version of openssl, 3250 * unless the legacy provider explicitly loaded. e.g. DES 3251 */ 3252 ossl_legacy_provider_load(); 3253 # endif 3254 return 0; 3255 3256 init_error: 3257 OPENSSL_LOG(ERR, "driver %s: create failed", 3258 init_params->name); 3259 3260 cryptodev_openssl_remove(vdev); 3261 return -EFAULT; 3262 } 3263 3264 /** Initialise OPENSSL crypto device */ 3265 static int 3266 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 3267 { 3268 struct rte_cryptodev_pmd_init_params init_params = { 3269 "", 3270 sizeof(struct openssl_private), 3271 rte_socket_id(), 3272 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 3273 }; 3274 const char *name; 3275 const char *input_args; 3276 3277 name = rte_vdev_device_name(vdev); 3278 if (name == NULL) 3279 return -EINVAL; 3280 input_args = rte_vdev_device_args(vdev); 3281 3282 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 3283 3284 return cryptodev_openssl_create(name, vdev, &init_params); 3285 } 3286 3287 /** Uninitialise OPENSSL crypto device */ 3288 static int 3289 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 3290 { 3291 struct rte_cryptodev *cryptodev; 3292 const char *name; 3293 3294 name = rte_vdev_device_name(vdev); 3295 if (name == NULL) 3296 return -EINVAL; 3297 3298 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 3299 if (cryptodev == NULL) 3300 return -ENODEV; 3301 3302 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3303 ossl_legacy_provider_unload(); 3304 # endif 3305 return rte_cryptodev_pmd_destroy(cryptodev); 3306 } 3307 3308 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 3309 .probe = cryptodev_openssl_probe, 3310 .remove = cryptodev_openssl_remove 3311 }; 3312 3313 static struct cryptodev_driver openssl_crypto_drv; 3314 3315 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 3316 cryptodev_openssl_pmd_drv); 3317 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 3318 "max_nb_queue_pairs=<int> " 3319 "socket_id=<int>"); 3320 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 3321 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 3322 RTE_LOG_REGISTER_DEFAULT(openssl_logtype_driver, INFO); 3323