1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_common.h> 6 #include <rte_hexdump.h> 7 #include <rte_cryptodev.h> 8 #include <cryptodev_pmd.h> 9 #include <bus_vdev_driver.h> 10 #include <rte_malloc.h> 11 #include <rte_cpuflags.h> 12 13 #include <openssl/cmac.h> 14 #include <openssl/hmac.h> 15 #include <openssl/evp.h> 16 #include <openssl/ec.h> 17 18 #include "openssl_pmd_private.h" 19 #include "compat.h" 20 21 #define DES_BLOCK_SIZE 8 22 23 static uint8_t cryptodev_driver_id; 24 25 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 26 static HMAC_CTX *HMAC_CTX_new(void) 27 { 28 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 29 30 if (ctx != NULL) 31 HMAC_CTX_init(ctx); 32 return ctx; 33 } 34 35 static void HMAC_CTX_free(HMAC_CTX *ctx) 36 { 37 if (ctx != NULL) { 38 HMAC_CTX_cleanup(ctx); 39 OPENSSL_free(ctx); 40 } 41 } 42 #endif 43 44 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 45 46 #include <openssl/provider.h> 47 #include <openssl/core_names.h> 48 #include <openssl/param_build.h> 49 50 #define MAX_OSSL_ALGO_NAME_SIZE 16 51 52 OSSL_PROVIDER *legacy; 53 OSSL_PROVIDER *deflt; 54 55 static void ossl_legacy_provider_load(void) 56 { 57 /* Load Multiple providers into the default (NULL) library context */ 58 legacy = OSSL_PROVIDER_load(NULL, "legacy"); 59 if (legacy == NULL) { 60 OPENSSL_LOG(ERR, "Failed to load Legacy provider\n"); 61 return; 62 } 63 64 deflt = OSSL_PROVIDER_load(NULL, "default"); 65 if (deflt == NULL) { 66 OPENSSL_LOG(ERR, "Failed to load Default provider\n"); 67 OSSL_PROVIDER_unload(legacy); 68 return; 69 } 70 } 71 72 static void ossl_legacy_provider_unload(void) 73 { 74 OSSL_PROVIDER_unload(legacy); 75 OSSL_PROVIDER_unload(deflt); 76 } 77 78 static __rte_always_inline const char * 79 digest_name_get(enum rte_crypto_auth_algorithm algo) 80 { 81 switch (algo) { 82 case RTE_CRYPTO_AUTH_MD5_HMAC: 83 return OSSL_DIGEST_NAME_MD5; 84 case RTE_CRYPTO_AUTH_SHA1_HMAC: 85 return OSSL_DIGEST_NAME_SHA1; 86 case RTE_CRYPTO_AUTH_SHA224_HMAC: 87 return OSSL_DIGEST_NAME_SHA2_224; 88 case RTE_CRYPTO_AUTH_SHA256_HMAC: 89 return OSSL_DIGEST_NAME_SHA2_256; 90 case RTE_CRYPTO_AUTH_SHA384_HMAC: 91 return OSSL_DIGEST_NAME_SHA2_384; 92 case RTE_CRYPTO_AUTH_SHA512_HMAC: 93 return OSSL_DIGEST_NAME_SHA2_512; 94 default: 95 return NULL; 96 } 97 } 98 #endif 99 100 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 101 102 /*----------------------------------------------------------------------------*/ 103 104 /** 105 * Increment counter by 1 106 * Counter is 64 bit array, big-endian 107 */ 108 static void 109 ctr_inc(uint8_t *ctr) 110 { 111 uint64_t *ctr64 = (uint64_t *)ctr; 112 113 *ctr64 = __builtin_bswap64(*ctr64); 114 (*ctr64)++; 115 *ctr64 = __builtin_bswap64(*ctr64); 116 } 117 118 /* 119 *------------------------------------------------------------------------------ 120 * Session Prepare 121 *------------------------------------------------------------------------------ 122 */ 123 124 /** Get xform chain order */ 125 static enum openssl_chain_order 126 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 127 { 128 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 129 130 if (xform != NULL) { 131 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 132 if (xform->next == NULL) 133 res = OPENSSL_CHAIN_ONLY_AUTH; 134 else if (xform->next->type == 135 RTE_CRYPTO_SYM_XFORM_CIPHER) 136 res = OPENSSL_CHAIN_AUTH_CIPHER; 137 } 138 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 139 if (xform->next == NULL) 140 res = OPENSSL_CHAIN_ONLY_CIPHER; 141 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 142 res = OPENSSL_CHAIN_CIPHER_AUTH; 143 } 144 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 145 res = OPENSSL_CHAIN_COMBINED; 146 } 147 148 return res; 149 } 150 151 /** Get session cipher key from input cipher key */ 152 static void 153 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key) 154 { 155 memcpy(session_key, input_key, keylen); 156 } 157 158 /** Get key ede 24 bytes standard from input key */ 159 static int 160 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede) 161 { 162 int res = 0; 163 164 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 165 switch (keylen) { 166 case 24: 167 memcpy(key_ede, key, 24); 168 break; 169 case 16: 170 /* K3 = K1 */ 171 memcpy(key_ede, key, 16); 172 memcpy(key_ede + 16, key, 8); 173 break; 174 case 8: 175 /* K1 = K2 = K3 (DES compatibility) */ 176 memcpy(key_ede, key, 8); 177 memcpy(key_ede + 8, key, 8); 178 memcpy(key_ede + 16, key, 8); 179 break; 180 default: 181 OPENSSL_LOG(ERR, "Unsupported key size"); 182 res = -EINVAL; 183 } 184 185 return res; 186 } 187 188 /** Get adequate openssl function for input cipher algorithm */ 189 static uint8_t 190 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 191 const EVP_CIPHER **algo) 192 { 193 int res = 0; 194 195 if (algo != NULL) { 196 switch (sess_algo) { 197 case RTE_CRYPTO_CIPHER_3DES_CBC: 198 switch (keylen) { 199 case 8: 200 *algo = EVP_des_cbc(); 201 break; 202 case 16: 203 *algo = EVP_des_ede_cbc(); 204 break; 205 case 24: 206 *algo = EVP_des_ede3_cbc(); 207 break; 208 default: 209 res = -EINVAL; 210 } 211 break; 212 case RTE_CRYPTO_CIPHER_3DES_CTR: 213 break; 214 case RTE_CRYPTO_CIPHER_AES_CBC: 215 switch (keylen) { 216 case 16: 217 *algo = EVP_aes_128_cbc(); 218 break; 219 case 24: 220 *algo = EVP_aes_192_cbc(); 221 break; 222 case 32: 223 *algo = EVP_aes_256_cbc(); 224 break; 225 default: 226 res = -EINVAL; 227 } 228 break; 229 case RTE_CRYPTO_CIPHER_AES_CTR: 230 switch (keylen) { 231 case 16: 232 *algo = EVP_aes_128_ctr(); 233 break; 234 case 24: 235 *algo = EVP_aes_192_ctr(); 236 break; 237 case 32: 238 *algo = EVP_aes_256_ctr(); 239 break; 240 default: 241 res = -EINVAL; 242 } 243 break; 244 default: 245 res = -EINVAL; 246 break; 247 } 248 } else { 249 res = -EINVAL; 250 } 251 252 return res; 253 } 254 255 /** Get adequate openssl function for input auth algorithm */ 256 static uint8_t 257 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 258 const EVP_MD **algo) 259 { 260 int res = 0; 261 262 if (algo != NULL) { 263 switch (sessalgo) { 264 case RTE_CRYPTO_AUTH_MD5: 265 case RTE_CRYPTO_AUTH_MD5_HMAC: 266 *algo = EVP_md5(); 267 break; 268 case RTE_CRYPTO_AUTH_SHA1: 269 case RTE_CRYPTO_AUTH_SHA1_HMAC: 270 *algo = EVP_sha1(); 271 break; 272 case RTE_CRYPTO_AUTH_SHA224: 273 case RTE_CRYPTO_AUTH_SHA224_HMAC: 274 *algo = EVP_sha224(); 275 break; 276 case RTE_CRYPTO_AUTH_SHA256: 277 case RTE_CRYPTO_AUTH_SHA256_HMAC: 278 *algo = EVP_sha256(); 279 break; 280 case RTE_CRYPTO_AUTH_SHA384: 281 case RTE_CRYPTO_AUTH_SHA384_HMAC: 282 *algo = EVP_sha384(); 283 break; 284 case RTE_CRYPTO_AUTH_SHA512: 285 case RTE_CRYPTO_AUTH_SHA512_HMAC: 286 *algo = EVP_sha512(); 287 break; 288 default: 289 res = -EINVAL; 290 break; 291 } 292 } else { 293 res = -EINVAL; 294 } 295 296 return res; 297 } 298 299 /** Get adequate openssl function for input cipher algorithm */ 300 static uint8_t 301 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 302 const EVP_CIPHER **algo) 303 { 304 int res = 0; 305 306 if (algo != NULL) { 307 switch (sess_algo) { 308 case RTE_CRYPTO_AEAD_AES_GCM: 309 switch (keylen) { 310 case 16: 311 *algo = EVP_aes_128_gcm(); 312 break; 313 case 24: 314 *algo = EVP_aes_192_gcm(); 315 break; 316 case 32: 317 *algo = EVP_aes_256_gcm(); 318 break; 319 default: 320 res = -EINVAL; 321 } 322 break; 323 case RTE_CRYPTO_AEAD_AES_CCM: 324 switch (keylen) { 325 case 16: 326 *algo = EVP_aes_128_ccm(); 327 break; 328 case 24: 329 *algo = EVP_aes_192_ccm(); 330 break; 331 case 32: 332 *algo = EVP_aes_256_ccm(); 333 break; 334 default: 335 res = -EINVAL; 336 } 337 break; 338 default: 339 res = -EINVAL; 340 break; 341 } 342 } else { 343 res = -EINVAL; 344 } 345 346 return res; 347 } 348 349 /* Set session AEAD encryption parameters */ 350 static int 351 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 352 enum rte_crypto_aead_algorithm algo, 353 uint8_t tag_len, const uint8_t *key) 354 { 355 int iv_type = 0; 356 unsigned int do_ccm; 357 358 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 359 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 360 361 /* Select AEAD algo */ 362 switch (algo) { 363 case RTE_CRYPTO_AEAD_AES_GCM: 364 iv_type = EVP_CTRL_GCM_SET_IVLEN; 365 if (tag_len != 16) 366 return -EINVAL; 367 do_ccm = 0; 368 break; 369 case RTE_CRYPTO_AEAD_AES_CCM: 370 iv_type = EVP_CTRL_CCM_SET_IVLEN; 371 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 372 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 373 return -EINVAL; 374 do_ccm = 1; 375 break; 376 default: 377 return -ENOTSUP; 378 } 379 380 sess->cipher.mode = OPENSSL_CIPHER_LIB; 381 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 382 383 if (get_aead_algo(algo, sess->cipher.key.length, 384 &sess->cipher.evp_algo) != 0) 385 return -EINVAL; 386 387 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 388 389 sess->chain_order = OPENSSL_CHAIN_COMBINED; 390 391 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 392 NULL, NULL, NULL) <= 0) 393 return -EINVAL; 394 395 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, 396 NULL) <= 0) 397 return -EINVAL; 398 399 if (do_ccm) 400 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 401 tag_len, NULL); 402 403 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 404 return -EINVAL; 405 406 return 0; 407 } 408 409 /* Set session AEAD decryption parameters */ 410 static int 411 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 412 enum rte_crypto_aead_algorithm algo, 413 uint8_t tag_len, const uint8_t *key) 414 { 415 int iv_type = 0; 416 unsigned int do_ccm = 0; 417 418 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 419 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 420 421 /* Select AEAD algo */ 422 switch (algo) { 423 case RTE_CRYPTO_AEAD_AES_GCM: 424 iv_type = EVP_CTRL_GCM_SET_IVLEN; 425 if (tag_len != 16) 426 return -EINVAL; 427 break; 428 case RTE_CRYPTO_AEAD_AES_CCM: 429 iv_type = EVP_CTRL_CCM_SET_IVLEN; 430 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 431 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 432 return -EINVAL; 433 do_ccm = 1; 434 break; 435 default: 436 return -ENOTSUP; 437 } 438 439 sess->cipher.mode = OPENSSL_CIPHER_LIB; 440 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 441 442 if (get_aead_algo(algo, sess->cipher.key.length, 443 &sess->cipher.evp_algo) != 0) 444 return -EINVAL; 445 446 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 447 448 sess->chain_order = OPENSSL_CHAIN_COMBINED; 449 450 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 451 NULL, NULL, NULL) <= 0) 452 return -EINVAL; 453 454 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, 455 sess->iv.length, NULL) <= 0) 456 return -EINVAL; 457 458 if (do_ccm) 459 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 460 tag_len, NULL); 461 462 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 463 return -EINVAL; 464 465 return 0; 466 } 467 468 /** Set session cipher parameters */ 469 static int 470 openssl_set_session_cipher_parameters(struct openssl_session *sess, 471 const struct rte_crypto_sym_xform *xform) 472 { 473 /* Select cipher direction */ 474 sess->cipher.direction = xform->cipher.op; 475 /* Select cipher key */ 476 sess->cipher.key.length = xform->cipher.key.length; 477 478 /* Set IV parameters */ 479 sess->iv.offset = xform->cipher.iv.offset; 480 sess->iv.length = xform->cipher.iv.length; 481 482 /* Select cipher algo */ 483 switch (xform->cipher.algo) { 484 case RTE_CRYPTO_CIPHER_3DES_CBC: 485 case RTE_CRYPTO_CIPHER_AES_CBC: 486 case RTE_CRYPTO_CIPHER_AES_CTR: 487 sess->cipher.mode = OPENSSL_CIPHER_LIB; 488 sess->cipher.algo = xform->cipher.algo; 489 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 490 491 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 492 &sess->cipher.evp_algo) != 0) 493 return -EINVAL; 494 495 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 496 sess->cipher.key.data); 497 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 498 if (EVP_EncryptInit_ex(sess->cipher.ctx, 499 sess->cipher.evp_algo, 500 NULL, xform->cipher.key.data, 501 NULL) != 1) { 502 return -EINVAL; 503 } 504 } else if (sess->cipher.direction == 505 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 506 if (EVP_DecryptInit_ex(sess->cipher.ctx, 507 sess->cipher.evp_algo, 508 NULL, xform->cipher.key.data, 509 NULL) != 1) { 510 return -EINVAL; 511 } 512 } 513 514 break; 515 516 case RTE_CRYPTO_CIPHER_3DES_CTR: 517 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 518 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 519 520 if (get_cipher_key_ede(xform->cipher.key.data, 521 sess->cipher.key.length, 522 sess->cipher.key.data) != 0) 523 return -EINVAL; 524 break; 525 526 case RTE_CRYPTO_CIPHER_DES_CBC: 527 sess->cipher.algo = xform->cipher.algo; 528 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 529 sess->cipher.evp_algo = EVP_des_cbc(); 530 531 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 532 sess->cipher.key.data); 533 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 534 if (EVP_EncryptInit_ex(sess->cipher.ctx, 535 sess->cipher.evp_algo, 536 NULL, xform->cipher.key.data, 537 NULL) != 1) { 538 return -EINVAL; 539 } 540 } else if (sess->cipher.direction == 541 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 542 if (EVP_DecryptInit_ex(sess->cipher.ctx, 543 sess->cipher.evp_algo, 544 NULL, xform->cipher.key.data, 545 NULL) != 1) { 546 return -EINVAL; 547 } 548 } 549 550 break; 551 552 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 553 sess->cipher.algo = xform->cipher.algo; 554 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 555 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 556 sess->cipher.evp_algo = EVP_des_cbc(); 557 558 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 559 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 560 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 561 NULL, xform->cipher.key.data, 0) != 1) 562 return -EINVAL; 563 564 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 565 sess->cipher.key.data); 566 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 567 if (EVP_EncryptInit_ex(sess->cipher.ctx, 568 sess->cipher.evp_algo, 569 NULL, xform->cipher.key.data, 570 NULL) != 1) { 571 return -EINVAL; 572 } 573 } else if (sess->cipher.direction == 574 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 575 if (EVP_DecryptInit_ex(sess->cipher.ctx, 576 sess->cipher.evp_algo, 577 NULL, xform->cipher.key.data, 578 NULL) != 1) { 579 return -EINVAL; 580 } 581 } 582 583 break; 584 default: 585 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 586 return -ENOTSUP; 587 } 588 589 return 0; 590 } 591 592 /* Set session auth parameters */ 593 static int 594 openssl_set_session_auth_parameters(struct openssl_session *sess, 595 const struct rte_crypto_sym_xform *xform) 596 { 597 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 598 char algo_name[MAX_OSSL_ALGO_NAME_SIZE]; 599 OSSL_PARAM params[2]; 600 const char *algo; 601 EVP_MAC *mac; 602 # endif 603 /* Select auth generate/verify */ 604 sess->auth.operation = xform->auth.op; 605 sess->auth.algo = xform->auth.algo; 606 607 sess->auth.digest_length = xform->auth.digest_length; 608 609 /* Select auth algo */ 610 switch (xform->auth.algo) { 611 case RTE_CRYPTO_AUTH_AES_GMAC: 612 /* 613 * OpenSSL requires GMAC to be a GCM operation 614 * with no cipher data length 615 */ 616 sess->cipher.key.length = xform->auth.key.length; 617 618 /* Set IV parameters */ 619 sess->iv.offset = xform->auth.iv.offset; 620 sess->iv.length = xform->auth.iv.length; 621 622 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 623 return openssl_set_sess_aead_enc_param(sess, 624 RTE_CRYPTO_AEAD_AES_GCM, 625 xform->auth.digest_length, 626 xform->auth.key.data); 627 else 628 return openssl_set_sess_aead_dec_param(sess, 629 RTE_CRYPTO_AEAD_AES_GCM, 630 xform->auth.digest_length, 631 xform->auth.key.data); 632 break; 633 634 case RTE_CRYPTO_AUTH_MD5: 635 case RTE_CRYPTO_AUTH_SHA1: 636 case RTE_CRYPTO_AUTH_SHA224: 637 case RTE_CRYPTO_AUTH_SHA256: 638 case RTE_CRYPTO_AUTH_SHA384: 639 case RTE_CRYPTO_AUTH_SHA512: 640 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 641 if (get_auth_algo(xform->auth.algo, 642 &sess->auth.auth.evp_algo) != 0) 643 return -EINVAL; 644 sess->auth.auth.ctx = EVP_MD_CTX_create(); 645 break; 646 647 case RTE_CRYPTO_AUTH_AES_CMAC: 648 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 649 if (xform->auth.key.length == 16) 650 algo = SN_aes_128_cbc; 651 else if (xform->auth.key.length == 24) 652 algo = SN_aes_192_cbc; 653 else if (xform->auth.key.length == 32) 654 algo = SN_aes_256_cbc; 655 else 656 return -EINVAL; 657 658 rte_memcpy(algo_name, algo, strlen(algo) + 1); 659 params[0] = OSSL_PARAM_construct_utf8_string( 660 OSSL_MAC_PARAM_CIPHER, algo_name, 0); 661 params[1] = OSSL_PARAM_construct_end(); 662 663 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 664 mac = EVP_MAC_fetch(NULL, OSSL_MAC_NAME_CMAC, NULL); 665 sess->auth.cmac.ctx = EVP_MAC_CTX_new(mac); 666 EVP_MAC_free(mac); 667 668 if (EVP_MAC_init(sess->auth.cmac.ctx, 669 xform->auth.key.data, 670 xform->auth.key.length, 671 params) != 1) 672 return -EINVAL; 673 # else 674 sess->auth.mode = OPENSSL_AUTH_AS_CMAC; 675 sess->auth.cmac.ctx = CMAC_CTX_new(); 676 if (get_cipher_algo(RTE_CRYPTO_CIPHER_AES_CBC, 677 xform->auth.key.length, 678 &sess->auth.cmac.evp_algo) != 0) 679 return -EINVAL; 680 if (CMAC_Init(sess->auth.cmac.ctx, 681 xform->auth.key.data, 682 xform->auth.key.length, 683 sess->auth.cmac.evp_algo, NULL) != 1) 684 return -EINVAL; 685 # endif 686 break; 687 688 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 689 case RTE_CRYPTO_AUTH_MD5_HMAC: 690 case RTE_CRYPTO_AUTH_SHA1_HMAC: 691 case RTE_CRYPTO_AUTH_SHA224_HMAC: 692 case RTE_CRYPTO_AUTH_SHA256_HMAC: 693 case RTE_CRYPTO_AUTH_SHA384_HMAC: 694 case RTE_CRYPTO_AUTH_SHA512_HMAC: 695 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 696 697 algo = digest_name_get(xform->auth.algo); 698 if (!algo) 699 return -EINVAL; 700 strlcpy(algo_name, algo, sizeof(algo_name)); 701 702 mac = EVP_MAC_fetch(NULL, "HMAC", NULL); 703 sess->auth.hmac.ctx = EVP_MAC_CTX_new(mac); 704 EVP_MAC_free(mac); 705 if (get_auth_algo(xform->auth.algo, 706 &sess->auth.hmac.evp_algo) != 0) 707 return -EINVAL; 708 709 params[0] = OSSL_PARAM_construct_utf8_string("digest", 710 algo_name, 0); 711 params[1] = OSSL_PARAM_construct_end(); 712 if (EVP_MAC_init(sess->auth.hmac.ctx, 713 xform->auth.key.data, 714 xform->auth.key.length, 715 params) != 1) 716 return -EINVAL; 717 break; 718 # else 719 case RTE_CRYPTO_AUTH_MD5_HMAC: 720 case RTE_CRYPTO_AUTH_SHA1_HMAC: 721 case RTE_CRYPTO_AUTH_SHA224_HMAC: 722 case RTE_CRYPTO_AUTH_SHA256_HMAC: 723 case RTE_CRYPTO_AUTH_SHA384_HMAC: 724 case RTE_CRYPTO_AUTH_SHA512_HMAC: 725 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 726 sess->auth.hmac.ctx = HMAC_CTX_new(); 727 if (get_auth_algo(xform->auth.algo, 728 &sess->auth.hmac.evp_algo) != 0) 729 return -EINVAL; 730 731 if (HMAC_Init_ex(sess->auth.hmac.ctx, 732 xform->auth.key.data, 733 xform->auth.key.length, 734 sess->auth.hmac.evp_algo, NULL) != 1) 735 return -EINVAL; 736 break; 737 # endif 738 default: 739 return -ENOTSUP; 740 } 741 742 return 0; 743 } 744 745 /* Set session AEAD parameters */ 746 static int 747 openssl_set_session_aead_parameters(struct openssl_session *sess, 748 const struct rte_crypto_sym_xform *xform) 749 { 750 /* Select cipher key */ 751 sess->cipher.key.length = xform->aead.key.length; 752 753 /* Set IV parameters */ 754 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 755 /* 756 * For AES-CCM, the actual IV is placed 757 * one byte after the start of the IV field, 758 * according to the API. 759 */ 760 sess->iv.offset = xform->aead.iv.offset + 1; 761 else 762 sess->iv.offset = xform->aead.iv.offset; 763 764 sess->iv.length = xform->aead.iv.length; 765 766 sess->auth.aad_length = xform->aead.aad_length; 767 sess->auth.digest_length = xform->aead.digest_length; 768 769 sess->aead_algo = xform->aead.algo; 770 /* Select cipher direction */ 771 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 772 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 773 xform->aead.digest_length, xform->aead.key.data); 774 else 775 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 776 xform->aead.digest_length, xform->aead.key.data); 777 } 778 779 /** Parse crypto xform chain and set private session parameters */ 780 int 781 openssl_set_session_parameters(struct openssl_session *sess, 782 const struct rte_crypto_sym_xform *xform) 783 { 784 const struct rte_crypto_sym_xform *cipher_xform = NULL; 785 const struct rte_crypto_sym_xform *auth_xform = NULL; 786 const struct rte_crypto_sym_xform *aead_xform = NULL; 787 int ret; 788 789 sess->chain_order = openssl_get_chain_order(xform); 790 switch (sess->chain_order) { 791 case OPENSSL_CHAIN_ONLY_CIPHER: 792 cipher_xform = xform; 793 break; 794 case OPENSSL_CHAIN_ONLY_AUTH: 795 auth_xform = xform; 796 break; 797 case OPENSSL_CHAIN_CIPHER_AUTH: 798 cipher_xform = xform; 799 auth_xform = xform->next; 800 break; 801 case OPENSSL_CHAIN_AUTH_CIPHER: 802 auth_xform = xform; 803 cipher_xform = xform->next; 804 break; 805 case OPENSSL_CHAIN_COMBINED: 806 aead_xform = xform; 807 break; 808 default: 809 return -EINVAL; 810 } 811 812 /* Default IV length = 0 */ 813 sess->iv.length = 0; 814 815 /* cipher_xform must be check before auth_xform */ 816 if (cipher_xform) { 817 ret = openssl_set_session_cipher_parameters( 818 sess, cipher_xform); 819 if (ret != 0) { 820 OPENSSL_LOG(ERR, 821 "Invalid/unsupported cipher parameters"); 822 return ret; 823 } 824 } 825 826 if (auth_xform) { 827 ret = openssl_set_session_auth_parameters(sess, auth_xform); 828 if (ret != 0) { 829 OPENSSL_LOG(ERR, 830 "Invalid/unsupported auth parameters"); 831 return ret; 832 } 833 } 834 835 if (aead_xform) { 836 ret = openssl_set_session_aead_parameters(sess, aead_xform); 837 if (ret != 0) { 838 OPENSSL_LOG(ERR, 839 "Invalid/unsupported AEAD parameters"); 840 return ret; 841 } 842 } 843 844 return 0; 845 } 846 847 /** Reset private session parameters */ 848 void 849 openssl_reset_session(struct openssl_session *sess) 850 { 851 EVP_CIPHER_CTX_free(sess->cipher.ctx); 852 853 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 854 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 855 856 switch (sess->auth.mode) { 857 case OPENSSL_AUTH_AS_AUTH: 858 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 859 break; 860 case OPENSSL_AUTH_AS_HMAC: 861 EVP_PKEY_free(sess->auth.hmac.pkey); 862 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 863 EVP_MAC_CTX_free(sess->auth.hmac.ctx); 864 # else 865 HMAC_CTX_free(sess->auth.hmac.ctx); 866 # endif 867 break; 868 case OPENSSL_AUTH_AS_CMAC: 869 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 870 EVP_MAC_CTX_free(sess->auth.cmac.ctx); 871 # else 872 CMAC_CTX_free(sess->auth.cmac.ctx); 873 # endif 874 break; 875 default: 876 break; 877 } 878 } 879 880 /** Provide session for operation */ 881 static void * 882 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 883 { 884 struct openssl_session *sess = NULL; 885 struct openssl_asym_session *asym_sess = NULL; 886 887 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 888 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 889 /* get existing session */ 890 if (likely(op->sym->session != NULL)) 891 sess = CRYPTODEV_GET_SYM_SESS_PRIV( 892 op->sym->session); 893 } else { 894 if (likely(op->asym->session != NULL)) 895 asym_sess = (struct openssl_asym_session *) 896 op->asym->session->sess_private_data; 897 if (asym_sess == NULL) 898 op->status = 899 RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 900 return asym_sess; 901 } 902 } else { 903 struct rte_cryptodev_sym_session *_sess; 904 /* sessionless asymmetric not supported */ 905 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) 906 return NULL; 907 908 /* provide internal session */ 909 rte_mempool_get(qp->sess_mp, (void **)&_sess); 910 911 if (_sess == NULL) 912 return NULL; 913 914 sess = (struct openssl_session *)_sess->driver_priv_data; 915 916 if (unlikely(openssl_set_session_parameters(sess, 917 op->sym->xform) != 0)) { 918 rte_mempool_put(qp->sess_mp, _sess); 919 sess = NULL; 920 } 921 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 922 923 } 924 925 if (sess == NULL) 926 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 927 928 return sess; 929 } 930 931 /* 932 *------------------------------------------------------------------------------ 933 * Process Operations 934 *------------------------------------------------------------------------------ 935 */ 936 static inline int 937 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 938 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 939 { 940 struct rte_mbuf *m; 941 int dstlen; 942 int l, n = srclen; 943 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 944 945 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 946 m = m->next) 947 offset -= rte_pktmbuf_data_len(m); 948 949 if (m == 0) 950 return -1; 951 952 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 953 if (inplace) 954 *dst = src; 955 956 l = rte_pktmbuf_data_len(m) - offset; 957 if (srclen <= l) { 958 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 959 return -1; 960 *dst += l; 961 return 0; 962 } 963 964 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 965 return -1; 966 967 *dst += dstlen; 968 n -= l; 969 970 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 971 uint8_t diff = l - dstlen, rem; 972 973 src = rte_pktmbuf_mtod(m, uint8_t *); 974 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 975 if (diff && inplace) { 976 rem = RTE_MIN(l, 977 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 978 if (EVP_EncryptUpdate(ctx, temp, 979 &dstlen, src, rem) <= 0) 980 return -1; 981 n -= rem; 982 rte_memcpy(*dst, temp, diff); 983 rte_memcpy(src, temp + diff, rem); 984 src += rem; 985 l -= rem; 986 } 987 if (inplace) 988 *dst = src; 989 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 990 return -1; 991 *dst += dstlen; 992 n -= l; 993 } 994 995 return 0; 996 } 997 998 static inline int 999 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 1000 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 1001 { 1002 struct rte_mbuf *m; 1003 int dstlen; 1004 int l, n = srclen; 1005 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 1006 1007 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1008 m = m->next) 1009 offset -= rte_pktmbuf_data_len(m); 1010 1011 if (m == 0) 1012 return -1; 1013 1014 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1015 if (inplace) 1016 *dst = src; 1017 1018 l = rte_pktmbuf_data_len(m) - offset; 1019 if (srclen <= l) { 1020 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 1021 return -1; 1022 *dst += l; 1023 return 0; 1024 } 1025 1026 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1027 return -1; 1028 1029 *dst += dstlen; 1030 n -= l; 1031 1032 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1033 uint8_t diff = l - dstlen, rem; 1034 1035 src = rte_pktmbuf_mtod(m, uint8_t *); 1036 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 1037 if (diff && inplace) { 1038 rem = RTE_MIN(l, 1039 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 1040 if (EVP_DecryptUpdate(ctx, temp, 1041 &dstlen, src, rem) <= 0) 1042 return -1; 1043 n -= rem; 1044 rte_memcpy(*dst, temp, diff); 1045 rte_memcpy(src, temp + diff, rem); 1046 src += rem; 1047 l -= rem; 1048 } 1049 if (inplace) 1050 *dst = src; 1051 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1052 return -1; 1053 *dst += dstlen; 1054 n -= l; 1055 } 1056 1057 return 0; 1058 } 1059 1060 /** Process standard openssl cipher encryption */ 1061 static int 1062 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1063 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1064 uint8_t inplace) 1065 { 1066 int totlen; 1067 1068 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1069 goto process_cipher_encrypt_err; 1070 1071 EVP_CIPHER_CTX_set_padding(ctx, 0); 1072 1073 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1074 srclen, ctx, inplace)) 1075 goto process_cipher_encrypt_err; 1076 1077 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 1078 goto process_cipher_encrypt_err; 1079 1080 return 0; 1081 1082 process_cipher_encrypt_err: 1083 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 1084 return -EINVAL; 1085 } 1086 1087 /** Process standard openssl cipher encryption */ 1088 static int 1089 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 1090 uint8_t *iv, int srclen, 1091 EVP_CIPHER_CTX *ctx) 1092 { 1093 uint8_t i; 1094 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 1095 int encrypted_ivlen; 1096 1097 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 1098 iv, DES_BLOCK_SIZE) <= 0) 1099 goto process_cipher_encrypt_err; 1100 1101 for (i = 0; i < srclen; i++) 1102 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 1103 1104 return 0; 1105 1106 process_cipher_encrypt_err: 1107 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 1108 return -EINVAL; 1109 } 1110 /** Process standard openssl cipher decryption */ 1111 static int 1112 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1113 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1114 uint8_t inplace) 1115 { 1116 int totlen; 1117 1118 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1119 goto process_cipher_decrypt_err; 1120 1121 EVP_CIPHER_CTX_set_padding(ctx, 0); 1122 1123 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1124 srclen, ctx, inplace)) 1125 goto process_cipher_decrypt_err; 1126 1127 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 1128 goto process_cipher_decrypt_err; 1129 return 0; 1130 1131 process_cipher_decrypt_err: 1132 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 1133 return -EINVAL; 1134 } 1135 1136 /** Process cipher des 3 ctr encryption, decryption algorithm */ 1137 static int 1138 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 1139 int offset, uint8_t *iv, uint8_t *key, int srclen, 1140 EVP_CIPHER_CTX *ctx) 1141 { 1142 uint8_t ebuf[8], ctr[8]; 1143 int unused, n; 1144 struct rte_mbuf *m; 1145 uint8_t *src; 1146 int l; 1147 1148 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1149 m = m->next) 1150 offset -= rte_pktmbuf_data_len(m); 1151 1152 if (m == 0) 1153 goto process_cipher_des3ctr_err; 1154 1155 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1156 l = rte_pktmbuf_data_len(m) - offset; 1157 1158 /* We use 3DES encryption also for decryption. 1159 * IV is not important for 3DES ecb 1160 */ 1161 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 1162 goto process_cipher_des3ctr_err; 1163 1164 memcpy(ctr, iv, 8); 1165 1166 for (n = 0; n < srclen; n++) { 1167 if (n % 8 == 0) { 1168 if (EVP_EncryptUpdate(ctx, 1169 (unsigned char *)&ebuf, &unused, 1170 (const unsigned char *)&ctr, 8) <= 0) 1171 goto process_cipher_des3ctr_err; 1172 ctr_inc(ctr); 1173 } 1174 dst[n] = *(src++) ^ ebuf[n % 8]; 1175 1176 l--; 1177 if (!l) { 1178 m = m->next; 1179 if (m) { 1180 src = rte_pktmbuf_mtod(m, uint8_t *); 1181 l = rte_pktmbuf_data_len(m); 1182 } 1183 } 1184 } 1185 1186 return 0; 1187 1188 process_cipher_des3ctr_err: 1189 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 1190 return -EINVAL; 1191 } 1192 1193 /** Process AES-GCM encrypt algorithm */ 1194 static int 1195 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1196 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1197 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1198 { 1199 int len = 0; 1200 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1201 int unused = 0; 1202 uint8_t empty[] = {}; 1203 #endif 1204 1205 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1206 goto process_auth_encryption_gcm_err; 1207 1208 if (aadlen > 0) 1209 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1210 goto process_auth_encryption_gcm_err; 1211 1212 if (srclen > 0) 1213 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1214 srclen, ctx, 0)) 1215 goto process_auth_encryption_gcm_err; 1216 1217 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1218 /* Workaround open ssl bug in version less then 1.0.1f */ 1219 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1220 goto process_auth_encryption_gcm_err; 1221 #endif 1222 1223 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1224 goto process_auth_encryption_gcm_err; 1225 1226 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1227 goto process_auth_encryption_gcm_err; 1228 1229 return 0; 1230 1231 process_auth_encryption_gcm_err: 1232 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1233 return -EINVAL; 1234 } 1235 1236 /** Process AES-CCM encrypt algorithm */ 1237 static int 1238 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1239 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1240 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1241 { 1242 int len = 0; 1243 1244 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1245 goto process_auth_encryption_ccm_err; 1246 1247 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1248 goto process_auth_encryption_ccm_err; 1249 1250 if (aadlen > 0) 1251 /* 1252 * For AES-CCM, the actual AAD is placed 1253 * 18 bytes after the start of the AAD field, 1254 * according to the API. 1255 */ 1256 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1257 goto process_auth_encryption_ccm_err; 1258 1259 if (srclen >= 0) 1260 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1261 srclen, ctx, 0)) 1262 goto process_auth_encryption_ccm_err; 1263 1264 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1265 goto process_auth_encryption_ccm_err; 1266 1267 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1268 goto process_auth_encryption_ccm_err; 1269 1270 return 0; 1271 1272 process_auth_encryption_ccm_err: 1273 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1274 return -EINVAL; 1275 } 1276 1277 /** Process AES-GCM decrypt algorithm */ 1278 static int 1279 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1280 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1281 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1282 { 1283 int len = 0; 1284 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1285 int unused = 0; 1286 uint8_t empty[] = {}; 1287 #endif 1288 1289 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1290 goto process_auth_decryption_gcm_err; 1291 1292 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1293 goto process_auth_decryption_gcm_err; 1294 1295 if (aadlen > 0) 1296 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1297 goto process_auth_decryption_gcm_err; 1298 1299 if (srclen > 0) 1300 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1301 srclen, ctx, 0)) 1302 goto process_auth_decryption_gcm_err; 1303 1304 #if OPENSSL_VERSION_NUMBER < 0x10100000L 1305 /* Workaround open ssl bug in version less then 1.0.1f */ 1306 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1307 goto process_auth_decryption_gcm_err; 1308 #endif 1309 1310 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1311 return -EFAULT; 1312 1313 return 0; 1314 1315 process_auth_decryption_gcm_err: 1316 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1317 return -EINVAL; 1318 } 1319 1320 /** Process AES-CCM decrypt algorithm */ 1321 static int 1322 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1323 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1324 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1325 EVP_CIPHER_CTX *ctx) 1326 { 1327 int len = 0; 1328 1329 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1330 goto process_auth_decryption_ccm_err; 1331 1332 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1333 goto process_auth_decryption_ccm_err; 1334 1335 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1336 goto process_auth_decryption_ccm_err; 1337 1338 if (aadlen > 0) 1339 /* 1340 * For AES-CCM, the actual AAD is placed 1341 * 18 bytes after the start of the AAD field, 1342 * according to the API. 1343 */ 1344 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1345 goto process_auth_decryption_ccm_err; 1346 1347 if (srclen >= 0) 1348 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1349 srclen, ctx, 0)) 1350 return -EFAULT; 1351 1352 return 0; 1353 1354 process_auth_decryption_ccm_err: 1355 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1356 return -EINVAL; 1357 } 1358 1359 /** Process standard openssl auth algorithms */ 1360 static int 1361 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1362 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1363 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1364 { 1365 size_t dstlen; 1366 struct rte_mbuf *m; 1367 int l, n = srclen; 1368 uint8_t *src; 1369 1370 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1371 m = m->next) 1372 offset -= rte_pktmbuf_data_len(m); 1373 1374 if (m == 0) 1375 goto process_auth_err; 1376 1377 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1378 goto process_auth_err; 1379 1380 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1381 1382 l = rte_pktmbuf_data_len(m) - offset; 1383 if (srclen <= l) { 1384 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1385 goto process_auth_err; 1386 goto process_auth_final; 1387 } 1388 1389 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1390 goto process_auth_err; 1391 1392 n -= l; 1393 1394 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1395 src = rte_pktmbuf_mtod(m, uint8_t *); 1396 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1397 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1398 goto process_auth_err; 1399 n -= l; 1400 } 1401 1402 process_auth_final: 1403 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1404 goto process_auth_err; 1405 return 0; 1406 1407 process_auth_err: 1408 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1409 return -EINVAL; 1410 } 1411 1412 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1413 /** Process standard openssl auth algorithms with hmac/cmac */ 1414 static int 1415 process_openssl_auth_mac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1416 int srclen, EVP_MAC_CTX *ctx) 1417 { 1418 size_t dstlen; 1419 struct rte_mbuf *m; 1420 int l, n = srclen; 1421 uint8_t *src; 1422 1423 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1424 m = m->next) 1425 offset -= rte_pktmbuf_data_len(m); 1426 1427 if (m == 0) 1428 goto process_auth_err; 1429 1430 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1431 1432 l = rte_pktmbuf_data_len(m) - offset; 1433 if (srclen <= l) { 1434 if (EVP_MAC_update(ctx, (unsigned char *)src, srclen) != 1) 1435 goto process_auth_err; 1436 goto process_auth_final; 1437 } 1438 1439 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1440 goto process_auth_err; 1441 1442 n -= l; 1443 1444 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1445 src = rte_pktmbuf_mtod(m, uint8_t *); 1446 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1447 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1448 goto process_auth_err; 1449 n -= l; 1450 } 1451 1452 process_auth_final: 1453 if (EVP_MAC_final(ctx, dst, &dstlen, DIGEST_LENGTH_MAX) != 1) 1454 goto process_auth_err; 1455 1456 EVP_MAC_CTX_free(ctx); 1457 return 0; 1458 1459 process_auth_err: 1460 EVP_MAC_CTX_free(ctx); 1461 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1462 return -EINVAL; 1463 } 1464 # else 1465 /** Process standard openssl auth algorithms with hmac */ 1466 static int 1467 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1468 int srclen, HMAC_CTX *ctx) 1469 { 1470 unsigned int dstlen; 1471 struct rte_mbuf *m; 1472 int l, n = srclen; 1473 uint8_t *src; 1474 1475 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1476 m = m->next) 1477 offset -= rte_pktmbuf_data_len(m); 1478 1479 if (m == 0) 1480 goto process_auth_err; 1481 1482 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1483 1484 l = rte_pktmbuf_data_len(m) - offset; 1485 if (srclen <= l) { 1486 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1487 goto process_auth_err; 1488 goto process_auth_final; 1489 } 1490 1491 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1492 goto process_auth_err; 1493 1494 n -= l; 1495 1496 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1497 src = rte_pktmbuf_mtod(m, uint8_t *); 1498 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1499 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1500 goto process_auth_err; 1501 n -= l; 1502 } 1503 1504 process_auth_final: 1505 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1506 goto process_auth_err; 1507 1508 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1509 goto process_auth_err; 1510 1511 return 0; 1512 1513 process_auth_err: 1514 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1515 return -EINVAL; 1516 } 1517 1518 /** Process standard openssl auth algorithms with cmac */ 1519 static int 1520 process_openssl_auth_cmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1521 int srclen, CMAC_CTX *ctx) 1522 { 1523 unsigned int dstlen; 1524 struct rte_mbuf *m; 1525 int l, n = srclen; 1526 uint8_t *src; 1527 1528 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1529 m = m->next) 1530 offset -= rte_pktmbuf_data_len(m); 1531 1532 if (m == 0) 1533 goto process_auth_err; 1534 1535 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1536 1537 l = rte_pktmbuf_data_len(m) - offset; 1538 if (srclen <= l) { 1539 if (CMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1540 goto process_auth_err; 1541 goto process_auth_final; 1542 } 1543 1544 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1545 goto process_auth_err; 1546 1547 n -= l; 1548 1549 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1550 src = rte_pktmbuf_mtod(m, uint8_t *); 1551 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1552 if (CMAC_Update(ctx, (unsigned char *)src, l) != 1) 1553 goto process_auth_err; 1554 n -= l; 1555 } 1556 1557 process_auth_final: 1558 if (CMAC_Final(ctx, dst, (size_t *)&dstlen) != 1) 1559 goto process_auth_err; 1560 1561 CMAC_CTX_cleanup(ctx); 1562 1563 return 0; 1564 1565 process_auth_err: 1566 OPENSSL_LOG(ERR, "Process openssl cmac auth failed"); 1567 return -EINVAL; 1568 } 1569 # endif 1570 /*----------------------------------------------------------------------------*/ 1571 1572 /** Process auth/cipher combined operation */ 1573 static void 1574 process_openssl_combined_op 1575 (struct rte_crypto_op *op, struct openssl_session *sess, 1576 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1577 { 1578 /* cipher */ 1579 uint8_t *dst = NULL, *iv, *tag, *aad; 1580 int srclen, aadlen, status = -1; 1581 uint32_t offset; 1582 uint8_t taglen; 1583 1584 /* 1585 * Segmented destination buffer is not supported for 1586 * encryption/decryption 1587 */ 1588 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1589 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1590 return; 1591 } 1592 1593 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1594 sess->iv.offset); 1595 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1596 srclen = 0; 1597 offset = op->sym->auth.data.offset; 1598 aadlen = op->sym->auth.data.length; 1599 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1600 op->sym->auth.data.offset); 1601 tag = op->sym->auth.digest.data; 1602 if (tag == NULL) 1603 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1604 offset + aadlen); 1605 } else { 1606 srclen = op->sym->aead.data.length; 1607 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1608 op->sym->aead.data.offset); 1609 offset = op->sym->aead.data.offset; 1610 aad = op->sym->aead.aad.data; 1611 aadlen = sess->auth.aad_length; 1612 tag = op->sym->aead.digest.data; 1613 if (tag == NULL) 1614 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1615 offset + srclen); 1616 } 1617 1618 taglen = sess->auth.digest_length; 1619 1620 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1621 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1622 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1623 status = process_openssl_auth_encryption_gcm( 1624 mbuf_src, offset, srclen, 1625 aad, aadlen, iv, 1626 dst, tag, sess->cipher.ctx); 1627 else 1628 status = process_openssl_auth_encryption_ccm( 1629 mbuf_src, offset, srclen, 1630 aad, aadlen, iv, 1631 dst, tag, taglen, sess->cipher.ctx); 1632 1633 } else { 1634 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1635 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1636 status = process_openssl_auth_decryption_gcm( 1637 mbuf_src, offset, srclen, 1638 aad, aadlen, iv, 1639 dst, tag, sess->cipher.ctx); 1640 else 1641 status = process_openssl_auth_decryption_ccm( 1642 mbuf_src, offset, srclen, 1643 aad, aadlen, iv, 1644 dst, tag, taglen, sess->cipher.ctx); 1645 } 1646 1647 if (status != 0) { 1648 if (status == (-EFAULT) && 1649 sess->auth.operation == 1650 RTE_CRYPTO_AUTH_OP_VERIFY) 1651 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1652 else 1653 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1654 } 1655 } 1656 1657 /** Process cipher operation */ 1658 static void 1659 process_openssl_cipher_op 1660 (struct rte_crypto_op *op, struct openssl_session *sess, 1661 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1662 { 1663 uint8_t *dst, *iv; 1664 int srclen, status; 1665 uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0; 1666 EVP_CIPHER_CTX *ctx_copy; 1667 1668 /* 1669 * Segmented OOP destination buffer is not supported for encryption/ 1670 * decryption. In case of des3ctr, even inplace segmented buffers are 1671 * not supported. 1672 */ 1673 if (!rte_pktmbuf_is_contiguous(mbuf_dst) && 1674 (!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) { 1675 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1676 return; 1677 } 1678 1679 srclen = op->sym->cipher.data.length; 1680 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1681 op->sym->cipher.data.offset); 1682 1683 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1684 sess->iv.offset); 1685 ctx_copy = EVP_CIPHER_CTX_new(); 1686 EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx); 1687 1688 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1689 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1690 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1691 op->sym->cipher.data.offset, iv, 1692 srclen, ctx_copy, inplace); 1693 else 1694 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1695 op->sym->cipher.data.offset, iv, 1696 srclen, ctx_copy, inplace); 1697 else 1698 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1699 op->sym->cipher.data.offset, iv, 1700 sess->cipher.key.data, srclen, 1701 ctx_copy); 1702 1703 EVP_CIPHER_CTX_free(ctx_copy); 1704 if (status != 0) 1705 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1706 } 1707 1708 /** Process cipher operation */ 1709 static void 1710 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1711 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1712 struct rte_mbuf *mbuf_dst) 1713 { 1714 uint8_t *src, *dst, *iv; 1715 uint8_t block_size, last_block_len; 1716 int srclen, status = 0; 1717 1718 srclen = op->sym->cipher.data.length; 1719 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1720 op->sym->cipher.data.offset); 1721 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1722 op->sym->cipher.data.offset); 1723 1724 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1725 sess->iv.offset); 1726 1727 block_size = DES_BLOCK_SIZE; 1728 1729 last_block_len = srclen % block_size; 1730 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1731 /* Encrypt only with ECB mode XOR IV */ 1732 if (srclen < block_size) { 1733 status = process_openssl_cipher_bpi_encrypt(src, dst, 1734 iv, srclen, 1735 sess->cipher.bpi_ctx); 1736 } else { 1737 srclen -= last_block_len; 1738 /* Encrypt with the block aligned stream with CBC mode */ 1739 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1740 op->sym->cipher.data.offset, iv, 1741 srclen, sess->cipher.ctx, 0); 1742 if (last_block_len) { 1743 /* Point at last block */ 1744 dst += srclen; 1745 /* 1746 * IV is the last encrypted block from 1747 * the previous operation 1748 */ 1749 iv = dst - block_size; 1750 src += srclen; 1751 srclen = last_block_len; 1752 /* Encrypt the last frame with ECB mode */ 1753 status |= process_openssl_cipher_bpi_encrypt(src, 1754 dst, iv, 1755 srclen, sess->cipher.bpi_ctx); 1756 } 1757 } 1758 } else { 1759 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1760 if (srclen < block_size) { 1761 status = process_openssl_cipher_bpi_encrypt(src, dst, 1762 iv, 1763 srclen, 1764 sess->cipher.bpi_ctx); 1765 } else { 1766 if (last_block_len) { 1767 /* Point at last block */ 1768 dst += srclen - last_block_len; 1769 src += srclen - last_block_len; 1770 /* 1771 * IV is the last full block 1772 */ 1773 iv = src - block_size; 1774 /* 1775 * Decrypt the last frame with ECB mode 1776 * (encrypt, as it is the same operation) 1777 */ 1778 status = process_openssl_cipher_bpi_encrypt(src, 1779 dst, iv, 1780 last_block_len, sess->cipher.bpi_ctx); 1781 /* Prepare parameters for CBC mode op */ 1782 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1783 sess->iv.offset); 1784 dst += last_block_len - srclen; 1785 srclen -= last_block_len; 1786 } 1787 1788 /* Decrypt with CBC mode */ 1789 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1790 op->sym->cipher.data.offset, iv, 1791 srclen, sess->cipher.ctx, 0); 1792 } 1793 } 1794 1795 if (status != 0) 1796 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1797 } 1798 1799 /** Process auth operation */ 1800 static void 1801 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1802 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1803 struct rte_mbuf *mbuf_dst) 1804 { 1805 uint8_t *dst; 1806 int srclen, status; 1807 EVP_MD_CTX *ctx_a; 1808 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1809 EVP_MAC_CTX *ctx_h; 1810 EVP_MAC_CTX *ctx_c; 1811 # else 1812 HMAC_CTX *ctx_h; 1813 CMAC_CTX *ctx_c; 1814 # endif 1815 1816 srclen = op->sym->auth.data.length; 1817 1818 dst = qp->temp_digest; 1819 1820 switch (sess->auth.mode) { 1821 case OPENSSL_AUTH_AS_AUTH: 1822 ctx_a = EVP_MD_CTX_create(); 1823 EVP_MD_CTX_copy_ex(ctx_a, sess->auth.auth.ctx); 1824 status = process_openssl_auth(mbuf_src, dst, 1825 op->sym->auth.data.offset, NULL, NULL, srclen, 1826 ctx_a, sess->auth.auth.evp_algo); 1827 EVP_MD_CTX_destroy(ctx_a); 1828 break; 1829 case OPENSSL_AUTH_AS_HMAC: 1830 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1831 ctx_h = EVP_MAC_CTX_dup(sess->auth.hmac.ctx); 1832 status = process_openssl_auth_mac(mbuf_src, dst, 1833 op->sym->auth.data.offset, srclen, 1834 ctx_h); 1835 # else 1836 ctx_h = HMAC_CTX_new(); 1837 HMAC_CTX_copy(ctx_h, sess->auth.hmac.ctx); 1838 status = process_openssl_auth_hmac(mbuf_src, dst, 1839 op->sym->auth.data.offset, srclen, 1840 ctx_h); 1841 HMAC_CTX_free(ctx_h); 1842 # endif 1843 break; 1844 case OPENSSL_AUTH_AS_CMAC: 1845 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1846 ctx_c = EVP_MAC_CTX_dup(sess->auth.cmac.ctx); 1847 status = process_openssl_auth_mac(mbuf_src, dst, 1848 op->sym->auth.data.offset, srclen, 1849 ctx_c); 1850 # else 1851 ctx_c = CMAC_CTX_new(); 1852 CMAC_CTX_copy(ctx_c, sess->auth.cmac.ctx); 1853 status = process_openssl_auth_cmac(mbuf_src, dst, 1854 op->sym->auth.data.offset, srclen, 1855 ctx_c); 1856 CMAC_CTX_free(ctx_c); 1857 # endif 1858 break; 1859 default: 1860 status = -1; 1861 break; 1862 } 1863 1864 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1865 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data, 1866 sess->auth.digest_length) != 0) { 1867 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1868 } 1869 } else { 1870 uint8_t *auth_dst; 1871 1872 auth_dst = op->sym->auth.digest.data; 1873 if (auth_dst == NULL) 1874 auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1875 op->sym->auth.data.offset + 1876 op->sym->auth.data.length); 1877 memcpy(auth_dst, dst, sess->auth.digest_length); 1878 } 1879 1880 if (status != 0) 1881 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1882 } 1883 1884 /* process dsa sign operation */ 1885 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 1886 static int 1887 process_openssl_dsa_sign_op_evp(struct rte_crypto_op *cop, 1888 struct openssl_asym_session *sess) 1889 { 1890 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1891 EVP_PKEY_CTX *dsa_ctx = NULL; 1892 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 1893 EVP_PKEY *pkey = NULL; 1894 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 1895 OSSL_PARAM *params = NULL; 1896 1897 size_t outlen; 1898 unsigned char *dsa_sign_data; 1899 const unsigned char *dsa_sign_data_p; 1900 1901 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1902 params = OSSL_PARAM_BLD_to_param(param_bld); 1903 if (!params) { 1904 OSSL_PARAM_BLD_free(param_bld); 1905 return -1; 1906 } 1907 1908 if (key_ctx == NULL 1909 || EVP_PKEY_fromdata_init(key_ctx) <= 0 1910 || EVP_PKEY_fromdata(key_ctx, &pkey, 1911 EVP_PKEY_KEYPAIR, params) <= 0) 1912 goto err_dsa_sign; 1913 1914 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 1915 if (!dsa_ctx) 1916 goto err_dsa_sign; 1917 1918 if (EVP_PKEY_sign_init(dsa_ctx) <= 0) 1919 goto err_dsa_sign; 1920 1921 if (EVP_PKEY_sign(dsa_ctx, NULL, &outlen, op->message.data, 1922 op->message.length) <= 0) 1923 goto err_dsa_sign; 1924 1925 if (outlen <= 0) 1926 goto err_dsa_sign; 1927 1928 dsa_sign_data = OPENSSL_malloc(outlen); 1929 if (!dsa_sign_data) 1930 goto err_dsa_sign; 1931 1932 if (EVP_PKEY_sign(dsa_ctx, dsa_sign_data, &outlen, op->message.data, 1933 op->message.length) <= 0) { 1934 OPENSSL_free(dsa_sign_data); 1935 goto err_dsa_sign; 1936 } 1937 1938 dsa_sign_data_p = (const unsigned char *)dsa_sign_data; 1939 DSA_SIG *sign = d2i_DSA_SIG(NULL, &dsa_sign_data_p, outlen); 1940 if (!sign) { 1941 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 1942 OPENSSL_free(dsa_sign_data); 1943 goto err_dsa_sign; 1944 } else { 1945 const BIGNUM *r = NULL, *s = NULL; 1946 get_dsa_sign(sign, &r, &s); 1947 1948 op->r.length = BN_bn2bin(r, op->r.data); 1949 op->s.length = BN_bn2bin(s, op->s.data); 1950 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1951 } 1952 1953 DSA_SIG_free(sign); 1954 OPENSSL_free(dsa_sign_data); 1955 return 0; 1956 1957 err_dsa_sign: 1958 if (params) 1959 OSSL_PARAM_free(params); 1960 if (key_ctx) 1961 EVP_PKEY_CTX_free(key_ctx); 1962 if (dsa_ctx) 1963 EVP_PKEY_CTX_free(dsa_ctx); 1964 return -1; 1965 } 1966 1967 /* process dsa verify operation */ 1968 static int 1969 process_openssl_dsa_verify_op_evp(struct rte_crypto_op *cop, 1970 struct openssl_asym_session *sess) 1971 { 1972 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1973 DSA_SIG *sign = DSA_SIG_new(); 1974 BIGNUM *r = NULL, *s = NULL; 1975 BIGNUM *pub_key = NULL; 1976 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 1977 OSSL_PARAM *params = NULL; 1978 EVP_PKEY *pkey = NULL; 1979 EVP_PKEY_CTX *dsa_ctx = NULL; 1980 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 1981 unsigned char *dsa_sig = NULL; 1982 size_t sig_len; 1983 int ret = -1; 1984 1985 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1986 if (!param_bld) { 1987 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 1988 return -1; 1989 } 1990 1991 r = BN_bin2bn(op->r.data, op->r.length, r); 1992 s = BN_bin2bn(op->s.data, op->s.length, s); 1993 pub_key = BN_bin2bn(op->y.data, op->y.length, pub_key); 1994 if (!r || !s || !pub_key) { 1995 BN_free(r); 1996 BN_free(s); 1997 BN_free(pub_key); 1998 OSSL_PARAM_BLD_free(param_bld); 1999 goto err_dsa_verify; 2000 } 2001 2002 set_dsa_sign(sign, r, s); 2003 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PUB_KEY, pub_key)) { 2004 OSSL_PARAM_BLD_free(param_bld); 2005 goto err_dsa_verify; 2006 } 2007 2008 params = OSSL_PARAM_BLD_to_param(param_bld); 2009 if (!params) { 2010 OSSL_PARAM_BLD_free(param_bld); 2011 goto err_dsa_verify; 2012 } 2013 2014 if (key_ctx == NULL 2015 || EVP_PKEY_fromdata_init(key_ctx) <= 0 2016 || EVP_PKEY_fromdata(key_ctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2017 goto err_dsa_verify; 2018 2019 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 2020 if (!dsa_ctx) 2021 goto err_dsa_verify; 2022 2023 if (!sign) 2024 goto err_dsa_verify; 2025 2026 sig_len = i2d_DSA_SIG(sign, &dsa_sig); 2027 if (EVP_PKEY_verify_init(dsa_ctx) <= 0) 2028 goto err_dsa_verify; 2029 2030 ret = EVP_PKEY_verify(dsa_ctx, dsa_sig, sig_len, 2031 op->message.data, op->message.length); 2032 if (ret == 1) { 2033 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2034 ret = 0; 2035 } 2036 2037 err_dsa_verify: 2038 if (sign) 2039 DSA_SIG_free(sign); 2040 if (params) 2041 OSSL_PARAM_free(params); 2042 if (key_ctx) 2043 EVP_PKEY_CTX_free(key_ctx); 2044 if (dsa_ctx) 2045 EVP_PKEY_CTX_free(dsa_ctx); 2046 2047 return ret; 2048 } 2049 #else 2050 static int 2051 process_openssl_dsa_sign_op(struct rte_crypto_op *cop, 2052 struct openssl_asym_session *sess) 2053 { 2054 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2055 DSA *dsa = sess->u.s.dsa; 2056 DSA_SIG *sign = NULL; 2057 2058 sign = DSA_do_sign(op->message.data, 2059 op->message.length, 2060 dsa); 2061 2062 if (sign == NULL) { 2063 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 2064 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2065 } else { 2066 const BIGNUM *r = NULL, *s = NULL; 2067 get_dsa_sign(sign, &r, &s); 2068 2069 op->r.length = BN_bn2bin(r, op->r.data); 2070 op->s.length = BN_bn2bin(s, op->s.data); 2071 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2072 } 2073 2074 DSA_SIG_free(sign); 2075 2076 return 0; 2077 } 2078 2079 /* process dsa verify operation */ 2080 static int 2081 process_openssl_dsa_verify_op(struct rte_crypto_op *cop, 2082 struct openssl_asym_session *sess) 2083 { 2084 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 2085 DSA *dsa = sess->u.s.dsa; 2086 int ret; 2087 DSA_SIG *sign = DSA_SIG_new(); 2088 BIGNUM *r = NULL, *s = NULL; 2089 BIGNUM *pub_key = NULL; 2090 2091 if (sign == NULL) { 2092 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 2093 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2094 return -1; 2095 } 2096 2097 r = BN_bin2bn(op->r.data, 2098 op->r.length, 2099 r); 2100 s = BN_bin2bn(op->s.data, 2101 op->s.length, 2102 s); 2103 pub_key = BN_bin2bn(op->y.data, 2104 op->y.length, 2105 pub_key); 2106 if (!r || !s || !pub_key) { 2107 BN_free(r); 2108 BN_free(s); 2109 BN_free(pub_key); 2110 2111 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2112 return -1; 2113 } 2114 set_dsa_sign(sign, r, s); 2115 set_dsa_pub_key(dsa, pub_key); 2116 2117 ret = DSA_do_verify(op->message.data, 2118 op->message.length, 2119 sign, 2120 dsa); 2121 2122 if (ret != 1) 2123 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2124 else 2125 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2126 2127 DSA_SIG_free(sign); 2128 2129 return 0; 2130 } 2131 #endif 2132 2133 /* process dh operation */ 2134 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2135 static int 2136 process_openssl_dh_op_evp(struct rte_crypto_op *cop, 2137 struct openssl_asym_session *sess) 2138 { 2139 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2140 OSSL_PARAM_BLD *param_bld = sess->u.dh.param_bld; 2141 OSSL_PARAM_BLD *param_bld_peer = sess->u.dh.param_bld_peer; 2142 OSSL_PARAM *params = NULL; 2143 EVP_PKEY *dhpkey = NULL; 2144 EVP_PKEY *peerkey = NULL; 2145 BIGNUM *priv_key = NULL; 2146 BIGNUM *pub_key = NULL; 2147 int ret = -1; 2148 2149 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2150 EVP_PKEY_CTX *dh_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2151 if (dh_ctx == NULL || param_bld == NULL) 2152 return ret; 2153 2154 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2155 OSSL_PARAM *params_peer = NULL; 2156 2157 if (!param_bld_peer) 2158 return ret; 2159 2160 pub_key = BN_bin2bn(op->pub_key.data, op->pub_key.length, 2161 pub_key); 2162 if (pub_key == NULL) { 2163 OSSL_PARAM_BLD_free(param_bld_peer); 2164 return ret; 2165 } 2166 2167 if (!OSSL_PARAM_BLD_push_BN(param_bld_peer, OSSL_PKEY_PARAM_PUB_KEY, 2168 pub_key)) { 2169 OPENSSL_LOG(ERR, "Failed to set public key\n"); 2170 OSSL_PARAM_BLD_free(param_bld_peer); 2171 BN_free(pub_key); 2172 return ret; 2173 } 2174 2175 params_peer = OSSL_PARAM_BLD_to_param(param_bld_peer); 2176 if (!params_peer) { 2177 OSSL_PARAM_BLD_free(param_bld_peer); 2178 BN_free(pub_key); 2179 return ret; 2180 } 2181 2182 EVP_PKEY_CTX *peer_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2183 if (EVP_PKEY_keygen_init(peer_ctx) != 1) { 2184 OSSL_PARAM_free(params_peer); 2185 BN_free(pub_key); 2186 return ret; 2187 } 2188 2189 if (EVP_PKEY_CTX_set_params(peer_ctx, params_peer) != 1) { 2190 EVP_PKEY_CTX_free(peer_ctx); 2191 OSSL_PARAM_free(params_peer); 2192 BN_free(pub_key); 2193 return ret; 2194 } 2195 2196 if (EVP_PKEY_keygen(peer_ctx, &peerkey) != 1) { 2197 EVP_PKEY_CTX_free(peer_ctx); 2198 OSSL_PARAM_free(params_peer); 2199 BN_free(pub_key); 2200 return ret; 2201 } 2202 2203 priv_key = BN_bin2bn(op->priv_key.data, op->priv_key.length, 2204 priv_key); 2205 if (priv_key == NULL) { 2206 EVP_PKEY_CTX_free(peer_ctx); 2207 OSSL_PARAM_free(params_peer); 2208 BN_free(pub_key); 2209 return ret; 2210 } 2211 2212 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY, 2213 priv_key)) { 2214 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2215 EVP_PKEY_CTX_free(peer_ctx); 2216 OSSL_PARAM_free(params_peer); 2217 BN_free(pub_key); 2218 BN_free(priv_key); 2219 return ret; 2220 } 2221 2222 OSSL_PARAM_free(params_peer); 2223 EVP_PKEY_CTX_free(peer_ctx); 2224 } 2225 2226 params = OSSL_PARAM_BLD_to_param(param_bld); 2227 if (!params) 2228 goto err_dh; 2229 2230 if (EVP_PKEY_keygen_init(dh_ctx) != 1) 2231 goto err_dh; 2232 2233 if (EVP_PKEY_CTX_set_params(dh_ctx, params) != 1) 2234 goto err_dh; 2235 2236 if (EVP_PKEY_keygen(dh_ctx, &dhpkey) != 1) 2237 goto err_dh; 2238 2239 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2240 OPENSSL_LOG(DEBUG, "%s:%d updated pub key\n", __func__, __LINE__); 2241 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PUB_KEY, &pub_key)) 2242 goto err_dh; 2243 /* output public key */ 2244 op->pub_key.length = BN_bn2bin(pub_key, op->pub_key.data); 2245 } 2246 2247 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2248 2249 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", __func__, __LINE__); 2250 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PRIV_KEY, &priv_key)) 2251 goto err_dh; 2252 2253 /* provide generated private key back to user */ 2254 op->priv_key.length = BN_bn2bin(priv_key, op->priv_key.data); 2255 } 2256 2257 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2258 size_t skey_len; 2259 EVP_PKEY_CTX *sc_ctx = EVP_PKEY_CTX_new(dhpkey, NULL); 2260 if (!sc_ctx) 2261 goto err_dh; 2262 2263 if (EVP_PKEY_derive_init(sc_ctx) <= 0) { 2264 EVP_PKEY_CTX_free(sc_ctx); 2265 goto err_dh; 2266 } 2267 2268 if (!peerkey) { 2269 EVP_PKEY_CTX_free(sc_ctx); 2270 goto err_dh; 2271 } 2272 2273 if (EVP_PKEY_derive_set_peer(sc_ctx, peerkey) <= 0) { 2274 EVP_PKEY_CTX_free(sc_ctx); 2275 goto err_dh; 2276 } 2277 2278 /* Determine buffer length */ 2279 if (EVP_PKEY_derive(sc_ctx, NULL, &skey_len) <= 0) { 2280 EVP_PKEY_CTX_free(sc_ctx); 2281 goto err_dh; 2282 } 2283 2284 if (EVP_PKEY_derive(sc_ctx, op->shared_secret.data, &skey_len) <= 0) { 2285 EVP_PKEY_CTX_free(sc_ctx); 2286 goto err_dh; 2287 } 2288 2289 op->shared_secret.length = skey_len; 2290 EVP_PKEY_CTX_free(sc_ctx); 2291 } 2292 2293 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2294 ret = 0; 2295 2296 err_dh: 2297 if (pub_key) 2298 BN_free(pub_key); 2299 if (priv_key) 2300 BN_free(priv_key); 2301 if (params) 2302 OSSL_PARAM_free(params); 2303 if (dhpkey) 2304 EVP_PKEY_free(dhpkey); 2305 if (peerkey) 2306 EVP_PKEY_free(peerkey); 2307 2308 EVP_PKEY_CTX_free(dh_ctx); 2309 2310 return ret; 2311 } 2312 #else 2313 static int 2314 process_openssl_dh_op(struct rte_crypto_op *cop, 2315 struct openssl_asym_session *sess) 2316 { 2317 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2318 struct rte_crypto_asym_op *asym_op = cop->asym; 2319 DH *dh_key = sess->u.dh.dh_key; 2320 BIGNUM *priv_key = NULL; 2321 int ret = 0; 2322 2323 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2324 /* compute shared secret using peer public key 2325 * and current private key 2326 * shared secret = peer_key ^ priv_key mod p 2327 */ 2328 BIGNUM *peer_key = NULL; 2329 2330 /* copy private key and peer key and compute shared secret */ 2331 peer_key = BN_bin2bn(op->pub_key.data, 2332 op->pub_key.length, 2333 peer_key); 2334 if (peer_key == NULL) { 2335 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2336 return -1; 2337 } 2338 priv_key = BN_bin2bn(op->priv_key.data, 2339 op->priv_key.length, 2340 priv_key); 2341 if (priv_key == NULL) { 2342 BN_free(peer_key); 2343 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2344 return -1; 2345 } 2346 ret = set_dh_priv_key(dh_key, priv_key); 2347 if (ret) { 2348 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2349 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2350 BN_free(peer_key); 2351 BN_free(priv_key); 2352 return 0; 2353 } 2354 2355 ret = DH_compute_key( 2356 op->shared_secret.data, 2357 peer_key, dh_key); 2358 if (ret < 0) { 2359 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2360 BN_free(peer_key); 2361 /* priv key is already loaded into dh, 2362 * let's not free that directly here. 2363 * DH_free() will auto free it later. 2364 */ 2365 return 0; 2366 } 2367 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2368 op->shared_secret.length = ret; 2369 BN_free(peer_key); 2370 return 0; 2371 } 2372 2373 /* 2374 * other options are public and private key generations. 2375 * 2376 * if user provides private key, 2377 * then first set DH with user provided private key 2378 */ 2379 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE && 2380 op->priv_key.length) { 2381 /* generate public key using user-provided private key 2382 * pub_key = g ^ priv_key mod p 2383 */ 2384 2385 /* load private key into DH */ 2386 priv_key = BN_bin2bn(op->priv_key.data, 2387 op->priv_key.length, 2388 priv_key); 2389 if (priv_key == NULL) { 2390 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2391 return -1; 2392 } 2393 ret = set_dh_priv_key(dh_key, priv_key); 2394 if (ret) { 2395 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2396 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2397 BN_free(priv_key); 2398 return 0; 2399 } 2400 } 2401 2402 /* generate public and private key pair. 2403 * 2404 * if private key already set, generates only public key. 2405 * 2406 * if private key is not already set, then set it to random value 2407 * and update internal private key. 2408 */ 2409 if (!DH_generate_key(dh_key)) { 2410 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2411 return 0; 2412 } 2413 2414 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2415 const BIGNUM *pub_key = NULL; 2416 2417 OPENSSL_LOG(DEBUG, "%s:%d update public key\n", 2418 __func__, __LINE__); 2419 2420 /* get the generated keys */ 2421 get_dh_pub_key(dh_key, &pub_key); 2422 2423 /* output public key */ 2424 op->pub_key.length = BN_bn2bin(pub_key, 2425 op->pub_key.data); 2426 } 2427 2428 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2429 const BIGNUM *priv_key = NULL; 2430 2431 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", 2432 __func__, __LINE__); 2433 2434 /* get the generated keys */ 2435 get_dh_priv_key(dh_key, &priv_key); 2436 2437 /* provide generated private key back to user */ 2438 op->priv_key.length = BN_bn2bin(priv_key, 2439 op->priv_key.data); 2440 } 2441 2442 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2443 2444 return 0; 2445 } 2446 #endif 2447 2448 /* process modinv operation */ 2449 static int 2450 process_openssl_modinv_op(struct rte_crypto_op *cop, 2451 struct openssl_asym_session *sess) 2452 { 2453 struct rte_crypto_asym_op *op = cop->asym; 2454 BIGNUM *base = BN_CTX_get(sess->u.m.ctx); 2455 BIGNUM *res = BN_CTX_get(sess->u.m.ctx); 2456 2457 if (unlikely(base == NULL || res == NULL)) { 2458 BN_free(base); 2459 BN_free(res); 2460 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2461 return -1; 2462 } 2463 2464 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 2465 op->modinv.base.length, base); 2466 2467 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) { 2468 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2469 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data); 2470 } else { 2471 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2472 } 2473 2474 BN_clear(res); 2475 BN_clear(base); 2476 2477 return 0; 2478 } 2479 2480 /* process modexp operation */ 2481 static int 2482 process_openssl_modexp_op(struct rte_crypto_op *cop, 2483 struct openssl_asym_session *sess) 2484 { 2485 struct rte_crypto_asym_op *op = cop->asym; 2486 BIGNUM *base = BN_CTX_get(sess->u.e.ctx); 2487 BIGNUM *res = BN_CTX_get(sess->u.e.ctx); 2488 2489 if (unlikely(base == NULL || res == NULL)) { 2490 BN_free(base); 2491 BN_free(res); 2492 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2493 return -1; 2494 } 2495 2496 base = BN_bin2bn((const unsigned char *)op->modex.base.data, 2497 op->modex.base.length, base); 2498 2499 if (BN_mod_exp(res, base, sess->u.e.exp, 2500 sess->u.e.mod, sess->u.e.ctx)) { 2501 op->modex.result.length = BN_bn2bin(res, op->modex.result.data); 2502 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2503 } else { 2504 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2505 } 2506 2507 BN_clear(res); 2508 BN_clear(base); 2509 2510 return 0; 2511 } 2512 2513 /* process rsa operations */ 2514 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2515 static int 2516 process_openssl_rsa_op_evp(struct rte_crypto_op *cop, 2517 struct openssl_asym_session *sess) 2518 { 2519 struct rte_crypto_asym_op *op = cop->asym; 2520 uint32_t pad = (op->rsa.padding.type); 2521 uint8_t *tmp; 2522 size_t outlen = 0; 2523 int ret = -1; 2524 2525 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2526 EVP_PKEY_CTX *rsa_ctx = sess->u.r.ctx; 2527 if (!rsa_ctx) 2528 return ret; 2529 2530 switch (pad) { 2531 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2532 pad = RSA_PKCS1_PADDING; 2533 break; 2534 case RTE_CRYPTO_RSA_PADDING_NONE: 2535 pad = RSA_NO_PADDING; 2536 break; 2537 default: 2538 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2539 OPENSSL_LOG(ERR, 2540 "rsa pad type not supported %d\n", pad); 2541 return ret; 2542 } 2543 2544 switch (op->rsa.op_type) { 2545 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2546 if (EVP_PKEY_encrypt_init(rsa_ctx) != 1) 2547 goto err_rsa; 2548 2549 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2550 goto err_rsa; 2551 2552 if (EVP_PKEY_encrypt(rsa_ctx, NULL, &outlen, 2553 op->rsa.message.data, 2554 op->rsa.message.length) <= 0) 2555 goto err_rsa; 2556 2557 if (outlen <= 0) 2558 goto err_rsa; 2559 2560 if (EVP_PKEY_encrypt(rsa_ctx, op->rsa.cipher.data, &outlen, 2561 op->rsa.message.data, 2562 op->rsa.message.length) <= 0) 2563 goto err_rsa; 2564 op->rsa.cipher.length = outlen; 2565 2566 OPENSSL_LOG(DEBUG, 2567 "length of encrypted text %zu\n", outlen); 2568 break; 2569 2570 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2571 if (EVP_PKEY_decrypt_init(rsa_ctx) != 1) 2572 goto err_rsa; 2573 2574 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2575 goto err_rsa; 2576 2577 if (EVP_PKEY_decrypt(rsa_ctx, NULL, &outlen, 2578 op->rsa.cipher.data, 2579 op->rsa.cipher.length) <= 0) 2580 goto err_rsa; 2581 2582 if (outlen <= 0) 2583 goto err_rsa; 2584 2585 if (EVP_PKEY_decrypt(rsa_ctx, op->rsa.message.data, &outlen, 2586 op->rsa.cipher.data, 2587 op->rsa.cipher.length) <= 0) 2588 goto err_rsa; 2589 op->rsa.message.length = outlen; 2590 2591 OPENSSL_LOG(DEBUG, "length of decrypted text %zu\n", outlen); 2592 break; 2593 2594 case RTE_CRYPTO_ASYM_OP_SIGN: 2595 if (EVP_PKEY_sign_init(rsa_ctx) <= 0) 2596 goto err_rsa; 2597 2598 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2599 goto err_rsa; 2600 2601 if (EVP_PKEY_sign(rsa_ctx, NULL, &outlen, 2602 op->rsa.message.data, 2603 op->rsa.message.length) <= 0) 2604 goto err_rsa; 2605 2606 if (outlen <= 0) 2607 goto err_rsa; 2608 2609 if (EVP_PKEY_sign(rsa_ctx, op->rsa.sign.data, &outlen, 2610 op->rsa.message.data, 2611 op->rsa.message.length) <= 0) 2612 goto err_rsa; 2613 op->rsa.sign.length = outlen; 2614 break; 2615 2616 case RTE_CRYPTO_ASYM_OP_VERIFY: 2617 if (EVP_PKEY_verify_recover_init(rsa_ctx) <= 0) 2618 goto err_rsa; 2619 2620 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2621 goto err_rsa; 2622 2623 if (EVP_PKEY_verify_recover(rsa_ctx, NULL, &outlen, 2624 op->rsa.sign.data, 2625 op->rsa.sign.length) <= 0) 2626 goto err_rsa; 2627 2628 if ((outlen <= 0) || (outlen != op->rsa.sign.length)) 2629 goto err_rsa; 2630 2631 tmp = OPENSSL_malloc(outlen); 2632 if (tmp == NULL) { 2633 OPENSSL_LOG(ERR, "Memory allocation failed"); 2634 goto err_rsa; 2635 } 2636 2637 if (EVP_PKEY_verify_recover(rsa_ctx, tmp, &outlen, 2638 op->rsa.sign.data, 2639 op->rsa.sign.length) <= 0) { 2640 OPENSSL_free(tmp); 2641 goto err_rsa; 2642 } 2643 2644 OPENSSL_LOG(DEBUG, 2645 "Length of public_decrypt %zu " 2646 "length of message %zd\n", 2647 outlen, op->rsa.message.length); 2648 if (CRYPTO_memcmp(tmp, op->rsa.message.data, 2649 op->rsa.message.length)) { 2650 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 2651 } 2652 OPENSSL_free(tmp); 2653 break; 2654 2655 default: 2656 /* allow ops with invalid args to be pushed to 2657 * completion queue 2658 */ 2659 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2660 goto err_rsa; 2661 } 2662 2663 ret = 0; 2664 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2665 err_rsa: 2666 return ret; 2667 2668 } 2669 2670 static int 2671 process_openssl_sm2_op_evp(struct rte_crypto_op *cop, 2672 struct openssl_asym_session *sess) 2673 { 2674 EVP_PKEY_CTX *kctx = NULL, *sctx = NULL, *cctx = NULL; 2675 struct rte_crypto_asym_op *op = cop->asym; 2676 OSSL_PARAM *params = sess->u.sm2.params; 2677 EVP_PKEY *pkey = NULL; 2678 int ret = -1; 2679 2680 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2681 2682 if (cop->asym->sm2.k.data != NULL) 2683 goto err_sm2; 2684 2685 switch (op->sm2.op_type) { 2686 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2687 { 2688 OSSL_PARAM *eparams = sess->u.sm2.params; 2689 size_t output_len = 0; 2690 2691 kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_SM2, NULL); 2692 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2693 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2694 goto err_sm2; 2695 2696 cctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2697 if (!cctx) 2698 goto err_sm2; 2699 2700 if (!EVP_PKEY_encrypt_init(cctx)) 2701 goto err_sm2; 2702 2703 if (!EVP_PKEY_CTX_set_params(cctx, eparams)) 2704 goto err_sm2; 2705 2706 if (!EVP_PKEY_encrypt(cctx, op->sm2.cipher.data, &output_len, 2707 op->sm2.message.data, 2708 op->sm2.message.length)) 2709 goto err_sm2; 2710 op->sm2.cipher.length = output_len; 2711 } 2712 break; 2713 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2714 { 2715 OSSL_PARAM *eparams = sess->u.sm2.params; 2716 2717 kctx = EVP_PKEY_CTX_new_id(EVP_PKEY_SM2, NULL); 2718 if (kctx == NULL 2719 || EVP_PKEY_fromdata_init(kctx) <= 0 2720 || EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2721 goto err_sm2; 2722 2723 cctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2724 if (!cctx) 2725 goto err_sm2; 2726 2727 if (!EVP_PKEY_decrypt_init(cctx)) 2728 goto err_sm2; 2729 2730 if (!EVP_PKEY_CTX_set_params(cctx, eparams)) 2731 goto err_sm2; 2732 2733 if (!EVP_PKEY_decrypt(cctx, op->sm2.message.data, &op->sm2.message.length, 2734 op->sm2.cipher.data, op->sm2.cipher.length)) 2735 goto err_sm2; 2736 } 2737 break; 2738 case RTE_CRYPTO_ASYM_OP_SIGN: 2739 { 2740 unsigned char signbuf[128] = {0}; 2741 const unsigned char *signptr; 2742 EVP_MD_CTX *md_ctx = NULL; 2743 const BIGNUM *r, *s; 2744 ECDSA_SIG *ec_sign; 2745 EVP_MD *check_md; 2746 size_t signlen; 2747 2748 kctx = EVP_PKEY_CTX_new_from_name(NULL, "SM2", NULL); 2749 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2750 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 2751 goto err_sm2; 2752 2753 md_ctx = EVP_MD_CTX_new(); 2754 if (!md_ctx) 2755 goto err_sm2; 2756 2757 sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2758 if (!sctx) 2759 goto err_sm2; 2760 2761 EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx); 2762 2763 check_md = EVP_MD_fetch(NULL, "sm3", NULL); 2764 if (!check_md) 2765 goto err_sm2; 2766 2767 if (!EVP_DigestSignInit(md_ctx, NULL, check_md, NULL, pkey)) 2768 goto err_sm2; 2769 2770 if (EVP_PKEY_CTX_set1_id(sctx, op->sm2.id.data, op->sm2.id.length) <= 0) 2771 goto err_sm2; 2772 2773 if (!EVP_DigestSignUpdate(md_ctx, op->sm2.message.data, 2774 op->sm2.message.length)) 2775 goto err_sm2; 2776 2777 if (!EVP_DigestSignFinal(md_ctx, NULL, &signlen)) 2778 goto err_sm2; 2779 2780 if (!EVP_DigestSignFinal(md_ctx, signbuf, &signlen)) 2781 goto err_sm2; 2782 2783 signptr = signbuf; 2784 ec_sign = d2i_ECDSA_SIG(NULL, &signptr, signlen); 2785 if (!ec_sign) 2786 goto err_sm2; 2787 2788 r = ECDSA_SIG_get0_r(ec_sign); 2789 s = ECDSA_SIG_get0_s(ec_sign); 2790 if (!r || !s) 2791 goto err_sm2; 2792 2793 op->sm2.r.length = BN_num_bytes(r); 2794 op->sm2.s.length = BN_num_bytes(s); 2795 BN_bn2bin(r, op->sm2.r.data); 2796 BN_bn2bin(s, op->sm2.s.data); 2797 2798 ECDSA_SIG_free(ec_sign); 2799 } 2800 break; 2801 case RTE_CRYPTO_ASYM_OP_VERIFY: 2802 { 2803 unsigned char signbuf[128] = {0}; 2804 BIGNUM *r = NULL, *s = NULL; 2805 EVP_MD_CTX *md_ctx = NULL; 2806 ECDSA_SIG *ec_sign; 2807 EVP_MD *check_md; 2808 size_t signlen; 2809 2810 kctx = EVP_PKEY_CTX_new_from_name(NULL, "SM2", NULL); 2811 if (kctx == NULL || EVP_PKEY_fromdata_init(kctx) <= 0 || 2812 EVP_PKEY_fromdata(kctx, &pkey, EVP_PKEY_PUBLIC_KEY, params) <= 0) 2813 goto err_sm2; 2814 2815 if (!EVP_PKEY_is_a(pkey, "SM2")) 2816 goto err_sm2; 2817 2818 md_ctx = EVP_MD_CTX_new(); 2819 if (!md_ctx) 2820 goto err_sm2; 2821 2822 sctx = EVP_PKEY_CTX_new_from_pkey(NULL, pkey, NULL); 2823 if (!sctx) 2824 goto err_sm2; 2825 2826 EVP_MD_CTX_set_pkey_ctx(md_ctx, sctx); 2827 2828 check_md = EVP_MD_fetch(NULL, "sm3", NULL); 2829 if (!check_md) 2830 goto err_sm2; 2831 2832 if (!EVP_DigestVerifyInit(md_ctx, NULL, check_md, NULL, pkey)) 2833 goto err_sm2; 2834 2835 if (EVP_PKEY_CTX_set1_id(sctx, op->sm2.id.data, op->sm2.id.length) <= 0) 2836 goto err_sm2; 2837 2838 if (!EVP_DigestVerifyUpdate(md_ctx, op->sm2.message.data, 2839 op->sm2.message.length)) 2840 goto err_sm2; 2841 2842 ec_sign = ECDSA_SIG_new(); 2843 if (!ec_sign) 2844 goto err_sm2; 2845 2846 r = BN_bin2bn(op->sm2.r.data, op->sm2.r.length, r); 2847 s = BN_bin2bn(op->sm2.s.data, op->sm2.s.length, s); 2848 if (!r || !s) 2849 goto err_sm2; 2850 2851 if (!ECDSA_SIG_set0(ec_sign, r, s)) { 2852 BN_free(r); 2853 BN_free(s); 2854 goto err_sm2; 2855 } 2856 2857 r = NULL; 2858 s = NULL; 2859 2860 signlen = i2d_ECDSA_SIG(ec_sign, (unsigned char **)&signbuf); 2861 if (signlen <= 0) 2862 goto err_sm2; 2863 2864 if (!EVP_DigestVerifyFinal(md_ctx, signbuf, signlen)) 2865 goto err_sm2; 2866 2867 BN_free(r); 2868 BN_free(s); 2869 ECDSA_SIG_free(ec_sign); 2870 } 2871 break; 2872 default: 2873 /* allow ops with invalid args to be pushed to 2874 * completion queue 2875 */ 2876 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2877 goto err_sm2; 2878 } 2879 2880 ret = 0; 2881 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2882 err_sm2: 2883 if (kctx) 2884 EVP_PKEY_CTX_free(kctx); 2885 2886 if (sctx) 2887 EVP_PKEY_CTX_free(sctx); 2888 2889 if (cctx) 2890 EVP_PKEY_CTX_free(cctx); 2891 2892 if (pkey) 2893 EVP_PKEY_free(pkey); 2894 2895 return ret; 2896 } 2897 2898 #else 2899 static int 2900 process_openssl_rsa_op(struct rte_crypto_op *cop, 2901 struct openssl_asym_session *sess) 2902 { 2903 int ret = 0; 2904 struct rte_crypto_asym_op *op = cop->asym; 2905 RSA *rsa = sess->u.r.rsa; 2906 uint32_t pad = (op->rsa.padding.type); 2907 uint8_t *tmp; 2908 2909 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2910 2911 switch (pad) { 2912 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2913 pad = RSA_PKCS1_PADDING; 2914 break; 2915 case RTE_CRYPTO_RSA_PADDING_NONE: 2916 pad = RSA_NO_PADDING; 2917 break; 2918 default: 2919 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2920 OPENSSL_LOG(ERR, 2921 "rsa pad type not supported %d\n", pad); 2922 return 0; 2923 } 2924 2925 switch (op->rsa.op_type) { 2926 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2927 ret = RSA_public_encrypt(op->rsa.message.length, 2928 op->rsa.message.data, 2929 op->rsa.cipher.data, 2930 rsa, 2931 pad); 2932 2933 if (ret > 0) 2934 op->rsa.cipher.length = ret; 2935 OPENSSL_LOG(DEBUG, 2936 "length of encrypted text %d\n", ret); 2937 break; 2938 2939 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2940 ret = RSA_private_decrypt(op->rsa.cipher.length, 2941 op->rsa.cipher.data, 2942 op->rsa.message.data, 2943 rsa, 2944 pad); 2945 if (ret > 0) 2946 op->rsa.message.length = ret; 2947 break; 2948 2949 case RTE_CRYPTO_ASYM_OP_SIGN: 2950 ret = RSA_private_encrypt(op->rsa.message.length, 2951 op->rsa.message.data, 2952 op->rsa.sign.data, 2953 rsa, 2954 pad); 2955 if (ret > 0) 2956 op->rsa.sign.length = ret; 2957 break; 2958 2959 case RTE_CRYPTO_ASYM_OP_VERIFY: 2960 tmp = rte_malloc(NULL, op->rsa.sign.length, 0); 2961 if (tmp == NULL) { 2962 OPENSSL_LOG(ERR, "Memory allocation failed"); 2963 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2964 break; 2965 } 2966 ret = RSA_public_decrypt(op->rsa.sign.length, 2967 op->rsa.sign.data, 2968 tmp, 2969 rsa, 2970 pad); 2971 2972 OPENSSL_LOG(DEBUG, 2973 "Length of public_decrypt %d " 2974 "length of message %zd\n", 2975 ret, op->rsa.message.length); 2976 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data, 2977 op->rsa.message.length))) { 2978 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 2979 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2980 } 2981 rte_free(tmp); 2982 break; 2983 2984 default: 2985 /* allow ops with invalid args to be pushed to 2986 * completion queue 2987 */ 2988 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2989 break; 2990 } 2991 2992 if (ret < 0) 2993 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2994 2995 return 0; 2996 } 2997 2998 static int 2999 process_openssl_sm2_op(struct rte_crypto_op *cop, 3000 struct openssl_asym_session *sess) 3001 { 3002 RTE_SET_USED(cop); 3003 RTE_SET_USED(sess); 3004 return -ENOTSUP; 3005 } 3006 #endif 3007 3008 static int 3009 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op, 3010 struct openssl_asym_session *sess) 3011 { 3012 int retval = 0; 3013 3014 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 3015 3016 switch (sess->xfrm_type) { 3017 case RTE_CRYPTO_ASYM_XFORM_RSA: 3018 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3019 retval = process_openssl_rsa_op_evp(op, sess); 3020 # else 3021 retval = process_openssl_rsa_op(op, sess); 3022 #endif 3023 break; 3024 case RTE_CRYPTO_ASYM_XFORM_MODEX: 3025 retval = process_openssl_modexp_op(op, sess); 3026 break; 3027 case RTE_CRYPTO_ASYM_XFORM_MODINV: 3028 retval = process_openssl_modinv_op(op, sess); 3029 break; 3030 case RTE_CRYPTO_ASYM_XFORM_DH: 3031 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3032 retval = process_openssl_dh_op_evp(op, sess); 3033 # else 3034 retval = process_openssl_dh_op(op, sess); 3035 #endif 3036 break; 3037 case RTE_CRYPTO_ASYM_XFORM_DSA: 3038 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3039 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 3040 retval = process_openssl_dsa_sign_op_evp(op, sess); 3041 else if (op->asym->dsa.op_type == 3042 RTE_CRYPTO_ASYM_OP_VERIFY) 3043 retval = 3044 process_openssl_dsa_verify_op_evp(op, sess); 3045 #else 3046 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 3047 retval = process_openssl_dsa_sign_op(op, sess); 3048 else if (op->asym->dsa.op_type == 3049 RTE_CRYPTO_ASYM_OP_VERIFY) 3050 retval = 3051 process_openssl_dsa_verify_op(op, sess); 3052 else 3053 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3054 #endif 3055 break; 3056 case RTE_CRYPTO_ASYM_XFORM_SM2: 3057 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3058 retval = process_openssl_sm2_op_evp(op, sess); 3059 #else 3060 retval = process_openssl_sm2_op(op, sess); 3061 #endif 3062 break; 3063 default: 3064 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 3065 break; 3066 } 3067 if (!retval) { 3068 /* op processed so push to completion queue as processed */ 3069 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 3070 if (retval) 3071 /* return error if failed to put in completion queue */ 3072 retval = -1; 3073 } 3074 3075 return retval; 3076 } 3077 3078 static void 3079 copy_plaintext(struct rte_mbuf *m_src, struct rte_mbuf *m_dst, 3080 struct rte_crypto_op *op) 3081 { 3082 uint8_t *p_src, *p_dst; 3083 3084 p_src = rte_pktmbuf_mtod(m_src, uint8_t *); 3085 p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *); 3086 3087 /** 3088 * Copy the content between cipher offset and auth offset 3089 * for generating correct digest. 3090 */ 3091 if (op->sym->cipher.data.offset > op->sym->auth.data.offset) 3092 memcpy(p_dst + op->sym->auth.data.offset, 3093 p_src + op->sym->auth.data.offset, 3094 op->sym->cipher.data.offset - 3095 op->sym->auth.data.offset); 3096 } 3097 3098 /** Process crypto operation for mbuf */ 3099 static int 3100 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 3101 struct openssl_session *sess) 3102 { 3103 struct rte_mbuf *msrc, *mdst; 3104 int retval; 3105 3106 msrc = op->sym->m_src; 3107 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 3108 3109 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 3110 3111 switch (sess->chain_order) { 3112 case OPENSSL_CHAIN_ONLY_CIPHER: 3113 process_openssl_cipher_op(op, sess, msrc, mdst); 3114 break; 3115 case OPENSSL_CHAIN_ONLY_AUTH: 3116 process_openssl_auth_op(qp, op, sess, msrc, mdst); 3117 break; 3118 case OPENSSL_CHAIN_CIPHER_AUTH: 3119 process_openssl_cipher_op(op, sess, msrc, mdst); 3120 /* OOP */ 3121 if (msrc != mdst) 3122 copy_plaintext(msrc, mdst, op); 3123 process_openssl_auth_op(qp, op, sess, mdst, mdst); 3124 break; 3125 case OPENSSL_CHAIN_AUTH_CIPHER: 3126 process_openssl_auth_op(qp, op, sess, msrc, mdst); 3127 process_openssl_cipher_op(op, sess, msrc, mdst); 3128 break; 3129 case OPENSSL_CHAIN_COMBINED: 3130 process_openssl_combined_op(op, sess, msrc, mdst); 3131 break; 3132 case OPENSSL_CHAIN_CIPHER_BPI: 3133 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 3134 break; 3135 default: 3136 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3137 break; 3138 } 3139 3140 /* Free session if a session-less crypto op */ 3141 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 3142 openssl_reset_session(sess); 3143 memset(sess, 0, sizeof(struct openssl_session)); 3144 rte_mempool_put(qp->sess_mp, op->sym->session); 3145 op->sym->session = NULL; 3146 } 3147 3148 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 3149 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3150 3151 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 3152 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 3153 else 3154 retval = -1; 3155 3156 return retval; 3157 } 3158 3159 /* 3160 *------------------------------------------------------------------------------ 3161 * PMD Framework 3162 *------------------------------------------------------------------------------ 3163 */ 3164 3165 /** Enqueue burst */ 3166 static uint16_t 3167 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 3168 uint16_t nb_ops) 3169 { 3170 void *sess; 3171 struct openssl_qp *qp = queue_pair; 3172 int i, retval; 3173 3174 for (i = 0; i < nb_ops; i++) { 3175 sess = get_session(qp, ops[i]); 3176 if (unlikely(sess == NULL)) 3177 goto enqueue_err; 3178 3179 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) 3180 retval = process_op(qp, ops[i], 3181 (struct openssl_session *) sess); 3182 else 3183 retval = process_asym_op(qp, ops[i], 3184 (struct openssl_asym_session *) sess); 3185 if (unlikely(retval < 0)) 3186 goto enqueue_err; 3187 } 3188 3189 qp->stats.enqueued_count += i; 3190 return i; 3191 3192 enqueue_err: 3193 qp->stats.enqueue_err_count++; 3194 return i; 3195 } 3196 3197 /** Dequeue burst */ 3198 static uint16_t 3199 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 3200 uint16_t nb_ops) 3201 { 3202 struct openssl_qp *qp = queue_pair; 3203 3204 unsigned int nb_dequeued = 0; 3205 3206 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 3207 (void **)ops, nb_ops, NULL); 3208 qp->stats.dequeued_count += nb_dequeued; 3209 3210 return nb_dequeued; 3211 } 3212 3213 /** Create OPENSSL crypto device */ 3214 static int 3215 cryptodev_openssl_create(const char *name, 3216 struct rte_vdev_device *vdev, 3217 struct rte_cryptodev_pmd_init_params *init_params) 3218 { 3219 struct rte_cryptodev *dev; 3220 struct openssl_private *internals; 3221 3222 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 3223 if (dev == NULL) { 3224 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 3225 goto init_error; 3226 } 3227 3228 dev->driver_id = cryptodev_driver_id; 3229 dev->dev_ops = rte_openssl_pmd_ops; 3230 3231 /* register rx/tx burst functions for data path */ 3232 dev->dequeue_burst = openssl_pmd_dequeue_burst; 3233 dev->enqueue_burst = openssl_pmd_enqueue_burst; 3234 3235 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 3236 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 3237 RTE_CRYPTODEV_FF_CPU_AESNI | 3238 RTE_CRYPTODEV_FF_IN_PLACE_SGL | 3239 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 3240 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | 3241 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO | 3242 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP | 3243 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT | 3244 RTE_CRYPTODEV_FF_SYM_SESSIONLESS; 3245 3246 internals = dev->data->dev_private; 3247 3248 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 3249 3250 rte_cryptodev_pmd_probing_finish(dev); 3251 3252 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3253 /* Load legacy provider 3254 * Some algorithms are no longer available in earlier version of openssl, 3255 * unless the legacy provider explicitly loaded. e.g. DES 3256 */ 3257 ossl_legacy_provider_load(); 3258 # endif 3259 return 0; 3260 3261 init_error: 3262 OPENSSL_LOG(ERR, "driver %s: create failed", 3263 init_params->name); 3264 3265 cryptodev_openssl_remove(vdev); 3266 return -EFAULT; 3267 } 3268 3269 /** Initialise OPENSSL crypto device */ 3270 static int 3271 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 3272 { 3273 struct rte_cryptodev_pmd_init_params init_params = { 3274 "", 3275 sizeof(struct openssl_private), 3276 rte_socket_id(), 3277 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 3278 }; 3279 const char *name; 3280 const char *input_args; 3281 3282 name = rte_vdev_device_name(vdev); 3283 if (name == NULL) 3284 return -EINVAL; 3285 input_args = rte_vdev_device_args(vdev); 3286 3287 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 3288 3289 return cryptodev_openssl_create(name, vdev, &init_params); 3290 } 3291 3292 /** Uninitialise OPENSSL crypto device */ 3293 static int 3294 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 3295 { 3296 struct rte_cryptodev *cryptodev; 3297 const char *name; 3298 3299 name = rte_vdev_device_name(vdev); 3300 if (name == NULL) 3301 return -EINVAL; 3302 3303 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 3304 if (cryptodev == NULL) 3305 return -ENODEV; 3306 3307 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 3308 ossl_legacy_provider_unload(); 3309 # endif 3310 return rte_cryptodev_pmd_destroy(cryptodev); 3311 } 3312 3313 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 3314 .probe = cryptodev_openssl_probe, 3315 .remove = cryptodev_openssl_remove 3316 }; 3317 3318 static struct cryptodev_driver openssl_crypto_drv; 3319 3320 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 3321 cryptodev_openssl_pmd_drv); 3322 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 3323 "max_nb_queue_pairs=<int> " 3324 "socket_id=<int>"); 3325 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 3326 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 3327 RTE_LOG_REGISTER_DEFAULT(openssl_logtype_driver, INFO); 3328