1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_common.h> 6 #include <rte_hexdump.h> 7 #include <rte_cryptodev.h> 8 #include <cryptodev_pmd.h> 9 #include <rte_bus_vdev.h> 10 #include <rte_malloc.h> 11 #include <rte_cpuflags.h> 12 13 #include <openssl/hmac.h> 14 #include <openssl/evp.h> 15 16 #include "openssl_pmd_private.h" 17 #include "compat.h" 18 19 #define DES_BLOCK_SIZE 8 20 21 static uint8_t cryptodev_driver_id; 22 23 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 24 static HMAC_CTX *HMAC_CTX_new(void) 25 { 26 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 27 28 if (ctx != NULL) 29 HMAC_CTX_init(ctx); 30 return ctx; 31 } 32 33 static void HMAC_CTX_free(HMAC_CTX *ctx) 34 { 35 if (ctx != NULL) { 36 HMAC_CTX_cleanup(ctx); 37 OPENSSL_free(ctx); 38 } 39 } 40 #endif 41 42 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 43 44 #include <openssl/provider.h> 45 #include <openssl/core_names.h> 46 #include <openssl/param_build.h> 47 48 #define MAX_OSSL_ALGO_NAME_SIZE 16 49 50 OSSL_PROVIDER *legacy; 51 OSSL_PROVIDER *deflt; 52 53 static void ossl_legacy_provider_load(void) 54 { 55 /* Load Multiple providers into the default (NULL) library context */ 56 legacy = OSSL_PROVIDER_load(NULL, "legacy"); 57 if (legacy == NULL) { 58 OPENSSL_LOG(ERR, "Failed to load Legacy provider\n"); 59 return; 60 } 61 62 deflt = OSSL_PROVIDER_load(NULL, "default"); 63 if (deflt == NULL) { 64 OPENSSL_LOG(ERR, "Failed to load Default provider\n"); 65 OSSL_PROVIDER_unload(legacy); 66 return; 67 } 68 } 69 70 static void ossl_legacy_provider_unload(void) 71 { 72 OSSL_PROVIDER_unload(legacy); 73 OSSL_PROVIDER_unload(deflt); 74 } 75 76 static __rte_always_inline const char * 77 digest_name_get(enum rte_crypto_auth_algorithm algo) 78 { 79 switch (algo) { 80 case RTE_CRYPTO_AUTH_MD5_HMAC: 81 return OSSL_DIGEST_NAME_MD5; 82 case RTE_CRYPTO_AUTH_SHA1_HMAC: 83 return OSSL_DIGEST_NAME_SHA1; 84 case RTE_CRYPTO_AUTH_SHA224_HMAC: 85 return OSSL_DIGEST_NAME_SHA2_224; 86 case RTE_CRYPTO_AUTH_SHA256_HMAC: 87 return OSSL_DIGEST_NAME_SHA2_256; 88 case RTE_CRYPTO_AUTH_SHA384_HMAC: 89 return OSSL_DIGEST_NAME_SHA2_384; 90 case RTE_CRYPTO_AUTH_SHA512_HMAC: 91 return OSSL_DIGEST_NAME_SHA2_512; 92 default: 93 return NULL; 94 } 95 } 96 #endif 97 98 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 99 100 /*----------------------------------------------------------------------------*/ 101 102 /** 103 * Increment counter by 1 104 * Counter is 64 bit array, big-endian 105 */ 106 static void 107 ctr_inc(uint8_t *ctr) 108 { 109 uint64_t *ctr64 = (uint64_t *)ctr; 110 111 *ctr64 = __builtin_bswap64(*ctr64); 112 (*ctr64)++; 113 *ctr64 = __builtin_bswap64(*ctr64); 114 } 115 116 /* 117 *------------------------------------------------------------------------------ 118 * Session Prepare 119 *------------------------------------------------------------------------------ 120 */ 121 122 /** Get xform chain order */ 123 static enum openssl_chain_order 124 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 125 { 126 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 127 128 if (xform != NULL) { 129 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 130 if (xform->next == NULL) 131 res = OPENSSL_CHAIN_ONLY_AUTH; 132 else if (xform->next->type == 133 RTE_CRYPTO_SYM_XFORM_CIPHER) 134 res = OPENSSL_CHAIN_AUTH_CIPHER; 135 } 136 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 137 if (xform->next == NULL) 138 res = OPENSSL_CHAIN_ONLY_CIPHER; 139 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 140 res = OPENSSL_CHAIN_CIPHER_AUTH; 141 } 142 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 143 res = OPENSSL_CHAIN_COMBINED; 144 } 145 146 return res; 147 } 148 149 /** Get session cipher key from input cipher key */ 150 static void 151 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key) 152 { 153 memcpy(session_key, input_key, keylen); 154 } 155 156 /** Get key ede 24 bytes standard from input key */ 157 static int 158 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede) 159 { 160 int res = 0; 161 162 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 163 switch (keylen) { 164 case 24: 165 memcpy(key_ede, key, 24); 166 break; 167 case 16: 168 /* K3 = K1 */ 169 memcpy(key_ede, key, 16); 170 memcpy(key_ede + 16, key, 8); 171 break; 172 case 8: 173 /* K1 = K2 = K3 (DES compatibility) */ 174 memcpy(key_ede, key, 8); 175 memcpy(key_ede + 8, key, 8); 176 memcpy(key_ede + 16, key, 8); 177 break; 178 default: 179 OPENSSL_LOG(ERR, "Unsupported key size"); 180 res = -EINVAL; 181 } 182 183 return res; 184 } 185 186 /** Get adequate openssl function for input cipher algorithm */ 187 static uint8_t 188 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 189 const EVP_CIPHER **algo) 190 { 191 int res = 0; 192 193 if (algo != NULL) { 194 switch (sess_algo) { 195 case RTE_CRYPTO_CIPHER_3DES_CBC: 196 switch (keylen) { 197 case 8: 198 *algo = EVP_des_cbc(); 199 break; 200 case 16: 201 *algo = EVP_des_ede_cbc(); 202 break; 203 case 24: 204 *algo = EVP_des_ede3_cbc(); 205 break; 206 default: 207 res = -EINVAL; 208 } 209 break; 210 case RTE_CRYPTO_CIPHER_3DES_CTR: 211 break; 212 case RTE_CRYPTO_CIPHER_AES_CBC: 213 switch (keylen) { 214 case 16: 215 *algo = EVP_aes_128_cbc(); 216 break; 217 case 24: 218 *algo = EVP_aes_192_cbc(); 219 break; 220 case 32: 221 *algo = EVP_aes_256_cbc(); 222 break; 223 default: 224 res = -EINVAL; 225 } 226 break; 227 case RTE_CRYPTO_CIPHER_AES_CTR: 228 switch (keylen) { 229 case 16: 230 *algo = EVP_aes_128_ctr(); 231 break; 232 case 24: 233 *algo = EVP_aes_192_ctr(); 234 break; 235 case 32: 236 *algo = EVP_aes_256_ctr(); 237 break; 238 default: 239 res = -EINVAL; 240 } 241 break; 242 default: 243 res = -EINVAL; 244 break; 245 } 246 } else { 247 res = -EINVAL; 248 } 249 250 return res; 251 } 252 253 /** Get adequate openssl function for input auth algorithm */ 254 static uint8_t 255 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 256 const EVP_MD **algo) 257 { 258 int res = 0; 259 260 if (algo != NULL) { 261 switch (sessalgo) { 262 case RTE_CRYPTO_AUTH_MD5: 263 case RTE_CRYPTO_AUTH_MD5_HMAC: 264 *algo = EVP_md5(); 265 break; 266 case RTE_CRYPTO_AUTH_SHA1: 267 case RTE_CRYPTO_AUTH_SHA1_HMAC: 268 *algo = EVP_sha1(); 269 break; 270 case RTE_CRYPTO_AUTH_SHA224: 271 case RTE_CRYPTO_AUTH_SHA224_HMAC: 272 *algo = EVP_sha224(); 273 break; 274 case RTE_CRYPTO_AUTH_SHA256: 275 case RTE_CRYPTO_AUTH_SHA256_HMAC: 276 *algo = EVP_sha256(); 277 break; 278 case RTE_CRYPTO_AUTH_SHA384: 279 case RTE_CRYPTO_AUTH_SHA384_HMAC: 280 *algo = EVP_sha384(); 281 break; 282 case RTE_CRYPTO_AUTH_SHA512: 283 case RTE_CRYPTO_AUTH_SHA512_HMAC: 284 *algo = EVP_sha512(); 285 break; 286 default: 287 res = -EINVAL; 288 break; 289 } 290 } else { 291 res = -EINVAL; 292 } 293 294 return res; 295 } 296 297 /** Get adequate openssl function for input cipher algorithm */ 298 static uint8_t 299 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 300 const EVP_CIPHER **algo) 301 { 302 int res = 0; 303 304 if (algo != NULL) { 305 switch (sess_algo) { 306 case RTE_CRYPTO_AEAD_AES_GCM: 307 switch (keylen) { 308 case 16: 309 *algo = EVP_aes_128_gcm(); 310 break; 311 case 24: 312 *algo = EVP_aes_192_gcm(); 313 break; 314 case 32: 315 *algo = EVP_aes_256_gcm(); 316 break; 317 default: 318 res = -EINVAL; 319 } 320 break; 321 case RTE_CRYPTO_AEAD_AES_CCM: 322 switch (keylen) { 323 case 16: 324 *algo = EVP_aes_128_ccm(); 325 break; 326 case 24: 327 *algo = EVP_aes_192_ccm(); 328 break; 329 case 32: 330 *algo = EVP_aes_256_ccm(); 331 break; 332 default: 333 res = -EINVAL; 334 } 335 break; 336 default: 337 res = -EINVAL; 338 break; 339 } 340 } else { 341 res = -EINVAL; 342 } 343 344 return res; 345 } 346 347 /* Set session AEAD encryption parameters */ 348 static int 349 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 350 enum rte_crypto_aead_algorithm algo, 351 uint8_t tag_len, const uint8_t *key) 352 { 353 int iv_type = 0; 354 unsigned int do_ccm; 355 356 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 357 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 358 359 /* Select AEAD algo */ 360 switch (algo) { 361 case RTE_CRYPTO_AEAD_AES_GCM: 362 iv_type = EVP_CTRL_GCM_SET_IVLEN; 363 if (tag_len != 16) 364 return -EINVAL; 365 do_ccm = 0; 366 break; 367 case RTE_CRYPTO_AEAD_AES_CCM: 368 iv_type = EVP_CTRL_CCM_SET_IVLEN; 369 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 370 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 371 return -EINVAL; 372 do_ccm = 1; 373 break; 374 default: 375 return -ENOTSUP; 376 } 377 378 sess->cipher.mode = OPENSSL_CIPHER_LIB; 379 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 380 381 if (get_aead_algo(algo, sess->cipher.key.length, 382 &sess->cipher.evp_algo) != 0) 383 return -EINVAL; 384 385 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 386 387 sess->chain_order = OPENSSL_CHAIN_COMBINED; 388 389 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 390 NULL, NULL, NULL) <= 0) 391 return -EINVAL; 392 393 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, 394 NULL) <= 0) 395 return -EINVAL; 396 397 if (do_ccm) 398 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 399 tag_len, NULL); 400 401 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 402 return -EINVAL; 403 404 return 0; 405 } 406 407 /* Set session AEAD decryption parameters */ 408 static int 409 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 410 enum rte_crypto_aead_algorithm algo, 411 uint8_t tag_len, const uint8_t *key) 412 { 413 int iv_type = 0; 414 unsigned int do_ccm = 0; 415 416 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 417 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 418 419 /* Select AEAD algo */ 420 switch (algo) { 421 case RTE_CRYPTO_AEAD_AES_GCM: 422 iv_type = EVP_CTRL_GCM_SET_IVLEN; 423 if (tag_len != 16) 424 return -EINVAL; 425 break; 426 case RTE_CRYPTO_AEAD_AES_CCM: 427 iv_type = EVP_CTRL_CCM_SET_IVLEN; 428 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 429 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 430 return -EINVAL; 431 do_ccm = 1; 432 break; 433 default: 434 return -ENOTSUP; 435 } 436 437 sess->cipher.mode = OPENSSL_CIPHER_LIB; 438 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 439 440 if (get_aead_algo(algo, sess->cipher.key.length, 441 &sess->cipher.evp_algo) != 0) 442 return -EINVAL; 443 444 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 445 446 sess->chain_order = OPENSSL_CHAIN_COMBINED; 447 448 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 449 NULL, NULL, NULL) <= 0) 450 return -EINVAL; 451 452 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, 453 sess->iv.length, NULL) <= 0) 454 return -EINVAL; 455 456 if (do_ccm) 457 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 458 tag_len, NULL); 459 460 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 461 return -EINVAL; 462 463 return 0; 464 } 465 466 /** Set session cipher parameters */ 467 static int 468 openssl_set_session_cipher_parameters(struct openssl_session *sess, 469 const struct rte_crypto_sym_xform *xform) 470 { 471 /* Select cipher direction */ 472 sess->cipher.direction = xform->cipher.op; 473 /* Select cipher key */ 474 sess->cipher.key.length = xform->cipher.key.length; 475 476 /* Set IV parameters */ 477 sess->iv.offset = xform->cipher.iv.offset; 478 sess->iv.length = xform->cipher.iv.length; 479 480 /* Select cipher algo */ 481 switch (xform->cipher.algo) { 482 case RTE_CRYPTO_CIPHER_3DES_CBC: 483 case RTE_CRYPTO_CIPHER_AES_CBC: 484 case RTE_CRYPTO_CIPHER_AES_CTR: 485 sess->cipher.mode = OPENSSL_CIPHER_LIB; 486 sess->cipher.algo = xform->cipher.algo; 487 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 488 489 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 490 &sess->cipher.evp_algo) != 0) 491 return -EINVAL; 492 493 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 494 sess->cipher.key.data); 495 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 496 if (EVP_EncryptInit_ex(sess->cipher.ctx, 497 sess->cipher.evp_algo, 498 NULL, xform->cipher.key.data, 499 NULL) != 1) { 500 return -EINVAL; 501 } 502 } else if (sess->cipher.direction == 503 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 504 if (EVP_DecryptInit_ex(sess->cipher.ctx, 505 sess->cipher.evp_algo, 506 NULL, xform->cipher.key.data, 507 NULL) != 1) { 508 return -EINVAL; 509 } 510 } 511 512 break; 513 514 case RTE_CRYPTO_CIPHER_3DES_CTR: 515 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 516 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 517 518 if (get_cipher_key_ede(xform->cipher.key.data, 519 sess->cipher.key.length, 520 sess->cipher.key.data) != 0) 521 return -EINVAL; 522 break; 523 524 case RTE_CRYPTO_CIPHER_DES_CBC: 525 sess->cipher.algo = xform->cipher.algo; 526 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 527 sess->cipher.evp_algo = EVP_des_cbc(); 528 529 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 530 sess->cipher.key.data); 531 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 532 if (EVP_EncryptInit_ex(sess->cipher.ctx, 533 sess->cipher.evp_algo, 534 NULL, xform->cipher.key.data, 535 NULL) != 1) { 536 return -EINVAL; 537 } 538 } else if (sess->cipher.direction == 539 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 540 if (EVP_DecryptInit_ex(sess->cipher.ctx, 541 sess->cipher.evp_algo, 542 NULL, xform->cipher.key.data, 543 NULL) != 1) { 544 return -EINVAL; 545 } 546 } 547 548 break; 549 550 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 551 sess->cipher.algo = xform->cipher.algo; 552 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 553 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 554 sess->cipher.evp_algo = EVP_des_cbc(); 555 556 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 557 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 558 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 559 NULL, xform->cipher.key.data, 0) != 1) 560 return -EINVAL; 561 562 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 563 sess->cipher.key.data); 564 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 565 if (EVP_EncryptInit_ex(sess->cipher.ctx, 566 sess->cipher.evp_algo, 567 NULL, xform->cipher.key.data, 568 NULL) != 1) { 569 return -EINVAL; 570 } 571 } else if (sess->cipher.direction == 572 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 573 if (EVP_DecryptInit_ex(sess->cipher.ctx, 574 sess->cipher.evp_algo, 575 NULL, xform->cipher.key.data, 576 NULL) != 1) { 577 return -EINVAL; 578 } 579 } 580 581 break; 582 default: 583 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 584 return -ENOTSUP; 585 } 586 587 return 0; 588 } 589 590 /* Set session auth parameters */ 591 static int 592 openssl_set_session_auth_parameters(struct openssl_session *sess, 593 const struct rte_crypto_sym_xform *xform) 594 { 595 /* Select auth generate/verify */ 596 sess->auth.operation = xform->auth.op; 597 sess->auth.algo = xform->auth.algo; 598 599 sess->auth.digest_length = xform->auth.digest_length; 600 601 /* Select auth algo */ 602 switch (xform->auth.algo) { 603 case RTE_CRYPTO_AUTH_AES_GMAC: 604 /* 605 * OpenSSL requires GMAC to be a GCM operation 606 * with no cipher data length 607 */ 608 sess->cipher.key.length = xform->auth.key.length; 609 610 /* Set IV parameters */ 611 sess->iv.offset = xform->auth.iv.offset; 612 sess->iv.length = xform->auth.iv.length; 613 614 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 615 return openssl_set_sess_aead_enc_param(sess, 616 RTE_CRYPTO_AEAD_AES_GCM, 617 xform->auth.digest_length, 618 xform->auth.key.data); 619 else 620 return openssl_set_sess_aead_dec_param(sess, 621 RTE_CRYPTO_AEAD_AES_GCM, 622 xform->auth.digest_length, 623 xform->auth.key.data); 624 break; 625 626 case RTE_CRYPTO_AUTH_MD5: 627 case RTE_CRYPTO_AUTH_SHA1: 628 case RTE_CRYPTO_AUTH_SHA224: 629 case RTE_CRYPTO_AUTH_SHA256: 630 case RTE_CRYPTO_AUTH_SHA384: 631 case RTE_CRYPTO_AUTH_SHA512: 632 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 633 if (get_auth_algo(xform->auth.algo, 634 &sess->auth.auth.evp_algo) != 0) 635 return -EINVAL; 636 sess->auth.auth.ctx = EVP_MD_CTX_create(); 637 break; 638 639 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 640 case RTE_CRYPTO_AUTH_MD5_HMAC: 641 case RTE_CRYPTO_AUTH_SHA1_HMAC: 642 case RTE_CRYPTO_AUTH_SHA224_HMAC: 643 case RTE_CRYPTO_AUTH_SHA256_HMAC: 644 case RTE_CRYPTO_AUTH_SHA384_HMAC: 645 case RTE_CRYPTO_AUTH_SHA512_HMAC: 646 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 647 648 OSSL_PARAM params[2]; 649 const char *algo; 650 algo = digest_name_get(xform->auth.algo); 651 if (!algo) 652 return -EINVAL; 653 char algo_name[MAX_OSSL_ALGO_NAME_SIZE]; 654 rte_memcpy(algo_name, algo, (sizeof(algo)+1)); 655 656 EVP_MAC *mac = EVP_MAC_fetch(NULL, "HMAC", NULL); 657 sess->auth.hmac.ctx = EVP_MAC_CTX_new(mac); 658 EVP_MAC_free(mac); 659 if (get_auth_algo(xform->auth.algo, 660 &sess->auth.hmac.evp_algo) != 0) 661 return -EINVAL; 662 663 params[0] = OSSL_PARAM_construct_utf8_string("digest", 664 algo_name, 0); 665 params[1] = OSSL_PARAM_construct_end(); 666 if (EVP_MAC_init(sess->auth.hmac.ctx, 667 xform->auth.key.data, 668 xform->auth.key.length, 669 params) != 1) 670 return -EINVAL; 671 break; 672 # else 673 case RTE_CRYPTO_AUTH_MD5_HMAC: 674 case RTE_CRYPTO_AUTH_SHA1_HMAC: 675 case RTE_CRYPTO_AUTH_SHA224_HMAC: 676 case RTE_CRYPTO_AUTH_SHA256_HMAC: 677 case RTE_CRYPTO_AUTH_SHA384_HMAC: 678 case RTE_CRYPTO_AUTH_SHA512_HMAC: 679 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 680 sess->auth.hmac.ctx = HMAC_CTX_new(); 681 if (get_auth_algo(xform->auth.algo, 682 &sess->auth.hmac.evp_algo) != 0) 683 return -EINVAL; 684 685 if (HMAC_Init_ex(sess->auth.hmac.ctx, 686 xform->auth.key.data, 687 xform->auth.key.length, 688 sess->auth.hmac.evp_algo, NULL) != 1) 689 return -EINVAL; 690 break; 691 # endif 692 default: 693 return -ENOTSUP; 694 } 695 696 return 0; 697 } 698 699 /* Set session AEAD parameters */ 700 static int 701 openssl_set_session_aead_parameters(struct openssl_session *sess, 702 const struct rte_crypto_sym_xform *xform) 703 { 704 /* Select cipher key */ 705 sess->cipher.key.length = xform->aead.key.length; 706 707 /* Set IV parameters */ 708 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 709 /* 710 * For AES-CCM, the actual IV is placed 711 * one byte after the start of the IV field, 712 * according to the API. 713 */ 714 sess->iv.offset = xform->aead.iv.offset + 1; 715 else 716 sess->iv.offset = xform->aead.iv.offset; 717 718 sess->iv.length = xform->aead.iv.length; 719 720 sess->auth.aad_length = xform->aead.aad_length; 721 sess->auth.digest_length = xform->aead.digest_length; 722 723 sess->aead_algo = xform->aead.algo; 724 /* Select cipher direction */ 725 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 726 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 727 xform->aead.digest_length, xform->aead.key.data); 728 else 729 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 730 xform->aead.digest_length, xform->aead.key.data); 731 } 732 733 /** Parse crypto xform chain and set private session parameters */ 734 int 735 openssl_set_session_parameters(struct openssl_session *sess, 736 const struct rte_crypto_sym_xform *xform) 737 { 738 const struct rte_crypto_sym_xform *cipher_xform = NULL; 739 const struct rte_crypto_sym_xform *auth_xform = NULL; 740 const struct rte_crypto_sym_xform *aead_xform = NULL; 741 int ret; 742 743 sess->chain_order = openssl_get_chain_order(xform); 744 switch (sess->chain_order) { 745 case OPENSSL_CHAIN_ONLY_CIPHER: 746 cipher_xform = xform; 747 break; 748 case OPENSSL_CHAIN_ONLY_AUTH: 749 auth_xform = xform; 750 break; 751 case OPENSSL_CHAIN_CIPHER_AUTH: 752 cipher_xform = xform; 753 auth_xform = xform->next; 754 break; 755 case OPENSSL_CHAIN_AUTH_CIPHER: 756 auth_xform = xform; 757 cipher_xform = xform->next; 758 break; 759 case OPENSSL_CHAIN_COMBINED: 760 aead_xform = xform; 761 break; 762 default: 763 return -EINVAL; 764 } 765 766 /* Default IV length = 0 */ 767 sess->iv.length = 0; 768 769 /* cipher_xform must be check before auth_xform */ 770 if (cipher_xform) { 771 ret = openssl_set_session_cipher_parameters( 772 sess, cipher_xform); 773 if (ret != 0) { 774 OPENSSL_LOG(ERR, 775 "Invalid/unsupported cipher parameters"); 776 return ret; 777 } 778 } 779 780 if (auth_xform) { 781 ret = openssl_set_session_auth_parameters(sess, auth_xform); 782 if (ret != 0) { 783 OPENSSL_LOG(ERR, 784 "Invalid/unsupported auth parameters"); 785 return ret; 786 } 787 } 788 789 if (aead_xform) { 790 ret = openssl_set_session_aead_parameters(sess, aead_xform); 791 if (ret != 0) { 792 OPENSSL_LOG(ERR, 793 "Invalid/unsupported AEAD parameters"); 794 return ret; 795 } 796 } 797 798 return 0; 799 } 800 801 /** Reset private session parameters */ 802 void 803 openssl_reset_session(struct openssl_session *sess) 804 { 805 EVP_CIPHER_CTX_free(sess->cipher.ctx); 806 807 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 808 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 809 810 switch (sess->auth.mode) { 811 case OPENSSL_AUTH_AS_AUTH: 812 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 813 break; 814 case OPENSSL_AUTH_AS_HMAC: 815 EVP_PKEY_free(sess->auth.hmac.pkey); 816 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 817 EVP_MAC_CTX_free(sess->auth.hmac.ctx); 818 # else 819 HMAC_CTX_free(sess->auth.hmac.ctx); 820 # endif 821 break; 822 default: 823 break; 824 } 825 } 826 827 /** Provide session for operation */ 828 static void * 829 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 830 { 831 struct openssl_session *sess = NULL; 832 struct openssl_asym_session *asym_sess = NULL; 833 834 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 835 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 836 /* get existing session */ 837 if (likely(op->sym->session != NULL)) 838 sess = (struct openssl_session *) 839 get_sym_session_private_data( 840 op->sym->session, 841 cryptodev_driver_id); 842 } else { 843 if (likely(op->asym->session != NULL)) 844 asym_sess = (struct openssl_asym_session *) 845 op->asym->session->sess_private_data; 846 if (asym_sess == NULL) 847 op->status = 848 RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 849 return asym_sess; 850 } 851 } else { 852 /* sessionless asymmetric not supported */ 853 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) 854 return NULL; 855 856 /* provide internal session */ 857 void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp); 858 void *_sess_private_data = NULL; 859 860 if (_sess == NULL) 861 return NULL; 862 863 if (rte_mempool_get(qp->sess_mp_priv, 864 (void **)&_sess_private_data)) 865 return NULL; 866 867 sess = (struct openssl_session *)_sess_private_data; 868 869 if (unlikely(openssl_set_session_parameters(sess, 870 op->sym->xform) != 0)) { 871 rte_mempool_put(qp->sess_mp, _sess); 872 rte_mempool_put(qp->sess_mp_priv, _sess_private_data); 873 sess = NULL; 874 } 875 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 876 set_sym_session_private_data(op->sym->session, 877 cryptodev_driver_id, _sess_private_data); 878 } 879 880 if (sess == NULL) 881 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 882 883 return sess; 884 } 885 886 /* 887 *------------------------------------------------------------------------------ 888 * Process Operations 889 *------------------------------------------------------------------------------ 890 */ 891 static inline int 892 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 893 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 894 { 895 struct rte_mbuf *m; 896 int dstlen; 897 int l, n = srclen; 898 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 899 900 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 901 m = m->next) 902 offset -= rte_pktmbuf_data_len(m); 903 904 if (m == 0) 905 return -1; 906 907 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 908 if (inplace) 909 *dst = src; 910 911 l = rte_pktmbuf_data_len(m) - offset; 912 if (srclen <= l) { 913 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 914 return -1; 915 *dst += l; 916 return 0; 917 } 918 919 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 920 return -1; 921 922 *dst += dstlen; 923 n -= l; 924 925 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 926 uint8_t diff = l - dstlen, rem; 927 928 src = rte_pktmbuf_mtod(m, uint8_t *); 929 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 930 if (diff && inplace) { 931 rem = RTE_MIN(l, 932 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 933 if (EVP_EncryptUpdate(ctx, temp, 934 &dstlen, src, rem) <= 0) 935 return -1; 936 n -= rem; 937 rte_memcpy(*dst, temp, diff); 938 rte_memcpy(src, temp + diff, rem); 939 src += rem; 940 l -= rem; 941 } 942 if (inplace) 943 *dst = src; 944 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 945 return -1; 946 *dst += dstlen; 947 n -= l; 948 } 949 950 return 0; 951 } 952 953 static inline int 954 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 955 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 956 { 957 struct rte_mbuf *m; 958 int dstlen; 959 int l, n = srclen; 960 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 961 962 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 963 m = m->next) 964 offset -= rte_pktmbuf_data_len(m); 965 966 if (m == 0) 967 return -1; 968 969 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 970 if (inplace) 971 *dst = src; 972 973 l = rte_pktmbuf_data_len(m) - offset; 974 if (srclen <= l) { 975 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 976 return -1; 977 *dst += l; 978 return 0; 979 } 980 981 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 982 return -1; 983 984 *dst += dstlen; 985 n -= l; 986 987 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 988 uint8_t diff = l - dstlen, rem; 989 990 src = rte_pktmbuf_mtod(m, uint8_t *); 991 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 992 if (diff && inplace) { 993 rem = RTE_MIN(l, 994 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 995 if (EVP_DecryptUpdate(ctx, temp, 996 &dstlen, src, rem) <= 0) 997 return -1; 998 n -= rem; 999 rte_memcpy(*dst, temp, diff); 1000 rte_memcpy(src, temp + diff, rem); 1001 src += rem; 1002 l -= rem; 1003 } 1004 if (inplace) 1005 *dst = src; 1006 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 1007 return -1; 1008 *dst += dstlen; 1009 n -= l; 1010 } 1011 1012 return 0; 1013 } 1014 1015 /** Process standard openssl cipher encryption */ 1016 static int 1017 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1018 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1019 uint8_t inplace) 1020 { 1021 int totlen; 1022 1023 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1024 goto process_cipher_encrypt_err; 1025 1026 EVP_CIPHER_CTX_set_padding(ctx, 0); 1027 1028 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1029 srclen, ctx, inplace)) 1030 goto process_cipher_encrypt_err; 1031 1032 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 1033 goto process_cipher_encrypt_err; 1034 1035 return 0; 1036 1037 process_cipher_encrypt_err: 1038 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 1039 return -EINVAL; 1040 } 1041 1042 /** Process standard openssl cipher encryption */ 1043 static int 1044 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 1045 uint8_t *iv, int srclen, 1046 EVP_CIPHER_CTX *ctx) 1047 { 1048 uint8_t i; 1049 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 1050 int encrypted_ivlen; 1051 1052 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 1053 iv, DES_BLOCK_SIZE) <= 0) 1054 goto process_cipher_encrypt_err; 1055 1056 for (i = 0; i < srclen; i++) 1057 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 1058 1059 return 0; 1060 1061 process_cipher_encrypt_err: 1062 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 1063 return -EINVAL; 1064 } 1065 /** Process standard openssl cipher decryption */ 1066 static int 1067 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 1068 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 1069 uint8_t inplace) 1070 { 1071 int totlen; 1072 1073 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1074 goto process_cipher_decrypt_err; 1075 1076 EVP_CIPHER_CTX_set_padding(ctx, 0); 1077 1078 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1079 srclen, ctx, inplace)) 1080 goto process_cipher_decrypt_err; 1081 1082 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 1083 goto process_cipher_decrypt_err; 1084 return 0; 1085 1086 process_cipher_decrypt_err: 1087 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 1088 return -EINVAL; 1089 } 1090 1091 /** Process cipher des 3 ctr encryption, decryption algorithm */ 1092 static int 1093 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 1094 int offset, uint8_t *iv, uint8_t *key, int srclen, 1095 EVP_CIPHER_CTX *ctx) 1096 { 1097 uint8_t ebuf[8], ctr[8]; 1098 int unused, n; 1099 struct rte_mbuf *m; 1100 uint8_t *src; 1101 int l; 1102 1103 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1104 m = m->next) 1105 offset -= rte_pktmbuf_data_len(m); 1106 1107 if (m == 0) 1108 goto process_cipher_des3ctr_err; 1109 1110 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1111 l = rte_pktmbuf_data_len(m) - offset; 1112 1113 /* We use 3DES encryption also for decryption. 1114 * IV is not important for 3DES ecb 1115 */ 1116 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 1117 goto process_cipher_des3ctr_err; 1118 1119 memcpy(ctr, iv, 8); 1120 1121 for (n = 0; n < srclen; n++) { 1122 if (n % 8 == 0) { 1123 if (EVP_EncryptUpdate(ctx, 1124 (unsigned char *)&ebuf, &unused, 1125 (const unsigned char *)&ctr, 8) <= 0) 1126 goto process_cipher_des3ctr_err; 1127 ctr_inc(ctr); 1128 } 1129 dst[n] = *(src++) ^ ebuf[n % 8]; 1130 1131 l--; 1132 if (!l) { 1133 m = m->next; 1134 if (m) { 1135 src = rte_pktmbuf_mtod(m, uint8_t *); 1136 l = rte_pktmbuf_data_len(m); 1137 } 1138 } 1139 } 1140 1141 return 0; 1142 1143 process_cipher_des3ctr_err: 1144 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 1145 return -EINVAL; 1146 } 1147 1148 /** Process AES-GCM encrypt algorithm */ 1149 static int 1150 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1151 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1152 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1153 { 1154 int len = 0, unused = 0; 1155 uint8_t empty[] = {}; 1156 1157 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1158 goto process_auth_encryption_gcm_err; 1159 1160 if (aadlen > 0) 1161 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1162 goto process_auth_encryption_gcm_err; 1163 1164 if (srclen > 0) 1165 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1166 srclen, ctx, 0)) 1167 goto process_auth_encryption_gcm_err; 1168 1169 /* Workaround open ssl bug in version less then 1.0.1f */ 1170 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1171 goto process_auth_encryption_gcm_err; 1172 1173 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1174 goto process_auth_encryption_gcm_err; 1175 1176 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1177 goto process_auth_encryption_gcm_err; 1178 1179 return 0; 1180 1181 process_auth_encryption_gcm_err: 1182 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1183 return -EINVAL; 1184 } 1185 1186 /** Process AES-CCM encrypt algorithm */ 1187 static int 1188 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1189 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1190 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1191 { 1192 int len = 0; 1193 1194 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1195 goto process_auth_encryption_ccm_err; 1196 1197 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1198 goto process_auth_encryption_ccm_err; 1199 1200 if (aadlen > 0) 1201 /* 1202 * For AES-CCM, the actual AAD is placed 1203 * 18 bytes after the start of the AAD field, 1204 * according to the API. 1205 */ 1206 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1207 goto process_auth_encryption_ccm_err; 1208 1209 if (srclen >= 0) 1210 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1211 srclen, ctx, 0)) 1212 goto process_auth_encryption_ccm_err; 1213 1214 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1215 goto process_auth_encryption_ccm_err; 1216 1217 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1218 goto process_auth_encryption_ccm_err; 1219 1220 return 0; 1221 1222 process_auth_encryption_ccm_err: 1223 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1224 return -EINVAL; 1225 } 1226 1227 /** Process AES-GCM decrypt algorithm */ 1228 static int 1229 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1230 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1231 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1232 { 1233 int len = 0, unused = 0; 1234 uint8_t empty[] = {}; 1235 1236 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1237 goto process_auth_decryption_gcm_err; 1238 1239 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1240 goto process_auth_decryption_gcm_err; 1241 1242 if (aadlen > 0) 1243 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1244 goto process_auth_decryption_gcm_err; 1245 1246 if (srclen > 0) 1247 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1248 srclen, ctx, 0)) 1249 goto process_auth_decryption_gcm_err; 1250 1251 /* Workaround open ssl bug in version less then 1.0.1f */ 1252 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1253 goto process_auth_decryption_gcm_err; 1254 1255 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1256 return -EFAULT; 1257 1258 return 0; 1259 1260 process_auth_decryption_gcm_err: 1261 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1262 return -EINVAL; 1263 } 1264 1265 /** Process AES-CCM decrypt algorithm */ 1266 static int 1267 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1268 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1269 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1270 EVP_CIPHER_CTX *ctx) 1271 { 1272 int len = 0; 1273 1274 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1275 goto process_auth_decryption_ccm_err; 1276 1277 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1278 goto process_auth_decryption_ccm_err; 1279 1280 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1281 goto process_auth_decryption_ccm_err; 1282 1283 if (aadlen > 0) 1284 /* 1285 * For AES-CCM, the actual AAD is placed 1286 * 18 bytes after the start of the AAD field, 1287 * according to the API. 1288 */ 1289 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1290 goto process_auth_decryption_ccm_err; 1291 1292 if (srclen >= 0) 1293 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1294 srclen, ctx, 0)) 1295 return -EFAULT; 1296 1297 return 0; 1298 1299 process_auth_decryption_ccm_err: 1300 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1301 return -EINVAL; 1302 } 1303 1304 /** Process standard openssl auth algorithms */ 1305 static int 1306 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1307 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1308 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1309 { 1310 size_t dstlen; 1311 struct rte_mbuf *m; 1312 int l, n = srclen; 1313 uint8_t *src; 1314 1315 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1316 m = m->next) 1317 offset -= rte_pktmbuf_data_len(m); 1318 1319 if (m == 0) 1320 goto process_auth_err; 1321 1322 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1323 goto process_auth_err; 1324 1325 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1326 1327 l = rte_pktmbuf_data_len(m) - offset; 1328 if (srclen <= l) { 1329 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1330 goto process_auth_err; 1331 goto process_auth_final; 1332 } 1333 1334 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1335 goto process_auth_err; 1336 1337 n -= l; 1338 1339 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1340 src = rte_pktmbuf_mtod(m, uint8_t *); 1341 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1342 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1343 goto process_auth_err; 1344 n -= l; 1345 } 1346 1347 process_auth_final: 1348 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1349 goto process_auth_err; 1350 return 0; 1351 1352 process_auth_err: 1353 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1354 return -EINVAL; 1355 } 1356 1357 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1358 /** Process standard openssl auth algorithms with hmac */ 1359 static int 1360 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1361 int srclen, EVP_MAC_CTX *ctx) 1362 { 1363 size_t dstlen; 1364 struct rte_mbuf *m; 1365 int l, n = srclen; 1366 uint8_t *src; 1367 1368 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1369 m = m->next) 1370 offset -= rte_pktmbuf_data_len(m); 1371 1372 if (m == 0) 1373 goto process_auth_err; 1374 1375 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1376 1377 l = rte_pktmbuf_data_len(m) - offset; 1378 if (srclen <= l) { 1379 if (EVP_MAC_update(ctx, (unsigned char *)src, srclen) != 1) 1380 goto process_auth_err; 1381 goto process_auth_final; 1382 } 1383 1384 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1385 goto process_auth_err; 1386 1387 n -= l; 1388 1389 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1390 src = rte_pktmbuf_mtod(m, uint8_t *); 1391 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1392 if (EVP_MAC_update(ctx, (unsigned char *)src, l) != 1) 1393 goto process_auth_err; 1394 n -= l; 1395 } 1396 1397 process_auth_final: 1398 if (EVP_MAC_final(ctx, dst, &dstlen, sizeof(dst)) != 1) 1399 goto process_auth_err; 1400 1401 EVP_MAC_CTX_free(ctx); 1402 return 0; 1403 1404 process_auth_err: 1405 EVP_MAC_CTX_free(ctx); 1406 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1407 return -EINVAL; 1408 } 1409 # else 1410 /** Process standard openssl auth algorithms with hmac */ 1411 static int 1412 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1413 int srclen, HMAC_CTX *ctx) 1414 { 1415 unsigned int dstlen; 1416 struct rte_mbuf *m; 1417 int l, n = srclen; 1418 uint8_t *src; 1419 1420 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1421 m = m->next) 1422 offset -= rte_pktmbuf_data_len(m); 1423 1424 if (m == 0) 1425 goto process_auth_err; 1426 1427 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1428 1429 l = rte_pktmbuf_data_len(m) - offset; 1430 if (srclen <= l) { 1431 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1432 goto process_auth_err; 1433 goto process_auth_final; 1434 } 1435 1436 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1437 goto process_auth_err; 1438 1439 n -= l; 1440 1441 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1442 src = rte_pktmbuf_mtod(m, uint8_t *); 1443 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1444 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1445 goto process_auth_err; 1446 n -= l; 1447 } 1448 1449 process_auth_final: 1450 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1451 goto process_auth_err; 1452 1453 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1454 goto process_auth_err; 1455 1456 return 0; 1457 1458 process_auth_err: 1459 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1460 return -EINVAL; 1461 } 1462 # endif 1463 /*----------------------------------------------------------------------------*/ 1464 1465 /** Process auth/cipher combined operation */ 1466 static void 1467 process_openssl_combined_op 1468 (struct rte_crypto_op *op, struct openssl_session *sess, 1469 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1470 { 1471 /* cipher */ 1472 uint8_t *dst = NULL, *iv, *tag, *aad; 1473 int srclen, aadlen, status = -1; 1474 uint32_t offset; 1475 uint8_t taglen; 1476 1477 /* 1478 * Segmented destination buffer is not supported for 1479 * encryption/decryption 1480 */ 1481 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1482 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1483 return; 1484 } 1485 1486 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1487 sess->iv.offset); 1488 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1489 srclen = 0; 1490 offset = op->sym->auth.data.offset; 1491 aadlen = op->sym->auth.data.length; 1492 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1493 op->sym->auth.data.offset); 1494 tag = op->sym->auth.digest.data; 1495 if (tag == NULL) 1496 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1497 offset + aadlen); 1498 } else { 1499 srclen = op->sym->aead.data.length; 1500 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1501 op->sym->aead.data.offset); 1502 offset = op->sym->aead.data.offset; 1503 aad = op->sym->aead.aad.data; 1504 aadlen = sess->auth.aad_length; 1505 tag = op->sym->aead.digest.data; 1506 if (tag == NULL) 1507 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1508 offset + srclen); 1509 } 1510 1511 taglen = sess->auth.digest_length; 1512 1513 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1514 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1515 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1516 status = process_openssl_auth_encryption_gcm( 1517 mbuf_src, offset, srclen, 1518 aad, aadlen, iv, 1519 dst, tag, sess->cipher.ctx); 1520 else 1521 status = process_openssl_auth_encryption_ccm( 1522 mbuf_src, offset, srclen, 1523 aad, aadlen, iv, 1524 dst, tag, taglen, sess->cipher.ctx); 1525 1526 } else { 1527 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1528 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1529 status = process_openssl_auth_decryption_gcm( 1530 mbuf_src, offset, srclen, 1531 aad, aadlen, iv, 1532 dst, tag, sess->cipher.ctx); 1533 else 1534 status = process_openssl_auth_decryption_ccm( 1535 mbuf_src, offset, srclen, 1536 aad, aadlen, iv, 1537 dst, tag, taglen, sess->cipher.ctx); 1538 } 1539 1540 if (status != 0) { 1541 if (status == (-EFAULT) && 1542 sess->auth.operation == 1543 RTE_CRYPTO_AUTH_OP_VERIFY) 1544 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1545 else 1546 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1547 } 1548 } 1549 1550 /** Process cipher operation */ 1551 static void 1552 process_openssl_cipher_op 1553 (struct rte_crypto_op *op, struct openssl_session *sess, 1554 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1555 { 1556 uint8_t *dst, *iv; 1557 int srclen, status; 1558 uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0; 1559 EVP_CIPHER_CTX *ctx_copy; 1560 1561 /* 1562 * Segmented OOP destination buffer is not supported for encryption/ 1563 * decryption. In case of des3ctr, even inplace segmented buffers are 1564 * not supported. 1565 */ 1566 if (!rte_pktmbuf_is_contiguous(mbuf_dst) && 1567 (!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) { 1568 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1569 return; 1570 } 1571 1572 srclen = op->sym->cipher.data.length; 1573 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1574 op->sym->cipher.data.offset); 1575 1576 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1577 sess->iv.offset); 1578 ctx_copy = EVP_CIPHER_CTX_new(); 1579 EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx); 1580 1581 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1582 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1583 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1584 op->sym->cipher.data.offset, iv, 1585 srclen, ctx_copy, inplace); 1586 else 1587 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1588 op->sym->cipher.data.offset, iv, 1589 srclen, ctx_copy, inplace); 1590 else 1591 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1592 op->sym->cipher.data.offset, iv, 1593 sess->cipher.key.data, srclen, 1594 ctx_copy); 1595 1596 EVP_CIPHER_CTX_free(ctx_copy); 1597 if (status != 0) 1598 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1599 } 1600 1601 /** Process cipher operation */ 1602 static void 1603 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1604 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1605 struct rte_mbuf *mbuf_dst) 1606 { 1607 uint8_t *src, *dst, *iv; 1608 uint8_t block_size, last_block_len; 1609 int srclen, status = 0; 1610 1611 srclen = op->sym->cipher.data.length; 1612 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1613 op->sym->cipher.data.offset); 1614 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1615 op->sym->cipher.data.offset); 1616 1617 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1618 sess->iv.offset); 1619 1620 block_size = DES_BLOCK_SIZE; 1621 1622 last_block_len = srclen % block_size; 1623 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1624 /* Encrypt only with ECB mode XOR IV */ 1625 if (srclen < block_size) { 1626 status = process_openssl_cipher_bpi_encrypt(src, dst, 1627 iv, srclen, 1628 sess->cipher.bpi_ctx); 1629 } else { 1630 srclen -= last_block_len; 1631 /* Encrypt with the block aligned stream with CBC mode */ 1632 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1633 op->sym->cipher.data.offset, iv, 1634 srclen, sess->cipher.ctx, 0); 1635 if (last_block_len) { 1636 /* Point at last block */ 1637 dst += srclen; 1638 /* 1639 * IV is the last encrypted block from 1640 * the previous operation 1641 */ 1642 iv = dst - block_size; 1643 src += srclen; 1644 srclen = last_block_len; 1645 /* Encrypt the last frame with ECB mode */ 1646 status |= process_openssl_cipher_bpi_encrypt(src, 1647 dst, iv, 1648 srclen, sess->cipher.bpi_ctx); 1649 } 1650 } 1651 } else { 1652 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1653 if (srclen < block_size) { 1654 status = process_openssl_cipher_bpi_encrypt(src, dst, 1655 iv, 1656 srclen, 1657 sess->cipher.bpi_ctx); 1658 } else { 1659 if (last_block_len) { 1660 /* Point at last block */ 1661 dst += srclen - last_block_len; 1662 src += srclen - last_block_len; 1663 /* 1664 * IV is the last full block 1665 */ 1666 iv = src - block_size; 1667 /* 1668 * Decrypt the last frame with ECB mode 1669 * (encrypt, as it is the same operation) 1670 */ 1671 status = process_openssl_cipher_bpi_encrypt(src, 1672 dst, iv, 1673 last_block_len, sess->cipher.bpi_ctx); 1674 /* Prepare parameters for CBC mode op */ 1675 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1676 sess->iv.offset); 1677 dst += last_block_len - srclen; 1678 srclen -= last_block_len; 1679 } 1680 1681 /* Decrypt with CBC mode */ 1682 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1683 op->sym->cipher.data.offset, iv, 1684 srclen, sess->cipher.ctx, 0); 1685 } 1686 } 1687 1688 if (status != 0) 1689 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1690 } 1691 1692 /** Process auth operation */ 1693 static void 1694 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1695 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1696 struct rte_mbuf *mbuf_dst) 1697 { 1698 uint8_t *dst; 1699 int srclen, status; 1700 EVP_MD_CTX *ctx_a; 1701 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1702 EVP_MAC_CTX *ctx_h; 1703 EVP_MAC *mac; 1704 # else 1705 HMAC_CTX *ctx_h; 1706 # endif 1707 1708 srclen = op->sym->auth.data.length; 1709 1710 dst = qp->temp_digest; 1711 1712 switch (sess->auth.mode) { 1713 case OPENSSL_AUTH_AS_AUTH: 1714 ctx_a = EVP_MD_CTX_create(); 1715 EVP_MD_CTX_copy_ex(ctx_a, sess->auth.auth.ctx); 1716 status = process_openssl_auth(mbuf_src, dst, 1717 op->sym->auth.data.offset, NULL, NULL, srclen, 1718 ctx_a, sess->auth.auth.evp_algo); 1719 EVP_MD_CTX_destroy(ctx_a); 1720 break; 1721 case OPENSSL_AUTH_AS_HMAC: 1722 # if OPENSSL_VERSION_NUMBER >= 0x30000000L 1723 mac = EVP_MAC_fetch(NULL, "HMAC", NULL); 1724 ctx_h = EVP_MAC_CTX_new(mac); 1725 ctx_h = EVP_MAC_CTX_dup(sess->auth.hmac.ctx); 1726 EVP_MAC_free(mac); 1727 status = process_openssl_auth_hmac(mbuf_src, dst, 1728 op->sym->auth.data.offset, srclen, 1729 ctx_h); 1730 # else 1731 ctx_h = HMAC_CTX_new(); 1732 HMAC_CTX_copy(ctx_h, sess->auth.hmac.ctx); 1733 status = process_openssl_auth_hmac(mbuf_src, dst, 1734 op->sym->auth.data.offset, srclen, 1735 ctx_h); 1736 HMAC_CTX_free(ctx_h); 1737 # endif 1738 break; 1739 default: 1740 status = -1; 1741 break; 1742 } 1743 1744 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1745 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data, 1746 sess->auth.digest_length) != 0) { 1747 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1748 } 1749 } else { 1750 uint8_t *auth_dst; 1751 1752 auth_dst = op->sym->auth.digest.data; 1753 if (auth_dst == NULL) 1754 auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1755 op->sym->auth.data.offset + 1756 op->sym->auth.data.length); 1757 memcpy(auth_dst, dst, sess->auth.digest_length); 1758 } 1759 1760 if (status != 0) 1761 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1762 } 1763 1764 /* process dsa sign operation */ 1765 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 1766 static int 1767 process_openssl_dsa_sign_op_evp(struct rte_crypto_op *cop, 1768 struct openssl_asym_session *sess) 1769 { 1770 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1771 EVP_PKEY_CTX *dsa_ctx = NULL; 1772 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 1773 EVP_PKEY *pkey = NULL; 1774 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 1775 OSSL_PARAM *params = NULL; 1776 1777 size_t outlen; 1778 unsigned char *dsa_sign_data; 1779 const unsigned char *dsa_sign_data_p; 1780 1781 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1782 params = OSSL_PARAM_BLD_to_param(param_bld); 1783 if (!params) { 1784 OSSL_PARAM_BLD_free(param_bld); 1785 return -1; 1786 } 1787 1788 if (key_ctx == NULL 1789 || EVP_PKEY_fromdata_init(key_ctx) <= 0 1790 || EVP_PKEY_fromdata(key_ctx, &pkey, 1791 EVP_PKEY_KEYPAIR, params) <= 0) 1792 goto err_dsa_sign; 1793 1794 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 1795 if (!dsa_ctx) 1796 goto err_dsa_sign; 1797 1798 if (EVP_PKEY_sign_init(dsa_ctx) <= 0) 1799 goto err_dsa_sign; 1800 1801 if (EVP_PKEY_sign(dsa_ctx, NULL, &outlen, op->message.data, 1802 op->message.length) <= 0) 1803 goto err_dsa_sign; 1804 1805 if (outlen <= 0) 1806 goto err_dsa_sign; 1807 1808 dsa_sign_data = OPENSSL_malloc(outlen); 1809 if (!dsa_sign_data) 1810 goto err_dsa_sign; 1811 1812 if (EVP_PKEY_sign(dsa_ctx, dsa_sign_data, &outlen, op->message.data, 1813 op->message.length) <= 0) { 1814 free(dsa_sign_data); 1815 goto err_dsa_sign; 1816 } 1817 1818 dsa_sign_data_p = (const unsigned char *)dsa_sign_data; 1819 DSA_SIG *sign = d2i_DSA_SIG(NULL, &dsa_sign_data_p, outlen); 1820 if (!sign) { 1821 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 1822 free(dsa_sign_data); 1823 goto err_dsa_sign; 1824 } else { 1825 const BIGNUM *r = NULL, *s = NULL; 1826 get_dsa_sign(sign, &r, &s); 1827 1828 op->r.length = BN_bn2bin(r, op->r.data); 1829 op->s.length = BN_bn2bin(s, op->s.data); 1830 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1831 } 1832 1833 DSA_SIG_free(sign); 1834 free(dsa_sign_data); 1835 return 0; 1836 1837 err_dsa_sign: 1838 if (params) 1839 OSSL_PARAM_free(params); 1840 if (key_ctx) 1841 EVP_PKEY_CTX_free(key_ctx); 1842 if (dsa_ctx) 1843 EVP_PKEY_CTX_free(dsa_ctx); 1844 return -1; 1845 } 1846 1847 /* process dsa verify operation */ 1848 static int 1849 process_openssl_dsa_verify_op_evp(struct rte_crypto_op *cop, 1850 struct openssl_asym_session *sess) 1851 { 1852 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1853 DSA_SIG *sign = DSA_SIG_new(); 1854 BIGNUM *r = NULL, *s = NULL; 1855 BIGNUM *pub_key = NULL; 1856 OSSL_PARAM_BLD *param_bld = sess->u.s.param_bld; 1857 OSSL_PARAM *params = NULL; 1858 EVP_PKEY *pkey = NULL; 1859 EVP_PKEY_CTX *dsa_ctx = NULL; 1860 EVP_PKEY_CTX *key_ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL); 1861 unsigned char *dsa_sig = NULL; 1862 size_t sig_len; 1863 int ret = -1; 1864 1865 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1866 if (!param_bld) { 1867 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 1868 return -1; 1869 } 1870 1871 r = BN_bin2bn(op->r.data, op->r.length, r); 1872 s = BN_bin2bn(op->s.data, op->s.length, s); 1873 pub_key = BN_bin2bn(op->y.data, op->y.length, pub_key); 1874 if (!r || !s || !pub_key) { 1875 BN_free(r); 1876 BN_free(s); 1877 BN_free(pub_key); 1878 OSSL_PARAM_BLD_free(param_bld); 1879 goto err_dsa_verify; 1880 } 1881 1882 set_dsa_sign(sign, r, s); 1883 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PUB_KEY, pub_key)) { 1884 OSSL_PARAM_BLD_free(param_bld); 1885 goto err_dsa_verify; 1886 } 1887 1888 params = OSSL_PARAM_BLD_to_param(param_bld); 1889 if (!params) { 1890 OSSL_PARAM_BLD_free(param_bld); 1891 goto err_dsa_verify; 1892 } 1893 1894 if (key_ctx == NULL 1895 || EVP_PKEY_fromdata_init(key_ctx) <= 0 1896 || EVP_PKEY_fromdata(key_ctx, &pkey, EVP_PKEY_KEYPAIR, params) <= 0) 1897 goto err_dsa_verify; 1898 1899 dsa_ctx = EVP_PKEY_CTX_new(pkey, NULL); 1900 if (!dsa_ctx) 1901 goto err_dsa_verify; 1902 1903 if (!sign) 1904 goto err_dsa_verify; 1905 1906 sig_len = i2d_DSA_SIG(sign, &dsa_sig); 1907 if (EVP_PKEY_verify_init(dsa_ctx) <= 0) 1908 goto err_dsa_verify; 1909 1910 ret = EVP_PKEY_verify(dsa_ctx, dsa_sig, sig_len, 1911 op->message.data, op->message.length); 1912 if (ret == 1) { 1913 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1914 ret = 0; 1915 } 1916 1917 err_dsa_verify: 1918 if (sign) 1919 DSA_SIG_free(sign); 1920 if (params) 1921 OSSL_PARAM_free(params); 1922 if (key_ctx) 1923 EVP_PKEY_CTX_free(key_ctx); 1924 if (dsa_ctx) 1925 EVP_PKEY_CTX_free(dsa_ctx); 1926 1927 return ret; 1928 } 1929 #else 1930 static int 1931 process_openssl_dsa_sign_op(struct rte_crypto_op *cop, 1932 struct openssl_asym_session *sess) 1933 { 1934 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1935 DSA *dsa = sess->u.s.dsa; 1936 DSA_SIG *sign = NULL; 1937 1938 sign = DSA_do_sign(op->message.data, 1939 op->message.length, 1940 dsa); 1941 1942 if (sign == NULL) { 1943 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 1944 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1945 } else { 1946 const BIGNUM *r = NULL, *s = NULL; 1947 get_dsa_sign(sign, &r, &s); 1948 1949 op->r.length = BN_bn2bin(r, op->r.data); 1950 op->s.length = BN_bn2bin(s, op->s.data); 1951 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1952 } 1953 1954 DSA_SIG_free(sign); 1955 1956 return 0; 1957 } 1958 1959 /* process dsa verify operation */ 1960 static int 1961 process_openssl_dsa_verify_op(struct rte_crypto_op *cop, 1962 struct openssl_asym_session *sess) 1963 { 1964 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1965 DSA *dsa = sess->u.s.dsa; 1966 int ret; 1967 DSA_SIG *sign = DSA_SIG_new(); 1968 BIGNUM *r = NULL, *s = NULL; 1969 BIGNUM *pub_key = NULL; 1970 1971 if (sign == NULL) { 1972 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 1973 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1974 return -1; 1975 } 1976 1977 r = BN_bin2bn(op->r.data, 1978 op->r.length, 1979 r); 1980 s = BN_bin2bn(op->s.data, 1981 op->s.length, 1982 s); 1983 pub_key = BN_bin2bn(op->y.data, 1984 op->y.length, 1985 pub_key); 1986 if (!r || !s || !pub_key) { 1987 BN_free(r); 1988 BN_free(s); 1989 BN_free(pub_key); 1990 1991 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1992 return -1; 1993 } 1994 set_dsa_sign(sign, r, s); 1995 set_dsa_pub_key(dsa, pub_key); 1996 1997 ret = DSA_do_verify(op->message.data, 1998 op->message.length, 1999 sign, 2000 dsa); 2001 2002 if (ret != 1) 2003 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2004 else 2005 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2006 2007 DSA_SIG_free(sign); 2008 2009 return 0; 2010 } 2011 #endif 2012 2013 /* process dh operation */ 2014 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2015 static int 2016 process_openssl_dh_op_evp(struct rte_crypto_op *cop, 2017 struct openssl_asym_session *sess) 2018 { 2019 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2020 OSSL_PARAM_BLD *param_bld = sess->u.dh.param_bld; 2021 OSSL_PARAM_BLD *param_bld_peer = sess->u.dh.param_bld_peer; 2022 OSSL_PARAM *params = NULL; 2023 EVP_PKEY *dhpkey = NULL; 2024 EVP_PKEY *peerkey = NULL; 2025 BIGNUM *priv_key = NULL; 2026 BIGNUM *pub_key = NULL; 2027 int ret = -1; 2028 2029 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2030 EVP_PKEY_CTX *dh_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2031 if (dh_ctx == NULL || param_bld == NULL) 2032 return ret; 2033 2034 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2035 OSSL_PARAM *params_peer = NULL; 2036 2037 if (!param_bld_peer) 2038 return ret; 2039 2040 pub_key = BN_bin2bn(op->pub_key.data, op->pub_key.length, 2041 pub_key); 2042 if (pub_key == NULL) { 2043 OSSL_PARAM_BLD_free(param_bld_peer); 2044 return ret; 2045 } 2046 2047 if (!OSSL_PARAM_BLD_push_BN(param_bld_peer, OSSL_PKEY_PARAM_PUB_KEY, 2048 pub_key)) { 2049 OPENSSL_LOG(ERR, "Failed to set public key\n"); 2050 OSSL_PARAM_BLD_free(param_bld_peer); 2051 BN_free(pub_key); 2052 return ret; 2053 } 2054 2055 params_peer = OSSL_PARAM_BLD_to_param(param_bld_peer); 2056 if (!params_peer) { 2057 OSSL_PARAM_BLD_free(param_bld_peer); 2058 BN_free(pub_key); 2059 return ret; 2060 } 2061 2062 EVP_PKEY_CTX *peer_ctx = EVP_PKEY_CTX_new_id(EVP_PKEY_DH, NULL); 2063 if (EVP_PKEY_keygen_init(peer_ctx) != 1) { 2064 OSSL_PARAM_free(params_peer); 2065 BN_free(pub_key); 2066 return ret; 2067 } 2068 2069 if (EVP_PKEY_CTX_set_params(peer_ctx, params_peer) != 1) { 2070 EVP_PKEY_CTX_free(peer_ctx); 2071 OSSL_PARAM_free(params_peer); 2072 BN_free(pub_key); 2073 return ret; 2074 } 2075 2076 if (EVP_PKEY_keygen(peer_ctx, &peerkey) != 1) { 2077 EVP_PKEY_CTX_free(peer_ctx); 2078 OSSL_PARAM_free(params_peer); 2079 BN_free(pub_key); 2080 return ret; 2081 } 2082 2083 priv_key = BN_bin2bn(op->priv_key.data, op->priv_key.length, 2084 priv_key); 2085 if (priv_key == NULL) { 2086 EVP_PKEY_CTX_free(peer_ctx); 2087 OSSL_PARAM_free(params_peer); 2088 BN_free(pub_key); 2089 return ret; 2090 } 2091 2092 if (!OSSL_PARAM_BLD_push_BN(param_bld, OSSL_PKEY_PARAM_PRIV_KEY, 2093 priv_key)) { 2094 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2095 EVP_PKEY_CTX_free(peer_ctx); 2096 OSSL_PARAM_free(params_peer); 2097 BN_free(pub_key); 2098 BN_free(priv_key); 2099 return ret; 2100 } 2101 2102 OSSL_PARAM_free(params_peer); 2103 EVP_PKEY_CTX_free(peer_ctx); 2104 } 2105 2106 params = OSSL_PARAM_BLD_to_param(param_bld); 2107 if (!params) 2108 goto err_dh; 2109 2110 if (EVP_PKEY_keygen_init(dh_ctx) != 1) 2111 goto err_dh; 2112 2113 if (EVP_PKEY_CTX_set_params(dh_ctx, params) != 1) 2114 goto err_dh; 2115 2116 if (EVP_PKEY_keygen(dh_ctx, &dhpkey) != 1) 2117 goto err_dh; 2118 2119 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2120 OPENSSL_LOG(DEBUG, "%s:%d updated pub key\n", __func__, __LINE__); 2121 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PUB_KEY, &pub_key)) 2122 goto err_dh; 2123 /* output public key */ 2124 op->pub_key.length = BN_bn2bin(pub_key, op->pub_key.data); 2125 } 2126 2127 if (op->ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2128 2129 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", __func__, __LINE__); 2130 if (!EVP_PKEY_get_bn_param(dhpkey, OSSL_PKEY_PARAM_PRIV_KEY, &priv_key)) 2131 goto err_dh; 2132 2133 /* provide generated private key back to user */ 2134 op->priv_key.length = BN_bn2bin(priv_key, op->priv_key.data); 2135 } 2136 2137 if (op->ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2138 size_t skey_len; 2139 EVP_PKEY_CTX *sc_ctx = EVP_PKEY_CTX_new(dhpkey, NULL); 2140 if (!sc_ctx) 2141 goto err_dh; 2142 2143 if (EVP_PKEY_derive_init(sc_ctx) <= 0) { 2144 EVP_PKEY_CTX_free(sc_ctx); 2145 goto err_dh; 2146 } 2147 2148 if (!peerkey) { 2149 EVP_PKEY_CTX_free(sc_ctx); 2150 goto err_dh; 2151 } 2152 2153 if (EVP_PKEY_derive_set_peer(sc_ctx, peerkey) <= 0) { 2154 EVP_PKEY_CTX_free(sc_ctx); 2155 goto err_dh; 2156 } 2157 2158 /* Determine buffer length */ 2159 if (EVP_PKEY_derive(sc_ctx, NULL, &skey_len) <= 0) { 2160 EVP_PKEY_CTX_free(sc_ctx); 2161 goto err_dh; 2162 } 2163 2164 if (EVP_PKEY_derive(sc_ctx, op->shared_secret.data, &skey_len) <= 0) { 2165 EVP_PKEY_CTX_free(sc_ctx); 2166 goto err_dh; 2167 } 2168 2169 op->shared_secret.length = skey_len; 2170 EVP_PKEY_CTX_free(sc_ctx); 2171 } 2172 2173 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2174 ret = 0; 2175 2176 err_dh: 2177 if (pub_key) 2178 BN_free(pub_key); 2179 if (priv_key) 2180 BN_free(priv_key); 2181 if (params) 2182 OSSL_PARAM_free(params); 2183 if (dhpkey) 2184 EVP_PKEY_free(dhpkey); 2185 if (peerkey) 2186 EVP_PKEY_free(peerkey); 2187 2188 EVP_PKEY_CTX_free(dh_ctx); 2189 2190 return ret; 2191 } 2192 #else 2193 static int 2194 process_openssl_dh_op(struct rte_crypto_op *cop, 2195 struct openssl_asym_session *sess) 2196 { 2197 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 2198 struct rte_crypto_asym_op *asym_op = cop->asym; 2199 DH *dh_key = sess->u.dh.dh_key; 2200 BIGNUM *priv_key = NULL; 2201 int ret = 0; 2202 2203 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_SHARED_SECRET_COMPUTE) { 2204 /* compute shared secret using peer public key 2205 * and current private key 2206 * shared secret = peer_key ^ priv_key mod p 2207 */ 2208 BIGNUM *peer_key = NULL; 2209 2210 /* copy private key and peer key and compute shared secret */ 2211 peer_key = BN_bin2bn(op->pub_key.data, 2212 op->pub_key.length, 2213 peer_key); 2214 if (peer_key == NULL) { 2215 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2216 return -1; 2217 } 2218 priv_key = BN_bin2bn(op->priv_key.data, 2219 op->priv_key.length, 2220 priv_key); 2221 if (priv_key == NULL) { 2222 BN_free(peer_key); 2223 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2224 return -1; 2225 } 2226 ret = set_dh_priv_key(dh_key, priv_key); 2227 if (ret) { 2228 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2229 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2230 BN_free(peer_key); 2231 BN_free(priv_key); 2232 return 0; 2233 } 2234 2235 ret = DH_compute_key( 2236 op->shared_secret.data, 2237 peer_key, dh_key); 2238 if (ret < 0) { 2239 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2240 BN_free(peer_key); 2241 /* priv key is already loaded into dh, 2242 * let's not free that directly here. 2243 * DH_free() will auto free it later. 2244 */ 2245 return 0; 2246 } 2247 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2248 op->shared_secret.length = ret; 2249 BN_free(peer_key); 2250 return 0; 2251 } 2252 2253 /* 2254 * other options are public and private key generations. 2255 * 2256 * if user provides private key, 2257 * then first set DH with user provided private key 2258 */ 2259 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE && 2260 op->priv_key.length) { 2261 /* generate public key using user-provided private key 2262 * pub_key = g ^ priv_key mod p 2263 */ 2264 2265 /* load private key into DH */ 2266 priv_key = BN_bin2bn(op->priv_key.data, 2267 op->priv_key.length, 2268 priv_key); 2269 if (priv_key == NULL) { 2270 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2271 return -1; 2272 } 2273 ret = set_dh_priv_key(dh_key, priv_key); 2274 if (ret) { 2275 OPENSSL_LOG(ERR, "Failed to set private key\n"); 2276 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2277 BN_free(priv_key); 2278 return 0; 2279 } 2280 } 2281 2282 /* generate public and private key pair. 2283 * 2284 * if private key already set, generates only public key. 2285 * 2286 * if private key is not already set, then set it to random value 2287 * and update internal private key. 2288 */ 2289 if (!DH_generate_key(dh_key)) { 2290 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2291 return 0; 2292 } 2293 2294 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PUB_KEY_GENERATE) { 2295 const BIGNUM *pub_key = NULL; 2296 2297 OPENSSL_LOG(DEBUG, "%s:%d update public key\n", 2298 __func__, __LINE__); 2299 2300 /* get the generated keys */ 2301 get_dh_pub_key(dh_key, &pub_key); 2302 2303 /* output public key */ 2304 op->pub_key.length = BN_bn2bin(pub_key, 2305 op->pub_key.data); 2306 } 2307 2308 if (asym_op->dh.ke_type == RTE_CRYPTO_ASYM_KE_PRIV_KEY_GENERATE) { 2309 const BIGNUM *priv_key = NULL; 2310 2311 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", 2312 __func__, __LINE__); 2313 2314 /* get the generated keys */ 2315 get_dh_priv_key(dh_key, &priv_key); 2316 2317 /* provide generated private key back to user */ 2318 op->priv_key.length = BN_bn2bin(priv_key, 2319 op->priv_key.data); 2320 } 2321 2322 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2323 2324 return 0; 2325 } 2326 #endif 2327 2328 /* process modinv operation */ 2329 static int 2330 process_openssl_modinv_op(struct rte_crypto_op *cop, 2331 struct openssl_asym_session *sess) 2332 { 2333 struct rte_crypto_asym_op *op = cop->asym; 2334 BIGNUM *base = BN_CTX_get(sess->u.m.ctx); 2335 BIGNUM *res = BN_CTX_get(sess->u.m.ctx); 2336 2337 if (unlikely(base == NULL || res == NULL)) { 2338 BN_free(base); 2339 BN_free(res); 2340 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2341 return -1; 2342 } 2343 2344 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 2345 op->modinv.base.length, base); 2346 2347 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) { 2348 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2349 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data); 2350 } else { 2351 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2352 } 2353 2354 BN_clear(res); 2355 BN_clear(base); 2356 2357 return 0; 2358 } 2359 2360 /* process modexp operation */ 2361 static int 2362 process_openssl_modexp_op(struct rte_crypto_op *cop, 2363 struct openssl_asym_session *sess) 2364 { 2365 struct rte_crypto_asym_op *op = cop->asym; 2366 BIGNUM *base = BN_CTX_get(sess->u.e.ctx); 2367 BIGNUM *res = BN_CTX_get(sess->u.e.ctx); 2368 2369 if (unlikely(base == NULL || res == NULL)) { 2370 BN_free(base); 2371 BN_free(res); 2372 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2373 return -1; 2374 } 2375 2376 base = BN_bin2bn((const unsigned char *)op->modex.base.data, 2377 op->modex.base.length, base); 2378 2379 if (BN_mod_exp(res, base, sess->u.e.exp, 2380 sess->u.e.mod, sess->u.e.ctx)) { 2381 op->modex.result.length = BN_bn2bin(res, op->modex.result.data); 2382 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2383 } else { 2384 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2385 } 2386 2387 BN_clear(res); 2388 BN_clear(base); 2389 2390 return 0; 2391 } 2392 2393 /* process rsa operations */ 2394 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2395 static int 2396 process_openssl_rsa_op_evp(struct rte_crypto_op *cop, 2397 struct openssl_asym_session *sess) 2398 { 2399 struct rte_crypto_asym_op *op = cop->asym; 2400 uint32_t pad = (op->rsa.padding.type); 2401 uint8_t *tmp; 2402 size_t outlen = 0; 2403 int ret = -1; 2404 2405 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2406 EVP_PKEY_CTX *rsa_ctx = sess->u.r.ctx; 2407 if (!rsa_ctx) 2408 return ret; 2409 2410 switch (pad) { 2411 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2412 pad = RSA_PKCS1_PADDING; 2413 break; 2414 case RTE_CRYPTO_RSA_PADDING_NONE: 2415 pad = RSA_NO_PADDING; 2416 break; 2417 default: 2418 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2419 OPENSSL_LOG(ERR, 2420 "rsa pad type not supported %d\n", pad); 2421 return ret; 2422 } 2423 2424 switch (op->rsa.op_type) { 2425 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2426 if (EVP_PKEY_encrypt_init(rsa_ctx) != 1) 2427 goto err_rsa; 2428 2429 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2430 goto err_rsa; 2431 2432 if (EVP_PKEY_encrypt(rsa_ctx, NULL, &outlen, 2433 op->rsa.message.data, 2434 op->rsa.message.length) <= 0) 2435 goto err_rsa; 2436 2437 if (outlen <= 0) 2438 goto err_rsa; 2439 2440 if (EVP_PKEY_encrypt(rsa_ctx, op->rsa.cipher.data, &outlen, 2441 op->rsa.message.data, 2442 op->rsa.message.length) <= 0) 2443 goto err_rsa; 2444 op->rsa.cipher.length = outlen; 2445 2446 OPENSSL_LOG(DEBUG, 2447 "length of encrypted text %zu\n", outlen); 2448 break; 2449 2450 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2451 if (EVP_PKEY_decrypt_init(rsa_ctx) != 1) 2452 goto err_rsa; 2453 2454 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2455 goto err_rsa; 2456 2457 if (EVP_PKEY_decrypt(rsa_ctx, NULL, &outlen, 2458 op->rsa.cipher.data, 2459 op->rsa.cipher.length) <= 0) 2460 goto err_rsa; 2461 2462 if (outlen <= 0) 2463 goto err_rsa; 2464 2465 if (EVP_PKEY_decrypt(rsa_ctx, op->rsa.message.data, &outlen, 2466 op->rsa.cipher.data, 2467 op->rsa.cipher.length) <= 0) 2468 goto err_rsa; 2469 op->rsa.message.length = outlen; 2470 2471 OPENSSL_LOG(DEBUG, "length of decrypted text %zu\n", outlen); 2472 break; 2473 2474 case RTE_CRYPTO_ASYM_OP_SIGN: 2475 if (EVP_PKEY_sign_init(rsa_ctx) <= 0) 2476 goto err_rsa; 2477 2478 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2479 goto err_rsa; 2480 2481 if (EVP_PKEY_sign(rsa_ctx, NULL, &outlen, 2482 op->rsa.message.data, 2483 op->rsa.message.length) <= 0) 2484 goto err_rsa; 2485 2486 if (outlen <= 0) 2487 goto err_rsa; 2488 2489 if (EVP_PKEY_sign(rsa_ctx, op->rsa.sign.data, &outlen, 2490 op->rsa.message.data, 2491 op->rsa.message.length) <= 0) 2492 goto err_rsa; 2493 op->rsa.sign.length = outlen; 2494 break; 2495 2496 case RTE_CRYPTO_ASYM_OP_VERIFY: 2497 if (EVP_PKEY_verify_recover_init(rsa_ctx) <= 0) 2498 goto err_rsa; 2499 2500 if (EVP_PKEY_CTX_set_rsa_padding(rsa_ctx, pad) <= 0) 2501 goto err_rsa; 2502 2503 if (EVP_PKEY_verify_recover(rsa_ctx, NULL, &outlen, 2504 op->rsa.sign.data, 2505 op->rsa.sign.length) <= 0) 2506 goto err_rsa; 2507 2508 if ((outlen <= 0) || (outlen != op->rsa.sign.length)) 2509 goto err_rsa; 2510 2511 tmp = OPENSSL_malloc(outlen); 2512 if (tmp == NULL) { 2513 OPENSSL_LOG(ERR, "Memory allocation failed"); 2514 goto err_rsa; 2515 } 2516 2517 if (EVP_PKEY_verify_recover(rsa_ctx, tmp, &outlen, 2518 op->rsa.sign.data, 2519 op->rsa.sign.length) <= 0) { 2520 rte_free(tmp); 2521 goto err_rsa; 2522 } 2523 2524 OPENSSL_LOG(DEBUG, 2525 "Length of public_decrypt %zu " 2526 "length of message %zd\n", 2527 outlen, op->rsa.message.length); 2528 if (CRYPTO_memcmp(tmp, op->rsa.message.data, 2529 op->rsa.message.length)) { 2530 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 2531 } 2532 rte_free(tmp); 2533 break; 2534 2535 default: 2536 /* allow ops with invalid args to be pushed to 2537 * completion queue 2538 */ 2539 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2540 goto err_rsa; 2541 } 2542 2543 ret = 0; 2544 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2545 err_rsa: 2546 return ret; 2547 2548 } 2549 #else 2550 static int 2551 process_openssl_rsa_op(struct rte_crypto_op *cop, 2552 struct openssl_asym_session *sess) 2553 { 2554 int ret = 0; 2555 struct rte_crypto_asym_op *op = cop->asym; 2556 RSA *rsa = sess->u.r.rsa; 2557 uint32_t pad = (op->rsa.padding.type); 2558 uint8_t *tmp; 2559 2560 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2561 2562 switch (pad) { 2563 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 2564 pad = RSA_PKCS1_PADDING; 2565 break; 2566 case RTE_CRYPTO_RSA_PADDING_NONE: 2567 pad = RSA_NO_PADDING; 2568 break; 2569 default: 2570 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2571 OPENSSL_LOG(ERR, 2572 "rsa pad type not supported %d\n", pad); 2573 return 0; 2574 } 2575 2576 switch (op->rsa.op_type) { 2577 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 2578 ret = RSA_public_encrypt(op->rsa.message.length, 2579 op->rsa.message.data, 2580 op->rsa.cipher.data, 2581 rsa, 2582 pad); 2583 2584 if (ret > 0) 2585 op->rsa.cipher.length = ret; 2586 OPENSSL_LOG(DEBUG, 2587 "length of encrypted text %d\n", ret); 2588 break; 2589 2590 case RTE_CRYPTO_ASYM_OP_DECRYPT: 2591 ret = RSA_private_decrypt(op->rsa.cipher.length, 2592 op->rsa.cipher.data, 2593 op->rsa.message.data, 2594 rsa, 2595 pad); 2596 if (ret > 0) 2597 op->rsa.message.length = ret; 2598 break; 2599 2600 case RTE_CRYPTO_ASYM_OP_SIGN: 2601 ret = RSA_private_encrypt(op->rsa.message.length, 2602 op->rsa.message.data, 2603 op->rsa.sign.data, 2604 rsa, 2605 pad); 2606 if (ret > 0) 2607 op->rsa.sign.length = ret; 2608 break; 2609 2610 case RTE_CRYPTO_ASYM_OP_VERIFY: 2611 tmp = rte_malloc(NULL, op->rsa.sign.length, 0); 2612 if (tmp == NULL) { 2613 OPENSSL_LOG(ERR, "Memory allocation failed"); 2614 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2615 break; 2616 } 2617 ret = RSA_public_decrypt(op->rsa.sign.length, 2618 op->rsa.sign.data, 2619 tmp, 2620 rsa, 2621 pad); 2622 2623 OPENSSL_LOG(DEBUG, 2624 "Length of public_decrypt %d " 2625 "length of message %zd\n", 2626 ret, op->rsa.message.length); 2627 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data, 2628 op->rsa.message.length))) { 2629 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 2630 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2631 } 2632 rte_free(tmp); 2633 break; 2634 2635 default: 2636 /* allow ops with invalid args to be pushed to 2637 * completion queue 2638 */ 2639 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2640 break; 2641 } 2642 2643 if (ret < 0) 2644 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 2645 2646 return 0; 2647 } 2648 #endif 2649 2650 static int 2651 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op, 2652 struct openssl_asym_session *sess) 2653 { 2654 int retval = 0; 2655 2656 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2657 2658 switch (sess->xfrm_type) { 2659 case RTE_CRYPTO_ASYM_XFORM_RSA: 2660 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2661 retval = process_openssl_rsa_op_evp(op, sess); 2662 # else 2663 retval = process_openssl_rsa_op(op, sess); 2664 #endif 2665 break; 2666 case RTE_CRYPTO_ASYM_XFORM_MODEX: 2667 retval = process_openssl_modexp_op(op, sess); 2668 break; 2669 case RTE_CRYPTO_ASYM_XFORM_MODINV: 2670 retval = process_openssl_modinv_op(op, sess); 2671 break; 2672 case RTE_CRYPTO_ASYM_XFORM_DH: 2673 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2674 retval = process_openssl_dh_op_evp(op, sess); 2675 # else 2676 retval = process_openssl_dh_op(op, sess); 2677 #endif 2678 break; 2679 case RTE_CRYPTO_ASYM_XFORM_DSA: 2680 #if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2681 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 2682 retval = process_openssl_dsa_sign_op_evp(op, sess); 2683 else if (op->asym->dsa.op_type == 2684 RTE_CRYPTO_ASYM_OP_VERIFY) 2685 retval = 2686 process_openssl_dsa_verify_op_evp(op, sess); 2687 #else 2688 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 2689 retval = process_openssl_dsa_sign_op(op, sess); 2690 else if (op->asym->dsa.op_type == 2691 RTE_CRYPTO_ASYM_OP_VERIFY) 2692 retval = 2693 process_openssl_dsa_verify_op(op, sess); 2694 else 2695 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2696 #endif 2697 break; 2698 default: 2699 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2700 break; 2701 } 2702 if (!retval) { 2703 /* op processed so push to completion queue as processed */ 2704 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 2705 if (retval) 2706 /* return error if failed to put in completion queue */ 2707 retval = -1; 2708 } 2709 2710 return retval; 2711 } 2712 2713 static void 2714 copy_plaintext(struct rte_mbuf *m_src, struct rte_mbuf *m_dst, 2715 struct rte_crypto_op *op) 2716 { 2717 uint8_t *p_src, *p_dst; 2718 2719 p_src = rte_pktmbuf_mtod(m_src, uint8_t *); 2720 p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *); 2721 2722 /** 2723 * Copy the content between cipher offset and auth offset 2724 * for generating correct digest. 2725 */ 2726 if (op->sym->cipher.data.offset > op->sym->auth.data.offset) 2727 memcpy(p_dst + op->sym->auth.data.offset, 2728 p_src + op->sym->auth.data.offset, 2729 op->sym->cipher.data.offset - 2730 op->sym->auth.data.offset); 2731 } 2732 2733 /** Process crypto operation for mbuf */ 2734 static int 2735 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 2736 struct openssl_session *sess) 2737 { 2738 struct rte_mbuf *msrc, *mdst; 2739 int retval; 2740 2741 msrc = op->sym->m_src; 2742 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 2743 2744 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2745 2746 switch (sess->chain_order) { 2747 case OPENSSL_CHAIN_ONLY_CIPHER: 2748 process_openssl_cipher_op(op, sess, msrc, mdst); 2749 break; 2750 case OPENSSL_CHAIN_ONLY_AUTH: 2751 process_openssl_auth_op(qp, op, sess, msrc, mdst); 2752 break; 2753 case OPENSSL_CHAIN_CIPHER_AUTH: 2754 process_openssl_cipher_op(op, sess, msrc, mdst); 2755 /* OOP */ 2756 if (msrc != mdst) 2757 copy_plaintext(msrc, mdst, op); 2758 process_openssl_auth_op(qp, op, sess, mdst, mdst); 2759 break; 2760 case OPENSSL_CHAIN_AUTH_CIPHER: 2761 process_openssl_auth_op(qp, op, sess, msrc, mdst); 2762 process_openssl_cipher_op(op, sess, msrc, mdst); 2763 break; 2764 case OPENSSL_CHAIN_COMBINED: 2765 process_openssl_combined_op(op, sess, msrc, mdst); 2766 break; 2767 case OPENSSL_CHAIN_CIPHER_BPI: 2768 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 2769 break; 2770 default: 2771 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 2772 break; 2773 } 2774 2775 /* Free session if a session-less crypto op */ 2776 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 2777 openssl_reset_session(sess); 2778 memset(sess, 0, sizeof(struct openssl_session)); 2779 memset(op->sym->session, 0, 2780 rte_cryptodev_sym_get_existing_header_session_size( 2781 op->sym->session)); 2782 rte_mempool_put(qp->sess_mp_priv, sess); 2783 rte_mempool_put(qp->sess_mp, op->sym->session); 2784 op->sym->session = NULL; 2785 } 2786 2787 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 2788 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2789 2790 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 2791 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 2792 else 2793 retval = -1; 2794 2795 return retval; 2796 } 2797 2798 /* 2799 *------------------------------------------------------------------------------ 2800 * PMD Framework 2801 *------------------------------------------------------------------------------ 2802 */ 2803 2804 /** Enqueue burst */ 2805 static uint16_t 2806 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 2807 uint16_t nb_ops) 2808 { 2809 void *sess; 2810 struct openssl_qp *qp = queue_pair; 2811 int i, retval; 2812 2813 for (i = 0; i < nb_ops; i++) { 2814 sess = get_session(qp, ops[i]); 2815 if (unlikely(sess == NULL)) 2816 goto enqueue_err; 2817 2818 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) 2819 retval = process_op(qp, ops[i], 2820 (struct openssl_session *) sess); 2821 else 2822 retval = process_asym_op(qp, ops[i], 2823 (struct openssl_asym_session *) sess); 2824 if (unlikely(retval < 0)) 2825 goto enqueue_err; 2826 } 2827 2828 qp->stats.enqueued_count += i; 2829 return i; 2830 2831 enqueue_err: 2832 qp->stats.enqueue_err_count++; 2833 return i; 2834 } 2835 2836 /** Dequeue burst */ 2837 static uint16_t 2838 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 2839 uint16_t nb_ops) 2840 { 2841 struct openssl_qp *qp = queue_pair; 2842 2843 unsigned int nb_dequeued = 0; 2844 2845 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 2846 (void **)ops, nb_ops, NULL); 2847 qp->stats.dequeued_count += nb_dequeued; 2848 2849 return nb_dequeued; 2850 } 2851 2852 /** Create OPENSSL crypto device */ 2853 static int 2854 cryptodev_openssl_create(const char *name, 2855 struct rte_vdev_device *vdev, 2856 struct rte_cryptodev_pmd_init_params *init_params) 2857 { 2858 struct rte_cryptodev *dev; 2859 struct openssl_private *internals; 2860 2861 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 2862 if (dev == NULL) { 2863 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 2864 goto init_error; 2865 } 2866 2867 dev->driver_id = cryptodev_driver_id; 2868 dev->dev_ops = rte_openssl_pmd_ops; 2869 2870 /* register rx/tx burst functions for data path */ 2871 dev->dequeue_burst = openssl_pmd_dequeue_burst; 2872 dev->enqueue_burst = openssl_pmd_enqueue_burst; 2873 2874 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 2875 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 2876 RTE_CRYPTODEV_FF_CPU_AESNI | 2877 RTE_CRYPTODEV_FF_IN_PLACE_SGL | 2878 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 2879 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | 2880 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO | 2881 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP | 2882 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT | 2883 RTE_CRYPTODEV_FF_SYM_SESSIONLESS; 2884 2885 internals = dev->data->dev_private; 2886 2887 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 2888 2889 rte_cryptodev_pmd_probing_finish(dev); 2890 2891 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2892 /* Load legacy provider 2893 * Some algorithms are no longer available in earlier version of openssl, 2894 * unless the legacy provider explicitly loaded. e.g. DES 2895 */ 2896 ossl_legacy_provider_load(); 2897 # endif 2898 return 0; 2899 2900 init_error: 2901 OPENSSL_LOG(ERR, "driver %s: create failed", 2902 init_params->name); 2903 2904 cryptodev_openssl_remove(vdev); 2905 return -EFAULT; 2906 } 2907 2908 /** Initialise OPENSSL crypto device */ 2909 static int 2910 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 2911 { 2912 struct rte_cryptodev_pmd_init_params init_params = { 2913 "", 2914 sizeof(struct openssl_private), 2915 rte_socket_id(), 2916 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 2917 }; 2918 const char *name; 2919 const char *input_args; 2920 2921 name = rte_vdev_device_name(vdev); 2922 if (name == NULL) 2923 return -EINVAL; 2924 input_args = rte_vdev_device_args(vdev); 2925 2926 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 2927 2928 return cryptodev_openssl_create(name, vdev, &init_params); 2929 } 2930 2931 /** Uninitialise OPENSSL crypto device */ 2932 static int 2933 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 2934 { 2935 struct rte_cryptodev *cryptodev; 2936 const char *name; 2937 2938 name = rte_vdev_device_name(vdev); 2939 if (name == NULL) 2940 return -EINVAL; 2941 2942 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 2943 if (cryptodev == NULL) 2944 return -ENODEV; 2945 2946 # if (OPENSSL_VERSION_NUMBER >= 0x30000000L) 2947 ossl_legacy_provider_unload(); 2948 # endif 2949 return rte_cryptodev_pmd_destroy(cryptodev); 2950 } 2951 2952 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 2953 .probe = cryptodev_openssl_probe, 2954 .remove = cryptodev_openssl_remove 2955 }; 2956 2957 static struct cryptodev_driver openssl_crypto_drv; 2958 2959 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 2960 cryptodev_openssl_pmd_drv); 2961 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 2962 "max_nb_queue_pairs=<int> " 2963 "socket_id=<int>"); 2964 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 2965 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 2966 RTE_LOG_REGISTER_DEFAULT(openssl_logtype_driver, INFO); 2967