1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_common.h> 6 #include <rte_hexdump.h> 7 #include <rte_cryptodev.h> 8 #include <rte_cryptodev_pmd.h> 9 #include <rte_bus_vdev.h> 10 #include <rte_malloc.h> 11 #include <rte_cpuflags.h> 12 13 #include <openssl/hmac.h> 14 #include <openssl/evp.h> 15 16 #include "openssl_pmd_private.h" 17 #include "compat.h" 18 19 #define DES_BLOCK_SIZE 8 20 21 int openssl_logtype_driver; 22 static uint8_t cryptodev_driver_id; 23 24 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 25 static HMAC_CTX *HMAC_CTX_new(void) 26 { 27 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 28 29 if (ctx != NULL) 30 HMAC_CTX_init(ctx); 31 return ctx; 32 } 33 34 static void HMAC_CTX_free(HMAC_CTX *ctx) 35 { 36 if (ctx != NULL) { 37 HMAC_CTX_cleanup(ctx); 38 OPENSSL_free(ctx); 39 } 40 } 41 #endif 42 43 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 44 45 /*----------------------------------------------------------------------------*/ 46 47 /** 48 * Increment counter by 1 49 * Counter is 64 bit array, big-endian 50 */ 51 static void 52 ctr_inc(uint8_t *ctr) 53 { 54 uint64_t *ctr64 = (uint64_t *)ctr; 55 56 *ctr64 = __builtin_bswap64(*ctr64); 57 (*ctr64)++; 58 *ctr64 = __builtin_bswap64(*ctr64); 59 } 60 61 /* 62 *------------------------------------------------------------------------------ 63 * Session Prepare 64 *------------------------------------------------------------------------------ 65 */ 66 67 /** Get xform chain order */ 68 static enum openssl_chain_order 69 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 70 { 71 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 72 73 if (xform != NULL) { 74 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 75 if (xform->next == NULL) 76 res = OPENSSL_CHAIN_ONLY_AUTH; 77 else if (xform->next->type == 78 RTE_CRYPTO_SYM_XFORM_CIPHER) 79 res = OPENSSL_CHAIN_AUTH_CIPHER; 80 } 81 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 82 if (xform->next == NULL) 83 res = OPENSSL_CHAIN_ONLY_CIPHER; 84 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 85 res = OPENSSL_CHAIN_CIPHER_AUTH; 86 } 87 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 88 res = OPENSSL_CHAIN_COMBINED; 89 } 90 91 return res; 92 } 93 94 /** Get session cipher key from input cipher key */ 95 static void 96 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key) 97 { 98 memcpy(session_key, input_key, keylen); 99 } 100 101 /** Get key ede 24 bytes standard from input key */ 102 static int 103 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede) 104 { 105 int res = 0; 106 107 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 108 switch (keylen) { 109 case 24: 110 memcpy(key_ede, key, 24); 111 break; 112 case 16: 113 /* K3 = K1 */ 114 memcpy(key_ede, key, 16); 115 memcpy(key_ede + 16, key, 8); 116 break; 117 case 8: 118 /* K1 = K2 = K3 (DES compatibility) */ 119 memcpy(key_ede, key, 8); 120 memcpy(key_ede + 8, key, 8); 121 memcpy(key_ede + 16, key, 8); 122 break; 123 default: 124 OPENSSL_LOG(ERR, "Unsupported key size"); 125 res = -EINVAL; 126 } 127 128 return res; 129 } 130 131 /** Get adequate openssl function for input cipher algorithm */ 132 static uint8_t 133 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 134 const EVP_CIPHER **algo) 135 { 136 int res = 0; 137 138 if (algo != NULL) { 139 switch (sess_algo) { 140 case RTE_CRYPTO_CIPHER_3DES_CBC: 141 switch (keylen) { 142 case 8: 143 *algo = EVP_des_cbc(); 144 break; 145 case 16: 146 *algo = EVP_des_ede_cbc(); 147 break; 148 case 24: 149 *algo = EVP_des_ede3_cbc(); 150 break; 151 default: 152 res = -EINVAL; 153 } 154 break; 155 case RTE_CRYPTO_CIPHER_3DES_CTR: 156 break; 157 case RTE_CRYPTO_CIPHER_AES_CBC: 158 switch (keylen) { 159 case 16: 160 *algo = EVP_aes_128_cbc(); 161 break; 162 case 24: 163 *algo = EVP_aes_192_cbc(); 164 break; 165 case 32: 166 *algo = EVP_aes_256_cbc(); 167 break; 168 default: 169 res = -EINVAL; 170 } 171 break; 172 case RTE_CRYPTO_CIPHER_AES_CTR: 173 switch (keylen) { 174 case 16: 175 *algo = EVP_aes_128_ctr(); 176 break; 177 case 24: 178 *algo = EVP_aes_192_ctr(); 179 break; 180 case 32: 181 *algo = EVP_aes_256_ctr(); 182 break; 183 default: 184 res = -EINVAL; 185 } 186 break; 187 default: 188 res = -EINVAL; 189 break; 190 } 191 } else { 192 res = -EINVAL; 193 } 194 195 return res; 196 } 197 198 /** Get adequate openssl function for input auth algorithm */ 199 static uint8_t 200 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 201 const EVP_MD **algo) 202 { 203 int res = 0; 204 205 if (algo != NULL) { 206 switch (sessalgo) { 207 case RTE_CRYPTO_AUTH_MD5: 208 case RTE_CRYPTO_AUTH_MD5_HMAC: 209 *algo = EVP_md5(); 210 break; 211 case RTE_CRYPTO_AUTH_SHA1: 212 case RTE_CRYPTO_AUTH_SHA1_HMAC: 213 *algo = EVP_sha1(); 214 break; 215 case RTE_CRYPTO_AUTH_SHA224: 216 case RTE_CRYPTO_AUTH_SHA224_HMAC: 217 *algo = EVP_sha224(); 218 break; 219 case RTE_CRYPTO_AUTH_SHA256: 220 case RTE_CRYPTO_AUTH_SHA256_HMAC: 221 *algo = EVP_sha256(); 222 break; 223 case RTE_CRYPTO_AUTH_SHA384: 224 case RTE_CRYPTO_AUTH_SHA384_HMAC: 225 *algo = EVP_sha384(); 226 break; 227 case RTE_CRYPTO_AUTH_SHA512: 228 case RTE_CRYPTO_AUTH_SHA512_HMAC: 229 *algo = EVP_sha512(); 230 break; 231 default: 232 res = -EINVAL; 233 break; 234 } 235 } else { 236 res = -EINVAL; 237 } 238 239 return res; 240 } 241 242 /** Get adequate openssl function for input cipher algorithm */ 243 static uint8_t 244 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 245 const EVP_CIPHER **algo) 246 { 247 int res = 0; 248 249 if (algo != NULL) { 250 switch (sess_algo) { 251 case RTE_CRYPTO_AEAD_AES_GCM: 252 switch (keylen) { 253 case 16: 254 *algo = EVP_aes_128_gcm(); 255 break; 256 case 24: 257 *algo = EVP_aes_192_gcm(); 258 break; 259 case 32: 260 *algo = EVP_aes_256_gcm(); 261 break; 262 default: 263 res = -EINVAL; 264 } 265 break; 266 case RTE_CRYPTO_AEAD_AES_CCM: 267 switch (keylen) { 268 case 16: 269 *algo = EVP_aes_128_ccm(); 270 break; 271 case 24: 272 *algo = EVP_aes_192_ccm(); 273 break; 274 case 32: 275 *algo = EVP_aes_256_ccm(); 276 break; 277 default: 278 res = -EINVAL; 279 } 280 break; 281 default: 282 res = -EINVAL; 283 break; 284 } 285 } else { 286 res = -EINVAL; 287 } 288 289 return res; 290 } 291 292 /* Set session AEAD encryption parameters */ 293 static int 294 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 295 enum rte_crypto_aead_algorithm algo, 296 uint8_t tag_len, const uint8_t *key) 297 { 298 int iv_type = 0; 299 unsigned int do_ccm; 300 301 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 302 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 303 304 /* Select AEAD algo */ 305 switch (algo) { 306 case RTE_CRYPTO_AEAD_AES_GCM: 307 iv_type = EVP_CTRL_GCM_SET_IVLEN; 308 if (tag_len != 16) 309 return -EINVAL; 310 do_ccm = 0; 311 break; 312 case RTE_CRYPTO_AEAD_AES_CCM: 313 iv_type = EVP_CTRL_CCM_SET_IVLEN; 314 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 315 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 316 return -EINVAL; 317 do_ccm = 1; 318 break; 319 default: 320 return -ENOTSUP; 321 } 322 323 sess->cipher.mode = OPENSSL_CIPHER_LIB; 324 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 325 326 if (get_aead_algo(algo, sess->cipher.key.length, 327 &sess->cipher.evp_algo) != 0) 328 return -EINVAL; 329 330 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 331 332 sess->chain_order = OPENSSL_CHAIN_COMBINED; 333 334 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 335 NULL, NULL, NULL) <= 0) 336 return -EINVAL; 337 338 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, 339 NULL) <= 0) 340 return -EINVAL; 341 342 if (do_ccm) 343 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 344 tag_len, NULL); 345 346 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 347 return -EINVAL; 348 349 return 0; 350 } 351 352 /* Set session AEAD decryption parameters */ 353 static int 354 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 355 enum rte_crypto_aead_algorithm algo, 356 uint8_t tag_len, const uint8_t *key) 357 { 358 int iv_type = 0; 359 unsigned int do_ccm = 0; 360 361 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 362 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 363 364 /* Select AEAD algo */ 365 switch (algo) { 366 case RTE_CRYPTO_AEAD_AES_GCM: 367 iv_type = EVP_CTRL_GCM_SET_IVLEN; 368 if (tag_len != 16) 369 return -EINVAL; 370 break; 371 case RTE_CRYPTO_AEAD_AES_CCM: 372 iv_type = EVP_CTRL_CCM_SET_IVLEN; 373 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 374 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 375 return -EINVAL; 376 do_ccm = 1; 377 break; 378 default: 379 return -ENOTSUP; 380 } 381 382 sess->cipher.mode = OPENSSL_CIPHER_LIB; 383 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 384 385 if (get_aead_algo(algo, sess->cipher.key.length, 386 &sess->cipher.evp_algo) != 0) 387 return -EINVAL; 388 389 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 390 391 sess->chain_order = OPENSSL_CHAIN_COMBINED; 392 393 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 394 NULL, NULL, NULL) <= 0) 395 return -EINVAL; 396 397 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, 398 sess->iv.length, NULL) <= 0) 399 return -EINVAL; 400 401 if (do_ccm) 402 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 403 tag_len, NULL); 404 405 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 406 return -EINVAL; 407 408 return 0; 409 } 410 411 /** Set session cipher parameters */ 412 static int 413 openssl_set_session_cipher_parameters(struct openssl_session *sess, 414 const struct rte_crypto_sym_xform *xform) 415 { 416 /* Select cipher direction */ 417 sess->cipher.direction = xform->cipher.op; 418 /* Select cipher key */ 419 sess->cipher.key.length = xform->cipher.key.length; 420 421 /* Set IV parameters */ 422 sess->iv.offset = xform->cipher.iv.offset; 423 sess->iv.length = xform->cipher.iv.length; 424 425 /* Select cipher algo */ 426 switch (xform->cipher.algo) { 427 case RTE_CRYPTO_CIPHER_3DES_CBC: 428 case RTE_CRYPTO_CIPHER_AES_CBC: 429 case RTE_CRYPTO_CIPHER_AES_CTR: 430 sess->cipher.mode = OPENSSL_CIPHER_LIB; 431 sess->cipher.algo = xform->cipher.algo; 432 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 433 434 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 435 &sess->cipher.evp_algo) != 0) 436 return -EINVAL; 437 438 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 439 sess->cipher.key.data); 440 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 441 if (EVP_EncryptInit_ex(sess->cipher.ctx, 442 sess->cipher.evp_algo, 443 NULL, xform->cipher.key.data, 444 NULL) != 1) { 445 return -EINVAL; 446 } 447 } else if (sess->cipher.direction == 448 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 449 if (EVP_DecryptInit_ex(sess->cipher.ctx, 450 sess->cipher.evp_algo, 451 NULL, xform->cipher.key.data, 452 NULL) != 1) { 453 return -EINVAL; 454 } 455 } 456 457 break; 458 459 case RTE_CRYPTO_CIPHER_3DES_CTR: 460 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 461 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 462 463 if (get_cipher_key_ede(xform->cipher.key.data, 464 sess->cipher.key.length, 465 sess->cipher.key.data) != 0) 466 return -EINVAL; 467 break; 468 469 case RTE_CRYPTO_CIPHER_DES_CBC: 470 sess->cipher.algo = xform->cipher.algo; 471 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 472 sess->cipher.evp_algo = EVP_des_cbc(); 473 474 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 475 sess->cipher.key.data); 476 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 477 if (EVP_EncryptInit_ex(sess->cipher.ctx, 478 sess->cipher.evp_algo, 479 NULL, xform->cipher.key.data, 480 NULL) != 1) { 481 return -EINVAL; 482 } 483 } else if (sess->cipher.direction == 484 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 485 if (EVP_DecryptInit_ex(sess->cipher.ctx, 486 sess->cipher.evp_algo, 487 NULL, xform->cipher.key.data, 488 NULL) != 1) { 489 return -EINVAL; 490 } 491 } 492 493 break; 494 495 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 496 sess->cipher.algo = xform->cipher.algo; 497 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 498 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 499 sess->cipher.evp_algo = EVP_des_cbc(); 500 501 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 502 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 503 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 504 NULL, xform->cipher.key.data, 0) != 1) 505 return -EINVAL; 506 507 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 508 sess->cipher.key.data); 509 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 510 if (EVP_EncryptInit_ex(sess->cipher.ctx, 511 sess->cipher.evp_algo, 512 NULL, xform->cipher.key.data, 513 NULL) != 1) { 514 return -EINVAL; 515 } 516 } else if (sess->cipher.direction == 517 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 518 if (EVP_DecryptInit_ex(sess->cipher.ctx, 519 sess->cipher.evp_algo, 520 NULL, xform->cipher.key.data, 521 NULL) != 1) { 522 return -EINVAL; 523 } 524 } 525 526 break; 527 default: 528 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 529 return -ENOTSUP; 530 } 531 532 return 0; 533 } 534 535 /* Set session auth parameters */ 536 static int 537 openssl_set_session_auth_parameters(struct openssl_session *sess, 538 const struct rte_crypto_sym_xform *xform) 539 { 540 /* Select auth generate/verify */ 541 sess->auth.operation = xform->auth.op; 542 sess->auth.algo = xform->auth.algo; 543 544 sess->auth.digest_length = xform->auth.digest_length; 545 546 /* Select auth algo */ 547 switch (xform->auth.algo) { 548 case RTE_CRYPTO_AUTH_AES_GMAC: 549 /* 550 * OpenSSL requires GMAC to be a GCM operation 551 * with no cipher data length 552 */ 553 sess->cipher.key.length = xform->auth.key.length; 554 555 /* Set IV parameters */ 556 sess->iv.offset = xform->auth.iv.offset; 557 sess->iv.length = xform->auth.iv.length; 558 559 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 560 return openssl_set_sess_aead_enc_param(sess, 561 RTE_CRYPTO_AEAD_AES_GCM, 562 xform->auth.digest_length, 563 xform->auth.key.data); 564 else 565 return openssl_set_sess_aead_dec_param(sess, 566 RTE_CRYPTO_AEAD_AES_GCM, 567 xform->auth.digest_length, 568 xform->auth.key.data); 569 break; 570 571 case RTE_CRYPTO_AUTH_MD5: 572 case RTE_CRYPTO_AUTH_SHA1: 573 case RTE_CRYPTO_AUTH_SHA224: 574 case RTE_CRYPTO_AUTH_SHA256: 575 case RTE_CRYPTO_AUTH_SHA384: 576 case RTE_CRYPTO_AUTH_SHA512: 577 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 578 if (get_auth_algo(xform->auth.algo, 579 &sess->auth.auth.evp_algo) != 0) 580 return -EINVAL; 581 sess->auth.auth.ctx = EVP_MD_CTX_create(); 582 break; 583 584 case RTE_CRYPTO_AUTH_MD5_HMAC: 585 case RTE_CRYPTO_AUTH_SHA1_HMAC: 586 case RTE_CRYPTO_AUTH_SHA224_HMAC: 587 case RTE_CRYPTO_AUTH_SHA256_HMAC: 588 case RTE_CRYPTO_AUTH_SHA384_HMAC: 589 case RTE_CRYPTO_AUTH_SHA512_HMAC: 590 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 591 sess->auth.hmac.ctx = HMAC_CTX_new(); 592 if (get_auth_algo(xform->auth.algo, 593 &sess->auth.hmac.evp_algo) != 0) 594 return -EINVAL; 595 596 if (HMAC_Init_ex(sess->auth.hmac.ctx, 597 xform->auth.key.data, 598 xform->auth.key.length, 599 sess->auth.hmac.evp_algo, NULL) != 1) 600 return -EINVAL; 601 break; 602 603 default: 604 return -ENOTSUP; 605 } 606 607 return 0; 608 } 609 610 /* Set session AEAD parameters */ 611 static int 612 openssl_set_session_aead_parameters(struct openssl_session *sess, 613 const struct rte_crypto_sym_xform *xform) 614 { 615 /* Select cipher key */ 616 sess->cipher.key.length = xform->aead.key.length; 617 618 /* Set IV parameters */ 619 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 620 /* 621 * For AES-CCM, the actual IV is placed 622 * one byte after the start of the IV field, 623 * according to the API. 624 */ 625 sess->iv.offset = xform->aead.iv.offset + 1; 626 else 627 sess->iv.offset = xform->aead.iv.offset; 628 629 sess->iv.length = xform->aead.iv.length; 630 631 sess->auth.aad_length = xform->aead.aad_length; 632 sess->auth.digest_length = xform->aead.digest_length; 633 634 sess->aead_algo = xform->aead.algo; 635 /* Select cipher direction */ 636 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 637 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 638 xform->aead.digest_length, xform->aead.key.data); 639 else 640 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 641 xform->aead.digest_length, xform->aead.key.data); 642 } 643 644 /** Parse crypto xform chain and set private session parameters */ 645 int 646 openssl_set_session_parameters(struct openssl_session *sess, 647 const struct rte_crypto_sym_xform *xform) 648 { 649 const struct rte_crypto_sym_xform *cipher_xform = NULL; 650 const struct rte_crypto_sym_xform *auth_xform = NULL; 651 const struct rte_crypto_sym_xform *aead_xform = NULL; 652 int ret; 653 654 sess->chain_order = openssl_get_chain_order(xform); 655 switch (sess->chain_order) { 656 case OPENSSL_CHAIN_ONLY_CIPHER: 657 cipher_xform = xform; 658 break; 659 case OPENSSL_CHAIN_ONLY_AUTH: 660 auth_xform = xform; 661 break; 662 case OPENSSL_CHAIN_CIPHER_AUTH: 663 cipher_xform = xform; 664 auth_xform = xform->next; 665 break; 666 case OPENSSL_CHAIN_AUTH_CIPHER: 667 auth_xform = xform; 668 cipher_xform = xform->next; 669 break; 670 case OPENSSL_CHAIN_COMBINED: 671 aead_xform = xform; 672 break; 673 default: 674 return -EINVAL; 675 } 676 677 /* Default IV length = 0 */ 678 sess->iv.length = 0; 679 680 /* cipher_xform must be check before auth_xform */ 681 if (cipher_xform) { 682 ret = openssl_set_session_cipher_parameters( 683 sess, cipher_xform); 684 if (ret != 0) { 685 OPENSSL_LOG(ERR, 686 "Invalid/unsupported cipher parameters"); 687 return ret; 688 } 689 } 690 691 if (auth_xform) { 692 ret = openssl_set_session_auth_parameters(sess, auth_xform); 693 if (ret != 0) { 694 OPENSSL_LOG(ERR, 695 "Invalid/unsupported auth parameters"); 696 return ret; 697 } 698 } 699 700 if (aead_xform) { 701 ret = openssl_set_session_aead_parameters(sess, aead_xform); 702 if (ret != 0) { 703 OPENSSL_LOG(ERR, 704 "Invalid/unsupported AEAD parameters"); 705 return ret; 706 } 707 } 708 709 return 0; 710 } 711 712 /** Reset private session parameters */ 713 void 714 openssl_reset_session(struct openssl_session *sess) 715 { 716 EVP_CIPHER_CTX_free(sess->cipher.ctx); 717 718 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 719 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 720 721 switch (sess->auth.mode) { 722 case OPENSSL_AUTH_AS_AUTH: 723 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 724 break; 725 case OPENSSL_AUTH_AS_HMAC: 726 EVP_PKEY_free(sess->auth.hmac.pkey); 727 HMAC_CTX_free(sess->auth.hmac.ctx); 728 break; 729 default: 730 break; 731 } 732 } 733 734 /** Provide session for operation */ 735 static void * 736 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 737 { 738 struct openssl_session *sess = NULL; 739 struct openssl_asym_session *asym_sess = NULL; 740 741 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 742 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 743 /* get existing session */ 744 if (likely(op->sym->session != NULL)) 745 sess = (struct openssl_session *) 746 get_sym_session_private_data( 747 op->sym->session, 748 cryptodev_driver_id); 749 } else { 750 if (likely(op->asym->session != NULL)) 751 asym_sess = (struct openssl_asym_session *) 752 get_asym_session_private_data( 753 op->asym->session, 754 cryptodev_driver_id); 755 if (asym_sess == NULL) 756 op->status = 757 RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 758 return asym_sess; 759 } 760 } else { 761 /* sessionless asymmetric not supported */ 762 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) 763 return NULL; 764 765 /* provide internal session */ 766 void *_sess = rte_cryptodev_sym_session_create(qp->sess_mp); 767 void *_sess_private_data = NULL; 768 769 if (_sess == NULL) 770 return NULL; 771 772 if (rte_mempool_get(qp->sess_mp_priv, 773 (void **)&_sess_private_data)) 774 return NULL; 775 776 sess = (struct openssl_session *)_sess_private_data; 777 778 if (unlikely(openssl_set_session_parameters(sess, 779 op->sym->xform) != 0)) { 780 rte_mempool_put(qp->sess_mp, _sess); 781 rte_mempool_put(qp->sess_mp_priv, _sess_private_data); 782 sess = NULL; 783 } 784 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 785 set_sym_session_private_data(op->sym->session, 786 cryptodev_driver_id, _sess_private_data); 787 } 788 789 if (sess == NULL) 790 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 791 792 return sess; 793 } 794 795 /* 796 *------------------------------------------------------------------------------ 797 * Process Operations 798 *------------------------------------------------------------------------------ 799 */ 800 static inline int 801 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 802 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 803 { 804 struct rte_mbuf *m; 805 int dstlen; 806 int l, n = srclen; 807 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 808 809 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 810 m = m->next) 811 offset -= rte_pktmbuf_data_len(m); 812 813 if (m == 0) 814 return -1; 815 816 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 817 if (inplace) 818 *dst = src; 819 820 l = rte_pktmbuf_data_len(m) - offset; 821 if (srclen <= l) { 822 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 823 return -1; 824 *dst += l; 825 return 0; 826 } 827 828 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 829 return -1; 830 831 *dst += dstlen; 832 n -= l; 833 834 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 835 uint8_t diff = l - dstlen, rem; 836 837 src = rte_pktmbuf_mtod(m, uint8_t *); 838 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 839 if (diff && inplace) { 840 rem = RTE_MIN(l, 841 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 842 if (EVP_EncryptUpdate(ctx, temp, 843 &dstlen, src, rem) <= 0) 844 return -1; 845 n -= rem; 846 rte_memcpy(*dst, temp, diff); 847 rte_memcpy(src, temp + diff, rem); 848 src += rem; 849 l -= rem; 850 } 851 if (inplace) 852 *dst = src; 853 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 854 return -1; 855 *dst += dstlen; 856 n -= l; 857 } 858 859 return 0; 860 } 861 862 static inline int 863 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 864 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx, uint8_t inplace) 865 { 866 struct rte_mbuf *m; 867 int dstlen; 868 int l, n = srclen; 869 uint8_t *src, temp[EVP_CIPHER_CTX_block_size(ctx)]; 870 871 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 872 m = m->next) 873 offset -= rte_pktmbuf_data_len(m); 874 875 if (m == 0) 876 return -1; 877 878 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 879 if (inplace) 880 *dst = src; 881 882 l = rte_pktmbuf_data_len(m) - offset; 883 if (srclen <= l) { 884 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 885 return -1; 886 *dst += l; 887 return 0; 888 } 889 890 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 891 return -1; 892 893 *dst += dstlen; 894 n -= l; 895 896 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 897 uint8_t diff = l - dstlen, rem; 898 899 src = rte_pktmbuf_mtod(m, uint8_t *); 900 l = RTE_MIN(rte_pktmbuf_data_len(m), n); 901 if (diff && inplace) { 902 rem = RTE_MIN(l, 903 (EVP_CIPHER_CTX_block_size(ctx) - diff)); 904 if (EVP_DecryptUpdate(ctx, temp, 905 &dstlen, src, rem) <= 0) 906 return -1; 907 n -= rem; 908 rte_memcpy(*dst, temp, diff); 909 rte_memcpy(src, temp + diff, rem); 910 src += rem; 911 l -= rem; 912 } 913 if (inplace) 914 *dst = src; 915 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 916 return -1; 917 *dst += dstlen; 918 n -= l; 919 } 920 921 return 0; 922 } 923 924 /** Process standard openssl cipher encryption */ 925 static int 926 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 927 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 928 uint8_t inplace) 929 { 930 int totlen; 931 932 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 933 goto process_cipher_encrypt_err; 934 935 EVP_CIPHER_CTX_set_padding(ctx, 0); 936 937 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 938 srclen, ctx, inplace)) 939 goto process_cipher_encrypt_err; 940 941 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 942 goto process_cipher_encrypt_err; 943 944 return 0; 945 946 process_cipher_encrypt_err: 947 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 948 return -EINVAL; 949 } 950 951 /** Process standard openssl cipher encryption */ 952 static int 953 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 954 uint8_t *iv, int srclen, 955 EVP_CIPHER_CTX *ctx) 956 { 957 uint8_t i; 958 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 959 int encrypted_ivlen; 960 961 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 962 iv, DES_BLOCK_SIZE) <= 0) 963 goto process_cipher_encrypt_err; 964 965 for (i = 0; i < srclen; i++) 966 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 967 968 return 0; 969 970 process_cipher_encrypt_err: 971 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 972 return -EINVAL; 973 } 974 /** Process standard openssl cipher decryption */ 975 static int 976 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 977 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx, 978 uint8_t inplace) 979 { 980 int totlen; 981 982 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 983 goto process_cipher_decrypt_err; 984 985 EVP_CIPHER_CTX_set_padding(ctx, 0); 986 987 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 988 srclen, ctx, inplace)) 989 goto process_cipher_decrypt_err; 990 991 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 992 goto process_cipher_decrypt_err; 993 return 0; 994 995 process_cipher_decrypt_err: 996 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 997 return -EINVAL; 998 } 999 1000 /** Process cipher des 3 ctr encryption, decryption algorithm */ 1001 static int 1002 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 1003 int offset, uint8_t *iv, uint8_t *key, int srclen, 1004 EVP_CIPHER_CTX *ctx) 1005 { 1006 uint8_t ebuf[8], ctr[8]; 1007 int unused, n; 1008 struct rte_mbuf *m; 1009 uint8_t *src; 1010 int l; 1011 1012 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1013 m = m->next) 1014 offset -= rte_pktmbuf_data_len(m); 1015 1016 if (m == 0) 1017 goto process_cipher_des3ctr_err; 1018 1019 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1020 l = rte_pktmbuf_data_len(m) - offset; 1021 1022 /* We use 3DES encryption also for decryption. 1023 * IV is not important for 3DES ecb 1024 */ 1025 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 1026 goto process_cipher_des3ctr_err; 1027 1028 memcpy(ctr, iv, 8); 1029 1030 for (n = 0; n < srclen; n++) { 1031 if (n % 8 == 0) { 1032 if (EVP_EncryptUpdate(ctx, 1033 (unsigned char *)&ebuf, &unused, 1034 (const unsigned char *)&ctr, 8) <= 0) 1035 goto process_cipher_des3ctr_err; 1036 ctr_inc(ctr); 1037 } 1038 dst[n] = *(src++) ^ ebuf[n % 8]; 1039 1040 l--; 1041 if (!l) { 1042 m = m->next; 1043 if (m) { 1044 src = rte_pktmbuf_mtod(m, uint8_t *); 1045 l = rte_pktmbuf_data_len(m); 1046 } 1047 } 1048 } 1049 1050 return 0; 1051 1052 process_cipher_des3ctr_err: 1053 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 1054 return -EINVAL; 1055 } 1056 1057 /** Process AES-GCM encrypt algorithm */ 1058 static int 1059 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1060 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1061 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1062 { 1063 int len = 0, unused = 0; 1064 uint8_t empty[] = {}; 1065 1066 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1067 goto process_auth_encryption_gcm_err; 1068 1069 if (aadlen > 0) 1070 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1071 goto process_auth_encryption_gcm_err; 1072 1073 if (srclen > 0) 1074 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1075 srclen, ctx, 0)) 1076 goto process_auth_encryption_gcm_err; 1077 1078 /* Workaround open ssl bug in version less then 1.0.1f */ 1079 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1080 goto process_auth_encryption_gcm_err; 1081 1082 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1083 goto process_auth_encryption_gcm_err; 1084 1085 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1086 goto process_auth_encryption_gcm_err; 1087 1088 return 0; 1089 1090 process_auth_encryption_gcm_err: 1091 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1092 return -EINVAL; 1093 } 1094 1095 /** Process AES-CCM encrypt algorithm */ 1096 static int 1097 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1098 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1099 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1100 { 1101 int len = 0; 1102 1103 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1104 goto process_auth_encryption_ccm_err; 1105 1106 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1107 goto process_auth_encryption_ccm_err; 1108 1109 if (aadlen > 0) 1110 /* 1111 * For AES-CCM, the actual AAD is placed 1112 * 18 bytes after the start of the AAD field, 1113 * according to the API. 1114 */ 1115 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1116 goto process_auth_encryption_ccm_err; 1117 1118 if (srclen > 0) 1119 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1120 srclen, ctx, 0)) 1121 goto process_auth_encryption_ccm_err; 1122 1123 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1124 goto process_auth_encryption_ccm_err; 1125 1126 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1127 goto process_auth_encryption_ccm_err; 1128 1129 return 0; 1130 1131 process_auth_encryption_ccm_err: 1132 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1133 return -EINVAL; 1134 } 1135 1136 /** Process AES-GCM decrypt algorithm */ 1137 static int 1138 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1139 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1140 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1141 { 1142 int len = 0, unused = 0; 1143 uint8_t empty[] = {}; 1144 1145 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1146 goto process_auth_decryption_gcm_err; 1147 1148 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1149 goto process_auth_decryption_gcm_err; 1150 1151 if (aadlen > 0) 1152 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1153 goto process_auth_decryption_gcm_err; 1154 1155 if (srclen > 0) 1156 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1157 srclen, ctx, 0)) 1158 goto process_auth_decryption_gcm_err; 1159 1160 /* Workaround open ssl bug in version less then 1.0.1f */ 1161 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1162 goto process_auth_decryption_gcm_err; 1163 1164 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1165 return -EFAULT; 1166 1167 return 0; 1168 1169 process_auth_decryption_gcm_err: 1170 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1171 return -EINVAL; 1172 } 1173 1174 /** Process AES-CCM decrypt algorithm */ 1175 static int 1176 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1177 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1178 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1179 EVP_CIPHER_CTX *ctx) 1180 { 1181 int len = 0; 1182 1183 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1184 goto process_auth_decryption_ccm_err; 1185 1186 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1187 goto process_auth_decryption_ccm_err; 1188 1189 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1190 goto process_auth_decryption_ccm_err; 1191 1192 if (aadlen > 0) 1193 /* 1194 * For AES-CCM, the actual AAD is placed 1195 * 18 bytes after the start of the AAD field, 1196 * according to the API. 1197 */ 1198 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1199 goto process_auth_decryption_ccm_err; 1200 1201 if (srclen > 0) 1202 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1203 srclen, ctx, 0)) 1204 return -EFAULT; 1205 1206 return 0; 1207 1208 process_auth_decryption_ccm_err: 1209 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1210 return -EINVAL; 1211 } 1212 1213 /** Process standard openssl auth algorithms */ 1214 static int 1215 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1216 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1217 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1218 { 1219 size_t dstlen; 1220 struct rte_mbuf *m; 1221 int l, n = srclen; 1222 uint8_t *src; 1223 1224 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1225 m = m->next) 1226 offset -= rte_pktmbuf_data_len(m); 1227 1228 if (m == 0) 1229 goto process_auth_err; 1230 1231 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1232 goto process_auth_err; 1233 1234 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1235 1236 l = rte_pktmbuf_data_len(m) - offset; 1237 if (srclen <= l) { 1238 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1239 goto process_auth_err; 1240 goto process_auth_final; 1241 } 1242 1243 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1244 goto process_auth_err; 1245 1246 n -= l; 1247 1248 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1249 src = rte_pktmbuf_mtod(m, uint8_t *); 1250 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1251 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1252 goto process_auth_err; 1253 n -= l; 1254 } 1255 1256 process_auth_final: 1257 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1258 goto process_auth_err; 1259 return 0; 1260 1261 process_auth_err: 1262 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1263 return -EINVAL; 1264 } 1265 1266 /** Process standard openssl auth algorithms with hmac */ 1267 static int 1268 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1269 int srclen, HMAC_CTX *ctx) 1270 { 1271 unsigned int dstlen; 1272 struct rte_mbuf *m; 1273 int l, n = srclen; 1274 uint8_t *src; 1275 1276 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1277 m = m->next) 1278 offset -= rte_pktmbuf_data_len(m); 1279 1280 if (m == 0) 1281 goto process_auth_err; 1282 1283 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1284 1285 l = rte_pktmbuf_data_len(m) - offset; 1286 if (srclen <= l) { 1287 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1288 goto process_auth_err; 1289 goto process_auth_final; 1290 } 1291 1292 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1293 goto process_auth_err; 1294 1295 n -= l; 1296 1297 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1298 src = rte_pktmbuf_mtod(m, uint8_t *); 1299 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1300 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1301 goto process_auth_err; 1302 n -= l; 1303 } 1304 1305 process_auth_final: 1306 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1307 goto process_auth_err; 1308 1309 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1310 goto process_auth_err; 1311 1312 return 0; 1313 1314 process_auth_err: 1315 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1316 return -EINVAL; 1317 } 1318 1319 /*----------------------------------------------------------------------------*/ 1320 1321 /** Process auth/cipher combined operation */ 1322 static void 1323 process_openssl_combined_op 1324 (struct rte_crypto_op *op, struct openssl_session *sess, 1325 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1326 { 1327 /* cipher */ 1328 uint8_t *dst = NULL, *iv, *tag, *aad; 1329 int srclen, aadlen, status = -1; 1330 uint32_t offset; 1331 uint8_t taglen; 1332 EVP_CIPHER_CTX *ctx_copy; 1333 1334 /* 1335 * Segmented destination buffer is not supported for 1336 * encryption/decryption 1337 */ 1338 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1339 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1340 return; 1341 } 1342 1343 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1344 sess->iv.offset); 1345 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1346 srclen = 0; 1347 offset = op->sym->auth.data.offset; 1348 aadlen = op->sym->auth.data.length; 1349 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1350 op->sym->auth.data.offset); 1351 tag = op->sym->auth.digest.data; 1352 if (tag == NULL) 1353 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1354 offset + aadlen); 1355 } else { 1356 srclen = op->sym->aead.data.length; 1357 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1358 op->sym->aead.data.offset); 1359 offset = op->sym->aead.data.offset; 1360 aad = op->sym->aead.aad.data; 1361 aadlen = sess->auth.aad_length; 1362 tag = op->sym->aead.digest.data; 1363 if (tag == NULL) 1364 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1365 offset + srclen); 1366 } 1367 1368 taglen = sess->auth.digest_length; 1369 ctx_copy = EVP_CIPHER_CTX_new(); 1370 EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx); 1371 1372 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1373 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1374 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1375 status = process_openssl_auth_encryption_gcm( 1376 mbuf_src, offset, srclen, 1377 aad, aadlen, iv, 1378 dst, tag, ctx_copy); 1379 else 1380 status = process_openssl_auth_encryption_ccm( 1381 mbuf_src, offset, srclen, 1382 aad, aadlen, iv, 1383 dst, tag, taglen, ctx_copy); 1384 1385 } else { 1386 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1387 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1388 status = process_openssl_auth_decryption_gcm( 1389 mbuf_src, offset, srclen, 1390 aad, aadlen, iv, 1391 dst, tag, ctx_copy); 1392 else 1393 status = process_openssl_auth_decryption_ccm( 1394 mbuf_src, offset, srclen, 1395 aad, aadlen, iv, 1396 dst, tag, taglen, ctx_copy); 1397 } 1398 1399 EVP_CIPHER_CTX_free(ctx_copy); 1400 if (status != 0) { 1401 if (status == (-EFAULT) && 1402 sess->auth.operation == 1403 RTE_CRYPTO_AUTH_OP_VERIFY) 1404 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1405 else 1406 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1407 } 1408 } 1409 1410 /** Process cipher operation */ 1411 static void 1412 process_openssl_cipher_op 1413 (struct rte_crypto_op *op, struct openssl_session *sess, 1414 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1415 { 1416 uint8_t *dst, *iv; 1417 int srclen, status; 1418 uint8_t inplace = (mbuf_src == mbuf_dst) ? 1 : 0; 1419 EVP_CIPHER_CTX *ctx_copy; 1420 1421 /* 1422 * Segmented OOP destination buffer is not supported for encryption/ 1423 * decryption. In case of des3ctr, even inplace segmented buffers are 1424 * not supported. 1425 */ 1426 if (!rte_pktmbuf_is_contiguous(mbuf_dst) && 1427 (!inplace || sess->cipher.mode != OPENSSL_CIPHER_LIB)) { 1428 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1429 return; 1430 } 1431 1432 srclen = op->sym->cipher.data.length; 1433 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1434 op->sym->cipher.data.offset); 1435 1436 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1437 sess->iv.offset); 1438 ctx_copy = EVP_CIPHER_CTX_new(); 1439 EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx); 1440 1441 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1442 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1443 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1444 op->sym->cipher.data.offset, iv, 1445 srclen, ctx_copy, inplace); 1446 else 1447 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1448 op->sym->cipher.data.offset, iv, 1449 srclen, ctx_copy, inplace); 1450 else 1451 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1452 op->sym->cipher.data.offset, iv, 1453 sess->cipher.key.data, srclen, 1454 ctx_copy); 1455 1456 EVP_CIPHER_CTX_free(ctx_copy); 1457 if (status != 0) 1458 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1459 } 1460 1461 /** Process cipher operation */ 1462 static void 1463 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1464 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1465 struct rte_mbuf *mbuf_dst) 1466 { 1467 uint8_t *src, *dst, *iv; 1468 uint8_t block_size, last_block_len; 1469 int srclen, status = 0; 1470 1471 srclen = op->sym->cipher.data.length; 1472 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1473 op->sym->cipher.data.offset); 1474 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1475 op->sym->cipher.data.offset); 1476 1477 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1478 sess->iv.offset); 1479 1480 block_size = DES_BLOCK_SIZE; 1481 1482 last_block_len = srclen % block_size; 1483 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1484 /* Encrypt only with ECB mode XOR IV */ 1485 if (srclen < block_size) { 1486 status = process_openssl_cipher_bpi_encrypt(src, dst, 1487 iv, srclen, 1488 sess->cipher.bpi_ctx); 1489 } else { 1490 srclen -= last_block_len; 1491 /* Encrypt with the block aligned stream with CBC mode */ 1492 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1493 op->sym->cipher.data.offset, iv, 1494 srclen, sess->cipher.ctx, 0); 1495 if (last_block_len) { 1496 /* Point at last block */ 1497 dst += srclen; 1498 /* 1499 * IV is the last encrypted block from 1500 * the previous operation 1501 */ 1502 iv = dst - block_size; 1503 src += srclen; 1504 srclen = last_block_len; 1505 /* Encrypt the last frame with ECB mode */ 1506 status |= process_openssl_cipher_bpi_encrypt(src, 1507 dst, iv, 1508 srclen, sess->cipher.bpi_ctx); 1509 } 1510 } 1511 } else { 1512 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1513 if (srclen < block_size) { 1514 status = process_openssl_cipher_bpi_encrypt(src, dst, 1515 iv, 1516 srclen, 1517 sess->cipher.bpi_ctx); 1518 } else { 1519 if (last_block_len) { 1520 /* Point at last block */ 1521 dst += srclen - last_block_len; 1522 src += srclen - last_block_len; 1523 /* 1524 * IV is the last full block 1525 */ 1526 iv = src - block_size; 1527 /* 1528 * Decrypt the last frame with ECB mode 1529 * (encrypt, as it is the same operation) 1530 */ 1531 status = process_openssl_cipher_bpi_encrypt(src, 1532 dst, iv, 1533 last_block_len, sess->cipher.bpi_ctx); 1534 /* Prepare parameters for CBC mode op */ 1535 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1536 sess->iv.offset); 1537 dst += last_block_len - srclen; 1538 srclen -= last_block_len; 1539 } 1540 1541 /* Decrypt with CBC mode */ 1542 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1543 op->sym->cipher.data.offset, iv, 1544 srclen, sess->cipher.ctx, 0); 1545 } 1546 } 1547 1548 if (status != 0) 1549 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1550 } 1551 1552 /** Process auth operation */ 1553 static void 1554 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1555 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1556 struct rte_mbuf *mbuf_dst) 1557 { 1558 uint8_t *dst; 1559 int srclen, status; 1560 EVP_MD_CTX *ctx_a; 1561 HMAC_CTX *ctx_h; 1562 1563 srclen = op->sym->auth.data.length; 1564 1565 dst = qp->temp_digest; 1566 1567 switch (sess->auth.mode) { 1568 case OPENSSL_AUTH_AS_AUTH: 1569 ctx_a = EVP_MD_CTX_create(); 1570 EVP_MD_CTX_copy_ex(ctx_a, sess->auth.auth.ctx); 1571 status = process_openssl_auth(mbuf_src, dst, 1572 op->sym->auth.data.offset, NULL, NULL, srclen, 1573 ctx_a, sess->auth.auth.evp_algo); 1574 EVP_MD_CTX_destroy(ctx_a); 1575 break; 1576 case OPENSSL_AUTH_AS_HMAC: 1577 ctx_h = HMAC_CTX_new(); 1578 HMAC_CTX_copy(ctx_h, sess->auth.hmac.ctx); 1579 status = process_openssl_auth_hmac(mbuf_src, dst, 1580 op->sym->auth.data.offset, srclen, 1581 ctx_h); 1582 HMAC_CTX_free(ctx_h); 1583 break; 1584 default: 1585 status = -1; 1586 break; 1587 } 1588 1589 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1590 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data, 1591 sess->auth.digest_length) != 0) { 1592 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1593 } 1594 } else { 1595 uint8_t *auth_dst; 1596 1597 auth_dst = op->sym->auth.digest.data; 1598 if (auth_dst == NULL) 1599 auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1600 op->sym->auth.data.offset + 1601 op->sym->auth.data.length); 1602 memcpy(auth_dst, dst, sess->auth.digest_length); 1603 } 1604 1605 if (status != 0) 1606 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1607 } 1608 1609 /* process dsa sign operation */ 1610 static int 1611 process_openssl_dsa_sign_op(struct rte_crypto_op *cop, 1612 struct openssl_asym_session *sess) 1613 { 1614 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1615 DSA *dsa = sess->u.s.dsa; 1616 DSA_SIG *sign = NULL; 1617 1618 sign = DSA_do_sign(op->message.data, 1619 op->message.length, 1620 dsa); 1621 1622 if (sign == NULL) { 1623 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 1624 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1625 } else { 1626 const BIGNUM *r = NULL, *s = NULL; 1627 get_dsa_sign(sign, &r, &s); 1628 1629 op->r.length = BN_bn2bin(r, op->r.data); 1630 op->s.length = BN_bn2bin(s, op->s.data); 1631 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1632 } 1633 1634 DSA_SIG_free(sign); 1635 1636 return 0; 1637 } 1638 1639 /* process dsa verify operation */ 1640 static int 1641 process_openssl_dsa_verify_op(struct rte_crypto_op *cop, 1642 struct openssl_asym_session *sess) 1643 { 1644 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1645 DSA *dsa = sess->u.s.dsa; 1646 int ret; 1647 DSA_SIG *sign = DSA_SIG_new(); 1648 BIGNUM *r = NULL, *s = NULL; 1649 BIGNUM *pub_key = NULL; 1650 1651 if (sign == NULL) { 1652 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 1653 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1654 return -1; 1655 } 1656 1657 r = BN_bin2bn(op->r.data, 1658 op->r.length, 1659 r); 1660 s = BN_bin2bn(op->s.data, 1661 op->s.length, 1662 s); 1663 pub_key = BN_bin2bn(op->y.data, 1664 op->y.length, 1665 pub_key); 1666 if (!r || !s || !pub_key) { 1667 BN_free(r); 1668 BN_free(s); 1669 BN_free(pub_key); 1670 1671 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1672 return -1; 1673 } 1674 set_dsa_sign(sign, r, s); 1675 set_dsa_pub_key(dsa, pub_key); 1676 1677 ret = DSA_do_verify(op->message.data, 1678 op->message.length, 1679 sign, 1680 dsa); 1681 1682 if (ret != 1) 1683 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1684 else 1685 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1686 1687 DSA_SIG_free(sign); 1688 1689 return 0; 1690 } 1691 1692 /* process dh operation */ 1693 static int 1694 process_openssl_dh_op(struct rte_crypto_op *cop, 1695 struct openssl_asym_session *sess) 1696 { 1697 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 1698 DH *dh_key = sess->u.dh.dh_key; 1699 BIGNUM *priv_key = NULL; 1700 int ret = 0; 1701 1702 if (sess->u.dh.key_op & 1703 (1 << RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE)) { 1704 /* compute shared secret using peer public key 1705 * and current private key 1706 * shared secret = peer_key ^ priv_key mod p 1707 */ 1708 BIGNUM *peer_key = NULL; 1709 1710 /* copy private key and peer key and compute shared secret */ 1711 peer_key = BN_bin2bn(op->pub_key.data, 1712 op->pub_key.length, 1713 peer_key); 1714 if (peer_key == NULL) { 1715 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1716 return -1; 1717 } 1718 priv_key = BN_bin2bn(op->priv_key.data, 1719 op->priv_key.length, 1720 priv_key); 1721 if (priv_key == NULL) { 1722 BN_free(peer_key); 1723 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1724 return -1; 1725 } 1726 ret = set_dh_priv_key(dh_key, priv_key); 1727 if (ret) { 1728 OPENSSL_LOG(ERR, "Failed to set private key\n"); 1729 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1730 BN_free(peer_key); 1731 BN_free(priv_key); 1732 return 0; 1733 } 1734 1735 ret = DH_compute_key( 1736 op->shared_secret.data, 1737 peer_key, dh_key); 1738 if (ret < 0) { 1739 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1740 BN_free(peer_key); 1741 /* priv key is already loaded into dh, 1742 * let's not free that directly here. 1743 * DH_free() will auto free it later. 1744 */ 1745 return 0; 1746 } 1747 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1748 op->shared_secret.length = ret; 1749 BN_free(peer_key); 1750 return 0; 1751 } 1752 1753 /* 1754 * other options are public and private key generations. 1755 * 1756 * if user provides private key, 1757 * then first set DH with user provided private key 1758 */ 1759 if ((sess->u.dh.key_op & 1760 (1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)) && 1761 !(sess->u.dh.key_op & 1762 (1 << RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE))) { 1763 /* generate public key using user-provided private key 1764 * pub_key = g ^ priv_key mod p 1765 */ 1766 1767 /* load private key into DH */ 1768 priv_key = BN_bin2bn(op->priv_key.data, 1769 op->priv_key.length, 1770 priv_key); 1771 if (priv_key == NULL) { 1772 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1773 return -1; 1774 } 1775 ret = set_dh_priv_key(dh_key, priv_key); 1776 if (ret) { 1777 OPENSSL_LOG(ERR, "Failed to set private key\n"); 1778 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1779 BN_free(priv_key); 1780 return 0; 1781 } 1782 } 1783 1784 /* generate public and private key pair. 1785 * 1786 * if private key already set, generates only public key. 1787 * 1788 * if private key is not already set, then set it to random value 1789 * and update internal private key. 1790 */ 1791 if (!DH_generate_key(dh_key)) { 1792 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1793 return 0; 1794 } 1795 1796 if (sess->u.dh.key_op & (1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)) { 1797 const BIGNUM *pub_key = NULL; 1798 1799 OPENSSL_LOG(DEBUG, "%s:%d update public key\n", 1800 __func__, __LINE__); 1801 1802 /* get the generated keys */ 1803 get_dh_pub_key(dh_key, &pub_key); 1804 1805 /* output public key */ 1806 op->pub_key.length = BN_bn2bin(pub_key, 1807 op->pub_key.data); 1808 } 1809 1810 if (sess->u.dh.key_op & 1811 (1 << RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE)) { 1812 const BIGNUM *priv_key = NULL; 1813 1814 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", 1815 __func__, __LINE__); 1816 1817 /* get the generated keys */ 1818 get_dh_priv_key(dh_key, &priv_key); 1819 1820 /* provide generated private key back to user */ 1821 op->priv_key.length = BN_bn2bin(priv_key, 1822 op->priv_key.data); 1823 } 1824 1825 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1826 1827 return 0; 1828 } 1829 1830 /* process modinv operation */ 1831 static int 1832 process_openssl_modinv_op(struct rte_crypto_op *cop, 1833 struct openssl_asym_session *sess) 1834 { 1835 struct rte_crypto_asym_op *op = cop->asym; 1836 BIGNUM *base = BN_CTX_get(sess->u.m.ctx); 1837 BIGNUM *res = BN_CTX_get(sess->u.m.ctx); 1838 1839 if (unlikely(base == NULL || res == NULL)) { 1840 BN_free(base); 1841 BN_free(res); 1842 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1843 return -1; 1844 } 1845 1846 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 1847 op->modinv.base.length, base); 1848 1849 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) { 1850 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1851 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data); 1852 } else { 1853 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1854 } 1855 1856 BN_clear(res); 1857 BN_clear(base); 1858 1859 return 0; 1860 } 1861 1862 /* process modexp operation */ 1863 static int 1864 process_openssl_modexp_op(struct rte_crypto_op *cop, 1865 struct openssl_asym_session *sess) 1866 { 1867 struct rte_crypto_asym_op *op = cop->asym; 1868 BIGNUM *base = BN_CTX_get(sess->u.e.ctx); 1869 BIGNUM *res = BN_CTX_get(sess->u.e.ctx); 1870 1871 if (unlikely(base == NULL || res == NULL)) { 1872 BN_free(base); 1873 BN_free(res); 1874 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1875 return -1; 1876 } 1877 1878 base = BN_bin2bn((const unsigned char *)op->modex.base.data, 1879 op->modex.base.length, base); 1880 1881 if (BN_mod_exp(res, base, sess->u.e.exp, 1882 sess->u.e.mod, sess->u.e.ctx)) { 1883 op->modex.result.length = BN_bn2bin(res, op->modex.result.data); 1884 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1885 } else { 1886 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1887 } 1888 1889 BN_clear(res); 1890 BN_clear(base); 1891 1892 return 0; 1893 } 1894 1895 /* process rsa operations */ 1896 static int 1897 process_openssl_rsa_op(struct rte_crypto_op *cop, 1898 struct openssl_asym_session *sess) 1899 { 1900 int ret = 0; 1901 struct rte_crypto_asym_op *op = cop->asym; 1902 RSA *rsa = sess->u.r.rsa; 1903 uint32_t pad = (op->rsa.pad); 1904 uint8_t *tmp; 1905 1906 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1907 1908 switch (pad) { 1909 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 1910 pad = RSA_PKCS1_PADDING; 1911 break; 1912 case RTE_CRYPTO_RSA_PADDING_NONE: 1913 pad = RSA_NO_PADDING; 1914 break; 1915 default: 1916 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1917 OPENSSL_LOG(ERR, 1918 "rsa pad type not supported %d\n", pad); 1919 return 0; 1920 } 1921 1922 switch (op->rsa.op_type) { 1923 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 1924 ret = RSA_public_encrypt(op->rsa.message.length, 1925 op->rsa.message.data, 1926 op->rsa.cipher.data, 1927 rsa, 1928 pad); 1929 1930 if (ret > 0) 1931 op->rsa.cipher.length = ret; 1932 OPENSSL_LOG(DEBUG, 1933 "length of encrypted text %d\n", ret); 1934 break; 1935 1936 case RTE_CRYPTO_ASYM_OP_DECRYPT: 1937 ret = RSA_private_decrypt(op->rsa.cipher.length, 1938 op->rsa.cipher.data, 1939 op->rsa.message.data, 1940 rsa, 1941 pad); 1942 if (ret > 0) 1943 op->rsa.message.length = ret; 1944 break; 1945 1946 case RTE_CRYPTO_ASYM_OP_SIGN: 1947 ret = RSA_private_encrypt(op->rsa.message.length, 1948 op->rsa.message.data, 1949 op->rsa.sign.data, 1950 rsa, 1951 pad); 1952 if (ret > 0) 1953 op->rsa.sign.length = ret; 1954 break; 1955 1956 case RTE_CRYPTO_ASYM_OP_VERIFY: 1957 tmp = rte_malloc(NULL, op->rsa.sign.length, 0); 1958 if (tmp == NULL) { 1959 OPENSSL_LOG(ERR, "Memory allocation failed"); 1960 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1961 break; 1962 } 1963 ret = RSA_public_decrypt(op->rsa.sign.length, 1964 op->rsa.sign.data, 1965 tmp, 1966 rsa, 1967 pad); 1968 1969 OPENSSL_LOG(DEBUG, 1970 "Length of public_decrypt %d " 1971 "length of message %zd\n", 1972 ret, op->rsa.message.length); 1973 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data, 1974 op->rsa.message.length))) { 1975 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 1976 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1977 } 1978 rte_free(tmp); 1979 break; 1980 1981 default: 1982 /* allow ops with invalid args to be pushed to 1983 * completion queue 1984 */ 1985 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1986 break; 1987 } 1988 1989 if (ret < 0) 1990 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1991 1992 return 0; 1993 } 1994 1995 static int 1996 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1997 struct openssl_asym_session *sess) 1998 { 1999 int retval = 0; 2000 2001 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2002 2003 switch (sess->xfrm_type) { 2004 case RTE_CRYPTO_ASYM_XFORM_RSA: 2005 retval = process_openssl_rsa_op(op, sess); 2006 break; 2007 case RTE_CRYPTO_ASYM_XFORM_MODEX: 2008 retval = process_openssl_modexp_op(op, sess); 2009 break; 2010 case RTE_CRYPTO_ASYM_XFORM_MODINV: 2011 retval = process_openssl_modinv_op(op, sess); 2012 break; 2013 case RTE_CRYPTO_ASYM_XFORM_DH: 2014 retval = process_openssl_dh_op(op, sess); 2015 break; 2016 case RTE_CRYPTO_ASYM_XFORM_DSA: 2017 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 2018 retval = process_openssl_dsa_sign_op(op, sess); 2019 else if (op->asym->dsa.op_type == 2020 RTE_CRYPTO_ASYM_OP_VERIFY) 2021 retval = 2022 process_openssl_dsa_verify_op(op, sess); 2023 else 2024 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2025 break; 2026 default: 2027 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 2028 break; 2029 } 2030 if (!retval) { 2031 /* op processed so push to completion queue as processed */ 2032 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 2033 if (retval) 2034 /* return error if failed to put in completion queue */ 2035 retval = -1; 2036 } 2037 2038 return retval; 2039 } 2040 2041 static void 2042 copy_plaintext(struct rte_mbuf *m_src, struct rte_mbuf *m_dst, 2043 struct rte_crypto_op *op) 2044 { 2045 uint8_t *p_src, *p_dst; 2046 2047 p_src = rte_pktmbuf_mtod(m_src, uint8_t *); 2048 p_dst = rte_pktmbuf_mtod(m_dst, uint8_t *); 2049 2050 /** 2051 * Copy the content between cipher offset and auth offset 2052 * for generating correct digest. 2053 */ 2054 if (op->sym->cipher.data.offset > op->sym->auth.data.offset) 2055 memcpy(p_dst + op->sym->auth.data.offset, 2056 p_src + op->sym->auth.data.offset, 2057 op->sym->cipher.data.offset - 2058 op->sym->auth.data.offset); 2059 } 2060 2061 /** Process crypto operation for mbuf */ 2062 static int 2063 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 2064 struct openssl_session *sess) 2065 { 2066 struct rte_mbuf *msrc, *mdst; 2067 int retval; 2068 2069 msrc = op->sym->m_src; 2070 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 2071 2072 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2073 2074 switch (sess->chain_order) { 2075 case OPENSSL_CHAIN_ONLY_CIPHER: 2076 process_openssl_cipher_op(op, sess, msrc, mdst); 2077 break; 2078 case OPENSSL_CHAIN_ONLY_AUTH: 2079 process_openssl_auth_op(qp, op, sess, msrc, mdst); 2080 break; 2081 case OPENSSL_CHAIN_CIPHER_AUTH: 2082 process_openssl_cipher_op(op, sess, msrc, mdst); 2083 /* OOP */ 2084 if (msrc != mdst) 2085 copy_plaintext(msrc, mdst, op); 2086 process_openssl_auth_op(qp, op, sess, mdst, mdst); 2087 break; 2088 case OPENSSL_CHAIN_AUTH_CIPHER: 2089 process_openssl_auth_op(qp, op, sess, msrc, mdst); 2090 process_openssl_cipher_op(op, sess, msrc, mdst); 2091 break; 2092 case OPENSSL_CHAIN_COMBINED: 2093 process_openssl_combined_op(op, sess, msrc, mdst); 2094 break; 2095 case OPENSSL_CHAIN_CIPHER_BPI: 2096 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 2097 break; 2098 default: 2099 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 2100 break; 2101 } 2102 2103 /* Free session if a session-less crypto op */ 2104 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 2105 openssl_reset_session(sess); 2106 memset(sess, 0, sizeof(struct openssl_session)); 2107 memset(op->sym->session, 0, 2108 rte_cryptodev_sym_get_existing_header_session_size( 2109 op->sym->session)); 2110 rte_mempool_put(qp->sess_mp_priv, sess); 2111 rte_mempool_put(qp->sess_mp, op->sym->session); 2112 op->sym->session = NULL; 2113 } 2114 2115 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 2116 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2117 2118 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 2119 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 2120 else 2121 retval = -1; 2122 2123 return retval; 2124 } 2125 2126 /* 2127 *------------------------------------------------------------------------------ 2128 * PMD Framework 2129 *------------------------------------------------------------------------------ 2130 */ 2131 2132 /** Enqueue burst */ 2133 static uint16_t 2134 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 2135 uint16_t nb_ops) 2136 { 2137 void *sess; 2138 struct openssl_qp *qp = queue_pair; 2139 int i, retval; 2140 2141 for (i = 0; i < nb_ops; i++) { 2142 sess = get_session(qp, ops[i]); 2143 if (unlikely(sess == NULL)) 2144 goto enqueue_err; 2145 2146 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) 2147 retval = process_op(qp, ops[i], 2148 (struct openssl_session *) sess); 2149 else 2150 retval = process_asym_op(qp, ops[i], 2151 (struct openssl_asym_session *) sess); 2152 if (unlikely(retval < 0)) 2153 goto enqueue_err; 2154 } 2155 2156 qp->stats.enqueued_count += i; 2157 return i; 2158 2159 enqueue_err: 2160 qp->stats.enqueue_err_count++; 2161 return i; 2162 } 2163 2164 /** Dequeue burst */ 2165 static uint16_t 2166 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 2167 uint16_t nb_ops) 2168 { 2169 struct openssl_qp *qp = queue_pair; 2170 2171 unsigned int nb_dequeued = 0; 2172 2173 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 2174 (void **)ops, nb_ops, NULL); 2175 qp->stats.dequeued_count += nb_dequeued; 2176 2177 return nb_dequeued; 2178 } 2179 2180 /** Create OPENSSL crypto device */ 2181 static int 2182 cryptodev_openssl_create(const char *name, 2183 struct rte_vdev_device *vdev, 2184 struct rte_cryptodev_pmd_init_params *init_params) 2185 { 2186 struct rte_cryptodev *dev; 2187 struct openssl_private *internals; 2188 2189 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 2190 if (dev == NULL) { 2191 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 2192 goto init_error; 2193 } 2194 2195 dev->driver_id = cryptodev_driver_id; 2196 dev->dev_ops = rte_openssl_pmd_ops; 2197 2198 /* register rx/tx burst functions for data path */ 2199 dev->dequeue_burst = openssl_pmd_dequeue_burst; 2200 dev->enqueue_burst = openssl_pmd_enqueue_burst; 2201 2202 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 2203 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 2204 RTE_CRYPTODEV_FF_CPU_AESNI | 2205 RTE_CRYPTODEV_FF_IN_PLACE_SGL | 2206 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 2207 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | 2208 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO | 2209 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP | 2210 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT | 2211 RTE_CRYPTODEV_FF_SYM_SESSIONLESS; 2212 2213 internals = dev->data->dev_private; 2214 2215 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 2216 2217 return 0; 2218 2219 init_error: 2220 OPENSSL_LOG(ERR, "driver %s: create failed", 2221 init_params->name); 2222 2223 cryptodev_openssl_remove(vdev); 2224 return -EFAULT; 2225 } 2226 2227 /** Initialise OPENSSL crypto device */ 2228 static int 2229 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 2230 { 2231 struct rte_cryptodev_pmd_init_params init_params = { 2232 "", 2233 sizeof(struct openssl_private), 2234 rte_socket_id(), 2235 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 2236 }; 2237 const char *name; 2238 const char *input_args; 2239 2240 name = rte_vdev_device_name(vdev); 2241 if (name == NULL) 2242 return -EINVAL; 2243 input_args = rte_vdev_device_args(vdev); 2244 2245 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 2246 2247 return cryptodev_openssl_create(name, vdev, &init_params); 2248 } 2249 2250 /** Uninitialise OPENSSL crypto device */ 2251 static int 2252 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 2253 { 2254 struct rte_cryptodev *cryptodev; 2255 const char *name; 2256 2257 name = rte_vdev_device_name(vdev); 2258 if (name == NULL) 2259 return -EINVAL; 2260 2261 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 2262 if (cryptodev == NULL) 2263 return -ENODEV; 2264 2265 return rte_cryptodev_pmd_destroy(cryptodev); 2266 } 2267 2268 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 2269 .probe = cryptodev_openssl_probe, 2270 .remove = cryptodev_openssl_remove 2271 }; 2272 2273 static struct cryptodev_driver openssl_crypto_drv; 2274 2275 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 2276 cryptodev_openssl_pmd_drv); 2277 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 2278 "max_nb_queue_pairs=<int> " 2279 "socket_id=<int>"); 2280 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 2281 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 2282 2283 RTE_INIT(openssl_init_log) 2284 { 2285 openssl_logtype_driver = rte_log_register("pmd.crypto.openssl"); 2286 } 2287