1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_common.h> 6 #include <rte_hexdump.h> 7 #include <rte_cryptodev.h> 8 #include <rte_cryptodev_pmd.h> 9 #include <rte_bus_vdev.h> 10 #include <rte_malloc.h> 11 #include <rte_cpuflags.h> 12 13 #include <openssl/hmac.h> 14 #include <openssl/evp.h> 15 16 #include "openssl_pmd_private.h" 17 #include "compat.h" 18 19 #define DES_BLOCK_SIZE 8 20 21 static uint8_t cryptodev_driver_id; 22 23 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 24 static HMAC_CTX *HMAC_CTX_new(void) 25 { 26 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 27 28 if (ctx != NULL) 29 HMAC_CTX_init(ctx); 30 return ctx; 31 } 32 33 static void HMAC_CTX_free(HMAC_CTX *ctx) 34 { 35 if (ctx != NULL) { 36 HMAC_CTX_cleanup(ctx); 37 OPENSSL_free(ctx); 38 } 39 } 40 #endif 41 42 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 43 44 /*----------------------------------------------------------------------------*/ 45 46 /** 47 * Increment counter by 1 48 * Counter is 64 bit array, big-endian 49 */ 50 static void 51 ctr_inc(uint8_t *ctr) 52 { 53 uint64_t *ctr64 = (uint64_t *)ctr; 54 55 *ctr64 = __builtin_bswap64(*ctr64); 56 (*ctr64)++; 57 *ctr64 = __builtin_bswap64(*ctr64); 58 } 59 60 /* 61 *------------------------------------------------------------------------------ 62 * Session Prepare 63 *------------------------------------------------------------------------------ 64 */ 65 66 /** Get xform chain order */ 67 static enum openssl_chain_order 68 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 69 { 70 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 71 72 if (xform != NULL) { 73 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 74 if (xform->next == NULL) 75 res = OPENSSL_CHAIN_ONLY_AUTH; 76 else if (xform->next->type == 77 RTE_CRYPTO_SYM_XFORM_CIPHER) 78 res = OPENSSL_CHAIN_AUTH_CIPHER; 79 } 80 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 81 if (xform->next == NULL) 82 res = OPENSSL_CHAIN_ONLY_CIPHER; 83 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 84 res = OPENSSL_CHAIN_CIPHER_AUTH; 85 } 86 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 87 res = OPENSSL_CHAIN_COMBINED; 88 } 89 90 return res; 91 } 92 93 /** Get session cipher key from input cipher key */ 94 static void 95 get_cipher_key(const uint8_t *input_key, int keylen, uint8_t *session_key) 96 { 97 memcpy(session_key, input_key, keylen); 98 } 99 100 /** Get key ede 24 bytes standard from input key */ 101 static int 102 get_cipher_key_ede(const uint8_t *key, int keylen, uint8_t *key_ede) 103 { 104 int res = 0; 105 106 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 107 switch (keylen) { 108 case 24: 109 memcpy(key_ede, key, 24); 110 break; 111 case 16: 112 /* K3 = K1 */ 113 memcpy(key_ede, key, 16); 114 memcpy(key_ede + 16, key, 8); 115 break; 116 case 8: 117 /* K1 = K2 = K3 (DES compatibility) */ 118 memcpy(key_ede, key, 8); 119 memcpy(key_ede + 8, key, 8); 120 memcpy(key_ede + 16, key, 8); 121 break; 122 default: 123 OPENSSL_LOG(ERR, "Unsupported key size"); 124 res = -EINVAL; 125 } 126 127 return res; 128 } 129 130 /** Get adequate openssl function for input cipher algorithm */ 131 static uint8_t 132 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 133 const EVP_CIPHER **algo) 134 { 135 int res = 0; 136 137 if (algo != NULL) { 138 switch (sess_algo) { 139 case RTE_CRYPTO_CIPHER_3DES_CBC: 140 switch (keylen) { 141 case 8: 142 *algo = EVP_des_cbc(); 143 break; 144 case 16: 145 *algo = EVP_des_ede_cbc(); 146 break; 147 case 24: 148 *algo = EVP_des_ede3_cbc(); 149 break; 150 default: 151 res = -EINVAL; 152 } 153 break; 154 case RTE_CRYPTO_CIPHER_3DES_CTR: 155 break; 156 case RTE_CRYPTO_CIPHER_AES_CBC: 157 switch (keylen) { 158 case 16: 159 *algo = EVP_aes_128_cbc(); 160 break; 161 case 24: 162 *algo = EVP_aes_192_cbc(); 163 break; 164 case 32: 165 *algo = EVP_aes_256_cbc(); 166 break; 167 default: 168 res = -EINVAL; 169 } 170 break; 171 case RTE_CRYPTO_CIPHER_AES_CTR: 172 switch (keylen) { 173 case 16: 174 *algo = EVP_aes_128_ctr(); 175 break; 176 case 24: 177 *algo = EVP_aes_192_ctr(); 178 break; 179 case 32: 180 *algo = EVP_aes_256_ctr(); 181 break; 182 default: 183 res = -EINVAL; 184 } 185 break; 186 default: 187 res = -EINVAL; 188 break; 189 } 190 } else { 191 res = -EINVAL; 192 } 193 194 return res; 195 } 196 197 /** Get adequate openssl function for input auth algorithm */ 198 static uint8_t 199 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 200 const EVP_MD **algo) 201 { 202 int res = 0; 203 204 if (algo != NULL) { 205 switch (sessalgo) { 206 case RTE_CRYPTO_AUTH_MD5: 207 case RTE_CRYPTO_AUTH_MD5_HMAC: 208 *algo = EVP_md5(); 209 break; 210 case RTE_CRYPTO_AUTH_SHA1: 211 case RTE_CRYPTO_AUTH_SHA1_HMAC: 212 *algo = EVP_sha1(); 213 break; 214 case RTE_CRYPTO_AUTH_SHA224: 215 case RTE_CRYPTO_AUTH_SHA224_HMAC: 216 *algo = EVP_sha224(); 217 break; 218 case RTE_CRYPTO_AUTH_SHA256: 219 case RTE_CRYPTO_AUTH_SHA256_HMAC: 220 *algo = EVP_sha256(); 221 break; 222 case RTE_CRYPTO_AUTH_SHA384: 223 case RTE_CRYPTO_AUTH_SHA384_HMAC: 224 *algo = EVP_sha384(); 225 break; 226 case RTE_CRYPTO_AUTH_SHA512: 227 case RTE_CRYPTO_AUTH_SHA512_HMAC: 228 *algo = EVP_sha512(); 229 break; 230 default: 231 res = -EINVAL; 232 break; 233 } 234 } else { 235 res = -EINVAL; 236 } 237 238 return res; 239 } 240 241 /** Get adequate openssl function for input cipher algorithm */ 242 static uint8_t 243 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 244 const EVP_CIPHER **algo) 245 { 246 int res = 0; 247 248 if (algo != NULL) { 249 switch (sess_algo) { 250 case RTE_CRYPTO_AEAD_AES_GCM: 251 switch (keylen) { 252 case 16: 253 *algo = EVP_aes_128_gcm(); 254 break; 255 case 24: 256 *algo = EVP_aes_192_gcm(); 257 break; 258 case 32: 259 *algo = EVP_aes_256_gcm(); 260 break; 261 default: 262 res = -EINVAL; 263 } 264 break; 265 case RTE_CRYPTO_AEAD_AES_CCM: 266 switch (keylen) { 267 case 16: 268 *algo = EVP_aes_128_ccm(); 269 break; 270 case 24: 271 *algo = EVP_aes_192_ccm(); 272 break; 273 case 32: 274 *algo = EVP_aes_256_ccm(); 275 break; 276 default: 277 res = -EINVAL; 278 } 279 break; 280 default: 281 res = -EINVAL; 282 break; 283 } 284 } else { 285 res = -EINVAL; 286 } 287 288 return res; 289 } 290 291 /* Set session AEAD encryption parameters */ 292 static int 293 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 294 enum rte_crypto_aead_algorithm algo, 295 uint8_t tag_len, const uint8_t *key) 296 { 297 int iv_type = 0; 298 unsigned int do_ccm; 299 300 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 301 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 302 303 /* Select AEAD algo */ 304 switch (algo) { 305 case RTE_CRYPTO_AEAD_AES_GCM: 306 iv_type = EVP_CTRL_GCM_SET_IVLEN; 307 if (tag_len != 16) 308 return -EINVAL; 309 do_ccm = 0; 310 break; 311 case RTE_CRYPTO_AEAD_AES_CCM: 312 iv_type = EVP_CTRL_CCM_SET_IVLEN; 313 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 314 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 315 return -EINVAL; 316 do_ccm = 1; 317 break; 318 default: 319 return -ENOTSUP; 320 } 321 322 sess->cipher.mode = OPENSSL_CIPHER_LIB; 323 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 324 325 if (get_aead_algo(algo, sess->cipher.key.length, 326 &sess->cipher.evp_algo) != 0) 327 return -EINVAL; 328 329 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 330 331 sess->chain_order = OPENSSL_CHAIN_COMBINED; 332 333 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 334 NULL, NULL, NULL) <= 0) 335 return -EINVAL; 336 337 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, 338 NULL) <= 0) 339 return -EINVAL; 340 341 if (do_ccm) 342 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 343 tag_len, NULL); 344 345 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 346 return -EINVAL; 347 348 return 0; 349 } 350 351 /* Set session AEAD decryption parameters */ 352 static int 353 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 354 enum rte_crypto_aead_algorithm algo, 355 uint8_t tag_len, const uint8_t *key) 356 { 357 int iv_type = 0; 358 unsigned int do_ccm = 0; 359 360 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 361 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 362 363 /* Select AEAD algo */ 364 switch (algo) { 365 case RTE_CRYPTO_AEAD_AES_GCM: 366 iv_type = EVP_CTRL_GCM_SET_IVLEN; 367 if (tag_len != 16) 368 return -EINVAL; 369 break; 370 case RTE_CRYPTO_AEAD_AES_CCM: 371 iv_type = EVP_CTRL_CCM_SET_IVLEN; 372 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 373 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 374 return -EINVAL; 375 do_ccm = 1; 376 break; 377 default: 378 return -ENOTSUP; 379 } 380 381 sess->cipher.mode = OPENSSL_CIPHER_LIB; 382 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 383 384 if (get_aead_algo(algo, sess->cipher.key.length, 385 &sess->cipher.evp_algo) != 0) 386 return -EINVAL; 387 388 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 389 390 sess->chain_order = OPENSSL_CHAIN_COMBINED; 391 392 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 393 NULL, NULL, NULL) <= 0) 394 return -EINVAL; 395 396 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, 397 sess->iv.length, NULL) <= 0) 398 return -EINVAL; 399 400 if (do_ccm) 401 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 402 tag_len, NULL); 403 404 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 405 return -EINVAL; 406 407 return 0; 408 } 409 410 /** Set session cipher parameters */ 411 static int 412 openssl_set_session_cipher_parameters(struct openssl_session *sess, 413 const struct rte_crypto_sym_xform *xform) 414 { 415 /* Select cipher direction */ 416 sess->cipher.direction = xform->cipher.op; 417 /* Select cipher key */ 418 sess->cipher.key.length = xform->cipher.key.length; 419 420 /* Set IV parameters */ 421 sess->iv.offset = xform->cipher.iv.offset; 422 sess->iv.length = xform->cipher.iv.length; 423 424 /* Select cipher algo */ 425 switch (xform->cipher.algo) { 426 case RTE_CRYPTO_CIPHER_3DES_CBC: 427 case RTE_CRYPTO_CIPHER_AES_CBC: 428 case RTE_CRYPTO_CIPHER_AES_CTR: 429 sess->cipher.mode = OPENSSL_CIPHER_LIB; 430 sess->cipher.algo = xform->cipher.algo; 431 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 432 433 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 434 &sess->cipher.evp_algo) != 0) 435 return -EINVAL; 436 437 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 438 sess->cipher.key.data); 439 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 440 if (EVP_EncryptInit_ex(sess->cipher.ctx, 441 sess->cipher.evp_algo, 442 NULL, xform->cipher.key.data, 443 NULL) != 1) { 444 return -EINVAL; 445 } 446 } else if (sess->cipher.direction == 447 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 448 if (EVP_DecryptInit_ex(sess->cipher.ctx, 449 sess->cipher.evp_algo, 450 NULL, xform->cipher.key.data, 451 NULL) != 1) { 452 return -EINVAL; 453 } 454 } 455 456 break; 457 458 case RTE_CRYPTO_CIPHER_3DES_CTR: 459 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 460 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 461 462 if (get_cipher_key_ede(xform->cipher.key.data, 463 sess->cipher.key.length, 464 sess->cipher.key.data) != 0) 465 return -EINVAL; 466 break; 467 468 case RTE_CRYPTO_CIPHER_DES_CBC: 469 sess->cipher.algo = xform->cipher.algo; 470 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 471 sess->cipher.evp_algo = EVP_des_cbc(); 472 473 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 474 sess->cipher.key.data); 475 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 476 if (EVP_EncryptInit_ex(sess->cipher.ctx, 477 sess->cipher.evp_algo, 478 NULL, xform->cipher.key.data, 479 NULL) != 1) { 480 return -EINVAL; 481 } 482 } else if (sess->cipher.direction == 483 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 484 if (EVP_DecryptInit_ex(sess->cipher.ctx, 485 sess->cipher.evp_algo, 486 NULL, xform->cipher.key.data, 487 NULL) != 1) { 488 return -EINVAL; 489 } 490 } 491 492 break; 493 494 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 495 sess->cipher.algo = xform->cipher.algo; 496 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 497 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 498 sess->cipher.evp_algo = EVP_des_cbc(); 499 500 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 501 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 502 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 503 NULL, xform->cipher.key.data, 0) != 1) 504 return -EINVAL; 505 506 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 507 sess->cipher.key.data); 508 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 509 if (EVP_EncryptInit_ex(sess->cipher.ctx, 510 sess->cipher.evp_algo, 511 NULL, xform->cipher.key.data, 512 NULL) != 1) { 513 return -EINVAL; 514 } 515 } else if (sess->cipher.direction == 516 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 517 if (EVP_DecryptInit_ex(sess->cipher.ctx, 518 sess->cipher.evp_algo, 519 NULL, xform->cipher.key.data, 520 NULL) != 1) { 521 return -EINVAL; 522 } 523 } 524 525 break; 526 default: 527 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 528 return -ENOTSUP; 529 } 530 531 return 0; 532 } 533 534 /* Set session auth parameters */ 535 static int 536 openssl_set_session_auth_parameters(struct openssl_session *sess, 537 const struct rte_crypto_sym_xform *xform) 538 { 539 /* Select auth generate/verify */ 540 sess->auth.operation = xform->auth.op; 541 sess->auth.algo = xform->auth.algo; 542 543 sess->auth.digest_length = xform->auth.digest_length; 544 545 /* Select auth algo */ 546 switch (xform->auth.algo) { 547 case RTE_CRYPTO_AUTH_AES_GMAC: 548 /* 549 * OpenSSL requires GMAC to be a GCM operation 550 * with no cipher data length 551 */ 552 sess->cipher.key.length = xform->auth.key.length; 553 554 /* Set IV parameters */ 555 sess->iv.offset = xform->auth.iv.offset; 556 sess->iv.length = xform->auth.iv.length; 557 558 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 559 return openssl_set_sess_aead_enc_param(sess, 560 RTE_CRYPTO_AEAD_AES_GCM, 561 xform->auth.digest_length, 562 xform->auth.key.data); 563 else 564 return openssl_set_sess_aead_dec_param(sess, 565 RTE_CRYPTO_AEAD_AES_GCM, 566 xform->auth.digest_length, 567 xform->auth.key.data); 568 break; 569 570 case RTE_CRYPTO_AUTH_MD5: 571 case RTE_CRYPTO_AUTH_SHA1: 572 case RTE_CRYPTO_AUTH_SHA224: 573 case RTE_CRYPTO_AUTH_SHA256: 574 case RTE_CRYPTO_AUTH_SHA384: 575 case RTE_CRYPTO_AUTH_SHA512: 576 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 577 if (get_auth_algo(xform->auth.algo, 578 &sess->auth.auth.evp_algo) != 0) 579 return -EINVAL; 580 sess->auth.auth.ctx = EVP_MD_CTX_create(); 581 break; 582 583 case RTE_CRYPTO_AUTH_MD5_HMAC: 584 case RTE_CRYPTO_AUTH_SHA1_HMAC: 585 case RTE_CRYPTO_AUTH_SHA224_HMAC: 586 case RTE_CRYPTO_AUTH_SHA256_HMAC: 587 case RTE_CRYPTO_AUTH_SHA384_HMAC: 588 case RTE_CRYPTO_AUTH_SHA512_HMAC: 589 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 590 sess->auth.hmac.ctx = HMAC_CTX_new(); 591 if (get_auth_algo(xform->auth.algo, 592 &sess->auth.hmac.evp_algo) != 0) 593 return -EINVAL; 594 595 if (HMAC_Init_ex(sess->auth.hmac.ctx, 596 xform->auth.key.data, 597 xform->auth.key.length, 598 sess->auth.hmac.evp_algo, NULL) != 1) 599 return -EINVAL; 600 break; 601 602 default: 603 return -ENOTSUP; 604 } 605 606 return 0; 607 } 608 609 /* Set session AEAD parameters */ 610 static int 611 openssl_set_session_aead_parameters(struct openssl_session *sess, 612 const struct rte_crypto_sym_xform *xform) 613 { 614 /* Select cipher key */ 615 sess->cipher.key.length = xform->aead.key.length; 616 617 /* Set IV parameters */ 618 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 619 /* 620 * For AES-CCM, the actual IV is placed 621 * one byte after the start of the IV field, 622 * according to the API. 623 */ 624 sess->iv.offset = xform->aead.iv.offset + 1; 625 else 626 sess->iv.offset = xform->aead.iv.offset; 627 628 sess->iv.length = xform->aead.iv.length; 629 630 sess->auth.aad_length = xform->aead.aad_length; 631 sess->auth.digest_length = xform->aead.digest_length; 632 633 sess->aead_algo = xform->aead.algo; 634 /* Select cipher direction */ 635 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 636 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 637 xform->aead.digest_length, xform->aead.key.data); 638 else 639 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 640 xform->aead.digest_length, xform->aead.key.data); 641 } 642 643 /** Parse crypto xform chain and set private session parameters */ 644 int 645 openssl_set_session_parameters(struct openssl_session *sess, 646 const struct rte_crypto_sym_xform *xform) 647 { 648 const struct rte_crypto_sym_xform *cipher_xform = NULL; 649 const struct rte_crypto_sym_xform *auth_xform = NULL; 650 const struct rte_crypto_sym_xform *aead_xform = NULL; 651 int ret; 652 653 sess->chain_order = openssl_get_chain_order(xform); 654 switch (sess->chain_order) { 655 case OPENSSL_CHAIN_ONLY_CIPHER: 656 cipher_xform = xform; 657 break; 658 case OPENSSL_CHAIN_ONLY_AUTH: 659 auth_xform = xform; 660 break; 661 case OPENSSL_CHAIN_CIPHER_AUTH: 662 cipher_xform = xform; 663 auth_xform = xform->next; 664 break; 665 case OPENSSL_CHAIN_AUTH_CIPHER: 666 auth_xform = xform; 667 cipher_xform = xform->next; 668 break; 669 case OPENSSL_CHAIN_COMBINED: 670 aead_xform = xform; 671 break; 672 default: 673 return -EINVAL; 674 } 675 676 /* Default IV length = 0 */ 677 sess->iv.length = 0; 678 679 /* cipher_xform must be check before auth_xform */ 680 if (cipher_xform) { 681 ret = openssl_set_session_cipher_parameters( 682 sess, cipher_xform); 683 if (ret != 0) { 684 OPENSSL_LOG(ERR, 685 "Invalid/unsupported cipher parameters"); 686 return ret; 687 } 688 } 689 690 if (auth_xform) { 691 ret = openssl_set_session_auth_parameters(sess, auth_xform); 692 if (ret != 0) { 693 OPENSSL_LOG(ERR, 694 "Invalid/unsupported auth parameters"); 695 return ret; 696 } 697 } 698 699 if (aead_xform) { 700 ret = openssl_set_session_aead_parameters(sess, aead_xform); 701 if (ret != 0) { 702 OPENSSL_LOG(ERR, 703 "Invalid/unsupported AEAD parameters"); 704 return ret; 705 } 706 } 707 708 return 0; 709 } 710 711 /** Reset private session parameters */ 712 void 713 openssl_reset_session(struct openssl_session *sess) 714 { 715 EVP_CIPHER_CTX_free(sess->cipher.ctx); 716 717 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 718 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 719 720 switch (sess->auth.mode) { 721 case OPENSSL_AUTH_AS_AUTH: 722 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 723 break; 724 case OPENSSL_AUTH_AS_HMAC: 725 EVP_PKEY_free(sess->auth.hmac.pkey); 726 HMAC_CTX_free(sess->auth.hmac.ctx); 727 break; 728 default: 729 break; 730 } 731 } 732 733 /** Provide session for operation */ 734 static void * 735 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 736 { 737 struct openssl_session *sess = NULL; 738 struct openssl_asym_session *asym_sess = NULL; 739 740 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 741 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 742 /* get existing session */ 743 if (likely(op->sym->session != NULL)) 744 sess = (struct openssl_session *) 745 get_sym_session_private_data( 746 op->sym->session, 747 cryptodev_driver_id); 748 } else { 749 if (likely(op->asym->session != NULL)) 750 asym_sess = (struct openssl_asym_session *) 751 get_asym_session_private_data( 752 op->asym->session, 753 cryptodev_driver_id); 754 if (asym_sess == NULL) 755 op->status = 756 RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 757 return asym_sess; 758 } 759 } else { 760 /* sessionless asymmetric not supported */ 761 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) 762 return NULL; 763 764 /* provide internal session */ 765 void *_sess = NULL; 766 void *_sess_private_data = NULL; 767 768 if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) 769 return NULL; 770 771 if (rte_mempool_get(qp->sess_mp_priv, 772 (void **)&_sess_private_data)) 773 return NULL; 774 775 sess = (struct openssl_session *)_sess_private_data; 776 777 if (unlikely(openssl_set_session_parameters(sess, 778 op->sym->xform) != 0)) { 779 rte_mempool_put(qp->sess_mp, _sess); 780 rte_mempool_put(qp->sess_mp_priv, _sess_private_data); 781 sess = NULL; 782 } 783 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 784 set_sym_session_private_data(op->sym->session, 785 cryptodev_driver_id, _sess_private_data); 786 } 787 788 if (sess == NULL) 789 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 790 791 return sess; 792 } 793 794 /* 795 *------------------------------------------------------------------------------ 796 * Process Operations 797 *------------------------------------------------------------------------------ 798 */ 799 static inline int 800 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 801 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 802 { 803 struct rte_mbuf *m; 804 int dstlen; 805 int l, n = srclen; 806 uint8_t *src; 807 808 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 809 m = m->next) 810 offset -= rte_pktmbuf_data_len(m); 811 812 if (m == 0) 813 return -1; 814 815 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 816 817 l = rte_pktmbuf_data_len(m) - offset; 818 if (srclen <= l) { 819 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 820 return -1; 821 *dst += l; 822 return 0; 823 } 824 825 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 826 return -1; 827 828 *dst += dstlen; 829 n -= l; 830 831 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 832 src = rte_pktmbuf_mtod(m, uint8_t *); 833 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 834 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 835 return -1; 836 *dst += dstlen; 837 n -= l; 838 } 839 840 return 0; 841 } 842 843 static inline int 844 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 845 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 846 { 847 struct rte_mbuf *m; 848 int dstlen; 849 int l, n = srclen; 850 uint8_t *src; 851 852 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 853 m = m->next) 854 offset -= rte_pktmbuf_data_len(m); 855 856 if (m == 0) 857 return -1; 858 859 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 860 861 l = rte_pktmbuf_data_len(m) - offset; 862 if (srclen <= l) { 863 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 864 return -1; 865 *dst += l; 866 return 0; 867 } 868 869 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 870 return -1; 871 872 *dst += dstlen; 873 n -= l; 874 875 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 876 src = rte_pktmbuf_mtod(m, uint8_t *); 877 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 878 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 879 return -1; 880 *dst += dstlen; 881 n -= l; 882 } 883 884 return 0; 885 } 886 887 /** Process standard openssl cipher encryption */ 888 static int 889 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 890 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) 891 { 892 int totlen; 893 894 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 895 goto process_cipher_encrypt_err; 896 897 EVP_CIPHER_CTX_set_padding(ctx, 0); 898 899 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 900 srclen, ctx)) 901 goto process_cipher_encrypt_err; 902 903 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 904 goto process_cipher_encrypt_err; 905 906 return 0; 907 908 process_cipher_encrypt_err: 909 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 910 return -EINVAL; 911 } 912 913 /** Process standard openssl cipher encryption */ 914 static int 915 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 916 uint8_t *iv, int srclen, 917 EVP_CIPHER_CTX *ctx) 918 { 919 uint8_t i; 920 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 921 int encrypted_ivlen; 922 923 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 924 iv, DES_BLOCK_SIZE) <= 0) 925 goto process_cipher_encrypt_err; 926 927 for (i = 0; i < srclen; i++) 928 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 929 930 return 0; 931 932 process_cipher_encrypt_err: 933 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 934 return -EINVAL; 935 } 936 /** Process standard openssl cipher decryption */ 937 static int 938 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 939 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) 940 { 941 int totlen; 942 943 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 944 goto process_cipher_decrypt_err; 945 946 EVP_CIPHER_CTX_set_padding(ctx, 0); 947 948 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 949 srclen, ctx)) 950 goto process_cipher_decrypt_err; 951 952 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 953 goto process_cipher_decrypt_err; 954 return 0; 955 956 process_cipher_decrypt_err: 957 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 958 return -EINVAL; 959 } 960 961 /** Process cipher des 3 ctr encryption, decryption algorithm */ 962 static int 963 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 964 int offset, uint8_t *iv, uint8_t *key, int srclen, 965 EVP_CIPHER_CTX *ctx) 966 { 967 uint8_t ebuf[8], ctr[8]; 968 int unused, n; 969 struct rte_mbuf *m; 970 uint8_t *src; 971 int l; 972 973 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 974 m = m->next) 975 offset -= rte_pktmbuf_data_len(m); 976 977 if (m == 0) 978 goto process_cipher_des3ctr_err; 979 980 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 981 l = rte_pktmbuf_data_len(m) - offset; 982 983 /* We use 3DES encryption also for decryption. 984 * IV is not important for 3DES ecb 985 */ 986 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 987 goto process_cipher_des3ctr_err; 988 989 memcpy(ctr, iv, 8); 990 991 for (n = 0; n < srclen; n++) { 992 if (n % 8 == 0) { 993 if (EVP_EncryptUpdate(ctx, 994 (unsigned char *)&ebuf, &unused, 995 (const unsigned char *)&ctr, 8) <= 0) 996 goto process_cipher_des3ctr_err; 997 ctr_inc(ctr); 998 } 999 dst[n] = *(src++) ^ ebuf[n % 8]; 1000 1001 l--; 1002 if (!l) { 1003 m = m->next; 1004 if (m) { 1005 src = rte_pktmbuf_mtod(m, uint8_t *); 1006 l = rte_pktmbuf_data_len(m); 1007 } 1008 } 1009 } 1010 1011 return 0; 1012 1013 process_cipher_des3ctr_err: 1014 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 1015 return -EINVAL; 1016 } 1017 1018 /** Process AES-GCM encrypt algorithm */ 1019 static int 1020 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1021 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1022 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1023 { 1024 int len = 0, unused = 0; 1025 uint8_t empty[] = {}; 1026 1027 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1028 goto process_auth_encryption_gcm_err; 1029 1030 if (aadlen > 0) 1031 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1032 goto process_auth_encryption_gcm_err; 1033 1034 if (srclen > 0) 1035 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1036 srclen, ctx)) 1037 goto process_auth_encryption_gcm_err; 1038 1039 /* Workaround open ssl bug in version less then 1.0.1f */ 1040 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1041 goto process_auth_encryption_gcm_err; 1042 1043 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1044 goto process_auth_encryption_gcm_err; 1045 1046 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1047 goto process_auth_encryption_gcm_err; 1048 1049 return 0; 1050 1051 process_auth_encryption_gcm_err: 1052 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1053 return -EINVAL; 1054 } 1055 1056 /** Process AES-CCM encrypt algorithm */ 1057 static int 1058 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1059 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1060 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1061 { 1062 int len = 0; 1063 1064 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1065 goto process_auth_encryption_ccm_err; 1066 1067 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1068 goto process_auth_encryption_ccm_err; 1069 1070 if (aadlen > 0) 1071 /* 1072 * For AES-CCM, the actual AAD is placed 1073 * 18 bytes after the start of the AAD field, 1074 * according to the API. 1075 */ 1076 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1077 goto process_auth_encryption_ccm_err; 1078 1079 if (srclen > 0) 1080 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1081 srclen, ctx)) 1082 goto process_auth_encryption_ccm_err; 1083 1084 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1085 goto process_auth_encryption_ccm_err; 1086 1087 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1088 goto process_auth_encryption_ccm_err; 1089 1090 return 0; 1091 1092 process_auth_encryption_ccm_err: 1093 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1094 return -EINVAL; 1095 } 1096 1097 /** Process AES-GCM decrypt algorithm */ 1098 static int 1099 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1100 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1101 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1102 { 1103 int len = 0, unused = 0; 1104 uint8_t empty[] = {}; 1105 1106 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1107 goto process_auth_decryption_gcm_err; 1108 1109 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1110 goto process_auth_decryption_gcm_err; 1111 1112 if (aadlen > 0) 1113 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1114 goto process_auth_decryption_gcm_err; 1115 1116 if (srclen > 0) 1117 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1118 srclen, ctx)) 1119 goto process_auth_decryption_gcm_err; 1120 1121 /* Workaround open ssl bug in version less then 1.0.1f */ 1122 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1123 goto process_auth_decryption_gcm_err; 1124 1125 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1126 return -EFAULT; 1127 1128 return 0; 1129 1130 process_auth_decryption_gcm_err: 1131 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1132 return -EINVAL; 1133 } 1134 1135 /** Process AES-CCM decrypt algorithm */ 1136 static int 1137 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1138 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1139 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1140 EVP_CIPHER_CTX *ctx) 1141 { 1142 int len = 0; 1143 1144 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1145 goto process_auth_decryption_ccm_err; 1146 1147 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1148 goto process_auth_decryption_ccm_err; 1149 1150 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1151 goto process_auth_decryption_ccm_err; 1152 1153 if (aadlen > 0) 1154 /* 1155 * For AES-CCM, the actual AAD is placed 1156 * 18 bytes after the start of the AAD field, 1157 * according to the API. 1158 */ 1159 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1160 goto process_auth_decryption_ccm_err; 1161 1162 if (srclen > 0) 1163 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1164 srclen, ctx)) 1165 return -EFAULT; 1166 1167 return 0; 1168 1169 process_auth_decryption_ccm_err: 1170 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1171 return -EINVAL; 1172 } 1173 1174 /** Process standard openssl auth algorithms */ 1175 static int 1176 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1177 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1178 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1179 { 1180 size_t dstlen; 1181 struct rte_mbuf *m; 1182 int l, n = srclen; 1183 uint8_t *src; 1184 1185 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1186 m = m->next) 1187 offset -= rte_pktmbuf_data_len(m); 1188 1189 if (m == 0) 1190 goto process_auth_err; 1191 1192 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1193 goto process_auth_err; 1194 1195 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1196 1197 l = rte_pktmbuf_data_len(m) - offset; 1198 if (srclen <= l) { 1199 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1200 goto process_auth_err; 1201 goto process_auth_final; 1202 } 1203 1204 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1205 goto process_auth_err; 1206 1207 n -= l; 1208 1209 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1210 src = rte_pktmbuf_mtod(m, uint8_t *); 1211 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1212 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1213 goto process_auth_err; 1214 n -= l; 1215 } 1216 1217 process_auth_final: 1218 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1219 goto process_auth_err; 1220 return 0; 1221 1222 process_auth_err: 1223 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1224 return -EINVAL; 1225 } 1226 1227 /** Process standard openssl auth algorithms with hmac */ 1228 static int 1229 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1230 int srclen, HMAC_CTX *ctx) 1231 { 1232 unsigned int dstlen; 1233 struct rte_mbuf *m; 1234 int l, n = srclen; 1235 uint8_t *src; 1236 1237 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1238 m = m->next) 1239 offset -= rte_pktmbuf_data_len(m); 1240 1241 if (m == 0) 1242 goto process_auth_err; 1243 1244 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1245 1246 l = rte_pktmbuf_data_len(m) - offset; 1247 if (srclen <= l) { 1248 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1249 goto process_auth_err; 1250 goto process_auth_final; 1251 } 1252 1253 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1254 goto process_auth_err; 1255 1256 n -= l; 1257 1258 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1259 src = rte_pktmbuf_mtod(m, uint8_t *); 1260 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1261 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1262 goto process_auth_err; 1263 n -= l; 1264 } 1265 1266 process_auth_final: 1267 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1268 goto process_auth_err; 1269 1270 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1271 goto process_auth_err; 1272 1273 return 0; 1274 1275 process_auth_err: 1276 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1277 return -EINVAL; 1278 } 1279 1280 /*----------------------------------------------------------------------------*/ 1281 1282 /** Process auth/cipher combined operation */ 1283 static void 1284 process_openssl_combined_op 1285 (struct rte_crypto_op *op, struct openssl_session *sess, 1286 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1287 { 1288 /* cipher */ 1289 uint8_t *dst = NULL, *iv, *tag, *aad; 1290 int srclen, aadlen, status = -1; 1291 uint32_t offset; 1292 uint8_t taglen; 1293 EVP_CIPHER_CTX *ctx_copy; 1294 1295 /* 1296 * Segmented destination buffer is not supported for 1297 * encryption/decryption 1298 */ 1299 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1300 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1301 return; 1302 } 1303 1304 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1305 sess->iv.offset); 1306 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1307 srclen = 0; 1308 offset = op->sym->auth.data.offset; 1309 aadlen = op->sym->auth.data.length; 1310 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1311 op->sym->auth.data.offset); 1312 tag = op->sym->auth.digest.data; 1313 if (tag == NULL) 1314 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1315 offset + aadlen); 1316 } else { 1317 srclen = op->sym->aead.data.length; 1318 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1319 op->sym->aead.data.offset); 1320 offset = op->sym->aead.data.offset; 1321 aad = op->sym->aead.aad.data; 1322 aadlen = sess->auth.aad_length; 1323 tag = op->sym->aead.digest.data; 1324 if (tag == NULL) 1325 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1326 offset + srclen); 1327 } 1328 1329 taglen = sess->auth.digest_length; 1330 ctx_copy = EVP_CIPHER_CTX_new(); 1331 EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx); 1332 1333 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1334 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1335 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1336 status = process_openssl_auth_encryption_gcm( 1337 mbuf_src, offset, srclen, 1338 aad, aadlen, iv, 1339 dst, tag, ctx_copy); 1340 else 1341 status = process_openssl_auth_encryption_ccm( 1342 mbuf_src, offset, srclen, 1343 aad, aadlen, iv, 1344 dst, tag, taglen, ctx_copy); 1345 1346 } else { 1347 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1348 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1349 status = process_openssl_auth_decryption_gcm( 1350 mbuf_src, offset, srclen, 1351 aad, aadlen, iv, 1352 dst, tag, ctx_copy); 1353 else 1354 status = process_openssl_auth_decryption_ccm( 1355 mbuf_src, offset, srclen, 1356 aad, aadlen, iv, 1357 dst, tag, taglen, ctx_copy); 1358 } 1359 1360 EVP_CIPHER_CTX_free(ctx_copy); 1361 if (status != 0) { 1362 if (status == (-EFAULT) && 1363 sess->auth.operation == 1364 RTE_CRYPTO_AUTH_OP_VERIFY) 1365 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1366 else 1367 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1368 } 1369 } 1370 1371 /** Process cipher operation */ 1372 static void 1373 process_openssl_cipher_op 1374 (struct rte_crypto_op *op, struct openssl_session *sess, 1375 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1376 { 1377 uint8_t *dst, *iv; 1378 int srclen, status; 1379 EVP_CIPHER_CTX *ctx_copy; 1380 1381 /* 1382 * Segmented destination buffer is not supported for 1383 * encryption/decryption 1384 */ 1385 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1386 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1387 return; 1388 } 1389 1390 srclen = op->sym->cipher.data.length; 1391 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1392 op->sym->cipher.data.offset); 1393 1394 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1395 sess->iv.offset); 1396 ctx_copy = EVP_CIPHER_CTX_new(); 1397 EVP_CIPHER_CTX_copy(ctx_copy, sess->cipher.ctx); 1398 1399 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1400 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1401 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1402 op->sym->cipher.data.offset, iv, 1403 srclen, ctx_copy); 1404 else 1405 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1406 op->sym->cipher.data.offset, iv, 1407 srclen, ctx_copy); 1408 else 1409 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1410 op->sym->cipher.data.offset, iv, 1411 sess->cipher.key.data, srclen, 1412 ctx_copy); 1413 1414 EVP_CIPHER_CTX_free(ctx_copy); 1415 if (status != 0) 1416 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1417 } 1418 1419 /** Process cipher operation */ 1420 static void 1421 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1422 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1423 struct rte_mbuf *mbuf_dst) 1424 { 1425 uint8_t *src, *dst, *iv; 1426 uint8_t block_size, last_block_len; 1427 int srclen, status = 0; 1428 1429 srclen = op->sym->cipher.data.length; 1430 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1431 op->sym->cipher.data.offset); 1432 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1433 op->sym->cipher.data.offset); 1434 1435 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1436 sess->iv.offset); 1437 1438 block_size = DES_BLOCK_SIZE; 1439 1440 last_block_len = srclen % block_size; 1441 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1442 /* Encrypt only with ECB mode XOR IV */ 1443 if (srclen < block_size) { 1444 status = process_openssl_cipher_bpi_encrypt(src, dst, 1445 iv, srclen, 1446 sess->cipher.bpi_ctx); 1447 } else { 1448 srclen -= last_block_len; 1449 /* Encrypt with the block aligned stream with CBC mode */ 1450 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1451 op->sym->cipher.data.offset, iv, 1452 srclen, sess->cipher.ctx); 1453 if (last_block_len) { 1454 /* Point at last block */ 1455 dst += srclen; 1456 /* 1457 * IV is the last encrypted block from 1458 * the previous operation 1459 */ 1460 iv = dst - block_size; 1461 src += srclen; 1462 srclen = last_block_len; 1463 /* Encrypt the last frame with ECB mode */ 1464 status |= process_openssl_cipher_bpi_encrypt(src, 1465 dst, iv, 1466 srclen, sess->cipher.bpi_ctx); 1467 } 1468 } 1469 } else { 1470 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1471 if (srclen < block_size) { 1472 status = process_openssl_cipher_bpi_encrypt(src, dst, 1473 iv, 1474 srclen, 1475 sess->cipher.bpi_ctx); 1476 } else { 1477 if (last_block_len) { 1478 /* Point at last block */ 1479 dst += srclen - last_block_len; 1480 src += srclen - last_block_len; 1481 /* 1482 * IV is the last full block 1483 */ 1484 iv = src - block_size; 1485 /* 1486 * Decrypt the last frame with ECB mode 1487 * (encrypt, as it is the same operation) 1488 */ 1489 status = process_openssl_cipher_bpi_encrypt(src, 1490 dst, iv, 1491 last_block_len, sess->cipher.bpi_ctx); 1492 /* Prepare parameters for CBC mode op */ 1493 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1494 sess->iv.offset); 1495 dst += last_block_len - srclen; 1496 srclen -= last_block_len; 1497 } 1498 1499 /* Decrypt with CBC mode */ 1500 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1501 op->sym->cipher.data.offset, iv, 1502 srclen, sess->cipher.ctx); 1503 } 1504 } 1505 1506 if (status != 0) 1507 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1508 } 1509 1510 /** Process auth operation */ 1511 static void 1512 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1513 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1514 struct rte_mbuf *mbuf_dst) 1515 { 1516 uint8_t *dst; 1517 int srclen, status; 1518 EVP_MD_CTX *ctx_a; 1519 HMAC_CTX *ctx_h; 1520 1521 srclen = op->sym->auth.data.length; 1522 1523 dst = qp->temp_digest; 1524 1525 switch (sess->auth.mode) { 1526 case OPENSSL_AUTH_AS_AUTH: 1527 ctx_a = EVP_MD_CTX_create(); 1528 EVP_MD_CTX_copy_ex(ctx_a, sess->auth.auth.ctx); 1529 status = process_openssl_auth(mbuf_src, dst, 1530 op->sym->auth.data.offset, NULL, NULL, srclen, 1531 ctx_a, sess->auth.auth.evp_algo); 1532 EVP_MD_CTX_destroy(ctx_a); 1533 break; 1534 case OPENSSL_AUTH_AS_HMAC: 1535 ctx_h = HMAC_CTX_new(); 1536 HMAC_CTX_copy(ctx_h, sess->auth.hmac.ctx); 1537 status = process_openssl_auth_hmac(mbuf_src, dst, 1538 op->sym->auth.data.offset, srclen, 1539 ctx_h); 1540 HMAC_CTX_free(ctx_h); 1541 break; 1542 default: 1543 status = -1; 1544 break; 1545 } 1546 1547 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1548 if (CRYPTO_memcmp(dst, op->sym->auth.digest.data, 1549 sess->auth.digest_length) != 0) { 1550 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1551 } 1552 } else { 1553 uint8_t *auth_dst; 1554 1555 auth_dst = op->sym->auth.digest.data; 1556 if (auth_dst == NULL) 1557 auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1558 op->sym->auth.data.offset + 1559 op->sym->auth.data.length); 1560 memcpy(auth_dst, dst, sess->auth.digest_length); 1561 } 1562 1563 if (status != 0) 1564 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1565 } 1566 1567 /* process dsa sign operation */ 1568 static int 1569 process_openssl_dsa_sign_op(struct rte_crypto_op *cop, 1570 struct openssl_asym_session *sess) 1571 { 1572 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1573 DSA *dsa = sess->u.s.dsa; 1574 DSA_SIG *sign = NULL; 1575 1576 sign = DSA_do_sign(op->message.data, 1577 op->message.length, 1578 dsa); 1579 1580 if (sign == NULL) { 1581 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 1582 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1583 } else { 1584 const BIGNUM *r = NULL, *s = NULL; 1585 get_dsa_sign(sign, &r, &s); 1586 1587 op->r.length = BN_bn2bin(r, op->r.data); 1588 op->s.length = BN_bn2bin(s, op->s.data); 1589 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1590 } 1591 1592 DSA_SIG_free(sign); 1593 1594 return 0; 1595 } 1596 1597 /* process dsa verify operation */ 1598 static int 1599 process_openssl_dsa_verify_op(struct rte_crypto_op *cop, 1600 struct openssl_asym_session *sess) 1601 { 1602 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1603 DSA *dsa = sess->u.s.dsa; 1604 int ret; 1605 DSA_SIG *sign = DSA_SIG_new(); 1606 BIGNUM *r = NULL, *s = NULL; 1607 BIGNUM *pub_key = NULL; 1608 1609 if (sign == NULL) { 1610 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 1611 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1612 return -1; 1613 } 1614 1615 r = BN_bin2bn(op->r.data, 1616 op->r.length, 1617 r); 1618 s = BN_bin2bn(op->s.data, 1619 op->s.length, 1620 s); 1621 pub_key = BN_bin2bn(op->y.data, 1622 op->y.length, 1623 pub_key); 1624 if (!r || !s || !pub_key) { 1625 BN_free(r); 1626 BN_free(s); 1627 BN_free(pub_key); 1628 1629 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1630 return -1; 1631 } 1632 set_dsa_sign(sign, r, s); 1633 set_dsa_pub_key(dsa, pub_key); 1634 1635 ret = DSA_do_verify(op->message.data, 1636 op->message.length, 1637 sign, 1638 dsa); 1639 1640 if (ret != 1) 1641 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1642 else 1643 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1644 1645 DSA_SIG_free(sign); 1646 1647 return 0; 1648 } 1649 1650 /* process dh operation */ 1651 static int 1652 process_openssl_dh_op(struct rte_crypto_op *cop, 1653 struct openssl_asym_session *sess) 1654 { 1655 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 1656 DH *dh_key = sess->u.dh.dh_key; 1657 BIGNUM *priv_key = NULL; 1658 int ret = 0; 1659 1660 if (sess->u.dh.key_op & 1661 (1 << RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE)) { 1662 /* compute shared secret using peer public key 1663 * and current private key 1664 * shared secret = peer_key ^ priv_key mod p 1665 */ 1666 BIGNUM *peer_key = NULL; 1667 1668 /* copy private key and peer key and compute shared secret */ 1669 peer_key = BN_bin2bn(op->pub_key.data, 1670 op->pub_key.length, 1671 peer_key); 1672 if (peer_key == NULL) { 1673 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1674 return -1; 1675 } 1676 priv_key = BN_bin2bn(op->priv_key.data, 1677 op->priv_key.length, 1678 priv_key); 1679 if (priv_key == NULL) { 1680 BN_free(peer_key); 1681 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1682 return -1; 1683 } 1684 ret = set_dh_priv_key(dh_key, priv_key); 1685 if (ret) { 1686 OPENSSL_LOG(ERR, "Failed to set private key\n"); 1687 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1688 BN_free(peer_key); 1689 BN_free(priv_key); 1690 return 0; 1691 } 1692 1693 ret = DH_compute_key( 1694 op->shared_secret.data, 1695 peer_key, dh_key); 1696 if (ret < 0) { 1697 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1698 BN_free(peer_key); 1699 /* priv key is already loaded into dh, 1700 * let's not free that directly here. 1701 * DH_free() will auto free it later. 1702 */ 1703 return 0; 1704 } 1705 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1706 op->shared_secret.length = ret; 1707 BN_free(peer_key); 1708 return 0; 1709 } 1710 1711 /* 1712 * other options are public and private key generations. 1713 * 1714 * if user provides private key, 1715 * then first set DH with user provided private key 1716 */ 1717 if ((sess->u.dh.key_op & 1718 (1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)) && 1719 !(sess->u.dh.key_op & 1720 (1 << RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE))) { 1721 /* generate public key using user-provided private key 1722 * pub_key = g ^ priv_key mod p 1723 */ 1724 1725 /* load private key into DH */ 1726 priv_key = BN_bin2bn(op->priv_key.data, 1727 op->priv_key.length, 1728 priv_key); 1729 if (priv_key == NULL) { 1730 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1731 return -1; 1732 } 1733 ret = set_dh_priv_key(dh_key, priv_key); 1734 if (ret) { 1735 OPENSSL_LOG(ERR, "Failed to set private key\n"); 1736 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1737 BN_free(priv_key); 1738 return 0; 1739 } 1740 } 1741 1742 /* generate public and private key pair. 1743 * 1744 * if private key already set, generates only public key. 1745 * 1746 * if private key is not already set, then set it to random value 1747 * and update internal private key. 1748 */ 1749 if (!DH_generate_key(dh_key)) { 1750 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1751 return 0; 1752 } 1753 1754 if (sess->u.dh.key_op & (1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)) { 1755 const BIGNUM *pub_key = NULL; 1756 1757 OPENSSL_LOG(DEBUG, "%s:%d update public key\n", 1758 __func__, __LINE__); 1759 1760 /* get the generated keys */ 1761 get_dh_pub_key(dh_key, &pub_key); 1762 1763 /* output public key */ 1764 op->pub_key.length = BN_bn2bin(pub_key, 1765 op->pub_key.data); 1766 } 1767 1768 if (sess->u.dh.key_op & 1769 (1 << RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE)) { 1770 const BIGNUM *priv_key = NULL; 1771 1772 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", 1773 __func__, __LINE__); 1774 1775 /* get the generated keys */ 1776 get_dh_priv_key(dh_key, &priv_key); 1777 1778 /* provide generated private key back to user */ 1779 op->priv_key.length = BN_bn2bin(priv_key, 1780 op->priv_key.data); 1781 } 1782 1783 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1784 1785 return 0; 1786 } 1787 1788 /* process modinv operation */ 1789 static int 1790 process_openssl_modinv_op(struct rte_crypto_op *cop, 1791 struct openssl_asym_session *sess) 1792 { 1793 struct rte_crypto_asym_op *op = cop->asym; 1794 BIGNUM *base = BN_CTX_get(sess->u.m.ctx); 1795 BIGNUM *res = BN_CTX_get(sess->u.m.ctx); 1796 1797 if (unlikely(base == NULL || res == NULL)) { 1798 BN_free(base); 1799 BN_free(res); 1800 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1801 return -1; 1802 } 1803 1804 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 1805 op->modinv.base.length, base); 1806 1807 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) { 1808 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1809 op->modinv.result.length = BN_bn2bin(res, op->modinv.result.data); 1810 } else { 1811 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1812 } 1813 1814 BN_clear(res); 1815 BN_clear(base); 1816 1817 return 0; 1818 } 1819 1820 /* process modexp operation */ 1821 static int 1822 process_openssl_modexp_op(struct rte_crypto_op *cop, 1823 struct openssl_asym_session *sess) 1824 { 1825 struct rte_crypto_asym_op *op = cop->asym; 1826 BIGNUM *base = BN_CTX_get(sess->u.e.ctx); 1827 BIGNUM *res = BN_CTX_get(sess->u.e.ctx); 1828 1829 if (unlikely(base == NULL || res == NULL)) { 1830 BN_free(base); 1831 BN_free(res); 1832 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1833 return -1; 1834 } 1835 1836 base = BN_bin2bn((const unsigned char *)op->modex.base.data, 1837 op->modex.base.length, base); 1838 1839 if (BN_mod_exp(res, base, sess->u.e.exp, 1840 sess->u.e.mod, sess->u.e.ctx)) { 1841 op->modex.result.length = BN_bn2bin(res, op->modex.result.data); 1842 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1843 } else { 1844 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1845 } 1846 1847 BN_clear(res); 1848 BN_clear(base); 1849 1850 return 0; 1851 } 1852 1853 /* process rsa operations */ 1854 static int 1855 process_openssl_rsa_op(struct rte_crypto_op *cop, 1856 struct openssl_asym_session *sess) 1857 { 1858 int ret = 0; 1859 struct rte_crypto_asym_op *op = cop->asym; 1860 RSA *rsa = sess->u.r.rsa; 1861 uint32_t pad = (op->rsa.pad); 1862 uint8_t *tmp; 1863 1864 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1865 1866 switch (pad) { 1867 case RTE_CRYPTO_RSA_PADDING_PKCS1_5: 1868 pad = RSA_PKCS1_PADDING; 1869 break; 1870 case RTE_CRYPTO_RSA_PADDING_NONE: 1871 pad = RSA_NO_PADDING; 1872 break; 1873 default: 1874 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1875 OPENSSL_LOG(ERR, 1876 "rsa pad type not supported %d\n", pad); 1877 return 0; 1878 } 1879 1880 switch (op->rsa.op_type) { 1881 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 1882 ret = RSA_public_encrypt(op->rsa.message.length, 1883 op->rsa.message.data, 1884 op->rsa.cipher.data, 1885 rsa, 1886 pad); 1887 1888 if (ret > 0) 1889 op->rsa.cipher.length = ret; 1890 OPENSSL_LOG(DEBUG, 1891 "length of encrypted text %d\n", ret); 1892 break; 1893 1894 case RTE_CRYPTO_ASYM_OP_DECRYPT: 1895 ret = RSA_private_decrypt(op->rsa.cipher.length, 1896 op->rsa.cipher.data, 1897 op->rsa.message.data, 1898 rsa, 1899 pad); 1900 if (ret > 0) 1901 op->rsa.message.length = ret; 1902 break; 1903 1904 case RTE_CRYPTO_ASYM_OP_SIGN: 1905 ret = RSA_private_encrypt(op->rsa.message.length, 1906 op->rsa.message.data, 1907 op->rsa.sign.data, 1908 rsa, 1909 pad); 1910 if (ret > 0) 1911 op->rsa.sign.length = ret; 1912 break; 1913 1914 case RTE_CRYPTO_ASYM_OP_VERIFY: 1915 tmp = rte_malloc(NULL, op->rsa.sign.length, 0); 1916 if (tmp == NULL) { 1917 OPENSSL_LOG(ERR, "Memory allocation failed"); 1918 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1919 break; 1920 } 1921 ret = RSA_public_decrypt(op->rsa.sign.length, 1922 op->rsa.sign.data, 1923 tmp, 1924 rsa, 1925 pad); 1926 1927 OPENSSL_LOG(DEBUG, 1928 "Length of public_decrypt %d " 1929 "length of message %zd\n", 1930 ret, op->rsa.message.length); 1931 if ((ret <= 0) || (CRYPTO_memcmp(tmp, op->rsa.message.data, 1932 op->rsa.message.length))) { 1933 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 1934 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1935 } 1936 rte_free(tmp); 1937 break; 1938 1939 default: 1940 /* allow ops with invalid args to be pushed to 1941 * completion queue 1942 */ 1943 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1944 break; 1945 } 1946 1947 if (ret < 0) 1948 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1949 1950 return 0; 1951 } 1952 1953 static int 1954 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1955 struct openssl_asym_session *sess) 1956 { 1957 int retval = 0; 1958 1959 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1960 1961 switch (sess->xfrm_type) { 1962 case RTE_CRYPTO_ASYM_XFORM_RSA: 1963 retval = process_openssl_rsa_op(op, sess); 1964 break; 1965 case RTE_CRYPTO_ASYM_XFORM_MODEX: 1966 retval = process_openssl_modexp_op(op, sess); 1967 break; 1968 case RTE_CRYPTO_ASYM_XFORM_MODINV: 1969 retval = process_openssl_modinv_op(op, sess); 1970 break; 1971 case RTE_CRYPTO_ASYM_XFORM_DH: 1972 retval = process_openssl_dh_op(op, sess); 1973 break; 1974 case RTE_CRYPTO_ASYM_XFORM_DSA: 1975 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 1976 retval = process_openssl_dsa_sign_op(op, sess); 1977 else if (op->asym->dsa.op_type == 1978 RTE_CRYPTO_ASYM_OP_VERIFY) 1979 retval = 1980 process_openssl_dsa_verify_op(op, sess); 1981 else 1982 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1983 break; 1984 default: 1985 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1986 break; 1987 } 1988 if (!retval) { 1989 /* op processed so push to completion queue as processed */ 1990 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 1991 if (retval) 1992 /* return error if failed to put in completion queue */ 1993 retval = -1; 1994 } 1995 1996 return retval; 1997 } 1998 1999 /** Process crypto operation for mbuf */ 2000 static int 2001 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 2002 struct openssl_session *sess) 2003 { 2004 struct rte_mbuf *msrc, *mdst; 2005 int retval; 2006 2007 msrc = op->sym->m_src; 2008 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 2009 2010 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 2011 2012 switch (sess->chain_order) { 2013 case OPENSSL_CHAIN_ONLY_CIPHER: 2014 process_openssl_cipher_op(op, sess, msrc, mdst); 2015 break; 2016 case OPENSSL_CHAIN_ONLY_AUTH: 2017 process_openssl_auth_op(qp, op, sess, msrc, mdst); 2018 break; 2019 case OPENSSL_CHAIN_CIPHER_AUTH: 2020 process_openssl_cipher_op(op, sess, msrc, mdst); 2021 process_openssl_auth_op(qp, op, sess, mdst, mdst); 2022 break; 2023 case OPENSSL_CHAIN_AUTH_CIPHER: 2024 process_openssl_auth_op(qp, op, sess, msrc, mdst); 2025 process_openssl_cipher_op(op, sess, msrc, mdst); 2026 break; 2027 case OPENSSL_CHAIN_COMBINED: 2028 process_openssl_combined_op(op, sess, msrc, mdst); 2029 break; 2030 case OPENSSL_CHAIN_CIPHER_BPI: 2031 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 2032 break; 2033 default: 2034 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 2035 break; 2036 } 2037 2038 /* Free session if a session-less crypto op */ 2039 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 2040 openssl_reset_session(sess); 2041 memset(sess, 0, sizeof(struct openssl_session)); 2042 memset(op->sym->session, 0, 2043 rte_cryptodev_sym_get_existing_header_session_size( 2044 op->sym->session)); 2045 rte_mempool_put(qp->sess_mp_priv, sess); 2046 rte_mempool_put(qp->sess_mp, op->sym->session); 2047 op->sym->session = NULL; 2048 } 2049 2050 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 2051 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2052 2053 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 2054 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 2055 else 2056 retval = -1; 2057 2058 return retval; 2059 } 2060 2061 /* 2062 *------------------------------------------------------------------------------ 2063 * PMD Framework 2064 *------------------------------------------------------------------------------ 2065 */ 2066 2067 /** Enqueue burst */ 2068 static uint16_t 2069 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 2070 uint16_t nb_ops) 2071 { 2072 void *sess; 2073 struct openssl_qp *qp = queue_pair; 2074 int i, retval; 2075 2076 for (i = 0; i < nb_ops; i++) { 2077 sess = get_session(qp, ops[i]); 2078 if (unlikely(sess == NULL)) 2079 goto enqueue_err; 2080 2081 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) 2082 retval = process_op(qp, ops[i], 2083 (struct openssl_session *) sess); 2084 else 2085 retval = process_asym_op(qp, ops[i], 2086 (struct openssl_asym_session *) sess); 2087 if (unlikely(retval < 0)) 2088 goto enqueue_err; 2089 } 2090 2091 qp->stats.enqueued_count += i; 2092 return i; 2093 2094 enqueue_err: 2095 qp->stats.enqueue_err_count++; 2096 return i; 2097 } 2098 2099 /** Dequeue burst */ 2100 static uint16_t 2101 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 2102 uint16_t nb_ops) 2103 { 2104 struct openssl_qp *qp = queue_pair; 2105 2106 unsigned int nb_dequeued = 0; 2107 2108 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 2109 (void **)ops, nb_ops, NULL); 2110 qp->stats.dequeued_count += nb_dequeued; 2111 2112 return nb_dequeued; 2113 } 2114 2115 /** Create OPENSSL crypto device */ 2116 static int 2117 cryptodev_openssl_create(const char *name, 2118 struct rte_vdev_device *vdev, 2119 struct rte_cryptodev_pmd_init_params *init_params) 2120 { 2121 struct rte_cryptodev *dev; 2122 struct openssl_private *internals; 2123 2124 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 2125 if (dev == NULL) { 2126 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 2127 goto init_error; 2128 } 2129 2130 dev->driver_id = cryptodev_driver_id; 2131 dev->dev_ops = rte_openssl_pmd_ops; 2132 2133 /* register rx/tx burst functions for data path */ 2134 dev->dequeue_burst = openssl_pmd_dequeue_burst; 2135 dev->enqueue_burst = openssl_pmd_enqueue_burst; 2136 2137 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 2138 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 2139 RTE_CRYPTODEV_FF_CPU_AESNI | 2140 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 2141 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | 2142 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO | 2143 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_EXP | 2144 RTE_CRYPTODEV_FF_RSA_PRIV_OP_KEY_QT; 2145 2146 internals = dev->data->dev_private; 2147 2148 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 2149 2150 return 0; 2151 2152 init_error: 2153 OPENSSL_LOG(ERR, "driver %s: create failed", 2154 init_params->name); 2155 2156 cryptodev_openssl_remove(vdev); 2157 return -EFAULT; 2158 } 2159 2160 /** Initialise OPENSSL crypto device */ 2161 static int 2162 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 2163 { 2164 struct rte_cryptodev_pmd_init_params init_params = { 2165 "", 2166 sizeof(struct openssl_private), 2167 rte_socket_id(), 2168 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 2169 }; 2170 const char *name; 2171 const char *input_args; 2172 2173 name = rte_vdev_device_name(vdev); 2174 if (name == NULL) 2175 return -EINVAL; 2176 input_args = rte_vdev_device_args(vdev); 2177 2178 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 2179 2180 return cryptodev_openssl_create(name, vdev, &init_params); 2181 } 2182 2183 /** Uninitialise OPENSSL crypto device */ 2184 static int 2185 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 2186 { 2187 struct rte_cryptodev *cryptodev; 2188 const char *name; 2189 2190 name = rte_vdev_device_name(vdev); 2191 if (name == NULL) 2192 return -EINVAL; 2193 2194 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 2195 if (cryptodev == NULL) 2196 return -ENODEV; 2197 2198 return rte_cryptodev_pmd_destroy(cryptodev); 2199 } 2200 2201 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 2202 .probe = cryptodev_openssl_probe, 2203 .remove = cryptodev_openssl_remove 2204 }; 2205 2206 static struct cryptodev_driver openssl_crypto_drv; 2207 2208 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 2209 cryptodev_openssl_pmd_drv); 2210 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 2211 "max_nb_queue_pairs=<int> " 2212 "socket_id=<int>"); 2213 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 2214 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 2215 2216 RTE_INIT(openssl_init_log) 2217 { 2218 openssl_logtype_driver = rte_log_register("pmd.crypto.openssl"); 2219 } 2220