1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_common.h> 6 #include <rte_hexdump.h> 7 #include <rte_cryptodev.h> 8 #include <rte_cryptodev_pmd.h> 9 #include <rte_bus_vdev.h> 10 #include <rte_malloc.h> 11 #include <rte_cpuflags.h> 12 13 #include <openssl/hmac.h> 14 #include <openssl/evp.h> 15 16 #include "rte_openssl_pmd_private.h" 17 #include "compat.h" 18 19 #define DES_BLOCK_SIZE 8 20 21 static uint8_t cryptodev_driver_id; 22 23 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 24 static HMAC_CTX *HMAC_CTX_new(void) 25 { 26 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 27 28 if (ctx != NULL) 29 HMAC_CTX_init(ctx); 30 return ctx; 31 } 32 33 static void HMAC_CTX_free(HMAC_CTX *ctx) 34 { 35 if (ctx != NULL) { 36 HMAC_CTX_cleanup(ctx); 37 OPENSSL_free(ctx); 38 } 39 } 40 #endif 41 42 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 43 44 /*----------------------------------------------------------------------------*/ 45 46 /** 47 * Increment counter by 1 48 * Counter is 64 bit array, big-endian 49 */ 50 static void 51 ctr_inc(uint8_t *ctr) 52 { 53 uint64_t *ctr64 = (uint64_t *)ctr; 54 55 *ctr64 = __builtin_bswap64(*ctr64); 56 (*ctr64)++; 57 *ctr64 = __builtin_bswap64(*ctr64); 58 } 59 60 /* 61 *------------------------------------------------------------------------------ 62 * Session Prepare 63 *------------------------------------------------------------------------------ 64 */ 65 66 /** Get xform chain order */ 67 static enum openssl_chain_order 68 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 69 { 70 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 71 72 if (xform != NULL) { 73 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 74 if (xform->next == NULL) 75 res = OPENSSL_CHAIN_ONLY_AUTH; 76 else if (xform->next->type == 77 RTE_CRYPTO_SYM_XFORM_CIPHER) 78 res = OPENSSL_CHAIN_AUTH_CIPHER; 79 } 80 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 81 if (xform->next == NULL) 82 res = OPENSSL_CHAIN_ONLY_CIPHER; 83 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 84 res = OPENSSL_CHAIN_CIPHER_AUTH; 85 } 86 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 87 res = OPENSSL_CHAIN_COMBINED; 88 } 89 90 return res; 91 } 92 93 /** Get session cipher key from input cipher key */ 94 static void 95 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key) 96 { 97 memcpy(session_key, input_key, keylen); 98 } 99 100 /** Get key ede 24 bytes standard from input key */ 101 static int 102 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede) 103 { 104 int res = 0; 105 106 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 107 switch (keylen) { 108 case 24: 109 memcpy(key_ede, key, 24); 110 break; 111 case 16: 112 /* K3 = K1 */ 113 memcpy(key_ede, key, 16); 114 memcpy(key_ede + 16, key, 8); 115 break; 116 case 8: 117 /* K1 = K2 = K3 (DES compatibility) */ 118 memcpy(key_ede, key, 8); 119 memcpy(key_ede + 8, key, 8); 120 memcpy(key_ede + 16, key, 8); 121 break; 122 default: 123 OPENSSL_LOG(ERR, "Unsupported key size"); 124 res = -EINVAL; 125 } 126 127 return res; 128 } 129 130 /** Get adequate openssl function for input cipher algorithm */ 131 static uint8_t 132 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 133 const EVP_CIPHER **algo) 134 { 135 int res = 0; 136 137 if (algo != NULL) { 138 switch (sess_algo) { 139 case RTE_CRYPTO_CIPHER_3DES_CBC: 140 switch (keylen) { 141 case 8: 142 *algo = EVP_des_cbc(); 143 break; 144 case 16: 145 *algo = EVP_des_ede_cbc(); 146 break; 147 case 24: 148 *algo = EVP_des_ede3_cbc(); 149 break; 150 default: 151 res = -EINVAL; 152 } 153 break; 154 case RTE_CRYPTO_CIPHER_3DES_CTR: 155 break; 156 case RTE_CRYPTO_CIPHER_AES_CBC: 157 switch (keylen) { 158 case 16: 159 *algo = EVP_aes_128_cbc(); 160 break; 161 case 24: 162 *algo = EVP_aes_192_cbc(); 163 break; 164 case 32: 165 *algo = EVP_aes_256_cbc(); 166 break; 167 default: 168 res = -EINVAL; 169 } 170 break; 171 case RTE_CRYPTO_CIPHER_AES_CTR: 172 switch (keylen) { 173 case 16: 174 *algo = EVP_aes_128_ctr(); 175 break; 176 case 24: 177 *algo = EVP_aes_192_ctr(); 178 break; 179 case 32: 180 *algo = EVP_aes_256_ctr(); 181 break; 182 default: 183 res = -EINVAL; 184 } 185 break; 186 default: 187 res = -EINVAL; 188 break; 189 } 190 } else { 191 res = -EINVAL; 192 } 193 194 return res; 195 } 196 197 /** Get adequate openssl function for input auth algorithm */ 198 static uint8_t 199 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 200 const EVP_MD **algo) 201 { 202 int res = 0; 203 204 if (algo != NULL) { 205 switch (sessalgo) { 206 case RTE_CRYPTO_AUTH_MD5: 207 case RTE_CRYPTO_AUTH_MD5_HMAC: 208 *algo = EVP_md5(); 209 break; 210 case RTE_CRYPTO_AUTH_SHA1: 211 case RTE_CRYPTO_AUTH_SHA1_HMAC: 212 *algo = EVP_sha1(); 213 break; 214 case RTE_CRYPTO_AUTH_SHA224: 215 case RTE_CRYPTO_AUTH_SHA224_HMAC: 216 *algo = EVP_sha224(); 217 break; 218 case RTE_CRYPTO_AUTH_SHA256: 219 case RTE_CRYPTO_AUTH_SHA256_HMAC: 220 *algo = EVP_sha256(); 221 break; 222 case RTE_CRYPTO_AUTH_SHA384: 223 case RTE_CRYPTO_AUTH_SHA384_HMAC: 224 *algo = EVP_sha384(); 225 break; 226 case RTE_CRYPTO_AUTH_SHA512: 227 case RTE_CRYPTO_AUTH_SHA512_HMAC: 228 *algo = EVP_sha512(); 229 break; 230 default: 231 res = -EINVAL; 232 break; 233 } 234 } else { 235 res = -EINVAL; 236 } 237 238 return res; 239 } 240 241 /** Get adequate openssl function for input cipher algorithm */ 242 static uint8_t 243 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 244 const EVP_CIPHER **algo) 245 { 246 int res = 0; 247 248 if (algo != NULL) { 249 switch (sess_algo) { 250 case RTE_CRYPTO_AEAD_AES_GCM: 251 switch (keylen) { 252 case 16: 253 *algo = EVP_aes_128_gcm(); 254 break; 255 case 24: 256 *algo = EVP_aes_192_gcm(); 257 break; 258 case 32: 259 *algo = EVP_aes_256_gcm(); 260 break; 261 default: 262 res = -EINVAL; 263 } 264 break; 265 case RTE_CRYPTO_AEAD_AES_CCM: 266 switch (keylen) { 267 case 16: 268 *algo = EVP_aes_128_ccm(); 269 break; 270 case 24: 271 *algo = EVP_aes_192_ccm(); 272 break; 273 case 32: 274 *algo = EVP_aes_256_ccm(); 275 break; 276 default: 277 res = -EINVAL; 278 } 279 break; 280 default: 281 res = -EINVAL; 282 break; 283 } 284 } else { 285 res = -EINVAL; 286 } 287 288 return res; 289 } 290 291 /* Set session AEAD encryption parameters */ 292 static int 293 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 294 enum rte_crypto_aead_algorithm algo, 295 uint8_t tag_len, uint8_t *key) 296 { 297 int iv_type = 0; 298 unsigned int do_ccm; 299 300 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 301 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 302 303 /* Select AEAD algo */ 304 switch (algo) { 305 case RTE_CRYPTO_AEAD_AES_GCM: 306 iv_type = EVP_CTRL_GCM_SET_IVLEN; 307 if (tag_len != 16) 308 return -EINVAL; 309 do_ccm = 0; 310 break; 311 case RTE_CRYPTO_AEAD_AES_CCM: 312 iv_type = EVP_CTRL_CCM_SET_IVLEN; 313 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 314 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 315 return -EINVAL; 316 do_ccm = 1; 317 break; 318 default: 319 return -ENOTSUP; 320 } 321 322 sess->cipher.mode = OPENSSL_CIPHER_LIB; 323 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 324 325 if (get_aead_algo(algo, sess->cipher.key.length, 326 &sess->cipher.evp_algo) != 0) 327 return -EINVAL; 328 329 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 330 331 sess->chain_order = OPENSSL_CHAIN_COMBINED; 332 333 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 334 NULL, NULL, NULL) <= 0) 335 return -EINVAL; 336 337 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, 338 NULL) <= 0) 339 return -EINVAL; 340 341 if (do_ccm) 342 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 343 tag_len, NULL); 344 345 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 346 return -EINVAL; 347 348 return 0; 349 } 350 351 /* Set session AEAD decryption parameters */ 352 static int 353 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 354 enum rte_crypto_aead_algorithm algo, 355 uint8_t tag_len, uint8_t *key) 356 { 357 int iv_type = 0; 358 unsigned int do_ccm = 0; 359 360 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 361 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 362 363 /* Select AEAD algo */ 364 switch (algo) { 365 case RTE_CRYPTO_AEAD_AES_GCM: 366 iv_type = EVP_CTRL_GCM_SET_IVLEN; 367 if (tag_len != 16) 368 return -EINVAL; 369 break; 370 case RTE_CRYPTO_AEAD_AES_CCM: 371 iv_type = EVP_CTRL_CCM_SET_IVLEN; 372 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 373 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 374 return -EINVAL; 375 do_ccm = 1; 376 break; 377 default: 378 return -ENOTSUP; 379 } 380 381 sess->cipher.mode = OPENSSL_CIPHER_LIB; 382 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 383 384 if (get_aead_algo(algo, sess->cipher.key.length, 385 &sess->cipher.evp_algo) != 0) 386 return -EINVAL; 387 388 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 389 390 sess->chain_order = OPENSSL_CHAIN_COMBINED; 391 392 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 393 NULL, NULL, NULL) <= 0) 394 return -EINVAL; 395 396 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, 397 sess->iv.length, NULL) <= 0) 398 return -EINVAL; 399 400 if (do_ccm) 401 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 402 tag_len, NULL); 403 404 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 405 return -EINVAL; 406 407 return 0; 408 } 409 410 /** Set session cipher parameters */ 411 static int 412 openssl_set_session_cipher_parameters(struct openssl_session *sess, 413 const struct rte_crypto_sym_xform *xform) 414 { 415 /* Select cipher direction */ 416 sess->cipher.direction = xform->cipher.op; 417 /* Select cipher key */ 418 sess->cipher.key.length = xform->cipher.key.length; 419 420 /* Set IV parameters */ 421 sess->iv.offset = xform->cipher.iv.offset; 422 sess->iv.length = xform->cipher.iv.length; 423 424 /* Select cipher algo */ 425 switch (xform->cipher.algo) { 426 case RTE_CRYPTO_CIPHER_3DES_CBC: 427 case RTE_CRYPTO_CIPHER_AES_CBC: 428 case RTE_CRYPTO_CIPHER_AES_CTR: 429 sess->cipher.mode = OPENSSL_CIPHER_LIB; 430 sess->cipher.algo = xform->cipher.algo; 431 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 432 433 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 434 &sess->cipher.evp_algo) != 0) 435 return -EINVAL; 436 437 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 438 sess->cipher.key.data); 439 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 440 if (EVP_EncryptInit_ex(sess->cipher.ctx, 441 sess->cipher.evp_algo, 442 NULL, xform->cipher.key.data, 443 NULL) != 1) { 444 return -EINVAL; 445 } 446 } else if (sess->cipher.direction == 447 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 448 if (EVP_DecryptInit_ex(sess->cipher.ctx, 449 sess->cipher.evp_algo, 450 NULL, xform->cipher.key.data, 451 NULL) != 1) { 452 return -EINVAL; 453 } 454 } 455 456 break; 457 458 case RTE_CRYPTO_CIPHER_3DES_CTR: 459 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 460 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 461 462 if (get_cipher_key_ede(xform->cipher.key.data, 463 sess->cipher.key.length, 464 sess->cipher.key.data) != 0) 465 return -EINVAL; 466 break; 467 468 case RTE_CRYPTO_CIPHER_DES_CBC: 469 sess->cipher.algo = xform->cipher.algo; 470 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 471 sess->cipher.evp_algo = EVP_des_cbc(); 472 473 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 474 sess->cipher.key.data); 475 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 476 if (EVP_EncryptInit_ex(sess->cipher.ctx, 477 sess->cipher.evp_algo, 478 NULL, xform->cipher.key.data, 479 NULL) != 1) { 480 return -EINVAL; 481 } 482 } else if (sess->cipher.direction == 483 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 484 if (EVP_DecryptInit_ex(sess->cipher.ctx, 485 sess->cipher.evp_algo, 486 NULL, xform->cipher.key.data, 487 NULL) != 1) { 488 return -EINVAL; 489 } 490 } 491 492 break; 493 494 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 495 sess->cipher.algo = xform->cipher.algo; 496 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 497 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 498 sess->cipher.evp_algo = EVP_des_cbc(); 499 500 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 501 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 502 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 503 NULL, xform->cipher.key.data, 0) != 1) 504 return -EINVAL; 505 506 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 507 sess->cipher.key.data); 508 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 509 if (EVP_EncryptInit_ex(sess->cipher.ctx, 510 sess->cipher.evp_algo, 511 NULL, xform->cipher.key.data, 512 NULL) != 1) { 513 return -EINVAL; 514 } 515 } else if (sess->cipher.direction == 516 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 517 if (EVP_DecryptInit_ex(sess->cipher.ctx, 518 sess->cipher.evp_algo, 519 NULL, xform->cipher.key.data, 520 NULL) != 1) { 521 return -EINVAL; 522 } 523 } 524 525 break; 526 default: 527 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 528 return -ENOTSUP; 529 } 530 531 return 0; 532 } 533 534 /* Set session auth parameters */ 535 static int 536 openssl_set_session_auth_parameters(struct openssl_session *sess, 537 const struct rte_crypto_sym_xform *xform) 538 { 539 /* Select auth generate/verify */ 540 sess->auth.operation = xform->auth.op; 541 sess->auth.algo = xform->auth.algo; 542 543 sess->auth.digest_length = xform->auth.digest_length; 544 545 /* Select auth algo */ 546 switch (xform->auth.algo) { 547 case RTE_CRYPTO_AUTH_AES_GMAC: 548 /* 549 * OpenSSL requires GMAC to be a GCM operation 550 * with no cipher data length 551 */ 552 sess->cipher.key.length = xform->auth.key.length; 553 554 /* Set IV parameters */ 555 sess->iv.offset = xform->auth.iv.offset; 556 sess->iv.length = xform->auth.iv.length; 557 558 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 559 return openssl_set_sess_aead_enc_param(sess, 560 RTE_CRYPTO_AEAD_AES_GCM, 561 xform->auth.digest_length, 562 xform->auth.key.data); 563 else 564 return openssl_set_sess_aead_dec_param(sess, 565 RTE_CRYPTO_AEAD_AES_GCM, 566 xform->auth.digest_length, 567 xform->auth.key.data); 568 break; 569 570 case RTE_CRYPTO_AUTH_MD5: 571 case RTE_CRYPTO_AUTH_SHA1: 572 case RTE_CRYPTO_AUTH_SHA224: 573 case RTE_CRYPTO_AUTH_SHA256: 574 case RTE_CRYPTO_AUTH_SHA384: 575 case RTE_CRYPTO_AUTH_SHA512: 576 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 577 if (get_auth_algo(xform->auth.algo, 578 &sess->auth.auth.evp_algo) != 0) 579 return -EINVAL; 580 sess->auth.auth.ctx = EVP_MD_CTX_create(); 581 break; 582 583 case RTE_CRYPTO_AUTH_MD5_HMAC: 584 case RTE_CRYPTO_AUTH_SHA1_HMAC: 585 case RTE_CRYPTO_AUTH_SHA224_HMAC: 586 case RTE_CRYPTO_AUTH_SHA256_HMAC: 587 case RTE_CRYPTO_AUTH_SHA384_HMAC: 588 case RTE_CRYPTO_AUTH_SHA512_HMAC: 589 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 590 sess->auth.hmac.ctx = HMAC_CTX_new(); 591 if (get_auth_algo(xform->auth.algo, 592 &sess->auth.hmac.evp_algo) != 0) 593 return -EINVAL; 594 595 if (HMAC_Init_ex(sess->auth.hmac.ctx, 596 xform->auth.key.data, 597 xform->auth.key.length, 598 sess->auth.hmac.evp_algo, NULL) != 1) 599 return -EINVAL; 600 break; 601 602 default: 603 return -ENOTSUP; 604 } 605 606 return 0; 607 } 608 609 /* Set session AEAD parameters */ 610 static int 611 openssl_set_session_aead_parameters(struct openssl_session *sess, 612 const struct rte_crypto_sym_xform *xform) 613 { 614 /* Select cipher key */ 615 sess->cipher.key.length = xform->aead.key.length; 616 617 /* Set IV parameters */ 618 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 619 /* 620 * For AES-CCM, the actual IV is placed 621 * one byte after the start of the IV field, 622 * according to the API. 623 */ 624 sess->iv.offset = xform->aead.iv.offset + 1; 625 else 626 sess->iv.offset = xform->aead.iv.offset; 627 628 sess->iv.length = xform->aead.iv.length; 629 630 sess->auth.aad_length = xform->aead.aad_length; 631 sess->auth.digest_length = xform->aead.digest_length; 632 633 sess->aead_algo = xform->aead.algo; 634 /* Select cipher direction */ 635 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 636 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 637 xform->aead.digest_length, xform->aead.key.data); 638 else 639 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 640 xform->aead.digest_length, xform->aead.key.data); 641 } 642 643 /** Parse crypto xform chain and set private session parameters */ 644 int 645 openssl_set_session_parameters(struct openssl_session *sess, 646 const struct rte_crypto_sym_xform *xform) 647 { 648 const struct rte_crypto_sym_xform *cipher_xform = NULL; 649 const struct rte_crypto_sym_xform *auth_xform = NULL; 650 const struct rte_crypto_sym_xform *aead_xform = NULL; 651 int ret; 652 653 sess->chain_order = openssl_get_chain_order(xform); 654 switch (sess->chain_order) { 655 case OPENSSL_CHAIN_ONLY_CIPHER: 656 cipher_xform = xform; 657 break; 658 case OPENSSL_CHAIN_ONLY_AUTH: 659 auth_xform = xform; 660 break; 661 case OPENSSL_CHAIN_CIPHER_AUTH: 662 cipher_xform = xform; 663 auth_xform = xform->next; 664 break; 665 case OPENSSL_CHAIN_AUTH_CIPHER: 666 auth_xform = xform; 667 cipher_xform = xform->next; 668 break; 669 case OPENSSL_CHAIN_COMBINED: 670 aead_xform = xform; 671 break; 672 default: 673 return -EINVAL; 674 } 675 676 /* Default IV length = 0 */ 677 sess->iv.length = 0; 678 679 /* cipher_xform must be check before auth_xform */ 680 if (cipher_xform) { 681 ret = openssl_set_session_cipher_parameters( 682 sess, cipher_xform); 683 if (ret != 0) { 684 OPENSSL_LOG(ERR, 685 "Invalid/unsupported cipher parameters"); 686 return ret; 687 } 688 } 689 690 if (auth_xform) { 691 ret = openssl_set_session_auth_parameters(sess, auth_xform); 692 if (ret != 0) { 693 OPENSSL_LOG(ERR, 694 "Invalid/unsupported auth parameters"); 695 return ret; 696 } 697 } 698 699 if (aead_xform) { 700 ret = openssl_set_session_aead_parameters(sess, aead_xform); 701 if (ret != 0) { 702 OPENSSL_LOG(ERR, 703 "Invalid/unsupported AEAD parameters"); 704 return ret; 705 } 706 } 707 708 return 0; 709 } 710 711 /** Reset private session parameters */ 712 void 713 openssl_reset_session(struct openssl_session *sess) 714 { 715 EVP_CIPHER_CTX_free(sess->cipher.ctx); 716 717 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 718 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 719 720 switch (sess->auth.mode) { 721 case OPENSSL_AUTH_AS_AUTH: 722 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 723 break; 724 case OPENSSL_AUTH_AS_HMAC: 725 EVP_PKEY_free(sess->auth.hmac.pkey); 726 HMAC_CTX_free(sess->auth.hmac.ctx); 727 break; 728 default: 729 break; 730 } 731 } 732 733 /** Provide session for operation */ 734 static void * 735 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 736 { 737 struct openssl_session *sess = NULL; 738 struct openssl_asym_session *asym_sess = NULL; 739 740 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 741 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 742 /* get existing session */ 743 if (likely(op->sym->session != NULL)) 744 sess = (struct openssl_session *) 745 get_sym_session_private_data( 746 op->sym->session, 747 cryptodev_driver_id); 748 } else { 749 if (likely(op->asym->session != NULL)) 750 asym_sess = (struct openssl_asym_session *) 751 get_asym_session_private_data( 752 op->asym->session, 753 cryptodev_driver_id); 754 if (asym_sess == NULL) 755 op->status = 756 RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 757 return asym_sess; 758 } 759 } else { 760 /* sessionless asymmetric not supported */ 761 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) 762 return NULL; 763 764 /* provide internal session */ 765 void *_sess = NULL; 766 void *_sess_private_data = NULL; 767 768 if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) 769 return NULL; 770 771 if (rte_mempool_get(qp->sess_mp_priv, 772 (void **)&_sess_private_data)) 773 return NULL; 774 775 sess = (struct openssl_session *)_sess_private_data; 776 777 if (unlikely(openssl_set_session_parameters(sess, 778 op->sym->xform) != 0)) { 779 rte_mempool_put(qp->sess_mp, _sess); 780 rte_mempool_put(qp->sess_mp_priv, _sess_private_data); 781 sess = NULL; 782 } 783 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 784 set_sym_session_private_data(op->sym->session, 785 cryptodev_driver_id, _sess_private_data); 786 } 787 788 if (sess == NULL) 789 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 790 791 return sess; 792 } 793 794 /* 795 *------------------------------------------------------------------------------ 796 * Process Operations 797 *------------------------------------------------------------------------------ 798 */ 799 static inline int 800 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 801 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 802 { 803 struct rte_mbuf *m; 804 int dstlen; 805 int l, n = srclen; 806 uint8_t *src; 807 808 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 809 m = m->next) 810 offset -= rte_pktmbuf_data_len(m); 811 812 if (m == 0) 813 return -1; 814 815 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 816 817 l = rte_pktmbuf_data_len(m) - offset; 818 if (srclen <= l) { 819 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 820 return -1; 821 *dst += l; 822 return 0; 823 } 824 825 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 826 return -1; 827 828 *dst += dstlen; 829 n -= l; 830 831 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 832 src = rte_pktmbuf_mtod(m, uint8_t *); 833 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 834 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 835 return -1; 836 *dst += dstlen; 837 n -= l; 838 } 839 840 return 0; 841 } 842 843 static inline int 844 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 845 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 846 { 847 struct rte_mbuf *m; 848 int dstlen; 849 int l, n = srclen; 850 uint8_t *src; 851 852 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 853 m = m->next) 854 offset -= rte_pktmbuf_data_len(m); 855 856 if (m == 0) 857 return -1; 858 859 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 860 861 l = rte_pktmbuf_data_len(m) - offset; 862 if (srclen <= l) { 863 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 864 return -1; 865 *dst += l; 866 return 0; 867 } 868 869 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 870 return -1; 871 872 *dst += dstlen; 873 n -= l; 874 875 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 876 src = rte_pktmbuf_mtod(m, uint8_t *); 877 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 878 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 879 return -1; 880 *dst += dstlen; 881 n -= l; 882 } 883 884 return 0; 885 } 886 887 /** Process standard openssl cipher encryption */ 888 static int 889 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 890 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) 891 { 892 int totlen; 893 894 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 895 goto process_cipher_encrypt_err; 896 897 EVP_CIPHER_CTX_set_padding(ctx, 0); 898 899 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 900 srclen, ctx)) 901 goto process_cipher_encrypt_err; 902 903 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 904 goto process_cipher_encrypt_err; 905 906 return 0; 907 908 process_cipher_encrypt_err: 909 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 910 return -EINVAL; 911 } 912 913 /** Process standard openssl cipher encryption */ 914 static int 915 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 916 uint8_t *iv, int srclen, 917 EVP_CIPHER_CTX *ctx) 918 { 919 uint8_t i; 920 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 921 int encrypted_ivlen; 922 923 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 924 iv, DES_BLOCK_SIZE) <= 0) 925 goto process_cipher_encrypt_err; 926 927 for (i = 0; i < srclen; i++) 928 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 929 930 return 0; 931 932 process_cipher_encrypt_err: 933 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 934 return -EINVAL; 935 } 936 /** Process standard openssl cipher decryption */ 937 static int 938 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 939 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) 940 { 941 int totlen; 942 943 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 944 goto process_cipher_decrypt_err; 945 946 EVP_CIPHER_CTX_set_padding(ctx, 0); 947 948 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 949 srclen, ctx)) 950 goto process_cipher_decrypt_err; 951 952 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 953 goto process_cipher_decrypt_err; 954 return 0; 955 956 process_cipher_decrypt_err: 957 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 958 return -EINVAL; 959 } 960 961 /** Process cipher des 3 ctr encryption, decryption algorithm */ 962 static int 963 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 964 int offset, uint8_t *iv, uint8_t *key, int srclen, 965 EVP_CIPHER_CTX *ctx) 966 { 967 uint8_t ebuf[8], ctr[8]; 968 int unused, n; 969 struct rte_mbuf *m; 970 uint8_t *src; 971 int l; 972 973 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 974 m = m->next) 975 offset -= rte_pktmbuf_data_len(m); 976 977 if (m == 0) 978 goto process_cipher_des3ctr_err; 979 980 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 981 l = rte_pktmbuf_data_len(m) - offset; 982 983 /* We use 3DES encryption also for decryption. 984 * IV is not important for 3DES ecb 985 */ 986 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 987 goto process_cipher_des3ctr_err; 988 989 memcpy(ctr, iv, 8); 990 991 for (n = 0; n < srclen; n++) { 992 if (n % 8 == 0) { 993 if (EVP_EncryptUpdate(ctx, 994 (unsigned char *)&ebuf, &unused, 995 (const unsigned char *)&ctr, 8) <= 0) 996 goto process_cipher_des3ctr_err; 997 ctr_inc(ctr); 998 } 999 dst[n] = *(src++) ^ ebuf[n % 8]; 1000 1001 l--; 1002 if (!l) { 1003 m = m->next; 1004 if (m) { 1005 src = rte_pktmbuf_mtod(m, uint8_t *); 1006 l = rte_pktmbuf_data_len(m); 1007 } 1008 } 1009 } 1010 1011 return 0; 1012 1013 process_cipher_des3ctr_err: 1014 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 1015 return -EINVAL; 1016 } 1017 1018 /** Process AES-GCM encrypt algorithm */ 1019 static int 1020 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1021 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1022 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1023 { 1024 int len = 0, unused = 0; 1025 uint8_t empty[] = {}; 1026 1027 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1028 goto process_auth_encryption_gcm_err; 1029 1030 if (aadlen > 0) 1031 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1032 goto process_auth_encryption_gcm_err; 1033 1034 if (srclen > 0) 1035 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1036 srclen, ctx)) 1037 goto process_auth_encryption_gcm_err; 1038 1039 /* Workaround open ssl bug in version less then 1.0.1f */ 1040 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1041 goto process_auth_encryption_gcm_err; 1042 1043 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1044 goto process_auth_encryption_gcm_err; 1045 1046 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1047 goto process_auth_encryption_gcm_err; 1048 1049 return 0; 1050 1051 process_auth_encryption_gcm_err: 1052 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1053 return -EINVAL; 1054 } 1055 1056 /** Process AES-CCM encrypt algorithm */ 1057 static int 1058 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1059 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1060 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1061 { 1062 int len = 0; 1063 1064 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1065 goto process_auth_encryption_ccm_err; 1066 1067 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1068 goto process_auth_encryption_ccm_err; 1069 1070 if (aadlen > 0) 1071 /* 1072 * For AES-CCM, the actual AAD is placed 1073 * 18 bytes after the start of the AAD field, 1074 * according to the API. 1075 */ 1076 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1077 goto process_auth_encryption_ccm_err; 1078 1079 if (srclen > 0) 1080 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1081 srclen, ctx)) 1082 goto process_auth_encryption_ccm_err; 1083 1084 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1085 goto process_auth_encryption_ccm_err; 1086 1087 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1088 goto process_auth_encryption_ccm_err; 1089 1090 return 0; 1091 1092 process_auth_encryption_ccm_err: 1093 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1094 return -EINVAL; 1095 } 1096 1097 /** Process AES-GCM decrypt algorithm */ 1098 static int 1099 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1100 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1101 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1102 { 1103 int len = 0, unused = 0; 1104 uint8_t empty[] = {}; 1105 1106 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1107 goto process_auth_decryption_gcm_err; 1108 1109 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1110 goto process_auth_decryption_gcm_err; 1111 1112 if (aadlen > 0) 1113 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1114 goto process_auth_decryption_gcm_err; 1115 1116 if (srclen > 0) 1117 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1118 srclen, ctx)) 1119 goto process_auth_decryption_gcm_err; 1120 1121 /* Workaround open ssl bug in version less then 1.0.1f */ 1122 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1123 goto process_auth_decryption_gcm_err; 1124 1125 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1126 return -EFAULT; 1127 1128 return 0; 1129 1130 process_auth_decryption_gcm_err: 1131 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1132 return -EINVAL; 1133 } 1134 1135 /** Process AES-CCM decrypt algorithm */ 1136 static int 1137 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1138 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1139 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1140 EVP_CIPHER_CTX *ctx) 1141 { 1142 int len = 0; 1143 1144 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1145 goto process_auth_decryption_ccm_err; 1146 1147 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1148 goto process_auth_decryption_ccm_err; 1149 1150 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1151 goto process_auth_decryption_ccm_err; 1152 1153 if (aadlen > 0) 1154 /* 1155 * For AES-CCM, the actual AAD is placed 1156 * 18 bytes after the start of the AAD field, 1157 * according to the API. 1158 */ 1159 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1160 goto process_auth_decryption_ccm_err; 1161 1162 if (srclen > 0) 1163 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1164 srclen, ctx)) 1165 return -EFAULT; 1166 1167 return 0; 1168 1169 process_auth_decryption_ccm_err: 1170 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1171 return -EINVAL; 1172 } 1173 1174 /** Process standard openssl auth algorithms */ 1175 static int 1176 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1177 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1178 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1179 { 1180 size_t dstlen; 1181 struct rte_mbuf *m; 1182 int l, n = srclen; 1183 uint8_t *src; 1184 1185 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1186 m = m->next) 1187 offset -= rte_pktmbuf_data_len(m); 1188 1189 if (m == 0) 1190 goto process_auth_err; 1191 1192 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1193 goto process_auth_err; 1194 1195 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1196 1197 l = rte_pktmbuf_data_len(m) - offset; 1198 if (srclen <= l) { 1199 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1200 goto process_auth_err; 1201 goto process_auth_final; 1202 } 1203 1204 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1205 goto process_auth_err; 1206 1207 n -= l; 1208 1209 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1210 src = rte_pktmbuf_mtod(m, uint8_t *); 1211 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1212 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1213 goto process_auth_err; 1214 n -= l; 1215 } 1216 1217 process_auth_final: 1218 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1219 goto process_auth_err; 1220 return 0; 1221 1222 process_auth_err: 1223 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1224 return -EINVAL; 1225 } 1226 1227 /** Process standard openssl auth algorithms with hmac */ 1228 static int 1229 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1230 int srclen, HMAC_CTX *ctx) 1231 { 1232 unsigned int dstlen; 1233 struct rte_mbuf *m; 1234 int l, n = srclen; 1235 uint8_t *src; 1236 1237 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1238 m = m->next) 1239 offset -= rte_pktmbuf_data_len(m); 1240 1241 if (m == 0) 1242 goto process_auth_err; 1243 1244 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1245 1246 l = rte_pktmbuf_data_len(m) - offset; 1247 if (srclen <= l) { 1248 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1249 goto process_auth_err; 1250 goto process_auth_final; 1251 } 1252 1253 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1254 goto process_auth_err; 1255 1256 n -= l; 1257 1258 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1259 src = rte_pktmbuf_mtod(m, uint8_t *); 1260 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1261 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1262 goto process_auth_err; 1263 n -= l; 1264 } 1265 1266 process_auth_final: 1267 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1268 goto process_auth_err; 1269 1270 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1271 goto process_auth_err; 1272 1273 return 0; 1274 1275 process_auth_err: 1276 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1277 return -EINVAL; 1278 } 1279 1280 /*----------------------------------------------------------------------------*/ 1281 1282 /** Process auth/cipher combined operation */ 1283 static void 1284 process_openssl_combined_op 1285 (struct rte_crypto_op *op, struct openssl_session *sess, 1286 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1287 { 1288 /* cipher */ 1289 uint8_t *dst = NULL, *iv, *tag, *aad; 1290 int srclen, aadlen, status = -1; 1291 uint32_t offset; 1292 uint8_t taglen; 1293 1294 /* 1295 * Segmented destination buffer is not supported for 1296 * encryption/decryption 1297 */ 1298 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1299 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1300 return; 1301 } 1302 1303 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1304 sess->iv.offset); 1305 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1306 srclen = 0; 1307 offset = op->sym->auth.data.offset; 1308 aadlen = op->sym->auth.data.length; 1309 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1310 op->sym->auth.data.offset); 1311 tag = op->sym->auth.digest.data; 1312 if (tag == NULL) 1313 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1314 offset + aadlen); 1315 } else { 1316 srclen = op->sym->aead.data.length; 1317 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1318 op->sym->aead.data.offset); 1319 offset = op->sym->aead.data.offset; 1320 aad = op->sym->aead.aad.data; 1321 aadlen = sess->auth.aad_length; 1322 tag = op->sym->aead.digest.data; 1323 if (tag == NULL) 1324 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1325 offset + srclen); 1326 } 1327 1328 taglen = sess->auth.digest_length; 1329 1330 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1331 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1332 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1333 status = process_openssl_auth_encryption_gcm( 1334 mbuf_src, offset, srclen, 1335 aad, aadlen, iv, 1336 dst, tag, sess->cipher.ctx); 1337 else 1338 status = process_openssl_auth_encryption_ccm( 1339 mbuf_src, offset, srclen, 1340 aad, aadlen, iv, 1341 dst, tag, taglen, sess->cipher.ctx); 1342 1343 } else { 1344 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1345 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1346 status = process_openssl_auth_decryption_gcm( 1347 mbuf_src, offset, srclen, 1348 aad, aadlen, iv, 1349 dst, tag, sess->cipher.ctx); 1350 else 1351 status = process_openssl_auth_decryption_ccm( 1352 mbuf_src, offset, srclen, 1353 aad, aadlen, iv, 1354 dst, tag, taglen, sess->cipher.ctx); 1355 } 1356 1357 if (status != 0) { 1358 if (status == (-EFAULT) && 1359 sess->auth.operation == 1360 RTE_CRYPTO_AUTH_OP_VERIFY) 1361 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1362 else 1363 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1364 } 1365 } 1366 1367 /** Process cipher operation */ 1368 static void 1369 process_openssl_cipher_op 1370 (struct rte_crypto_op *op, struct openssl_session *sess, 1371 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1372 { 1373 uint8_t *dst, *iv; 1374 int srclen, status; 1375 1376 /* 1377 * Segmented destination buffer is not supported for 1378 * encryption/decryption 1379 */ 1380 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1381 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1382 return; 1383 } 1384 1385 srclen = op->sym->cipher.data.length; 1386 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1387 op->sym->cipher.data.offset); 1388 1389 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1390 sess->iv.offset); 1391 1392 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1393 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1394 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1395 op->sym->cipher.data.offset, iv, 1396 srclen, sess->cipher.ctx); 1397 else 1398 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1399 op->sym->cipher.data.offset, iv, 1400 srclen, sess->cipher.ctx); 1401 else 1402 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1403 op->sym->cipher.data.offset, iv, 1404 sess->cipher.key.data, srclen, 1405 sess->cipher.ctx); 1406 1407 if (status != 0) 1408 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1409 } 1410 1411 /** Process cipher operation */ 1412 static void 1413 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1414 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1415 struct rte_mbuf *mbuf_dst) 1416 { 1417 uint8_t *src, *dst, *iv; 1418 uint8_t block_size, last_block_len; 1419 int srclen, status = 0; 1420 1421 srclen = op->sym->cipher.data.length; 1422 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1423 op->sym->cipher.data.offset); 1424 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1425 op->sym->cipher.data.offset); 1426 1427 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1428 sess->iv.offset); 1429 1430 block_size = DES_BLOCK_SIZE; 1431 1432 last_block_len = srclen % block_size; 1433 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1434 /* Encrypt only with ECB mode XOR IV */ 1435 if (srclen < block_size) { 1436 status = process_openssl_cipher_bpi_encrypt(src, dst, 1437 iv, srclen, 1438 sess->cipher.bpi_ctx); 1439 } else { 1440 srclen -= last_block_len; 1441 /* Encrypt with the block aligned stream with CBC mode */ 1442 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1443 op->sym->cipher.data.offset, iv, 1444 srclen, sess->cipher.ctx); 1445 if (last_block_len) { 1446 /* Point at last block */ 1447 dst += srclen; 1448 /* 1449 * IV is the last encrypted block from 1450 * the previous operation 1451 */ 1452 iv = dst - block_size; 1453 src += srclen; 1454 srclen = last_block_len; 1455 /* Encrypt the last frame with ECB mode */ 1456 status |= process_openssl_cipher_bpi_encrypt(src, 1457 dst, iv, 1458 srclen, sess->cipher.bpi_ctx); 1459 } 1460 } 1461 } else { 1462 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1463 if (srclen < block_size) { 1464 status = process_openssl_cipher_bpi_encrypt(src, dst, 1465 iv, 1466 srclen, 1467 sess->cipher.bpi_ctx); 1468 } else { 1469 if (last_block_len) { 1470 /* Point at last block */ 1471 dst += srclen - last_block_len; 1472 src += srclen - last_block_len; 1473 /* 1474 * IV is the last full block 1475 */ 1476 iv = src - block_size; 1477 /* 1478 * Decrypt the last frame with ECB mode 1479 * (encrypt, as it is the same operation) 1480 */ 1481 status = process_openssl_cipher_bpi_encrypt(src, 1482 dst, iv, 1483 last_block_len, sess->cipher.bpi_ctx); 1484 /* Prepare parameters for CBC mode op */ 1485 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1486 sess->iv.offset); 1487 dst += last_block_len - srclen; 1488 srclen -= last_block_len; 1489 } 1490 1491 /* Decrypt with CBC mode */ 1492 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1493 op->sym->cipher.data.offset, iv, 1494 srclen, sess->cipher.ctx); 1495 } 1496 } 1497 1498 if (status != 0) 1499 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1500 } 1501 1502 /** Process auth operation */ 1503 static void 1504 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1505 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1506 struct rte_mbuf *mbuf_dst) 1507 { 1508 uint8_t *dst; 1509 int srclen, status; 1510 1511 srclen = op->sym->auth.data.length; 1512 1513 dst = qp->temp_digest; 1514 1515 switch (sess->auth.mode) { 1516 case OPENSSL_AUTH_AS_AUTH: 1517 status = process_openssl_auth(mbuf_src, dst, 1518 op->sym->auth.data.offset, NULL, NULL, srclen, 1519 sess->auth.auth.ctx, sess->auth.auth.evp_algo); 1520 break; 1521 case OPENSSL_AUTH_AS_HMAC: 1522 status = process_openssl_auth_hmac(mbuf_src, dst, 1523 op->sym->auth.data.offset, srclen, 1524 sess->auth.hmac.ctx); 1525 break; 1526 default: 1527 status = -1; 1528 break; 1529 } 1530 1531 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1532 if (memcmp(dst, op->sym->auth.digest.data, 1533 sess->auth.digest_length) != 0) { 1534 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1535 } 1536 } else { 1537 uint8_t *auth_dst; 1538 1539 auth_dst = op->sym->auth.digest.data; 1540 if (auth_dst == NULL) 1541 auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1542 op->sym->auth.data.offset + 1543 op->sym->auth.data.length); 1544 memcpy(auth_dst, dst, sess->auth.digest_length); 1545 } 1546 1547 if (status != 0) 1548 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1549 } 1550 1551 /* process dsa sign operation */ 1552 static int 1553 process_openssl_dsa_sign_op(struct rte_crypto_op *cop, 1554 struct openssl_asym_session *sess) 1555 { 1556 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1557 DSA *dsa = sess->u.s.dsa; 1558 DSA_SIG *sign = NULL; 1559 1560 sign = DSA_do_sign(op->message.data, 1561 op->message.length, 1562 dsa); 1563 1564 if (sign == NULL) { 1565 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 1566 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1567 } else { 1568 const BIGNUM *r = NULL, *s = NULL; 1569 get_dsa_sign(sign, &r, &s); 1570 1571 op->r.length = BN_bn2bin(r, op->r.data); 1572 op->s.length = BN_bn2bin(s, op->s.data); 1573 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1574 } 1575 1576 DSA_SIG_free(sign); 1577 1578 return 0; 1579 } 1580 1581 /* process dsa verify operation */ 1582 static int 1583 process_openssl_dsa_verify_op(struct rte_crypto_op *cop, 1584 struct openssl_asym_session *sess) 1585 { 1586 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1587 DSA *dsa = sess->u.s.dsa; 1588 int ret; 1589 DSA_SIG *sign = DSA_SIG_new(); 1590 BIGNUM *r = NULL, *s = NULL; 1591 BIGNUM *pub_key = NULL; 1592 1593 if (sign == NULL) { 1594 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 1595 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1596 return -1; 1597 } 1598 1599 r = BN_bin2bn(op->r.data, 1600 op->r.length, 1601 r); 1602 s = BN_bin2bn(op->s.data, 1603 op->s.length, 1604 s); 1605 pub_key = BN_bin2bn(op->y.data, 1606 op->y.length, 1607 pub_key); 1608 if (!r || !s || !pub_key) { 1609 BN_free(r); 1610 BN_free(s); 1611 BN_free(pub_key); 1612 1613 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1614 return -1; 1615 } 1616 set_dsa_sign(sign, r, s); 1617 set_dsa_pub_key(dsa, pub_key); 1618 1619 ret = DSA_do_verify(op->message.data, 1620 op->message.length, 1621 sign, 1622 dsa); 1623 1624 if (ret != 1) 1625 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1626 else 1627 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1628 1629 DSA_SIG_free(sign); 1630 1631 return 0; 1632 } 1633 1634 /* process dh operation */ 1635 static int 1636 process_openssl_dh_op(struct rte_crypto_op *cop, 1637 struct openssl_asym_session *sess) 1638 { 1639 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 1640 DH *dh_key = sess->u.dh.dh_key; 1641 BIGNUM *priv_key = NULL; 1642 int ret = 0; 1643 1644 if (sess->u.dh.key_op & 1645 (1 << RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE)) { 1646 /* compute shared secret using peer public key 1647 * and current private key 1648 * shared secret = peer_key ^ priv_key mod p 1649 */ 1650 BIGNUM *peer_key = NULL; 1651 1652 /* copy private key and peer key and compute shared secret */ 1653 peer_key = BN_bin2bn(op->pub_key.data, 1654 op->pub_key.length, 1655 peer_key); 1656 if (peer_key == NULL) { 1657 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1658 return -1; 1659 } 1660 priv_key = BN_bin2bn(op->priv_key.data, 1661 op->priv_key.length, 1662 priv_key); 1663 if (priv_key == NULL) { 1664 BN_free(peer_key); 1665 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1666 return -1; 1667 } 1668 ret = set_dh_priv_key(dh_key, priv_key); 1669 if (ret) { 1670 OPENSSL_LOG(ERR, "Failed to set private key\n"); 1671 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1672 BN_free(peer_key); 1673 BN_free(priv_key); 1674 return 0; 1675 } 1676 1677 ret = DH_compute_key( 1678 op->shared_secret.data, 1679 peer_key, dh_key); 1680 if (ret < 0) { 1681 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1682 BN_free(peer_key); 1683 /* priv key is already loaded into dh, 1684 * let's not free that directly here. 1685 * DH_free() will auto free it later. 1686 */ 1687 return 0; 1688 } 1689 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1690 op->shared_secret.length = ret; 1691 BN_free(peer_key); 1692 return 0; 1693 } 1694 1695 /* 1696 * other options are public and private key generations. 1697 * 1698 * if user provides private key, 1699 * then first set DH with user provided private key 1700 */ 1701 if ((sess->u.dh.key_op & 1702 (1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)) && 1703 !(sess->u.dh.key_op & 1704 (1 << RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE))) { 1705 /* generate public key using user-provided private key 1706 * pub_key = g ^ priv_key mod p 1707 */ 1708 1709 /* load private key into DH */ 1710 priv_key = BN_bin2bn(op->priv_key.data, 1711 op->priv_key.length, 1712 priv_key); 1713 if (priv_key == NULL) { 1714 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1715 return -1; 1716 } 1717 ret = set_dh_priv_key(dh_key, priv_key); 1718 if (ret) { 1719 OPENSSL_LOG(ERR, "Failed to set private key\n"); 1720 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1721 BN_free(priv_key); 1722 return 0; 1723 } 1724 } 1725 1726 /* generate public and private key pair. 1727 * 1728 * if private key already set, generates only public key. 1729 * 1730 * if private key is not already set, then set it to random value 1731 * and update internal private key. 1732 */ 1733 if (!DH_generate_key(dh_key)) { 1734 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1735 return 0; 1736 } 1737 1738 if (sess->u.dh.key_op & (1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)) { 1739 const BIGNUM *pub_key = NULL; 1740 1741 OPENSSL_LOG(DEBUG, "%s:%d update public key\n", 1742 __func__, __LINE__); 1743 1744 /* get the generated keys */ 1745 get_dh_pub_key(dh_key, &pub_key); 1746 1747 /* output public key */ 1748 op->pub_key.length = BN_bn2bin(pub_key, 1749 op->pub_key.data); 1750 } 1751 1752 if (sess->u.dh.key_op & 1753 (1 << RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE)) { 1754 const BIGNUM *priv_key = NULL; 1755 1756 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", 1757 __func__, __LINE__); 1758 1759 /* get the generated keys */ 1760 get_dh_priv_key(dh_key, &priv_key); 1761 1762 /* provide generated private key back to user */ 1763 op->priv_key.length = BN_bn2bin(priv_key, 1764 op->priv_key.data); 1765 } 1766 1767 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1768 1769 return 0; 1770 } 1771 1772 /* process modinv operation */ 1773 static int 1774 process_openssl_modinv_op(struct rte_crypto_op *cop, 1775 struct openssl_asym_session *sess) 1776 { 1777 struct rte_crypto_asym_op *op = cop->asym; 1778 BIGNUM *base = BN_CTX_get(sess->u.m.ctx); 1779 BIGNUM *res = BN_CTX_get(sess->u.m.ctx); 1780 1781 if (unlikely(base == NULL || res == NULL)) { 1782 BN_free(base); 1783 BN_free(res); 1784 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1785 return -1; 1786 } 1787 1788 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 1789 op->modinv.base.length, base); 1790 1791 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) { 1792 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1793 op->modinv.base.length = BN_bn2bin(res, op->modinv.base.data); 1794 } else { 1795 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1796 } 1797 1798 return 0; 1799 } 1800 1801 /* process modexp operation */ 1802 static int 1803 process_openssl_modexp_op(struct rte_crypto_op *cop, 1804 struct openssl_asym_session *sess) 1805 { 1806 struct rte_crypto_asym_op *op = cop->asym; 1807 BIGNUM *base = BN_CTX_get(sess->u.e.ctx); 1808 BIGNUM *res = BN_CTX_get(sess->u.e.ctx); 1809 1810 if (unlikely(base == NULL || res == NULL)) { 1811 BN_free(base); 1812 BN_free(res); 1813 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1814 return -1; 1815 } 1816 1817 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 1818 op->modinv.base.length, base); 1819 1820 if (BN_mod_exp(res, base, sess->u.e.exp, 1821 sess->u.e.mod, sess->u.e.ctx)) { 1822 op->modinv.base.length = BN_bn2bin(res, op->modinv.base.data); 1823 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1824 } else { 1825 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1826 } 1827 1828 return 0; 1829 } 1830 1831 /* process rsa operations */ 1832 static int 1833 process_openssl_rsa_op(struct rte_crypto_op *cop, 1834 struct openssl_asym_session *sess) 1835 { 1836 int ret = 0; 1837 struct rte_crypto_asym_op *op = cop->asym; 1838 RSA *rsa = sess->u.r.rsa; 1839 uint32_t pad = (op->rsa.pad); 1840 uint8_t *tmp; 1841 1842 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1843 1844 switch (pad) { 1845 case RTE_CRYPTO_RSA_PKCS1_V1_5_BT0: 1846 case RTE_CRYPTO_RSA_PKCS1_V1_5_BT1: 1847 case RTE_CRYPTO_RSA_PKCS1_V1_5_BT2: 1848 pad = RSA_PKCS1_PADDING; 1849 break; 1850 case RTE_CRYPTO_RSA_PADDING_NONE: 1851 pad = RSA_NO_PADDING; 1852 break; 1853 default: 1854 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1855 OPENSSL_LOG(ERR, 1856 "rsa pad type not supported %d\n", pad); 1857 return 0; 1858 } 1859 1860 switch (op->rsa.op_type) { 1861 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 1862 ret = RSA_public_encrypt(op->rsa.message.length, 1863 op->rsa.message.data, 1864 op->rsa.message.data, 1865 rsa, 1866 pad); 1867 1868 if (ret > 0) 1869 op->rsa.message.length = ret; 1870 OPENSSL_LOG(DEBUG, 1871 "length of encrypted text %d\n", ret); 1872 break; 1873 1874 case RTE_CRYPTO_ASYM_OP_DECRYPT: 1875 ret = RSA_private_decrypt(op->rsa.message.length, 1876 op->rsa.message.data, 1877 op->rsa.message.data, 1878 rsa, 1879 pad); 1880 if (ret > 0) 1881 op->rsa.message.length = ret; 1882 break; 1883 1884 case RTE_CRYPTO_ASYM_OP_SIGN: 1885 ret = RSA_private_encrypt(op->rsa.message.length, 1886 op->rsa.message.data, 1887 op->rsa.sign.data, 1888 rsa, 1889 pad); 1890 if (ret > 0) 1891 op->rsa.sign.length = ret; 1892 break; 1893 1894 case RTE_CRYPTO_ASYM_OP_VERIFY: 1895 tmp = rte_malloc(NULL, op->rsa.sign.length, 0); 1896 if (tmp == NULL) { 1897 OPENSSL_LOG(ERR, "Memory allocation failed"); 1898 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1899 break; 1900 } 1901 ret = RSA_public_decrypt(op->rsa.sign.length, 1902 op->rsa.sign.data, 1903 tmp, 1904 rsa, 1905 pad); 1906 1907 OPENSSL_LOG(DEBUG, 1908 "Length of public_decrypt %d " 1909 "length of message %zd\n", 1910 ret, op->rsa.message.length); 1911 if ((ret <= 0) || (memcmp(tmp, op->rsa.message.data, 1912 op->rsa.message.length))) { 1913 OPENSSL_LOG(ERR, "RSA sign Verification failed"); 1914 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1915 } 1916 rte_free(tmp); 1917 break; 1918 1919 default: 1920 /* allow ops with invalid args to be pushed to 1921 * completion queue 1922 */ 1923 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1924 break; 1925 } 1926 1927 if (ret < 0) 1928 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1929 1930 return 0; 1931 } 1932 1933 static int 1934 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1935 struct openssl_asym_session *sess) 1936 { 1937 int retval = 0; 1938 1939 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1940 1941 switch (sess->xfrm_type) { 1942 case RTE_CRYPTO_ASYM_XFORM_RSA: 1943 retval = process_openssl_rsa_op(op, sess); 1944 break; 1945 case RTE_CRYPTO_ASYM_XFORM_MODEX: 1946 retval = process_openssl_modexp_op(op, sess); 1947 break; 1948 case RTE_CRYPTO_ASYM_XFORM_MODINV: 1949 retval = process_openssl_modinv_op(op, sess); 1950 break; 1951 case RTE_CRYPTO_ASYM_XFORM_DH: 1952 retval = process_openssl_dh_op(op, sess); 1953 break; 1954 case RTE_CRYPTO_ASYM_XFORM_DSA: 1955 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 1956 retval = process_openssl_dsa_sign_op(op, sess); 1957 else if (op->asym->dsa.op_type == 1958 RTE_CRYPTO_ASYM_OP_VERIFY) 1959 retval = 1960 process_openssl_dsa_verify_op(op, sess); 1961 else 1962 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1963 break; 1964 default: 1965 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1966 break; 1967 } 1968 if (!retval) { 1969 /* op processed so push to completion queue as processed */ 1970 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 1971 if (retval) 1972 /* return error if failed to put in completion queue */ 1973 retval = -1; 1974 } 1975 1976 return retval; 1977 } 1978 1979 /** Process crypto operation for mbuf */ 1980 static int 1981 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1982 struct openssl_session *sess) 1983 { 1984 struct rte_mbuf *msrc, *mdst; 1985 int retval; 1986 1987 msrc = op->sym->m_src; 1988 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 1989 1990 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1991 1992 switch (sess->chain_order) { 1993 case OPENSSL_CHAIN_ONLY_CIPHER: 1994 process_openssl_cipher_op(op, sess, msrc, mdst); 1995 break; 1996 case OPENSSL_CHAIN_ONLY_AUTH: 1997 process_openssl_auth_op(qp, op, sess, msrc, mdst); 1998 break; 1999 case OPENSSL_CHAIN_CIPHER_AUTH: 2000 process_openssl_cipher_op(op, sess, msrc, mdst); 2001 process_openssl_auth_op(qp, op, sess, mdst, mdst); 2002 break; 2003 case OPENSSL_CHAIN_AUTH_CIPHER: 2004 process_openssl_auth_op(qp, op, sess, msrc, mdst); 2005 process_openssl_cipher_op(op, sess, msrc, mdst); 2006 break; 2007 case OPENSSL_CHAIN_COMBINED: 2008 process_openssl_combined_op(op, sess, msrc, mdst); 2009 break; 2010 case OPENSSL_CHAIN_CIPHER_BPI: 2011 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 2012 break; 2013 default: 2014 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 2015 break; 2016 } 2017 2018 /* Free session if a session-less crypto op */ 2019 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 2020 openssl_reset_session(sess); 2021 memset(sess, 0, sizeof(struct openssl_session)); 2022 memset(op->sym->session, 0, 2023 rte_cryptodev_sym_get_existing_header_session_size( 2024 op->sym->session)); 2025 rte_mempool_put(qp->sess_mp_priv, sess); 2026 rte_mempool_put(qp->sess_mp, op->sym->session); 2027 op->sym->session = NULL; 2028 } 2029 2030 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 2031 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2032 2033 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 2034 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 2035 else 2036 retval = -1; 2037 2038 return retval; 2039 } 2040 2041 /* 2042 *------------------------------------------------------------------------------ 2043 * PMD Framework 2044 *------------------------------------------------------------------------------ 2045 */ 2046 2047 /** Enqueue burst */ 2048 static uint16_t 2049 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 2050 uint16_t nb_ops) 2051 { 2052 void *sess; 2053 struct openssl_qp *qp = queue_pair; 2054 int i, retval; 2055 2056 for (i = 0; i < nb_ops; i++) { 2057 sess = get_session(qp, ops[i]); 2058 if (unlikely(sess == NULL)) 2059 goto enqueue_err; 2060 2061 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) 2062 retval = process_op(qp, ops[i], 2063 (struct openssl_session *) sess); 2064 else 2065 retval = process_asym_op(qp, ops[i], 2066 (struct openssl_asym_session *) sess); 2067 if (unlikely(retval < 0)) 2068 goto enqueue_err; 2069 } 2070 2071 qp->stats.enqueued_count += i; 2072 return i; 2073 2074 enqueue_err: 2075 qp->stats.enqueue_err_count++; 2076 return i; 2077 } 2078 2079 /** Dequeue burst */ 2080 static uint16_t 2081 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 2082 uint16_t nb_ops) 2083 { 2084 struct openssl_qp *qp = queue_pair; 2085 2086 unsigned int nb_dequeued = 0; 2087 2088 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 2089 (void **)ops, nb_ops, NULL); 2090 qp->stats.dequeued_count += nb_dequeued; 2091 2092 return nb_dequeued; 2093 } 2094 2095 /** Create OPENSSL crypto device */ 2096 static int 2097 cryptodev_openssl_create(const char *name, 2098 struct rte_vdev_device *vdev, 2099 struct rte_cryptodev_pmd_init_params *init_params) 2100 { 2101 struct rte_cryptodev *dev; 2102 struct openssl_private *internals; 2103 2104 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 2105 if (dev == NULL) { 2106 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 2107 goto init_error; 2108 } 2109 2110 dev->driver_id = cryptodev_driver_id; 2111 dev->dev_ops = rte_openssl_pmd_ops; 2112 2113 /* register rx/tx burst functions for data path */ 2114 dev->dequeue_burst = openssl_pmd_dequeue_burst; 2115 dev->enqueue_burst = openssl_pmd_enqueue_burst; 2116 2117 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 2118 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 2119 RTE_CRYPTODEV_FF_CPU_AESNI | 2120 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 2121 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | 2122 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO; 2123 2124 /* Set vector instructions mode supported */ 2125 internals = dev->data->dev_private; 2126 2127 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 2128 2129 return 0; 2130 2131 init_error: 2132 OPENSSL_LOG(ERR, "driver %s: create failed", 2133 init_params->name); 2134 2135 cryptodev_openssl_remove(vdev); 2136 return -EFAULT; 2137 } 2138 2139 /** Initialise OPENSSL crypto device */ 2140 static int 2141 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 2142 { 2143 struct rte_cryptodev_pmd_init_params init_params = { 2144 "", 2145 sizeof(struct openssl_private), 2146 rte_socket_id(), 2147 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 2148 }; 2149 const char *name; 2150 const char *input_args; 2151 2152 name = rte_vdev_device_name(vdev); 2153 if (name == NULL) 2154 return -EINVAL; 2155 input_args = rte_vdev_device_args(vdev); 2156 2157 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 2158 2159 return cryptodev_openssl_create(name, vdev, &init_params); 2160 } 2161 2162 /** Uninitialise OPENSSL crypto device */ 2163 static int 2164 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 2165 { 2166 struct rte_cryptodev *cryptodev; 2167 const char *name; 2168 2169 name = rte_vdev_device_name(vdev); 2170 if (name == NULL) 2171 return -EINVAL; 2172 2173 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 2174 if (cryptodev == NULL) 2175 return -ENODEV; 2176 2177 return rte_cryptodev_pmd_destroy(cryptodev); 2178 } 2179 2180 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 2181 .probe = cryptodev_openssl_probe, 2182 .remove = cryptodev_openssl_remove 2183 }; 2184 2185 static struct cryptodev_driver openssl_crypto_drv; 2186 2187 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 2188 cryptodev_openssl_pmd_drv); 2189 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 2190 "max_nb_queue_pairs=<int> " 2191 "socket_id=<int>"); 2192 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 2193 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 2194 2195 RTE_INIT(openssl_init_log) 2196 { 2197 openssl_logtype_driver = rte_log_register("pmd.crypto.openssl"); 2198 } 2199