1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_common.h> 6 #include <rte_hexdump.h> 7 #include <rte_cryptodev.h> 8 #include <rte_cryptodev_pmd.h> 9 #include <rte_bus_vdev.h> 10 #include <rte_malloc.h> 11 #include <rte_cpuflags.h> 12 13 #include <openssl/hmac.h> 14 #include <openssl/evp.h> 15 16 #include "rte_openssl_pmd_private.h" 17 #include "compat.h" 18 19 #define DES_BLOCK_SIZE 8 20 21 static uint8_t cryptodev_driver_id; 22 23 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 24 static HMAC_CTX *HMAC_CTX_new(void) 25 { 26 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 27 28 if (ctx != NULL) 29 HMAC_CTX_init(ctx); 30 return ctx; 31 } 32 33 static void HMAC_CTX_free(HMAC_CTX *ctx) 34 { 35 if (ctx != NULL) { 36 HMAC_CTX_cleanup(ctx); 37 OPENSSL_free(ctx); 38 } 39 } 40 #endif 41 42 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 43 44 /*----------------------------------------------------------------------------*/ 45 46 /** 47 * Increment counter by 1 48 * Counter is 64 bit array, big-endian 49 */ 50 static void 51 ctr_inc(uint8_t *ctr) 52 { 53 uint64_t *ctr64 = (uint64_t *)ctr; 54 55 *ctr64 = __builtin_bswap64(*ctr64); 56 (*ctr64)++; 57 *ctr64 = __builtin_bswap64(*ctr64); 58 } 59 60 /* 61 *------------------------------------------------------------------------------ 62 * Session Prepare 63 *------------------------------------------------------------------------------ 64 */ 65 66 /** Get xform chain order */ 67 static enum openssl_chain_order 68 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 69 { 70 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 71 72 if (xform != NULL) { 73 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 74 if (xform->next == NULL) 75 res = OPENSSL_CHAIN_ONLY_AUTH; 76 else if (xform->next->type == 77 RTE_CRYPTO_SYM_XFORM_CIPHER) 78 res = OPENSSL_CHAIN_AUTH_CIPHER; 79 } 80 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 81 if (xform->next == NULL) 82 res = OPENSSL_CHAIN_ONLY_CIPHER; 83 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 84 res = OPENSSL_CHAIN_CIPHER_AUTH; 85 } 86 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 87 res = OPENSSL_CHAIN_COMBINED; 88 } 89 90 return res; 91 } 92 93 /** Get session cipher key from input cipher key */ 94 static void 95 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key) 96 { 97 memcpy(session_key, input_key, keylen); 98 } 99 100 /** Get key ede 24 bytes standard from input key */ 101 static int 102 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede) 103 { 104 int res = 0; 105 106 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 107 switch (keylen) { 108 case 24: 109 memcpy(key_ede, key, 24); 110 break; 111 case 16: 112 /* K3 = K1 */ 113 memcpy(key_ede, key, 16); 114 memcpy(key_ede + 16, key, 8); 115 break; 116 case 8: 117 /* K1 = K2 = K3 (DES compatibility) */ 118 memcpy(key_ede, key, 8); 119 memcpy(key_ede + 8, key, 8); 120 memcpy(key_ede + 16, key, 8); 121 break; 122 default: 123 OPENSSL_LOG(ERR, "Unsupported key size"); 124 res = -EINVAL; 125 } 126 127 return res; 128 } 129 130 /** Get adequate openssl function for input cipher algorithm */ 131 static uint8_t 132 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 133 const EVP_CIPHER **algo) 134 { 135 int res = 0; 136 137 if (algo != NULL) { 138 switch (sess_algo) { 139 case RTE_CRYPTO_CIPHER_3DES_CBC: 140 switch (keylen) { 141 case 8: 142 *algo = EVP_des_cbc(); 143 break; 144 case 16: 145 *algo = EVP_des_ede_cbc(); 146 break; 147 case 24: 148 *algo = EVP_des_ede3_cbc(); 149 break; 150 default: 151 res = -EINVAL; 152 } 153 break; 154 case RTE_CRYPTO_CIPHER_3DES_CTR: 155 break; 156 case RTE_CRYPTO_CIPHER_AES_CBC: 157 switch (keylen) { 158 case 16: 159 *algo = EVP_aes_128_cbc(); 160 break; 161 case 24: 162 *algo = EVP_aes_192_cbc(); 163 break; 164 case 32: 165 *algo = EVP_aes_256_cbc(); 166 break; 167 default: 168 res = -EINVAL; 169 } 170 break; 171 case RTE_CRYPTO_CIPHER_AES_CTR: 172 switch (keylen) { 173 case 16: 174 *algo = EVP_aes_128_ctr(); 175 break; 176 case 24: 177 *algo = EVP_aes_192_ctr(); 178 break; 179 case 32: 180 *algo = EVP_aes_256_ctr(); 181 break; 182 default: 183 res = -EINVAL; 184 } 185 break; 186 default: 187 res = -EINVAL; 188 break; 189 } 190 } else { 191 res = -EINVAL; 192 } 193 194 return res; 195 } 196 197 /** Get adequate openssl function for input auth algorithm */ 198 static uint8_t 199 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 200 const EVP_MD **algo) 201 { 202 int res = 0; 203 204 if (algo != NULL) { 205 switch (sessalgo) { 206 case RTE_CRYPTO_AUTH_MD5: 207 case RTE_CRYPTO_AUTH_MD5_HMAC: 208 *algo = EVP_md5(); 209 break; 210 case RTE_CRYPTO_AUTH_SHA1: 211 case RTE_CRYPTO_AUTH_SHA1_HMAC: 212 *algo = EVP_sha1(); 213 break; 214 case RTE_CRYPTO_AUTH_SHA224: 215 case RTE_CRYPTO_AUTH_SHA224_HMAC: 216 *algo = EVP_sha224(); 217 break; 218 case RTE_CRYPTO_AUTH_SHA256: 219 case RTE_CRYPTO_AUTH_SHA256_HMAC: 220 *algo = EVP_sha256(); 221 break; 222 case RTE_CRYPTO_AUTH_SHA384: 223 case RTE_CRYPTO_AUTH_SHA384_HMAC: 224 *algo = EVP_sha384(); 225 break; 226 case RTE_CRYPTO_AUTH_SHA512: 227 case RTE_CRYPTO_AUTH_SHA512_HMAC: 228 *algo = EVP_sha512(); 229 break; 230 default: 231 res = -EINVAL; 232 break; 233 } 234 } else { 235 res = -EINVAL; 236 } 237 238 return res; 239 } 240 241 /** Get adequate openssl function for input cipher algorithm */ 242 static uint8_t 243 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 244 const EVP_CIPHER **algo) 245 { 246 int res = 0; 247 248 if (algo != NULL) { 249 switch (sess_algo) { 250 case RTE_CRYPTO_AEAD_AES_GCM: 251 switch (keylen) { 252 case 16: 253 *algo = EVP_aes_128_gcm(); 254 break; 255 case 24: 256 *algo = EVP_aes_192_gcm(); 257 break; 258 case 32: 259 *algo = EVP_aes_256_gcm(); 260 break; 261 default: 262 res = -EINVAL; 263 } 264 break; 265 case RTE_CRYPTO_AEAD_AES_CCM: 266 switch (keylen) { 267 case 16: 268 *algo = EVP_aes_128_ccm(); 269 break; 270 case 24: 271 *algo = EVP_aes_192_ccm(); 272 break; 273 case 32: 274 *algo = EVP_aes_256_ccm(); 275 break; 276 default: 277 res = -EINVAL; 278 } 279 break; 280 default: 281 res = -EINVAL; 282 break; 283 } 284 } else { 285 res = -EINVAL; 286 } 287 288 return res; 289 } 290 291 /* Set session AEAD encryption parameters */ 292 static int 293 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 294 enum rte_crypto_aead_algorithm algo, 295 uint8_t tag_len, uint8_t *key) 296 { 297 int iv_type = 0; 298 unsigned int do_ccm; 299 300 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 301 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 302 303 /* Select AEAD algo */ 304 switch (algo) { 305 case RTE_CRYPTO_AEAD_AES_GCM: 306 iv_type = EVP_CTRL_GCM_SET_IVLEN; 307 if (tag_len != 16) 308 return -EINVAL; 309 do_ccm = 0; 310 break; 311 case RTE_CRYPTO_AEAD_AES_CCM: 312 iv_type = EVP_CTRL_CCM_SET_IVLEN; 313 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 314 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 315 return -EINVAL; 316 do_ccm = 1; 317 break; 318 default: 319 return -ENOTSUP; 320 } 321 322 sess->cipher.mode = OPENSSL_CIPHER_LIB; 323 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 324 325 if (get_aead_algo(algo, sess->cipher.key.length, 326 &sess->cipher.evp_algo) != 0) 327 return -EINVAL; 328 329 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 330 331 sess->chain_order = OPENSSL_CHAIN_COMBINED; 332 333 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 334 NULL, NULL, NULL) <= 0) 335 return -EINVAL; 336 337 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, 338 NULL) <= 0) 339 return -EINVAL; 340 341 if (do_ccm) 342 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 343 tag_len, NULL); 344 345 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 346 return -EINVAL; 347 348 return 0; 349 } 350 351 /* Set session AEAD decryption parameters */ 352 static int 353 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 354 enum rte_crypto_aead_algorithm algo, 355 uint8_t tag_len, uint8_t *key) 356 { 357 int iv_type = 0; 358 unsigned int do_ccm = 0; 359 360 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 361 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 362 363 /* Select AEAD algo */ 364 switch (algo) { 365 case RTE_CRYPTO_AEAD_AES_GCM: 366 iv_type = EVP_CTRL_GCM_SET_IVLEN; 367 if (tag_len != 16) 368 return -EINVAL; 369 break; 370 case RTE_CRYPTO_AEAD_AES_CCM: 371 iv_type = EVP_CTRL_CCM_SET_IVLEN; 372 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 373 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 374 return -EINVAL; 375 do_ccm = 1; 376 break; 377 default: 378 return -ENOTSUP; 379 } 380 381 sess->cipher.mode = OPENSSL_CIPHER_LIB; 382 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 383 384 if (get_aead_algo(algo, sess->cipher.key.length, 385 &sess->cipher.evp_algo) != 0) 386 return -EINVAL; 387 388 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 389 390 sess->chain_order = OPENSSL_CHAIN_COMBINED; 391 392 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 393 NULL, NULL, NULL) <= 0) 394 return -EINVAL; 395 396 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, 397 sess->iv.length, NULL) <= 0) 398 return -EINVAL; 399 400 if (do_ccm) 401 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 402 tag_len, NULL); 403 404 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 405 return -EINVAL; 406 407 return 0; 408 } 409 410 /** Set session cipher parameters */ 411 static int 412 openssl_set_session_cipher_parameters(struct openssl_session *sess, 413 const struct rte_crypto_sym_xform *xform) 414 { 415 /* Select cipher direction */ 416 sess->cipher.direction = xform->cipher.op; 417 /* Select cipher key */ 418 sess->cipher.key.length = xform->cipher.key.length; 419 420 /* Set IV parameters */ 421 sess->iv.offset = xform->cipher.iv.offset; 422 sess->iv.length = xform->cipher.iv.length; 423 424 /* Select cipher algo */ 425 switch (xform->cipher.algo) { 426 case RTE_CRYPTO_CIPHER_3DES_CBC: 427 case RTE_CRYPTO_CIPHER_AES_CBC: 428 case RTE_CRYPTO_CIPHER_AES_CTR: 429 sess->cipher.mode = OPENSSL_CIPHER_LIB; 430 sess->cipher.algo = xform->cipher.algo; 431 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 432 433 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 434 &sess->cipher.evp_algo) != 0) 435 return -EINVAL; 436 437 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 438 sess->cipher.key.data); 439 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 440 if (EVP_EncryptInit_ex(sess->cipher.ctx, 441 sess->cipher.evp_algo, 442 NULL, xform->cipher.key.data, 443 NULL) != 1) { 444 return -EINVAL; 445 } 446 } else if (sess->cipher.direction == 447 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 448 if (EVP_DecryptInit_ex(sess->cipher.ctx, 449 sess->cipher.evp_algo, 450 NULL, xform->cipher.key.data, 451 NULL) != 1) { 452 return -EINVAL; 453 } 454 } 455 456 break; 457 458 case RTE_CRYPTO_CIPHER_3DES_CTR: 459 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 460 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 461 462 if (get_cipher_key_ede(xform->cipher.key.data, 463 sess->cipher.key.length, 464 sess->cipher.key.data) != 0) 465 return -EINVAL; 466 break; 467 468 case RTE_CRYPTO_CIPHER_DES_CBC: 469 sess->cipher.algo = xform->cipher.algo; 470 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 471 sess->cipher.evp_algo = EVP_des_cbc(); 472 473 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 474 sess->cipher.key.data); 475 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 476 if (EVP_EncryptInit_ex(sess->cipher.ctx, 477 sess->cipher.evp_algo, 478 NULL, xform->cipher.key.data, 479 NULL) != 1) { 480 return -EINVAL; 481 } 482 } else if (sess->cipher.direction == 483 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 484 if (EVP_DecryptInit_ex(sess->cipher.ctx, 485 sess->cipher.evp_algo, 486 NULL, xform->cipher.key.data, 487 NULL) != 1) { 488 return -EINVAL; 489 } 490 } 491 492 break; 493 494 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 495 sess->cipher.algo = xform->cipher.algo; 496 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 497 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 498 sess->cipher.evp_algo = EVP_des_cbc(); 499 500 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 501 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 502 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 503 NULL, xform->cipher.key.data, 0) != 1) 504 return -EINVAL; 505 506 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 507 sess->cipher.key.data); 508 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 509 if (EVP_EncryptInit_ex(sess->cipher.ctx, 510 sess->cipher.evp_algo, 511 NULL, xform->cipher.key.data, 512 NULL) != 1) { 513 return -EINVAL; 514 } 515 } else if (sess->cipher.direction == 516 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 517 if (EVP_DecryptInit_ex(sess->cipher.ctx, 518 sess->cipher.evp_algo, 519 NULL, xform->cipher.key.data, 520 NULL) != 1) { 521 return -EINVAL; 522 } 523 } 524 525 break; 526 default: 527 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 528 return -ENOTSUP; 529 } 530 531 return 0; 532 } 533 534 /* Set session auth parameters */ 535 static int 536 openssl_set_session_auth_parameters(struct openssl_session *sess, 537 const struct rte_crypto_sym_xform *xform) 538 { 539 /* Select auth generate/verify */ 540 sess->auth.operation = xform->auth.op; 541 sess->auth.algo = xform->auth.algo; 542 543 sess->auth.digest_length = xform->auth.digest_length; 544 545 /* Select auth algo */ 546 switch (xform->auth.algo) { 547 case RTE_CRYPTO_AUTH_AES_GMAC: 548 /* 549 * OpenSSL requires GMAC to be a GCM operation 550 * with no cipher data length 551 */ 552 sess->cipher.key.length = xform->auth.key.length; 553 554 /* Set IV parameters */ 555 sess->iv.offset = xform->auth.iv.offset; 556 sess->iv.length = xform->auth.iv.length; 557 558 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 559 return openssl_set_sess_aead_enc_param(sess, 560 RTE_CRYPTO_AEAD_AES_GCM, 561 xform->auth.digest_length, 562 xform->auth.key.data); 563 else 564 return openssl_set_sess_aead_dec_param(sess, 565 RTE_CRYPTO_AEAD_AES_GCM, 566 xform->auth.digest_length, 567 xform->auth.key.data); 568 break; 569 570 case RTE_CRYPTO_AUTH_MD5: 571 case RTE_CRYPTO_AUTH_SHA1: 572 case RTE_CRYPTO_AUTH_SHA224: 573 case RTE_CRYPTO_AUTH_SHA256: 574 case RTE_CRYPTO_AUTH_SHA384: 575 case RTE_CRYPTO_AUTH_SHA512: 576 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 577 if (get_auth_algo(xform->auth.algo, 578 &sess->auth.auth.evp_algo) != 0) 579 return -EINVAL; 580 sess->auth.auth.ctx = EVP_MD_CTX_create(); 581 break; 582 583 case RTE_CRYPTO_AUTH_MD5_HMAC: 584 case RTE_CRYPTO_AUTH_SHA1_HMAC: 585 case RTE_CRYPTO_AUTH_SHA224_HMAC: 586 case RTE_CRYPTO_AUTH_SHA256_HMAC: 587 case RTE_CRYPTO_AUTH_SHA384_HMAC: 588 case RTE_CRYPTO_AUTH_SHA512_HMAC: 589 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 590 sess->auth.hmac.ctx = HMAC_CTX_new(); 591 if (get_auth_algo(xform->auth.algo, 592 &sess->auth.hmac.evp_algo) != 0) 593 return -EINVAL; 594 595 if (HMAC_Init_ex(sess->auth.hmac.ctx, 596 xform->auth.key.data, 597 xform->auth.key.length, 598 sess->auth.hmac.evp_algo, NULL) != 1) 599 return -EINVAL; 600 break; 601 602 default: 603 return -ENOTSUP; 604 } 605 606 return 0; 607 } 608 609 /* Set session AEAD parameters */ 610 static int 611 openssl_set_session_aead_parameters(struct openssl_session *sess, 612 const struct rte_crypto_sym_xform *xform) 613 { 614 /* Select cipher key */ 615 sess->cipher.key.length = xform->aead.key.length; 616 617 /* Set IV parameters */ 618 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 619 /* 620 * For AES-CCM, the actual IV is placed 621 * one byte after the start of the IV field, 622 * according to the API. 623 */ 624 sess->iv.offset = xform->aead.iv.offset + 1; 625 else 626 sess->iv.offset = xform->aead.iv.offset; 627 628 sess->iv.length = xform->aead.iv.length; 629 630 sess->auth.aad_length = xform->aead.aad_length; 631 sess->auth.digest_length = xform->aead.digest_length; 632 633 sess->aead_algo = xform->aead.algo; 634 /* Select cipher direction */ 635 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 636 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 637 xform->aead.digest_length, xform->aead.key.data); 638 else 639 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 640 xform->aead.digest_length, xform->aead.key.data); 641 } 642 643 /** Parse crypto xform chain and set private session parameters */ 644 int 645 openssl_set_session_parameters(struct openssl_session *sess, 646 const struct rte_crypto_sym_xform *xform) 647 { 648 const struct rte_crypto_sym_xform *cipher_xform = NULL; 649 const struct rte_crypto_sym_xform *auth_xform = NULL; 650 const struct rte_crypto_sym_xform *aead_xform = NULL; 651 int ret; 652 653 sess->chain_order = openssl_get_chain_order(xform); 654 switch (sess->chain_order) { 655 case OPENSSL_CHAIN_ONLY_CIPHER: 656 cipher_xform = xform; 657 break; 658 case OPENSSL_CHAIN_ONLY_AUTH: 659 auth_xform = xform; 660 break; 661 case OPENSSL_CHAIN_CIPHER_AUTH: 662 cipher_xform = xform; 663 auth_xform = xform->next; 664 break; 665 case OPENSSL_CHAIN_AUTH_CIPHER: 666 auth_xform = xform; 667 cipher_xform = xform->next; 668 break; 669 case OPENSSL_CHAIN_COMBINED: 670 aead_xform = xform; 671 break; 672 default: 673 return -EINVAL; 674 } 675 676 /* Default IV length = 0 */ 677 sess->iv.length = 0; 678 679 /* cipher_xform must be check before auth_xform */ 680 if (cipher_xform) { 681 ret = openssl_set_session_cipher_parameters( 682 sess, cipher_xform); 683 if (ret != 0) { 684 OPENSSL_LOG(ERR, 685 "Invalid/unsupported cipher parameters"); 686 return ret; 687 } 688 } 689 690 if (auth_xform) { 691 ret = openssl_set_session_auth_parameters(sess, auth_xform); 692 if (ret != 0) { 693 OPENSSL_LOG(ERR, 694 "Invalid/unsupported auth parameters"); 695 return ret; 696 } 697 } 698 699 if (aead_xform) { 700 ret = openssl_set_session_aead_parameters(sess, aead_xform); 701 if (ret != 0) { 702 OPENSSL_LOG(ERR, 703 "Invalid/unsupported AEAD parameters"); 704 return ret; 705 } 706 } 707 708 return 0; 709 } 710 711 /** Reset private session parameters */ 712 void 713 openssl_reset_session(struct openssl_session *sess) 714 { 715 EVP_CIPHER_CTX_free(sess->cipher.ctx); 716 717 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 718 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 719 720 switch (sess->auth.mode) { 721 case OPENSSL_AUTH_AS_AUTH: 722 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 723 break; 724 case OPENSSL_AUTH_AS_HMAC: 725 EVP_PKEY_free(sess->auth.hmac.pkey); 726 HMAC_CTX_free(sess->auth.hmac.ctx); 727 break; 728 default: 729 break; 730 } 731 } 732 733 /** Provide session for operation */ 734 static void * 735 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 736 { 737 struct openssl_session *sess = NULL; 738 struct openssl_asym_session *asym_sess = NULL; 739 740 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 741 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 742 /* get existing session */ 743 if (likely(op->sym->session != NULL)) 744 sess = (struct openssl_session *) 745 get_sym_session_private_data( 746 op->sym->session, 747 cryptodev_driver_id); 748 } else { 749 if (likely(op->asym->session != NULL)) 750 asym_sess = (struct openssl_asym_session *) 751 get_asym_session_private_data( 752 op->asym->session, 753 cryptodev_driver_id); 754 if (asym_sess == NULL) 755 op->status = 756 RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 757 return asym_sess; 758 } 759 } else { 760 /* sessionless asymmetric not supported */ 761 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) 762 return NULL; 763 764 /* provide internal session */ 765 void *_sess = NULL; 766 void *_sess_private_data = NULL; 767 768 if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) 769 return NULL; 770 771 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) 772 return NULL; 773 774 sess = (struct openssl_session *)_sess_private_data; 775 776 if (unlikely(openssl_set_session_parameters(sess, 777 op->sym->xform) != 0)) { 778 rte_mempool_put(qp->sess_mp, _sess); 779 rte_mempool_put(qp->sess_mp, _sess_private_data); 780 sess = NULL; 781 } 782 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 783 set_sym_session_private_data(op->sym->session, 784 cryptodev_driver_id, _sess_private_data); 785 } 786 787 if (sess == NULL) 788 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 789 790 return sess; 791 } 792 793 /* 794 *------------------------------------------------------------------------------ 795 * Process Operations 796 *------------------------------------------------------------------------------ 797 */ 798 static inline int 799 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 800 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 801 { 802 struct rte_mbuf *m; 803 int dstlen; 804 int l, n = srclen; 805 uint8_t *src; 806 807 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 808 m = m->next) 809 offset -= rte_pktmbuf_data_len(m); 810 811 if (m == 0) 812 return -1; 813 814 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 815 816 l = rte_pktmbuf_data_len(m) - offset; 817 if (srclen <= l) { 818 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 819 return -1; 820 *dst += l; 821 return 0; 822 } 823 824 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 825 return -1; 826 827 *dst += dstlen; 828 n -= l; 829 830 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 831 src = rte_pktmbuf_mtod(m, uint8_t *); 832 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 833 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 834 return -1; 835 *dst += dstlen; 836 n -= l; 837 } 838 839 return 0; 840 } 841 842 static inline int 843 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 844 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 845 { 846 struct rte_mbuf *m; 847 int dstlen; 848 int l, n = srclen; 849 uint8_t *src; 850 851 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 852 m = m->next) 853 offset -= rte_pktmbuf_data_len(m); 854 855 if (m == 0) 856 return -1; 857 858 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 859 860 l = rte_pktmbuf_data_len(m) - offset; 861 if (srclen <= l) { 862 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 863 return -1; 864 *dst += l; 865 return 0; 866 } 867 868 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 869 return -1; 870 871 *dst += dstlen; 872 n -= l; 873 874 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 875 src = rte_pktmbuf_mtod(m, uint8_t *); 876 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 877 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 878 return -1; 879 *dst += dstlen; 880 n -= l; 881 } 882 883 return 0; 884 } 885 886 /** Process standard openssl cipher encryption */ 887 static int 888 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 889 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) 890 { 891 int totlen; 892 893 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 894 goto process_cipher_encrypt_err; 895 896 EVP_CIPHER_CTX_set_padding(ctx, 0); 897 898 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 899 srclen, ctx)) 900 goto process_cipher_encrypt_err; 901 902 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 903 goto process_cipher_encrypt_err; 904 905 return 0; 906 907 process_cipher_encrypt_err: 908 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 909 return -EINVAL; 910 } 911 912 /** Process standard openssl cipher encryption */ 913 static int 914 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 915 uint8_t *iv, int srclen, 916 EVP_CIPHER_CTX *ctx) 917 { 918 uint8_t i; 919 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 920 int encrypted_ivlen; 921 922 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 923 iv, DES_BLOCK_SIZE) <= 0) 924 goto process_cipher_encrypt_err; 925 926 for (i = 0; i < srclen; i++) 927 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 928 929 return 0; 930 931 process_cipher_encrypt_err: 932 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 933 return -EINVAL; 934 } 935 /** Process standard openssl cipher decryption */ 936 static int 937 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 938 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) 939 { 940 int totlen; 941 942 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 943 goto process_cipher_decrypt_err; 944 945 EVP_CIPHER_CTX_set_padding(ctx, 0); 946 947 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 948 srclen, ctx)) 949 goto process_cipher_decrypt_err; 950 951 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 952 goto process_cipher_decrypt_err; 953 return 0; 954 955 process_cipher_decrypt_err: 956 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 957 return -EINVAL; 958 } 959 960 /** Process cipher des 3 ctr encryption, decryption algorithm */ 961 static int 962 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 963 int offset, uint8_t *iv, uint8_t *key, int srclen, 964 EVP_CIPHER_CTX *ctx) 965 { 966 uint8_t ebuf[8], ctr[8]; 967 int unused, n; 968 struct rte_mbuf *m; 969 uint8_t *src; 970 int l; 971 972 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 973 m = m->next) 974 offset -= rte_pktmbuf_data_len(m); 975 976 if (m == 0) 977 goto process_cipher_des3ctr_err; 978 979 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 980 l = rte_pktmbuf_data_len(m) - offset; 981 982 /* We use 3DES encryption also for decryption. 983 * IV is not important for 3DES ecb 984 */ 985 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 986 goto process_cipher_des3ctr_err; 987 988 memcpy(ctr, iv, 8); 989 990 for (n = 0; n < srclen; n++) { 991 if (n % 8 == 0) { 992 if (EVP_EncryptUpdate(ctx, 993 (unsigned char *)&ebuf, &unused, 994 (const unsigned char *)&ctr, 8) <= 0) 995 goto process_cipher_des3ctr_err; 996 ctr_inc(ctr); 997 } 998 dst[n] = *(src++) ^ ebuf[n % 8]; 999 1000 l--; 1001 if (!l) { 1002 m = m->next; 1003 if (m) { 1004 src = rte_pktmbuf_mtod(m, uint8_t *); 1005 l = rte_pktmbuf_data_len(m); 1006 } 1007 } 1008 } 1009 1010 return 0; 1011 1012 process_cipher_des3ctr_err: 1013 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 1014 return -EINVAL; 1015 } 1016 1017 /** Process AES-GCM encrypt algorithm */ 1018 static int 1019 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1020 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1021 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1022 { 1023 int len = 0, unused = 0; 1024 uint8_t empty[] = {}; 1025 1026 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1027 goto process_auth_encryption_gcm_err; 1028 1029 if (aadlen > 0) 1030 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1031 goto process_auth_encryption_gcm_err; 1032 1033 if (srclen > 0) 1034 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1035 srclen, ctx)) 1036 goto process_auth_encryption_gcm_err; 1037 1038 /* Workaround open ssl bug in version less then 1.0.1f */ 1039 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1040 goto process_auth_encryption_gcm_err; 1041 1042 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1043 goto process_auth_encryption_gcm_err; 1044 1045 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1046 goto process_auth_encryption_gcm_err; 1047 1048 return 0; 1049 1050 process_auth_encryption_gcm_err: 1051 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1052 return -EINVAL; 1053 } 1054 1055 /** Process AES-CCM encrypt algorithm */ 1056 static int 1057 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1058 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1059 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1060 { 1061 int len = 0; 1062 1063 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1064 goto process_auth_encryption_ccm_err; 1065 1066 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1067 goto process_auth_encryption_ccm_err; 1068 1069 if (aadlen > 0) 1070 /* 1071 * For AES-CCM, the actual AAD is placed 1072 * 18 bytes after the start of the AAD field, 1073 * according to the API. 1074 */ 1075 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1076 goto process_auth_encryption_ccm_err; 1077 1078 if (srclen > 0) 1079 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1080 srclen, ctx)) 1081 goto process_auth_encryption_ccm_err; 1082 1083 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1084 goto process_auth_encryption_ccm_err; 1085 1086 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1087 goto process_auth_encryption_ccm_err; 1088 1089 return 0; 1090 1091 process_auth_encryption_ccm_err: 1092 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1093 return -EINVAL; 1094 } 1095 1096 /** Process AES-GCM decrypt algorithm */ 1097 static int 1098 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1099 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1100 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1101 { 1102 int len = 0, unused = 0; 1103 uint8_t empty[] = {}; 1104 1105 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1106 goto process_auth_decryption_gcm_err; 1107 1108 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1109 goto process_auth_decryption_gcm_err; 1110 1111 if (aadlen > 0) 1112 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1113 goto process_auth_decryption_gcm_err; 1114 1115 if (srclen > 0) 1116 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1117 srclen, ctx)) 1118 goto process_auth_decryption_gcm_err; 1119 1120 /* Workaround open ssl bug in version less then 1.0.1f */ 1121 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1122 goto process_auth_decryption_gcm_err; 1123 1124 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1125 return -EFAULT; 1126 1127 return 0; 1128 1129 process_auth_decryption_gcm_err: 1130 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1131 return -EINVAL; 1132 } 1133 1134 /** Process AES-CCM decrypt algorithm */ 1135 static int 1136 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1137 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1138 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1139 EVP_CIPHER_CTX *ctx) 1140 { 1141 int len = 0; 1142 1143 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1144 goto process_auth_decryption_ccm_err; 1145 1146 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1147 goto process_auth_decryption_ccm_err; 1148 1149 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1150 goto process_auth_decryption_ccm_err; 1151 1152 if (aadlen > 0) 1153 /* 1154 * For AES-CCM, the actual AAD is placed 1155 * 18 bytes after the start of the AAD field, 1156 * according to the API. 1157 */ 1158 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1159 goto process_auth_decryption_ccm_err; 1160 1161 if (srclen > 0) 1162 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1163 srclen, ctx)) 1164 return -EFAULT; 1165 1166 return 0; 1167 1168 process_auth_decryption_ccm_err: 1169 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1170 return -EINVAL; 1171 } 1172 1173 /** Process standard openssl auth algorithms */ 1174 static int 1175 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1176 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1177 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1178 { 1179 size_t dstlen; 1180 struct rte_mbuf *m; 1181 int l, n = srclen; 1182 uint8_t *src; 1183 1184 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1185 m = m->next) 1186 offset -= rte_pktmbuf_data_len(m); 1187 1188 if (m == 0) 1189 goto process_auth_err; 1190 1191 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1192 goto process_auth_err; 1193 1194 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1195 1196 l = rte_pktmbuf_data_len(m) - offset; 1197 if (srclen <= l) { 1198 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1199 goto process_auth_err; 1200 goto process_auth_final; 1201 } 1202 1203 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1204 goto process_auth_err; 1205 1206 n -= l; 1207 1208 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1209 src = rte_pktmbuf_mtod(m, uint8_t *); 1210 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1211 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1212 goto process_auth_err; 1213 n -= l; 1214 } 1215 1216 process_auth_final: 1217 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1218 goto process_auth_err; 1219 return 0; 1220 1221 process_auth_err: 1222 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1223 return -EINVAL; 1224 } 1225 1226 /** Process standard openssl auth algorithms with hmac */ 1227 static int 1228 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1229 int srclen, HMAC_CTX *ctx) 1230 { 1231 unsigned int dstlen; 1232 struct rte_mbuf *m; 1233 int l, n = srclen; 1234 uint8_t *src; 1235 1236 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1237 m = m->next) 1238 offset -= rte_pktmbuf_data_len(m); 1239 1240 if (m == 0) 1241 goto process_auth_err; 1242 1243 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1244 1245 l = rte_pktmbuf_data_len(m) - offset; 1246 if (srclen <= l) { 1247 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1248 goto process_auth_err; 1249 goto process_auth_final; 1250 } 1251 1252 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1253 goto process_auth_err; 1254 1255 n -= l; 1256 1257 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1258 src = rte_pktmbuf_mtod(m, uint8_t *); 1259 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1260 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1261 goto process_auth_err; 1262 n -= l; 1263 } 1264 1265 process_auth_final: 1266 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1267 goto process_auth_err; 1268 1269 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1270 goto process_auth_err; 1271 1272 return 0; 1273 1274 process_auth_err: 1275 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1276 return -EINVAL; 1277 } 1278 1279 /*----------------------------------------------------------------------------*/ 1280 1281 /** Process auth/cipher combined operation */ 1282 static void 1283 process_openssl_combined_op 1284 (struct rte_crypto_op *op, struct openssl_session *sess, 1285 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1286 { 1287 /* cipher */ 1288 uint8_t *dst = NULL, *iv, *tag, *aad; 1289 int srclen, aadlen, status = -1; 1290 uint32_t offset; 1291 uint8_t taglen; 1292 1293 /* 1294 * Segmented destination buffer is not supported for 1295 * encryption/decryption 1296 */ 1297 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1298 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1299 return; 1300 } 1301 1302 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1303 sess->iv.offset); 1304 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1305 srclen = 0; 1306 offset = op->sym->auth.data.offset; 1307 aadlen = op->sym->auth.data.length; 1308 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1309 op->sym->auth.data.offset); 1310 tag = op->sym->auth.digest.data; 1311 if (tag == NULL) 1312 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1313 offset + aadlen); 1314 } else { 1315 srclen = op->sym->aead.data.length; 1316 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1317 op->sym->aead.data.offset); 1318 offset = op->sym->aead.data.offset; 1319 aad = op->sym->aead.aad.data; 1320 aadlen = sess->auth.aad_length; 1321 tag = op->sym->aead.digest.data; 1322 if (tag == NULL) 1323 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1324 offset + srclen); 1325 } 1326 1327 taglen = sess->auth.digest_length; 1328 1329 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1330 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1331 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1332 status = process_openssl_auth_encryption_gcm( 1333 mbuf_src, offset, srclen, 1334 aad, aadlen, iv, 1335 dst, tag, sess->cipher.ctx); 1336 else 1337 status = process_openssl_auth_encryption_ccm( 1338 mbuf_src, offset, srclen, 1339 aad, aadlen, iv, 1340 dst, tag, taglen, sess->cipher.ctx); 1341 1342 } else { 1343 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1344 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1345 status = process_openssl_auth_decryption_gcm( 1346 mbuf_src, offset, srclen, 1347 aad, aadlen, iv, 1348 dst, tag, sess->cipher.ctx); 1349 else 1350 status = process_openssl_auth_decryption_ccm( 1351 mbuf_src, offset, srclen, 1352 aad, aadlen, iv, 1353 dst, tag, taglen, sess->cipher.ctx); 1354 } 1355 1356 if (status != 0) { 1357 if (status == (-EFAULT) && 1358 sess->auth.operation == 1359 RTE_CRYPTO_AUTH_OP_VERIFY) 1360 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1361 else 1362 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1363 } 1364 } 1365 1366 /** Process cipher operation */ 1367 static void 1368 process_openssl_cipher_op 1369 (struct rte_crypto_op *op, struct openssl_session *sess, 1370 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1371 { 1372 uint8_t *dst, *iv; 1373 int srclen, status; 1374 1375 /* 1376 * Segmented destination buffer is not supported for 1377 * encryption/decryption 1378 */ 1379 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1380 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1381 return; 1382 } 1383 1384 srclen = op->sym->cipher.data.length; 1385 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1386 op->sym->cipher.data.offset); 1387 1388 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1389 sess->iv.offset); 1390 1391 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1392 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1393 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1394 op->sym->cipher.data.offset, iv, 1395 srclen, sess->cipher.ctx); 1396 else 1397 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1398 op->sym->cipher.data.offset, iv, 1399 srclen, sess->cipher.ctx); 1400 else 1401 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1402 op->sym->cipher.data.offset, iv, 1403 sess->cipher.key.data, srclen, 1404 sess->cipher.ctx); 1405 1406 if (status != 0) 1407 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1408 } 1409 1410 /** Process cipher operation */ 1411 static void 1412 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1413 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1414 struct rte_mbuf *mbuf_dst) 1415 { 1416 uint8_t *src, *dst, *iv; 1417 uint8_t block_size, last_block_len; 1418 int srclen, status = 0; 1419 1420 srclen = op->sym->cipher.data.length; 1421 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1422 op->sym->cipher.data.offset); 1423 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1424 op->sym->cipher.data.offset); 1425 1426 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1427 sess->iv.offset); 1428 1429 block_size = DES_BLOCK_SIZE; 1430 1431 last_block_len = srclen % block_size; 1432 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1433 /* Encrypt only with ECB mode XOR IV */ 1434 if (srclen < block_size) { 1435 status = process_openssl_cipher_bpi_encrypt(src, dst, 1436 iv, srclen, 1437 sess->cipher.bpi_ctx); 1438 } else { 1439 srclen -= last_block_len; 1440 /* Encrypt with the block aligned stream with CBC mode */ 1441 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1442 op->sym->cipher.data.offset, iv, 1443 srclen, sess->cipher.ctx); 1444 if (last_block_len) { 1445 /* Point at last block */ 1446 dst += srclen; 1447 /* 1448 * IV is the last encrypted block from 1449 * the previous operation 1450 */ 1451 iv = dst - block_size; 1452 src += srclen; 1453 srclen = last_block_len; 1454 /* Encrypt the last frame with ECB mode */ 1455 status |= process_openssl_cipher_bpi_encrypt(src, 1456 dst, iv, 1457 srclen, sess->cipher.bpi_ctx); 1458 } 1459 } 1460 } else { 1461 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1462 if (srclen < block_size) { 1463 status = process_openssl_cipher_bpi_encrypt(src, dst, 1464 iv, 1465 srclen, 1466 sess->cipher.bpi_ctx); 1467 } else { 1468 if (last_block_len) { 1469 /* Point at last block */ 1470 dst += srclen - last_block_len; 1471 src += srclen - last_block_len; 1472 /* 1473 * IV is the last full block 1474 */ 1475 iv = src - block_size; 1476 /* 1477 * Decrypt the last frame with ECB mode 1478 * (encrypt, as it is the same operation) 1479 */ 1480 status = process_openssl_cipher_bpi_encrypt(src, 1481 dst, iv, 1482 last_block_len, sess->cipher.bpi_ctx); 1483 /* Prepare parameters for CBC mode op */ 1484 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1485 sess->iv.offset); 1486 dst += last_block_len - srclen; 1487 srclen -= last_block_len; 1488 } 1489 1490 /* Decrypt with CBC mode */ 1491 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1492 op->sym->cipher.data.offset, iv, 1493 srclen, sess->cipher.ctx); 1494 } 1495 } 1496 1497 if (status != 0) 1498 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1499 } 1500 1501 /** Process auth operation */ 1502 static void 1503 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1504 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1505 struct rte_mbuf *mbuf_dst) 1506 { 1507 uint8_t *dst; 1508 int srclen, status; 1509 1510 srclen = op->sym->auth.data.length; 1511 1512 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) 1513 dst = qp->temp_digest; 1514 else { 1515 dst = op->sym->auth.digest.data; 1516 if (dst == NULL) 1517 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1518 op->sym->auth.data.offset + 1519 op->sym->auth.data.length); 1520 } 1521 1522 switch (sess->auth.mode) { 1523 case OPENSSL_AUTH_AS_AUTH: 1524 status = process_openssl_auth(mbuf_src, dst, 1525 op->sym->auth.data.offset, NULL, NULL, srclen, 1526 sess->auth.auth.ctx, sess->auth.auth.evp_algo); 1527 break; 1528 case OPENSSL_AUTH_AS_HMAC: 1529 status = process_openssl_auth_hmac(mbuf_src, dst, 1530 op->sym->auth.data.offset, srclen, 1531 sess->auth.hmac.ctx); 1532 break; 1533 default: 1534 status = -1; 1535 break; 1536 } 1537 1538 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1539 if (memcmp(dst, op->sym->auth.digest.data, 1540 sess->auth.digest_length) != 0) { 1541 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1542 } 1543 } 1544 1545 if (status != 0) 1546 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1547 } 1548 1549 /* process dsa sign operation */ 1550 static int 1551 process_openssl_dsa_sign_op(struct rte_crypto_op *cop, 1552 struct openssl_asym_session *sess) 1553 { 1554 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1555 DSA *dsa = sess->u.s.dsa; 1556 DSA_SIG *sign = NULL; 1557 1558 sign = DSA_do_sign(op->message.data, 1559 op->message.length, 1560 dsa); 1561 1562 if (sign == NULL) { 1563 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 1564 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1565 } else { 1566 const BIGNUM *r = NULL, *s = NULL; 1567 get_dsa_sign(sign, r, s); 1568 1569 op->r.length = BN_bn2bin(r, op->r.data); 1570 op->s.length = BN_bn2bin(s, op->s.data); 1571 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1572 } 1573 1574 DSA_SIG_free(sign); 1575 1576 return 0; 1577 } 1578 1579 /* process dsa verify operation */ 1580 static int 1581 process_openssl_dsa_verify_op(struct rte_crypto_op *cop, 1582 struct openssl_asym_session *sess) 1583 { 1584 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1585 DSA *dsa = sess->u.s.dsa; 1586 int ret; 1587 DSA_SIG *sign = DSA_SIG_new(); 1588 BIGNUM *r = NULL, *s = NULL; 1589 BIGNUM *pub_key = NULL; 1590 1591 if (sign == NULL) { 1592 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 1593 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1594 return -1; 1595 } 1596 1597 r = BN_bin2bn(op->r.data, 1598 op->r.length, 1599 r); 1600 s = BN_bin2bn(op->s.data, 1601 op->s.length, 1602 s); 1603 pub_key = BN_bin2bn(op->y.data, 1604 op->y.length, 1605 pub_key); 1606 if (!r || !s || !pub_key) { 1607 if (r) 1608 BN_free(r); 1609 if (s) 1610 BN_free(s); 1611 if (pub_key) 1612 BN_free(pub_key); 1613 1614 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1615 return -1; 1616 } 1617 set_dsa_sign(sign, r, s); 1618 set_dsa_pub_key(dsa, pub_key); 1619 1620 ret = DSA_do_verify(op->message.data, 1621 op->message.length, 1622 sign, 1623 dsa); 1624 1625 if (ret != 1) 1626 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1627 else 1628 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1629 1630 DSA_SIG_free(sign); 1631 1632 return 0; 1633 } 1634 1635 /* process dh operation */ 1636 static int 1637 process_openssl_dh_op(struct rte_crypto_op *cop, 1638 struct openssl_asym_session *sess) 1639 { 1640 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 1641 DH *dh_key = sess->u.dh.dh_key; 1642 BIGNUM *priv_key = NULL; 1643 int ret = 0; 1644 1645 if (sess->u.dh.key_op & 1646 (1 << RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE)) { 1647 /* compute shared secret using peer public key 1648 * and current private key 1649 * shared secret = peer_key ^ priv_key mod p 1650 */ 1651 BIGNUM *peer_key = NULL; 1652 1653 /* copy private key and peer key and compute shared secret */ 1654 peer_key = BN_bin2bn(op->pub_key.data, 1655 op->pub_key.length, 1656 peer_key); 1657 if (peer_key == NULL) { 1658 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1659 return -1; 1660 } 1661 priv_key = BN_bin2bn(op->priv_key.data, 1662 op->priv_key.length, 1663 priv_key); 1664 if (priv_key == NULL) { 1665 BN_free(peer_key); 1666 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1667 return -1; 1668 } 1669 set_dh_priv_key(dh_key, priv_key, ret); 1670 if (ret) { 1671 OPENSSL_LOG(ERR, "Failed to set private key\n"); 1672 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1673 BN_free(peer_key); 1674 BN_free(priv_key); 1675 return 0; 1676 } 1677 1678 ret = DH_compute_key( 1679 op->shared_secret.data, 1680 peer_key, dh_key); 1681 if (ret < 0) { 1682 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1683 BN_free(peer_key); 1684 /* priv key is already loaded into dh, 1685 * let's not free that directly here. 1686 * DH_free() will auto free it later. 1687 */ 1688 return 0; 1689 } 1690 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1691 op->shared_secret.length = ret; 1692 BN_free(peer_key); 1693 return 0; 1694 } 1695 1696 /* 1697 * other options are public and private key generations. 1698 * 1699 * if user provides private key, 1700 * then first set DH with user provided private key 1701 */ 1702 if ((sess->u.dh.key_op & 1703 (1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)) && 1704 !(sess->u.dh.key_op & 1705 (1 << RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE))) { 1706 /* generate public key using user-provided private key 1707 * pub_key = g ^ priv_key mod p 1708 */ 1709 1710 /* load private key into DH */ 1711 priv_key = BN_bin2bn(op->priv_key.data, 1712 op->priv_key.length, 1713 priv_key); 1714 if (priv_key == NULL) { 1715 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1716 return -1; 1717 } 1718 set_dh_priv_key(dh_key, priv_key, ret); 1719 if (ret) { 1720 OPENSSL_LOG(ERR, "Failed to set private key\n"); 1721 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1722 BN_free(priv_key); 1723 return 0; 1724 } 1725 } 1726 1727 /* generate public and private key pair. 1728 * 1729 * if private key already set, generates only public key. 1730 * 1731 * if private key is not already set, then set it to random value 1732 * and update internal private key. 1733 */ 1734 if (!DH_generate_key(dh_key)) { 1735 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1736 return 0; 1737 } 1738 1739 if (sess->u.dh.key_op & (1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)) { 1740 const BIGNUM *pub_key = NULL; 1741 1742 OPENSSL_LOG(DEBUG, "%s:%d update public key\n", 1743 __func__, __LINE__); 1744 1745 /* get the generated keys */ 1746 get_dh_pub_key(dh_key, pub_key); 1747 1748 /* output public key */ 1749 op->pub_key.length = BN_bn2bin(pub_key, 1750 op->pub_key.data); 1751 } 1752 1753 if (sess->u.dh.key_op & 1754 (1 << RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE)) { 1755 const BIGNUM *priv_key = NULL; 1756 1757 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", 1758 __func__, __LINE__); 1759 1760 /* get the generated keys */ 1761 get_dh_priv_key(dh_key, priv_key); 1762 1763 /* provide generated private key back to user */ 1764 op->priv_key.length = BN_bn2bin(priv_key, 1765 op->priv_key.data); 1766 } 1767 1768 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1769 1770 return 0; 1771 } 1772 1773 /* process modinv operation */ 1774 static int 1775 process_openssl_modinv_op(struct rte_crypto_op *cop, 1776 struct openssl_asym_session *sess) 1777 { 1778 struct rte_crypto_asym_op *op = cop->asym; 1779 BIGNUM *base = BN_CTX_get(sess->u.m.ctx); 1780 BIGNUM *res = BN_CTX_get(sess->u.m.ctx); 1781 1782 if (unlikely(base == NULL || res == NULL)) { 1783 if (base) 1784 BN_free(base); 1785 if (res) 1786 BN_free(res); 1787 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1788 return -1; 1789 } 1790 1791 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 1792 op->modinv.base.length, base); 1793 1794 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) { 1795 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1796 op->modinv.base.length = BN_bn2bin(res, op->modinv.base.data); 1797 } else { 1798 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1799 } 1800 1801 return 0; 1802 } 1803 1804 /* process modexp operation */ 1805 static int 1806 process_openssl_modexp_op(struct rte_crypto_op *cop, 1807 struct openssl_asym_session *sess) 1808 { 1809 struct rte_crypto_asym_op *op = cop->asym; 1810 BIGNUM *base = BN_CTX_get(sess->u.e.ctx); 1811 BIGNUM *res = BN_CTX_get(sess->u.e.ctx); 1812 1813 if (unlikely(base == NULL || res == NULL)) { 1814 if (base) 1815 BN_free(base); 1816 if (res) 1817 BN_free(res); 1818 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1819 return -1; 1820 } 1821 1822 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 1823 op->modinv.base.length, base); 1824 1825 if (BN_mod_exp(res, base, sess->u.e.exp, 1826 sess->u.e.mod, sess->u.e.ctx)) { 1827 op->modinv.base.length = BN_bn2bin(res, op->modinv.base.data); 1828 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1829 } else { 1830 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1831 } 1832 1833 return 0; 1834 } 1835 1836 /* process rsa operations */ 1837 static int 1838 process_openssl_rsa_op(struct rte_crypto_op *cop, 1839 struct openssl_asym_session *sess) 1840 { 1841 int ret = 0; 1842 struct rte_crypto_asym_op *op = cop->asym; 1843 RSA *rsa = sess->u.r.rsa; 1844 uint32_t pad = (op->rsa.pad); 1845 1846 switch (pad) { 1847 case RTE_CRYPTO_RSA_PKCS1_V1_5_BT0: 1848 case RTE_CRYPTO_RSA_PKCS1_V1_5_BT1: 1849 case RTE_CRYPTO_RSA_PKCS1_V1_5_BT2: 1850 pad = RSA_PKCS1_PADDING; 1851 break; 1852 case RTE_CRYPTO_RSA_PADDING_NONE: 1853 pad = RSA_NO_PADDING; 1854 break; 1855 default: 1856 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1857 OPENSSL_LOG(ERR, 1858 "rsa pad type not supported %d\n", pad); 1859 return 0; 1860 } 1861 1862 switch (op->rsa.op_type) { 1863 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 1864 ret = RSA_public_encrypt(op->rsa.message.length, 1865 op->rsa.message.data, 1866 op->rsa.message.data, 1867 rsa, 1868 pad); 1869 1870 if (ret > 0) 1871 op->rsa.message.length = ret; 1872 OPENSSL_LOG(DEBUG, 1873 "length of encrypted text %d\n", ret); 1874 break; 1875 1876 case RTE_CRYPTO_ASYM_OP_DECRYPT: 1877 ret = RSA_private_decrypt(op->rsa.message.length, 1878 op->rsa.message.data, 1879 op->rsa.message.data, 1880 rsa, 1881 pad); 1882 if (ret > 0) 1883 op->rsa.message.length = ret; 1884 break; 1885 1886 case RTE_CRYPTO_ASYM_OP_SIGN: 1887 ret = RSA_private_encrypt(op->rsa.message.length, 1888 op->rsa.message.data, 1889 op->rsa.sign.data, 1890 rsa, 1891 pad); 1892 if (ret > 0) 1893 op->rsa.sign.length = ret; 1894 break; 1895 1896 case RTE_CRYPTO_ASYM_OP_VERIFY: 1897 ret = RSA_public_decrypt(op->rsa.sign.length, 1898 op->rsa.sign.data, 1899 op->rsa.sign.data, 1900 rsa, 1901 pad); 1902 1903 OPENSSL_LOG(DEBUG, 1904 "Length of public_decrypt %d " 1905 "length of message %zd\n", 1906 ret, op->rsa.message.length); 1907 1908 if (memcmp(op->rsa.sign.data, op->rsa.message.data, 1909 op->rsa.message.length)) { 1910 OPENSSL_LOG(ERR, 1911 "RSA sign Verification failed"); 1912 return -1; 1913 } 1914 break; 1915 1916 default: 1917 /* allow ops with invalid args to be pushed to 1918 * completion queue 1919 */ 1920 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1921 break; 1922 } 1923 1924 if (ret < 0) 1925 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1926 1927 return 0; 1928 } 1929 1930 static int 1931 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1932 struct openssl_asym_session *sess) 1933 { 1934 int retval = 0; 1935 1936 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1937 1938 switch (sess->xfrm_type) { 1939 case RTE_CRYPTO_ASYM_XFORM_RSA: 1940 retval = process_openssl_rsa_op(op, sess); 1941 break; 1942 case RTE_CRYPTO_ASYM_XFORM_MODEX: 1943 retval = process_openssl_modexp_op(op, sess); 1944 break; 1945 case RTE_CRYPTO_ASYM_XFORM_MODINV: 1946 retval = process_openssl_modinv_op(op, sess); 1947 break; 1948 case RTE_CRYPTO_ASYM_XFORM_DH: 1949 retval = process_openssl_dh_op(op, sess); 1950 break; 1951 case RTE_CRYPTO_ASYM_XFORM_DSA: 1952 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 1953 retval = process_openssl_dsa_sign_op(op, sess); 1954 else if (op->asym->dsa.op_type == 1955 RTE_CRYPTO_ASYM_OP_VERIFY) 1956 retval = 1957 process_openssl_dsa_verify_op(op, sess); 1958 else 1959 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1960 break; 1961 default: 1962 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1963 break; 1964 } 1965 if (!retval) { 1966 /* op processed so push to completion queue as processed */ 1967 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 1968 if (retval) 1969 /* return error if failed to put in completion queue */ 1970 retval = -1; 1971 } 1972 1973 return retval; 1974 } 1975 1976 /** Process crypto operation for mbuf */ 1977 static int 1978 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1979 struct openssl_session *sess) 1980 { 1981 struct rte_mbuf *msrc, *mdst; 1982 int retval; 1983 1984 msrc = op->sym->m_src; 1985 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 1986 1987 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1988 1989 switch (sess->chain_order) { 1990 case OPENSSL_CHAIN_ONLY_CIPHER: 1991 process_openssl_cipher_op(op, sess, msrc, mdst); 1992 break; 1993 case OPENSSL_CHAIN_ONLY_AUTH: 1994 process_openssl_auth_op(qp, op, sess, msrc, mdst); 1995 break; 1996 case OPENSSL_CHAIN_CIPHER_AUTH: 1997 process_openssl_cipher_op(op, sess, msrc, mdst); 1998 process_openssl_auth_op(qp, op, sess, mdst, mdst); 1999 break; 2000 case OPENSSL_CHAIN_AUTH_CIPHER: 2001 process_openssl_auth_op(qp, op, sess, msrc, mdst); 2002 process_openssl_cipher_op(op, sess, msrc, mdst); 2003 break; 2004 case OPENSSL_CHAIN_COMBINED: 2005 process_openssl_combined_op(op, sess, msrc, mdst); 2006 break; 2007 case OPENSSL_CHAIN_CIPHER_BPI: 2008 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 2009 break; 2010 default: 2011 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 2012 break; 2013 } 2014 2015 /* Free session if a session-less crypto op */ 2016 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 2017 openssl_reset_session(sess); 2018 memset(sess, 0, sizeof(struct openssl_session)); 2019 memset(op->sym->session, 0, 2020 rte_cryptodev_sym_get_header_session_size()); 2021 rte_mempool_put(qp->sess_mp, sess); 2022 rte_mempool_put(qp->sess_mp, op->sym->session); 2023 op->sym->session = NULL; 2024 } 2025 2026 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 2027 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2028 2029 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 2030 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 2031 else 2032 retval = -1; 2033 2034 return retval; 2035 } 2036 2037 /* 2038 *------------------------------------------------------------------------------ 2039 * PMD Framework 2040 *------------------------------------------------------------------------------ 2041 */ 2042 2043 /** Enqueue burst */ 2044 static uint16_t 2045 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 2046 uint16_t nb_ops) 2047 { 2048 void *sess; 2049 struct openssl_qp *qp = queue_pair; 2050 int i, retval; 2051 2052 for (i = 0; i < nb_ops; i++) { 2053 sess = get_session(qp, ops[i]); 2054 if (unlikely(sess == NULL)) 2055 goto enqueue_err; 2056 2057 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) 2058 retval = process_op(qp, ops[i], 2059 (struct openssl_session *) sess); 2060 else 2061 retval = process_asym_op(qp, ops[i], 2062 (struct openssl_asym_session *) sess); 2063 if (unlikely(retval < 0)) 2064 goto enqueue_err; 2065 } 2066 2067 qp->stats.enqueued_count += i; 2068 return i; 2069 2070 enqueue_err: 2071 qp->stats.enqueue_err_count++; 2072 return i; 2073 } 2074 2075 /** Dequeue burst */ 2076 static uint16_t 2077 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 2078 uint16_t nb_ops) 2079 { 2080 struct openssl_qp *qp = queue_pair; 2081 2082 unsigned int nb_dequeued = 0; 2083 2084 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 2085 (void **)ops, nb_ops, NULL); 2086 qp->stats.dequeued_count += nb_dequeued; 2087 2088 return nb_dequeued; 2089 } 2090 2091 /** Create OPENSSL crypto device */ 2092 static int 2093 cryptodev_openssl_create(const char *name, 2094 struct rte_vdev_device *vdev, 2095 struct rte_cryptodev_pmd_init_params *init_params) 2096 { 2097 struct rte_cryptodev *dev; 2098 struct openssl_private *internals; 2099 2100 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 2101 if (dev == NULL) { 2102 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 2103 goto init_error; 2104 } 2105 2106 dev->driver_id = cryptodev_driver_id; 2107 dev->dev_ops = rte_openssl_pmd_ops; 2108 2109 /* register rx/tx burst functions for data path */ 2110 dev->dequeue_burst = openssl_pmd_dequeue_burst; 2111 dev->enqueue_burst = openssl_pmd_enqueue_burst; 2112 2113 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 2114 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 2115 RTE_CRYPTODEV_FF_CPU_AESNI | 2116 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 2117 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | 2118 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO; 2119 2120 /* Set vector instructions mode supported */ 2121 internals = dev->data->dev_private; 2122 2123 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 2124 2125 return 0; 2126 2127 init_error: 2128 OPENSSL_LOG(ERR, "driver %s: create failed", 2129 init_params->name); 2130 2131 cryptodev_openssl_remove(vdev); 2132 return -EFAULT; 2133 } 2134 2135 /** Initialise OPENSSL crypto device */ 2136 static int 2137 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 2138 { 2139 struct rte_cryptodev_pmd_init_params init_params = { 2140 "", 2141 sizeof(struct openssl_private), 2142 rte_socket_id(), 2143 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 2144 }; 2145 const char *name; 2146 const char *input_args; 2147 2148 name = rte_vdev_device_name(vdev); 2149 if (name == NULL) 2150 return -EINVAL; 2151 input_args = rte_vdev_device_args(vdev); 2152 2153 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 2154 2155 return cryptodev_openssl_create(name, vdev, &init_params); 2156 } 2157 2158 /** Uninitialise OPENSSL crypto device */ 2159 static int 2160 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 2161 { 2162 struct rte_cryptodev *cryptodev; 2163 const char *name; 2164 2165 name = rte_vdev_device_name(vdev); 2166 if (name == NULL) 2167 return -EINVAL; 2168 2169 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 2170 if (cryptodev == NULL) 2171 return -ENODEV; 2172 2173 return rte_cryptodev_pmd_destroy(cryptodev); 2174 } 2175 2176 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 2177 .probe = cryptodev_openssl_probe, 2178 .remove = cryptodev_openssl_remove 2179 }; 2180 2181 static struct cryptodev_driver openssl_crypto_drv; 2182 2183 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 2184 cryptodev_openssl_pmd_drv); 2185 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 2186 "max_nb_queue_pairs=<int> " 2187 "socket_id=<int>"); 2188 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 2189 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 2190 2191 RTE_INIT(openssl_init_log) 2192 { 2193 openssl_logtype_driver = rte_log_register("pmd.crypto.openssl"); 2194 } 2195