1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_common.h> 6 #include <rte_hexdump.h> 7 #include <rte_cryptodev.h> 8 #include <rte_cryptodev_pmd.h> 9 #include <rte_bus_vdev.h> 10 #include <rte_malloc.h> 11 #include <rte_cpuflags.h> 12 13 #include <openssl/hmac.h> 14 #include <openssl/evp.h> 15 16 #include "rte_openssl_pmd_private.h" 17 #include "compat.h" 18 19 #define DES_BLOCK_SIZE 8 20 21 static uint8_t cryptodev_driver_id; 22 23 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 24 static HMAC_CTX *HMAC_CTX_new(void) 25 { 26 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 27 28 if (ctx != NULL) 29 HMAC_CTX_init(ctx); 30 return ctx; 31 } 32 33 static void HMAC_CTX_free(HMAC_CTX *ctx) 34 { 35 if (ctx != NULL) { 36 HMAC_CTX_cleanup(ctx); 37 OPENSSL_free(ctx); 38 } 39 } 40 #endif 41 42 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 43 44 /*----------------------------------------------------------------------------*/ 45 46 /** 47 * Increment counter by 1 48 * Counter is 64 bit array, big-endian 49 */ 50 static void 51 ctr_inc(uint8_t *ctr) 52 { 53 uint64_t *ctr64 = (uint64_t *)ctr; 54 55 *ctr64 = __builtin_bswap64(*ctr64); 56 (*ctr64)++; 57 *ctr64 = __builtin_bswap64(*ctr64); 58 } 59 60 /* 61 *------------------------------------------------------------------------------ 62 * Session Prepare 63 *------------------------------------------------------------------------------ 64 */ 65 66 /** Get xform chain order */ 67 static enum openssl_chain_order 68 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 69 { 70 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 71 72 if (xform != NULL) { 73 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 74 if (xform->next == NULL) 75 res = OPENSSL_CHAIN_ONLY_AUTH; 76 else if (xform->next->type == 77 RTE_CRYPTO_SYM_XFORM_CIPHER) 78 res = OPENSSL_CHAIN_AUTH_CIPHER; 79 } 80 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 81 if (xform->next == NULL) 82 res = OPENSSL_CHAIN_ONLY_CIPHER; 83 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 84 res = OPENSSL_CHAIN_CIPHER_AUTH; 85 } 86 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 87 res = OPENSSL_CHAIN_COMBINED; 88 } 89 90 return res; 91 } 92 93 /** Get session cipher key from input cipher key */ 94 static void 95 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key) 96 { 97 memcpy(session_key, input_key, keylen); 98 } 99 100 /** Get key ede 24 bytes standard from input key */ 101 static int 102 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede) 103 { 104 int res = 0; 105 106 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 107 switch (keylen) { 108 case 24: 109 memcpy(key_ede, key, 24); 110 break; 111 case 16: 112 /* K3 = K1 */ 113 memcpy(key_ede, key, 16); 114 memcpy(key_ede + 16, key, 8); 115 break; 116 case 8: 117 /* K1 = K2 = K3 (DES compatibility) */ 118 memcpy(key_ede, key, 8); 119 memcpy(key_ede + 8, key, 8); 120 memcpy(key_ede + 16, key, 8); 121 break; 122 default: 123 OPENSSL_LOG(ERR, "Unsupported key size"); 124 res = -EINVAL; 125 } 126 127 return res; 128 } 129 130 /** Get adequate openssl function for input cipher algorithm */ 131 static uint8_t 132 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 133 const EVP_CIPHER **algo) 134 { 135 int res = 0; 136 137 if (algo != NULL) { 138 switch (sess_algo) { 139 case RTE_CRYPTO_CIPHER_3DES_CBC: 140 switch (keylen) { 141 case 8: 142 *algo = EVP_des_cbc(); 143 break; 144 case 16: 145 *algo = EVP_des_ede_cbc(); 146 break; 147 case 24: 148 *algo = EVP_des_ede3_cbc(); 149 break; 150 default: 151 res = -EINVAL; 152 } 153 break; 154 case RTE_CRYPTO_CIPHER_3DES_CTR: 155 break; 156 case RTE_CRYPTO_CIPHER_AES_CBC: 157 switch (keylen) { 158 case 16: 159 *algo = EVP_aes_128_cbc(); 160 break; 161 case 24: 162 *algo = EVP_aes_192_cbc(); 163 break; 164 case 32: 165 *algo = EVP_aes_256_cbc(); 166 break; 167 default: 168 res = -EINVAL; 169 } 170 break; 171 case RTE_CRYPTO_CIPHER_AES_CTR: 172 switch (keylen) { 173 case 16: 174 *algo = EVP_aes_128_ctr(); 175 break; 176 case 24: 177 *algo = EVP_aes_192_ctr(); 178 break; 179 case 32: 180 *algo = EVP_aes_256_ctr(); 181 break; 182 default: 183 res = -EINVAL; 184 } 185 break; 186 default: 187 res = -EINVAL; 188 break; 189 } 190 } else { 191 res = -EINVAL; 192 } 193 194 return res; 195 } 196 197 /** Get adequate openssl function for input auth algorithm */ 198 static uint8_t 199 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 200 const EVP_MD **algo) 201 { 202 int res = 0; 203 204 if (algo != NULL) { 205 switch (sessalgo) { 206 case RTE_CRYPTO_AUTH_MD5: 207 case RTE_CRYPTO_AUTH_MD5_HMAC: 208 *algo = EVP_md5(); 209 break; 210 case RTE_CRYPTO_AUTH_SHA1: 211 case RTE_CRYPTO_AUTH_SHA1_HMAC: 212 *algo = EVP_sha1(); 213 break; 214 case RTE_CRYPTO_AUTH_SHA224: 215 case RTE_CRYPTO_AUTH_SHA224_HMAC: 216 *algo = EVP_sha224(); 217 break; 218 case RTE_CRYPTO_AUTH_SHA256: 219 case RTE_CRYPTO_AUTH_SHA256_HMAC: 220 *algo = EVP_sha256(); 221 break; 222 case RTE_CRYPTO_AUTH_SHA384: 223 case RTE_CRYPTO_AUTH_SHA384_HMAC: 224 *algo = EVP_sha384(); 225 break; 226 case RTE_CRYPTO_AUTH_SHA512: 227 case RTE_CRYPTO_AUTH_SHA512_HMAC: 228 *algo = EVP_sha512(); 229 break; 230 default: 231 res = -EINVAL; 232 break; 233 } 234 } else { 235 res = -EINVAL; 236 } 237 238 return res; 239 } 240 241 /** Get adequate openssl function for input cipher algorithm */ 242 static uint8_t 243 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 244 const EVP_CIPHER **algo) 245 { 246 int res = 0; 247 248 if (algo != NULL) { 249 switch (sess_algo) { 250 case RTE_CRYPTO_AEAD_AES_GCM: 251 switch (keylen) { 252 case 16: 253 *algo = EVP_aes_128_gcm(); 254 break; 255 case 24: 256 *algo = EVP_aes_192_gcm(); 257 break; 258 case 32: 259 *algo = EVP_aes_256_gcm(); 260 break; 261 default: 262 res = -EINVAL; 263 } 264 break; 265 case RTE_CRYPTO_AEAD_AES_CCM: 266 switch (keylen) { 267 case 16: 268 *algo = EVP_aes_128_ccm(); 269 break; 270 case 24: 271 *algo = EVP_aes_192_ccm(); 272 break; 273 case 32: 274 *algo = EVP_aes_256_ccm(); 275 break; 276 default: 277 res = -EINVAL; 278 } 279 break; 280 default: 281 res = -EINVAL; 282 break; 283 } 284 } else { 285 res = -EINVAL; 286 } 287 288 return res; 289 } 290 291 /* Set session AEAD encryption parameters */ 292 static int 293 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 294 enum rte_crypto_aead_algorithm algo, 295 uint8_t tag_len, uint8_t *key) 296 { 297 int iv_type = 0; 298 unsigned int do_ccm; 299 300 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 301 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 302 303 /* Select AEAD algo */ 304 switch (algo) { 305 case RTE_CRYPTO_AEAD_AES_GCM: 306 iv_type = EVP_CTRL_GCM_SET_IVLEN; 307 if (tag_len != 16) 308 return -EINVAL; 309 do_ccm = 0; 310 break; 311 case RTE_CRYPTO_AEAD_AES_CCM: 312 iv_type = EVP_CTRL_CCM_SET_IVLEN; 313 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 314 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 315 return -EINVAL; 316 do_ccm = 1; 317 break; 318 default: 319 return -ENOTSUP; 320 } 321 322 sess->cipher.mode = OPENSSL_CIPHER_LIB; 323 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 324 325 if (get_aead_algo(algo, sess->cipher.key.length, 326 &sess->cipher.evp_algo) != 0) 327 return -EINVAL; 328 329 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 330 331 sess->chain_order = OPENSSL_CHAIN_COMBINED; 332 333 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 334 NULL, NULL, NULL) <= 0) 335 return -EINVAL; 336 337 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, 338 NULL) <= 0) 339 return -EINVAL; 340 341 if (do_ccm) 342 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 343 tag_len, NULL); 344 345 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 346 return -EINVAL; 347 348 return 0; 349 } 350 351 /* Set session AEAD decryption parameters */ 352 static int 353 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 354 enum rte_crypto_aead_algorithm algo, 355 uint8_t tag_len, uint8_t *key) 356 { 357 int iv_type = 0; 358 unsigned int do_ccm = 0; 359 360 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 361 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 362 363 /* Select AEAD algo */ 364 switch (algo) { 365 case RTE_CRYPTO_AEAD_AES_GCM: 366 iv_type = EVP_CTRL_GCM_SET_IVLEN; 367 if (tag_len != 16) 368 return -EINVAL; 369 break; 370 case RTE_CRYPTO_AEAD_AES_CCM: 371 iv_type = EVP_CTRL_CCM_SET_IVLEN; 372 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 373 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 374 return -EINVAL; 375 do_ccm = 1; 376 break; 377 default: 378 return -ENOTSUP; 379 } 380 381 sess->cipher.mode = OPENSSL_CIPHER_LIB; 382 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 383 384 if (get_aead_algo(algo, sess->cipher.key.length, 385 &sess->cipher.evp_algo) != 0) 386 return -EINVAL; 387 388 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 389 390 sess->chain_order = OPENSSL_CHAIN_COMBINED; 391 392 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 393 NULL, NULL, NULL) <= 0) 394 return -EINVAL; 395 396 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, 397 sess->iv.length, NULL) <= 0) 398 return -EINVAL; 399 400 if (do_ccm) 401 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 402 tag_len, NULL); 403 404 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 405 return -EINVAL; 406 407 return 0; 408 } 409 410 /** Set session cipher parameters */ 411 static int 412 openssl_set_session_cipher_parameters(struct openssl_session *sess, 413 const struct rte_crypto_sym_xform *xform) 414 { 415 /* Select cipher direction */ 416 sess->cipher.direction = xform->cipher.op; 417 /* Select cipher key */ 418 sess->cipher.key.length = xform->cipher.key.length; 419 420 /* Set IV parameters */ 421 sess->iv.offset = xform->cipher.iv.offset; 422 sess->iv.length = xform->cipher.iv.length; 423 424 /* Select cipher algo */ 425 switch (xform->cipher.algo) { 426 case RTE_CRYPTO_CIPHER_3DES_CBC: 427 case RTE_CRYPTO_CIPHER_AES_CBC: 428 case RTE_CRYPTO_CIPHER_AES_CTR: 429 sess->cipher.mode = OPENSSL_CIPHER_LIB; 430 sess->cipher.algo = xform->cipher.algo; 431 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 432 433 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 434 &sess->cipher.evp_algo) != 0) 435 return -EINVAL; 436 437 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 438 sess->cipher.key.data); 439 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 440 if (EVP_EncryptInit_ex(sess->cipher.ctx, 441 sess->cipher.evp_algo, 442 NULL, xform->cipher.key.data, 443 NULL) != 1) { 444 return -EINVAL; 445 } 446 } else if (sess->cipher.direction == 447 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 448 if (EVP_DecryptInit_ex(sess->cipher.ctx, 449 sess->cipher.evp_algo, 450 NULL, xform->cipher.key.data, 451 NULL) != 1) { 452 return -EINVAL; 453 } 454 } 455 456 break; 457 458 case RTE_CRYPTO_CIPHER_3DES_CTR: 459 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 460 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 461 462 if (get_cipher_key_ede(xform->cipher.key.data, 463 sess->cipher.key.length, 464 sess->cipher.key.data) != 0) 465 return -EINVAL; 466 break; 467 468 case RTE_CRYPTO_CIPHER_DES_CBC: 469 sess->cipher.algo = xform->cipher.algo; 470 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 471 sess->cipher.evp_algo = EVP_des_cbc(); 472 473 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 474 sess->cipher.key.data); 475 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 476 if (EVP_EncryptInit_ex(sess->cipher.ctx, 477 sess->cipher.evp_algo, 478 NULL, xform->cipher.key.data, 479 NULL) != 1) { 480 return -EINVAL; 481 } 482 } else if (sess->cipher.direction == 483 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 484 if (EVP_DecryptInit_ex(sess->cipher.ctx, 485 sess->cipher.evp_algo, 486 NULL, xform->cipher.key.data, 487 NULL) != 1) { 488 return -EINVAL; 489 } 490 } 491 492 break; 493 494 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 495 sess->cipher.algo = xform->cipher.algo; 496 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 497 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 498 sess->cipher.evp_algo = EVP_des_cbc(); 499 500 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 501 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 502 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 503 NULL, xform->cipher.key.data, 0) != 1) 504 return -EINVAL; 505 506 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 507 sess->cipher.key.data); 508 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 509 if (EVP_EncryptInit_ex(sess->cipher.ctx, 510 sess->cipher.evp_algo, 511 NULL, xform->cipher.key.data, 512 NULL) != 1) { 513 return -EINVAL; 514 } 515 } else if (sess->cipher.direction == 516 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 517 if (EVP_DecryptInit_ex(sess->cipher.ctx, 518 sess->cipher.evp_algo, 519 NULL, xform->cipher.key.data, 520 NULL) != 1) { 521 return -EINVAL; 522 } 523 } 524 525 break; 526 default: 527 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 528 return -ENOTSUP; 529 } 530 531 return 0; 532 } 533 534 /* Set session auth parameters */ 535 static int 536 openssl_set_session_auth_parameters(struct openssl_session *sess, 537 const struct rte_crypto_sym_xform *xform) 538 { 539 /* Select auth generate/verify */ 540 sess->auth.operation = xform->auth.op; 541 sess->auth.algo = xform->auth.algo; 542 543 sess->auth.digest_length = xform->auth.digest_length; 544 545 /* Select auth algo */ 546 switch (xform->auth.algo) { 547 case RTE_CRYPTO_AUTH_AES_GMAC: 548 /* 549 * OpenSSL requires GMAC to be a GCM operation 550 * with no cipher data length 551 */ 552 sess->cipher.key.length = xform->auth.key.length; 553 554 /* Set IV parameters */ 555 sess->iv.offset = xform->auth.iv.offset; 556 sess->iv.length = xform->auth.iv.length; 557 558 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 559 return openssl_set_sess_aead_enc_param(sess, 560 RTE_CRYPTO_AEAD_AES_GCM, 561 xform->auth.digest_length, 562 xform->auth.key.data); 563 else 564 return openssl_set_sess_aead_dec_param(sess, 565 RTE_CRYPTO_AEAD_AES_GCM, 566 xform->auth.digest_length, 567 xform->auth.key.data); 568 break; 569 570 case RTE_CRYPTO_AUTH_MD5: 571 case RTE_CRYPTO_AUTH_SHA1: 572 case RTE_CRYPTO_AUTH_SHA224: 573 case RTE_CRYPTO_AUTH_SHA256: 574 case RTE_CRYPTO_AUTH_SHA384: 575 case RTE_CRYPTO_AUTH_SHA512: 576 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 577 if (get_auth_algo(xform->auth.algo, 578 &sess->auth.auth.evp_algo) != 0) 579 return -EINVAL; 580 sess->auth.auth.ctx = EVP_MD_CTX_create(); 581 break; 582 583 case RTE_CRYPTO_AUTH_MD5_HMAC: 584 case RTE_CRYPTO_AUTH_SHA1_HMAC: 585 case RTE_CRYPTO_AUTH_SHA224_HMAC: 586 case RTE_CRYPTO_AUTH_SHA256_HMAC: 587 case RTE_CRYPTO_AUTH_SHA384_HMAC: 588 case RTE_CRYPTO_AUTH_SHA512_HMAC: 589 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 590 sess->auth.hmac.ctx = HMAC_CTX_new(); 591 if (get_auth_algo(xform->auth.algo, 592 &sess->auth.hmac.evp_algo) != 0) 593 return -EINVAL; 594 595 if (HMAC_Init_ex(sess->auth.hmac.ctx, 596 xform->auth.key.data, 597 xform->auth.key.length, 598 sess->auth.hmac.evp_algo, NULL) != 1) 599 return -EINVAL; 600 break; 601 602 default: 603 return -ENOTSUP; 604 } 605 606 return 0; 607 } 608 609 /* Set session AEAD parameters */ 610 static int 611 openssl_set_session_aead_parameters(struct openssl_session *sess, 612 const struct rte_crypto_sym_xform *xform) 613 { 614 /* Select cipher key */ 615 sess->cipher.key.length = xform->aead.key.length; 616 617 /* Set IV parameters */ 618 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 619 /* 620 * For AES-CCM, the actual IV is placed 621 * one byte after the start of the IV field, 622 * according to the API. 623 */ 624 sess->iv.offset = xform->aead.iv.offset + 1; 625 else 626 sess->iv.offset = xform->aead.iv.offset; 627 628 sess->iv.length = xform->aead.iv.length; 629 630 sess->auth.aad_length = xform->aead.aad_length; 631 sess->auth.digest_length = xform->aead.digest_length; 632 633 sess->aead_algo = xform->aead.algo; 634 /* Select cipher direction */ 635 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 636 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 637 xform->aead.digest_length, xform->aead.key.data); 638 else 639 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 640 xform->aead.digest_length, xform->aead.key.data); 641 } 642 643 /** Parse crypto xform chain and set private session parameters */ 644 int 645 openssl_set_session_parameters(struct openssl_session *sess, 646 const struct rte_crypto_sym_xform *xform) 647 { 648 const struct rte_crypto_sym_xform *cipher_xform = NULL; 649 const struct rte_crypto_sym_xform *auth_xform = NULL; 650 const struct rte_crypto_sym_xform *aead_xform = NULL; 651 int ret; 652 653 sess->chain_order = openssl_get_chain_order(xform); 654 switch (sess->chain_order) { 655 case OPENSSL_CHAIN_ONLY_CIPHER: 656 cipher_xform = xform; 657 break; 658 case OPENSSL_CHAIN_ONLY_AUTH: 659 auth_xform = xform; 660 break; 661 case OPENSSL_CHAIN_CIPHER_AUTH: 662 cipher_xform = xform; 663 auth_xform = xform->next; 664 break; 665 case OPENSSL_CHAIN_AUTH_CIPHER: 666 auth_xform = xform; 667 cipher_xform = xform->next; 668 break; 669 case OPENSSL_CHAIN_COMBINED: 670 aead_xform = xform; 671 break; 672 default: 673 return -EINVAL; 674 } 675 676 /* Default IV length = 0 */ 677 sess->iv.length = 0; 678 679 /* cipher_xform must be check before auth_xform */ 680 if (cipher_xform) { 681 ret = openssl_set_session_cipher_parameters( 682 sess, cipher_xform); 683 if (ret != 0) { 684 OPENSSL_LOG(ERR, 685 "Invalid/unsupported cipher parameters"); 686 return ret; 687 } 688 } 689 690 if (auth_xform) { 691 ret = openssl_set_session_auth_parameters(sess, auth_xform); 692 if (ret != 0) { 693 OPENSSL_LOG(ERR, 694 "Invalid/unsupported auth parameters"); 695 return ret; 696 } 697 } 698 699 if (aead_xform) { 700 ret = openssl_set_session_aead_parameters(sess, aead_xform); 701 if (ret != 0) { 702 OPENSSL_LOG(ERR, 703 "Invalid/unsupported AEAD parameters"); 704 return ret; 705 } 706 } 707 708 return 0; 709 } 710 711 /** Reset private session parameters */ 712 void 713 openssl_reset_session(struct openssl_session *sess) 714 { 715 EVP_CIPHER_CTX_free(sess->cipher.ctx); 716 717 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 718 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 719 720 switch (sess->auth.mode) { 721 case OPENSSL_AUTH_AS_AUTH: 722 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 723 break; 724 case OPENSSL_AUTH_AS_HMAC: 725 EVP_PKEY_free(sess->auth.hmac.pkey); 726 HMAC_CTX_free(sess->auth.hmac.ctx); 727 break; 728 default: 729 break; 730 } 731 } 732 733 /** Provide session for operation */ 734 static void * 735 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 736 { 737 struct openssl_session *sess = NULL; 738 struct openssl_asym_session *asym_sess = NULL; 739 740 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 741 if (op->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) { 742 /* get existing session */ 743 if (likely(op->sym->session != NULL)) 744 sess = (struct openssl_session *) 745 get_sym_session_private_data( 746 op->sym->session, 747 cryptodev_driver_id); 748 } else { 749 if (likely(op->asym->session != NULL)) 750 asym_sess = (struct openssl_asym_session *) 751 get_asym_session_private_data( 752 op->asym->session, 753 cryptodev_driver_id); 754 if (asym_sess == NULL) 755 op->status = 756 RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 757 return asym_sess; 758 } 759 } else { 760 /* sessionless asymmetric not supported */ 761 if (op->type == RTE_CRYPTO_OP_TYPE_ASYMMETRIC) 762 return NULL; 763 764 /* provide internal session */ 765 void *_sess = NULL; 766 void *_sess_private_data = NULL; 767 768 if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) 769 return NULL; 770 771 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) 772 return NULL; 773 774 sess = (struct openssl_session *)_sess_private_data; 775 776 if (unlikely(openssl_set_session_parameters(sess, 777 op->sym->xform) != 0)) { 778 rte_mempool_put(qp->sess_mp, _sess); 779 rte_mempool_put(qp->sess_mp, _sess_private_data); 780 sess = NULL; 781 } 782 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 783 set_sym_session_private_data(op->sym->session, 784 cryptodev_driver_id, _sess_private_data); 785 } 786 787 if (sess == NULL) 788 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 789 790 return sess; 791 } 792 793 /* 794 *------------------------------------------------------------------------------ 795 * Process Operations 796 *------------------------------------------------------------------------------ 797 */ 798 static inline int 799 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 800 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 801 { 802 struct rte_mbuf *m; 803 int dstlen; 804 int l, n = srclen; 805 uint8_t *src; 806 807 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 808 m = m->next) 809 offset -= rte_pktmbuf_data_len(m); 810 811 if (m == 0) 812 return -1; 813 814 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 815 816 l = rte_pktmbuf_data_len(m) - offset; 817 if (srclen <= l) { 818 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 819 return -1; 820 *dst += l; 821 return 0; 822 } 823 824 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 825 return -1; 826 827 *dst += dstlen; 828 n -= l; 829 830 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 831 src = rte_pktmbuf_mtod(m, uint8_t *); 832 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 833 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 834 return -1; 835 *dst += dstlen; 836 n -= l; 837 } 838 839 return 0; 840 } 841 842 static inline int 843 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 844 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 845 { 846 struct rte_mbuf *m; 847 int dstlen; 848 int l, n = srclen; 849 uint8_t *src; 850 851 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 852 m = m->next) 853 offset -= rte_pktmbuf_data_len(m); 854 855 if (m == 0) 856 return -1; 857 858 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 859 860 l = rte_pktmbuf_data_len(m) - offset; 861 if (srclen <= l) { 862 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 863 return -1; 864 *dst += l; 865 return 0; 866 } 867 868 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 869 return -1; 870 871 *dst += dstlen; 872 n -= l; 873 874 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 875 src = rte_pktmbuf_mtod(m, uint8_t *); 876 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 877 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 878 return -1; 879 *dst += dstlen; 880 n -= l; 881 } 882 883 return 0; 884 } 885 886 /** Process standard openssl cipher encryption */ 887 static int 888 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 889 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) 890 { 891 int totlen; 892 893 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 894 goto process_cipher_encrypt_err; 895 896 EVP_CIPHER_CTX_set_padding(ctx, 0); 897 898 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 899 srclen, ctx)) 900 goto process_cipher_encrypt_err; 901 902 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 903 goto process_cipher_encrypt_err; 904 905 return 0; 906 907 process_cipher_encrypt_err: 908 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 909 return -EINVAL; 910 } 911 912 /** Process standard openssl cipher encryption */ 913 static int 914 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 915 uint8_t *iv, int srclen, 916 EVP_CIPHER_CTX *ctx) 917 { 918 uint8_t i; 919 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 920 int encrypted_ivlen; 921 922 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 923 iv, DES_BLOCK_SIZE) <= 0) 924 goto process_cipher_encrypt_err; 925 926 for (i = 0; i < srclen; i++) 927 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 928 929 return 0; 930 931 process_cipher_encrypt_err: 932 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 933 return -EINVAL; 934 } 935 /** Process standard openssl cipher decryption */ 936 static int 937 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 938 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) 939 { 940 int totlen; 941 942 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 943 goto process_cipher_decrypt_err; 944 945 EVP_CIPHER_CTX_set_padding(ctx, 0); 946 947 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 948 srclen, ctx)) 949 goto process_cipher_decrypt_err; 950 951 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 952 goto process_cipher_decrypt_err; 953 return 0; 954 955 process_cipher_decrypt_err: 956 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 957 return -EINVAL; 958 } 959 960 /** Process cipher des 3 ctr encryption, decryption algorithm */ 961 static int 962 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 963 int offset, uint8_t *iv, uint8_t *key, int srclen, 964 EVP_CIPHER_CTX *ctx) 965 { 966 uint8_t ebuf[8], ctr[8]; 967 int unused, n; 968 struct rte_mbuf *m; 969 uint8_t *src; 970 int l; 971 972 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 973 m = m->next) 974 offset -= rte_pktmbuf_data_len(m); 975 976 if (m == 0) 977 goto process_cipher_des3ctr_err; 978 979 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 980 l = rte_pktmbuf_data_len(m) - offset; 981 982 /* We use 3DES encryption also for decryption. 983 * IV is not important for 3DES ecb 984 */ 985 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 986 goto process_cipher_des3ctr_err; 987 988 memcpy(ctr, iv, 8); 989 990 for (n = 0; n < srclen; n++) { 991 if (n % 8 == 0) { 992 if (EVP_EncryptUpdate(ctx, 993 (unsigned char *)&ebuf, &unused, 994 (const unsigned char *)&ctr, 8) <= 0) 995 goto process_cipher_des3ctr_err; 996 ctr_inc(ctr); 997 } 998 dst[n] = *(src++) ^ ebuf[n % 8]; 999 1000 l--; 1001 if (!l) { 1002 m = m->next; 1003 if (m) { 1004 src = rte_pktmbuf_mtod(m, uint8_t *); 1005 l = rte_pktmbuf_data_len(m); 1006 } 1007 } 1008 } 1009 1010 return 0; 1011 1012 process_cipher_des3ctr_err: 1013 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 1014 return -EINVAL; 1015 } 1016 1017 /** Process AES-GCM encrypt algorithm */ 1018 static int 1019 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1020 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1021 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1022 { 1023 int len = 0, unused = 0; 1024 uint8_t empty[] = {}; 1025 1026 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1027 goto process_auth_encryption_gcm_err; 1028 1029 if (aadlen > 0) 1030 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1031 goto process_auth_encryption_gcm_err; 1032 1033 if (srclen > 0) 1034 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1035 srclen, ctx)) 1036 goto process_auth_encryption_gcm_err; 1037 1038 /* Workaround open ssl bug in version less then 1.0.1f */ 1039 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1040 goto process_auth_encryption_gcm_err; 1041 1042 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1043 goto process_auth_encryption_gcm_err; 1044 1045 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1046 goto process_auth_encryption_gcm_err; 1047 1048 return 0; 1049 1050 process_auth_encryption_gcm_err: 1051 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1052 return -EINVAL; 1053 } 1054 1055 /** Process AES-CCM encrypt algorithm */ 1056 static int 1057 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1058 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1059 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1060 { 1061 int len = 0; 1062 1063 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1064 goto process_auth_encryption_ccm_err; 1065 1066 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1067 goto process_auth_encryption_ccm_err; 1068 1069 if (aadlen > 0) 1070 /* 1071 * For AES-CCM, the actual AAD is placed 1072 * 18 bytes after the start of the AAD field, 1073 * according to the API. 1074 */ 1075 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1076 goto process_auth_encryption_ccm_err; 1077 1078 if (srclen > 0) 1079 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1080 srclen, ctx)) 1081 goto process_auth_encryption_ccm_err; 1082 1083 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1084 goto process_auth_encryption_ccm_err; 1085 1086 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1087 goto process_auth_encryption_ccm_err; 1088 1089 return 0; 1090 1091 process_auth_encryption_ccm_err: 1092 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1093 return -EINVAL; 1094 } 1095 1096 /** Process AES-GCM decrypt algorithm */ 1097 static int 1098 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1099 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1100 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1101 { 1102 int len = 0, unused = 0; 1103 uint8_t empty[] = {}; 1104 1105 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1106 goto process_auth_decryption_gcm_err; 1107 1108 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1109 goto process_auth_decryption_gcm_err; 1110 1111 if (aadlen > 0) 1112 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1113 goto process_auth_decryption_gcm_err; 1114 1115 if (srclen > 0) 1116 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1117 srclen, ctx)) 1118 goto process_auth_decryption_gcm_err; 1119 1120 /* Workaround open ssl bug in version less then 1.0.1f */ 1121 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1122 goto process_auth_decryption_gcm_err; 1123 1124 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1125 return -EFAULT; 1126 1127 return 0; 1128 1129 process_auth_decryption_gcm_err: 1130 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1131 return -EINVAL; 1132 } 1133 1134 /** Process AES-CCM decrypt algorithm */ 1135 static int 1136 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1137 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1138 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1139 EVP_CIPHER_CTX *ctx) 1140 { 1141 int len = 0; 1142 1143 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1144 goto process_auth_decryption_ccm_err; 1145 1146 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1147 goto process_auth_decryption_ccm_err; 1148 1149 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1150 goto process_auth_decryption_ccm_err; 1151 1152 if (aadlen > 0) 1153 /* 1154 * For AES-CCM, the actual AAD is placed 1155 * 18 bytes after the start of the AAD field, 1156 * according to the API. 1157 */ 1158 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1159 goto process_auth_decryption_ccm_err; 1160 1161 if (srclen > 0) 1162 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1163 srclen, ctx)) 1164 return -EFAULT; 1165 1166 return 0; 1167 1168 process_auth_decryption_ccm_err: 1169 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1170 return -EINVAL; 1171 } 1172 1173 /** Process standard openssl auth algorithms */ 1174 static int 1175 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1176 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1177 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1178 { 1179 size_t dstlen; 1180 struct rte_mbuf *m; 1181 int l, n = srclen; 1182 uint8_t *src; 1183 1184 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1185 m = m->next) 1186 offset -= rte_pktmbuf_data_len(m); 1187 1188 if (m == 0) 1189 goto process_auth_err; 1190 1191 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1192 goto process_auth_err; 1193 1194 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1195 1196 l = rte_pktmbuf_data_len(m) - offset; 1197 if (srclen <= l) { 1198 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1199 goto process_auth_err; 1200 goto process_auth_final; 1201 } 1202 1203 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1204 goto process_auth_err; 1205 1206 n -= l; 1207 1208 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1209 src = rte_pktmbuf_mtod(m, uint8_t *); 1210 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1211 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1212 goto process_auth_err; 1213 n -= l; 1214 } 1215 1216 process_auth_final: 1217 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1218 goto process_auth_err; 1219 return 0; 1220 1221 process_auth_err: 1222 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1223 return -EINVAL; 1224 } 1225 1226 /** Process standard openssl auth algorithms with hmac */ 1227 static int 1228 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1229 int srclen, HMAC_CTX *ctx) 1230 { 1231 unsigned int dstlen; 1232 struct rte_mbuf *m; 1233 int l, n = srclen; 1234 uint8_t *src; 1235 1236 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1237 m = m->next) 1238 offset -= rte_pktmbuf_data_len(m); 1239 1240 if (m == 0) 1241 goto process_auth_err; 1242 1243 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1244 1245 l = rte_pktmbuf_data_len(m) - offset; 1246 if (srclen <= l) { 1247 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1248 goto process_auth_err; 1249 goto process_auth_final; 1250 } 1251 1252 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1253 goto process_auth_err; 1254 1255 n -= l; 1256 1257 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1258 src = rte_pktmbuf_mtod(m, uint8_t *); 1259 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1260 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1261 goto process_auth_err; 1262 n -= l; 1263 } 1264 1265 process_auth_final: 1266 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1267 goto process_auth_err; 1268 1269 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1270 goto process_auth_err; 1271 1272 return 0; 1273 1274 process_auth_err: 1275 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1276 return -EINVAL; 1277 } 1278 1279 /*----------------------------------------------------------------------------*/ 1280 1281 /** Process auth/cipher combined operation */ 1282 static void 1283 process_openssl_combined_op 1284 (struct rte_crypto_op *op, struct openssl_session *sess, 1285 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1286 { 1287 /* cipher */ 1288 uint8_t *dst = NULL, *iv, *tag, *aad; 1289 int srclen, aadlen, status = -1; 1290 uint32_t offset; 1291 uint8_t taglen; 1292 1293 /* 1294 * Segmented destination buffer is not supported for 1295 * encryption/decryption 1296 */ 1297 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1298 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1299 return; 1300 } 1301 1302 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1303 sess->iv.offset); 1304 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1305 srclen = 0; 1306 offset = op->sym->auth.data.offset; 1307 aadlen = op->sym->auth.data.length; 1308 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1309 op->sym->auth.data.offset); 1310 tag = op->sym->auth.digest.data; 1311 if (tag == NULL) 1312 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1313 offset + aadlen); 1314 } else { 1315 srclen = op->sym->aead.data.length; 1316 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1317 op->sym->aead.data.offset); 1318 offset = op->sym->aead.data.offset; 1319 aad = op->sym->aead.aad.data; 1320 aadlen = sess->auth.aad_length; 1321 tag = op->sym->aead.digest.data; 1322 if (tag == NULL) 1323 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1324 offset + srclen); 1325 } 1326 1327 taglen = sess->auth.digest_length; 1328 1329 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1330 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1331 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1332 status = process_openssl_auth_encryption_gcm( 1333 mbuf_src, offset, srclen, 1334 aad, aadlen, iv, 1335 dst, tag, sess->cipher.ctx); 1336 else 1337 status = process_openssl_auth_encryption_ccm( 1338 mbuf_src, offset, srclen, 1339 aad, aadlen, iv, 1340 dst, tag, taglen, sess->cipher.ctx); 1341 1342 } else { 1343 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1344 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1345 status = process_openssl_auth_decryption_gcm( 1346 mbuf_src, offset, srclen, 1347 aad, aadlen, iv, 1348 dst, tag, sess->cipher.ctx); 1349 else 1350 status = process_openssl_auth_decryption_ccm( 1351 mbuf_src, offset, srclen, 1352 aad, aadlen, iv, 1353 dst, tag, taglen, sess->cipher.ctx); 1354 } 1355 1356 if (status != 0) { 1357 if (status == (-EFAULT) && 1358 sess->auth.operation == 1359 RTE_CRYPTO_AUTH_OP_VERIFY) 1360 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1361 else 1362 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1363 } 1364 } 1365 1366 /** Process cipher operation */ 1367 static void 1368 process_openssl_cipher_op 1369 (struct rte_crypto_op *op, struct openssl_session *sess, 1370 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1371 { 1372 uint8_t *dst, *iv; 1373 int srclen, status; 1374 1375 /* 1376 * Segmented destination buffer is not supported for 1377 * encryption/decryption 1378 */ 1379 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1380 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1381 return; 1382 } 1383 1384 srclen = op->sym->cipher.data.length; 1385 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1386 op->sym->cipher.data.offset); 1387 1388 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1389 sess->iv.offset); 1390 1391 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1392 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1393 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1394 op->sym->cipher.data.offset, iv, 1395 srclen, sess->cipher.ctx); 1396 else 1397 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1398 op->sym->cipher.data.offset, iv, 1399 srclen, sess->cipher.ctx); 1400 else 1401 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1402 op->sym->cipher.data.offset, iv, 1403 sess->cipher.key.data, srclen, 1404 sess->cipher.ctx); 1405 1406 if (status != 0) 1407 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1408 } 1409 1410 /** Process cipher operation */ 1411 static void 1412 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1413 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1414 struct rte_mbuf *mbuf_dst) 1415 { 1416 uint8_t *src, *dst, *iv; 1417 uint8_t block_size, last_block_len; 1418 int srclen, status = 0; 1419 1420 srclen = op->sym->cipher.data.length; 1421 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1422 op->sym->cipher.data.offset); 1423 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1424 op->sym->cipher.data.offset); 1425 1426 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1427 sess->iv.offset); 1428 1429 block_size = DES_BLOCK_SIZE; 1430 1431 last_block_len = srclen % block_size; 1432 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1433 /* Encrypt only with ECB mode XOR IV */ 1434 if (srclen < block_size) { 1435 status = process_openssl_cipher_bpi_encrypt(src, dst, 1436 iv, srclen, 1437 sess->cipher.bpi_ctx); 1438 } else { 1439 srclen -= last_block_len; 1440 /* Encrypt with the block aligned stream with CBC mode */ 1441 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1442 op->sym->cipher.data.offset, iv, 1443 srclen, sess->cipher.ctx); 1444 if (last_block_len) { 1445 /* Point at last block */ 1446 dst += srclen; 1447 /* 1448 * IV is the last encrypted block from 1449 * the previous operation 1450 */ 1451 iv = dst - block_size; 1452 src += srclen; 1453 srclen = last_block_len; 1454 /* Encrypt the last frame with ECB mode */ 1455 status |= process_openssl_cipher_bpi_encrypt(src, 1456 dst, iv, 1457 srclen, sess->cipher.bpi_ctx); 1458 } 1459 } 1460 } else { 1461 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1462 if (srclen < block_size) { 1463 status = process_openssl_cipher_bpi_encrypt(src, dst, 1464 iv, 1465 srclen, 1466 sess->cipher.bpi_ctx); 1467 } else { 1468 if (last_block_len) { 1469 /* Point at last block */ 1470 dst += srclen - last_block_len; 1471 src += srclen - last_block_len; 1472 /* 1473 * IV is the last full block 1474 */ 1475 iv = src - block_size; 1476 /* 1477 * Decrypt the last frame with ECB mode 1478 * (encrypt, as it is the same operation) 1479 */ 1480 status = process_openssl_cipher_bpi_encrypt(src, 1481 dst, iv, 1482 last_block_len, sess->cipher.bpi_ctx); 1483 /* Prepare parameters for CBC mode op */ 1484 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1485 sess->iv.offset); 1486 dst += last_block_len - srclen; 1487 srclen -= last_block_len; 1488 } 1489 1490 /* Decrypt with CBC mode */ 1491 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1492 op->sym->cipher.data.offset, iv, 1493 srclen, sess->cipher.ctx); 1494 } 1495 } 1496 1497 if (status != 0) 1498 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1499 } 1500 1501 /** Process auth operation */ 1502 static void 1503 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1504 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1505 struct rte_mbuf *mbuf_dst) 1506 { 1507 uint8_t *dst; 1508 int srclen, status; 1509 1510 srclen = op->sym->auth.data.length; 1511 1512 dst = qp->temp_digest; 1513 1514 switch (sess->auth.mode) { 1515 case OPENSSL_AUTH_AS_AUTH: 1516 status = process_openssl_auth(mbuf_src, dst, 1517 op->sym->auth.data.offset, NULL, NULL, srclen, 1518 sess->auth.auth.ctx, sess->auth.auth.evp_algo); 1519 break; 1520 case OPENSSL_AUTH_AS_HMAC: 1521 status = process_openssl_auth_hmac(mbuf_src, dst, 1522 op->sym->auth.data.offset, srclen, 1523 sess->auth.hmac.ctx); 1524 break; 1525 default: 1526 status = -1; 1527 break; 1528 } 1529 1530 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1531 if (memcmp(dst, op->sym->auth.digest.data, 1532 sess->auth.digest_length) != 0) { 1533 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1534 } 1535 } else { 1536 uint8_t *auth_dst; 1537 1538 auth_dst = op->sym->auth.digest.data; 1539 if (auth_dst == NULL) 1540 auth_dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1541 op->sym->auth.data.offset + 1542 op->sym->auth.data.length); 1543 memcpy(auth_dst, dst, sess->auth.digest_length); 1544 } 1545 1546 if (status != 0) 1547 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1548 } 1549 1550 /* process dsa sign operation */ 1551 static int 1552 process_openssl_dsa_sign_op(struct rte_crypto_op *cop, 1553 struct openssl_asym_session *sess) 1554 { 1555 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1556 DSA *dsa = sess->u.s.dsa; 1557 DSA_SIG *sign = NULL; 1558 1559 sign = DSA_do_sign(op->message.data, 1560 op->message.length, 1561 dsa); 1562 1563 if (sign == NULL) { 1564 OPENSSL_LOG(ERR, "%s:%d\n", __func__, __LINE__); 1565 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1566 } else { 1567 const BIGNUM *r = NULL, *s = NULL; 1568 get_dsa_sign(sign, &r, &s); 1569 1570 op->r.length = BN_bn2bin(r, op->r.data); 1571 op->s.length = BN_bn2bin(s, op->s.data); 1572 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1573 } 1574 1575 DSA_SIG_free(sign); 1576 1577 return 0; 1578 } 1579 1580 /* process dsa verify operation */ 1581 static int 1582 process_openssl_dsa_verify_op(struct rte_crypto_op *cop, 1583 struct openssl_asym_session *sess) 1584 { 1585 struct rte_crypto_dsa_op_param *op = &cop->asym->dsa; 1586 DSA *dsa = sess->u.s.dsa; 1587 int ret; 1588 DSA_SIG *sign = DSA_SIG_new(); 1589 BIGNUM *r = NULL, *s = NULL; 1590 BIGNUM *pub_key = NULL; 1591 1592 if (sign == NULL) { 1593 OPENSSL_LOG(ERR, " %s:%d\n", __func__, __LINE__); 1594 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1595 return -1; 1596 } 1597 1598 r = BN_bin2bn(op->r.data, 1599 op->r.length, 1600 r); 1601 s = BN_bin2bn(op->s.data, 1602 op->s.length, 1603 s); 1604 pub_key = BN_bin2bn(op->y.data, 1605 op->y.length, 1606 pub_key); 1607 if (!r || !s || !pub_key) { 1608 if (r) 1609 BN_free(r); 1610 if (s) 1611 BN_free(s); 1612 if (pub_key) 1613 BN_free(pub_key); 1614 1615 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1616 return -1; 1617 } 1618 set_dsa_sign(sign, r, s); 1619 set_dsa_pub_key(dsa, pub_key); 1620 1621 ret = DSA_do_verify(op->message.data, 1622 op->message.length, 1623 sign, 1624 dsa); 1625 1626 if (ret != 1) 1627 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1628 else 1629 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1630 1631 DSA_SIG_free(sign); 1632 1633 return 0; 1634 } 1635 1636 /* process dh operation */ 1637 static int 1638 process_openssl_dh_op(struct rte_crypto_op *cop, 1639 struct openssl_asym_session *sess) 1640 { 1641 struct rte_crypto_dh_op_param *op = &cop->asym->dh; 1642 DH *dh_key = sess->u.dh.dh_key; 1643 BIGNUM *priv_key = NULL; 1644 int ret = 0; 1645 1646 if (sess->u.dh.key_op & 1647 (1 << RTE_CRYPTO_ASYM_OP_SHARED_SECRET_COMPUTE)) { 1648 /* compute shared secret using peer public key 1649 * and current private key 1650 * shared secret = peer_key ^ priv_key mod p 1651 */ 1652 BIGNUM *peer_key = NULL; 1653 1654 /* copy private key and peer key and compute shared secret */ 1655 peer_key = BN_bin2bn(op->pub_key.data, 1656 op->pub_key.length, 1657 peer_key); 1658 if (peer_key == NULL) { 1659 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1660 return -1; 1661 } 1662 priv_key = BN_bin2bn(op->priv_key.data, 1663 op->priv_key.length, 1664 priv_key); 1665 if (priv_key == NULL) { 1666 BN_free(peer_key); 1667 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1668 return -1; 1669 } 1670 ret = set_dh_priv_key(dh_key, priv_key); 1671 if (ret) { 1672 OPENSSL_LOG(ERR, "Failed to set private key\n"); 1673 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1674 BN_free(peer_key); 1675 BN_free(priv_key); 1676 return 0; 1677 } 1678 1679 ret = DH_compute_key( 1680 op->shared_secret.data, 1681 peer_key, dh_key); 1682 if (ret < 0) { 1683 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1684 BN_free(peer_key); 1685 /* priv key is already loaded into dh, 1686 * let's not free that directly here. 1687 * DH_free() will auto free it later. 1688 */ 1689 return 0; 1690 } 1691 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1692 op->shared_secret.length = ret; 1693 BN_free(peer_key); 1694 return 0; 1695 } 1696 1697 /* 1698 * other options are public and private key generations. 1699 * 1700 * if user provides private key, 1701 * then first set DH with user provided private key 1702 */ 1703 if ((sess->u.dh.key_op & 1704 (1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)) && 1705 !(sess->u.dh.key_op & 1706 (1 << RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE))) { 1707 /* generate public key using user-provided private key 1708 * pub_key = g ^ priv_key mod p 1709 */ 1710 1711 /* load private key into DH */ 1712 priv_key = BN_bin2bn(op->priv_key.data, 1713 op->priv_key.length, 1714 priv_key); 1715 if (priv_key == NULL) { 1716 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1717 return -1; 1718 } 1719 ret = set_dh_priv_key(dh_key, priv_key); 1720 if (ret) { 1721 OPENSSL_LOG(ERR, "Failed to set private key\n"); 1722 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1723 BN_free(priv_key); 1724 return 0; 1725 } 1726 } 1727 1728 /* generate public and private key pair. 1729 * 1730 * if private key already set, generates only public key. 1731 * 1732 * if private key is not already set, then set it to random value 1733 * and update internal private key. 1734 */ 1735 if (!DH_generate_key(dh_key)) { 1736 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1737 return 0; 1738 } 1739 1740 if (sess->u.dh.key_op & (1 << RTE_CRYPTO_ASYM_OP_PUBLIC_KEY_GENERATE)) { 1741 const BIGNUM *pub_key = NULL; 1742 1743 OPENSSL_LOG(DEBUG, "%s:%d update public key\n", 1744 __func__, __LINE__); 1745 1746 /* get the generated keys */ 1747 get_dh_pub_key(dh_key, &pub_key); 1748 1749 /* output public key */ 1750 op->pub_key.length = BN_bn2bin(pub_key, 1751 op->pub_key.data); 1752 } 1753 1754 if (sess->u.dh.key_op & 1755 (1 << RTE_CRYPTO_ASYM_OP_PRIVATE_KEY_GENERATE)) { 1756 const BIGNUM *priv_key = NULL; 1757 1758 OPENSSL_LOG(DEBUG, "%s:%d updated priv key\n", 1759 __func__, __LINE__); 1760 1761 /* get the generated keys */ 1762 get_dh_priv_key(dh_key, &priv_key); 1763 1764 /* provide generated private key back to user */ 1765 op->priv_key.length = BN_bn2bin(priv_key, 1766 op->priv_key.data); 1767 } 1768 1769 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1770 1771 return 0; 1772 } 1773 1774 /* process modinv operation */ 1775 static int 1776 process_openssl_modinv_op(struct rte_crypto_op *cop, 1777 struct openssl_asym_session *sess) 1778 { 1779 struct rte_crypto_asym_op *op = cop->asym; 1780 BIGNUM *base = BN_CTX_get(sess->u.m.ctx); 1781 BIGNUM *res = BN_CTX_get(sess->u.m.ctx); 1782 1783 if (unlikely(base == NULL || res == NULL)) { 1784 if (base) 1785 BN_free(base); 1786 if (res) 1787 BN_free(res); 1788 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1789 return -1; 1790 } 1791 1792 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 1793 op->modinv.base.length, base); 1794 1795 if (BN_mod_inverse(res, base, sess->u.m.modulus, sess->u.m.ctx)) { 1796 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1797 op->modinv.base.length = BN_bn2bin(res, op->modinv.base.data); 1798 } else { 1799 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1800 } 1801 1802 return 0; 1803 } 1804 1805 /* process modexp operation */ 1806 static int 1807 process_openssl_modexp_op(struct rte_crypto_op *cop, 1808 struct openssl_asym_session *sess) 1809 { 1810 struct rte_crypto_asym_op *op = cop->asym; 1811 BIGNUM *base = BN_CTX_get(sess->u.e.ctx); 1812 BIGNUM *res = BN_CTX_get(sess->u.e.ctx); 1813 1814 if (unlikely(base == NULL || res == NULL)) { 1815 if (base) 1816 BN_free(base); 1817 if (res) 1818 BN_free(res); 1819 cop->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1820 return -1; 1821 } 1822 1823 base = BN_bin2bn((const unsigned char *)op->modinv.base.data, 1824 op->modinv.base.length, base); 1825 1826 if (BN_mod_exp(res, base, sess->u.e.exp, 1827 sess->u.e.mod, sess->u.e.ctx)) { 1828 op->modinv.base.length = BN_bn2bin(res, op->modinv.base.data); 1829 cop->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1830 } else { 1831 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1832 } 1833 1834 return 0; 1835 } 1836 1837 /* process rsa operations */ 1838 static int 1839 process_openssl_rsa_op(struct rte_crypto_op *cop, 1840 struct openssl_asym_session *sess) 1841 { 1842 int ret = 0; 1843 struct rte_crypto_asym_op *op = cop->asym; 1844 RSA *rsa = sess->u.r.rsa; 1845 uint32_t pad = (op->rsa.pad); 1846 1847 switch (pad) { 1848 case RTE_CRYPTO_RSA_PKCS1_V1_5_BT0: 1849 case RTE_CRYPTO_RSA_PKCS1_V1_5_BT1: 1850 case RTE_CRYPTO_RSA_PKCS1_V1_5_BT2: 1851 pad = RSA_PKCS1_PADDING; 1852 break; 1853 case RTE_CRYPTO_RSA_PADDING_NONE: 1854 pad = RSA_NO_PADDING; 1855 break; 1856 default: 1857 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1858 OPENSSL_LOG(ERR, 1859 "rsa pad type not supported %d\n", pad); 1860 return 0; 1861 } 1862 1863 switch (op->rsa.op_type) { 1864 case RTE_CRYPTO_ASYM_OP_ENCRYPT: 1865 ret = RSA_public_encrypt(op->rsa.message.length, 1866 op->rsa.message.data, 1867 op->rsa.message.data, 1868 rsa, 1869 pad); 1870 1871 if (ret > 0) 1872 op->rsa.message.length = ret; 1873 OPENSSL_LOG(DEBUG, 1874 "length of encrypted text %d\n", ret); 1875 break; 1876 1877 case RTE_CRYPTO_ASYM_OP_DECRYPT: 1878 ret = RSA_private_decrypt(op->rsa.message.length, 1879 op->rsa.message.data, 1880 op->rsa.message.data, 1881 rsa, 1882 pad); 1883 if (ret > 0) 1884 op->rsa.message.length = ret; 1885 break; 1886 1887 case RTE_CRYPTO_ASYM_OP_SIGN: 1888 ret = RSA_private_encrypt(op->rsa.message.length, 1889 op->rsa.message.data, 1890 op->rsa.sign.data, 1891 rsa, 1892 pad); 1893 if (ret > 0) 1894 op->rsa.sign.length = ret; 1895 break; 1896 1897 case RTE_CRYPTO_ASYM_OP_VERIFY: 1898 ret = RSA_public_decrypt(op->rsa.sign.length, 1899 op->rsa.sign.data, 1900 op->rsa.sign.data, 1901 rsa, 1902 pad); 1903 1904 OPENSSL_LOG(DEBUG, 1905 "Length of public_decrypt %d " 1906 "length of message %zd\n", 1907 ret, op->rsa.message.length); 1908 1909 if (memcmp(op->rsa.sign.data, op->rsa.message.data, 1910 op->rsa.message.length)) { 1911 OPENSSL_LOG(ERR, 1912 "RSA sign Verification failed"); 1913 return -1; 1914 } 1915 break; 1916 1917 default: 1918 /* allow ops with invalid args to be pushed to 1919 * completion queue 1920 */ 1921 cop->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1922 break; 1923 } 1924 1925 if (ret < 0) 1926 cop->status = RTE_CRYPTO_OP_STATUS_ERROR; 1927 1928 return 0; 1929 } 1930 1931 static int 1932 process_asym_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1933 struct openssl_asym_session *sess) 1934 { 1935 int retval = 0; 1936 1937 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1938 1939 switch (sess->xfrm_type) { 1940 case RTE_CRYPTO_ASYM_XFORM_RSA: 1941 retval = process_openssl_rsa_op(op, sess); 1942 break; 1943 case RTE_CRYPTO_ASYM_XFORM_MODEX: 1944 retval = process_openssl_modexp_op(op, sess); 1945 break; 1946 case RTE_CRYPTO_ASYM_XFORM_MODINV: 1947 retval = process_openssl_modinv_op(op, sess); 1948 break; 1949 case RTE_CRYPTO_ASYM_XFORM_DH: 1950 retval = process_openssl_dh_op(op, sess); 1951 break; 1952 case RTE_CRYPTO_ASYM_XFORM_DSA: 1953 if (op->asym->dsa.op_type == RTE_CRYPTO_ASYM_OP_SIGN) 1954 retval = process_openssl_dsa_sign_op(op, sess); 1955 else if (op->asym->dsa.op_type == 1956 RTE_CRYPTO_ASYM_OP_VERIFY) 1957 retval = 1958 process_openssl_dsa_verify_op(op, sess); 1959 else 1960 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1961 break; 1962 default: 1963 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 1964 break; 1965 } 1966 if (!retval) { 1967 /* op processed so push to completion queue as processed */ 1968 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 1969 if (retval) 1970 /* return error if failed to put in completion queue */ 1971 retval = -1; 1972 } 1973 1974 return retval; 1975 } 1976 1977 /** Process crypto operation for mbuf */ 1978 static int 1979 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1980 struct openssl_session *sess) 1981 { 1982 struct rte_mbuf *msrc, *mdst; 1983 int retval; 1984 1985 msrc = op->sym->m_src; 1986 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 1987 1988 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1989 1990 switch (sess->chain_order) { 1991 case OPENSSL_CHAIN_ONLY_CIPHER: 1992 process_openssl_cipher_op(op, sess, msrc, mdst); 1993 break; 1994 case OPENSSL_CHAIN_ONLY_AUTH: 1995 process_openssl_auth_op(qp, op, sess, msrc, mdst); 1996 break; 1997 case OPENSSL_CHAIN_CIPHER_AUTH: 1998 process_openssl_cipher_op(op, sess, msrc, mdst); 1999 process_openssl_auth_op(qp, op, sess, mdst, mdst); 2000 break; 2001 case OPENSSL_CHAIN_AUTH_CIPHER: 2002 process_openssl_auth_op(qp, op, sess, msrc, mdst); 2003 process_openssl_cipher_op(op, sess, msrc, mdst); 2004 break; 2005 case OPENSSL_CHAIN_COMBINED: 2006 process_openssl_combined_op(op, sess, msrc, mdst); 2007 break; 2008 case OPENSSL_CHAIN_CIPHER_BPI: 2009 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 2010 break; 2011 default: 2012 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 2013 break; 2014 } 2015 2016 /* Free session if a session-less crypto op */ 2017 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 2018 openssl_reset_session(sess); 2019 memset(sess, 0, sizeof(struct openssl_session)); 2020 memset(op->sym->session, 0, 2021 rte_cryptodev_sym_get_header_session_size()); 2022 rte_mempool_put(qp->sess_mp, sess); 2023 rte_mempool_put(qp->sess_mp, op->sym->session); 2024 op->sym->session = NULL; 2025 } 2026 2027 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 2028 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 2029 2030 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 2031 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 2032 else 2033 retval = -1; 2034 2035 return retval; 2036 } 2037 2038 /* 2039 *------------------------------------------------------------------------------ 2040 * PMD Framework 2041 *------------------------------------------------------------------------------ 2042 */ 2043 2044 /** Enqueue burst */ 2045 static uint16_t 2046 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 2047 uint16_t nb_ops) 2048 { 2049 void *sess; 2050 struct openssl_qp *qp = queue_pair; 2051 int i, retval; 2052 2053 for (i = 0; i < nb_ops; i++) { 2054 sess = get_session(qp, ops[i]); 2055 if (unlikely(sess == NULL)) 2056 goto enqueue_err; 2057 2058 if (ops[i]->type == RTE_CRYPTO_OP_TYPE_SYMMETRIC) 2059 retval = process_op(qp, ops[i], 2060 (struct openssl_session *) sess); 2061 else 2062 retval = process_asym_op(qp, ops[i], 2063 (struct openssl_asym_session *) sess); 2064 if (unlikely(retval < 0)) 2065 goto enqueue_err; 2066 } 2067 2068 qp->stats.enqueued_count += i; 2069 return i; 2070 2071 enqueue_err: 2072 qp->stats.enqueue_err_count++; 2073 return i; 2074 } 2075 2076 /** Dequeue burst */ 2077 static uint16_t 2078 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 2079 uint16_t nb_ops) 2080 { 2081 struct openssl_qp *qp = queue_pair; 2082 2083 unsigned int nb_dequeued = 0; 2084 2085 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 2086 (void **)ops, nb_ops, NULL); 2087 qp->stats.dequeued_count += nb_dequeued; 2088 2089 return nb_dequeued; 2090 } 2091 2092 /** Create OPENSSL crypto device */ 2093 static int 2094 cryptodev_openssl_create(const char *name, 2095 struct rte_vdev_device *vdev, 2096 struct rte_cryptodev_pmd_init_params *init_params) 2097 { 2098 struct rte_cryptodev *dev; 2099 struct openssl_private *internals; 2100 2101 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 2102 if (dev == NULL) { 2103 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 2104 goto init_error; 2105 } 2106 2107 dev->driver_id = cryptodev_driver_id; 2108 dev->dev_ops = rte_openssl_pmd_ops; 2109 2110 /* register rx/tx burst functions for data path */ 2111 dev->dequeue_burst = openssl_pmd_dequeue_burst; 2112 dev->enqueue_burst = openssl_pmd_enqueue_burst; 2113 2114 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 2115 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 2116 RTE_CRYPTODEV_FF_CPU_AESNI | 2117 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 2118 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT | 2119 RTE_CRYPTODEV_FF_ASYMMETRIC_CRYPTO; 2120 2121 /* Set vector instructions mode supported */ 2122 internals = dev->data->dev_private; 2123 2124 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 2125 2126 return 0; 2127 2128 init_error: 2129 OPENSSL_LOG(ERR, "driver %s: create failed", 2130 init_params->name); 2131 2132 cryptodev_openssl_remove(vdev); 2133 return -EFAULT; 2134 } 2135 2136 /** Initialise OPENSSL crypto device */ 2137 static int 2138 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 2139 { 2140 struct rte_cryptodev_pmd_init_params init_params = { 2141 "", 2142 sizeof(struct openssl_private), 2143 rte_socket_id(), 2144 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 2145 }; 2146 const char *name; 2147 const char *input_args; 2148 2149 name = rte_vdev_device_name(vdev); 2150 if (name == NULL) 2151 return -EINVAL; 2152 input_args = rte_vdev_device_args(vdev); 2153 2154 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 2155 2156 return cryptodev_openssl_create(name, vdev, &init_params); 2157 } 2158 2159 /** Uninitialise OPENSSL crypto device */ 2160 static int 2161 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 2162 { 2163 struct rte_cryptodev *cryptodev; 2164 const char *name; 2165 2166 name = rte_vdev_device_name(vdev); 2167 if (name == NULL) 2168 return -EINVAL; 2169 2170 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 2171 if (cryptodev == NULL) 2172 return -ENODEV; 2173 2174 return rte_cryptodev_pmd_destroy(cryptodev); 2175 } 2176 2177 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 2178 .probe = cryptodev_openssl_probe, 2179 .remove = cryptodev_openssl_remove 2180 }; 2181 2182 static struct cryptodev_driver openssl_crypto_drv; 2183 2184 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 2185 cryptodev_openssl_pmd_drv); 2186 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 2187 "max_nb_queue_pairs=<int> " 2188 "socket_id=<int>"); 2189 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 2190 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 2191 2192 RTE_INIT(openssl_init_log) 2193 { 2194 openssl_logtype_driver = rte_log_register("pmd.crypto.openssl"); 2195 } 2196