1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_common.h> 6 #include <rte_hexdump.h> 7 #include <rte_cryptodev.h> 8 #include <rte_cryptodev_pmd.h> 9 #include <rte_bus_vdev.h> 10 #include <rte_malloc.h> 11 #include <rte_cpuflags.h> 12 13 #include <openssl/hmac.h> 14 #include <openssl/evp.h> 15 16 #include "rte_openssl_pmd_private.h" 17 18 #define DES_BLOCK_SIZE 8 19 20 static uint8_t cryptodev_driver_id; 21 22 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 23 static HMAC_CTX *HMAC_CTX_new(void) 24 { 25 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 26 27 if (ctx != NULL) 28 HMAC_CTX_init(ctx); 29 return ctx; 30 } 31 32 static void HMAC_CTX_free(HMAC_CTX *ctx) 33 { 34 if (ctx != NULL) { 35 HMAC_CTX_cleanup(ctx); 36 OPENSSL_free(ctx); 37 } 38 } 39 #endif 40 41 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 42 43 /*----------------------------------------------------------------------------*/ 44 45 /** 46 * Increment counter by 1 47 * Counter is 64 bit array, big-endian 48 */ 49 static void 50 ctr_inc(uint8_t *ctr) 51 { 52 uint64_t *ctr64 = (uint64_t *)ctr; 53 54 *ctr64 = __builtin_bswap64(*ctr64); 55 (*ctr64)++; 56 *ctr64 = __builtin_bswap64(*ctr64); 57 } 58 59 /* 60 *------------------------------------------------------------------------------ 61 * Session Prepare 62 *------------------------------------------------------------------------------ 63 */ 64 65 /** Get xform chain order */ 66 static enum openssl_chain_order 67 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 68 { 69 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 70 71 if (xform != NULL) { 72 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 73 if (xform->next == NULL) 74 res = OPENSSL_CHAIN_ONLY_AUTH; 75 else if (xform->next->type == 76 RTE_CRYPTO_SYM_XFORM_CIPHER) 77 res = OPENSSL_CHAIN_AUTH_CIPHER; 78 } 79 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 80 if (xform->next == NULL) 81 res = OPENSSL_CHAIN_ONLY_CIPHER; 82 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 83 res = OPENSSL_CHAIN_CIPHER_AUTH; 84 } 85 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 86 res = OPENSSL_CHAIN_COMBINED; 87 } 88 89 return res; 90 } 91 92 /** Get session cipher key from input cipher key */ 93 static void 94 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key) 95 { 96 memcpy(session_key, input_key, keylen); 97 } 98 99 /** Get key ede 24 bytes standard from input key */ 100 static int 101 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede) 102 { 103 int res = 0; 104 105 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 106 switch (keylen) { 107 case 24: 108 memcpy(key_ede, key, 24); 109 break; 110 case 16: 111 /* K3 = K1 */ 112 memcpy(key_ede, key, 16); 113 memcpy(key_ede + 16, key, 8); 114 break; 115 case 8: 116 /* K1 = K2 = K3 (DES compatibility) */ 117 memcpy(key_ede, key, 8); 118 memcpy(key_ede + 8, key, 8); 119 memcpy(key_ede + 16, key, 8); 120 break; 121 default: 122 OPENSSL_LOG(ERR, "Unsupported key size"); 123 res = -EINVAL; 124 } 125 126 return res; 127 } 128 129 /** Get adequate openssl function for input cipher algorithm */ 130 static uint8_t 131 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 132 const EVP_CIPHER **algo) 133 { 134 int res = 0; 135 136 if (algo != NULL) { 137 switch (sess_algo) { 138 case RTE_CRYPTO_CIPHER_3DES_CBC: 139 switch (keylen) { 140 case 8: 141 *algo = EVP_des_cbc(); 142 break; 143 case 16: 144 *algo = EVP_des_ede_cbc(); 145 break; 146 case 24: 147 *algo = EVP_des_ede3_cbc(); 148 break; 149 default: 150 res = -EINVAL; 151 } 152 break; 153 case RTE_CRYPTO_CIPHER_3DES_CTR: 154 break; 155 case RTE_CRYPTO_CIPHER_AES_CBC: 156 switch (keylen) { 157 case 16: 158 *algo = EVP_aes_128_cbc(); 159 break; 160 case 24: 161 *algo = EVP_aes_192_cbc(); 162 break; 163 case 32: 164 *algo = EVP_aes_256_cbc(); 165 break; 166 default: 167 res = -EINVAL; 168 } 169 break; 170 case RTE_CRYPTO_CIPHER_AES_CTR: 171 switch (keylen) { 172 case 16: 173 *algo = EVP_aes_128_ctr(); 174 break; 175 case 24: 176 *algo = EVP_aes_192_ctr(); 177 break; 178 case 32: 179 *algo = EVP_aes_256_ctr(); 180 break; 181 default: 182 res = -EINVAL; 183 } 184 break; 185 default: 186 res = -EINVAL; 187 break; 188 } 189 } else { 190 res = -EINVAL; 191 } 192 193 return res; 194 } 195 196 /** Get adequate openssl function for input auth algorithm */ 197 static uint8_t 198 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 199 const EVP_MD **algo) 200 { 201 int res = 0; 202 203 if (algo != NULL) { 204 switch (sessalgo) { 205 case RTE_CRYPTO_AUTH_MD5: 206 case RTE_CRYPTO_AUTH_MD5_HMAC: 207 *algo = EVP_md5(); 208 break; 209 case RTE_CRYPTO_AUTH_SHA1: 210 case RTE_CRYPTO_AUTH_SHA1_HMAC: 211 *algo = EVP_sha1(); 212 break; 213 case RTE_CRYPTO_AUTH_SHA224: 214 case RTE_CRYPTO_AUTH_SHA224_HMAC: 215 *algo = EVP_sha224(); 216 break; 217 case RTE_CRYPTO_AUTH_SHA256: 218 case RTE_CRYPTO_AUTH_SHA256_HMAC: 219 *algo = EVP_sha256(); 220 break; 221 case RTE_CRYPTO_AUTH_SHA384: 222 case RTE_CRYPTO_AUTH_SHA384_HMAC: 223 *algo = EVP_sha384(); 224 break; 225 case RTE_CRYPTO_AUTH_SHA512: 226 case RTE_CRYPTO_AUTH_SHA512_HMAC: 227 *algo = EVP_sha512(); 228 break; 229 default: 230 res = -EINVAL; 231 break; 232 } 233 } else { 234 res = -EINVAL; 235 } 236 237 return res; 238 } 239 240 /** Get adequate openssl function for input cipher algorithm */ 241 static uint8_t 242 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 243 const EVP_CIPHER **algo) 244 { 245 int res = 0; 246 247 if (algo != NULL) { 248 switch (sess_algo) { 249 case RTE_CRYPTO_AEAD_AES_GCM: 250 switch (keylen) { 251 case 16: 252 *algo = EVP_aes_128_gcm(); 253 break; 254 case 24: 255 *algo = EVP_aes_192_gcm(); 256 break; 257 case 32: 258 *algo = EVP_aes_256_gcm(); 259 break; 260 default: 261 res = -EINVAL; 262 } 263 break; 264 case RTE_CRYPTO_AEAD_AES_CCM: 265 switch (keylen) { 266 case 16: 267 *algo = EVP_aes_128_ccm(); 268 break; 269 case 24: 270 *algo = EVP_aes_192_ccm(); 271 break; 272 case 32: 273 *algo = EVP_aes_256_ccm(); 274 break; 275 default: 276 res = -EINVAL; 277 } 278 break; 279 default: 280 res = -EINVAL; 281 break; 282 } 283 } else { 284 res = -EINVAL; 285 } 286 287 return res; 288 } 289 290 /* Set session AEAD encryption parameters */ 291 static int 292 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 293 enum rte_crypto_aead_algorithm algo, 294 uint8_t tag_len, uint8_t *key) 295 { 296 int iv_type = 0; 297 unsigned int do_ccm; 298 299 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 300 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 301 302 /* Select AEAD algo */ 303 switch (algo) { 304 case RTE_CRYPTO_AEAD_AES_GCM: 305 iv_type = EVP_CTRL_GCM_SET_IVLEN; 306 if (tag_len != 16) 307 return -EINVAL; 308 do_ccm = 0; 309 break; 310 case RTE_CRYPTO_AEAD_AES_CCM: 311 iv_type = EVP_CTRL_CCM_SET_IVLEN; 312 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 313 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 314 return -EINVAL; 315 do_ccm = 1; 316 break; 317 default: 318 return -ENOTSUP; 319 } 320 321 sess->cipher.mode = OPENSSL_CIPHER_LIB; 322 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 323 324 if (get_aead_algo(algo, sess->cipher.key.length, 325 &sess->cipher.evp_algo) != 0) 326 return -EINVAL; 327 328 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 329 330 sess->chain_order = OPENSSL_CHAIN_COMBINED; 331 332 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 333 NULL, NULL, NULL) <= 0) 334 return -EINVAL; 335 336 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, 337 NULL) <= 0) 338 return -EINVAL; 339 340 if (do_ccm) 341 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 342 tag_len, NULL); 343 344 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 345 return -EINVAL; 346 347 return 0; 348 } 349 350 /* Set session AEAD decryption parameters */ 351 static int 352 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 353 enum rte_crypto_aead_algorithm algo, 354 uint8_t tag_len, uint8_t *key) 355 { 356 int iv_type = 0; 357 unsigned int do_ccm = 0; 358 359 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 360 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 361 362 /* Select AEAD algo */ 363 switch (algo) { 364 case RTE_CRYPTO_AEAD_AES_GCM: 365 iv_type = EVP_CTRL_GCM_SET_IVLEN; 366 if (tag_len != 16) 367 return -EINVAL; 368 break; 369 case RTE_CRYPTO_AEAD_AES_CCM: 370 iv_type = EVP_CTRL_CCM_SET_IVLEN; 371 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 372 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 373 return -EINVAL; 374 do_ccm = 1; 375 break; 376 default: 377 return -ENOTSUP; 378 } 379 380 sess->cipher.mode = OPENSSL_CIPHER_LIB; 381 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 382 383 if (get_aead_algo(algo, sess->cipher.key.length, 384 &sess->cipher.evp_algo) != 0) 385 return -EINVAL; 386 387 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 388 389 sess->chain_order = OPENSSL_CHAIN_COMBINED; 390 391 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 392 NULL, NULL, NULL) <= 0) 393 return -EINVAL; 394 395 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, 396 sess->iv.length, NULL) <= 0) 397 return -EINVAL; 398 399 if (do_ccm) 400 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 401 tag_len, NULL); 402 403 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 404 return -EINVAL; 405 406 return 0; 407 } 408 409 /** Set session cipher parameters */ 410 static int 411 openssl_set_session_cipher_parameters(struct openssl_session *sess, 412 const struct rte_crypto_sym_xform *xform) 413 { 414 /* Select cipher direction */ 415 sess->cipher.direction = xform->cipher.op; 416 /* Select cipher key */ 417 sess->cipher.key.length = xform->cipher.key.length; 418 419 /* Set IV parameters */ 420 sess->iv.offset = xform->cipher.iv.offset; 421 sess->iv.length = xform->cipher.iv.length; 422 423 /* Select cipher algo */ 424 switch (xform->cipher.algo) { 425 case RTE_CRYPTO_CIPHER_3DES_CBC: 426 case RTE_CRYPTO_CIPHER_AES_CBC: 427 case RTE_CRYPTO_CIPHER_AES_CTR: 428 sess->cipher.mode = OPENSSL_CIPHER_LIB; 429 sess->cipher.algo = xform->cipher.algo; 430 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 431 432 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 433 &sess->cipher.evp_algo) != 0) 434 return -EINVAL; 435 436 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 437 sess->cipher.key.data); 438 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 439 if (EVP_EncryptInit_ex(sess->cipher.ctx, 440 sess->cipher.evp_algo, 441 NULL, xform->cipher.key.data, 442 NULL) != 1) { 443 return -EINVAL; 444 } 445 } else if (sess->cipher.direction == 446 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 447 if (EVP_DecryptInit_ex(sess->cipher.ctx, 448 sess->cipher.evp_algo, 449 NULL, xform->cipher.key.data, 450 NULL) != 1) { 451 return -EINVAL; 452 } 453 } 454 455 break; 456 457 case RTE_CRYPTO_CIPHER_3DES_CTR: 458 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 459 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 460 461 if (get_cipher_key_ede(xform->cipher.key.data, 462 sess->cipher.key.length, 463 sess->cipher.key.data) != 0) 464 return -EINVAL; 465 break; 466 467 case RTE_CRYPTO_CIPHER_DES_CBC: 468 sess->cipher.algo = xform->cipher.algo; 469 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 470 sess->cipher.evp_algo = EVP_des_cbc(); 471 472 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 473 sess->cipher.key.data); 474 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 475 if (EVP_EncryptInit_ex(sess->cipher.ctx, 476 sess->cipher.evp_algo, 477 NULL, xform->cipher.key.data, 478 NULL) != 1) { 479 return -EINVAL; 480 } 481 } else if (sess->cipher.direction == 482 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 483 if (EVP_DecryptInit_ex(sess->cipher.ctx, 484 sess->cipher.evp_algo, 485 NULL, xform->cipher.key.data, 486 NULL) != 1) { 487 return -EINVAL; 488 } 489 } 490 491 break; 492 493 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 494 sess->cipher.algo = xform->cipher.algo; 495 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 496 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 497 sess->cipher.evp_algo = EVP_des_cbc(); 498 499 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 500 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 501 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 502 NULL, xform->cipher.key.data, 0) != 1) 503 return -EINVAL; 504 505 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 506 sess->cipher.key.data); 507 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 508 if (EVP_EncryptInit_ex(sess->cipher.ctx, 509 sess->cipher.evp_algo, 510 NULL, xform->cipher.key.data, 511 NULL) != 1) { 512 return -EINVAL; 513 } 514 } else if (sess->cipher.direction == 515 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 516 if (EVP_DecryptInit_ex(sess->cipher.ctx, 517 sess->cipher.evp_algo, 518 NULL, xform->cipher.key.data, 519 NULL) != 1) { 520 return -EINVAL; 521 } 522 } 523 524 break; 525 default: 526 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 527 return -ENOTSUP; 528 } 529 530 return 0; 531 } 532 533 /* Set session auth parameters */ 534 static int 535 openssl_set_session_auth_parameters(struct openssl_session *sess, 536 const struct rte_crypto_sym_xform *xform) 537 { 538 /* Select auth generate/verify */ 539 sess->auth.operation = xform->auth.op; 540 sess->auth.algo = xform->auth.algo; 541 542 sess->auth.digest_length = xform->auth.digest_length; 543 544 /* Select auth algo */ 545 switch (xform->auth.algo) { 546 case RTE_CRYPTO_AUTH_AES_GMAC: 547 /* 548 * OpenSSL requires GMAC to be a GCM operation 549 * with no cipher data length 550 */ 551 sess->cipher.key.length = xform->auth.key.length; 552 553 /* Set IV parameters */ 554 sess->iv.offset = xform->auth.iv.offset; 555 sess->iv.length = xform->auth.iv.length; 556 557 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 558 return openssl_set_sess_aead_enc_param(sess, 559 RTE_CRYPTO_AEAD_AES_GCM, 560 xform->auth.digest_length, 561 xform->auth.key.data); 562 else 563 return openssl_set_sess_aead_dec_param(sess, 564 RTE_CRYPTO_AEAD_AES_GCM, 565 xform->auth.digest_length, 566 xform->auth.key.data); 567 break; 568 569 case RTE_CRYPTO_AUTH_MD5: 570 case RTE_CRYPTO_AUTH_SHA1: 571 case RTE_CRYPTO_AUTH_SHA224: 572 case RTE_CRYPTO_AUTH_SHA256: 573 case RTE_CRYPTO_AUTH_SHA384: 574 case RTE_CRYPTO_AUTH_SHA512: 575 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 576 if (get_auth_algo(xform->auth.algo, 577 &sess->auth.auth.evp_algo) != 0) 578 return -EINVAL; 579 sess->auth.auth.ctx = EVP_MD_CTX_create(); 580 break; 581 582 case RTE_CRYPTO_AUTH_MD5_HMAC: 583 case RTE_CRYPTO_AUTH_SHA1_HMAC: 584 case RTE_CRYPTO_AUTH_SHA224_HMAC: 585 case RTE_CRYPTO_AUTH_SHA256_HMAC: 586 case RTE_CRYPTO_AUTH_SHA384_HMAC: 587 case RTE_CRYPTO_AUTH_SHA512_HMAC: 588 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 589 sess->auth.hmac.ctx = HMAC_CTX_new(); 590 if (get_auth_algo(xform->auth.algo, 591 &sess->auth.hmac.evp_algo) != 0) 592 return -EINVAL; 593 594 if (HMAC_Init_ex(sess->auth.hmac.ctx, 595 xform->auth.key.data, 596 xform->auth.key.length, 597 sess->auth.hmac.evp_algo, NULL) != 1) 598 return -EINVAL; 599 break; 600 601 default: 602 return -ENOTSUP; 603 } 604 605 return 0; 606 } 607 608 /* Set session AEAD parameters */ 609 static int 610 openssl_set_session_aead_parameters(struct openssl_session *sess, 611 const struct rte_crypto_sym_xform *xform) 612 { 613 /* Select cipher key */ 614 sess->cipher.key.length = xform->aead.key.length; 615 616 /* Set IV parameters */ 617 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 618 /* 619 * For AES-CCM, the actual IV is placed 620 * one byte after the start of the IV field, 621 * according to the API. 622 */ 623 sess->iv.offset = xform->aead.iv.offset + 1; 624 else 625 sess->iv.offset = xform->aead.iv.offset; 626 627 sess->iv.length = xform->aead.iv.length; 628 629 sess->auth.aad_length = xform->aead.aad_length; 630 sess->auth.digest_length = xform->aead.digest_length; 631 632 sess->aead_algo = xform->aead.algo; 633 /* Select cipher direction */ 634 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 635 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 636 xform->aead.digest_length, xform->aead.key.data); 637 else 638 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 639 xform->aead.digest_length, xform->aead.key.data); 640 } 641 642 /** Parse crypto xform chain and set private session parameters */ 643 int 644 openssl_set_session_parameters(struct openssl_session *sess, 645 const struct rte_crypto_sym_xform *xform) 646 { 647 const struct rte_crypto_sym_xform *cipher_xform = NULL; 648 const struct rte_crypto_sym_xform *auth_xform = NULL; 649 const struct rte_crypto_sym_xform *aead_xform = NULL; 650 int ret; 651 652 sess->chain_order = openssl_get_chain_order(xform); 653 switch (sess->chain_order) { 654 case OPENSSL_CHAIN_ONLY_CIPHER: 655 cipher_xform = xform; 656 break; 657 case OPENSSL_CHAIN_ONLY_AUTH: 658 auth_xform = xform; 659 break; 660 case OPENSSL_CHAIN_CIPHER_AUTH: 661 cipher_xform = xform; 662 auth_xform = xform->next; 663 break; 664 case OPENSSL_CHAIN_AUTH_CIPHER: 665 auth_xform = xform; 666 cipher_xform = xform->next; 667 break; 668 case OPENSSL_CHAIN_COMBINED: 669 aead_xform = xform; 670 break; 671 default: 672 return -EINVAL; 673 } 674 675 /* Default IV length = 0 */ 676 sess->iv.length = 0; 677 678 /* cipher_xform must be check before auth_xform */ 679 if (cipher_xform) { 680 ret = openssl_set_session_cipher_parameters( 681 sess, cipher_xform); 682 if (ret != 0) { 683 OPENSSL_LOG(ERR, 684 "Invalid/unsupported cipher parameters"); 685 return ret; 686 } 687 } 688 689 if (auth_xform) { 690 ret = openssl_set_session_auth_parameters(sess, auth_xform); 691 if (ret != 0) { 692 OPENSSL_LOG(ERR, 693 "Invalid/unsupported auth parameters"); 694 return ret; 695 } 696 } 697 698 if (aead_xform) { 699 ret = openssl_set_session_aead_parameters(sess, aead_xform); 700 if (ret != 0) { 701 OPENSSL_LOG(ERR, 702 "Invalid/unsupported AEAD parameters"); 703 return ret; 704 } 705 } 706 707 return 0; 708 } 709 710 /** Reset private session parameters */ 711 void 712 openssl_reset_session(struct openssl_session *sess) 713 { 714 EVP_CIPHER_CTX_free(sess->cipher.ctx); 715 716 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 717 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 718 719 switch (sess->auth.mode) { 720 case OPENSSL_AUTH_AS_AUTH: 721 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 722 break; 723 case OPENSSL_AUTH_AS_HMAC: 724 EVP_PKEY_free(sess->auth.hmac.pkey); 725 HMAC_CTX_free(sess->auth.hmac.ctx); 726 break; 727 default: 728 break; 729 } 730 } 731 732 /** Provide session for operation */ 733 static struct openssl_session * 734 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 735 { 736 struct openssl_session *sess = NULL; 737 738 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 739 /* get existing session */ 740 if (likely(op->sym->session != NULL)) 741 sess = (struct openssl_session *) 742 get_sym_session_private_data( 743 op->sym->session, 744 cryptodev_driver_id); 745 } else { 746 /* provide internal session */ 747 void *_sess = NULL; 748 void *_sess_private_data = NULL; 749 750 if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) 751 return NULL; 752 753 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) 754 return NULL; 755 756 sess = (struct openssl_session *)_sess_private_data; 757 758 if (unlikely(openssl_set_session_parameters(sess, 759 op->sym->xform) != 0)) { 760 rte_mempool_put(qp->sess_mp, _sess); 761 rte_mempool_put(qp->sess_mp, _sess_private_data); 762 sess = NULL; 763 } 764 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 765 set_sym_session_private_data(op->sym->session, 766 cryptodev_driver_id, _sess_private_data); 767 } 768 769 if (sess == NULL) 770 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 771 772 return sess; 773 } 774 775 /* 776 *------------------------------------------------------------------------------ 777 * Process Operations 778 *------------------------------------------------------------------------------ 779 */ 780 static inline int 781 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 782 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 783 { 784 struct rte_mbuf *m; 785 int dstlen; 786 int l, n = srclen; 787 uint8_t *src; 788 789 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 790 m = m->next) 791 offset -= rte_pktmbuf_data_len(m); 792 793 if (m == 0) 794 return -1; 795 796 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 797 798 l = rte_pktmbuf_data_len(m) - offset; 799 if (srclen <= l) { 800 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 801 return -1; 802 *dst += l; 803 return 0; 804 } 805 806 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 807 return -1; 808 809 *dst += dstlen; 810 n -= l; 811 812 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 813 src = rte_pktmbuf_mtod(m, uint8_t *); 814 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 815 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 816 return -1; 817 *dst += dstlen; 818 n -= l; 819 } 820 821 return 0; 822 } 823 824 static inline int 825 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 826 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 827 { 828 struct rte_mbuf *m; 829 int dstlen; 830 int l, n = srclen; 831 uint8_t *src; 832 833 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 834 m = m->next) 835 offset -= rte_pktmbuf_data_len(m); 836 837 if (m == 0) 838 return -1; 839 840 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 841 842 l = rte_pktmbuf_data_len(m) - offset; 843 if (srclen <= l) { 844 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 845 return -1; 846 *dst += l; 847 return 0; 848 } 849 850 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 851 return -1; 852 853 *dst += dstlen; 854 n -= l; 855 856 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 857 src = rte_pktmbuf_mtod(m, uint8_t *); 858 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 859 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 860 return -1; 861 *dst += dstlen; 862 n -= l; 863 } 864 865 return 0; 866 } 867 868 /** Process standard openssl cipher encryption */ 869 static int 870 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 871 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) 872 { 873 int totlen; 874 875 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 876 goto process_cipher_encrypt_err; 877 878 EVP_CIPHER_CTX_set_padding(ctx, 0); 879 880 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 881 srclen, ctx)) 882 goto process_cipher_encrypt_err; 883 884 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 885 goto process_cipher_encrypt_err; 886 887 return 0; 888 889 process_cipher_encrypt_err: 890 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 891 return -EINVAL; 892 } 893 894 /** Process standard openssl cipher encryption */ 895 static int 896 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 897 uint8_t *iv, int srclen, 898 EVP_CIPHER_CTX *ctx) 899 { 900 uint8_t i; 901 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 902 int encrypted_ivlen; 903 904 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 905 iv, DES_BLOCK_SIZE) <= 0) 906 goto process_cipher_encrypt_err; 907 908 for (i = 0; i < srclen; i++) 909 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 910 911 return 0; 912 913 process_cipher_encrypt_err: 914 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 915 return -EINVAL; 916 } 917 /** Process standard openssl cipher decryption */ 918 static int 919 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 920 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) 921 { 922 int totlen; 923 924 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 925 goto process_cipher_decrypt_err; 926 927 EVP_CIPHER_CTX_set_padding(ctx, 0); 928 929 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 930 srclen, ctx)) 931 goto process_cipher_decrypt_err; 932 933 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 934 goto process_cipher_decrypt_err; 935 return 0; 936 937 process_cipher_decrypt_err: 938 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 939 return -EINVAL; 940 } 941 942 /** Process cipher des 3 ctr encryption, decryption algorithm */ 943 static int 944 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 945 int offset, uint8_t *iv, uint8_t *key, int srclen, 946 EVP_CIPHER_CTX *ctx) 947 { 948 uint8_t ebuf[8], ctr[8]; 949 int unused, n; 950 struct rte_mbuf *m; 951 uint8_t *src; 952 int l; 953 954 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 955 m = m->next) 956 offset -= rte_pktmbuf_data_len(m); 957 958 if (m == 0) 959 goto process_cipher_des3ctr_err; 960 961 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 962 l = rte_pktmbuf_data_len(m) - offset; 963 964 /* We use 3DES encryption also for decryption. 965 * IV is not important for 3DES ecb 966 */ 967 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 968 goto process_cipher_des3ctr_err; 969 970 memcpy(ctr, iv, 8); 971 972 for (n = 0; n < srclen; n++) { 973 if (n % 8 == 0) { 974 if (EVP_EncryptUpdate(ctx, 975 (unsigned char *)&ebuf, &unused, 976 (const unsigned char *)&ctr, 8) <= 0) 977 goto process_cipher_des3ctr_err; 978 ctr_inc(ctr); 979 } 980 dst[n] = *(src++) ^ ebuf[n % 8]; 981 982 l--; 983 if (!l) { 984 m = m->next; 985 if (m) { 986 src = rte_pktmbuf_mtod(m, uint8_t *); 987 l = rte_pktmbuf_data_len(m); 988 } 989 } 990 } 991 992 return 0; 993 994 process_cipher_des3ctr_err: 995 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 996 return -EINVAL; 997 } 998 999 /** Process AES-GCM encrypt algorithm */ 1000 static int 1001 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1002 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1003 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1004 { 1005 int len = 0, unused = 0; 1006 uint8_t empty[] = {}; 1007 1008 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1009 goto process_auth_encryption_gcm_err; 1010 1011 if (aadlen > 0) 1012 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1013 goto process_auth_encryption_gcm_err; 1014 1015 if (srclen > 0) 1016 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1017 srclen, ctx)) 1018 goto process_auth_encryption_gcm_err; 1019 1020 /* Workaround open ssl bug in version less then 1.0.1f */ 1021 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1022 goto process_auth_encryption_gcm_err; 1023 1024 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1025 goto process_auth_encryption_gcm_err; 1026 1027 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1028 goto process_auth_encryption_gcm_err; 1029 1030 return 0; 1031 1032 process_auth_encryption_gcm_err: 1033 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1034 return -EINVAL; 1035 } 1036 1037 /** Process AES-CCM encrypt algorithm */ 1038 static int 1039 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1040 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1041 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1042 { 1043 int len = 0; 1044 1045 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1046 goto process_auth_encryption_ccm_err; 1047 1048 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1049 goto process_auth_encryption_ccm_err; 1050 1051 if (aadlen > 0) 1052 /* 1053 * For AES-CCM, the actual AAD is placed 1054 * 18 bytes after the start of the AAD field, 1055 * according to the API. 1056 */ 1057 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1058 goto process_auth_encryption_ccm_err; 1059 1060 if (srclen > 0) 1061 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1062 srclen, ctx)) 1063 goto process_auth_encryption_ccm_err; 1064 1065 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1066 goto process_auth_encryption_ccm_err; 1067 1068 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1069 goto process_auth_encryption_ccm_err; 1070 1071 return 0; 1072 1073 process_auth_encryption_ccm_err: 1074 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1075 return -EINVAL; 1076 } 1077 1078 /** Process AES-GCM decrypt algorithm */ 1079 static int 1080 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1081 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1082 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1083 { 1084 int len = 0, unused = 0; 1085 uint8_t empty[] = {}; 1086 1087 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1088 goto process_auth_decryption_gcm_err; 1089 1090 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1091 goto process_auth_decryption_gcm_err; 1092 1093 if (aadlen > 0) 1094 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1095 goto process_auth_decryption_gcm_err; 1096 1097 if (srclen > 0) 1098 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1099 srclen, ctx)) 1100 goto process_auth_decryption_gcm_err; 1101 1102 /* Workaround open ssl bug in version less then 1.0.1f */ 1103 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1104 goto process_auth_decryption_gcm_err; 1105 1106 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1107 return -EFAULT; 1108 1109 return 0; 1110 1111 process_auth_decryption_gcm_err: 1112 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1113 return -EINVAL; 1114 } 1115 1116 /** Process AES-CCM decrypt algorithm */ 1117 static int 1118 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1119 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1120 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1121 EVP_CIPHER_CTX *ctx) 1122 { 1123 int len = 0; 1124 1125 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1126 goto process_auth_decryption_ccm_err; 1127 1128 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1129 goto process_auth_decryption_ccm_err; 1130 1131 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1132 goto process_auth_decryption_ccm_err; 1133 1134 if (aadlen > 0) 1135 /* 1136 * For AES-CCM, the actual AAD is placed 1137 * 18 bytes after the start of the AAD field, 1138 * according to the API. 1139 */ 1140 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1141 goto process_auth_decryption_ccm_err; 1142 1143 if (srclen > 0) 1144 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1145 srclen, ctx)) 1146 return -EFAULT; 1147 1148 return 0; 1149 1150 process_auth_decryption_ccm_err: 1151 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1152 return -EINVAL; 1153 } 1154 1155 /** Process standard openssl auth algorithms */ 1156 static int 1157 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1158 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1159 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1160 { 1161 size_t dstlen; 1162 struct rte_mbuf *m; 1163 int l, n = srclen; 1164 uint8_t *src; 1165 1166 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1167 m = m->next) 1168 offset -= rte_pktmbuf_data_len(m); 1169 1170 if (m == 0) 1171 goto process_auth_err; 1172 1173 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1174 goto process_auth_err; 1175 1176 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1177 1178 l = rte_pktmbuf_data_len(m) - offset; 1179 if (srclen <= l) { 1180 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1181 goto process_auth_err; 1182 goto process_auth_final; 1183 } 1184 1185 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1186 goto process_auth_err; 1187 1188 n -= l; 1189 1190 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1191 src = rte_pktmbuf_mtod(m, uint8_t *); 1192 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1193 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1194 goto process_auth_err; 1195 n -= l; 1196 } 1197 1198 process_auth_final: 1199 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1200 goto process_auth_err; 1201 return 0; 1202 1203 process_auth_err: 1204 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1205 return -EINVAL; 1206 } 1207 1208 /** Process standard openssl auth algorithms with hmac */ 1209 static int 1210 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1211 int srclen, HMAC_CTX *ctx) 1212 { 1213 unsigned int dstlen; 1214 struct rte_mbuf *m; 1215 int l, n = srclen; 1216 uint8_t *src; 1217 1218 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1219 m = m->next) 1220 offset -= rte_pktmbuf_data_len(m); 1221 1222 if (m == 0) 1223 goto process_auth_err; 1224 1225 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1226 1227 l = rte_pktmbuf_data_len(m) - offset; 1228 if (srclen <= l) { 1229 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1230 goto process_auth_err; 1231 goto process_auth_final; 1232 } 1233 1234 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1235 goto process_auth_err; 1236 1237 n -= l; 1238 1239 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1240 src = rte_pktmbuf_mtod(m, uint8_t *); 1241 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1242 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1243 goto process_auth_err; 1244 n -= l; 1245 } 1246 1247 process_auth_final: 1248 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1249 goto process_auth_err; 1250 1251 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1252 goto process_auth_err; 1253 1254 return 0; 1255 1256 process_auth_err: 1257 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1258 return -EINVAL; 1259 } 1260 1261 /*----------------------------------------------------------------------------*/ 1262 1263 /** Process auth/cipher combined operation */ 1264 static void 1265 process_openssl_combined_op 1266 (struct rte_crypto_op *op, struct openssl_session *sess, 1267 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1268 { 1269 /* cipher */ 1270 uint8_t *dst = NULL, *iv, *tag, *aad; 1271 int srclen, aadlen, status = -1; 1272 uint32_t offset; 1273 uint8_t taglen; 1274 1275 /* 1276 * Segmented destination buffer is not supported for 1277 * encryption/decryption 1278 */ 1279 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1280 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1281 return; 1282 } 1283 1284 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1285 sess->iv.offset); 1286 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1287 srclen = 0; 1288 offset = op->sym->auth.data.offset; 1289 aadlen = op->sym->auth.data.length; 1290 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1291 op->sym->auth.data.offset); 1292 tag = op->sym->auth.digest.data; 1293 if (tag == NULL) 1294 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1295 offset + aadlen); 1296 } else { 1297 srclen = op->sym->aead.data.length; 1298 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1299 op->sym->aead.data.offset); 1300 offset = op->sym->aead.data.offset; 1301 aad = op->sym->aead.aad.data; 1302 aadlen = sess->auth.aad_length; 1303 tag = op->sym->aead.digest.data; 1304 if (tag == NULL) 1305 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1306 offset + srclen); 1307 } 1308 1309 taglen = sess->auth.digest_length; 1310 1311 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1312 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1313 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1314 status = process_openssl_auth_encryption_gcm( 1315 mbuf_src, offset, srclen, 1316 aad, aadlen, iv, 1317 dst, tag, sess->cipher.ctx); 1318 else 1319 status = process_openssl_auth_encryption_ccm( 1320 mbuf_src, offset, srclen, 1321 aad, aadlen, iv, 1322 dst, tag, taglen, sess->cipher.ctx); 1323 1324 } else { 1325 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1326 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1327 status = process_openssl_auth_decryption_gcm( 1328 mbuf_src, offset, srclen, 1329 aad, aadlen, iv, 1330 dst, tag, sess->cipher.ctx); 1331 else 1332 status = process_openssl_auth_decryption_ccm( 1333 mbuf_src, offset, srclen, 1334 aad, aadlen, iv, 1335 dst, tag, taglen, sess->cipher.ctx); 1336 } 1337 1338 if (status != 0) { 1339 if (status == (-EFAULT) && 1340 sess->auth.operation == 1341 RTE_CRYPTO_AUTH_OP_VERIFY) 1342 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1343 else 1344 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1345 } 1346 } 1347 1348 /** Process cipher operation */ 1349 static void 1350 process_openssl_cipher_op 1351 (struct rte_crypto_op *op, struct openssl_session *sess, 1352 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1353 { 1354 uint8_t *dst, *iv; 1355 int srclen, status; 1356 1357 /* 1358 * Segmented destination buffer is not supported for 1359 * encryption/decryption 1360 */ 1361 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1362 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1363 return; 1364 } 1365 1366 srclen = op->sym->cipher.data.length; 1367 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1368 op->sym->cipher.data.offset); 1369 1370 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1371 sess->iv.offset); 1372 1373 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1374 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1375 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1376 op->sym->cipher.data.offset, iv, 1377 srclen, sess->cipher.ctx); 1378 else 1379 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1380 op->sym->cipher.data.offset, iv, 1381 srclen, sess->cipher.ctx); 1382 else 1383 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1384 op->sym->cipher.data.offset, iv, 1385 sess->cipher.key.data, srclen, 1386 sess->cipher.ctx); 1387 1388 if (status != 0) 1389 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1390 } 1391 1392 /** Process cipher operation */ 1393 static void 1394 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1395 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1396 struct rte_mbuf *mbuf_dst) 1397 { 1398 uint8_t *src, *dst, *iv; 1399 uint8_t block_size, last_block_len; 1400 int srclen, status = 0; 1401 1402 srclen = op->sym->cipher.data.length; 1403 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1404 op->sym->cipher.data.offset); 1405 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1406 op->sym->cipher.data.offset); 1407 1408 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1409 sess->iv.offset); 1410 1411 block_size = DES_BLOCK_SIZE; 1412 1413 last_block_len = srclen % block_size; 1414 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1415 /* Encrypt only with ECB mode XOR IV */ 1416 if (srclen < block_size) { 1417 status = process_openssl_cipher_bpi_encrypt(src, dst, 1418 iv, srclen, 1419 sess->cipher.bpi_ctx); 1420 } else { 1421 srclen -= last_block_len; 1422 /* Encrypt with the block aligned stream with CBC mode */ 1423 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1424 op->sym->cipher.data.offset, iv, 1425 srclen, sess->cipher.ctx); 1426 if (last_block_len) { 1427 /* Point at last block */ 1428 dst += srclen; 1429 /* 1430 * IV is the last encrypted block from 1431 * the previous operation 1432 */ 1433 iv = dst - block_size; 1434 src += srclen; 1435 srclen = last_block_len; 1436 /* Encrypt the last frame with ECB mode */ 1437 status |= process_openssl_cipher_bpi_encrypt(src, 1438 dst, iv, 1439 srclen, sess->cipher.bpi_ctx); 1440 } 1441 } 1442 } else { 1443 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1444 if (srclen < block_size) { 1445 status = process_openssl_cipher_bpi_encrypt(src, dst, 1446 iv, 1447 srclen, 1448 sess->cipher.bpi_ctx); 1449 } else { 1450 if (last_block_len) { 1451 /* Point at last block */ 1452 dst += srclen - last_block_len; 1453 src += srclen - last_block_len; 1454 /* 1455 * IV is the last full block 1456 */ 1457 iv = src - block_size; 1458 /* 1459 * Decrypt the last frame with ECB mode 1460 * (encrypt, as it is the same operation) 1461 */ 1462 status = process_openssl_cipher_bpi_encrypt(src, 1463 dst, iv, 1464 last_block_len, sess->cipher.bpi_ctx); 1465 /* Prepare parameters for CBC mode op */ 1466 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1467 sess->iv.offset); 1468 dst += last_block_len - srclen; 1469 srclen -= last_block_len; 1470 } 1471 1472 /* Decrypt with CBC mode */ 1473 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1474 op->sym->cipher.data.offset, iv, 1475 srclen, sess->cipher.ctx); 1476 } 1477 } 1478 1479 if (status != 0) 1480 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1481 } 1482 1483 /** Process auth operation */ 1484 static void 1485 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1486 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1487 struct rte_mbuf *mbuf_dst) 1488 { 1489 uint8_t *dst; 1490 int srclen, status; 1491 1492 srclen = op->sym->auth.data.length; 1493 1494 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) 1495 dst = qp->temp_digest; 1496 else { 1497 dst = op->sym->auth.digest.data; 1498 if (dst == NULL) 1499 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1500 op->sym->auth.data.offset + 1501 op->sym->auth.data.length); 1502 } 1503 1504 switch (sess->auth.mode) { 1505 case OPENSSL_AUTH_AS_AUTH: 1506 status = process_openssl_auth(mbuf_src, dst, 1507 op->sym->auth.data.offset, NULL, NULL, srclen, 1508 sess->auth.auth.ctx, sess->auth.auth.evp_algo); 1509 break; 1510 case OPENSSL_AUTH_AS_HMAC: 1511 status = process_openssl_auth_hmac(mbuf_src, dst, 1512 op->sym->auth.data.offset, srclen, 1513 sess->auth.hmac.ctx); 1514 break; 1515 default: 1516 status = -1; 1517 break; 1518 } 1519 1520 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1521 if (memcmp(dst, op->sym->auth.digest.data, 1522 sess->auth.digest_length) != 0) { 1523 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1524 } 1525 } 1526 1527 if (status != 0) 1528 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1529 } 1530 1531 /** Process crypto operation for mbuf */ 1532 static int 1533 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1534 struct openssl_session *sess) 1535 { 1536 struct rte_mbuf *msrc, *mdst; 1537 int retval; 1538 1539 msrc = op->sym->m_src; 1540 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 1541 1542 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1543 1544 switch (sess->chain_order) { 1545 case OPENSSL_CHAIN_ONLY_CIPHER: 1546 process_openssl_cipher_op(op, sess, msrc, mdst); 1547 break; 1548 case OPENSSL_CHAIN_ONLY_AUTH: 1549 process_openssl_auth_op(qp, op, sess, msrc, mdst); 1550 break; 1551 case OPENSSL_CHAIN_CIPHER_AUTH: 1552 process_openssl_cipher_op(op, sess, msrc, mdst); 1553 process_openssl_auth_op(qp, op, sess, mdst, mdst); 1554 break; 1555 case OPENSSL_CHAIN_AUTH_CIPHER: 1556 process_openssl_auth_op(qp, op, sess, msrc, mdst); 1557 process_openssl_cipher_op(op, sess, msrc, mdst); 1558 break; 1559 case OPENSSL_CHAIN_COMBINED: 1560 process_openssl_combined_op(op, sess, msrc, mdst); 1561 break; 1562 case OPENSSL_CHAIN_CIPHER_BPI: 1563 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 1564 break; 1565 default: 1566 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1567 break; 1568 } 1569 1570 /* Free session if a session-less crypto op */ 1571 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 1572 openssl_reset_session(sess); 1573 memset(sess, 0, sizeof(struct openssl_session)); 1574 memset(op->sym->session, 0, 1575 rte_cryptodev_sym_get_header_session_size()); 1576 rte_mempool_put(qp->sess_mp, sess); 1577 rte_mempool_put(qp->sess_mp, op->sym->session); 1578 op->sym->session = NULL; 1579 } 1580 1581 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 1582 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1583 1584 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 1585 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 1586 else 1587 retval = -1; 1588 1589 return retval; 1590 } 1591 1592 /* 1593 *------------------------------------------------------------------------------ 1594 * PMD Framework 1595 *------------------------------------------------------------------------------ 1596 */ 1597 1598 /** Enqueue burst */ 1599 static uint16_t 1600 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 1601 uint16_t nb_ops) 1602 { 1603 struct openssl_session *sess; 1604 struct openssl_qp *qp = queue_pair; 1605 int i, retval; 1606 1607 for (i = 0; i < nb_ops; i++) { 1608 sess = get_session(qp, ops[i]); 1609 if (unlikely(sess == NULL)) 1610 goto enqueue_err; 1611 1612 retval = process_op(qp, ops[i], sess); 1613 if (unlikely(retval < 0)) 1614 goto enqueue_err; 1615 } 1616 1617 qp->stats.enqueued_count += i; 1618 return i; 1619 1620 enqueue_err: 1621 qp->stats.enqueue_err_count++; 1622 return i; 1623 } 1624 1625 /** Dequeue burst */ 1626 static uint16_t 1627 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 1628 uint16_t nb_ops) 1629 { 1630 struct openssl_qp *qp = queue_pair; 1631 1632 unsigned int nb_dequeued = 0; 1633 1634 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 1635 (void **)ops, nb_ops, NULL); 1636 qp->stats.dequeued_count += nb_dequeued; 1637 1638 return nb_dequeued; 1639 } 1640 1641 /** Create OPENSSL crypto device */ 1642 static int 1643 cryptodev_openssl_create(const char *name, 1644 struct rte_vdev_device *vdev, 1645 struct rte_cryptodev_pmd_init_params *init_params) 1646 { 1647 struct rte_cryptodev *dev; 1648 struct openssl_private *internals; 1649 1650 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 1651 if (dev == NULL) { 1652 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 1653 goto init_error; 1654 } 1655 1656 dev->driver_id = cryptodev_driver_id; 1657 dev->dev_ops = rte_openssl_pmd_ops; 1658 1659 /* register rx/tx burst functions for data path */ 1660 dev->dequeue_burst = openssl_pmd_dequeue_burst; 1661 dev->enqueue_burst = openssl_pmd_enqueue_burst; 1662 1663 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 1664 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 1665 RTE_CRYPTODEV_FF_CPU_AESNI | 1666 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 1667 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; 1668 1669 /* Set vector instructions mode supported */ 1670 internals = dev->data->dev_private; 1671 1672 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 1673 1674 return 0; 1675 1676 init_error: 1677 OPENSSL_LOG(ERR, "driver %s: create failed", 1678 init_params->name); 1679 1680 cryptodev_openssl_remove(vdev); 1681 return -EFAULT; 1682 } 1683 1684 /** Initialise OPENSSL crypto device */ 1685 static int 1686 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 1687 { 1688 struct rte_cryptodev_pmd_init_params init_params = { 1689 "", 1690 sizeof(struct openssl_private), 1691 rte_socket_id(), 1692 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 1693 }; 1694 const char *name; 1695 const char *input_args; 1696 1697 name = rte_vdev_device_name(vdev); 1698 if (name == NULL) 1699 return -EINVAL; 1700 input_args = rte_vdev_device_args(vdev); 1701 1702 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 1703 1704 return cryptodev_openssl_create(name, vdev, &init_params); 1705 } 1706 1707 /** Uninitialise OPENSSL crypto device */ 1708 static int 1709 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 1710 { 1711 struct rte_cryptodev *cryptodev; 1712 const char *name; 1713 1714 name = rte_vdev_device_name(vdev); 1715 if (name == NULL) 1716 return -EINVAL; 1717 1718 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 1719 if (cryptodev == NULL) 1720 return -ENODEV; 1721 1722 return rte_cryptodev_pmd_destroy(cryptodev); 1723 } 1724 1725 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 1726 .probe = cryptodev_openssl_probe, 1727 .remove = cryptodev_openssl_remove 1728 }; 1729 1730 static struct cryptodev_driver openssl_crypto_drv; 1731 1732 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 1733 cryptodev_openssl_pmd_drv); 1734 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 1735 "max_nb_queue_pairs=<int> " 1736 "socket_id=<int>"); 1737 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 1738 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 1739 1740 RTE_INIT(openssl_init_log) 1741 { 1742 openssl_logtype_driver = rte_log_register("pmd.crypto.openssl"); 1743 } 1744