1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Intel Corporation nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <rte_common.h> 34 #include <rte_hexdump.h> 35 #include <rte_cryptodev.h> 36 #include <rte_cryptodev_pmd.h> 37 #include <rte_vdev.h> 38 #include <rte_malloc.h> 39 #include <rte_cpuflags.h> 40 41 #include <openssl/hmac.h> 42 #include <openssl/evp.h> 43 44 #include "rte_openssl_pmd_private.h" 45 46 #define DES_BLOCK_SIZE 8 47 48 static uint8_t cryptodev_driver_id; 49 50 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 51 static HMAC_CTX *HMAC_CTX_new(void) 52 { 53 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 54 55 if (ctx != NULL) 56 HMAC_CTX_init(ctx); 57 return ctx; 58 } 59 60 static void HMAC_CTX_free(HMAC_CTX *ctx) 61 { 62 if (ctx != NULL) { 63 HMAC_CTX_cleanup(ctx); 64 OPENSSL_free(ctx); 65 } 66 } 67 #endif 68 69 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 70 71 /*----------------------------------------------------------------------------*/ 72 73 /** 74 * Increment counter by 1 75 * Counter is 64 bit array, big-endian 76 */ 77 static void 78 ctr_inc(uint8_t *ctr) 79 { 80 uint64_t *ctr64 = (uint64_t *)ctr; 81 82 *ctr64 = __builtin_bswap64(*ctr64); 83 (*ctr64)++; 84 *ctr64 = __builtin_bswap64(*ctr64); 85 } 86 87 /* 88 *------------------------------------------------------------------------------ 89 * Session Prepare 90 *------------------------------------------------------------------------------ 91 */ 92 93 /** Get xform chain order */ 94 static enum openssl_chain_order 95 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 96 { 97 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 98 99 if (xform != NULL) { 100 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 101 if (xform->next == NULL) 102 res = OPENSSL_CHAIN_ONLY_AUTH; 103 else if (xform->next->type == 104 RTE_CRYPTO_SYM_XFORM_CIPHER) 105 res = OPENSSL_CHAIN_AUTH_CIPHER; 106 } 107 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 108 if (xform->next == NULL) 109 res = OPENSSL_CHAIN_ONLY_CIPHER; 110 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 111 res = OPENSSL_CHAIN_CIPHER_AUTH; 112 } 113 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 114 res = OPENSSL_CHAIN_COMBINED; 115 } 116 117 return res; 118 } 119 120 /** Get session cipher key from input cipher key */ 121 static void 122 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key) 123 { 124 memcpy(session_key, input_key, keylen); 125 } 126 127 /** Get key ede 24 bytes standard from input key */ 128 static int 129 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede) 130 { 131 int res = 0; 132 133 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 134 switch (keylen) { 135 case 24: 136 memcpy(key_ede, key, 24); 137 break; 138 case 16: 139 /* K3 = K1 */ 140 memcpy(key_ede, key, 16); 141 memcpy(key_ede + 16, key, 8); 142 break; 143 case 8: 144 /* K1 = K2 = K3 (DES compatibility) */ 145 memcpy(key_ede, key, 8); 146 memcpy(key_ede + 8, key, 8); 147 memcpy(key_ede + 16, key, 8); 148 break; 149 default: 150 OPENSSL_LOG_ERR("Unsupported key size"); 151 res = -EINVAL; 152 } 153 154 return res; 155 } 156 157 /** Get adequate openssl function for input cipher algorithm */ 158 static uint8_t 159 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 160 const EVP_CIPHER **algo) 161 { 162 int res = 0; 163 164 if (algo != NULL) { 165 switch (sess_algo) { 166 case RTE_CRYPTO_CIPHER_3DES_CBC: 167 switch (keylen) { 168 case 16: 169 *algo = EVP_des_ede_cbc(); 170 break; 171 case 24: 172 *algo = EVP_des_ede3_cbc(); 173 break; 174 default: 175 res = -EINVAL; 176 } 177 break; 178 case RTE_CRYPTO_CIPHER_3DES_CTR: 179 break; 180 case RTE_CRYPTO_CIPHER_AES_CBC: 181 switch (keylen) { 182 case 16: 183 *algo = EVP_aes_128_cbc(); 184 break; 185 case 24: 186 *algo = EVP_aes_192_cbc(); 187 break; 188 case 32: 189 *algo = EVP_aes_256_cbc(); 190 break; 191 default: 192 res = -EINVAL; 193 } 194 break; 195 case RTE_CRYPTO_CIPHER_AES_CTR: 196 switch (keylen) { 197 case 16: 198 *algo = EVP_aes_128_ctr(); 199 break; 200 case 24: 201 *algo = EVP_aes_192_ctr(); 202 break; 203 case 32: 204 *algo = EVP_aes_256_ctr(); 205 break; 206 default: 207 res = -EINVAL; 208 } 209 break; 210 default: 211 res = -EINVAL; 212 break; 213 } 214 } else { 215 res = -EINVAL; 216 } 217 218 return res; 219 } 220 221 /** Get adequate openssl function for input auth algorithm */ 222 static uint8_t 223 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 224 const EVP_MD **algo) 225 { 226 int res = 0; 227 228 if (algo != NULL) { 229 switch (sessalgo) { 230 case RTE_CRYPTO_AUTH_MD5: 231 case RTE_CRYPTO_AUTH_MD5_HMAC: 232 *algo = EVP_md5(); 233 break; 234 case RTE_CRYPTO_AUTH_SHA1: 235 case RTE_CRYPTO_AUTH_SHA1_HMAC: 236 *algo = EVP_sha1(); 237 break; 238 case RTE_CRYPTO_AUTH_SHA224: 239 case RTE_CRYPTO_AUTH_SHA224_HMAC: 240 *algo = EVP_sha224(); 241 break; 242 case RTE_CRYPTO_AUTH_SHA256: 243 case RTE_CRYPTO_AUTH_SHA256_HMAC: 244 *algo = EVP_sha256(); 245 break; 246 case RTE_CRYPTO_AUTH_SHA384: 247 case RTE_CRYPTO_AUTH_SHA384_HMAC: 248 *algo = EVP_sha384(); 249 break; 250 case RTE_CRYPTO_AUTH_SHA512: 251 case RTE_CRYPTO_AUTH_SHA512_HMAC: 252 *algo = EVP_sha512(); 253 break; 254 default: 255 res = -EINVAL; 256 break; 257 } 258 } else { 259 res = -EINVAL; 260 } 261 262 return res; 263 } 264 265 /** Get adequate openssl function for input cipher algorithm */ 266 static uint8_t 267 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 268 const EVP_CIPHER **algo) 269 { 270 int res = 0; 271 272 if (algo != NULL) { 273 switch (sess_algo) { 274 case RTE_CRYPTO_AEAD_AES_GCM: 275 switch (keylen) { 276 case 16: 277 *algo = EVP_aes_128_gcm(); 278 break; 279 case 24: 280 *algo = EVP_aes_192_gcm(); 281 break; 282 case 32: 283 *algo = EVP_aes_256_gcm(); 284 break; 285 default: 286 res = -EINVAL; 287 } 288 break; 289 case RTE_CRYPTO_AEAD_AES_CCM: 290 switch (keylen) { 291 case 16: 292 *algo = EVP_aes_128_ccm(); 293 break; 294 case 24: 295 *algo = EVP_aes_192_ccm(); 296 break; 297 case 32: 298 *algo = EVP_aes_256_ccm(); 299 break; 300 default: 301 res = -EINVAL; 302 } 303 break; 304 default: 305 res = -EINVAL; 306 break; 307 } 308 } else { 309 res = -EINVAL; 310 } 311 312 return res; 313 } 314 315 /* Set session AEAD encryption parameters */ 316 static int 317 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 318 enum rte_crypto_aead_algorithm algo, 319 uint8_t tag_len, uint8_t *key) 320 { 321 int iv_type = 0; 322 unsigned int do_ccm; 323 324 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 325 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 326 327 /* Select AEAD algo */ 328 switch (algo) { 329 case RTE_CRYPTO_AEAD_AES_GCM: 330 iv_type = EVP_CTRL_GCM_SET_IVLEN; 331 if (tag_len != 16) 332 return -EINVAL; 333 do_ccm = 0; 334 break; 335 case RTE_CRYPTO_AEAD_AES_CCM: 336 iv_type = EVP_CTRL_CCM_SET_IVLEN; 337 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 338 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 339 return -EINVAL; 340 do_ccm = 1; 341 break; 342 default: 343 return -ENOTSUP; 344 } 345 346 sess->cipher.mode = OPENSSL_CIPHER_LIB; 347 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 348 349 if (get_aead_algo(algo, sess->cipher.key.length, 350 &sess->cipher.evp_algo) != 0) 351 return -EINVAL; 352 353 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 354 355 sess->chain_order = OPENSSL_CHAIN_COMBINED; 356 357 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 358 NULL, NULL, NULL) <= 0) 359 return -EINVAL; 360 361 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, 362 NULL) <= 0) 363 return -EINVAL; 364 365 if (do_ccm) 366 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 367 tag_len, NULL); 368 369 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 370 return -EINVAL; 371 372 return 0; 373 } 374 375 /* Set session AEAD decryption parameters */ 376 static int 377 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 378 enum rte_crypto_aead_algorithm algo, 379 uint8_t tag_len, uint8_t *key) 380 { 381 int iv_type = 0; 382 unsigned int do_ccm = 0; 383 384 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 385 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 386 387 /* Select AEAD algo */ 388 switch (algo) { 389 case RTE_CRYPTO_AEAD_AES_GCM: 390 iv_type = EVP_CTRL_GCM_SET_IVLEN; 391 if (tag_len != 16) 392 return -EINVAL; 393 break; 394 case RTE_CRYPTO_AEAD_AES_CCM: 395 iv_type = EVP_CTRL_CCM_SET_IVLEN; 396 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 397 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 398 return -EINVAL; 399 do_ccm = 1; 400 break; 401 default: 402 return -ENOTSUP; 403 } 404 405 sess->cipher.mode = OPENSSL_CIPHER_LIB; 406 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 407 408 if (get_aead_algo(algo, sess->cipher.key.length, 409 &sess->cipher.evp_algo) != 0) 410 return -EINVAL; 411 412 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 413 414 sess->chain_order = OPENSSL_CHAIN_COMBINED; 415 416 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 417 NULL, NULL, NULL) <= 0) 418 return -EINVAL; 419 420 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, 421 sess->iv.length, NULL) <= 0) 422 return -EINVAL; 423 424 if (do_ccm) 425 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 426 tag_len, NULL); 427 428 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 429 return -EINVAL; 430 431 return 0; 432 } 433 434 /** Set session cipher parameters */ 435 static int 436 openssl_set_session_cipher_parameters(struct openssl_session *sess, 437 const struct rte_crypto_sym_xform *xform) 438 { 439 /* Select cipher direction */ 440 sess->cipher.direction = xform->cipher.op; 441 /* Select cipher key */ 442 sess->cipher.key.length = xform->cipher.key.length; 443 444 /* Set IV parameters */ 445 sess->iv.offset = xform->cipher.iv.offset; 446 sess->iv.length = xform->cipher.iv.length; 447 448 /* Select cipher algo */ 449 switch (xform->cipher.algo) { 450 case RTE_CRYPTO_CIPHER_3DES_CBC: 451 case RTE_CRYPTO_CIPHER_AES_CBC: 452 case RTE_CRYPTO_CIPHER_AES_CTR: 453 sess->cipher.mode = OPENSSL_CIPHER_LIB; 454 sess->cipher.algo = xform->cipher.algo; 455 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 456 457 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 458 &sess->cipher.evp_algo) != 0) 459 return -EINVAL; 460 461 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 462 sess->cipher.key.data); 463 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 464 if (EVP_EncryptInit_ex(sess->cipher.ctx, 465 sess->cipher.evp_algo, 466 NULL, xform->cipher.key.data, 467 NULL) != 1) { 468 return -EINVAL; 469 } 470 } else if (sess->cipher.direction == 471 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 472 if (EVP_DecryptInit_ex(sess->cipher.ctx, 473 sess->cipher.evp_algo, 474 NULL, xform->cipher.key.data, 475 NULL) != 1) { 476 return -EINVAL; 477 } 478 } 479 480 break; 481 482 case RTE_CRYPTO_CIPHER_3DES_CTR: 483 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 484 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 485 486 if (get_cipher_key_ede(xform->cipher.key.data, 487 sess->cipher.key.length, 488 sess->cipher.key.data) != 0) 489 return -EINVAL; 490 break; 491 492 case RTE_CRYPTO_CIPHER_DES_CBC: 493 sess->cipher.algo = xform->cipher.algo; 494 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 495 sess->cipher.evp_algo = EVP_des_cbc(); 496 497 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 498 sess->cipher.key.data); 499 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 500 if (EVP_EncryptInit_ex(sess->cipher.ctx, 501 sess->cipher.evp_algo, 502 NULL, xform->cipher.key.data, 503 NULL) != 1) { 504 return -EINVAL; 505 } 506 } else if (sess->cipher.direction == 507 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 508 if (EVP_DecryptInit_ex(sess->cipher.ctx, 509 sess->cipher.evp_algo, 510 NULL, xform->cipher.key.data, 511 NULL) != 1) { 512 return -EINVAL; 513 } 514 } 515 516 break; 517 518 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 519 sess->cipher.algo = xform->cipher.algo; 520 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 521 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 522 sess->cipher.evp_algo = EVP_des_cbc(); 523 524 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 525 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 526 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 527 NULL, xform->cipher.key.data, 0) != 1) 528 return -EINVAL; 529 530 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 531 sess->cipher.key.data); 532 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 533 if (EVP_EncryptInit_ex(sess->cipher.ctx, 534 sess->cipher.evp_algo, 535 NULL, xform->cipher.key.data, 536 NULL) != 1) { 537 return -EINVAL; 538 } 539 } else if (sess->cipher.direction == 540 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 541 if (EVP_DecryptInit_ex(sess->cipher.ctx, 542 sess->cipher.evp_algo, 543 NULL, xform->cipher.key.data, 544 NULL) != 1) { 545 return -EINVAL; 546 } 547 } 548 549 break; 550 default: 551 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 552 return -ENOTSUP; 553 } 554 555 return 0; 556 } 557 558 /* Set session auth parameters */ 559 static int 560 openssl_set_session_auth_parameters(struct openssl_session *sess, 561 const struct rte_crypto_sym_xform *xform) 562 { 563 /* Select auth generate/verify */ 564 sess->auth.operation = xform->auth.op; 565 sess->auth.algo = xform->auth.algo; 566 567 sess->auth.digest_length = xform->auth.digest_length; 568 569 /* Select auth algo */ 570 switch (xform->auth.algo) { 571 case RTE_CRYPTO_AUTH_AES_GMAC: 572 /* 573 * OpenSSL requires GMAC to be a GCM operation 574 * with no cipher data length 575 */ 576 sess->cipher.key.length = xform->auth.key.length; 577 578 /* Set IV parameters */ 579 sess->iv.offset = xform->auth.iv.offset; 580 sess->iv.length = xform->auth.iv.length; 581 582 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 583 return openssl_set_sess_aead_enc_param(sess, 584 RTE_CRYPTO_AEAD_AES_GCM, 585 xform->auth.digest_length, 586 xform->auth.key.data); 587 else 588 return openssl_set_sess_aead_dec_param(sess, 589 RTE_CRYPTO_AEAD_AES_GCM, 590 xform->auth.digest_length, 591 xform->auth.key.data); 592 break; 593 594 case RTE_CRYPTO_AUTH_MD5: 595 case RTE_CRYPTO_AUTH_SHA1: 596 case RTE_CRYPTO_AUTH_SHA224: 597 case RTE_CRYPTO_AUTH_SHA256: 598 case RTE_CRYPTO_AUTH_SHA384: 599 case RTE_CRYPTO_AUTH_SHA512: 600 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 601 if (get_auth_algo(xform->auth.algo, 602 &sess->auth.auth.evp_algo) != 0) 603 return -EINVAL; 604 sess->auth.auth.ctx = EVP_MD_CTX_create(); 605 break; 606 607 case RTE_CRYPTO_AUTH_MD5_HMAC: 608 case RTE_CRYPTO_AUTH_SHA1_HMAC: 609 case RTE_CRYPTO_AUTH_SHA224_HMAC: 610 case RTE_CRYPTO_AUTH_SHA256_HMAC: 611 case RTE_CRYPTO_AUTH_SHA384_HMAC: 612 case RTE_CRYPTO_AUTH_SHA512_HMAC: 613 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 614 sess->auth.hmac.ctx = HMAC_CTX_new(); 615 if (get_auth_algo(xform->auth.algo, 616 &sess->auth.hmac.evp_algo) != 0) 617 return -EINVAL; 618 619 if (HMAC_Init_ex(sess->auth.hmac.ctx, 620 xform->auth.key.data, 621 xform->auth.key.length, 622 sess->auth.hmac.evp_algo, NULL) != 1) 623 return -EINVAL; 624 break; 625 626 default: 627 return -ENOTSUP; 628 } 629 630 return 0; 631 } 632 633 /* Set session AEAD parameters */ 634 static int 635 openssl_set_session_aead_parameters(struct openssl_session *sess, 636 const struct rte_crypto_sym_xform *xform) 637 { 638 /* Select cipher key */ 639 sess->cipher.key.length = xform->aead.key.length; 640 641 /* Set IV parameters */ 642 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 643 /* 644 * For AES-CCM, the actual IV is placed 645 * one byte after the start of the IV field, 646 * according to the API. 647 */ 648 sess->iv.offset = xform->aead.iv.offset + 1; 649 else 650 sess->iv.offset = xform->aead.iv.offset; 651 652 sess->iv.length = xform->aead.iv.length; 653 654 sess->auth.aad_length = xform->aead.aad_length; 655 sess->auth.digest_length = xform->aead.digest_length; 656 657 sess->aead_algo = xform->aead.algo; 658 /* Select cipher direction */ 659 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 660 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 661 xform->aead.digest_length, xform->aead.key.data); 662 else 663 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 664 xform->aead.digest_length, xform->aead.key.data); 665 } 666 667 /** Parse crypto xform chain and set private session parameters */ 668 int 669 openssl_set_session_parameters(struct openssl_session *sess, 670 const struct rte_crypto_sym_xform *xform) 671 { 672 const struct rte_crypto_sym_xform *cipher_xform = NULL; 673 const struct rte_crypto_sym_xform *auth_xform = NULL; 674 const struct rte_crypto_sym_xform *aead_xform = NULL; 675 int ret; 676 677 sess->chain_order = openssl_get_chain_order(xform); 678 switch (sess->chain_order) { 679 case OPENSSL_CHAIN_ONLY_CIPHER: 680 cipher_xform = xform; 681 break; 682 case OPENSSL_CHAIN_ONLY_AUTH: 683 auth_xform = xform; 684 break; 685 case OPENSSL_CHAIN_CIPHER_AUTH: 686 cipher_xform = xform; 687 auth_xform = xform->next; 688 break; 689 case OPENSSL_CHAIN_AUTH_CIPHER: 690 auth_xform = xform; 691 cipher_xform = xform->next; 692 break; 693 case OPENSSL_CHAIN_COMBINED: 694 aead_xform = xform; 695 break; 696 default: 697 return -EINVAL; 698 } 699 700 /* Default IV length = 0 */ 701 sess->iv.length = 0; 702 703 /* cipher_xform must be check before auth_xform */ 704 if (cipher_xform) { 705 ret = openssl_set_session_cipher_parameters( 706 sess, cipher_xform); 707 if (ret != 0) { 708 OPENSSL_LOG_ERR( 709 "Invalid/unsupported cipher parameters"); 710 return ret; 711 } 712 } 713 714 if (auth_xform) { 715 ret = openssl_set_session_auth_parameters(sess, auth_xform); 716 if (ret != 0) { 717 OPENSSL_LOG_ERR( 718 "Invalid/unsupported auth parameters"); 719 return ret; 720 } 721 } 722 723 if (aead_xform) { 724 ret = openssl_set_session_aead_parameters(sess, aead_xform); 725 if (ret != 0) { 726 OPENSSL_LOG_ERR( 727 "Invalid/unsupported AEAD parameters"); 728 return ret; 729 } 730 } 731 732 return 0; 733 } 734 735 /** Reset private session parameters */ 736 void 737 openssl_reset_session(struct openssl_session *sess) 738 { 739 EVP_CIPHER_CTX_free(sess->cipher.ctx); 740 741 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 742 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 743 744 switch (sess->auth.mode) { 745 case OPENSSL_AUTH_AS_AUTH: 746 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 747 break; 748 case OPENSSL_AUTH_AS_HMAC: 749 EVP_PKEY_free(sess->auth.hmac.pkey); 750 HMAC_CTX_free(sess->auth.hmac.ctx); 751 break; 752 default: 753 break; 754 } 755 } 756 757 /** Provide session for operation */ 758 static struct openssl_session * 759 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 760 { 761 struct openssl_session *sess = NULL; 762 763 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 764 /* get existing session */ 765 if (likely(op->sym->session != NULL)) 766 sess = (struct openssl_session *) 767 get_session_private_data( 768 op->sym->session, 769 cryptodev_driver_id); 770 } else { 771 /* provide internal session */ 772 void *_sess = NULL; 773 void *_sess_private_data = NULL; 774 775 if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) 776 return NULL; 777 778 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) 779 return NULL; 780 781 sess = (struct openssl_session *)_sess_private_data; 782 783 if (unlikely(openssl_set_session_parameters(sess, 784 op->sym->xform) != 0)) { 785 rte_mempool_put(qp->sess_mp, _sess); 786 rte_mempool_put(qp->sess_mp, _sess_private_data); 787 sess = NULL; 788 } 789 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 790 set_session_private_data(op->sym->session, cryptodev_driver_id, 791 _sess_private_data); 792 } 793 794 if (sess == NULL) 795 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 796 797 return sess; 798 } 799 800 /* 801 *------------------------------------------------------------------------------ 802 * Process Operations 803 *------------------------------------------------------------------------------ 804 */ 805 static inline int 806 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 807 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 808 { 809 struct rte_mbuf *m; 810 int dstlen; 811 int l, n = srclen; 812 uint8_t *src; 813 814 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 815 m = m->next) 816 offset -= rte_pktmbuf_data_len(m); 817 818 if (m == 0) 819 return -1; 820 821 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 822 823 l = rte_pktmbuf_data_len(m) - offset; 824 if (srclen <= l) { 825 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 826 return -1; 827 *dst += l; 828 return 0; 829 } 830 831 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 832 return -1; 833 834 *dst += dstlen; 835 n -= l; 836 837 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 838 src = rte_pktmbuf_mtod(m, uint8_t *); 839 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 840 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 841 return -1; 842 *dst += dstlen; 843 n -= l; 844 } 845 846 return 0; 847 } 848 849 static inline int 850 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 851 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 852 { 853 struct rte_mbuf *m; 854 int dstlen; 855 int l, n = srclen; 856 uint8_t *src; 857 858 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 859 m = m->next) 860 offset -= rte_pktmbuf_data_len(m); 861 862 if (m == 0) 863 return -1; 864 865 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 866 867 l = rte_pktmbuf_data_len(m) - offset; 868 if (srclen <= l) { 869 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 870 return -1; 871 *dst += l; 872 return 0; 873 } 874 875 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 876 return -1; 877 878 *dst += dstlen; 879 n -= l; 880 881 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 882 src = rte_pktmbuf_mtod(m, uint8_t *); 883 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 884 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 885 return -1; 886 *dst += dstlen; 887 n -= l; 888 } 889 890 return 0; 891 } 892 893 /** Process standard openssl cipher encryption */ 894 static int 895 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 896 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) 897 { 898 int totlen; 899 900 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 901 goto process_cipher_encrypt_err; 902 903 EVP_CIPHER_CTX_set_padding(ctx, 0); 904 905 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 906 srclen, ctx)) 907 goto process_cipher_encrypt_err; 908 909 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 910 goto process_cipher_encrypt_err; 911 912 return 0; 913 914 process_cipher_encrypt_err: 915 OPENSSL_LOG_ERR("Process openssl cipher encrypt failed"); 916 return -EINVAL; 917 } 918 919 /** Process standard openssl cipher encryption */ 920 static int 921 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 922 uint8_t *iv, int srclen, 923 EVP_CIPHER_CTX *ctx) 924 { 925 uint8_t i; 926 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 927 int encrypted_ivlen; 928 929 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 930 iv, DES_BLOCK_SIZE) <= 0) 931 goto process_cipher_encrypt_err; 932 933 for (i = 0; i < srclen; i++) 934 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 935 936 return 0; 937 938 process_cipher_encrypt_err: 939 OPENSSL_LOG_ERR("Process openssl cipher bpi encrypt failed"); 940 return -EINVAL; 941 } 942 /** Process standard openssl cipher decryption */ 943 static int 944 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 945 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) 946 { 947 int totlen; 948 949 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 950 goto process_cipher_decrypt_err; 951 952 EVP_CIPHER_CTX_set_padding(ctx, 0); 953 954 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 955 srclen, ctx)) 956 goto process_cipher_decrypt_err; 957 958 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 959 goto process_cipher_decrypt_err; 960 return 0; 961 962 process_cipher_decrypt_err: 963 OPENSSL_LOG_ERR("Process openssl cipher decrypt failed"); 964 return -EINVAL; 965 } 966 967 /** Process cipher des 3 ctr encryption, decryption algorithm */ 968 static int 969 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 970 int offset, uint8_t *iv, uint8_t *key, int srclen, 971 EVP_CIPHER_CTX *ctx) 972 { 973 uint8_t ebuf[8], ctr[8]; 974 int unused, n; 975 struct rte_mbuf *m; 976 uint8_t *src; 977 int l; 978 979 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 980 m = m->next) 981 offset -= rte_pktmbuf_data_len(m); 982 983 if (m == 0) 984 goto process_cipher_des3ctr_err; 985 986 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 987 l = rte_pktmbuf_data_len(m) - offset; 988 989 /* We use 3DES encryption also for decryption. 990 * IV is not important for 3DES ecb 991 */ 992 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 993 goto process_cipher_des3ctr_err; 994 995 memcpy(ctr, iv, 8); 996 997 for (n = 0; n < srclen; n++) { 998 if (n % 8 == 0) { 999 if (EVP_EncryptUpdate(ctx, 1000 (unsigned char *)&ebuf, &unused, 1001 (const unsigned char *)&ctr, 8) <= 0) 1002 goto process_cipher_des3ctr_err; 1003 ctr_inc(ctr); 1004 } 1005 dst[n] = *(src++) ^ ebuf[n % 8]; 1006 1007 l--; 1008 if (!l) { 1009 m = m->next; 1010 if (m) { 1011 src = rte_pktmbuf_mtod(m, uint8_t *); 1012 l = rte_pktmbuf_data_len(m); 1013 } 1014 } 1015 } 1016 1017 return 0; 1018 1019 process_cipher_des3ctr_err: 1020 OPENSSL_LOG_ERR("Process openssl cipher des 3 ede ctr failed"); 1021 return -EINVAL; 1022 } 1023 1024 /** Process AES-GCM encrypt algorithm */ 1025 static int 1026 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1027 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1028 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1029 { 1030 int len = 0, unused = 0; 1031 uint8_t empty[] = {}; 1032 1033 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1034 goto process_auth_encryption_gcm_err; 1035 1036 if (aadlen > 0) 1037 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1038 goto process_auth_encryption_gcm_err; 1039 1040 if (srclen > 0) 1041 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1042 srclen, ctx)) 1043 goto process_auth_encryption_gcm_err; 1044 1045 /* Workaround open ssl bug in version less then 1.0.1f */ 1046 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1047 goto process_auth_encryption_gcm_err; 1048 1049 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1050 goto process_auth_encryption_gcm_err; 1051 1052 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1053 goto process_auth_encryption_gcm_err; 1054 1055 return 0; 1056 1057 process_auth_encryption_gcm_err: 1058 OPENSSL_LOG_ERR("Process openssl auth encryption gcm failed"); 1059 return -EINVAL; 1060 } 1061 1062 /** Process AES-CCM encrypt algorithm */ 1063 static int 1064 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1065 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1066 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1067 { 1068 int len = 0; 1069 1070 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1071 goto process_auth_encryption_ccm_err; 1072 1073 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1074 goto process_auth_encryption_ccm_err; 1075 1076 if (aadlen > 0) 1077 /* 1078 * For AES-CCM, the actual AAD is placed 1079 * 18 bytes after the start of the AAD field, 1080 * according to the API. 1081 */ 1082 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1083 goto process_auth_encryption_ccm_err; 1084 1085 if (srclen > 0) 1086 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1087 srclen, ctx)) 1088 goto process_auth_encryption_ccm_err; 1089 1090 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1091 goto process_auth_encryption_ccm_err; 1092 1093 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1094 goto process_auth_encryption_ccm_err; 1095 1096 return 0; 1097 1098 process_auth_encryption_ccm_err: 1099 OPENSSL_LOG_ERR("Process openssl auth encryption ccm failed"); 1100 return -EINVAL; 1101 } 1102 1103 /** Process AES-GCM decrypt algorithm */ 1104 static int 1105 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1106 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1107 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1108 { 1109 int len = 0, unused = 0; 1110 uint8_t empty[] = {}; 1111 1112 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1113 goto process_auth_decryption_gcm_err; 1114 1115 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1116 goto process_auth_decryption_gcm_err; 1117 1118 if (aadlen > 0) 1119 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1120 goto process_auth_decryption_gcm_err; 1121 1122 if (srclen > 0) 1123 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1124 srclen, ctx)) 1125 goto process_auth_decryption_gcm_err; 1126 1127 /* Workaround open ssl bug in version less then 1.0.1f */ 1128 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1129 goto process_auth_decryption_gcm_err; 1130 1131 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1132 return -EFAULT; 1133 1134 return 0; 1135 1136 process_auth_decryption_gcm_err: 1137 OPENSSL_LOG_ERR("Process openssl auth decryption gcm failed"); 1138 return -EINVAL; 1139 } 1140 1141 /** Process AES-CCM decrypt algorithm */ 1142 static int 1143 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1144 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1145 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1146 EVP_CIPHER_CTX *ctx) 1147 { 1148 int len = 0; 1149 1150 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1151 goto process_auth_decryption_ccm_err; 1152 1153 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1154 goto process_auth_decryption_ccm_err; 1155 1156 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1157 goto process_auth_decryption_ccm_err; 1158 1159 if (aadlen > 0) 1160 /* 1161 * For AES-CCM, the actual AAD is placed 1162 * 18 bytes after the start of the AAD field, 1163 * according to the API. 1164 */ 1165 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1166 goto process_auth_decryption_ccm_err; 1167 1168 if (srclen > 0) 1169 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1170 srclen, ctx)) 1171 return -EFAULT; 1172 1173 return 0; 1174 1175 process_auth_decryption_ccm_err: 1176 OPENSSL_LOG_ERR("Process openssl auth decryption ccm failed"); 1177 return -EINVAL; 1178 } 1179 1180 /** Process standard openssl auth algorithms */ 1181 static int 1182 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1183 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1184 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1185 { 1186 size_t dstlen; 1187 struct rte_mbuf *m; 1188 int l, n = srclen; 1189 uint8_t *src; 1190 1191 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1192 m = m->next) 1193 offset -= rte_pktmbuf_data_len(m); 1194 1195 if (m == 0) 1196 goto process_auth_err; 1197 1198 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1199 goto process_auth_err; 1200 1201 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1202 1203 l = rte_pktmbuf_data_len(m) - offset; 1204 if (srclen <= l) { 1205 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1206 goto process_auth_err; 1207 goto process_auth_final; 1208 } 1209 1210 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1211 goto process_auth_err; 1212 1213 n -= l; 1214 1215 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1216 src = rte_pktmbuf_mtod(m, uint8_t *); 1217 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1218 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1219 goto process_auth_err; 1220 n -= l; 1221 } 1222 1223 process_auth_final: 1224 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1225 goto process_auth_err; 1226 return 0; 1227 1228 process_auth_err: 1229 OPENSSL_LOG_ERR("Process openssl auth failed"); 1230 return -EINVAL; 1231 } 1232 1233 /** Process standard openssl auth algorithms with hmac */ 1234 static int 1235 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1236 int srclen, HMAC_CTX *ctx) 1237 { 1238 unsigned int dstlen; 1239 struct rte_mbuf *m; 1240 int l, n = srclen; 1241 uint8_t *src; 1242 1243 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1244 m = m->next) 1245 offset -= rte_pktmbuf_data_len(m); 1246 1247 if (m == 0) 1248 goto process_auth_err; 1249 1250 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1251 1252 l = rte_pktmbuf_data_len(m) - offset; 1253 if (srclen <= l) { 1254 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1255 goto process_auth_err; 1256 goto process_auth_final; 1257 } 1258 1259 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1260 goto process_auth_err; 1261 1262 n -= l; 1263 1264 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1265 src = rte_pktmbuf_mtod(m, uint8_t *); 1266 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1267 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1268 goto process_auth_err; 1269 n -= l; 1270 } 1271 1272 process_auth_final: 1273 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1274 goto process_auth_err; 1275 1276 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1277 goto process_auth_err; 1278 1279 return 0; 1280 1281 process_auth_err: 1282 OPENSSL_LOG_ERR("Process openssl auth failed"); 1283 return -EINVAL; 1284 } 1285 1286 /*----------------------------------------------------------------------------*/ 1287 1288 /** Process auth/cipher combined operation */ 1289 static void 1290 process_openssl_combined_op 1291 (struct rte_crypto_op *op, struct openssl_session *sess, 1292 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1293 { 1294 /* cipher */ 1295 uint8_t *dst = NULL, *iv, *tag, *aad; 1296 int srclen, aadlen, status = -1; 1297 uint32_t offset; 1298 uint8_t taglen; 1299 1300 /* 1301 * Segmented destination buffer is not supported for 1302 * encryption/decryption 1303 */ 1304 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1305 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1306 return; 1307 } 1308 1309 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1310 sess->iv.offset); 1311 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1312 srclen = 0; 1313 offset = op->sym->auth.data.offset; 1314 aadlen = op->sym->auth.data.length; 1315 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1316 op->sym->auth.data.offset); 1317 tag = op->sym->auth.digest.data; 1318 if (tag == NULL) 1319 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1320 offset + aadlen); 1321 } else { 1322 srclen = op->sym->aead.data.length; 1323 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1324 op->sym->aead.data.offset); 1325 offset = op->sym->aead.data.offset; 1326 aad = op->sym->aead.aad.data; 1327 aadlen = sess->auth.aad_length; 1328 tag = op->sym->aead.digest.data; 1329 if (tag == NULL) 1330 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1331 offset + srclen); 1332 } 1333 1334 taglen = sess->auth.digest_length; 1335 1336 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1337 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1338 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1339 status = process_openssl_auth_encryption_gcm( 1340 mbuf_src, offset, srclen, 1341 aad, aadlen, iv, 1342 dst, tag, sess->cipher.ctx); 1343 else 1344 status = process_openssl_auth_encryption_ccm( 1345 mbuf_src, offset, srclen, 1346 aad, aadlen, iv, 1347 dst, tag, taglen, sess->cipher.ctx); 1348 1349 } else { 1350 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1351 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1352 status = process_openssl_auth_decryption_gcm( 1353 mbuf_src, offset, srclen, 1354 aad, aadlen, iv, 1355 dst, tag, sess->cipher.ctx); 1356 else 1357 status = process_openssl_auth_decryption_ccm( 1358 mbuf_src, offset, srclen, 1359 aad, aadlen, iv, 1360 dst, tag, taglen, sess->cipher.ctx); 1361 } 1362 1363 if (status != 0) { 1364 if (status == (-EFAULT) && 1365 sess->auth.operation == 1366 RTE_CRYPTO_AUTH_OP_VERIFY) 1367 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1368 else 1369 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1370 } 1371 } 1372 1373 /** Process cipher operation */ 1374 static void 1375 process_openssl_cipher_op 1376 (struct rte_crypto_op *op, struct openssl_session *sess, 1377 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1378 { 1379 uint8_t *dst, *iv; 1380 int srclen, status; 1381 1382 /* 1383 * Segmented destination buffer is not supported for 1384 * encryption/decryption 1385 */ 1386 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1387 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1388 return; 1389 } 1390 1391 srclen = op->sym->cipher.data.length; 1392 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1393 op->sym->cipher.data.offset); 1394 1395 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1396 sess->iv.offset); 1397 1398 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1399 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1400 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1401 op->sym->cipher.data.offset, iv, 1402 srclen, sess->cipher.ctx); 1403 else 1404 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1405 op->sym->cipher.data.offset, iv, 1406 srclen, sess->cipher.ctx); 1407 else 1408 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1409 op->sym->cipher.data.offset, iv, 1410 sess->cipher.key.data, srclen, 1411 sess->cipher.ctx); 1412 1413 if (status != 0) 1414 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1415 } 1416 1417 /** Process cipher operation */ 1418 static void 1419 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1420 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1421 struct rte_mbuf *mbuf_dst) 1422 { 1423 uint8_t *src, *dst, *iv; 1424 uint8_t block_size, last_block_len; 1425 int srclen, status = 0; 1426 1427 srclen = op->sym->cipher.data.length; 1428 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1429 op->sym->cipher.data.offset); 1430 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1431 op->sym->cipher.data.offset); 1432 1433 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1434 sess->iv.offset); 1435 1436 block_size = DES_BLOCK_SIZE; 1437 1438 last_block_len = srclen % block_size; 1439 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1440 /* Encrypt only with ECB mode XOR IV */ 1441 if (srclen < block_size) { 1442 status = process_openssl_cipher_bpi_encrypt(src, dst, 1443 iv, srclen, 1444 sess->cipher.bpi_ctx); 1445 } else { 1446 srclen -= last_block_len; 1447 /* Encrypt with the block aligned stream with CBC mode */ 1448 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1449 op->sym->cipher.data.offset, iv, 1450 srclen, sess->cipher.ctx); 1451 if (last_block_len) { 1452 /* Point at last block */ 1453 dst += srclen; 1454 /* 1455 * IV is the last encrypted block from 1456 * the previous operation 1457 */ 1458 iv = dst - block_size; 1459 src += srclen; 1460 srclen = last_block_len; 1461 /* Encrypt the last frame with ECB mode */ 1462 status |= process_openssl_cipher_bpi_encrypt(src, 1463 dst, iv, 1464 srclen, sess->cipher.bpi_ctx); 1465 } 1466 } 1467 } else { 1468 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1469 if (srclen < block_size) { 1470 status = process_openssl_cipher_bpi_encrypt(src, dst, 1471 iv, 1472 srclen, 1473 sess->cipher.bpi_ctx); 1474 } else { 1475 if (last_block_len) { 1476 /* Point at last block */ 1477 dst += srclen - last_block_len; 1478 src += srclen - last_block_len; 1479 /* 1480 * IV is the last full block 1481 */ 1482 iv = src - block_size; 1483 /* 1484 * Decrypt the last frame with ECB mode 1485 * (encrypt, as it is the same operation) 1486 */ 1487 status = process_openssl_cipher_bpi_encrypt(src, 1488 dst, iv, 1489 last_block_len, sess->cipher.bpi_ctx); 1490 /* Prepare parameters for CBC mode op */ 1491 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1492 sess->iv.offset); 1493 dst += last_block_len - srclen; 1494 srclen -= last_block_len; 1495 } 1496 1497 /* Decrypt with CBC mode */ 1498 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1499 op->sym->cipher.data.offset, iv, 1500 srclen, sess->cipher.ctx); 1501 } 1502 } 1503 1504 if (status != 0) 1505 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1506 } 1507 1508 /** Process auth operation */ 1509 static void 1510 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1511 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1512 struct rte_mbuf *mbuf_dst) 1513 { 1514 uint8_t *dst; 1515 int srclen, status; 1516 1517 srclen = op->sym->auth.data.length; 1518 1519 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) 1520 dst = qp->temp_digest; 1521 else { 1522 dst = op->sym->auth.digest.data; 1523 if (dst == NULL) 1524 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1525 op->sym->auth.data.offset + 1526 op->sym->auth.data.length); 1527 } 1528 1529 switch (sess->auth.mode) { 1530 case OPENSSL_AUTH_AS_AUTH: 1531 status = process_openssl_auth(mbuf_src, dst, 1532 op->sym->auth.data.offset, NULL, NULL, srclen, 1533 sess->auth.auth.ctx, sess->auth.auth.evp_algo); 1534 break; 1535 case OPENSSL_AUTH_AS_HMAC: 1536 status = process_openssl_auth_hmac(mbuf_src, dst, 1537 op->sym->auth.data.offset, srclen, 1538 sess->auth.hmac.ctx); 1539 break; 1540 default: 1541 status = -1; 1542 break; 1543 } 1544 1545 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1546 if (memcmp(dst, op->sym->auth.digest.data, 1547 sess->auth.digest_length) != 0) { 1548 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1549 } 1550 } 1551 1552 if (status != 0) 1553 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1554 } 1555 1556 /** Process crypto operation for mbuf */ 1557 static int 1558 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1559 struct openssl_session *sess) 1560 { 1561 struct rte_mbuf *msrc, *mdst; 1562 int retval; 1563 1564 msrc = op->sym->m_src; 1565 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 1566 1567 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1568 1569 switch (sess->chain_order) { 1570 case OPENSSL_CHAIN_ONLY_CIPHER: 1571 process_openssl_cipher_op(op, sess, msrc, mdst); 1572 break; 1573 case OPENSSL_CHAIN_ONLY_AUTH: 1574 process_openssl_auth_op(qp, op, sess, msrc, mdst); 1575 break; 1576 case OPENSSL_CHAIN_CIPHER_AUTH: 1577 process_openssl_cipher_op(op, sess, msrc, mdst); 1578 process_openssl_auth_op(qp, op, sess, mdst, mdst); 1579 break; 1580 case OPENSSL_CHAIN_AUTH_CIPHER: 1581 process_openssl_auth_op(qp, op, sess, msrc, mdst); 1582 process_openssl_cipher_op(op, sess, msrc, mdst); 1583 break; 1584 case OPENSSL_CHAIN_COMBINED: 1585 process_openssl_combined_op(op, sess, msrc, mdst); 1586 break; 1587 case OPENSSL_CHAIN_CIPHER_BPI: 1588 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 1589 break; 1590 default: 1591 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1592 break; 1593 } 1594 1595 /* Free session if a session-less crypto op */ 1596 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 1597 openssl_reset_session(sess); 1598 memset(sess, 0, sizeof(struct openssl_session)); 1599 memset(op->sym->session, 0, 1600 rte_cryptodev_get_header_session_size()); 1601 rte_mempool_put(qp->sess_mp, sess); 1602 rte_mempool_put(qp->sess_mp, op->sym->session); 1603 op->sym->session = NULL; 1604 } 1605 1606 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 1607 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1608 1609 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 1610 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 1611 else 1612 retval = -1; 1613 1614 return retval; 1615 } 1616 1617 /* 1618 *------------------------------------------------------------------------------ 1619 * PMD Framework 1620 *------------------------------------------------------------------------------ 1621 */ 1622 1623 /** Enqueue burst */ 1624 static uint16_t 1625 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 1626 uint16_t nb_ops) 1627 { 1628 struct openssl_session *sess; 1629 struct openssl_qp *qp = queue_pair; 1630 int i, retval; 1631 1632 for (i = 0; i < nb_ops; i++) { 1633 sess = get_session(qp, ops[i]); 1634 if (unlikely(sess == NULL)) 1635 goto enqueue_err; 1636 1637 retval = process_op(qp, ops[i], sess); 1638 if (unlikely(retval < 0)) 1639 goto enqueue_err; 1640 } 1641 1642 qp->stats.enqueued_count += i; 1643 return i; 1644 1645 enqueue_err: 1646 qp->stats.enqueue_err_count++; 1647 return i; 1648 } 1649 1650 /** Dequeue burst */ 1651 static uint16_t 1652 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 1653 uint16_t nb_ops) 1654 { 1655 struct openssl_qp *qp = queue_pair; 1656 1657 unsigned int nb_dequeued = 0; 1658 1659 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 1660 (void **)ops, nb_ops, NULL); 1661 qp->stats.dequeued_count += nb_dequeued; 1662 1663 return nb_dequeued; 1664 } 1665 1666 /** Create OPENSSL crypto device */ 1667 static int 1668 cryptodev_openssl_create(const char *name, 1669 struct rte_vdev_device *vdev, 1670 struct rte_cryptodev_pmd_init_params *init_params) 1671 { 1672 struct rte_cryptodev *dev; 1673 struct openssl_private *internals; 1674 1675 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 1676 if (dev == NULL) { 1677 OPENSSL_LOG_ERR("failed to create cryptodev vdev"); 1678 goto init_error; 1679 } 1680 1681 dev->driver_id = cryptodev_driver_id; 1682 dev->dev_ops = rte_openssl_pmd_ops; 1683 1684 /* register rx/tx burst functions for data path */ 1685 dev->dequeue_burst = openssl_pmd_dequeue_burst; 1686 dev->enqueue_burst = openssl_pmd_enqueue_burst; 1687 1688 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 1689 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 1690 RTE_CRYPTODEV_FF_CPU_AESNI | 1691 RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER; 1692 1693 /* Set vector instructions mode supported */ 1694 internals = dev->data->dev_private; 1695 1696 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 1697 internals->max_nb_sessions = init_params->max_nb_sessions; 1698 1699 return 0; 1700 1701 init_error: 1702 OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed", 1703 init_params->name); 1704 1705 cryptodev_openssl_remove(vdev); 1706 return -EFAULT; 1707 } 1708 1709 /** Initialise OPENSSL crypto device */ 1710 static int 1711 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 1712 { 1713 struct rte_cryptodev_pmd_init_params init_params = { 1714 "", 1715 sizeof(struct openssl_private), 1716 rte_socket_id(), 1717 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, 1718 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS 1719 }; 1720 const char *name; 1721 const char *input_args; 1722 1723 name = rte_vdev_device_name(vdev); 1724 if (name == NULL) 1725 return -EINVAL; 1726 input_args = rte_vdev_device_args(vdev); 1727 1728 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 1729 1730 return cryptodev_openssl_create(name, vdev, &init_params); 1731 } 1732 1733 /** Uninitialise OPENSSL crypto device */ 1734 static int 1735 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 1736 { 1737 struct rte_cryptodev *cryptodev; 1738 const char *name; 1739 1740 name = rte_vdev_device_name(vdev); 1741 if (name == NULL) 1742 return -EINVAL; 1743 1744 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 1745 if (cryptodev == NULL) 1746 return -ENODEV; 1747 1748 return rte_cryptodev_pmd_destroy(cryptodev); 1749 } 1750 1751 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 1752 .probe = cryptodev_openssl_probe, 1753 .remove = cryptodev_openssl_remove 1754 }; 1755 1756 static struct cryptodev_driver openssl_crypto_drv; 1757 1758 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 1759 cryptodev_openssl_pmd_drv); 1760 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 1761 "max_nb_queue_pairs=<int> " 1762 "max_nb_sessions=<int> " 1763 "socket_id=<int>"); 1764 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, cryptodev_openssl_pmd_drv, 1765 cryptodev_driver_id); 1766