1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2016-2017 Intel Corporation 3 */ 4 5 #include <rte_common.h> 6 #include <rte_hexdump.h> 7 #include <rte_cryptodev.h> 8 #include <rte_cryptodev_pmd.h> 9 #include <rte_bus_vdev.h> 10 #include <rte_malloc.h> 11 #include <rte_cpuflags.h> 12 13 #include <openssl/hmac.h> 14 #include <openssl/evp.h> 15 16 #include "rte_openssl_pmd_private.h" 17 18 #define DES_BLOCK_SIZE 8 19 20 static uint8_t cryptodev_driver_id; 21 22 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) 23 static HMAC_CTX *HMAC_CTX_new(void) 24 { 25 HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx)); 26 27 if (ctx != NULL) 28 HMAC_CTX_init(ctx); 29 return ctx; 30 } 31 32 static void HMAC_CTX_free(HMAC_CTX *ctx) 33 { 34 if (ctx != NULL) { 35 HMAC_CTX_cleanup(ctx); 36 OPENSSL_free(ctx); 37 } 38 } 39 #endif 40 41 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 42 43 /*----------------------------------------------------------------------------*/ 44 45 /** 46 * Increment counter by 1 47 * Counter is 64 bit array, big-endian 48 */ 49 static void 50 ctr_inc(uint8_t *ctr) 51 { 52 uint64_t *ctr64 = (uint64_t *)ctr; 53 54 *ctr64 = __builtin_bswap64(*ctr64); 55 (*ctr64)++; 56 *ctr64 = __builtin_bswap64(*ctr64); 57 } 58 59 /* 60 *------------------------------------------------------------------------------ 61 * Session Prepare 62 *------------------------------------------------------------------------------ 63 */ 64 65 /** Get xform chain order */ 66 static enum openssl_chain_order 67 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 68 { 69 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 70 71 if (xform != NULL) { 72 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 73 if (xform->next == NULL) 74 res = OPENSSL_CHAIN_ONLY_AUTH; 75 else if (xform->next->type == 76 RTE_CRYPTO_SYM_XFORM_CIPHER) 77 res = OPENSSL_CHAIN_AUTH_CIPHER; 78 } 79 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 80 if (xform->next == NULL) 81 res = OPENSSL_CHAIN_ONLY_CIPHER; 82 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 83 res = OPENSSL_CHAIN_CIPHER_AUTH; 84 } 85 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 86 res = OPENSSL_CHAIN_COMBINED; 87 } 88 89 return res; 90 } 91 92 /** Get session cipher key from input cipher key */ 93 static void 94 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key) 95 { 96 memcpy(session_key, input_key, keylen); 97 } 98 99 /** Get key ede 24 bytes standard from input key */ 100 static int 101 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede) 102 { 103 int res = 0; 104 105 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 106 switch (keylen) { 107 case 24: 108 memcpy(key_ede, key, 24); 109 break; 110 case 16: 111 /* K3 = K1 */ 112 memcpy(key_ede, key, 16); 113 memcpy(key_ede + 16, key, 8); 114 break; 115 case 8: 116 /* K1 = K2 = K3 (DES compatibility) */ 117 memcpy(key_ede, key, 8); 118 memcpy(key_ede + 8, key, 8); 119 memcpy(key_ede + 16, key, 8); 120 break; 121 default: 122 OPENSSL_LOG(ERR, "Unsupported key size"); 123 res = -EINVAL; 124 } 125 126 return res; 127 } 128 129 /** Get adequate openssl function for input cipher algorithm */ 130 static uint8_t 131 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 132 const EVP_CIPHER **algo) 133 { 134 int res = 0; 135 136 if (algo != NULL) { 137 switch (sess_algo) { 138 case RTE_CRYPTO_CIPHER_3DES_CBC: 139 switch (keylen) { 140 case 16: 141 *algo = EVP_des_ede_cbc(); 142 break; 143 case 24: 144 *algo = EVP_des_ede3_cbc(); 145 break; 146 default: 147 res = -EINVAL; 148 } 149 break; 150 case RTE_CRYPTO_CIPHER_3DES_CTR: 151 break; 152 case RTE_CRYPTO_CIPHER_AES_CBC: 153 switch (keylen) { 154 case 16: 155 *algo = EVP_aes_128_cbc(); 156 break; 157 case 24: 158 *algo = EVP_aes_192_cbc(); 159 break; 160 case 32: 161 *algo = EVP_aes_256_cbc(); 162 break; 163 default: 164 res = -EINVAL; 165 } 166 break; 167 case RTE_CRYPTO_CIPHER_AES_CTR: 168 switch (keylen) { 169 case 16: 170 *algo = EVP_aes_128_ctr(); 171 break; 172 case 24: 173 *algo = EVP_aes_192_ctr(); 174 break; 175 case 32: 176 *algo = EVP_aes_256_ctr(); 177 break; 178 default: 179 res = -EINVAL; 180 } 181 break; 182 default: 183 res = -EINVAL; 184 break; 185 } 186 } else { 187 res = -EINVAL; 188 } 189 190 return res; 191 } 192 193 /** Get adequate openssl function for input auth algorithm */ 194 static uint8_t 195 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 196 const EVP_MD **algo) 197 { 198 int res = 0; 199 200 if (algo != NULL) { 201 switch (sessalgo) { 202 case RTE_CRYPTO_AUTH_MD5: 203 case RTE_CRYPTO_AUTH_MD5_HMAC: 204 *algo = EVP_md5(); 205 break; 206 case RTE_CRYPTO_AUTH_SHA1: 207 case RTE_CRYPTO_AUTH_SHA1_HMAC: 208 *algo = EVP_sha1(); 209 break; 210 case RTE_CRYPTO_AUTH_SHA224: 211 case RTE_CRYPTO_AUTH_SHA224_HMAC: 212 *algo = EVP_sha224(); 213 break; 214 case RTE_CRYPTO_AUTH_SHA256: 215 case RTE_CRYPTO_AUTH_SHA256_HMAC: 216 *algo = EVP_sha256(); 217 break; 218 case RTE_CRYPTO_AUTH_SHA384: 219 case RTE_CRYPTO_AUTH_SHA384_HMAC: 220 *algo = EVP_sha384(); 221 break; 222 case RTE_CRYPTO_AUTH_SHA512: 223 case RTE_CRYPTO_AUTH_SHA512_HMAC: 224 *algo = EVP_sha512(); 225 break; 226 default: 227 res = -EINVAL; 228 break; 229 } 230 } else { 231 res = -EINVAL; 232 } 233 234 return res; 235 } 236 237 /** Get adequate openssl function for input cipher algorithm */ 238 static uint8_t 239 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 240 const EVP_CIPHER **algo) 241 { 242 int res = 0; 243 244 if (algo != NULL) { 245 switch (sess_algo) { 246 case RTE_CRYPTO_AEAD_AES_GCM: 247 switch (keylen) { 248 case 16: 249 *algo = EVP_aes_128_gcm(); 250 break; 251 case 24: 252 *algo = EVP_aes_192_gcm(); 253 break; 254 case 32: 255 *algo = EVP_aes_256_gcm(); 256 break; 257 default: 258 res = -EINVAL; 259 } 260 break; 261 case RTE_CRYPTO_AEAD_AES_CCM: 262 switch (keylen) { 263 case 16: 264 *algo = EVP_aes_128_ccm(); 265 break; 266 case 24: 267 *algo = EVP_aes_192_ccm(); 268 break; 269 case 32: 270 *algo = EVP_aes_256_ccm(); 271 break; 272 default: 273 res = -EINVAL; 274 } 275 break; 276 default: 277 res = -EINVAL; 278 break; 279 } 280 } else { 281 res = -EINVAL; 282 } 283 284 return res; 285 } 286 287 /* Set session AEAD encryption parameters */ 288 static int 289 openssl_set_sess_aead_enc_param(struct openssl_session *sess, 290 enum rte_crypto_aead_algorithm algo, 291 uint8_t tag_len, uint8_t *key) 292 { 293 int iv_type = 0; 294 unsigned int do_ccm; 295 296 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 297 sess->auth.operation = RTE_CRYPTO_AUTH_OP_GENERATE; 298 299 /* Select AEAD algo */ 300 switch (algo) { 301 case RTE_CRYPTO_AEAD_AES_GCM: 302 iv_type = EVP_CTRL_GCM_SET_IVLEN; 303 if (tag_len != 16) 304 return -EINVAL; 305 do_ccm = 0; 306 break; 307 case RTE_CRYPTO_AEAD_AES_CCM: 308 iv_type = EVP_CTRL_CCM_SET_IVLEN; 309 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 310 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 311 return -EINVAL; 312 do_ccm = 1; 313 break; 314 default: 315 return -ENOTSUP; 316 } 317 318 sess->cipher.mode = OPENSSL_CIPHER_LIB; 319 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 320 321 if (get_aead_algo(algo, sess->cipher.key.length, 322 &sess->cipher.evp_algo) != 0) 323 return -EINVAL; 324 325 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 326 327 sess->chain_order = OPENSSL_CHAIN_COMBINED; 328 329 if (EVP_EncryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 330 NULL, NULL, NULL) <= 0) 331 return -EINVAL; 332 333 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, sess->iv.length, 334 NULL) <= 0) 335 return -EINVAL; 336 337 if (do_ccm) 338 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 339 tag_len, NULL); 340 341 if (EVP_EncryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 342 return -EINVAL; 343 344 return 0; 345 } 346 347 /* Set session AEAD decryption parameters */ 348 static int 349 openssl_set_sess_aead_dec_param(struct openssl_session *sess, 350 enum rte_crypto_aead_algorithm algo, 351 uint8_t tag_len, uint8_t *key) 352 { 353 int iv_type = 0; 354 unsigned int do_ccm = 0; 355 356 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 357 sess->auth.operation = RTE_CRYPTO_AUTH_OP_VERIFY; 358 359 /* Select AEAD algo */ 360 switch (algo) { 361 case RTE_CRYPTO_AEAD_AES_GCM: 362 iv_type = EVP_CTRL_GCM_SET_IVLEN; 363 if (tag_len != 16) 364 return -EINVAL; 365 break; 366 case RTE_CRYPTO_AEAD_AES_CCM: 367 iv_type = EVP_CTRL_CCM_SET_IVLEN; 368 /* Digest size can be 4, 6, 8, 10, 12, 14 or 16 bytes */ 369 if (tag_len < 4 || tag_len > 16 || (tag_len & 1) == 1) 370 return -EINVAL; 371 do_ccm = 1; 372 break; 373 default: 374 return -ENOTSUP; 375 } 376 377 sess->cipher.mode = OPENSSL_CIPHER_LIB; 378 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 379 380 if (get_aead_algo(algo, sess->cipher.key.length, 381 &sess->cipher.evp_algo) != 0) 382 return -EINVAL; 383 384 get_cipher_key(key, sess->cipher.key.length, sess->cipher.key.data); 385 386 sess->chain_order = OPENSSL_CHAIN_COMBINED; 387 388 if (EVP_DecryptInit_ex(sess->cipher.ctx, sess->cipher.evp_algo, 389 NULL, NULL, NULL) <= 0) 390 return -EINVAL; 391 392 if (EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, iv_type, 393 sess->iv.length, NULL) <= 0) 394 return -EINVAL; 395 396 if (do_ccm) 397 EVP_CIPHER_CTX_ctrl(sess->cipher.ctx, EVP_CTRL_CCM_SET_TAG, 398 tag_len, NULL); 399 400 if (EVP_DecryptInit_ex(sess->cipher.ctx, NULL, NULL, key, NULL) <= 0) 401 return -EINVAL; 402 403 return 0; 404 } 405 406 /** Set session cipher parameters */ 407 static int 408 openssl_set_session_cipher_parameters(struct openssl_session *sess, 409 const struct rte_crypto_sym_xform *xform) 410 { 411 /* Select cipher direction */ 412 sess->cipher.direction = xform->cipher.op; 413 /* Select cipher key */ 414 sess->cipher.key.length = xform->cipher.key.length; 415 416 /* Set IV parameters */ 417 sess->iv.offset = xform->cipher.iv.offset; 418 sess->iv.length = xform->cipher.iv.length; 419 420 /* Select cipher algo */ 421 switch (xform->cipher.algo) { 422 case RTE_CRYPTO_CIPHER_3DES_CBC: 423 case RTE_CRYPTO_CIPHER_AES_CBC: 424 case RTE_CRYPTO_CIPHER_AES_CTR: 425 sess->cipher.mode = OPENSSL_CIPHER_LIB; 426 sess->cipher.algo = xform->cipher.algo; 427 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 428 429 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 430 &sess->cipher.evp_algo) != 0) 431 return -EINVAL; 432 433 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 434 sess->cipher.key.data); 435 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 436 if (EVP_EncryptInit_ex(sess->cipher.ctx, 437 sess->cipher.evp_algo, 438 NULL, xform->cipher.key.data, 439 NULL) != 1) { 440 return -EINVAL; 441 } 442 } else if (sess->cipher.direction == 443 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 444 if (EVP_DecryptInit_ex(sess->cipher.ctx, 445 sess->cipher.evp_algo, 446 NULL, xform->cipher.key.data, 447 NULL) != 1) { 448 return -EINVAL; 449 } 450 } 451 452 break; 453 454 case RTE_CRYPTO_CIPHER_3DES_CTR: 455 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 456 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 457 458 if (get_cipher_key_ede(xform->cipher.key.data, 459 sess->cipher.key.length, 460 sess->cipher.key.data) != 0) 461 return -EINVAL; 462 break; 463 464 case RTE_CRYPTO_CIPHER_DES_CBC: 465 sess->cipher.algo = xform->cipher.algo; 466 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 467 sess->cipher.evp_algo = EVP_des_cbc(); 468 469 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 470 sess->cipher.key.data); 471 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 472 if (EVP_EncryptInit_ex(sess->cipher.ctx, 473 sess->cipher.evp_algo, 474 NULL, xform->cipher.key.data, 475 NULL) != 1) { 476 return -EINVAL; 477 } 478 } else if (sess->cipher.direction == 479 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 480 if (EVP_DecryptInit_ex(sess->cipher.ctx, 481 sess->cipher.evp_algo, 482 NULL, xform->cipher.key.data, 483 NULL) != 1) { 484 return -EINVAL; 485 } 486 } 487 488 break; 489 490 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 491 sess->cipher.algo = xform->cipher.algo; 492 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 493 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 494 sess->cipher.evp_algo = EVP_des_cbc(); 495 496 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 497 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 498 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 499 NULL, xform->cipher.key.data, 0) != 1) 500 return -EINVAL; 501 502 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 503 sess->cipher.key.data); 504 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 505 if (EVP_EncryptInit_ex(sess->cipher.ctx, 506 sess->cipher.evp_algo, 507 NULL, xform->cipher.key.data, 508 NULL) != 1) { 509 return -EINVAL; 510 } 511 } else if (sess->cipher.direction == 512 RTE_CRYPTO_CIPHER_OP_DECRYPT) { 513 if (EVP_DecryptInit_ex(sess->cipher.ctx, 514 sess->cipher.evp_algo, 515 NULL, xform->cipher.key.data, 516 NULL) != 1) { 517 return -EINVAL; 518 } 519 } 520 521 break; 522 default: 523 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 524 return -ENOTSUP; 525 } 526 527 return 0; 528 } 529 530 /* Set session auth parameters */ 531 static int 532 openssl_set_session_auth_parameters(struct openssl_session *sess, 533 const struct rte_crypto_sym_xform *xform) 534 { 535 /* Select auth generate/verify */ 536 sess->auth.operation = xform->auth.op; 537 sess->auth.algo = xform->auth.algo; 538 539 sess->auth.digest_length = xform->auth.digest_length; 540 541 /* Select auth algo */ 542 switch (xform->auth.algo) { 543 case RTE_CRYPTO_AUTH_AES_GMAC: 544 /* 545 * OpenSSL requires GMAC to be a GCM operation 546 * with no cipher data length 547 */ 548 sess->cipher.key.length = xform->auth.key.length; 549 550 /* Set IV parameters */ 551 sess->iv.offset = xform->auth.iv.offset; 552 sess->iv.length = xform->auth.iv.length; 553 554 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 555 return openssl_set_sess_aead_enc_param(sess, 556 RTE_CRYPTO_AEAD_AES_GCM, 557 xform->auth.digest_length, 558 xform->auth.key.data); 559 else 560 return openssl_set_sess_aead_dec_param(sess, 561 RTE_CRYPTO_AEAD_AES_GCM, 562 xform->auth.digest_length, 563 xform->auth.key.data); 564 break; 565 566 case RTE_CRYPTO_AUTH_MD5: 567 case RTE_CRYPTO_AUTH_SHA1: 568 case RTE_CRYPTO_AUTH_SHA224: 569 case RTE_CRYPTO_AUTH_SHA256: 570 case RTE_CRYPTO_AUTH_SHA384: 571 case RTE_CRYPTO_AUTH_SHA512: 572 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 573 if (get_auth_algo(xform->auth.algo, 574 &sess->auth.auth.evp_algo) != 0) 575 return -EINVAL; 576 sess->auth.auth.ctx = EVP_MD_CTX_create(); 577 break; 578 579 case RTE_CRYPTO_AUTH_MD5_HMAC: 580 case RTE_CRYPTO_AUTH_SHA1_HMAC: 581 case RTE_CRYPTO_AUTH_SHA224_HMAC: 582 case RTE_CRYPTO_AUTH_SHA256_HMAC: 583 case RTE_CRYPTO_AUTH_SHA384_HMAC: 584 case RTE_CRYPTO_AUTH_SHA512_HMAC: 585 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 586 sess->auth.hmac.ctx = HMAC_CTX_new(); 587 if (get_auth_algo(xform->auth.algo, 588 &sess->auth.hmac.evp_algo) != 0) 589 return -EINVAL; 590 591 if (HMAC_Init_ex(sess->auth.hmac.ctx, 592 xform->auth.key.data, 593 xform->auth.key.length, 594 sess->auth.hmac.evp_algo, NULL) != 1) 595 return -EINVAL; 596 break; 597 598 default: 599 return -ENOTSUP; 600 } 601 602 return 0; 603 } 604 605 /* Set session AEAD parameters */ 606 static int 607 openssl_set_session_aead_parameters(struct openssl_session *sess, 608 const struct rte_crypto_sym_xform *xform) 609 { 610 /* Select cipher key */ 611 sess->cipher.key.length = xform->aead.key.length; 612 613 /* Set IV parameters */ 614 if (xform->aead.algo == RTE_CRYPTO_AEAD_AES_CCM) 615 /* 616 * For AES-CCM, the actual IV is placed 617 * one byte after the start of the IV field, 618 * according to the API. 619 */ 620 sess->iv.offset = xform->aead.iv.offset + 1; 621 else 622 sess->iv.offset = xform->aead.iv.offset; 623 624 sess->iv.length = xform->aead.iv.length; 625 626 sess->auth.aad_length = xform->aead.aad_length; 627 sess->auth.digest_length = xform->aead.digest_length; 628 629 sess->aead_algo = xform->aead.algo; 630 /* Select cipher direction */ 631 if (xform->aead.op == RTE_CRYPTO_AEAD_OP_ENCRYPT) 632 return openssl_set_sess_aead_enc_param(sess, xform->aead.algo, 633 xform->aead.digest_length, xform->aead.key.data); 634 else 635 return openssl_set_sess_aead_dec_param(sess, xform->aead.algo, 636 xform->aead.digest_length, xform->aead.key.data); 637 } 638 639 /** Parse crypto xform chain and set private session parameters */ 640 int 641 openssl_set_session_parameters(struct openssl_session *sess, 642 const struct rte_crypto_sym_xform *xform) 643 { 644 const struct rte_crypto_sym_xform *cipher_xform = NULL; 645 const struct rte_crypto_sym_xform *auth_xform = NULL; 646 const struct rte_crypto_sym_xform *aead_xform = NULL; 647 int ret; 648 649 sess->chain_order = openssl_get_chain_order(xform); 650 switch (sess->chain_order) { 651 case OPENSSL_CHAIN_ONLY_CIPHER: 652 cipher_xform = xform; 653 break; 654 case OPENSSL_CHAIN_ONLY_AUTH: 655 auth_xform = xform; 656 break; 657 case OPENSSL_CHAIN_CIPHER_AUTH: 658 cipher_xform = xform; 659 auth_xform = xform->next; 660 break; 661 case OPENSSL_CHAIN_AUTH_CIPHER: 662 auth_xform = xform; 663 cipher_xform = xform->next; 664 break; 665 case OPENSSL_CHAIN_COMBINED: 666 aead_xform = xform; 667 break; 668 default: 669 return -EINVAL; 670 } 671 672 /* Default IV length = 0 */ 673 sess->iv.length = 0; 674 675 /* cipher_xform must be check before auth_xform */ 676 if (cipher_xform) { 677 ret = openssl_set_session_cipher_parameters( 678 sess, cipher_xform); 679 if (ret != 0) { 680 OPENSSL_LOG(ERR, 681 "Invalid/unsupported cipher parameters"); 682 return ret; 683 } 684 } 685 686 if (auth_xform) { 687 ret = openssl_set_session_auth_parameters(sess, auth_xform); 688 if (ret != 0) { 689 OPENSSL_LOG(ERR, 690 "Invalid/unsupported auth parameters"); 691 return ret; 692 } 693 } 694 695 if (aead_xform) { 696 ret = openssl_set_session_aead_parameters(sess, aead_xform); 697 if (ret != 0) { 698 OPENSSL_LOG(ERR, 699 "Invalid/unsupported AEAD parameters"); 700 return ret; 701 } 702 } 703 704 return 0; 705 } 706 707 /** Reset private session parameters */ 708 void 709 openssl_reset_session(struct openssl_session *sess) 710 { 711 EVP_CIPHER_CTX_free(sess->cipher.ctx); 712 713 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 714 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 715 716 switch (sess->auth.mode) { 717 case OPENSSL_AUTH_AS_AUTH: 718 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 719 break; 720 case OPENSSL_AUTH_AS_HMAC: 721 EVP_PKEY_free(sess->auth.hmac.pkey); 722 HMAC_CTX_free(sess->auth.hmac.ctx); 723 break; 724 default: 725 break; 726 } 727 } 728 729 /** Provide session for operation */ 730 static struct openssl_session * 731 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 732 { 733 struct openssl_session *sess = NULL; 734 735 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 736 /* get existing session */ 737 if (likely(op->sym->session != NULL)) 738 sess = (struct openssl_session *) 739 get_sym_session_private_data( 740 op->sym->session, 741 cryptodev_driver_id); 742 } else { 743 /* provide internal session */ 744 void *_sess = NULL; 745 void *_sess_private_data = NULL; 746 747 if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) 748 return NULL; 749 750 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) 751 return NULL; 752 753 sess = (struct openssl_session *)_sess_private_data; 754 755 if (unlikely(openssl_set_session_parameters(sess, 756 op->sym->xform) != 0)) { 757 rte_mempool_put(qp->sess_mp, _sess); 758 rte_mempool_put(qp->sess_mp, _sess_private_data); 759 sess = NULL; 760 } 761 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 762 set_sym_session_private_data(op->sym->session, 763 cryptodev_driver_id, _sess_private_data); 764 } 765 766 if (sess == NULL) 767 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 768 769 return sess; 770 } 771 772 /* 773 *------------------------------------------------------------------------------ 774 * Process Operations 775 *------------------------------------------------------------------------------ 776 */ 777 static inline int 778 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 779 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 780 { 781 struct rte_mbuf *m; 782 int dstlen; 783 int l, n = srclen; 784 uint8_t *src; 785 786 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 787 m = m->next) 788 offset -= rte_pktmbuf_data_len(m); 789 790 if (m == 0) 791 return -1; 792 793 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 794 795 l = rte_pktmbuf_data_len(m) - offset; 796 if (srclen <= l) { 797 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 798 return -1; 799 *dst += l; 800 return 0; 801 } 802 803 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 804 return -1; 805 806 *dst += dstlen; 807 n -= l; 808 809 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 810 src = rte_pktmbuf_mtod(m, uint8_t *); 811 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 812 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 813 return -1; 814 *dst += dstlen; 815 n -= l; 816 } 817 818 return 0; 819 } 820 821 static inline int 822 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 823 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 824 { 825 struct rte_mbuf *m; 826 int dstlen; 827 int l, n = srclen; 828 uint8_t *src; 829 830 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 831 m = m->next) 832 offset -= rte_pktmbuf_data_len(m); 833 834 if (m == 0) 835 return -1; 836 837 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 838 839 l = rte_pktmbuf_data_len(m) - offset; 840 if (srclen <= l) { 841 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 842 return -1; 843 *dst += l; 844 return 0; 845 } 846 847 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 848 return -1; 849 850 *dst += dstlen; 851 n -= l; 852 853 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 854 src = rte_pktmbuf_mtod(m, uint8_t *); 855 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 856 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 857 return -1; 858 *dst += dstlen; 859 n -= l; 860 } 861 862 return 0; 863 } 864 865 /** Process standard openssl cipher encryption */ 866 static int 867 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 868 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) 869 { 870 int totlen; 871 872 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 873 goto process_cipher_encrypt_err; 874 875 EVP_CIPHER_CTX_set_padding(ctx, 0); 876 877 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 878 srclen, ctx)) 879 goto process_cipher_encrypt_err; 880 881 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 882 goto process_cipher_encrypt_err; 883 884 return 0; 885 886 process_cipher_encrypt_err: 887 OPENSSL_LOG(ERR, "Process openssl cipher encrypt failed"); 888 return -EINVAL; 889 } 890 891 /** Process standard openssl cipher encryption */ 892 static int 893 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 894 uint8_t *iv, int srclen, 895 EVP_CIPHER_CTX *ctx) 896 { 897 uint8_t i; 898 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 899 int encrypted_ivlen; 900 901 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 902 iv, DES_BLOCK_SIZE) <= 0) 903 goto process_cipher_encrypt_err; 904 905 for (i = 0; i < srclen; i++) 906 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 907 908 return 0; 909 910 process_cipher_encrypt_err: 911 OPENSSL_LOG(ERR, "Process openssl cipher bpi encrypt failed"); 912 return -EINVAL; 913 } 914 /** Process standard openssl cipher decryption */ 915 static int 916 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 917 int offset, uint8_t *iv, int srclen, EVP_CIPHER_CTX *ctx) 918 { 919 int totlen; 920 921 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 922 goto process_cipher_decrypt_err; 923 924 EVP_CIPHER_CTX_set_padding(ctx, 0); 925 926 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 927 srclen, ctx)) 928 goto process_cipher_decrypt_err; 929 930 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 931 goto process_cipher_decrypt_err; 932 return 0; 933 934 process_cipher_decrypt_err: 935 OPENSSL_LOG(ERR, "Process openssl cipher decrypt failed"); 936 return -EINVAL; 937 } 938 939 /** Process cipher des 3 ctr encryption, decryption algorithm */ 940 static int 941 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 942 int offset, uint8_t *iv, uint8_t *key, int srclen, 943 EVP_CIPHER_CTX *ctx) 944 { 945 uint8_t ebuf[8], ctr[8]; 946 int unused, n; 947 struct rte_mbuf *m; 948 uint8_t *src; 949 int l; 950 951 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 952 m = m->next) 953 offset -= rte_pktmbuf_data_len(m); 954 955 if (m == 0) 956 goto process_cipher_des3ctr_err; 957 958 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 959 l = rte_pktmbuf_data_len(m) - offset; 960 961 /* We use 3DES encryption also for decryption. 962 * IV is not important for 3DES ecb 963 */ 964 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 965 goto process_cipher_des3ctr_err; 966 967 memcpy(ctr, iv, 8); 968 969 for (n = 0; n < srclen; n++) { 970 if (n % 8 == 0) { 971 if (EVP_EncryptUpdate(ctx, 972 (unsigned char *)&ebuf, &unused, 973 (const unsigned char *)&ctr, 8) <= 0) 974 goto process_cipher_des3ctr_err; 975 ctr_inc(ctr); 976 } 977 dst[n] = *(src++) ^ ebuf[n % 8]; 978 979 l--; 980 if (!l) { 981 m = m->next; 982 if (m) { 983 src = rte_pktmbuf_mtod(m, uint8_t *); 984 l = rte_pktmbuf_data_len(m); 985 } 986 } 987 } 988 989 return 0; 990 991 process_cipher_des3ctr_err: 992 OPENSSL_LOG(ERR, "Process openssl cipher des 3 ede ctr failed"); 993 return -EINVAL; 994 } 995 996 /** Process AES-GCM encrypt algorithm */ 997 static int 998 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 999 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1000 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1001 { 1002 int len = 0, unused = 0; 1003 uint8_t empty[] = {}; 1004 1005 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1006 goto process_auth_encryption_gcm_err; 1007 1008 if (aadlen > 0) 1009 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1010 goto process_auth_encryption_gcm_err; 1011 1012 if (srclen > 0) 1013 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1014 srclen, ctx)) 1015 goto process_auth_encryption_gcm_err; 1016 1017 /* Workaround open ssl bug in version less then 1.0.1f */ 1018 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1019 goto process_auth_encryption_gcm_err; 1020 1021 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1022 goto process_auth_encryption_gcm_err; 1023 1024 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 1025 goto process_auth_encryption_gcm_err; 1026 1027 return 0; 1028 1029 process_auth_encryption_gcm_err: 1030 OPENSSL_LOG(ERR, "Process openssl auth encryption gcm failed"); 1031 return -EINVAL; 1032 } 1033 1034 /** Process AES-CCM encrypt algorithm */ 1035 static int 1036 process_openssl_auth_encryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1037 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1038 uint8_t *dst, uint8_t *tag, uint8_t taglen, EVP_CIPHER_CTX *ctx) 1039 { 1040 int len = 0; 1041 1042 if (EVP_EncryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1043 goto process_auth_encryption_ccm_err; 1044 1045 if (EVP_EncryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1046 goto process_auth_encryption_ccm_err; 1047 1048 if (aadlen > 0) 1049 /* 1050 * For AES-CCM, the actual AAD is placed 1051 * 18 bytes after the start of the AAD field, 1052 * according to the API. 1053 */ 1054 if (EVP_EncryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1055 goto process_auth_encryption_ccm_err; 1056 1057 if (srclen > 0) 1058 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 1059 srclen, ctx)) 1060 goto process_auth_encryption_ccm_err; 1061 1062 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 1063 goto process_auth_encryption_ccm_err; 1064 1065 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_GET_TAG, taglen, tag) <= 0) 1066 goto process_auth_encryption_ccm_err; 1067 1068 return 0; 1069 1070 process_auth_encryption_ccm_err: 1071 OPENSSL_LOG(ERR, "Process openssl auth encryption ccm failed"); 1072 return -EINVAL; 1073 } 1074 1075 /** Process AES-GCM decrypt algorithm */ 1076 static int 1077 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 1078 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1079 uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx) 1080 { 1081 int len = 0, unused = 0; 1082 uint8_t empty[] = {}; 1083 1084 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 1085 goto process_auth_decryption_gcm_err; 1086 1087 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1088 goto process_auth_decryption_gcm_err; 1089 1090 if (aadlen > 0) 1091 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 1092 goto process_auth_decryption_gcm_err; 1093 1094 if (srclen > 0) 1095 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1096 srclen, ctx)) 1097 goto process_auth_decryption_gcm_err; 1098 1099 /* Workaround open ssl bug in version less then 1.0.1f */ 1100 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 1101 goto process_auth_decryption_gcm_err; 1102 1103 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 1104 return -EFAULT; 1105 1106 return 0; 1107 1108 process_auth_decryption_gcm_err: 1109 OPENSSL_LOG(ERR, "Process openssl auth decryption gcm failed"); 1110 return -EINVAL; 1111 } 1112 1113 /** Process AES-CCM decrypt algorithm */ 1114 static int 1115 process_openssl_auth_decryption_ccm(struct rte_mbuf *mbuf_src, int offset, 1116 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, 1117 uint8_t *dst, uint8_t *tag, uint8_t tag_len, 1118 EVP_CIPHER_CTX *ctx) 1119 { 1120 int len = 0; 1121 1122 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_CCM_SET_TAG, tag_len, tag) <= 0) 1123 goto process_auth_decryption_ccm_err; 1124 1125 if (EVP_DecryptInit_ex(ctx, NULL, NULL, NULL, iv) <= 0) 1126 goto process_auth_decryption_ccm_err; 1127 1128 if (EVP_DecryptUpdate(ctx, NULL, &len, NULL, srclen) <= 0) 1129 goto process_auth_decryption_ccm_err; 1130 1131 if (aadlen > 0) 1132 /* 1133 * For AES-CCM, the actual AAD is placed 1134 * 18 bytes after the start of the AAD field, 1135 * according to the API. 1136 */ 1137 if (EVP_DecryptUpdate(ctx, NULL, &len, aad + 18, aadlen) <= 0) 1138 goto process_auth_decryption_ccm_err; 1139 1140 if (srclen > 0) 1141 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 1142 srclen, ctx)) 1143 return -EFAULT; 1144 1145 return 0; 1146 1147 process_auth_decryption_ccm_err: 1148 OPENSSL_LOG(ERR, "Process openssl auth decryption ccm failed"); 1149 return -EINVAL; 1150 } 1151 1152 /** Process standard openssl auth algorithms */ 1153 static int 1154 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1155 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 1156 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 1157 { 1158 size_t dstlen; 1159 struct rte_mbuf *m; 1160 int l, n = srclen; 1161 uint8_t *src; 1162 1163 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1164 m = m->next) 1165 offset -= rte_pktmbuf_data_len(m); 1166 1167 if (m == 0) 1168 goto process_auth_err; 1169 1170 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 1171 goto process_auth_err; 1172 1173 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1174 1175 l = rte_pktmbuf_data_len(m) - offset; 1176 if (srclen <= l) { 1177 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 1178 goto process_auth_err; 1179 goto process_auth_final; 1180 } 1181 1182 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1183 goto process_auth_err; 1184 1185 n -= l; 1186 1187 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1188 src = rte_pktmbuf_mtod(m, uint8_t *); 1189 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1190 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 1191 goto process_auth_err; 1192 n -= l; 1193 } 1194 1195 process_auth_final: 1196 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 1197 goto process_auth_err; 1198 return 0; 1199 1200 process_auth_err: 1201 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1202 return -EINVAL; 1203 } 1204 1205 /** Process standard openssl auth algorithms with hmac */ 1206 static int 1207 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 1208 int srclen, HMAC_CTX *ctx) 1209 { 1210 unsigned int dstlen; 1211 struct rte_mbuf *m; 1212 int l, n = srclen; 1213 uint8_t *src; 1214 1215 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 1216 m = m->next) 1217 offset -= rte_pktmbuf_data_len(m); 1218 1219 if (m == 0) 1220 goto process_auth_err; 1221 1222 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 1223 1224 l = rte_pktmbuf_data_len(m) - offset; 1225 if (srclen <= l) { 1226 if (HMAC_Update(ctx, (unsigned char *)src, srclen) != 1) 1227 goto process_auth_err; 1228 goto process_auth_final; 1229 } 1230 1231 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1232 goto process_auth_err; 1233 1234 n -= l; 1235 1236 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 1237 src = rte_pktmbuf_mtod(m, uint8_t *); 1238 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 1239 if (HMAC_Update(ctx, (unsigned char *)src, l) != 1) 1240 goto process_auth_err; 1241 n -= l; 1242 } 1243 1244 process_auth_final: 1245 if (HMAC_Final(ctx, dst, &dstlen) != 1) 1246 goto process_auth_err; 1247 1248 if (unlikely(HMAC_Init_ex(ctx, NULL, 0, NULL, NULL) != 1)) 1249 goto process_auth_err; 1250 1251 return 0; 1252 1253 process_auth_err: 1254 OPENSSL_LOG(ERR, "Process openssl auth failed"); 1255 return -EINVAL; 1256 } 1257 1258 /*----------------------------------------------------------------------------*/ 1259 1260 /** Process auth/cipher combined operation */ 1261 static void 1262 process_openssl_combined_op 1263 (struct rte_crypto_op *op, struct openssl_session *sess, 1264 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1265 { 1266 /* cipher */ 1267 uint8_t *dst = NULL, *iv, *tag, *aad; 1268 int srclen, aadlen, status = -1; 1269 uint32_t offset; 1270 uint8_t taglen; 1271 1272 /* 1273 * Segmented destination buffer is not supported for 1274 * encryption/decryption 1275 */ 1276 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1277 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1278 return; 1279 } 1280 1281 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1282 sess->iv.offset); 1283 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1284 srclen = 0; 1285 offset = op->sym->auth.data.offset; 1286 aadlen = op->sym->auth.data.length; 1287 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1288 op->sym->auth.data.offset); 1289 tag = op->sym->auth.digest.data; 1290 if (tag == NULL) 1291 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1292 offset + aadlen); 1293 } else { 1294 srclen = op->sym->aead.data.length; 1295 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1296 op->sym->aead.data.offset); 1297 offset = op->sym->aead.data.offset; 1298 aad = op->sym->aead.aad.data; 1299 aadlen = sess->auth.aad_length; 1300 tag = op->sym->aead.digest.data; 1301 if (tag == NULL) 1302 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1303 offset + srclen); 1304 } 1305 1306 taglen = sess->auth.digest_length; 1307 1308 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1309 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1310 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1311 status = process_openssl_auth_encryption_gcm( 1312 mbuf_src, offset, srclen, 1313 aad, aadlen, iv, 1314 dst, tag, sess->cipher.ctx); 1315 else 1316 status = process_openssl_auth_encryption_ccm( 1317 mbuf_src, offset, srclen, 1318 aad, aadlen, iv, 1319 dst, tag, taglen, sess->cipher.ctx); 1320 1321 } else { 1322 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC || 1323 sess->aead_algo == RTE_CRYPTO_AEAD_AES_GCM) 1324 status = process_openssl_auth_decryption_gcm( 1325 mbuf_src, offset, srclen, 1326 aad, aadlen, iv, 1327 dst, tag, sess->cipher.ctx); 1328 else 1329 status = process_openssl_auth_decryption_ccm( 1330 mbuf_src, offset, srclen, 1331 aad, aadlen, iv, 1332 dst, tag, taglen, sess->cipher.ctx); 1333 } 1334 1335 if (status != 0) { 1336 if (status == (-EFAULT) && 1337 sess->auth.operation == 1338 RTE_CRYPTO_AUTH_OP_VERIFY) 1339 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1340 else 1341 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1342 } 1343 } 1344 1345 /** Process cipher operation */ 1346 static void 1347 process_openssl_cipher_op 1348 (struct rte_crypto_op *op, struct openssl_session *sess, 1349 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1350 { 1351 uint8_t *dst, *iv; 1352 int srclen, status; 1353 1354 /* 1355 * Segmented destination buffer is not supported for 1356 * encryption/decryption 1357 */ 1358 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1359 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1360 return; 1361 } 1362 1363 srclen = op->sym->cipher.data.length; 1364 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1365 op->sym->cipher.data.offset); 1366 1367 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1368 sess->iv.offset); 1369 1370 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1371 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1372 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1373 op->sym->cipher.data.offset, iv, 1374 srclen, sess->cipher.ctx); 1375 else 1376 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1377 op->sym->cipher.data.offset, iv, 1378 srclen, sess->cipher.ctx); 1379 else 1380 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1381 op->sym->cipher.data.offset, iv, 1382 sess->cipher.key.data, srclen, 1383 sess->cipher.ctx); 1384 1385 if (status != 0) 1386 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1387 } 1388 1389 /** Process cipher operation */ 1390 static void 1391 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1392 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1393 struct rte_mbuf *mbuf_dst) 1394 { 1395 uint8_t *src, *dst, *iv; 1396 uint8_t block_size, last_block_len; 1397 int srclen, status = 0; 1398 1399 srclen = op->sym->cipher.data.length; 1400 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1401 op->sym->cipher.data.offset); 1402 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1403 op->sym->cipher.data.offset); 1404 1405 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1406 sess->iv.offset); 1407 1408 block_size = DES_BLOCK_SIZE; 1409 1410 last_block_len = srclen % block_size; 1411 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1412 /* Encrypt only with ECB mode XOR IV */ 1413 if (srclen < block_size) { 1414 status = process_openssl_cipher_bpi_encrypt(src, dst, 1415 iv, srclen, 1416 sess->cipher.bpi_ctx); 1417 } else { 1418 srclen -= last_block_len; 1419 /* Encrypt with the block aligned stream with CBC mode */ 1420 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1421 op->sym->cipher.data.offset, iv, 1422 srclen, sess->cipher.ctx); 1423 if (last_block_len) { 1424 /* Point at last block */ 1425 dst += srclen; 1426 /* 1427 * IV is the last encrypted block from 1428 * the previous operation 1429 */ 1430 iv = dst - block_size; 1431 src += srclen; 1432 srclen = last_block_len; 1433 /* Encrypt the last frame with ECB mode */ 1434 status |= process_openssl_cipher_bpi_encrypt(src, 1435 dst, iv, 1436 srclen, sess->cipher.bpi_ctx); 1437 } 1438 } 1439 } else { 1440 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1441 if (srclen < block_size) { 1442 status = process_openssl_cipher_bpi_encrypt(src, dst, 1443 iv, 1444 srclen, 1445 sess->cipher.bpi_ctx); 1446 } else { 1447 if (last_block_len) { 1448 /* Point at last block */ 1449 dst += srclen - last_block_len; 1450 src += srclen - last_block_len; 1451 /* 1452 * IV is the last full block 1453 */ 1454 iv = src - block_size; 1455 /* 1456 * Decrypt the last frame with ECB mode 1457 * (encrypt, as it is the same operation) 1458 */ 1459 status = process_openssl_cipher_bpi_encrypt(src, 1460 dst, iv, 1461 last_block_len, sess->cipher.bpi_ctx); 1462 /* Prepare parameters for CBC mode op */ 1463 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1464 sess->iv.offset); 1465 dst += last_block_len - srclen; 1466 srclen -= last_block_len; 1467 } 1468 1469 /* Decrypt with CBC mode */ 1470 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1471 op->sym->cipher.data.offset, iv, 1472 srclen, sess->cipher.ctx); 1473 } 1474 } 1475 1476 if (status != 0) 1477 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1478 } 1479 1480 /** Process auth operation */ 1481 static void 1482 process_openssl_auth_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1483 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1484 struct rte_mbuf *mbuf_dst) 1485 { 1486 uint8_t *dst; 1487 int srclen, status; 1488 1489 srclen = op->sym->auth.data.length; 1490 1491 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) 1492 dst = qp->temp_digest; 1493 else { 1494 dst = op->sym->auth.digest.data; 1495 if (dst == NULL) 1496 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1497 op->sym->auth.data.offset + 1498 op->sym->auth.data.length); 1499 } 1500 1501 switch (sess->auth.mode) { 1502 case OPENSSL_AUTH_AS_AUTH: 1503 status = process_openssl_auth(mbuf_src, dst, 1504 op->sym->auth.data.offset, NULL, NULL, srclen, 1505 sess->auth.auth.ctx, sess->auth.auth.evp_algo); 1506 break; 1507 case OPENSSL_AUTH_AS_HMAC: 1508 status = process_openssl_auth_hmac(mbuf_src, dst, 1509 op->sym->auth.data.offset, srclen, 1510 sess->auth.hmac.ctx); 1511 break; 1512 default: 1513 status = -1; 1514 break; 1515 } 1516 1517 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1518 if (memcmp(dst, op->sym->auth.digest.data, 1519 sess->auth.digest_length) != 0) { 1520 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1521 } 1522 } 1523 1524 if (status != 0) 1525 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1526 } 1527 1528 /** Process crypto operation for mbuf */ 1529 static int 1530 process_op(struct openssl_qp *qp, struct rte_crypto_op *op, 1531 struct openssl_session *sess) 1532 { 1533 struct rte_mbuf *msrc, *mdst; 1534 int retval; 1535 1536 msrc = op->sym->m_src; 1537 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 1538 1539 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1540 1541 switch (sess->chain_order) { 1542 case OPENSSL_CHAIN_ONLY_CIPHER: 1543 process_openssl_cipher_op(op, sess, msrc, mdst); 1544 break; 1545 case OPENSSL_CHAIN_ONLY_AUTH: 1546 process_openssl_auth_op(qp, op, sess, msrc, mdst); 1547 break; 1548 case OPENSSL_CHAIN_CIPHER_AUTH: 1549 process_openssl_cipher_op(op, sess, msrc, mdst); 1550 process_openssl_auth_op(qp, op, sess, mdst, mdst); 1551 break; 1552 case OPENSSL_CHAIN_AUTH_CIPHER: 1553 process_openssl_auth_op(qp, op, sess, msrc, mdst); 1554 process_openssl_cipher_op(op, sess, msrc, mdst); 1555 break; 1556 case OPENSSL_CHAIN_COMBINED: 1557 process_openssl_combined_op(op, sess, msrc, mdst); 1558 break; 1559 case OPENSSL_CHAIN_CIPHER_BPI: 1560 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 1561 break; 1562 default: 1563 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1564 break; 1565 } 1566 1567 /* Free session if a session-less crypto op */ 1568 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 1569 openssl_reset_session(sess); 1570 memset(sess, 0, sizeof(struct openssl_session)); 1571 memset(op->sym->session, 0, 1572 rte_cryptodev_sym_get_header_session_size()); 1573 rte_mempool_put(qp->sess_mp, sess); 1574 rte_mempool_put(qp->sess_mp, op->sym->session); 1575 op->sym->session = NULL; 1576 } 1577 1578 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 1579 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1580 1581 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 1582 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 1583 else 1584 retval = -1; 1585 1586 return retval; 1587 } 1588 1589 /* 1590 *------------------------------------------------------------------------------ 1591 * PMD Framework 1592 *------------------------------------------------------------------------------ 1593 */ 1594 1595 /** Enqueue burst */ 1596 static uint16_t 1597 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 1598 uint16_t nb_ops) 1599 { 1600 struct openssl_session *sess; 1601 struct openssl_qp *qp = queue_pair; 1602 int i, retval; 1603 1604 for (i = 0; i < nb_ops; i++) { 1605 sess = get_session(qp, ops[i]); 1606 if (unlikely(sess == NULL)) 1607 goto enqueue_err; 1608 1609 retval = process_op(qp, ops[i], sess); 1610 if (unlikely(retval < 0)) 1611 goto enqueue_err; 1612 } 1613 1614 qp->stats.enqueued_count += i; 1615 return i; 1616 1617 enqueue_err: 1618 qp->stats.enqueue_err_count++; 1619 return i; 1620 } 1621 1622 /** Dequeue burst */ 1623 static uint16_t 1624 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 1625 uint16_t nb_ops) 1626 { 1627 struct openssl_qp *qp = queue_pair; 1628 1629 unsigned int nb_dequeued = 0; 1630 1631 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 1632 (void **)ops, nb_ops, NULL); 1633 qp->stats.dequeued_count += nb_dequeued; 1634 1635 return nb_dequeued; 1636 } 1637 1638 /** Create OPENSSL crypto device */ 1639 static int 1640 cryptodev_openssl_create(const char *name, 1641 struct rte_vdev_device *vdev, 1642 struct rte_cryptodev_pmd_init_params *init_params) 1643 { 1644 struct rte_cryptodev *dev; 1645 struct openssl_private *internals; 1646 1647 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 1648 if (dev == NULL) { 1649 OPENSSL_LOG(ERR, "failed to create cryptodev vdev"); 1650 goto init_error; 1651 } 1652 1653 dev->driver_id = cryptodev_driver_id; 1654 dev->dev_ops = rte_openssl_pmd_ops; 1655 1656 /* register rx/tx burst functions for data path */ 1657 dev->dequeue_burst = openssl_pmd_dequeue_burst; 1658 dev->enqueue_burst = openssl_pmd_enqueue_burst; 1659 1660 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 1661 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 1662 RTE_CRYPTODEV_FF_CPU_AESNI | 1663 RTE_CRYPTODEV_FF_OOP_SGL_IN_LB_OUT | 1664 RTE_CRYPTODEV_FF_OOP_LB_IN_LB_OUT; 1665 1666 /* Set vector instructions mode supported */ 1667 internals = dev->data->dev_private; 1668 1669 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 1670 1671 return 0; 1672 1673 init_error: 1674 OPENSSL_LOG(ERR, "driver %s: create failed", 1675 init_params->name); 1676 1677 cryptodev_openssl_remove(vdev); 1678 return -EFAULT; 1679 } 1680 1681 /** Initialise OPENSSL crypto device */ 1682 static int 1683 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 1684 { 1685 struct rte_cryptodev_pmd_init_params init_params = { 1686 "", 1687 sizeof(struct openssl_private), 1688 rte_socket_id(), 1689 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 1690 }; 1691 const char *name; 1692 const char *input_args; 1693 1694 name = rte_vdev_device_name(vdev); 1695 if (name == NULL) 1696 return -EINVAL; 1697 input_args = rte_vdev_device_args(vdev); 1698 1699 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 1700 1701 return cryptodev_openssl_create(name, vdev, &init_params); 1702 } 1703 1704 /** Uninitialise OPENSSL crypto device */ 1705 static int 1706 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 1707 { 1708 struct rte_cryptodev *cryptodev; 1709 const char *name; 1710 1711 name = rte_vdev_device_name(vdev); 1712 if (name == NULL) 1713 return -EINVAL; 1714 1715 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 1716 if (cryptodev == NULL) 1717 return -ENODEV; 1718 1719 return rte_cryptodev_pmd_destroy(cryptodev); 1720 } 1721 1722 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 1723 .probe = cryptodev_openssl_probe, 1724 .remove = cryptodev_openssl_remove 1725 }; 1726 1727 static struct cryptodev_driver openssl_crypto_drv; 1728 1729 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 1730 cryptodev_openssl_pmd_drv); 1731 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 1732 "max_nb_queue_pairs=<int> " 1733 "socket_id=<int>"); 1734 RTE_PMD_REGISTER_CRYPTO_DRIVER(openssl_crypto_drv, 1735 cryptodev_openssl_pmd_drv.driver, cryptodev_driver_id); 1736 1737 RTE_INIT(openssl_init_log) 1738 { 1739 openssl_logtype_driver = rte_log_register("pmd.crypto.openssl"); 1740 } 1741