1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Intel Corporation nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <rte_common.h> 34 #include <rte_hexdump.h> 35 #include <rte_cryptodev.h> 36 #include <rte_cryptodev_pmd.h> 37 #include <rte_cryptodev_vdev.h> 38 #include <rte_vdev.h> 39 #include <rte_malloc.h> 40 #include <rte_cpuflags.h> 41 42 #include <openssl/evp.h> 43 44 #include "rte_openssl_pmd_private.h" 45 46 #define DES_BLOCK_SIZE 8 47 48 static uint8_t cryptodev_driver_id; 49 50 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 51 52 /*----------------------------------------------------------------------------*/ 53 54 /** 55 * Increment counter by 1 56 * Counter is 64 bit array, big-endian 57 */ 58 static void 59 ctr_inc(uint8_t *ctr) 60 { 61 uint64_t *ctr64 = (uint64_t *)ctr; 62 63 *ctr64 = __builtin_bswap64(*ctr64); 64 (*ctr64)++; 65 *ctr64 = __builtin_bswap64(*ctr64); 66 } 67 68 /* 69 *------------------------------------------------------------------------------ 70 * Session Prepare 71 *------------------------------------------------------------------------------ 72 */ 73 74 /** Get xform chain order */ 75 static enum openssl_chain_order 76 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 77 { 78 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 79 80 if (xform != NULL) { 81 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 82 if (xform->next == NULL) 83 res = OPENSSL_CHAIN_ONLY_AUTH; 84 else if (xform->next->type == 85 RTE_CRYPTO_SYM_XFORM_CIPHER) 86 res = OPENSSL_CHAIN_AUTH_CIPHER; 87 } 88 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 89 if (xform->next == NULL) 90 res = OPENSSL_CHAIN_ONLY_CIPHER; 91 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 92 res = OPENSSL_CHAIN_CIPHER_AUTH; 93 } 94 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 95 res = OPENSSL_CHAIN_COMBINED; 96 } 97 98 return res; 99 } 100 101 /** Get session cipher key from input cipher key */ 102 static void 103 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key) 104 { 105 memcpy(session_key, input_key, keylen); 106 } 107 108 /** Get key ede 24 bytes standard from input key */ 109 static int 110 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede) 111 { 112 int res = 0; 113 114 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 115 switch (keylen) { 116 case 24: 117 memcpy(key_ede, key, 24); 118 break; 119 case 16: 120 /* K3 = K1 */ 121 memcpy(key_ede, key, 16); 122 memcpy(key_ede + 16, key, 8); 123 break; 124 case 8: 125 /* K1 = K2 = K3 (DES compatibility) */ 126 memcpy(key_ede, key, 8); 127 memcpy(key_ede + 8, key, 8); 128 memcpy(key_ede + 16, key, 8); 129 break; 130 default: 131 OPENSSL_LOG_ERR("Unsupported key size"); 132 res = -EINVAL; 133 } 134 135 return res; 136 } 137 138 /** Get adequate openssl function for input cipher algorithm */ 139 static uint8_t 140 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 141 const EVP_CIPHER **algo) 142 { 143 int res = 0; 144 145 if (algo != NULL) { 146 switch (sess_algo) { 147 case RTE_CRYPTO_CIPHER_3DES_CBC: 148 switch (keylen) { 149 case 16: 150 *algo = EVP_des_ede_cbc(); 151 break; 152 case 24: 153 *algo = EVP_des_ede3_cbc(); 154 break; 155 default: 156 res = -EINVAL; 157 } 158 break; 159 case RTE_CRYPTO_CIPHER_3DES_CTR: 160 break; 161 case RTE_CRYPTO_CIPHER_AES_CBC: 162 switch (keylen) { 163 case 16: 164 *algo = EVP_aes_128_cbc(); 165 break; 166 case 24: 167 *algo = EVP_aes_192_cbc(); 168 break; 169 case 32: 170 *algo = EVP_aes_256_cbc(); 171 break; 172 default: 173 res = -EINVAL; 174 } 175 break; 176 case RTE_CRYPTO_CIPHER_AES_CTR: 177 switch (keylen) { 178 case 16: 179 *algo = EVP_aes_128_ctr(); 180 break; 181 case 24: 182 *algo = EVP_aes_192_ctr(); 183 break; 184 case 32: 185 *algo = EVP_aes_256_ctr(); 186 break; 187 default: 188 res = -EINVAL; 189 } 190 break; 191 default: 192 res = -EINVAL; 193 break; 194 } 195 } else { 196 res = -EINVAL; 197 } 198 199 return res; 200 } 201 202 /** Get adequate openssl function for input auth algorithm */ 203 static uint8_t 204 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 205 const EVP_MD **algo) 206 { 207 int res = 0; 208 209 if (algo != NULL) { 210 switch (sessalgo) { 211 case RTE_CRYPTO_AUTH_MD5: 212 case RTE_CRYPTO_AUTH_MD5_HMAC: 213 *algo = EVP_md5(); 214 break; 215 case RTE_CRYPTO_AUTH_SHA1: 216 case RTE_CRYPTO_AUTH_SHA1_HMAC: 217 *algo = EVP_sha1(); 218 break; 219 case RTE_CRYPTO_AUTH_SHA224: 220 case RTE_CRYPTO_AUTH_SHA224_HMAC: 221 *algo = EVP_sha224(); 222 break; 223 case RTE_CRYPTO_AUTH_SHA256: 224 case RTE_CRYPTO_AUTH_SHA256_HMAC: 225 *algo = EVP_sha256(); 226 break; 227 case RTE_CRYPTO_AUTH_SHA384: 228 case RTE_CRYPTO_AUTH_SHA384_HMAC: 229 *algo = EVP_sha384(); 230 break; 231 case RTE_CRYPTO_AUTH_SHA512: 232 case RTE_CRYPTO_AUTH_SHA512_HMAC: 233 *algo = EVP_sha512(); 234 break; 235 default: 236 res = -EINVAL; 237 break; 238 } 239 } else { 240 res = -EINVAL; 241 } 242 243 return res; 244 } 245 246 /** Get adequate openssl function for input cipher algorithm */ 247 static uint8_t 248 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 249 const EVP_CIPHER **algo) 250 { 251 int res = 0; 252 253 if (algo != NULL) { 254 switch (sess_algo) { 255 case RTE_CRYPTO_AEAD_AES_GCM: 256 switch (keylen) { 257 case 16: 258 *algo = EVP_aes_128_gcm(); 259 break; 260 case 24: 261 *algo = EVP_aes_192_gcm(); 262 break; 263 case 32: 264 *algo = EVP_aes_256_gcm(); 265 break; 266 default: 267 res = -EINVAL; 268 } 269 break; 270 default: 271 res = -EINVAL; 272 break; 273 } 274 } else { 275 res = -EINVAL; 276 } 277 278 return res; 279 } 280 281 /** Set session cipher parameters */ 282 static int 283 openssl_set_session_cipher_parameters(struct openssl_session *sess, 284 const struct rte_crypto_sym_xform *xform) 285 { 286 /* Select cipher direction */ 287 sess->cipher.direction = xform->cipher.op; 288 /* Select cipher key */ 289 sess->cipher.key.length = xform->cipher.key.length; 290 291 /* Set IV parameters */ 292 sess->iv.offset = xform->cipher.iv.offset; 293 sess->iv.length = xform->cipher.iv.length; 294 295 /* Select cipher algo */ 296 switch (xform->cipher.algo) { 297 case RTE_CRYPTO_CIPHER_3DES_CBC: 298 case RTE_CRYPTO_CIPHER_AES_CBC: 299 case RTE_CRYPTO_CIPHER_AES_CTR: 300 sess->cipher.mode = OPENSSL_CIPHER_LIB; 301 sess->cipher.algo = xform->cipher.algo; 302 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 303 304 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 305 &sess->cipher.evp_algo) != 0) 306 return -EINVAL; 307 308 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 309 sess->cipher.key.data); 310 311 break; 312 313 case RTE_CRYPTO_CIPHER_3DES_CTR: 314 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 315 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 316 317 if (get_cipher_key_ede(xform->cipher.key.data, 318 sess->cipher.key.length, 319 sess->cipher.key.data) != 0) 320 return -EINVAL; 321 break; 322 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 323 sess->cipher.algo = xform->cipher.algo; 324 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 325 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 326 sess->cipher.evp_algo = EVP_des_cbc(); 327 328 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 329 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 330 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 331 NULL, xform->cipher.key.data, 0) != 1) 332 return -EINVAL; 333 334 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 335 sess->cipher.key.data); 336 break; 337 default: 338 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 339 return -EINVAL; 340 } 341 342 return 0; 343 } 344 345 /* Set session auth parameters */ 346 static int 347 openssl_set_session_auth_parameters(struct openssl_session *sess, 348 const struct rte_crypto_sym_xform *xform) 349 { 350 /* Select auth generate/verify */ 351 sess->auth.operation = xform->auth.op; 352 sess->auth.algo = xform->auth.algo; 353 354 /* Select auth algo */ 355 switch (xform->auth.algo) { 356 case RTE_CRYPTO_AUTH_AES_GMAC: 357 sess->chain_order = OPENSSL_CHAIN_COMBINED; 358 359 /* Set IV parameters */ 360 sess->iv.offset = xform->auth.iv.offset; 361 sess->iv.length = xform->auth.iv.length; 362 363 /* 364 * OpenSSL requires GMAC to be a GCM operation 365 * with no cipher data length 366 */ 367 sess->cipher.mode = OPENSSL_CIPHER_LIB; 368 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 369 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 370 else 371 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 372 373 sess->cipher.key.length = xform->auth.key.length; 374 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 375 376 if (get_aead_algo(RTE_CRYPTO_AEAD_AES_GCM, 377 sess->cipher.key.length, 378 &sess->cipher.evp_algo) != 0) 379 return -EINVAL; 380 381 get_cipher_key(xform->auth.key.data, xform->auth.key.length, 382 sess->cipher.key.data); 383 384 break; 385 386 case RTE_CRYPTO_AUTH_MD5: 387 case RTE_CRYPTO_AUTH_SHA1: 388 case RTE_CRYPTO_AUTH_SHA224: 389 case RTE_CRYPTO_AUTH_SHA256: 390 case RTE_CRYPTO_AUTH_SHA384: 391 case RTE_CRYPTO_AUTH_SHA512: 392 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 393 if (get_auth_algo(xform->auth.algo, 394 &sess->auth.auth.evp_algo) != 0) 395 return -EINVAL; 396 sess->auth.auth.ctx = EVP_MD_CTX_create(); 397 break; 398 399 case RTE_CRYPTO_AUTH_MD5_HMAC: 400 case RTE_CRYPTO_AUTH_SHA1_HMAC: 401 case RTE_CRYPTO_AUTH_SHA224_HMAC: 402 case RTE_CRYPTO_AUTH_SHA256_HMAC: 403 case RTE_CRYPTO_AUTH_SHA384_HMAC: 404 case RTE_CRYPTO_AUTH_SHA512_HMAC: 405 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 406 sess->auth.hmac.ctx = EVP_MD_CTX_create(); 407 if (get_auth_algo(xform->auth.algo, 408 &sess->auth.hmac.evp_algo) != 0) 409 return -EINVAL; 410 sess->auth.hmac.pkey = EVP_PKEY_new_mac_key(EVP_PKEY_HMAC, NULL, 411 xform->auth.key.data, xform->auth.key.length); 412 break; 413 414 default: 415 return -EINVAL; 416 } 417 418 sess->auth.digest_length = xform->auth.digest_length; 419 420 return 0; 421 } 422 423 /* Set session AEAD parameters */ 424 static int 425 openssl_set_session_aead_parameters(struct openssl_session *sess, 426 const struct rte_crypto_sym_xform *xform) 427 { 428 /* Select cipher direction */ 429 sess->cipher.direction = xform->cipher.op; 430 /* Select cipher key */ 431 sess->cipher.key.length = xform->aead.key.length; 432 433 /* Set IV parameters */ 434 sess->iv.offset = xform->aead.iv.offset; 435 sess->iv.length = xform->aead.iv.length; 436 437 /* Select auth generate/verify */ 438 sess->auth.operation = xform->auth.op; 439 sess->auth.algo = xform->auth.algo; 440 441 /* Select auth algo */ 442 switch (xform->aead.algo) { 443 case RTE_CRYPTO_AEAD_AES_GCM: 444 sess->cipher.mode = OPENSSL_CIPHER_LIB; 445 sess->aead_algo = xform->aead.algo; 446 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 447 448 if (get_aead_algo(sess->aead_algo, sess->cipher.key.length, 449 &sess->cipher.evp_algo) != 0) 450 return -EINVAL; 451 452 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 453 sess->cipher.key.data); 454 455 sess->chain_order = OPENSSL_CHAIN_COMBINED; 456 break; 457 default: 458 return -EINVAL; 459 } 460 461 sess->auth.aad_length = xform->aead.add_auth_data_length; 462 sess->auth.digest_length = xform->aead.digest_length; 463 464 return 0; 465 } 466 467 /** Parse crypto xform chain and set private session parameters */ 468 int 469 openssl_set_session_parameters(struct openssl_session *sess, 470 const struct rte_crypto_sym_xform *xform) 471 { 472 const struct rte_crypto_sym_xform *cipher_xform = NULL; 473 const struct rte_crypto_sym_xform *auth_xform = NULL; 474 const struct rte_crypto_sym_xform *aead_xform = NULL; 475 476 sess->chain_order = openssl_get_chain_order(xform); 477 switch (sess->chain_order) { 478 case OPENSSL_CHAIN_ONLY_CIPHER: 479 cipher_xform = xform; 480 break; 481 case OPENSSL_CHAIN_ONLY_AUTH: 482 auth_xform = xform; 483 break; 484 case OPENSSL_CHAIN_CIPHER_AUTH: 485 cipher_xform = xform; 486 auth_xform = xform->next; 487 break; 488 case OPENSSL_CHAIN_AUTH_CIPHER: 489 auth_xform = xform; 490 cipher_xform = xform->next; 491 break; 492 case OPENSSL_CHAIN_COMBINED: 493 aead_xform = xform; 494 break; 495 default: 496 return -EINVAL; 497 } 498 499 /* Default IV length = 0 */ 500 sess->iv.length = 0; 501 502 /* cipher_xform must be check before auth_xform */ 503 if (cipher_xform) { 504 if (openssl_set_session_cipher_parameters( 505 sess, cipher_xform)) { 506 OPENSSL_LOG_ERR( 507 "Invalid/unsupported cipher parameters"); 508 return -EINVAL; 509 } 510 } 511 512 if (auth_xform) { 513 if (openssl_set_session_auth_parameters(sess, auth_xform)) { 514 OPENSSL_LOG_ERR( 515 "Invalid/unsupported auth parameters"); 516 return -EINVAL; 517 } 518 } 519 520 if (aead_xform) { 521 if (openssl_set_session_aead_parameters(sess, aead_xform)) { 522 OPENSSL_LOG_ERR( 523 "Invalid/unsupported auth parameters"); 524 return -EINVAL; 525 } 526 } 527 528 return 0; 529 } 530 531 /** Reset private session parameters */ 532 void 533 openssl_reset_session(struct openssl_session *sess) 534 { 535 EVP_CIPHER_CTX_free(sess->cipher.ctx); 536 537 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 538 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 539 540 switch (sess->auth.mode) { 541 case OPENSSL_AUTH_AS_AUTH: 542 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 543 break; 544 case OPENSSL_AUTH_AS_HMAC: 545 EVP_PKEY_free(sess->auth.hmac.pkey); 546 EVP_MD_CTX_destroy(sess->auth.hmac.ctx); 547 break; 548 default: 549 break; 550 } 551 } 552 553 /** Provide session for operation */ 554 static struct openssl_session * 555 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 556 { 557 struct openssl_session *sess = NULL; 558 559 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 560 /* get existing session */ 561 if (likely(op->sym->session != NULL && 562 op->sym->session->driver_id == 563 cryptodev_driver_id)) 564 sess = (struct openssl_session *) 565 op->sym->session->_private; 566 } else { 567 /* provide internal session */ 568 void *_sess = NULL; 569 570 if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) { 571 sess = (struct openssl_session *) 572 ((struct rte_cryptodev_sym_session *)_sess) 573 ->_private; 574 575 if (unlikely(openssl_set_session_parameters( 576 sess, op->sym->xform) != 0)) { 577 rte_mempool_put(qp->sess_mp, _sess); 578 sess = NULL; 579 } else 580 op->sym->session = _sess; 581 } 582 } 583 584 if (sess == NULL) 585 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 586 587 return sess; 588 } 589 590 /* 591 *------------------------------------------------------------------------------ 592 * Process Operations 593 *------------------------------------------------------------------------------ 594 */ 595 static inline int 596 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 597 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 598 { 599 struct rte_mbuf *m; 600 int dstlen; 601 int l, n = srclen; 602 uint8_t *src; 603 604 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 605 m = m->next) 606 offset -= rte_pktmbuf_data_len(m); 607 608 if (m == 0) 609 return -1; 610 611 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 612 613 l = rte_pktmbuf_data_len(m) - offset; 614 if (srclen <= l) { 615 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 616 return -1; 617 *dst += l; 618 return 0; 619 } 620 621 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 622 return -1; 623 624 *dst += dstlen; 625 n -= l; 626 627 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 628 src = rte_pktmbuf_mtod(m, uint8_t *); 629 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 630 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 631 return -1; 632 *dst += dstlen; 633 n -= l; 634 } 635 636 return 0; 637 } 638 639 static inline int 640 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 641 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 642 { 643 struct rte_mbuf *m; 644 int dstlen; 645 int l, n = srclen; 646 uint8_t *src; 647 648 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 649 m = m->next) 650 offset -= rte_pktmbuf_data_len(m); 651 652 if (m == 0) 653 return -1; 654 655 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 656 657 l = rte_pktmbuf_data_len(m) - offset; 658 if (srclen <= l) { 659 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 660 return -1; 661 *dst += l; 662 return 0; 663 } 664 665 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 666 return -1; 667 668 *dst += dstlen; 669 n -= l; 670 671 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 672 src = rte_pktmbuf_mtod(m, uint8_t *); 673 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 674 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 675 return -1; 676 *dst += dstlen; 677 n -= l; 678 } 679 680 return 0; 681 } 682 683 /** Process standard openssl cipher encryption */ 684 static int 685 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 686 int offset, uint8_t *iv, uint8_t *key, int srclen, 687 EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) 688 { 689 int totlen; 690 691 if (EVP_EncryptInit_ex(ctx, algo, NULL, key, iv) <= 0) 692 goto process_cipher_encrypt_err; 693 694 EVP_CIPHER_CTX_set_padding(ctx, 0); 695 696 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 697 srclen, ctx)) 698 goto process_cipher_encrypt_err; 699 700 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 701 goto process_cipher_encrypt_err; 702 703 return 0; 704 705 process_cipher_encrypt_err: 706 OPENSSL_LOG_ERR("Process openssl cipher encrypt failed"); 707 return -EINVAL; 708 } 709 710 /** Process standard openssl cipher encryption */ 711 static int 712 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 713 uint8_t *iv, int srclen, 714 EVP_CIPHER_CTX *ctx) 715 { 716 uint8_t i; 717 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 718 int encrypted_ivlen; 719 720 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 721 iv, DES_BLOCK_SIZE) <= 0) 722 goto process_cipher_encrypt_err; 723 724 for (i = 0; i < srclen; i++) 725 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 726 727 return 0; 728 729 process_cipher_encrypt_err: 730 OPENSSL_LOG_ERR("Process openssl cipher bpi encrypt failed"); 731 return -EINVAL; 732 } 733 /** Process standard openssl cipher decryption */ 734 static int 735 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 736 int offset, uint8_t *iv, uint8_t *key, int srclen, 737 EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) 738 { 739 int totlen; 740 741 if (EVP_DecryptInit_ex(ctx, algo, NULL, key, iv) <= 0) 742 goto process_cipher_decrypt_err; 743 744 EVP_CIPHER_CTX_set_padding(ctx, 0); 745 746 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 747 srclen, ctx)) 748 goto process_cipher_decrypt_err; 749 750 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 751 goto process_cipher_decrypt_err; 752 return 0; 753 754 process_cipher_decrypt_err: 755 OPENSSL_LOG_ERR("Process openssl cipher decrypt failed"); 756 return -EINVAL; 757 } 758 759 /** Process cipher des 3 ctr encryption, decryption algorithm */ 760 static int 761 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 762 int offset, uint8_t *iv, uint8_t *key, int srclen, 763 EVP_CIPHER_CTX *ctx) 764 { 765 uint8_t ebuf[8], ctr[8]; 766 int unused, n; 767 struct rte_mbuf *m; 768 uint8_t *src; 769 int l; 770 771 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 772 m = m->next) 773 offset -= rte_pktmbuf_data_len(m); 774 775 if (m == 0) 776 goto process_cipher_des3ctr_err; 777 778 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 779 l = rte_pktmbuf_data_len(m) - offset; 780 781 /* We use 3DES encryption also for decryption. 782 * IV is not important for 3DES ecb 783 */ 784 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 785 goto process_cipher_des3ctr_err; 786 787 memcpy(ctr, iv, 8); 788 789 for (n = 0; n < srclen; n++) { 790 if (n % 8 == 0) { 791 if (EVP_EncryptUpdate(ctx, 792 (unsigned char *)&ebuf, &unused, 793 (const unsigned char *)&ctr, 8) <= 0) 794 goto process_cipher_des3ctr_err; 795 ctr_inc(ctr); 796 } 797 dst[n] = *(src++) ^ ebuf[n % 8]; 798 799 l--; 800 if (!l) { 801 m = m->next; 802 if (m) { 803 src = rte_pktmbuf_mtod(m, uint8_t *); 804 l = rte_pktmbuf_data_len(m); 805 } 806 } 807 } 808 809 return 0; 810 811 process_cipher_des3ctr_err: 812 OPENSSL_LOG_ERR("Process openssl cipher des 3 ede ctr failed"); 813 return -EINVAL; 814 } 815 816 /** Process auth/encription aes-gcm algorithm */ 817 static int 818 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 819 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, 820 uint8_t *key, uint8_t *dst, uint8_t *tag, 821 EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) 822 { 823 int len = 0, unused = 0; 824 uint8_t empty[] = {}; 825 826 if (EVP_EncryptInit_ex(ctx, algo, NULL, NULL, NULL) <= 0) 827 goto process_auth_encryption_gcm_err; 828 829 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL) <= 0) 830 goto process_auth_encryption_gcm_err; 831 832 if (EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv) <= 0) 833 goto process_auth_encryption_gcm_err; 834 835 if (aadlen > 0) 836 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 837 goto process_auth_encryption_gcm_err; 838 839 if (srclen > 0) 840 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 841 srclen, ctx)) 842 goto process_auth_encryption_gcm_err; 843 844 /* Workaround open ssl bug in version less then 1.0.1f */ 845 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 846 goto process_auth_encryption_gcm_err; 847 848 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 849 goto process_auth_encryption_gcm_err; 850 851 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 852 goto process_auth_encryption_gcm_err; 853 854 return 0; 855 856 process_auth_encryption_gcm_err: 857 OPENSSL_LOG_ERR("Process openssl auth encryption gcm failed"); 858 return -EINVAL; 859 } 860 861 static int 862 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 863 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, 864 uint8_t *key, uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx, 865 const EVP_CIPHER *algo) 866 { 867 int len = 0, unused = 0; 868 uint8_t empty[] = {}; 869 870 if (EVP_DecryptInit_ex(ctx, algo, NULL, NULL, NULL) <= 0) 871 goto process_auth_decryption_gcm_err; 872 873 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL) <= 0) 874 goto process_auth_decryption_gcm_err; 875 876 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 877 goto process_auth_decryption_gcm_err; 878 879 if (EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv) <= 0) 880 goto process_auth_decryption_gcm_err; 881 882 if (aadlen > 0) 883 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 884 goto process_auth_decryption_gcm_err; 885 886 if (srclen > 0) 887 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 888 srclen, ctx)) 889 goto process_auth_decryption_gcm_err; 890 891 /* Workaround open ssl bug in version less then 1.0.1f */ 892 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 893 goto process_auth_decryption_gcm_err; 894 895 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 896 goto process_auth_decryption_gcm_final_err; 897 898 return 0; 899 900 process_auth_decryption_gcm_err: 901 OPENSSL_LOG_ERR("Process openssl auth description gcm failed"); 902 return -EINVAL; 903 904 process_auth_decryption_gcm_final_err: 905 return -EFAULT; 906 } 907 908 /** Process standard openssl auth algorithms */ 909 static int 910 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 911 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 912 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 913 { 914 size_t dstlen; 915 struct rte_mbuf *m; 916 int l, n = srclen; 917 uint8_t *src; 918 919 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 920 m = m->next) 921 offset -= rte_pktmbuf_data_len(m); 922 923 if (m == 0) 924 goto process_auth_err; 925 926 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 927 goto process_auth_err; 928 929 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 930 931 l = rte_pktmbuf_data_len(m) - offset; 932 if (srclen <= l) { 933 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 934 goto process_auth_err; 935 goto process_auth_final; 936 } 937 938 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 939 goto process_auth_err; 940 941 n -= l; 942 943 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 944 src = rte_pktmbuf_mtod(m, uint8_t *); 945 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 946 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 947 goto process_auth_err; 948 n -= l; 949 } 950 951 process_auth_final: 952 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 953 goto process_auth_err; 954 return 0; 955 956 process_auth_err: 957 OPENSSL_LOG_ERR("Process openssl auth failed"); 958 return -EINVAL; 959 } 960 961 /** Process standard openssl auth algorithms with hmac */ 962 static int 963 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 964 __rte_unused uint8_t *iv, EVP_PKEY *pkey, 965 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 966 { 967 size_t dstlen; 968 struct rte_mbuf *m; 969 int l, n = srclen; 970 uint8_t *src; 971 972 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 973 m = m->next) 974 offset -= rte_pktmbuf_data_len(m); 975 976 if (m == 0) 977 goto process_auth_err; 978 979 if (EVP_DigestSignInit(ctx, NULL, algo, NULL, pkey) <= 0) 980 goto process_auth_err; 981 982 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 983 984 l = rte_pktmbuf_data_len(m) - offset; 985 if (srclen <= l) { 986 if (EVP_DigestSignUpdate(ctx, (char *)src, srclen) <= 0) 987 goto process_auth_err; 988 goto process_auth_final; 989 } 990 991 if (EVP_DigestSignUpdate(ctx, (char *)src, l) <= 0) 992 goto process_auth_err; 993 994 n -= l; 995 996 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 997 src = rte_pktmbuf_mtod(m, uint8_t *); 998 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 999 if (EVP_DigestSignUpdate(ctx, (char *)src, l) <= 0) 1000 goto process_auth_err; 1001 n -= l; 1002 } 1003 1004 process_auth_final: 1005 if (EVP_DigestSignFinal(ctx, dst, &dstlen) <= 0) 1006 goto process_auth_err; 1007 1008 return 0; 1009 1010 process_auth_err: 1011 OPENSSL_LOG_ERR("Process openssl auth failed"); 1012 return -EINVAL; 1013 } 1014 1015 /*----------------------------------------------------------------------------*/ 1016 1017 /** Process auth/cipher combined operation */ 1018 static void 1019 process_openssl_combined_op 1020 (struct rte_crypto_op *op, struct openssl_session *sess, 1021 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1022 { 1023 /* cipher */ 1024 uint8_t *dst = NULL, *iv, *tag, *aad; 1025 int srclen, ivlen, aadlen, status = -1; 1026 uint32_t offset; 1027 1028 /* 1029 * Segmented destination buffer is not supported for 1030 * encryption/decryption 1031 */ 1032 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1033 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1034 return; 1035 } 1036 1037 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1038 sess->iv.offset); 1039 ivlen = sess->iv.length; 1040 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1041 srclen = 0; 1042 offset = op->sym->auth.data.offset; 1043 aadlen = op->sym->auth.data.length; 1044 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1045 op->sym->auth.data.offset); 1046 tag = op->sym->auth.digest.data; 1047 if (tag == NULL) 1048 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1049 offset + aadlen); 1050 } else { 1051 srclen = op->sym->aead.data.length; 1052 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1053 op->sym->aead.data.offset); 1054 offset = op->sym->aead.data.offset; 1055 aad = op->sym->aead.aad.data; 1056 aadlen = sess->auth.aad_length; 1057 tag = op->sym->aead.digest.data; 1058 if (tag == NULL) 1059 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1060 offset + srclen); 1061 } 1062 1063 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1064 status = process_openssl_auth_encryption_gcm( 1065 mbuf_src, offset, srclen, 1066 aad, aadlen, iv, ivlen, sess->cipher.key.data, 1067 dst, tag, sess->cipher.ctx, 1068 sess->cipher.evp_algo); 1069 else 1070 status = process_openssl_auth_decryption_gcm( 1071 mbuf_src, offset, srclen, 1072 aad, aadlen, iv, ivlen, sess->cipher.key.data, 1073 dst, tag, sess->cipher.ctx, 1074 sess->cipher.evp_algo); 1075 1076 if (status != 0) { 1077 if (status == (-EFAULT) && 1078 sess->auth.operation == 1079 RTE_CRYPTO_AUTH_OP_VERIFY) 1080 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1081 else 1082 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1083 } 1084 } 1085 1086 /** Process cipher operation */ 1087 static void 1088 process_openssl_cipher_op 1089 (struct rte_crypto_op *op, struct openssl_session *sess, 1090 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1091 { 1092 uint8_t *dst, *iv; 1093 int srclen, status; 1094 1095 /* 1096 * Segmented destination buffer is not supported for 1097 * encryption/decryption 1098 */ 1099 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1100 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1101 return; 1102 } 1103 1104 srclen = op->sym->cipher.data.length; 1105 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1106 op->sym->cipher.data.offset); 1107 1108 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1109 sess->iv.offset); 1110 1111 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1112 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1113 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1114 op->sym->cipher.data.offset, iv, 1115 sess->cipher.key.data, srclen, 1116 sess->cipher.ctx, 1117 sess->cipher.evp_algo); 1118 else 1119 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1120 op->sym->cipher.data.offset, iv, 1121 sess->cipher.key.data, srclen, 1122 sess->cipher.ctx, 1123 sess->cipher.evp_algo); 1124 else 1125 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1126 op->sym->cipher.data.offset, iv, 1127 sess->cipher.key.data, srclen, 1128 sess->cipher.ctx); 1129 1130 if (status != 0) 1131 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1132 } 1133 1134 /** Process cipher operation */ 1135 static void 1136 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1137 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1138 struct rte_mbuf *mbuf_dst) 1139 { 1140 uint8_t *src, *dst, *iv; 1141 uint8_t block_size, last_block_len; 1142 int srclen, status = 0; 1143 1144 srclen = op->sym->cipher.data.length; 1145 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1146 op->sym->cipher.data.offset); 1147 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1148 op->sym->cipher.data.offset); 1149 1150 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1151 sess->iv.offset); 1152 1153 block_size = DES_BLOCK_SIZE; 1154 1155 last_block_len = srclen % block_size; 1156 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1157 /* Encrypt only with ECB mode XOR IV */ 1158 if (srclen < block_size) { 1159 status = process_openssl_cipher_bpi_encrypt(src, dst, 1160 iv, srclen, 1161 sess->cipher.bpi_ctx); 1162 } else { 1163 srclen -= last_block_len; 1164 /* Encrypt with the block aligned stream with CBC mode */ 1165 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1166 op->sym->cipher.data.offset, iv, 1167 sess->cipher.key.data, srclen, 1168 sess->cipher.ctx, sess->cipher.evp_algo); 1169 if (last_block_len) { 1170 /* Point at last block */ 1171 dst += srclen; 1172 /* 1173 * IV is the last encrypted block from 1174 * the previous operation 1175 */ 1176 iv = dst - block_size; 1177 src += srclen; 1178 srclen = last_block_len; 1179 /* Encrypt the last frame with ECB mode */ 1180 status |= process_openssl_cipher_bpi_encrypt(src, 1181 dst, iv, 1182 srclen, sess->cipher.bpi_ctx); 1183 } 1184 } 1185 } else { 1186 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1187 if (srclen < block_size) { 1188 status = process_openssl_cipher_bpi_encrypt(src, dst, 1189 iv, 1190 srclen, 1191 sess->cipher.bpi_ctx); 1192 } else { 1193 if (last_block_len) { 1194 /* Point at last block */ 1195 dst += srclen - last_block_len; 1196 src += srclen - last_block_len; 1197 /* 1198 * IV is the last full block 1199 */ 1200 iv = src - block_size; 1201 /* 1202 * Decrypt the last frame with ECB mode 1203 * (encrypt, as it is the same operation) 1204 */ 1205 status = process_openssl_cipher_bpi_encrypt(src, 1206 dst, iv, 1207 last_block_len, sess->cipher.bpi_ctx); 1208 /* Prepare parameters for CBC mode op */ 1209 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1210 sess->iv.offset); 1211 dst += last_block_len - srclen; 1212 srclen -= last_block_len; 1213 } 1214 1215 /* Decrypt with CBC mode */ 1216 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1217 op->sym->cipher.data.offset, iv, 1218 sess->cipher.key.data, srclen, 1219 sess->cipher.ctx, 1220 sess->cipher.evp_algo); 1221 } 1222 } 1223 1224 if (status != 0) 1225 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1226 } 1227 1228 /** Process auth operation */ 1229 static void 1230 process_openssl_auth_op 1231 (struct rte_crypto_op *op, struct openssl_session *sess, 1232 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1233 { 1234 uint8_t *dst; 1235 int srclen, status; 1236 1237 srclen = op->sym->auth.data.length; 1238 1239 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) 1240 dst = (uint8_t *)rte_pktmbuf_append(mbuf_src, 1241 sess->auth.digest_length); 1242 else { 1243 dst = op->sym->auth.digest.data; 1244 if (dst == NULL) 1245 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1246 op->sym->auth.data.offset + 1247 op->sym->auth.data.length); 1248 } 1249 1250 switch (sess->auth.mode) { 1251 case OPENSSL_AUTH_AS_AUTH: 1252 status = process_openssl_auth(mbuf_src, dst, 1253 op->sym->auth.data.offset, NULL, NULL, srclen, 1254 sess->auth.auth.ctx, sess->auth.auth.evp_algo); 1255 break; 1256 case OPENSSL_AUTH_AS_HMAC: 1257 status = process_openssl_auth_hmac(mbuf_src, dst, 1258 op->sym->auth.data.offset, NULL, 1259 sess->auth.hmac.pkey, srclen, 1260 sess->auth.hmac.ctx, sess->auth.hmac.evp_algo); 1261 break; 1262 default: 1263 status = -1; 1264 break; 1265 } 1266 1267 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1268 if (memcmp(dst, op->sym->auth.digest.data, 1269 sess->auth.digest_length) != 0) { 1270 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1271 } 1272 /* Trim area used for digest from mbuf. */ 1273 rte_pktmbuf_trim(mbuf_src, sess->auth.digest_length); 1274 } 1275 1276 if (status != 0) 1277 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1278 } 1279 1280 /** Process crypto operation for mbuf */ 1281 static int 1282 process_op(const struct openssl_qp *qp, struct rte_crypto_op *op, 1283 struct openssl_session *sess) 1284 { 1285 struct rte_mbuf *msrc, *mdst; 1286 int retval; 1287 1288 msrc = op->sym->m_src; 1289 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 1290 1291 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1292 1293 switch (sess->chain_order) { 1294 case OPENSSL_CHAIN_ONLY_CIPHER: 1295 process_openssl_cipher_op(op, sess, msrc, mdst); 1296 break; 1297 case OPENSSL_CHAIN_ONLY_AUTH: 1298 process_openssl_auth_op(op, sess, msrc, mdst); 1299 break; 1300 case OPENSSL_CHAIN_CIPHER_AUTH: 1301 process_openssl_cipher_op(op, sess, msrc, mdst); 1302 process_openssl_auth_op(op, sess, mdst, mdst); 1303 break; 1304 case OPENSSL_CHAIN_AUTH_CIPHER: 1305 process_openssl_auth_op(op, sess, msrc, mdst); 1306 process_openssl_cipher_op(op, sess, msrc, mdst); 1307 break; 1308 case OPENSSL_CHAIN_COMBINED: 1309 process_openssl_combined_op(op, sess, msrc, mdst); 1310 break; 1311 case OPENSSL_CHAIN_CIPHER_BPI: 1312 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 1313 break; 1314 default: 1315 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1316 break; 1317 } 1318 1319 /* Free session if a session-less crypto op */ 1320 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 1321 openssl_reset_session(sess); 1322 memset(sess, 0, sizeof(struct openssl_session)); 1323 rte_mempool_put(qp->sess_mp, op->sym->session); 1324 op->sym->session = NULL; 1325 } 1326 1327 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 1328 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1329 1330 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 1331 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 1332 else 1333 retval = -1; 1334 1335 return retval; 1336 } 1337 1338 /* 1339 *------------------------------------------------------------------------------ 1340 * PMD Framework 1341 *------------------------------------------------------------------------------ 1342 */ 1343 1344 /** Enqueue burst */ 1345 static uint16_t 1346 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 1347 uint16_t nb_ops) 1348 { 1349 struct openssl_session *sess; 1350 struct openssl_qp *qp = queue_pair; 1351 int i, retval; 1352 1353 for (i = 0; i < nb_ops; i++) { 1354 sess = get_session(qp, ops[i]); 1355 if (unlikely(sess == NULL)) 1356 goto enqueue_err; 1357 1358 retval = process_op(qp, ops[i], sess); 1359 if (unlikely(retval < 0)) 1360 goto enqueue_err; 1361 } 1362 1363 qp->stats.enqueued_count += i; 1364 return i; 1365 1366 enqueue_err: 1367 qp->stats.enqueue_err_count++; 1368 return i; 1369 } 1370 1371 /** Dequeue burst */ 1372 static uint16_t 1373 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 1374 uint16_t nb_ops) 1375 { 1376 struct openssl_qp *qp = queue_pair; 1377 1378 unsigned int nb_dequeued = 0; 1379 1380 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 1381 (void **)ops, nb_ops, NULL); 1382 qp->stats.dequeued_count += nb_dequeued; 1383 1384 return nb_dequeued; 1385 } 1386 1387 /** Create OPENSSL crypto device */ 1388 static int 1389 cryptodev_openssl_create(const char *name, 1390 struct rte_vdev_device *vdev, 1391 struct rte_crypto_vdev_init_params *init_params) 1392 { 1393 struct rte_cryptodev *dev; 1394 struct openssl_private *internals; 1395 1396 if (init_params->name[0] == '\0') 1397 snprintf(init_params->name, sizeof(init_params->name), 1398 "%s", name); 1399 1400 dev = rte_cryptodev_vdev_pmd_init(init_params->name, 1401 sizeof(struct openssl_private), 1402 init_params->socket_id, 1403 vdev); 1404 if (dev == NULL) { 1405 OPENSSL_LOG_ERR("failed to create cryptodev vdev"); 1406 goto init_error; 1407 } 1408 1409 dev->driver_id = cryptodev_driver_id; 1410 dev->dev_ops = rte_openssl_pmd_ops; 1411 1412 /* register rx/tx burst functions for data path */ 1413 dev->dequeue_burst = openssl_pmd_dequeue_burst; 1414 dev->enqueue_burst = openssl_pmd_enqueue_burst; 1415 1416 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 1417 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 1418 RTE_CRYPTODEV_FF_CPU_AESNI | 1419 RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER; 1420 1421 /* Set vector instructions mode supported */ 1422 internals = dev->data->dev_private; 1423 1424 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 1425 internals->max_nb_sessions = init_params->max_nb_sessions; 1426 1427 return 0; 1428 1429 init_error: 1430 OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed", 1431 init_params->name); 1432 1433 cryptodev_openssl_remove(vdev); 1434 return -EFAULT; 1435 } 1436 1437 /** Initialise OPENSSL crypto device */ 1438 static int 1439 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 1440 { 1441 struct rte_crypto_vdev_init_params init_params = { 1442 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, 1443 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, 1444 rte_socket_id(), 1445 {0} 1446 }; 1447 const char *name; 1448 const char *input_args; 1449 1450 name = rte_vdev_device_name(vdev); 1451 if (name == NULL) 1452 return -EINVAL; 1453 input_args = rte_vdev_device_args(vdev); 1454 1455 rte_cryptodev_vdev_parse_init_params(&init_params, input_args); 1456 1457 RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, 1458 init_params.socket_id); 1459 if (init_params.name[0] != '\0') 1460 RTE_LOG(INFO, PMD, " User defined name = %s\n", 1461 init_params.name); 1462 RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", 1463 init_params.max_nb_queue_pairs); 1464 RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", 1465 init_params.max_nb_sessions); 1466 1467 return cryptodev_openssl_create(name, vdev, &init_params); 1468 } 1469 1470 /** Uninitialise OPENSSL crypto device */ 1471 static int 1472 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 1473 { 1474 const char *name; 1475 1476 name = rte_vdev_device_name(vdev); 1477 if (name == NULL) 1478 return -EINVAL; 1479 1480 RTE_LOG(INFO, PMD, 1481 "Closing OPENSSL crypto device %s on numa socket %u\n", 1482 name, rte_socket_id()); 1483 1484 return 0; 1485 } 1486 1487 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 1488 .probe = cryptodev_openssl_probe, 1489 .remove = cryptodev_openssl_remove 1490 }; 1491 1492 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 1493 cryptodev_openssl_pmd_drv); 1494 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 1495 "max_nb_queue_pairs=<int> " 1496 "max_nb_sessions=<int> " 1497 "socket_id=<int>"); 1498 RTE_PMD_REGISTER_CRYPTO_DRIVER(cryptodev_openssl_pmd_drv, cryptodev_driver_id); 1499