1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Intel Corporation nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <rte_common.h> 34 #include <rte_hexdump.h> 35 #include <rte_cryptodev.h> 36 #include <rte_cryptodev_pmd.h> 37 #include <rte_cryptodev_vdev.h> 38 #include <rte_vdev.h> 39 #include <rte_malloc.h> 40 #include <rte_cpuflags.h> 41 42 #include <openssl/evp.h> 43 44 #include "rte_openssl_pmd_private.h" 45 46 #define DES_BLOCK_SIZE 8 47 48 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 49 50 /*----------------------------------------------------------------------------*/ 51 52 /** 53 * Increment counter by 1 54 * Counter is 64 bit array, big-endian 55 */ 56 static void 57 ctr_inc(uint8_t *ctr) 58 { 59 uint64_t *ctr64 = (uint64_t *)ctr; 60 61 *ctr64 = __builtin_bswap64(*ctr64); 62 (*ctr64)++; 63 *ctr64 = __builtin_bswap64(*ctr64); 64 } 65 66 /* 67 *------------------------------------------------------------------------------ 68 * Session Prepare 69 *------------------------------------------------------------------------------ 70 */ 71 72 /** Get xform chain order */ 73 static enum openssl_chain_order 74 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 75 { 76 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 77 78 if (xform != NULL) { 79 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 80 if (xform->next == NULL) 81 res = OPENSSL_CHAIN_ONLY_AUTH; 82 else if (xform->next->type == 83 RTE_CRYPTO_SYM_XFORM_CIPHER) 84 res = OPENSSL_CHAIN_AUTH_CIPHER; 85 } 86 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 87 if (xform->next == NULL) 88 res = OPENSSL_CHAIN_ONLY_CIPHER; 89 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 90 res = OPENSSL_CHAIN_CIPHER_AUTH; 91 } 92 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 93 res = OPENSSL_CHAIN_COMBINED; 94 } 95 96 return res; 97 } 98 99 /** Get session cipher key from input cipher key */ 100 static void 101 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key) 102 { 103 memcpy(session_key, input_key, keylen); 104 } 105 106 /** Get key ede 24 bytes standard from input key */ 107 static int 108 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede) 109 { 110 int res = 0; 111 112 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 113 switch (keylen) { 114 case 24: 115 memcpy(key_ede, key, 24); 116 break; 117 case 16: 118 /* K3 = K1 */ 119 memcpy(key_ede, key, 16); 120 memcpy(key_ede + 16, key, 8); 121 break; 122 case 8: 123 /* K1 = K2 = K3 (DES compatibility) */ 124 memcpy(key_ede, key, 8); 125 memcpy(key_ede + 8, key, 8); 126 memcpy(key_ede + 16, key, 8); 127 break; 128 default: 129 OPENSSL_LOG_ERR("Unsupported key size"); 130 res = -EINVAL; 131 } 132 133 return res; 134 } 135 136 /** Get adequate openssl function for input cipher algorithm */ 137 static uint8_t 138 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 139 const EVP_CIPHER **algo) 140 { 141 int res = 0; 142 143 if (algo != NULL) { 144 switch (sess_algo) { 145 case RTE_CRYPTO_CIPHER_3DES_CBC: 146 switch (keylen) { 147 case 16: 148 *algo = EVP_des_ede_cbc(); 149 break; 150 case 24: 151 *algo = EVP_des_ede3_cbc(); 152 break; 153 default: 154 res = -EINVAL; 155 } 156 break; 157 case RTE_CRYPTO_CIPHER_3DES_CTR: 158 break; 159 case RTE_CRYPTO_CIPHER_AES_CBC: 160 switch (keylen) { 161 case 16: 162 *algo = EVP_aes_128_cbc(); 163 break; 164 case 24: 165 *algo = EVP_aes_192_cbc(); 166 break; 167 case 32: 168 *algo = EVP_aes_256_cbc(); 169 break; 170 default: 171 res = -EINVAL; 172 } 173 break; 174 case RTE_CRYPTO_CIPHER_AES_CTR: 175 switch (keylen) { 176 case 16: 177 *algo = EVP_aes_128_ctr(); 178 break; 179 case 24: 180 *algo = EVP_aes_192_ctr(); 181 break; 182 case 32: 183 *algo = EVP_aes_256_ctr(); 184 break; 185 default: 186 res = -EINVAL; 187 } 188 break; 189 default: 190 res = -EINVAL; 191 break; 192 } 193 } else { 194 res = -EINVAL; 195 } 196 197 return res; 198 } 199 200 /** Get adequate openssl function for input auth algorithm */ 201 static uint8_t 202 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 203 const EVP_MD **algo) 204 { 205 int res = 0; 206 207 if (algo != NULL) { 208 switch (sessalgo) { 209 case RTE_CRYPTO_AUTH_MD5: 210 case RTE_CRYPTO_AUTH_MD5_HMAC: 211 *algo = EVP_md5(); 212 break; 213 case RTE_CRYPTO_AUTH_SHA1: 214 case RTE_CRYPTO_AUTH_SHA1_HMAC: 215 *algo = EVP_sha1(); 216 break; 217 case RTE_CRYPTO_AUTH_SHA224: 218 case RTE_CRYPTO_AUTH_SHA224_HMAC: 219 *algo = EVP_sha224(); 220 break; 221 case RTE_CRYPTO_AUTH_SHA256: 222 case RTE_CRYPTO_AUTH_SHA256_HMAC: 223 *algo = EVP_sha256(); 224 break; 225 case RTE_CRYPTO_AUTH_SHA384: 226 case RTE_CRYPTO_AUTH_SHA384_HMAC: 227 *algo = EVP_sha384(); 228 break; 229 case RTE_CRYPTO_AUTH_SHA512: 230 case RTE_CRYPTO_AUTH_SHA512_HMAC: 231 *algo = EVP_sha512(); 232 break; 233 default: 234 res = -EINVAL; 235 break; 236 } 237 } else { 238 res = -EINVAL; 239 } 240 241 return res; 242 } 243 244 /** Get adequate openssl function for input cipher algorithm */ 245 static uint8_t 246 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 247 const EVP_CIPHER **algo) 248 { 249 int res = 0; 250 251 if (algo != NULL) { 252 switch (sess_algo) { 253 case RTE_CRYPTO_AEAD_AES_GCM: 254 switch (keylen) { 255 case 16: 256 *algo = EVP_aes_128_gcm(); 257 break; 258 case 24: 259 *algo = EVP_aes_192_gcm(); 260 break; 261 case 32: 262 *algo = EVP_aes_256_gcm(); 263 break; 264 default: 265 res = -EINVAL; 266 } 267 break; 268 default: 269 res = -EINVAL; 270 break; 271 } 272 } else { 273 res = -EINVAL; 274 } 275 276 return res; 277 } 278 279 /** Set session cipher parameters */ 280 static int 281 openssl_set_session_cipher_parameters(struct openssl_session *sess, 282 const struct rte_crypto_sym_xform *xform) 283 { 284 /* Select cipher direction */ 285 sess->cipher.direction = xform->cipher.op; 286 /* Select cipher key */ 287 sess->cipher.key.length = xform->cipher.key.length; 288 289 /* Set IV parameters */ 290 sess->iv.offset = xform->cipher.iv.offset; 291 sess->iv.length = xform->cipher.iv.length; 292 293 /* Select cipher algo */ 294 switch (xform->cipher.algo) { 295 case RTE_CRYPTO_CIPHER_3DES_CBC: 296 case RTE_CRYPTO_CIPHER_AES_CBC: 297 case RTE_CRYPTO_CIPHER_AES_CTR: 298 sess->cipher.mode = OPENSSL_CIPHER_LIB; 299 sess->cipher.algo = xform->cipher.algo; 300 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 301 302 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 303 &sess->cipher.evp_algo) != 0) 304 return -EINVAL; 305 306 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 307 sess->cipher.key.data); 308 309 break; 310 311 case RTE_CRYPTO_CIPHER_3DES_CTR: 312 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 313 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 314 315 if (get_cipher_key_ede(xform->cipher.key.data, 316 sess->cipher.key.length, 317 sess->cipher.key.data) != 0) 318 return -EINVAL; 319 break; 320 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 321 sess->cipher.algo = xform->cipher.algo; 322 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 323 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 324 sess->cipher.evp_algo = EVP_des_cbc(); 325 326 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 327 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 328 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 329 NULL, xform->cipher.key.data, 0) != 1) 330 return -EINVAL; 331 332 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 333 sess->cipher.key.data); 334 break; 335 default: 336 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 337 return -EINVAL; 338 } 339 340 return 0; 341 } 342 343 /* Set session auth parameters */ 344 static int 345 openssl_set_session_auth_parameters(struct openssl_session *sess, 346 const struct rte_crypto_sym_xform *xform) 347 { 348 /* Select auth generate/verify */ 349 sess->auth.operation = xform->auth.op; 350 sess->auth.algo = xform->auth.algo; 351 352 /* Select auth algo */ 353 switch (xform->auth.algo) { 354 case RTE_CRYPTO_AUTH_AES_GMAC: 355 sess->chain_order = OPENSSL_CHAIN_COMBINED; 356 357 /* Set IV parameters */ 358 sess->iv.offset = xform->auth.iv.offset; 359 sess->iv.length = xform->auth.iv.length; 360 361 /* 362 * OpenSSL requires GMAC to be a GCM operation 363 * with no cipher data length 364 */ 365 sess->cipher.mode = OPENSSL_CIPHER_LIB; 366 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 367 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 368 else 369 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 370 371 sess->cipher.key.length = xform->auth.key.length; 372 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 373 374 if (get_aead_algo(RTE_CRYPTO_AEAD_AES_GCM, 375 sess->cipher.key.length, 376 &sess->cipher.evp_algo) != 0) 377 return -EINVAL; 378 379 get_cipher_key(xform->auth.key.data, xform->auth.key.length, 380 sess->cipher.key.data); 381 382 break; 383 384 case RTE_CRYPTO_AUTH_MD5: 385 case RTE_CRYPTO_AUTH_SHA1: 386 case RTE_CRYPTO_AUTH_SHA224: 387 case RTE_CRYPTO_AUTH_SHA256: 388 case RTE_CRYPTO_AUTH_SHA384: 389 case RTE_CRYPTO_AUTH_SHA512: 390 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 391 if (get_auth_algo(xform->auth.algo, 392 &sess->auth.auth.evp_algo) != 0) 393 return -EINVAL; 394 sess->auth.auth.ctx = EVP_MD_CTX_create(); 395 break; 396 397 case RTE_CRYPTO_AUTH_MD5_HMAC: 398 case RTE_CRYPTO_AUTH_SHA1_HMAC: 399 case RTE_CRYPTO_AUTH_SHA224_HMAC: 400 case RTE_CRYPTO_AUTH_SHA256_HMAC: 401 case RTE_CRYPTO_AUTH_SHA384_HMAC: 402 case RTE_CRYPTO_AUTH_SHA512_HMAC: 403 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 404 sess->auth.hmac.ctx = EVP_MD_CTX_create(); 405 if (get_auth_algo(xform->auth.algo, 406 &sess->auth.hmac.evp_algo) != 0) 407 return -EINVAL; 408 sess->auth.hmac.pkey = EVP_PKEY_new_mac_key(EVP_PKEY_HMAC, NULL, 409 xform->auth.key.data, xform->auth.key.length); 410 break; 411 412 default: 413 return -EINVAL; 414 } 415 416 sess->auth.aad_length = xform->auth.add_auth_data_length; 417 sess->auth.digest_length = xform->auth.digest_length; 418 419 return 0; 420 } 421 422 /* Set session AEAD parameters */ 423 static int 424 openssl_set_session_aead_parameters(struct openssl_session *sess, 425 const struct rte_crypto_sym_xform *xform) 426 { 427 /* Select cipher direction */ 428 sess->cipher.direction = xform->cipher.op; 429 /* Select cipher key */ 430 sess->cipher.key.length = xform->aead.key.length; 431 432 /* Set IV parameters */ 433 sess->iv.offset = xform->aead.iv.offset; 434 sess->iv.length = xform->aead.iv.length; 435 436 /* Select auth generate/verify */ 437 sess->auth.operation = xform->auth.op; 438 sess->auth.algo = xform->auth.algo; 439 440 /* Select auth algo */ 441 switch (xform->aead.algo) { 442 case RTE_CRYPTO_AEAD_AES_GCM: 443 sess->cipher.mode = OPENSSL_CIPHER_LIB; 444 sess->aead_algo = xform->aead.algo; 445 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 446 447 if (get_aead_algo(sess->aead_algo, sess->cipher.key.length, 448 &sess->cipher.evp_algo) != 0) 449 return -EINVAL; 450 451 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 452 sess->cipher.key.data); 453 454 sess->chain_order = OPENSSL_CHAIN_COMBINED; 455 break; 456 default: 457 return -EINVAL; 458 } 459 460 sess->auth.aad_length = xform->aead.add_auth_data_length; 461 sess->auth.digest_length = xform->aead.digest_length; 462 463 return 0; 464 } 465 466 /** Parse crypto xform chain and set private session parameters */ 467 int 468 openssl_set_session_parameters(struct openssl_session *sess, 469 const struct rte_crypto_sym_xform *xform) 470 { 471 const struct rte_crypto_sym_xform *cipher_xform = NULL; 472 const struct rte_crypto_sym_xform *auth_xform = NULL; 473 const struct rte_crypto_sym_xform *aead_xform = NULL; 474 475 sess->chain_order = openssl_get_chain_order(xform); 476 switch (sess->chain_order) { 477 case OPENSSL_CHAIN_ONLY_CIPHER: 478 cipher_xform = xform; 479 break; 480 case OPENSSL_CHAIN_ONLY_AUTH: 481 auth_xform = xform; 482 break; 483 case OPENSSL_CHAIN_CIPHER_AUTH: 484 cipher_xform = xform; 485 auth_xform = xform->next; 486 break; 487 case OPENSSL_CHAIN_AUTH_CIPHER: 488 auth_xform = xform; 489 cipher_xform = xform->next; 490 break; 491 case OPENSSL_CHAIN_COMBINED: 492 aead_xform = xform; 493 break; 494 default: 495 return -EINVAL; 496 } 497 498 /* Default IV length = 0 */ 499 sess->iv.length = 0; 500 501 /* cipher_xform must be check before auth_xform */ 502 if (cipher_xform) { 503 if (openssl_set_session_cipher_parameters( 504 sess, cipher_xform)) { 505 OPENSSL_LOG_ERR( 506 "Invalid/unsupported cipher parameters"); 507 return -EINVAL; 508 } 509 } 510 511 if (auth_xform) { 512 if (openssl_set_session_auth_parameters(sess, auth_xform)) { 513 OPENSSL_LOG_ERR( 514 "Invalid/unsupported auth parameters"); 515 return -EINVAL; 516 } 517 } 518 519 if (aead_xform) { 520 if (openssl_set_session_aead_parameters(sess, aead_xform)) { 521 OPENSSL_LOG_ERR( 522 "Invalid/unsupported auth parameters"); 523 return -EINVAL; 524 } 525 } 526 527 return 0; 528 } 529 530 /** Reset private session parameters */ 531 void 532 openssl_reset_session(struct openssl_session *sess) 533 { 534 EVP_CIPHER_CTX_free(sess->cipher.ctx); 535 536 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 537 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 538 539 switch (sess->auth.mode) { 540 case OPENSSL_AUTH_AS_AUTH: 541 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 542 break; 543 case OPENSSL_AUTH_AS_HMAC: 544 EVP_PKEY_free(sess->auth.hmac.pkey); 545 EVP_MD_CTX_destroy(sess->auth.hmac.ctx); 546 break; 547 default: 548 break; 549 } 550 } 551 552 /** Provide session for operation */ 553 static struct openssl_session * 554 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 555 { 556 struct openssl_session *sess = NULL; 557 558 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 559 /* get existing session */ 560 if (likely(op->sym->session != NULL && 561 op->sym->session->dev_type == 562 RTE_CRYPTODEV_OPENSSL_PMD)) 563 sess = (struct openssl_session *) 564 op->sym->session->_private; 565 } else { 566 /* provide internal session */ 567 void *_sess = NULL; 568 569 if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) { 570 sess = (struct openssl_session *) 571 ((struct rte_cryptodev_sym_session *)_sess) 572 ->_private; 573 574 if (unlikely(openssl_set_session_parameters( 575 sess, op->sym->xform) != 0)) { 576 rte_mempool_put(qp->sess_mp, _sess); 577 sess = NULL; 578 } else 579 op->sym->session = _sess; 580 } 581 } 582 583 if (sess == NULL) 584 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 585 586 return sess; 587 } 588 589 /* 590 *------------------------------------------------------------------------------ 591 * Process Operations 592 *------------------------------------------------------------------------------ 593 */ 594 static inline int 595 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 596 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 597 { 598 struct rte_mbuf *m; 599 int dstlen; 600 int l, n = srclen; 601 uint8_t *src; 602 603 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 604 m = m->next) 605 offset -= rte_pktmbuf_data_len(m); 606 607 if (m == 0) 608 return -1; 609 610 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 611 612 l = rte_pktmbuf_data_len(m) - offset; 613 if (srclen <= l) { 614 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 615 return -1; 616 *dst += l; 617 return 0; 618 } 619 620 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 621 return -1; 622 623 *dst += dstlen; 624 n -= l; 625 626 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 627 src = rte_pktmbuf_mtod(m, uint8_t *); 628 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 629 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 630 return -1; 631 *dst += dstlen; 632 n -= l; 633 } 634 635 return 0; 636 } 637 638 static inline int 639 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 640 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 641 { 642 struct rte_mbuf *m; 643 int dstlen; 644 int l, n = srclen; 645 uint8_t *src; 646 647 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 648 m = m->next) 649 offset -= rte_pktmbuf_data_len(m); 650 651 if (m == 0) 652 return -1; 653 654 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 655 656 l = rte_pktmbuf_data_len(m) - offset; 657 if (srclen <= l) { 658 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 659 return -1; 660 *dst += l; 661 return 0; 662 } 663 664 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 665 return -1; 666 667 *dst += dstlen; 668 n -= l; 669 670 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 671 src = rte_pktmbuf_mtod(m, uint8_t *); 672 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 673 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 674 return -1; 675 *dst += dstlen; 676 n -= l; 677 } 678 679 return 0; 680 } 681 682 /** Process standard openssl cipher encryption */ 683 static int 684 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 685 int offset, uint8_t *iv, uint8_t *key, int srclen, 686 EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) 687 { 688 int totlen; 689 690 if (EVP_EncryptInit_ex(ctx, algo, NULL, key, iv) <= 0) 691 goto process_cipher_encrypt_err; 692 693 EVP_CIPHER_CTX_set_padding(ctx, 0); 694 695 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 696 srclen, ctx)) 697 goto process_cipher_encrypt_err; 698 699 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 700 goto process_cipher_encrypt_err; 701 702 return 0; 703 704 process_cipher_encrypt_err: 705 OPENSSL_LOG_ERR("Process openssl cipher encrypt failed"); 706 return -EINVAL; 707 } 708 709 /** Process standard openssl cipher encryption */ 710 static int 711 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 712 uint8_t *iv, int srclen, 713 EVP_CIPHER_CTX *ctx) 714 { 715 uint8_t i; 716 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 717 int encrypted_ivlen; 718 719 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 720 iv, DES_BLOCK_SIZE) <= 0) 721 goto process_cipher_encrypt_err; 722 723 for (i = 0; i < srclen; i++) 724 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 725 726 return 0; 727 728 process_cipher_encrypt_err: 729 OPENSSL_LOG_ERR("Process openssl cipher bpi encrypt failed"); 730 return -EINVAL; 731 } 732 /** Process standard openssl cipher decryption */ 733 static int 734 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 735 int offset, uint8_t *iv, uint8_t *key, int srclen, 736 EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) 737 { 738 int totlen; 739 740 if (EVP_DecryptInit_ex(ctx, algo, NULL, key, iv) <= 0) 741 goto process_cipher_decrypt_err; 742 743 EVP_CIPHER_CTX_set_padding(ctx, 0); 744 745 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 746 srclen, ctx)) 747 goto process_cipher_decrypt_err; 748 749 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 750 goto process_cipher_decrypt_err; 751 return 0; 752 753 process_cipher_decrypt_err: 754 OPENSSL_LOG_ERR("Process openssl cipher decrypt failed"); 755 return -EINVAL; 756 } 757 758 /** Process cipher des 3 ctr encryption, decryption algorithm */ 759 static int 760 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 761 int offset, uint8_t *iv, uint8_t *key, int srclen, 762 EVP_CIPHER_CTX *ctx) 763 { 764 uint8_t ebuf[8], ctr[8]; 765 int unused, n; 766 struct rte_mbuf *m; 767 uint8_t *src; 768 int l; 769 770 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 771 m = m->next) 772 offset -= rte_pktmbuf_data_len(m); 773 774 if (m == 0) 775 goto process_cipher_des3ctr_err; 776 777 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 778 l = rte_pktmbuf_data_len(m) - offset; 779 780 /* We use 3DES encryption also for decryption. 781 * IV is not important for 3DES ecb 782 */ 783 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 784 goto process_cipher_des3ctr_err; 785 786 memcpy(ctr, iv, 8); 787 788 for (n = 0; n < srclen; n++) { 789 if (n % 8 == 0) { 790 if (EVP_EncryptUpdate(ctx, 791 (unsigned char *)&ebuf, &unused, 792 (const unsigned char *)&ctr, 8) <= 0) 793 goto process_cipher_des3ctr_err; 794 ctr_inc(ctr); 795 } 796 dst[n] = *(src++) ^ ebuf[n % 8]; 797 798 l--; 799 if (!l) { 800 m = m->next; 801 if (m) { 802 src = rte_pktmbuf_mtod(m, uint8_t *); 803 l = rte_pktmbuf_data_len(m); 804 } 805 } 806 } 807 808 return 0; 809 810 process_cipher_des3ctr_err: 811 OPENSSL_LOG_ERR("Process openssl cipher des 3 ede ctr failed"); 812 return -EINVAL; 813 } 814 815 /** Process auth/encription aes-gcm algorithm */ 816 static int 817 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 818 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, 819 uint8_t *key, uint8_t *dst, uint8_t *tag, 820 EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) 821 { 822 int len = 0, unused = 0; 823 uint8_t empty[] = {}; 824 825 if (EVP_EncryptInit_ex(ctx, algo, NULL, NULL, NULL) <= 0) 826 goto process_auth_encryption_gcm_err; 827 828 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL) <= 0) 829 goto process_auth_encryption_gcm_err; 830 831 if (EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv) <= 0) 832 goto process_auth_encryption_gcm_err; 833 834 if (aadlen > 0) 835 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 836 goto process_auth_encryption_gcm_err; 837 838 if (srclen > 0) 839 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 840 srclen, ctx)) 841 goto process_auth_encryption_gcm_err; 842 843 /* Workaround open ssl bug in version less then 1.0.1f */ 844 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 845 goto process_auth_encryption_gcm_err; 846 847 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 848 goto process_auth_encryption_gcm_err; 849 850 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 851 goto process_auth_encryption_gcm_err; 852 853 return 0; 854 855 process_auth_encryption_gcm_err: 856 OPENSSL_LOG_ERR("Process openssl auth encryption gcm failed"); 857 return -EINVAL; 858 } 859 860 static int 861 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 862 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, 863 uint8_t *key, uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx, 864 const EVP_CIPHER *algo) 865 { 866 int len = 0, unused = 0; 867 uint8_t empty[] = {}; 868 869 if (EVP_DecryptInit_ex(ctx, algo, NULL, NULL, NULL) <= 0) 870 goto process_auth_decryption_gcm_err; 871 872 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL) <= 0) 873 goto process_auth_decryption_gcm_err; 874 875 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 876 goto process_auth_decryption_gcm_err; 877 878 if (EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv) <= 0) 879 goto process_auth_decryption_gcm_err; 880 881 if (aadlen > 0) 882 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 883 goto process_auth_decryption_gcm_err; 884 885 if (srclen > 0) 886 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 887 srclen, ctx)) 888 goto process_auth_decryption_gcm_err; 889 890 /* Workaround open ssl bug in version less then 1.0.1f */ 891 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 892 goto process_auth_decryption_gcm_err; 893 894 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 895 goto process_auth_decryption_gcm_final_err; 896 897 return 0; 898 899 process_auth_decryption_gcm_err: 900 OPENSSL_LOG_ERR("Process openssl auth description gcm failed"); 901 return -EINVAL; 902 903 process_auth_decryption_gcm_final_err: 904 return -EFAULT; 905 } 906 907 /** Process standard openssl auth algorithms */ 908 static int 909 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 910 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 911 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 912 { 913 size_t dstlen; 914 struct rte_mbuf *m; 915 int l, n = srclen; 916 uint8_t *src; 917 918 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 919 m = m->next) 920 offset -= rte_pktmbuf_data_len(m); 921 922 if (m == 0) 923 goto process_auth_err; 924 925 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 926 goto process_auth_err; 927 928 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 929 930 l = rte_pktmbuf_data_len(m) - offset; 931 if (srclen <= l) { 932 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 933 goto process_auth_err; 934 goto process_auth_final; 935 } 936 937 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 938 goto process_auth_err; 939 940 n -= l; 941 942 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 943 src = rte_pktmbuf_mtod(m, uint8_t *); 944 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 945 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 946 goto process_auth_err; 947 n -= l; 948 } 949 950 process_auth_final: 951 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 952 goto process_auth_err; 953 return 0; 954 955 process_auth_err: 956 OPENSSL_LOG_ERR("Process openssl auth failed"); 957 return -EINVAL; 958 } 959 960 /** Process standard openssl auth algorithms with hmac */ 961 static int 962 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 963 __rte_unused uint8_t *iv, EVP_PKEY *pkey, 964 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 965 { 966 size_t dstlen; 967 struct rte_mbuf *m; 968 int l, n = srclen; 969 uint8_t *src; 970 971 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 972 m = m->next) 973 offset -= rte_pktmbuf_data_len(m); 974 975 if (m == 0) 976 goto process_auth_err; 977 978 if (EVP_DigestSignInit(ctx, NULL, algo, NULL, pkey) <= 0) 979 goto process_auth_err; 980 981 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 982 983 l = rte_pktmbuf_data_len(m) - offset; 984 if (srclen <= l) { 985 if (EVP_DigestSignUpdate(ctx, (char *)src, srclen) <= 0) 986 goto process_auth_err; 987 goto process_auth_final; 988 } 989 990 if (EVP_DigestSignUpdate(ctx, (char *)src, l) <= 0) 991 goto process_auth_err; 992 993 n -= l; 994 995 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 996 src = rte_pktmbuf_mtod(m, uint8_t *); 997 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 998 if (EVP_DigestSignUpdate(ctx, (char *)src, l) <= 0) 999 goto process_auth_err; 1000 n -= l; 1001 } 1002 1003 process_auth_final: 1004 if (EVP_DigestSignFinal(ctx, dst, &dstlen) <= 0) 1005 goto process_auth_err; 1006 1007 return 0; 1008 1009 process_auth_err: 1010 OPENSSL_LOG_ERR("Process openssl auth failed"); 1011 return -EINVAL; 1012 } 1013 1014 /*----------------------------------------------------------------------------*/ 1015 1016 /** Process auth/cipher combined operation */ 1017 static void 1018 process_openssl_combined_op 1019 (struct rte_crypto_op *op, struct openssl_session *sess, 1020 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1021 { 1022 /* cipher */ 1023 uint8_t *dst = NULL, *iv, *tag, *aad; 1024 int srclen, ivlen, aadlen, status = -1; 1025 uint32_t offset; 1026 1027 /* 1028 * Segmented destination buffer is not supported for 1029 * encryption/decryption 1030 */ 1031 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1032 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1033 return; 1034 } 1035 1036 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1037 sess->iv.offset); 1038 ivlen = sess->iv.length; 1039 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1040 srclen = 0; 1041 offset = op->sym->auth.data.offset; 1042 aadlen = op->sym->auth.data.length; 1043 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1044 op->sym->auth.data.offset); 1045 tag = op->sym->auth.digest.data; 1046 if (tag == NULL) 1047 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1048 offset + aadlen); 1049 } else { 1050 srclen = op->sym->aead.data.length; 1051 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1052 op->sym->aead.data.offset); 1053 offset = op->sym->aead.data.offset; 1054 aad = op->sym->aead.aad.data; 1055 aadlen = sess->auth.aad_length; 1056 tag = op->sym->aead.digest.data; 1057 if (tag == NULL) 1058 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1059 offset + srclen); 1060 } 1061 1062 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1063 status = process_openssl_auth_encryption_gcm( 1064 mbuf_src, offset, srclen, 1065 aad, aadlen, iv, ivlen, sess->cipher.key.data, 1066 dst, tag, sess->cipher.ctx, 1067 sess->cipher.evp_algo); 1068 else 1069 status = process_openssl_auth_decryption_gcm( 1070 mbuf_src, offset, srclen, 1071 aad, aadlen, iv, ivlen, sess->cipher.key.data, 1072 dst, tag, sess->cipher.ctx, 1073 sess->cipher.evp_algo); 1074 1075 if (status != 0) { 1076 if (status == (-EFAULT) && 1077 sess->auth.operation == 1078 RTE_CRYPTO_AUTH_OP_VERIFY) 1079 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1080 else 1081 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1082 } 1083 } 1084 1085 /** Process cipher operation */ 1086 static void 1087 process_openssl_cipher_op 1088 (struct rte_crypto_op *op, struct openssl_session *sess, 1089 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1090 { 1091 uint8_t *dst, *iv; 1092 int srclen, status; 1093 1094 /* 1095 * Segmented destination buffer is not supported for 1096 * encryption/decryption 1097 */ 1098 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1099 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1100 return; 1101 } 1102 1103 srclen = op->sym->cipher.data.length; 1104 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1105 op->sym->cipher.data.offset); 1106 1107 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1108 sess->iv.offset); 1109 1110 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1111 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1112 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1113 op->sym->cipher.data.offset, iv, 1114 sess->cipher.key.data, srclen, 1115 sess->cipher.ctx, 1116 sess->cipher.evp_algo); 1117 else 1118 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1119 op->sym->cipher.data.offset, iv, 1120 sess->cipher.key.data, srclen, 1121 sess->cipher.ctx, 1122 sess->cipher.evp_algo); 1123 else 1124 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1125 op->sym->cipher.data.offset, iv, 1126 sess->cipher.key.data, srclen, 1127 sess->cipher.ctx); 1128 1129 if (status != 0) 1130 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1131 } 1132 1133 /** Process cipher operation */ 1134 static void 1135 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1136 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1137 struct rte_mbuf *mbuf_dst) 1138 { 1139 uint8_t *src, *dst, *iv; 1140 uint8_t block_size, last_block_len; 1141 int srclen, status = 0; 1142 1143 srclen = op->sym->cipher.data.length; 1144 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1145 op->sym->cipher.data.offset); 1146 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1147 op->sym->cipher.data.offset); 1148 1149 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1150 sess->iv.offset); 1151 1152 block_size = DES_BLOCK_SIZE; 1153 1154 last_block_len = srclen % block_size; 1155 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1156 /* Encrypt only with ECB mode XOR IV */ 1157 if (srclen < block_size) { 1158 status = process_openssl_cipher_bpi_encrypt(src, dst, 1159 iv, srclen, 1160 sess->cipher.bpi_ctx); 1161 } else { 1162 srclen -= last_block_len; 1163 /* Encrypt with the block aligned stream with CBC mode */ 1164 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1165 op->sym->cipher.data.offset, iv, 1166 sess->cipher.key.data, srclen, 1167 sess->cipher.ctx, sess->cipher.evp_algo); 1168 if (last_block_len) { 1169 /* Point at last block */ 1170 dst += srclen; 1171 /* 1172 * IV is the last encrypted block from 1173 * the previous operation 1174 */ 1175 iv = dst - block_size; 1176 src += srclen; 1177 srclen = last_block_len; 1178 /* Encrypt the last frame with ECB mode */ 1179 status |= process_openssl_cipher_bpi_encrypt(src, 1180 dst, iv, 1181 srclen, sess->cipher.bpi_ctx); 1182 } 1183 } 1184 } else { 1185 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1186 if (srclen < block_size) { 1187 status = process_openssl_cipher_bpi_encrypt(src, dst, 1188 iv, 1189 srclen, 1190 sess->cipher.bpi_ctx); 1191 } else { 1192 if (last_block_len) { 1193 /* Point at last block */ 1194 dst += srclen - last_block_len; 1195 src += srclen - last_block_len; 1196 /* 1197 * IV is the last full block 1198 */ 1199 iv = src - block_size; 1200 /* 1201 * Decrypt the last frame with ECB mode 1202 * (encrypt, as it is the same operation) 1203 */ 1204 status = process_openssl_cipher_bpi_encrypt(src, 1205 dst, iv, 1206 last_block_len, sess->cipher.bpi_ctx); 1207 /* Prepare parameters for CBC mode op */ 1208 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1209 sess->iv.offset); 1210 dst += last_block_len - srclen; 1211 srclen -= last_block_len; 1212 } 1213 1214 /* Decrypt with CBC mode */ 1215 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1216 op->sym->cipher.data.offset, iv, 1217 sess->cipher.key.data, srclen, 1218 sess->cipher.ctx, 1219 sess->cipher.evp_algo); 1220 } 1221 } 1222 1223 if (status != 0) 1224 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1225 } 1226 1227 /** Process auth operation */ 1228 static void 1229 process_openssl_auth_op 1230 (struct rte_crypto_op *op, struct openssl_session *sess, 1231 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1232 { 1233 uint8_t *dst; 1234 int srclen, status; 1235 1236 srclen = op->sym->auth.data.length; 1237 1238 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) 1239 dst = (uint8_t *)rte_pktmbuf_append(mbuf_src, 1240 sess->auth.digest_length); 1241 else { 1242 dst = op->sym->auth.digest.data; 1243 if (dst == NULL) 1244 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1245 op->sym->auth.data.offset + 1246 op->sym->auth.data.length); 1247 } 1248 1249 switch (sess->auth.mode) { 1250 case OPENSSL_AUTH_AS_AUTH: 1251 status = process_openssl_auth(mbuf_src, dst, 1252 op->sym->auth.data.offset, NULL, NULL, srclen, 1253 sess->auth.auth.ctx, sess->auth.auth.evp_algo); 1254 break; 1255 case OPENSSL_AUTH_AS_HMAC: 1256 status = process_openssl_auth_hmac(mbuf_src, dst, 1257 op->sym->auth.data.offset, NULL, 1258 sess->auth.hmac.pkey, srclen, 1259 sess->auth.hmac.ctx, sess->auth.hmac.evp_algo); 1260 break; 1261 default: 1262 status = -1; 1263 break; 1264 } 1265 1266 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1267 if (memcmp(dst, op->sym->auth.digest.data, 1268 sess->auth.digest_length) != 0) { 1269 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1270 } 1271 /* Trim area used for digest from mbuf. */ 1272 rte_pktmbuf_trim(mbuf_src, sess->auth.digest_length); 1273 } 1274 1275 if (status != 0) 1276 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1277 } 1278 1279 /** Process crypto operation for mbuf */ 1280 static int 1281 process_op(const struct openssl_qp *qp, struct rte_crypto_op *op, 1282 struct openssl_session *sess) 1283 { 1284 struct rte_mbuf *msrc, *mdst; 1285 int retval; 1286 1287 msrc = op->sym->m_src; 1288 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 1289 1290 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1291 1292 switch (sess->chain_order) { 1293 case OPENSSL_CHAIN_ONLY_CIPHER: 1294 process_openssl_cipher_op(op, sess, msrc, mdst); 1295 break; 1296 case OPENSSL_CHAIN_ONLY_AUTH: 1297 process_openssl_auth_op(op, sess, msrc, mdst); 1298 break; 1299 case OPENSSL_CHAIN_CIPHER_AUTH: 1300 process_openssl_cipher_op(op, sess, msrc, mdst); 1301 process_openssl_auth_op(op, sess, mdst, mdst); 1302 break; 1303 case OPENSSL_CHAIN_AUTH_CIPHER: 1304 process_openssl_auth_op(op, sess, msrc, mdst); 1305 process_openssl_cipher_op(op, sess, msrc, mdst); 1306 break; 1307 case OPENSSL_CHAIN_COMBINED: 1308 process_openssl_combined_op(op, sess, msrc, mdst); 1309 break; 1310 case OPENSSL_CHAIN_CIPHER_BPI: 1311 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 1312 break; 1313 default: 1314 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1315 break; 1316 } 1317 1318 /* Free session if a session-less crypto op */ 1319 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 1320 openssl_reset_session(sess); 1321 memset(sess, 0, sizeof(struct openssl_session)); 1322 rte_mempool_put(qp->sess_mp, op->sym->session); 1323 op->sym->session = NULL; 1324 } 1325 1326 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 1327 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1328 1329 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 1330 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 1331 else 1332 retval = -1; 1333 1334 return retval; 1335 } 1336 1337 /* 1338 *------------------------------------------------------------------------------ 1339 * PMD Framework 1340 *------------------------------------------------------------------------------ 1341 */ 1342 1343 /** Enqueue burst */ 1344 static uint16_t 1345 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 1346 uint16_t nb_ops) 1347 { 1348 struct openssl_session *sess; 1349 struct openssl_qp *qp = queue_pair; 1350 int i, retval; 1351 1352 for (i = 0; i < nb_ops; i++) { 1353 sess = get_session(qp, ops[i]); 1354 if (unlikely(sess == NULL)) 1355 goto enqueue_err; 1356 1357 retval = process_op(qp, ops[i], sess); 1358 if (unlikely(retval < 0)) 1359 goto enqueue_err; 1360 } 1361 1362 qp->stats.enqueued_count += i; 1363 return i; 1364 1365 enqueue_err: 1366 qp->stats.enqueue_err_count++; 1367 return i; 1368 } 1369 1370 /** Dequeue burst */ 1371 static uint16_t 1372 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 1373 uint16_t nb_ops) 1374 { 1375 struct openssl_qp *qp = queue_pair; 1376 1377 unsigned int nb_dequeued = 0; 1378 1379 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 1380 (void **)ops, nb_ops, NULL); 1381 qp->stats.dequeued_count += nb_dequeued; 1382 1383 return nb_dequeued; 1384 } 1385 1386 /** Create OPENSSL crypto device */ 1387 static int 1388 cryptodev_openssl_create(const char *name, 1389 struct rte_vdev_device *vdev, 1390 struct rte_crypto_vdev_init_params *init_params) 1391 { 1392 struct rte_cryptodev *dev; 1393 struct openssl_private *internals; 1394 1395 if (init_params->name[0] == '\0') 1396 snprintf(init_params->name, sizeof(init_params->name), 1397 "%s", name); 1398 1399 dev = rte_cryptodev_vdev_pmd_init(init_params->name, 1400 sizeof(struct openssl_private), 1401 init_params->socket_id, 1402 vdev); 1403 if (dev == NULL) { 1404 OPENSSL_LOG_ERR("failed to create cryptodev vdev"); 1405 goto init_error; 1406 } 1407 1408 dev->dev_type = RTE_CRYPTODEV_OPENSSL_PMD; 1409 dev->dev_ops = rte_openssl_pmd_ops; 1410 1411 /* register rx/tx burst functions for data path */ 1412 dev->dequeue_burst = openssl_pmd_dequeue_burst; 1413 dev->enqueue_burst = openssl_pmd_enqueue_burst; 1414 1415 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 1416 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 1417 RTE_CRYPTODEV_FF_CPU_AESNI | 1418 RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER; 1419 1420 /* Set vector instructions mode supported */ 1421 internals = dev->data->dev_private; 1422 1423 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 1424 internals->max_nb_sessions = init_params->max_nb_sessions; 1425 1426 return 0; 1427 1428 init_error: 1429 OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed", 1430 init_params->name); 1431 1432 cryptodev_openssl_remove(vdev); 1433 return -EFAULT; 1434 } 1435 1436 /** Initialise OPENSSL crypto device */ 1437 static int 1438 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 1439 { 1440 struct rte_crypto_vdev_init_params init_params = { 1441 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, 1442 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, 1443 rte_socket_id(), 1444 {0} 1445 }; 1446 const char *name; 1447 const char *input_args; 1448 1449 name = rte_vdev_device_name(vdev); 1450 if (name == NULL) 1451 return -EINVAL; 1452 input_args = rte_vdev_device_args(vdev); 1453 1454 rte_cryptodev_vdev_parse_init_params(&init_params, input_args); 1455 1456 RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, 1457 init_params.socket_id); 1458 if (init_params.name[0] != '\0') 1459 RTE_LOG(INFO, PMD, " User defined name = %s\n", 1460 init_params.name); 1461 RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", 1462 init_params.max_nb_queue_pairs); 1463 RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", 1464 init_params.max_nb_sessions); 1465 1466 return cryptodev_openssl_create(name, vdev, &init_params); 1467 } 1468 1469 /** Uninitialise OPENSSL crypto device */ 1470 static int 1471 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 1472 { 1473 const char *name; 1474 1475 name = rte_vdev_device_name(vdev); 1476 if (name == NULL) 1477 return -EINVAL; 1478 1479 RTE_LOG(INFO, PMD, 1480 "Closing OPENSSL crypto device %s on numa socket %u\n", 1481 name, rte_socket_id()); 1482 1483 return 0; 1484 } 1485 1486 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 1487 .probe = cryptodev_openssl_probe, 1488 .remove = cryptodev_openssl_remove 1489 }; 1490 1491 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 1492 cryptodev_openssl_pmd_drv); 1493 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 1494 "max_nb_queue_pairs=<int> " 1495 "max_nb_sessions=<int> " 1496 "socket_id=<int>"); 1497