1 /*- 2 * BSD LICENSE 3 * 4 * Copyright(c) 2016-2017 Intel Corporation. All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Intel Corporation nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <rte_common.h> 34 #include <rte_hexdump.h> 35 #include <rte_cryptodev.h> 36 #include <rte_cryptodev_pmd.h> 37 #include <rte_cryptodev_vdev.h> 38 #include <rte_vdev.h> 39 #include <rte_malloc.h> 40 #include <rte_cpuflags.h> 41 42 #include <openssl/evp.h> 43 44 #include "rte_openssl_pmd_private.h" 45 46 #define DES_BLOCK_SIZE 8 47 48 static int cryptodev_openssl_remove(struct rte_vdev_device *vdev); 49 50 /*----------------------------------------------------------------------------*/ 51 52 /** 53 * Increment counter by 1 54 * Counter is 64 bit array, big-endian 55 */ 56 static void 57 ctr_inc(uint8_t *ctr) 58 { 59 uint64_t *ctr64 = (uint64_t *)ctr; 60 61 *ctr64 = __builtin_bswap64(*ctr64); 62 (*ctr64)++; 63 *ctr64 = __builtin_bswap64(*ctr64); 64 } 65 66 /* 67 *------------------------------------------------------------------------------ 68 * Session Prepare 69 *------------------------------------------------------------------------------ 70 */ 71 72 /** Get xform chain order */ 73 static enum openssl_chain_order 74 openssl_get_chain_order(const struct rte_crypto_sym_xform *xform) 75 { 76 enum openssl_chain_order res = OPENSSL_CHAIN_NOT_SUPPORTED; 77 78 if (xform != NULL) { 79 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 80 if (xform->next == NULL) 81 res = OPENSSL_CHAIN_ONLY_AUTH; 82 else if (xform->next->type == 83 RTE_CRYPTO_SYM_XFORM_CIPHER) 84 res = OPENSSL_CHAIN_AUTH_CIPHER; 85 } 86 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 87 if (xform->next == NULL) 88 res = OPENSSL_CHAIN_ONLY_CIPHER; 89 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 90 res = OPENSSL_CHAIN_CIPHER_AUTH; 91 } 92 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 93 res = OPENSSL_CHAIN_COMBINED; 94 } 95 96 return res; 97 } 98 99 /** Get session cipher key from input cipher key */ 100 static void 101 get_cipher_key(uint8_t *input_key, int keylen, uint8_t *session_key) 102 { 103 memcpy(session_key, input_key, keylen); 104 } 105 106 /** Get key ede 24 bytes standard from input key */ 107 static int 108 get_cipher_key_ede(uint8_t *key, int keylen, uint8_t *key_ede) 109 { 110 int res = 0; 111 112 /* Initialize keys - 24 bytes: [key1-key2-key3] */ 113 switch (keylen) { 114 case 24: 115 memcpy(key_ede, key, 24); 116 break; 117 case 16: 118 /* K3 = K1 */ 119 memcpy(key_ede, key, 16); 120 memcpy(key_ede + 16, key, 8); 121 break; 122 case 8: 123 /* K1 = K2 = K3 (DES compatibility) */ 124 memcpy(key_ede, key, 8); 125 memcpy(key_ede + 8, key, 8); 126 memcpy(key_ede + 16, key, 8); 127 break; 128 default: 129 OPENSSL_LOG_ERR("Unsupported key size"); 130 res = -EINVAL; 131 } 132 133 return res; 134 } 135 136 /** Get adequate openssl function for input cipher algorithm */ 137 static uint8_t 138 get_cipher_algo(enum rte_crypto_cipher_algorithm sess_algo, size_t keylen, 139 const EVP_CIPHER **algo) 140 { 141 int res = 0; 142 143 if (algo != NULL) { 144 switch (sess_algo) { 145 case RTE_CRYPTO_CIPHER_3DES_CBC: 146 switch (keylen) { 147 case 16: 148 *algo = EVP_des_ede_cbc(); 149 break; 150 case 24: 151 *algo = EVP_des_ede3_cbc(); 152 break; 153 default: 154 res = -EINVAL; 155 } 156 break; 157 case RTE_CRYPTO_CIPHER_3DES_CTR: 158 break; 159 case RTE_CRYPTO_CIPHER_AES_CBC: 160 switch (keylen) { 161 case 16: 162 *algo = EVP_aes_128_cbc(); 163 break; 164 case 24: 165 *algo = EVP_aes_192_cbc(); 166 break; 167 case 32: 168 *algo = EVP_aes_256_cbc(); 169 break; 170 default: 171 res = -EINVAL; 172 } 173 break; 174 case RTE_CRYPTO_CIPHER_AES_CTR: 175 switch (keylen) { 176 case 16: 177 *algo = EVP_aes_128_ctr(); 178 break; 179 case 24: 180 *algo = EVP_aes_192_ctr(); 181 break; 182 case 32: 183 *algo = EVP_aes_256_ctr(); 184 break; 185 default: 186 res = -EINVAL; 187 } 188 break; 189 default: 190 res = -EINVAL; 191 break; 192 } 193 } else { 194 res = -EINVAL; 195 } 196 197 return res; 198 } 199 200 /** Get adequate openssl function for input auth algorithm */ 201 static uint8_t 202 get_auth_algo(enum rte_crypto_auth_algorithm sessalgo, 203 const EVP_MD **algo) 204 { 205 int res = 0; 206 207 if (algo != NULL) { 208 switch (sessalgo) { 209 case RTE_CRYPTO_AUTH_MD5: 210 case RTE_CRYPTO_AUTH_MD5_HMAC: 211 *algo = EVP_md5(); 212 break; 213 case RTE_CRYPTO_AUTH_SHA1: 214 case RTE_CRYPTO_AUTH_SHA1_HMAC: 215 *algo = EVP_sha1(); 216 break; 217 case RTE_CRYPTO_AUTH_SHA224: 218 case RTE_CRYPTO_AUTH_SHA224_HMAC: 219 *algo = EVP_sha224(); 220 break; 221 case RTE_CRYPTO_AUTH_SHA256: 222 case RTE_CRYPTO_AUTH_SHA256_HMAC: 223 *algo = EVP_sha256(); 224 break; 225 case RTE_CRYPTO_AUTH_SHA384: 226 case RTE_CRYPTO_AUTH_SHA384_HMAC: 227 *algo = EVP_sha384(); 228 break; 229 case RTE_CRYPTO_AUTH_SHA512: 230 case RTE_CRYPTO_AUTH_SHA512_HMAC: 231 *algo = EVP_sha512(); 232 break; 233 default: 234 res = -EINVAL; 235 break; 236 } 237 } else { 238 res = -EINVAL; 239 } 240 241 return res; 242 } 243 244 /** Get adequate openssl function for input cipher algorithm */ 245 static uint8_t 246 get_aead_algo(enum rte_crypto_aead_algorithm sess_algo, size_t keylen, 247 const EVP_CIPHER **algo) 248 { 249 int res = 0; 250 251 if (algo != NULL) { 252 switch (sess_algo) { 253 case RTE_CRYPTO_AEAD_AES_GCM: 254 switch (keylen) { 255 case 16: 256 *algo = EVP_aes_128_gcm(); 257 break; 258 case 24: 259 *algo = EVP_aes_192_gcm(); 260 break; 261 case 32: 262 *algo = EVP_aes_256_gcm(); 263 break; 264 default: 265 res = -EINVAL; 266 } 267 break; 268 default: 269 res = -EINVAL; 270 break; 271 } 272 } else { 273 res = -EINVAL; 274 } 275 276 return res; 277 } 278 279 /** Set session cipher parameters */ 280 static int 281 openssl_set_session_cipher_parameters(struct openssl_session *sess, 282 const struct rte_crypto_sym_xform *xform) 283 { 284 /* Select cipher direction */ 285 sess->cipher.direction = xform->cipher.op; 286 /* Select cipher key */ 287 sess->cipher.key.length = xform->cipher.key.length; 288 289 /* Set IV parameters */ 290 sess->iv.offset = xform->cipher.iv.offset; 291 sess->iv.length = xform->cipher.iv.length; 292 293 /* Select cipher algo */ 294 switch (xform->cipher.algo) { 295 case RTE_CRYPTO_CIPHER_3DES_CBC: 296 case RTE_CRYPTO_CIPHER_AES_CBC: 297 case RTE_CRYPTO_CIPHER_AES_CTR: 298 sess->cipher.mode = OPENSSL_CIPHER_LIB; 299 sess->cipher.algo = xform->cipher.algo; 300 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 301 302 if (get_cipher_algo(sess->cipher.algo, sess->cipher.key.length, 303 &sess->cipher.evp_algo) != 0) 304 return -EINVAL; 305 306 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 307 sess->cipher.key.data); 308 309 break; 310 311 case RTE_CRYPTO_CIPHER_3DES_CTR: 312 sess->cipher.mode = OPENSSL_CIPHER_DES3CTR; 313 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 314 315 if (get_cipher_key_ede(xform->cipher.key.data, 316 sess->cipher.key.length, 317 sess->cipher.key.data) != 0) 318 return -EINVAL; 319 break; 320 case RTE_CRYPTO_CIPHER_DES_DOCSISBPI: 321 sess->cipher.algo = xform->cipher.algo; 322 sess->chain_order = OPENSSL_CHAIN_CIPHER_BPI; 323 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 324 sess->cipher.evp_algo = EVP_des_cbc(); 325 326 sess->cipher.bpi_ctx = EVP_CIPHER_CTX_new(); 327 /* IV will be ECB encrypted whether direction is encrypt or decrypt */ 328 if (EVP_EncryptInit_ex(sess->cipher.bpi_ctx, EVP_des_ecb(), 329 NULL, xform->cipher.key.data, 0) != 1) 330 return -EINVAL; 331 332 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 333 sess->cipher.key.data); 334 break; 335 default: 336 sess->cipher.algo = RTE_CRYPTO_CIPHER_NULL; 337 return -EINVAL; 338 } 339 340 return 0; 341 } 342 343 /* Set session auth parameters */ 344 static int 345 openssl_set_session_auth_parameters(struct openssl_session *sess, 346 const struct rte_crypto_sym_xform *xform) 347 { 348 /* Select auth generate/verify */ 349 sess->auth.operation = xform->auth.op; 350 sess->auth.algo = xform->auth.algo; 351 352 /* Select auth algo */ 353 switch (xform->auth.algo) { 354 case RTE_CRYPTO_AUTH_AES_GMAC: 355 sess->chain_order = OPENSSL_CHAIN_COMBINED; 356 357 /* Set IV parameters */ 358 sess->iv.offset = xform->auth.iv.offset; 359 sess->iv.length = xform->auth.iv.length; 360 361 /* 362 * OpenSSL requires GMAC to be a GCM operation 363 * with no cipher data length 364 */ 365 sess->cipher.mode = OPENSSL_CIPHER_LIB; 366 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) 367 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_ENCRYPT; 368 else 369 sess->cipher.direction = RTE_CRYPTO_CIPHER_OP_DECRYPT; 370 371 sess->cipher.key.length = xform->auth.key.length; 372 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 373 374 if (get_aead_algo(RTE_CRYPTO_AEAD_AES_GCM, 375 sess->cipher.key.length, 376 &sess->cipher.evp_algo) != 0) 377 return -EINVAL; 378 379 get_cipher_key(xform->auth.key.data, xform->auth.key.length, 380 sess->cipher.key.data); 381 382 break; 383 384 case RTE_CRYPTO_AUTH_MD5: 385 case RTE_CRYPTO_AUTH_SHA1: 386 case RTE_CRYPTO_AUTH_SHA224: 387 case RTE_CRYPTO_AUTH_SHA256: 388 case RTE_CRYPTO_AUTH_SHA384: 389 case RTE_CRYPTO_AUTH_SHA512: 390 sess->auth.mode = OPENSSL_AUTH_AS_AUTH; 391 if (get_auth_algo(xform->auth.algo, 392 &sess->auth.auth.evp_algo) != 0) 393 return -EINVAL; 394 sess->auth.auth.ctx = EVP_MD_CTX_create(); 395 break; 396 397 case RTE_CRYPTO_AUTH_MD5_HMAC: 398 case RTE_CRYPTO_AUTH_SHA1_HMAC: 399 case RTE_CRYPTO_AUTH_SHA224_HMAC: 400 case RTE_CRYPTO_AUTH_SHA256_HMAC: 401 case RTE_CRYPTO_AUTH_SHA384_HMAC: 402 case RTE_CRYPTO_AUTH_SHA512_HMAC: 403 sess->auth.mode = OPENSSL_AUTH_AS_HMAC; 404 sess->auth.hmac.ctx = EVP_MD_CTX_create(); 405 if (get_auth_algo(xform->auth.algo, 406 &sess->auth.hmac.evp_algo) != 0) 407 return -EINVAL; 408 sess->auth.hmac.pkey = EVP_PKEY_new_mac_key(EVP_PKEY_HMAC, NULL, 409 xform->auth.key.data, xform->auth.key.length); 410 break; 411 412 default: 413 return -EINVAL; 414 } 415 416 sess->auth.digest_length = xform->auth.digest_length; 417 418 return 0; 419 } 420 421 /* Set session AEAD parameters */ 422 static int 423 openssl_set_session_aead_parameters(struct openssl_session *sess, 424 const struct rte_crypto_sym_xform *xform) 425 { 426 /* Select cipher direction */ 427 sess->cipher.direction = xform->cipher.op; 428 /* Select cipher key */ 429 sess->cipher.key.length = xform->aead.key.length; 430 431 /* Set IV parameters */ 432 sess->iv.offset = xform->aead.iv.offset; 433 sess->iv.length = xform->aead.iv.length; 434 435 /* Select auth generate/verify */ 436 sess->auth.operation = xform->auth.op; 437 sess->auth.algo = xform->auth.algo; 438 439 /* Select auth algo */ 440 switch (xform->aead.algo) { 441 case RTE_CRYPTO_AEAD_AES_GCM: 442 sess->cipher.mode = OPENSSL_CIPHER_LIB; 443 sess->aead_algo = xform->aead.algo; 444 sess->cipher.ctx = EVP_CIPHER_CTX_new(); 445 446 if (get_aead_algo(sess->aead_algo, sess->cipher.key.length, 447 &sess->cipher.evp_algo) != 0) 448 return -EINVAL; 449 450 get_cipher_key(xform->cipher.key.data, sess->cipher.key.length, 451 sess->cipher.key.data); 452 453 sess->chain_order = OPENSSL_CHAIN_COMBINED; 454 break; 455 default: 456 return -EINVAL; 457 } 458 459 sess->auth.aad_length = xform->aead.add_auth_data_length; 460 sess->auth.digest_length = xform->aead.digest_length; 461 462 return 0; 463 } 464 465 /** Parse crypto xform chain and set private session parameters */ 466 int 467 openssl_set_session_parameters(struct openssl_session *sess, 468 const struct rte_crypto_sym_xform *xform) 469 { 470 const struct rte_crypto_sym_xform *cipher_xform = NULL; 471 const struct rte_crypto_sym_xform *auth_xform = NULL; 472 const struct rte_crypto_sym_xform *aead_xform = NULL; 473 474 sess->chain_order = openssl_get_chain_order(xform); 475 switch (sess->chain_order) { 476 case OPENSSL_CHAIN_ONLY_CIPHER: 477 cipher_xform = xform; 478 break; 479 case OPENSSL_CHAIN_ONLY_AUTH: 480 auth_xform = xform; 481 break; 482 case OPENSSL_CHAIN_CIPHER_AUTH: 483 cipher_xform = xform; 484 auth_xform = xform->next; 485 break; 486 case OPENSSL_CHAIN_AUTH_CIPHER: 487 auth_xform = xform; 488 cipher_xform = xform->next; 489 break; 490 case OPENSSL_CHAIN_COMBINED: 491 aead_xform = xform; 492 break; 493 default: 494 return -EINVAL; 495 } 496 497 /* Default IV length = 0 */ 498 sess->iv.length = 0; 499 500 /* cipher_xform must be check before auth_xform */ 501 if (cipher_xform) { 502 if (openssl_set_session_cipher_parameters( 503 sess, cipher_xform)) { 504 OPENSSL_LOG_ERR( 505 "Invalid/unsupported cipher parameters"); 506 return -EINVAL; 507 } 508 } 509 510 if (auth_xform) { 511 if (openssl_set_session_auth_parameters(sess, auth_xform)) { 512 OPENSSL_LOG_ERR( 513 "Invalid/unsupported auth parameters"); 514 return -EINVAL; 515 } 516 } 517 518 if (aead_xform) { 519 if (openssl_set_session_aead_parameters(sess, aead_xform)) { 520 OPENSSL_LOG_ERR( 521 "Invalid/unsupported auth parameters"); 522 return -EINVAL; 523 } 524 } 525 526 return 0; 527 } 528 529 /** Reset private session parameters */ 530 void 531 openssl_reset_session(struct openssl_session *sess) 532 { 533 EVP_CIPHER_CTX_free(sess->cipher.ctx); 534 535 if (sess->chain_order == OPENSSL_CHAIN_CIPHER_BPI) 536 EVP_CIPHER_CTX_free(sess->cipher.bpi_ctx); 537 538 switch (sess->auth.mode) { 539 case OPENSSL_AUTH_AS_AUTH: 540 EVP_MD_CTX_destroy(sess->auth.auth.ctx); 541 break; 542 case OPENSSL_AUTH_AS_HMAC: 543 EVP_PKEY_free(sess->auth.hmac.pkey); 544 EVP_MD_CTX_destroy(sess->auth.hmac.ctx); 545 break; 546 default: 547 break; 548 } 549 } 550 551 /** Provide session for operation */ 552 static struct openssl_session * 553 get_session(struct openssl_qp *qp, struct rte_crypto_op *op) 554 { 555 struct openssl_session *sess = NULL; 556 557 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 558 /* get existing session */ 559 if (likely(op->sym->session != NULL && 560 op->sym->session->dev_type == 561 RTE_CRYPTODEV_OPENSSL_PMD)) 562 sess = (struct openssl_session *) 563 op->sym->session->_private; 564 } else { 565 /* provide internal session */ 566 void *_sess = NULL; 567 568 if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) { 569 sess = (struct openssl_session *) 570 ((struct rte_cryptodev_sym_session *)_sess) 571 ->_private; 572 573 if (unlikely(openssl_set_session_parameters( 574 sess, op->sym->xform) != 0)) { 575 rte_mempool_put(qp->sess_mp, _sess); 576 sess = NULL; 577 } else 578 op->sym->session = _sess; 579 } 580 } 581 582 if (sess == NULL) 583 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 584 585 return sess; 586 } 587 588 /* 589 *------------------------------------------------------------------------------ 590 * Process Operations 591 *------------------------------------------------------------------------------ 592 */ 593 static inline int 594 process_openssl_encryption_update(struct rte_mbuf *mbuf_src, int offset, 595 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 596 { 597 struct rte_mbuf *m; 598 int dstlen; 599 int l, n = srclen; 600 uint8_t *src; 601 602 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 603 m = m->next) 604 offset -= rte_pktmbuf_data_len(m); 605 606 if (m == 0) 607 return -1; 608 609 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 610 611 l = rte_pktmbuf_data_len(m) - offset; 612 if (srclen <= l) { 613 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 614 return -1; 615 *dst += l; 616 return 0; 617 } 618 619 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 620 return -1; 621 622 *dst += dstlen; 623 n -= l; 624 625 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 626 src = rte_pktmbuf_mtod(m, uint8_t *); 627 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 628 if (EVP_EncryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 629 return -1; 630 *dst += dstlen; 631 n -= l; 632 } 633 634 return 0; 635 } 636 637 static inline int 638 process_openssl_decryption_update(struct rte_mbuf *mbuf_src, int offset, 639 uint8_t **dst, int srclen, EVP_CIPHER_CTX *ctx) 640 { 641 struct rte_mbuf *m; 642 int dstlen; 643 int l, n = srclen; 644 uint8_t *src; 645 646 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 647 m = m->next) 648 offset -= rte_pktmbuf_data_len(m); 649 650 if (m == 0) 651 return -1; 652 653 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 654 655 l = rte_pktmbuf_data_len(m) - offset; 656 if (srclen <= l) { 657 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, srclen) <= 0) 658 return -1; 659 *dst += l; 660 return 0; 661 } 662 663 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 664 return -1; 665 666 *dst += dstlen; 667 n -= l; 668 669 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 670 src = rte_pktmbuf_mtod(m, uint8_t *); 671 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 672 if (EVP_DecryptUpdate(ctx, *dst, &dstlen, src, l) <= 0) 673 return -1; 674 *dst += dstlen; 675 n -= l; 676 } 677 678 return 0; 679 } 680 681 /** Process standard openssl cipher encryption */ 682 static int 683 process_openssl_cipher_encrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 684 int offset, uint8_t *iv, uint8_t *key, int srclen, 685 EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) 686 { 687 int totlen; 688 689 if (EVP_EncryptInit_ex(ctx, algo, NULL, key, iv) <= 0) 690 goto process_cipher_encrypt_err; 691 692 EVP_CIPHER_CTX_set_padding(ctx, 0); 693 694 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 695 srclen, ctx)) 696 goto process_cipher_encrypt_err; 697 698 if (EVP_EncryptFinal_ex(ctx, dst, &totlen) <= 0) 699 goto process_cipher_encrypt_err; 700 701 return 0; 702 703 process_cipher_encrypt_err: 704 OPENSSL_LOG_ERR("Process openssl cipher encrypt failed"); 705 return -EINVAL; 706 } 707 708 /** Process standard openssl cipher encryption */ 709 static int 710 process_openssl_cipher_bpi_encrypt(uint8_t *src, uint8_t *dst, 711 uint8_t *iv, int srclen, 712 EVP_CIPHER_CTX *ctx) 713 { 714 uint8_t i; 715 uint8_t encrypted_iv[DES_BLOCK_SIZE]; 716 int encrypted_ivlen; 717 718 if (EVP_EncryptUpdate(ctx, encrypted_iv, &encrypted_ivlen, 719 iv, DES_BLOCK_SIZE) <= 0) 720 goto process_cipher_encrypt_err; 721 722 for (i = 0; i < srclen; i++) 723 *(dst + i) = *(src + i) ^ (encrypted_iv[i]); 724 725 return 0; 726 727 process_cipher_encrypt_err: 728 OPENSSL_LOG_ERR("Process openssl cipher bpi encrypt failed"); 729 return -EINVAL; 730 } 731 /** Process standard openssl cipher decryption */ 732 static int 733 process_openssl_cipher_decrypt(struct rte_mbuf *mbuf_src, uint8_t *dst, 734 int offset, uint8_t *iv, uint8_t *key, int srclen, 735 EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) 736 { 737 int totlen; 738 739 if (EVP_DecryptInit_ex(ctx, algo, NULL, key, iv) <= 0) 740 goto process_cipher_decrypt_err; 741 742 EVP_CIPHER_CTX_set_padding(ctx, 0); 743 744 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 745 srclen, ctx)) 746 goto process_cipher_decrypt_err; 747 748 if (EVP_DecryptFinal_ex(ctx, dst, &totlen) <= 0) 749 goto process_cipher_decrypt_err; 750 return 0; 751 752 process_cipher_decrypt_err: 753 OPENSSL_LOG_ERR("Process openssl cipher decrypt failed"); 754 return -EINVAL; 755 } 756 757 /** Process cipher des 3 ctr encryption, decryption algorithm */ 758 static int 759 process_openssl_cipher_des3ctr(struct rte_mbuf *mbuf_src, uint8_t *dst, 760 int offset, uint8_t *iv, uint8_t *key, int srclen, 761 EVP_CIPHER_CTX *ctx) 762 { 763 uint8_t ebuf[8], ctr[8]; 764 int unused, n; 765 struct rte_mbuf *m; 766 uint8_t *src; 767 int l; 768 769 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 770 m = m->next) 771 offset -= rte_pktmbuf_data_len(m); 772 773 if (m == 0) 774 goto process_cipher_des3ctr_err; 775 776 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 777 l = rte_pktmbuf_data_len(m) - offset; 778 779 /* We use 3DES encryption also for decryption. 780 * IV is not important for 3DES ecb 781 */ 782 if (EVP_EncryptInit_ex(ctx, EVP_des_ede3_ecb(), NULL, key, NULL) <= 0) 783 goto process_cipher_des3ctr_err; 784 785 memcpy(ctr, iv, 8); 786 787 for (n = 0; n < srclen; n++) { 788 if (n % 8 == 0) { 789 if (EVP_EncryptUpdate(ctx, 790 (unsigned char *)&ebuf, &unused, 791 (const unsigned char *)&ctr, 8) <= 0) 792 goto process_cipher_des3ctr_err; 793 ctr_inc(ctr); 794 } 795 dst[n] = *(src++) ^ ebuf[n % 8]; 796 797 l--; 798 if (!l) { 799 m = m->next; 800 if (m) { 801 src = rte_pktmbuf_mtod(m, uint8_t *); 802 l = rte_pktmbuf_data_len(m); 803 } 804 } 805 } 806 807 return 0; 808 809 process_cipher_des3ctr_err: 810 OPENSSL_LOG_ERR("Process openssl cipher des 3 ede ctr failed"); 811 return -EINVAL; 812 } 813 814 /** Process auth/encription aes-gcm algorithm */ 815 static int 816 process_openssl_auth_encryption_gcm(struct rte_mbuf *mbuf_src, int offset, 817 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, 818 uint8_t *key, uint8_t *dst, uint8_t *tag, 819 EVP_CIPHER_CTX *ctx, const EVP_CIPHER *algo) 820 { 821 int len = 0, unused = 0; 822 uint8_t empty[] = {}; 823 824 if (EVP_EncryptInit_ex(ctx, algo, NULL, NULL, NULL) <= 0) 825 goto process_auth_encryption_gcm_err; 826 827 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL) <= 0) 828 goto process_auth_encryption_gcm_err; 829 830 if (EVP_EncryptInit_ex(ctx, NULL, NULL, key, iv) <= 0) 831 goto process_auth_encryption_gcm_err; 832 833 if (aadlen > 0) 834 if (EVP_EncryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 835 goto process_auth_encryption_gcm_err; 836 837 if (srclen > 0) 838 if (process_openssl_encryption_update(mbuf_src, offset, &dst, 839 srclen, ctx)) 840 goto process_auth_encryption_gcm_err; 841 842 /* Workaround open ssl bug in version less then 1.0.1f */ 843 if (EVP_EncryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 844 goto process_auth_encryption_gcm_err; 845 846 if (EVP_EncryptFinal_ex(ctx, dst, &len) <= 0) 847 goto process_auth_encryption_gcm_err; 848 849 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_GET_TAG, 16, tag) <= 0) 850 goto process_auth_encryption_gcm_err; 851 852 return 0; 853 854 process_auth_encryption_gcm_err: 855 OPENSSL_LOG_ERR("Process openssl auth encryption gcm failed"); 856 return -EINVAL; 857 } 858 859 static int 860 process_openssl_auth_decryption_gcm(struct rte_mbuf *mbuf_src, int offset, 861 int srclen, uint8_t *aad, int aadlen, uint8_t *iv, int ivlen, 862 uint8_t *key, uint8_t *dst, uint8_t *tag, EVP_CIPHER_CTX *ctx, 863 const EVP_CIPHER *algo) 864 { 865 int len = 0, unused = 0; 866 uint8_t empty[] = {}; 867 868 if (EVP_DecryptInit_ex(ctx, algo, NULL, NULL, NULL) <= 0) 869 goto process_auth_decryption_gcm_err; 870 871 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_IVLEN, ivlen, NULL) <= 0) 872 goto process_auth_decryption_gcm_err; 873 874 if (EVP_CIPHER_CTX_ctrl(ctx, EVP_CTRL_GCM_SET_TAG, 16, tag) <= 0) 875 goto process_auth_decryption_gcm_err; 876 877 if (EVP_DecryptInit_ex(ctx, NULL, NULL, key, iv) <= 0) 878 goto process_auth_decryption_gcm_err; 879 880 if (aadlen > 0) 881 if (EVP_DecryptUpdate(ctx, NULL, &len, aad, aadlen) <= 0) 882 goto process_auth_decryption_gcm_err; 883 884 if (srclen > 0) 885 if (process_openssl_decryption_update(mbuf_src, offset, &dst, 886 srclen, ctx)) 887 goto process_auth_decryption_gcm_err; 888 889 /* Workaround open ssl bug in version less then 1.0.1f */ 890 if (EVP_DecryptUpdate(ctx, empty, &unused, empty, 0) <= 0) 891 goto process_auth_decryption_gcm_err; 892 893 if (EVP_DecryptFinal_ex(ctx, dst, &len) <= 0) 894 goto process_auth_decryption_gcm_final_err; 895 896 return 0; 897 898 process_auth_decryption_gcm_err: 899 OPENSSL_LOG_ERR("Process openssl auth description gcm failed"); 900 return -EINVAL; 901 902 process_auth_decryption_gcm_final_err: 903 return -EFAULT; 904 } 905 906 /** Process standard openssl auth algorithms */ 907 static int 908 process_openssl_auth(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 909 __rte_unused uint8_t *iv, __rte_unused EVP_PKEY * pkey, 910 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 911 { 912 size_t dstlen; 913 struct rte_mbuf *m; 914 int l, n = srclen; 915 uint8_t *src; 916 917 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 918 m = m->next) 919 offset -= rte_pktmbuf_data_len(m); 920 921 if (m == 0) 922 goto process_auth_err; 923 924 if (EVP_DigestInit_ex(ctx, algo, NULL) <= 0) 925 goto process_auth_err; 926 927 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 928 929 l = rte_pktmbuf_data_len(m) - offset; 930 if (srclen <= l) { 931 if (EVP_DigestUpdate(ctx, (char *)src, srclen) <= 0) 932 goto process_auth_err; 933 goto process_auth_final; 934 } 935 936 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 937 goto process_auth_err; 938 939 n -= l; 940 941 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 942 src = rte_pktmbuf_mtod(m, uint8_t *); 943 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 944 if (EVP_DigestUpdate(ctx, (char *)src, l) <= 0) 945 goto process_auth_err; 946 n -= l; 947 } 948 949 process_auth_final: 950 if (EVP_DigestFinal_ex(ctx, dst, (unsigned int *)&dstlen) <= 0) 951 goto process_auth_err; 952 return 0; 953 954 process_auth_err: 955 OPENSSL_LOG_ERR("Process openssl auth failed"); 956 return -EINVAL; 957 } 958 959 /** Process standard openssl auth algorithms with hmac */ 960 static int 961 process_openssl_auth_hmac(struct rte_mbuf *mbuf_src, uint8_t *dst, int offset, 962 __rte_unused uint8_t *iv, EVP_PKEY *pkey, 963 int srclen, EVP_MD_CTX *ctx, const EVP_MD *algo) 964 { 965 size_t dstlen; 966 struct rte_mbuf *m; 967 int l, n = srclen; 968 uint8_t *src; 969 970 for (m = mbuf_src; m != NULL && offset > rte_pktmbuf_data_len(m); 971 m = m->next) 972 offset -= rte_pktmbuf_data_len(m); 973 974 if (m == 0) 975 goto process_auth_err; 976 977 if (EVP_DigestSignInit(ctx, NULL, algo, NULL, pkey) <= 0) 978 goto process_auth_err; 979 980 src = rte_pktmbuf_mtod_offset(m, uint8_t *, offset); 981 982 l = rte_pktmbuf_data_len(m) - offset; 983 if (srclen <= l) { 984 if (EVP_DigestSignUpdate(ctx, (char *)src, srclen) <= 0) 985 goto process_auth_err; 986 goto process_auth_final; 987 } 988 989 if (EVP_DigestSignUpdate(ctx, (char *)src, l) <= 0) 990 goto process_auth_err; 991 992 n -= l; 993 994 for (m = m->next; (m != NULL) && (n > 0); m = m->next) { 995 src = rte_pktmbuf_mtod(m, uint8_t *); 996 l = rte_pktmbuf_data_len(m) < n ? rte_pktmbuf_data_len(m) : n; 997 if (EVP_DigestSignUpdate(ctx, (char *)src, l) <= 0) 998 goto process_auth_err; 999 n -= l; 1000 } 1001 1002 process_auth_final: 1003 if (EVP_DigestSignFinal(ctx, dst, &dstlen) <= 0) 1004 goto process_auth_err; 1005 1006 return 0; 1007 1008 process_auth_err: 1009 OPENSSL_LOG_ERR("Process openssl auth failed"); 1010 return -EINVAL; 1011 } 1012 1013 /*----------------------------------------------------------------------------*/ 1014 1015 /** Process auth/cipher combined operation */ 1016 static void 1017 process_openssl_combined_op 1018 (struct rte_crypto_op *op, struct openssl_session *sess, 1019 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1020 { 1021 /* cipher */ 1022 uint8_t *dst = NULL, *iv, *tag, *aad; 1023 int srclen, ivlen, aadlen, status = -1; 1024 uint32_t offset; 1025 1026 /* 1027 * Segmented destination buffer is not supported for 1028 * encryption/decryption 1029 */ 1030 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1031 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1032 return; 1033 } 1034 1035 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1036 sess->iv.offset); 1037 ivlen = sess->iv.length; 1038 if (sess->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 1039 srclen = 0; 1040 offset = op->sym->auth.data.offset; 1041 aadlen = op->sym->auth.data.length; 1042 aad = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1043 op->sym->auth.data.offset); 1044 tag = op->sym->auth.digest.data; 1045 if (tag == NULL) 1046 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1047 offset + aadlen); 1048 } else { 1049 srclen = op->sym->aead.data.length; 1050 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1051 op->sym->aead.data.offset); 1052 offset = op->sym->aead.data.offset; 1053 aad = op->sym->aead.aad.data; 1054 aadlen = sess->auth.aad_length; 1055 tag = op->sym->aead.digest.data; 1056 if (tag == NULL) 1057 tag = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1058 offset + srclen); 1059 } 1060 1061 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1062 status = process_openssl_auth_encryption_gcm( 1063 mbuf_src, offset, srclen, 1064 aad, aadlen, iv, ivlen, sess->cipher.key.data, 1065 dst, tag, sess->cipher.ctx, 1066 sess->cipher.evp_algo); 1067 else 1068 status = process_openssl_auth_decryption_gcm( 1069 mbuf_src, offset, srclen, 1070 aad, aadlen, iv, ivlen, sess->cipher.key.data, 1071 dst, tag, sess->cipher.ctx, 1072 sess->cipher.evp_algo); 1073 1074 if (status != 0) { 1075 if (status == (-EFAULT) && 1076 sess->auth.operation == 1077 RTE_CRYPTO_AUTH_OP_VERIFY) 1078 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1079 else 1080 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1081 } 1082 } 1083 1084 /** Process cipher operation */ 1085 static void 1086 process_openssl_cipher_op 1087 (struct rte_crypto_op *op, struct openssl_session *sess, 1088 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1089 { 1090 uint8_t *dst, *iv; 1091 int srclen, status; 1092 1093 /* 1094 * Segmented destination buffer is not supported for 1095 * encryption/decryption 1096 */ 1097 if (!rte_pktmbuf_is_contiguous(mbuf_dst)) { 1098 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1099 return; 1100 } 1101 1102 srclen = op->sym->cipher.data.length; 1103 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1104 op->sym->cipher.data.offset); 1105 1106 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1107 sess->iv.offset); 1108 1109 if (sess->cipher.mode == OPENSSL_CIPHER_LIB) 1110 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 1111 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1112 op->sym->cipher.data.offset, iv, 1113 sess->cipher.key.data, srclen, 1114 sess->cipher.ctx, 1115 sess->cipher.evp_algo); 1116 else 1117 status = process_openssl_cipher_decrypt(mbuf_src, dst, 1118 op->sym->cipher.data.offset, iv, 1119 sess->cipher.key.data, srclen, 1120 sess->cipher.ctx, 1121 sess->cipher.evp_algo); 1122 else 1123 status = process_openssl_cipher_des3ctr(mbuf_src, dst, 1124 op->sym->cipher.data.offset, iv, 1125 sess->cipher.key.data, srclen, 1126 sess->cipher.ctx); 1127 1128 if (status != 0) 1129 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1130 } 1131 1132 /** Process cipher operation */ 1133 static void 1134 process_openssl_docsis_bpi_op(struct rte_crypto_op *op, 1135 struct openssl_session *sess, struct rte_mbuf *mbuf_src, 1136 struct rte_mbuf *mbuf_dst) 1137 { 1138 uint8_t *src, *dst, *iv; 1139 uint8_t block_size, last_block_len; 1140 int srclen, status = 0; 1141 1142 srclen = op->sym->cipher.data.length; 1143 src = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 1144 op->sym->cipher.data.offset); 1145 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1146 op->sym->cipher.data.offset); 1147 1148 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1149 sess->iv.offset); 1150 1151 block_size = DES_BLOCK_SIZE; 1152 1153 last_block_len = srclen % block_size; 1154 if (sess->cipher.direction == RTE_CRYPTO_CIPHER_OP_ENCRYPT) { 1155 /* Encrypt only with ECB mode XOR IV */ 1156 if (srclen < block_size) { 1157 status = process_openssl_cipher_bpi_encrypt(src, dst, 1158 iv, srclen, 1159 sess->cipher.bpi_ctx); 1160 } else { 1161 srclen -= last_block_len; 1162 /* Encrypt with the block aligned stream with CBC mode */ 1163 status = process_openssl_cipher_encrypt(mbuf_src, dst, 1164 op->sym->cipher.data.offset, iv, 1165 sess->cipher.key.data, srclen, 1166 sess->cipher.ctx, sess->cipher.evp_algo); 1167 if (last_block_len) { 1168 /* Point at last block */ 1169 dst += srclen; 1170 /* 1171 * IV is the last encrypted block from 1172 * the previous operation 1173 */ 1174 iv = dst - block_size; 1175 src += srclen; 1176 srclen = last_block_len; 1177 /* Encrypt the last frame with ECB mode */ 1178 status |= process_openssl_cipher_bpi_encrypt(src, 1179 dst, iv, 1180 srclen, sess->cipher.bpi_ctx); 1181 } 1182 } 1183 } else { 1184 /* Decrypt only with ECB mode (encrypt, as it is same operation) */ 1185 if (srclen < block_size) { 1186 status = process_openssl_cipher_bpi_encrypt(src, dst, 1187 iv, 1188 srclen, 1189 sess->cipher.bpi_ctx); 1190 } else { 1191 if (last_block_len) { 1192 /* Point at last block */ 1193 dst += srclen - last_block_len; 1194 src += srclen - last_block_len; 1195 /* 1196 * IV is the last full block 1197 */ 1198 iv = src - block_size; 1199 /* 1200 * Decrypt the last frame with ECB mode 1201 * (encrypt, as it is the same operation) 1202 */ 1203 status = process_openssl_cipher_bpi_encrypt(src, 1204 dst, iv, 1205 last_block_len, sess->cipher.bpi_ctx); 1206 /* Prepare parameters for CBC mode op */ 1207 iv = rte_crypto_op_ctod_offset(op, uint8_t *, 1208 sess->iv.offset); 1209 dst += last_block_len - srclen; 1210 srclen -= last_block_len; 1211 } 1212 1213 /* Decrypt with CBC mode */ 1214 status |= process_openssl_cipher_decrypt(mbuf_src, dst, 1215 op->sym->cipher.data.offset, iv, 1216 sess->cipher.key.data, srclen, 1217 sess->cipher.ctx, 1218 sess->cipher.evp_algo); 1219 } 1220 } 1221 1222 if (status != 0) 1223 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1224 } 1225 1226 /** Process auth operation */ 1227 static void 1228 process_openssl_auth_op 1229 (struct rte_crypto_op *op, struct openssl_session *sess, 1230 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 1231 { 1232 uint8_t *dst; 1233 int srclen, status; 1234 1235 srclen = op->sym->auth.data.length; 1236 1237 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) 1238 dst = (uint8_t *)rte_pktmbuf_append(mbuf_src, 1239 sess->auth.digest_length); 1240 else { 1241 dst = op->sym->auth.digest.data; 1242 if (dst == NULL) 1243 dst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 1244 op->sym->auth.data.offset + 1245 op->sym->auth.data.length); 1246 } 1247 1248 switch (sess->auth.mode) { 1249 case OPENSSL_AUTH_AS_AUTH: 1250 status = process_openssl_auth(mbuf_src, dst, 1251 op->sym->auth.data.offset, NULL, NULL, srclen, 1252 sess->auth.auth.ctx, sess->auth.auth.evp_algo); 1253 break; 1254 case OPENSSL_AUTH_AS_HMAC: 1255 status = process_openssl_auth_hmac(mbuf_src, dst, 1256 op->sym->auth.data.offset, NULL, 1257 sess->auth.hmac.pkey, srclen, 1258 sess->auth.hmac.ctx, sess->auth.hmac.evp_algo); 1259 break; 1260 default: 1261 status = -1; 1262 break; 1263 } 1264 1265 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 1266 if (memcmp(dst, op->sym->auth.digest.data, 1267 sess->auth.digest_length) != 0) { 1268 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 1269 } 1270 /* Trim area used for digest from mbuf. */ 1271 rte_pktmbuf_trim(mbuf_src, sess->auth.digest_length); 1272 } 1273 1274 if (status != 0) 1275 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1276 } 1277 1278 /** Process crypto operation for mbuf */ 1279 static int 1280 process_op(const struct openssl_qp *qp, struct rte_crypto_op *op, 1281 struct openssl_session *sess) 1282 { 1283 struct rte_mbuf *msrc, *mdst; 1284 int retval; 1285 1286 msrc = op->sym->m_src; 1287 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 1288 1289 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 1290 1291 switch (sess->chain_order) { 1292 case OPENSSL_CHAIN_ONLY_CIPHER: 1293 process_openssl_cipher_op(op, sess, msrc, mdst); 1294 break; 1295 case OPENSSL_CHAIN_ONLY_AUTH: 1296 process_openssl_auth_op(op, sess, msrc, mdst); 1297 break; 1298 case OPENSSL_CHAIN_CIPHER_AUTH: 1299 process_openssl_cipher_op(op, sess, msrc, mdst); 1300 process_openssl_auth_op(op, sess, mdst, mdst); 1301 break; 1302 case OPENSSL_CHAIN_AUTH_CIPHER: 1303 process_openssl_auth_op(op, sess, msrc, mdst); 1304 process_openssl_cipher_op(op, sess, msrc, mdst); 1305 break; 1306 case OPENSSL_CHAIN_COMBINED: 1307 process_openssl_combined_op(op, sess, msrc, mdst); 1308 break; 1309 case OPENSSL_CHAIN_CIPHER_BPI: 1310 process_openssl_docsis_bpi_op(op, sess, msrc, mdst); 1311 break; 1312 default: 1313 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 1314 break; 1315 } 1316 1317 /* Free session if a session-less crypto op */ 1318 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 1319 openssl_reset_session(sess); 1320 memset(sess, 0, sizeof(struct openssl_session)); 1321 rte_mempool_put(qp->sess_mp, op->sym->session); 1322 op->sym->session = NULL; 1323 } 1324 1325 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 1326 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 1327 1328 if (op->status != RTE_CRYPTO_OP_STATUS_ERROR) 1329 retval = rte_ring_enqueue(qp->processed_ops, (void *)op); 1330 else 1331 retval = -1; 1332 1333 return retval; 1334 } 1335 1336 /* 1337 *------------------------------------------------------------------------------ 1338 * PMD Framework 1339 *------------------------------------------------------------------------------ 1340 */ 1341 1342 /** Enqueue burst */ 1343 static uint16_t 1344 openssl_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 1345 uint16_t nb_ops) 1346 { 1347 struct openssl_session *sess; 1348 struct openssl_qp *qp = queue_pair; 1349 int i, retval; 1350 1351 for (i = 0; i < nb_ops; i++) { 1352 sess = get_session(qp, ops[i]); 1353 if (unlikely(sess == NULL)) 1354 goto enqueue_err; 1355 1356 retval = process_op(qp, ops[i], sess); 1357 if (unlikely(retval < 0)) 1358 goto enqueue_err; 1359 } 1360 1361 qp->stats.enqueued_count += i; 1362 return i; 1363 1364 enqueue_err: 1365 qp->stats.enqueue_err_count++; 1366 return i; 1367 } 1368 1369 /** Dequeue burst */ 1370 static uint16_t 1371 openssl_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 1372 uint16_t nb_ops) 1373 { 1374 struct openssl_qp *qp = queue_pair; 1375 1376 unsigned int nb_dequeued = 0; 1377 1378 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 1379 (void **)ops, nb_ops, NULL); 1380 qp->stats.dequeued_count += nb_dequeued; 1381 1382 return nb_dequeued; 1383 } 1384 1385 /** Create OPENSSL crypto device */ 1386 static int 1387 cryptodev_openssl_create(const char *name, 1388 struct rte_vdev_device *vdev, 1389 struct rte_crypto_vdev_init_params *init_params) 1390 { 1391 struct rte_cryptodev *dev; 1392 struct openssl_private *internals; 1393 1394 if (init_params->name[0] == '\0') 1395 snprintf(init_params->name, sizeof(init_params->name), 1396 "%s", name); 1397 1398 dev = rte_cryptodev_vdev_pmd_init(init_params->name, 1399 sizeof(struct openssl_private), 1400 init_params->socket_id, 1401 vdev); 1402 if (dev == NULL) { 1403 OPENSSL_LOG_ERR("failed to create cryptodev vdev"); 1404 goto init_error; 1405 } 1406 1407 dev->dev_type = RTE_CRYPTODEV_OPENSSL_PMD; 1408 dev->dev_ops = rte_openssl_pmd_ops; 1409 1410 /* register rx/tx burst functions for data path */ 1411 dev->dequeue_burst = openssl_pmd_dequeue_burst; 1412 dev->enqueue_burst = openssl_pmd_enqueue_burst; 1413 1414 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 1415 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 1416 RTE_CRYPTODEV_FF_CPU_AESNI | 1417 RTE_CRYPTODEV_FF_MBUF_SCATTER_GATHER; 1418 1419 /* Set vector instructions mode supported */ 1420 internals = dev->data->dev_private; 1421 1422 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 1423 internals->max_nb_sessions = init_params->max_nb_sessions; 1424 1425 return 0; 1426 1427 init_error: 1428 OPENSSL_LOG_ERR("driver %s: cryptodev_openssl_create failed", 1429 init_params->name); 1430 1431 cryptodev_openssl_remove(vdev); 1432 return -EFAULT; 1433 } 1434 1435 /** Initialise OPENSSL crypto device */ 1436 static int 1437 cryptodev_openssl_probe(struct rte_vdev_device *vdev) 1438 { 1439 struct rte_crypto_vdev_init_params init_params = { 1440 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, 1441 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, 1442 rte_socket_id(), 1443 {0} 1444 }; 1445 const char *name; 1446 const char *input_args; 1447 1448 name = rte_vdev_device_name(vdev); 1449 if (name == NULL) 1450 return -EINVAL; 1451 input_args = rte_vdev_device_args(vdev); 1452 1453 rte_cryptodev_vdev_parse_init_params(&init_params, input_args); 1454 1455 RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, 1456 init_params.socket_id); 1457 if (init_params.name[0] != '\0') 1458 RTE_LOG(INFO, PMD, " User defined name = %s\n", 1459 init_params.name); 1460 RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", 1461 init_params.max_nb_queue_pairs); 1462 RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", 1463 init_params.max_nb_sessions); 1464 1465 return cryptodev_openssl_create(name, vdev, &init_params); 1466 } 1467 1468 /** Uninitialise OPENSSL crypto device */ 1469 static int 1470 cryptodev_openssl_remove(struct rte_vdev_device *vdev) 1471 { 1472 const char *name; 1473 1474 name = rte_vdev_device_name(vdev); 1475 if (name == NULL) 1476 return -EINVAL; 1477 1478 RTE_LOG(INFO, PMD, 1479 "Closing OPENSSL crypto device %s on numa socket %u\n", 1480 name, rte_socket_id()); 1481 1482 return 0; 1483 } 1484 1485 static struct rte_vdev_driver cryptodev_openssl_pmd_drv = { 1486 .probe = cryptodev_openssl_probe, 1487 .remove = cryptodev_openssl_remove 1488 }; 1489 1490 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_OPENSSL_PMD, 1491 cryptodev_openssl_pmd_drv); 1492 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_OPENSSL_PMD, 1493 "max_nb_queue_pairs=<int> " 1494 "max_nb_sessions=<int> " 1495 "socket_id=<int>"); 1496