1 /* 2 * BSD LICENSE 3 * 4 * Copyright (C) Cavium, Inc. 2017. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Cavium, Inc nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <stdbool.h> 34 35 #include <rte_common.h> 36 #include <rte_hexdump.h> 37 #include <rte_cryptodev.h> 38 #include <rte_cryptodev_pmd.h> 39 #include <rte_bus_vdev.h> 40 #include <rte_malloc.h> 41 #include <rte_cpuflags.h> 42 43 #include "armv8_crypto_defs.h" 44 45 #include "rte_armv8_pmd_private.h" 46 47 static uint8_t cryptodev_driver_id; 48 49 static int cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev); 50 51 /** 52 * Pointers to the supported combined mode crypto functions are stored 53 * in the static tables. Each combined (chained) cryptographic operation 54 * can be described by a set of numbers: 55 * - order: order of operations (cipher, auth) or (auth, cipher) 56 * - direction: encryption or decryption 57 * - calg: cipher algorithm such as AES_CBC, AES_CTR, etc. 58 * - aalg: authentication algorithm such as SHA1, SHA256, etc. 59 * - keyl: cipher key length, for example 128, 192, 256 bits 60 * 61 * In order to quickly acquire each function pointer based on those numbers, 62 * a hierarchy of arrays is maintained. The final level, 3D array is indexed 63 * by the combined mode function parameters only (cipher algorithm, 64 * authentication algorithm and key length). 65 * 66 * This gives 3 memory accesses to obtain a function pointer instead of 67 * traversing the array manually and comparing function parameters on each loop. 68 * 69 * +--+CRYPTO_FUNC 70 * +--+ENC| 71 * +--+CA| 72 * | +--+DEC 73 * ORDER| 74 * | +--+ENC 75 * +--+AC| 76 * +--+DEC 77 * 78 */ 79 80 /** 81 * 3D array type for ARM Combined Mode crypto functions pointers. 82 * CRYPTO_CIPHER_MAX: max cipher ID number 83 * CRYPTO_AUTH_MAX: max auth ID number 84 * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number 85 */ 86 typedef const crypto_func_t 87 crypto_func_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_AUTH_MAX][CRYPTO_CIPHER_KEYLEN_MAX]; 88 89 /* Evaluate to key length definition */ 90 #define KEYL(keyl) (ARMV8_CRYPTO_CIPHER_KEYLEN_ ## keyl) 91 92 /* Local aliases for supported ciphers */ 93 #define CIPH_AES_CBC RTE_CRYPTO_CIPHER_AES_CBC 94 /* Local aliases for supported hashes */ 95 #define AUTH_SHA1_HMAC RTE_CRYPTO_AUTH_SHA1_HMAC 96 #define AUTH_SHA256_HMAC RTE_CRYPTO_AUTH_SHA256_HMAC 97 98 /** 99 * Arrays containing pointers to particular cryptographic, 100 * combined mode functions. 101 * crypto_op_ca_encrypt: cipher (encrypt), authenticate 102 * crypto_op_ca_decrypt: cipher (decrypt), authenticate 103 * crypto_op_ac_encrypt: authenticate, cipher (encrypt) 104 * crypto_op_ac_decrypt: authenticate, cipher (decrypt) 105 */ 106 static const crypto_func_tbl_t 107 crypto_op_ca_encrypt = { 108 /* [cipher alg][auth alg][key length] = crypto_function, */ 109 [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = aes128cbc_sha1_hmac, 110 [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = aes128cbc_sha256_hmac, 111 }; 112 113 static const crypto_func_tbl_t 114 crypto_op_ca_decrypt = { 115 NULL 116 }; 117 118 static const crypto_func_tbl_t 119 crypto_op_ac_encrypt = { 120 NULL 121 }; 122 123 static const crypto_func_tbl_t 124 crypto_op_ac_decrypt = { 125 /* [cipher alg][auth alg][key length] = crypto_function, */ 126 [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = sha1_hmac_aes128cbc_dec, 127 [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = sha256_hmac_aes128cbc_dec, 128 }; 129 130 /** 131 * Arrays containing pointers to particular cryptographic function sets, 132 * covering given cipher operation directions (encrypt, decrypt) 133 * for each order of cipher and authentication pairs. 134 */ 135 static const crypto_func_tbl_t * 136 crypto_cipher_auth[] = { 137 &crypto_op_ca_encrypt, 138 &crypto_op_ca_decrypt, 139 NULL 140 }; 141 142 static const crypto_func_tbl_t * 143 crypto_auth_cipher[] = { 144 &crypto_op_ac_encrypt, 145 &crypto_op_ac_decrypt, 146 NULL 147 }; 148 149 /** 150 * Top level array containing pointers to particular cryptographic 151 * function sets, covering given order of chained operations. 152 * crypto_cipher_auth: cipher first, authenticate after 153 * crypto_auth_cipher: authenticate first, cipher after 154 */ 155 static const crypto_func_tbl_t ** 156 crypto_chain_order[] = { 157 crypto_cipher_auth, 158 crypto_auth_cipher, 159 NULL 160 }; 161 162 /** 163 * Extract particular combined mode crypto function from the 3D array. 164 */ 165 #define CRYPTO_GET_ALGO(order, cop, calg, aalg, keyl) \ 166 ({ \ 167 crypto_func_tbl_t *func_tbl = \ 168 (crypto_chain_order[(order)])[(cop)]; \ 169 \ 170 ((*func_tbl)[(calg)][(aalg)][KEYL(keyl)]); \ 171 }) 172 173 /*----------------------------------------------------------------------------*/ 174 175 /** 176 * 2D array type for ARM key schedule functions pointers. 177 * CRYPTO_CIPHER_MAX: max cipher ID number 178 * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number 179 */ 180 typedef const crypto_key_sched_t 181 crypto_key_sched_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_CIPHER_KEYLEN_MAX]; 182 183 static const crypto_key_sched_tbl_t 184 crypto_key_sched_encrypt = { 185 /* [cipher alg][key length] = key_expand_func, */ 186 [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_enc, 187 }; 188 189 static const crypto_key_sched_tbl_t 190 crypto_key_sched_decrypt = { 191 /* [cipher alg][key length] = key_expand_func, */ 192 [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_dec, 193 }; 194 195 /** 196 * Top level array containing pointers to particular key generation 197 * function sets, covering given operation direction. 198 * crypto_key_sched_encrypt: keys for encryption 199 * crypto_key_sched_decrypt: keys for decryption 200 */ 201 static const crypto_key_sched_tbl_t * 202 crypto_key_sched_dir[] = { 203 &crypto_key_sched_encrypt, 204 &crypto_key_sched_decrypt, 205 NULL 206 }; 207 208 /** 209 * Extract particular combined mode crypto function from the 3D array. 210 */ 211 #define CRYPTO_GET_KEY_SCHED(cop, calg, keyl) \ 212 ({ \ 213 crypto_key_sched_tbl_t *ks_tbl = crypto_key_sched_dir[(cop)]; \ 214 \ 215 ((*ks_tbl)[(calg)][KEYL(keyl)]); \ 216 }) 217 218 /*----------------------------------------------------------------------------*/ 219 220 /* 221 *------------------------------------------------------------------------------ 222 * Session Prepare 223 *------------------------------------------------------------------------------ 224 */ 225 226 /** Get xform chain order */ 227 static enum armv8_crypto_chain_order 228 armv8_crypto_get_chain_order(const struct rte_crypto_sym_xform *xform) 229 { 230 231 /* 232 * This driver currently covers only chained operations. 233 * Ignore only cipher or only authentication operations 234 * or chains longer than 2 xform structures. 235 */ 236 if (xform->next == NULL || xform->next->next != NULL) 237 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED; 238 239 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 240 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) 241 return ARMV8_CRYPTO_CHAIN_AUTH_CIPHER; 242 } 243 244 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 245 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 246 return ARMV8_CRYPTO_CHAIN_CIPHER_AUTH; 247 } 248 249 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED; 250 } 251 252 static inline void 253 auth_hmac_pad_prepare(struct armv8_crypto_session *sess, 254 const struct rte_crypto_sym_xform *xform) 255 { 256 size_t i; 257 258 /* Generate i_key_pad and o_key_pad */ 259 memset(sess->auth.hmac.i_key_pad, 0, sizeof(sess->auth.hmac.i_key_pad)); 260 rte_memcpy(sess->auth.hmac.i_key_pad, sess->auth.hmac.key, 261 xform->auth.key.length); 262 memset(sess->auth.hmac.o_key_pad, 0, sizeof(sess->auth.hmac.o_key_pad)); 263 rte_memcpy(sess->auth.hmac.o_key_pad, sess->auth.hmac.key, 264 xform->auth.key.length); 265 /* 266 * XOR key with IPAD/OPAD values to obtain i_key_pad 267 * and o_key_pad. 268 * Byte-by-byte operation may seem to be the less efficient 269 * here but in fact it's the opposite. 270 * The result ASM code is likely operate on NEON registers 271 * (load auth key to Qx, load IPAD/OPAD to multiple 272 * elements of Qy, eor 128 bits at once). 273 */ 274 for (i = 0; i < SHA_BLOCK_MAX; i++) { 275 sess->auth.hmac.i_key_pad[i] ^= HMAC_IPAD_VALUE; 276 sess->auth.hmac.o_key_pad[i] ^= HMAC_OPAD_VALUE; 277 } 278 } 279 280 static inline int 281 auth_set_prerequisites(struct armv8_crypto_session *sess, 282 const struct rte_crypto_sym_xform *xform) 283 { 284 uint8_t partial[64] = { 0 }; 285 int error; 286 287 switch (xform->auth.algo) { 288 case RTE_CRYPTO_AUTH_SHA1_HMAC: 289 /* 290 * Generate authentication key, i_key_pad and o_key_pad. 291 */ 292 /* Zero memory under key */ 293 memset(sess->auth.hmac.key, 0, SHA1_BLOCK_SIZE); 294 295 /* 296 * Now copy the given authentication key to the session 297 * key. 298 */ 299 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, 300 xform->auth.key.length); 301 302 /* Prepare HMAC padding: key|pattern */ 303 auth_hmac_pad_prepare(sess, xform); 304 /* 305 * Calculate partial hash values for i_key_pad and o_key_pad. 306 * Will be used as initialization state for final HMAC. 307 */ 308 error = sha1_block_partial(NULL, sess->auth.hmac.i_key_pad, 309 partial, SHA1_BLOCK_SIZE); 310 if (error != 0) 311 return -1; 312 memcpy(sess->auth.hmac.i_key_pad, partial, SHA1_BLOCK_SIZE); 313 314 error = sha1_block_partial(NULL, sess->auth.hmac.o_key_pad, 315 partial, SHA1_BLOCK_SIZE); 316 if (error != 0) 317 return -1; 318 memcpy(sess->auth.hmac.o_key_pad, partial, SHA1_BLOCK_SIZE); 319 320 break; 321 case RTE_CRYPTO_AUTH_SHA256_HMAC: 322 /* 323 * Generate authentication key, i_key_pad and o_key_pad. 324 */ 325 /* Zero memory under key */ 326 memset(sess->auth.hmac.key, 0, SHA256_BLOCK_SIZE); 327 328 /* 329 * Now copy the given authentication key to the session 330 * key. 331 */ 332 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, 333 xform->auth.key.length); 334 335 /* Prepare HMAC padding: key|pattern */ 336 auth_hmac_pad_prepare(sess, xform); 337 /* 338 * Calculate partial hash values for i_key_pad and o_key_pad. 339 * Will be used as initialization state for final HMAC. 340 */ 341 error = sha256_block_partial(NULL, sess->auth.hmac.i_key_pad, 342 partial, SHA256_BLOCK_SIZE); 343 if (error != 0) 344 return -1; 345 memcpy(sess->auth.hmac.i_key_pad, partial, SHA256_BLOCK_SIZE); 346 347 error = sha256_block_partial(NULL, sess->auth.hmac.o_key_pad, 348 partial, SHA256_BLOCK_SIZE); 349 if (error != 0) 350 return -1; 351 memcpy(sess->auth.hmac.o_key_pad, partial, SHA256_BLOCK_SIZE); 352 353 break; 354 default: 355 break; 356 } 357 358 return 0; 359 } 360 361 static inline int 362 cipher_set_prerequisites(struct armv8_crypto_session *sess, 363 const struct rte_crypto_sym_xform *xform) 364 { 365 crypto_key_sched_t cipher_key_sched; 366 367 cipher_key_sched = sess->cipher.key_sched; 368 if (likely(cipher_key_sched != NULL)) { 369 /* Set up cipher session key */ 370 cipher_key_sched(sess->cipher.key.data, xform->cipher.key.data); 371 } 372 373 return 0; 374 } 375 376 static int 377 armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess, 378 const struct rte_crypto_sym_xform *cipher_xform, 379 const struct rte_crypto_sym_xform *auth_xform) 380 { 381 enum armv8_crypto_chain_order order; 382 enum armv8_crypto_cipher_operation cop; 383 enum rte_crypto_cipher_algorithm calg; 384 enum rte_crypto_auth_algorithm aalg; 385 386 /* Validate and prepare scratch order of combined operations */ 387 switch (sess->chain_order) { 388 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: 389 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: 390 order = sess->chain_order; 391 break; 392 default: 393 return -ENOTSUP; 394 } 395 /* Select cipher direction */ 396 sess->cipher.direction = cipher_xform->cipher.op; 397 /* Select cipher key */ 398 sess->cipher.key.length = cipher_xform->cipher.key.length; 399 /* Set cipher direction */ 400 cop = sess->cipher.direction; 401 /* Set cipher algorithm */ 402 calg = cipher_xform->cipher.algo; 403 404 /* Select cipher algo */ 405 switch (calg) { 406 /* Cover supported cipher algorithms */ 407 case RTE_CRYPTO_CIPHER_AES_CBC: 408 sess->cipher.algo = calg; 409 /* IV len is always 16 bytes (block size) for AES CBC */ 410 sess->cipher.iv.length = 16; 411 break; 412 default: 413 return -ENOTSUP; 414 } 415 /* Select auth generate/verify */ 416 sess->auth.operation = auth_xform->auth.op; 417 418 /* Select auth algo */ 419 switch (auth_xform->auth.algo) { 420 /* Cover supported hash algorithms */ 421 case RTE_CRYPTO_AUTH_SHA1_HMAC: 422 case RTE_CRYPTO_AUTH_SHA256_HMAC: /* Fall through */ 423 aalg = auth_xform->auth.algo; 424 sess->auth.mode = ARMV8_CRYPTO_AUTH_AS_HMAC; 425 break; 426 default: 427 return -ENOTSUP; 428 } 429 430 /* Set the digest length */ 431 sess->auth.digest_length = auth_xform->auth.digest_length; 432 433 /* Verify supported key lengths and extract proper algorithm */ 434 switch (cipher_xform->cipher.key.length << 3) { 435 case 128: 436 sess->crypto_func = 437 CRYPTO_GET_ALGO(order, cop, calg, aalg, 128); 438 sess->cipher.key_sched = 439 CRYPTO_GET_KEY_SCHED(cop, calg, 128); 440 break; 441 case 192: 442 case 256: 443 /* These key lengths are not supported yet */ 444 default: /* Fall through */ 445 sess->crypto_func = NULL; 446 sess->cipher.key_sched = NULL; 447 return -ENOTSUP; 448 } 449 450 if (unlikely(sess->crypto_func == NULL)) { 451 /* 452 * If we got here that means that there must be a bug 453 * in the algorithms selection above. Nevertheless keep 454 * it here to catch bug immediately and avoid NULL pointer 455 * dereference in OPs processing. 456 */ 457 ARMV8_CRYPTO_LOG_ERR( 458 "No appropriate crypto function for given parameters"); 459 return -EINVAL; 460 } 461 462 /* Set up cipher session prerequisites */ 463 if (cipher_set_prerequisites(sess, cipher_xform) != 0) 464 return -EINVAL; 465 466 /* Set up authentication session prerequisites */ 467 if (auth_set_prerequisites(sess, auth_xform) != 0) 468 return -EINVAL; 469 470 return 0; 471 } 472 473 /** Parse crypto xform chain and set private session parameters */ 474 int 475 armv8_crypto_set_session_parameters(struct armv8_crypto_session *sess, 476 const struct rte_crypto_sym_xform *xform) 477 { 478 const struct rte_crypto_sym_xform *cipher_xform = NULL; 479 const struct rte_crypto_sym_xform *auth_xform = NULL; 480 bool is_chained_op; 481 int ret; 482 483 /* Filter out spurious/broken requests */ 484 if (xform == NULL) 485 return -EINVAL; 486 487 sess->chain_order = armv8_crypto_get_chain_order(xform); 488 switch (sess->chain_order) { 489 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: 490 cipher_xform = xform; 491 auth_xform = xform->next; 492 is_chained_op = true; 493 break; 494 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: 495 auth_xform = xform; 496 cipher_xform = xform->next; 497 is_chained_op = true; 498 break; 499 default: 500 is_chained_op = false; 501 return -ENOTSUP; 502 } 503 504 /* Set IV offset */ 505 sess->cipher.iv.offset = cipher_xform->cipher.iv.offset; 506 507 if (is_chained_op) { 508 ret = armv8_crypto_set_session_chained_parameters(sess, 509 cipher_xform, auth_xform); 510 if (unlikely(ret != 0)) { 511 ARMV8_CRYPTO_LOG_ERR( 512 "Invalid/unsupported chained (cipher/auth) parameters"); 513 return ret; 514 } 515 } else { 516 ARMV8_CRYPTO_LOG_ERR("Invalid/unsupported operation"); 517 return -ENOTSUP; 518 } 519 520 return 0; 521 } 522 523 /** Provide session for operation */ 524 static inline struct armv8_crypto_session * 525 get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op) 526 { 527 struct armv8_crypto_session *sess = NULL; 528 529 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 530 /* get existing session */ 531 if (likely(op->sym->session != NULL)) { 532 sess = (struct armv8_crypto_session *) 533 get_session_private_data( 534 op->sym->session, 535 cryptodev_driver_id); 536 } 537 } else { 538 /* provide internal session */ 539 void *_sess = NULL; 540 void *_sess_private_data = NULL; 541 542 if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) 543 return NULL; 544 545 if (rte_mempool_get(qp->sess_mp, (void **)&_sess_private_data)) 546 return NULL; 547 548 sess = (struct armv8_crypto_session *)_sess_private_data; 549 550 if (unlikely(armv8_crypto_set_session_parameters(sess, 551 op->sym->xform) != 0)) { 552 rte_mempool_put(qp->sess_mp, _sess); 553 rte_mempool_put(qp->sess_mp, _sess_private_data); 554 sess = NULL; 555 } 556 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 557 set_session_private_data(op->sym->session, cryptodev_driver_id, 558 _sess_private_data); 559 } 560 561 if (unlikely(sess == NULL)) 562 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 563 564 return sess; 565 } 566 567 /* 568 *------------------------------------------------------------------------------ 569 * Process Operations 570 *------------------------------------------------------------------------------ 571 */ 572 573 /*----------------------------------------------------------------------------*/ 574 575 /** Process cipher operation */ 576 static inline void 577 process_armv8_chained_op(struct armv8_crypto_qp *qp, struct rte_crypto_op *op, 578 struct armv8_crypto_session *sess, 579 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 580 { 581 crypto_func_t crypto_func; 582 crypto_arg_t arg; 583 struct rte_mbuf *m_asrc, *m_adst; 584 uint8_t *csrc, *cdst; 585 uint8_t *adst, *asrc; 586 uint64_t clen, alen; 587 int error; 588 589 clen = op->sym->cipher.data.length; 590 alen = op->sym->auth.data.length; 591 592 csrc = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 593 op->sym->cipher.data.offset); 594 cdst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 595 op->sym->cipher.data.offset); 596 597 switch (sess->chain_order) { 598 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: 599 m_asrc = m_adst = mbuf_dst; 600 break; 601 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: 602 m_asrc = mbuf_src; 603 m_adst = mbuf_dst; 604 break; 605 default: 606 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 607 return; 608 } 609 asrc = rte_pktmbuf_mtod_offset(m_asrc, uint8_t *, 610 op->sym->auth.data.offset); 611 612 switch (sess->auth.mode) { 613 case ARMV8_CRYPTO_AUTH_AS_AUTH: 614 /* Nothing to do here, just verify correct option */ 615 break; 616 case ARMV8_CRYPTO_AUTH_AS_HMAC: 617 arg.digest.hmac.key = sess->auth.hmac.key; 618 arg.digest.hmac.i_key_pad = sess->auth.hmac.i_key_pad; 619 arg.digest.hmac.o_key_pad = sess->auth.hmac.o_key_pad; 620 break; 621 default: 622 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 623 return; 624 } 625 626 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) { 627 adst = op->sym->auth.digest.data; 628 if (adst == NULL) { 629 adst = rte_pktmbuf_mtod_offset(m_adst, 630 uint8_t *, 631 op->sym->auth.data.offset + 632 op->sym->auth.data.length); 633 } 634 } else { 635 adst = qp->temp_digest; 636 } 637 638 arg.cipher.iv = rte_crypto_op_ctod_offset(op, uint8_t *, 639 sess->cipher.iv.offset); 640 arg.cipher.key = sess->cipher.key.data; 641 /* Acquire combined mode function */ 642 crypto_func = sess->crypto_func; 643 ARMV8_CRYPTO_ASSERT(crypto_func != NULL); 644 error = crypto_func(csrc, cdst, clen, asrc, adst, alen, &arg); 645 if (error != 0) { 646 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 647 return; 648 } 649 650 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 651 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 652 if (memcmp(adst, op->sym->auth.digest.data, 653 sess->auth.digest_length) != 0) { 654 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 655 } 656 } 657 } 658 659 /** Process crypto operation for mbuf */ 660 static inline int 661 process_op(struct armv8_crypto_qp *qp, struct rte_crypto_op *op, 662 struct armv8_crypto_session *sess) 663 { 664 struct rte_mbuf *msrc, *mdst; 665 666 msrc = op->sym->m_src; 667 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 668 669 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 670 671 switch (sess->chain_order) { 672 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: 673 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: /* Fall through */ 674 process_armv8_chained_op(qp, op, sess, msrc, mdst); 675 break; 676 default: 677 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 678 break; 679 } 680 681 /* Free session if a session-less crypto op */ 682 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 683 memset(sess, 0, sizeof(struct armv8_crypto_session)); 684 memset(op->sym->session, 0, 685 rte_cryptodev_get_header_session_size()); 686 rte_mempool_put(qp->sess_mp, sess); 687 rte_mempool_put(qp->sess_mp, op->sym->session); 688 op->sym->session = NULL; 689 } 690 691 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 692 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 693 694 if (unlikely(op->status == RTE_CRYPTO_OP_STATUS_ERROR)) 695 return -1; 696 697 return 0; 698 } 699 700 /* 701 *------------------------------------------------------------------------------ 702 * PMD Framework 703 *------------------------------------------------------------------------------ 704 */ 705 706 /** Enqueue burst */ 707 static uint16_t 708 armv8_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 709 uint16_t nb_ops) 710 { 711 struct armv8_crypto_session *sess; 712 struct armv8_crypto_qp *qp = queue_pair; 713 int i, retval; 714 715 for (i = 0; i < nb_ops; i++) { 716 sess = get_session(qp, ops[i]); 717 if (unlikely(sess == NULL)) 718 goto enqueue_err; 719 720 retval = process_op(qp, ops[i], sess); 721 if (unlikely(retval < 0)) 722 goto enqueue_err; 723 } 724 725 retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i, 726 NULL); 727 qp->stats.enqueued_count += retval; 728 729 return retval; 730 731 enqueue_err: 732 retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i, 733 NULL); 734 if (ops[i] != NULL) 735 ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 736 737 qp->stats.enqueue_err_count++; 738 return retval; 739 } 740 741 /** Dequeue burst */ 742 static uint16_t 743 armv8_crypto_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 744 uint16_t nb_ops) 745 { 746 struct armv8_crypto_qp *qp = queue_pair; 747 748 unsigned int nb_dequeued = 0; 749 750 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 751 (void **)ops, nb_ops, NULL); 752 qp->stats.dequeued_count += nb_dequeued; 753 754 return nb_dequeued; 755 } 756 757 /** Create ARMv8 crypto device */ 758 static int 759 cryptodev_armv8_crypto_create(const char *name, 760 struct rte_vdev_device *vdev, 761 struct rte_cryptodev_pmd_init_params *init_params) 762 { 763 struct rte_cryptodev *dev; 764 struct armv8_crypto_private *internals; 765 766 /* Check CPU for support for AES instruction set */ 767 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) { 768 ARMV8_CRYPTO_LOG_ERR( 769 "AES instructions not supported by CPU"); 770 return -EFAULT; 771 } 772 773 /* Check CPU for support for SHA instruction set */ 774 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA1) || 775 !rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA2)) { 776 ARMV8_CRYPTO_LOG_ERR( 777 "SHA1/SHA2 instructions not supported by CPU"); 778 return -EFAULT; 779 } 780 781 /* Check CPU for support for Advance SIMD instruction set */ 782 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) { 783 ARMV8_CRYPTO_LOG_ERR( 784 "Advanced SIMD instructions not supported by CPU"); 785 return -EFAULT; 786 } 787 788 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 789 if (dev == NULL) { 790 ARMV8_CRYPTO_LOG_ERR("failed to create cryptodev vdev"); 791 goto init_error; 792 } 793 794 dev->driver_id = cryptodev_driver_id; 795 dev->dev_ops = rte_armv8_crypto_pmd_ops; 796 797 /* register rx/tx burst functions for data path */ 798 dev->dequeue_burst = armv8_crypto_pmd_dequeue_burst; 799 dev->enqueue_burst = armv8_crypto_pmd_enqueue_burst; 800 801 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 802 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 803 RTE_CRYPTODEV_FF_CPU_NEON | 804 RTE_CRYPTODEV_FF_CPU_ARM_CE; 805 806 /* Set vector instructions mode supported */ 807 internals = dev->data->dev_private; 808 809 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 810 internals->max_nb_sessions = init_params->max_nb_sessions; 811 812 return 0; 813 814 init_error: 815 ARMV8_CRYPTO_LOG_ERR( 816 "driver %s: cryptodev_armv8_crypto_create failed", 817 init_params->name); 818 819 cryptodev_armv8_crypto_uninit(vdev); 820 return -EFAULT; 821 } 822 823 /** Initialise ARMv8 crypto device */ 824 static int 825 cryptodev_armv8_crypto_init(struct rte_vdev_device *vdev) 826 { 827 struct rte_cryptodev_pmd_init_params init_params = { 828 "", 829 sizeof(struct armv8_crypto_private), 830 rte_socket_id(), 831 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS, 832 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_SESSIONS 833 }; 834 const char *name; 835 const char *input_args; 836 837 name = rte_vdev_device_name(vdev); 838 if (name == NULL) 839 return -EINVAL; 840 input_args = rte_vdev_device_args(vdev); 841 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 842 843 return cryptodev_armv8_crypto_create(name, vdev, &init_params); 844 } 845 846 /** Uninitialise ARMv8 crypto device */ 847 static int 848 cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev) 849 { 850 struct rte_cryptodev *cryptodev; 851 const char *name; 852 853 name = rte_vdev_device_name(vdev); 854 if (name == NULL) 855 return -EINVAL; 856 857 RTE_LOG(INFO, PMD, 858 "Closing ARMv8 crypto device %s on numa socket %u\n", 859 name, rte_socket_id()); 860 861 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 862 if (cryptodev == NULL) 863 return -ENODEV; 864 865 return rte_cryptodev_pmd_destroy(cryptodev); 866 } 867 868 static struct rte_vdev_driver armv8_crypto_pmd_drv = { 869 .probe = cryptodev_armv8_crypto_init, 870 .remove = cryptodev_armv8_crypto_uninit 871 }; 872 873 static struct cryptodev_driver armv8_crypto_drv; 874 875 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ARMV8_PMD, armv8_crypto_pmd_drv); 876 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_ARMV8_PMD, cryptodev_armv8_pmd); 877 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ARMV8_PMD, 878 "max_nb_queue_pairs=<int> " 879 "max_nb_sessions=<int> " 880 "socket_id=<int>"); 881 RTE_PMD_REGISTER_CRYPTO_DRIVER(armv8_crypto_drv, armv8_crypto_pmd_drv, 882 cryptodev_driver_id); 883