1 /* 2 * BSD LICENSE 3 * 4 * Copyright (C) Cavium networks Ltd. 2017. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 10 * * Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * * Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in 14 * the documentation and/or other materials provided with the 15 * distribution. 16 * * Neither the name of Cavium networks nor the names of its 17 * contributors may be used to endorse or promote products derived 18 * from this software without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 21 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 22 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 23 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 24 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 25 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 26 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 30 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #include <stdbool.h> 34 35 #include <rte_common.h> 36 #include <rte_hexdump.h> 37 #include <rte_cryptodev.h> 38 #include <rte_cryptodev_pmd.h> 39 #include <rte_cryptodev_vdev.h> 40 #include <rte_vdev.h> 41 #include <rte_malloc.h> 42 #include <rte_cpuflags.h> 43 44 #include "armv8_crypto_defs.h" 45 46 #include "rte_armv8_pmd_private.h" 47 48 static int cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev); 49 50 /** 51 * Pointers to the supported combined mode crypto functions are stored 52 * in the static tables. Each combined (chained) cryptographic operation 53 * can be described by a set of numbers: 54 * - order: order of operations (cipher, auth) or (auth, cipher) 55 * - direction: encryption or decryption 56 * - calg: cipher algorithm such as AES_CBC, AES_CTR, etc. 57 * - aalg: authentication algorithm such as SHA1, SHA256, etc. 58 * - keyl: cipher key length, for example 128, 192, 256 bits 59 * 60 * In order to quickly acquire each function pointer based on those numbers, 61 * a hierarchy of arrays is maintained. The final level, 3D array is indexed 62 * by the combined mode function parameters only (cipher algorithm, 63 * authentication algorithm and key length). 64 * 65 * This gives 3 memory accesses to obtain a function pointer instead of 66 * traversing the array manually and comparing function parameters on each loop. 67 * 68 * +--+CRYPTO_FUNC 69 * +--+ENC| 70 * +--+CA| 71 * | +--+DEC 72 * ORDER| 73 * | +--+ENC 74 * +--+AC| 75 * +--+DEC 76 * 77 */ 78 79 /** 80 * 3D array type for ARM Combined Mode crypto functions pointers. 81 * CRYPTO_CIPHER_MAX: max cipher ID number 82 * CRYPTO_AUTH_MAX: max auth ID number 83 * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number 84 */ 85 typedef const crypto_func_t 86 crypto_func_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_AUTH_MAX][CRYPTO_CIPHER_KEYLEN_MAX]; 87 88 /* Evaluate to key length definition */ 89 #define KEYL(keyl) (ARMV8_CRYPTO_CIPHER_KEYLEN_ ## keyl) 90 91 /* Local aliases for supported ciphers */ 92 #define CIPH_AES_CBC RTE_CRYPTO_CIPHER_AES_CBC 93 /* Local aliases for supported hashes */ 94 #define AUTH_SHA1_HMAC RTE_CRYPTO_AUTH_SHA1_HMAC 95 #define AUTH_SHA256_HMAC RTE_CRYPTO_AUTH_SHA256_HMAC 96 97 /** 98 * Arrays containing pointers to particular cryptographic, 99 * combined mode functions. 100 * crypto_op_ca_encrypt: cipher (encrypt), authenticate 101 * crypto_op_ca_decrypt: cipher (decrypt), authenticate 102 * crypto_op_ac_encrypt: authenticate, cipher (encrypt) 103 * crypto_op_ac_decrypt: authenticate, cipher (decrypt) 104 */ 105 static const crypto_func_tbl_t 106 crypto_op_ca_encrypt = { 107 /* [cipher alg][auth alg][key length] = crypto_function, */ 108 [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = aes128cbc_sha1_hmac, 109 [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = aes128cbc_sha256_hmac, 110 }; 111 112 static const crypto_func_tbl_t 113 crypto_op_ca_decrypt = { 114 NULL 115 }; 116 117 static const crypto_func_tbl_t 118 crypto_op_ac_encrypt = { 119 NULL 120 }; 121 122 static const crypto_func_tbl_t 123 crypto_op_ac_decrypt = { 124 /* [cipher alg][auth alg][key length] = crypto_function, */ 125 [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = sha1_hmac_aes128cbc_dec, 126 [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = sha256_hmac_aes128cbc_dec, 127 }; 128 129 /** 130 * Arrays containing pointers to particular cryptographic function sets, 131 * covering given cipher operation directions (encrypt, decrypt) 132 * for each order of cipher and authentication pairs. 133 */ 134 static const crypto_func_tbl_t * 135 crypto_cipher_auth[] = { 136 &crypto_op_ca_encrypt, 137 &crypto_op_ca_decrypt, 138 NULL 139 }; 140 141 static const crypto_func_tbl_t * 142 crypto_auth_cipher[] = { 143 &crypto_op_ac_encrypt, 144 &crypto_op_ac_decrypt, 145 NULL 146 }; 147 148 /** 149 * Top level array containing pointers to particular cryptographic 150 * function sets, covering given order of chained operations. 151 * crypto_cipher_auth: cipher first, authenticate after 152 * crypto_auth_cipher: authenticate first, cipher after 153 */ 154 static const crypto_func_tbl_t ** 155 crypto_chain_order[] = { 156 crypto_cipher_auth, 157 crypto_auth_cipher, 158 NULL 159 }; 160 161 /** 162 * Extract particular combined mode crypto function from the 3D array. 163 */ 164 #define CRYPTO_GET_ALGO(order, cop, calg, aalg, keyl) \ 165 ({ \ 166 crypto_func_tbl_t *func_tbl = \ 167 (crypto_chain_order[(order)])[(cop)]; \ 168 \ 169 ((*func_tbl)[(calg)][(aalg)][KEYL(keyl)]); \ 170 }) 171 172 /*----------------------------------------------------------------------------*/ 173 174 /** 175 * 2D array type for ARM key schedule functions pointers. 176 * CRYPTO_CIPHER_MAX: max cipher ID number 177 * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number 178 */ 179 typedef const crypto_key_sched_t 180 crypto_key_sched_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_CIPHER_KEYLEN_MAX]; 181 182 static const crypto_key_sched_tbl_t 183 crypto_key_sched_encrypt = { 184 /* [cipher alg][key length] = key_expand_func, */ 185 [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_enc, 186 }; 187 188 static const crypto_key_sched_tbl_t 189 crypto_key_sched_decrypt = { 190 /* [cipher alg][key length] = key_expand_func, */ 191 [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_dec, 192 }; 193 194 /** 195 * Top level array containing pointers to particular key generation 196 * function sets, covering given operation direction. 197 * crypto_key_sched_encrypt: keys for encryption 198 * crypto_key_sched_decrypt: keys for decryption 199 */ 200 static const crypto_key_sched_tbl_t * 201 crypto_key_sched_dir[] = { 202 &crypto_key_sched_encrypt, 203 &crypto_key_sched_decrypt, 204 NULL 205 }; 206 207 /** 208 * Extract particular combined mode crypto function from the 3D array. 209 */ 210 #define CRYPTO_GET_KEY_SCHED(cop, calg, keyl) \ 211 ({ \ 212 crypto_key_sched_tbl_t *ks_tbl = crypto_key_sched_dir[(cop)]; \ 213 \ 214 ((*ks_tbl)[(calg)][KEYL(keyl)]); \ 215 }) 216 217 /*----------------------------------------------------------------------------*/ 218 219 /* 220 *------------------------------------------------------------------------------ 221 * Session Prepare 222 *------------------------------------------------------------------------------ 223 */ 224 225 /** Get xform chain order */ 226 static enum armv8_crypto_chain_order 227 armv8_crypto_get_chain_order(const struct rte_crypto_sym_xform *xform) 228 { 229 230 /* 231 * This driver currently covers only chained operations. 232 * Ignore only cipher or only authentication operations 233 * or chains longer than 2 xform structures. 234 */ 235 if (xform->next == NULL || xform->next->next != NULL) 236 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED; 237 238 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 239 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) 240 return ARMV8_CRYPTO_CHAIN_AUTH_CIPHER; 241 } 242 243 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 244 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 245 return ARMV8_CRYPTO_CHAIN_CIPHER_AUTH; 246 } 247 248 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED; 249 } 250 251 static inline void 252 auth_hmac_pad_prepare(struct armv8_crypto_session *sess, 253 const struct rte_crypto_sym_xform *xform) 254 { 255 size_t i; 256 257 /* Generate i_key_pad and o_key_pad */ 258 memset(sess->auth.hmac.i_key_pad, 0, sizeof(sess->auth.hmac.i_key_pad)); 259 rte_memcpy(sess->auth.hmac.i_key_pad, sess->auth.hmac.key, 260 xform->auth.key.length); 261 memset(sess->auth.hmac.o_key_pad, 0, sizeof(sess->auth.hmac.o_key_pad)); 262 rte_memcpy(sess->auth.hmac.o_key_pad, sess->auth.hmac.key, 263 xform->auth.key.length); 264 /* 265 * XOR key with IPAD/OPAD values to obtain i_key_pad 266 * and o_key_pad. 267 * Byte-by-byte operation may seem to be the less efficient 268 * here but in fact it's the opposite. 269 * The result ASM code is likely operate on NEON registers 270 * (load auth key to Qx, load IPAD/OPAD to multiple 271 * elements of Qy, eor 128 bits at once). 272 */ 273 for (i = 0; i < SHA_BLOCK_MAX; i++) { 274 sess->auth.hmac.i_key_pad[i] ^= HMAC_IPAD_VALUE; 275 sess->auth.hmac.o_key_pad[i] ^= HMAC_OPAD_VALUE; 276 } 277 } 278 279 static inline int 280 auth_set_prerequisites(struct armv8_crypto_session *sess, 281 const struct rte_crypto_sym_xform *xform) 282 { 283 uint8_t partial[64] = { 0 }; 284 int error; 285 286 switch (xform->auth.algo) { 287 case RTE_CRYPTO_AUTH_SHA1_HMAC: 288 /* 289 * Generate authentication key, i_key_pad and o_key_pad. 290 */ 291 /* Zero memory under key */ 292 memset(sess->auth.hmac.key, 0, SHA1_AUTH_KEY_LENGTH); 293 294 if (xform->auth.key.length > SHA1_AUTH_KEY_LENGTH) { 295 /* 296 * In case the key is longer than 160 bits 297 * the algorithm will use SHA1(key) instead. 298 */ 299 error = sha1_block(NULL, xform->auth.key.data, 300 sess->auth.hmac.key, xform->auth.key.length); 301 if (error != 0) 302 return -1; 303 } else { 304 /* 305 * Now copy the given authentication key to the session 306 * key assuming that the session key is zeroed there is 307 * no need for additional zero padding if the key is 308 * shorter than SHA1_AUTH_KEY_LENGTH. 309 */ 310 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, 311 xform->auth.key.length); 312 } 313 314 /* Prepare HMAC padding: key|pattern */ 315 auth_hmac_pad_prepare(sess, xform); 316 /* 317 * Calculate partial hash values for i_key_pad and o_key_pad. 318 * Will be used as initialization state for final HMAC. 319 */ 320 error = sha1_block_partial(NULL, sess->auth.hmac.i_key_pad, 321 partial, SHA1_BLOCK_SIZE); 322 if (error != 0) 323 return -1; 324 memcpy(sess->auth.hmac.i_key_pad, partial, SHA1_BLOCK_SIZE); 325 326 error = sha1_block_partial(NULL, sess->auth.hmac.o_key_pad, 327 partial, SHA1_BLOCK_SIZE); 328 if (error != 0) 329 return -1; 330 memcpy(sess->auth.hmac.o_key_pad, partial, SHA1_BLOCK_SIZE); 331 332 break; 333 case RTE_CRYPTO_AUTH_SHA256_HMAC: 334 /* 335 * Generate authentication key, i_key_pad and o_key_pad. 336 */ 337 /* Zero memory under key */ 338 memset(sess->auth.hmac.key, 0, SHA256_AUTH_KEY_LENGTH); 339 340 if (xform->auth.key.length > SHA256_AUTH_KEY_LENGTH) { 341 /* 342 * In case the key is longer than 256 bits 343 * the algorithm will use SHA256(key) instead. 344 */ 345 error = sha256_block(NULL, xform->auth.key.data, 346 sess->auth.hmac.key, xform->auth.key.length); 347 if (error != 0) 348 return -1; 349 } else { 350 /* 351 * Now copy the given authentication key to the session 352 * key assuming that the session key is zeroed there is 353 * no need for additional zero padding if the key is 354 * shorter than SHA256_AUTH_KEY_LENGTH. 355 */ 356 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, 357 xform->auth.key.length); 358 } 359 360 /* Prepare HMAC padding: key|pattern */ 361 auth_hmac_pad_prepare(sess, xform); 362 /* 363 * Calculate partial hash values for i_key_pad and o_key_pad. 364 * Will be used as initialization state for final HMAC. 365 */ 366 error = sha256_block_partial(NULL, sess->auth.hmac.i_key_pad, 367 partial, SHA256_BLOCK_SIZE); 368 if (error != 0) 369 return -1; 370 memcpy(sess->auth.hmac.i_key_pad, partial, SHA256_BLOCK_SIZE); 371 372 error = sha256_block_partial(NULL, sess->auth.hmac.o_key_pad, 373 partial, SHA256_BLOCK_SIZE); 374 if (error != 0) 375 return -1; 376 memcpy(sess->auth.hmac.o_key_pad, partial, SHA256_BLOCK_SIZE); 377 378 break; 379 default: 380 break; 381 } 382 383 return 0; 384 } 385 386 static inline int 387 cipher_set_prerequisites(struct armv8_crypto_session *sess, 388 const struct rte_crypto_sym_xform *xform) 389 { 390 crypto_key_sched_t cipher_key_sched; 391 392 cipher_key_sched = sess->cipher.key_sched; 393 if (likely(cipher_key_sched != NULL)) { 394 /* Set up cipher session key */ 395 cipher_key_sched(sess->cipher.key.data, xform->cipher.key.data); 396 } 397 398 return 0; 399 } 400 401 static int 402 armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess, 403 const struct rte_crypto_sym_xform *cipher_xform, 404 const struct rte_crypto_sym_xform *auth_xform) 405 { 406 enum armv8_crypto_chain_order order; 407 enum armv8_crypto_cipher_operation cop; 408 enum rte_crypto_cipher_algorithm calg; 409 enum rte_crypto_auth_algorithm aalg; 410 411 /* Validate and prepare scratch order of combined operations */ 412 switch (sess->chain_order) { 413 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: 414 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: 415 order = sess->chain_order; 416 break; 417 default: 418 return -EINVAL; 419 } 420 /* Select cipher direction */ 421 sess->cipher.direction = cipher_xform->cipher.op; 422 /* Select cipher key */ 423 sess->cipher.key.length = cipher_xform->cipher.key.length; 424 /* Set cipher direction */ 425 cop = sess->cipher.direction; 426 /* Set cipher algorithm */ 427 calg = cipher_xform->cipher.algo; 428 429 /* Select cipher algo */ 430 switch (calg) { 431 /* Cover supported cipher algorithms */ 432 case RTE_CRYPTO_CIPHER_AES_CBC: 433 sess->cipher.algo = calg; 434 /* IV len is always 16 bytes (block size) for AES CBC */ 435 sess->cipher.iv.length = 16; 436 break; 437 default: 438 return -EINVAL; 439 } 440 /* Select auth generate/verify */ 441 sess->auth.operation = auth_xform->auth.op; 442 443 /* Select auth algo */ 444 switch (auth_xform->auth.algo) { 445 /* Cover supported hash algorithms */ 446 case RTE_CRYPTO_AUTH_SHA1_HMAC: 447 case RTE_CRYPTO_AUTH_SHA256_HMAC: /* Fall through */ 448 aalg = auth_xform->auth.algo; 449 sess->auth.mode = ARMV8_CRYPTO_AUTH_AS_HMAC; 450 break; 451 default: 452 return -EINVAL; 453 } 454 455 /* Set the digest length */ 456 sess->auth.digest_length = auth_xform->auth.digest_length; 457 458 /* Verify supported key lengths and extract proper algorithm */ 459 switch (cipher_xform->cipher.key.length << 3) { 460 case 128: 461 sess->crypto_func = 462 CRYPTO_GET_ALGO(order, cop, calg, aalg, 128); 463 sess->cipher.key_sched = 464 CRYPTO_GET_KEY_SCHED(cop, calg, 128); 465 break; 466 case 192: 467 case 256: 468 /* These key lengths are not supported yet */ 469 default: /* Fall through */ 470 sess->crypto_func = NULL; 471 sess->cipher.key_sched = NULL; 472 return -EINVAL; 473 } 474 475 if (unlikely(sess->crypto_func == NULL)) { 476 /* 477 * If we got here that means that there must be a bug 478 * in the algorithms selection above. Nevertheless keep 479 * it here to catch bug immediately and avoid NULL pointer 480 * dereference in OPs processing. 481 */ 482 ARMV8_CRYPTO_LOG_ERR( 483 "No appropriate crypto function for given parameters"); 484 return -EINVAL; 485 } 486 487 /* Set up cipher session prerequisites */ 488 if (cipher_set_prerequisites(sess, cipher_xform) != 0) 489 return -EINVAL; 490 491 /* Set up authentication session prerequisites */ 492 if (auth_set_prerequisites(sess, auth_xform) != 0) 493 return -EINVAL; 494 495 return 0; 496 } 497 498 /** Parse crypto xform chain and set private session parameters */ 499 int 500 armv8_crypto_set_session_parameters(struct armv8_crypto_session *sess, 501 const struct rte_crypto_sym_xform *xform) 502 { 503 const struct rte_crypto_sym_xform *cipher_xform = NULL; 504 const struct rte_crypto_sym_xform *auth_xform = NULL; 505 bool is_chained_op; 506 int ret; 507 508 /* Filter out spurious/broken requests */ 509 if (xform == NULL) 510 return -EINVAL; 511 512 sess->chain_order = armv8_crypto_get_chain_order(xform); 513 switch (sess->chain_order) { 514 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: 515 cipher_xform = xform; 516 auth_xform = xform->next; 517 is_chained_op = true; 518 break; 519 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: 520 auth_xform = xform; 521 cipher_xform = xform->next; 522 is_chained_op = true; 523 break; 524 default: 525 is_chained_op = false; 526 return -EINVAL; 527 } 528 529 /* Set IV offset */ 530 sess->cipher.iv.offset = cipher_xform->cipher.iv.offset; 531 532 if (is_chained_op) { 533 ret = armv8_crypto_set_session_chained_parameters(sess, 534 cipher_xform, auth_xform); 535 if (unlikely(ret != 0)) { 536 ARMV8_CRYPTO_LOG_ERR( 537 "Invalid/unsupported chained (cipher/auth) parameters"); 538 return -EINVAL; 539 } 540 } else { 541 ARMV8_CRYPTO_LOG_ERR("Invalid/unsupported operation"); 542 return -EINVAL; 543 } 544 545 return 0; 546 } 547 548 /** Provide session for operation */ 549 static inline struct armv8_crypto_session * 550 get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op) 551 { 552 struct armv8_crypto_session *sess = NULL; 553 554 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 555 /* get existing session */ 556 if (likely(op->sym->session != NULL && 557 op->sym->session->dev_type == 558 RTE_CRYPTODEV_ARMV8_PMD)) { 559 sess = (struct armv8_crypto_session *) 560 op->sym->session->_private; 561 } 562 } else { 563 /* provide internal session */ 564 void *_sess = NULL; 565 566 if (!rte_mempool_get(qp->sess_mp, (void **)&_sess)) { 567 sess = (struct armv8_crypto_session *) 568 ((struct rte_cryptodev_sym_session *)_sess) 569 ->_private; 570 571 if (unlikely(armv8_crypto_set_session_parameters( 572 sess, op->sym->xform) != 0)) { 573 rte_mempool_put(qp->sess_mp, _sess); 574 sess = NULL; 575 } else 576 op->sym->session = _sess; 577 } 578 } 579 580 if (unlikely(sess == NULL)) 581 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 582 583 return sess; 584 } 585 586 /* 587 *------------------------------------------------------------------------------ 588 * Process Operations 589 *------------------------------------------------------------------------------ 590 */ 591 592 /*----------------------------------------------------------------------------*/ 593 594 /** Process cipher operation */ 595 static inline void 596 process_armv8_chained_op 597 (struct rte_crypto_op *op, struct armv8_crypto_session *sess, 598 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 599 { 600 crypto_func_t crypto_func; 601 crypto_arg_t arg; 602 struct rte_mbuf *m_asrc, *m_adst; 603 uint8_t *csrc, *cdst; 604 uint8_t *adst, *asrc; 605 uint64_t clen, alen; 606 int error; 607 608 clen = op->sym->cipher.data.length; 609 alen = op->sym->auth.data.length; 610 611 csrc = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 612 op->sym->cipher.data.offset); 613 cdst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 614 op->sym->cipher.data.offset); 615 616 switch (sess->chain_order) { 617 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: 618 m_asrc = m_adst = mbuf_dst; 619 break; 620 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: 621 m_asrc = mbuf_src; 622 m_adst = mbuf_dst; 623 break; 624 default: 625 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 626 return; 627 } 628 asrc = rte_pktmbuf_mtod_offset(m_asrc, uint8_t *, 629 op->sym->auth.data.offset); 630 631 switch (sess->auth.mode) { 632 case ARMV8_CRYPTO_AUTH_AS_AUTH: 633 /* Nothing to do here, just verify correct option */ 634 break; 635 case ARMV8_CRYPTO_AUTH_AS_HMAC: 636 arg.digest.hmac.key = sess->auth.hmac.key; 637 arg.digest.hmac.i_key_pad = sess->auth.hmac.i_key_pad; 638 arg.digest.hmac.o_key_pad = sess->auth.hmac.o_key_pad; 639 break; 640 default: 641 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 642 return; 643 } 644 645 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) { 646 adst = op->sym->auth.digest.data; 647 if (adst == NULL) { 648 adst = rte_pktmbuf_mtod_offset(m_adst, 649 uint8_t *, 650 op->sym->auth.data.offset + 651 op->sym->auth.data.length); 652 } 653 } else { 654 adst = (uint8_t *)rte_pktmbuf_append(m_asrc, 655 sess->auth.digest_length); 656 } 657 658 arg.cipher.iv = rte_crypto_op_ctod_offset(op, uint8_t *, 659 sess->cipher.iv.offset); 660 arg.cipher.key = sess->cipher.key.data; 661 /* Acquire combined mode function */ 662 crypto_func = sess->crypto_func; 663 ARMV8_CRYPTO_ASSERT(crypto_func != NULL); 664 error = crypto_func(csrc, cdst, clen, asrc, adst, alen, &arg); 665 if (error != 0) { 666 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 667 return; 668 } 669 670 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 671 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 672 if (memcmp(adst, op->sym->auth.digest.data, 673 sess->auth.digest_length) != 0) { 674 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 675 } 676 /* Trim area used for digest from mbuf. */ 677 rte_pktmbuf_trim(m_asrc, 678 sess->auth.digest_length); 679 } 680 } 681 682 /** Process crypto operation for mbuf */ 683 static inline int 684 process_op(const struct armv8_crypto_qp *qp, struct rte_crypto_op *op, 685 struct armv8_crypto_session *sess) 686 { 687 struct rte_mbuf *msrc, *mdst; 688 689 msrc = op->sym->m_src; 690 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 691 692 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 693 694 switch (sess->chain_order) { 695 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: 696 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: /* Fall through */ 697 process_armv8_chained_op(op, sess, msrc, mdst); 698 break; 699 default: 700 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 701 break; 702 } 703 704 /* Free session if a session-less crypto op */ 705 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 706 memset(sess, 0, sizeof(struct armv8_crypto_session)); 707 rte_mempool_put(qp->sess_mp, op->sym->session); 708 op->sym->session = NULL; 709 } 710 711 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 712 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 713 714 if (unlikely(op->status == RTE_CRYPTO_OP_STATUS_ERROR)) 715 return -1; 716 717 return 0; 718 } 719 720 /* 721 *------------------------------------------------------------------------------ 722 * PMD Framework 723 *------------------------------------------------------------------------------ 724 */ 725 726 /** Enqueue burst */ 727 static uint16_t 728 armv8_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 729 uint16_t nb_ops) 730 { 731 struct armv8_crypto_session *sess; 732 struct armv8_crypto_qp *qp = queue_pair; 733 int i, retval; 734 735 for (i = 0; i < nb_ops; i++) { 736 sess = get_session(qp, ops[i]); 737 if (unlikely(sess == NULL)) 738 goto enqueue_err; 739 740 retval = process_op(qp, ops[i], sess); 741 if (unlikely(retval < 0)) 742 goto enqueue_err; 743 } 744 745 retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i, 746 NULL); 747 qp->stats.enqueued_count += retval; 748 749 return retval; 750 751 enqueue_err: 752 retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i, 753 NULL); 754 if (ops[i] != NULL) 755 ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 756 757 qp->stats.enqueue_err_count++; 758 return retval; 759 } 760 761 /** Dequeue burst */ 762 static uint16_t 763 armv8_crypto_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 764 uint16_t nb_ops) 765 { 766 struct armv8_crypto_qp *qp = queue_pair; 767 768 unsigned int nb_dequeued = 0; 769 770 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 771 (void **)ops, nb_ops, NULL); 772 qp->stats.dequeued_count += nb_dequeued; 773 774 return nb_dequeued; 775 } 776 777 /** Create ARMv8 crypto device */ 778 static int 779 cryptodev_armv8_crypto_create(const char *name, 780 struct rte_vdev_device *vdev, 781 struct rte_crypto_vdev_init_params *init_params) 782 { 783 struct rte_cryptodev *dev; 784 struct armv8_crypto_private *internals; 785 786 /* Check CPU for support for AES instruction set */ 787 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) { 788 ARMV8_CRYPTO_LOG_ERR( 789 "AES instructions not supported by CPU"); 790 return -EFAULT; 791 } 792 793 /* Check CPU for support for SHA instruction set */ 794 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA1) || 795 !rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA2)) { 796 ARMV8_CRYPTO_LOG_ERR( 797 "SHA1/SHA2 instructions not supported by CPU"); 798 return -EFAULT; 799 } 800 801 /* Check CPU for support for Advance SIMD instruction set */ 802 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) { 803 ARMV8_CRYPTO_LOG_ERR( 804 "Advanced SIMD instructions not supported by CPU"); 805 return -EFAULT; 806 } 807 808 if (init_params->name[0] == '\0') 809 snprintf(init_params->name, sizeof(init_params->name), 810 "%s", name); 811 812 dev = rte_cryptodev_vdev_pmd_init(init_params->name, 813 sizeof(struct armv8_crypto_private), 814 init_params->socket_id, 815 vdev); 816 if (dev == NULL) { 817 ARMV8_CRYPTO_LOG_ERR("failed to create cryptodev vdev"); 818 goto init_error; 819 } 820 821 dev->dev_type = RTE_CRYPTODEV_ARMV8_PMD; 822 dev->dev_ops = rte_armv8_crypto_pmd_ops; 823 824 /* register rx/tx burst functions for data path */ 825 dev->dequeue_burst = armv8_crypto_pmd_dequeue_burst; 826 dev->enqueue_burst = armv8_crypto_pmd_enqueue_burst; 827 828 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 829 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 830 RTE_CRYPTODEV_FF_CPU_NEON | 831 RTE_CRYPTODEV_FF_CPU_ARM_CE; 832 833 /* Set vector instructions mode supported */ 834 internals = dev->data->dev_private; 835 836 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 837 internals->max_nb_sessions = init_params->max_nb_sessions; 838 839 return 0; 840 841 init_error: 842 ARMV8_CRYPTO_LOG_ERR( 843 "driver %s: cryptodev_armv8_crypto_create failed", 844 init_params->name); 845 846 cryptodev_armv8_crypto_uninit(vdev); 847 return -EFAULT; 848 } 849 850 /** Initialise ARMv8 crypto device */ 851 static int 852 cryptodev_armv8_crypto_init(struct rte_vdev_device *vdev) 853 { 854 struct rte_crypto_vdev_init_params init_params = { 855 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_QUEUE_PAIRS, 856 RTE_CRYPTODEV_VDEV_DEFAULT_MAX_NB_SESSIONS, 857 rte_socket_id(), 858 {0} 859 }; 860 const char *name; 861 const char *input_args; 862 863 name = rte_vdev_device_name(vdev); 864 if (name == NULL) 865 return -EINVAL; 866 input_args = rte_vdev_device_args(vdev); 867 rte_cryptodev_vdev_parse_init_params(&init_params, input_args); 868 869 RTE_LOG(INFO, PMD, "Initialising %s on NUMA node %d\n", name, 870 init_params.socket_id); 871 if (init_params.name[0] != '\0') { 872 RTE_LOG(INFO, PMD, " User defined name = %s\n", 873 init_params.name); 874 } 875 RTE_LOG(INFO, PMD, " Max number of queue pairs = %d\n", 876 init_params.max_nb_queue_pairs); 877 RTE_LOG(INFO, PMD, " Max number of sessions = %d\n", 878 init_params.max_nb_sessions); 879 880 return cryptodev_armv8_crypto_create(name, vdev, &init_params); 881 } 882 883 /** Uninitialise ARMv8 crypto device */ 884 static int 885 cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev) 886 { 887 const char *name; 888 889 name = rte_vdev_device_name(vdev); 890 if (name == NULL) 891 return -EINVAL; 892 893 RTE_LOG(INFO, PMD, 894 "Closing ARMv8 crypto device %s on numa socket %u\n", 895 name, rte_socket_id()); 896 897 return 0; 898 } 899 900 static struct rte_vdev_driver armv8_crypto_drv = { 901 .probe = cryptodev_armv8_crypto_init, 902 .remove = cryptodev_armv8_crypto_uninit 903 }; 904 905 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ARMV8_PMD, armv8_crypto_drv); 906 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_ARMV8_PMD, cryptodev_armv8_pmd); 907 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ARMV8_PMD, 908 "max_nb_queue_pairs=<int> " 909 "max_nb_sessions=<int> " 910 "socket_id=<int>"); 911