1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Cavium, Inc 3 */ 4 5 #include <stdbool.h> 6 7 #include <rte_common.h> 8 #include <rte_hexdump.h> 9 #include <rte_cryptodev.h> 10 #include <rte_cryptodev_pmd.h> 11 #include <rte_bus_vdev.h> 12 #include <rte_malloc.h> 13 #include <rte_cpuflags.h> 14 15 #include "AArch64cryptolib.h" 16 17 #include "armv8_pmd_private.h" 18 19 static uint8_t cryptodev_driver_id; 20 21 static int cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev); 22 23 /** 24 * Pointers to the supported combined mode crypto functions are stored 25 * in the static tables. Each combined (chained) cryptographic operation 26 * can be described by a set of numbers: 27 * - order: order of operations (cipher, auth) or (auth, cipher) 28 * - direction: encryption or decryption 29 * - calg: cipher algorithm such as AES_CBC, AES_CTR, etc. 30 * - aalg: authentication algorithm such as SHA1, SHA256, etc. 31 * - keyl: cipher key length, for example 128, 192, 256 bits 32 * 33 * In order to quickly acquire each function pointer based on those numbers, 34 * a hierarchy of arrays is maintained. The final level, 3D array is indexed 35 * by the combined mode function parameters only (cipher algorithm, 36 * authentication algorithm and key length). 37 * 38 * This gives 3 memory accesses to obtain a function pointer instead of 39 * traversing the array manually and comparing function parameters on each loop. 40 * 41 * +--+CRYPTO_FUNC 42 * +--+ENC| 43 * +--+CA| 44 * | +--+DEC 45 * ORDER| 46 * | +--+ENC 47 * +--+AC| 48 * +--+DEC 49 * 50 */ 51 52 /** 53 * 3D array type for ARM Combined Mode crypto functions pointers. 54 * CRYPTO_CIPHER_MAX: max cipher ID number 55 * CRYPTO_AUTH_MAX: max auth ID number 56 * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number 57 */ 58 typedef const crypto_func_t 59 crypto_func_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_AUTH_MAX][CRYPTO_CIPHER_KEYLEN_MAX]; 60 61 /* Evaluate to key length definition */ 62 #define KEYL(keyl) (ARMV8_CRYPTO_CIPHER_KEYLEN_ ## keyl) 63 64 /* Local aliases for supported ciphers */ 65 #define CIPH_AES_CBC RTE_CRYPTO_CIPHER_AES_CBC 66 /* Local aliases for supported hashes */ 67 #define AUTH_SHA1_HMAC RTE_CRYPTO_AUTH_SHA1_HMAC 68 #define AUTH_SHA256_HMAC RTE_CRYPTO_AUTH_SHA256_HMAC 69 70 /** 71 * Arrays containing pointers to particular cryptographic, 72 * combined mode functions. 73 * crypto_op_ca_encrypt: cipher (encrypt), authenticate 74 * crypto_op_ca_decrypt: cipher (decrypt), authenticate 75 * crypto_op_ac_encrypt: authenticate, cipher (encrypt) 76 * crypto_op_ac_decrypt: authenticate, cipher (decrypt) 77 */ 78 static const crypto_func_tbl_t 79 crypto_op_ca_encrypt = { 80 /* [cipher alg][auth alg][key length] = crypto_function, */ 81 [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = 82 armv8_enc_aes_cbc_sha1_128, 83 [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = 84 armv8_enc_aes_cbc_sha256_128, 85 }; 86 87 static const crypto_func_tbl_t 88 crypto_op_ca_decrypt = { 89 { {NULL} } 90 }; 91 92 static const crypto_func_tbl_t 93 crypto_op_ac_encrypt = { 94 { {NULL} } 95 }; 96 97 static const crypto_func_tbl_t 98 crypto_op_ac_decrypt = { 99 /* [cipher alg][auth alg][key length] = crypto_function, */ 100 [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = 101 armv8_dec_aes_cbc_sha1_128, 102 [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = 103 armv8_dec_aes_cbc_sha256_128, 104 }; 105 106 /** 107 * Arrays containing pointers to particular cryptographic function sets, 108 * covering given cipher operation directions (encrypt, decrypt) 109 * for each order of cipher and authentication pairs. 110 */ 111 static const crypto_func_tbl_t * 112 crypto_cipher_auth[] = { 113 &crypto_op_ca_encrypt, 114 &crypto_op_ca_decrypt, 115 NULL 116 }; 117 118 static const crypto_func_tbl_t * 119 crypto_auth_cipher[] = { 120 &crypto_op_ac_encrypt, 121 &crypto_op_ac_decrypt, 122 NULL 123 }; 124 125 /** 126 * Top level array containing pointers to particular cryptographic 127 * function sets, covering given order of chained operations. 128 * crypto_cipher_auth: cipher first, authenticate after 129 * crypto_auth_cipher: authenticate first, cipher after 130 */ 131 static const crypto_func_tbl_t ** 132 crypto_chain_order[] = { 133 crypto_cipher_auth, 134 crypto_auth_cipher, 135 NULL 136 }; 137 138 /** 139 * Extract particular combined mode crypto function from the 3D array. 140 */ 141 #define CRYPTO_GET_ALGO(order, cop, calg, aalg, keyl) \ 142 ({ \ 143 crypto_func_tbl_t *func_tbl = \ 144 (crypto_chain_order[(order)])[(cop)]; \ 145 \ 146 ((*func_tbl)[(calg)][(aalg)][KEYL(keyl)]); \ 147 }) 148 149 /*----------------------------------------------------------------------------*/ 150 151 /** 152 * 2D array type for ARM key schedule functions pointers. 153 * CRYPTO_CIPHER_MAX: max cipher ID number 154 * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number 155 */ 156 typedef const crypto_key_sched_t 157 crypto_key_sched_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_CIPHER_KEYLEN_MAX]; 158 159 static const crypto_key_sched_tbl_t 160 crypto_key_sched_encrypt = { 161 /* [cipher alg][key length] = key_expand_func, */ 162 [CIPH_AES_CBC][KEYL(128)] = armv8_expandkeys_enc_aes_cbc_128, 163 }; 164 165 static const crypto_key_sched_tbl_t 166 crypto_key_sched_decrypt = { 167 /* [cipher alg][key length] = key_expand_func, */ 168 [CIPH_AES_CBC][KEYL(128)] = armv8_expandkeys_dec_aes_cbc_128, 169 }; 170 171 /** 172 * Top level array containing pointers to particular key generation 173 * function sets, covering given operation direction. 174 * crypto_key_sched_encrypt: keys for encryption 175 * crypto_key_sched_decrypt: keys for decryption 176 */ 177 static const crypto_key_sched_tbl_t * 178 crypto_key_sched_dir[] = { 179 &crypto_key_sched_encrypt, 180 &crypto_key_sched_decrypt, 181 NULL 182 }; 183 184 /** 185 * Extract particular combined mode crypto function from the 3D array. 186 */ 187 #define CRYPTO_GET_KEY_SCHED(cop, calg, keyl) \ 188 ({ \ 189 crypto_key_sched_tbl_t *ks_tbl = crypto_key_sched_dir[(cop)]; \ 190 \ 191 ((*ks_tbl)[(calg)][KEYL(keyl)]); \ 192 }) 193 194 /*----------------------------------------------------------------------------*/ 195 196 /* 197 *------------------------------------------------------------------------------ 198 * Session Prepare 199 *------------------------------------------------------------------------------ 200 */ 201 202 /** Get xform chain order */ 203 static enum armv8_crypto_chain_order 204 armv8_crypto_get_chain_order(const struct rte_crypto_sym_xform *xform) 205 { 206 207 /* 208 * This driver currently covers only chained operations. 209 * Ignore only cipher or only authentication operations 210 * or chains longer than 2 xform structures. 211 */ 212 if (xform->next == NULL || xform->next->next != NULL) 213 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED; 214 215 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 216 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) 217 return ARMV8_CRYPTO_CHAIN_AUTH_CIPHER; 218 } 219 220 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 221 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 222 return ARMV8_CRYPTO_CHAIN_CIPHER_AUTH; 223 } 224 225 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED; 226 } 227 228 static inline void 229 auth_hmac_pad_prepare(struct armv8_crypto_session *sess, 230 const struct rte_crypto_sym_xform *xform) 231 { 232 size_t i; 233 234 /* Generate i_key_pad and o_key_pad */ 235 memset(sess->auth.hmac.i_key_pad, 0, sizeof(sess->auth.hmac.i_key_pad)); 236 rte_memcpy(sess->auth.hmac.i_key_pad, sess->auth.hmac.key, 237 xform->auth.key.length); 238 memset(sess->auth.hmac.o_key_pad, 0, sizeof(sess->auth.hmac.o_key_pad)); 239 rte_memcpy(sess->auth.hmac.o_key_pad, sess->auth.hmac.key, 240 xform->auth.key.length); 241 /* 242 * XOR key with IPAD/OPAD values to obtain i_key_pad 243 * and o_key_pad. 244 * Byte-by-byte operation may seem to be the less efficient 245 * here but in fact it's the opposite. 246 * The result ASM code is likely operate on NEON registers 247 * (load auth key to Qx, load IPAD/OPAD to multiple 248 * elements of Qy, eor 128 bits at once). 249 */ 250 for (i = 0; i < SHA_BLOCK_MAX; i++) { 251 sess->auth.hmac.i_key_pad[i] ^= HMAC_IPAD_VALUE; 252 sess->auth.hmac.o_key_pad[i] ^= HMAC_OPAD_VALUE; 253 } 254 } 255 256 static inline int 257 auth_set_prerequisites(struct armv8_crypto_session *sess, 258 const struct rte_crypto_sym_xform *xform) 259 { 260 uint8_t partial[64] = { 0 }; 261 int error; 262 263 switch (xform->auth.algo) { 264 case RTE_CRYPTO_AUTH_SHA1_HMAC: 265 /* 266 * Generate authentication key, i_key_pad and o_key_pad. 267 */ 268 /* Zero memory under key */ 269 memset(sess->auth.hmac.key, 0, SHA1_BLOCK_SIZE); 270 271 /* 272 * Now copy the given authentication key to the session 273 * key. 274 */ 275 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, 276 xform->auth.key.length); 277 278 /* Prepare HMAC padding: key|pattern */ 279 auth_hmac_pad_prepare(sess, xform); 280 /* 281 * Calculate partial hash values for i_key_pad and o_key_pad. 282 * Will be used as initialization state for final HMAC. 283 */ 284 error = armv8_sha1_block_partial(NULL, 285 sess->auth.hmac.i_key_pad, 286 partial, SHA1_BLOCK_SIZE); 287 if (error != 0) 288 return -1; 289 memcpy(sess->auth.hmac.i_key_pad, partial, SHA1_BLOCK_SIZE); 290 291 error = armv8_sha1_block_partial(NULL, 292 sess->auth.hmac.o_key_pad, 293 partial, SHA1_BLOCK_SIZE); 294 if (error != 0) 295 return -1; 296 memcpy(sess->auth.hmac.o_key_pad, partial, SHA1_BLOCK_SIZE); 297 298 break; 299 case RTE_CRYPTO_AUTH_SHA256_HMAC: 300 /* 301 * Generate authentication key, i_key_pad and o_key_pad. 302 */ 303 /* Zero memory under key */ 304 memset(sess->auth.hmac.key, 0, SHA256_BLOCK_SIZE); 305 306 /* 307 * Now copy the given authentication key to the session 308 * key. 309 */ 310 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, 311 xform->auth.key.length); 312 313 /* Prepare HMAC padding: key|pattern */ 314 auth_hmac_pad_prepare(sess, xform); 315 /* 316 * Calculate partial hash values for i_key_pad and o_key_pad. 317 * Will be used as initialization state for final HMAC. 318 */ 319 error = armv8_sha256_block_partial(NULL, 320 sess->auth.hmac.i_key_pad, 321 partial, SHA256_BLOCK_SIZE); 322 if (error != 0) 323 return -1; 324 memcpy(sess->auth.hmac.i_key_pad, partial, SHA256_BLOCK_SIZE); 325 326 error = armv8_sha256_block_partial(NULL, 327 sess->auth.hmac.o_key_pad, 328 partial, SHA256_BLOCK_SIZE); 329 if (error != 0) 330 return -1; 331 memcpy(sess->auth.hmac.o_key_pad, partial, SHA256_BLOCK_SIZE); 332 333 break; 334 default: 335 break; 336 } 337 338 return 0; 339 } 340 341 static inline int 342 cipher_set_prerequisites(struct armv8_crypto_session *sess, 343 const struct rte_crypto_sym_xform *xform) 344 { 345 crypto_key_sched_t cipher_key_sched; 346 347 cipher_key_sched = sess->cipher.key_sched; 348 if (likely(cipher_key_sched != NULL)) { 349 /* Set up cipher session key */ 350 cipher_key_sched(sess->cipher.key.data, xform->cipher.key.data); 351 } 352 353 return 0; 354 } 355 356 static int 357 armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess, 358 const struct rte_crypto_sym_xform *cipher_xform, 359 const struct rte_crypto_sym_xform *auth_xform) 360 { 361 enum armv8_crypto_chain_order order; 362 enum armv8_crypto_cipher_operation cop; 363 enum rte_crypto_cipher_algorithm calg; 364 enum rte_crypto_auth_algorithm aalg; 365 366 /* Validate and prepare scratch order of combined operations */ 367 switch (sess->chain_order) { 368 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: 369 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: 370 order = sess->chain_order; 371 break; 372 default: 373 return -ENOTSUP; 374 } 375 /* Select cipher direction */ 376 sess->cipher.direction = cipher_xform->cipher.op; 377 /* Select cipher key */ 378 sess->cipher.key.length = cipher_xform->cipher.key.length; 379 /* Set cipher direction */ 380 switch (sess->cipher.direction) { 381 case RTE_CRYPTO_CIPHER_OP_ENCRYPT: 382 cop = ARMV8_CRYPTO_CIPHER_OP_ENCRYPT; 383 break; 384 case RTE_CRYPTO_CIPHER_OP_DECRYPT: 385 cop = ARMV8_CRYPTO_CIPHER_OP_DECRYPT; 386 break; 387 default: 388 return -ENOTSUP; 389 } 390 /* Set cipher algorithm */ 391 calg = cipher_xform->cipher.algo; 392 393 /* Select cipher algo */ 394 switch (calg) { 395 /* Cover supported cipher algorithms */ 396 case RTE_CRYPTO_CIPHER_AES_CBC: 397 sess->cipher.algo = calg; 398 /* IV len is always 16 bytes (block size) for AES CBC */ 399 sess->cipher.iv.length = 16; 400 break; 401 default: 402 return -ENOTSUP; 403 } 404 /* Select auth generate/verify */ 405 sess->auth.operation = auth_xform->auth.op; 406 407 /* Select auth algo */ 408 switch (auth_xform->auth.algo) { 409 /* Cover supported hash algorithms */ 410 case RTE_CRYPTO_AUTH_SHA1_HMAC: 411 case RTE_CRYPTO_AUTH_SHA256_HMAC: /* Fall through */ 412 aalg = auth_xform->auth.algo; 413 sess->auth.mode = ARMV8_CRYPTO_AUTH_AS_HMAC; 414 break; 415 default: 416 return -ENOTSUP; 417 } 418 419 /* Set the digest length */ 420 sess->auth.digest_length = auth_xform->auth.digest_length; 421 422 /* Verify supported key lengths and extract proper algorithm */ 423 switch (cipher_xform->cipher.key.length << 3) { 424 case 128: 425 sess->crypto_func = 426 CRYPTO_GET_ALGO(order, cop, calg, aalg, 128); 427 sess->cipher.key_sched = 428 CRYPTO_GET_KEY_SCHED(cop, calg, 128); 429 break; 430 case 192: 431 case 256: 432 /* These key lengths are not supported yet */ 433 default: /* Fall through */ 434 sess->crypto_func = NULL; 435 sess->cipher.key_sched = NULL; 436 return -ENOTSUP; 437 } 438 439 if (unlikely(sess->crypto_func == NULL)) { 440 /* 441 * If we got here that means that there must be a bug 442 * in the algorithms selection above. Nevertheless keep 443 * it here to catch bug immediately and avoid NULL pointer 444 * dereference in OPs processing. 445 */ 446 ARMV8_CRYPTO_LOG_ERR( 447 "No appropriate crypto function for given parameters"); 448 return -EINVAL; 449 } 450 451 /* Set up cipher session prerequisites */ 452 if (cipher_set_prerequisites(sess, cipher_xform) != 0) 453 return -EINVAL; 454 455 /* Set up authentication session prerequisites */ 456 if (auth_set_prerequisites(sess, auth_xform) != 0) 457 return -EINVAL; 458 459 return 0; 460 } 461 462 /** Parse crypto xform chain and set private session parameters */ 463 int 464 armv8_crypto_set_session_parameters(struct armv8_crypto_session *sess, 465 const struct rte_crypto_sym_xform *xform) 466 { 467 const struct rte_crypto_sym_xform *cipher_xform = NULL; 468 const struct rte_crypto_sym_xform *auth_xform = NULL; 469 bool is_chained_op; 470 int ret; 471 472 /* Filter out spurious/broken requests */ 473 if (xform == NULL) 474 return -EINVAL; 475 476 sess->chain_order = armv8_crypto_get_chain_order(xform); 477 switch (sess->chain_order) { 478 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: 479 cipher_xform = xform; 480 auth_xform = xform->next; 481 is_chained_op = true; 482 break; 483 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: 484 auth_xform = xform; 485 cipher_xform = xform->next; 486 is_chained_op = true; 487 break; 488 default: 489 is_chained_op = false; 490 return -ENOTSUP; 491 } 492 493 /* Set IV offset */ 494 sess->cipher.iv.offset = cipher_xform->cipher.iv.offset; 495 496 if (is_chained_op) { 497 ret = armv8_crypto_set_session_chained_parameters(sess, 498 cipher_xform, auth_xform); 499 if (unlikely(ret != 0)) { 500 ARMV8_CRYPTO_LOG_ERR( 501 "Invalid/unsupported chained (cipher/auth) parameters"); 502 return ret; 503 } 504 } else { 505 ARMV8_CRYPTO_LOG_ERR("Invalid/unsupported operation"); 506 return -ENOTSUP; 507 } 508 509 return 0; 510 } 511 512 /** Provide session for operation */ 513 static inline struct armv8_crypto_session * 514 get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op) 515 { 516 struct armv8_crypto_session *sess = NULL; 517 518 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 519 /* get existing session */ 520 if (likely(op->sym->session != NULL)) { 521 sess = (struct armv8_crypto_session *) 522 get_sym_session_private_data( 523 op->sym->session, 524 cryptodev_driver_id); 525 } 526 } else { 527 /* provide internal session */ 528 void *_sess = NULL; 529 void *_sess_private_data = NULL; 530 531 if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) 532 return NULL; 533 534 if (rte_mempool_get(qp->sess_mp_priv, 535 (void **)&_sess_private_data)) 536 return NULL; 537 538 sess = (struct armv8_crypto_session *)_sess_private_data; 539 540 if (unlikely(armv8_crypto_set_session_parameters(sess, 541 op->sym->xform) != 0)) { 542 rte_mempool_put(qp->sess_mp, _sess); 543 rte_mempool_put(qp->sess_mp_priv, _sess_private_data); 544 sess = NULL; 545 } 546 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 547 set_sym_session_private_data(op->sym->session, 548 cryptodev_driver_id, _sess_private_data); 549 } 550 551 if (unlikely(sess == NULL)) 552 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 553 554 return sess; 555 } 556 557 /* 558 *------------------------------------------------------------------------------ 559 * Process Operations 560 *------------------------------------------------------------------------------ 561 */ 562 563 /*----------------------------------------------------------------------------*/ 564 565 /** Process cipher operation */ 566 static inline void 567 process_armv8_chained_op(struct armv8_crypto_qp *qp, struct rte_crypto_op *op, 568 struct armv8_crypto_session *sess, 569 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 570 { 571 crypto_func_t crypto_func; 572 armv8_cipher_digest_t arg; 573 struct rte_mbuf *m_asrc, *m_adst; 574 uint8_t *csrc, *cdst; 575 uint8_t *adst, *asrc; 576 uint64_t clen, alen; 577 int error; 578 579 clen = op->sym->cipher.data.length; 580 alen = op->sym->auth.data.length; 581 582 csrc = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 583 op->sym->cipher.data.offset); 584 cdst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 585 op->sym->cipher.data.offset); 586 587 switch (sess->chain_order) { 588 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: 589 m_asrc = m_adst = mbuf_dst; 590 break; 591 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: 592 m_asrc = mbuf_src; 593 m_adst = mbuf_dst; 594 break; 595 default: 596 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 597 return; 598 } 599 asrc = rte_pktmbuf_mtod_offset(m_asrc, uint8_t *, 600 op->sym->auth.data.offset); 601 602 switch (sess->auth.mode) { 603 case ARMV8_CRYPTO_AUTH_AS_AUTH: 604 /* Nothing to do here, just verify correct option */ 605 break; 606 case ARMV8_CRYPTO_AUTH_AS_HMAC: 607 arg.digest.hmac.key = sess->auth.hmac.key; 608 arg.digest.hmac.i_key_pad = sess->auth.hmac.i_key_pad; 609 arg.digest.hmac.o_key_pad = sess->auth.hmac.o_key_pad; 610 break; 611 default: 612 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 613 return; 614 } 615 616 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) { 617 adst = op->sym->auth.digest.data; 618 if (adst == NULL) { 619 adst = rte_pktmbuf_mtod_offset(m_adst, 620 uint8_t *, 621 op->sym->auth.data.offset + 622 op->sym->auth.data.length); 623 } 624 } else { 625 adst = qp->temp_digest; 626 } 627 628 arg.cipher.iv = rte_crypto_op_ctod_offset(op, uint8_t *, 629 sess->cipher.iv.offset); 630 arg.cipher.key = sess->cipher.key.data; 631 /* Acquire combined mode function */ 632 crypto_func = sess->crypto_func; 633 RTE_VERIFY(crypto_func != NULL); 634 error = crypto_func(csrc, cdst, clen, asrc, adst, alen, &arg); 635 if (error != 0) { 636 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 637 return; 638 } 639 640 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 641 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 642 if (memcmp(adst, op->sym->auth.digest.data, 643 sess->auth.digest_length) != 0) { 644 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 645 } 646 } 647 } 648 649 /** Process crypto operation for mbuf */ 650 static inline int 651 process_op(struct armv8_crypto_qp *qp, struct rte_crypto_op *op, 652 struct armv8_crypto_session *sess) 653 { 654 struct rte_mbuf *msrc, *mdst; 655 656 msrc = op->sym->m_src; 657 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 658 659 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 660 661 switch (sess->chain_order) { 662 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: 663 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: /* Fall through */ 664 process_armv8_chained_op(qp, op, sess, msrc, mdst); 665 break; 666 default: 667 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 668 break; 669 } 670 671 /* Free session if a session-less crypto op */ 672 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 673 memset(sess, 0, sizeof(struct armv8_crypto_session)); 674 memset(op->sym->session, 0, 675 rte_cryptodev_sym_get_existing_header_session_size( 676 op->sym->session)); 677 rte_mempool_put(qp->sess_mp, sess); 678 rte_mempool_put(qp->sess_mp_priv, op->sym->session); 679 op->sym->session = NULL; 680 } 681 682 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 683 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 684 685 if (unlikely(op->status == RTE_CRYPTO_OP_STATUS_ERROR)) 686 return -1; 687 688 return 0; 689 } 690 691 /* 692 *------------------------------------------------------------------------------ 693 * PMD Framework 694 *------------------------------------------------------------------------------ 695 */ 696 697 /** Enqueue burst */ 698 static uint16_t 699 armv8_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 700 uint16_t nb_ops) 701 { 702 struct armv8_crypto_session *sess; 703 struct armv8_crypto_qp *qp = queue_pair; 704 int i, retval; 705 706 for (i = 0; i < nb_ops; i++) { 707 sess = get_session(qp, ops[i]); 708 if (unlikely(sess == NULL)) 709 goto enqueue_err; 710 711 retval = process_op(qp, ops[i], sess); 712 if (unlikely(retval < 0)) 713 goto enqueue_err; 714 } 715 716 retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i, 717 NULL); 718 qp->stats.enqueued_count += retval; 719 720 return retval; 721 722 enqueue_err: 723 retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i, 724 NULL); 725 if (ops[i] != NULL) 726 ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 727 728 qp->stats.enqueue_err_count++; 729 return retval; 730 } 731 732 /** Dequeue burst */ 733 static uint16_t 734 armv8_crypto_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 735 uint16_t nb_ops) 736 { 737 struct armv8_crypto_qp *qp = queue_pair; 738 739 unsigned int nb_dequeued = 0; 740 741 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 742 (void **)ops, nb_ops, NULL); 743 qp->stats.dequeued_count += nb_dequeued; 744 745 return nb_dequeued; 746 } 747 748 /** Create ARMv8 crypto device */ 749 static int 750 cryptodev_armv8_crypto_create(const char *name, 751 struct rte_vdev_device *vdev, 752 struct rte_cryptodev_pmd_init_params *init_params) 753 { 754 struct rte_cryptodev *dev; 755 struct armv8_crypto_private *internals; 756 757 /* Check CPU for support for AES instruction set */ 758 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) { 759 ARMV8_CRYPTO_LOG_ERR( 760 "AES instructions not supported by CPU"); 761 return -EFAULT; 762 } 763 764 /* Check CPU for support for SHA instruction set */ 765 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA1) || 766 !rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA2)) { 767 ARMV8_CRYPTO_LOG_ERR( 768 "SHA1/SHA2 instructions not supported by CPU"); 769 return -EFAULT; 770 } 771 772 /* Check CPU for support for Advance SIMD instruction set */ 773 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) { 774 ARMV8_CRYPTO_LOG_ERR( 775 "Advanced SIMD instructions not supported by CPU"); 776 return -EFAULT; 777 } 778 779 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 780 if (dev == NULL) { 781 ARMV8_CRYPTO_LOG_ERR("failed to create cryptodev vdev"); 782 goto init_error; 783 } 784 785 dev->driver_id = cryptodev_driver_id; 786 dev->dev_ops = rte_armv8_crypto_pmd_ops; 787 788 /* register rx/tx burst functions for data path */ 789 dev->dequeue_burst = armv8_crypto_pmd_dequeue_burst; 790 dev->enqueue_burst = armv8_crypto_pmd_enqueue_burst; 791 792 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 793 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 794 RTE_CRYPTODEV_FF_CPU_NEON | 795 RTE_CRYPTODEV_FF_CPU_ARM_CE | 796 RTE_CRYPTODEV_FF_SYM_SESSIONLESS; 797 798 internals = dev->data->dev_private; 799 800 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 801 802 return 0; 803 804 init_error: 805 ARMV8_CRYPTO_LOG_ERR( 806 "driver %s: cryptodev_armv8_crypto_create failed", 807 init_params->name); 808 809 cryptodev_armv8_crypto_uninit(vdev); 810 return -EFAULT; 811 } 812 813 /** Initialise ARMv8 crypto device */ 814 static int 815 cryptodev_armv8_crypto_init(struct rte_vdev_device *vdev) 816 { 817 struct rte_cryptodev_pmd_init_params init_params = { 818 "", 819 sizeof(struct armv8_crypto_private), 820 rte_socket_id(), 821 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 822 }; 823 const char *name; 824 const char *input_args; 825 826 name = rte_vdev_device_name(vdev); 827 if (name == NULL) 828 return -EINVAL; 829 input_args = rte_vdev_device_args(vdev); 830 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 831 832 return cryptodev_armv8_crypto_create(name, vdev, &init_params); 833 } 834 835 /** Uninitialise ARMv8 crypto device */ 836 static int 837 cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev) 838 { 839 struct rte_cryptodev *cryptodev; 840 const char *name; 841 842 name = rte_vdev_device_name(vdev); 843 if (name == NULL) 844 return -EINVAL; 845 846 RTE_LOG(INFO, PMD, 847 "Closing ARMv8 crypto device %s on numa socket %u\n", 848 name, rte_socket_id()); 849 850 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 851 if (cryptodev == NULL) 852 return -ENODEV; 853 854 return rte_cryptodev_pmd_destroy(cryptodev); 855 } 856 857 static struct rte_vdev_driver armv8_crypto_pmd_drv = { 858 .probe = cryptodev_armv8_crypto_init, 859 .remove = cryptodev_armv8_crypto_uninit 860 }; 861 862 static struct cryptodev_driver armv8_crypto_drv; 863 864 RTE_LOG_REGISTER(crypto_armv8_log_type, pmd.crypto.armv8, ERR); 865 866 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ARMV8_PMD, armv8_crypto_pmd_drv); 867 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_ARMV8_PMD, cryptodev_armv8_pmd); 868 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ARMV8_PMD, 869 "max_nb_queue_pairs=<int> " 870 "socket_id=<int>"); 871 RTE_PMD_REGISTER_CRYPTO_DRIVER(armv8_crypto_drv, armv8_crypto_pmd_drv.driver, 872 cryptodev_driver_id); 873