1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2017 Cavium, Inc 3 */ 4 5 #include <stdbool.h> 6 7 #include <rte_common.h> 8 #include <rte_hexdump.h> 9 #include <rte_cryptodev.h> 10 #include <rte_cryptodev_pmd.h> 11 #include <rte_bus_vdev.h> 12 #include <rte_malloc.h> 13 #include <rte_cpuflags.h> 14 15 #include "armv8_crypto_defs.h" 16 17 #include "rte_armv8_pmd_private.h" 18 19 static uint8_t cryptodev_driver_id; 20 21 static int cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev); 22 23 /** 24 * Pointers to the supported combined mode crypto functions are stored 25 * in the static tables. Each combined (chained) cryptographic operation 26 * can be described by a set of numbers: 27 * - order: order of operations (cipher, auth) or (auth, cipher) 28 * - direction: encryption or decryption 29 * - calg: cipher algorithm such as AES_CBC, AES_CTR, etc. 30 * - aalg: authentication algorithm such as SHA1, SHA256, etc. 31 * - keyl: cipher key length, for example 128, 192, 256 bits 32 * 33 * In order to quickly acquire each function pointer based on those numbers, 34 * a hierarchy of arrays is maintained. The final level, 3D array is indexed 35 * by the combined mode function parameters only (cipher algorithm, 36 * authentication algorithm and key length). 37 * 38 * This gives 3 memory accesses to obtain a function pointer instead of 39 * traversing the array manually and comparing function parameters on each loop. 40 * 41 * +--+CRYPTO_FUNC 42 * +--+ENC| 43 * +--+CA| 44 * | +--+DEC 45 * ORDER| 46 * | +--+ENC 47 * +--+AC| 48 * +--+DEC 49 * 50 */ 51 52 /** 53 * 3D array type for ARM Combined Mode crypto functions pointers. 54 * CRYPTO_CIPHER_MAX: max cipher ID number 55 * CRYPTO_AUTH_MAX: max auth ID number 56 * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number 57 */ 58 typedef const crypto_func_t 59 crypto_func_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_AUTH_MAX][CRYPTO_CIPHER_KEYLEN_MAX]; 60 61 /* Evaluate to key length definition */ 62 #define KEYL(keyl) (ARMV8_CRYPTO_CIPHER_KEYLEN_ ## keyl) 63 64 /* Local aliases for supported ciphers */ 65 #define CIPH_AES_CBC RTE_CRYPTO_CIPHER_AES_CBC 66 /* Local aliases for supported hashes */ 67 #define AUTH_SHA1_HMAC RTE_CRYPTO_AUTH_SHA1_HMAC 68 #define AUTH_SHA256_HMAC RTE_CRYPTO_AUTH_SHA256_HMAC 69 70 /** 71 * Arrays containing pointers to particular cryptographic, 72 * combined mode functions. 73 * crypto_op_ca_encrypt: cipher (encrypt), authenticate 74 * crypto_op_ca_decrypt: cipher (decrypt), authenticate 75 * crypto_op_ac_encrypt: authenticate, cipher (encrypt) 76 * crypto_op_ac_decrypt: authenticate, cipher (decrypt) 77 */ 78 static const crypto_func_tbl_t 79 crypto_op_ca_encrypt = { 80 /* [cipher alg][auth alg][key length] = crypto_function, */ 81 [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = aes128cbc_sha1_hmac, 82 [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = aes128cbc_sha256_hmac, 83 }; 84 85 static const crypto_func_tbl_t 86 crypto_op_ca_decrypt = { 87 NULL 88 }; 89 90 static const crypto_func_tbl_t 91 crypto_op_ac_encrypt = { 92 NULL 93 }; 94 95 static const crypto_func_tbl_t 96 crypto_op_ac_decrypt = { 97 /* [cipher alg][auth alg][key length] = crypto_function, */ 98 [CIPH_AES_CBC][AUTH_SHA1_HMAC][KEYL(128)] = sha1_hmac_aes128cbc_dec, 99 [CIPH_AES_CBC][AUTH_SHA256_HMAC][KEYL(128)] = sha256_hmac_aes128cbc_dec, 100 }; 101 102 /** 103 * Arrays containing pointers to particular cryptographic function sets, 104 * covering given cipher operation directions (encrypt, decrypt) 105 * for each order of cipher and authentication pairs. 106 */ 107 static const crypto_func_tbl_t * 108 crypto_cipher_auth[] = { 109 &crypto_op_ca_encrypt, 110 &crypto_op_ca_decrypt, 111 NULL 112 }; 113 114 static const crypto_func_tbl_t * 115 crypto_auth_cipher[] = { 116 &crypto_op_ac_encrypt, 117 &crypto_op_ac_decrypt, 118 NULL 119 }; 120 121 /** 122 * Top level array containing pointers to particular cryptographic 123 * function sets, covering given order of chained operations. 124 * crypto_cipher_auth: cipher first, authenticate after 125 * crypto_auth_cipher: authenticate first, cipher after 126 */ 127 static const crypto_func_tbl_t ** 128 crypto_chain_order[] = { 129 crypto_cipher_auth, 130 crypto_auth_cipher, 131 NULL 132 }; 133 134 /** 135 * Extract particular combined mode crypto function from the 3D array. 136 */ 137 #define CRYPTO_GET_ALGO(order, cop, calg, aalg, keyl) \ 138 ({ \ 139 crypto_func_tbl_t *func_tbl = \ 140 (crypto_chain_order[(order)])[(cop)]; \ 141 \ 142 ((*func_tbl)[(calg)][(aalg)][KEYL(keyl)]); \ 143 }) 144 145 /*----------------------------------------------------------------------------*/ 146 147 /** 148 * 2D array type for ARM key schedule functions pointers. 149 * CRYPTO_CIPHER_MAX: max cipher ID number 150 * CRYPTO_CIPHER_KEYLEN_MAX: max key length ID number 151 */ 152 typedef const crypto_key_sched_t 153 crypto_key_sched_tbl_t[CRYPTO_CIPHER_MAX][CRYPTO_CIPHER_KEYLEN_MAX]; 154 155 static const crypto_key_sched_tbl_t 156 crypto_key_sched_encrypt = { 157 /* [cipher alg][key length] = key_expand_func, */ 158 [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_enc, 159 }; 160 161 static const crypto_key_sched_tbl_t 162 crypto_key_sched_decrypt = { 163 /* [cipher alg][key length] = key_expand_func, */ 164 [CIPH_AES_CBC][KEYL(128)] = aes128_key_sched_dec, 165 }; 166 167 /** 168 * Top level array containing pointers to particular key generation 169 * function sets, covering given operation direction. 170 * crypto_key_sched_encrypt: keys for encryption 171 * crypto_key_sched_decrypt: keys for decryption 172 */ 173 static const crypto_key_sched_tbl_t * 174 crypto_key_sched_dir[] = { 175 &crypto_key_sched_encrypt, 176 &crypto_key_sched_decrypt, 177 NULL 178 }; 179 180 /** 181 * Extract particular combined mode crypto function from the 3D array. 182 */ 183 #define CRYPTO_GET_KEY_SCHED(cop, calg, keyl) \ 184 ({ \ 185 crypto_key_sched_tbl_t *ks_tbl = crypto_key_sched_dir[(cop)]; \ 186 \ 187 ((*ks_tbl)[(calg)][KEYL(keyl)]); \ 188 }) 189 190 /*----------------------------------------------------------------------------*/ 191 192 /* 193 *------------------------------------------------------------------------------ 194 * Session Prepare 195 *------------------------------------------------------------------------------ 196 */ 197 198 /** Get xform chain order */ 199 static enum armv8_crypto_chain_order 200 armv8_crypto_get_chain_order(const struct rte_crypto_sym_xform *xform) 201 { 202 203 /* 204 * This driver currently covers only chained operations. 205 * Ignore only cipher or only authentication operations 206 * or chains longer than 2 xform structures. 207 */ 208 if (xform->next == NULL || xform->next->next != NULL) 209 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED; 210 211 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 212 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) 213 return ARMV8_CRYPTO_CHAIN_AUTH_CIPHER; 214 } 215 216 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 217 if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 218 return ARMV8_CRYPTO_CHAIN_CIPHER_AUTH; 219 } 220 221 return ARMV8_CRYPTO_CHAIN_NOT_SUPPORTED; 222 } 223 224 static inline void 225 auth_hmac_pad_prepare(struct armv8_crypto_session *sess, 226 const struct rte_crypto_sym_xform *xform) 227 { 228 size_t i; 229 230 /* Generate i_key_pad and o_key_pad */ 231 memset(sess->auth.hmac.i_key_pad, 0, sizeof(sess->auth.hmac.i_key_pad)); 232 rte_memcpy(sess->auth.hmac.i_key_pad, sess->auth.hmac.key, 233 xform->auth.key.length); 234 memset(sess->auth.hmac.o_key_pad, 0, sizeof(sess->auth.hmac.o_key_pad)); 235 rte_memcpy(sess->auth.hmac.o_key_pad, sess->auth.hmac.key, 236 xform->auth.key.length); 237 /* 238 * XOR key with IPAD/OPAD values to obtain i_key_pad 239 * and o_key_pad. 240 * Byte-by-byte operation may seem to be the less efficient 241 * here but in fact it's the opposite. 242 * The result ASM code is likely operate on NEON registers 243 * (load auth key to Qx, load IPAD/OPAD to multiple 244 * elements of Qy, eor 128 bits at once). 245 */ 246 for (i = 0; i < SHA_BLOCK_MAX; i++) { 247 sess->auth.hmac.i_key_pad[i] ^= HMAC_IPAD_VALUE; 248 sess->auth.hmac.o_key_pad[i] ^= HMAC_OPAD_VALUE; 249 } 250 } 251 252 static inline int 253 auth_set_prerequisites(struct armv8_crypto_session *sess, 254 const struct rte_crypto_sym_xform *xform) 255 { 256 uint8_t partial[64] = { 0 }; 257 int error; 258 259 switch (xform->auth.algo) { 260 case RTE_CRYPTO_AUTH_SHA1_HMAC: 261 /* 262 * Generate authentication key, i_key_pad and o_key_pad. 263 */ 264 /* Zero memory under key */ 265 memset(sess->auth.hmac.key, 0, SHA1_BLOCK_SIZE); 266 267 /* 268 * Now copy the given authentication key to the session 269 * key. 270 */ 271 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, 272 xform->auth.key.length); 273 274 /* Prepare HMAC padding: key|pattern */ 275 auth_hmac_pad_prepare(sess, xform); 276 /* 277 * Calculate partial hash values for i_key_pad and o_key_pad. 278 * Will be used as initialization state for final HMAC. 279 */ 280 error = sha1_block_partial(NULL, sess->auth.hmac.i_key_pad, 281 partial, SHA1_BLOCK_SIZE); 282 if (error != 0) 283 return -1; 284 memcpy(sess->auth.hmac.i_key_pad, partial, SHA1_BLOCK_SIZE); 285 286 error = sha1_block_partial(NULL, sess->auth.hmac.o_key_pad, 287 partial, SHA1_BLOCK_SIZE); 288 if (error != 0) 289 return -1; 290 memcpy(sess->auth.hmac.o_key_pad, partial, SHA1_BLOCK_SIZE); 291 292 break; 293 case RTE_CRYPTO_AUTH_SHA256_HMAC: 294 /* 295 * Generate authentication key, i_key_pad and o_key_pad. 296 */ 297 /* Zero memory under key */ 298 memset(sess->auth.hmac.key, 0, SHA256_BLOCK_SIZE); 299 300 /* 301 * Now copy the given authentication key to the session 302 * key. 303 */ 304 rte_memcpy(sess->auth.hmac.key, xform->auth.key.data, 305 xform->auth.key.length); 306 307 /* Prepare HMAC padding: key|pattern */ 308 auth_hmac_pad_prepare(sess, xform); 309 /* 310 * Calculate partial hash values for i_key_pad and o_key_pad. 311 * Will be used as initialization state for final HMAC. 312 */ 313 error = sha256_block_partial(NULL, sess->auth.hmac.i_key_pad, 314 partial, SHA256_BLOCK_SIZE); 315 if (error != 0) 316 return -1; 317 memcpy(sess->auth.hmac.i_key_pad, partial, SHA256_BLOCK_SIZE); 318 319 error = sha256_block_partial(NULL, sess->auth.hmac.o_key_pad, 320 partial, SHA256_BLOCK_SIZE); 321 if (error != 0) 322 return -1; 323 memcpy(sess->auth.hmac.o_key_pad, partial, SHA256_BLOCK_SIZE); 324 325 break; 326 default: 327 break; 328 } 329 330 return 0; 331 } 332 333 static inline int 334 cipher_set_prerequisites(struct armv8_crypto_session *sess, 335 const struct rte_crypto_sym_xform *xform) 336 { 337 crypto_key_sched_t cipher_key_sched; 338 339 cipher_key_sched = sess->cipher.key_sched; 340 if (likely(cipher_key_sched != NULL)) { 341 /* Set up cipher session key */ 342 cipher_key_sched(sess->cipher.key.data, xform->cipher.key.data); 343 } 344 345 return 0; 346 } 347 348 static int 349 armv8_crypto_set_session_chained_parameters(struct armv8_crypto_session *sess, 350 const struct rte_crypto_sym_xform *cipher_xform, 351 const struct rte_crypto_sym_xform *auth_xform) 352 { 353 enum armv8_crypto_chain_order order; 354 enum armv8_crypto_cipher_operation cop; 355 enum rte_crypto_cipher_algorithm calg; 356 enum rte_crypto_auth_algorithm aalg; 357 358 /* Validate and prepare scratch order of combined operations */ 359 switch (sess->chain_order) { 360 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: 361 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: 362 order = sess->chain_order; 363 break; 364 default: 365 return -ENOTSUP; 366 } 367 /* Select cipher direction */ 368 sess->cipher.direction = cipher_xform->cipher.op; 369 /* Select cipher key */ 370 sess->cipher.key.length = cipher_xform->cipher.key.length; 371 /* Set cipher direction */ 372 cop = sess->cipher.direction; 373 /* Set cipher algorithm */ 374 calg = cipher_xform->cipher.algo; 375 376 /* Select cipher algo */ 377 switch (calg) { 378 /* Cover supported cipher algorithms */ 379 case RTE_CRYPTO_CIPHER_AES_CBC: 380 sess->cipher.algo = calg; 381 /* IV len is always 16 bytes (block size) for AES CBC */ 382 sess->cipher.iv.length = 16; 383 break; 384 default: 385 return -ENOTSUP; 386 } 387 /* Select auth generate/verify */ 388 sess->auth.operation = auth_xform->auth.op; 389 390 /* Select auth algo */ 391 switch (auth_xform->auth.algo) { 392 /* Cover supported hash algorithms */ 393 case RTE_CRYPTO_AUTH_SHA1_HMAC: 394 case RTE_CRYPTO_AUTH_SHA256_HMAC: /* Fall through */ 395 aalg = auth_xform->auth.algo; 396 sess->auth.mode = ARMV8_CRYPTO_AUTH_AS_HMAC; 397 break; 398 default: 399 return -ENOTSUP; 400 } 401 402 /* Set the digest length */ 403 sess->auth.digest_length = auth_xform->auth.digest_length; 404 405 /* Verify supported key lengths and extract proper algorithm */ 406 switch (cipher_xform->cipher.key.length << 3) { 407 case 128: 408 sess->crypto_func = 409 CRYPTO_GET_ALGO(order, cop, calg, aalg, 128); 410 sess->cipher.key_sched = 411 CRYPTO_GET_KEY_SCHED(cop, calg, 128); 412 break; 413 case 192: 414 case 256: 415 /* These key lengths are not supported yet */ 416 default: /* Fall through */ 417 sess->crypto_func = NULL; 418 sess->cipher.key_sched = NULL; 419 return -ENOTSUP; 420 } 421 422 if (unlikely(sess->crypto_func == NULL)) { 423 /* 424 * If we got here that means that there must be a bug 425 * in the algorithms selection above. Nevertheless keep 426 * it here to catch bug immediately and avoid NULL pointer 427 * dereference in OPs processing. 428 */ 429 ARMV8_CRYPTO_LOG_ERR( 430 "No appropriate crypto function for given parameters"); 431 return -EINVAL; 432 } 433 434 /* Set up cipher session prerequisites */ 435 if (cipher_set_prerequisites(sess, cipher_xform) != 0) 436 return -EINVAL; 437 438 /* Set up authentication session prerequisites */ 439 if (auth_set_prerequisites(sess, auth_xform) != 0) 440 return -EINVAL; 441 442 return 0; 443 } 444 445 /** Parse crypto xform chain and set private session parameters */ 446 int 447 armv8_crypto_set_session_parameters(struct armv8_crypto_session *sess, 448 const struct rte_crypto_sym_xform *xform) 449 { 450 const struct rte_crypto_sym_xform *cipher_xform = NULL; 451 const struct rte_crypto_sym_xform *auth_xform = NULL; 452 bool is_chained_op; 453 int ret; 454 455 /* Filter out spurious/broken requests */ 456 if (xform == NULL) 457 return -EINVAL; 458 459 sess->chain_order = armv8_crypto_get_chain_order(xform); 460 switch (sess->chain_order) { 461 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: 462 cipher_xform = xform; 463 auth_xform = xform->next; 464 is_chained_op = true; 465 break; 466 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: 467 auth_xform = xform; 468 cipher_xform = xform->next; 469 is_chained_op = true; 470 break; 471 default: 472 is_chained_op = false; 473 return -ENOTSUP; 474 } 475 476 /* Set IV offset */ 477 sess->cipher.iv.offset = cipher_xform->cipher.iv.offset; 478 479 if (is_chained_op) { 480 ret = armv8_crypto_set_session_chained_parameters(sess, 481 cipher_xform, auth_xform); 482 if (unlikely(ret != 0)) { 483 ARMV8_CRYPTO_LOG_ERR( 484 "Invalid/unsupported chained (cipher/auth) parameters"); 485 return ret; 486 } 487 } else { 488 ARMV8_CRYPTO_LOG_ERR("Invalid/unsupported operation"); 489 return -ENOTSUP; 490 } 491 492 return 0; 493 } 494 495 /** Provide session for operation */ 496 static inline struct armv8_crypto_session * 497 get_session(struct armv8_crypto_qp *qp, struct rte_crypto_op *op) 498 { 499 struct armv8_crypto_session *sess = NULL; 500 501 if (op->sess_type == RTE_CRYPTO_OP_WITH_SESSION) { 502 /* get existing session */ 503 if (likely(op->sym->session != NULL)) { 504 sess = (struct armv8_crypto_session *) 505 get_sym_session_private_data( 506 op->sym->session, 507 cryptodev_driver_id); 508 } 509 } else { 510 /* provide internal session */ 511 void *_sess = NULL; 512 void *_sess_private_data = NULL; 513 514 if (rte_mempool_get(qp->sess_mp, (void **)&_sess)) 515 return NULL; 516 517 if (rte_mempool_get(qp->sess_mp_priv, 518 (void **)&_sess_private_data)) 519 return NULL; 520 521 sess = (struct armv8_crypto_session *)_sess_private_data; 522 523 if (unlikely(armv8_crypto_set_session_parameters(sess, 524 op->sym->xform) != 0)) { 525 rte_mempool_put(qp->sess_mp, _sess); 526 rte_mempool_put(qp->sess_mp_priv, _sess_private_data); 527 sess = NULL; 528 } 529 op->sym->session = (struct rte_cryptodev_sym_session *)_sess; 530 set_sym_session_private_data(op->sym->session, 531 cryptodev_driver_id, _sess_private_data); 532 } 533 534 if (unlikely(sess == NULL)) 535 op->status = RTE_CRYPTO_OP_STATUS_INVALID_SESSION; 536 537 return sess; 538 } 539 540 /* 541 *------------------------------------------------------------------------------ 542 * Process Operations 543 *------------------------------------------------------------------------------ 544 */ 545 546 /*----------------------------------------------------------------------------*/ 547 548 /** Process cipher operation */ 549 static inline void 550 process_armv8_chained_op(struct armv8_crypto_qp *qp, struct rte_crypto_op *op, 551 struct armv8_crypto_session *sess, 552 struct rte_mbuf *mbuf_src, struct rte_mbuf *mbuf_dst) 553 { 554 crypto_func_t crypto_func; 555 crypto_arg_t arg; 556 struct rte_mbuf *m_asrc, *m_adst; 557 uint8_t *csrc, *cdst; 558 uint8_t *adst, *asrc; 559 uint64_t clen, alen; 560 int error; 561 562 clen = op->sym->cipher.data.length; 563 alen = op->sym->auth.data.length; 564 565 csrc = rte_pktmbuf_mtod_offset(mbuf_src, uint8_t *, 566 op->sym->cipher.data.offset); 567 cdst = rte_pktmbuf_mtod_offset(mbuf_dst, uint8_t *, 568 op->sym->cipher.data.offset); 569 570 switch (sess->chain_order) { 571 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: 572 m_asrc = m_adst = mbuf_dst; 573 break; 574 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: 575 m_asrc = mbuf_src; 576 m_adst = mbuf_dst; 577 break; 578 default: 579 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 580 return; 581 } 582 asrc = rte_pktmbuf_mtod_offset(m_asrc, uint8_t *, 583 op->sym->auth.data.offset); 584 585 switch (sess->auth.mode) { 586 case ARMV8_CRYPTO_AUTH_AS_AUTH: 587 /* Nothing to do here, just verify correct option */ 588 break; 589 case ARMV8_CRYPTO_AUTH_AS_HMAC: 590 arg.digest.hmac.key = sess->auth.hmac.key; 591 arg.digest.hmac.i_key_pad = sess->auth.hmac.i_key_pad; 592 arg.digest.hmac.o_key_pad = sess->auth.hmac.o_key_pad; 593 break; 594 default: 595 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 596 return; 597 } 598 599 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_GENERATE) { 600 adst = op->sym->auth.digest.data; 601 if (adst == NULL) { 602 adst = rte_pktmbuf_mtod_offset(m_adst, 603 uint8_t *, 604 op->sym->auth.data.offset + 605 op->sym->auth.data.length); 606 } 607 } else { 608 adst = qp->temp_digest; 609 } 610 611 arg.cipher.iv = rte_crypto_op_ctod_offset(op, uint8_t *, 612 sess->cipher.iv.offset); 613 arg.cipher.key = sess->cipher.key.data; 614 /* Acquire combined mode function */ 615 crypto_func = sess->crypto_func; 616 ARMV8_CRYPTO_ASSERT(crypto_func != NULL); 617 error = crypto_func(csrc, cdst, clen, asrc, adst, alen, &arg); 618 if (error != 0) { 619 op->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 620 return; 621 } 622 623 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 624 if (sess->auth.operation == RTE_CRYPTO_AUTH_OP_VERIFY) { 625 if (memcmp(adst, op->sym->auth.digest.data, 626 sess->auth.digest_length) != 0) { 627 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 628 } 629 } 630 } 631 632 /** Process crypto operation for mbuf */ 633 static inline int 634 process_op(struct armv8_crypto_qp *qp, struct rte_crypto_op *op, 635 struct armv8_crypto_session *sess) 636 { 637 struct rte_mbuf *msrc, *mdst; 638 639 msrc = op->sym->m_src; 640 mdst = op->sym->m_dst ? op->sym->m_dst : op->sym->m_src; 641 642 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 643 644 switch (sess->chain_order) { 645 case ARMV8_CRYPTO_CHAIN_CIPHER_AUTH: 646 case ARMV8_CRYPTO_CHAIN_AUTH_CIPHER: /* Fall through */ 647 process_armv8_chained_op(qp, op, sess, msrc, mdst); 648 break; 649 default: 650 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 651 break; 652 } 653 654 /* Free session if a session-less crypto op */ 655 if (op->sess_type == RTE_CRYPTO_OP_SESSIONLESS) { 656 memset(sess, 0, sizeof(struct armv8_crypto_session)); 657 memset(op->sym->session, 0, 658 rte_cryptodev_sym_get_existing_header_session_size( 659 op->sym->session)); 660 rte_mempool_put(qp->sess_mp, sess); 661 rte_mempool_put(qp->sess_mp_priv, op->sym->session); 662 op->sym->session = NULL; 663 } 664 665 if (op->status == RTE_CRYPTO_OP_STATUS_NOT_PROCESSED) 666 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 667 668 if (unlikely(op->status == RTE_CRYPTO_OP_STATUS_ERROR)) 669 return -1; 670 671 return 0; 672 } 673 674 /* 675 *------------------------------------------------------------------------------ 676 * PMD Framework 677 *------------------------------------------------------------------------------ 678 */ 679 680 /** Enqueue burst */ 681 static uint16_t 682 armv8_crypto_pmd_enqueue_burst(void *queue_pair, struct rte_crypto_op **ops, 683 uint16_t nb_ops) 684 { 685 struct armv8_crypto_session *sess; 686 struct armv8_crypto_qp *qp = queue_pair; 687 int i, retval; 688 689 for (i = 0; i < nb_ops; i++) { 690 sess = get_session(qp, ops[i]); 691 if (unlikely(sess == NULL)) 692 goto enqueue_err; 693 694 retval = process_op(qp, ops[i], sess); 695 if (unlikely(retval < 0)) 696 goto enqueue_err; 697 } 698 699 retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i, 700 NULL); 701 qp->stats.enqueued_count += retval; 702 703 return retval; 704 705 enqueue_err: 706 retval = rte_ring_enqueue_burst(qp->processed_ops, (void *)ops, i, 707 NULL); 708 if (ops[i] != NULL) 709 ops[i]->status = RTE_CRYPTO_OP_STATUS_INVALID_ARGS; 710 711 qp->stats.enqueue_err_count++; 712 return retval; 713 } 714 715 /** Dequeue burst */ 716 static uint16_t 717 armv8_crypto_pmd_dequeue_burst(void *queue_pair, struct rte_crypto_op **ops, 718 uint16_t nb_ops) 719 { 720 struct armv8_crypto_qp *qp = queue_pair; 721 722 unsigned int nb_dequeued = 0; 723 724 nb_dequeued = rte_ring_dequeue_burst(qp->processed_ops, 725 (void **)ops, nb_ops, NULL); 726 qp->stats.dequeued_count += nb_dequeued; 727 728 return nb_dequeued; 729 } 730 731 /** Create ARMv8 crypto device */ 732 static int 733 cryptodev_armv8_crypto_create(const char *name, 734 struct rte_vdev_device *vdev, 735 struct rte_cryptodev_pmd_init_params *init_params) 736 { 737 struct rte_cryptodev *dev; 738 struct armv8_crypto_private *internals; 739 740 /* Check CPU for support for AES instruction set */ 741 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_AES)) { 742 ARMV8_CRYPTO_LOG_ERR( 743 "AES instructions not supported by CPU"); 744 return -EFAULT; 745 } 746 747 /* Check CPU for support for SHA instruction set */ 748 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA1) || 749 !rte_cpu_get_flag_enabled(RTE_CPUFLAG_SHA2)) { 750 ARMV8_CRYPTO_LOG_ERR( 751 "SHA1/SHA2 instructions not supported by CPU"); 752 return -EFAULT; 753 } 754 755 /* Check CPU for support for Advance SIMD instruction set */ 756 if (!rte_cpu_get_flag_enabled(RTE_CPUFLAG_NEON)) { 757 ARMV8_CRYPTO_LOG_ERR( 758 "Advanced SIMD instructions not supported by CPU"); 759 return -EFAULT; 760 } 761 762 dev = rte_cryptodev_pmd_create(name, &vdev->device, init_params); 763 if (dev == NULL) { 764 ARMV8_CRYPTO_LOG_ERR("failed to create cryptodev vdev"); 765 goto init_error; 766 } 767 768 dev->driver_id = cryptodev_driver_id; 769 dev->dev_ops = rte_armv8_crypto_pmd_ops; 770 771 /* register rx/tx burst functions for data path */ 772 dev->dequeue_burst = armv8_crypto_pmd_dequeue_burst; 773 dev->enqueue_burst = armv8_crypto_pmd_enqueue_burst; 774 775 dev->feature_flags = RTE_CRYPTODEV_FF_SYMMETRIC_CRYPTO | 776 RTE_CRYPTODEV_FF_SYM_OPERATION_CHAINING | 777 RTE_CRYPTODEV_FF_CPU_NEON | 778 RTE_CRYPTODEV_FF_CPU_ARM_CE; 779 780 /* Set vector instructions mode supported */ 781 internals = dev->data->dev_private; 782 783 internals->max_nb_qpairs = init_params->max_nb_queue_pairs; 784 785 return 0; 786 787 init_error: 788 ARMV8_CRYPTO_LOG_ERR( 789 "driver %s: cryptodev_armv8_crypto_create failed", 790 init_params->name); 791 792 cryptodev_armv8_crypto_uninit(vdev); 793 return -EFAULT; 794 } 795 796 /** Initialise ARMv8 crypto device */ 797 static int 798 cryptodev_armv8_crypto_init(struct rte_vdev_device *vdev) 799 { 800 struct rte_cryptodev_pmd_init_params init_params = { 801 "", 802 sizeof(struct armv8_crypto_private), 803 rte_socket_id(), 804 RTE_CRYPTODEV_PMD_DEFAULT_MAX_NB_QUEUE_PAIRS 805 }; 806 const char *name; 807 const char *input_args; 808 809 name = rte_vdev_device_name(vdev); 810 if (name == NULL) 811 return -EINVAL; 812 input_args = rte_vdev_device_args(vdev); 813 rte_cryptodev_pmd_parse_input_args(&init_params, input_args); 814 815 return cryptodev_armv8_crypto_create(name, vdev, &init_params); 816 } 817 818 /** Uninitialise ARMv8 crypto device */ 819 static int 820 cryptodev_armv8_crypto_uninit(struct rte_vdev_device *vdev) 821 { 822 struct rte_cryptodev *cryptodev; 823 const char *name; 824 825 name = rte_vdev_device_name(vdev); 826 if (name == NULL) 827 return -EINVAL; 828 829 RTE_LOG(INFO, PMD, 830 "Closing ARMv8 crypto device %s on numa socket %u\n", 831 name, rte_socket_id()); 832 833 cryptodev = rte_cryptodev_pmd_get_named_dev(name); 834 if (cryptodev == NULL) 835 return -ENODEV; 836 837 return rte_cryptodev_pmd_destroy(cryptodev); 838 } 839 840 static struct rte_vdev_driver armv8_crypto_pmd_drv = { 841 .probe = cryptodev_armv8_crypto_init, 842 .remove = cryptodev_armv8_crypto_uninit 843 }; 844 845 static struct cryptodev_driver armv8_crypto_drv; 846 847 RTE_PMD_REGISTER_VDEV(CRYPTODEV_NAME_ARMV8_PMD, armv8_crypto_pmd_drv); 848 RTE_PMD_REGISTER_ALIAS(CRYPTODEV_NAME_ARMV8_PMD, cryptodev_armv8_pmd); 849 RTE_PMD_REGISTER_PARAM_STRING(CRYPTODEV_NAME_ARMV8_PMD, 850 "max_nb_queue_pairs=<int> " 851 "socket_id=<int>"); 852 RTE_PMD_REGISTER_CRYPTO_DRIVER(armv8_crypto_drv, armv8_crypto_pmd_drv.driver, 853 cryptodev_driver_id); 854