1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Advanced Micro Devices, Inc. All rights reserved. 3 */ 4 5 #include <dirent.h> 6 #include <fcntl.h> 7 #include <stdio.h> 8 #include <string.h> 9 #include <sys/mman.h> 10 #include <sys/queue.h> 11 #include <sys/types.h> 12 #include <unistd.h> 13 14 #include <rte_hexdump.h> 15 #include <rte_memzone.h> 16 #include <rte_malloc.h> 17 #include <rte_memory.h> 18 #include <rte_spinlock.h> 19 #include <rte_string_fns.h> 20 #include <rte_cryptodev_pmd.h> 21 22 #include "ccp_dev.h" 23 #include "ccp_crypto.h" 24 #include "ccp_pci.h" 25 #include "ccp_pmd_private.h" 26 27 static enum ccp_cmd_order 28 ccp_get_cmd_id(const struct rte_crypto_sym_xform *xform) 29 { 30 enum ccp_cmd_order res = CCP_CMD_NOT_SUPPORTED; 31 32 if (xform == NULL) 33 return res; 34 if (xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) { 35 if (xform->next == NULL) 36 return CCP_CMD_AUTH; 37 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) 38 return CCP_CMD_HASH_CIPHER; 39 } 40 if (xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) { 41 if (xform->next == NULL) 42 return CCP_CMD_CIPHER; 43 else if (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) 44 return CCP_CMD_CIPHER_HASH; 45 } 46 if (xform->type == RTE_CRYPTO_SYM_XFORM_AEAD) 47 return CCP_CMD_COMBINED; 48 return res; 49 } 50 51 /* configure session */ 52 static int 53 ccp_configure_session_cipher(struct ccp_session *sess, 54 const struct rte_crypto_sym_xform *xform) 55 { 56 const struct rte_crypto_cipher_xform *cipher_xform = NULL; 57 size_t i, j, x; 58 59 cipher_xform = &xform->cipher; 60 61 /* set cipher direction */ 62 if (cipher_xform->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 63 sess->cipher.dir = CCP_CIPHER_DIR_ENCRYPT; 64 else 65 sess->cipher.dir = CCP_CIPHER_DIR_DECRYPT; 66 67 /* set cipher key */ 68 sess->cipher.key_length = cipher_xform->key.length; 69 rte_memcpy(sess->cipher.key, cipher_xform->key.data, 70 cipher_xform->key.length); 71 72 /* set iv parameters */ 73 sess->iv.offset = cipher_xform->iv.offset; 74 sess->iv.length = cipher_xform->iv.length; 75 76 switch (cipher_xform->algo) { 77 case RTE_CRYPTO_CIPHER_AES_CTR: 78 sess->cipher.algo = CCP_CIPHER_ALGO_AES_CTR; 79 sess->cipher.um.aes_mode = CCP_AES_MODE_CTR; 80 sess->cipher.engine = CCP_ENGINE_AES; 81 break; 82 case RTE_CRYPTO_CIPHER_AES_ECB: 83 sess->cipher.algo = CCP_CIPHER_ALGO_AES_CBC; 84 sess->cipher.um.aes_mode = CCP_AES_MODE_ECB; 85 sess->cipher.engine = CCP_ENGINE_AES; 86 break; 87 case RTE_CRYPTO_CIPHER_AES_CBC: 88 sess->cipher.algo = CCP_CIPHER_ALGO_AES_CBC; 89 sess->cipher.um.aes_mode = CCP_AES_MODE_CBC; 90 sess->cipher.engine = CCP_ENGINE_AES; 91 break; 92 case RTE_CRYPTO_CIPHER_3DES_CBC: 93 sess->cipher.algo = CCP_CIPHER_ALGO_3DES_CBC; 94 sess->cipher.um.des_mode = CCP_DES_MODE_CBC; 95 sess->cipher.engine = CCP_ENGINE_3DES; 96 break; 97 default: 98 CCP_LOG_ERR("Unsupported cipher algo"); 99 return -1; 100 } 101 102 103 switch (sess->cipher.engine) { 104 case CCP_ENGINE_AES: 105 if (sess->cipher.key_length == 16) 106 sess->cipher.ut.aes_type = CCP_AES_TYPE_128; 107 else if (sess->cipher.key_length == 24) 108 sess->cipher.ut.aes_type = CCP_AES_TYPE_192; 109 else if (sess->cipher.key_length == 32) 110 sess->cipher.ut.aes_type = CCP_AES_TYPE_256; 111 else { 112 CCP_LOG_ERR("Invalid cipher key length"); 113 return -1; 114 } 115 for (i = 0; i < sess->cipher.key_length ; i++) 116 sess->cipher.key_ccp[sess->cipher.key_length - i - 1] = 117 sess->cipher.key[i]; 118 break; 119 case CCP_ENGINE_3DES: 120 if (sess->cipher.key_length == 16) 121 sess->cipher.ut.des_type = CCP_DES_TYPE_128; 122 else if (sess->cipher.key_length == 24) 123 sess->cipher.ut.des_type = CCP_DES_TYPE_192; 124 else { 125 CCP_LOG_ERR("Invalid cipher key length"); 126 return -1; 127 } 128 for (j = 0, x = 0; j < sess->cipher.key_length/8; j++, x += 8) 129 for (i = 0; i < 8; i++) 130 sess->cipher.key_ccp[(8 + x) - i - 1] = 131 sess->cipher.key[i + x]; 132 break; 133 default: 134 CCP_LOG_ERR("Invalid CCP Engine"); 135 return -ENOTSUP; 136 } 137 sess->cipher.nonce_phys = rte_mem_virt2phy(sess->cipher.nonce); 138 sess->cipher.key_phys = rte_mem_virt2phy(sess->cipher.key_ccp); 139 return 0; 140 } 141 142 static int 143 ccp_configure_session_auth(struct ccp_session *sess, 144 const struct rte_crypto_sym_xform *xform) 145 { 146 const struct rte_crypto_auth_xform *auth_xform = NULL; 147 148 auth_xform = &xform->auth; 149 150 sess->auth.digest_length = auth_xform->digest_length; 151 if (auth_xform->op == RTE_CRYPTO_AUTH_OP_GENERATE) 152 sess->auth.op = CCP_AUTH_OP_GENERATE; 153 else 154 sess->auth.op = CCP_AUTH_OP_VERIFY; 155 switch (auth_xform->algo) { 156 default: 157 CCP_LOG_ERR("Unsupported hash algo"); 158 return -ENOTSUP; 159 } 160 return 0; 161 } 162 163 static int 164 ccp_configure_session_aead(struct ccp_session *sess, 165 const struct rte_crypto_sym_xform *xform) 166 { 167 const struct rte_crypto_aead_xform *aead_xform = NULL; 168 169 aead_xform = &xform->aead; 170 171 sess->cipher.key_length = aead_xform->key.length; 172 rte_memcpy(sess->cipher.key, aead_xform->key.data, 173 aead_xform->key.length); 174 175 if (aead_xform->op == RTE_CRYPTO_AEAD_OP_ENCRYPT) { 176 sess->cipher.dir = CCP_CIPHER_DIR_ENCRYPT; 177 sess->auth.op = CCP_AUTH_OP_GENERATE; 178 } else { 179 sess->cipher.dir = CCP_CIPHER_DIR_DECRYPT; 180 sess->auth.op = CCP_AUTH_OP_VERIFY; 181 } 182 sess->auth.aad_length = aead_xform->aad_length; 183 sess->auth.digest_length = aead_xform->digest_length; 184 185 /* set iv parameters */ 186 sess->iv.offset = aead_xform->iv.offset; 187 sess->iv.length = aead_xform->iv.length; 188 189 switch (aead_xform->algo) { 190 default: 191 CCP_LOG_ERR("Unsupported aead algo"); 192 return -ENOTSUP; 193 } 194 return 0; 195 } 196 197 int 198 ccp_set_session_parameters(struct ccp_session *sess, 199 const struct rte_crypto_sym_xform *xform) 200 { 201 const struct rte_crypto_sym_xform *cipher_xform = NULL; 202 const struct rte_crypto_sym_xform *auth_xform = NULL; 203 const struct rte_crypto_sym_xform *aead_xform = NULL; 204 int ret = 0; 205 206 sess->cmd_id = ccp_get_cmd_id(xform); 207 208 switch (sess->cmd_id) { 209 case CCP_CMD_CIPHER: 210 cipher_xform = xform; 211 break; 212 case CCP_CMD_AUTH: 213 auth_xform = xform; 214 break; 215 case CCP_CMD_CIPHER_HASH: 216 cipher_xform = xform; 217 auth_xform = xform->next; 218 break; 219 case CCP_CMD_HASH_CIPHER: 220 auth_xform = xform; 221 cipher_xform = xform->next; 222 break; 223 case CCP_CMD_COMBINED: 224 aead_xform = xform; 225 break; 226 default: 227 CCP_LOG_ERR("Unsupported cmd_id"); 228 return -1; 229 } 230 231 /* Default IV length = 0 */ 232 sess->iv.length = 0; 233 if (cipher_xform) { 234 ret = ccp_configure_session_cipher(sess, cipher_xform); 235 if (ret != 0) { 236 CCP_LOG_ERR("Invalid/unsupported cipher parameters"); 237 return ret; 238 } 239 } 240 if (auth_xform) { 241 ret = ccp_configure_session_auth(sess, auth_xform); 242 if (ret != 0) { 243 CCP_LOG_ERR("Invalid/unsupported auth parameters"); 244 return ret; 245 } 246 } 247 if (aead_xform) { 248 ret = ccp_configure_session_aead(sess, aead_xform); 249 if (ret != 0) { 250 CCP_LOG_ERR("Invalid/unsupported aead parameters"); 251 return ret; 252 } 253 } 254 return ret; 255 } 256 257 /* calculate CCP descriptors requirement */ 258 static inline int 259 ccp_cipher_slot(struct ccp_session *session) 260 { 261 int count = 0; 262 263 switch (session->cipher.algo) { 264 case CCP_CIPHER_ALGO_AES_CBC: 265 count = 2; 266 /**< op + passthrough for iv */ 267 break; 268 case CCP_CIPHER_ALGO_AES_ECB: 269 count = 1; 270 /**<only op*/ 271 break; 272 case CCP_CIPHER_ALGO_AES_CTR: 273 count = 2; 274 /**< op + passthrough for iv */ 275 break; 276 case CCP_CIPHER_ALGO_3DES_CBC: 277 count = 2; 278 /**< op + passthrough for iv */ 279 break; 280 default: 281 CCP_LOG_ERR("Unsupported cipher algo %d", 282 session->cipher.algo); 283 } 284 return count; 285 } 286 287 static inline int 288 ccp_auth_slot(struct ccp_session *session) 289 { 290 int count = 0; 291 292 switch (session->auth.algo) { 293 default: 294 CCP_LOG_ERR("Unsupported auth algo %d", 295 session->auth.algo); 296 } 297 298 return count; 299 } 300 301 static int 302 ccp_aead_slot(struct ccp_session *session) 303 { 304 int count = 0; 305 306 switch (session->aead_algo) { 307 default: 308 CCP_LOG_ERR("Unsupported aead algo %d", 309 session->aead_algo); 310 } 311 return count; 312 } 313 314 int 315 ccp_compute_slot_count(struct ccp_session *session) 316 { 317 int count = 0; 318 319 switch (session->cmd_id) { 320 case CCP_CMD_CIPHER: 321 count = ccp_cipher_slot(session); 322 break; 323 case CCP_CMD_AUTH: 324 count = ccp_auth_slot(session); 325 break; 326 case CCP_CMD_CIPHER_HASH: 327 case CCP_CMD_HASH_CIPHER: 328 count = ccp_cipher_slot(session); 329 count += ccp_auth_slot(session); 330 break; 331 case CCP_CMD_COMBINED: 332 count = ccp_aead_slot(session); 333 break; 334 default: 335 CCP_LOG_ERR("Unsupported cmd_id"); 336 337 } 338 339 return count; 340 } 341 342 static void 343 ccp_perform_passthru(struct ccp_passthru *pst, 344 struct ccp_queue *cmd_q) 345 { 346 struct ccp_desc *desc; 347 union ccp_function function; 348 349 desc = &cmd_q->qbase_desc[cmd_q->qidx]; 350 351 CCP_CMD_ENGINE(desc) = CCP_ENGINE_PASSTHRU; 352 353 CCP_CMD_SOC(desc) = 0; 354 CCP_CMD_IOC(desc) = 0; 355 CCP_CMD_INIT(desc) = 0; 356 CCP_CMD_EOM(desc) = 0; 357 CCP_CMD_PROT(desc) = 0; 358 359 function.raw = 0; 360 CCP_PT_BYTESWAP(&function) = pst->byte_swap; 361 CCP_PT_BITWISE(&function) = pst->bit_mod; 362 CCP_CMD_FUNCTION(desc) = function.raw; 363 364 CCP_CMD_LEN(desc) = pst->len; 365 366 if (pst->dir) { 367 CCP_CMD_SRC_LO(desc) = (uint32_t)(pst->src_addr); 368 CCP_CMD_SRC_HI(desc) = high32_value(pst->src_addr); 369 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM; 370 371 CCP_CMD_DST_LO(desc) = (uint32_t)(pst->dest_addr); 372 CCP_CMD_DST_HI(desc) = 0; 373 CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SB; 374 375 if (pst->bit_mod != CCP_PASSTHRU_BITWISE_NOOP) 376 CCP_CMD_LSB_ID(desc) = cmd_q->sb_key; 377 } else { 378 379 CCP_CMD_SRC_LO(desc) = (uint32_t)(pst->src_addr); 380 CCP_CMD_SRC_HI(desc) = 0; 381 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SB; 382 383 CCP_CMD_DST_LO(desc) = (uint32_t)(pst->dest_addr); 384 CCP_CMD_DST_HI(desc) = high32_value(pst->dest_addr); 385 CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM; 386 } 387 388 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE; 389 } 390 391 static int 392 ccp_perform_aes(struct rte_crypto_op *op, 393 struct ccp_queue *cmd_q, 394 struct ccp_batch_info *b_info) 395 { 396 struct ccp_session *session; 397 union ccp_function function; 398 uint8_t *lsb_buf; 399 struct ccp_passthru pst = {0}; 400 struct ccp_desc *desc; 401 phys_addr_t src_addr, dest_addr, key_addr; 402 uint8_t *iv; 403 404 session = (struct ccp_session *)get_session_private_data( 405 op->sym->session, 406 ccp_cryptodev_driver_id); 407 function.raw = 0; 408 409 iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset); 410 if (session->cipher.um.aes_mode != CCP_AES_MODE_ECB) { 411 if (session->cipher.um.aes_mode == CCP_AES_MODE_CTR) { 412 rte_memcpy(session->cipher.nonce + AES_BLOCK_SIZE, 413 iv, session->iv.length); 414 pst.src_addr = (phys_addr_t)session->cipher.nonce_phys; 415 CCP_AES_SIZE(&function) = 0x1F; 416 } else { 417 lsb_buf = 418 &(b_info->lsb_buf[b_info->lsb_buf_idx*CCP_SB_BYTES]); 419 rte_memcpy(lsb_buf + 420 (CCP_SB_BYTES - session->iv.length), 421 iv, session->iv.length); 422 pst.src_addr = b_info->lsb_buf_phys + 423 (b_info->lsb_buf_idx * CCP_SB_BYTES); 424 b_info->lsb_buf_idx++; 425 } 426 427 pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES); 428 pst.len = CCP_SB_BYTES; 429 pst.dir = 1; 430 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP; 431 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT; 432 ccp_perform_passthru(&pst, cmd_q); 433 } 434 435 desc = &cmd_q->qbase_desc[cmd_q->qidx]; 436 437 src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src, 438 op->sym->cipher.data.offset); 439 if (likely(op->sym->m_dst != NULL)) 440 dest_addr = rte_pktmbuf_mtophys_offset(op->sym->m_dst, 441 op->sym->cipher.data.offset); 442 else 443 dest_addr = src_addr; 444 key_addr = session->cipher.key_phys; 445 446 /* prepare desc for aes command */ 447 CCP_CMD_ENGINE(desc) = CCP_ENGINE_AES; 448 CCP_CMD_INIT(desc) = 1; 449 CCP_CMD_EOM(desc) = 1; 450 451 CCP_AES_ENCRYPT(&function) = session->cipher.dir; 452 CCP_AES_MODE(&function) = session->cipher.um.aes_mode; 453 CCP_AES_TYPE(&function) = session->cipher.ut.aes_type; 454 CCP_CMD_FUNCTION(desc) = function.raw; 455 456 CCP_CMD_LEN(desc) = op->sym->cipher.data.length; 457 458 CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr); 459 CCP_CMD_SRC_HI(desc) = high32_value(src_addr); 460 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM; 461 462 CCP_CMD_DST_LO(desc) = ((uint32_t)dest_addr); 463 CCP_CMD_DST_HI(desc) = high32_value(dest_addr); 464 CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM; 465 466 CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr); 467 CCP_CMD_KEY_HI(desc) = high32_value(key_addr); 468 CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM; 469 470 if (session->cipher.um.aes_mode != CCP_AES_MODE_ECB) 471 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv; 472 473 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE; 474 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 475 return 0; 476 } 477 478 static int 479 ccp_perform_3des(struct rte_crypto_op *op, 480 struct ccp_queue *cmd_q, 481 struct ccp_batch_info *b_info) 482 { 483 struct ccp_session *session; 484 union ccp_function function; 485 unsigned char *lsb_buf; 486 struct ccp_passthru pst; 487 struct ccp_desc *desc; 488 uint32_t tail; 489 uint8_t *iv; 490 phys_addr_t src_addr, dest_addr, key_addr; 491 492 session = (struct ccp_session *)get_session_private_data( 493 op->sym->session, 494 ccp_cryptodev_driver_id); 495 496 iv = rte_crypto_op_ctod_offset(op, uint8_t *, session->iv.offset); 497 switch (session->cipher.um.des_mode) { 498 case CCP_DES_MODE_CBC: 499 lsb_buf = &(b_info->lsb_buf[b_info->lsb_buf_idx*CCP_SB_BYTES]); 500 b_info->lsb_buf_idx++; 501 502 rte_memcpy(lsb_buf + (CCP_SB_BYTES - session->iv.length), 503 iv, session->iv.length); 504 505 pst.src_addr = (phys_addr_t)rte_mem_virt2phy((void *) lsb_buf); 506 pst.dest_addr = (phys_addr_t)(cmd_q->sb_iv * CCP_SB_BYTES); 507 pst.len = CCP_SB_BYTES; 508 pst.dir = 1; 509 pst.bit_mod = CCP_PASSTHRU_BITWISE_NOOP; 510 pst.byte_swap = CCP_PASSTHRU_BYTESWAP_256BIT; 511 ccp_perform_passthru(&pst, cmd_q); 512 break; 513 case CCP_DES_MODE_CFB: 514 case CCP_DES_MODE_ECB: 515 CCP_LOG_ERR("Unsupported DES cipher mode"); 516 return -ENOTSUP; 517 } 518 519 src_addr = rte_pktmbuf_mtophys_offset(op->sym->m_src, 520 op->sym->cipher.data.offset); 521 if (unlikely(op->sym->m_dst != NULL)) 522 dest_addr = 523 rte_pktmbuf_mtophys_offset(op->sym->m_dst, 524 op->sym->cipher.data.offset); 525 else 526 dest_addr = src_addr; 527 528 key_addr = rte_mem_virt2phy(session->cipher.key_ccp); 529 530 desc = &cmd_q->qbase_desc[cmd_q->qidx]; 531 532 memset(desc, 0, Q_DESC_SIZE); 533 534 /* prepare desc for des command */ 535 CCP_CMD_ENGINE(desc) = CCP_ENGINE_3DES; 536 537 CCP_CMD_SOC(desc) = 0; 538 CCP_CMD_IOC(desc) = 0; 539 CCP_CMD_INIT(desc) = 1; 540 CCP_CMD_EOM(desc) = 1; 541 CCP_CMD_PROT(desc) = 0; 542 543 function.raw = 0; 544 CCP_DES_ENCRYPT(&function) = session->cipher.dir; 545 CCP_DES_MODE(&function) = session->cipher.um.des_mode; 546 CCP_DES_TYPE(&function) = session->cipher.ut.des_type; 547 CCP_CMD_FUNCTION(desc) = function.raw; 548 549 CCP_CMD_LEN(desc) = op->sym->cipher.data.length; 550 551 CCP_CMD_SRC_LO(desc) = ((uint32_t)src_addr); 552 CCP_CMD_SRC_HI(desc) = high32_value(src_addr); 553 CCP_CMD_SRC_MEM(desc) = CCP_MEMTYPE_SYSTEM; 554 555 CCP_CMD_DST_LO(desc) = ((uint32_t)dest_addr); 556 CCP_CMD_DST_HI(desc) = high32_value(dest_addr); 557 CCP_CMD_DST_MEM(desc) = CCP_MEMTYPE_SYSTEM; 558 559 CCP_CMD_KEY_LO(desc) = ((uint32_t)key_addr); 560 CCP_CMD_KEY_HI(desc) = high32_value(key_addr); 561 CCP_CMD_KEY_MEM(desc) = CCP_MEMTYPE_SYSTEM; 562 563 if (session->cipher.um.des_mode) 564 CCP_CMD_LSB_ID(desc) = cmd_q->sb_iv; 565 566 cmd_q->qidx = (cmd_q->qidx + 1) % COMMANDS_PER_QUEUE; 567 568 rte_wmb(); 569 570 /* Write the new tail address back to the queue register */ 571 tail = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * Q_DESC_SIZE); 572 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, tail); 573 /* Turn the queue back on using our cached control register */ 574 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE, 575 cmd_q->qcontrol | CMD_Q_RUN); 576 577 op->status = RTE_CRYPTO_OP_STATUS_NOT_PROCESSED; 578 return 0; 579 } 580 581 static inline int 582 ccp_crypto_cipher(struct rte_crypto_op *op, 583 struct ccp_queue *cmd_q, 584 struct ccp_batch_info *b_info) 585 { 586 int result = 0; 587 struct ccp_session *session; 588 589 session = (struct ccp_session *)get_session_private_data( 590 op->sym->session, 591 ccp_cryptodev_driver_id); 592 593 switch (session->cipher.algo) { 594 case CCP_CIPHER_ALGO_AES_CBC: 595 result = ccp_perform_aes(op, cmd_q, b_info); 596 b_info->desccnt += 2; 597 break; 598 case CCP_CIPHER_ALGO_AES_CTR: 599 result = ccp_perform_aes(op, cmd_q, b_info); 600 b_info->desccnt += 2; 601 break; 602 case CCP_CIPHER_ALGO_AES_ECB: 603 result = ccp_perform_aes(op, cmd_q, b_info); 604 b_info->desccnt += 1; 605 break; 606 case CCP_CIPHER_ALGO_3DES_CBC: 607 result = ccp_perform_3des(op, cmd_q, b_info); 608 b_info->desccnt += 2; 609 break; 610 default: 611 CCP_LOG_ERR("Unsupported cipher algo %d", 612 session->cipher.algo); 613 return -ENOTSUP; 614 } 615 return result; 616 } 617 618 static inline int 619 ccp_crypto_auth(struct rte_crypto_op *op, 620 struct ccp_queue *cmd_q __rte_unused, 621 struct ccp_batch_info *b_info __rte_unused) 622 { 623 624 int result = 0; 625 struct ccp_session *session; 626 627 session = (struct ccp_session *)get_session_private_data( 628 op->sym->session, 629 ccp_cryptodev_driver_id); 630 631 switch (session->auth.algo) { 632 default: 633 CCP_LOG_ERR("Unsupported auth algo %d", 634 session->auth.algo); 635 return -ENOTSUP; 636 } 637 638 return result; 639 } 640 641 static inline int 642 ccp_crypto_aead(struct rte_crypto_op *op, 643 struct ccp_queue *cmd_q __rte_unused, 644 struct ccp_batch_info *b_info __rte_unused) 645 { 646 int result = 0; 647 struct ccp_session *session; 648 649 session = (struct ccp_session *)get_session_private_data( 650 op->sym->session, 651 ccp_cryptodev_driver_id); 652 653 switch (session->aead_algo) { 654 default: 655 CCP_LOG_ERR("Unsupported aead algo %d", 656 session->aead_algo); 657 return -ENOTSUP; 658 } 659 return result; 660 } 661 662 int 663 process_ops_to_enqueue(const struct ccp_qp *qp, 664 struct rte_crypto_op **op, 665 struct ccp_queue *cmd_q, 666 uint16_t nb_ops, 667 int slots_req) 668 { 669 int i, result = 0; 670 struct ccp_batch_info *b_info; 671 struct ccp_session *session; 672 673 if (rte_mempool_get(qp->batch_mp, (void **)&b_info)) { 674 CCP_LOG_ERR("batch info allocation failed"); 675 return 0; 676 } 677 /* populate batch info necessary for dequeue */ 678 b_info->op_idx = 0; 679 b_info->lsb_buf_idx = 0; 680 b_info->desccnt = 0; 681 b_info->cmd_q = cmd_q; 682 b_info->lsb_buf_phys = 683 (phys_addr_t)rte_mem_virt2phy((void *)b_info->lsb_buf); 684 rte_atomic64_sub(&b_info->cmd_q->free_slots, slots_req); 685 686 b_info->head_offset = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * 687 Q_DESC_SIZE); 688 for (i = 0; i < nb_ops; i++) { 689 session = (struct ccp_session *)get_session_private_data( 690 op[i]->sym->session, 691 ccp_cryptodev_driver_id); 692 switch (session->cmd_id) { 693 case CCP_CMD_CIPHER: 694 result = ccp_crypto_cipher(op[i], cmd_q, b_info); 695 break; 696 case CCP_CMD_AUTH: 697 result = ccp_crypto_auth(op[i], cmd_q, b_info); 698 break; 699 case CCP_CMD_CIPHER_HASH: 700 result = ccp_crypto_cipher(op[i], cmd_q, b_info); 701 if (result) 702 break; 703 result = ccp_crypto_auth(op[i], cmd_q, b_info); 704 break; 705 case CCP_CMD_HASH_CIPHER: 706 result = ccp_crypto_auth(op[i], cmd_q, b_info); 707 if (result) 708 break; 709 result = ccp_crypto_cipher(op[i], cmd_q, b_info); 710 break; 711 case CCP_CMD_COMBINED: 712 result = ccp_crypto_aead(op[i], cmd_q, b_info); 713 break; 714 default: 715 CCP_LOG_ERR("Unsupported cmd_id"); 716 result = -1; 717 } 718 if (unlikely(result < 0)) { 719 rte_atomic64_add(&b_info->cmd_q->free_slots, 720 (slots_req - b_info->desccnt)); 721 break; 722 } 723 b_info->op[i] = op[i]; 724 } 725 726 b_info->opcnt = i; 727 b_info->tail_offset = (uint32_t)(cmd_q->qbase_phys_addr + cmd_q->qidx * 728 Q_DESC_SIZE); 729 730 rte_wmb(); 731 /* Write the new tail address back to the queue register */ 732 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_TAIL_LO_BASE, 733 b_info->tail_offset); 734 /* Turn the queue back on using our cached control register */ 735 CCP_WRITE_REG(cmd_q->reg_base, CMD_Q_CONTROL_BASE, 736 cmd_q->qcontrol | CMD_Q_RUN); 737 738 rte_ring_enqueue(qp->processed_pkts, (void *)b_info); 739 740 return i; 741 } 742 743 static inline void ccp_auth_dq_prepare(struct rte_crypto_op *op) 744 { 745 struct ccp_session *session; 746 uint8_t *digest_data, *addr; 747 struct rte_mbuf *m_last; 748 int offset, digest_offset; 749 uint8_t digest_le[64]; 750 751 session = (struct ccp_session *)get_session_private_data( 752 op->sym->session, 753 ccp_cryptodev_driver_id); 754 755 if (session->cmd_id == CCP_CMD_COMBINED) { 756 digest_data = op->sym->aead.digest.data; 757 digest_offset = op->sym->aead.data.offset + 758 op->sym->aead.data.length; 759 } else { 760 digest_data = op->sym->auth.digest.data; 761 digest_offset = op->sym->auth.data.offset + 762 op->sym->auth.data.length; 763 } 764 m_last = rte_pktmbuf_lastseg(op->sym->m_src); 765 addr = (uint8_t *)((char *)m_last->buf_addr + m_last->data_off + 766 m_last->data_len - session->auth.ctx_len); 767 768 rte_mb(); 769 offset = session->auth.offset; 770 771 if (session->auth.engine == CCP_ENGINE_SHA) 772 if ((session->auth.ut.sha_type != CCP_SHA_TYPE_1) && 773 (session->auth.ut.sha_type != CCP_SHA_TYPE_224) && 774 (session->auth.ut.sha_type != CCP_SHA_TYPE_256)) { 775 /* All other algorithms require byte 776 * swap done by host 777 */ 778 unsigned int i; 779 780 offset = session->auth.ctx_len - 781 session->auth.offset - 1; 782 for (i = 0; i < session->auth.digest_length; i++) 783 digest_le[i] = addr[offset - i]; 784 offset = 0; 785 addr = digest_le; 786 } 787 788 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 789 if (session->auth.op == CCP_AUTH_OP_VERIFY) { 790 if (memcmp(addr + offset, digest_data, 791 session->auth.digest_length) != 0) 792 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 793 794 } else { 795 if (unlikely(digest_data == 0)) 796 digest_data = rte_pktmbuf_mtod_offset( 797 op->sym->m_dst, uint8_t *, 798 digest_offset); 799 rte_memcpy(digest_data, addr + offset, 800 session->auth.digest_length); 801 } 802 /* Trim area used for digest from mbuf. */ 803 rte_pktmbuf_trim(op->sym->m_src, 804 session->auth.ctx_len); 805 } 806 807 static int 808 ccp_prepare_ops(struct rte_crypto_op **op_d, 809 struct ccp_batch_info *b_info, 810 uint16_t nb_ops) 811 { 812 int i, min_ops; 813 struct ccp_session *session; 814 815 min_ops = RTE_MIN(nb_ops, b_info->opcnt); 816 817 for (i = 0; i < min_ops; i++) { 818 op_d[i] = b_info->op[b_info->op_idx++]; 819 session = (struct ccp_session *)get_session_private_data( 820 op_d[i]->sym->session, 821 ccp_cryptodev_driver_id); 822 switch (session->cmd_id) { 823 case CCP_CMD_CIPHER: 824 op_d[i]->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 825 break; 826 case CCP_CMD_AUTH: 827 case CCP_CMD_CIPHER_HASH: 828 case CCP_CMD_HASH_CIPHER: 829 case CCP_CMD_COMBINED: 830 ccp_auth_dq_prepare(op_d[i]); 831 break; 832 default: 833 CCP_LOG_ERR("Unsupported cmd_id"); 834 } 835 } 836 837 b_info->opcnt -= min_ops; 838 return min_ops; 839 } 840 841 int 842 process_ops_to_dequeue(struct ccp_qp *qp, 843 struct rte_crypto_op **op, 844 uint16_t nb_ops) 845 { 846 struct ccp_batch_info *b_info; 847 uint32_t cur_head_offset; 848 849 if (qp->b_info != NULL) { 850 b_info = qp->b_info; 851 if (unlikely(b_info->op_idx > 0)) 852 goto success; 853 } else if (rte_ring_dequeue(qp->processed_pkts, 854 (void **)&b_info)) 855 return 0; 856 cur_head_offset = CCP_READ_REG(b_info->cmd_q->reg_base, 857 CMD_Q_HEAD_LO_BASE); 858 859 if (b_info->head_offset < b_info->tail_offset) { 860 if ((cur_head_offset >= b_info->head_offset) && 861 (cur_head_offset < b_info->tail_offset)) { 862 qp->b_info = b_info; 863 return 0; 864 } 865 } else { 866 if ((cur_head_offset >= b_info->head_offset) || 867 (cur_head_offset < b_info->tail_offset)) { 868 qp->b_info = b_info; 869 return 0; 870 } 871 } 872 873 874 success: 875 nb_ops = ccp_prepare_ops(op, b_info, nb_ops); 876 rte_atomic64_add(&b_info->cmd_q->free_slots, b_info->desccnt); 877 b_info->desccnt = 0; 878 if (b_info->opcnt > 0) { 879 qp->b_info = b_info; 880 } else { 881 rte_mempool_put(qp->batch_mp, (void *)b_info); 882 qp->b_info = NULL; 883 } 884 885 return nb_ops; 886 } 887