1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #ifndef _CPT_UCODE_H_ 6 #define _CPT_UCODE_H_ 7 #include <stdbool.h> 8 9 #include "cpt_common.h" 10 #include "cpt_hw_types.h" 11 #include "cpt_mcode_defines.h" 12 13 /* 14 * This file defines functions that are interfaces to microcode spec. 15 * 16 */ 17 18 static uint8_t zuc_d[32] = { 19 0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E, 20 0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF, 21 0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1, 22 0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC 23 }; 24 25 static __rte_always_inline int 26 cpt_is_algo_supported(struct rte_crypto_sym_xform *xform) 27 { 28 /* 29 * Microcode only supports the following combination. 30 * Encryption followed by authentication 31 * Authentication followed by decryption 32 */ 33 if (xform->next) { 34 if ((xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) && 35 (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 36 (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)) { 37 /* Unsupported as of now by microcode */ 38 CPT_LOG_DP_ERR("Unsupported combination"); 39 return -1; 40 } 41 if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 42 (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) && 43 (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT)) { 44 /* For GMAC auth there is no cipher operation */ 45 if (xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM || 46 xform->next->auth.algo != 47 RTE_CRYPTO_AUTH_AES_GMAC) { 48 /* Unsupported as of now by microcode */ 49 CPT_LOG_DP_ERR("Unsupported combination"); 50 return -1; 51 } 52 } 53 } 54 return 0; 55 } 56 57 static __rte_always_inline void 58 gen_key_snow3g(const uint8_t *ck, uint32_t *keyx) 59 { 60 int i, base; 61 62 for (i = 0; i < 4; i++) { 63 base = 4 * i; 64 keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) | 65 (ck[base + 2] << 8) | (ck[base + 3]); 66 keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]); 67 } 68 } 69 70 static __rte_always_inline void 71 cpt_fc_salt_update(void *ctx, 72 uint8_t *salt) 73 { 74 struct cpt_ctx *cpt_ctx = ctx; 75 memcpy(&cpt_ctx->fctx.enc.encr_iv, salt, 4); 76 } 77 78 static __rte_always_inline int 79 cpt_fc_ciph_validate_key_aes(uint16_t key_len) 80 { 81 switch (key_len) { 82 case CPT_BYTE_16: 83 case CPT_BYTE_24: 84 case CPT_BYTE_32: 85 return 0; 86 default: 87 return -1; 88 } 89 } 90 91 static __rte_always_inline int 92 cpt_fc_ciph_set_type(cipher_type_t type, struct cpt_ctx *ctx, uint16_t key_len) 93 { 94 int fc_type = 0; 95 switch (type) { 96 case PASSTHROUGH: 97 fc_type = FC_GEN; 98 break; 99 case DES3_CBC: 100 case DES3_ECB: 101 fc_type = FC_GEN; 102 break; 103 case AES_CBC: 104 case AES_ECB: 105 case AES_CFB: 106 case AES_CTR: 107 case AES_GCM: 108 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 109 return -1; 110 fc_type = FC_GEN; 111 break; 112 case AES_XTS: 113 key_len = key_len / 2; 114 if (unlikely(key_len == CPT_BYTE_24)) { 115 CPT_LOG_DP_ERR("Invalid AES key len for XTS"); 116 return -1; 117 } 118 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 119 return -1; 120 fc_type = FC_GEN; 121 break; 122 case ZUC_EEA3: 123 case SNOW3G_UEA2: 124 if (unlikely(key_len != 16)) 125 return -1; 126 /* No support for AEAD yet */ 127 if (unlikely(ctx->hash_type)) 128 return -1; 129 fc_type = ZUC_SNOW3G; 130 break; 131 case KASUMI_F8_CBC: 132 case KASUMI_F8_ECB: 133 if (unlikely(key_len != 16)) 134 return -1; 135 /* No support for AEAD yet */ 136 if (unlikely(ctx->hash_type)) 137 return -1; 138 fc_type = KASUMI; 139 break; 140 default: 141 return -1; 142 } 143 144 ctx->fc_type = fc_type; 145 return 0; 146 } 147 148 static __rte_always_inline void 149 cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx) 150 { 151 cpt_ctx->enc_cipher = 0; 152 CPT_P_ENC_CTRL(fctx).enc_cipher = 0; 153 } 154 155 static __rte_always_inline void 156 cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len) 157 { 158 mc_aes_type_t aes_key_type = 0; 159 switch (key_len) { 160 case CPT_BYTE_16: 161 aes_key_type = AES_128_BIT; 162 break; 163 case CPT_BYTE_24: 164 aes_key_type = AES_192_BIT; 165 break; 166 case CPT_BYTE_32: 167 aes_key_type = AES_256_BIT; 168 break; 169 default: 170 /* This should not happen */ 171 CPT_LOG_DP_ERR("Invalid AES key len"); 172 return; 173 } 174 CPT_P_ENC_CTRL(fctx).aes_key = aes_key_type; 175 } 176 177 static __rte_always_inline void 178 cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key, 179 uint16_t key_len) 180 { 181 uint32_t keyx[4]; 182 cpt_ctx->snow3g = 1; 183 gen_key_snow3g(key, keyx); 184 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 185 cpt_ctx->zsk_flags = 0; 186 } 187 188 static __rte_always_inline void 189 cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key, 190 uint16_t key_len) 191 { 192 cpt_ctx->snow3g = 0; 193 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 194 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 195 cpt_ctx->zsk_flags = 0; 196 } 197 198 static __rte_always_inline void 199 cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key, 200 uint16_t key_len) 201 { 202 cpt_ctx->k_ecb = 1; 203 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 204 cpt_ctx->zsk_flags = 0; 205 } 206 207 static __rte_always_inline void 208 cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key, 209 uint16_t key_len) 210 { 211 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 212 cpt_ctx->zsk_flags = 0; 213 } 214 215 static __rte_always_inline int 216 cpt_fc_ciph_set_key(void *ctx, cipher_type_t type, const uint8_t *key, 217 uint16_t key_len, uint8_t *salt) 218 { 219 struct cpt_ctx *cpt_ctx = ctx; 220 mc_fc_context_t *fctx = &cpt_ctx->fctx; 221 uint64_t *ctrl_flags = NULL; 222 int ret; 223 224 ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len); 225 if (unlikely(ret)) 226 return -1; 227 228 if (cpt_ctx->fc_type == FC_GEN) { 229 ctrl_flags = (uint64_t *)&(fctx->enc.enc_ctrl.flags); 230 *ctrl_flags = rte_be_to_cpu_64(*ctrl_flags); 231 /* 232 * We need to always say IV is from DPTR as user can 233 * sometimes iverride IV per operation. 234 */ 235 CPT_P_ENC_CTRL(fctx).iv_source = CPT_FROM_DPTR; 236 } 237 238 switch (type) { 239 case PASSTHROUGH: 240 cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx); 241 goto fc_success; 242 case DES3_CBC: 243 /* CPT performs DES using 3DES with the 8B DES-key 244 * replicated 2 more times to match the 24B 3DES-key. 245 * Eg. If org. key is "0x0a 0x0b", then new key is 246 * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b" 247 */ 248 if (key_len == 8) { 249 /* Skipping the first 8B as it will be copied 250 * in the regular code flow 251 */ 252 memcpy(fctx->enc.encr_key+key_len, key, key_len); 253 memcpy(fctx->enc.encr_key+2*key_len, key, key_len); 254 } 255 break; 256 case DES3_ECB: 257 /* For DES3_ECB IV need to be from CTX. */ 258 CPT_P_ENC_CTRL(fctx).iv_source = CPT_FROM_CTX; 259 break; 260 case AES_CBC: 261 case AES_ECB: 262 case AES_CFB: 263 case AES_CTR: 264 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 265 break; 266 case AES_GCM: 267 /* Even though iv source is from dptr, 268 * aes_gcm salt is taken from ctx 269 */ 270 if (salt) { 271 memcpy(fctx->enc.encr_iv, salt, 4); 272 /* Assuming it was just salt update 273 * and nothing else 274 */ 275 if (!key) 276 goto fc_success; 277 } 278 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 279 break; 280 case AES_XTS: 281 key_len = key_len / 2; 282 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 283 284 /* Copy key2 for XTS into ipad */ 285 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 286 memcpy(fctx->hmac.ipad, &key[key_len], key_len); 287 break; 288 case SNOW3G_UEA2: 289 cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len); 290 goto success; 291 case ZUC_EEA3: 292 cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len); 293 goto success; 294 case KASUMI_F8_ECB: 295 cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len); 296 goto success; 297 case KASUMI_F8_CBC: 298 cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len); 299 goto success; 300 default: 301 break; 302 } 303 304 /* Only for FC_GEN case */ 305 306 /* For GMAC auth, cipher must be NULL */ 307 if (cpt_ctx->hash_type != GMAC_TYPE) 308 CPT_P_ENC_CTRL(fctx).enc_cipher = type; 309 310 memcpy(fctx->enc.encr_key, key, key_len); 311 312 fc_success: 313 if (ctrl_flags != NULL) 314 *ctrl_flags = rte_cpu_to_be_64(*ctrl_flags); 315 316 success: 317 cpt_ctx->enc_cipher = type; 318 319 return 0; 320 } 321 322 static __rte_always_inline uint32_t 323 fill_sg_comp(sg_comp_t *list, 324 uint32_t i, 325 phys_addr_t dma_addr, 326 uint32_t size) 327 { 328 sg_comp_t *to = &list[i>>2]; 329 330 to->u.s.len[i%4] = rte_cpu_to_be_16(size); 331 to->ptr[i%4] = rte_cpu_to_be_64(dma_addr); 332 i++; 333 return i; 334 } 335 336 static __rte_always_inline uint32_t 337 fill_sg_comp_from_buf(sg_comp_t *list, 338 uint32_t i, 339 buf_ptr_t *from) 340 { 341 sg_comp_t *to = &list[i>>2]; 342 343 to->u.s.len[i%4] = rte_cpu_to_be_16(from->size); 344 to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr); 345 i++; 346 return i; 347 } 348 349 static __rte_always_inline uint32_t 350 fill_sg_comp_from_buf_min(sg_comp_t *list, 351 uint32_t i, 352 buf_ptr_t *from, 353 uint32_t *psize) 354 { 355 sg_comp_t *to = &list[i >> 2]; 356 uint32_t size = *psize; 357 uint32_t e_len; 358 359 e_len = (size > from->size) ? from->size : size; 360 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 361 to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr); 362 *psize -= e_len; 363 i++; 364 return i; 365 } 366 367 /* 368 * This fills the MC expected SGIO list 369 * from IOV given by user. 370 */ 371 static __rte_always_inline uint32_t 372 fill_sg_comp_from_iov(sg_comp_t *list, 373 uint32_t i, 374 iov_ptr_t *from, uint32_t from_offset, 375 uint32_t *psize, buf_ptr_t *extra_buf, 376 uint32_t extra_offset) 377 { 378 int32_t j; 379 uint32_t extra_len = extra_buf ? extra_buf->size : 0; 380 uint32_t size = *psize - extra_len; 381 buf_ptr_t *bufs; 382 383 bufs = from->bufs; 384 for (j = 0; (j < from->buf_cnt) && size; j++) { 385 phys_addr_t e_dma_addr; 386 uint32_t e_len; 387 sg_comp_t *to = &list[i >> 2]; 388 389 if (!bufs[j].size) 390 continue; 391 392 if (unlikely(from_offset)) { 393 if (from_offset >= bufs[j].size) { 394 from_offset -= bufs[j].size; 395 continue; 396 } 397 e_dma_addr = bufs[j].dma_addr + from_offset; 398 e_len = (size > (bufs[j].size - from_offset)) ? 399 (bufs[j].size - from_offset) : size; 400 from_offset = 0; 401 } else { 402 e_dma_addr = bufs[j].dma_addr; 403 e_len = (size > bufs[j].size) ? 404 bufs[j].size : size; 405 } 406 407 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 408 to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr); 409 410 if (extra_len && (e_len >= extra_offset)) { 411 /* Break the data at given offset */ 412 uint32_t next_len = e_len - extra_offset; 413 phys_addr_t next_dma = e_dma_addr + extra_offset; 414 415 if (!extra_offset) { 416 i--; 417 } else { 418 e_len = extra_offset; 419 size -= e_len; 420 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 421 } 422 423 /* Insert extra data ptr */ 424 if (extra_len) { 425 i++; 426 to = &list[i >> 2]; 427 to->u.s.len[i % 4] = 428 rte_cpu_to_be_16(extra_buf->size); 429 to->ptr[i % 4] = 430 rte_cpu_to_be_64(extra_buf->dma_addr); 431 432 /* size already decremented by extra len */ 433 } 434 435 /* insert the rest of the data */ 436 if (next_len) { 437 i++; 438 to = &list[i >> 2]; 439 to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len); 440 to->ptr[i % 4] = rte_cpu_to_be_64(next_dma); 441 size -= next_len; 442 } 443 extra_len = 0; 444 445 } else { 446 size -= e_len; 447 } 448 if (extra_offset) 449 extra_offset -= size; 450 i++; 451 } 452 453 *psize = size; 454 return (uint32_t)i; 455 } 456 457 static __rte_always_inline void 458 cpt_digest_gen_prep(uint32_t flags, 459 uint64_t d_lens, 460 digest_params_t *params, 461 void *op, 462 void **prep_req) 463 { 464 struct cpt_request_info *req; 465 uint32_t size, i; 466 uint16_t data_len, mac_len, key_len; 467 auth_type_t hash_type; 468 buf_ptr_t *meta_p; 469 struct cpt_ctx *ctx; 470 sg_comp_t *gather_comp; 471 sg_comp_t *scatter_comp; 472 uint8_t *in_buffer; 473 uint32_t g_size_bytes, s_size_bytes; 474 uint64_t dptr_dma, rptr_dma; 475 vq_cmd_word0_t vq_cmd_w0; 476 vq_cmd_word3_t vq_cmd_w3; 477 void *c_vaddr, *m_vaddr; 478 uint64_t c_dma, m_dma; 479 opcode_info_t opcode; 480 481 ctx = params->ctx_buf.vaddr; 482 meta_p = ¶ms->meta_buf; 483 484 m_vaddr = meta_p->vaddr; 485 m_dma = meta_p->dma_addr; 486 487 /* 488 * Save initial space that followed app data for completion code & 489 * alternate completion code to fall in same cache line as app data 490 */ 491 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 492 m_dma += COMPLETION_CODE_SIZE; 493 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 494 (uint8_t *)m_vaddr; 495 c_vaddr = (uint8_t *)m_vaddr + size; 496 c_dma = m_dma + size; 497 size += sizeof(cpt_res_s_t); 498 499 m_vaddr = (uint8_t *)m_vaddr + size; 500 m_dma += size; 501 502 req = m_vaddr; 503 504 size = sizeof(struct cpt_request_info); 505 m_vaddr = (uint8_t *)m_vaddr + size; 506 m_dma += size; 507 508 hash_type = ctx->hash_type; 509 mac_len = ctx->mac_len; 510 key_len = ctx->auth_key_len; 511 data_len = AUTH_DLEN(d_lens); 512 513 /*GP op header */ 514 vq_cmd_w0.u64 = 0; 515 vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8); 516 if (ctx->hmac) { 517 opcode.s.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE; 518 vq_cmd_w0.s.param1 = key_len; 519 vq_cmd_w0.s.dlen = data_len + ROUNDUP8(key_len); 520 } else { 521 opcode.s.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE; 522 vq_cmd_w0.s.param1 = 0; 523 vq_cmd_w0.s.dlen = data_len; 524 } 525 526 opcode.s.minor = 0; 527 528 /* Null auth only case enters the if */ 529 if (unlikely(!hash_type && !ctx->enc_cipher)) { 530 opcode.s.major = CPT_MAJOR_OP_MISC; 531 /* Minor op is passthrough */ 532 opcode.s.minor = 0x03; 533 /* Send out completion code only */ 534 vq_cmd_w0.s.param2 = 0x1; 535 } 536 537 vq_cmd_w0.s.opcode = opcode.flags; 538 539 /* DPTR has SG list */ 540 in_buffer = m_vaddr; 541 dptr_dma = m_dma; 542 543 ((uint16_t *)in_buffer)[0] = 0; 544 ((uint16_t *)in_buffer)[1] = 0; 545 546 /* TODO Add error check if space will be sufficient */ 547 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 548 549 /* 550 * Input gather list 551 */ 552 553 i = 0; 554 555 if (ctx->hmac) { 556 uint64_t k_dma = params->ctx_buf.dma_addr + 557 offsetof(struct cpt_ctx, auth_key); 558 /* Key */ 559 i = fill_sg_comp(gather_comp, i, k_dma, ROUNDUP8(key_len)); 560 } 561 562 /* input data */ 563 size = data_len; 564 if (size) { 565 i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 566 0, &size, NULL, 0); 567 if (unlikely(size)) { 568 CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short" 569 " by %dB", size); 570 return; 571 } 572 } else { 573 /* 574 * Looks like we need to support zero data 575 * gather ptr in case of hash & hmac 576 */ 577 i++; 578 } 579 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 580 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 581 582 /* 583 * Output Gather list 584 */ 585 586 i = 0; 587 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 588 589 if (flags & VALID_MAC_BUF) { 590 if (unlikely(params->mac_buf.size < mac_len)) { 591 CPT_LOG_DP_ERR("Insufficient MAC size"); 592 return; 593 } 594 595 size = mac_len; 596 i = fill_sg_comp_from_buf_min(scatter_comp, i, 597 ¶ms->mac_buf, &size); 598 } else { 599 size = mac_len; 600 i = fill_sg_comp_from_iov(scatter_comp, i, 601 params->src_iov, data_len, 602 &size, NULL, 0); 603 if (unlikely(size)) { 604 CPT_LOG_DP_ERR("Insufficient dst IOV size, short by" 605 " %dB", size); 606 return; 607 } 608 } 609 610 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 611 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 612 613 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 614 615 /* This is DPTR len incase of SG mode */ 616 vq_cmd_w0.s.dlen = size; 617 618 m_vaddr = (uint8_t *)m_vaddr + size; 619 m_dma += size; 620 621 /* cpt alternate completion address saved earlier */ 622 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 623 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 624 rptr_dma = c_dma - 8; 625 626 req->ist.ei1 = dptr_dma; 627 req->ist.ei2 = rptr_dma; 628 629 /* vq command w3 */ 630 vq_cmd_w3.u64 = 0; 631 632 /* 16 byte aligned cpt res address */ 633 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 634 *req->completion_addr = COMPLETION_CODE_INIT; 635 req->comp_baddr = c_dma; 636 637 /* Fill microcode part of instruction */ 638 req->ist.ei0 = vq_cmd_w0.u64; 639 req->ist.ei3 = vq_cmd_w3.u64; 640 641 req->op = op; 642 643 *prep_req = req; 644 return; 645 } 646 647 static __rte_always_inline void 648 cpt_enc_hmac_prep(uint32_t flags, 649 uint64_t d_offs, 650 uint64_t d_lens, 651 fc_params_t *fc_params, 652 void *op, 653 void **prep_req) 654 { 655 uint32_t iv_offset = 0; 656 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 657 struct cpt_ctx *cpt_ctx; 658 uint32_t cipher_type, hash_type; 659 uint32_t mac_len, size; 660 uint8_t iv_len = 16; 661 struct cpt_request_info *req; 662 buf_ptr_t *meta_p, *aad_buf = NULL; 663 uint32_t encr_offset, auth_offset; 664 uint32_t encr_data_len, auth_data_len, aad_len = 0; 665 uint32_t passthrough_len = 0; 666 void *m_vaddr, *offset_vaddr; 667 uint64_t m_dma, offset_dma, ctx_dma; 668 vq_cmd_word0_t vq_cmd_w0; 669 vq_cmd_word3_t vq_cmd_w3; 670 void *c_vaddr; 671 uint64_t c_dma; 672 opcode_info_t opcode; 673 674 meta_p = &fc_params->meta_buf; 675 m_vaddr = meta_p->vaddr; 676 m_dma = meta_p->dma_addr; 677 678 encr_offset = ENCR_OFFSET(d_offs); 679 auth_offset = AUTH_OFFSET(d_offs); 680 encr_data_len = ENCR_DLEN(d_lens); 681 auth_data_len = AUTH_DLEN(d_lens); 682 if (unlikely(flags & VALID_AAD_BUF)) { 683 /* 684 * We dont support both aad 685 * and auth data separately 686 */ 687 auth_data_len = 0; 688 auth_offset = 0; 689 aad_len = fc_params->aad_buf.size; 690 aad_buf = &fc_params->aad_buf; 691 } 692 cpt_ctx = fc_params->ctx_buf.vaddr; 693 cipher_type = cpt_ctx->enc_cipher; 694 hash_type = cpt_ctx->hash_type; 695 mac_len = cpt_ctx->mac_len; 696 697 /* 698 * Save initial space that followed app data for completion code & 699 * alternate completion code to fall in same cache line as app data 700 */ 701 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 702 m_dma += COMPLETION_CODE_SIZE; 703 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 704 (uint8_t *)m_vaddr; 705 706 c_vaddr = (uint8_t *)m_vaddr + size; 707 c_dma = m_dma + size; 708 size += sizeof(cpt_res_s_t); 709 710 m_vaddr = (uint8_t *)m_vaddr + size; 711 m_dma += size; 712 713 /* start cpt request info struct at 8 byte boundary */ 714 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 715 (uint8_t *)m_vaddr; 716 717 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 718 719 size += sizeof(struct cpt_request_info); 720 m_vaddr = (uint8_t *)m_vaddr + size; 721 m_dma += size; 722 723 if (hash_type == GMAC_TYPE) 724 encr_data_len = 0; 725 726 if (unlikely(!(flags & VALID_IV_BUF))) { 727 iv_len = 0; 728 iv_offset = ENCR_IV_OFFSET(d_offs); 729 } 730 731 if (unlikely(flags & VALID_AAD_BUF)) { 732 /* 733 * When AAD is given, data above encr_offset is pass through 734 * Since AAD is given as separate pointer and not as offset, 735 * this is a special case as we need to fragment input data 736 * into passthrough + encr_data and then insert AAD in between. 737 */ 738 if (hash_type != GMAC_TYPE) { 739 passthrough_len = encr_offset; 740 auth_offset = passthrough_len + iv_len; 741 encr_offset = passthrough_len + aad_len + iv_len; 742 auth_data_len = aad_len + encr_data_len; 743 } else { 744 passthrough_len = 16 + aad_len; 745 auth_offset = passthrough_len + iv_len; 746 auth_data_len = aad_len; 747 } 748 } else { 749 encr_offset += iv_len; 750 auth_offset += iv_len; 751 } 752 753 /* Encryption */ 754 opcode.s.major = CPT_MAJOR_OP_FC; 755 opcode.s.minor = 0; 756 757 auth_dlen = auth_offset + auth_data_len; 758 enc_dlen = encr_data_len + encr_offset; 759 if (unlikely(encr_data_len & 0xf)) { 760 if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB)) 761 enc_dlen = ROUNDUP8(encr_data_len) + encr_offset; 762 else if (likely((cipher_type == AES_CBC) || 763 (cipher_type == AES_ECB))) 764 enc_dlen = ROUNDUP16(encr_data_len) + encr_offset; 765 } 766 767 if (unlikely(hash_type == GMAC_TYPE)) { 768 encr_offset = auth_dlen; 769 enc_dlen = 0; 770 } 771 772 if (unlikely(auth_dlen > enc_dlen)) { 773 inputlen = auth_dlen; 774 outputlen = auth_dlen + mac_len; 775 } else { 776 inputlen = enc_dlen; 777 outputlen = enc_dlen + mac_len; 778 } 779 780 /* GP op header */ 781 vq_cmd_w0.u64 = 0; 782 vq_cmd_w0.s.param1 = encr_data_len; 783 vq_cmd_w0.s.param2 = auth_data_len; 784 /* 785 * In 83XX since we have a limitation of 786 * IV & Offset control word not part of instruction 787 * and need to be part of Data Buffer, we check if 788 * head room is there and then only do the Direct mode processing 789 */ 790 if (likely((flags & SINGLE_BUF_INPLACE) && 791 (flags & SINGLE_BUF_HEADTAILROOM))) { 792 void *dm_vaddr = fc_params->bufs[0].vaddr; 793 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 794 /* 795 * This flag indicates that there is 24 bytes head room and 796 * 8 bytes tail room available, so that we get to do 797 * DIRECT MODE with limitation 798 */ 799 800 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 801 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 802 803 /* DPTR */ 804 req->ist.ei1 = offset_dma; 805 /* RPTR should just exclude offset control word */ 806 req->ist.ei2 = dm_dma_addr - iv_len; 807 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 808 + outputlen - iv_len); 809 810 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 811 812 vq_cmd_w0.s.opcode = opcode.flags; 813 814 if (likely(iv_len)) { 815 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 816 + OFF_CTRL_LEN); 817 uint64_t *src = fc_params->iv_buf; 818 dest[0] = src[0]; 819 dest[1] = src[1]; 820 } 821 822 *(uint64_t *)offset_vaddr = 823 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 824 ((uint64_t)iv_offset << 8) | 825 ((uint64_t)auth_offset)); 826 827 } else { 828 uint32_t i, g_size_bytes, s_size_bytes; 829 uint64_t dptr_dma, rptr_dma; 830 sg_comp_t *gather_comp; 831 sg_comp_t *scatter_comp; 832 uint8_t *in_buffer; 833 834 /* This falls under strict SG mode */ 835 offset_vaddr = m_vaddr; 836 offset_dma = m_dma; 837 size = OFF_CTRL_LEN + iv_len; 838 839 m_vaddr = (uint8_t *)m_vaddr + size; 840 m_dma += size; 841 842 opcode.s.major |= CPT_DMA_MODE; 843 844 vq_cmd_w0.s.opcode = opcode.flags; 845 846 if (likely(iv_len)) { 847 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 848 + OFF_CTRL_LEN); 849 uint64_t *src = fc_params->iv_buf; 850 dest[0] = src[0]; 851 dest[1] = src[1]; 852 } 853 854 *(uint64_t *)offset_vaddr = 855 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 856 ((uint64_t)iv_offset << 8) | 857 ((uint64_t)auth_offset)); 858 859 /* DPTR has SG list */ 860 in_buffer = m_vaddr; 861 dptr_dma = m_dma; 862 863 ((uint16_t *)in_buffer)[0] = 0; 864 ((uint16_t *)in_buffer)[1] = 0; 865 866 /* TODO Add error check if space will be sufficient */ 867 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 868 869 /* 870 * Input Gather List 871 */ 872 873 i = 0; 874 875 /* Offset control word that includes iv */ 876 i = fill_sg_comp(gather_comp, i, offset_dma, 877 OFF_CTRL_LEN + iv_len); 878 879 /* Add input data */ 880 size = inputlen - iv_len; 881 if (likely(size)) { 882 uint32_t aad_offset = aad_len ? passthrough_len : 0; 883 884 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 885 i = fill_sg_comp_from_buf_min(gather_comp, i, 886 fc_params->bufs, 887 &size); 888 } else { 889 i = fill_sg_comp_from_iov(gather_comp, i, 890 fc_params->src_iov, 891 0, &size, 892 aad_buf, aad_offset); 893 } 894 895 if (unlikely(size)) { 896 CPT_LOG_DP_ERR("Insufficient buffer space," 897 " size %d needed", size); 898 return; 899 } 900 } 901 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 902 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 903 904 /* 905 * Output Scatter list 906 */ 907 i = 0; 908 scatter_comp = 909 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 910 911 /* Add IV */ 912 if (likely(iv_len)) { 913 i = fill_sg_comp(scatter_comp, i, 914 offset_dma + OFF_CTRL_LEN, 915 iv_len); 916 } 917 918 /* output data or output data + digest*/ 919 if (unlikely(flags & VALID_MAC_BUF)) { 920 size = outputlen - iv_len - mac_len; 921 if (size) { 922 uint32_t aad_offset = 923 aad_len ? passthrough_len : 0; 924 925 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 926 i = fill_sg_comp_from_buf_min( 927 scatter_comp, 928 i, 929 fc_params->bufs, 930 &size); 931 } else { 932 i = fill_sg_comp_from_iov(scatter_comp, 933 i, 934 fc_params->dst_iov, 935 0, 936 &size, 937 aad_buf, 938 aad_offset); 939 } 940 if (unlikely(size)) { 941 CPT_LOG_DP_ERR("Insufficient buffer" 942 " space, size %d needed", 943 size); 944 return; 945 } 946 } 947 /* mac_data */ 948 if (mac_len) { 949 i = fill_sg_comp_from_buf(scatter_comp, i, 950 &fc_params->mac_buf); 951 } 952 } else { 953 /* Output including mac */ 954 size = outputlen - iv_len; 955 if (likely(size)) { 956 uint32_t aad_offset = 957 aad_len ? passthrough_len : 0; 958 959 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 960 i = fill_sg_comp_from_buf_min( 961 scatter_comp, 962 i, 963 fc_params->bufs, 964 &size); 965 } else { 966 i = fill_sg_comp_from_iov(scatter_comp, 967 i, 968 fc_params->dst_iov, 969 0, 970 &size, 971 aad_buf, 972 aad_offset); 973 } 974 if (unlikely(size)) { 975 CPT_LOG_DP_ERR("Insufficient buffer" 976 " space, size %d needed", 977 size); 978 return; 979 } 980 } 981 } 982 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 983 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 984 985 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 986 987 /* This is DPTR len incase of SG mode */ 988 vq_cmd_w0.s.dlen = size; 989 990 m_vaddr = (uint8_t *)m_vaddr + size; 991 m_dma += size; 992 993 /* cpt alternate completion address saved earlier */ 994 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 995 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 996 rptr_dma = c_dma - 8; 997 998 req->ist.ei1 = dptr_dma; 999 req->ist.ei2 = rptr_dma; 1000 } 1001 1002 ctx_dma = fc_params->ctx_buf.dma_addr + 1003 offsetof(struct cpt_ctx, fctx); 1004 /* vq command w3 */ 1005 vq_cmd_w3.u64 = 0; 1006 vq_cmd_w3.s.grp = 0; 1007 vq_cmd_w3.s.cptr = ctx_dma; 1008 1009 /* 16 byte aligned cpt res address */ 1010 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1011 *req->completion_addr = COMPLETION_CODE_INIT; 1012 req->comp_baddr = c_dma; 1013 1014 /* Fill microcode part of instruction */ 1015 req->ist.ei0 = vq_cmd_w0.u64; 1016 req->ist.ei3 = vq_cmd_w3.u64; 1017 1018 req->op = op; 1019 1020 *prep_req = req; 1021 return; 1022 } 1023 1024 static __rte_always_inline void 1025 cpt_dec_hmac_prep(uint32_t flags, 1026 uint64_t d_offs, 1027 uint64_t d_lens, 1028 fc_params_t *fc_params, 1029 void *op, 1030 void **prep_req) 1031 { 1032 uint32_t iv_offset = 0, size; 1033 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 1034 struct cpt_ctx *cpt_ctx; 1035 int32_t hash_type, mac_len; 1036 uint8_t iv_len = 16; 1037 struct cpt_request_info *req; 1038 buf_ptr_t *meta_p, *aad_buf = NULL; 1039 uint32_t encr_offset, auth_offset; 1040 uint32_t encr_data_len, auth_data_len, aad_len = 0; 1041 uint32_t passthrough_len = 0; 1042 void *m_vaddr, *offset_vaddr; 1043 uint64_t m_dma, offset_dma, ctx_dma; 1044 opcode_info_t opcode; 1045 vq_cmd_word0_t vq_cmd_w0; 1046 vq_cmd_word3_t vq_cmd_w3; 1047 void *c_vaddr; 1048 uint64_t c_dma; 1049 1050 meta_p = &fc_params->meta_buf; 1051 m_vaddr = meta_p->vaddr; 1052 m_dma = meta_p->dma_addr; 1053 1054 encr_offset = ENCR_OFFSET(d_offs); 1055 auth_offset = AUTH_OFFSET(d_offs); 1056 encr_data_len = ENCR_DLEN(d_lens); 1057 auth_data_len = AUTH_DLEN(d_lens); 1058 1059 if (unlikely(flags & VALID_AAD_BUF)) { 1060 /* 1061 * We dont support both aad 1062 * and auth data separately 1063 */ 1064 auth_data_len = 0; 1065 auth_offset = 0; 1066 aad_len = fc_params->aad_buf.size; 1067 aad_buf = &fc_params->aad_buf; 1068 } 1069 1070 cpt_ctx = fc_params->ctx_buf.vaddr; 1071 hash_type = cpt_ctx->hash_type; 1072 mac_len = cpt_ctx->mac_len; 1073 1074 if (hash_type == GMAC_TYPE) 1075 encr_data_len = 0; 1076 1077 if (unlikely(!(flags & VALID_IV_BUF))) { 1078 iv_len = 0; 1079 iv_offset = ENCR_IV_OFFSET(d_offs); 1080 } 1081 1082 if (unlikely(flags & VALID_AAD_BUF)) { 1083 /* 1084 * When AAD is given, data above encr_offset is pass through 1085 * Since AAD is given as separate pointer and not as offset, 1086 * this is a special case as we need to fragment input data 1087 * into passthrough + encr_data and then insert AAD in between. 1088 */ 1089 if (hash_type != GMAC_TYPE) { 1090 passthrough_len = encr_offset; 1091 auth_offset = passthrough_len + iv_len; 1092 encr_offset = passthrough_len + aad_len + iv_len; 1093 auth_data_len = aad_len + encr_data_len; 1094 } else { 1095 passthrough_len = 16 + aad_len; 1096 auth_offset = passthrough_len + iv_len; 1097 auth_data_len = aad_len; 1098 } 1099 } else { 1100 encr_offset += iv_len; 1101 auth_offset += iv_len; 1102 } 1103 1104 /* 1105 * Save initial space that followed app data for completion code & 1106 * alternate completion code to fall in same cache line as app data 1107 */ 1108 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1109 m_dma += COMPLETION_CODE_SIZE; 1110 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1111 (uint8_t *)m_vaddr; 1112 c_vaddr = (uint8_t *)m_vaddr + size; 1113 c_dma = m_dma + size; 1114 size += sizeof(cpt_res_s_t); 1115 1116 m_vaddr = (uint8_t *)m_vaddr + size; 1117 m_dma += size; 1118 1119 /* start cpt request info structure at 8 byte alignment */ 1120 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 1121 (uint8_t *)m_vaddr; 1122 1123 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 1124 1125 size += sizeof(struct cpt_request_info); 1126 m_vaddr = (uint8_t *)m_vaddr + size; 1127 m_dma += size; 1128 1129 /* Decryption */ 1130 opcode.s.major = CPT_MAJOR_OP_FC; 1131 opcode.s.minor = 1; 1132 1133 enc_dlen = encr_offset + encr_data_len; 1134 auth_dlen = auth_offset + auth_data_len; 1135 1136 if (auth_dlen > enc_dlen) { 1137 inputlen = auth_dlen + mac_len; 1138 outputlen = auth_dlen; 1139 } else { 1140 inputlen = enc_dlen + mac_len; 1141 outputlen = enc_dlen; 1142 } 1143 1144 if (hash_type == GMAC_TYPE) 1145 encr_offset = inputlen; 1146 1147 vq_cmd_w0.u64 = 0; 1148 vq_cmd_w0.s.param1 = encr_data_len; 1149 vq_cmd_w0.s.param2 = auth_data_len; 1150 1151 /* 1152 * In 83XX since we have a limitation of 1153 * IV & Offset control word not part of instruction 1154 * and need to be part of Data Buffer, we check if 1155 * head room is there and then only do the Direct mode processing 1156 */ 1157 if (likely((flags & SINGLE_BUF_INPLACE) && 1158 (flags & SINGLE_BUF_HEADTAILROOM))) { 1159 void *dm_vaddr = fc_params->bufs[0].vaddr; 1160 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 1161 /* 1162 * This flag indicates that there is 24 bytes head room and 1163 * 8 bytes tail room available, so that we get to do 1164 * DIRECT MODE with limitation 1165 */ 1166 1167 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 1168 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1169 req->ist.ei1 = offset_dma; 1170 1171 /* RPTR should just exclude offset control word */ 1172 req->ist.ei2 = dm_dma_addr - iv_len; 1173 1174 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr + 1175 outputlen - iv_len); 1176 /* since this is decryption, 1177 * don't touch the content of 1178 * alternate ccode space as it contains 1179 * hmac. 1180 */ 1181 1182 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1183 1184 vq_cmd_w0.s.opcode = opcode.flags; 1185 1186 if (likely(iv_len)) { 1187 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1188 OFF_CTRL_LEN); 1189 uint64_t *src = fc_params->iv_buf; 1190 dest[0] = src[0]; 1191 dest[1] = src[1]; 1192 } 1193 1194 *(uint64_t *)offset_vaddr = 1195 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1196 ((uint64_t)iv_offset << 8) | 1197 ((uint64_t)auth_offset)); 1198 1199 } else { 1200 uint64_t dptr_dma, rptr_dma; 1201 uint32_t g_size_bytes, s_size_bytes; 1202 sg_comp_t *gather_comp; 1203 sg_comp_t *scatter_comp; 1204 uint8_t *in_buffer; 1205 uint8_t i = 0; 1206 1207 /* This falls under strict SG mode */ 1208 offset_vaddr = m_vaddr; 1209 offset_dma = m_dma; 1210 size = OFF_CTRL_LEN + iv_len; 1211 1212 m_vaddr = (uint8_t *)m_vaddr + size; 1213 m_dma += size; 1214 1215 opcode.s.major |= CPT_DMA_MODE; 1216 1217 vq_cmd_w0.s.opcode = opcode.flags; 1218 1219 if (likely(iv_len)) { 1220 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1221 OFF_CTRL_LEN); 1222 uint64_t *src = fc_params->iv_buf; 1223 dest[0] = src[0]; 1224 dest[1] = src[1]; 1225 } 1226 1227 *(uint64_t *)offset_vaddr = 1228 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1229 ((uint64_t)iv_offset << 8) | 1230 ((uint64_t)auth_offset)); 1231 1232 /* DPTR has SG list */ 1233 in_buffer = m_vaddr; 1234 dptr_dma = m_dma; 1235 1236 ((uint16_t *)in_buffer)[0] = 0; 1237 ((uint16_t *)in_buffer)[1] = 0; 1238 1239 /* TODO Add error check if space will be sufficient */ 1240 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1241 1242 /* 1243 * Input Gather List 1244 */ 1245 i = 0; 1246 1247 /* Offset control word that includes iv */ 1248 i = fill_sg_comp(gather_comp, i, offset_dma, 1249 OFF_CTRL_LEN + iv_len); 1250 1251 /* Add input data */ 1252 if (flags & VALID_MAC_BUF) { 1253 size = inputlen - iv_len - mac_len; 1254 if (size) { 1255 /* input data only */ 1256 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1257 i = fill_sg_comp_from_buf_min( 1258 gather_comp, i, 1259 fc_params->bufs, 1260 &size); 1261 } else { 1262 uint32_t aad_offset = aad_len ? 1263 passthrough_len : 0; 1264 1265 i = fill_sg_comp_from_iov(gather_comp, 1266 i, 1267 fc_params->src_iov, 1268 0, &size, 1269 aad_buf, 1270 aad_offset); 1271 } 1272 if (unlikely(size)) { 1273 CPT_LOG_DP_ERR("Insufficient buffer" 1274 " space, size %d needed", 1275 size); 1276 return; 1277 } 1278 } 1279 1280 /* mac data */ 1281 if (mac_len) { 1282 i = fill_sg_comp_from_buf(gather_comp, i, 1283 &fc_params->mac_buf); 1284 } 1285 } else { 1286 /* input data + mac */ 1287 size = inputlen - iv_len; 1288 if (size) { 1289 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1290 i = fill_sg_comp_from_buf_min( 1291 gather_comp, i, 1292 fc_params->bufs, 1293 &size); 1294 } else { 1295 uint32_t aad_offset = aad_len ? 1296 passthrough_len : 0; 1297 1298 if (unlikely(!fc_params->src_iov)) { 1299 CPT_LOG_DP_ERR("Bad input args"); 1300 return; 1301 } 1302 1303 i = fill_sg_comp_from_iov( 1304 gather_comp, i, 1305 fc_params->src_iov, 1306 0, &size, 1307 aad_buf, 1308 aad_offset); 1309 } 1310 1311 if (unlikely(size)) { 1312 CPT_LOG_DP_ERR("Insufficient buffer" 1313 " space, size %d needed", 1314 size); 1315 return; 1316 } 1317 } 1318 } 1319 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1320 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1321 1322 /* 1323 * Output Scatter List 1324 */ 1325 1326 i = 0; 1327 scatter_comp = 1328 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1329 1330 /* Add iv */ 1331 if (iv_len) { 1332 i = fill_sg_comp(scatter_comp, i, 1333 offset_dma + OFF_CTRL_LEN, 1334 iv_len); 1335 } 1336 1337 /* Add output data */ 1338 size = outputlen - iv_len; 1339 if (size) { 1340 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1341 /* handle single buffer here */ 1342 i = fill_sg_comp_from_buf_min(scatter_comp, i, 1343 fc_params->bufs, 1344 &size); 1345 } else { 1346 uint32_t aad_offset = aad_len ? 1347 passthrough_len : 0; 1348 1349 if (unlikely(!fc_params->dst_iov)) { 1350 CPT_LOG_DP_ERR("Bad input args"); 1351 return; 1352 } 1353 1354 i = fill_sg_comp_from_iov(scatter_comp, i, 1355 fc_params->dst_iov, 0, 1356 &size, aad_buf, 1357 aad_offset); 1358 } 1359 1360 if (unlikely(size)) { 1361 CPT_LOG_DP_ERR("Insufficient buffer space," 1362 " size %d needed", size); 1363 return; 1364 } 1365 } 1366 1367 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1368 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1369 1370 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1371 1372 /* This is DPTR len incase of SG mode */ 1373 vq_cmd_w0.s.dlen = size; 1374 1375 m_vaddr = (uint8_t *)m_vaddr + size; 1376 m_dma += size; 1377 1378 /* cpt alternate completion address saved earlier */ 1379 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1380 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1381 rptr_dma = c_dma - 8; 1382 size += COMPLETION_CODE_SIZE; 1383 1384 req->ist.ei1 = dptr_dma; 1385 req->ist.ei2 = rptr_dma; 1386 } 1387 1388 ctx_dma = fc_params->ctx_buf.dma_addr + 1389 offsetof(struct cpt_ctx, fctx); 1390 /* vq command w3 */ 1391 vq_cmd_w3.u64 = 0; 1392 vq_cmd_w3.s.grp = 0; 1393 vq_cmd_w3.s.cptr = ctx_dma; 1394 1395 /* 16 byte aligned cpt res address */ 1396 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1397 *req->completion_addr = COMPLETION_CODE_INIT; 1398 req->comp_baddr = c_dma; 1399 1400 /* Fill microcode part of instruction */ 1401 req->ist.ei0 = vq_cmd_w0.u64; 1402 req->ist.ei3 = vq_cmd_w3.u64; 1403 1404 req->op = op; 1405 1406 *prep_req = req; 1407 return; 1408 } 1409 1410 static __rte_always_inline void 1411 cpt_zuc_snow3g_enc_prep(uint32_t req_flags, 1412 uint64_t d_offs, 1413 uint64_t d_lens, 1414 fc_params_t *params, 1415 void *op, 1416 void **prep_req) 1417 { 1418 uint32_t size; 1419 int32_t inputlen, outputlen; 1420 struct cpt_ctx *cpt_ctx; 1421 uint32_t mac_len = 0; 1422 uint8_t snow3g, j; 1423 struct cpt_request_info *req; 1424 buf_ptr_t *buf_p; 1425 uint32_t encr_offset = 0, auth_offset = 0; 1426 uint32_t encr_data_len = 0, auth_data_len = 0; 1427 int flags, iv_len = 16; 1428 void *m_vaddr, *c_vaddr; 1429 uint64_t m_dma, c_dma, offset_ctrl; 1430 uint64_t *offset_vaddr, offset_dma; 1431 uint32_t *iv_s, iv[4]; 1432 vq_cmd_word0_t vq_cmd_w0; 1433 vq_cmd_word3_t vq_cmd_w3; 1434 opcode_info_t opcode; 1435 1436 buf_p = ¶ms->meta_buf; 1437 m_vaddr = buf_p->vaddr; 1438 m_dma = buf_p->dma_addr; 1439 1440 cpt_ctx = params->ctx_buf.vaddr; 1441 flags = cpt_ctx->zsk_flags; 1442 mac_len = cpt_ctx->mac_len; 1443 snow3g = cpt_ctx->snow3g; 1444 1445 /* 1446 * Save initial space that followed app data for completion code & 1447 * alternate completion code to fall in same cache line as app data 1448 */ 1449 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1450 m_dma += COMPLETION_CODE_SIZE; 1451 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1452 (uint8_t *)m_vaddr; 1453 1454 c_vaddr = (uint8_t *)m_vaddr + size; 1455 c_dma = m_dma + size; 1456 size += sizeof(cpt_res_s_t); 1457 1458 m_vaddr = (uint8_t *)m_vaddr + size; 1459 m_dma += size; 1460 1461 /* Reserve memory for cpt request info */ 1462 req = m_vaddr; 1463 1464 size = sizeof(struct cpt_request_info); 1465 m_vaddr = (uint8_t *)m_vaddr + size; 1466 m_dma += size; 1467 1468 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1469 1470 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1471 1472 opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1473 (0 << 3) | (flags & 0x7)); 1474 1475 if (flags == 0x1) { 1476 /* 1477 * Microcode expects offsets in bytes 1478 * TODO: Rounding off 1479 */ 1480 auth_data_len = AUTH_DLEN(d_lens); 1481 1482 /* EIA3 or UIA2 */ 1483 auth_offset = AUTH_OFFSET(d_offs); 1484 auth_offset = auth_offset / 8; 1485 1486 /* consider iv len */ 1487 auth_offset += iv_len; 1488 1489 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 1490 outputlen = mac_len; 1491 1492 offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset); 1493 1494 } else { 1495 /* EEA3 or UEA2 */ 1496 /* 1497 * Microcode expects offsets in bytes 1498 * TODO: Rounding off 1499 */ 1500 encr_data_len = ENCR_DLEN(d_lens); 1501 1502 encr_offset = ENCR_OFFSET(d_offs); 1503 encr_offset = encr_offset / 8; 1504 /* consider iv len */ 1505 encr_offset += iv_len; 1506 1507 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 1508 outputlen = inputlen; 1509 1510 /* iv offset is 0 */ 1511 offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1512 } 1513 1514 /* IV */ 1515 iv_s = (flags == 0x1) ? params->auth_iv_buf : 1516 params->iv_buf; 1517 1518 if (snow3g) { 1519 /* 1520 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1521 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1522 */ 1523 1524 for (j = 0; j < 4; j++) 1525 iv[j] = iv_s[3 - j]; 1526 } else { 1527 /* ZUC doesn't need a swap */ 1528 for (j = 0; j < 4; j++) 1529 iv[j] = iv_s[j]; 1530 } 1531 1532 /* 1533 * GP op header, lengths are expected in bits. 1534 */ 1535 vq_cmd_w0.u64 = 0; 1536 vq_cmd_w0.s.param1 = encr_data_len; 1537 vq_cmd_w0.s.param2 = auth_data_len; 1538 1539 /* 1540 * In 83XX since we have a limitation of 1541 * IV & Offset control word not part of instruction 1542 * and need to be part of Data Buffer, we check if 1543 * head room is there and then only do the Direct mode processing 1544 */ 1545 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1546 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1547 void *dm_vaddr = params->bufs[0].vaddr; 1548 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1549 /* 1550 * This flag indicates that there is 24 bytes head room and 1551 * 8 bytes tail room available, so that we get to do 1552 * DIRECT MODE with limitation 1553 */ 1554 1555 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1556 OFF_CTRL_LEN - iv_len); 1557 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1558 1559 /* DPTR */ 1560 req->ist.ei1 = offset_dma; 1561 /* RPTR should just exclude offset control word */ 1562 req->ist.ei2 = dm_dma_addr - iv_len; 1563 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1564 + outputlen - iv_len); 1565 1566 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1567 1568 vq_cmd_w0.s.opcode = opcode.flags; 1569 1570 if (likely(iv_len)) { 1571 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1572 + OFF_CTRL_LEN); 1573 memcpy(iv_d, iv, 16); 1574 } 1575 1576 *offset_vaddr = offset_ctrl; 1577 } else { 1578 uint32_t i, g_size_bytes, s_size_bytes; 1579 uint64_t dptr_dma, rptr_dma; 1580 sg_comp_t *gather_comp; 1581 sg_comp_t *scatter_comp; 1582 uint8_t *in_buffer; 1583 uint32_t *iv_d; 1584 1585 /* save space for iv */ 1586 offset_vaddr = m_vaddr; 1587 offset_dma = m_dma; 1588 1589 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1590 m_dma += OFF_CTRL_LEN + iv_len; 1591 1592 opcode.s.major |= CPT_DMA_MODE; 1593 1594 vq_cmd_w0.s.opcode = opcode.flags; 1595 1596 /* DPTR has SG list */ 1597 in_buffer = m_vaddr; 1598 dptr_dma = m_dma; 1599 1600 ((uint16_t *)in_buffer)[0] = 0; 1601 ((uint16_t *)in_buffer)[1] = 0; 1602 1603 /* TODO Add error check if space will be sufficient */ 1604 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1605 1606 /* 1607 * Input Gather List 1608 */ 1609 i = 0; 1610 1611 /* Offset control word followed by iv */ 1612 1613 i = fill_sg_comp(gather_comp, i, offset_dma, 1614 OFF_CTRL_LEN + iv_len); 1615 1616 /* iv offset is 0 */ 1617 *offset_vaddr = offset_ctrl; 1618 1619 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1620 memcpy(iv_d, iv, 16); 1621 1622 /* input data */ 1623 size = inputlen - iv_len; 1624 if (size) { 1625 i = fill_sg_comp_from_iov(gather_comp, i, 1626 params->src_iov, 1627 0, &size, NULL, 0); 1628 if (unlikely(size)) { 1629 CPT_LOG_DP_ERR("Insufficient buffer space," 1630 " size %d needed", size); 1631 return; 1632 } 1633 } 1634 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1635 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1636 1637 /* 1638 * Output Scatter List 1639 */ 1640 1641 i = 0; 1642 scatter_comp = 1643 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1644 1645 if (flags == 0x1) { 1646 /* IV in SLIST only for EEA3 & UEA2 */ 1647 iv_len = 0; 1648 } 1649 1650 if (iv_len) { 1651 i = fill_sg_comp(scatter_comp, i, 1652 offset_dma + OFF_CTRL_LEN, iv_len); 1653 } 1654 1655 /* Add output data */ 1656 if (req_flags & VALID_MAC_BUF) { 1657 size = outputlen - iv_len - mac_len; 1658 if (size) { 1659 i = fill_sg_comp_from_iov(scatter_comp, i, 1660 params->dst_iov, 0, 1661 &size, NULL, 0); 1662 1663 if (unlikely(size)) { 1664 CPT_LOG_DP_ERR("Insufficient buffer space," 1665 " size %d needed", size); 1666 return; 1667 } 1668 } 1669 1670 /* mac data */ 1671 if (mac_len) { 1672 i = fill_sg_comp_from_buf(scatter_comp, i, 1673 ¶ms->mac_buf); 1674 } 1675 } else { 1676 /* Output including mac */ 1677 size = outputlen - iv_len; 1678 if (size) { 1679 i = fill_sg_comp_from_iov(scatter_comp, i, 1680 params->dst_iov, 0, 1681 &size, NULL, 0); 1682 1683 if (unlikely(size)) { 1684 CPT_LOG_DP_ERR("Insufficient buffer space," 1685 " size %d needed", size); 1686 return; 1687 } 1688 } 1689 } 1690 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1691 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1692 1693 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1694 1695 /* This is DPTR len incase of SG mode */ 1696 vq_cmd_w0.s.dlen = size; 1697 1698 m_vaddr = (uint8_t *)m_vaddr + size; 1699 m_dma += size; 1700 1701 /* cpt alternate completion address saved earlier */ 1702 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1703 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1704 rptr_dma = c_dma - 8; 1705 1706 req->ist.ei1 = dptr_dma; 1707 req->ist.ei2 = rptr_dma; 1708 } 1709 1710 /* vq command w3 */ 1711 vq_cmd_w3.u64 = 0; 1712 vq_cmd_w3.s.grp = 0; 1713 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1714 offsetof(struct cpt_ctx, zs_ctx); 1715 1716 /* 16 byte aligned cpt res address */ 1717 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1718 *req->completion_addr = COMPLETION_CODE_INIT; 1719 req->comp_baddr = c_dma; 1720 1721 /* Fill microcode part of instruction */ 1722 req->ist.ei0 = vq_cmd_w0.u64; 1723 req->ist.ei3 = vq_cmd_w3.u64; 1724 1725 req->op = op; 1726 1727 *prep_req = req; 1728 return; 1729 } 1730 1731 static __rte_always_inline void 1732 cpt_zuc_snow3g_dec_prep(uint32_t req_flags, 1733 uint64_t d_offs, 1734 uint64_t d_lens, 1735 fc_params_t *params, 1736 void *op, 1737 void **prep_req) 1738 { 1739 uint32_t size; 1740 int32_t inputlen = 0, outputlen; 1741 struct cpt_ctx *cpt_ctx; 1742 uint8_t snow3g, iv_len = 16; 1743 struct cpt_request_info *req; 1744 buf_ptr_t *buf_p; 1745 uint32_t encr_offset; 1746 uint32_t encr_data_len; 1747 int flags; 1748 void *m_vaddr, *c_vaddr; 1749 uint64_t m_dma, c_dma; 1750 uint64_t *offset_vaddr, offset_dma; 1751 uint32_t *iv_s, iv[4], j; 1752 vq_cmd_word0_t vq_cmd_w0; 1753 vq_cmd_word3_t vq_cmd_w3; 1754 opcode_info_t opcode; 1755 1756 buf_p = ¶ms->meta_buf; 1757 m_vaddr = buf_p->vaddr; 1758 m_dma = buf_p->dma_addr; 1759 1760 /* 1761 * Microcode expects offsets in bytes 1762 * TODO: Rounding off 1763 */ 1764 encr_offset = ENCR_OFFSET(d_offs) / 8; 1765 encr_data_len = ENCR_DLEN(d_lens); 1766 1767 cpt_ctx = params->ctx_buf.vaddr; 1768 flags = cpt_ctx->zsk_flags; 1769 snow3g = cpt_ctx->snow3g; 1770 /* 1771 * Save initial space that followed app data for completion code & 1772 * alternate completion code to fall in same cache line as app data 1773 */ 1774 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1775 m_dma += COMPLETION_CODE_SIZE; 1776 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1777 (uint8_t *)m_vaddr; 1778 1779 c_vaddr = (uint8_t *)m_vaddr + size; 1780 c_dma = m_dma + size; 1781 size += sizeof(cpt_res_s_t); 1782 1783 m_vaddr = (uint8_t *)m_vaddr + size; 1784 m_dma += size; 1785 1786 /* Reserve memory for cpt request info */ 1787 req = m_vaddr; 1788 1789 size = sizeof(struct cpt_request_info); 1790 m_vaddr = (uint8_t *)m_vaddr + size; 1791 m_dma += size; 1792 1793 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1794 1795 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1796 1797 opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1798 (0 << 3) | (flags & 0x7)); 1799 1800 /* consider iv len */ 1801 encr_offset += iv_len; 1802 1803 inputlen = encr_offset + 1804 (RTE_ALIGN(encr_data_len, 8) / 8); 1805 outputlen = inputlen; 1806 1807 /* IV */ 1808 iv_s = params->iv_buf; 1809 if (snow3g) { 1810 /* 1811 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1812 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1813 */ 1814 1815 for (j = 0; j < 4; j++) 1816 iv[j] = iv_s[3 - j]; 1817 } else { 1818 /* ZUC doesn't need a swap */ 1819 for (j = 0; j < 4; j++) 1820 iv[j] = iv_s[j]; 1821 } 1822 1823 /* 1824 * GP op header, lengths are expected in bits. 1825 */ 1826 vq_cmd_w0.u64 = 0; 1827 vq_cmd_w0.s.param1 = encr_data_len; 1828 1829 /* 1830 * In 83XX since we have a limitation of 1831 * IV & Offset control word not part of instruction 1832 * and need to be part of Data Buffer, we check if 1833 * head room is there and then only do the Direct mode processing 1834 */ 1835 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1836 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1837 void *dm_vaddr = params->bufs[0].vaddr; 1838 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1839 /* 1840 * This flag indicates that there is 24 bytes head room and 1841 * 8 bytes tail room available, so that we get to do 1842 * DIRECT MODE with limitation 1843 */ 1844 1845 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1846 OFF_CTRL_LEN - iv_len); 1847 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1848 1849 /* DPTR */ 1850 req->ist.ei1 = offset_dma; 1851 /* RPTR should just exclude offset control word */ 1852 req->ist.ei2 = dm_dma_addr - iv_len; 1853 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1854 + outputlen - iv_len); 1855 1856 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1857 1858 vq_cmd_w0.s.opcode = opcode.flags; 1859 1860 if (likely(iv_len)) { 1861 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1862 + OFF_CTRL_LEN); 1863 memcpy(iv_d, iv, 16); 1864 } 1865 1866 /* iv offset is 0 */ 1867 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1868 } else { 1869 uint32_t i, g_size_bytes, s_size_bytes; 1870 uint64_t dptr_dma, rptr_dma; 1871 sg_comp_t *gather_comp; 1872 sg_comp_t *scatter_comp; 1873 uint8_t *in_buffer; 1874 uint32_t *iv_d; 1875 1876 /* save space for offset and iv... */ 1877 offset_vaddr = m_vaddr; 1878 offset_dma = m_dma; 1879 1880 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1881 m_dma += OFF_CTRL_LEN + iv_len; 1882 1883 opcode.s.major |= CPT_DMA_MODE; 1884 1885 vq_cmd_w0.s.opcode = opcode.flags; 1886 1887 /* DPTR has SG list */ 1888 in_buffer = m_vaddr; 1889 dptr_dma = m_dma; 1890 1891 ((uint16_t *)in_buffer)[0] = 0; 1892 ((uint16_t *)in_buffer)[1] = 0; 1893 1894 /* TODO Add error check if space will be sufficient */ 1895 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1896 1897 /* 1898 * Input Gather List 1899 */ 1900 i = 0; 1901 1902 /* Offset control word */ 1903 1904 /* iv offset is 0 */ 1905 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1906 1907 i = fill_sg_comp(gather_comp, i, offset_dma, 1908 OFF_CTRL_LEN + iv_len); 1909 1910 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1911 memcpy(iv_d, iv, 16); 1912 1913 /* Add input data */ 1914 size = inputlen - iv_len; 1915 if (size) { 1916 i = fill_sg_comp_from_iov(gather_comp, i, 1917 params->src_iov, 1918 0, &size, NULL, 0); 1919 if (unlikely(size)) { 1920 CPT_LOG_DP_ERR("Insufficient buffer space," 1921 " size %d needed", size); 1922 return; 1923 } 1924 } 1925 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1926 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1927 1928 /* 1929 * Output Scatter List 1930 */ 1931 1932 i = 0; 1933 scatter_comp = 1934 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1935 1936 /* IV */ 1937 i = fill_sg_comp(scatter_comp, i, 1938 offset_dma + OFF_CTRL_LEN, 1939 iv_len); 1940 1941 /* Add output data */ 1942 size = outputlen - iv_len; 1943 if (size) { 1944 i = fill_sg_comp_from_iov(scatter_comp, i, 1945 params->dst_iov, 0, 1946 &size, NULL, 0); 1947 1948 if (unlikely(size)) { 1949 CPT_LOG_DP_ERR("Insufficient buffer space," 1950 " size %d needed", size); 1951 return; 1952 } 1953 } 1954 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1955 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1956 1957 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1958 1959 /* This is DPTR len incase of SG mode */ 1960 vq_cmd_w0.s.dlen = size; 1961 1962 m_vaddr = (uint8_t *)m_vaddr + size; 1963 m_dma += size; 1964 1965 /* cpt alternate completion address saved earlier */ 1966 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1967 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1968 rptr_dma = c_dma - 8; 1969 1970 req->ist.ei1 = dptr_dma; 1971 req->ist.ei2 = rptr_dma; 1972 } 1973 1974 /* vq command w3 */ 1975 vq_cmd_w3.u64 = 0; 1976 vq_cmd_w3.s.grp = 0; 1977 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1978 offsetof(struct cpt_ctx, zs_ctx); 1979 1980 /* 16 byte aligned cpt res address */ 1981 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1982 *req->completion_addr = COMPLETION_CODE_INIT; 1983 req->comp_baddr = c_dma; 1984 1985 /* Fill microcode part of instruction */ 1986 req->ist.ei0 = vq_cmd_w0.u64; 1987 req->ist.ei3 = vq_cmd_w3.u64; 1988 1989 req->op = op; 1990 1991 *prep_req = req; 1992 return; 1993 } 1994 1995 static __rte_always_inline void 1996 cpt_kasumi_enc_prep(uint32_t req_flags, 1997 uint64_t d_offs, 1998 uint64_t d_lens, 1999 fc_params_t *params, 2000 void *op, 2001 void **prep_req) 2002 { 2003 uint32_t size; 2004 int32_t inputlen = 0, outputlen = 0; 2005 struct cpt_ctx *cpt_ctx; 2006 uint32_t mac_len = 0; 2007 uint8_t i = 0; 2008 struct cpt_request_info *req; 2009 buf_ptr_t *buf_p; 2010 uint32_t encr_offset, auth_offset; 2011 uint32_t encr_data_len, auth_data_len; 2012 int flags; 2013 uint8_t *iv_s, *iv_d, iv_len = 8; 2014 uint8_t dir = 0; 2015 void *m_vaddr, *c_vaddr; 2016 uint64_t m_dma, c_dma; 2017 uint64_t *offset_vaddr, offset_dma; 2018 vq_cmd_word0_t vq_cmd_w0; 2019 vq_cmd_word3_t vq_cmd_w3; 2020 opcode_info_t opcode; 2021 uint8_t *in_buffer; 2022 uint32_t g_size_bytes, s_size_bytes; 2023 uint64_t dptr_dma, rptr_dma; 2024 sg_comp_t *gather_comp; 2025 sg_comp_t *scatter_comp; 2026 2027 buf_p = ¶ms->meta_buf; 2028 m_vaddr = buf_p->vaddr; 2029 m_dma = buf_p->dma_addr; 2030 2031 encr_offset = ENCR_OFFSET(d_offs) / 8; 2032 auth_offset = AUTH_OFFSET(d_offs) / 8; 2033 encr_data_len = ENCR_DLEN(d_lens); 2034 auth_data_len = AUTH_DLEN(d_lens); 2035 2036 cpt_ctx = params->ctx_buf.vaddr; 2037 flags = cpt_ctx->zsk_flags; 2038 mac_len = cpt_ctx->mac_len; 2039 2040 if (flags == 0x0) 2041 iv_s = params->iv_buf; 2042 else 2043 iv_s = params->auth_iv_buf; 2044 2045 dir = iv_s[8] & 0x1; 2046 2047 /* 2048 * Save initial space that followed app data for completion code & 2049 * alternate completion code to fall in same cache line as app data 2050 */ 2051 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2052 m_dma += COMPLETION_CODE_SIZE; 2053 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2054 (uint8_t *)m_vaddr; 2055 2056 c_vaddr = (uint8_t *)m_vaddr + size; 2057 c_dma = m_dma + size; 2058 size += sizeof(cpt_res_s_t); 2059 2060 m_vaddr = (uint8_t *)m_vaddr + size; 2061 m_dma += size; 2062 2063 /* Reserve memory for cpt request info */ 2064 req = m_vaddr; 2065 2066 size = sizeof(struct cpt_request_info); 2067 m_vaddr = (uint8_t *)m_vaddr + size; 2068 m_dma += size; 2069 2070 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2071 2072 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2073 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2074 (dir << 4) | (0 << 3) | (flags & 0x7)); 2075 2076 /* 2077 * GP op header, lengths are expected in bits. 2078 */ 2079 vq_cmd_w0.u64 = 0; 2080 vq_cmd_w0.s.param1 = encr_data_len; 2081 vq_cmd_w0.s.param2 = auth_data_len; 2082 vq_cmd_w0.s.opcode = opcode.flags; 2083 2084 /* consider iv len */ 2085 if (flags == 0x0) { 2086 encr_offset += iv_len; 2087 auth_offset += iv_len; 2088 } 2089 2090 /* save space for offset ctrl and iv */ 2091 offset_vaddr = m_vaddr; 2092 offset_dma = m_dma; 2093 2094 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2095 m_dma += OFF_CTRL_LEN + iv_len; 2096 2097 /* DPTR has SG list */ 2098 in_buffer = m_vaddr; 2099 dptr_dma = m_dma; 2100 2101 ((uint16_t *)in_buffer)[0] = 0; 2102 ((uint16_t *)in_buffer)[1] = 0; 2103 2104 /* TODO Add error check if space will be sufficient */ 2105 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2106 2107 /* 2108 * Input Gather List 2109 */ 2110 i = 0; 2111 2112 /* Offset control word followed by iv */ 2113 2114 if (flags == 0x0) { 2115 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 2116 outputlen = inputlen; 2117 /* iv offset is 0 */ 2118 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2119 } else { 2120 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 2121 outputlen = mac_len; 2122 /* iv offset is 0 */ 2123 *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset); 2124 } 2125 2126 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2127 2128 /* IV */ 2129 iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN; 2130 memcpy(iv_d, iv_s, iv_len); 2131 2132 /* input data */ 2133 size = inputlen - iv_len; 2134 if (size) { 2135 i = fill_sg_comp_from_iov(gather_comp, i, 2136 params->src_iov, 0, 2137 &size, NULL, 0); 2138 2139 if (unlikely(size)) { 2140 CPT_LOG_DP_ERR("Insufficient buffer space," 2141 " size %d needed", size); 2142 return; 2143 } 2144 } 2145 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2146 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2147 2148 /* 2149 * Output Scatter List 2150 */ 2151 2152 i = 0; 2153 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2154 2155 if (flags == 0x1) { 2156 /* IV in SLIST only for F8 */ 2157 iv_len = 0; 2158 } 2159 2160 /* IV */ 2161 if (iv_len) { 2162 i = fill_sg_comp(scatter_comp, i, 2163 offset_dma + OFF_CTRL_LEN, 2164 iv_len); 2165 } 2166 2167 /* Add output data */ 2168 if (req_flags & VALID_MAC_BUF) { 2169 size = outputlen - iv_len - mac_len; 2170 if (size) { 2171 i = fill_sg_comp_from_iov(scatter_comp, i, 2172 params->dst_iov, 0, 2173 &size, NULL, 0); 2174 2175 if (unlikely(size)) { 2176 CPT_LOG_DP_ERR("Insufficient buffer space," 2177 " size %d needed", size); 2178 return; 2179 } 2180 } 2181 2182 /* mac data */ 2183 if (mac_len) { 2184 i = fill_sg_comp_from_buf(scatter_comp, i, 2185 ¶ms->mac_buf); 2186 } 2187 } else { 2188 /* Output including mac */ 2189 size = outputlen - iv_len; 2190 if (size) { 2191 i = fill_sg_comp_from_iov(scatter_comp, i, 2192 params->dst_iov, 0, 2193 &size, NULL, 0); 2194 2195 if (unlikely(size)) { 2196 CPT_LOG_DP_ERR("Insufficient buffer space," 2197 " size %d needed", size); 2198 return; 2199 } 2200 } 2201 } 2202 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2203 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2204 2205 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2206 2207 /* This is DPTR len incase of SG mode */ 2208 vq_cmd_w0.s.dlen = size; 2209 2210 m_vaddr = (uint8_t *)m_vaddr + size; 2211 m_dma += size; 2212 2213 /* cpt alternate completion address saved earlier */ 2214 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2215 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2216 rptr_dma = c_dma - 8; 2217 2218 req->ist.ei1 = dptr_dma; 2219 req->ist.ei2 = rptr_dma; 2220 2221 /* vq command w3 */ 2222 vq_cmd_w3.u64 = 0; 2223 vq_cmd_w3.s.grp = 0; 2224 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2225 offsetof(struct cpt_ctx, k_ctx); 2226 2227 /* 16 byte aligned cpt res address */ 2228 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2229 *req->completion_addr = COMPLETION_CODE_INIT; 2230 req->comp_baddr = c_dma; 2231 2232 /* Fill microcode part of instruction */ 2233 req->ist.ei0 = vq_cmd_w0.u64; 2234 req->ist.ei3 = vq_cmd_w3.u64; 2235 2236 req->op = op; 2237 2238 *prep_req = req; 2239 return; 2240 } 2241 2242 static __rte_always_inline void 2243 cpt_kasumi_dec_prep(uint64_t d_offs, 2244 uint64_t d_lens, 2245 fc_params_t *params, 2246 void *op, 2247 void **prep_req) 2248 { 2249 uint32_t size; 2250 int32_t inputlen = 0, outputlen; 2251 struct cpt_ctx *cpt_ctx; 2252 uint8_t i = 0, iv_len = 8; 2253 struct cpt_request_info *req; 2254 buf_ptr_t *buf_p; 2255 uint32_t encr_offset; 2256 uint32_t encr_data_len; 2257 int flags; 2258 uint8_t dir = 0; 2259 void *m_vaddr, *c_vaddr; 2260 uint64_t m_dma, c_dma; 2261 uint64_t *offset_vaddr, offset_dma; 2262 vq_cmd_word0_t vq_cmd_w0; 2263 vq_cmd_word3_t vq_cmd_w3; 2264 opcode_info_t opcode; 2265 uint8_t *in_buffer; 2266 uint32_t g_size_bytes, s_size_bytes; 2267 uint64_t dptr_dma, rptr_dma; 2268 sg_comp_t *gather_comp; 2269 sg_comp_t *scatter_comp; 2270 2271 buf_p = ¶ms->meta_buf; 2272 m_vaddr = buf_p->vaddr; 2273 m_dma = buf_p->dma_addr; 2274 2275 encr_offset = ENCR_OFFSET(d_offs) / 8; 2276 encr_data_len = ENCR_DLEN(d_lens); 2277 2278 cpt_ctx = params->ctx_buf.vaddr; 2279 flags = cpt_ctx->zsk_flags; 2280 /* 2281 * Save initial space that followed app data for completion code & 2282 * alternate completion code to fall in same cache line as app data 2283 */ 2284 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2285 m_dma += COMPLETION_CODE_SIZE; 2286 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2287 (uint8_t *)m_vaddr; 2288 2289 c_vaddr = (uint8_t *)m_vaddr + size; 2290 c_dma = m_dma + size; 2291 size += sizeof(cpt_res_s_t); 2292 2293 m_vaddr = (uint8_t *)m_vaddr + size; 2294 m_dma += size; 2295 2296 /* Reserve memory for cpt request info */ 2297 req = m_vaddr; 2298 2299 size = sizeof(struct cpt_request_info); 2300 m_vaddr = (uint8_t *)m_vaddr + size; 2301 m_dma += size; 2302 2303 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2304 2305 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2306 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2307 (dir << 4) | (0 << 3) | (flags & 0x7)); 2308 2309 /* 2310 * GP op header, lengths are expected in bits. 2311 */ 2312 vq_cmd_w0.u64 = 0; 2313 vq_cmd_w0.s.param1 = encr_data_len; 2314 vq_cmd_w0.s.opcode = opcode.flags; 2315 2316 /* consider iv len */ 2317 encr_offset += iv_len; 2318 2319 inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8); 2320 outputlen = inputlen; 2321 2322 /* save space for offset ctrl & iv */ 2323 offset_vaddr = m_vaddr; 2324 offset_dma = m_dma; 2325 2326 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2327 m_dma += OFF_CTRL_LEN + iv_len; 2328 2329 /* DPTR has SG list */ 2330 in_buffer = m_vaddr; 2331 dptr_dma = m_dma; 2332 2333 ((uint16_t *)in_buffer)[0] = 0; 2334 ((uint16_t *)in_buffer)[1] = 0; 2335 2336 /* TODO Add error check if space will be sufficient */ 2337 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2338 2339 /* 2340 * Input Gather List 2341 */ 2342 i = 0; 2343 2344 /* Offset control word followed by iv */ 2345 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2346 2347 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2348 2349 /* IV */ 2350 memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN, 2351 params->iv_buf, iv_len); 2352 2353 /* Add input data */ 2354 size = inputlen - iv_len; 2355 if (size) { 2356 i = fill_sg_comp_from_iov(gather_comp, i, 2357 params->src_iov, 2358 0, &size, NULL, 0); 2359 if (unlikely(size)) { 2360 CPT_LOG_DP_ERR("Insufficient buffer space," 2361 " size %d needed", size); 2362 return; 2363 } 2364 } 2365 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2366 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2367 2368 /* 2369 * Output Scatter List 2370 */ 2371 2372 i = 0; 2373 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2374 2375 /* IV */ 2376 i = fill_sg_comp(scatter_comp, i, 2377 offset_dma + OFF_CTRL_LEN, 2378 iv_len); 2379 2380 /* Add output data */ 2381 size = outputlen - iv_len; 2382 if (size) { 2383 i = fill_sg_comp_from_iov(scatter_comp, i, 2384 params->dst_iov, 0, 2385 &size, NULL, 0); 2386 if (unlikely(size)) { 2387 CPT_LOG_DP_ERR("Insufficient buffer space," 2388 " size %d needed", size); 2389 return; 2390 } 2391 } 2392 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2393 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2394 2395 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2396 2397 /* This is DPTR len incase of SG mode */ 2398 vq_cmd_w0.s.dlen = size; 2399 2400 m_vaddr = (uint8_t *)m_vaddr + size; 2401 m_dma += size; 2402 2403 /* cpt alternate completion address saved earlier */ 2404 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2405 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2406 rptr_dma = c_dma - 8; 2407 2408 req->ist.ei1 = dptr_dma; 2409 req->ist.ei2 = rptr_dma; 2410 2411 /* vq command w3 */ 2412 vq_cmd_w3.u64 = 0; 2413 vq_cmd_w3.s.grp = 0; 2414 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2415 offsetof(struct cpt_ctx, k_ctx); 2416 2417 /* 16 byte aligned cpt res address */ 2418 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2419 *req->completion_addr = COMPLETION_CODE_INIT; 2420 req->comp_baddr = c_dma; 2421 2422 /* Fill microcode part of instruction */ 2423 req->ist.ei0 = vq_cmd_w0.u64; 2424 req->ist.ei3 = vq_cmd_w3.u64; 2425 2426 req->op = op; 2427 2428 *prep_req = req; 2429 return; 2430 } 2431 2432 static __rte_always_inline void * 2433 cpt_fc_dec_hmac_prep(uint32_t flags, 2434 uint64_t d_offs, 2435 uint64_t d_lens, 2436 fc_params_t *fc_params, 2437 void *op) 2438 { 2439 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2440 uint8_t fc_type; 2441 void *prep_req = NULL; 2442 2443 fc_type = ctx->fc_type; 2444 2445 if (likely(fc_type == FC_GEN)) { 2446 cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2447 &prep_req); 2448 } else if (fc_type == ZUC_SNOW3G) { 2449 cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, op, 2450 &prep_req); 2451 } else if (fc_type == KASUMI) { 2452 cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, &prep_req); 2453 } 2454 2455 /* 2456 * For AUTH_ONLY case, 2457 * MC only supports digest generation and verification 2458 * should be done in software by memcmp() 2459 */ 2460 2461 return prep_req; 2462 } 2463 2464 static __rte_always_inline void *__hot 2465 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, 2466 fc_params_t *fc_params, void *op) 2467 { 2468 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2469 uint8_t fc_type; 2470 void *prep_req = NULL; 2471 2472 fc_type = ctx->fc_type; 2473 2474 /* Common api for rest of the ops */ 2475 if (likely(fc_type == FC_GEN)) { 2476 cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2477 &prep_req); 2478 } else if (fc_type == ZUC_SNOW3G) { 2479 cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, op, 2480 &prep_req); 2481 } else if (fc_type == KASUMI) { 2482 cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, op, 2483 &prep_req); 2484 } else if (fc_type == HASH_HMAC) { 2485 cpt_digest_gen_prep(flags, d_lens, fc_params, op, &prep_req); 2486 } 2487 2488 return prep_req; 2489 } 2490 2491 static __rte_always_inline int 2492 cpt_fc_auth_set_key(void *ctx, auth_type_t type, const uint8_t *key, 2493 uint16_t key_len, uint16_t mac_len) 2494 { 2495 struct cpt_ctx *cpt_ctx = ctx; 2496 mc_fc_context_t *fctx = &cpt_ctx->fctx; 2497 uint64_t *ctrl_flags = NULL; 2498 2499 if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) { 2500 uint32_t keyx[4]; 2501 2502 if (key_len != 16) 2503 return -1; 2504 /* No support for AEAD yet */ 2505 if (cpt_ctx->enc_cipher) 2506 return -1; 2507 /* For ZUC/SNOW3G/Kasumi */ 2508 switch (type) { 2509 case SNOW3G_UIA2: 2510 cpt_ctx->snow3g = 1; 2511 gen_key_snow3g(key, keyx); 2512 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 2513 cpt_ctx->fc_type = ZUC_SNOW3G; 2514 cpt_ctx->zsk_flags = 0x1; 2515 break; 2516 case ZUC_EIA3: 2517 cpt_ctx->snow3g = 0; 2518 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 2519 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 2520 cpt_ctx->fc_type = ZUC_SNOW3G; 2521 cpt_ctx->zsk_flags = 0x1; 2522 break; 2523 case KASUMI_F9_ECB: 2524 /* Kasumi ECB mode */ 2525 cpt_ctx->k_ecb = 1; 2526 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2527 cpt_ctx->fc_type = KASUMI; 2528 cpt_ctx->zsk_flags = 0x1; 2529 break; 2530 case KASUMI_F9_CBC: 2531 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2532 cpt_ctx->fc_type = KASUMI; 2533 cpt_ctx->zsk_flags = 0x1; 2534 break; 2535 default: 2536 return -1; 2537 } 2538 cpt_ctx->mac_len = 4; 2539 cpt_ctx->hash_type = type; 2540 return 0; 2541 } 2542 2543 if (!(cpt_ctx->fc_type == FC_GEN && !type)) { 2544 if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher) 2545 cpt_ctx->fc_type = HASH_HMAC; 2546 } 2547 2548 ctrl_flags = (uint64_t *)&fctx->enc.enc_ctrl.flags; 2549 *ctrl_flags = rte_be_to_cpu_64(*ctrl_flags); 2550 2551 /* For GMAC auth, cipher must be NULL */ 2552 if (type == GMAC_TYPE) 2553 CPT_P_ENC_CTRL(fctx).enc_cipher = 0; 2554 2555 CPT_P_ENC_CTRL(fctx).hash_type = cpt_ctx->hash_type = type; 2556 CPT_P_ENC_CTRL(fctx).mac_len = cpt_ctx->mac_len = mac_len; 2557 2558 if (key_len) { 2559 cpt_ctx->hmac = 1; 2560 memset(cpt_ctx->auth_key, 0, sizeof(cpt_ctx->auth_key)); 2561 memcpy(cpt_ctx->auth_key, key, key_len); 2562 cpt_ctx->auth_key_len = key_len; 2563 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 2564 memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad)); 2565 memcpy(fctx->hmac.opad, key, key_len); 2566 CPT_P_ENC_CTRL(fctx).auth_input_type = 1; 2567 } 2568 *ctrl_flags = rte_cpu_to_be_64(*ctrl_flags); 2569 return 0; 2570 } 2571 2572 static __rte_always_inline int 2573 fill_sess_aead(struct rte_crypto_sym_xform *xform, 2574 struct cpt_sess_misc *sess) 2575 { 2576 struct rte_crypto_aead_xform *aead_form; 2577 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2578 auth_type_t auth_type = 0; /* NULL Auth type */ 2579 uint32_t cipher_key_len = 0; 2580 uint8_t aes_gcm = 0; 2581 aead_form = &xform->aead; 2582 void *ctx = SESS_PRIV(sess); 2583 2584 if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT && 2585 aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) { 2586 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2587 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2588 } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT && 2589 aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) { 2590 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2591 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2592 } else { 2593 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2594 return -1; 2595 } 2596 switch (aead_form->algo) { 2597 case RTE_CRYPTO_AEAD_AES_GCM: 2598 enc_type = AES_GCM; 2599 cipher_key_len = 16; 2600 aes_gcm = 1; 2601 break; 2602 case RTE_CRYPTO_AEAD_AES_CCM: 2603 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2604 aead_form->algo); 2605 return -1; 2606 default: 2607 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2608 aead_form->algo); 2609 return -1; 2610 } 2611 if (aead_form->key.length < cipher_key_len) { 2612 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2613 (unsigned int long)aead_form->key.length); 2614 return -1; 2615 } 2616 sess->zsk_flag = 0; 2617 sess->aes_gcm = aes_gcm; 2618 sess->mac_len = aead_form->digest_length; 2619 sess->iv_offset = aead_form->iv.offset; 2620 sess->iv_length = aead_form->iv.length; 2621 sess->aad_length = aead_form->aad_length; 2622 2623 cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data, 2624 aead_form->key.length, NULL); 2625 2626 cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, aead_form->digest_length); 2627 2628 return 0; 2629 } 2630 2631 static __rte_always_inline int 2632 fill_sess_cipher(struct rte_crypto_sym_xform *xform, 2633 struct cpt_sess_misc *sess) 2634 { 2635 struct rte_crypto_cipher_xform *c_form; 2636 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2637 uint32_t cipher_key_len = 0; 2638 uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0; 2639 2640 c_form = &xform->cipher; 2641 2642 if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2643 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2644 else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT) 2645 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2646 else { 2647 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2648 return -1; 2649 } 2650 2651 switch (c_form->algo) { 2652 case RTE_CRYPTO_CIPHER_AES_CBC: 2653 enc_type = AES_CBC; 2654 cipher_key_len = 16; 2655 break; 2656 case RTE_CRYPTO_CIPHER_3DES_CBC: 2657 enc_type = DES3_CBC; 2658 cipher_key_len = 24; 2659 break; 2660 case RTE_CRYPTO_CIPHER_DES_CBC: 2661 /* DES is implemented using 3DES in hardware */ 2662 enc_type = DES3_CBC; 2663 cipher_key_len = 8; 2664 break; 2665 case RTE_CRYPTO_CIPHER_AES_CTR: 2666 enc_type = AES_CTR; 2667 cipher_key_len = 16; 2668 aes_ctr = 1; 2669 break; 2670 case RTE_CRYPTO_CIPHER_NULL: 2671 enc_type = 0; 2672 is_null = 1; 2673 break; 2674 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2675 enc_type = KASUMI_F8_ECB; 2676 cipher_key_len = 16; 2677 zsk_flag = K_F8; 2678 break; 2679 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2680 enc_type = SNOW3G_UEA2; 2681 cipher_key_len = 16; 2682 zsk_flag = ZS_EA; 2683 break; 2684 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 2685 enc_type = ZUC_EEA3; 2686 cipher_key_len = 16; 2687 zsk_flag = ZS_EA; 2688 break; 2689 case RTE_CRYPTO_CIPHER_AES_XTS: 2690 enc_type = AES_XTS; 2691 cipher_key_len = 16; 2692 break; 2693 case RTE_CRYPTO_CIPHER_3DES_ECB: 2694 enc_type = DES3_ECB; 2695 cipher_key_len = 24; 2696 break; 2697 case RTE_CRYPTO_CIPHER_AES_ECB: 2698 enc_type = AES_ECB; 2699 cipher_key_len = 16; 2700 break; 2701 case RTE_CRYPTO_CIPHER_3DES_CTR: 2702 case RTE_CRYPTO_CIPHER_AES_F8: 2703 case RTE_CRYPTO_CIPHER_ARC4: 2704 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2705 c_form->algo); 2706 return -1; 2707 default: 2708 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2709 c_form->algo); 2710 return -1; 2711 } 2712 2713 if (c_form->key.length < cipher_key_len) { 2714 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2715 (unsigned long) c_form->key.length); 2716 return -1; 2717 } 2718 2719 sess->zsk_flag = zsk_flag; 2720 sess->aes_gcm = 0; 2721 sess->aes_ctr = aes_ctr; 2722 sess->iv_offset = c_form->iv.offset; 2723 sess->iv_length = c_form->iv.length; 2724 sess->is_null = is_null; 2725 2726 cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type, c_form->key.data, 2727 c_form->key.length, NULL); 2728 2729 return 0; 2730 } 2731 2732 static __rte_always_inline int 2733 fill_sess_auth(struct rte_crypto_sym_xform *xform, 2734 struct cpt_sess_misc *sess) 2735 { 2736 struct rte_crypto_auth_xform *a_form; 2737 auth_type_t auth_type = 0; /* NULL Auth type */ 2738 uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0; 2739 2740 a_form = &xform->auth; 2741 2742 if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2743 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2744 else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2745 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2746 else { 2747 CPT_LOG_DP_ERR("Unknown auth operation"); 2748 return -1; 2749 } 2750 2751 if (a_form->key.length > 64) { 2752 CPT_LOG_DP_ERR("Auth key length is big"); 2753 return -1; 2754 } 2755 2756 switch (a_form->algo) { 2757 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2758 /* Fall through */ 2759 case RTE_CRYPTO_AUTH_SHA1: 2760 auth_type = SHA1_TYPE; 2761 break; 2762 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2763 case RTE_CRYPTO_AUTH_SHA256: 2764 auth_type = SHA2_SHA256; 2765 break; 2766 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2767 case RTE_CRYPTO_AUTH_SHA512: 2768 auth_type = SHA2_SHA512; 2769 break; 2770 case RTE_CRYPTO_AUTH_AES_GMAC: 2771 auth_type = GMAC_TYPE; 2772 aes_gcm = 1; 2773 break; 2774 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2775 case RTE_CRYPTO_AUTH_SHA224: 2776 auth_type = SHA2_SHA224; 2777 break; 2778 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2779 case RTE_CRYPTO_AUTH_SHA384: 2780 auth_type = SHA2_SHA384; 2781 break; 2782 case RTE_CRYPTO_AUTH_MD5_HMAC: 2783 case RTE_CRYPTO_AUTH_MD5: 2784 auth_type = MD5_TYPE; 2785 break; 2786 case RTE_CRYPTO_AUTH_KASUMI_F9: 2787 auth_type = KASUMI_F9_ECB; 2788 /* 2789 * Indicate that direction needs to be taken out 2790 * from end of src 2791 */ 2792 zsk_flag = K_F9; 2793 break; 2794 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2795 auth_type = SNOW3G_UIA2; 2796 zsk_flag = ZS_IA; 2797 break; 2798 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2799 auth_type = ZUC_EIA3; 2800 zsk_flag = ZS_IA; 2801 break; 2802 case RTE_CRYPTO_AUTH_NULL: 2803 auth_type = 0; 2804 is_null = 1; 2805 break; 2806 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2807 case RTE_CRYPTO_AUTH_AES_CMAC: 2808 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2809 CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u", 2810 a_form->algo); 2811 return -1; 2812 default: 2813 CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified", 2814 a_form->algo); 2815 return -1; 2816 } 2817 2818 sess->zsk_flag = zsk_flag; 2819 sess->aes_gcm = aes_gcm; 2820 sess->mac_len = a_form->digest_length; 2821 sess->is_null = is_null; 2822 if (zsk_flag) { 2823 sess->auth_iv_offset = a_form->iv.offset; 2824 sess->auth_iv_length = a_form->iv.length; 2825 } 2826 cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type, a_form->key.data, 2827 a_form->key.length, a_form->digest_length); 2828 2829 return 0; 2830 } 2831 2832 static __rte_always_inline int 2833 fill_sess_gmac(struct rte_crypto_sym_xform *xform, 2834 struct cpt_sess_misc *sess) 2835 { 2836 struct rte_crypto_auth_xform *a_form; 2837 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2838 auth_type_t auth_type = 0; /* NULL Auth type */ 2839 void *ctx = SESS_PRIV(sess); 2840 2841 a_form = &xform->auth; 2842 2843 if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2844 sess->cpt_op |= CPT_OP_ENCODE; 2845 else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2846 sess->cpt_op |= CPT_OP_DECODE; 2847 else { 2848 CPT_LOG_DP_ERR("Unknown auth operation"); 2849 return -1; 2850 } 2851 2852 switch (a_form->algo) { 2853 case RTE_CRYPTO_AUTH_AES_GMAC: 2854 enc_type = AES_GCM; 2855 auth_type = GMAC_TYPE; 2856 break; 2857 default: 2858 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2859 a_form->algo); 2860 return -1; 2861 } 2862 2863 sess->zsk_flag = 0; 2864 sess->aes_gcm = 0; 2865 sess->is_gmac = 1; 2866 sess->iv_offset = a_form->iv.offset; 2867 sess->iv_length = a_form->iv.length; 2868 sess->mac_len = a_form->digest_length; 2869 2870 cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data, 2871 a_form->key.length, NULL); 2872 cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, a_form->digest_length); 2873 2874 return 0; 2875 } 2876 2877 static __rte_always_inline void * 2878 alloc_op_meta(struct rte_mbuf *m_src, 2879 buf_ptr_t *buf, 2880 int32_t len, 2881 struct rte_mempool *cpt_meta_pool) 2882 { 2883 uint8_t *mdata; 2884 2885 #ifndef CPT_ALWAYS_USE_SEPARATE_BUF 2886 if (likely(m_src && (m_src->nb_segs == 1))) { 2887 int32_t tailroom; 2888 phys_addr_t mphys; 2889 2890 /* Check if tailroom is sufficient to hold meta data */ 2891 tailroom = rte_pktmbuf_tailroom(m_src); 2892 if (likely(tailroom > len + 8)) { 2893 mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len; 2894 mphys = m_src->buf_physaddr + m_src->buf_len; 2895 mdata -= len; 2896 mphys -= len; 2897 buf->vaddr = mdata; 2898 buf->dma_addr = mphys; 2899 buf->size = len; 2900 /* Indicate that this is a mbuf allocated mdata */ 2901 mdata = (uint8_t *)((uint64_t)mdata | 1ull); 2902 return mdata; 2903 } 2904 } 2905 #else 2906 RTE_SET_USED(m_src); 2907 #endif 2908 2909 if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0)) 2910 return NULL; 2911 2912 buf->vaddr = mdata; 2913 buf->dma_addr = rte_mempool_virt2iova(mdata); 2914 buf->size = len; 2915 2916 return mdata; 2917 } 2918 2919 /** 2920 * cpt_free_metabuf - free metabuf to mempool. 2921 * @param instance: pointer to instance. 2922 * @param objp: pointer to the metabuf. 2923 */ 2924 static __rte_always_inline void 2925 free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool) 2926 { 2927 bool nofree = ((uintptr_t)mdata & 1ull); 2928 2929 if (likely(nofree)) 2930 return; 2931 rte_mempool_put(cpt_meta_pool, mdata); 2932 } 2933 2934 static __rte_always_inline uint32_t 2935 prepare_iov_from_pkt(struct rte_mbuf *pkt, 2936 iov_ptr_t *iovec, uint32_t start_offset) 2937 { 2938 uint16_t index = 0; 2939 void *seg_data = NULL; 2940 phys_addr_t seg_phys; 2941 int32_t seg_size = 0; 2942 2943 if (!pkt) { 2944 iovec->buf_cnt = 0; 2945 return 0; 2946 } 2947 2948 if (!start_offset) { 2949 seg_data = rte_pktmbuf_mtod(pkt, void *); 2950 seg_phys = rte_pktmbuf_mtophys(pkt); 2951 seg_size = pkt->data_len; 2952 } else { 2953 while (start_offset >= pkt->data_len) { 2954 start_offset -= pkt->data_len; 2955 pkt = pkt->next; 2956 } 2957 2958 seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset); 2959 seg_phys = rte_pktmbuf_mtophys_offset(pkt, start_offset); 2960 seg_size = pkt->data_len - start_offset; 2961 if (!seg_size) 2962 return 1; 2963 } 2964 2965 /* first seg */ 2966 iovec->bufs[index].vaddr = seg_data; 2967 iovec->bufs[index].dma_addr = seg_phys; 2968 iovec->bufs[index].size = seg_size; 2969 index++; 2970 pkt = pkt->next; 2971 2972 while (unlikely(pkt != NULL)) { 2973 seg_data = rte_pktmbuf_mtod(pkt, void *); 2974 seg_phys = rte_pktmbuf_mtophys(pkt); 2975 seg_size = pkt->data_len; 2976 if (!seg_size) 2977 break; 2978 2979 iovec->bufs[index].vaddr = seg_data; 2980 iovec->bufs[index].dma_addr = seg_phys; 2981 iovec->bufs[index].size = seg_size; 2982 2983 index++; 2984 2985 pkt = pkt->next; 2986 } 2987 2988 iovec->buf_cnt = index; 2989 return 0; 2990 } 2991 2992 static __rte_always_inline uint32_t 2993 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt, 2994 fc_params_t *param, 2995 uint32_t *flags) 2996 { 2997 uint16_t index = 0; 2998 void *seg_data = NULL; 2999 phys_addr_t seg_phys; 3000 uint32_t seg_size = 0; 3001 iov_ptr_t *iovec; 3002 3003 seg_data = rte_pktmbuf_mtod(pkt, void *); 3004 seg_phys = rte_pktmbuf_mtophys(pkt); 3005 seg_size = pkt->data_len; 3006 3007 /* first seg */ 3008 if (likely(!pkt->next)) { 3009 uint32_t headroom, tailroom; 3010 3011 *flags |= SINGLE_BUF_INPLACE; 3012 headroom = rte_pktmbuf_headroom(pkt); 3013 tailroom = rte_pktmbuf_tailroom(pkt); 3014 if (likely((headroom >= 24) && 3015 (tailroom >= 8))) { 3016 /* In 83XX this is prerequivisit for Direct mode */ 3017 *flags |= SINGLE_BUF_HEADTAILROOM; 3018 } 3019 param->bufs[0].vaddr = seg_data; 3020 param->bufs[0].dma_addr = seg_phys; 3021 param->bufs[0].size = seg_size; 3022 return 0; 3023 } 3024 iovec = param->src_iov; 3025 iovec->bufs[index].vaddr = seg_data; 3026 iovec->bufs[index].dma_addr = seg_phys; 3027 iovec->bufs[index].size = seg_size; 3028 index++; 3029 pkt = pkt->next; 3030 3031 while (unlikely(pkt != NULL)) { 3032 seg_data = rte_pktmbuf_mtod(pkt, void *); 3033 seg_phys = rte_pktmbuf_mtophys(pkt); 3034 seg_size = pkt->data_len; 3035 3036 if (!seg_size) 3037 break; 3038 3039 iovec->bufs[index].vaddr = seg_data; 3040 iovec->bufs[index].dma_addr = seg_phys; 3041 iovec->bufs[index].size = seg_size; 3042 3043 index++; 3044 3045 pkt = pkt->next; 3046 } 3047 3048 iovec->buf_cnt = index; 3049 return 0; 3050 } 3051 3052 static __rte_always_inline int 3053 fill_fc_params(struct rte_crypto_op *cop, 3054 struct cpt_sess_misc *sess_misc, 3055 struct cpt_qp_meta_info *m_info, 3056 void **mdata_ptr, 3057 void **prep_req) 3058 { 3059 uint32_t space = 0; 3060 struct rte_crypto_sym_op *sym_op = cop->sym; 3061 void *mdata = NULL; 3062 uintptr_t *op; 3063 uint32_t mc_hash_off; 3064 uint32_t flags = 0; 3065 uint64_t d_offs, d_lens; 3066 struct rte_mbuf *m_src, *m_dst; 3067 uint8_t cpt_op = sess_misc->cpt_op; 3068 #ifdef CPT_ALWAYS_USE_SG_MODE 3069 uint8_t inplace = 0; 3070 #else 3071 uint8_t inplace = 1; 3072 #endif 3073 fc_params_t fc_params; 3074 char src[SRC_IOV_SIZE]; 3075 char dst[SRC_IOV_SIZE]; 3076 uint32_t iv_buf[4]; 3077 int ret; 3078 3079 if (likely(sess_misc->iv_length)) { 3080 flags |= VALID_IV_BUF; 3081 fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, 3082 uint8_t *, sess_misc->iv_offset); 3083 if (sess_misc->aes_ctr && 3084 unlikely(sess_misc->iv_length != 16)) { 3085 memcpy((uint8_t *)iv_buf, 3086 rte_crypto_op_ctod_offset(cop, 3087 uint8_t *, sess_misc->iv_offset), 12); 3088 iv_buf[3] = rte_cpu_to_be_32(0x1); 3089 fc_params.iv_buf = iv_buf; 3090 } 3091 } 3092 3093 if (sess_misc->zsk_flag) { 3094 fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3095 uint8_t *, 3096 sess_misc->auth_iv_offset); 3097 if (sess_misc->zsk_flag != ZS_EA) 3098 inplace = 0; 3099 } 3100 m_src = sym_op->m_src; 3101 m_dst = sym_op->m_dst; 3102 3103 if (sess_misc->aes_gcm) { 3104 uint8_t *salt; 3105 uint8_t *aad_data; 3106 uint16_t aad_len; 3107 3108 d_offs = sym_op->aead.data.offset; 3109 d_lens = sym_op->aead.data.length; 3110 mc_hash_off = sym_op->aead.data.offset + 3111 sym_op->aead.data.length; 3112 3113 aad_data = sym_op->aead.aad.data; 3114 aad_len = sess_misc->aad_length; 3115 if (likely((aad_data + aad_len) == 3116 rte_pktmbuf_mtod_offset(m_src, 3117 uint8_t *, 3118 sym_op->aead.data.offset))) { 3119 d_offs = (d_offs - aad_len) | (d_offs << 16); 3120 d_lens = (d_lens + aad_len) | (d_lens << 32); 3121 } else { 3122 fc_params.aad_buf.vaddr = sym_op->aead.aad.data; 3123 fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr; 3124 fc_params.aad_buf.size = aad_len; 3125 flags |= VALID_AAD_BUF; 3126 inplace = 0; 3127 d_offs = d_offs << 16; 3128 d_lens = d_lens << 32; 3129 } 3130 3131 salt = fc_params.iv_buf; 3132 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3133 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3134 sess_misc->salt = *(uint32_t *)salt; 3135 } 3136 fc_params.iv_buf = salt + 4; 3137 if (likely(sess_misc->mac_len)) { 3138 struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst : 3139 m_src; 3140 3141 if (!m) 3142 m = m_src; 3143 3144 /* hmac immediately following data is best case */ 3145 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3146 mc_hash_off != 3147 (uint8_t *)sym_op->aead.digest.data)) { 3148 flags |= VALID_MAC_BUF; 3149 fc_params.mac_buf.size = sess_misc->mac_len; 3150 fc_params.mac_buf.vaddr = 3151 sym_op->aead.digest.data; 3152 fc_params.mac_buf.dma_addr = 3153 sym_op->aead.digest.phys_addr; 3154 inplace = 0; 3155 } 3156 } 3157 } else { 3158 d_offs = sym_op->cipher.data.offset; 3159 d_lens = sym_op->cipher.data.length; 3160 mc_hash_off = sym_op->cipher.data.offset + 3161 sym_op->cipher.data.length; 3162 d_offs = (d_offs << 16) | sym_op->auth.data.offset; 3163 d_lens = (d_lens << 32) | sym_op->auth.data.length; 3164 3165 if (mc_hash_off < (sym_op->auth.data.offset + 3166 sym_op->auth.data.length)){ 3167 mc_hash_off = (sym_op->auth.data.offset + 3168 sym_op->auth.data.length); 3169 } 3170 /* for gmac, salt should be updated like in gcm */ 3171 if (unlikely(sess_misc->is_gmac)) { 3172 uint8_t *salt; 3173 salt = fc_params.iv_buf; 3174 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3175 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3176 sess_misc->salt = *(uint32_t *)salt; 3177 } 3178 fc_params.iv_buf = salt + 4; 3179 } 3180 if (likely(sess_misc->mac_len)) { 3181 struct rte_mbuf *m; 3182 3183 m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src; 3184 if (!m) 3185 m = m_src; 3186 3187 /* hmac immediately following data is best case */ 3188 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3189 mc_hash_off != 3190 (uint8_t *)sym_op->auth.digest.data)) { 3191 flags |= VALID_MAC_BUF; 3192 fc_params.mac_buf.size = 3193 sess_misc->mac_len; 3194 fc_params.mac_buf.vaddr = 3195 sym_op->auth.digest.data; 3196 fc_params.mac_buf.dma_addr = 3197 sym_op->auth.digest.phys_addr; 3198 inplace = 0; 3199 } 3200 } 3201 } 3202 fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc); 3203 fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr; 3204 3205 if (unlikely(sess_misc->is_null || sess_misc->cpt_op == CPT_OP_DECODE)) 3206 inplace = 0; 3207 3208 if (likely(!m_dst && inplace)) { 3209 /* Case of single buffer without AAD buf or 3210 * separate mac buf in place and 3211 * not air crypto 3212 */ 3213 fc_params.dst_iov = fc_params.src_iov = (void *)src; 3214 3215 if (unlikely(prepare_iov_from_pkt_inplace(m_src, 3216 &fc_params, 3217 &flags))) { 3218 CPT_LOG_DP_ERR("Prepare inplace src iov failed"); 3219 ret = -EINVAL; 3220 goto err_exit; 3221 } 3222 3223 } else { 3224 /* Out of place processing */ 3225 fc_params.src_iov = (void *)src; 3226 fc_params.dst_iov = (void *)dst; 3227 3228 /* Store SG I/O in the api for reuse */ 3229 if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) { 3230 CPT_LOG_DP_ERR("Prepare src iov failed"); 3231 ret = -EINVAL; 3232 goto err_exit; 3233 } 3234 3235 if (unlikely(m_dst != NULL)) { 3236 uint32_t pkt_len; 3237 3238 /* Try to make room as much as src has */ 3239 pkt_len = rte_pktmbuf_pkt_len(m_dst); 3240 3241 if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) { 3242 pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len; 3243 if (!rte_pktmbuf_append(m_dst, pkt_len)) { 3244 CPT_LOG_DP_ERR("Not enough space in " 3245 "m_dst %p, need %u" 3246 " more", 3247 m_dst, pkt_len); 3248 ret = -EINVAL; 3249 goto err_exit; 3250 } 3251 } 3252 3253 if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) { 3254 CPT_LOG_DP_ERR("Prepare dst iov failed for " 3255 "m_dst %p", m_dst); 3256 ret = -EINVAL; 3257 goto err_exit; 3258 } 3259 } else { 3260 fc_params.dst_iov = (void *)src; 3261 } 3262 } 3263 3264 if (likely(flags & SINGLE_BUF_HEADTAILROOM)) 3265 mdata = alloc_op_meta(m_src, &fc_params.meta_buf, 3266 m_info->lb_mlen, m_info->pool); 3267 else 3268 mdata = alloc_op_meta(NULL, &fc_params.meta_buf, 3269 m_info->sg_mlen, m_info->pool); 3270 3271 if (unlikely(mdata == NULL)) { 3272 CPT_LOG_DP_ERR("Error allocating meta buffer for request"); 3273 ret = -ENOMEM; 3274 goto err_exit; 3275 } 3276 3277 op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull); 3278 op[0] = (uintptr_t)mdata; 3279 op[1] = (uintptr_t)cop; 3280 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3281 space += 4 * sizeof(uint64_t); 3282 3283 fc_params.meta_buf.vaddr = (uint8_t *)op + space; 3284 fc_params.meta_buf.dma_addr += space; 3285 fc_params.meta_buf.size -= space; 3286 3287 /* Finally prepare the instruction */ 3288 if (cpt_op & CPT_OP_ENCODE) 3289 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, 3290 &fc_params, op); 3291 else 3292 *prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens, 3293 &fc_params, op); 3294 3295 if (unlikely(*prep_req == NULL)) { 3296 CPT_LOG_DP_ERR("Preparing request failed due to bad input arg"); 3297 ret = -EINVAL; 3298 goto free_mdata_and_exit; 3299 } 3300 3301 *mdata_ptr = mdata; 3302 3303 return 0; 3304 3305 free_mdata_and_exit: 3306 free_op_meta(mdata, m_info->pool); 3307 err_exit: 3308 return ret; 3309 } 3310 3311 static __rte_always_inline void 3312 compl_auth_verify(struct rte_crypto_op *op, 3313 uint8_t *gen_mac, 3314 uint64_t mac_len) 3315 { 3316 uint8_t *mac; 3317 struct rte_crypto_sym_op *sym_op = op->sym; 3318 3319 if (sym_op->auth.digest.data) 3320 mac = sym_op->auth.digest.data; 3321 else 3322 mac = rte_pktmbuf_mtod_offset(sym_op->m_src, 3323 uint8_t *, 3324 sym_op->auth.data.length + 3325 sym_op->auth.data.offset); 3326 if (!mac) { 3327 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3328 return; 3329 } 3330 3331 if (memcmp(mac, gen_mac, mac_len)) 3332 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 3333 else 3334 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3335 } 3336 3337 static __rte_always_inline int 3338 instance_session_cfg(struct rte_crypto_sym_xform *xform, void *sess) 3339 { 3340 struct rte_crypto_sym_xform *chain; 3341 3342 CPT_PMD_INIT_FUNC_TRACE(); 3343 3344 if (cpt_is_algo_supported(xform)) 3345 goto err; 3346 3347 chain = xform; 3348 while (chain) { 3349 switch (chain->type) { 3350 case RTE_CRYPTO_SYM_XFORM_AEAD: 3351 if (fill_sess_aead(chain, sess)) 3352 goto err; 3353 break; 3354 case RTE_CRYPTO_SYM_XFORM_CIPHER: 3355 if (fill_sess_cipher(chain, sess)) 3356 goto err; 3357 break; 3358 case RTE_CRYPTO_SYM_XFORM_AUTH: 3359 if (chain->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 3360 if (fill_sess_gmac(chain, sess)) 3361 goto err; 3362 } else { 3363 if (fill_sess_auth(chain, sess)) 3364 goto err; 3365 } 3366 break; 3367 default: 3368 CPT_LOG_DP_ERR("Invalid crypto xform type"); 3369 break; 3370 } 3371 chain = chain->next; 3372 } 3373 3374 return 0; 3375 3376 err: 3377 return -1; 3378 } 3379 3380 static __rte_always_inline void 3381 find_kasumif9_direction_and_length(uint8_t *src, 3382 uint32_t counter_num_bytes, 3383 uint32_t *addr_length_in_bits, 3384 uint8_t *addr_direction) 3385 { 3386 uint8_t found = 0; 3387 uint32_t pos; 3388 uint8_t last_byte; 3389 while (!found && counter_num_bytes > 0) { 3390 counter_num_bytes--; 3391 if (src[counter_num_bytes] == 0x00) 3392 continue; 3393 pos = rte_bsf32(src[counter_num_bytes]); 3394 if (pos == 7) { 3395 if (likely(counter_num_bytes > 0)) { 3396 last_byte = src[counter_num_bytes - 1]; 3397 *addr_direction = last_byte & 0x1; 3398 *addr_length_in_bits = counter_num_bytes * 8 3399 - 1; 3400 } 3401 } else { 3402 last_byte = src[counter_num_bytes]; 3403 *addr_direction = (last_byte >> (pos + 1)) & 0x1; 3404 *addr_length_in_bits = counter_num_bytes * 8 3405 + (8 - (pos + 2)); 3406 } 3407 found = 1; 3408 } 3409 } 3410 3411 /* 3412 * This handles all auth only except AES_GMAC 3413 */ 3414 static __rte_always_inline int 3415 fill_digest_params(struct rte_crypto_op *cop, 3416 struct cpt_sess_misc *sess, 3417 struct cpt_qp_meta_info *m_info, 3418 void **mdata_ptr, 3419 void **prep_req) 3420 { 3421 uint32_t space = 0; 3422 struct rte_crypto_sym_op *sym_op = cop->sym; 3423 void *mdata; 3424 phys_addr_t mphys; 3425 uint64_t *op; 3426 uint32_t auth_range_off; 3427 uint32_t flags = 0; 3428 uint64_t d_offs = 0, d_lens; 3429 struct rte_mbuf *m_src, *m_dst; 3430 uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK; 3431 uint16_t mac_len = sess->mac_len; 3432 fc_params_t params; 3433 char src[SRC_IOV_SIZE]; 3434 uint8_t iv_buf[16]; 3435 int ret; 3436 3437 memset(¶ms, 0, sizeof(fc_params_t)); 3438 3439 m_src = sym_op->m_src; 3440 3441 /* For just digest lets force mempool alloc */ 3442 mdata = alloc_op_meta(NULL, ¶ms.meta_buf, m_info->sg_mlen, 3443 m_info->pool); 3444 if (mdata == NULL) { 3445 ret = -ENOMEM; 3446 goto err_exit; 3447 } 3448 3449 mphys = params.meta_buf.dma_addr; 3450 3451 op = mdata; 3452 op[0] = (uintptr_t)mdata; 3453 op[1] = (uintptr_t)cop; 3454 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3455 space += 4 * sizeof(uint64_t); 3456 3457 auth_range_off = sym_op->auth.data.offset; 3458 3459 flags = VALID_MAC_BUF; 3460 params.src_iov = (void *)src; 3461 if (unlikely(sess->zsk_flag)) { 3462 /* 3463 * Since for Zuc, Kasumi, Snow3g offsets are in bits 3464 * we will send pass through even for auth only case, 3465 * let MC handle it 3466 */ 3467 d_offs = auth_range_off; 3468 auth_range_off = 0; 3469 params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3470 uint8_t *, sess->auth_iv_offset); 3471 if (sess->zsk_flag == K_F9) { 3472 uint32_t length_in_bits, num_bytes; 3473 uint8_t *src, direction = 0; 3474 3475 memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src, 3476 uint8_t *), 8); 3477 /* 3478 * This is kasumi f9, take direction from 3479 * source buffer 3480 */ 3481 length_in_bits = cop->sym->auth.data.length; 3482 num_bytes = (length_in_bits >> 3); 3483 src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *); 3484 find_kasumif9_direction_and_length(src, 3485 num_bytes, 3486 &length_in_bits, 3487 &direction); 3488 length_in_bits -= 64; 3489 cop->sym->auth.data.offset += 64; 3490 d_offs = cop->sym->auth.data.offset; 3491 auth_range_off = d_offs / 8; 3492 cop->sym->auth.data.length = length_in_bits; 3493 3494 /* Store it at end of auth iv */ 3495 iv_buf[8] = direction; 3496 params.auth_iv_buf = iv_buf; 3497 } 3498 } 3499 3500 d_lens = sym_op->auth.data.length; 3501 3502 params.ctx_buf.vaddr = SESS_PRIV(sess); 3503 params.ctx_buf.dma_addr = sess->ctx_dma_addr; 3504 3505 if (auth_op == CPT_OP_AUTH_GENERATE) { 3506 if (sym_op->auth.digest.data) { 3507 /* 3508 * Digest to be generated 3509 * in separate buffer 3510 */ 3511 params.mac_buf.size = 3512 sess->mac_len; 3513 params.mac_buf.vaddr = 3514 sym_op->auth.digest.data; 3515 params.mac_buf.dma_addr = 3516 sym_op->auth.digest.phys_addr; 3517 } else { 3518 uint32_t off = sym_op->auth.data.offset + 3519 sym_op->auth.data.length; 3520 int32_t dlen, space; 3521 3522 m_dst = sym_op->m_dst ? 3523 sym_op->m_dst : sym_op->m_src; 3524 dlen = rte_pktmbuf_pkt_len(m_dst); 3525 3526 space = off + mac_len - dlen; 3527 if (space > 0) 3528 if (!rte_pktmbuf_append(m_dst, space)) { 3529 CPT_LOG_DP_ERR("Failed to extend " 3530 "mbuf by %uB", space); 3531 ret = -EINVAL; 3532 goto free_mdata_and_exit; 3533 } 3534 3535 params.mac_buf.vaddr = 3536 rte_pktmbuf_mtod_offset(m_dst, void *, off); 3537 params.mac_buf.dma_addr = 3538 rte_pktmbuf_mtophys_offset(m_dst, off); 3539 params.mac_buf.size = mac_len; 3540 } 3541 } else { 3542 /* Need space for storing generated mac */ 3543 params.mac_buf.vaddr = (uint8_t *)mdata + space; 3544 params.mac_buf.dma_addr = mphys + space; 3545 params.mac_buf.size = mac_len; 3546 space += RTE_ALIGN_CEIL(mac_len, 8); 3547 op[2] = (uintptr_t)params.mac_buf.vaddr; 3548 op[3] = mac_len; 3549 } 3550 3551 params.meta_buf.vaddr = (uint8_t *)mdata + space; 3552 params.meta_buf.dma_addr = mphys + space; 3553 params.meta_buf.size -= space; 3554 3555 /* Out of place processing */ 3556 params.src_iov = (void *)src; 3557 3558 /*Store SG I/O in the api for reuse */ 3559 if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) { 3560 CPT_LOG_DP_ERR("Prepare src iov failed"); 3561 ret = -EINVAL; 3562 goto free_mdata_and_exit; 3563 } 3564 3565 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, ¶ms, op); 3566 if (unlikely(*prep_req == NULL)) { 3567 ret = -EINVAL; 3568 goto free_mdata_and_exit; 3569 } 3570 3571 *mdata_ptr = mdata; 3572 3573 return 0; 3574 3575 free_mdata_and_exit: 3576 free_op_meta(mdata, m_info->pool); 3577 err_exit: 3578 return ret; 3579 } 3580 3581 #endif /*_CPT_UCODE_H_ */ 3582