1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #ifndef _CPT_UCODE_H_ 6 #define _CPT_UCODE_H_ 7 #include <stdbool.h> 8 9 #include "cpt_common.h" 10 #include "cpt_hw_types.h" 11 #include "cpt_mcode_defines.h" 12 13 /* 14 * This file defines functions that are interfaces to microcode spec. 15 * 16 */ 17 18 static uint8_t zuc_d[32] = { 19 0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E, 20 0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF, 21 0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1, 22 0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC 23 }; 24 25 static __rte_always_inline int 26 cpt_is_algo_supported(struct rte_crypto_sym_xform *xform) 27 { 28 /* 29 * Microcode only supports the following combination. 30 * Encryption followed by authentication 31 * Authentication followed by decryption 32 */ 33 if (xform->next) { 34 if ((xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) && 35 (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 36 (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)) { 37 /* Unsupported as of now by microcode */ 38 CPT_LOG_DP_ERR("Unsupported combination"); 39 return -1; 40 } 41 if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 42 (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) && 43 (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT)) { 44 /* For GMAC auth there is no cipher operation */ 45 if (xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM || 46 xform->next->auth.algo != 47 RTE_CRYPTO_AUTH_AES_GMAC) { 48 /* Unsupported as of now by microcode */ 49 CPT_LOG_DP_ERR("Unsupported combination"); 50 return -1; 51 } 52 } 53 } 54 return 0; 55 } 56 57 static __rte_always_inline void 58 gen_key_snow3g(const uint8_t *ck, uint32_t *keyx) 59 { 60 int i, base; 61 62 for (i = 0; i < 4; i++) { 63 base = 4 * i; 64 keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) | 65 (ck[base + 2] << 8) | (ck[base + 3]); 66 keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]); 67 } 68 } 69 70 static __rte_always_inline void 71 cpt_fc_salt_update(void *ctx, 72 uint8_t *salt) 73 { 74 struct cpt_ctx *cpt_ctx = ctx; 75 memcpy(&cpt_ctx->fctx.enc.encr_iv, salt, 4); 76 } 77 78 static __rte_always_inline int 79 cpt_fc_ciph_validate_key_aes(uint16_t key_len) 80 { 81 switch (key_len) { 82 case CPT_BYTE_16: 83 case CPT_BYTE_24: 84 case CPT_BYTE_32: 85 return 0; 86 default: 87 return -1; 88 } 89 } 90 91 static __rte_always_inline int 92 cpt_fc_ciph_set_type(cipher_type_t type, struct cpt_ctx *ctx, uint16_t key_len) 93 { 94 int fc_type = 0; 95 switch (type) { 96 case PASSTHROUGH: 97 fc_type = FC_GEN; 98 break; 99 case DES3_CBC: 100 case DES3_ECB: 101 fc_type = FC_GEN; 102 break; 103 case AES_CBC: 104 case AES_ECB: 105 case AES_CFB: 106 case AES_CTR: 107 case AES_GCM: 108 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 109 return -1; 110 fc_type = FC_GEN; 111 break; 112 case AES_XTS: 113 key_len = key_len / 2; 114 if (unlikely(key_len == CPT_BYTE_24)) { 115 CPT_LOG_DP_ERR("Invalid AES key len for XTS"); 116 return -1; 117 } 118 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 119 return -1; 120 fc_type = FC_GEN; 121 break; 122 case ZUC_EEA3: 123 case SNOW3G_UEA2: 124 if (unlikely(key_len != 16)) 125 return -1; 126 /* No support for AEAD yet */ 127 if (unlikely(ctx->hash_type)) 128 return -1; 129 fc_type = ZUC_SNOW3G; 130 break; 131 case KASUMI_F8_CBC: 132 case KASUMI_F8_ECB: 133 if (unlikely(key_len != 16)) 134 return -1; 135 /* No support for AEAD yet */ 136 if (unlikely(ctx->hash_type)) 137 return -1; 138 fc_type = KASUMI; 139 break; 140 default: 141 return -1; 142 } 143 144 ctx->fc_type = fc_type; 145 return 0; 146 } 147 148 static __rte_always_inline void 149 cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx) 150 { 151 cpt_ctx->enc_cipher = 0; 152 CPT_P_ENC_CTRL(fctx).enc_cipher = 0; 153 } 154 155 static __rte_always_inline void 156 cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len) 157 { 158 mc_aes_type_t aes_key_type = 0; 159 switch (key_len) { 160 case CPT_BYTE_16: 161 aes_key_type = AES_128_BIT; 162 break; 163 case CPT_BYTE_24: 164 aes_key_type = AES_192_BIT; 165 break; 166 case CPT_BYTE_32: 167 aes_key_type = AES_256_BIT; 168 break; 169 default: 170 /* This should not happen */ 171 CPT_LOG_DP_ERR("Invalid AES key len"); 172 return; 173 } 174 CPT_P_ENC_CTRL(fctx).aes_key = aes_key_type; 175 } 176 177 static __rte_always_inline void 178 cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key, 179 uint16_t key_len) 180 { 181 uint32_t keyx[4]; 182 cpt_ctx->snow3g = 1; 183 gen_key_snow3g(key, keyx); 184 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 185 cpt_ctx->zsk_flags = 0; 186 } 187 188 static __rte_always_inline void 189 cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key, 190 uint16_t key_len) 191 { 192 cpt_ctx->snow3g = 0; 193 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 194 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 195 cpt_ctx->zsk_flags = 0; 196 } 197 198 static __rte_always_inline void 199 cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key, 200 uint16_t key_len) 201 { 202 cpt_ctx->k_ecb = 1; 203 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 204 cpt_ctx->zsk_flags = 0; 205 } 206 207 static __rte_always_inline void 208 cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key, 209 uint16_t key_len) 210 { 211 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 212 cpt_ctx->zsk_flags = 0; 213 } 214 215 static __rte_always_inline int 216 cpt_fc_ciph_set_key(void *ctx, cipher_type_t type, const uint8_t *key, 217 uint16_t key_len, uint8_t *salt) 218 { 219 struct cpt_ctx *cpt_ctx = ctx; 220 mc_fc_context_t *fctx = &cpt_ctx->fctx; 221 uint64_t *ctrl_flags = NULL; 222 int ret; 223 224 ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len); 225 if (unlikely(ret)) 226 return -1; 227 228 if (cpt_ctx->fc_type == FC_GEN) { 229 ctrl_flags = (uint64_t *)&(fctx->enc.enc_ctrl.flags); 230 *ctrl_flags = rte_be_to_cpu_64(*ctrl_flags); 231 /* 232 * We need to always say IV is from DPTR as user can 233 * sometimes iverride IV per operation. 234 */ 235 CPT_P_ENC_CTRL(fctx).iv_source = CPT_FROM_DPTR; 236 } 237 238 switch (type) { 239 case PASSTHROUGH: 240 cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx); 241 goto fc_success; 242 case DES3_CBC: 243 /* CPT performs DES using 3DES with the 8B DES-key 244 * replicated 2 more times to match the 24B 3DES-key. 245 * Eg. If org. key is "0x0a 0x0b", then new key is 246 * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b" 247 */ 248 if (key_len == 8) { 249 /* Skipping the first 8B as it will be copied 250 * in the regular code flow 251 */ 252 memcpy(fctx->enc.encr_key+key_len, key, key_len); 253 memcpy(fctx->enc.encr_key+2*key_len, key, key_len); 254 } 255 break; 256 case DES3_ECB: 257 /* For DES3_ECB IV need to be from CTX. */ 258 CPT_P_ENC_CTRL(fctx).iv_source = CPT_FROM_CTX; 259 break; 260 case AES_CBC: 261 case AES_ECB: 262 case AES_CFB: 263 case AES_CTR: 264 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 265 break; 266 case AES_GCM: 267 /* Even though iv source is from dptr, 268 * aes_gcm salt is taken from ctx 269 */ 270 if (salt) { 271 memcpy(fctx->enc.encr_iv, salt, 4); 272 /* Assuming it was just salt update 273 * and nothing else 274 */ 275 if (!key) 276 goto fc_success; 277 } 278 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 279 break; 280 case AES_XTS: 281 key_len = key_len / 2; 282 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 283 284 /* Copy key2 for XTS into ipad */ 285 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 286 memcpy(fctx->hmac.ipad, &key[key_len], key_len); 287 break; 288 case SNOW3G_UEA2: 289 cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len); 290 goto success; 291 case ZUC_EEA3: 292 cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len); 293 goto success; 294 case KASUMI_F8_ECB: 295 cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len); 296 goto success; 297 case KASUMI_F8_CBC: 298 cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len); 299 goto success; 300 default: 301 break; 302 } 303 304 /* Only for FC_GEN case */ 305 306 /* For GMAC auth, cipher must be NULL */ 307 if (cpt_ctx->hash_type != GMAC_TYPE) 308 CPT_P_ENC_CTRL(fctx).enc_cipher = type; 309 310 memcpy(fctx->enc.encr_key, key, key_len); 311 312 fc_success: 313 *ctrl_flags = rte_cpu_to_be_64(*ctrl_flags); 314 315 success: 316 cpt_ctx->enc_cipher = type; 317 318 return 0; 319 } 320 321 static __rte_always_inline uint32_t 322 fill_sg_comp(sg_comp_t *list, 323 uint32_t i, 324 phys_addr_t dma_addr, 325 uint32_t size) 326 { 327 sg_comp_t *to = &list[i>>2]; 328 329 to->u.s.len[i%4] = rte_cpu_to_be_16(size); 330 to->ptr[i%4] = rte_cpu_to_be_64(dma_addr); 331 i++; 332 return i; 333 } 334 335 static __rte_always_inline uint32_t 336 fill_sg_comp_from_buf(sg_comp_t *list, 337 uint32_t i, 338 buf_ptr_t *from) 339 { 340 sg_comp_t *to = &list[i>>2]; 341 342 to->u.s.len[i%4] = rte_cpu_to_be_16(from->size); 343 to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr); 344 i++; 345 return i; 346 } 347 348 static __rte_always_inline uint32_t 349 fill_sg_comp_from_buf_min(sg_comp_t *list, 350 uint32_t i, 351 buf_ptr_t *from, 352 uint32_t *psize) 353 { 354 sg_comp_t *to = &list[i >> 2]; 355 uint32_t size = *psize; 356 uint32_t e_len; 357 358 e_len = (size > from->size) ? from->size : size; 359 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 360 to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr); 361 *psize -= e_len; 362 i++; 363 return i; 364 } 365 366 /* 367 * This fills the MC expected SGIO list 368 * from IOV given by user. 369 */ 370 static __rte_always_inline uint32_t 371 fill_sg_comp_from_iov(sg_comp_t *list, 372 uint32_t i, 373 iov_ptr_t *from, uint32_t from_offset, 374 uint32_t *psize, buf_ptr_t *extra_buf, 375 uint32_t extra_offset) 376 { 377 int32_t j; 378 uint32_t extra_len = extra_buf ? extra_buf->size : 0; 379 uint32_t size = *psize - extra_len; 380 buf_ptr_t *bufs; 381 382 bufs = from->bufs; 383 for (j = 0; (j < from->buf_cnt) && size; j++) { 384 phys_addr_t e_dma_addr; 385 uint32_t e_len; 386 sg_comp_t *to = &list[i >> 2]; 387 388 if (!bufs[j].size) 389 continue; 390 391 if (unlikely(from_offset)) { 392 if (from_offset >= bufs[j].size) { 393 from_offset -= bufs[j].size; 394 continue; 395 } 396 e_dma_addr = bufs[j].dma_addr + from_offset; 397 e_len = (size > (bufs[j].size - from_offset)) ? 398 (bufs[j].size - from_offset) : size; 399 from_offset = 0; 400 } else { 401 e_dma_addr = bufs[j].dma_addr; 402 e_len = (size > bufs[j].size) ? 403 bufs[j].size : size; 404 } 405 406 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 407 to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr); 408 409 if (extra_len && (e_len >= extra_offset)) { 410 /* Break the data at given offset */ 411 uint32_t next_len = e_len - extra_offset; 412 phys_addr_t next_dma = e_dma_addr + extra_offset; 413 414 if (!extra_offset) { 415 i--; 416 } else { 417 e_len = extra_offset; 418 size -= e_len; 419 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 420 } 421 422 /* Insert extra data ptr */ 423 if (extra_len) { 424 i++; 425 to = &list[i >> 2]; 426 to->u.s.len[i % 4] = 427 rte_cpu_to_be_16(extra_buf->size); 428 to->ptr[i % 4] = 429 rte_cpu_to_be_64(extra_buf->dma_addr); 430 431 /* size already decremented by extra len */ 432 } 433 434 /* insert the rest of the data */ 435 if (next_len) { 436 i++; 437 to = &list[i >> 2]; 438 to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len); 439 to->ptr[i % 4] = rte_cpu_to_be_64(next_dma); 440 size -= next_len; 441 } 442 extra_len = 0; 443 444 } else { 445 size -= e_len; 446 } 447 if (extra_offset) 448 extra_offset -= size; 449 i++; 450 } 451 452 *psize = size; 453 return (uint32_t)i; 454 } 455 456 static __rte_always_inline void 457 cpt_digest_gen_prep(uint32_t flags, 458 uint64_t d_lens, 459 digest_params_t *params, 460 void *op, 461 void **prep_req) 462 { 463 struct cpt_request_info *req; 464 uint32_t size, i; 465 uint16_t data_len, mac_len, key_len; 466 auth_type_t hash_type; 467 buf_ptr_t *meta_p; 468 struct cpt_ctx *ctx; 469 sg_comp_t *gather_comp; 470 sg_comp_t *scatter_comp; 471 uint8_t *in_buffer; 472 uint32_t g_size_bytes, s_size_bytes; 473 uint64_t dptr_dma, rptr_dma; 474 vq_cmd_word0_t vq_cmd_w0; 475 vq_cmd_word3_t vq_cmd_w3; 476 void *c_vaddr, *m_vaddr; 477 uint64_t c_dma, m_dma; 478 opcode_info_t opcode; 479 480 ctx = params->ctx_buf.vaddr; 481 meta_p = ¶ms->meta_buf; 482 483 m_vaddr = meta_p->vaddr; 484 m_dma = meta_p->dma_addr; 485 486 /* 487 * Save initial space that followed app data for completion code & 488 * alternate completion code to fall in same cache line as app data 489 */ 490 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 491 m_dma += COMPLETION_CODE_SIZE; 492 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 493 (uint8_t *)m_vaddr; 494 c_vaddr = (uint8_t *)m_vaddr + size; 495 c_dma = m_dma + size; 496 size += sizeof(cpt_res_s_t); 497 498 m_vaddr = (uint8_t *)m_vaddr + size; 499 m_dma += size; 500 501 req = m_vaddr; 502 503 size = sizeof(struct cpt_request_info); 504 m_vaddr = (uint8_t *)m_vaddr + size; 505 m_dma += size; 506 507 hash_type = ctx->hash_type; 508 mac_len = ctx->mac_len; 509 key_len = ctx->auth_key_len; 510 data_len = AUTH_DLEN(d_lens); 511 512 /*GP op header */ 513 vq_cmd_w0.u64 = 0; 514 vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8); 515 if (ctx->hmac) { 516 opcode.s.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE; 517 vq_cmd_w0.s.param1 = key_len; 518 vq_cmd_w0.s.dlen = data_len + ROUNDUP8(key_len); 519 } else { 520 opcode.s.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE; 521 vq_cmd_w0.s.param1 = 0; 522 vq_cmd_w0.s.dlen = data_len; 523 } 524 525 opcode.s.minor = 0; 526 527 /* Null auth only case enters the if */ 528 if (unlikely(!hash_type && !ctx->enc_cipher)) { 529 opcode.s.major = CPT_MAJOR_OP_MISC; 530 /* Minor op is passthrough */ 531 opcode.s.minor = 0x03; 532 /* Send out completion code only */ 533 vq_cmd_w0.s.param2 = 0x1; 534 } 535 536 vq_cmd_w0.s.opcode = opcode.flags; 537 538 /* DPTR has SG list */ 539 in_buffer = m_vaddr; 540 dptr_dma = m_dma; 541 542 ((uint16_t *)in_buffer)[0] = 0; 543 ((uint16_t *)in_buffer)[1] = 0; 544 545 /* TODO Add error check if space will be sufficient */ 546 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 547 548 /* 549 * Input gather list 550 */ 551 552 i = 0; 553 554 if (ctx->hmac) { 555 uint64_t k_dma = params->ctx_buf.dma_addr + 556 offsetof(struct cpt_ctx, auth_key); 557 /* Key */ 558 i = fill_sg_comp(gather_comp, i, k_dma, ROUNDUP8(key_len)); 559 } 560 561 /* input data */ 562 size = data_len; 563 if (size) { 564 i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 565 0, &size, NULL, 0); 566 if (unlikely(size)) { 567 CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short" 568 " by %dB", size); 569 return; 570 } 571 } else { 572 /* 573 * Looks like we need to support zero data 574 * gather ptr in case of hash & hmac 575 */ 576 i++; 577 } 578 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 579 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 580 581 /* 582 * Output Gather list 583 */ 584 585 i = 0; 586 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 587 588 if (flags & VALID_MAC_BUF) { 589 if (unlikely(params->mac_buf.size < mac_len)) { 590 CPT_LOG_DP_ERR("Insufficient MAC size"); 591 return; 592 } 593 594 size = mac_len; 595 i = fill_sg_comp_from_buf_min(scatter_comp, i, 596 ¶ms->mac_buf, &size); 597 } else { 598 size = mac_len; 599 i = fill_sg_comp_from_iov(scatter_comp, i, 600 params->src_iov, data_len, 601 &size, NULL, 0); 602 if (unlikely(size)) { 603 CPT_LOG_DP_ERR("Insufficient dst IOV size, short by" 604 " %dB", size); 605 return; 606 } 607 } 608 609 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 610 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 611 612 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 613 614 /* This is DPTR len incase of SG mode */ 615 vq_cmd_w0.s.dlen = size; 616 617 m_vaddr = (uint8_t *)m_vaddr + size; 618 m_dma += size; 619 620 /* cpt alternate completion address saved earlier */ 621 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 622 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 623 rptr_dma = c_dma - 8; 624 625 req->ist.ei1 = dptr_dma; 626 req->ist.ei2 = rptr_dma; 627 628 /* vq command w3 */ 629 vq_cmd_w3.u64 = 0; 630 631 /* 16 byte aligned cpt res address */ 632 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 633 *req->completion_addr = COMPLETION_CODE_INIT; 634 req->comp_baddr = c_dma; 635 636 /* Fill microcode part of instruction */ 637 req->ist.ei0 = vq_cmd_w0.u64; 638 req->ist.ei3 = vq_cmd_w3.u64; 639 640 req->op = op; 641 642 *prep_req = req; 643 return; 644 } 645 646 static __rte_always_inline void 647 cpt_enc_hmac_prep(uint32_t flags, 648 uint64_t d_offs, 649 uint64_t d_lens, 650 fc_params_t *fc_params, 651 void *op, 652 void **prep_req) 653 { 654 uint32_t iv_offset = 0; 655 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 656 struct cpt_ctx *cpt_ctx; 657 uint32_t cipher_type, hash_type; 658 uint32_t mac_len, size; 659 uint8_t iv_len = 16; 660 struct cpt_request_info *req; 661 buf_ptr_t *meta_p, *aad_buf = NULL; 662 uint32_t encr_offset, auth_offset; 663 uint32_t encr_data_len, auth_data_len, aad_len = 0; 664 uint32_t passthrough_len = 0; 665 void *m_vaddr, *offset_vaddr; 666 uint64_t m_dma, offset_dma, ctx_dma; 667 vq_cmd_word0_t vq_cmd_w0; 668 vq_cmd_word3_t vq_cmd_w3; 669 void *c_vaddr; 670 uint64_t c_dma; 671 opcode_info_t opcode; 672 673 meta_p = &fc_params->meta_buf; 674 m_vaddr = meta_p->vaddr; 675 m_dma = meta_p->dma_addr; 676 677 encr_offset = ENCR_OFFSET(d_offs); 678 auth_offset = AUTH_OFFSET(d_offs); 679 encr_data_len = ENCR_DLEN(d_lens); 680 auth_data_len = AUTH_DLEN(d_lens); 681 if (unlikely(flags & VALID_AAD_BUF)) { 682 /* 683 * We dont support both aad 684 * and auth data separately 685 */ 686 auth_data_len = 0; 687 auth_offset = 0; 688 aad_len = fc_params->aad_buf.size; 689 aad_buf = &fc_params->aad_buf; 690 } 691 cpt_ctx = fc_params->ctx_buf.vaddr; 692 cipher_type = cpt_ctx->enc_cipher; 693 hash_type = cpt_ctx->hash_type; 694 mac_len = cpt_ctx->mac_len; 695 696 /* 697 * Save initial space that followed app data for completion code & 698 * alternate completion code to fall in same cache line as app data 699 */ 700 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 701 m_dma += COMPLETION_CODE_SIZE; 702 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 703 (uint8_t *)m_vaddr; 704 705 c_vaddr = (uint8_t *)m_vaddr + size; 706 c_dma = m_dma + size; 707 size += sizeof(cpt_res_s_t); 708 709 m_vaddr = (uint8_t *)m_vaddr + size; 710 m_dma += size; 711 712 /* start cpt request info struct at 8 byte boundary */ 713 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 714 (uint8_t *)m_vaddr; 715 716 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 717 718 size += sizeof(struct cpt_request_info); 719 m_vaddr = (uint8_t *)m_vaddr + size; 720 m_dma += size; 721 722 if (hash_type == GMAC_TYPE) 723 encr_data_len = 0; 724 725 if (unlikely(!(flags & VALID_IV_BUF))) { 726 iv_len = 0; 727 iv_offset = ENCR_IV_OFFSET(d_offs); 728 } 729 730 if (unlikely(flags & VALID_AAD_BUF)) { 731 /* 732 * When AAD is given, data above encr_offset is pass through 733 * Since AAD is given as separate pointer and not as offset, 734 * this is a special case as we need to fragment input data 735 * into passthrough + encr_data and then insert AAD in between. 736 */ 737 if (hash_type != GMAC_TYPE) { 738 passthrough_len = encr_offset; 739 auth_offset = passthrough_len + iv_len; 740 encr_offset = passthrough_len + aad_len + iv_len; 741 auth_data_len = aad_len + encr_data_len; 742 } else { 743 passthrough_len = 16 + aad_len; 744 auth_offset = passthrough_len + iv_len; 745 auth_data_len = aad_len; 746 } 747 } else { 748 encr_offset += iv_len; 749 auth_offset += iv_len; 750 } 751 752 /* Encryption */ 753 opcode.s.major = CPT_MAJOR_OP_FC; 754 opcode.s.minor = 0; 755 756 auth_dlen = auth_offset + auth_data_len; 757 enc_dlen = encr_data_len + encr_offset; 758 if (unlikely(encr_data_len & 0xf)) { 759 if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB)) 760 enc_dlen = ROUNDUP8(encr_data_len) + encr_offset; 761 else if (likely((cipher_type == AES_CBC) || 762 (cipher_type == AES_ECB))) 763 enc_dlen = ROUNDUP16(encr_data_len) + encr_offset; 764 } 765 766 if (unlikely(hash_type == GMAC_TYPE)) { 767 encr_offset = auth_dlen; 768 enc_dlen = 0; 769 } 770 771 if (unlikely(auth_dlen > enc_dlen)) { 772 inputlen = auth_dlen; 773 outputlen = auth_dlen + mac_len; 774 } else { 775 inputlen = enc_dlen; 776 outputlen = enc_dlen + mac_len; 777 } 778 779 /* GP op header */ 780 vq_cmd_w0.u64 = 0; 781 vq_cmd_w0.s.param1 = encr_data_len; 782 vq_cmd_w0.s.param2 = auth_data_len; 783 /* 784 * In 83XX since we have a limitation of 785 * IV & Offset control word not part of instruction 786 * and need to be part of Data Buffer, we check if 787 * head room is there and then only do the Direct mode processing 788 */ 789 if (likely((flags & SINGLE_BUF_INPLACE) && 790 (flags & SINGLE_BUF_HEADTAILROOM))) { 791 void *dm_vaddr = fc_params->bufs[0].vaddr; 792 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 793 /* 794 * This flag indicates that there is 24 bytes head room and 795 * 8 bytes tail room available, so that we get to do 796 * DIRECT MODE with limitation 797 */ 798 799 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 800 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 801 802 /* DPTR */ 803 req->ist.ei1 = offset_dma; 804 /* RPTR should just exclude offset control word */ 805 req->ist.ei2 = dm_dma_addr - iv_len; 806 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 807 + outputlen - iv_len); 808 809 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 810 811 vq_cmd_w0.s.opcode = opcode.flags; 812 813 if (likely(iv_len)) { 814 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 815 + OFF_CTRL_LEN); 816 uint64_t *src = fc_params->iv_buf; 817 dest[0] = src[0]; 818 dest[1] = src[1]; 819 } 820 821 *(uint64_t *)offset_vaddr = 822 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 823 ((uint64_t)iv_offset << 8) | 824 ((uint64_t)auth_offset)); 825 826 } else { 827 uint32_t i, g_size_bytes, s_size_bytes; 828 uint64_t dptr_dma, rptr_dma; 829 sg_comp_t *gather_comp; 830 sg_comp_t *scatter_comp; 831 uint8_t *in_buffer; 832 833 /* This falls under strict SG mode */ 834 offset_vaddr = m_vaddr; 835 offset_dma = m_dma; 836 size = OFF_CTRL_LEN + iv_len; 837 838 m_vaddr = (uint8_t *)m_vaddr + size; 839 m_dma += size; 840 841 opcode.s.major |= CPT_DMA_MODE; 842 843 vq_cmd_w0.s.opcode = opcode.flags; 844 845 if (likely(iv_len)) { 846 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 847 + OFF_CTRL_LEN); 848 uint64_t *src = fc_params->iv_buf; 849 dest[0] = src[0]; 850 dest[1] = src[1]; 851 } 852 853 *(uint64_t *)offset_vaddr = 854 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 855 ((uint64_t)iv_offset << 8) | 856 ((uint64_t)auth_offset)); 857 858 /* DPTR has SG list */ 859 in_buffer = m_vaddr; 860 dptr_dma = m_dma; 861 862 ((uint16_t *)in_buffer)[0] = 0; 863 ((uint16_t *)in_buffer)[1] = 0; 864 865 /* TODO Add error check if space will be sufficient */ 866 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 867 868 /* 869 * Input Gather List 870 */ 871 872 i = 0; 873 874 /* Offset control word that includes iv */ 875 i = fill_sg_comp(gather_comp, i, offset_dma, 876 OFF_CTRL_LEN + iv_len); 877 878 /* Add input data */ 879 size = inputlen - iv_len; 880 if (likely(size)) { 881 uint32_t aad_offset = aad_len ? passthrough_len : 0; 882 883 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 884 i = fill_sg_comp_from_buf_min(gather_comp, i, 885 fc_params->bufs, 886 &size); 887 } else { 888 i = fill_sg_comp_from_iov(gather_comp, i, 889 fc_params->src_iov, 890 0, &size, 891 aad_buf, aad_offset); 892 } 893 894 if (unlikely(size)) { 895 CPT_LOG_DP_ERR("Insufficient buffer space," 896 " size %d needed", size); 897 return; 898 } 899 } 900 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 901 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 902 903 /* 904 * Output Scatter list 905 */ 906 i = 0; 907 scatter_comp = 908 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 909 910 /* Add IV */ 911 if (likely(iv_len)) { 912 i = fill_sg_comp(scatter_comp, i, 913 offset_dma + OFF_CTRL_LEN, 914 iv_len); 915 } 916 917 /* output data or output data + digest*/ 918 if (unlikely(flags & VALID_MAC_BUF)) { 919 size = outputlen - iv_len - mac_len; 920 if (size) { 921 uint32_t aad_offset = 922 aad_len ? passthrough_len : 0; 923 924 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 925 i = fill_sg_comp_from_buf_min( 926 scatter_comp, 927 i, 928 fc_params->bufs, 929 &size); 930 } else { 931 i = fill_sg_comp_from_iov(scatter_comp, 932 i, 933 fc_params->dst_iov, 934 0, 935 &size, 936 aad_buf, 937 aad_offset); 938 } 939 if (unlikely(size)) { 940 CPT_LOG_DP_ERR("Insufficient buffer" 941 " space, size %d needed", 942 size); 943 return; 944 } 945 } 946 /* mac_data */ 947 if (mac_len) { 948 i = fill_sg_comp_from_buf(scatter_comp, i, 949 &fc_params->mac_buf); 950 } 951 } else { 952 /* Output including mac */ 953 size = outputlen - iv_len; 954 if (likely(size)) { 955 uint32_t aad_offset = 956 aad_len ? passthrough_len : 0; 957 958 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 959 i = fill_sg_comp_from_buf_min( 960 scatter_comp, 961 i, 962 fc_params->bufs, 963 &size); 964 } else { 965 i = fill_sg_comp_from_iov(scatter_comp, 966 i, 967 fc_params->dst_iov, 968 0, 969 &size, 970 aad_buf, 971 aad_offset); 972 } 973 if (unlikely(size)) { 974 CPT_LOG_DP_ERR("Insufficient buffer" 975 " space, size %d needed", 976 size); 977 return; 978 } 979 } 980 } 981 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 982 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 983 984 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 985 986 /* This is DPTR len incase of SG mode */ 987 vq_cmd_w0.s.dlen = size; 988 989 m_vaddr = (uint8_t *)m_vaddr + size; 990 m_dma += size; 991 992 /* cpt alternate completion address saved earlier */ 993 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 994 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 995 rptr_dma = c_dma - 8; 996 997 req->ist.ei1 = dptr_dma; 998 req->ist.ei2 = rptr_dma; 999 } 1000 1001 ctx_dma = fc_params->ctx_buf.dma_addr + 1002 offsetof(struct cpt_ctx, fctx); 1003 /* vq command w3 */ 1004 vq_cmd_w3.u64 = 0; 1005 vq_cmd_w3.s.grp = 0; 1006 vq_cmd_w3.s.cptr = ctx_dma; 1007 1008 /* 16 byte aligned cpt res address */ 1009 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1010 *req->completion_addr = COMPLETION_CODE_INIT; 1011 req->comp_baddr = c_dma; 1012 1013 /* Fill microcode part of instruction */ 1014 req->ist.ei0 = vq_cmd_w0.u64; 1015 req->ist.ei3 = vq_cmd_w3.u64; 1016 1017 req->op = op; 1018 1019 *prep_req = req; 1020 return; 1021 } 1022 1023 static __rte_always_inline void 1024 cpt_dec_hmac_prep(uint32_t flags, 1025 uint64_t d_offs, 1026 uint64_t d_lens, 1027 fc_params_t *fc_params, 1028 void *op, 1029 void **prep_req) 1030 { 1031 uint32_t iv_offset = 0, size; 1032 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 1033 struct cpt_ctx *cpt_ctx; 1034 int32_t hash_type, mac_len; 1035 uint8_t iv_len = 16; 1036 struct cpt_request_info *req; 1037 buf_ptr_t *meta_p, *aad_buf = NULL; 1038 uint32_t encr_offset, auth_offset; 1039 uint32_t encr_data_len, auth_data_len, aad_len = 0; 1040 uint32_t passthrough_len = 0; 1041 void *m_vaddr, *offset_vaddr; 1042 uint64_t m_dma, offset_dma, ctx_dma; 1043 opcode_info_t opcode; 1044 vq_cmd_word0_t vq_cmd_w0; 1045 vq_cmd_word3_t vq_cmd_w3; 1046 void *c_vaddr; 1047 uint64_t c_dma; 1048 1049 meta_p = &fc_params->meta_buf; 1050 m_vaddr = meta_p->vaddr; 1051 m_dma = meta_p->dma_addr; 1052 1053 encr_offset = ENCR_OFFSET(d_offs); 1054 auth_offset = AUTH_OFFSET(d_offs); 1055 encr_data_len = ENCR_DLEN(d_lens); 1056 auth_data_len = AUTH_DLEN(d_lens); 1057 1058 if (unlikely(flags & VALID_AAD_BUF)) { 1059 /* 1060 * We dont support both aad 1061 * and auth data separately 1062 */ 1063 auth_data_len = 0; 1064 auth_offset = 0; 1065 aad_len = fc_params->aad_buf.size; 1066 aad_buf = &fc_params->aad_buf; 1067 } 1068 1069 cpt_ctx = fc_params->ctx_buf.vaddr; 1070 hash_type = cpt_ctx->hash_type; 1071 mac_len = cpt_ctx->mac_len; 1072 1073 if (hash_type == GMAC_TYPE) 1074 encr_data_len = 0; 1075 1076 if (unlikely(!(flags & VALID_IV_BUF))) { 1077 iv_len = 0; 1078 iv_offset = ENCR_IV_OFFSET(d_offs); 1079 } 1080 1081 if (unlikely(flags & VALID_AAD_BUF)) { 1082 /* 1083 * When AAD is given, data above encr_offset is pass through 1084 * Since AAD is given as separate pointer and not as offset, 1085 * this is a special case as we need to fragment input data 1086 * into passthrough + encr_data and then insert AAD in between. 1087 */ 1088 if (hash_type != GMAC_TYPE) { 1089 passthrough_len = encr_offset; 1090 auth_offset = passthrough_len + iv_len; 1091 encr_offset = passthrough_len + aad_len + iv_len; 1092 auth_data_len = aad_len + encr_data_len; 1093 } else { 1094 passthrough_len = 16 + aad_len; 1095 auth_offset = passthrough_len + iv_len; 1096 auth_data_len = aad_len; 1097 } 1098 } else { 1099 encr_offset += iv_len; 1100 auth_offset += iv_len; 1101 } 1102 1103 /* 1104 * Save initial space that followed app data for completion code & 1105 * alternate completion code to fall in same cache line as app data 1106 */ 1107 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1108 m_dma += COMPLETION_CODE_SIZE; 1109 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1110 (uint8_t *)m_vaddr; 1111 c_vaddr = (uint8_t *)m_vaddr + size; 1112 c_dma = m_dma + size; 1113 size += sizeof(cpt_res_s_t); 1114 1115 m_vaddr = (uint8_t *)m_vaddr + size; 1116 m_dma += size; 1117 1118 /* start cpt request info structure at 8 byte alignment */ 1119 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 1120 (uint8_t *)m_vaddr; 1121 1122 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 1123 1124 size += sizeof(struct cpt_request_info); 1125 m_vaddr = (uint8_t *)m_vaddr + size; 1126 m_dma += size; 1127 1128 /* Decryption */ 1129 opcode.s.major = CPT_MAJOR_OP_FC; 1130 opcode.s.minor = 1; 1131 1132 enc_dlen = encr_offset + encr_data_len; 1133 auth_dlen = auth_offset + auth_data_len; 1134 1135 if (auth_dlen > enc_dlen) { 1136 inputlen = auth_dlen + mac_len; 1137 outputlen = auth_dlen; 1138 } else { 1139 inputlen = enc_dlen + mac_len; 1140 outputlen = enc_dlen; 1141 } 1142 1143 if (hash_type == GMAC_TYPE) 1144 encr_offset = inputlen; 1145 1146 vq_cmd_w0.u64 = 0; 1147 vq_cmd_w0.s.param1 = encr_data_len; 1148 vq_cmd_w0.s.param2 = auth_data_len; 1149 1150 /* 1151 * In 83XX since we have a limitation of 1152 * IV & Offset control word not part of instruction 1153 * and need to be part of Data Buffer, we check if 1154 * head room is there and then only do the Direct mode processing 1155 */ 1156 if (likely((flags & SINGLE_BUF_INPLACE) && 1157 (flags & SINGLE_BUF_HEADTAILROOM))) { 1158 void *dm_vaddr = fc_params->bufs[0].vaddr; 1159 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 1160 /* 1161 * This flag indicates that there is 24 bytes head room and 1162 * 8 bytes tail room available, so that we get to do 1163 * DIRECT MODE with limitation 1164 */ 1165 1166 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 1167 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1168 req->ist.ei1 = offset_dma; 1169 1170 /* RPTR should just exclude offset control word */ 1171 req->ist.ei2 = dm_dma_addr - iv_len; 1172 1173 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr + 1174 outputlen - iv_len); 1175 /* since this is decryption, 1176 * don't touch the content of 1177 * alternate ccode space as it contains 1178 * hmac. 1179 */ 1180 1181 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1182 1183 vq_cmd_w0.s.opcode = opcode.flags; 1184 1185 if (likely(iv_len)) { 1186 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1187 OFF_CTRL_LEN); 1188 uint64_t *src = fc_params->iv_buf; 1189 dest[0] = src[0]; 1190 dest[1] = src[1]; 1191 } 1192 1193 *(uint64_t *)offset_vaddr = 1194 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1195 ((uint64_t)iv_offset << 8) | 1196 ((uint64_t)auth_offset)); 1197 1198 } else { 1199 uint64_t dptr_dma, rptr_dma; 1200 uint32_t g_size_bytes, s_size_bytes; 1201 sg_comp_t *gather_comp; 1202 sg_comp_t *scatter_comp; 1203 uint8_t *in_buffer; 1204 uint8_t i = 0; 1205 1206 /* This falls under strict SG mode */ 1207 offset_vaddr = m_vaddr; 1208 offset_dma = m_dma; 1209 size = OFF_CTRL_LEN + iv_len; 1210 1211 m_vaddr = (uint8_t *)m_vaddr + size; 1212 m_dma += size; 1213 1214 opcode.s.major |= CPT_DMA_MODE; 1215 1216 vq_cmd_w0.s.opcode = opcode.flags; 1217 1218 if (likely(iv_len)) { 1219 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1220 OFF_CTRL_LEN); 1221 uint64_t *src = fc_params->iv_buf; 1222 dest[0] = src[0]; 1223 dest[1] = src[1]; 1224 } 1225 1226 *(uint64_t *)offset_vaddr = 1227 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1228 ((uint64_t)iv_offset << 8) | 1229 ((uint64_t)auth_offset)); 1230 1231 /* DPTR has SG list */ 1232 in_buffer = m_vaddr; 1233 dptr_dma = m_dma; 1234 1235 ((uint16_t *)in_buffer)[0] = 0; 1236 ((uint16_t *)in_buffer)[1] = 0; 1237 1238 /* TODO Add error check if space will be sufficient */ 1239 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1240 1241 /* 1242 * Input Gather List 1243 */ 1244 i = 0; 1245 1246 /* Offset control word that includes iv */ 1247 i = fill_sg_comp(gather_comp, i, offset_dma, 1248 OFF_CTRL_LEN + iv_len); 1249 1250 /* Add input data */ 1251 if (flags & VALID_MAC_BUF) { 1252 size = inputlen - iv_len - mac_len; 1253 if (size) { 1254 /* input data only */ 1255 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1256 i = fill_sg_comp_from_buf_min( 1257 gather_comp, i, 1258 fc_params->bufs, 1259 &size); 1260 } else { 1261 uint32_t aad_offset = aad_len ? 1262 passthrough_len : 0; 1263 1264 i = fill_sg_comp_from_iov(gather_comp, 1265 i, 1266 fc_params->src_iov, 1267 0, &size, 1268 aad_buf, 1269 aad_offset); 1270 } 1271 if (unlikely(size)) { 1272 CPT_LOG_DP_ERR("Insufficient buffer" 1273 " space, size %d needed", 1274 size); 1275 return; 1276 } 1277 } 1278 1279 /* mac data */ 1280 if (mac_len) { 1281 i = fill_sg_comp_from_buf(gather_comp, i, 1282 &fc_params->mac_buf); 1283 } 1284 } else { 1285 /* input data + mac */ 1286 size = inputlen - iv_len; 1287 if (size) { 1288 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1289 i = fill_sg_comp_from_buf_min( 1290 gather_comp, i, 1291 fc_params->bufs, 1292 &size); 1293 } else { 1294 uint32_t aad_offset = aad_len ? 1295 passthrough_len : 0; 1296 1297 if (unlikely(!fc_params->src_iov)) { 1298 CPT_LOG_DP_ERR("Bad input args"); 1299 return; 1300 } 1301 1302 i = fill_sg_comp_from_iov( 1303 gather_comp, i, 1304 fc_params->src_iov, 1305 0, &size, 1306 aad_buf, 1307 aad_offset); 1308 } 1309 1310 if (unlikely(size)) { 1311 CPT_LOG_DP_ERR("Insufficient buffer" 1312 " space, size %d needed", 1313 size); 1314 return; 1315 } 1316 } 1317 } 1318 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1319 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1320 1321 /* 1322 * Output Scatter List 1323 */ 1324 1325 i = 0; 1326 scatter_comp = 1327 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1328 1329 /* Add iv */ 1330 if (iv_len) { 1331 i = fill_sg_comp(scatter_comp, i, 1332 offset_dma + OFF_CTRL_LEN, 1333 iv_len); 1334 } 1335 1336 /* Add output data */ 1337 size = outputlen - iv_len; 1338 if (size) { 1339 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1340 /* handle single buffer here */ 1341 i = fill_sg_comp_from_buf_min(scatter_comp, i, 1342 fc_params->bufs, 1343 &size); 1344 } else { 1345 uint32_t aad_offset = aad_len ? 1346 passthrough_len : 0; 1347 1348 if (unlikely(!fc_params->dst_iov)) { 1349 CPT_LOG_DP_ERR("Bad input args"); 1350 return; 1351 } 1352 1353 i = fill_sg_comp_from_iov(scatter_comp, i, 1354 fc_params->dst_iov, 0, 1355 &size, aad_buf, 1356 aad_offset); 1357 } 1358 1359 if (unlikely(size)) { 1360 CPT_LOG_DP_ERR("Insufficient buffer space," 1361 " size %d needed", size); 1362 return; 1363 } 1364 } 1365 1366 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1367 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1368 1369 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1370 1371 /* This is DPTR len incase of SG mode */ 1372 vq_cmd_w0.s.dlen = size; 1373 1374 m_vaddr = (uint8_t *)m_vaddr + size; 1375 m_dma += size; 1376 1377 /* cpt alternate completion address saved earlier */ 1378 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1379 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1380 rptr_dma = c_dma - 8; 1381 size += COMPLETION_CODE_SIZE; 1382 1383 req->ist.ei1 = dptr_dma; 1384 req->ist.ei2 = rptr_dma; 1385 } 1386 1387 ctx_dma = fc_params->ctx_buf.dma_addr + 1388 offsetof(struct cpt_ctx, fctx); 1389 /* vq command w3 */ 1390 vq_cmd_w3.u64 = 0; 1391 vq_cmd_w3.s.grp = 0; 1392 vq_cmd_w3.s.cptr = ctx_dma; 1393 1394 /* 16 byte aligned cpt res address */ 1395 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1396 *req->completion_addr = COMPLETION_CODE_INIT; 1397 req->comp_baddr = c_dma; 1398 1399 /* Fill microcode part of instruction */ 1400 req->ist.ei0 = vq_cmd_w0.u64; 1401 req->ist.ei3 = vq_cmd_w3.u64; 1402 1403 req->op = op; 1404 1405 *prep_req = req; 1406 return; 1407 } 1408 1409 static __rte_always_inline void 1410 cpt_zuc_snow3g_enc_prep(uint32_t req_flags, 1411 uint64_t d_offs, 1412 uint64_t d_lens, 1413 fc_params_t *params, 1414 void *op, 1415 void **prep_req) 1416 { 1417 uint32_t size; 1418 int32_t inputlen, outputlen; 1419 struct cpt_ctx *cpt_ctx; 1420 uint32_t mac_len = 0; 1421 uint8_t snow3g, j; 1422 struct cpt_request_info *req; 1423 buf_ptr_t *buf_p; 1424 uint32_t encr_offset = 0, auth_offset = 0; 1425 uint32_t encr_data_len = 0, auth_data_len = 0; 1426 int flags, iv_len = 16; 1427 void *m_vaddr, *c_vaddr; 1428 uint64_t m_dma, c_dma, offset_ctrl; 1429 uint64_t *offset_vaddr, offset_dma; 1430 uint32_t *iv_s, iv[4]; 1431 vq_cmd_word0_t vq_cmd_w0; 1432 vq_cmd_word3_t vq_cmd_w3; 1433 opcode_info_t opcode; 1434 1435 buf_p = ¶ms->meta_buf; 1436 m_vaddr = buf_p->vaddr; 1437 m_dma = buf_p->dma_addr; 1438 1439 cpt_ctx = params->ctx_buf.vaddr; 1440 flags = cpt_ctx->zsk_flags; 1441 mac_len = cpt_ctx->mac_len; 1442 snow3g = cpt_ctx->snow3g; 1443 1444 /* 1445 * Save initial space that followed app data for completion code & 1446 * alternate completion code to fall in same cache line as app data 1447 */ 1448 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1449 m_dma += COMPLETION_CODE_SIZE; 1450 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1451 (uint8_t *)m_vaddr; 1452 1453 c_vaddr = (uint8_t *)m_vaddr + size; 1454 c_dma = m_dma + size; 1455 size += sizeof(cpt_res_s_t); 1456 1457 m_vaddr = (uint8_t *)m_vaddr + size; 1458 m_dma += size; 1459 1460 /* Reserve memory for cpt request info */ 1461 req = m_vaddr; 1462 1463 size = sizeof(struct cpt_request_info); 1464 m_vaddr = (uint8_t *)m_vaddr + size; 1465 m_dma += size; 1466 1467 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1468 1469 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1470 opcode.s.minor = ((1 << 6) | (snow3g << 5) | (0 << 4) | 1471 (0 << 3) | (flags & 0x7)); 1472 1473 if (flags == 0x1) { 1474 /* 1475 * Microcode expects offsets in bytes 1476 * TODO: Rounding off 1477 */ 1478 auth_data_len = AUTH_DLEN(d_lens); 1479 1480 /* EIA3 or UIA2 */ 1481 auth_offset = AUTH_OFFSET(d_offs); 1482 auth_offset = auth_offset / 8; 1483 1484 /* consider iv len */ 1485 auth_offset += iv_len; 1486 1487 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 1488 outputlen = mac_len; 1489 1490 offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset); 1491 1492 } else { 1493 /* EEA3 or UEA2 */ 1494 /* 1495 * Microcode expects offsets in bytes 1496 * TODO: Rounding off 1497 */ 1498 encr_data_len = ENCR_DLEN(d_lens); 1499 1500 encr_offset = ENCR_OFFSET(d_offs); 1501 encr_offset = encr_offset / 8; 1502 /* consider iv len */ 1503 encr_offset += iv_len; 1504 1505 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 1506 outputlen = inputlen; 1507 1508 /* iv offset is 0 */ 1509 offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1510 } 1511 1512 /* IV */ 1513 iv_s = (flags == 0x1) ? params->auth_iv_buf : 1514 params->iv_buf; 1515 1516 if (snow3g) { 1517 /* 1518 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1519 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1520 */ 1521 1522 for (j = 0; j < 4; j++) 1523 iv[j] = iv_s[3 - j]; 1524 } else { 1525 /* ZUC doesn't need a swap */ 1526 for (j = 0; j < 4; j++) 1527 iv[j] = iv_s[j]; 1528 } 1529 1530 /* 1531 * GP op header, lengths are expected in bits. 1532 */ 1533 vq_cmd_w0.u64 = 0; 1534 vq_cmd_w0.s.param1 = encr_data_len; 1535 vq_cmd_w0.s.param2 = auth_data_len; 1536 1537 /* 1538 * In 83XX since we have a limitation of 1539 * IV & Offset control word not part of instruction 1540 * and need to be part of Data Buffer, we check if 1541 * head room is there and then only do the Direct mode processing 1542 */ 1543 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1544 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1545 void *dm_vaddr = params->bufs[0].vaddr; 1546 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1547 /* 1548 * This flag indicates that there is 24 bytes head room and 1549 * 8 bytes tail room available, so that we get to do 1550 * DIRECT MODE with limitation 1551 */ 1552 1553 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1554 OFF_CTRL_LEN - iv_len); 1555 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1556 1557 /* DPTR */ 1558 req->ist.ei1 = offset_dma; 1559 /* RPTR should just exclude offset control word */ 1560 req->ist.ei2 = dm_dma_addr - iv_len; 1561 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1562 + outputlen - iv_len); 1563 1564 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1565 1566 vq_cmd_w0.s.opcode = opcode.flags; 1567 1568 if (likely(iv_len)) { 1569 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1570 + OFF_CTRL_LEN); 1571 memcpy(iv_d, iv, 16); 1572 } 1573 1574 *offset_vaddr = offset_ctrl; 1575 } else { 1576 uint32_t i, g_size_bytes, s_size_bytes; 1577 uint64_t dptr_dma, rptr_dma; 1578 sg_comp_t *gather_comp; 1579 sg_comp_t *scatter_comp; 1580 uint8_t *in_buffer; 1581 uint32_t *iv_d; 1582 1583 /* save space for iv */ 1584 offset_vaddr = m_vaddr; 1585 offset_dma = m_dma; 1586 1587 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1588 m_dma += OFF_CTRL_LEN + iv_len; 1589 1590 opcode.s.major |= CPT_DMA_MODE; 1591 1592 vq_cmd_w0.s.opcode = opcode.flags; 1593 1594 /* DPTR has SG list */ 1595 in_buffer = m_vaddr; 1596 dptr_dma = m_dma; 1597 1598 ((uint16_t *)in_buffer)[0] = 0; 1599 ((uint16_t *)in_buffer)[1] = 0; 1600 1601 /* TODO Add error check if space will be sufficient */ 1602 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1603 1604 /* 1605 * Input Gather List 1606 */ 1607 i = 0; 1608 1609 /* Offset control word followed by iv */ 1610 1611 i = fill_sg_comp(gather_comp, i, offset_dma, 1612 OFF_CTRL_LEN + iv_len); 1613 1614 /* iv offset is 0 */ 1615 *offset_vaddr = offset_ctrl; 1616 1617 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1618 memcpy(iv_d, iv, 16); 1619 1620 /* input data */ 1621 size = inputlen - iv_len; 1622 if (size) { 1623 i = fill_sg_comp_from_iov(gather_comp, i, 1624 params->src_iov, 1625 0, &size, NULL, 0); 1626 if (unlikely(size)) { 1627 CPT_LOG_DP_ERR("Insufficient buffer space," 1628 " size %d needed", size); 1629 return; 1630 } 1631 } 1632 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1633 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1634 1635 /* 1636 * Output Scatter List 1637 */ 1638 1639 i = 0; 1640 scatter_comp = 1641 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1642 1643 if (flags == 0x1) { 1644 /* IV in SLIST only for EEA3 & UEA2 */ 1645 iv_len = 0; 1646 } 1647 1648 if (iv_len) { 1649 i = fill_sg_comp(scatter_comp, i, 1650 offset_dma + OFF_CTRL_LEN, iv_len); 1651 } 1652 1653 /* Add output data */ 1654 if (req_flags & VALID_MAC_BUF) { 1655 size = outputlen - iv_len - mac_len; 1656 if (size) { 1657 i = fill_sg_comp_from_iov(scatter_comp, i, 1658 params->dst_iov, 0, 1659 &size, NULL, 0); 1660 1661 if (unlikely(size)) { 1662 CPT_LOG_DP_ERR("Insufficient buffer space," 1663 " size %d needed", size); 1664 return; 1665 } 1666 } 1667 1668 /* mac data */ 1669 if (mac_len) { 1670 i = fill_sg_comp_from_buf(scatter_comp, i, 1671 ¶ms->mac_buf); 1672 } 1673 } else { 1674 /* Output including mac */ 1675 size = outputlen - iv_len; 1676 if (size) { 1677 i = fill_sg_comp_from_iov(scatter_comp, i, 1678 params->dst_iov, 0, 1679 &size, NULL, 0); 1680 1681 if (unlikely(size)) { 1682 CPT_LOG_DP_ERR("Insufficient buffer space," 1683 " size %d needed", size); 1684 return; 1685 } 1686 } 1687 } 1688 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1689 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1690 1691 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1692 1693 /* This is DPTR len incase of SG mode */ 1694 vq_cmd_w0.s.dlen = size; 1695 1696 m_vaddr = (uint8_t *)m_vaddr + size; 1697 m_dma += size; 1698 1699 /* cpt alternate completion address saved earlier */ 1700 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1701 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1702 rptr_dma = c_dma - 8; 1703 1704 req->ist.ei1 = dptr_dma; 1705 req->ist.ei2 = rptr_dma; 1706 } 1707 1708 /* vq command w3 */ 1709 vq_cmd_w3.u64 = 0; 1710 vq_cmd_w3.s.grp = 0; 1711 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1712 offsetof(struct cpt_ctx, zs_ctx); 1713 1714 /* 16 byte aligned cpt res address */ 1715 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1716 *req->completion_addr = COMPLETION_CODE_INIT; 1717 req->comp_baddr = c_dma; 1718 1719 /* Fill microcode part of instruction */ 1720 req->ist.ei0 = vq_cmd_w0.u64; 1721 req->ist.ei3 = vq_cmd_w3.u64; 1722 1723 req->op = op; 1724 1725 *prep_req = req; 1726 return; 1727 } 1728 1729 static __rte_always_inline void 1730 cpt_zuc_snow3g_dec_prep(uint32_t req_flags, 1731 uint64_t d_offs, 1732 uint64_t d_lens, 1733 fc_params_t *params, 1734 void *op, 1735 void **prep_req) 1736 { 1737 uint32_t size; 1738 int32_t inputlen = 0, outputlen; 1739 struct cpt_ctx *cpt_ctx; 1740 uint8_t snow3g, iv_len = 16; 1741 struct cpt_request_info *req; 1742 buf_ptr_t *buf_p; 1743 uint32_t encr_offset; 1744 uint32_t encr_data_len; 1745 int flags; 1746 void *m_vaddr, *c_vaddr; 1747 uint64_t m_dma, c_dma; 1748 uint64_t *offset_vaddr, offset_dma; 1749 uint32_t *iv_s, iv[4], j; 1750 vq_cmd_word0_t vq_cmd_w0; 1751 vq_cmd_word3_t vq_cmd_w3; 1752 opcode_info_t opcode; 1753 1754 buf_p = ¶ms->meta_buf; 1755 m_vaddr = buf_p->vaddr; 1756 m_dma = buf_p->dma_addr; 1757 1758 /* 1759 * Microcode expects offsets in bytes 1760 * TODO: Rounding off 1761 */ 1762 encr_offset = ENCR_OFFSET(d_offs) / 8; 1763 encr_data_len = ENCR_DLEN(d_lens); 1764 1765 cpt_ctx = params->ctx_buf.vaddr; 1766 flags = cpt_ctx->zsk_flags; 1767 snow3g = cpt_ctx->snow3g; 1768 /* 1769 * Save initial space that followed app data for completion code & 1770 * alternate completion code to fall in same cache line as app data 1771 */ 1772 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1773 m_dma += COMPLETION_CODE_SIZE; 1774 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1775 (uint8_t *)m_vaddr; 1776 1777 c_vaddr = (uint8_t *)m_vaddr + size; 1778 c_dma = m_dma + size; 1779 size += sizeof(cpt_res_s_t); 1780 1781 m_vaddr = (uint8_t *)m_vaddr + size; 1782 m_dma += size; 1783 1784 /* Reserve memory for cpt request info */ 1785 req = m_vaddr; 1786 1787 size = sizeof(struct cpt_request_info); 1788 m_vaddr = (uint8_t *)m_vaddr + size; 1789 m_dma += size; 1790 1791 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1792 1793 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1794 opcode.s.minor = ((1 << 6) | (snow3g << 5) | (0 << 4) | 1795 (0 << 3) | (flags & 0x7)); 1796 1797 /* consider iv len */ 1798 encr_offset += iv_len; 1799 1800 inputlen = encr_offset + 1801 (RTE_ALIGN(encr_data_len, 8) / 8); 1802 outputlen = inputlen; 1803 1804 /* IV */ 1805 iv_s = params->iv_buf; 1806 if (snow3g) { 1807 /* 1808 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1809 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1810 */ 1811 1812 for (j = 0; j < 4; j++) 1813 iv[j] = iv_s[3 - j]; 1814 } else { 1815 /* ZUC doesn't need a swap */ 1816 for (j = 0; j < 4; j++) 1817 iv[j] = iv_s[j]; 1818 } 1819 1820 /* 1821 * GP op header, lengths are expected in bits. 1822 */ 1823 vq_cmd_w0.u64 = 0; 1824 vq_cmd_w0.s.param1 = encr_data_len; 1825 1826 /* 1827 * In 83XX since we have a limitation of 1828 * IV & Offset control word not part of instruction 1829 * and need to be part of Data Buffer, we check if 1830 * head room is there and then only do the Direct mode processing 1831 */ 1832 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1833 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1834 void *dm_vaddr = params->bufs[0].vaddr; 1835 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1836 /* 1837 * This flag indicates that there is 24 bytes head room and 1838 * 8 bytes tail room available, so that we get to do 1839 * DIRECT MODE with limitation 1840 */ 1841 1842 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1843 OFF_CTRL_LEN - iv_len); 1844 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1845 1846 /* DPTR */ 1847 req->ist.ei1 = offset_dma; 1848 /* RPTR should just exclude offset control word */ 1849 req->ist.ei2 = dm_dma_addr - iv_len; 1850 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1851 + outputlen - iv_len); 1852 1853 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1854 1855 vq_cmd_w0.s.opcode = opcode.flags; 1856 1857 if (likely(iv_len)) { 1858 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1859 + OFF_CTRL_LEN); 1860 memcpy(iv_d, iv, 16); 1861 } 1862 1863 /* iv offset is 0 */ 1864 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1865 } else { 1866 uint32_t i, g_size_bytes, s_size_bytes; 1867 uint64_t dptr_dma, rptr_dma; 1868 sg_comp_t *gather_comp; 1869 sg_comp_t *scatter_comp; 1870 uint8_t *in_buffer; 1871 uint32_t *iv_d; 1872 1873 /* save space for offset and iv... */ 1874 offset_vaddr = m_vaddr; 1875 offset_dma = m_dma; 1876 1877 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1878 m_dma += OFF_CTRL_LEN + iv_len; 1879 1880 opcode.s.major |= CPT_DMA_MODE; 1881 1882 vq_cmd_w0.s.opcode = opcode.flags; 1883 1884 /* DPTR has SG list */ 1885 in_buffer = m_vaddr; 1886 dptr_dma = m_dma; 1887 1888 ((uint16_t *)in_buffer)[0] = 0; 1889 ((uint16_t *)in_buffer)[1] = 0; 1890 1891 /* TODO Add error check if space will be sufficient */ 1892 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1893 1894 /* 1895 * Input Gather List 1896 */ 1897 i = 0; 1898 1899 /* Offset control word */ 1900 1901 /* iv offset is 0 */ 1902 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1903 1904 i = fill_sg_comp(gather_comp, i, offset_dma, 1905 OFF_CTRL_LEN + iv_len); 1906 1907 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1908 memcpy(iv_d, iv, 16); 1909 1910 /* Add input data */ 1911 size = inputlen - iv_len; 1912 if (size) { 1913 i = fill_sg_comp_from_iov(gather_comp, i, 1914 params->src_iov, 1915 0, &size, NULL, 0); 1916 if (unlikely(size)) { 1917 CPT_LOG_DP_ERR("Insufficient buffer space," 1918 " size %d needed", size); 1919 return; 1920 } 1921 } 1922 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1923 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1924 1925 /* 1926 * Output Scatter List 1927 */ 1928 1929 i = 0; 1930 scatter_comp = 1931 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1932 1933 /* IV */ 1934 i = fill_sg_comp(scatter_comp, i, 1935 offset_dma + OFF_CTRL_LEN, 1936 iv_len); 1937 1938 /* Add output data */ 1939 size = outputlen - iv_len; 1940 if (size) { 1941 i = fill_sg_comp_from_iov(scatter_comp, i, 1942 params->dst_iov, 0, 1943 &size, NULL, 0); 1944 1945 if (unlikely(size)) { 1946 CPT_LOG_DP_ERR("Insufficient buffer space," 1947 " size %d needed", size); 1948 return; 1949 } 1950 } 1951 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1952 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1953 1954 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1955 1956 /* This is DPTR len incase of SG mode */ 1957 vq_cmd_w0.s.dlen = size; 1958 1959 m_vaddr = (uint8_t *)m_vaddr + size; 1960 m_dma += size; 1961 1962 /* cpt alternate completion address saved earlier */ 1963 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1964 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1965 rptr_dma = c_dma - 8; 1966 1967 req->ist.ei1 = dptr_dma; 1968 req->ist.ei2 = rptr_dma; 1969 } 1970 1971 /* vq command w3 */ 1972 vq_cmd_w3.u64 = 0; 1973 vq_cmd_w3.s.grp = 0; 1974 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1975 offsetof(struct cpt_ctx, zs_ctx); 1976 1977 /* 16 byte aligned cpt res address */ 1978 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1979 *req->completion_addr = COMPLETION_CODE_INIT; 1980 req->comp_baddr = c_dma; 1981 1982 /* Fill microcode part of instruction */ 1983 req->ist.ei0 = vq_cmd_w0.u64; 1984 req->ist.ei3 = vq_cmd_w3.u64; 1985 1986 req->op = op; 1987 1988 *prep_req = req; 1989 return; 1990 } 1991 1992 static __rte_always_inline void 1993 cpt_kasumi_enc_prep(uint32_t req_flags, 1994 uint64_t d_offs, 1995 uint64_t d_lens, 1996 fc_params_t *params, 1997 void *op, 1998 void **prep_req) 1999 { 2000 uint32_t size; 2001 int32_t inputlen = 0, outputlen = 0; 2002 struct cpt_ctx *cpt_ctx; 2003 uint32_t mac_len = 0; 2004 uint8_t i = 0; 2005 struct cpt_request_info *req; 2006 buf_ptr_t *buf_p; 2007 uint32_t encr_offset, auth_offset; 2008 uint32_t encr_data_len, auth_data_len; 2009 int flags; 2010 uint8_t *iv_s, *iv_d, iv_len = 8; 2011 uint8_t dir = 0; 2012 void *m_vaddr, *c_vaddr; 2013 uint64_t m_dma, c_dma; 2014 uint64_t *offset_vaddr, offset_dma; 2015 vq_cmd_word0_t vq_cmd_w0; 2016 vq_cmd_word3_t vq_cmd_w3; 2017 opcode_info_t opcode; 2018 uint8_t *in_buffer; 2019 uint32_t g_size_bytes, s_size_bytes; 2020 uint64_t dptr_dma, rptr_dma; 2021 sg_comp_t *gather_comp; 2022 sg_comp_t *scatter_comp; 2023 2024 buf_p = ¶ms->meta_buf; 2025 m_vaddr = buf_p->vaddr; 2026 m_dma = buf_p->dma_addr; 2027 2028 encr_offset = ENCR_OFFSET(d_offs) / 8; 2029 auth_offset = AUTH_OFFSET(d_offs) / 8; 2030 encr_data_len = ENCR_DLEN(d_lens); 2031 auth_data_len = AUTH_DLEN(d_lens); 2032 2033 cpt_ctx = params->ctx_buf.vaddr; 2034 flags = cpt_ctx->zsk_flags; 2035 mac_len = cpt_ctx->mac_len; 2036 2037 if (flags == 0x0) 2038 iv_s = params->iv_buf; 2039 else 2040 iv_s = params->auth_iv_buf; 2041 2042 dir = iv_s[8] & 0x1; 2043 2044 /* 2045 * Save initial space that followed app data for completion code & 2046 * alternate completion code to fall in same cache line as app data 2047 */ 2048 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2049 m_dma += COMPLETION_CODE_SIZE; 2050 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2051 (uint8_t *)m_vaddr; 2052 2053 c_vaddr = (uint8_t *)m_vaddr + size; 2054 c_dma = m_dma + size; 2055 size += sizeof(cpt_res_s_t); 2056 2057 m_vaddr = (uint8_t *)m_vaddr + size; 2058 m_dma += size; 2059 2060 /* Reserve memory for cpt request info */ 2061 req = m_vaddr; 2062 2063 size = sizeof(struct cpt_request_info); 2064 m_vaddr = (uint8_t *)m_vaddr + size; 2065 m_dma += size; 2066 2067 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2068 2069 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2070 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2071 (dir << 4) | (0 << 3) | (flags & 0x7)); 2072 2073 /* 2074 * GP op header, lengths are expected in bits. 2075 */ 2076 vq_cmd_w0.u64 = 0; 2077 vq_cmd_w0.s.param1 = encr_data_len; 2078 vq_cmd_w0.s.param2 = auth_data_len; 2079 vq_cmd_w0.s.opcode = opcode.flags; 2080 2081 /* consider iv len */ 2082 if (flags == 0x0) { 2083 encr_offset += iv_len; 2084 auth_offset += iv_len; 2085 } 2086 2087 /* save space for offset ctrl and iv */ 2088 offset_vaddr = m_vaddr; 2089 offset_dma = m_dma; 2090 2091 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2092 m_dma += OFF_CTRL_LEN + iv_len; 2093 2094 /* DPTR has SG list */ 2095 in_buffer = m_vaddr; 2096 dptr_dma = m_dma; 2097 2098 ((uint16_t *)in_buffer)[0] = 0; 2099 ((uint16_t *)in_buffer)[1] = 0; 2100 2101 /* TODO Add error check if space will be sufficient */ 2102 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2103 2104 /* 2105 * Input Gather List 2106 */ 2107 i = 0; 2108 2109 /* Offset control word followed by iv */ 2110 2111 if (flags == 0x0) { 2112 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 2113 outputlen = inputlen; 2114 /* iv offset is 0 */ 2115 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2116 } else { 2117 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 2118 outputlen = mac_len; 2119 /* iv offset is 0 */ 2120 *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset); 2121 } 2122 2123 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2124 2125 /* IV */ 2126 iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN; 2127 memcpy(iv_d, iv_s, iv_len); 2128 2129 /* input data */ 2130 size = inputlen - iv_len; 2131 if (size) { 2132 i = fill_sg_comp_from_iov(gather_comp, i, 2133 params->src_iov, 0, 2134 &size, NULL, 0); 2135 2136 if (unlikely(size)) { 2137 CPT_LOG_DP_ERR("Insufficient buffer space," 2138 " size %d needed", size); 2139 return; 2140 } 2141 } 2142 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2143 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2144 2145 /* 2146 * Output Scatter List 2147 */ 2148 2149 i = 0; 2150 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2151 2152 if (flags == 0x1) { 2153 /* IV in SLIST only for F8 */ 2154 iv_len = 0; 2155 } 2156 2157 /* IV */ 2158 if (iv_len) { 2159 i = fill_sg_comp(scatter_comp, i, 2160 offset_dma + OFF_CTRL_LEN, 2161 iv_len); 2162 } 2163 2164 /* Add output data */ 2165 if (req_flags & VALID_MAC_BUF) { 2166 size = outputlen - iv_len - mac_len; 2167 if (size) { 2168 i = fill_sg_comp_from_iov(scatter_comp, i, 2169 params->dst_iov, 0, 2170 &size, NULL, 0); 2171 2172 if (unlikely(size)) { 2173 CPT_LOG_DP_ERR("Insufficient buffer space," 2174 " size %d needed", size); 2175 return; 2176 } 2177 } 2178 2179 /* mac data */ 2180 if (mac_len) { 2181 i = fill_sg_comp_from_buf(scatter_comp, i, 2182 ¶ms->mac_buf); 2183 } 2184 } else { 2185 /* Output including mac */ 2186 size = outputlen - iv_len; 2187 if (size) { 2188 i = fill_sg_comp_from_iov(scatter_comp, i, 2189 params->dst_iov, 0, 2190 &size, NULL, 0); 2191 2192 if (unlikely(size)) { 2193 CPT_LOG_DP_ERR("Insufficient buffer space," 2194 " size %d needed", size); 2195 return; 2196 } 2197 } 2198 } 2199 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2200 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2201 2202 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2203 2204 /* This is DPTR len incase of SG mode */ 2205 vq_cmd_w0.s.dlen = size; 2206 2207 m_vaddr = (uint8_t *)m_vaddr + size; 2208 m_dma += size; 2209 2210 /* cpt alternate completion address saved earlier */ 2211 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2212 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2213 rptr_dma = c_dma - 8; 2214 2215 req->ist.ei1 = dptr_dma; 2216 req->ist.ei2 = rptr_dma; 2217 2218 /* vq command w3 */ 2219 vq_cmd_w3.u64 = 0; 2220 vq_cmd_w3.s.grp = 0; 2221 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2222 offsetof(struct cpt_ctx, k_ctx); 2223 2224 /* 16 byte aligned cpt res address */ 2225 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2226 *req->completion_addr = COMPLETION_CODE_INIT; 2227 req->comp_baddr = c_dma; 2228 2229 /* Fill microcode part of instruction */ 2230 req->ist.ei0 = vq_cmd_w0.u64; 2231 req->ist.ei3 = vq_cmd_w3.u64; 2232 2233 req->op = op; 2234 2235 *prep_req = req; 2236 return; 2237 } 2238 2239 static __rte_always_inline void 2240 cpt_kasumi_dec_prep(uint64_t d_offs, 2241 uint64_t d_lens, 2242 fc_params_t *params, 2243 void *op, 2244 void **prep_req) 2245 { 2246 uint32_t size; 2247 int32_t inputlen = 0, outputlen; 2248 struct cpt_ctx *cpt_ctx; 2249 uint8_t i = 0, iv_len = 8; 2250 struct cpt_request_info *req; 2251 buf_ptr_t *buf_p; 2252 uint32_t encr_offset; 2253 uint32_t encr_data_len; 2254 int flags; 2255 uint8_t dir = 0; 2256 void *m_vaddr, *c_vaddr; 2257 uint64_t m_dma, c_dma; 2258 uint64_t *offset_vaddr, offset_dma; 2259 vq_cmd_word0_t vq_cmd_w0; 2260 vq_cmd_word3_t vq_cmd_w3; 2261 opcode_info_t opcode; 2262 uint8_t *in_buffer; 2263 uint32_t g_size_bytes, s_size_bytes; 2264 uint64_t dptr_dma, rptr_dma; 2265 sg_comp_t *gather_comp; 2266 sg_comp_t *scatter_comp; 2267 2268 buf_p = ¶ms->meta_buf; 2269 m_vaddr = buf_p->vaddr; 2270 m_dma = buf_p->dma_addr; 2271 2272 encr_offset = ENCR_OFFSET(d_offs) / 8; 2273 encr_data_len = ENCR_DLEN(d_lens); 2274 2275 cpt_ctx = params->ctx_buf.vaddr; 2276 flags = cpt_ctx->zsk_flags; 2277 /* 2278 * Save initial space that followed app data for completion code & 2279 * alternate completion code to fall in same cache line as app data 2280 */ 2281 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2282 m_dma += COMPLETION_CODE_SIZE; 2283 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2284 (uint8_t *)m_vaddr; 2285 2286 c_vaddr = (uint8_t *)m_vaddr + size; 2287 c_dma = m_dma + size; 2288 size += sizeof(cpt_res_s_t); 2289 2290 m_vaddr = (uint8_t *)m_vaddr + size; 2291 m_dma += size; 2292 2293 /* Reserve memory for cpt request info */ 2294 req = m_vaddr; 2295 2296 size = sizeof(struct cpt_request_info); 2297 m_vaddr = (uint8_t *)m_vaddr + size; 2298 m_dma += size; 2299 2300 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2301 2302 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2303 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2304 (dir << 4) | (0 << 3) | (flags & 0x7)); 2305 2306 /* 2307 * GP op header, lengths are expected in bits. 2308 */ 2309 vq_cmd_w0.u64 = 0; 2310 vq_cmd_w0.s.param1 = encr_data_len; 2311 vq_cmd_w0.s.opcode = opcode.flags; 2312 2313 /* consider iv len */ 2314 encr_offset += iv_len; 2315 2316 inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8); 2317 outputlen = inputlen; 2318 2319 /* save space for offset ctrl & iv */ 2320 offset_vaddr = m_vaddr; 2321 offset_dma = m_dma; 2322 2323 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2324 m_dma += OFF_CTRL_LEN + iv_len; 2325 2326 /* DPTR has SG list */ 2327 in_buffer = m_vaddr; 2328 dptr_dma = m_dma; 2329 2330 ((uint16_t *)in_buffer)[0] = 0; 2331 ((uint16_t *)in_buffer)[1] = 0; 2332 2333 /* TODO Add error check if space will be sufficient */ 2334 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2335 2336 /* 2337 * Input Gather List 2338 */ 2339 i = 0; 2340 2341 /* Offset control word followed by iv */ 2342 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2343 2344 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2345 2346 /* IV */ 2347 memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN, 2348 params->iv_buf, iv_len); 2349 2350 /* Add input data */ 2351 size = inputlen - iv_len; 2352 if (size) { 2353 i = fill_sg_comp_from_iov(gather_comp, i, 2354 params->src_iov, 2355 0, &size, NULL, 0); 2356 if (unlikely(size)) { 2357 CPT_LOG_DP_ERR("Insufficient buffer space," 2358 " size %d needed", size); 2359 return; 2360 } 2361 } 2362 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2363 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2364 2365 /* 2366 * Output Scatter List 2367 */ 2368 2369 i = 0; 2370 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2371 2372 /* IV */ 2373 i = fill_sg_comp(scatter_comp, i, 2374 offset_dma + OFF_CTRL_LEN, 2375 iv_len); 2376 2377 /* Add output data */ 2378 size = outputlen - iv_len; 2379 if (size) { 2380 i = fill_sg_comp_from_iov(scatter_comp, i, 2381 params->dst_iov, 0, 2382 &size, NULL, 0); 2383 if (unlikely(size)) { 2384 CPT_LOG_DP_ERR("Insufficient buffer space," 2385 " size %d needed", size); 2386 return; 2387 } 2388 } 2389 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2390 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2391 2392 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2393 2394 /* This is DPTR len incase of SG mode */ 2395 vq_cmd_w0.s.dlen = size; 2396 2397 m_vaddr = (uint8_t *)m_vaddr + size; 2398 m_dma += size; 2399 2400 /* cpt alternate completion address saved earlier */ 2401 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2402 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2403 rptr_dma = c_dma - 8; 2404 2405 req->ist.ei1 = dptr_dma; 2406 req->ist.ei2 = rptr_dma; 2407 2408 /* vq command w3 */ 2409 vq_cmd_w3.u64 = 0; 2410 vq_cmd_w3.s.grp = 0; 2411 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2412 offsetof(struct cpt_ctx, k_ctx); 2413 2414 /* 16 byte aligned cpt res address */ 2415 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2416 *req->completion_addr = COMPLETION_CODE_INIT; 2417 req->comp_baddr = c_dma; 2418 2419 /* Fill microcode part of instruction */ 2420 req->ist.ei0 = vq_cmd_w0.u64; 2421 req->ist.ei3 = vq_cmd_w3.u64; 2422 2423 req->op = op; 2424 2425 *prep_req = req; 2426 return; 2427 } 2428 2429 static __rte_always_inline void * 2430 cpt_fc_dec_hmac_prep(uint32_t flags, 2431 uint64_t d_offs, 2432 uint64_t d_lens, 2433 fc_params_t *fc_params, 2434 void *op) 2435 { 2436 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2437 uint8_t fc_type; 2438 void *prep_req = NULL; 2439 2440 fc_type = ctx->fc_type; 2441 2442 if (likely(fc_type == FC_GEN)) { 2443 cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2444 &prep_req); 2445 } else if (fc_type == ZUC_SNOW3G) { 2446 cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, op, 2447 &prep_req); 2448 } else if (fc_type == KASUMI) { 2449 cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, &prep_req); 2450 } 2451 2452 /* 2453 * For AUTH_ONLY case, 2454 * MC only supports digest generation and verification 2455 * should be done in software by memcmp() 2456 */ 2457 2458 return prep_req; 2459 } 2460 2461 static __rte_always_inline void *__hot 2462 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, 2463 fc_params_t *fc_params, void *op) 2464 { 2465 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2466 uint8_t fc_type; 2467 void *prep_req = NULL; 2468 2469 fc_type = ctx->fc_type; 2470 2471 /* Common api for rest of the ops */ 2472 if (likely(fc_type == FC_GEN)) { 2473 cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2474 &prep_req); 2475 } else if (fc_type == ZUC_SNOW3G) { 2476 cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, op, 2477 &prep_req); 2478 } else if (fc_type == KASUMI) { 2479 cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, op, 2480 &prep_req); 2481 } else if (fc_type == HASH_HMAC) { 2482 cpt_digest_gen_prep(flags, d_lens, fc_params, op, &prep_req); 2483 } 2484 2485 return prep_req; 2486 } 2487 2488 static __rte_always_inline int 2489 cpt_fc_auth_set_key(void *ctx, auth_type_t type, const uint8_t *key, 2490 uint16_t key_len, uint16_t mac_len) 2491 { 2492 struct cpt_ctx *cpt_ctx = ctx; 2493 mc_fc_context_t *fctx = &cpt_ctx->fctx; 2494 uint64_t *ctrl_flags = NULL; 2495 2496 if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) { 2497 uint32_t keyx[4]; 2498 2499 if (key_len != 16) 2500 return -1; 2501 /* No support for AEAD yet */ 2502 if (cpt_ctx->enc_cipher) 2503 return -1; 2504 /* For ZUC/SNOW3G/Kasumi */ 2505 switch (type) { 2506 case SNOW3G_UIA2: 2507 cpt_ctx->snow3g = 1; 2508 gen_key_snow3g(key, keyx); 2509 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 2510 cpt_ctx->fc_type = ZUC_SNOW3G; 2511 cpt_ctx->zsk_flags = 0x1; 2512 break; 2513 case ZUC_EIA3: 2514 cpt_ctx->snow3g = 0; 2515 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 2516 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 2517 cpt_ctx->fc_type = ZUC_SNOW3G; 2518 cpt_ctx->zsk_flags = 0x1; 2519 break; 2520 case KASUMI_F9_ECB: 2521 /* Kasumi ECB mode */ 2522 cpt_ctx->k_ecb = 1; 2523 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2524 cpt_ctx->fc_type = KASUMI; 2525 cpt_ctx->zsk_flags = 0x1; 2526 break; 2527 case KASUMI_F9_CBC: 2528 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2529 cpt_ctx->fc_type = KASUMI; 2530 cpt_ctx->zsk_flags = 0x1; 2531 break; 2532 default: 2533 return -1; 2534 } 2535 cpt_ctx->mac_len = 4; 2536 cpt_ctx->hash_type = type; 2537 return 0; 2538 } 2539 2540 if (!(cpt_ctx->fc_type == FC_GEN && !type)) { 2541 if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher) 2542 cpt_ctx->fc_type = HASH_HMAC; 2543 } 2544 2545 ctrl_flags = (uint64_t *)&fctx->enc.enc_ctrl.flags; 2546 *ctrl_flags = rte_be_to_cpu_64(*ctrl_flags); 2547 2548 /* For GMAC auth, cipher must be NULL */ 2549 if (type == GMAC_TYPE) 2550 CPT_P_ENC_CTRL(fctx).enc_cipher = 0; 2551 2552 CPT_P_ENC_CTRL(fctx).hash_type = cpt_ctx->hash_type = type; 2553 CPT_P_ENC_CTRL(fctx).mac_len = cpt_ctx->mac_len = mac_len; 2554 2555 if (key_len) { 2556 cpt_ctx->hmac = 1; 2557 memset(cpt_ctx->auth_key, 0, sizeof(cpt_ctx->auth_key)); 2558 memcpy(cpt_ctx->auth_key, key, key_len); 2559 cpt_ctx->auth_key_len = key_len; 2560 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 2561 memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad)); 2562 memcpy(fctx->hmac.opad, key, key_len); 2563 CPT_P_ENC_CTRL(fctx).auth_input_type = 1; 2564 } 2565 *ctrl_flags = rte_cpu_to_be_64(*ctrl_flags); 2566 return 0; 2567 } 2568 2569 static __rte_always_inline int 2570 fill_sess_aead(struct rte_crypto_sym_xform *xform, 2571 struct cpt_sess_misc *sess) 2572 { 2573 struct rte_crypto_aead_xform *aead_form; 2574 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2575 auth_type_t auth_type = 0; /* NULL Auth type */ 2576 uint32_t cipher_key_len = 0; 2577 uint8_t aes_gcm = 0; 2578 aead_form = &xform->aead; 2579 void *ctx = SESS_PRIV(sess); 2580 2581 if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT && 2582 aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) { 2583 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2584 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2585 } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT && 2586 aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) { 2587 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2588 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2589 } else { 2590 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2591 return -1; 2592 } 2593 switch (aead_form->algo) { 2594 case RTE_CRYPTO_AEAD_AES_GCM: 2595 enc_type = AES_GCM; 2596 cipher_key_len = 16; 2597 aes_gcm = 1; 2598 break; 2599 case RTE_CRYPTO_AEAD_AES_CCM: 2600 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2601 aead_form->algo); 2602 return -1; 2603 default: 2604 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2605 aead_form->algo); 2606 return -1; 2607 } 2608 if (aead_form->key.length < cipher_key_len) { 2609 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2610 (unsigned int long)aead_form->key.length); 2611 return -1; 2612 } 2613 sess->zsk_flag = 0; 2614 sess->aes_gcm = aes_gcm; 2615 sess->mac_len = aead_form->digest_length; 2616 sess->iv_offset = aead_form->iv.offset; 2617 sess->iv_length = aead_form->iv.length; 2618 sess->aad_length = aead_form->aad_length; 2619 2620 cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data, 2621 aead_form->key.length, NULL); 2622 2623 cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, aead_form->digest_length); 2624 2625 return 0; 2626 } 2627 2628 static __rte_always_inline int 2629 fill_sess_cipher(struct rte_crypto_sym_xform *xform, 2630 struct cpt_sess_misc *sess) 2631 { 2632 struct rte_crypto_cipher_xform *c_form; 2633 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2634 uint32_t cipher_key_len = 0; 2635 uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0; 2636 2637 c_form = &xform->cipher; 2638 2639 if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2640 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2641 else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT) 2642 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2643 else { 2644 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2645 return -1; 2646 } 2647 2648 switch (c_form->algo) { 2649 case RTE_CRYPTO_CIPHER_AES_CBC: 2650 enc_type = AES_CBC; 2651 cipher_key_len = 16; 2652 break; 2653 case RTE_CRYPTO_CIPHER_3DES_CBC: 2654 enc_type = DES3_CBC; 2655 cipher_key_len = 24; 2656 break; 2657 case RTE_CRYPTO_CIPHER_DES_CBC: 2658 /* DES is implemented using 3DES in hardware */ 2659 enc_type = DES3_CBC; 2660 cipher_key_len = 8; 2661 break; 2662 case RTE_CRYPTO_CIPHER_AES_CTR: 2663 enc_type = AES_CTR; 2664 cipher_key_len = 16; 2665 aes_ctr = 1; 2666 break; 2667 case RTE_CRYPTO_CIPHER_NULL: 2668 enc_type = 0; 2669 is_null = 1; 2670 break; 2671 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2672 enc_type = KASUMI_F8_ECB; 2673 cipher_key_len = 16; 2674 zsk_flag = K_F8; 2675 break; 2676 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2677 enc_type = SNOW3G_UEA2; 2678 cipher_key_len = 16; 2679 zsk_flag = ZS_EA; 2680 break; 2681 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 2682 enc_type = ZUC_EEA3; 2683 cipher_key_len = 16; 2684 zsk_flag = ZS_EA; 2685 break; 2686 case RTE_CRYPTO_CIPHER_AES_XTS: 2687 enc_type = AES_XTS; 2688 cipher_key_len = 16; 2689 break; 2690 case RTE_CRYPTO_CIPHER_3DES_ECB: 2691 enc_type = DES3_ECB; 2692 cipher_key_len = 24; 2693 break; 2694 case RTE_CRYPTO_CIPHER_AES_ECB: 2695 enc_type = AES_ECB; 2696 cipher_key_len = 16; 2697 break; 2698 case RTE_CRYPTO_CIPHER_3DES_CTR: 2699 case RTE_CRYPTO_CIPHER_AES_F8: 2700 case RTE_CRYPTO_CIPHER_ARC4: 2701 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2702 c_form->algo); 2703 return -1; 2704 default: 2705 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2706 c_form->algo); 2707 return -1; 2708 } 2709 2710 if (c_form->key.length < cipher_key_len) { 2711 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2712 (unsigned long) c_form->key.length); 2713 return -1; 2714 } 2715 2716 sess->zsk_flag = zsk_flag; 2717 sess->aes_gcm = 0; 2718 sess->aes_ctr = aes_ctr; 2719 sess->iv_offset = c_form->iv.offset; 2720 sess->iv_length = c_form->iv.length; 2721 sess->is_null = is_null; 2722 2723 cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type, c_form->key.data, 2724 c_form->key.length, NULL); 2725 2726 return 0; 2727 } 2728 2729 static __rte_always_inline int 2730 fill_sess_auth(struct rte_crypto_sym_xform *xform, 2731 struct cpt_sess_misc *sess) 2732 { 2733 struct rte_crypto_auth_xform *a_form; 2734 auth_type_t auth_type = 0; /* NULL Auth type */ 2735 uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0; 2736 2737 a_form = &xform->auth; 2738 2739 if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2740 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2741 else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2742 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2743 else { 2744 CPT_LOG_DP_ERR("Unknown auth operation"); 2745 return -1; 2746 } 2747 2748 if (a_form->key.length > 64) { 2749 CPT_LOG_DP_ERR("Auth key length is big"); 2750 return -1; 2751 } 2752 2753 switch (a_form->algo) { 2754 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2755 /* Fall through */ 2756 case RTE_CRYPTO_AUTH_SHA1: 2757 auth_type = SHA1_TYPE; 2758 break; 2759 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2760 case RTE_CRYPTO_AUTH_SHA256: 2761 auth_type = SHA2_SHA256; 2762 break; 2763 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2764 case RTE_CRYPTO_AUTH_SHA512: 2765 auth_type = SHA2_SHA512; 2766 break; 2767 case RTE_CRYPTO_AUTH_AES_GMAC: 2768 auth_type = GMAC_TYPE; 2769 aes_gcm = 1; 2770 break; 2771 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2772 case RTE_CRYPTO_AUTH_SHA224: 2773 auth_type = SHA2_SHA224; 2774 break; 2775 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2776 case RTE_CRYPTO_AUTH_SHA384: 2777 auth_type = SHA2_SHA384; 2778 break; 2779 case RTE_CRYPTO_AUTH_MD5_HMAC: 2780 case RTE_CRYPTO_AUTH_MD5: 2781 auth_type = MD5_TYPE; 2782 break; 2783 case RTE_CRYPTO_AUTH_KASUMI_F9: 2784 auth_type = KASUMI_F9_ECB; 2785 /* 2786 * Indicate that direction needs to be taken out 2787 * from end of src 2788 */ 2789 zsk_flag = K_F9; 2790 break; 2791 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2792 auth_type = SNOW3G_UIA2; 2793 zsk_flag = ZS_IA; 2794 break; 2795 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2796 auth_type = ZUC_EIA3; 2797 zsk_flag = ZS_IA; 2798 break; 2799 case RTE_CRYPTO_AUTH_NULL: 2800 auth_type = 0; 2801 is_null = 1; 2802 break; 2803 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2804 case RTE_CRYPTO_AUTH_AES_CMAC: 2805 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2806 CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u", 2807 a_form->algo); 2808 return -1; 2809 default: 2810 CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified", 2811 a_form->algo); 2812 return -1; 2813 } 2814 2815 sess->zsk_flag = zsk_flag; 2816 sess->aes_gcm = aes_gcm; 2817 sess->mac_len = a_form->digest_length; 2818 sess->is_null = is_null; 2819 if (zsk_flag) { 2820 sess->auth_iv_offset = a_form->iv.offset; 2821 sess->auth_iv_length = a_form->iv.length; 2822 } 2823 cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type, a_form->key.data, 2824 a_form->key.length, a_form->digest_length); 2825 2826 return 0; 2827 } 2828 2829 static __rte_always_inline int 2830 fill_sess_gmac(struct rte_crypto_sym_xform *xform, 2831 struct cpt_sess_misc *sess) 2832 { 2833 struct rte_crypto_auth_xform *a_form; 2834 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2835 auth_type_t auth_type = 0; /* NULL Auth type */ 2836 void *ctx = SESS_PRIV(sess); 2837 2838 a_form = &xform->auth; 2839 2840 if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2841 sess->cpt_op |= CPT_OP_ENCODE; 2842 else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2843 sess->cpt_op |= CPT_OP_DECODE; 2844 else { 2845 CPT_LOG_DP_ERR("Unknown auth operation"); 2846 return -1; 2847 } 2848 2849 switch (a_form->algo) { 2850 case RTE_CRYPTO_AUTH_AES_GMAC: 2851 enc_type = AES_GCM; 2852 auth_type = GMAC_TYPE; 2853 break; 2854 default: 2855 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2856 a_form->algo); 2857 return -1; 2858 } 2859 2860 sess->zsk_flag = 0; 2861 sess->aes_gcm = 0; 2862 sess->is_gmac = 1; 2863 sess->iv_offset = a_form->iv.offset; 2864 sess->iv_length = a_form->iv.length; 2865 sess->mac_len = a_form->digest_length; 2866 2867 cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data, 2868 a_form->key.length, NULL); 2869 cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, a_form->digest_length); 2870 2871 return 0; 2872 } 2873 2874 static __rte_always_inline void * 2875 alloc_op_meta(struct rte_mbuf *m_src, 2876 buf_ptr_t *buf, 2877 int32_t len, 2878 struct rte_mempool *cpt_meta_pool) 2879 { 2880 uint8_t *mdata; 2881 2882 #ifndef CPT_ALWAYS_USE_SEPARATE_BUF 2883 if (likely(m_src && (m_src->nb_segs == 1))) { 2884 int32_t tailroom; 2885 phys_addr_t mphys; 2886 2887 /* Check if tailroom is sufficient to hold meta data */ 2888 tailroom = rte_pktmbuf_tailroom(m_src); 2889 if (likely(tailroom > len + 8)) { 2890 mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len; 2891 mphys = m_src->buf_physaddr + m_src->buf_len; 2892 mdata -= len; 2893 mphys -= len; 2894 buf->vaddr = mdata; 2895 buf->dma_addr = mphys; 2896 buf->size = len; 2897 /* Indicate that this is a mbuf allocated mdata */ 2898 mdata = (uint8_t *)((uint64_t)mdata | 1ull); 2899 return mdata; 2900 } 2901 } 2902 #else 2903 RTE_SET_USED(m_src); 2904 #endif 2905 2906 if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0)) 2907 return NULL; 2908 2909 buf->vaddr = mdata; 2910 buf->dma_addr = rte_mempool_virt2iova(mdata); 2911 buf->size = len; 2912 2913 return mdata; 2914 } 2915 2916 /** 2917 * cpt_free_metabuf - free metabuf to mempool. 2918 * @param instance: pointer to instance. 2919 * @param objp: pointer to the metabuf. 2920 */ 2921 static __rte_always_inline void 2922 free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool) 2923 { 2924 bool nofree = ((uintptr_t)mdata & 1ull); 2925 2926 if (likely(nofree)) 2927 return; 2928 rte_mempool_put(cpt_meta_pool, mdata); 2929 } 2930 2931 static __rte_always_inline uint32_t 2932 prepare_iov_from_pkt(struct rte_mbuf *pkt, 2933 iov_ptr_t *iovec, uint32_t start_offset) 2934 { 2935 uint16_t index = 0; 2936 void *seg_data = NULL; 2937 phys_addr_t seg_phys; 2938 int32_t seg_size = 0; 2939 2940 if (!pkt) { 2941 iovec->buf_cnt = 0; 2942 return 0; 2943 } 2944 2945 if (!start_offset) { 2946 seg_data = rte_pktmbuf_mtod(pkt, void *); 2947 seg_phys = rte_pktmbuf_mtophys(pkt); 2948 seg_size = pkt->data_len; 2949 } else { 2950 while (start_offset >= pkt->data_len) { 2951 start_offset -= pkt->data_len; 2952 pkt = pkt->next; 2953 } 2954 2955 seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset); 2956 seg_phys = rte_pktmbuf_mtophys_offset(pkt, start_offset); 2957 seg_size = pkt->data_len - start_offset; 2958 if (!seg_size) 2959 return 1; 2960 } 2961 2962 /* first seg */ 2963 iovec->bufs[index].vaddr = seg_data; 2964 iovec->bufs[index].dma_addr = seg_phys; 2965 iovec->bufs[index].size = seg_size; 2966 index++; 2967 pkt = pkt->next; 2968 2969 while (unlikely(pkt != NULL)) { 2970 seg_data = rte_pktmbuf_mtod(pkt, void *); 2971 seg_phys = rte_pktmbuf_mtophys(pkt); 2972 seg_size = pkt->data_len; 2973 if (!seg_size) 2974 break; 2975 2976 iovec->bufs[index].vaddr = seg_data; 2977 iovec->bufs[index].dma_addr = seg_phys; 2978 iovec->bufs[index].size = seg_size; 2979 2980 index++; 2981 2982 pkt = pkt->next; 2983 } 2984 2985 iovec->buf_cnt = index; 2986 return 0; 2987 } 2988 2989 static __rte_always_inline uint32_t 2990 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt, 2991 fc_params_t *param, 2992 uint32_t *flags) 2993 { 2994 uint16_t index = 0; 2995 void *seg_data = NULL; 2996 phys_addr_t seg_phys; 2997 uint32_t seg_size = 0; 2998 iov_ptr_t *iovec; 2999 3000 seg_data = rte_pktmbuf_mtod(pkt, void *); 3001 seg_phys = rte_pktmbuf_mtophys(pkt); 3002 seg_size = pkt->data_len; 3003 3004 /* first seg */ 3005 if (likely(!pkt->next)) { 3006 uint32_t headroom, tailroom; 3007 3008 *flags |= SINGLE_BUF_INPLACE; 3009 headroom = rte_pktmbuf_headroom(pkt); 3010 tailroom = rte_pktmbuf_tailroom(pkt); 3011 if (likely((headroom >= 24) && 3012 (tailroom >= 8))) { 3013 /* In 83XX this is prerequivisit for Direct mode */ 3014 *flags |= SINGLE_BUF_HEADTAILROOM; 3015 } 3016 param->bufs[0].vaddr = seg_data; 3017 param->bufs[0].dma_addr = seg_phys; 3018 param->bufs[0].size = seg_size; 3019 return 0; 3020 } 3021 iovec = param->src_iov; 3022 iovec->bufs[index].vaddr = seg_data; 3023 iovec->bufs[index].dma_addr = seg_phys; 3024 iovec->bufs[index].size = seg_size; 3025 index++; 3026 pkt = pkt->next; 3027 3028 while (unlikely(pkt != NULL)) { 3029 seg_data = rte_pktmbuf_mtod(pkt, void *); 3030 seg_phys = rte_pktmbuf_mtophys(pkt); 3031 seg_size = pkt->data_len; 3032 3033 if (!seg_size) 3034 break; 3035 3036 iovec->bufs[index].vaddr = seg_data; 3037 iovec->bufs[index].dma_addr = seg_phys; 3038 iovec->bufs[index].size = seg_size; 3039 3040 index++; 3041 3042 pkt = pkt->next; 3043 } 3044 3045 iovec->buf_cnt = index; 3046 return 0; 3047 } 3048 3049 static __rte_always_inline int 3050 fill_fc_params(struct rte_crypto_op *cop, 3051 struct cpt_sess_misc *sess_misc, 3052 struct cpt_qp_meta_info *m_info, 3053 void **mdata_ptr, 3054 void **prep_req) 3055 { 3056 uint32_t space = 0; 3057 struct rte_crypto_sym_op *sym_op = cop->sym; 3058 void *mdata = NULL; 3059 uintptr_t *op; 3060 uint32_t mc_hash_off; 3061 uint32_t flags = 0; 3062 uint64_t d_offs, d_lens; 3063 struct rte_mbuf *m_src, *m_dst; 3064 uint8_t cpt_op = sess_misc->cpt_op; 3065 #ifdef CPT_ALWAYS_USE_SG_MODE 3066 uint8_t inplace = 0; 3067 #else 3068 uint8_t inplace = 1; 3069 #endif 3070 fc_params_t fc_params; 3071 char src[SRC_IOV_SIZE]; 3072 char dst[SRC_IOV_SIZE]; 3073 uint32_t iv_buf[4]; 3074 int ret; 3075 3076 if (likely(sess_misc->iv_length)) { 3077 flags |= VALID_IV_BUF; 3078 fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, 3079 uint8_t *, sess_misc->iv_offset); 3080 if (sess_misc->aes_ctr && 3081 unlikely(sess_misc->iv_length != 16)) { 3082 memcpy((uint8_t *)iv_buf, 3083 rte_crypto_op_ctod_offset(cop, 3084 uint8_t *, sess_misc->iv_offset), 12); 3085 iv_buf[3] = rte_cpu_to_be_32(0x1); 3086 fc_params.iv_buf = iv_buf; 3087 } 3088 } 3089 3090 if (sess_misc->zsk_flag) { 3091 fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3092 uint8_t *, 3093 sess_misc->auth_iv_offset); 3094 if (sess_misc->zsk_flag != ZS_EA) 3095 inplace = 0; 3096 } 3097 m_src = sym_op->m_src; 3098 m_dst = sym_op->m_dst; 3099 3100 if (sess_misc->aes_gcm) { 3101 uint8_t *salt; 3102 uint8_t *aad_data; 3103 uint16_t aad_len; 3104 3105 d_offs = sym_op->aead.data.offset; 3106 d_lens = sym_op->aead.data.length; 3107 mc_hash_off = sym_op->aead.data.offset + 3108 sym_op->aead.data.length; 3109 3110 aad_data = sym_op->aead.aad.data; 3111 aad_len = sess_misc->aad_length; 3112 if (likely((aad_data + aad_len) == 3113 rte_pktmbuf_mtod_offset(m_src, 3114 uint8_t *, 3115 sym_op->aead.data.offset))) { 3116 d_offs = (d_offs - aad_len) | (d_offs << 16); 3117 d_lens = (d_lens + aad_len) | (d_lens << 32); 3118 } else { 3119 fc_params.aad_buf.vaddr = sym_op->aead.aad.data; 3120 fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr; 3121 fc_params.aad_buf.size = aad_len; 3122 flags |= VALID_AAD_BUF; 3123 inplace = 0; 3124 d_offs = d_offs << 16; 3125 d_lens = d_lens << 32; 3126 } 3127 3128 salt = fc_params.iv_buf; 3129 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3130 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3131 sess_misc->salt = *(uint32_t *)salt; 3132 } 3133 fc_params.iv_buf = salt + 4; 3134 if (likely(sess_misc->mac_len)) { 3135 struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst : 3136 m_src; 3137 3138 if (!m) 3139 m = m_src; 3140 3141 /* hmac immediately following data is best case */ 3142 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3143 mc_hash_off != 3144 (uint8_t *)sym_op->aead.digest.data)) { 3145 flags |= VALID_MAC_BUF; 3146 fc_params.mac_buf.size = sess_misc->mac_len; 3147 fc_params.mac_buf.vaddr = 3148 sym_op->aead.digest.data; 3149 fc_params.mac_buf.dma_addr = 3150 sym_op->aead.digest.phys_addr; 3151 inplace = 0; 3152 } 3153 } 3154 } else { 3155 d_offs = sym_op->cipher.data.offset; 3156 d_lens = sym_op->cipher.data.length; 3157 mc_hash_off = sym_op->cipher.data.offset + 3158 sym_op->cipher.data.length; 3159 d_offs = (d_offs << 16) | sym_op->auth.data.offset; 3160 d_lens = (d_lens << 32) | sym_op->auth.data.length; 3161 3162 if (mc_hash_off < (sym_op->auth.data.offset + 3163 sym_op->auth.data.length)){ 3164 mc_hash_off = (sym_op->auth.data.offset + 3165 sym_op->auth.data.length); 3166 } 3167 /* for gmac, salt should be updated like in gcm */ 3168 if (unlikely(sess_misc->is_gmac)) { 3169 uint8_t *salt; 3170 salt = fc_params.iv_buf; 3171 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3172 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3173 sess_misc->salt = *(uint32_t *)salt; 3174 } 3175 fc_params.iv_buf = salt + 4; 3176 } 3177 if (likely(sess_misc->mac_len)) { 3178 struct rte_mbuf *m; 3179 3180 m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src; 3181 if (!m) 3182 m = m_src; 3183 3184 /* hmac immediately following data is best case */ 3185 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3186 mc_hash_off != 3187 (uint8_t *)sym_op->auth.digest.data)) { 3188 flags |= VALID_MAC_BUF; 3189 fc_params.mac_buf.size = 3190 sess_misc->mac_len; 3191 fc_params.mac_buf.vaddr = 3192 sym_op->auth.digest.data; 3193 fc_params.mac_buf.dma_addr = 3194 sym_op->auth.digest.phys_addr; 3195 inplace = 0; 3196 } 3197 } 3198 } 3199 fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc); 3200 fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr; 3201 3202 if (unlikely(sess_misc->is_null || sess_misc->cpt_op == CPT_OP_DECODE)) 3203 inplace = 0; 3204 3205 if (likely(!m_dst && inplace)) { 3206 /* Case of single buffer without AAD buf or 3207 * separate mac buf in place and 3208 * not air crypto 3209 */ 3210 fc_params.dst_iov = fc_params.src_iov = (void *)src; 3211 3212 if (unlikely(prepare_iov_from_pkt_inplace(m_src, 3213 &fc_params, 3214 &flags))) { 3215 CPT_LOG_DP_ERR("Prepare inplace src iov failed"); 3216 ret = -EINVAL; 3217 goto err_exit; 3218 } 3219 3220 } else { 3221 /* Out of place processing */ 3222 fc_params.src_iov = (void *)src; 3223 fc_params.dst_iov = (void *)dst; 3224 3225 /* Store SG I/O in the api for reuse */ 3226 if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) { 3227 CPT_LOG_DP_ERR("Prepare src iov failed"); 3228 ret = -EINVAL; 3229 goto err_exit; 3230 } 3231 3232 if (unlikely(m_dst != NULL)) { 3233 uint32_t pkt_len; 3234 3235 /* Try to make room as much as src has */ 3236 pkt_len = rte_pktmbuf_pkt_len(m_dst); 3237 3238 if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) { 3239 pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len; 3240 if (!rte_pktmbuf_append(m_dst, pkt_len)) { 3241 CPT_LOG_DP_ERR("Not enough space in " 3242 "m_dst %p, need %u" 3243 " more", 3244 m_dst, pkt_len); 3245 ret = -EINVAL; 3246 goto err_exit; 3247 } 3248 } 3249 3250 if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) { 3251 CPT_LOG_DP_ERR("Prepare dst iov failed for " 3252 "m_dst %p", m_dst); 3253 ret = -EINVAL; 3254 goto err_exit; 3255 } 3256 } else { 3257 fc_params.dst_iov = (void *)src; 3258 } 3259 } 3260 3261 if (likely(flags & SINGLE_BUF_HEADTAILROOM)) 3262 mdata = alloc_op_meta(m_src, &fc_params.meta_buf, 3263 m_info->lb_mlen, m_info->pool); 3264 else 3265 mdata = alloc_op_meta(NULL, &fc_params.meta_buf, 3266 m_info->sg_mlen, m_info->pool); 3267 3268 if (unlikely(mdata == NULL)) { 3269 CPT_LOG_DP_ERR("Error allocating meta buffer for request"); 3270 ret = -ENOMEM; 3271 goto err_exit; 3272 } 3273 3274 op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull); 3275 op[0] = (uintptr_t)mdata; 3276 op[1] = (uintptr_t)cop; 3277 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3278 space += 4 * sizeof(uint64_t); 3279 3280 fc_params.meta_buf.vaddr = (uint8_t *)op + space; 3281 fc_params.meta_buf.dma_addr += space; 3282 fc_params.meta_buf.size -= space; 3283 3284 /* Finally prepare the instruction */ 3285 if (cpt_op & CPT_OP_ENCODE) 3286 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, 3287 &fc_params, op); 3288 else 3289 *prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens, 3290 &fc_params, op); 3291 3292 if (unlikely(*prep_req == NULL)) { 3293 CPT_LOG_DP_ERR("Preparing request failed due to bad input arg"); 3294 ret = -EINVAL; 3295 goto free_mdata_and_exit; 3296 } 3297 3298 *mdata_ptr = mdata; 3299 3300 return 0; 3301 3302 free_mdata_and_exit: 3303 free_op_meta(mdata, m_info->pool); 3304 err_exit: 3305 return ret; 3306 } 3307 3308 static __rte_always_inline void 3309 compl_auth_verify(struct rte_crypto_op *op, 3310 uint8_t *gen_mac, 3311 uint64_t mac_len) 3312 { 3313 uint8_t *mac; 3314 struct rte_crypto_sym_op *sym_op = op->sym; 3315 3316 if (sym_op->auth.digest.data) 3317 mac = sym_op->auth.digest.data; 3318 else 3319 mac = rte_pktmbuf_mtod_offset(sym_op->m_src, 3320 uint8_t *, 3321 sym_op->auth.data.length + 3322 sym_op->auth.data.offset); 3323 if (!mac) { 3324 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3325 return; 3326 } 3327 3328 if (memcmp(mac, gen_mac, mac_len)) 3329 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 3330 else 3331 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3332 } 3333 3334 static __rte_always_inline int 3335 instance_session_cfg(struct rte_crypto_sym_xform *xform, void *sess) 3336 { 3337 struct rte_crypto_sym_xform *chain; 3338 3339 CPT_PMD_INIT_FUNC_TRACE(); 3340 3341 if (cpt_is_algo_supported(xform)) 3342 goto err; 3343 3344 chain = xform; 3345 while (chain) { 3346 switch (chain->type) { 3347 case RTE_CRYPTO_SYM_XFORM_AEAD: 3348 if (fill_sess_aead(chain, sess)) 3349 goto err; 3350 break; 3351 case RTE_CRYPTO_SYM_XFORM_CIPHER: 3352 if (fill_sess_cipher(chain, sess)) 3353 goto err; 3354 break; 3355 case RTE_CRYPTO_SYM_XFORM_AUTH: 3356 if (chain->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 3357 if (fill_sess_gmac(chain, sess)) 3358 goto err; 3359 } else { 3360 if (fill_sess_auth(chain, sess)) 3361 goto err; 3362 } 3363 break; 3364 default: 3365 CPT_LOG_DP_ERR("Invalid crypto xform type"); 3366 break; 3367 } 3368 chain = chain->next; 3369 } 3370 3371 return 0; 3372 3373 err: 3374 return -1; 3375 } 3376 3377 static __rte_always_inline void 3378 find_kasumif9_direction_and_length(uint8_t *src, 3379 uint32_t counter_num_bytes, 3380 uint32_t *addr_length_in_bits, 3381 uint8_t *addr_direction) 3382 { 3383 uint8_t found = 0; 3384 uint32_t pos; 3385 uint8_t last_byte; 3386 while (!found && counter_num_bytes > 0) { 3387 counter_num_bytes--; 3388 if (src[counter_num_bytes] == 0x00) 3389 continue; 3390 pos = rte_bsf32(src[counter_num_bytes]); 3391 if (pos == 7) { 3392 if (likely(counter_num_bytes > 0)) { 3393 last_byte = src[counter_num_bytes - 1]; 3394 *addr_direction = last_byte & 0x1; 3395 *addr_length_in_bits = counter_num_bytes * 8 3396 - 1; 3397 } 3398 } else { 3399 last_byte = src[counter_num_bytes]; 3400 *addr_direction = (last_byte >> (pos + 1)) & 0x1; 3401 *addr_length_in_bits = counter_num_bytes * 8 3402 + (8 - (pos + 2)); 3403 } 3404 found = 1; 3405 } 3406 } 3407 3408 /* 3409 * This handles all auth only except AES_GMAC 3410 */ 3411 static __rte_always_inline int 3412 fill_digest_params(struct rte_crypto_op *cop, 3413 struct cpt_sess_misc *sess, 3414 struct cpt_qp_meta_info *m_info, 3415 void **mdata_ptr, 3416 void **prep_req) 3417 { 3418 uint32_t space = 0; 3419 struct rte_crypto_sym_op *sym_op = cop->sym; 3420 void *mdata; 3421 phys_addr_t mphys; 3422 uint64_t *op; 3423 uint32_t auth_range_off; 3424 uint32_t flags = 0; 3425 uint64_t d_offs = 0, d_lens; 3426 struct rte_mbuf *m_src, *m_dst; 3427 uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK; 3428 uint16_t mac_len = sess->mac_len; 3429 fc_params_t params; 3430 char src[SRC_IOV_SIZE]; 3431 uint8_t iv_buf[16]; 3432 int ret; 3433 3434 memset(¶ms, 0, sizeof(fc_params_t)); 3435 3436 m_src = sym_op->m_src; 3437 3438 /* For just digest lets force mempool alloc */ 3439 mdata = alloc_op_meta(NULL, ¶ms.meta_buf, m_info->sg_mlen, 3440 m_info->pool); 3441 if (mdata == NULL) { 3442 ret = -ENOMEM; 3443 goto err_exit; 3444 } 3445 3446 mphys = params.meta_buf.dma_addr; 3447 3448 op = mdata; 3449 op[0] = (uintptr_t)mdata; 3450 op[1] = (uintptr_t)cop; 3451 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3452 space += 4 * sizeof(uint64_t); 3453 3454 auth_range_off = sym_op->auth.data.offset; 3455 3456 flags = VALID_MAC_BUF; 3457 params.src_iov = (void *)src; 3458 if (unlikely(sess->zsk_flag)) { 3459 /* 3460 * Since for Zuc, Kasumi, Snow3g offsets are in bits 3461 * we will send pass through even for auth only case, 3462 * let MC handle it 3463 */ 3464 d_offs = auth_range_off; 3465 auth_range_off = 0; 3466 params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3467 uint8_t *, sess->auth_iv_offset); 3468 if (sess->zsk_flag == K_F9) { 3469 uint32_t length_in_bits, num_bytes; 3470 uint8_t *src, direction = 0; 3471 3472 memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src, 3473 uint8_t *), 8); 3474 /* 3475 * This is kasumi f9, take direction from 3476 * source buffer 3477 */ 3478 length_in_bits = cop->sym->auth.data.length; 3479 num_bytes = (length_in_bits >> 3); 3480 src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *); 3481 find_kasumif9_direction_and_length(src, 3482 num_bytes, 3483 &length_in_bits, 3484 &direction); 3485 length_in_bits -= 64; 3486 cop->sym->auth.data.offset += 64; 3487 d_offs = cop->sym->auth.data.offset; 3488 auth_range_off = d_offs / 8; 3489 cop->sym->auth.data.length = length_in_bits; 3490 3491 /* Store it at end of auth iv */ 3492 iv_buf[8] = direction; 3493 params.auth_iv_buf = iv_buf; 3494 } 3495 } 3496 3497 d_lens = sym_op->auth.data.length; 3498 3499 params.ctx_buf.vaddr = SESS_PRIV(sess); 3500 params.ctx_buf.dma_addr = sess->ctx_dma_addr; 3501 3502 if (auth_op == CPT_OP_AUTH_GENERATE) { 3503 if (sym_op->auth.digest.data) { 3504 /* 3505 * Digest to be generated 3506 * in separate buffer 3507 */ 3508 params.mac_buf.size = 3509 sess->mac_len; 3510 params.mac_buf.vaddr = 3511 sym_op->auth.digest.data; 3512 params.mac_buf.dma_addr = 3513 sym_op->auth.digest.phys_addr; 3514 } else { 3515 uint32_t off = sym_op->auth.data.offset + 3516 sym_op->auth.data.length; 3517 int32_t dlen, space; 3518 3519 m_dst = sym_op->m_dst ? 3520 sym_op->m_dst : sym_op->m_src; 3521 dlen = rte_pktmbuf_pkt_len(m_dst); 3522 3523 space = off + mac_len - dlen; 3524 if (space > 0) 3525 if (!rte_pktmbuf_append(m_dst, space)) { 3526 CPT_LOG_DP_ERR("Failed to extend " 3527 "mbuf by %uB", space); 3528 ret = -EINVAL; 3529 goto free_mdata_and_exit; 3530 } 3531 3532 params.mac_buf.vaddr = 3533 rte_pktmbuf_mtod_offset(m_dst, void *, off); 3534 params.mac_buf.dma_addr = 3535 rte_pktmbuf_mtophys_offset(m_dst, off); 3536 params.mac_buf.size = mac_len; 3537 } 3538 } else { 3539 /* Need space for storing generated mac */ 3540 params.mac_buf.vaddr = (uint8_t *)mdata + space; 3541 params.mac_buf.dma_addr = mphys + space; 3542 params.mac_buf.size = mac_len; 3543 space += RTE_ALIGN_CEIL(mac_len, 8); 3544 op[2] = (uintptr_t)params.mac_buf.vaddr; 3545 op[3] = mac_len; 3546 } 3547 3548 params.meta_buf.vaddr = (uint8_t *)mdata + space; 3549 params.meta_buf.dma_addr = mphys + space; 3550 params.meta_buf.size -= space; 3551 3552 /* Out of place processing */ 3553 params.src_iov = (void *)src; 3554 3555 /*Store SG I/O in the api for reuse */ 3556 if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) { 3557 CPT_LOG_DP_ERR("Prepare src iov failed"); 3558 ret = -EINVAL; 3559 goto free_mdata_and_exit; 3560 } 3561 3562 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, ¶ms, op); 3563 if (unlikely(*prep_req == NULL)) { 3564 ret = -EINVAL; 3565 goto free_mdata_and_exit; 3566 } 3567 3568 *mdata_ptr = mdata; 3569 3570 return 0; 3571 3572 free_mdata_and_exit: 3573 free_op_meta(mdata, m_info->pool); 3574 err_exit: 3575 return ret; 3576 } 3577 3578 #endif /*_CPT_UCODE_H_ */ 3579