1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #ifndef _CPT_UCODE_H_ 6 #define _CPT_UCODE_H_ 7 #include <stdbool.h> 8 9 #include "cpt_common.h" 10 #include "cpt_hw_types.h" 11 #include "cpt_mcode_defines.h" 12 13 /* 14 * This file defines functions that are interfaces to microcode spec. 15 * 16 */ 17 18 static uint8_t zuc_d[32] = { 19 0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E, 20 0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF, 21 0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1, 22 0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC 23 }; 24 25 static __rte_always_inline int 26 cpt_is_algo_supported(struct rte_crypto_sym_xform *xform) 27 { 28 /* 29 * Microcode only supports the following combination. 30 * Encryption followed by authentication 31 * Authentication followed by decryption 32 */ 33 if (xform->next) { 34 if ((xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) && 35 (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 36 (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)) { 37 /* Unsupported as of now by microcode */ 38 CPT_LOG_DP_ERR("Unsupported combination"); 39 return -1; 40 } 41 if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 42 (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) && 43 (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT)) { 44 /* For GMAC auth there is no cipher operation */ 45 if (xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM || 46 xform->next->auth.algo != 47 RTE_CRYPTO_AUTH_AES_GMAC) { 48 /* Unsupported as of now by microcode */ 49 CPT_LOG_DP_ERR("Unsupported combination"); 50 return -1; 51 } 52 } 53 } 54 return 0; 55 } 56 57 static __rte_always_inline void 58 gen_key_snow3g(const uint8_t *ck, uint32_t *keyx) 59 { 60 int i, base; 61 62 for (i = 0; i < 4; i++) { 63 base = 4 * i; 64 keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) | 65 (ck[base + 2] << 8) | (ck[base + 3]); 66 keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]); 67 } 68 } 69 70 static __rte_always_inline void 71 cpt_fc_salt_update(void *ctx, 72 uint8_t *salt) 73 { 74 struct cpt_ctx *cpt_ctx = ctx; 75 memcpy(&cpt_ctx->fctx.enc.encr_iv, salt, 4); 76 } 77 78 static __rte_always_inline int 79 cpt_fc_ciph_validate_key_aes(uint16_t key_len) 80 { 81 switch (key_len) { 82 case CPT_BYTE_16: 83 case CPT_BYTE_24: 84 case CPT_BYTE_32: 85 return 0; 86 default: 87 return -1; 88 } 89 } 90 91 static __rte_always_inline int 92 cpt_fc_ciph_set_type(cipher_type_t type, struct cpt_ctx *ctx, uint16_t key_len) 93 { 94 int fc_type = 0; 95 switch (type) { 96 case PASSTHROUGH: 97 fc_type = FC_GEN; 98 break; 99 case DES3_CBC: 100 case DES3_ECB: 101 fc_type = FC_GEN; 102 break; 103 case AES_CBC: 104 case AES_ECB: 105 case AES_CFB: 106 case AES_CTR: 107 case AES_GCM: 108 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 109 return -1; 110 fc_type = FC_GEN; 111 break; 112 case AES_XTS: 113 key_len = key_len / 2; 114 if (unlikely(key_len == CPT_BYTE_24)) { 115 CPT_LOG_DP_ERR("Invalid AES key len for XTS"); 116 return -1; 117 } 118 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 119 return -1; 120 fc_type = FC_GEN; 121 break; 122 case ZUC_EEA3: 123 case SNOW3G_UEA2: 124 if (unlikely(key_len != 16)) 125 return -1; 126 /* No support for AEAD yet */ 127 if (unlikely(ctx->hash_type)) 128 return -1; 129 fc_type = ZUC_SNOW3G; 130 break; 131 case KASUMI_F8_CBC: 132 case KASUMI_F8_ECB: 133 if (unlikely(key_len != 16)) 134 return -1; 135 /* No support for AEAD yet */ 136 if (unlikely(ctx->hash_type)) 137 return -1; 138 fc_type = KASUMI; 139 break; 140 default: 141 return -1; 142 } 143 144 ctx->fc_type = fc_type; 145 return 0; 146 } 147 148 static __rte_always_inline void 149 cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx) 150 { 151 cpt_ctx->enc_cipher = 0; 152 CPT_P_ENC_CTRL(fctx).enc_cipher = 0; 153 } 154 155 static __rte_always_inline void 156 cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len) 157 { 158 mc_aes_type_t aes_key_type = 0; 159 switch (key_len) { 160 case CPT_BYTE_16: 161 aes_key_type = AES_128_BIT; 162 break; 163 case CPT_BYTE_24: 164 aes_key_type = AES_192_BIT; 165 break; 166 case CPT_BYTE_32: 167 aes_key_type = AES_256_BIT; 168 break; 169 default: 170 /* This should not happen */ 171 CPT_LOG_DP_ERR("Invalid AES key len"); 172 return; 173 } 174 CPT_P_ENC_CTRL(fctx).aes_key = aes_key_type; 175 } 176 177 static __rte_always_inline void 178 cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key, 179 uint16_t key_len) 180 { 181 uint32_t keyx[4]; 182 cpt_ctx->snow3g = 1; 183 gen_key_snow3g(key, keyx); 184 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 185 cpt_ctx->zsk_flags = 0; 186 } 187 188 static __rte_always_inline void 189 cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key, 190 uint16_t key_len) 191 { 192 cpt_ctx->snow3g = 0; 193 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 194 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 195 cpt_ctx->zsk_flags = 0; 196 } 197 198 static __rte_always_inline void 199 cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key, 200 uint16_t key_len) 201 { 202 cpt_ctx->k_ecb = 1; 203 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 204 cpt_ctx->zsk_flags = 0; 205 } 206 207 static __rte_always_inline void 208 cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key, 209 uint16_t key_len) 210 { 211 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 212 cpt_ctx->zsk_flags = 0; 213 } 214 215 static __rte_always_inline int 216 cpt_fc_ciph_set_key(void *ctx, cipher_type_t type, const uint8_t *key, 217 uint16_t key_len, uint8_t *salt) 218 { 219 struct cpt_ctx *cpt_ctx = ctx; 220 mc_fc_context_t *fctx = &cpt_ctx->fctx; 221 uint64_t *ctrl_flags = NULL; 222 int ret; 223 224 ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len); 225 if (unlikely(ret)) 226 return -1; 227 228 if (cpt_ctx->fc_type == FC_GEN) { 229 ctrl_flags = (uint64_t *)&(fctx->enc.enc_ctrl.flags); 230 *ctrl_flags = rte_be_to_cpu_64(*ctrl_flags); 231 /* 232 * We need to always say IV is from DPTR as user can 233 * sometimes iverride IV per operation. 234 */ 235 CPT_P_ENC_CTRL(fctx).iv_source = CPT_FROM_DPTR; 236 } 237 238 switch (type) { 239 case PASSTHROUGH: 240 cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx); 241 goto fc_success; 242 case DES3_CBC: 243 /* CPT performs DES using 3DES with the 8B DES-key 244 * replicated 2 more times to match the 24B 3DES-key. 245 * Eg. If org. key is "0x0a 0x0b", then new key is 246 * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b" 247 */ 248 if (key_len == 8) { 249 /* Skipping the first 8B as it will be copied 250 * in the regular code flow 251 */ 252 memcpy(fctx->enc.encr_key+key_len, key, key_len); 253 memcpy(fctx->enc.encr_key+2*key_len, key, key_len); 254 } 255 break; 256 case DES3_ECB: 257 /* For DES3_ECB IV need to be from CTX. */ 258 CPT_P_ENC_CTRL(fctx).iv_source = CPT_FROM_CTX; 259 break; 260 case AES_CBC: 261 case AES_ECB: 262 case AES_CFB: 263 case AES_CTR: 264 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 265 break; 266 case AES_GCM: 267 /* Even though iv source is from dptr, 268 * aes_gcm salt is taken from ctx 269 */ 270 if (salt) { 271 memcpy(fctx->enc.encr_iv, salt, 4); 272 /* Assuming it was just salt update 273 * and nothing else 274 */ 275 if (!key) 276 goto fc_success; 277 } 278 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 279 break; 280 case AES_XTS: 281 key_len = key_len / 2; 282 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 283 284 /* Copy key2 for XTS into ipad */ 285 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 286 memcpy(fctx->hmac.ipad, &key[key_len], key_len); 287 break; 288 case SNOW3G_UEA2: 289 cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len); 290 goto success; 291 case ZUC_EEA3: 292 cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len); 293 goto success; 294 case KASUMI_F8_ECB: 295 cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len); 296 goto success; 297 case KASUMI_F8_CBC: 298 cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len); 299 goto success; 300 default: 301 break; 302 } 303 304 /* Only for FC_GEN case */ 305 306 /* For GMAC auth, cipher must be NULL */ 307 if (cpt_ctx->hash_type != GMAC_TYPE) 308 CPT_P_ENC_CTRL(fctx).enc_cipher = type; 309 310 memcpy(fctx->enc.encr_key, key, key_len); 311 312 fc_success: 313 *ctrl_flags = rte_cpu_to_be_64(*ctrl_flags); 314 315 success: 316 cpt_ctx->enc_cipher = type; 317 318 return 0; 319 } 320 321 static __rte_always_inline uint32_t 322 fill_sg_comp(sg_comp_t *list, 323 uint32_t i, 324 phys_addr_t dma_addr, 325 uint32_t size) 326 { 327 sg_comp_t *to = &list[i>>2]; 328 329 to->u.s.len[i%4] = rte_cpu_to_be_16(size); 330 to->ptr[i%4] = rte_cpu_to_be_64(dma_addr); 331 i++; 332 return i; 333 } 334 335 static __rte_always_inline uint32_t 336 fill_sg_comp_from_buf(sg_comp_t *list, 337 uint32_t i, 338 buf_ptr_t *from) 339 { 340 sg_comp_t *to = &list[i>>2]; 341 342 to->u.s.len[i%4] = rte_cpu_to_be_16(from->size); 343 to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr); 344 i++; 345 return i; 346 } 347 348 static __rte_always_inline uint32_t 349 fill_sg_comp_from_buf_min(sg_comp_t *list, 350 uint32_t i, 351 buf_ptr_t *from, 352 uint32_t *psize) 353 { 354 sg_comp_t *to = &list[i >> 2]; 355 uint32_t size = *psize; 356 uint32_t e_len; 357 358 e_len = (size > from->size) ? from->size : size; 359 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 360 to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr); 361 *psize -= e_len; 362 i++; 363 return i; 364 } 365 366 /* 367 * This fills the MC expected SGIO list 368 * from IOV given by user. 369 */ 370 static __rte_always_inline uint32_t 371 fill_sg_comp_from_iov(sg_comp_t *list, 372 uint32_t i, 373 iov_ptr_t *from, uint32_t from_offset, 374 uint32_t *psize, buf_ptr_t *extra_buf, 375 uint32_t extra_offset) 376 { 377 int32_t j; 378 uint32_t extra_len = extra_buf ? extra_buf->size : 0; 379 uint32_t size = *psize - extra_len; 380 buf_ptr_t *bufs; 381 382 bufs = from->bufs; 383 for (j = 0; (j < from->buf_cnt) && size; j++) { 384 phys_addr_t e_dma_addr; 385 uint32_t e_len; 386 sg_comp_t *to = &list[i >> 2]; 387 388 if (!bufs[j].size) 389 continue; 390 391 if (unlikely(from_offset)) { 392 if (from_offset >= bufs[j].size) { 393 from_offset -= bufs[j].size; 394 continue; 395 } 396 e_dma_addr = bufs[j].dma_addr + from_offset; 397 e_len = (size > (bufs[j].size - from_offset)) ? 398 (bufs[j].size - from_offset) : size; 399 from_offset = 0; 400 } else { 401 e_dma_addr = bufs[j].dma_addr; 402 e_len = (size > bufs[j].size) ? 403 bufs[j].size : size; 404 } 405 406 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 407 to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr); 408 409 if (extra_len && (e_len >= extra_offset)) { 410 /* Break the data at given offset */ 411 uint32_t next_len = e_len - extra_offset; 412 phys_addr_t next_dma = e_dma_addr + extra_offset; 413 414 if (!extra_offset) { 415 i--; 416 } else { 417 e_len = extra_offset; 418 size -= e_len; 419 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 420 } 421 422 /* Insert extra data ptr */ 423 if (extra_len) { 424 i++; 425 to = &list[i >> 2]; 426 to->u.s.len[i % 4] = 427 rte_cpu_to_be_16(extra_buf->size); 428 to->ptr[i % 4] = 429 rte_cpu_to_be_64(extra_buf->dma_addr); 430 431 /* size already decremented by extra len */ 432 } 433 434 /* insert the rest of the data */ 435 if (next_len) { 436 i++; 437 to = &list[i >> 2]; 438 to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len); 439 to->ptr[i % 4] = rte_cpu_to_be_64(next_dma); 440 size -= next_len; 441 } 442 extra_len = 0; 443 444 } else { 445 size -= e_len; 446 } 447 if (extra_offset) 448 extra_offset -= size; 449 i++; 450 } 451 452 *psize = size; 453 return (uint32_t)i; 454 } 455 456 static __rte_always_inline void 457 cpt_digest_gen_prep(uint32_t flags, 458 uint64_t d_lens, 459 digest_params_t *params, 460 void *op, 461 void **prep_req) 462 { 463 struct cpt_request_info *req; 464 uint32_t size, i; 465 uint16_t data_len, mac_len, key_len; 466 auth_type_t hash_type; 467 buf_ptr_t *meta_p; 468 struct cpt_ctx *ctx; 469 sg_comp_t *gather_comp; 470 sg_comp_t *scatter_comp; 471 uint8_t *in_buffer; 472 uint32_t g_size_bytes, s_size_bytes; 473 uint64_t dptr_dma, rptr_dma; 474 vq_cmd_word0_t vq_cmd_w0; 475 vq_cmd_word3_t vq_cmd_w3; 476 void *c_vaddr, *m_vaddr; 477 uint64_t c_dma, m_dma; 478 opcode_info_t opcode; 479 480 ctx = params->ctx_buf.vaddr; 481 meta_p = ¶ms->meta_buf; 482 483 m_vaddr = meta_p->vaddr; 484 m_dma = meta_p->dma_addr; 485 486 /* 487 * Save initial space that followed app data for completion code & 488 * alternate completion code to fall in same cache line as app data 489 */ 490 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 491 m_dma += COMPLETION_CODE_SIZE; 492 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 493 (uint8_t *)m_vaddr; 494 c_vaddr = (uint8_t *)m_vaddr + size; 495 c_dma = m_dma + size; 496 size += sizeof(cpt_res_s_t); 497 498 m_vaddr = (uint8_t *)m_vaddr + size; 499 m_dma += size; 500 501 req = m_vaddr; 502 503 size = sizeof(struct cpt_request_info); 504 m_vaddr = (uint8_t *)m_vaddr + size; 505 m_dma += size; 506 507 hash_type = ctx->hash_type; 508 mac_len = ctx->mac_len; 509 key_len = ctx->auth_key_len; 510 data_len = AUTH_DLEN(d_lens); 511 512 /*GP op header */ 513 vq_cmd_w0.u64 = 0; 514 vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8); 515 if (ctx->hmac) { 516 opcode.s.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE; 517 vq_cmd_w0.s.param1 = key_len; 518 vq_cmd_w0.s.dlen = data_len + ROUNDUP8(key_len); 519 } else { 520 opcode.s.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE; 521 vq_cmd_w0.s.param1 = 0; 522 vq_cmd_w0.s.dlen = data_len; 523 } 524 525 opcode.s.minor = 0; 526 527 /* Null auth only case enters the if */ 528 if (unlikely(!hash_type && !ctx->enc_cipher)) { 529 opcode.s.major = CPT_MAJOR_OP_MISC; 530 /* Minor op is passthrough */ 531 opcode.s.minor = 0x03; 532 /* Send out completion code only */ 533 vq_cmd_w0.s.param2 = 0x1; 534 } 535 536 vq_cmd_w0.s.opcode = opcode.flags; 537 538 /* DPTR has SG list */ 539 in_buffer = m_vaddr; 540 dptr_dma = m_dma; 541 542 ((uint16_t *)in_buffer)[0] = 0; 543 ((uint16_t *)in_buffer)[1] = 0; 544 545 /* TODO Add error check if space will be sufficient */ 546 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 547 548 /* 549 * Input gather list 550 */ 551 552 i = 0; 553 554 if (ctx->hmac) { 555 uint64_t k_dma = params->ctx_buf.dma_addr + 556 offsetof(struct cpt_ctx, auth_key); 557 /* Key */ 558 i = fill_sg_comp(gather_comp, i, k_dma, ROUNDUP8(key_len)); 559 } 560 561 /* input data */ 562 size = data_len; 563 if (size) { 564 i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 565 0, &size, NULL, 0); 566 if (unlikely(size)) { 567 CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short" 568 " by %dB", size); 569 return; 570 } 571 } else { 572 /* 573 * Looks like we need to support zero data 574 * gather ptr in case of hash & hmac 575 */ 576 i++; 577 } 578 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 579 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 580 581 /* 582 * Output Gather list 583 */ 584 585 i = 0; 586 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 587 588 if (flags & VALID_MAC_BUF) { 589 if (unlikely(params->mac_buf.size < mac_len)) { 590 CPT_LOG_DP_ERR("Insufficient MAC size"); 591 return; 592 } 593 594 size = mac_len; 595 i = fill_sg_comp_from_buf_min(scatter_comp, i, 596 ¶ms->mac_buf, &size); 597 } else { 598 size = mac_len; 599 i = fill_sg_comp_from_iov(scatter_comp, i, 600 params->src_iov, data_len, 601 &size, NULL, 0); 602 if (unlikely(size)) { 603 CPT_LOG_DP_ERR("Insufficient dst IOV size, short by" 604 " %dB", size); 605 return; 606 } 607 } 608 609 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 610 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 611 612 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 613 614 /* This is DPTR len incase of SG mode */ 615 vq_cmd_w0.s.dlen = size; 616 617 m_vaddr = (uint8_t *)m_vaddr + size; 618 m_dma += size; 619 620 /* cpt alternate completion address saved earlier */ 621 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 622 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 623 rptr_dma = c_dma - 8; 624 625 req->ist.ei1 = dptr_dma; 626 req->ist.ei2 = rptr_dma; 627 628 /* vq command w3 */ 629 vq_cmd_w3.u64 = 0; 630 631 /* 16 byte aligned cpt res address */ 632 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 633 *req->completion_addr = COMPLETION_CODE_INIT; 634 req->comp_baddr = c_dma; 635 636 /* Fill microcode part of instruction */ 637 req->ist.ei0 = vq_cmd_w0.u64; 638 req->ist.ei3 = vq_cmd_w3.u64; 639 640 req->op = op; 641 642 *prep_req = req; 643 return; 644 } 645 646 static __rte_always_inline void 647 cpt_enc_hmac_prep(uint32_t flags, 648 uint64_t d_offs, 649 uint64_t d_lens, 650 fc_params_t *fc_params, 651 void *op, 652 void **prep_req) 653 { 654 uint32_t iv_offset = 0; 655 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 656 struct cpt_ctx *cpt_ctx; 657 uint32_t cipher_type, hash_type; 658 uint32_t mac_len, size; 659 uint8_t iv_len = 16; 660 struct cpt_request_info *req; 661 buf_ptr_t *meta_p, *aad_buf = NULL; 662 uint32_t encr_offset, auth_offset; 663 uint32_t encr_data_len, auth_data_len, aad_len = 0; 664 uint32_t passthrough_len = 0; 665 void *m_vaddr, *offset_vaddr; 666 uint64_t m_dma, offset_dma, ctx_dma; 667 vq_cmd_word0_t vq_cmd_w0; 668 vq_cmd_word3_t vq_cmd_w3; 669 void *c_vaddr; 670 uint64_t c_dma; 671 opcode_info_t opcode; 672 673 meta_p = &fc_params->meta_buf; 674 m_vaddr = meta_p->vaddr; 675 m_dma = meta_p->dma_addr; 676 677 encr_offset = ENCR_OFFSET(d_offs); 678 auth_offset = AUTH_OFFSET(d_offs); 679 encr_data_len = ENCR_DLEN(d_lens); 680 auth_data_len = AUTH_DLEN(d_lens); 681 if (unlikely(flags & VALID_AAD_BUF)) { 682 /* 683 * We dont support both aad 684 * and auth data separately 685 */ 686 auth_data_len = 0; 687 auth_offset = 0; 688 aad_len = fc_params->aad_buf.size; 689 aad_buf = &fc_params->aad_buf; 690 } 691 cpt_ctx = fc_params->ctx_buf.vaddr; 692 cipher_type = cpt_ctx->enc_cipher; 693 hash_type = cpt_ctx->hash_type; 694 mac_len = cpt_ctx->mac_len; 695 696 /* 697 * Save initial space that followed app data for completion code & 698 * alternate completion code to fall in same cache line as app data 699 */ 700 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 701 m_dma += COMPLETION_CODE_SIZE; 702 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 703 (uint8_t *)m_vaddr; 704 705 c_vaddr = (uint8_t *)m_vaddr + size; 706 c_dma = m_dma + size; 707 size += sizeof(cpt_res_s_t); 708 709 m_vaddr = (uint8_t *)m_vaddr + size; 710 m_dma += size; 711 712 /* start cpt request info struct at 8 byte boundary */ 713 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 714 (uint8_t *)m_vaddr; 715 716 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 717 718 size += sizeof(struct cpt_request_info); 719 m_vaddr = (uint8_t *)m_vaddr + size; 720 m_dma += size; 721 722 if (hash_type == GMAC_TYPE) 723 encr_data_len = 0; 724 725 if (unlikely(!(flags & VALID_IV_BUF))) { 726 iv_len = 0; 727 iv_offset = ENCR_IV_OFFSET(d_offs); 728 } 729 730 if (unlikely(flags & VALID_AAD_BUF)) { 731 /* 732 * When AAD is given, data above encr_offset is pass through 733 * Since AAD is given as separate pointer and not as offset, 734 * this is a special case as we need to fragment input data 735 * into passthrough + encr_data and then insert AAD in between. 736 */ 737 if (hash_type != GMAC_TYPE) { 738 passthrough_len = encr_offset; 739 auth_offset = passthrough_len + iv_len; 740 encr_offset = passthrough_len + aad_len + iv_len; 741 auth_data_len = aad_len + encr_data_len; 742 } else { 743 passthrough_len = 16 + aad_len; 744 auth_offset = passthrough_len + iv_len; 745 auth_data_len = aad_len; 746 } 747 } else { 748 encr_offset += iv_len; 749 auth_offset += iv_len; 750 } 751 752 /* Encryption */ 753 opcode.s.major = CPT_MAJOR_OP_FC; 754 opcode.s.minor = 0; 755 756 auth_dlen = auth_offset + auth_data_len; 757 enc_dlen = encr_data_len + encr_offset; 758 if (unlikely(encr_data_len & 0xf)) { 759 if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB)) 760 enc_dlen = ROUNDUP8(encr_data_len) + encr_offset; 761 else if (likely((cipher_type == AES_CBC) || 762 (cipher_type == AES_ECB))) 763 enc_dlen = ROUNDUP16(encr_data_len) + encr_offset; 764 } 765 766 if (unlikely(hash_type == GMAC_TYPE)) { 767 encr_offset = auth_dlen; 768 enc_dlen = 0; 769 } 770 771 if (unlikely(auth_dlen > enc_dlen)) { 772 inputlen = auth_dlen; 773 outputlen = auth_dlen + mac_len; 774 } else { 775 inputlen = enc_dlen; 776 outputlen = enc_dlen + mac_len; 777 } 778 779 /* GP op header */ 780 vq_cmd_w0.u64 = 0; 781 vq_cmd_w0.s.param1 = encr_data_len; 782 vq_cmd_w0.s.param2 = auth_data_len; 783 /* 784 * In 83XX since we have a limitation of 785 * IV & Offset control word not part of instruction 786 * and need to be part of Data Buffer, we check if 787 * head room is there and then only do the Direct mode processing 788 */ 789 if (likely((flags & SINGLE_BUF_INPLACE) && 790 (flags & SINGLE_BUF_HEADTAILROOM))) { 791 void *dm_vaddr = fc_params->bufs[0].vaddr; 792 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 793 /* 794 * This flag indicates that there is 24 bytes head room and 795 * 8 bytes tail room available, so that we get to do 796 * DIRECT MODE with limitation 797 */ 798 799 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 800 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 801 802 /* DPTR */ 803 req->ist.ei1 = offset_dma; 804 /* RPTR should just exclude offset control word */ 805 req->ist.ei2 = dm_dma_addr - iv_len; 806 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 807 + outputlen - iv_len); 808 809 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 810 811 vq_cmd_w0.s.opcode = opcode.flags; 812 813 if (likely(iv_len)) { 814 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 815 + OFF_CTRL_LEN); 816 uint64_t *src = fc_params->iv_buf; 817 dest[0] = src[0]; 818 dest[1] = src[1]; 819 } 820 821 *(uint64_t *)offset_vaddr = 822 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 823 ((uint64_t)iv_offset << 8) | 824 ((uint64_t)auth_offset)); 825 826 } else { 827 uint32_t i, g_size_bytes, s_size_bytes; 828 uint64_t dptr_dma, rptr_dma; 829 sg_comp_t *gather_comp; 830 sg_comp_t *scatter_comp; 831 uint8_t *in_buffer; 832 833 /* This falls under strict SG mode */ 834 offset_vaddr = m_vaddr; 835 offset_dma = m_dma; 836 size = OFF_CTRL_LEN + iv_len; 837 838 m_vaddr = (uint8_t *)m_vaddr + size; 839 m_dma += size; 840 841 opcode.s.major |= CPT_DMA_MODE; 842 843 vq_cmd_w0.s.opcode = opcode.flags; 844 845 if (likely(iv_len)) { 846 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 847 + OFF_CTRL_LEN); 848 uint64_t *src = fc_params->iv_buf; 849 dest[0] = src[0]; 850 dest[1] = src[1]; 851 } 852 853 *(uint64_t *)offset_vaddr = 854 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 855 ((uint64_t)iv_offset << 8) | 856 ((uint64_t)auth_offset)); 857 858 /* DPTR has SG list */ 859 in_buffer = m_vaddr; 860 dptr_dma = m_dma; 861 862 ((uint16_t *)in_buffer)[0] = 0; 863 ((uint16_t *)in_buffer)[1] = 0; 864 865 /* TODO Add error check if space will be sufficient */ 866 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 867 868 /* 869 * Input Gather List 870 */ 871 872 i = 0; 873 874 /* Offset control word that includes iv */ 875 i = fill_sg_comp(gather_comp, i, offset_dma, 876 OFF_CTRL_LEN + iv_len); 877 878 /* Add input data */ 879 size = inputlen - iv_len; 880 if (likely(size)) { 881 uint32_t aad_offset = aad_len ? passthrough_len : 0; 882 883 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 884 i = fill_sg_comp_from_buf_min(gather_comp, i, 885 fc_params->bufs, 886 &size); 887 } else { 888 i = fill_sg_comp_from_iov(gather_comp, i, 889 fc_params->src_iov, 890 0, &size, 891 aad_buf, aad_offset); 892 } 893 894 if (unlikely(size)) { 895 CPT_LOG_DP_ERR("Insufficient buffer space," 896 " size %d needed", size); 897 return; 898 } 899 } 900 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 901 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 902 903 /* 904 * Output Scatter list 905 */ 906 i = 0; 907 scatter_comp = 908 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 909 910 /* Add IV */ 911 if (likely(iv_len)) { 912 i = fill_sg_comp(scatter_comp, i, 913 offset_dma + OFF_CTRL_LEN, 914 iv_len); 915 } 916 917 /* output data or output data + digest*/ 918 if (unlikely(flags & VALID_MAC_BUF)) { 919 size = outputlen - iv_len - mac_len; 920 if (size) { 921 uint32_t aad_offset = 922 aad_len ? passthrough_len : 0; 923 924 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 925 i = fill_sg_comp_from_buf_min( 926 scatter_comp, 927 i, 928 fc_params->bufs, 929 &size); 930 } else { 931 i = fill_sg_comp_from_iov(scatter_comp, 932 i, 933 fc_params->dst_iov, 934 0, 935 &size, 936 aad_buf, 937 aad_offset); 938 } 939 if (unlikely(size)) { 940 CPT_LOG_DP_ERR("Insufficient buffer" 941 " space, size %d needed", 942 size); 943 return; 944 } 945 } 946 /* mac_data */ 947 if (mac_len) { 948 i = fill_sg_comp_from_buf(scatter_comp, i, 949 &fc_params->mac_buf); 950 } 951 } else { 952 /* Output including mac */ 953 size = outputlen - iv_len; 954 if (likely(size)) { 955 uint32_t aad_offset = 956 aad_len ? passthrough_len : 0; 957 958 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 959 i = fill_sg_comp_from_buf_min( 960 scatter_comp, 961 i, 962 fc_params->bufs, 963 &size); 964 } else { 965 i = fill_sg_comp_from_iov(scatter_comp, 966 i, 967 fc_params->dst_iov, 968 0, 969 &size, 970 aad_buf, 971 aad_offset); 972 } 973 if (unlikely(size)) { 974 CPT_LOG_DP_ERR("Insufficient buffer" 975 " space, size %d needed", 976 size); 977 return; 978 } 979 } 980 } 981 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 982 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 983 984 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 985 986 /* This is DPTR len incase of SG mode */ 987 vq_cmd_w0.s.dlen = size; 988 989 m_vaddr = (uint8_t *)m_vaddr + size; 990 m_dma += size; 991 992 /* cpt alternate completion address saved earlier */ 993 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 994 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 995 rptr_dma = c_dma - 8; 996 997 req->ist.ei1 = dptr_dma; 998 req->ist.ei2 = rptr_dma; 999 } 1000 1001 ctx_dma = fc_params->ctx_buf.dma_addr + 1002 offsetof(struct cpt_ctx, fctx); 1003 /* vq command w3 */ 1004 vq_cmd_w3.u64 = 0; 1005 vq_cmd_w3.s.grp = 0; 1006 vq_cmd_w3.s.cptr = ctx_dma; 1007 1008 /* 16 byte aligned cpt res address */ 1009 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1010 *req->completion_addr = COMPLETION_CODE_INIT; 1011 req->comp_baddr = c_dma; 1012 1013 /* Fill microcode part of instruction */ 1014 req->ist.ei0 = vq_cmd_w0.u64; 1015 req->ist.ei3 = vq_cmd_w3.u64; 1016 1017 req->op = op; 1018 1019 *prep_req = req; 1020 return; 1021 } 1022 1023 static __rte_always_inline void 1024 cpt_dec_hmac_prep(uint32_t flags, 1025 uint64_t d_offs, 1026 uint64_t d_lens, 1027 fc_params_t *fc_params, 1028 void *op, 1029 void **prep_req) 1030 { 1031 uint32_t iv_offset = 0, size; 1032 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 1033 struct cpt_ctx *cpt_ctx; 1034 int32_t hash_type, mac_len; 1035 uint8_t iv_len = 16; 1036 struct cpt_request_info *req; 1037 buf_ptr_t *meta_p, *aad_buf = NULL; 1038 uint32_t encr_offset, auth_offset; 1039 uint32_t encr_data_len, auth_data_len, aad_len = 0; 1040 uint32_t passthrough_len = 0; 1041 void *m_vaddr, *offset_vaddr; 1042 uint64_t m_dma, offset_dma, ctx_dma; 1043 opcode_info_t opcode; 1044 vq_cmd_word0_t vq_cmd_w0; 1045 vq_cmd_word3_t vq_cmd_w3; 1046 void *c_vaddr; 1047 uint64_t c_dma; 1048 1049 meta_p = &fc_params->meta_buf; 1050 m_vaddr = meta_p->vaddr; 1051 m_dma = meta_p->dma_addr; 1052 1053 encr_offset = ENCR_OFFSET(d_offs); 1054 auth_offset = AUTH_OFFSET(d_offs); 1055 encr_data_len = ENCR_DLEN(d_lens); 1056 auth_data_len = AUTH_DLEN(d_lens); 1057 1058 if (unlikely(flags & VALID_AAD_BUF)) { 1059 /* 1060 * We dont support both aad 1061 * and auth data separately 1062 */ 1063 auth_data_len = 0; 1064 auth_offset = 0; 1065 aad_len = fc_params->aad_buf.size; 1066 aad_buf = &fc_params->aad_buf; 1067 } 1068 1069 cpt_ctx = fc_params->ctx_buf.vaddr; 1070 hash_type = cpt_ctx->hash_type; 1071 mac_len = cpt_ctx->mac_len; 1072 1073 if (hash_type == GMAC_TYPE) 1074 encr_data_len = 0; 1075 1076 if (unlikely(!(flags & VALID_IV_BUF))) { 1077 iv_len = 0; 1078 iv_offset = ENCR_IV_OFFSET(d_offs); 1079 } 1080 1081 if (unlikely(flags & VALID_AAD_BUF)) { 1082 /* 1083 * When AAD is given, data above encr_offset is pass through 1084 * Since AAD is given as separate pointer and not as offset, 1085 * this is a special case as we need to fragment input data 1086 * into passthrough + encr_data and then insert AAD in between. 1087 */ 1088 if (hash_type != GMAC_TYPE) { 1089 passthrough_len = encr_offset; 1090 auth_offset = passthrough_len + iv_len; 1091 encr_offset = passthrough_len + aad_len + iv_len; 1092 auth_data_len = aad_len + encr_data_len; 1093 } else { 1094 passthrough_len = 16 + aad_len; 1095 auth_offset = passthrough_len + iv_len; 1096 auth_data_len = aad_len; 1097 } 1098 } else { 1099 encr_offset += iv_len; 1100 auth_offset += iv_len; 1101 } 1102 1103 /* 1104 * Save initial space that followed app data for completion code & 1105 * alternate completion code to fall in same cache line as app data 1106 */ 1107 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1108 m_dma += COMPLETION_CODE_SIZE; 1109 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1110 (uint8_t *)m_vaddr; 1111 c_vaddr = (uint8_t *)m_vaddr + size; 1112 c_dma = m_dma + size; 1113 size += sizeof(cpt_res_s_t); 1114 1115 m_vaddr = (uint8_t *)m_vaddr + size; 1116 m_dma += size; 1117 1118 /* start cpt request info structure at 8 byte alignment */ 1119 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 1120 (uint8_t *)m_vaddr; 1121 1122 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 1123 1124 size += sizeof(struct cpt_request_info); 1125 m_vaddr = (uint8_t *)m_vaddr + size; 1126 m_dma += size; 1127 1128 /* Decryption */ 1129 opcode.s.major = CPT_MAJOR_OP_FC; 1130 opcode.s.minor = 1; 1131 1132 enc_dlen = encr_offset + encr_data_len; 1133 auth_dlen = auth_offset + auth_data_len; 1134 1135 if (auth_dlen > enc_dlen) { 1136 inputlen = auth_dlen + mac_len; 1137 outputlen = auth_dlen; 1138 } else { 1139 inputlen = enc_dlen + mac_len; 1140 outputlen = enc_dlen; 1141 } 1142 1143 if (hash_type == GMAC_TYPE) 1144 encr_offset = inputlen; 1145 1146 vq_cmd_w0.u64 = 0; 1147 vq_cmd_w0.s.param1 = encr_data_len; 1148 vq_cmd_w0.s.param2 = auth_data_len; 1149 1150 /* 1151 * In 83XX since we have a limitation of 1152 * IV & Offset control word not part of instruction 1153 * and need to be part of Data Buffer, we check if 1154 * head room is there and then only do the Direct mode processing 1155 */ 1156 if (likely((flags & SINGLE_BUF_INPLACE) && 1157 (flags & SINGLE_BUF_HEADTAILROOM))) { 1158 void *dm_vaddr = fc_params->bufs[0].vaddr; 1159 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 1160 /* 1161 * This flag indicates that there is 24 bytes head room and 1162 * 8 bytes tail room available, so that we get to do 1163 * DIRECT MODE with limitation 1164 */ 1165 1166 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 1167 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1168 req->ist.ei1 = offset_dma; 1169 1170 /* RPTR should just exclude offset control word */ 1171 req->ist.ei2 = dm_dma_addr - iv_len; 1172 1173 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr + 1174 outputlen - iv_len); 1175 /* since this is decryption, 1176 * don't touch the content of 1177 * alternate ccode space as it contains 1178 * hmac. 1179 */ 1180 1181 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1182 1183 vq_cmd_w0.s.opcode = opcode.flags; 1184 1185 if (likely(iv_len)) { 1186 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1187 OFF_CTRL_LEN); 1188 uint64_t *src = fc_params->iv_buf; 1189 dest[0] = src[0]; 1190 dest[1] = src[1]; 1191 } 1192 1193 *(uint64_t *)offset_vaddr = 1194 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1195 ((uint64_t)iv_offset << 8) | 1196 ((uint64_t)auth_offset)); 1197 1198 } else { 1199 uint64_t dptr_dma, rptr_dma; 1200 uint32_t g_size_bytes, s_size_bytes; 1201 sg_comp_t *gather_comp; 1202 sg_comp_t *scatter_comp; 1203 uint8_t *in_buffer; 1204 uint8_t i = 0; 1205 1206 /* This falls under strict SG mode */ 1207 offset_vaddr = m_vaddr; 1208 offset_dma = m_dma; 1209 size = OFF_CTRL_LEN + iv_len; 1210 1211 m_vaddr = (uint8_t *)m_vaddr + size; 1212 m_dma += size; 1213 1214 opcode.s.major |= CPT_DMA_MODE; 1215 1216 vq_cmd_w0.s.opcode = opcode.flags; 1217 1218 if (likely(iv_len)) { 1219 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1220 OFF_CTRL_LEN); 1221 uint64_t *src = fc_params->iv_buf; 1222 dest[0] = src[0]; 1223 dest[1] = src[1]; 1224 } 1225 1226 *(uint64_t *)offset_vaddr = 1227 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1228 ((uint64_t)iv_offset << 8) | 1229 ((uint64_t)auth_offset)); 1230 1231 /* DPTR has SG list */ 1232 in_buffer = m_vaddr; 1233 dptr_dma = m_dma; 1234 1235 ((uint16_t *)in_buffer)[0] = 0; 1236 ((uint16_t *)in_buffer)[1] = 0; 1237 1238 /* TODO Add error check if space will be sufficient */ 1239 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1240 1241 /* 1242 * Input Gather List 1243 */ 1244 i = 0; 1245 1246 /* Offset control word that includes iv */ 1247 i = fill_sg_comp(gather_comp, i, offset_dma, 1248 OFF_CTRL_LEN + iv_len); 1249 1250 /* Add input data */ 1251 if (flags & VALID_MAC_BUF) { 1252 size = inputlen - iv_len - mac_len; 1253 if (size) { 1254 /* input data only */ 1255 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1256 i = fill_sg_comp_from_buf_min( 1257 gather_comp, i, 1258 fc_params->bufs, 1259 &size); 1260 } else { 1261 uint32_t aad_offset = aad_len ? 1262 passthrough_len : 0; 1263 1264 i = fill_sg_comp_from_iov(gather_comp, 1265 i, 1266 fc_params->src_iov, 1267 0, &size, 1268 aad_buf, 1269 aad_offset); 1270 } 1271 if (unlikely(size)) { 1272 CPT_LOG_DP_ERR("Insufficient buffer" 1273 " space, size %d needed", 1274 size); 1275 return; 1276 } 1277 } 1278 1279 /* mac data */ 1280 if (mac_len) { 1281 i = fill_sg_comp_from_buf(gather_comp, i, 1282 &fc_params->mac_buf); 1283 } 1284 } else { 1285 /* input data + mac */ 1286 size = inputlen - iv_len; 1287 if (size) { 1288 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1289 i = fill_sg_comp_from_buf_min( 1290 gather_comp, i, 1291 fc_params->bufs, 1292 &size); 1293 } else { 1294 uint32_t aad_offset = aad_len ? 1295 passthrough_len : 0; 1296 1297 if (unlikely(!fc_params->src_iov)) { 1298 CPT_LOG_DP_ERR("Bad input args"); 1299 return; 1300 } 1301 1302 i = fill_sg_comp_from_iov( 1303 gather_comp, i, 1304 fc_params->src_iov, 1305 0, &size, 1306 aad_buf, 1307 aad_offset); 1308 } 1309 1310 if (unlikely(size)) { 1311 CPT_LOG_DP_ERR("Insufficient buffer" 1312 " space, size %d needed", 1313 size); 1314 return; 1315 } 1316 } 1317 } 1318 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1319 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1320 1321 /* 1322 * Output Scatter List 1323 */ 1324 1325 i = 0; 1326 scatter_comp = 1327 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1328 1329 /* Add iv */ 1330 if (iv_len) { 1331 i = fill_sg_comp(scatter_comp, i, 1332 offset_dma + OFF_CTRL_LEN, 1333 iv_len); 1334 } 1335 1336 /* Add output data */ 1337 size = outputlen - iv_len; 1338 if (size) { 1339 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1340 /* handle single buffer here */ 1341 i = fill_sg_comp_from_buf_min(scatter_comp, i, 1342 fc_params->bufs, 1343 &size); 1344 } else { 1345 uint32_t aad_offset = aad_len ? 1346 passthrough_len : 0; 1347 1348 if (unlikely(!fc_params->dst_iov)) { 1349 CPT_LOG_DP_ERR("Bad input args"); 1350 return; 1351 } 1352 1353 i = fill_sg_comp_from_iov(scatter_comp, i, 1354 fc_params->dst_iov, 0, 1355 &size, aad_buf, 1356 aad_offset); 1357 } 1358 1359 if (unlikely(size)) { 1360 CPT_LOG_DP_ERR("Insufficient buffer space," 1361 " size %d needed", size); 1362 return; 1363 } 1364 } 1365 1366 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1367 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1368 1369 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1370 1371 /* This is DPTR len incase of SG mode */ 1372 vq_cmd_w0.s.dlen = size; 1373 1374 m_vaddr = (uint8_t *)m_vaddr + size; 1375 m_dma += size; 1376 1377 /* cpt alternate completion address saved earlier */ 1378 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1379 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1380 rptr_dma = c_dma - 8; 1381 size += COMPLETION_CODE_SIZE; 1382 1383 req->ist.ei1 = dptr_dma; 1384 req->ist.ei2 = rptr_dma; 1385 } 1386 1387 ctx_dma = fc_params->ctx_buf.dma_addr + 1388 offsetof(struct cpt_ctx, fctx); 1389 /* vq command w3 */ 1390 vq_cmd_w3.u64 = 0; 1391 vq_cmd_w3.s.grp = 0; 1392 vq_cmd_w3.s.cptr = ctx_dma; 1393 1394 /* 16 byte aligned cpt res address */ 1395 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1396 *req->completion_addr = COMPLETION_CODE_INIT; 1397 req->comp_baddr = c_dma; 1398 1399 /* Fill microcode part of instruction */ 1400 req->ist.ei0 = vq_cmd_w0.u64; 1401 req->ist.ei3 = vq_cmd_w3.u64; 1402 1403 req->op = op; 1404 1405 *prep_req = req; 1406 return; 1407 } 1408 1409 static __rte_always_inline void 1410 cpt_zuc_snow3g_enc_prep(uint32_t req_flags, 1411 uint64_t d_offs, 1412 uint64_t d_lens, 1413 fc_params_t *params, 1414 void *op, 1415 void **prep_req) 1416 { 1417 uint32_t size; 1418 int32_t inputlen, outputlen; 1419 struct cpt_ctx *cpt_ctx; 1420 uint32_t mac_len = 0; 1421 uint8_t snow3g, j; 1422 struct cpt_request_info *req; 1423 buf_ptr_t *buf_p; 1424 uint32_t encr_offset = 0, auth_offset = 0; 1425 uint32_t encr_data_len = 0, auth_data_len = 0; 1426 int flags, iv_len = 16; 1427 void *m_vaddr, *c_vaddr; 1428 uint64_t m_dma, c_dma, offset_ctrl; 1429 uint64_t *offset_vaddr, offset_dma; 1430 uint32_t *iv_s, iv[4]; 1431 vq_cmd_word0_t vq_cmd_w0; 1432 vq_cmd_word3_t vq_cmd_w3; 1433 opcode_info_t opcode; 1434 1435 buf_p = ¶ms->meta_buf; 1436 m_vaddr = buf_p->vaddr; 1437 m_dma = buf_p->dma_addr; 1438 1439 cpt_ctx = params->ctx_buf.vaddr; 1440 flags = cpt_ctx->zsk_flags; 1441 mac_len = cpt_ctx->mac_len; 1442 snow3g = cpt_ctx->snow3g; 1443 1444 /* 1445 * Save initial space that followed app data for completion code & 1446 * alternate completion code to fall in same cache line as app data 1447 */ 1448 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1449 m_dma += COMPLETION_CODE_SIZE; 1450 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1451 (uint8_t *)m_vaddr; 1452 1453 c_vaddr = (uint8_t *)m_vaddr + size; 1454 c_dma = m_dma + size; 1455 size += sizeof(cpt_res_s_t); 1456 1457 m_vaddr = (uint8_t *)m_vaddr + size; 1458 m_dma += size; 1459 1460 /* Reserve memory for cpt request info */ 1461 req = m_vaddr; 1462 1463 size = sizeof(struct cpt_request_info); 1464 m_vaddr = (uint8_t *)m_vaddr + size; 1465 m_dma += size; 1466 1467 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1468 1469 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1470 1471 opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1472 (0 << 3) | (flags & 0x7)); 1473 1474 if (flags == 0x1) { 1475 /* 1476 * Microcode expects offsets in bytes 1477 * TODO: Rounding off 1478 */ 1479 auth_data_len = AUTH_DLEN(d_lens); 1480 1481 /* EIA3 or UIA2 */ 1482 auth_offset = AUTH_OFFSET(d_offs); 1483 auth_offset = auth_offset / 8; 1484 1485 /* consider iv len */ 1486 auth_offset += iv_len; 1487 1488 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 1489 outputlen = mac_len; 1490 1491 offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset); 1492 1493 } else { 1494 /* EEA3 or UEA2 */ 1495 /* 1496 * Microcode expects offsets in bytes 1497 * TODO: Rounding off 1498 */ 1499 encr_data_len = ENCR_DLEN(d_lens); 1500 1501 encr_offset = ENCR_OFFSET(d_offs); 1502 encr_offset = encr_offset / 8; 1503 /* consider iv len */ 1504 encr_offset += iv_len; 1505 1506 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 1507 outputlen = inputlen; 1508 1509 /* iv offset is 0 */ 1510 offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1511 } 1512 1513 /* IV */ 1514 iv_s = (flags == 0x1) ? params->auth_iv_buf : 1515 params->iv_buf; 1516 1517 if (snow3g) { 1518 /* 1519 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1520 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1521 */ 1522 1523 for (j = 0; j < 4; j++) 1524 iv[j] = iv_s[3 - j]; 1525 } else { 1526 /* ZUC doesn't need a swap */ 1527 for (j = 0; j < 4; j++) 1528 iv[j] = iv_s[j]; 1529 } 1530 1531 /* 1532 * GP op header, lengths are expected in bits. 1533 */ 1534 vq_cmd_w0.u64 = 0; 1535 vq_cmd_w0.s.param1 = encr_data_len; 1536 vq_cmd_w0.s.param2 = auth_data_len; 1537 1538 /* 1539 * In 83XX since we have a limitation of 1540 * IV & Offset control word not part of instruction 1541 * and need to be part of Data Buffer, we check if 1542 * head room is there and then only do the Direct mode processing 1543 */ 1544 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1545 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1546 void *dm_vaddr = params->bufs[0].vaddr; 1547 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1548 /* 1549 * This flag indicates that there is 24 bytes head room and 1550 * 8 bytes tail room available, so that we get to do 1551 * DIRECT MODE with limitation 1552 */ 1553 1554 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1555 OFF_CTRL_LEN - iv_len); 1556 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1557 1558 /* DPTR */ 1559 req->ist.ei1 = offset_dma; 1560 /* RPTR should just exclude offset control word */ 1561 req->ist.ei2 = dm_dma_addr - iv_len; 1562 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1563 + outputlen - iv_len); 1564 1565 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1566 1567 vq_cmd_w0.s.opcode = opcode.flags; 1568 1569 if (likely(iv_len)) { 1570 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1571 + OFF_CTRL_LEN); 1572 memcpy(iv_d, iv, 16); 1573 } 1574 1575 *offset_vaddr = offset_ctrl; 1576 } else { 1577 uint32_t i, g_size_bytes, s_size_bytes; 1578 uint64_t dptr_dma, rptr_dma; 1579 sg_comp_t *gather_comp; 1580 sg_comp_t *scatter_comp; 1581 uint8_t *in_buffer; 1582 uint32_t *iv_d; 1583 1584 /* save space for iv */ 1585 offset_vaddr = m_vaddr; 1586 offset_dma = m_dma; 1587 1588 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1589 m_dma += OFF_CTRL_LEN + iv_len; 1590 1591 opcode.s.major |= CPT_DMA_MODE; 1592 1593 vq_cmd_w0.s.opcode = opcode.flags; 1594 1595 /* DPTR has SG list */ 1596 in_buffer = m_vaddr; 1597 dptr_dma = m_dma; 1598 1599 ((uint16_t *)in_buffer)[0] = 0; 1600 ((uint16_t *)in_buffer)[1] = 0; 1601 1602 /* TODO Add error check if space will be sufficient */ 1603 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1604 1605 /* 1606 * Input Gather List 1607 */ 1608 i = 0; 1609 1610 /* Offset control word followed by iv */ 1611 1612 i = fill_sg_comp(gather_comp, i, offset_dma, 1613 OFF_CTRL_LEN + iv_len); 1614 1615 /* iv offset is 0 */ 1616 *offset_vaddr = offset_ctrl; 1617 1618 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1619 memcpy(iv_d, iv, 16); 1620 1621 /* input data */ 1622 size = inputlen - iv_len; 1623 if (size) { 1624 i = fill_sg_comp_from_iov(gather_comp, i, 1625 params->src_iov, 1626 0, &size, NULL, 0); 1627 if (unlikely(size)) { 1628 CPT_LOG_DP_ERR("Insufficient buffer space," 1629 " size %d needed", size); 1630 return; 1631 } 1632 } 1633 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1634 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1635 1636 /* 1637 * Output Scatter List 1638 */ 1639 1640 i = 0; 1641 scatter_comp = 1642 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1643 1644 if (flags == 0x1) { 1645 /* IV in SLIST only for EEA3 & UEA2 */ 1646 iv_len = 0; 1647 } 1648 1649 if (iv_len) { 1650 i = fill_sg_comp(scatter_comp, i, 1651 offset_dma + OFF_CTRL_LEN, iv_len); 1652 } 1653 1654 /* Add output data */ 1655 if (req_flags & VALID_MAC_BUF) { 1656 size = outputlen - iv_len - mac_len; 1657 if (size) { 1658 i = fill_sg_comp_from_iov(scatter_comp, i, 1659 params->dst_iov, 0, 1660 &size, NULL, 0); 1661 1662 if (unlikely(size)) { 1663 CPT_LOG_DP_ERR("Insufficient buffer space," 1664 " size %d needed", size); 1665 return; 1666 } 1667 } 1668 1669 /* mac data */ 1670 if (mac_len) { 1671 i = fill_sg_comp_from_buf(scatter_comp, i, 1672 ¶ms->mac_buf); 1673 } 1674 } else { 1675 /* Output including mac */ 1676 size = outputlen - iv_len; 1677 if (size) { 1678 i = fill_sg_comp_from_iov(scatter_comp, i, 1679 params->dst_iov, 0, 1680 &size, NULL, 0); 1681 1682 if (unlikely(size)) { 1683 CPT_LOG_DP_ERR("Insufficient buffer space," 1684 " size %d needed", size); 1685 return; 1686 } 1687 } 1688 } 1689 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1690 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1691 1692 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1693 1694 /* This is DPTR len incase of SG mode */ 1695 vq_cmd_w0.s.dlen = size; 1696 1697 m_vaddr = (uint8_t *)m_vaddr + size; 1698 m_dma += size; 1699 1700 /* cpt alternate completion address saved earlier */ 1701 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1702 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1703 rptr_dma = c_dma - 8; 1704 1705 req->ist.ei1 = dptr_dma; 1706 req->ist.ei2 = rptr_dma; 1707 } 1708 1709 /* vq command w3 */ 1710 vq_cmd_w3.u64 = 0; 1711 vq_cmd_w3.s.grp = 0; 1712 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1713 offsetof(struct cpt_ctx, zs_ctx); 1714 1715 /* 16 byte aligned cpt res address */ 1716 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1717 *req->completion_addr = COMPLETION_CODE_INIT; 1718 req->comp_baddr = c_dma; 1719 1720 /* Fill microcode part of instruction */ 1721 req->ist.ei0 = vq_cmd_w0.u64; 1722 req->ist.ei3 = vq_cmd_w3.u64; 1723 1724 req->op = op; 1725 1726 *prep_req = req; 1727 return; 1728 } 1729 1730 static __rte_always_inline void 1731 cpt_zuc_snow3g_dec_prep(uint32_t req_flags, 1732 uint64_t d_offs, 1733 uint64_t d_lens, 1734 fc_params_t *params, 1735 void *op, 1736 void **prep_req) 1737 { 1738 uint32_t size; 1739 int32_t inputlen = 0, outputlen; 1740 struct cpt_ctx *cpt_ctx; 1741 uint8_t snow3g, iv_len = 16; 1742 struct cpt_request_info *req; 1743 buf_ptr_t *buf_p; 1744 uint32_t encr_offset; 1745 uint32_t encr_data_len; 1746 int flags; 1747 void *m_vaddr, *c_vaddr; 1748 uint64_t m_dma, c_dma; 1749 uint64_t *offset_vaddr, offset_dma; 1750 uint32_t *iv_s, iv[4], j; 1751 vq_cmd_word0_t vq_cmd_w0; 1752 vq_cmd_word3_t vq_cmd_w3; 1753 opcode_info_t opcode; 1754 1755 buf_p = ¶ms->meta_buf; 1756 m_vaddr = buf_p->vaddr; 1757 m_dma = buf_p->dma_addr; 1758 1759 /* 1760 * Microcode expects offsets in bytes 1761 * TODO: Rounding off 1762 */ 1763 encr_offset = ENCR_OFFSET(d_offs) / 8; 1764 encr_data_len = ENCR_DLEN(d_lens); 1765 1766 cpt_ctx = params->ctx_buf.vaddr; 1767 flags = cpt_ctx->zsk_flags; 1768 snow3g = cpt_ctx->snow3g; 1769 /* 1770 * Save initial space that followed app data for completion code & 1771 * alternate completion code to fall in same cache line as app data 1772 */ 1773 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1774 m_dma += COMPLETION_CODE_SIZE; 1775 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1776 (uint8_t *)m_vaddr; 1777 1778 c_vaddr = (uint8_t *)m_vaddr + size; 1779 c_dma = m_dma + size; 1780 size += sizeof(cpt_res_s_t); 1781 1782 m_vaddr = (uint8_t *)m_vaddr + size; 1783 m_dma += size; 1784 1785 /* Reserve memory for cpt request info */ 1786 req = m_vaddr; 1787 1788 size = sizeof(struct cpt_request_info); 1789 m_vaddr = (uint8_t *)m_vaddr + size; 1790 m_dma += size; 1791 1792 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1793 1794 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1795 1796 opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1797 (0 << 3) | (flags & 0x7)); 1798 1799 /* consider iv len */ 1800 encr_offset += iv_len; 1801 1802 inputlen = encr_offset + 1803 (RTE_ALIGN(encr_data_len, 8) / 8); 1804 outputlen = inputlen; 1805 1806 /* IV */ 1807 iv_s = params->iv_buf; 1808 if (snow3g) { 1809 /* 1810 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1811 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1812 */ 1813 1814 for (j = 0; j < 4; j++) 1815 iv[j] = iv_s[3 - j]; 1816 } else { 1817 /* ZUC doesn't need a swap */ 1818 for (j = 0; j < 4; j++) 1819 iv[j] = iv_s[j]; 1820 } 1821 1822 /* 1823 * GP op header, lengths are expected in bits. 1824 */ 1825 vq_cmd_w0.u64 = 0; 1826 vq_cmd_w0.s.param1 = encr_data_len; 1827 1828 /* 1829 * In 83XX since we have a limitation of 1830 * IV & Offset control word not part of instruction 1831 * and need to be part of Data Buffer, we check if 1832 * head room is there and then only do the Direct mode processing 1833 */ 1834 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1835 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1836 void *dm_vaddr = params->bufs[0].vaddr; 1837 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1838 /* 1839 * This flag indicates that there is 24 bytes head room and 1840 * 8 bytes tail room available, so that we get to do 1841 * DIRECT MODE with limitation 1842 */ 1843 1844 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1845 OFF_CTRL_LEN - iv_len); 1846 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1847 1848 /* DPTR */ 1849 req->ist.ei1 = offset_dma; 1850 /* RPTR should just exclude offset control word */ 1851 req->ist.ei2 = dm_dma_addr - iv_len; 1852 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1853 + outputlen - iv_len); 1854 1855 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1856 1857 vq_cmd_w0.s.opcode = opcode.flags; 1858 1859 if (likely(iv_len)) { 1860 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1861 + OFF_CTRL_LEN); 1862 memcpy(iv_d, iv, 16); 1863 } 1864 1865 /* iv offset is 0 */ 1866 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1867 } else { 1868 uint32_t i, g_size_bytes, s_size_bytes; 1869 uint64_t dptr_dma, rptr_dma; 1870 sg_comp_t *gather_comp; 1871 sg_comp_t *scatter_comp; 1872 uint8_t *in_buffer; 1873 uint32_t *iv_d; 1874 1875 /* save space for offset and iv... */ 1876 offset_vaddr = m_vaddr; 1877 offset_dma = m_dma; 1878 1879 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1880 m_dma += OFF_CTRL_LEN + iv_len; 1881 1882 opcode.s.major |= CPT_DMA_MODE; 1883 1884 vq_cmd_w0.s.opcode = opcode.flags; 1885 1886 /* DPTR has SG list */ 1887 in_buffer = m_vaddr; 1888 dptr_dma = m_dma; 1889 1890 ((uint16_t *)in_buffer)[0] = 0; 1891 ((uint16_t *)in_buffer)[1] = 0; 1892 1893 /* TODO Add error check if space will be sufficient */ 1894 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1895 1896 /* 1897 * Input Gather List 1898 */ 1899 i = 0; 1900 1901 /* Offset control word */ 1902 1903 /* iv offset is 0 */ 1904 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1905 1906 i = fill_sg_comp(gather_comp, i, offset_dma, 1907 OFF_CTRL_LEN + iv_len); 1908 1909 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1910 memcpy(iv_d, iv, 16); 1911 1912 /* Add input data */ 1913 size = inputlen - iv_len; 1914 if (size) { 1915 i = fill_sg_comp_from_iov(gather_comp, i, 1916 params->src_iov, 1917 0, &size, NULL, 0); 1918 if (unlikely(size)) { 1919 CPT_LOG_DP_ERR("Insufficient buffer space," 1920 " size %d needed", size); 1921 return; 1922 } 1923 } 1924 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1925 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1926 1927 /* 1928 * Output Scatter List 1929 */ 1930 1931 i = 0; 1932 scatter_comp = 1933 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1934 1935 /* IV */ 1936 i = fill_sg_comp(scatter_comp, i, 1937 offset_dma + OFF_CTRL_LEN, 1938 iv_len); 1939 1940 /* Add output data */ 1941 size = outputlen - iv_len; 1942 if (size) { 1943 i = fill_sg_comp_from_iov(scatter_comp, i, 1944 params->dst_iov, 0, 1945 &size, NULL, 0); 1946 1947 if (unlikely(size)) { 1948 CPT_LOG_DP_ERR("Insufficient buffer space," 1949 " size %d needed", size); 1950 return; 1951 } 1952 } 1953 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1954 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1955 1956 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1957 1958 /* This is DPTR len incase of SG mode */ 1959 vq_cmd_w0.s.dlen = size; 1960 1961 m_vaddr = (uint8_t *)m_vaddr + size; 1962 m_dma += size; 1963 1964 /* cpt alternate completion address saved earlier */ 1965 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1966 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1967 rptr_dma = c_dma - 8; 1968 1969 req->ist.ei1 = dptr_dma; 1970 req->ist.ei2 = rptr_dma; 1971 } 1972 1973 /* vq command w3 */ 1974 vq_cmd_w3.u64 = 0; 1975 vq_cmd_w3.s.grp = 0; 1976 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1977 offsetof(struct cpt_ctx, zs_ctx); 1978 1979 /* 16 byte aligned cpt res address */ 1980 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1981 *req->completion_addr = COMPLETION_CODE_INIT; 1982 req->comp_baddr = c_dma; 1983 1984 /* Fill microcode part of instruction */ 1985 req->ist.ei0 = vq_cmd_w0.u64; 1986 req->ist.ei3 = vq_cmd_w3.u64; 1987 1988 req->op = op; 1989 1990 *prep_req = req; 1991 return; 1992 } 1993 1994 static __rte_always_inline void 1995 cpt_kasumi_enc_prep(uint32_t req_flags, 1996 uint64_t d_offs, 1997 uint64_t d_lens, 1998 fc_params_t *params, 1999 void *op, 2000 void **prep_req) 2001 { 2002 uint32_t size; 2003 int32_t inputlen = 0, outputlen = 0; 2004 struct cpt_ctx *cpt_ctx; 2005 uint32_t mac_len = 0; 2006 uint8_t i = 0; 2007 struct cpt_request_info *req; 2008 buf_ptr_t *buf_p; 2009 uint32_t encr_offset, auth_offset; 2010 uint32_t encr_data_len, auth_data_len; 2011 int flags; 2012 uint8_t *iv_s, *iv_d, iv_len = 8; 2013 uint8_t dir = 0; 2014 void *m_vaddr, *c_vaddr; 2015 uint64_t m_dma, c_dma; 2016 uint64_t *offset_vaddr, offset_dma; 2017 vq_cmd_word0_t vq_cmd_w0; 2018 vq_cmd_word3_t vq_cmd_w3; 2019 opcode_info_t opcode; 2020 uint8_t *in_buffer; 2021 uint32_t g_size_bytes, s_size_bytes; 2022 uint64_t dptr_dma, rptr_dma; 2023 sg_comp_t *gather_comp; 2024 sg_comp_t *scatter_comp; 2025 2026 buf_p = ¶ms->meta_buf; 2027 m_vaddr = buf_p->vaddr; 2028 m_dma = buf_p->dma_addr; 2029 2030 encr_offset = ENCR_OFFSET(d_offs) / 8; 2031 auth_offset = AUTH_OFFSET(d_offs) / 8; 2032 encr_data_len = ENCR_DLEN(d_lens); 2033 auth_data_len = AUTH_DLEN(d_lens); 2034 2035 cpt_ctx = params->ctx_buf.vaddr; 2036 flags = cpt_ctx->zsk_flags; 2037 mac_len = cpt_ctx->mac_len; 2038 2039 if (flags == 0x0) 2040 iv_s = params->iv_buf; 2041 else 2042 iv_s = params->auth_iv_buf; 2043 2044 dir = iv_s[8] & 0x1; 2045 2046 /* 2047 * Save initial space that followed app data for completion code & 2048 * alternate completion code to fall in same cache line as app data 2049 */ 2050 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2051 m_dma += COMPLETION_CODE_SIZE; 2052 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2053 (uint8_t *)m_vaddr; 2054 2055 c_vaddr = (uint8_t *)m_vaddr + size; 2056 c_dma = m_dma + size; 2057 size += sizeof(cpt_res_s_t); 2058 2059 m_vaddr = (uint8_t *)m_vaddr + size; 2060 m_dma += size; 2061 2062 /* Reserve memory for cpt request info */ 2063 req = m_vaddr; 2064 2065 size = sizeof(struct cpt_request_info); 2066 m_vaddr = (uint8_t *)m_vaddr + size; 2067 m_dma += size; 2068 2069 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2070 2071 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2072 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2073 (dir << 4) | (0 << 3) | (flags & 0x7)); 2074 2075 /* 2076 * GP op header, lengths are expected in bits. 2077 */ 2078 vq_cmd_w0.u64 = 0; 2079 vq_cmd_w0.s.param1 = encr_data_len; 2080 vq_cmd_w0.s.param2 = auth_data_len; 2081 vq_cmd_w0.s.opcode = opcode.flags; 2082 2083 /* consider iv len */ 2084 if (flags == 0x0) { 2085 encr_offset += iv_len; 2086 auth_offset += iv_len; 2087 } 2088 2089 /* save space for offset ctrl and iv */ 2090 offset_vaddr = m_vaddr; 2091 offset_dma = m_dma; 2092 2093 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2094 m_dma += OFF_CTRL_LEN + iv_len; 2095 2096 /* DPTR has SG list */ 2097 in_buffer = m_vaddr; 2098 dptr_dma = m_dma; 2099 2100 ((uint16_t *)in_buffer)[0] = 0; 2101 ((uint16_t *)in_buffer)[1] = 0; 2102 2103 /* TODO Add error check if space will be sufficient */ 2104 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2105 2106 /* 2107 * Input Gather List 2108 */ 2109 i = 0; 2110 2111 /* Offset control word followed by iv */ 2112 2113 if (flags == 0x0) { 2114 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 2115 outputlen = inputlen; 2116 /* iv offset is 0 */ 2117 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2118 } else { 2119 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 2120 outputlen = mac_len; 2121 /* iv offset is 0 */ 2122 *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset); 2123 } 2124 2125 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2126 2127 /* IV */ 2128 iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN; 2129 memcpy(iv_d, iv_s, iv_len); 2130 2131 /* input data */ 2132 size = inputlen - iv_len; 2133 if (size) { 2134 i = fill_sg_comp_from_iov(gather_comp, i, 2135 params->src_iov, 0, 2136 &size, NULL, 0); 2137 2138 if (unlikely(size)) { 2139 CPT_LOG_DP_ERR("Insufficient buffer space," 2140 " size %d needed", size); 2141 return; 2142 } 2143 } 2144 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2145 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2146 2147 /* 2148 * Output Scatter List 2149 */ 2150 2151 i = 0; 2152 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2153 2154 if (flags == 0x1) { 2155 /* IV in SLIST only for F8 */ 2156 iv_len = 0; 2157 } 2158 2159 /* IV */ 2160 if (iv_len) { 2161 i = fill_sg_comp(scatter_comp, i, 2162 offset_dma + OFF_CTRL_LEN, 2163 iv_len); 2164 } 2165 2166 /* Add output data */ 2167 if (req_flags & VALID_MAC_BUF) { 2168 size = outputlen - iv_len - mac_len; 2169 if (size) { 2170 i = fill_sg_comp_from_iov(scatter_comp, i, 2171 params->dst_iov, 0, 2172 &size, NULL, 0); 2173 2174 if (unlikely(size)) { 2175 CPT_LOG_DP_ERR("Insufficient buffer space," 2176 " size %d needed", size); 2177 return; 2178 } 2179 } 2180 2181 /* mac data */ 2182 if (mac_len) { 2183 i = fill_sg_comp_from_buf(scatter_comp, i, 2184 ¶ms->mac_buf); 2185 } 2186 } else { 2187 /* Output including mac */ 2188 size = outputlen - iv_len; 2189 if (size) { 2190 i = fill_sg_comp_from_iov(scatter_comp, i, 2191 params->dst_iov, 0, 2192 &size, NULL, 0); 2193 2194 if (unlikely(size)) { 2195 CPT_LOG_DP_ERR("Insufficient buffer space," 2196 " size %d needed", size); 2197 return; 2198 } 2199 } 2200 } 2201 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2202 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2203 2204 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2205 2206 /* This is DPTR len incase of SG mode */ 2207 vq_cmd_w0.s.dlen = size; 2208 2209 m_vaddr = (uint8_t *)m_vaddr + size; 2210 m_dma += size; 2211 2212 /* cpt alternate completion address saved earlier */ 2213 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2214 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2215 rptr_dma = c_dma - 8; 2216 2217 req->ist.ei1 = dptr_dma; 2218 req->ist.ei2 = rptr_dma; 2219 2220 /* vq command w3 */ 2221 vq_cmd_w3.u64 = 0; 2222 vq_cmd_w3.s.grp = 0; 2223 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2224 offsetof(struct cpt_ctx, k_ctx); 2225 2226 /* 16 byte aligned cpt res address */ 2227 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2228 *req->completion_addr = COMPLETION_CODE_INIT; 2229 req->comp_baddr = c_dma; 2230 2231 /* Fill microcode part of instruction */ 2232 req->ist.ei0 = vq_cmd_w0.u64; 2233 req->ist.ei3 = vq_cmd_w3.u64; 2234 2235 req->op = op; 2236 2237 *prep_req = req; 2238 return; 2239 } 2240 2241 static __rte_always_inline void 2242 cpt_kasumi_dec_prep(uint64_t d_offs, 2243 uint64_t d_lens, 2244 fc_params_t *params, 2245 void *op, 2246 void **prep_req) 2247 { 2248 uint32_t size; 2249 int32_t inputlen = 0, outputlen; 2250 struct cpt_ctx *cpt_ctx; 2251 uint8_t i = 0, iv_len = 8; 2252 struct cpt_request_info *req; 2253 buf_ptr_t *buf_p; 2254 uint32_t encr_offset; 2255 uint32_t encr_data_len; 2256 int flags; 2257 uint8_t dir = 0; 2258 void *m_vaddr, *c_vaddr; 2259 uint64_t m_dma, c_dma; 2260 uint64_t *offset_vaddr, offset_dma; 2261 vq_cmd_word0_t vq_cmd_w0; 2262 vq_cmd_word3_t vq_cmd_w3; 2263 opcode_info_t opcode; 2264 uint8_t *in_buffer; 2265 uint32_t g_size_bytes, s_size_bytes; 2266 uint64_t dptr_dma, rptr_dma; 2267 sg_comp_t *gather_comp; 2268 sg_comp_t *scatter_comp; 2269 2270 buf_p = ¶ms->meta_buf; 2271 m_vaddr = buf_p->vaddr; 2272 m_dma = buf_p->dma_addr; 2273 2274 encr_offset = ENCR_OFFSET(d_offs) / 8; 2275 encr_data_len = ENCR_DLEN(d_lens); 2276 2277 cpt_ctx = params->ctx_buf.vaddr; 2278 flags = cpt_ctx->zsk_flags; 2279 /* 2280 * Save initial space that followed app data for completion code & 2281 * alternate completion code to fall in same cache line as app data 2282 */ 2283 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2284 m_dma += COMPLETION_CODE_SIZE; 2285 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2286 (uint8_t *)m_vaddr; 2287 2288 c_vaddr = (uint8_t *)m_vaddr + size; 2289 c_dma = m_dma + size; 2290 size += sizeof(cpt_res_s_t); 2291 2292 m_vaddr = (uint8_t *)m_vaddr + size; 2293 m_dma += size; 2294 2295 /* Reserve memory for cpt request info */ 2296 req = m_vaddr; 2297 2298 size = sizeof(struct cpt_request_info); 2299 m_vaddr = (uint8_t *)m_vaddr + size; 2300 m_dma += size; 2301 2302 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2303 2304 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2305 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2306 (dir << 4) | (0 << 3) | (flags & 0x7)); 2307 2308 /* 2309 * GP op header, lengths are expected in bits. 2310 */ 2311 vq_cmd_w0.u64 = 0; 2312 vq_cmd_w0.s.param1 = encr_data_len; 2313 vq_cmd_w0.s.opcode = opcode.flags; 2314 2315 /* consider iv len */ 2316 encr_offset += iv_len; 2317 2318 inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8); 2319 outputlen = inputlen; 2320 2321 /* save space for offset ctrl & iv */ 2322 offset_vaddr = m_vaddr; 2323 offset_dma = m_dma; 2324 2325 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2326 m_dma += OFF_CTRL_LEN + iv_len; 2327 2328 /* DPTR has SG list */ 2329 in_buffer = m_vaddr; 2330 dptr_dma = m_dma; 2331 2332 ((uint16_t *)in_buffer)[0] = 0; 2333 ((uint16_t *)in_buffer)[1] = 0; 2334 2335 /* TODO Add error check if space will be sufficient */ 2336 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2337 2338 /* 2339 * Input Gather List 2340 */ 2341 i = 0; 2342 2343 /* Offset control word followed by iv */ 2344 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2345 2346 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2347 2348 /* IV */ 2349 memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN, 2350 params->iv_buf, iv_len); 2351 2352 /* Add input data */ 2353 size = inputlen - iv_len; 2354 if (size) { 2355 i = fill_sg_comp_from_iov(gather_comp, i, 2356 params->src_iov, 2357 0, &size, NULL, 0); 2358 if (unlikely(size)) { 2359 CPT_LOG_DP_ERR("Insufficient buffer space," 2360 " size %d needed", size); 2361 return; 2362 } 2363 } 2364 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2365 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2366 2367 /* 2368 * Output Scatter List 2369 */ 2370 2371 i = 0; 2372 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2373 2374 /* IV */ 2375 i = fill_sg_comp(scatter_comp, i, 2376 offset_dma + OFF_CTRL_LEN, 2377 iv_len); 2378 2379 /* Add output data */ 2380 size = outputlen - iv_len; 2381 if (size) { 2382 i = fill_sg_comp_from_iov(scatter_comp, i, 2383 params->dst_iov, 0, 2384 &size, NULL, 0); 2385 if (unlikely(size)) { 2386 CPT_LOG_DP_ERR("Insufficient buffer space," 2387 " size %d needed", size); 2388 return; 2389 } 2390 } 2391 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2392 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2393 2394 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2395 2396 /* This is DPTR len incase of SG mode */ 2397 vq_cmd_w0.s.dlen = size; 2398 2399 m_vaddr = (uint8_t *)m_vaddr + size; 2400 m_dma += size; 2401 2402 /* cpt alternate completion address saved earlier */ 2403 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2404 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2405 rptr_dma = c_dma - 8; 2406 2407 req->ist.ei1 = dptr_dma; 2408 req->ist.ei2 = rptr_dma; 2409 2410 /* vq command w3 */ 2411 vq_cmd_w3.u64 = 0; 2412 vq_cmd_w3.s.grp = 0; 2413 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2414 offsetof(struct cpt_ctx, k_ctx); 2415 2416 /* 16 byte aligned cpt res address */ 2417 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2418 *req->completion_addr = COMPLETION_CODE_INIT; 2419 req->comp_baddr = c_dma; 2420 2421 /* Fill microcode part of instruction */ 2422 req->ist.ei0 = vq_cmd_w0.u64; 2423 req->ist.ei3 = vq_cmd_w3.u64; 2424 2425 req->op = op; 2426 2427 *prep_req = req; 2428 return; 2429 } 2430 2431 static __rte_always_inline void * 2432 cpt_fc_dec_hmac_prep(uint32_t flags, 2433 uint64_t d_offs, 2434 uint64_t d_lens, 2435 fc_params_t *fc_params, 2436 void *op) 2437 { 2438 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2439 uint8_t fc_type; 2440 void *prep_req = NULL; 2441 2442 fc_type = ctx->fc_type; 2443 2444 if (likely(fc_type == FC_GEN)) { 2445 cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2446 &prep_req); 2447 } else if (fc_type == ZUC_SNOW3G) { 2448 cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, op, 2449 &prep_req); 2450 } else if (fc_type == KASUMI) { 2451 cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, &prep_req); 2452 } 2453 2454 /* 2455 * For AUTH_ONLY case, 2456 * MC only supports digest generation and verification 2457 * should be done in software by memcmp() 2458 */ 2459 2460 return prep_req; 2461 } 2462 2463 static __rte_always_inline void *__hot 2464 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, 2465 fc_params_t *fc_params, void *op) 2466 { 2467 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2468 uint8_t fc_type; 2469 void *prep_req = NULL; 2470 2471 fc_type = ctx->fc_type; 2472 2473 /* Common api for rest of the ops */ 2474 if (likely(fc_type == FC_GEN)) { 2475 cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2476 &prep_req); 2477 } else if (fc_type == ZUC_SNOW3G) { 2478 cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, op, 2479 &prep_req); 2480 } else if (fc_type == KASUMI) { 2481 cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, op, 2482 &prep_req); 2483 } else if (fc_type == HASH_HMAC) { 2484 cpt_digest_gen_prep(flags, d_lens, fc_params, op, &prep_req); 2485 } 2486 2487 return prep_req; 2488 } 2489 2490 static __rte_always_inline int 2491 cpt_fc_auth_set_key(void *ctx, auth_type_t type, const uint8_t *key, 2492 uint16_t key_len, uint16_t mac_len) 2493 { 2494 struct cpt_ctx *cpt_ctx = ctx; 2495 mc_fc_context_t *fctx = &cpt_ctx->fctx; 2496 uint64_t *ctrl_flags = NULL; 2497 2498 if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) { 2499 uint32_t keyx[4]; 2500 2501 if (key_len != 16) 2502 return -1; 2503 /* No support for AEAD yet */ 2504 if (cpt_ctx->enc_cipher) 2505 return -1; 2506 /* For ZUC/SNOW3G/Kasumi */ 2507 switch (type) { 2508 case SNOW3G_UIA2: 2509 cpt_ctx->snow3g = 1; 2510 gen_key_snow3g(key, keyx); 2511 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 2512 cpt_ctx->fc_type = ZUC_SNOW3G; 2513 cpt_ctx->zsk_flags = 0x1; 2514 break; 2515 case ZUC_EIA3: 2516 cpt_ctx->snow3g = 0; 2517 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 2518 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 2519 cpt_ctx->fc_type = ZUC_SNOW3G; 2520 cpt_ctx->zsk_flags = 0x1; 2521 break; 2522 case KASUMI_F9_ECB: 2523 /* Kasumi ECB mode */ 2524 cpt_ctx->k_ecb = 1; 2525 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2526 cpt_ctx->fc_type = KASUMI; 2527 cpt_ctx->zsk_flags = 0x1; 2528 break; 2529 case KASUMI_F9_CBC: 2530 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2531 cpt_ctx->fc_type = KASUMI; 2532 cpt_ctx->zsk_flags = 0x1; 2533 break; 2534 default: 2535 return -1; 2536 } 2537 cpt_ctx->mac_len = 4; 2538 cpt_ctx->hash_type = type; 2539 return 0; 2540 } 2541 2542 if (!(cpt_ctx->fc_type == FC_GEN && !type)) { 2543 if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher) 2544 cpt_ctx->fc_type = HASH_HMAC; 2545 } 2546 2547 ctrl_flags = (uint64_t *)&fctx->enc.enc_ctrl.flags; 2548 *ctrl_flags = rte_be_to_cpu_64(*ctrl_flags); 2549 2550 /* For GMAC auth, cipher must be NULL */ 2551 if (type == GMAC_TYPE) 2552 CPT_P_ENC_CTRL(fctx).enc_cipher = 0; 2553 2554 CPT_P_ENC_CTRL(fctx).hash_type = cpt_ctx->hash_type = type; 2555 CPT_P_ENC_CTRL(fctx).mac_len = cpt_ctx->mac_len = mac_len; 2556 2557 if (key_len) { 2558 cpt_ctx->hmac = 1; 2559 memset(cpt_ctx->auth_key, 0, sizeof(cpt_ctx->auth_key)); 2560 memcpy(cpt_ctx->auth_key, key, key_len); 2561 cpt_ctx->auth_key_len = key_len; 2562 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 2563 memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad)); 2564 memcpy(fctx->hmac.opad, key, key_len); 2565 CPT_P_ENC_CTRL(fctx).auth_input_type = 1; 2566 } 2567 *ctrl_flags = rte_cpu_to_be_64(*ctrl_flags); 2568 return 0; 2569 } 2570 2571 static __rte_always_inline int 2572 fill_sess_aead(struct rte_crypto_sym_xform *xform, 2573 struct cpt_sess_misc *sess) 2574 { 2575 struct rte_crypto_aead_xform *aead_form; 2576 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2577 auth_type_t auth_type = 0; /* NULL Auth type */ 2578 uint32_t cipher_key_len = 0; 2579 uint8_t aes_gcm = 0; 2580 aead_form = &xform->aead; 2581 void *ctx = SESS_PRIV(sess); 2582 2583 if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT && 2584 aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) { 2585 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2586 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2587 } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT && 2588 aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) { 2589 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2590 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2591 } else { 2592 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2593 return -1; 2594 } 2595 switch (aead_form->algo) { 2596 case RTE_CRYPTO_AEAD_AES_GCM: 2597 enc_type = AES_GCM; 2598 cipher_key_len = 16; 2599 aes_gcm = 1; 2600 break; 2601 case RTE_CRYPTO_AEAD_AES_CCM: 2602 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2603 aead_form->algo); 2604 return -1; 2605 default: 2606 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2607 aead_form->algo); 2608 return -1; 2609 } 2610 if (aead_form->key.length < cipher_key_len) { 2611 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2612 (unsigned int long)aead_form->key.length); 2613 return -1; 2614 } 2615 sess->zsk_flag = 0; 2616 sess->aes_gcm = aes_gcm; 2617 sess->mac_len = aead_form->digest_length; 2618 sess->iv_offset = aead_form->iv.offset; 2619 sess->iv_length = aead_form->iv.length; 2620 sess->aad_length = aead_form->aad_length; 2621 2622 cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data, 2623 aead_form->key.length, NULL); 2624 2625 cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, aead_form->digest_length); 2626 2627 return 0; 2628 } 2629 2630 static __rte_always_inline int 2631 fill_sess_cipher(struct rte_crypto_sym_xform *xform, 2632 struct cpt_sess_misc *sess) 2633 { 2634 struct rte_crypto_cipher_xform *c_form; 2635 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2636 uint32_t cipher_key_len = 0; 2637 uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0; 2638 2639 c_form = &xform->cipher; 2640 2641 if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2642 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2643 else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT) 2644 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2645 else { 2646 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2647 return -1; 2648 } 2649 2650 switch (c_form->algo) { 2651 case RTE_CRYPTO_CIPHER_AES_CBC: 2652 enc_type = AES_CBC; 2653 cipher_key_len = 16; 2654 break; 2655 case RTE_CRYPTO_CIPHER_3DES_CBC: 2656 enc_type = DES3_CBC; 2657 cipher_key_len = 24; 2658 break; 2659 case RTE_CRYPTO_CIPHER_DES_CBC: 2660 /* DES is implemented using 3DES in hardware */ 2661 enc_type = DES3_CBC; 2662 cipher_key_len = 8; 2663 break; 2664 case RTE_CRYPTO_CIPHER_AES_CTR: 2665 enc_type = AES_CTR; 2666 cipher_key_len = 16; 2667 aes_ctr = 1; 2668 break; 2669 case RTE_CRYPTO_CIPHER_NULL: 2670 enc_type = 0; 2671 is_null = 1; 2672 break; 2673 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2674 enc_type = KASUMI_F8_ECB; 2675 cipher_key_len = 16; 2676 zsk_flag = K_F8; 2677 break; 2678 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2679 enc_type = SNOW3G_UEA2; 2680 cipher_key_len = 16; 2681 zsk_flag = ZS_EA; 2682 break; 2683 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 2684 enc_type = ZUC_EEA3; 2685 cipher_key_len = 16; 2686 zsk_flag = ZS_EA; 2687 break; 2688 case RTE_CRYPTO_CIPHER_AES_XTS: 2689 enc_type = AES_XTS; 2690 cipher_key_len = 16; 2691 break; 2692 case RTE_CRYPTO_CIPHER_3DES_ECB: 2693 enc_type = DES3_ECB; 2694 cipher_key_len = 24; 2695 break; 2696 case RTE_CRYPTO_CIPHER_AES_ECB: 2697 enc_type = AES_ECB; 2698 cipher_key_len = 16; 2699 break; 2700 case RTE_CRYPTO_CIPHER_3DES_CTR: 2701 case RTE_CRYPTO_CIPHER_AES_F8: 2702 case RTE_CRYPTO_CIPHER_ARC4: 2703 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2704 c_form->algo); 2705 return -1; 2706 default: 2707 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2708 c_form->algo); 2709 return -1; 2710 } 2711 2712 if (c_form->key.length < cipher_key_len) { 2713 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2714 (unsigned long) c_form->key.length); 2715 return -1; 2716 } 2717 2718 sess->zsk_flag = zsk_flag; 2719 sess->aes_gcm = 0; 2720 sess->aes_ctr = aes_ctr; 2721 sess->iv_offset = c_form->iv.offset; 2722 sess->iv_length = c_form->iv.length; 2723 sess->is_null = is_null; 2724 2725 cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type, c_form->key.data, 2726 c_form->key.length, NULL); 2727 2728 return 0; 2729 } 2730 2731 static __rte_always_inline int 2732 fill_sess_auth(struct rte_crypto_sym_xform *xform, 2733 struct cpt_sess_misc *sess) 2734 { 2735 struct rte_crypto_auth_xform *a_form; 2736 auth_type_t auth_type = 0; /* NULL Auth type */ 2737 uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0; 2738 2739 a_form = &xform->auth; 2740 2741 if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2742 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2743 else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2744 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2745 else { 2746 CPT_LOG_DP_ERR("Unknown auth operation"); 2747 return -1; 2748 } 2749 2750 if (a_form->key.length > 64) { 2751 CPT_LOG_DP_ERR("Auth key length is big"); 2752 return -1; 2753 } 2754 2755 switch (a_form->algo) { 2756 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2757 /* Fall through */ 2758 case RTE_CRYPTO_AUTH_SHA1: 2759 auth_type = SHA1_TYPE; 2760 break; 2761 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2762 case RTE_CRYPTO_AUTH_SHA256: 2763 auth_type = SHA2_SHA256; 2764 break; 2765 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2766 case RTE_CRYPTO_AUTH_SHA512: 2767 auth_type = SHA2_SHA512; 2768 break; 2769 case RTE_CRYPTO_AUTH_AES_GMAC: 2770 auth_type = GMAC_TYPE; 2771 aes_gcm = 1; 2772 break; 2773 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2774 case RTE_CRYPTO_AUTH_SHA224: 2775 auth_type = SHA2_SHA224; 2776 break; 2777 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2778 case RTE_CRYPTO_AUTH_SHA384: 2779 auth_type = SHA2_SHA384; 2780 break; 2781 case RTE_CRYPTO_AUTH_MD5_HMAC: 2782 case RTE_CRYPTO_AUTH_MD5: 2783 auth_type = MD5_TYPE; 2784 break; 2785 case RTE_CRYPTO_AUTH_KASUMI_F9: 2786 auth_type = KASUMI_F9_ECB; 2787 /* 2788 * Indicate that direction needs to be taken out 2789 * from end of src 2790 */ 2791 zsk_flag = K_F9; 2792 break; 2793 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2794 auth_type = SNOW3G_UIA2; 2795 zsk_flag = ZS_IA; 2796 break; 2797 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2798 auth_type = ZUC_EIA3; 2799 zsk_flag = ZS_IA; 2800 break; 2801 case RTE_CRYPTO_AUTH_NULL: 2802 auth_type = 0; 2803 is_null = 1; 2804 break; 2805 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2806 case RTE_CRYPTO_AUTH_AES_CMAC: 2807 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2808 CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u", 2809 a_form->algo); 2810 return -1; 2811 default: 2812 CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified", 2813 a_form->algo); 2814 return -1; 2815 } 2816 2817 sess->zsk_flag = zsk_flag; 2818 sess->aes_gcm = aes_gcm; 2819 sess->mac_len = a_form->digest_length; 2820 sess->is_null = is_null; 2821 if (zsk_flag) { 2822 sess->auth_iv_offset = a_form->iv.offset; 2823 sess->auth_iv_length = a_form->iv.length; 2824 } 2825 cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type, a_form->key.data, 2826 a_form->key.length, a_form->digest_length); 2827 2828 return 0; 2829 } 2830 2831 static __rte_always_inline int 2832 fill_sess_gmac(struct rte_crypto_sym_xform *xform, 2833 struct cpt_sess_misc *sess) 2834 { 2835 struct rte_crypto_auth_xform *a_form; 2836 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2837 auth_type_t auth_type = 0; /* NULL Auth type */ 2838 void *ctx = SESS_PRIV(sess); 2839 2840 a_form = &xform->auth; 2841 2842 if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2843 sess->cpt_op |= CPT_OP_ENCODE; 2844 else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2845 sess->cpt_op |= CPT_OP_DECODE; 2846 else { 2847 CPT_LOG_DP_ERR("Unknown auth operation"); 2848 return -1; 2849 } 2850 2851 switch (a_form->algo) { 2852 case RTE_CRYPTO_AUTH_AES_GMAC: 2853 enc_type = AES_GCM; 2854 auth_type = GMAC_TYPE; 2855 break; 2856 default: 2857 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2858 a_form->algo); 2859 return -1; 2860 } 2861 2862 sess->zsk_flag = 0; 2863 sess->aes_gcm = 0; 2864 sess->is_gmac = 1; 2865 sess->iv_offset = a_form->iv.offset; 2866 sess->iv_length = a_form->iv.length; 2867 sess->mac_len = a_form->digest_length; 2868 2869 cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data, 2870 a_form->key.length, NULL); 2871 cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, a_form->digest_length); 2872 2873 return 0; 2874 } 2875 2876 static __rte_always_inline void * 2877 alloc_op_meta(struct rte_mbuf *m_src, 2878 buf_ptr_t *buf, 2879 int32_t len, 2880 struct rte_mempool *cpt_meta_pool) 2881 { 2882 uint8_t *mdata; 2883 2884 #ifndef CPT_ALWAYS_USE_SEPARATE_BUF 2885 if (likely(m_src && (m_src->nb_segs == 1))) { 2886 int32_t tailroom; 2887 phys_addr_t mphys; 2888 2889 /* Check if tailroom is sufficient to hold meta data */ 2890 tailroom = rte_pktmbuf_tailroom(m_src); 2891 if (likely(tailroom > len + 8)) { 2892 mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len; 2893 mphys = m_src->buf_physaddr + m_src->buf_len; 2894 mdata -= len; 2895 mphys -= len; 2896 buf->vaddr = mdata; 2897 buf->dma_addr = mphys; 2898 buf->size = len; 2899 /* Indicate that this is a mbuf allocated mdata */ 2900 mdata = (uint8_t *)((uint64_t)mdata | 1ull); 2901 return mdata; 2902 } 2903 } 2904 #else 2905 RTE_SET_USED(m_src); 2906 #endif 2907 2908 if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0)) 2909 return NULL; 2910 2911 buf->vaddr = mdata; 2912 buf->dma_addr = rte_mempool_virt2iova(mdata); 2913 buf->size = len; 2914 2915 return mdata; 2916 } 2917 2918 /** 2919 * cpt_free_metabuf - free metabuf to mempool. 2920 * @param instance: pointer to instance. 2921 * @param objp: pointer to the metabuf. 2922 */ 2923 static __rte_always_inline void 2924 free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool) 2925 { 2926 bool nofree = ((uintptr_t)mdata & 1ull); 2927 2928 if (likely(nofree)) 2929 return; 2930 rte_mempool_put(cpt_meta_pool, mdata); 2931 } 2932 2933 static __rte_always_inline uint32_t 2934 prepare_iov_from_pkt(struct rte_mbuf *pkt, 2935 iov_ptr_t *iovec, uint32_t start_offset) 2936 { 2937 uint16_t index = 0; 2938 void *seg_data = NULL; 2939 phys_addr_t seg_phys; 2940 int32_t seg_size = 0; 2941 2942 if (!pkt) { 2943 iovec->buf_cnt = 0; 2944 return 0; 2945 } 2946 2947 if (!start_offset) { 2948 seg_data = rte_pktmbuf_mtod(pkt, void *); 2949 seg_phys = rte_pktmbuf_mtophys(pkt); 2950 seg_size = pkt->data_len; 2951 } else { 2952 while (start_offset >= pkt->data_len) { 2953 start_offset -= pkt->data_len; 2954 pkt = pkt->next; 2955 } 2956 2957 seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset); 2958 seg_phys = rte_pktmbuf_mtophys_offset(pkt, start_offset); 2959 seg_size = pkt->data_len - start_offset; 2960 if (!seg_size) 2961 return 1; 2962 } 2963 2964 /* first seg */ 2965 iovec->bufs[index].vaddr = seg_data; 2966 iovec->bufs[index].dma_addr = seg_phys; 2967 iovec->bufs[index].size = seg_size; 2968 index++; 2969 pkt = pkt->next; 2970 2971 while (unlikely(pkt != NULL)) { 2972 seg_data = rte_pktmbuf_mtod(pkt, void *); 2973 seg_phys = rte_pktmbuf_mtophys(pkt); 2974 seg_size = pkt->data_len; 2975 if (!seg_size) 2976 break; 2977 2978 iovec->bufs[index].vaddr = seg_data; 2979 iovec->bufs[index].dma_addr = seg_phys; 2980 iovec->bufs[index].size = seg_size; 2981 2982 index++; 2983 2984 pkt = pkt->next; 2985 } 2986 2987 iovec->buf_cnt = index; 2988 return 0; 2989 } 2990 2991 static __rte_always_inline uint32_t 2992 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt, 2993 fc_params_t *param, 2994 uint32_t *flags) 2995 { 2996 uint16_t index = 0; 2997 void *seg_data = NULL; 2998 phys_addr_t seg_phys; 2999 uint32_t seg_size = 0; 3000 iov_ptr_t *iovec; 3001 3002 seg_data = rte_pktmbuf_mtod(pkt, void *); 3003 seg_phys = rte_pktmbuf_mtophys(pkt); 3004 seg_size = pkt->data_len; 3005 3006 /* first seg */ 3007 if (likely(!pkt->next)) { 3008 uint32_t headroom, tailroom; 3009 3010 *flags |= SINGLE_BUF_INPLACE; 3011 headroom = rte_pktmbuf_headroom(pkt); 3012 tailroom = rte_pktmbuf_tailroom(pkt); 3013 if (likely((headroom >= 24) && 3014 (tailroom >= 8))) { 3015 /* In 83XX this is prerequivisit for Direct mode */ 3016 *flags |= SINGLE_BUF_HEADTAILROOM; 3017 } 3018 param->bufs[0].vaddr = seg_data; 3019 param->bufs[0].dma_addr = seg_phys; 3020 param->bufs[0].size = seg_size; 3021 return 0; 3022 } 3023 iovec = param->src_iov; 3024 iovec->bufs[index].vaddr = seg_data; 3025 iovec->bufs[index].dma_addr = seg_phys; 3026 iovec->bufs[index].size = seg_size; 3027 index++; 3028 pkt = pkt->next; 3029 3030 while (unlikely(pkt != NULL)) { 3031 seg_data = rte_pktmbuf_mtod(pkt, void *); 3032 seg_phys = rte_pktmbuf_mtophys(pkt); 3033 seg_size = pkt->data_len; 3034 3035 if (!seg_size) 3036 break; 3037 3038 iovec->bufs[index].vaddr = seg_data; 3039 iovec->bufs[index].dma_addr = seg_phys; 3040 iovec->bufs[index].size = seg_size; 3041 3042 index++; 3043 3044 pkt = pkt->next; 3045 } 3046 3047 iovec->buf_cnt = index; 3048 return 0; 3049 } 3050 3051 static __rte_always_inline int 3052 fill_fc_params(struct rte_crypto_op *cop, 3053 struct cpt_sess_misc *sess_misc, 3054 struct cpt_qp_meta_info *m_info, 3055 void **mdata_ptr, 3056 void **prep_req) 3057 { 3058 uint32_t space = 0; 3059 struct rte_crypto_sym_op *sym_op = cop->sym; 3060 void *mdata = NULL; 3061 uintptr_t *op; 3062 uint32_t mc_hash_off; 3063 uint32_t flags = 0; 3064 uint64_t d_offs, d_lens; 3065 struct rte_mbuf *m_src, *m_dst; 3066 uint8_t cpt_op = sess_misc->cpt_op; 3067 #ifdef CPT_ALWAYS_USE_SG_MODE 3068 uint8_t inplace = 0; 3069 #else 3070 uint8_t inplace = 1; 3071 #endif 3072 fc_params_t fc_params; 3073 char src[SRC_IOV_SIZE]; 3074 char dst[SRC_IOV_SIZE]; 3075 uint32_t iv_buf[4]; 3076 int ret; 3077 3078 if (likely(sess_misc->iv_length)) { 3079 flags |= VALID_IV_BUF; 3080 fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, 3081 uint8_t *, sess_misc->iv_offset); 3082 if (sess_misc->aes_ctr && 3083 unlikely(sess_misc->iv_length != 16)) { 3084 memcpy((uint8_t *)iv_buf, 3085 rte_crypto_op_ctod_offset(cop, 3086 uint8_t *, sess_misc->iv_offset), 12); 3087 iv_buf[3] = rte_cpu_to_be_32(0x1); 3088 fc_params.iv_buf = iv_buf; 3089 } 3090 } 3091 3092 if (sess_misc->zsk_flag) { 3093 fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3094 uint8_t *, 3095 sess_misc->auth_iv_offset); 3096 if (sess_misc->zsk_flag != ZS_EA) 3097 inplace = 0; 3098 } 3099 m_src = sym_op->m_src; 3100 m_dst = sym_op->m_dst; 3101 3102 if (sess_misc->aes_gcm) { 3103 uint8_t *salt; 3104 uint8_t *aad_data; 3105 uint16_t aad_len; 3106 3107 d_offs = sym_op->aead.data.offset; 3108 d_lens = sym_op->aead.data.length; 3109 mc_hash_off = sym_op->aead.data.offset + 3110 sym_op->aead.data.length; 3111 3112 aad_data = sym_op->aead.aad.data; 3113 aad_len = sess_misc->aad_length; 3114 if (likely((aad_data + aad_len) == 3115 rte_pktmbuf_mtod_offset(m_src, 3116 uint8_t *, 3117 sym_op->aead.data.offset))) { 3118 d_offs = (d_offs - aad_len) | (d_offs << 16); 3119 d_lens = (d_lens + aad_len) | (d_lens << 32); 3120 } else { 3121 fc_params.aad_buf.vaddr = sym_op->aead.aad.data; 3122 fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr; 3123 fc_params.aad_buf.size = aad_len; 3124 flags |= VALID_AAD_BUF; 3125 inplace = 0; 3126 d_offs = d_offs << 16; 3127 d_lens = d_lens << 32; 3128 } 3129 3130 salt = fc_params.iv_buf; 3131 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3132 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3133 sess_misc->salt = *(uint32_t *)salt; 3134 } 3135 fc_params.iv_buf = salt + 4; 3136 if (likely(sess_misc->mac_len)) { 3137 struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst : 3138 m_src; 3139 3140 if (!m) 3141 m = m_src; 3142 3143 /* hmac immediately following data is best case */ 3144 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3145 mc_hash_off != 3146 (uint8_t *)sym_op->aead.digest.data)) { 3147 flags |= VALID_MAC_BUF; 3148 fc_params.mac_buf.size = sess_misc->mac_len; 3149 fc_params.mac_buf.vaddr = 3150 sym_op->aead.digest.data; 3151 fc_params.mac_buf.dma_addr = 3152 sym_op->aead.digest.phys_addr; 3153 inplace = 0; 3154 } 3155 } 3156 } else { 3157 d_offs = sym_op->cipher.data.offset; 3158 d_lens = sym_op->cipher.data.length; 3159 mc_hash_off = sym_op->cipher.data.offset + 3160 sym_op->cipher.data.length; 3161 d_offs = (d_offs << 16) | sym_op->auth.data.offset; 3162 d_lens = (d_lens << 32) | sym_op->auth.data.length; 3163 3164 if (mc_hash_off < (sym_op->auth.data.offset + 3165 sym_op->auth.data.length)){ 3166 mc_hash_off = (sym_op->auth.data.offset + 3167 sym_op->auth.data.length); 3168 } 3169 /* for gmac, salt should be updated like in gcm */ 3170 if (unlikely(sess_misc->is_gmac)) { 3171 uint8_t *salt; 3172 salt = fc_params.iv_buf; 3173 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3174 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3175 sess_misc->salt = *(uint32_t *)salt; 3176 } 3177 fc_params.iv_buf = salt + 4; 3178 } 3179 if (likely(sess_misc->mac_len)) { 3180 struct rte_mbuf *m; 3181 3182 m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src; 3183 if (!m) 3184 m = m_src; 3185 3186 /* hmac immediately following data is best case */ 3187 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3188 mc_hash_off != 3189 (uint8_t *)sym_op->auth.digest.data)) { 3190 flags |= VALID_MAC_BUF; 3191 fc_params.mac_buf.size = 3192 sess_misc->mac_len; 3193 fc_params.mac_buf.vaddr = 3194 sym_op->auth.digest.data; 3195 fc_params.mac_buf.dma_addr = 3196 sym_op->auth.digest.phys_addr; 3197 inplace = 0; 3198 } 3199 } 3200 } 3201 fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc); 3202 fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr; 3203 3204 if (unlikely(sess_misc->is_null || sess_misc->cpt_op == CPT_OP_DECODE)) 3205 inplace = 0; 3206 3207 if (likely(!m_dst && inplace)) { 3208 /* Case of single buffer without AAD buf or 3209 * separate mac buf in place and 3210 * not air crypto 3211 */ 3212 fc_params.dst_iov = fc_params.src_iov = (void *)src; 3213 3214 if (unlikely(prepare_iov_from_pkt_inplace(m_src, 3215 &fc_params, 3216 &flags))) { 3217 CPT_LOG_DP_ERR("Prepare inplace src iov failed"); 3218 ret = -EINVAL; 3219 goto err_exit; 3220 } 3221 3222 } else { 3223 /* Out of place processing */ 3224 fc_params.src_iov = (void *)src; 3225 fc_params.dst_iov = (void *)dst; 3226 3227 /* Store SG I/O in the api for reuse */ 3228 if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) { 3229 CPT_LOG_DP_ERR("Prepare src iov failed"); 3230 ret = -EINVAL; 3231 goto err_exit; 3232 } 3233 3234 if (unlikely(m_dst != NULL)) { 3235 uint32_t pkt_len; 3236 3237 /* Try to make room as much as src has */ 3238 pkt_len = rte_pktmbuf_pkt_len(m_dst); 3239 3240 if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) { 3241 pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len; 3242 if (!rte_pktmbuf_append(m_dst, pkt_len)) { 3243 CPT_LOG_DP_ERR("Not enough space in " 3244 "m_dst %p, need %u" 3245 " more", 3246 m_dst, pkt_len); 3247 ret = -EINVAL; 3248 goto err_exit; 3249 } 3250 } 3251 3252 if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) { 3253 CPT_LOG_DP_ERR("Prepare dst iov failed for " 3254 "m_dst %p", m_dst); 3255 ret = -EINVAL; 3256 goto err_exit; 3257 } 3258 } else { 3259 fc_params.dst_iov = (void *)src; 3260 } 3261 } 3262 3263 if (likely(flags & SINGLE_BUF_HEADTAILROOM)) 3264 mdata = alloc_op_meta(m_src, &fc_params.meta_buf, 3265 m_info->lb_mlen, m_info->pool); 3266 else 3267 mdata = alloc_op_meta(NULL, &fc_params.meta_buf, 3268 m_info->sg_mlen, m_info->pool); 3269 3270 if (unlikely(mdata == NULL)) { 3271 CPT_LOG_DP_ERR("Error allocating meta buffer for request"); 3272 ret = -ENOMEM; 3273 goto err_exit; 3274 } 3275 3276 op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull); 3277 op[0] = (uintptr_t)mdata; 3278 op[1] = (uintptr_t)cop; 3279 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3280 space += 4 * sizeof(uint64_t); 3281 3282 fc_params.meta_buf.vaddr = (uint8_t *)op + space; 3283 fc_params.meta_buf.dma_addr += space; 3284 fc_params.meta_buf.size -= space; 3285 3286 /* Finally prepare the instruction */ 3287 if (cpt_op & CPT_OP_ENCODE) 3288 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, 3289 &fc_params, op); 3290 else 3291 *prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens, 3292 &fc_params, op); 3293 3294 if (unlikely(*prep_req == NULL)) { 3295 CPT_LOG_DP_ERR("Preparing request failed due to bad input arg"); 3296 ret = -EINVAL; 3297 goto free_mdata_and_exit; 3298 } 3299 3300 *mdata_ptr = mdata; 3301 3302 return 0; 3303 3304 free_mdata_and_exit: 3305 free_op_meta(mdata, m_info->pool); 3306 err_exit: 3307 return ret; 3308 } 3309 3310 static __rte_always_inline void 3311 compl_auth_verify(struct rte_crypto_op *op, 3312 uint8_t *gen_mac, 3313 uint64_t mac_len) 3314 { 3315 uint8_t *mac; 3316 struct rte_crypto_sym_op *sym_op = op->sym; 3317 3318 if (sym_op->auth.digest.data) 3319 mac = sym_op->auth.digest.data; 3320 else 3321 mac = rte_pktmbuf_mtod_offset(sym_op->m_src, 3322 uint8_t *, 3323 sym_op->auth.data.length + 3324 sym_op->auth.data.offset); 3325 if (!mac) { 3326 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3327 return; 3328 } 3329 3330 if (memcmp(mac, gen_mac, mac_len)) 3331 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 3332 else 3333 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3334 } 3335 3336 static __rte_always_inline int 3337 instance_session_cfg(struct rte_crypto_sym_xform *xform, void *sess) 3338 { 3339 struct rte_crypto_sym_xform *chain; 3340 3341 CPT_PMD_INIT_FUNC_TRACE(); 3342 3343 if (cpt_is_algo_supported(xform)) 3344 goto err; 3345 3346 chain = xform; 3347 while (chain) { 3348 switch (chain->type) { 3349 case RTE_CRYPTO_SYM_XFORM_AEAD: 3350 if (fill_sess_aead(chain, sess)) 3351 goto err; 3352 break; 3353 case RTE_CRYPTO_SYM_XFORM_CIPHER: 3354 if (fill_sess_cipher(chain, sess)) 3355 goto err; 3356 break; 3357 case RTE_CRYPTO_SYM_XFORM_AUTH: 3358 if (chain->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 3359 if (fill_sess_gmac(chain, sess)) 3360 goto err; 3361 } else { 3362 if (fill_sess_auth(chain, sess)) 3363 goto err; 3364 } 3365 break; 3366 default: 3367 CPT_LOG_DP_ERR("Invalid crypto xform type"); 3368 break; 3369 } 3370 chain = chain->next; 3371 } 3372 3373 return 0; 3374 3375 err: 3376 return -1; 3377 } 3378 3379 static __rte_always_inline void 3380 find_kasumif9_direction_and_length(uint8_t *src, 3381 uint32_t counter_num_bytes, 3382 uint32_t *addr_length_in_bits, 3383 uint8_t *addr_direction) 3384 { 3385 uint8_t found = 0; 3386 uint32_t pos; 3387 uint8_t last_byte; 3388 while (!found && counter_num_bytes > 0) { 3389 counter_num_bytes--; 3390 if (src[counter_num_bytes] == 0x00) 3391 continue; 3392 pos = rte_bsf32(src[counter_num_bytes]); 3393 if (pos == 7) { 3394 if (likely(counter_num_bytes > 0)) { 3395 last_byte = src[counter_num_bytes - 1]; 3396 *addr_direction = last_byte & 0x1; 3397 *addr_length_in_bits = counter_num_bytes * 8 3398 - 1; 3399 } 3400 } else { 3401 last_byte = src[counter_num_bytes]; 3402 *addr_direction = (last_byte >> (pos + 1)) & 0x1; 3403 *addr_length_in_bits = counter_num_bytes * 8 3404 + (8 - (pos + 2)); 3405 } 3406 found = 1; 3407 } 3408 } 3409 3410 /* 3411 * This handles all auth only except AES_GMAC 3412 */ 3413 static __rte_always_inline int 3414 fill_digest_params(struct rte_crypto_op *cop, 3415 struct cpt_sess_misc *sess, 3416 struct cpt_qp_meta_info *m_info, 3417 void **mdata_ptr, 3418 void **prep_req) 3419 { 3420 uint32_t space = 0; 3421 struct rte_crypto_sym_op *sym_op = cop->sym; 3422 void *mdata; 3423 phys_addr_t mphys; 3424 uint64_t *op; 3425 uint32_t auth_range_off; 3426 uint32_t flags = 0; 3427 uint64_t d_offs = 0, d_lens; 3428 struct rte_mbuf *m_src, *m_dst; 3429 uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK; 3430 uint16_t mac_len = sess->mac_len; 3431 fc_params_t params; 3432 char src[SRC_IOV_SIZE]; 3433 uint8_t iv_buf[16]; 3434 int ret; 3435 3436 memset(¶ms, 0, sizeof(fc_params_t)); 3437 3438 m_src = sym_op->m_src; 3439 3440 /* For just digest lets force mempool alloc */ 3441 mdata = alloc_op_meta(NULL, ¶ms.meta_buf, m_info->sg_mlen, 3442 m_info->pool); 3443 if (mdata == NULL) { 3444 ret = -ENOMEM; 3445 goto err_exit; 3446 } 3447 3448 mphys = params.meta_buf.dma_addr; 3449 3450 op = mdata; 3451 op[0] = (uintptr_t)mdata; 3452 op[1] = (uintptr_t)cop; 3453 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3454 space += 4 * sizeof(uint64_t); 3455 3456 auth_range_off = sym_op->auth.data.offset; 3457 3458 flags = VALID_MAC_BUF; 3459 params.src_iov = (void *)src; 3460 if (unlikely(sess->zsk_flag)) { 3461 /* 3462 * Since for Zuc, Kasumi, Snow3g offsets are in bits 3463 * we will send pass through even for auth only case, 3464 * let MC handle it 3465 */ 3466 d_offs = auth_range_off; 3467 auth_range_off = 0; 3468 params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3469 uint8_t *, sess->auth_iv_offset); 3470 if (sess->zsk_flag == K_F9) { 3471 uint32_t length_in_bits, num_bytes; 3472 uint8_t *src, direction = 0; 3473 3474 memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src, 3475 uint8_t *), 8); 3476 /* 3477 * This is kasumi f9, take direction from 3478 * source buffer 3479 */ 3480 length_in_bits = cop->sym->auth.data.length; 3481 num_bytes = (length_in_bits >> 3); 3482 src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *); 3483 find_kasumif9_direction_and_length(src, 3484 num_bytes, 3485 &length_in_bits, 3486 &direction); 3487 length_in_bits -= 64; 3488 cop->sym->auth.data.offset += 64; 3489 d_offs = cop->sym->auth.data.offset; 3490 auth_range_off = d_offs / 8; 3491 cop->sym->auth.data.length = length_in_bits; 3492 3493 /* Store it at end of auth iv */ 3494 iv_buf[8] = direction; 3495 params.auth_iv_buf = iv_buf; 3496 } 3497 } 3498 3499 d_lens = sym_op->auth.data.length; 3500 3501 params.ctx_buf.vaddr = SESS_PRIV(sess); 3502 params.ctx_buf.dma_addr = sess->ctx_dma_addr; 3503 3504 if (auth_op == CPT_OP_AUTH_GENERATE) { 3505 if (sym_op->auth.digest.data) { 3506 /* 3507 * Digest to be generated 3508 * in separate buffer 3509 */ 3510 params.mac_buf.size = 3511 sess->mac_len; 3512 params.mac_buf.vaddr = 3513 sym_op->auth.digest.data; 3514 params.mac_buf.dma_addr = 3515 sym_op->auth.digest.phys_addr; 3516 } else { 3517 uint32_t off = sym_op->auth.data.offset + 3518 sym_op->auth.data.length; 3519 int32_t dlen, space; 3520 3521 m_dst = sym_op->m_dst ? 3522 sym_op->m_dst : sym_op->m_src; 3523 dlen = rte_pktmbuf_pkt_len(m_dst); 3524 3525 space = off + mac_len - dlen; 3526 if (space > 0) 3527 if (!rte_pktmbuf_append(m_dst, space)) { 3528 CPT_LOG_DP_ERR("Failed to extend " 3529 "mbuf by %uB", space); 3530 ret = -EINVAL; 3531 goto free_mdata_and_exit; 3532 } 3533 3534 params.mac_buf.vaddr = 3535 rte_pktmbuf_mtod_offset(m_dst, void *, off); 3536 params.mac_buf.dma_addr = 3537 rte_pktmbuf_mtophys_offset(m_dst, off); 3538 params.mac_buf.size = mac_len; 3539 } 3540 } else { 3541 /* Need space for storing generated mac */ 3542 params.mac_buf.vaddr = (uint8_t *)mdata + space; 3543 params.mac_buf.dma_addr = mphys + space; 3544 params.mac_buf.size = mac_len; 3545 space += RTE_ALIGN_CEIL(mac_len, 8); 3546 op[2] = (uintptr_t)params.mac_buf.vaddr; 3547 op[3] = mac_len; 3548 } 3549 3550 params.meta_buf.vaddr = (uint8_t *)mdata + space; 3551 params.meta_buf.dma_addr = mphys + space; 3552 params.meta_buf.size -= space; 3553 3554 /* Out of place processing */ 3555 params.src_iov = (void *)src; 3556 3557 /*Store SG I/O in the api for reuse */ 3558 if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) { 3559 CPT_LOG_DP_ERR("Prepare src iov failed"); 3560 ret = -EINVAL; 3561 goto free_mdata_and_exit; 3562 } 3563 3564 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, ¶ms, op); 3565 if (unlikely(*prep_req == NULL)) { 3566 ret = -EINVAL; 3567 goto free_mdata_and_exit; 3568 } 3569 3570 *mdata_ptr = mdata; 3571 3572 return 0; 3573 3574 free_mdata_and_exit: 3575 free_op_meta(mdata, m_info->pool); 3576 err_exit: 3577 return ret; 3578 } 3579 3580 #endif /*_CPT_UCODE_H_ */ 3581