1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #ifndef _CPT_UCODE_H_ 6 #define _CPT_UCODE_H_ 7 #include <stdbool.h> 8 9 #include "cpt_common.h" 10 #include "cpt_hw_types.h" 11 #include "cpt_mcode_defines.h" 12 13 /* 14 * This file defines functions that are interfaces to microcode spec. 15 * 16 */ 17 18 static uint8_t zuc_d[32] = { 19 0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E, 20 0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF, 21 0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1, 22 0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC 23 }; 24 25 static __rte_always_inline int 26 cpt_is_algo_supported(struct rte_crypto_sym_xform *xform) 27 { 28 /* 29 * Microcode only supports the following combination. 30 * Encryption followed by authentication 31 * Authentication followed by decryption 32 */ 33 if (xform->next) { 34 if ((xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) && 35 (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 36 (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)) { 37 /* Unsupported as of now by microcode */ 38 CPT_LOG_DP_ERR("Unsupported combination"); 39 return -1; 40 } 41 if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 42 (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) && 43 (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT)) { 44 /* For GMAC auth there is no cipher operation */ 45 if (xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM || 46 xform->next->auth.algo != 47 RTE_CRYPTO_AUTH_AES_GMAC) { 48 /* Unsupported as of now by microcode */ 49 CPT_LOG_DP_ERR("Unsupported combination"); 50 return -1; 51 } 52 } 53 } 54 return 0; 55 } 56 57 static __rte_always_inline void 58 gen_key_snow3g(const uint8_t *ck, uint32_t *keyx) 59 { 60 int i, base; 61 62 for (i = 0; i < 4; i++) { 63 base = 4 * i; 64 keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) | 65 (ck[base + 2] << 8) | (ck[base + 3]); 66 keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]); 67 } 68 } 69 70 static __rte_always_inline void 71 cpt_fc_salt_update(void *ctx, 72 uint8_t *salt) 73 { 74 struct cpt_ctx *cpt_ctx = ctx; 75 memcpy(&cpt_ctx->fctx.enc.encr_iv, salt, 4); 76 } 77 78 static __rte_always_inline int 79 cpt_fc_ciph_validate_key_aes(uint16_t key_len) 80 { 81 switch (key_len) { 82 case CPT_BYTE_16: 83 case CPT_BYTE_24: 84 case CPT_BYTE_32: 85 return 0; 86 default: 87 return -1; 88 } 89 } 90 91 static __rte_always_inline int 92 cpt_fc_ciph_set_type(cipher_type_t type, struct cpt_ctx *ctx, uint16_t key_len) 93 { 94 int fc_type = 0; 95 switch (type) { 96 case PASSTHROUGH: 97 fc_type = FC_GEN; 98 break; 99 case DES3_CBC: 100 case DES3_ECB: 101 fc_type = FC_GEN; 102 break; 103 case AES_CBC: 104 case AES_ECB: 105 case AES_CFB: 106 case AES_CTR: 107 case AES_GCM: 108 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 109 return -1; 110 fc_type = FC_GEN; 111 break; 112 case AES_XTS: 113 key_len = key_len / 2; 114 if (unlikely(key_len == CPT_BYTE_24)) { 115 CPT_LOG_DP_ERR("Invalid AES key len for XTS"); 116 return -1; 117 } 118 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 119 return -1; 120 fc_type = FC_GEN; 121 break; 122 case ZUC_EEA3: 123 case SNOW3G_UEA2: 124 if (unlikely(key_len != 16)) 125 return -1; 126 /* No support for AEAD yet */ 127 if (unlikely(ctx->hash_type)) 128 return -1; 129 fc_type = ZUC_SNOW3G; 130 break; 131 case KASUMI_F8_CBC: 132 case KASUMI_F8_ECB: 133 if (unlikely(key_len != 16)) 134 return -1; 135 /* No support for AEAD yet */ 136 if (unlikely(ctx->hash_type)) 137 return -1; 138 fc_type = KASUMI; 139 break; 140 default: 141 return -1; 142 } 143 144 ctx->fc_type = fc_type; 145 return 0; 146 } 147 148 static __rte_always_inline void 149 cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx) 150 { 151 cpt_ctx->enc_cipher = 0; 152 fctx->enc.enc_cipher = 0; 153 } 154 155 static __rte_always_inline void 156 cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len) 157 { 158 mc_aes_type_t aes_key_type = 0; 159 switch (key_len) { 160 case CPT_BYTE_16: 161 aes_key_type = AES_128_BIT; 162 break; 163 case CPT_BYTE_24: 164 aes_key_type = AES_192_BIT; 165 break; 166 case CPT_BYTE_32: 167 aes_key_type = AES_256_BIT; 168 break; 169 default: 170 /* This should not happen */ 171 CPT_LOG_DP_ERR("Invalid AES key len"); 172 return; 173 } 174 fctx->enc.aes_key = aes_key_type; 175 } 176 177 static __rte_always_inline void 178 cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key, 179 uint16_t key_len) 180 { 181 uint32_t keyx[4]; 182 cpt_ctx->snow3g = 1; 183 gen_key_snow3g(key, keyx); 184 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 185 cpt_ctx->zsk_flags = 0; 186 } 187 188 static __rte_always_inline void 189 cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key, 190 uint16_t key_len) 191 { 192 cpt_ctx->snow3g = 0; 193 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 194 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 195 cpt_ctx->zsk_flags = 0; 196 } 197 198 static __rte_always_inline void 199 cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key, 200 uint16_t key_len) 201 { 202 cpt_ctx->k_ecb = 1; 203 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 204 cpt_ctx->zsk_flags = 0; 205 } 206 207 static __rte_always_inline void 208 cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key, 209 uint16_t key_len) 210 { 211 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 212 cpt_ctx->zsk_flags = 0; 213 } 214 215 static __rte_always_inline int 216 cpt_fc_ciph_set_key(void *ctx, cipher_type_t type, const uint8_t *key, 217 uint16_t key_len, uint8_t *salt) 218 { 219 struct cpt_ctx *cpt_ctx = ctx; 220 mc_fc_context_t *fctx = &cpt_ctx->fctx; 221 int ret; 222 223 ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len); 224 if (unlikely(ret)) 225 return -1; 226 227 if (cpt_ctx->fc_type == FC_GEN) { 228 /* 229 * We need to always say IV is from DPTR as user can 230 * sometimes iverride IV per operation. 231 */ 232 fctx->enc.iv_source = CPT_FROM_DPTR; 233 234 if (cpt_ctx->auth_key_len > 64) 235 return -1; 236 } 237 238 switch (type) { 239 case PASSTHROUGH: 240 cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx); 241 goto success; 242 case DES3_CBC: 243 /* CPT performs DES using 3DES with the 8B DES-key 244 * replicated 2 more times to match the 24B 3DES-key. 245 * Eg. If org. key is "0x0a 0x0b", then new key is 246 * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b" 247 */ 248 if (key_len == 8) { 249 /* Skipping the first 8B as it will be copied 250 * in the regular code flow 251 */ 252 memcpy(fctx->enc.encr_key+key_len, key, key_len); 253 memcpy(fctx->enc.encr_key+2*key_len, key, key_len); 254 } 255 break; 256 case DES3_ECB: 257 /* For DES3_ECB IV need to be from CTX. */ 258 fctx->enc.iv_source = CPT_FROM_CTX; 259 break; 260 case AES_CBC: 261 case AES_ECB: 262 case AES_CFB: 263 case AES_CTR: 264 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 265 break; 266 case AES_GCM: 267 /* Even though iv source is from dptr, 268 * aes_gcm salt is taken from ctx 269 */ 270 if (salt) { 271 memcpy(fctx->enc.encr_iv, salt, 4); 272 /* Assuming it was just salt update 273 * and nothing else 274 */ 275 if (!key) 276 goto success; 277 } 278 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 279 break; 280 case AES_XTS: 281 key_len = key_len / 2; 282 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 283 284 /* Copy key2 for XTS into ipad */ 285 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 286 memcpy(fctx->hmac.ipad, &key[key_len], key_len); 287 break; 288 case SNOW3G_UEA2: 289 cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len); 290 goto success; 291 case ZUC_EEA3: 292 cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len); 293 goto success; 294 case KASUMI_F8_ECB: 295 cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len); 296 goto success; 297 case KASUMI_F8_CBC: 298 cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len); 299 goto success; 300 default: 301 return -1; 302 } 303 304 /* Only for FC_GEN case */ 305 306 /* For GMAC auth, cipher must be NULL */ 307 if (cpt_ctx->hash_type != GMAC_TYPE) 308 fctx->enc.enc_cipher = type; 309 310 memcpy(fctx->enc.encr_key, key, key_len); 311 312 success: 313 cpt_ctx->enc_cipher = type; 314 315 return 0; 316 } 317 318 static __rte_always_inline uint32_t 319 fill_sg_comp(sg_comp_t *list, 320 uint32_t i, 321 phys_addr_t dma_addr, 322 uint32_t size) 323 { 324 sg_comp_t *to = &list[i>>2]; 325 326 to->u.s.len[i%4] = rte_cpu_to_be_16(size); 327 to->ptr[i%4] = rte_cpu_to_be_64(dma_addr); 328 i++; 329 return i; 330 } 331 332 static __rte_always_inline uint32_t 333 fill_sg_comp_from_buf(sg_comp_t *list, 334 uint32_t i, 335 buf_ptr_t *from) 336 { 337 sg_comp_t *to = &list[i>>2]; 338 339 to->u.s.len[i%4] = rte_cpu_to_be_16(from->size); 340 to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr); 341 i++; 342 return i; 343 } 344 345 static __rte_always_inline uint32_t 346 fill_sg_comp_from_buf_min(sg_comp_t *list, 347 uint32_t i, 348 buf_ptr_t *from, 349 uint32_t *psize) 350 { 351 sg_comp_t *to = &list[i >> 2]; 352 uint32_t size = *psize; 353 uint32_t e_len; 354 355 e_len = (size > from->size) ? from->size : size; 356 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 357 to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr); 358 *psize -= e_len; 359 i++; 360 return i; 361 } 362 363 /* 364 * This fills the MC expected SGIO list 365 * from IOV given by user. 366 */ 367 static __rte_always_inline uint32_t 368 fill_sg_comp_from_iov(sg_comp_t *list, 369 uint32_t i, 370 iov_ptr_t *from, uint32_t from_offset, 371 uint32_t *psize, buf_ptr_t *extra_buf, 372 uint32_t extra_offset) 373 { 374 int32_t j; 375 uint32_t extra_len = extra_buf ? extra_buf->size : 0; 376 uint32_t size = *psize - extra_len; 377 buf_ptr_t *bufs; 378 379 bufs = from->bufs; 380 for (j = 0; (j < from->buf_cnt) && size; j++) { 381 phys_addr_t e_dma_addr; 382 uint32_t e_len; 383 sg_comp_t *to = &list[i >> 2]; 384 385 if (!bufs[j].size) 386 continue; 387 388 if (unlikely(from_offset)) { 389 if (from_offset >= bufs[j].size) { 390 from_offset -= bufs[j].size; 391 continue; 392 } 393 e_dma_addr = bufs[j].dma_addr + from_offset; 394 e_len = (size > (bufs[j].size - from_offset)) ? 395 (bufs[j].size - from_offset) : size; 396 from_offset = 0; 397 } else { 398 e_dma_addr = bufs[j].dma_addr; 399 e_len = (size > bufs[j].size) ? 400 bufs[j].size : size; 401 } 402 403 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 404 to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr); 405 406 if (extra_len && (e_len >= extra_offset)) { 407 /* Break the data at given offset */ 408 uint32_t next_len = e_len - extra_offset; 409 phys_addr_t next_dma = e_dma_addr + extra_offset; 410 411 if (!extra_offset) { 412 i--; 413 } else { 414 e_len = extra_offset; 415 size -= e_len; 416 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 417 } 418 419 /* Insert extra data ptr */ 420 if (extra_len) { 421 i++; 422 to = &list[i >> 2]; 423 to->u.s.len[i % 4] = 424 rte_cpu_to_be_16(extra_buf->size); 425 to->ptr[i % 4] = 426 rte_cpu_to_be_64(extra_buf->dma_addr); 427 428 /* size already decremented by extra len */ 429 } 430 431 /* insert the rest of the data */ 432 if (next_len) { 433 i++; 434 to = &list[i >> 2]; 435 to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len); 436 to->ptr[i % 4] = rte_cpu_to_be_64(next_dma); 437 size -= next_len; 438 } 439 extra_len = 0; 440 441 } else { 442 size -= e_len; 443 } 444 if (extra_offset) 445 extra_offset -= size; 446 i++; 447 } 448 449 *psize = size; 450 return (uint32_t)i; 451 } 452 453 static __rte_always_inline void 454 cpt_digest_gen_prep(uint32_t flags, 455 uint64_t d_lens, 456 digest_params_t *params, 457 void *op, 458 void **prep_req) 459 { 460 struct cpt_request_info *req; 461 uint32_t size, i; 462 uint16_t data_len, mac_len, key_len; 463 auth_type_t hash_type; 464 buf_ptr_t *meta_p; 465 struct cpt_ctx *ctx; 466 sg_comp_t *gather_comp; 467 sg_comp_t *scatter_comp; 468 uint8_t *in_buffer; 469 uint32_t g_size_bytes, s_size_bytes; 470 uint64_t dptr_dma, rptr_dma; 471 vq_cmd_word0_t vq_cmd_w0; 472 vq_cmd_word3_t vq_cmd_w3; 473 void *c_vaddr, *m_vaddr; 474 uint64_t c_dma, m_dma; 475 opcode_info_t opcode; 476 477 ctx = params->ctx_buf.vaddr; 478 meta_p = ¶ms->meta_buf; 479 480 m_vaddr = meta_p->vaddr; 481 m_dma = meta_p->dma_addr; 482 483 /* 484 * Save initial space that followed app data for completion code & 485 * alternate completion code to fall in same cache line as app data 486 */ 487 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 488 m_dma += COMPLETION_CODE_SIZE; 489 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 490 (uint8_t *)m_vaddr; 491 c_vaddr = (uint8_t *)m_vaddr + size; 492 c_dma = m_dma + size; 493 size += sizeof(cpt_res_s_t); 494 495 m_vaddr = (uint8_t *)m_vaddr + size; 496 m_dma += size; 497 498 req = m_vaddr; 499 500 size = sizeof(struct cpt_request_info); 501 m_vaddr = (uint8_t *)m_vaddr + size; 502 m_dma += size; 503 504 hash_type = ctx->hash_type; 505 mac_len = ctx->mac_len; 506 key_len = ctx->auth_key_len; 507 data_len = AUTH_DLEN(d_lens); 508 509 /*GP op header */ 510 vq_cmd_w0.u64 = 0; 511 vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8); 512 if (ctx->hmac) { 513 opcode.s.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE; 514 vq_cmd_w0.s.param1 = key_len; 515 vq_cmd_w0.s.dlen = data_len + ROUNDUP8(key_len); 516 } else { 517 opcode.s.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE; 518 vq_cmd_w0.s.param1 = 0; 519 vq_cmd_w0.s.dlen = data_len; 520 } 521 522 opcode.s.minor = 0; 523 524 /* Null auth only case enters the if */ 525 if (unlikely(!hash_type && !ctx->enc_cipher)) { 526 opcode.s.major = CPT_MAJOR_OP_MISC; 527 /* Minor op is passthrough */ 528 opcode.s.minor = 0x03; 529 /* Send out completion code only */ 530 vq_cmd_w0.s.param2 = 0x1; 531 } 532 533 vq_cmd_w0.s.opcode = opcode.flags; 534 535 /* DPTR has SG list */ 536 in_buffer = m_vaddr; 537 dptr_dma = m_dma; 538 539 ((uint16_t *)in_buffer)[0] = 0; 540 ((uint16_t *)in_buffer)[1] = 0; 541 542 /* TODO Add error check if space will be sufficient */ 543 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 544 545 /* 546 * Input gather list 547 */ 548 549 i = 0; 550 551 if (ctx->hmac) { 552 uint64_t k_dma = params->ctx_buf.dma_addr + 553 offsetof(struct cpt_ctx, auth_key); 554 /* Key */ 555 i = fill_sg_comp(gather_comp, i, k_dma, ROUNDUP8(key_len)); 556 } 557 558 /* input data */ 559 size = data_len; 560 if (size) { 561 i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 562 0, &size, NULL, 0); 563 if (unlikely(size)) { 564 CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short" 565 " by %dB", size); 566 return; 567 } 568 } else { 569 /* 570 * Looks like we need to support zero data 571 * gather ptr in case of hash & hmac 572 */ 573 i++; 574 } 575 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 576 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 577 578 /* 579 * Output Gather list 580 */ 581 582 i = 0; 583 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 584 585 if (flags & VALID_MAC_BUF) { 586 if (unlikely(params->mac_buf.size < mac_len)) { 587 CPT_LOG_DP_ERR("Insufficient MAC size"); 588 return; 589 } 590 591 size = mac_len; 592 i = fill_sg_comp_from_buf_min(scatter_comp, i, 593 ¶ms->mac_buf, &size); 594 } else { 595 size = mac_len; 596 i = fill_sg_comp_from_iov(scatter_comp, i, 597 params->src_iov, data_len, 598 &size, NULL, 0); 599 if (unlikely(size)) { 600 CPT_LOG_DP_ERR("Insufficient dst IOV size, short by" 601 " %dB", size); 602 return; 603 } 604 } 605 606 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 607 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 608 609 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 610 611 /* This is DPTR len incase of SG mode */ 612 vq_cmd_w0.s.dlen = size; 613 614 m_vaddr = (uint8_t *)m_vaddr + size; 615 m_dma += size; 616 617 /* cpt alternate completion address saved earlier */ 618 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 619 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 620 rptr_dma = c_dma - 8; 621 622 req->ist.ei1 = dptr_dma; 623 req->ist.ei2 = rptr_dma; 624 625 /* vq command w3 */ 626 vq_cmd_w3.u64 = 0; 627 628 /* 16 byte aligned cpt res address */ 629 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 630 *req->completion_addr = COMPLETION_CODE_INIT; 631 req->comp_baddr = c_dma; 632 633 /* Fill microcode part of instruction */ 634 req->ist.ei0 = vq_cmd_w0.u64; 635 req->ist.ei3 = vq_cmd_w3.u64; 636 637 req->op = op; 638 639 *prep_req = req; 640 return; 641 } 642 643 static __rte_always_inline void 644 cpt_enc_hmac_prep(uint32_t flags, 645 uint64_t d_offs, 646 uint64_t d_lens, 647 fc_params_t *fc_params, 648 void *op, 649 void **prep_req) 650 { 651 uint32_t iv_offset = 0; 652 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 653 struct cpt_ctx *cpt_ctx; 654 uint32_t cipher_type, hash_type; 655 uint32_t mac_len, size; 656 uint8_t iv_len = 16; 657 struct cpt_request_info *req; 658 buf_ptr_t *meta_p, *aad_buf = NULL; 659 uint32_t encr_offset, auth_offset; 660 uint32_t encr_data_len, auth_data_len, aad_len = 0; 661 uint32_t passthrough_len = 0; 662 void *m_vaddr, *offset_vaddr; 663 uint64_t m_dma, offset_dma, ctx_dma; 664 vq_cmd_word0_t vq_cmd_w0; 665 vq_cmd_word3_t vq_cmd_w3; 666 void *c_vaddr; 667 uint64_t c_dma; 668 opcode_info_t opcode; 669 670 meta_p = &fc_params->meta_buf; 671 m_vaddr = meta_p->vaddr; 672 m_dma = meta_p->dma_addr; 673 674 encr_offset = ENCR_OFFSET(d_offs); 675 auth_offset = AUTH_OFFSET(d_offs); 676 encr_data_len = ENCR_DLEN(d_lens); 677 auth_data_len = AUTH_DLEN(d_lens); 678 if (unlikely(flags & VALID_AAD_BUF)) { 679 /* 680 * We dont support both aad 681 * and auth data separately 682 */ 683 auth_data_len = 0; 684 auth_offset = 0; 685 aad_len = fc_params->aad_buf.size; 686 aad_buf = &fc_params->aad_buf; 687 } 688 cpt_ctx = fc_params->ctx_buf.vaddr; 689 cipher_type = cpt_ctx->enc_cipher; 690 hash_type = cpt_ctx->hash_type; 691 mac_len = cpt_ctx->mac_len; 692 693 /* 694 * Save initial space that followed app data for completion code & 695 * alternate completion code to fall in same cache line as app data 696 */ 697 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 698 m_dma += COMPLETION_CODE_SIZE; 699 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 700 (uint8_t *)m_vaddr; 701 702 c_vaddr = (uint8_t *)m_vaddr + size; 703 c_dma = m_dma + size; 704 size += sizeof(cpt_res_s_t); 705 706 m_vaddr = (uint8_t *)m_vaddr + size; 707 m_dma += size; 708 709 /* start cpt request info struct at 8 byte boundary */ 710 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 711 (uint8_t *)m_vaddr; 712 713 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 714 715 size += sizeof(struct cpt_request_info); 716 m_vaddr = (uint8_t *)m_vaddr + size; 717 m_dma += size; 718 719 if (hash_type == GMAC_TYPE) 720 encr_data_len = 0; 721 722 if (unlikely(!(flags & VALID_IV_BUF))) { 723 iv_len = 0; 724 iv_offset = ENCR_IV_OFFSET(d_offs); 725 } 726 727 if (unlikely(flags & VALID_AAD_BUF)) { 728 /* 729 * When AAD is given, data above encr_offset is pass through 730 * Since AAD is given as separate pointer and not as offset, 731 * this is a special case as we need to fragment input data 732 * into passthrough + encr_data and then insert AAD in between. 733 */ 734 if (hash_type != GMAC_TYPE) { 735 passthrough_len = encr_offset; 736 auth_offset = passthrough_len + iv_len; 737 encr_offset = passthrough_len + aad_len + iv_len; 738 auth_data_len = aad_len + encr_data_len; 739 } else { 740 passthrough_len = 16 + aad_len; 741 auth_offset = passthrough_len + iv_len; 742 auth_data_len = aad_len; 743 } 744 } else { 745 encr_offset += iv_len; 746 auth_offset += iv_len; 747 } 748 749 /* Encryption */ 750 opcode.s.major = CPT_MAJOR_OP_FC; 751 opcode.s.minor = 0; 752 753 auth_dlen = auth_offset + auth_data_len; 754 enc_dlen = encr_data_len + encr_offset; 755 if (unlikely(encr_data_len & 0xf)) { 756 if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB)) 757 enc_dlen = ROUNDUP8(encr_data_len) + encr_offset; 758 else if (likely((cipher_type == AES_CBC) || 759 (cipher_type == AES_ECB))) 760 enc_dlen = ROUNDUP16(encr_data_len) + encr_offset; 761 } 762 763 if (unlikely(hash_type == GMAC_TYPE)) { 764 encr_offset = auth_dlen; 765 enc_dlen = 0; 766 } 767 768 if (unlikely(auth_dlen > enc_dlen)) { 769 inputlen = auth_dlen; 770 outputlen = auth_dlen + mac_len; 771 } else { 772 inputlen = enc_dlen; 773 outputlen = enc_dlen + mac_len; 774 } 775 776 /* GP op header */ 777 vq_cmd_w0.u64 = 0; 778 vq_cmd_w0.s.param1 = encr_data_len; 779 vq_cmd_w0.s.param2 = auth_data_len; 780 /* 781 * In 83XX since we have a limitation of 782 * IV & Offset control word not part of instruction 783 * and need to be part of Data Buffer, we check if 784 * head room is there and then only do the Direct mode processing 785 */ 786 if (likely((flags & SINGLE_BUF_INPLACE) && 787 (flags & SINGLE_BUF_HEADTAILROOM))) { 788 void *dm_vaddr = fc_params->bufs[0].vaddr; 789 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 790 /* 791 * This flag indicates that there is 24 bytes head room and 792 * 8 bytes tail room available, so that we get to do 793 * DIRECT MODE with limitation 794 */ 795 796 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 797 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 798 799 /* DPTR */ 800 req->ist.ei1 = offset_dma; 801 /* RPTR should just exclude offset control word */ 802 req->ist.ei2 = dm_dma_addr - iv_len; 803 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 804 + outputlen - iv_len); 805 806 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 807 808 vq_cmd_w0.s.opcode = opcode.flags; 809 810 if (likely(iv_len)) { 811 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 812 + OFF_CTRL_LEN); 813 uint64_t *src = fc_params->iv_buf; 814 dest[0] = src[0]; 815 dest[1] = src[1]; 816 } 817 818 *(uint64_t *)offset_vaddr = 819 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 820 ((uint64_t)iv_offset << 8) | 821 ((uint64_t)auth_offset)); 822 823 } else { 824 uint32_t i, g_size_bytes, s_size_bytes; 825 uint64_t dptr_dma, rptr_dma; 826 sg_comp_t *gather_comp; 827 sg_comp_t *scatter_comp; 828 uint8_t *in_buffer; 829 830 /* This falls under strict SG mode */ 831 offset_vaddr = m_vaddr; 832 offset_dma = m_dma; 833 size = OFF_CTRL_LEN + iv_len; 834 835 m_vaddr = (uint8_t *)m_vaddr + size; 836 m_dma += size; 837 838 opcode.s.major |= CPT_DMA_MODE; 839 840 vq_cmd_w0.s.opcode = opcode.flags; 841 842 if (likely(iv_len)) { 843 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 844 + OFF_CTRL_LEN); 845 uint64_t *src = fc_params->iv_buf; 846 dest[0] = src[0]; 847 dest[1] = src[1]; 848 } 849 850 *(uint64_t *)offset_vaddr = 851 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 852 ((uint64_t)iv_offset << 8) | 853 ((uint64_t)auth_offset)); 854 855 /* DPTR has SG list */ 856 in_buffer = m_vaddr; 857 dptr_dma = m_dma; 858 859 ((uint16_t *)in_buffer)[0] = 0; 860 ((uint16_t *)in_buffer)[1] = 0; 861 862 /* TODO Add error check if space will be sufficient */ 863 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 864 865 /* 866 * Input Gather List 867 */ 868 869 i = 0; 870 871 /* Offset control word that includes iv */ 872 i = fill_sg_comp(gather_comp, i, offset_dma, 873 OFF_CTRL_LEN + iv_len); 874 875 /* Add input data */ 876 size = inputlen - iv_len; 877 if (likely(size)) { 878 uint32_t aad_offset = aad_len ? passthrough_len : 0; 879 880 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 881 i = fill_sg_comp_from_buf_min(gather_comp, i, 882 fc_params->bufs, 883 &size); 884 } else { 885 i = fill_sg_comp_from_iov(gather_comp, i, 886 fc_params->src_iov, 887 0, &size, 888 aad_buf, aad_offset); 889 } 890 891 if (unlikely(size)) { 892 CPT_LOG_DP_ERR("Insufficient buffer space," 893 " size %d needed", size); 894 return; 895 } 896 } 897 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 898 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 899 900 /* 901 * Output Scatter list 902 */ 903 i = 0; 904 scatter_comp = 905 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 906 907 /* Add IV */ 908 if (likely(iv_len)) { 909 i = fill_sg_comp(scatter_comp, i, 910 offset_dma + OFF_CTRL_LEN, 911 iv_len); 912 } 913 914 /* output data or output data + digest*/ 915 if (unlikely(flags & VALID_MAC_BUF)) { 916 size = outputlen - iv_len - mac_len; 917 if (size) { 918 uint32_t aad_offset = 919 aad_len ? passthrough_len : 0; 920 921 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 922 i = fill_sg_comp_from_buf_min( 923 scatter_comp, 924 i, 925 fc_params->bufs, 926 &size); 927 } else { 928 i = fill_sg_comp_from_iov(scatter_comp, 929 i, 930 fc_params->dst_iov, 931 0, 932 &size, 933 aad_buf, 934 aad_offset); 935 } 936 if (unlikely(size)) { 937 CPT_LOG_DP_ERR("Insufficient buffer" 938 " space, size %d needed", 939 size); 940 return; 941 } 942 } 943 /* mac_data */ 944 if (mac_len) { 945 i = fill_sg_comp_from_buf(scatter_comp, i, 946 &fc_params->mac_buf); 947 } 948 } else { 949 /* Output including mac */ 950 size = outputlen - iv_len; 951 if (likely(size)) { 952 uint32_t aad_offset = 953 aad_len ? passthrough_len : 0; 954 955 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 956 i = fill_sg_comp_from_buf_min( 957 scatter_comp, 958 i, 959 fc_params->bufs, 960 &size); 961 } else { 962 i = fill_sg_comp_from_iov(scatter_comp, 963 i, 964 fc_params->dst_iov, 965 0, 966 &size, 967 aad_buf, 968 aad_offset); 969 } 970 if (unlikely(size)) { 971 CPT_LOG_DP_ERR("Insufficient buffer" 972 " space, size %d needed", 973 size); 974 return; 975 } 976 } 977 } 978 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 979 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 980 981 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 982 983 /* This is DPTR len incase of SG mode */ 984 vq_cmd_w0.s.dlen = size; 985 986 m_vaddr = (uint8_t *)m_vaddr + size; 987 m_dma += size; 988 989 /* cpt alternate completion address saved earlier */ 990 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 991 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 992 rptr_dma = c_dma - 8; 993 994 req->ist.ei1 = dptr_dma; 995 req->ist.ei2 = rptr_dma; 996 } 997 998 ctx_dma = fc_params->ctx_buf.dma_addr + 999 offsetof(struct cpt_ctx, fctx); 1000 /* vq command w3 */ 1001 vq_cmd_w3.u64 = 0; 1002 vq_cmd_w3.s.grp = 0; 1003 vq_cmd_w3.s.cptr = ctx_dma; 1004 1005 /* 16 byte aligned cpt res address */ 1006 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1007 *req->completion_addr = COMPLETION_CODE_INIT; 1008 req->comp_baddr = c_dma; 1009 1010 /* Fill microcode part of instruction */ 1011 req->ist.ei0 = vq_cmd_w0.u64; 1012 req->ist.ei3 = vq_cmd_w3.u64; 1013 1014 req->op = op; 1015 1016 *prep_req = req; 1017 return; 1018 } 1019 1020 static __rte_always_inline void 1021 cpt_dec_hmac_prep(uint32_t flags, 1022 uint64_t d_offs, 1023 uint64_t d_lens, 1024 fc_params_t *fc_params, 1025 void *op, 1026 void **prep_req) 1027 { 1028 uint32_t iv_offset = 0, size; 1029 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 1030 struct cpt_ctx *cpt_ctx; 1031 int32_t hash_type, mac_len; 1032 uint8_t iv_len = 16; 1033 struct cpt_request_info *req; 1034 buf_ptr_t *meta_p, *aad_buf = NULL; 1035 uint32_t encr_offset, auth_offset; 1036 uint32_t encr_data_len, auth_data_len, aad_len = 0; 1037 uint32_t passthrough_len = 0; 1038 void *m_vaddr, *offset_vaddr; 1039 uint64_t m_dma, offset_dma, ctx_dma; 1040 opcode_info_t opcode; 1041 vq_cmd_word0_t vq_cmd_w0; 1042 vq_cmd_word3_t vq_cmd_w3; 1043 void *c_vaddr; 1044 uint64_t c_dma; 1045 1046 meta_p = &fc_params->meta_buf; 1047 m_vaddr = meta_p->vaddr; 1048 m_dma = meta_p->dma_addr; 1049 1050 encr_offset = ENCR_OFFSET(d_offs); 1051 auth_offset = AUTH_OFFSET(d_offs); 1052 encr_data_len = ENCR_DLEN(d_lens); 1053 auth_data_len = AUTH_DLEN(d_lens); 1054 1055 if (unlikely(flags & VALID_AAD_BUF)) { 1056 /* 1057 * We dont support both aad 1058 * and auth data separately 1059 */ 1060 auth_data_len = 0; 1061 auth_offset = 0; 1062 aad_len = fc_params->aad_buf.size; 1063 aad_buf = &fc_params->aad_buf; 1064 } 1065 1066 cpt_ctx = fc_params->ctx_buf.vaddr; 1067 hash_type = cpt_ctx->hash_type; 1068 mac_len = cpt_ctx->mac_len; 1069 1070 if (hash_type == GMAC_TYPE) 1071 encr_data_len = 0; 1072 1073 if (unlikely(!(flags & VALID_IV_BUF))) { 1074 iv_len = 0; 1075 iv_offset = ENCR_IV_OFFSET(d_offs); 1076 } 1077 1078 if (unlikely(flags & VALID_AAD_BUF)) { 1079 /* 1080 * When AAD is given, data above encr_offset is pass through 1081 * Since AAD is given as separate pointer and not as offset, 1082 * this is a special case as we need to fragment input data 1083 * into passthrough + encr_data and then insert AAD in between. 1084 */ 1085 if (hash_type != GMAC_TYPE) { 1086 passthrough_len = encr_offset; 1087 auth_offset = passthrough_len + iv_len; 1088 encr_offset = passthrough_len + aad_len + iv_len; 1089 auth_data_len = aad_len + encr_data_len; 1090 } else { 1091 passthrough_len = 16 + aad_len; 1092 auth_offset = passthrough_len + iv_len; 1093 auth_data_len = aad_len; 1094 } 1095 } else { 1096 encr_offset += iv_len; 1097 auth_offset += iv_len; 1098 } 1099 1100 /* 1101 * Save initial space that followed app data for completion code & 1102 * alternate completion code to fall in same cache line as app data 1103 */ 1104 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1105 m_dma += COMPLETION_CODE_SIZE; 1106 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1107 (uint8_t *)m_vaddr; 1108 c_vaddr = (uint8_t *)m_vaddr + size; 1109 c_dma = m_dma + size; 1110 size += sizeof(cpt_res_s_t); 1111 1112 m_vaddr = (uint8_t *)m_vaddr + size; 1113 m_dma += size; 1114 1115 /* start cpt request info structure at 8 byte alignment */ 1116 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 1117 (uint8_t *)m_vaddr; 1118 1119 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 1120 1121 size += sizeof(struct cpt_request_info); 1122 m_vaddr = (uint8_t *)m_vaddr + size; 1123 m_dma += size; 1124 1125 /* Decryption */ 1126 opcode.s.major = CPT_MAJOR_OP_FC; 1127 opcode.s.minor = 1; 1128 1129 enc_dlen = encr_offset + encr_data_len; 1130 auth_dlen = auth_offset + auth_data_len; 1131 1132 if (auth_dlen > enc_dlen) { 1133 inputlen = auth_dlen + mac_len; 1134 outputlen = auth_dlen; 1135 } else { 1136 inputlen = enc_dlen + mac_len; 1137 outputlen = enc_dlen; 1138 } 1139 1140 if (hash_type == GMAC_TYPE) 1141 encr_offset = inputlen; 1142 1143 vq_cmd_w0.u64 = 0; 1144 vq_cmd_w0.s.param1 = encr_data_len; 1145 vq_cmd_w0.s.param2 = auth_data_len; 1146 1147 /* 1148 * In 83XX since we have a limitation of 1149 * IV & Offset control word not part of instruction 1150 * and need to be part of Data Buffer, we check if 1151 * head room is there and then only do the Direct mode processing 1152 */ 1153 if (likely((flags & SINGLE_BUF_INPLACE) && 1154 (flags & SINGLE_BUF_HEADTAILROOM))) { 1155 void *dm_vaddr = fc_params->bufs[0].vaddr; 1156 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 1157 /* 1158 * This flag indicates that there is 24 bytes head room and 1159 * 8 bytes tail room available, so that we get to do 1160 * DIRECT MODE with limitation 1161 */ 1162 1163 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 1164 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1165 req->ist.ei1 = offset_dma; 1166 1167 /* RPTR should just exclude offset control word */ 1168 req->ist.ei2 = dm_dma_addr - iv_len; 1169 1170 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr + 1171 outputlen - iv_len); 1172 /* since this is decryption, 1173 * don't touch the content of 1174 * alternate ccode space as it contains 1175 * hmac. 1176 */ 1177 1178 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1179 1180 vq_cmd_w0.s.opcode = opcode.flags; 1181 1182 if (likely(iv_len)) { 1183 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1184 OFF_CTRL_LEN); 1185 uint64_t *src = fc_params->iv_buf; 1186 dest[0] = src[0]; 1187 dest[1] = src[1]; 1188 } 1189 1190 *(uint64_t *)offset_vaddr = 1191 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1192 ((uint64_t)iv_offset << 8) | 1193 ((uint64_t)auth_offset)); 1194 1195 } else { 1196 uint64_t dptr_dma, rptr_dma; 1197 uint32_t g_size_bytes, s_size_bytes; 1198 sg_comp_t *gather_comp; 1199 sg_comp_t *scatter_comp; 1200 uint8_t *in_buffer; 1201 uint8_t i = 0; 1202 1203 /* This falls under strict SG mode */ 1204 offset_vaddr = m_vaddr; 1205 offset_dma = m_dma; 1206 size = OFF_CTRL_LEN + iv_len; 1207 1208 m_vaddr = (uint8_t *)m_vaddr + size; 1209 m_dma += size; 1210 1211 opcode.s.major |= CPT_DMA_MODE; 1212 1213 vq_cmd_w0.s.opcode = opcode.flags; 1214 1215 if (likely(iv_len)) { 1216 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1217 OFF_CTRL_LEN); 1218 uint64_t *src = fc_params->iv_buf; 1219 dest[0] = src[0]; 1220 dest[1] = src[1]; 1221 } 1222 1223 *(uint64_t *)offset_vaddr = 1224 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1225 ((uint64_t)iv_offset << 8) | 1226 ((uint64_t)auth_offset)); 1227 1228 /* DPTR has SG list */ 1229 in_buffer = m_vaddr; 1230 dptr_dma = m_dma; 1231 1232 ((uint16_t *)in_buffer)[0] = 0; 1233 ((uint16_t *)in_buffer)[1] = 0; 1234 1235 /* TODO Add error check if space will be sufficient */ 1236 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1237 1238 /* 1239 * Input Gather List 1240 */ 1241 i = 0; 1242 1243 /* Offset control word that includes iv */ 1244 i = fill_sg_comp(gather_comp, i, offset_dma, 1245 OFF_CTRL_LEN + iv_len); 1246 1247 /* Add input data */ 1248 if (flags & VALID_MAC_BUF) { 1249 size = inputlen - iv_len - mac_len; 1250 if (size) { 1251 /* input data only */ 1252 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1253 i = fill_sg_comp_from_buf_min( 1254 gather_comp, i, 1255 fc_params->bufs, 1256 &size); 1257 } else { 1258 uint32_t aad_offset = aad_len ? 1259 passthrough_len : 0; 1260 1261 i = fill_sg_comp_from_iov(gather_comp, 1262 i, 1263 fc_params->src_iov, 1264 0, &size, 1265 aad_buf, 1266 aad_offset); 1267 } 1268 if (unlikely(size)) { 1269 CPT_LOG_DP_ERR("Insufficient buffer" 1270 " space, size %d needed", 1271 size); 1272 return; 1273 } 1274 } 1275 1276 /* mac data */ 1277 if (mac_len) { 1278 i = fill_sg_comp_from_buf(gather_comp, i, 1279 &fc_params->mac_buf); 1280 } 1281 } else { 1282 /* input data + mac */ 1283 size = inputlen - iv_len; 1284 if (size) { 1285 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1286 i = fill_sg_comp_from_buf_min( 1287 gather_comp, i, 1288 fc_params->bufs, 1289 &size); 1290 } else { 1291 uint32_t aad_offset = aad_len ? 1292 passthrough_len : 0; 1293 1294 if (unlikely(!fc_params->src_iov)) { 1295 CPT_LOG_DP_ERR("Bad input args"); 1296 return; 1297 } 1298 1299 i = fill_sg_comp_from_iov( 1300 gather_comp, i, 1301 fc_params->src_iov, 1302 0, &size, 1303 aad_buf, 1304 aad_offset); 1305 } 1306 1307 if (unlikely(size)) { 1308 CPT_LOG_DP_ERR("Insufficient buffer" 1309 " space, size %d needed", 1310 size); 1311 return; 1312 } 1313 } 1314 } 1315 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1316 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1317 1318 /* 1319 * Output Scatter List 1320 */ 1321 1322 i = 0; 1323 scatter_comp = 1324 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1325 1326 /* Add iv */ 1327 if (iv_len) { 1328 i = fill_sg_comp(scatter_comp, i, 1329 offset_dma + OFF_CTRL_LEN, 1330 iv_len); 1331 } 1332 1333 /* Add output data */ 1334 size = outputlen - iv_len; 1335 if (size) { 1336 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1337 /* handle single buffer here */ 1338 i = fill_sg_comp_from_buf_min(scatter_comp, i, 1339 fc_params->bufs, 1340 &size); 1341 } else { 1342 uint32_t aad_offset = aad_len ? 1343 passthrough_len : 0; 1344 1345 if (unlikely(!fc_params->dst_iov)) { 1346 CPT_LOG_DP_ERR("Bad input args"); 1347 return; 1348 } 1349 1350 i = fill_sg_comp_from_iov(scatter_comp, i, 1351 fc_params->dst_iov, 0, 1352 &size, aad_buf, 1353 aad_offset); 1354 } 1355 1356 if (unlikely(size)) { 1357 CPT_LOG_DP_ERR("Insufficient buffer space," 1358 " size %d needed", size); 1359 return; 1360 } 1361 } 1362 1363 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1364 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1365 1366 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1367 1368 /* This is DPTR len incase of SG mode */ 1369 vq_cmd_w0.s.dlen = size; 1370 1371 m_vaddr = (uint8_t *)m_vaddr + size; 1372 m_dma += size; 1373 1374 /* cpt alternate completion address saved earlier */ 1375 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1376 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1377 rptr_dma = c_dma - 8; 1378 size += COMPLETION_CODE_SIZE; 1379 1380 req->ist.ei1 = dptr_dma; 1381 req->ist.ei2 = rptr_dma; 1382 } 1383 1384 ctx_dma = fc_params->ctx_buf.dma_addr + 1385 offsetof(struct cpt_ctx, fctx); 1386 /* vq command w3 */ 1387 vq_cmd_w3.u64 = 0; 1388 vq_cmd_w3.s.grp = 0; 1389 vq_cmd_w3.s.cptr = ctx_dma; 1390 1391 /* 16 byte aligned cpt res address */ 1392 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1393 *req->completion_addr = COMPLETION_CODE_INIT; 1394 req->comp_baddr = c_dma; 1395 1396 /* Fill microcode part of instruction */ 1397 req->ist.ei0 = vq_cmd_w0.u64; 1398 req->ist.ei3 = vq_cmd_w3.u64; 1399 1400 req->op = op; 1401 1402 *prep_req = req; 1403 return; 1404 } 1405 1406 static __rte_always_inline void 1407 cpt_zuc_snow3g_enc_prep(uint32_t req_flags, 1408 uint64_t d_offs, 1409 uint64_t d_lens, 1410 fc_params_t *params, 1411 void *op, 1412 void **prep_req) 1413 { 1414 uint32_t size; 1415 int32_t inputlen, outputlen; 1416 struct cpt_ctx *cpt_ctx; 1417 uint32_t mac_len = 0; 1418 uint8_t snow3g, j; 1419 struct cpt_request_info *req; 1420 buf_ptr_t *buf_p; 1421 uint32_t encr_offset = 0, auth_offset = 0; 1422 uint32_t encr_data_len = 0, auth_data_len = 0; 1423 int flags, iv_len = 16; 1424 void *m_vaddr, *c_vaddr; 1425 uint64_t m_dma, c_dma, offset_ctrl; 1426 uint64_t *offset_vaddr, offset_dma; 1427 uint32_t *iv_s, iv[4]; 1428 vq_cmd_word0_t vq_cmd_w0; 1429 vq_cmd_word3_t vq_cmd_w3; 1430 opcode_info_t opcode; 1431 1432 buf_p = ¶ms->meta_buf; 1433 m_vaddr = buf_p->vaddr; 1434 m_dma = buf_p->dma_addr; 1435 1436 cpt_ctx = params->ctx_buf.vaddr; 1437 flags = cpt_ctx->zsk_flags; 1438 mac_len = cpt_ctx->mac_len; 1439 snow3g = cpt_ctx->snow3g; 1440 1441 /* 1442 * Save initial space that followed app data for completion code & 1443 * alternate completion code to fall in same cache line as app data 1444 */ 1445 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1446 m_dma += COMPLETION_CODE_SIZE; 1447 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1448 (uint8_t *)m_vaddr; 1449 1450 c_vaddr = (uint8_t *)m_vaddr + size; 1451 c_dma = m_dma + size; 1452 size += sizeof(cpt_res_s_t); 1453 1454 m_vaddr = (uint8_t *)m_vaddr + size; 1455 m_dma += size; 1456 1457 /* Reserve memory for cpt request info */ 1458 req = m_vaddr; 1459 1460 size = sizeof(struct cpt_request_info); 1461 m_vaddr = (uint8_t *)m_vaddr + size; 1462 m_dma += size; 1463 1464 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1465 1466 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1467 1468 opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1469 (0 << 3) | (flags & 0x7)); 1470 1471 if (flags == 0x1) { 1472 /* 1473 * Microcode expects offsets in bytes 1474 * TODO: Rounding off 1475 */ 1476 auth_data_len = AUTH_DLEN(d_lens); 1477 1478 /* EIA3 or UIA2 */ 1479 auth_offset = AUTH_OFFSET(d_offs); 1480 auth_offset = auth_offset / 8; 1481 1482 /* consider iv len */ 1483 auth_offset += iv_len; 1484 1485 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 1486 outputlen = mac_len; 1487 1488 offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset); 1489 1490 } else { 1491 /* EEA3 or UEA2 */ 1492 /* 1493 * Microcode expects offsets in bytes 1494 * TODO: Rounding off 1495 */ 1496 encr_data_len = ENCR_DLEN(d_lens); 1497 1498 encr_offset = ENCR_OFFSET(d_offs); 1499 encr_offset = encr_offset / 8; 1500 /* consider iv len */ 1501 encr_offset += iv_len; 1502 1503 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 1504 outputlen = inputlen; 1505 1506 /* iv offset is 0 */ 1507 offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1508 } 1509 1510 /* IV */ 1511 iv_s = (flags == 0x1) ? params->auth_iv_buf : 1512 params->iv_buf; 1513 1514 if (snow3g) { 1515 /* 1516 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1517 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1518 */ 1519 1520 for (j = 0; j < 4; j++) 1521 iv[j] = iv_s[3 - j]; 1522 } else { 1523 /* ZUC doesn't need a swap */ 1524 for (j = 0; j < 4; j++) 1525 iv[j] = iv_s[j]; 1526 } 1527 1528 /* 1529 * GP op header, lengths are expected in bits. 1530 */ 1531 vq_cmd_w0.u64 = 0; 1532 vq_cmd_w0.s.param1 = encr_data_len; 1533 vq_cmd_w0.s.param2 = auth_data_len; 1534 1535 /* 1536 * In 83XX since we have a limitation of 1537 * IV & Offset control word not part of instruction 1538 * and need to be part of Data Buffer, we check if 1539 * head room is there and then only do the Direct mode processing 1540 */ 1541 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1542 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1543 void *dm_vaddr = params->bufs[0].vaddr; 1544 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1545 /* 1546 * This flag indicates that there is 24 bytes head room and 1547 * 8 bytes tail room available, so that we get to do 1548 * DIRECT MODE with limitation 1549 */ 1550 1551 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1552 OFF_CTRL_LEN - iv_len); 1553 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1554 1555 /* DPTR */ 1556 req->ist.ei1 = offset_dma; 1557 /* RPTR should just exclude offset control word */ 1558 req->ist.ei2 = dm_dma_addr - iv_len; 1559 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1560 + outputlen - iv_len); 1561 1562 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1563 1564 vq_cmd_w0.s.opcode = opcode.flags; 1565 1566 if (likely(iv_len)) { 1567 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1568 + OFF_CTRL_LEN); 1569 memcpy(iv_d, iv, 16); 1570 } 1571 1572 *offset_vaddr = offset_ctrl; 1573 } else { 1574 uint32_t i, g_size_bytes, s_size_bytes; 1575 uint64_t dptr_dma, rptr_dma; 1576 sg_comp_t *gather_comp; 1577 sg_comp_t *scatter_comp; 1578 uint8_t *in_buffer; 1579 uint32_t *iv_d; 1580 1581 /* save space for iv */ 1582 offset_vaddr = m_vaddr; 1583 offset_dma = m_dma; 1584 1585 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1586 m_dma += OFF_CTRL_LEN + iv_len; 1587 1588 opcode.s.major |= CPT_DMA_MODE; 1589 1590 vq_cmd_w0.s.opcode = opcode.flags; 1591 1592 /* DPTR has SG list */ 1593 in_buffer = m_vaddr; 1594 dptr_dma = m_dma; 1595 1596 ((uint16_t *)in_buffer)[0] = 0; 1597 ((uint16_t *)in_buffer)[1] = 0; 1598 1599 /* TODO Add error check if space will be sufficient */ 1600 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1601 1602 /* 1603 * Input Gather List 1604 */ 1605 i = 0; 1606 1607 /* Offset control word followed by iv */ 1608 1609 i = fill_sg_comp(gather_comp, i, offset_dma, 1610 OFF_CTRL_LEN + iv_len); 1611 1612 /* iv offset is 0 */ 1613 *offset_vaddr = offset_ctrl; 1614 1615 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1616 memcpy(iv_d, iv, 16); 1617 1618 /* input data */ 1619 size = inputlen - iv_len; 1620 if (size) { 1621 i = fill_sg_comp_from_iov(gather_comp, i, 1622 params->src_iov, 1623 0, &size, NULL, 0); 1624 if (unlikely(size)) { 1625 CPT_LOG_DP_ERR("Insufficient buffer space," 1626 " size %d needed", size); 1627 return; 1628 } 1629 } 1630 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1631 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1632 1633 /* 1634 * Output Scatter List 1635 */ 1636 1637 i = 0; 1638 scatter_comp = 1639 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1640 1641 if (flags == 0x1) { 1642 /* IV in SLIST only for EEA3 & UEA2 */ 1643 iv_len = 0; 1644 } 1645 1646 if (iv_len) { 1647 i = fill_sg_comp(scatter_comp, i, 1648 offset_dma + OFF_CTRL_LEN, iv_len); 1649 } 1650 1651 /* Add output data */ 1652 if (req_flags & VALID_MAC_BUF) { 1653 size = outputlen - iv_len - mac_len; 1654 if (size) { 1655 i = fill_sg_comp_from_iov(scatter_comp, i, 1656 params->dst_iov, 0, 1657 &size, NULL, 0); 1658 1659 if (unlikely(size)) { 1660 CPT_LOG_DP_ERR("Insufficient buffer space," 1661 " size %d needed", size); 1662 return; 1663 } 1664 } 1665 1666 /* mac data */ 1667 if (mac_len) { 1668 i = fill_sg_comp_from_buf(scatter_comp, i, 1669 ¶ms->mac_buf); 1670 } 1671 } else { 1672 /* Output including mac */ 1673 size = outputlen - iv_len; 1674 if (size) { 1675 i = fill_sg_comp_from_iov(scatter_comp, i, 1676 params->dst_iov, 0, 1677 &size, NULL, 0); 1678 1679 if (unlikely(size)) { 1680 CPT_LOG_DP_ERR("Insufficient buffer space," 1681 " size %d needed", size); 1682 return; 1683 } 1684 } 1685 } 1686 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1687 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1688 1689 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1690 1691 /* This is DPTR len incase of SG mode */ 1692 vq_cmd_w0.s.dlen = size; 1693 1694 m_vaddr = (uint8_t *)m_vaddr + size; 1695 m_dma += size; 1696 1697 /* cpt alternate completion address saved earlier */ 1698 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1699 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1700 rptr_dma = c_dma - 8; 1701 1702 req->ist.ei1 = dptr_dma; 1703 req->ist.ei2 = rptr_dma; 1704 } 1705 1706 /* vq command w3 */ 1707 vq_cmd_w3.u64 = 0; 1708 vq_cmd_w3.s.grp = 0; 1709 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1710 offsetof(struct cpt_ctx, zs_ctx); 1711 1712 /* 16 byte aligned cpt res address */ 1713 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1714 *req->completion_addr = COMPLETION_CODE_INIT; 1715 req->comp_baddr = c_dma; 1716 1717 /* Fill microcode part of instruction */ 1718 req->ist.ei0 = vq_cmd_w0.u64; 1719 req->ist.ei3 = vq_cmd_w3.u64; 1720 1721 req->op = op; 1722 1723 *prep_req = req; 1724 return; 1725 } 1726 1727 static __rte_always_inline void 1728 cpt_zuc_snow3g_dec_prep(uint32_t req_flags, 1729 uint64_t d_offs, 1730 uint64_t d_lens, 1731 fc_params_t *params, 1732 void *op, 1733 void **prep_req) 1734 { 1735 uint32_t size; 1736 int32_t inputlen = 0, outputlen; 1737 struct cpt_ctx *cpt_ctx; 1738 uint8_t snow3g, iv_len = 16; 1739 struct cpt_request_info *req; 1740 buf_ptr_t *buf_p; 1741 uint32_t encr_offset; 1742 uint32_t encr_data_len; 1743 int flags; 1744 void *m_vaddr, *c_vaddr; 1745 uint64_t m_dma, c_dma; 1746 uint64_t *offset_vaddr, offset_dma; 1747 uint32_t *iv_s, iv[4], j; 1748 vq_cmd_word0_t vq_cmd_w0; 1749 vq_cmd_word3_t vq_cmd_w3; 1750 opcode_info_t opcode; 1751 1752 buf_p = ¶ms->meta_buf; 1753 m_vaddr = buf_p->vaddr; 1754 m_dma = buf_p->dma_addr; 1755 1756 /* 1757 * Microcode expects offsets in bytes 1758 * TODO: Rounding off 1759 */ 1760 encr_offset = ENCR_OFFSET(d_offs) / 8; 1761 encr_data_len = ENCR_DLEN(d_lens); 1762 1763 cpt_ctx = params->ctx_buf.vaddr; 1764 flags = cpt_ctx->zsk_flags; 1765 snow3g = cpt_ctx->snow3g; 1766 /* 1767 * Save initial space that followed app data for completion code & 1768 * alternate completion code to fall in same cache line as app data 1769 */ 1770 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1771 m_dma += COMPLETION_CODE_SIZE; 1772 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1773 (uint8_t *)m_vaddr; 1774 1775 c_vaddr = (uint8_t *)m_vaddr + size; 1776 c_dma = m_dma + size; 1777 size += sizeof(cpt_res_s_t); 1778 1779 m_vaddr = (uint8_t *)m_vaddr + size; 1780 m_dma += size; 1781 1782 /* Reserve memory for cpt request info */ 1783 req = m_vaddr; 1784 1785 size = sizeof(struct cpt_request_info); 1786 m_vaddr = (uint8_t *)m_vaddr + size; 1787 m_dma += size; 1788 1789 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1790 1791 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1792 1793 opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1794 (0 << 3) | (flags & 0x7)); 1795 1796 /* consider iv len */ 1797 encr_offset += iv_len; 1798 1799 inputlen = encr_offset + 1800 (RTE_ALIGN(encr_data_len, 8) / 8); 1801 outputlen = inputlen; 1802 1803 /* IV */ 1804 iv_s = params->iv_buf; 1805 if (snow3g) { 1806 /* 1807 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1808 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1809 */ 1810 1811 for (j = 0; j < 4; j++) 1812 iv[j] = iv_s[3 - j]; 1813 } else { 1814 /* ZUC doesn't need a swap */ 1815 for (j = 0; j < 4; j++) 1816 iv[j] = iv_s[j]; 1817 } 1818 1819 /* 1820 * GP op header, lengths are expected in bits. 1821 */ 1822 vq_cmd_w0.u64 = 0; 1823 vq_cmd_w0.s.param1 = encr_data_len; 1824 1825 /* 1826 * In 83XX since we have a limitation of 1827 * IV & Offset control word not part of instruction 1828 * and need to be part of Data Buffer, we check if 1829 * head room is there and then only do the Direct mode processing 1830 */ 1831 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1832 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1833 void *dm_vaddr = params->bufs[0].vaddr; 1834 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1835 /* 1836 * This flag indicates that there is 24 bytes head room and 1837 * 8 bytes tail room available, so that we get to do 1838 * DIRECT MODE with limitation 1839 */ 1840 1841 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1842 OFF_CTRL_LEN - iv_len); 1843 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1844 1845 /* DPTR */ 1846 req->ist.ei1 = offset_dma; 1847 /* RPTR should just exclude offset control word */ 1848 req->ist.ei2 = dm_dma_addr - iv_len; 1849 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1850 + outputlen - iv_len); 1851 1852 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1853 1854 vq_cmd_w0.s.opcode = opcode.flags; 1855 1856 if (likely(iv_len)) { 1857 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1858 + OFF_CTRL_LEN); 1859 memcpy(iv_d, iv, 16); 1860 } 1861 1862 /* iv offset is 0 */ 1863 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1864 } else { 1865 uint32_t i, g_size_bytes, s_size_bytes; 1866 uint64_t dptr_dma, rptr_dma; 1867 sg_comp_t *gather_comp; 1868 sg_comp_t *scatter_comp; 1869 uint8_t *in_buffer; 1870 uint32_t *iv_d; 1871 1872 /* save space for offset and iv... */ 1873 offset_vaddr = m_vaddr; 1874 offset_dma = m_dma; 1875 1876 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1877 m_dma += OFF_CTRL_LEN + iv_len; 1878 1879 opcode.s.major |= CPT_DMA_MODE; 1880 1881 vq_cmd_w0.s.opcode = opcode.flags; 1882 1883 /* DPTR has SG list */ 1884 in_buffer = m_vaddr; 1885 dptr_dma = m_dma; 1886 1887 ((uint16_t *)in_buffer)[0] = 0; 1888 ((uint16_t *)in_buffer)[1] = 0; 1889 1890 /* TODO Add error check if space will be sufficient */ 1891 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1892 1893 /* 1894 * Input Gather List 1895 */ 1896 i = 0; 1897 1898 /* Offset control word */ 1899 1900 /* iv offset is 0 */ 1901 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1902 1903 i = fill_sg_comp(gather_comp, i, offset_dma, 1904 OFF_CTRL_LEN + iv_len); 1905 1906 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1907 memcpy(iv_d, iv, 16); 1908 1909 /* Add input data */ 1910 size = inputlen - iv_len; 1911 if (size) { 1912 i = fill_sg_comp_from_iov(gather_comp, i, 1913 params->src_iov, 1914 0, &size, NULL, 0); 1915 if (unlikely(size)) { 1916 CPT_LOG_DP_ERR("Insufficient buffer space," 1917 " size %d needed", size); 1918 return; 1919 } 1920 } 1921 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1922 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1923 1924 /* 1925 * Output Scatter List 1926 */ 1927 1928 i = 0; 1929 scatter_comp = 1930 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1931 1932 /* IV */ 1933 i = fill_sg_comp(scatter_comp, i, 1934 offset_dma + OFF_CTRL_LEN, 1935 iv_len); 1936 1937 /* Add output data */ 1938 size = outputlen - iv_len; 1939 if (size) { 1940 i = fill_sg_comp_from_iov(scatter_comp, i, 1941 params->dst_iov, 0, 1942 &size, NULL, 0); 1943 1944 if (unlikely(size)) { 1945 CPT_LOG_DP_ERR("Insufficient buffer space," 1946 " size %d needed", size); 1947 return; 1948 } 1949 } 1950 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1951 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1952 1953 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1954 1955 /* This is DPTR len incase of SG mode */ 1956 vq_cmd_w0.s.dlen = size; 1957 1958 m_vaddr = (uint8_t *)m_vaddr + size; 1959 m_dma += size; 1960 1961 /* cpt alternate completion address saved earlier */ 1962 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1963 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1964 rptr_dma = c_dma - 8; 1965 1966 req->ist.ei1 = dptr_dma; 1967 req->ist.ei2 = rptr_dma; 1968 } 1969 1970 /* vq command w3 */ 1971 vq_cmd_w3.u64 = 0; 1972 vq_cmd_w3.s.grp = 0; 1973 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1974 offsetof(struct cpt_ctx, zs_ctx); 1975 1976 /* 16 byte aligned cpt res address */ 1977 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1978 *req->completion_addr = COMPLETION_CODE_INIT; 1979 req->comp_baddr = c_dma; 1980 1981 /* Fill microcode part of instruction */ 1982 req->ist.ei0 = vq_cmd_w0.u64; 1983 req->ist.ei3 = vq_cmd_w3.u64; 1984 1985 req->op = op; 1986 1987 *prep_req = req; 1988 return; 1989 } 1990 1991 static __rte_always_inline void 1992 cpt_kasumi_enc_prep(uint32_t req_flags, 1993 uint64_t d_offs, 1994 uint64_t d_lens, 1995 fc_params_t *params, 1996 void *op, 1997 void **prep_req) 1998 { 1999 uint32_t size; 2000 int32_t inputlen = 0, outputlen = 0; 2001 struct cpt_ctx *cpt_ctx; 2002 uint32_t mac_len = 0; 2003 uint8_t i = 0; 2004 struct cpt_request_info *req; 2005 buf_ptr_t *buf_p; 2006 uint32_t encr_offset, auth_offset; 2007 uint32_t encr_data_len, auth_data_len; 2008 int flags; 2009 uint8_t *iv_s, *iv_d, iv_len = 8; 2010 uint8_t dir = 0; 2011 void *m_vaddr, *c_vaddr; 2012 uint64_t m_dma, c_dma; 2013 uint64_t *offset_vaddr, offset_dma; 2014 vq_cmd_word0_t vq_cmd_w0; 2015 vq_cmd_word3_t vq_cmd_w3; 2016 opcode_info_t opcode; 2017 uint8_t *in_buffer; 2018 uint32_t g_size_bytes, s_size_bytes; 2019 uint64_t dptr_dma, rptr_dma; 2020 sg_comp_t *gather_comp; 2021 sg_comp_t *scatter_comp; 2022 2023 buf_p = ¶ms->meta_buf; 2024 m_vaddr = buf_p->vaddr; 2025 m_dma = buf_p->dma_addr; 2026 2027 encr_offset = ENCR_OFFSET(d_offs) / 8; 2028 auth_offset = AUTH_OFFSET(d_offs) / 8; 2029 encr_data_len = ENCR_DLEN(d_lens); 2030 auth_data_len = AUTH_DLEN(d_lens); 2031 2032 cpt_ctx = params->ctx_buf.vaddr; 2033 flags = cpt_ctx->zsk_flags; 2034 mac_len = cpt_ctx->mac_len; 2035 2036 if (flags == 0x0) 2037 iv_s = params->iv_buf; 2038 else 2039 iv_s = params->auth_iv_buf; 2040 2041 dir = iv_s[8] & 0x1; 2042 2043 /* 2044 * Save initial space that followed app data for completion code & 2045 * alternate completion code to fall in same cache line as app data 2046 */ 2047 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2048 m_dma += COMPLETION_CODE_SIZE; 2049 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2050 (uint8_t *)m_vaddr; 2051 2052 c_vaddr = (uint8_t *)m_vaddr + size; 2053 c_dma = m_dma + size; 2054 size += sizeof(cpt_res_s_t); 2055 2056 m_vaddr = (uint8_t *)m_vaddr + size; 2057 m_dma += size; 2058 2059 /* Reserve memory for cpt request info */ 2060 req = m_vaddr; 2061 2062 size = sizeof(struct cpt_request_info); 2063 m_vaddr = (uint8_t *)m_vaddr + size; 2064 m_dma += size; 2065 2066 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2067 2068 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2069 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2070 (dir << 4) | (0 << 3) | (flags & 0x7)); 2071 2072 /* 2073 * GP op header, lengths are expected in bits. 2074 */ 2075 vq_cmd_w0.u64 = 0; 2076 vq_cmd_w0.s.param1 = encr_data_len; 2077 vq_cmd_w0.s.param2 = auth_data_len; 2078 vq_cmd_w0.s.opcode = opcode.flags; 2079 2080 /* consider iv len */ 2081 if (flags == 0x0) { 2082 encr_offset += iv_len; 2083 auth_offset += iv_len; 2084 } 2085 2086 /* save space for offset ctrl and iv */ 2087 offset_vaddr = m_vaddr; 2088 offset_dma = m_dma; 2089 2090 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2091 m_dma += OFF_CTRL_LEN + iv_len; 2092 2093 /* DPTR has SG list */ 2094 in_buffer = m_vaddr; 2095 dptr_dma = m_dma; 2096 2097 ((uint16_t *)in_buffer)[0] = 0; 2098 ((uint16_t *)in_buffer)[1] = 0; 2099 2100 /* TODO Add error check if space will be sufficient */ 2101 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2102 2103 /* 2104 * Input Gather List 2105 */ 2106 i = 0; 2107 2108 /* Offset control word followed by iv */ 2109 2110 if (flags == 0x0) { 2111 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 2112 outputlen = inputlen; 2113 /* iv offset is 0 */ 2114 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2115 } else { 2116 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 2117 outputlen = mac_len; 2118 /* iv offset is 0 */ 2119 *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset); 2120 } 2121 2122 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2123 2124 /* IV */ 2125 iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN; 2126 memcpy(iv_d, iv_s, iv_len); 2127 2128 /* input data */ 2129 size = inputlen - iv_len; 2130 if (size) { 2131 i = fill_sg_comp_from_iov(gather_comp, i, 2132 params->src_iov, 0, 2133 &size, NULL, 0); 2134 2135 if (unlikely(size)) { 2136 CPT_LOG_DP_ERR("Insufficient buffer space," 2137 " size %d needed", size); 2138 return; 2139 } 2140 } 2141 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2142 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2143 2144 /* 2145 * Output Scatter List 2146 */ 2147 2148 i = 0; 2149 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2150 2151 if (flags == 0x1) { 2152 /* IV in SLIST only for F8 */ 2153 iv_len = 0; 2154 } 2155 2156 /* IV */ 2157 if (iv_len) { 2158 i = fill_sg_comp(scatter_comp, i, 2159 offset_dma + OFF_CTRL_LEN, 2160 iv_len); 2161 } 2162 2163 /* Add output data */ 2164 if (req_flags & VALID_MAC_BUF) { 2165 size = outputlen - iv_len - mac_len; 2166 if (size) { 2167 i = fill_sg_comp_from_iov(scatter_comp, i, 2168 params->dst_iov, 0, 2169 &size, NULL, 0); 2170 2171 if (unlikely(size)) { 2172 CPT_LOG_DP_ERR("Insufficient buffer space," 2173 " size %d needed", size); 2174 return; 2175 } 2176 } 2177 2178 /* mac data */ 2179 if (mac_len) { 2180 i = fill_sg_comp_from_buf(scatter_comp, i, 2181 ¶ms->mac_buf); 2182 } 2183 } else { 2184 /* Output including mac */ 2185 size = outputlen - iv_len; 2186 if (size) { 2187 i = fill_sg_comp_from_iov(scatter_comp, i, 2188 params->dst_iov, 0, 2189 &size, NULL, 0); 2190 2191 if (unlikely(size)) { 2192 CPT_LOG_DP_ERR("Insufficient buffer space," 2193 " size %d needed", size); 2194 return; 2195 } 2196 } 2197 } 2198 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2199 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2200 2201 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2202 2203 /* This is DPTR len incase of SG mode */ 2204 vq_cmd_w0.s.dlen = size; 2205 2206 m_vaddr = (uint8_t *)m_vaddr + size; 2207 m_dma += size; 2208 2209 /* cpt alternate completion address saved earlier */ 2210 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2211 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2212 rptr_dma = c_dma - 8; 2213 2214 req->ist.ei1 = dptr_dma; 2215 req->ist.ei2 = rptr_dma; 2216 2217 /* vq command w3 */ 2218 vq_cmd_w3.u64 = 0; 2219 vq_cmd_w3.s.grp = 0; 2220 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2221 offsetof(struct cpt_ctx, k_ctx); 2222 2223 /* 16 byte aligned cpt res address */ 2224 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2225 *req->completion_addr = COMPLETION_CODE_INIT; 2226 req->comp_baddr = c_dma; 2227 2228 /* Fill microcode part of instruction */ 2229 req->ist.ei0 = vq_cmd_w0.u64; 2230 req->ist.ei3 = vq_cmd_w3.u64; 2231 2232 req->op = op; 2233 2234 *prep_req = req; 2235 return; 2236 } 2237 2238 static __rte_always_inline void 2239 cpt_kasumi_dec_prep(uint64_t d_offs, 2240 uint64_t d_lens, 2241 fc_params_t *params, 2242 void *op, 2243 void **prep_req) 2244 { 2245 uint32_t size; 2246 int32_t inputlen = 0, outputlen; 2247 struct cpt_ctx *cpt_ctx; 2248 uint8_t i = 0, iv_len = 8; 2249 struct cpt_request_info *req; 2250 buf_ptr_t *buf_p; 2251 uint32_t encr_offset; 2252 uint32_t encr_data_len; 2253 int flags; 2254 uint8_t dir = 0; 2255 void *m_vaddr, *c_vaddr; 2256 uint64_t m_dma, c_dma; 2257 uint64_t *offset_vaddr, offset_dma; 2258 vq_cmd_word0_t vq_cmd_w0; 2259 vq_cmd_word3_t vq_cmd_w3; 2260 opcode_info_t opcode; 2261 uint8_t *in_buffer; 2262 uint32_t g_size_bytes, s_size_bytes; 2263 uint64_t dptr_dma, rptr_dma; 2264 sg_comp_t *gather_comp; 2265 sg_comp_t *scatter_comp; 2266 2267 buf_p = ¶ms->meta_buf; 2268 m_vaddr = buf_p->vaddr; 2269 m_dma = buf_p->dma_addr; 2270 2271 encr_offset = ENCR_OFFSET(d_offs) / 8; 2272 encr_data_len = ENCR_DLEN(d_lens); 2273 2274 cpt_ctx = params->ctx_buf.vaddr; 2275 flags = cpt_ctx->zsk_flags; 2276 /* 2277 * Save initial space that followed app data for completion code & 2278 * alternate completion code to fall in same cache line as app data 2279 */ 2280 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2281 m_dma += COMPLETION_CODE_SIZE; 2282 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2283 (uint8_t *)m_vaddr; 2284 2285 c_vaddr = (uint8_t *)m_vaddr + size; 2286 c_dma = m_dma + size; 2287 size += sizeof(cpt_res_s_t); 2288 2289 m_vaddr = (uint8_t *)m_vaddr + size; 2290 m_dma += size; 2291 2292 /* Reserve memory for cpt request info */ 2293 req = m_vaddr; 2294 2295 size = sizeof(struct cpt_request_info); 2296 m_vaddr = (uint8_t *)m_vaddr + size; 2297 m_dma += size; 2298 2299 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2300 2301 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2302 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2303 (dir << 4) | (0 << 3) | (flags & 0x7)); 2304 2305 /* 2306 * GP op header, lengths are expected in bits. 2307 */ 2308 vq_cmd_w0.u64 = 0; 2309 vq_cmd_w0.s.param1 = encr_data_len; 2310 vq_cmd_w0.s.opcode = opcode.flags; 2311 2312 /* consider iv len */ 2313 encr_offset += iv_len; 2314 2315 inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8); 2316 outputlen = inputlen; 2317 2318 /* save space for offset ctrl & iv */ 2319 offset_vaddr = m_vaddr; 2320 offset_dma = m_dma; 2321 2322 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2323 m_dma += OFF_CTRL_LEN + iv_len; 2324 2325 /* DPTR has SG list */ 2326 in_buffer = m_vaddr; 2327 dptr_dma = m_dma; 2328 2329 ((uint16_t *)in_buffer)[0] = 0; 2330 ((uint16_t *)in_buffer)[1] = 0; 2331 2332 /* TODO Add error check if space will be sufficient */ 2333 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2334 2335 /* 2336 * Input Gather List 2337 */ 2338 i = 0; 2339 2340 /* Offset control word followed by iv */ 2341 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2342 2343 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2344 2345 /* IV */ 2346 memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN, 2347 params->iv_buf, iv_len); 2348 2349 /* Add input data */ 2350 size = inputlen - iv_len; 2351 if (size) { 2352 i = fill_sg_comp_from_iov(gather_comp, i, 2353 params->src_iov, 2354 0, &size, NULL, 0); 2355 if (unlikely(size)) { 2356 CPT_LOG_DP_ERR("Insufficient buffer space," 2357 " size %d needed", size); 2358 return; 2359 } 2360 } 2361 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2362 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2363 2364 /* 2365 * Output Scatter List 2366 */ 2367 2368 i = 0; 2369 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2370 2371 /* IV */ 2372 i = fill_sg_comp(scatter_comp, i, 2373 offset_dma + OFF_CTRL_LEN, 2374 iv_len); 2375 2376 /* Add output data */ 2377 size = outputlen - iv_len; 2378 if (size) { 2379 i = fill_sg_comp_from_iov(scatter_comp, i, 2380 params->dst_iov, 0, 2381 &size, NULL, 0); 2382 if (unlikely(size)) { 2383 CPT_LOG_DP_ERR("Insufficient buffer space," 2384 " size %d needed", size); 2385 return; 2386 } 2387 } 2388 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2389 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2390 2391 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2392 2393 /* This is DPTR len incase of SG mode */ 2394 vq_cmd_w0.s.dlen = size; 2395 2396 m_vaddr = (uint8_t *)m_vaddr + size; 2397 m_dma += size; 2398 2399 /* cpt alternate completion address saved earlier */ 2400 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2401 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2402 rptr_dma = c_dma - 8; 2403 2404 req->ist.ei1 = dptr_dma; 2405 req->ist.ei2 = rptr_dma; 2406 2407 /* vq command w3 */ 2408 vq_cmd_w3.u64 = 0; 2409 vq_cmd_w3.s.grp = 0; 2410 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2411 offsetof(struct cpt_ctx, k_ctx); 2412 2413 /* 16 byte aligned cpt res address */ 2414 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2415 *req->completion_addr = COMPLETION_CODE_INIT; 2416 req->comp_baddr = c_dma; 2417 2418 /* Fill microcode part of instruction */ 2419 req->ist.ei0 = vq_cmd_w0.u64; 2420 req->ist.ei3 = vq_cmd_w3.u64; 2421 2422 req->op = op; 2423 2424 *prep_req = req; 2425 return; 2426 } 2427 2428 static __rte_always_inline void * 2429 cpt_fc_dec_hmac_prep(uint32_t flags, 2430 uint64_t d_offs, 2431 uint64_t d_lens, 2432 fc_params_t *fc_params, 2433 void *op) 2434 { 2435 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2436 uint8_t fc_type; 2437 void *prep_req = NULL; 2438 2439 fc_type = ctx->fc_type; 2440 2441 if (likely(fc_type == FC_GEN)) { 2442 cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2443 &prep_req); 2444 } else if (fc_type == ZUC_SNOW3G) { 2445 cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, op, 2446 &prep_req); 2447 } else if (fc_type == KASUMI) { 2448 cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, &prep_req); 2449 } 2450 2451 /* 2452 * For AUTH_ONLY case, 2453 * MC only supports digest generation and verification 2454 * should be done in software by memcmp() 2455 */ 2456 2457 return prep_req; 2458 } 2459 2460 static __rte_always_inline void *__hot 2461 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, 2462 fc_params_t *fc_params, void *op) 2463 { 2464 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2465 uint8_t fc_type; 2466 void *prep_req = NULL; 2467 2468 fc_type = ctx->fc_type; 2469 2470 /* Common api for rest of the ops */ 2471 if (likely(fc_type == FC_GEN)) { 2472 cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2473 &prep_req); 2474 } else if (fc_type == ZUC_SNOW3G) { 2475 cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, op, 2476 &prep_req); 2477 } else if (fc_type == KASUMI) { 2478 cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, op, 2479 &prep_req); 2480 } else if (fc_type == HASH_HMAC) { 2481 cpt_digest_gen_prep(flags, d_lens, fc_params, op, &prep_req); 2482 } 2483 2484 return prep_req; 2485 } 2486 2487 static __rte_always_inline int 2488 cpt_fc_auth_set_key(void *ctx, auth_type_t type, const uint8_t *key, 2489 uint16_t key_len, uint16_t mac_len) 2490 { 2491 struct cpt_ctx *cpt_ctx = ctx; 2492 mc_fc_context_t *fctx = &cpt_ctx->fctx; 2493 2494 if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) { 2495 uint32_t keyx[4]; 2496 2497 if (key_len != 16) 2498 return -1; 2499 /* No support for AEAD yet */ 2500 if (cpt_ctx->enc_cipher) 2501 return -1; 2502 /* For ZUC/SNOW3G/Kasumi */ 2503 switch (type) { 2504 case SNOW3G_UIA2: 2505 cpt_ctx->snow3g = 1; 2506 gen_key_snow3g(key, keyx); 2507 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 2508 cpt_ctx->fc_type = ZUC_SNOW3G; 2509 cpt_ctx->zsk_flags = 0x1; 2510 break; 2511 case ZUC_EIA3: 2512 cpt_ctx->snow3g = 0; 2513 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 2514 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 2515 cpt_ctx->fc_type = ZUC_SNOW3G; 2516 cpt_ctx->zsk_flags = 0x1; 2517 break; 2518 case KASUMI_F9_ECB: 2519 /* Kasumi ECB mode */ 2520 cpt_ctx->k_ecb = 1; 2521 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2522 cpt_ctx->fc_type = KASUMI; 2523 cpt_ctx->zsk_flags = 0x1; 2524 break; 2525 case KASUMI_F9_CBC: 2526 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2527 cpt_ctx->fc_type = KASUMI; 2528 cpt_ctx->zsk_flags = 0x1; 2529 break; 2530 default: 2531 return -1; 2532 } 2533 cpt_ctx->mac_len = 4; 2534 cpt_ctx->hash_type = type; 2535 return 0; 2536 } 2537 2538 if (!(cpt_ctx->fc_type == FC_GEN && !type)) { 2539 if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher) 2540 cpt_ctx->fc_type = HASH_HMAC; 2541 } 2542 2543 if (cpt_ctx->fc_type == FC_GEN && key_len > 64) 2544 return -1; 2545 2546 /* For GMAC auth, cipher must be NULL */ 2547 if (type == GMAC_TYPE) 2548 fctx->enc.enc_cipher = 0; 2549 2550 fctx->enc.hash_type = cpt_ctx->hash_type = type; 2551 fctx->enc.mac_len = cpt_ctx->mac_len = mac_len; 2552 2553 if (key_len) { 2554 cpt_ctx->hmac = 1; 2555 memset(cpt_ctx->auth_key, 0, sizeof(cpt_ctx->auth_key)); 2556 memcpy(cpt_ctx->auth_key, key, key_len); 2557 cpt_ctx->auth_key_len = key_len; 2558 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 2559 memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad)); 2560 2561 if (key_len <= 64) 2562 memcpy(fctx->hmac.opad, key, key_len); 2563 fctx->enc.auth_input_type = 1; 2564 } 2565 return 0; 2566 } 2567 2568 static __rte_always_inline int 2569 fill_sess_aead(struct rte_crypto_sym_xform *xform, 2570 struct cpt_sess_misc *sess) 2571 { 2572 struct rte_crypto_aead_xform *aead_form; 2573 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2574 auth_type_t auth_type = 0; /* NULL Auth type */ 2575 uint32_t cipher_key_len = 0; 2576 uint8_t aes_gcm = 0; 2577 aead_form = &xform->aead; 2578 void *ctx = SESS_PRIV(sess); 2579 2580 if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT && 2581 aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) { 2582 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2583 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2584 } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT && 2585 aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) { 2586 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2587 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2588 } else { 2589 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2590 return -1; 2591 } 2592 switch (aead_form->algo) { 2593 case RTE_CRYPTO_AEAD_AES_GCM: 2594 enc_type = AES_GCM; 2595 cipher_key_len = 16; 2596 aes_gcm = 1; 2597 break; 2598 case RTE_CRYPTO_AEAD_AES_CCM: 2599 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2600 aead_form->algo); 2601 return -1; 2602 default: 2603 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2604 aead_form->algo); 2605 return -1; 2606 } 2607 if (aead_form->key.length < cipher_key_len) { 2608 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2609 (unsigned int long)aead_form->key.length); 2610 return -1; 2611 } 2612 sess->zsk_flag = 0; 2613 sess->aes_gcm = aes_gcm; 2614 sess->mac_len = aead_form->digest_length; 2615 sess->iv_offset = aead_form->iv.offset; 2616 sess->iv_length = aead_form->iv.length; 2617 sess->aad_length = aead_form->aad_length; 2618 2619 if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data, 2620 aead_form->key.length, NULL))) 2621 return -1; 2622 2623 if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, 2624 aead_form->digest_length))) 2625 return -1; 2626 2627 return 0; 2628 } 2629 2630 static __rte_always_inline int 2631 fill_sess_cipher(struct rte_crypto_sym_xform *xform, 2632 struct cpt_sess_misc *sess) 2633 { 2634 struct rte_crypto_cipher_xform *c_form; 2635 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2636 uint32_t cipher_key_len = 0; 2637 uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0; 2638 2639 c_form = &xform->cipher; 2640 2641 if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2642 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2643 else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT) 2644 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2645 else { 2646 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2647 return -1; 2648 } 2649 2650 switch (c_form->algo) { 2651 case RTE_CRYPTO_CIPHER_AES_CBC: 2652 enc_type = AES_CBC; 2653 cipher_key_len = 16; 2654 break; 2655 case RTE_CRYPTO_CIPHER_3DES_CBC: 2656 enc_type = DES3_CBC; 2657 cipher_key_len = 24; 2658 break; 2659 case RTE_CRYPTO_CIPHER_DES_CBC: 2660 /* DES is implemented using 3DES in hardware */ 2661 enc_type = DES3_CBC; 2662 cipher_key_len = 8; 2663 break; 2664 case RTE_CRYPTO_CIPHER_AES_CTR: 2665 enc_type = AES_CTR; 2666 cipher_key_len = 16; 2667 aes_ctr = 1; 2668 break; 2669 case RTE_CRYPTO_CIPHER_NULL: 2670 enc_type = 0; 2671 is_null = 1; 2672 break; 2673 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2674 enc_type = KASUMI_F8_ECB; 2675 cipher_key_len = 16; 2676 zsk_flag = K_F8; 2677 break; 2678 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2679 enc_type = SNOW3G_UEA2; 2680 cipher_key_len = 16; 2681 zsk_flag = ZS_EA; 2682 break; 2683 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 2684 enc_type = ZUC_EEA3; 2685 cipher_key_len = 16; 2686 zsk_flag = ZS_EA; 2687 break; 2688 case RTE_CRYPTO_CIPHER_AES_XTS: 2689 enc_type = AES_XTS; 2690 cipher_key_len = 16; 2691 break; 2692 case RTE_CRYPTO_CIPHER_3DES_ECB: 2693 enc_type = DES3_ECB; 2694 cipher_key_len = 24; 2695 break; 2696 case RTE_CRYPTO_CIPHER_AES_ECB: 2697 enc_type = AES_ECB; 2698 cipher_key_len = 16; 2699 break; 2700 case RTE_CRYPTO_CIPHER_3DES_CTR: 2701 case RTE_CRYPTO_CIPHER_AES_F8: 2702 case RTE_CRYPTO_CIPHER_ARC4: 2703 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2704 c_form->algo); 2705 return -1; 2706 default: 2707 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2708 c_form->algo); 2709 return -1; 2710 } 2711 2712 if (c_form->key.length < cipher_key_len) { 2713 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2714 (unsigned long) c_form->key.length); 2715 return -1; 2716 } 2717 2718 sess->zsk_flag = zsk_flag; 2719 sess->aes_gcm = 0; 2720 sess->aes_ctr = aes_ctr; 2721 sess->iv_offset = c_form->iv.offset; 2722 sess->iv_length = c_form->iv.length; 2723 sess->is_null = is_null; 2724 2725 if (unlikely(cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type, 2726 c_form->key.data, c_form->key.length, NULL))) 2727 return -1; 2728 2729 return 0; 2730 } 2731 2732 static __rte_always_inline int 2733 fill_sess_auth(struct rte_crypto_sym_xform *xform, 2734 struct cpt_sess_misc *sess) 2735 { 2736 struct rte_crypto_auth_xform *a_form; 2737 auth_type_t auth_type = 0; /* NULL Auth type */ 2738 uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0; 2739 2740 a_form = &xform->auth; 2741 2742 if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2743 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2744 else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2745 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2746 else { 2747 CPT_LOG_DP_ERR("Unknown auth operation"); 2748 return -1; 2749 } 2750 2751 switch (a_form->algo) { 2752 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2753 /* Fall through */ 2754 case RTE_CRYPTO_AUTH_SHA1: 2755 auth_type = SHA1_TYPE; 2756 break; 2757 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2758 case RTE_CRYPTO_AUTH_SHA256: 2759 auth_type = SHA2_SHA256; 2760 break; 2761 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2762 case RTE_CRYPTO_AUTH_SHA512: 2763 auth_type = SHA2_SHA512; 2764 break; 2765 case RTE_CRYPTO_AUTH_AES_GMAC: 2766 auth_type = GMAC_TYPE; 2767 aes_gcm = 1; 2768 break; 2769 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2770 case RTE_CRYPTO_AUTH_SHA224: 2771 auth_type = SHA2_SHA224; 2772 break; 2773 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2774 case RTE_CRYPTO_AUTH_SHA384: 2775 auth_type = SHA2_SHA384; 2776 break; 2777 case RTE_CRYPTO_AUTH_MD5_HMAC: 2778 case RTE_CRYPTO_AUTH_MD5: 2779 auth_type = MD5_TYPE; 2780 break; 2781 case RTE_CRYPTO_AUTH_KASUMI_F9: 2782 auth_type = KASUMI_F9_ECB; 2783 /* 2784 * Indicate that direction needs to be taken out 2785 * from end of src 2786 */ 2787 zsk_flag = K_F9; 2788 break; 2789 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2790 auth_type = SNOW3G_UIA2; 2791 zsk_flag = ZS_IA; 2792 break; 2793 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2794 auth_type = ZUC_EIA3; 2795 zsk_flag = ZS_IA; 2796 break; 2797 case RTE_CRYPTO_AUTH_NULL: 2798 auth_type = 0; 2799 is_null = 1; 2800 break; 2801 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2802 case RTE_CRYPTO_AUTH_AES_CMAC: 2803 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2804 CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u", 2805 a_form->algo); 2806 return -1; 2807 default: 2808 CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified", 2809 a_form->algo); 2810 return -1; 2811 } 2812 2813 sess->zsk_flag = zsk_flag; 2814 sess->aes_gcm = aes_gcm; 2815 sess->mac_len = a_form->digest_length; 2816 sess->is_null = is_null; 2817 if (zsk_flag) { 2818 sess->auth_iv_offset = a_form->iv.offset; 2819 sess->auth_iv_length = a_form->iv.length; 2820 } 2821 if (unlikely(cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type, 2822 a_form->key.data, a_form->key.length, 2823 a_form->digest_length))) 2824 return -1; 2825 2826 return 0; 2827 } 2828 2829 static __rte_always_inline int 2830 fill_sess_gmac(struct rte_crypto_sym_xform *xform, 2831 struct cpt_sess_misc *sess) 2832 { 2833 struct rte_crypto_auth_xform *a_form; 2834 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2835 auth_type_t auth_type = 0; /* NULL Auth type */ 2836 void *ctx = SESS_PRIV(sess); 2837 2838 a_form = &xform->auth; 2839 2840 if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2841 sess->cpt_op |= CPT_OP_ENCODE; 2842 else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2843 sess->cpt_op |= CPT_OP_DECODE; 2844 else { 2845 CPT_LOG_DP_ERR("Unknown auth operation"); 2846 return -1; 2847 } 2848 2849 switch (a_form->algo) { 2850 case RTE_CRYPTO_AUTH_AES_GMAC: 2851 enc_type = AES_GCM; 2852 auth_type = GMAC_TYPE; 2853 break; 2854 default: 2855 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2856 a_form->algo); 2857 return -1; 2858 } 2859 2860 sess->zsk_flag = 0; 2861 sess->aes_gcm = 0; 2862 sess->is_gmac = 1; 2863 sess->iv_offset = a_form->iv.offset; 2864 sess->iv_length = a_form->iv.length; 2865 sess->mac_len = a_form->digest_length; 2866 2867 if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data, 2868 a_form->key.length, NULL))) 2869 return -1; 2870 2871 if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, 2872 a_form->digest_length))) 2873 return -1; 2874 2875 return 0; 2876 } 2877 2878 static __rte_always_inline void * 2879 alloc_op_meta(struct rte_mbuf *m_src, 2880 buf_ptr_t *buf, 2881 int32_t len, 2882 struct rte_mempool *cpt_meta_pool) 2883 { 2884 uint8_t *mdata; 2885 2886 #ifndef CPT_ALWAYS_USE_SEPARATE_BUF 2887 if (likely(m_src && (m_src->nb_segs == 1))) { 2888 int32_t tailroom; 2889 phys_addr_t mphys; 2890 2891 /* Check if tailroom is sufficient to hold meta data */ 2892 tailroom = rte_pktmbuf_tailroom(m_src); 2893 if (likely(tailroom > len + 8)) { 2894 mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len; 2895 mphys = m_src->buf_physaddr + m_src->buf_len; 2896 mdata -= len; 2897 mphys -= len; 2898 buf->vaddr = mdata; 2899 buf->dma_addr = mphys; 2900 buf->size = len; 2901 /* Indicate that this is a mbuf allocated mdata */ 2902 mdata = (uint8_t *)((uint64_t)mdata | 1ull); 2903 return mdata; 2904 } 2905 } 2906 #else 2907 RTE_SET_USED(m_src); 2908 #endif 2909 2910 if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0)) 2911 return NULL; 2912 2913 buf->vaddr = mdata; 2914 buf->dma_addr = rte_mempool_virt2iova(mdata); 2915 buf->size = len; 2916 2917 return mdata; 2918 } 2919 2920 /** 2921 * cpt_free_metabuf - free metabuf to mempool. 2922 * @param instance: pointer to instance. 2923 * @param objp: pointer to the metabuf. 2924 */ 2925 static __rte_always_inline void 2926 free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool) 2927 { 2928 bool nofree = ((uintptr_t)mdata & 1ull); 2929 2930 if (likely(nofree)) 2931 return; 2932 rte_mempool_put(cpt_meta_pool, mdata); 2933 } 2934 2935 static __rte_always_inline uint32_t 2936 prepare_iov_from_pkt(struct rte_mbuf *pkt, 2937 iov_ptr_t *iovec, uint32_t start_offset) 2938 { 2939 uint16_t index = 0; 2940 void *seg_data = NULL; 2941 phys_addr_t seg_phys; 2942 int32_t seg_size = 0; 2943 2944 if (!pkt) { 2945 iovec->buf_cnt = 0; 2946 return 0; 2947 } 2948 2949 if (!start_offset) { 2950 seg_data = rte_pktmbuf_mtod(pkt, void *); 2951 seg_phys = rte_pktmbuf_mtophys(pkt); 2952 seg_size = pkt->data_len; 2953 } else { 2954 while (start_offset >= pkt->data_len) { 2955 start_offset -= pkt->data_len; 2956 pkt = pkt->next; 2957 } 2958 2959 seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset); 2960 seg_phys = rte_pktmbuf_mtophys_offset(pkt, start_offset); 2961 seg_size = pkt->data_len - start_offset; 2962 if (!seg_size) 2963 return 1; 2964 } 2965 2966 /* first seg */ 2967 iovec->bufs[index].vaddr = seg_data; 2968 iovec->bufs[index].dma_addr = seg_phys; 2969 iovec->bufs[index].size = seg_size; 2970 index++; 2971 pkt = pkt->next; 2972 2973 while (unlikely(pkt != NULL)) { 2974 seg_data = rte_pktmbuf_mtod(pkt, void *); 2975 seg_phys = rte_pktmbuf_mtophys(pkt); 2976 seg_size = pkt->data_len; 2977 if (!seg_size) 2978 break; 2979 2980 iovec->bufs[index].vaddr = seg_data; 2981 iovec->bufs[index].dma_addr = seg_phys; 2982 iovec->bufs[index].size = seg_size; 2983 2984 index++; 2985 2986 pkt = pkt->next; 2987 } 2988 2989 iovec->buf_cnt = index; 2990 return 0; 2991 } 2992 2993 static __rte_always_inline uint32_t 2994 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt, 2995 fc_params_t *param, 2996 uint32_t *flags) 2997 { 2998 uint16_t index = 0; 2999 void *seg_data = NULL; 3000 phys_addr_t seg_phys; 3001 uint32_t seg_size = 0; 3002 iov_ptr_t *iovec; 3003 3004 seg_data = rte_pktmbuf_mtod(pkt, void *); 3005 seg_phys = rte_pktmbuf_mtophys(pkt); 3006 seg_size = pkt->data_len; 3007 3008 /* first seg */ 3009 if (likely(!pkt->next)) { 3010 uint32_t headroom, tailroom; 3011 3012 *flags |= SINGLE_BUF_INPLACE; 3013 headroom = rte_pktmbuf_headroom(pkt); 3014 tailroom = rte_pktmbuf_tailroom(pkt); 3015 if (likely((headroom >= 24) && 3016 (tailroom >= 8))) { 3017 /* In 83XX this is prerequivisit for Direct mode */ 3018 *flags |= SINGLE_BUF_HEADTAILROOM; 3019 } 3020 param->bufs[0].vaddr = seg_data; 3021 param->bufs[0].dma_addr = seg_phys; 3022 param->bufs[0].size = seg_size; 3023 return 0; 3024 } 3025 iovec = param->src_iov; 3026 iovec->bufs[index].vaddr = seg_data; 3027 iovec->bufs[index].dma_addr = seg_phys; 3028 iovec->bufs[index].size = seg_size; 3029 index++; 3030 pkt = pkt->next; 3031 3032 while (unlikely(pkt != NULL)) { 3033 seg_data = rte_pktmbuf_mtod(pkt, void *); 3034 seg_phys = rte_pktmbuf_mtophys(pkt); 3035 seg_size = pkt->data_len; 3036 3037 if (!seg_size) 3038 break; 3039 3040 iovec->bufs[index].vaddr = seg_data; 3041 iovec->bufs[index].dma_addr = seg_phys; 3042 iovec->bufs[index].size = seg_size; 3043 3044 index++; 3045 3046 pkt = pkt->next; 3047 } 3048 3049 iovec->buf_cnt = index; 3050 return 0; 3051 } 3052 3053 static __rte_always_inline int 3054 fill_fc_params(struct rte_crypto_op *cop, 3055 struct cpt_sess_misc *sess_misc, 3056 struct cpt_qp_meta_info *m_info, 3057 void **mdata_ptr, 3058 void **prep_req) 3059 { 3060 uint32_t space = 0; 3061 struct rte_crypto_sym_op *sym_op = cop->sym; 3062 void *mdata = NULL; 3063 uintptr_t *op; 3064 uint32_t mc_hash_off; 3065 uint32_t flags = 0; 3066 uint64_t d_offs, d_lens; 3067 struct rte_mbuf *m_src, *m_dst; 3068 uint8_t cpt_op = sess_misc->cpt_op; 3069 #ifdef CPT_ALWAYS_USE_SG_MODE 3070 uint8_t inplace = 0; 3071 #else 3072 uint8_t inplace = 1; 3073 #endif 3074 fc_params_t fc_params; 3075 char src[SRC_IOV_SIZE]; 3076 char dst[SRC_IOV_SIZE]; 3077 uint32_t iv_buf[4]; 3078 int ret; 3079 3080 if (likely(sess_misc->iv_length)) { 3081 flags |= VALID_IV_BUF; 3082 fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, 3083 uint8_t *, sess_misc->iv_offset); 3084 if (sess_misc->aes_ctr && 3085 unlikely(sess_misc->iv_length != 16)) { 3086 memcpy((uint8_t *)iv_buf, 3087 rte_crypto_op_ctod_offset(cop, 3088 uint8_t *, sess_misc->iv_offset), 12); 3089 iv_buf[3] = rte_cpu_to_be_32(0x1); 3090 fc_params.iv_buf = iv_buf; 3091 } 3092 } 3093 3094 if (sess_misc->zsk_flag) { 3095 fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3096 uint8_t *, 3097 sess_misc->auth_iv_offset); 3098 if (sess_misc->zsk_flag != ZS_EA) 3099 inplace = 0; 3100 } 3101 m_src = sym_op->m_src; 3102 m_dst = sym_op->m_dst; 3103 3104 if (sess_misc->aes_gcm) { 3105 uint8_t *salt; 3106 uint8_t *aad_data; 3107 uint16_t aad_len; 3108 3109 d_offs = sym_op->aead.data.offset; 3110 d_lens = sym_op->aead.data.length; 3111 mc_hash_off = sym_op->aead.data.offset + 3112 sym_op->aead.data.length; 3113 3114 aad_data = sym_op->aead.aad.data; 3115 aad_len = sess_misc->aad_length; 3116 if (likely((aad_data + aad_len) == 3117 rte_pktmbuf_mtod_offset(m_src, 3118 uint8_t *, 3119 sym_op->aead.data.offset))) { 3120 d_offs = (d_offs - aad_len) | (d_offs << 16); 3121 d_lens = (d_lens + aad_len) | (d_lens << 32); 3122 } else { 3123 fc_params.aad_buf.vaddr = sym_op->aead.aad.data; 3124 fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr; 3125 fc_params.aad_buf.size = aad_len; 3126 flags |= VALID_AAD_BUF; 3127 inplace = 0; 3128 d_offs = d_offs << 16; 3129 d_lens = d_lens << 32; 3130 } 3131 3132 salt = fc_params.iv_buf; 3133 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3134 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3135 sess_misc->salt = *(uint32_t *)salt; 3136 } 3137 fc_params.iv_buf = salt + 4; 3138 if (likely(sess_misc->mac_len)) { 3139 struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst : 3140 m_src; 3141 3142 if (!m) 3143 m = m_src; 3144 3145 /* hmac immediately following data is best case */ 3146 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3147 mc_hash_off != 3148 (uint8_t *)sym_op->aead.digest.data)) { 3149 flags |= VALID_MAC_BUF; 3150 fc_params.mac_buf.size = sess_misc->mac_len; 3151 fc_params.mac_buf.vaddr = 3152 sym_op->aead.digest.data; 3153 fc_params.mac_buf.dma_addr = 3154 sym_op->aead.digest.phys_addr; 3155 inplace = 0; 3156 } 3157 } 3158 } else { 3159 d_offs = sym_op->cipher.data.offset; 3160 d_lens = sym_op->cipher.data.length; 3161 mc_hash_off = sym_op->cipher.data.offset + 3162 sym_op->cipher.data.length; 3163 d_offs = (d_offs << 16) | sym_op->auth.data.offset; 3164 d_lens = (d_lens << 32) | sym_op->auth.data.length; 3165 3166 if (mc_hash_off < (sym_op->auth.data.offset + 3167 sym_op->auth.data.length)){ 3168 mc_hash_off = (sym_op->auth.data.offset + 3169 sym_op->auth.data.length); 3170 } 3171 /* for gmac, salt should be updated like in gcm */ 3172 if (unlikely(sess_misc->is_gmac)) { 3173 uint8_t *salt; 3174 salt = fc_params.iv_buf; 3175 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3176 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3177 sess_misc->salt = *(uint32_t *)salt; 3178 } 3179 fc_params.iv_buf = salt + 4; 3180 } 3181 if (likely(sess_misc->mac_len)) { 3182 struct rte_mbuf *m; 3183 3184 m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src; 3185 if (!m) 3186 m = m_src; 3187 3188 /* hmac immediately following data is best case */ 3189 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3190 mc_hash_off != 3191 (uint8_t *)sym_op->auth.digest.data)) { 3192 flags |= VALID_MAC_BUF; 3193 fc_params.mac_buf.size = 3194 sess_misc->mac_len; 3195 fc_params.mac_buf.vaddr = 3196 sym_op->auth.digest.data; 3197 fc_params.mac_buf.dma_addr = 3198 sym_op->auth.digest.phys_addr; 3199 inplace = 0; 3200 } 3201 } 3202 } 3203 fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc); 3204 fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr; 3205 3206 if (unlikely(sess_misc->is_null || sess_misc->cpt_op == CPT_OP_DECODE)) 3207 inplace = 0; 3208 3209 if (likely(!m_dst && inplace)) { 3210 /* Case of single buffer without AAD buf or 3211 * separate mac buf in place and 3212 * not air crypto 3213 */ 3214 fc_params.dst_iov = fc_params.src_iov = (void *)src; 3215 3216 if (unlikely(prepare_iov_from_pkt_inplace(m_src, 3217 &fc_params, 3218 &flags))) { 3219 CPT_LOG_DP_ERR("Prepare inplace src iov failed"); 3220 ret = -EINVAL; 3221 goto err_exit; 3222 } 3223 3224 } else { 3225 /* Out of place processing */ 3226 fc_params.src_iov = (void *)src; 3227 fc_params.dst_iov = (void *)dst; 3228 3229 /* Store SG I/O in the api for reuse */ 3230 if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) { 3231 CPT_LOG_DP_ERR("Prepare src iov failed"); 3232 ret = -EINVAL; 3233 goto err_exit; 3234 } 3235 3236 if (unlikely(m_dst != NULL)) { 3237 uint32_t pkt_len; 3238 3239 /* Try to make room as much as src has */ 3240 pkt_len = rte_pktmbuf_pkt_len(m_dst); 3241 3242 if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) { 3243 pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len; 3244 if (!rte_pktmbuf_append(m_dst, pkt_len)) { 3245 CPT_LOG_DP_ERR("Not enough space in " 3246 "m_dst %p, need %u" 3247 " more", 3248 m_dst, pkt_len); 3249 ret = -EINVAL; 3250 goto err_exit; 3251 } 3252 } 3253 3254 if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) { 3255 CPT_LOG_DP_ERR("Prepare dst iov failed for " 3256 "m_dst %p", m_dst); 3257 ret = -EINVAL; 3258 goto err_exit; 3259 } 3260 } else { 3261 fc_params.dst_iov = (void *)src; 3262 } 3263 } 3264 3265 if (likely(flags & SINGLE_BUF_HEADTAILROOM)) 3266 mdata = alloc_op_meta(m_src, &fc_params.meta_buf, 3267 m_info->lb_mlen, m_info->pool); 3268 else 3269 mdata = alloc_op_meta(NULL, &fc_params.meta_buf, 3270 m_info->sg_mlen, m_info->pool); 3271 3272 if (unlikely(mdata == NULL)) { 3273 CPT_LOG_DP_ERR("Error allocating meta buffer for request"); 3274 ret = -ENOMEM; 3275 goto err_exit; 3276 } 3277 3278 op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull); 3279 op[0] = (uintptr_t)mdata; 3280 op[1] = (uintptr_t)cop; 3281 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3282 space += 4 * sizeof(uint64_t); 3283 3284 fc_params.meta_buf.vaddr = (uint8_t *)op + space; 3285 fc_params.meta_buf.dma_addr += space; 3286 fc_params.meta_buf.size -= space; 3287 3288 /* Finally prepare the instruction */ 3289 if (cpt_op & CPT_OP_ENCODE) 3290 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, 3291 &fc_params, op); 3292 else 3293 *prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens, 3294 &fc_params, op); 3295 3296 if (unlikely(*prep_req == NULL)) { 3297 CPT_LOG_DP_ERR("Preparing request failed due to bad input arg"); 3298 ret = -EINVAL; 3299 goto free_mdata_and_exit; 3300 } 3301 3302 *mdata_ptr = mdata; 3303 3304 return 0; 3305 3306 free_mdata_and_exit: 3307 free_op_meta(mdata, m_info->pool); 3308 err_exit: 3309 return ret; 3310 } 3311 3312 static __rte_always_inline void 3313 compl_auth_verify(struct rte_crypto_op *op, 3314 uint8_t *gen_mac, 3315 uint64_t mac_len) 3316 { 3317 uint8_t *mac; 3318 struct rte_crypto_sym_op *sym_op = op->sym; 3319 3320 if (sym_op->auth.digest.data) 3321 mac = sym_op->auth.digest.data; 3322 else 3323 mac = rte_pktmbuf_mtod_offset(sym_op->m_src, 3324 uint8_t *, 3325 sym_op->auth.data.length + 3326 sym_op->auth.data.offset); 3327 if (!mac) { 3328 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3329 return; 3330 } 3331 3332 if (memcmp(mac, gen_mac, mac_len)) 3333 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 3334 else 3335 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3336 } 3337 3338 static __rte_always_inline int 3339 instance_session_cfg(struct rte_crypto_sym_xform *xform, void *sess) 3340 { 3341 struct rte_crypto_sym_xform *chain; 3342 3343 CPT_PMD_INIT_FUNC_TRACE(); 3344 3345 if (cpt_is_algo_supported(xform)) 3346 goto err; 3347 3348 chain = xform; 3349 while (chain) { 3350 switch (chain->type) { 3351 case RTE_CRYPTO_SYM_XFORM_AEAD: 3352 if (fill_sess_aead(chain, sess)) 3353 goto err; 3354 break; 3355 case RTE_CRYPTO_SYM_XFORM_CIPHER: 3356 if (fill_sess_cipher(chain, sess)) 3357 goto err; 3358 break; 3359 case RTE_CRYPTO_SYM_XFORM_AUTH: 3360 if (chain->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 3361 if (fill_sess_gmac(chain, sess)) 3362 goto err; 3363 } else { 3364 if (fill_sess_auth(chain, sess)) 3365 goto err; 3366 } 3367 break; 3368 default: 3369 CPT_LOG_DP_ERR("Invalid crypto xform type"); 3370 break; 3371 } 3372 chain = chain->next; 3373 } 3374 3375 return 0; 3376 3377 err: 3378 return -1; 3379 } 3380 3381 static __rte_always_inline void 3382 find_kasumif9_direction_and_length(uint8_t *src, 3383 uint32_t counter_num_bytes, 3384 uint32_t *addr_length_in_bits, 3385 uint8_t *addr_direction) 3386 { 3387 uint8_t found = 0; 3388 uint32_t pos; 3389 uint8_t last_byte; 3390 while (!found && counter_num_bytes > 0) { 3391 counter_num_bytes--; 3392 if (src[counter_num_bytes] == 0x00) 3393 continue; 3394 pos = rte_bsf32(src[counter_num_bytes]); 3395 if (pos == 7) { 3396 if (likely(counter_num_bytes > 0)) { 3397 last_byte = src[counter_num_bytes - 1]; 3398 *addr_direction = last_byte & 0x1; 3399 *addr_length_in_bits = counter_num_bytes * 8 3400 - 1; 3401 } 3402 } else { 3403 last_byte = src[counter_num_bytes]; 3404 *addr_direction = (last_byte >> (pos + 1)) & 0x1; 3405 *addr_length_in_bits = counter_num_bytes * 8 3406 + (8 - (pos + 2)); 3407 } 3408 found = 1; 3409 } 3410 } 3411 3412 /* 3413 * This handles all auth only except AES_GMAC 3414 */ 3415 static __rte_always_inline int 3416 fill_digest_params(struct rte_crypto_op *cop, 3417 struct cpt_sess_misc *sess, 3418 struct cpt_qp_meta_info *m_info, 3419 void **mdata_ptr, 3420 void **prep_req) 3421 { 3422 uint32_t space = 0; 3423 struct rte_crypto_sym_op *sym_op = cop->sym; 3424 void *mdata; 3425 phys_addr_t mphys; 3426 uint64_t *op; 3427 uint32_t auth_range_off; 3428 uint32_t flags = 0; 3429 uint64_t d_offs = 0, d_lens; 3430 struct rte_mbuf *m_src, *m_dst; 3431 uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK; 3432 uint16_t mac_len = sess->mac_len; 3433 fc_params_t params; 3434 char src[SRC_IOV_SIZE]; 3435 uint8_t iv_buf[16]; 3436 int ret; 3437 3438 memset(¶ms, 0, sizeof(fc_params_t)); 3439 3440 m_src = sym_op->m_src; 3441 3442 /* For just digest lets force mempool alloc */ 3443 mdata = alloc_op_meta(NULL, ¶ms.meta_buf, m_info->sg_mlen, 3444 m_info->pool); 3445 if (mdata == NULL) { 3446 ret = -ENOMEM; 3447 goto err_exit; 3448 } 3449 3450 mphys = params.meta_buf.dma_addr; 3451 3452 op = mdata; 3453 op[0] = (uintptr_t)mdata; 3454 op[1] = (uintptr_t)cop; 3455 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3456 space += 4 * sizeof(uint64_t); 3457 3458 auth_range_off = sym_op->auth.data.offset; 3459 3460 flags = VALID_MAC_BUF; 3461 params.src_iov = (void *)src; 3462 if (unlikely(sess->zsk_flag)) { 3463 /* 3464 * Since for Zuc, Kasumi, Snow3g offsets are in bits 3465 * we will send pass through even for auth only case, 3466 * let MC handle it 3467 */ 3468 d_offs = auth_range_off; 3469 auth_range_off = 0; 3470 params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3471 uint8_t *, sess->auth_iv_offset); 3472 if (sess->zsk_flag == K_F9) { 3473 uint32_t length_in_bits, num_bytes; 3474 uint8_t *src, direction = 0; 3475 3476 memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src, 3477 uint8_t *), 8); 3478 /* 3479 * This is kasumi f9, take direction from 3480 * source buffer 3481 */ 3482 length_in_bits = cop->sym->auth.data.length; 3483 num_bytes = (length_in_bits >> 3); 3484 src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *); 3485 find_kasumif9_direction_and_length(src, 3486 num_bytes, 3487 &length_in_bits, 3488 &direction); 3489 length_in_bits -= 64; 3490 cop->sym->auth.data.offset += 64; 3491 d_offs = cop->sym->auth.data.offset; 3492 auth_range_off = d_offs / 8; 3493 cop->sym->auth.data.length = length_in_bits; 3494 3495 /* Store it at end of auth iv */ 3496 iv_buf[8] = direction; 3497 params.auth_iv_buf = iv_buf; 3498 } 3499 } 3500 3501 d_lens = sym_op->auth.data.length; 3502 3503 params.ctx_buf.vaddr = SESS_PRIV(sess); 3504 params.ctx_buf.dma_addr = sess->ctx_dma_addr; 3505 3506 if (auth_op == CPT_OP_AUTH_GENERATE) { 3507 if (sym_op->auth.digest.data) { 3508 /* 3509 * Digest to be generated 3510 * in separate buffer 3511 */ 3512 params.mac_buf.size = 3513 sess->mac_len; 3514 params.mac_buf.vaddr = 3515 sym_op->auth.digest.data; 3516 params.mac_buf.dma_addr = 3517 sym_op->auth.digest.phys_addr; 3518 } else { 3519 uint32_t off = sym_op->auth.data.offset + 3520 sym_op->auth.data.length; 3521 int32_t dlen, space; 3522 3523 m_dst = sym_op->m_dst ? 3524 sym_op->m_dst : sym_op->m_src; 3525 dlen = rte_pktmbuf_pkt_len(m_dst); 3526 3527 space = off + mac_len - dlen; 3528 if (space > 0) 3529 if (!rte_pktmbuf_append(m_dst, space)) { 3530 CPT_LOG_DP_ERR("Failed to extend " 3531 "mbuf by %uB", space); 3532 ret = -EINVAL; 3533 goto free_mdata_and_exit; 3534 } 3535 3536 params.mac_buf.vaddr = 3537 rte_pktmbuf_mtod_offset(m_dst, void *, off); 3538 params.mac_buf.dma_addr = 3539 rte_pktmbuf_mtophys_offset(m_dst, off); 3540 params.mac_buf.size = mac_len; 3541 } 3542 } else { 3543 /* Need space for storing generated mac */ 3544 params.mac_buf.vaddr = (uint8_t *)mdata + space; 3545 params.mac_buf.dma_addr = mphys + space; 3546 params.mac_buf.size = mac_len; 3547 space += RTE_ALIGN_CEIL(mac_len, 8); 3548 op[2] = (uintptr_t)params.mac_buf.vaddr; 3549 op[3] = mac_len; 3550 } 3551 3552 params.meta_buf.vaddr = (uint8_t *)mdata + space; 3553 params.meta_buf.dma_addr = mphys + space; 3554 params.meta_buf.size -= space; 3555 3556 /* Out of place processing */ 3557 params.src_iov = (void *)src; 3558 3559 /*Store SG I/O in the api for reuse */ 3560 if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) { 3561 CPT_LOG_DP_ERR("Prepare src iov failed"); 3562 ret = -EINVAL; 3563 goto free_mdata_and_exit; 3564 } 3565 3566 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, ¶ms, op); 3567 if (unlikely(*prep_req == NULL)) { 3568 ret = -EINVAL; 3569 goto free_mdata_and_exit; 3570 } 3571 3572 *mdata_ptr = mdata; 3573 3574 return 0; 3575 3576 free_mdata_and_exit: 3577 free_op_meta(mdata, m_info->pool); 3578 err_exit: 3579 return ret; 3580 } 3581 3582 #endif /*_CPT_UCODE_H_ */ 3583