1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2018 Cavium, Inc 3 */ 4 5 #ifndef _CPT_UCODE_H_ 6 #define _CPT_UCODE_H_ 7 #include <stdbool.h> 8 9 #include "cpt_common.h" 10 #include "cpt_hw_types.h" 11 #include "cpt_mcode_defines.h" 12 13 /* 14 * This file defines functions that are interfaces to microcode spec. 15 * 16 */ 17 18 static uint8_t zuc_d[32] = { 19 0x44, 0xD7, 0x26, 0xBC, 0x62, 0x6B, 0x13, 0x5E, 20 0x57, 0x89, 0x35, 0xE2, 0x71, 0x35, 0x09, 0xAF, 21 0x4D, 0x78, 0x2F, 0x13, 0x6B, 0xC4, 0x1A, 0xF1, 22 0x5E, 0x26, 0x3C, 0x4D, 0x78, 0x9A, 0x47, 0xAC 23 }; 24 25 static __rte_always_inline int 26 cpt_is_algo_supported(struct rte_crypto_sym_xform *xform) 27 { 28 /* 29 * Microcode only supports the following combination. 30 * Encryption followed by authentication 31 * Authentication followed by decryption 32 */ 33 if (xform->next) { 34 if ((xform->type == RTE_CRYPTO_SYM_XFORM_AUTH) && 35 (xform->next->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 36 (xform->next->cipher.op == RTE_CRYPTO_CIPHER_OP_ENCRYPT)) { 37 /* Unsupported as of now by microcode */ 38 CPT_LOG_DP_ERR("Unsupported combination"); 39 return -1; 40 } 41 if ((xform->type == RTE_CRYPTO_SYM_XFORM_CIPHER) && 42 (xform->next->type == RTE_CRYPTO_SYM_XFORM_AUTH) && 43 (xform->cipher.op == RTE_CRYPTO_CIPHER_OP_DECRYPT)) { 44 /* For GMAC auth there is no cipher operation */ 45 if (xform->aead.algo != RTE_CRYPTO_AEAD_AES_GCM || 46 xform->next->auth.algo != 47 RTE_CRYPTO_AUTH_AES_GMAC) { 48 /* Unsupported as of now by microcode */ 49 CPT_LOG_DP_ERR("Unsupported combination"); 50 return -1; 51 } 52 } 53 } 54 return 0; 55 } 56 57 static __rte_always_inline void 58 gen_key_snow3g(const uint8_t *ck, uint32_t *keyx) 59 { 60 int i, base; 61 62 for (i = 0; i < 4; i++) { 63 base = 4 * i; 64 keyx[3 - i] = (ck[base] << 24) | (ck[base + 1] << 16) | 65 (ck[base + 2] << 8) | (ck[base + 3]); 66 keyx[3 - i] = rte_cpu_to_be_32(keyx[3 - i]); 67 } 68 } 69 70 static __rte_always_inline void 71 cpt_fc_salt_update(void *ctx, 72 uint8_t *salt) 73 { 74 struct cpt_ctx *cpt_ctx = ctx; 75 memcpy(&cpt_ctx->fctx.enc.encr_iv, salt, 4); 76 } 77 78 static __rte_always_inline int 79 cpt_fc_ciph_validate_key_aes(uint16_t key_len) 80 { 81 switch (key_len) { 82 case CPT_BYTE_16: 83 case CPT_BYTE_24: 84 case CPT_BYTE_32: 85 return 0; 86 default: 87 return -1; 88 } 89 } 90 91 static __rte_always_inline int 92 cpt_fc_ciph_set_type(cipher_type_t type, struct cpt_ctx *ctx, uint16_t key_len) 93 { 94 int fc_type = 0; 95 switch (type) { 96 case PASSTHROUGH: 97 fc_type = FC_GEN; 98 break; 99 case DES3_CBC: 100 case DES3_ECB: 101 fc_type = FC_GEN; 102 break; 103 case AES_CBC: 104 case AES_ECB: 105 case AES_CFB: 106 case AES_CTR: 107 case AES_GCM: 108 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 109 return -1; 110 fc_type = FC_GEN; 111 break; 112 case AES_XTS: 113 key_len = key_len / 2; 114 if (unlikely(key_len == CPT_BYTE_24)) { 115 CPT_LOG_DP_ERR("Invalid AES key len for XTS"); 116 return -1; 117 } 118 if (unlikely(cpt_fc_ciph_validate_key_aes(key_len) != 0)) 119 return -1; 120 fc_type = FC_GEN; 121 break; 122 case ZUC_EEA3: 123 case SNOW3G_UEA2: 124 if (unlikely(key_len != 16)) 125 return -1; 126 /* No support for AEAD yet */ 127 if (unlikely(ctx->hash_type)) 128 return -1; 129 fc_type = ZUC_SNOW3G; 130 break; 131 case KASUMI_F8_CBC: 132 case KASUMI_F8_ECB: 133 if (unlikely(key_len != 16)) 134 return -1; 135 /* No support for AEAD yet */ 136 if (unlikely(ctx->hash_type)) 137 return -1; 138 fc_type = KASUMI; 139 break; 140 default: 141 return -1; 142 } 143 144 ctx->fc_type = fc_type; 145 return 0; 146 } 147 148 static __rte_always_inline void 149 cpt_fc_ciph_set_key_passthrough(struct cpt_ctx *cpt_ctx, mc_fc_context_t *fctx) 150 { 151 cpt_ctx->enc_cipher = 0; 152 fctx->enc.enc_cipher = 0; 153 } 154 155 static __rte_always_inline void 156 cpt_fc_ciph_set_key_set_aes_key_type(mc_fc_context_t *fctx, uint16_t key_len) 157 { 158 mc_aes_type_t aes_key_type = 0; 159 switch (key_len) { 160 case CPT_BYTE_16: 161 aes_key_type = AES_128_BIT; 162 break; 163 case CPT_BYTE_24: 164 aes_key_type = AES_192_BIT; 165 break; 166 case CPT_BYTE_32: 167 aes_key_type = AES_256_BIT; 168 break; 169 default: 170 /* This should not happen */ 171 CPT_LOG_DP_ERR("Invalid AES key len"); 172 return; 173 } 174 fctx->enc.aes_key = aes_key_type; 175 } 176 177 static __rte_always_inline void 178 cpt_fc_ciph_set_key_snow3g_uea2(struct cpt_ctx *cpt_ctx, const uint8_t *key, 179 uint16_t key_len) 180 { 181 uint32_t keyx[4]; 182 cpt_ctx->snow3g = 1; 183 gen_key_snow3g(key, keyx); 184 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 185 cpt_ctx->zsk_flags = 0; 186 } 187 188 static __rte_always_inline void 189 cpt_fc_ciph_set_key_zuc_eea3(struct cpt_ctx *cpt_ctx, const uint8_t *key, 190 uint16_t key_len) 191 { 192 cpt_ctx->snow3g = 0; 193 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 194 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 195 cpt_ctx->zsk_flags = 0; 196 } 197 198 static __rte_always_inline void 199 cpt_fc_ciph_set_key_kasumi_f8_ecb(struct cpt_ctx *cpt_ctx, const uint8_t *key, 200 uint16_t key_len) 201 { 202 cpt_ctx->k_ecb = 1; 203 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 204 cpt_ctx->zsk_flags = 0; 205 } 206 207 static __rte_always_inline void 208 cpt_fc_ciph_set_key_kasumi_f8_cbc(struct cpt_ctx *cpt_ctx, const uint8_t *key, 209 uint16_t key_len) 210 { 211 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 212 cpt_ctx->zsk_flags = 0; 213 } 214 215 static __rte_always_inline int 216 cpt_fc_ciph_set_key(void *ctx, cipher_type_t type, const uint8_t *key, 217 uint16_t key_len, uint8_t *salt) 218 { 219 struct cpt_ctx *cpt_ctx = ctx; 220 mc_fc_context_t *fctx = &cpt_ctx->fctx; 221 int ret; 222 223 ret = cpt_fc_ciph_set_type(type, cpt_ctx, key_len); 224 if (unlikely(ret)) 225 return -1; 226 227 if (cpt_ctx->fc_type == FC_GEN) { 228 /* 229 * We need to always say IV is from DPTR as user can 230 * sometimes iverride IV per operation. 231 */ 232 fctx->enc.iv_source = CPT_FROM_DPTR; 233 234 if (cpt_ctx->auth_key_len > 64) 235 return -1; 236 } 237 238 switch (type) { 239 case PASSTHROUGH: 240 cpt_fc_ciph_set_key_passthrough(cpt_ctx, fctx); 241 goto success; 242 case DES3_CBC: 243 /* CPT performs DES using 3DES with the 8B DES-key 244 * replicated 2 more times to match the 24B 3DES-key. 245 * Eg. If org. key is "0x0a 0x0b", then new key is 246 * "0x0a 0x0b 0x0a 0x0b 0x0a 0x0b" 247 */ 248 if (key_len == 8) { 249 /* Skipping the first 8B as it will be copied 250 * in the regular code flow 251 */ 252 memcpy(fctx->enc.encr_key+key_len, key, key_len); 253 memcpy(fctx->enc.encr_key+2*key_len, key, key_len); 254 } 255 break; 256 case DES3_ECB: 257 /* For DES3_ECB IV need to be from CTX. */ 258 fctx->enc.iv_source = CPT_FROM_CTX; 259 break; 260 case AES_CBC: 261 case AES_ECB: 262 case AES_CFB: 263 case AES_CTR: 264 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 265 break; 266 case AES_GCM: 267 /* Even though iv source is from dptr, 268 * aes_gcm salt is taken from ctx 269 */ 270 if (salt) { 271 memcpy(fctx->enc.encr_iv, salt, 4); 272 /* Assuming it was just salt update 273 * and nothing else 274 */ 275 if (!key) 276 goto success; 277 } 278 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 279 break; 280 case AES_XTS: 281 key_len = key_len / 2; 282 cpt_fc_ciph_set_key_set_aes_key_type(fctx, key_len); 283 284 /* Copy key2 for XTS into ipad */ 285 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 286 memcpy(fctx->hmac.ipad, &key[key_len], key_len); 287 break; 288 case SNOW3G_UEA2: 289 cpt_fc_ciph_set_key_snow3g_uea2(cpt_ctx, key, key_len); 290 goto success; 291 case ZUC_EEA3: 292 cpt_fc_ciph_set_key_zuc_eea3(cpt_ctx, key, key_len); 293 goto success; 294 case KASUMI_F8_ECB: 295 cpt_fc_ciph_set_key_kasumi_f8_ecb(cpt_ctx, key, key_len); 296 goto success; 297 case KASUMI_F8_CBC: 298 cpt_fc_ciph_set_key_kasumi_f8_cbc(cpt_ctx, key, key_len); 299 goto success; 300 default: 301 return -1; 302 } 303 304 /* Only for FC_GEN case */ 305 306 /* For GMAC auth, cipher must be NULL */ 307 if (cpt_ctx->hash_type != GMAC_TYPE) 308 fctx->enc.enc_cipher = type; 309 310 memcpy(fctx->enc.encr_key, key, key_len); 311 312 success: 313 cpt_ctx->enc_cipher = type; 314 315 return 0; 316 } 317 318 static __rte_always_inline uint32_t 319 fill_sg_comp(sg_comp_t *list, 320 uint32_t i, 321 phys_addr_t dma_addr, 322 uint32_t size) 323 { 324 sg_comp_t *to = &list[i>>2]; 325 326 to->u.s.len[i%4] = rte_cpu_to_be_16(size); 327 to->ptr[i%4] = rte_cpu_to_be_64(dma_addr); 328 i++; 329 return i; 330 } 331 332 static __rte_always_inline uint32_t 333 fill_sg_comp_from_buf(sg_comp_t *list, 334 uint32_t i, 335 buf_ptr_t *from) 336 { 337 sg_comp_t *to = &list[i>>2]; 338 339 to->u.s.len[i%4] = rte_cpu_to_be_16(from->size); 340 to->ptr[i%4] = rte_cpu_to_be_64(from->dma_addr); 341 i++; 342 return i; 343 } 344 345 static __rte_always_inline uint32_t 346 fill_sg_comp_from_buf_min(sg_comp_t *list, 347 uint32_t i, 348 buf_ptr_t *from, 349 uint32_t *psize) 350 { 351 sg_comp_t *to = &list[i >> 2]; 352 uint32_t size = *psize; 353 uint32_t e_len; 354 355 e_len = (size > from->size) ? from->size : size; 356 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 357 to->ptr[i % 4] = rte_cpu_to_be_64(from->dma_addr); 358 *psize -= e_len; 359 i++; 360 return i; 361 } 362 363 /* 364 * This fills the MC expected SGIO list 365 * from IOV given by user. 366 */ 367 static __rte_always_inline uint32_t 368 fill_sg_comp_from_iov(sg_comp_t *list, 369 uint32_t i, 370 iov_ptr_t *from, uint32_t from_offset, 371 uint32_t *psize, buf_ptr_t *extra_buf, 372 uint32_t extra_offset) 373 { 374 int32_t j; 375 uint32_t extra_len = extra_buf ? extra_buf->size : 0; 376 uint32_t size = *psize; 377 buf_ptr_t *bufs; 378 379 bufs = from->bufs; 380 for (j = 0; (j < from->buf_cnt) && size; j++) { 381 phys_addr_t e_dma_addr; 382 uint32_t e_len; 383 sg_comp_t *to = &list[i >> 2]; 384 385 if (unlikely(from_offset)) { 386 if (from_offset >= bufs[j].size) { 387 from_offset -= bufs[j].size; 388 continue; 389 } 390 e_dma_addr = bufs[j].dma_addr + from_offset; 391 e_len = (size > (bufs[j].size - from_offset)) ? 392 (bufs[j].size - from_offset) : size; 393 from_offset = 0; 394 } else { 395 e_dma_addr = bufs[j].dma_addr; 396 e_len = (size > bufs[j].size) ? 397 bufs[j].size : size; 398 } 399 400 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 401 to->ptr[i % 4] = rte_cpu_to_be_64(e_dma_addr); 402 403 if (extra_len && (e_len >= extra_offset)) { 404 /* Break the data at given offset */ 405 uint32_t next_len = e_len - extra_offset; 406 phys_addr_t next_dma = e_dma_addr + extra_offset; 407 408 if (!extra_offset) { 409 i--; 410 } else { 411 e_len = extra_offset; 412 size -= e_len; 413 to->u.s.len[i % 4] = rte_cpu_to_be_16(e_len); 414 } 415 416 extra_len = RTE_MIN(extra_len, size); 417 /* Insert extra data ptr */ 418 if (extra_len) { 419 i++; 420 to = &list[i >> 2]; 421 to->u.s.len[i % 4] = 422 rte_cpu_to_be_16(extra_len); 423 to->ptr[i % 4] = 424 rte_cpu_to_be_64(extra_buf->dma_addr); 425 size -= extra_len; 426 } 427 428 next_len = RTE_MIN(next_len, size); 429 /* insert the rest of the data */ 430 if (next_len) { 431 i++; 432 to = &list[i >> 2]; 433 to->u.s.len[i % 4] = rte_cpu_to_be_16(next_len); 434 to->ptr[i % 4] = rte_cpu_to_be_64(next_dma); 435 size -= next_len; 436 } 437 extra_len = 0; 438 439 } else { 440 size -= e_len; 441 } 442 if (extra_offset) 443 extra_offset -= size; 444 i++; 445 } 446 447 *psize = size; 448 return (uint32_t)i; 449 } 450 451 static __rte_always_inline void 452 cpt_digest_gen_prep(uint32_t flags, 453 uint64_t d_lens, 454 digest_params_t *params, 455 void *op, 456 void **prep_req) 457 { 458 struct cpt_request_info *req; 459 uint32_t size, i; 460 uint16_t data_len, mac_len, key_len; 461 auth_type_t hash_type; 462 buf_ptr_t *meta_p; 463 struct cpt_ctx *ctx; 464 sg_comp_t *gather_comp; 465 sg_comp_t *scatter_comp; 466 uint8_t *in_buffer; 467 uint32_t g_size_bytes, s_size_bytes; 468 uint64_t dptr_dma, rptr_dma; 469 vq_cmd_word0_t vq_cmd_w0; 470 vq_cmd_word3_t vq_cmd_w3; 471 void *c_vaddr, *m_vaddr; 472 uint64_t c_dma, m_dma; 473 opcode_info_t opcode; 474 475 ctx = params->ctx_buf.vaddr; 476 meta_p = ¶ms->meta_buf; 477 478 m_vaddr = meta_p->vaddr; 479 m_dma = meta_p->dma_addr; 480 481 /* 482 * Save initial space that followed app data for completion code & 483 * alternate completion code to fall in same cache line as app data 484 */ 485 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 486 m_dma += COMPLETION_CODE_SIZE; 487 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 488 (uint8_t *)m_vaddr; 489 c_vaddr = (uint8_t *)m_vaddr + size; 490 c_dma = m_dma + size; 491 size += sizeof(cpt_res_s_t); 492 493 m_vaddr = (uint8_t *)m_vaddr + size; 494 m_dma += size; 495 496 req = m_vaddr; 497 498 size = sizeof(struct cpt_request_info); 499 m_vaddr = (uint8_t *)m_vaddr + size; 500 m_dma += size; 501 502 hash_type = ctx->hash_type; 503 mac_len = ctx->mac_len; 504 key_len = ctx->auth_key_len; 505 data_len = AUTH_DLEN(d_lens); 506 507 /*GP op header */ 508 vq_cmd_w0.u64 = 0; 509 vq_cmd_w0.s.param2 = ((uint16_t)hash_type << 8); 510 if (ctx->hmac) { 511 opcode.s.major = CPT_MAJOR_OP_HMAC | CPT_DMA_MODE; 512 vq_cmd_w0.s.param1 = key_len; 513 vq_cmd_w0.s.dlen = data_len + ROUNDUP8(key_len); 514 } else { 515 opcode.s.major = CPT_MAJOR_OP_HASH | CPT_DMA_MODE; 516 vq_cmd_w0.s.param1 = 0; 517 vq_cmd_w0.s.dlen = data_len; 518 } 519 520 opcode.s.minor = 0; 521 522 /* Null auth only case enters the if */ 523 if (unlikely(!hash_type && !ctx->enc_cipher)) { 524 opcode.s.major = CPT_MAJOR_OP_MISC; 525 /* Minor op is passthrough */ 526 opcode.s.minor = 0x03; 527 /* Send out completion code only */ 528 vq_cmd_w0.s.param2 = 0x1; 529 } 530 531 vq_cmd_w0.s.opcode = opcode.flags; 532 533 /* DPTR has SG list */ 534 in_buffer = m_vaddr; 535 dptr_dma = m_dma; 536 537 ((uint16_t *)in_buffer)[0] = 0; 538 ((uint16_t *)in_buffer)[1] = 0; 539 540 /* TODO Add error check if space will be sufficient */ 541 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 542 543 /* 544 * Input gather list 545 */ 546 547 i = 0; 548 549 if (ctx->hmac) { 550 uint64_t k_dma = params->ctx_buf.dma_addr + 551 offsetof(struct cpt_ctx, auth_key); 552 /* Key */ 553 i = fill_sg_comp(gather_comp, i, k_dma, ROUNDUP8(key_len)); 554 } 555 556 /* input data */ 557 size = data_len; 558 if (size) { 559 i = fill_sg_comp_from_iov(gather_comp, i, params->src_iov, 560 0, &size, NULL, 0); 561 if (unlikely(size)) { 562 CPT_LOG_DP_DEBUG("Insufficient dst IOV size, short" 563 " by %dB", size); 564 return; 565 } 566 } else { 567 /* 568 * Looks like we need to support zero data 569 * gather ptr in case of hash & hmac 570 */ 571 i++; 572 } 573 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 574 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 575 576 /* 577 * Output Gather list 578 */ 579 580 i = 0; 581 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 582 583 if (flags & VALID_MAC_BUF) { 584 if (unlikely(params->mac_buf.size < mac_len)) { 585 CPT_LOG_DP_ERR("Insufficient MAC size"); 586 return; 587 } 588 589 size = mac_len; 590 i = fill_sg_comp_from_buf_min(scatter_comp, i, 591 ¶ms->mac_buf, &size); 592 } else { 593 size = mac_len; 594 i = fill_sg_comp_from_iov(scatter_comp, i, 595 params->src_iov, data_len, 596 &size, NULL, 0); 597 if (unlikely(size)) { 598 CPT_LOG_DP_ERR("Insufficient dst IOV size, short by" 599 " %dB", size); 600 return; 601 } 602 } 603 604 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 605 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 606 607 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 608 609 /* This is DPTR len incase of SG mode */ 610 vq_cmd_w0.s.dlen = size; 611 612 m_vaddr = (uint8_t *)m_vaddr + size; 613 m_dma += size; 614 615 /* cpt alternate completion address saved earlier */ 616 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 617 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 618 rptr_dma = c_dma - 8; 619 620 req->ist.ei1 = dptr_dma; 621 req->ist.ei2 = rptr_dma; 622 623 /* vq command w3 */ 624 vq_cmd_w3.u64 = 0; 625 626 /* 16 byte aligned cpt res address */ 627 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 628 *req->completion_addr = COMPLETION_CODE_INIT; 629 req->comp_baddr = c_dma; 630 631 /* Fill microcode part of instruction */ 632 req->ist.ei0 = vq_cmd_w0.u64; 633 req->ist.ei3 = vq_cmd_w3.u64; 634 635 req->op = op; 636 637 *prep_req = req; 638 return; 639 } 640 641 static __rte_always_inline void 642 cpt_enc_hmac_prep(uint32_t flags, 643 uint64_t d_offs, 644 uint64_t d_lens, 645 fc_params_t *fc_params, 646 void *op, 647 void **prep_req) 648 { 649 uint32_t iv_offset = 0; 650 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 651 struct cpt_ctx *cpt_ctx; 652 uint32_t cipher_type, hash_type; 653 uint32_t mac_len, size; 654 uint8_t iv_len = 16; 655 struct cpt_request_info *req; 656 buf_ptr_t *meta_p, *aad_buf = NULL; 657 uint32_t encr_offset, auth_offset; 658 uint32_t encr_data_len, auth_data_len, aad_len = 0; 659 uint32_t passthrough_len = 0; 660 void *m_vaddr, *offset_vaddr; 661 uint64_t m_dma, offset_dma, ctx_dma; 662 vq_cmd_word0_t vq_cmd_w0; 663 vq_cmd_word3_t vq_cmd_w3; 664 void *c_vaddr; 665 uint64_t c_dma; 666 opcode_info_t opcode; 667 668 meta_p = &fc_params->meta_buf; 669 m_vaddr = meta_p->vaddr; 670 m_dma = meta_p->dma_addr; 671 672 encr_offset = ENCR_OFFSET(d_offs); 673 auth_offset = AUTH_OFFSET(d_offs); 674 encr_data_len = ENCR_DLEN(d_lens); 675 auth_data_len = AUTH_DLEN(d_lens); 676 if (unlikely(flags & VALID_AAD_BUF)) { 677 /* 678 * We dont support both aad 679 * and auth data separately 680 */ 681 auth_data_len = 0; 682 auth_offset = 0; 683 aad_len = fc_params->aad_buf.size; 684 aad_buf = &fc_params->aad_buf; 685 } 686 cpt_ctx = fc_params->ctx_buf.vaddr; 687 cipher_type = cpt_ctx->enc_cipher; 688 hash_type = cpt_ctx->hash_type; 689 mac_len = cpt_ctx->mac_len; 690 691 /* 692 * Save initial space that followed app data for completion code & 693 * alternate completion code to fall in same cache line as app data 694 */ 695 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 696 m_dma += COMPLETION_CODE_SIZE; 697 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 698 (uint8_t *)m_vaddr; 699 700 c_vaddr = (uint8_t *)m_vaddr + size; 701 c_dma = m_dma + size; 702 size += sizeof(cpt_res_s_t); 703 704 m_vaddr = (uint8_t *)m_vaddr + size; 705 m_dma += size; 706 707 /* start cpt request info struct at 8 byte boundary */ 708 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 709 (uint8_t *)m_vaddr; 710 711 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 712 713 size += sizeof(struct cpt_request_info); 714 m_vaddr = (uint8_t *)m_vaddr + size; 715 m_dma += size; 716 717 if (hash_type == GMAC_TYPE) 718 encr_data_len = 0; 719 720 if (unlikely(!(flags & VALID_IV_BUF))) { 721 iv_len = 0; 722 iv_offset = ENCR_IV_OFFSET(d_offs); 723 } 724 725 if (unlikely(flags & VALID_AAD_BUF)) { 726 /* 727 * When AAD is given, data above encr_offset is pass through 728 * Since AAD is given as separate pointer and not as offset, 729 * this is a special case as we need to fragment input data 730 * into passthrough + encr_data and then insert AAD in between. 731 */ 732 if (hash_type != GMAC_TYPE) { 733 passthrough_len = encr_offset; 734 auth_offset = passthrough_len + iv_len; 735 encr_offset = passthrough_len + aad_len + iv_len; 736 auth_data_len = aad_len + encr_data_len; 737 } else { 738 passthrough_len = 16 + aad_len; 739 auth_offset = passthrough_len + iv_len; 740 auth_data_len = aad_len; 741 } 742 } else { 743 encr_offset += iv_len; 744 auth_offset += iv_len; 745 } 746 747 /* Encryption */ 748 opcode.s.major = CPT_MAJOR_OP_FC; 749 opcode.s.minor = 0; 750 751 auth_dlen = auth_offset + auth_data_len; 752 enc_dlen = encr_data_len + encr_offset; 753 if (unlikely(encr_data_len & 0xf)) { 754 if ((cipher_type == DES3_CBC) || (cipher_type == DES3_ECB)) 755 enc_dlen = ROUNDUP8(encr_data_len) + encr_offset; 756 else if (likely((cipher_type == AES_CBC) || 757 (cipher_type == AES_ECB))) 758 enc_dlen = ROUNDUP16(encr_data_len) + encr_offset; 759 } 760 761 if (unlikely(hash_type == GMAC_TYPE)) { 762 encr_offset = auth_dlen; 763 enc_dlen = 0; 764 } 765 766 if (unlikely(auth_dlen > enc_dlen)) { 767 inputlen = auth_dlen; 768 outputlen = auth_dlen + mac_len; 769 } else { 770 inputlen = enc_dlen; 771 outputlen = enc_dlen + mac_len; 772 } 773 774 /* GP op header */ 775 vq_cmd_w0.u64 = 0; 776 vq_cmd_w0.s.param1 = encr_data_len; 777 vq_cmd_w0.s.param2 = auth_data_len; 778 /* 779 * In 83XX since we have a limitation of 780 * IV & Offset control word not part of instruction 781 * and need to be part of Data Buffer, we check if 782 * head room is there and then only do the Direct mode processing 783 */ 784 if (likely((flags & SINGLE_BUF_INPLACE) && 785 (flags & SINGLE_BUF_HEADTAILROOM))) { 786 void *dm_vaddr = fc_params->bufs[0].vaddr; 787 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 788 /* 789 * This flag indicates that there is 24 bytes head room and 790 * 8 bytes tail room available, so that we get to do 791 * DIRECT MODE with limitation 792 */ 793 794 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 795 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 796 797 /* DPTR */ 798 req->ist.ei1 = offset_dma; 799 /* RPTR should just exclude offset control word */ 800 req->ist.ei2 = dm_dma_addr - iv_len; 801 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 802 + outputlen - iv_len); 803 804 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 805 806 vq_cmd_w0.s.opcode = opcode.flags; 807 808 if (likely(iv_len)) { 809 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 810 + OFF_CTRL_LEN); 811 uint64_t *src = fc_params->iv_buf; 812 dest[0] = src[0]; 813 dest[1] = src[1]; 814 } 815 816 *(uint64_t *)offset_vaddr = 817 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 818 ((uint64_t)iv_offset << 8) | 819 ((uint64_t)auth_offset)); 820 821 } else { 822 uint32_t i, g_size_bytes, s_size_bytes; 823 uint64_t dptr_dma, rptr_dma; 824 sg_comp_t *gather_comp; 825 sg_comp_t *scatter_comp; 826 uint8_t *in_buffer; 827 828 /* This falls under strict SG mode */ 829 offset_vaddr = m_vaddr; 830 offset_dma = m_dma; 831 size = OFF_CTRL_LEN + iv_len; 832 833 m_vaddr = (uint8_t *)m_vaddr + size; 834 m_dma += size; 835 836 opcode.s.major |= CPT_DMA_MODE; 837 838 vq_cmd_w0.s.opcode = opcode.flags; 839 840 if (likely(iv_len)) { 841 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr 842 + OFF_CTRL_LEN); 843 uint64_t *src = fc_params->iv_buf; 844 dest[0] = src[0]; 845 dest[1] = src[1]; 846 } 847 848 *(uint64_t *)offset_vaddr = 849 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 850 ((uint64_t)iv_offset << 8) | 851 ((uint64_t)auth_offset)); 852 853 /* DPTR has SG list */ 854 in_buffer = m_vaddr; 855 dptr_dma = m_dma; 856 857 ((uint16_t *)in_buffer)[0] = 0; 858 ((uint16_t *)in_buffer)[1] = 0; 859 860 /* TODO Add error check if space will be sufficient */ 861 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 862 863 /* 864 * Input Gather List 865 */ 866 867 i = 0; 868 869 /* Offset control word that includes iv */ 870 i = fill_sg_comp(gather_comp, i, offset_dma, 871 OFF_CTRL_LEN + iv_len); 872 873 /* Add input data */ 874 size = inputlen - iv_len; 875 if (likely(size)) { 876 uint32_t aad_offset = aad_len ? passthrough_len : 0; 877 878 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 879 i = fill_sg_comp_from_buf_min(gather_comp, i, 880 fc_params->bufs, 881 &size); 882 } else { 883 i = fill_sg_comp_from_iov(gather_comp, i, 884 fc_params->src_iov, 885 0, &size, 886 aad_buf, aad_offset); 887 } 888 889 if (unlikely(size)) { 890 CPT_LOG_DP_ERR("Insufficient buffer space," 891 " size %d needed", size); 892 return; 893 } 894 } 895 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 896 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 897 898 /* 899 * Output Scatter list 900 */ 901 i = 0; 902 scatter_comp = 903 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 904 905 /* Add IV */ 906 if (likely(iv_len)) { 907 i = fill_sg_comp(scatter_comp, i, 908 offset_dma + OFF_CTRL_LEN, 909 iv_len); 910 } 911 912 /* output data or output data + digest*/ 913 if (unlikely(flags & VALID_MAC_BUF)) { 914 size = outputlen - iv_len - mac_len; 915 if (size) { 916 uint32_t aad_offset = 917 aad_len ? passthrough_len : 0; 918 919 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 920 i = fill_sg_comp_from_buf_min( 921 scatter_comp, 922 i, 923 fc_params->bufs, 924 &size); 925 } else { 926 i = fill_sg_comp_from_iov(scatter_comp, 927 i, 928 fc_params->dst_iov, 929 0, 930 &size, 931 aad_buf, 932 aad_offset); 933 } 934 if (unlikely(size)) { 935 CPT_LOG_DP_ERR("Insufficient buffer" 936 " space, size %d needed", 937 size); 938 return; 939 } 940 } 941 /* mac_data */ 942 if (mac_len) { 943 i = fill_sg_comp_from_buf(scatter_comp, i, 944 &fc_params->mac_buf); 945 } 946 } else { 947 /* Output including mac */ 948 size = outputlen - iv_len; 949 if (likely(size)) { 950 uint32_t aad_offset = 951 aad_len ? passthrough_len : 0; 952 953 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 954 i = fill_sg_comp_from_buf_min( 955 scatter_comp, 956 i, 957 fc_params->bufs, 958 &size); 959 } else { 960 i = fill_sg_comp_from_iov(scatter_comp, 961 i, 962 fc_params->dst_iov, 963 0, 964 &size, 965 aad_buf, 966 aad_offset); 967 } 968 if (unlikely(size)) { 969 CPT_LOG_DP_ERR("Insufficient buffer" 970 " space, size %d needed", 971 size); 972 return; 973 } 974 } 975 } 976 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 977 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 978 979 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 980 981 /* This is DPTR len incase of SG mode */ 982 vq_cmd_w0.s.dlen = size; 983 984 m_vaddr = (uint8_t *)m_vaddr + size; 985 m_dma += size; 986 987 /* cpt alternate completion address saved earlier */ 988 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 989 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 990 rptr_dma = c_dma - 8; 991 992 req->ist.ei1 = dptr_dma; 993 req->ist.ei2 = rptr_dma; 994 } 995 996 ctx_dma = fc_params->ctx_buf.dma_addr + 997 offsetof(struct cpt_ctx, fctx); 998 /* vq command w3 */ 999 vq_cmd_w3.u64 = 0; 1000 vq_cmd_w3.s.grp = 0; 1001 vq_cmd_w3.s.cptr = ctx_dma; 1002 1003 /* 16 byte aligned cpt res address */ 1004 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1005 *req->completion_addr = COMPLETION_CODE_INIT; 1006 req->comp_baddr = c_dma; 1007 1008 /* Fill microcode part of instruction */ 1009 req->ist.ei0 = vq_cmd_w0.u64; 1010 req->ist.ei3 = vq_cmd_w3.u64; 1011 1012 req->op = op; 1013 1014 *prep_req = req; 1015 return; 1016 } 1017 1018 static __rte_always_inline void 1019 cpt_dec_hmac_prep(uint32_t flags, 1020 uint64_t d_offs, 1021 uint64_t d_lens, 1022 fc_params_t *fc_params, 1023 void *op, 1024 void **prep_req) 1025 { 1026 uint32_t iv_offset = 0, size; 1027 int32_t inputlen, outputlen, enc_dlen, auth_dlen; 1028 struct cpt_ctx *cpt_ctx; 1029 int32_t hash_type, mac_len; 1030 uint8_t iv_len = 16; 1031 struct cpt_request_info *req; 1032 buf_ptr_t *meta_p, *aad_buf = NULL; 1033 uint32_t encr_offset, auth_offset; 1034 uint32_t encr_data_len, auth_data_len, aad_len = 0; 1035 uint32_t passthrough_len = 0; 1036 void *m_vaddr, *offset_vaddr; 1037 uint64_t m_dma, offset_dma, ctx_dma; 1038 opcode_info_t opcode; 1039 vq_cmd_word0_t vq_cmd_w0; 1040 vq_cmd_word3_t vq_cmd_w3; 1041 void *c_vaddr; 1042 uint64_t c_dma; 1043 1044 meta_p = &fc_params->meta_buf; 1045 m_vaddr = meta_p->vaddr; 1046 m_dma = meta_p->dma_addr; 1047 1048 encr_offset = ENCR_OFFSET(d_offs); 1049 auth_offset = AUTH_OFFSET(d_offs); 1050 encr_data_len = ENCR_DLEN(d_lens); 1051 auth_data_len = AUTH_DLEN(d_lens); 1052 1053 if (unlikely(flags & VALID_AAD_BUF)) { 1054 /* 1055 * We dont support both aad 1056 * and auth data separately 1057 */ 1058 auth_data_len = 0; 1059 auth_offset = 0; 1060 aad_len = fc_params->aad_buf.size; 1061 aad_buf = &fc_params->aad_buf; 1062 } 1063 1064 cpt_ctx = fc_params->ctx_buf.vaddr; 1065 hash_type = cpt_ctx->hash_type; 1066 mac_len = cpt_ctx->mac_len; 1067 1068 if (hash_type == GMAC_TYPE) 1069 encr_data_len = 0; 1070 1071 if (unlikely(!(flags & VALID_IV_BUF))) { 1072 iv_len = 0; 1073 iv_offset = ENCR_IV_OFFSET(d_offs); 1074 } 1075 1076 if (unlikely(flags & VALID_AAD_BUF)) { 1077 /* 1078 * When AAD is given, data above encr_offset is pass through 1079 * Since AAD is given as separate pointer and not as offset, 1080 * this is a special case as we need to fragment input data 1081 * into passthrough + encr_data and then insert AAD in between. 1082 */ 1083 if (hash_type != GMAC_TYPE) { 1084 passthrough_len = encr_offset; 1085 auth_offset = passthrough_len + iv_len; 1086 encr_offset = passthrough_len + aad_len + iv_len; 1087 auth_data_len = aad_len + encr_data_len; 1088 } else { 1089 passthrough_len = 16 + aad_len; 1090 auth_offset = passthrough_len + iv_len; 1091 auth_data_len = aad_len; 1092 } 1093 } else { 1094 encr_offset += iv_len; 1095 auth_offset += iv_len; 1096 } 1097 1098 /* 1099 * Save initial space that followed app data for completion code & 1100 * alternate completion code to fall in same cache line as app data 1101 */ 1102 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1103 m_dma += COMPLETION_CODE_SIZE; 1104 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1105 (uint8_t *)m_vaddr; 1106 c_vaddr = (uint8_t *)m_vaddr + size; 1107 c_dma = m_dma + size; 1108 size += sizeof(cpt_res_s_t); 1109 1110 m_vaddr = (uint8_t *)m_vaddr + size; 1111 m_dma += size; 1112 1113 /* start cpt request info structure at 8 byte alignment */ 1114 size = (uint8_t *)RTE_PTR_ALIGN(m_vaddr, 8) - 1115 (uint8_t *)m_vaddr; 1116 1117 req = (struct cpt_request_info *)((uint8_t *)m_vaddr + size); 1118 1119 size += sizeof(struct cpt_request_info); 1120 m_vaddr = (uint8_t *)m_vaddr + size; 1121 m_dma += size; 1122 1123 /* Decryption */ 1124 opcode.s.major = CPT_MAJOR_OP_FC; 1125 opcode.s.minor = 1; 1126 1127 enc_dlen = encr_offset + encr_data_len; 1128 auth_dlen = auth_offset + auth_data_len; 1129 1130 if (auth_dlen > enc_dlen) { 1131 inputlen = auth_dlen + mac_len; 1132 outputlen = auth_dlen; 1133 } else { 1134 inputlen = enc_dlen + mac_len; 1135 outputlen = enc_dlen; 1136 } 1137 1138 if (hash_type == GMAC_TYPE) 1139 encr_offset = inputlen; 1140 1141 vq_cmd_w0.u64 = 0; 1142 vq_cmd_w0.s.param1 = encr_data_len; 1143 vq_cmd_w0.s.param2 = auth_data_len; 1144 1145 /* 1146 * In 83XX since we have a limitation of 1147 * IV & Offset control word not part of instruction 1148 * and need to be part of Data Buffer, we check if 1149 * head room is there and then only do the Direct mode processing 1150 */ 1151 if (likely((flags & SINGLE_BUF_INPLACE) && 1152 (flags & SINGLE_BUF_HEADTAILROOM))) { 1153 void *dm_vaddr = fc_params->bufs[0].vaddr; 1154 uint64_t dm_dma_addr = fc_params->bufs[0].dma_addr; 1155 /* 1156 * This flag indicates that there is 24 bytes head room and 1157 * 8 bytes tail room available, so that we get to do 1158 * DIRECT MODE with limitation 1159 */ 1160 1161 offset_vaddr = (uint8_t *)dm_vaddr - OFF_CTRL_LEN - iv_len; 1162 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1163 req->ist.ei1 = offset_dma; 1164 1165 /* RPTR should just exclude offset control word */ 1166 req->ist.ei2 = dm_dma_addr - iv_len; 1167 1168 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr + 1169 outputlen - iv_len); 1170 /* since this is decryption, 1171 * don't touch the content of 1172 * alternate ccode space as it contains 1173 * hmac. 1174 */ 1175 1176 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1177 1178 vq_cmd_w0.s.opcode = opcode.flags; 1179 1180 if (likely(iv_len)) { 1181 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1182 OFF_CTRL_LEN); 1183 uint64_t *src = fc_params->iv_buf; 1184 dest[0] = src[0]; 1185 dest[1] = src[1]; 1186 } 1187 1188 *(uint64_t *)offset_vaddr = 1189 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1190 ((uint64_t)iv_offset << 8) | 1191 ((uint64_t)auth_offset)); 1192 1193 } else { 1194 uint64_t dptr_dma, rptr_dma; 1195 uint32_t g_size_bytes, s_size_bytes; 1196 sg_comp_t *gather_comp; 1197 sg_comp_t *scatter_comp; 1198 uint8_t *in_buffer; 1199 uint8_t i = 0; 1200 1201 /* This falls under strict SG mode */ 1202 offset_vaddr = m_vaddr; 1203 offset_dma = m_dma; 1204 size = OFF_CTRL_LEN + iv_len; 1205 1206 m_vaddr = (uint8_t *)m_vaddr + size; 1207 m_dma += size; 1208 1209 opcode.s.major |= CPT_DMA_MODE; 1210 1211 vq_cmd_w0.s.opcode = opcode.flags; 1212 1213 if (likely(iv_len)) { 1214 uint64_t *dest = (uint64_t *)((uint8_t *)offset_vaddr + 1215 OFF_CTRL_LEN); 1216 uint64_t *src = fc_params->iv_buf; 1217 dest[0] = src[0]; 1218 dest[1] = src[1]; 1219 } 1220 1221 *(uint64_t *)offset_vaddr = 1222 rte_cpu_to_be_64(((uint64_t)encr_offset << 16) | 1223 ((uint64_t)iv_offset << 8) | 1224 ((uint64_t)auth_offset)); 1225 1226 /* DPTR has SG list */ 1227 in_buffer = m_vaddr; 1228 dptr_dma = m_dma; 1229 1230 ((uint16_t *)in_buffer)[0] = 0; 1231 ((uint16_t *)in_buffer)[1] = 0; 1232 1233 /* TODO Add error check if space will be sufficient */ 1234 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1235 1236 /* 1237 * Input Gather List 1238 */ 1239 i = 0; 1240 1241 /* Offset control word that includes iv */ 1242 i = fill_sg_comp(gather_comp, i, offset_dma, 1243 OFF_CTRL_LEN + iv_len); 1244 1245 /* Add input data */ 1246 if (flags & VALID_MAC_BUF) { 1247 size = inputlen - iv_len - mac_len; 1248 if (size) { 1249 /* input data only */ 1250 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1251 i = fill_sg_comp_from_buf_min( 1252 gather_comp, i, 1253 fc_params->bufs, 1254 &size); 1255 } else { 1256 uint32_t aad_offset = aad_len ? 1257 passthrough_len : 0; 1258 1259 i = fill_sg_comp_from_iov(gather_comp, 1260 i, 1261 fc_params->src_iov, 1262 0, &size, 1263 aad_buf, 1264 aad_offset); 1265 } 1266 if (unlikely(size)) { 1267 CPT_LOG_DP_ERR("Insufficient buffer" 1268 " space, size %d needed", 1269 size); 1270 return; 1271 } 1272 } 1273 1274 /* mac data */ 1275 if (mac_len) { 1276 i = fill_sg_comp_from_buf(gather_comp, i, 1277 &fc_params->mac_buf); 1278 } 1279 } else { 1280 /* input data + mac */ 1281 size = inputlen - iv_len; 1282 if (size) { 1283 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1284 i = fill_sg_comp_from_buf_min( 1285 gather_comp, i, 1286 fc_params->bufs, 1287 &size); 1288 } else { 1289 uint32_t aad_offset = aad_len ? 1290 passthrough_len : 0; 1291 1292 if (unlikely(!fc_params->src_iov)) { 1293 CPT_LOG_DP_ERR("Bad input args"); 1294 return; 1295 } 1296 1297 i = fill_sg_comp_from_iov( 1298 gather_comp, i, 1299 fc_params->src_iov, 1300 0, &size, 1301 aad_buf, 1302 aad_offset); 1303 } 1304 1305 if (unlikely(size)) { 1306 CPT_LOG_DP_ERR("Insufficient buffer" 1307 " space, size %d needed", 1308 size); 1309 return; 1310 } 1311 } 1312 } 1313 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1314 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1315 1316 /* 1317 * Output Scatter List 1318 */ 1319 1320 i = 0; 1321 scatter_comp = 1322 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1323 1324 /* Add iv */ 1325 if (iv_len) { 1326 i = fill_sg_comp(scatter_comp, i, 1327 offset_dma + OFF_CTRL_LEN, 1328 iv_len); 1329 } 1330 1331 /* Add output data */ 1332 size = outputlen - iv_len; 1333 if (size) { 1334 if (unlikely(flags & SINGLE_BUF_INPLACE)) { 1335 /* handle single buffer here */ 1336 i = fill_sg_comp_from_buf_min(scatter_comp, i, 1337 fc_params->bufs, 1338 &size); 1339 } else { 1340 uint32_t aad_offset = aad_len ? 1341 passthrough_len : 0; 1342 1343 if (unlikely(!fc_params->dst_iov)) { 1344 CPT_LOG_DP_ERR("Bad input args"); 1345 return; 1346 } 1347 1348 i = fill_sg_comp_from_iov(scatter_comp, i, 1349 fc_params->dst_iov, 0, 1350 &size, aad_buf, 1351 aad_offset); 1352 } 1353 1354 if (unlikely(size)) { 1355 CPT_LOG_DP_ERR("Insufficient buffer space," 1356 " size %d needed", size); 1357 return; 1358 } 1359 } 1360 1361 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1362 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1363 1364 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1365 1366 /* This is DPTR len incase of SG mode */ 1367 vq_cmd_w0.s.dlen = size; 1368 1369 m_vaddr = (uint8_t *)m_vaddr + size; 1370 m_dma += size; 1371 1372 /* cpt alternate completion address saved earlier */ 1373 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1374 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1375 rptr_dma = c_dma - 8; 1376 size += COMPLETION_CODE_SIZE; 1377 1378 req->ist.ei1 = dptr_dma; 1379 req->ist.ei2 = rptr_dma; 1380 } 1381 1382 ctx_dma = fc_params->ctx_buf.dma_addr + 1383 offsetof(struct cpt_ctx, fctx); 1384 /* vq command w3 */ 1385 vq_cmd_w3.u64 = 0; 1386 vq_cmd_w3.s.grp = 0; 1387 vq_cmd_w3.s.cptr = ctx_dma; 1388 1389 /* 16 byte aligned cpt res address */ 1390 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1391 *req->completion_addr = COMPLETION_CODE_INIT; 1392 req->comp_baddr = c_dma; 1393 1394 /* Fill microcode part of instruction */ 1395 req->ist.ei0 = vq_cmd_w0.u64; 1396 req->ist.ei3 = vq_cmd_w3.u64; 1397 1398 req->op = op; 1399 1400 *prep_req = req; 1401 return; 1402 } 1403 1404 static __rte_always_inline void 1405 cpt_zuc_snow3g_enc_prep(uint32_t req_flags, 1406 uint64_t d_offs, 1407 uint64_t d_lens, 1408 fc_params_t *params, 1409 void *op, 1410 void **prep_req) 1411 { 1412 uint32_t size; 1413 int32_t inputlen, outputlen; 1414 struct cpt_ctx *cpt_ctx; 1415 uint32_t mac_len = 0; 1416 uint8_t snow3g, j; 1417 struct cpt_request_info *req; 1418 buf_ptr_t *buf_p; 1419 uint32_t encr_offset = 0, auth_offset = 0; 1420 uint32_t encr_data_len = 0, auth_data_len = 0; 1421 int flags, iv_len = 16; 1422 void *m_vaddr, *c_vaddr; 1423 uint64_t m_dma, c_dma, offset_ctrl; 1424 uint64_t *offset_vaddr, offset_dma; 1425 uint32_t *iv_s, iv[4]; 1426 vq_cmd_word0_t vq_cmd_w0; 1427 vq_cmd_word3_t vq_cmd_w3; 1428 opcode_info_t opcode; 1429 1430 buf_p = ¶ms->meta_buf; 1431 m_vaddr = buf_p->vaddr; 1432 m_dma = buf_p->dma_addr; 1433 1434 cpt_ctx = params->ctx_buf.vaddr; 1435 flags = cpt_ctx->zsk_flags; 1436 mac_len = cpt_ctx->mac_len; 1437 snow3g = cpt_ctx->snow3g; 1438 1439 /* 1440 * Save initial space that followed app data for completion code & 1441 * alternate completion code to fall in same cache line as app data 1442 */ 1443 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1444 m_dma += COMPLETION_CODE_SIZE; 1445 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1446 (uint8_t *)m_vaddr; 1447 1448 c_vaddr = (uint8_t *)m_vaddr + size; 1449 c_dma = m_dma + size; 1450 size += sizeof(cpt_res_s_t); 1451 1452 m_vaddr = (uint8_t *)m_vaddr + size; 1453 m_dma += size; 1454 1455 /* Reserve memory for cpt request info */ 1456 req = m_vaddr; 1457 1458 size = sizeof(struct cpt_request_info); 1459 m_vaddr = (uint8_t *)m_vaddr + size; 1460 m_dma += size; 1461 1462 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1463 1464 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1465 1466 opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1467 (0 << 3) | (flags & 0x7)); 1468 1469 if (flags == 0x1) { 1470 /* 1471 * Microcode expects offsets in bytes 1472 * TODO: Rounding off 1473 */ 1474 auth_data_len = AUTH_DLEN(d_lens); 1475 1476 /* EIA3 or UIA2 */ 1477 auth_offset = AUTH_OFFSET(d_offs); 1478 auth_offset = auth_offset / 8; 1479 1480 /* consider iv len */ 1481 auth_offset += iv_len; 1482 1483 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 1484 outputlen = mac_len; 1485 1486 offset_ctrl = rte_cpu_to_be_64((uint64_t)auth_offset); 1487 1488 } else { 1489 /* EEA3 or UEA2 */ 1490 /* 1491 * Microcode expects offsets in bytes 1492 * TODO: Rounding off 1493 */ 1494 encr_data_len = ENCR_DLEN(d_lens); 1495 1496 encr_offset = ENCR_OFFSET(d_offs); 1497 encr_offset = encr_offset / 8; 1498 /* consider iv len */ 1499 encr_offset += iv_len; 1500 1501 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 1502 outputlen = inputlen; 1503 1504 /* iv offset is 0 */ 1505 offset_ctrl = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1506 } 1507 1508 /* IV */ 1509 iv_s = (flags == 0x1) ? params->auth_iv_buf : 1510 params->iv_buf; 1511 1512 if (snow3g) { 1513 /* 1514 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1515 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1516 */ 1517 1518 for (j = 0; j < 4; j++) 1519 iv[j] = iv_s[3 - j]; 1520 } else { 1521 /* ZUC doesn't need a swap */ 1522 for (j = 0; j < 4; j++) 1523 iv[j] = iv_s[j]; 1524 } 1525 1526 /* 1527 * GP op header, lengths are expected in bits. 1528 */ 1529 vq_cmd_w0.u64 = 0; 1530 vq_cmd_w0.s.param1 = encr_data_len; 1531 vq_cmd_w0.s.param2 = auth_data_len; 1532 1533 /* 1534 * In 83XX since we have a limitation of 1535 * IV & Offset control word not part of instruction 1536 * and need to be part of Data Buffer, we check if 1537 * head room is there and then only do the Direct mode processing 1538 */ 1539 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1540 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1541 void *dm_vaddr = params->bufs[0].vaddr; 1542 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1543 /* 1544 * This flag indicates that there is 24 bytes head room and 1545 * 8 bytes tail room available, so that we get to do 1546 * DIRECT MODE with limitation 1547 */ 1548 1549 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1550 OFF_CTRL_LEN - iv_len); 1551 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1552 1553 /* DPTR */ 1554 req->ist.ei1 = offset_dma; 1555 /* RPTR should just exclude offset control word */ 1556 req->ist.ei2 = dm_dma_addr - iv_len; 1557 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1558 + outputlen - iv_len); 1559 1560 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1561 1562 vq_cmd_w0.s.opcode = opcode.flags; 1563 1564 if (likely(iv_len)) { 1565 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1566 + OFF_CTRL_LEN); 1567 memcpy(iv_d, iv, 16); 1568 } 1569 1570 *offset_vaddr = offset_ctrl; 1571 } else { 1572 uint32_t i, g_size_bytes, s_size_bytes; 1573 uint64_t dptr_dma, rptr_dma; 1574 sg_comp_t *gather_comp; 1575 sg_comp_t *scatter_comp; 1576 uint8_t *in_buffer; 1577 uint32_t *iv_d; 1578 1579 /* save space for iv */ 1580 offset_vaddr = m_vaddr; 1581 offset_dma = m_dma; 1582 1583 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1584 m_dma += OFF_CTRL_LEN + iv_len; 1585 1586 opcode.s.major |= CPT_DMA_MODE; 1587 1588 vq_cmd_w0.s.opcode = opcode.flags; 1589 1590 /* DPTR has SG list */ 1591 in_buffer = m_vaddr; 1592 dptr_dma = m_dma; 1593 1594 ((uint16_t *)in_buffer)[0] = 0; 1595 ((uint16_t *)in_buffer)[1] = 0; 1596 1597 /* TODO Add error check if space will be sufficient */ 1598 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1599 1600 /* 1601 * Input Gather List 1602 */ 1603 i = 0; 1604 1605 /* Offset control word followed by iv */ 1606 1607 i = fill_sg_comp(gather_comp, i, offset_dma, 1608 OFF_CTRL_LEN + iv_len); 1609 1610 /* iv offset is 0 */ 1611 *offset_vaddr = offset_ctrl; 1612 1613 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1614 memcpy(iv_d, iv, 16); 1615 1616 /* input data */ 1617 size = inputlen - iv_len; 1618 if (size) { 1619 i = fill_sg_comp_from_iov(gather_comp, i, 1620 params->src_iov, 1621 0, &size, NULL, 0); 1622 if (unlikely(size)) { 1623 CPT_LOG_DP_ERR("Insufficient buffer space," 1624 " size %d needed", size); 1625 return; 1626 } 1627 } 1628 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1629 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1630 1631 /* 1632 * Output Scatter List 1633 */ 1634 1635 i = 0; 1636 scatter_comp = 1637 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1638 1639 if (flags == 0x1) { 1640 /* IV in SLIST only for EEA3 & UEA2 */ 1641 iv_len = 0; 1642 } 1643 1644 if (iv_len) { 1645 i = fill_sg_comp(scatter_comp, i, 1646 offset_dma + OFF_CTRL_LEN, iv_len); 1647 } 1648 1649 /* Add output data */ 1650 if (req_flags & VALID_MAC_BUF) { 1651 size = outputlen - iv_len - mac_len; 1652 if (size) { 1653 i = fill_sg_comp_from_iov(scatter_comp, i, 1654 params->dst_iov, 0, 1655 &size, NULL, 0); 1656 1657 if (unlikely(size)) { 1658 CPT_LOG_DP_ERR("Insufficient buffer space," 1659 " size %d needed", size); 1660 return; 1661 } 1662 } 1663 1664 /* mac data */ 1665 if (mac_len) { 1666 i = fill_sg_comp_from_buf(scatter_comp, i, 1667 ¶ms->mac_buf); 1668 } 1669 } else { 1670 /* Output including mac */ 1671 size = outputlen - iv_len; 1672 if (size) { 1673 i = fill_sg_comp_from_iov(scatter_comp, i, 1674 params->dst_iov, 0, 1675 &size, NULL, 0); 1676 1677 if (unlikely(size)) { 1678 CPT_LOG_DP_ERR("Insufficient buffer space," 1679 " size %d needed", size); 1680 return; 1681 } 1682 } 1683 } 1684 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1685 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1686 1687 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1688 1689 /* This is DPTR len incase of SG mode */ 1690 vq_cmd_w0.s.dlen = size; 1691 1692 m_vaddr = (uint8_t *)m_vaddr + size; 1693 m_dma += size; 1694 1695 /* cpt alternate completion address saved earlier */ 1696 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1697 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1698 rptr_dma = c_dma - 8; 1699 1700 req->ist.ei1 = dptr_dma; 1701 req->ist.ei2 = rptr_dma; 1702 } 1703 1704 /* vq command w3 */ 1705 vq_cmd_w3.u64 = 0; 1706 vq_cmd_w3.s.grp = 0; 1707 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1708 offsetof(struct cpt_ctx, zs_ctx); 1709 1710 /* 16 byte aligned cpt res address */ 1711 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1712 *req->completion_addr = COMPLETION_CODE_INIT; 1713 req->comp_baddr = c_dma; 1714 1715 /* Fill microcode part of instruction */ 1716 req->ist.ei0 = vq_cmd_w0.u64; 1717 req->ist.ei3 = vq_cmd_w3.u64; 1718 1719 req->op = op; 1720 1721 *prep_req = req; 1722 return; 1723 } 1724 1725 static __rte_always_inline void 1726 cpt_zuc_snow3g_dec_prep(uint32_t req_flags, 1727 uint64_t d_offs, 1728 uint64_t d_lens, 1729 fc_params_t *params, 1730 void *op, 1731 void **prep_req) 1732 { 1733 uint32_t size; 1734 int32_t inputlen = 0, outputlen; 1735 struct cpt_ctx *cpt_ctx; 1736 uint8_t snow3g, iv_len = 16; 1737 struct cpt_request_info *req; 1738 buf_ptr_t *buf_p; 1739 uint32_t encr_offset; 1740 uint32_t encr_data_len; 1741 int flags; 1742 void *m_vaddr, *c_vaddr; 1743 uint64_t m_dma, c_dma; 1744 uint64_t *offset_vaddr, offset_dma; 1745 uint32_t *iv_s, iv[4], j; 1746 vq_cmd_word0_t vq_cmd_w0; 1747 vq_cmd_word3_t vq_cmd_w3; 1748 opcode_info_t opcode; 1749 1750 buf_p = ¶ms->meta_buf; 1751 m_vaddr = buf_p->vaddr; 1752 m_dma = buf_p->dma_addr; 1753 1754 /* 1755 * Microcode expects offsets in bytes 1756 * TODO: Rounding off 1757 */ 1758 encr_offset = ENCR_OFFSET(d_offs) / 8; 1759 encr_data_len = ENCR_DLEN(d_lens); 1760 1761 cpt_ctx = params->ctx_buf.vaddr; 1762 flags = cpt_ctx->zsk_flags; 1763 snow3g = cpt_ctx->snow3g; 1764 /* 1765 * Save initial space that followed app data for completion code & 1766 * alternate completion code to fall in same cache line as app data 1767 */ 1768 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 1769 m_dma += COMPLETION_CODE_SIZE; 1770 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 1771 (uint8_t *)m_vaddr; 1772 1773 c_vaddr = (uint8_t *)m_vaddr + size; 1774 c_dma = m_dma + size; 1775 size += sizeof(cpt_res_s_t); 1776 1777 m_vaddr = (uint8_t *)m_vaddr + size; 1778 m_dma += size; 1779 1780 /* Reserve memory for cpt request info */ 1781 req = m_vaddr; 1782 1783 size = sizeof(struct cpt_request_info); 1784 m_vaddr = (uint8_t *)m_vaddr + size; 1785 m_dma += size; 1786 1787 opcode.s.major = CPT_MAJOR_OP_ZUC_SNOW3G; 1788 1789 /* indicates CPTR ctx, operation type, KEY & IV mode from DPTR */ 1790 1791 opcode.s.minor = ((1 << 7) | (snow3g << 5) | (0 << 4) | 1792 (0 << 3) | (flags & 0x7)); 1793 1794 /* consider iv len */ 1795 encr_offset += iv_len; 1796 1797 inputlen = encr_offset + 1798 (RTE_ALIGN(encr_data_len, 8) / 8); 1799 outputlen = inputlen; 1800 1801 /* IV */ 1802 iv_s = params->iv_buf; 1803 if (snow3g) { 1804 /* 1805 * DPDK seems to provide it in form of IV3 IV2 IV1 IV0 1806 * and BigEndian, MC needs it as IV0 IV1 IV2 IV3 1807 */ 1808 1809 for (j = 0; j < 4; j++) 1810 iv[j] = iv_s[3 - j]; 1811 } else { 1812 /* ZUC doesn't need a swap */ 1813 for (j = 0; j < 4; j++) 1814 iv[j] = iv_s[j]; 1815 } 1816 1817 /* 1818 * GP op header, lengths are expected in bits. 1819 */ 1820 vq_cmd_w0.u64 = 0; 1821 vq_cmd_w0.s.param1 = encr_data_len; 1822 1823 /* 1824 * In 83XX since we have a limitation of 1825 * IV & Offset control word not part of instruction 1826 * and need to be part of Data Buffer, we check if 1827 * head room is there and then only do the Direct mode processing 1828 */ 1829 if (likely((req_flags & SINGLE_BUF_INPLACE) && 1830 (req_flags & SINGLE_BUF_HEADTAILROOM))) { 1831 void *dm_vaddr = params->bufs[0].vaddr; 1832 uint64_t dm_dma_addr = params->bufs[0].dma_addr; 1833 /* 1834 * This flag indicates that there is 24 bytes head room and 1835 * 8 bytes tail room available, so that we get to do 1836 * DIRECT MODE with limitation 1837 */ 1838 1839 offset_vaddr = (uint64_t *)((uint8_t *)dm_vaddr - 1840 OFF_CTRL_LEN - iv_len); 1841 offset_dma = dm_dma_addr - OFF_CTRL_LEN - iv_len; 1842 1843 /* DPTR */ 1844 req->ist.ei1 = offset_dma; 1845 /* RPTR should just exclude offset control word */ 1846 req->ist.ei2 = dm_dma_addr - iv_len; 1847 req->alternate_caddr = (uint64_t *)((uint8_t *)dm_vaddr 1848 + outputlen - iv_len); 1849 1850 vq_cmd_w0.s.dlen = inputlen + OFF_CTRL_LEN; 1851 1852 vq_cmd_w0.s.opcode = opcode.flags; 1853 1854 if (likely(iv_len)) { 1855 uint32_t *iv_d = (uint32_t *)((uint8_t *)offset_vaddr 1856 + OFF_CTRL_LEN); 1857 memcpy(iv_d, iv, 16); 1858 } 1859 1860 /* iv offset is 0 */ 1861 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1862 } else { 1863 uint32_t i, g_size_bytes, s_size_bytes; 1864 uint64_t dptr_dma, rptr_dma; 1865 sg_comp_t *gather_comp; 1866 sg_comp_t *scatter_comp; 1867 uint8_t *in_buffer; 1868 uint32_t *iv_d; 1869 1870 /* save space for offset and iv... */ 1871 offset_vaddr = m_vaddr; 1872 offset_dma = m_dma; 1873 1874 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 1875 m_dma += OFF_CTRL_LEN + iv_len; 1876 1877 opcode.s.major |= CPT_DMA_MODE; 1878 1879 vq_cmd_w0.s.opcode = opcode.flags; 1880 1881 /* DPTR has SG list */ 1882 in_buffer = m_vaddr; 1883 dptr_dma = m_dma; 1884 1885 ((uint16_t *)in_buffer)[0] = 0; 1886 ((uint16_t *)in_buffer)[1] = 0; 1887 1888 /* TODO Add error check if space will be sufficient */ 1889 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 1890 1891 /* 1892 * Input Gather List 1893 */ 1894 i = 0; 1895 1896 /* Offset control word */ 1897 1898 /* iv offset is 0 */ 1899 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 1900 1901 i = fill_sg_comp(gather_comp, i, offset_dma, 1902 OFF_CTRL_LEN + iv_len); 1903 1904 iv_d = (uint32_t *)((uint8_t *)offset_vaddr + OFF_CTRL_LEN); 1905 memcpy(iv_d, iv, 16); 1906 1907 /* Add input data */ 1908 size = inputlen - iv_len; 1909 if (size) { 1910 i = fill_sg_comp_from_iov(gather_comp, i, 1911 params->src_iov, 1912 0, &size, NULL, 0); 1913 if (unlikely(size)) { 1914 CPT_LOG_DP_ERR("Insufficient buffer space," 1915 " size %d needed", size); 1916 return; 1917 } 1918 } 1919 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 1920 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1921 1922 /* 1923 * Output Scatter List 1924 */ 1925 1926 i = 0; 1927 scatter_comp = 1928 (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 1929 1930 /* IV */ 1931 i = fill_sg_comp(scatter_comp, i, 1932 offset_dma + OFF_CTRL_LEN, 1933 iv_len); 1934 1935 /* Add output data */ 1936 size = outputlen - iv_len; 1937 if (size) { 1938 i = fill_sg_comp_from_iov(scatter_comp, i, 1939 params->dst_iov, 0, 1940 &size, NULL, 0); 1941 1942 if (unlikely(size)) { 1943 CPT_LOG_DP_ERR("Insufficient buffer space," 1944 " size %d needed", size); 1945 return; 1946 } 1947 } 1948 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 1949 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 1950 1951 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 1952 1953 /* This is DPTR len incase of SG mode */ 1954 vq_cmd_w0.s.dlen = size; 1955 1956 m_vaddr = (uint8_t *)m_vaddr + size; 1957 m_dma += size; 1958 1959 /* cpt alternate completion address saved earlier */ 1960 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 1961 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 1962 rptr_dma = c_dma - 8; 1963 1964 req->ist.ei1 = dptr_dma; 1965 req->ist.ei2 = rptr_dma; 1966 } 1967 1968 /* vq command w3 */ 1969 vq_cmd_w3.u64 = 0; 1970 vq_cmd_w3.s.grp = 0; 1971 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 1972 offsetof(struct cpt_ctx, zs_ctx); 1973 1974 /* 16 byte aligned cpt res address */ 1975 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 1976 *req->completion_addr = COMPLETION_CODE_INIT; 1977 req->comp_baddr = c_dma; 1978 1979 /* Fill microcode part of instruction */ 1980 req->ist.ei0 = vq_cmd_w0.u64; 1981 req->ist.ei3 = vq_cmd_w3.u64; 1982 1983 req->op = op; 1984 1985 *prep_req = req; 1986 return; 1987 } 1988 1989 static __rte_always_inline void 1990 cpt_kasumi_enc_prep(uint32_t req_flags, 1991 uint64_t d_offs, 1992 uint64_t d_lens, 1993 fc_params_t *params, 1994 void *op, 1995 void **prep_req) 1996 { 1997 uint32_t size; 1998 int32_t inputlen = 0, outputlen = 0; 1999 struct cpt_ctx *cpt_ctx; 2000 uint32_t mac_len = 0; 2001 uint8_t i = 0; 2002 struct cpt_request_info *req; 2003 buf_ptr_t *buf_p; 2004 uint32_t encr_offset, auth_offset; 2005 uint32_t encr_data_len, auth_data_len; 2006 int flags; 2007 uint8_t *iv_s, *iv_d, iv_len = 8; 2008 uint8_t dir = 0; 2009 void *m_vaddr, *c_vaddr; 2010 uint64_t m_dma, c_dma; 2011 uint64_t *offset_vaddr, offset_dma; 2012 vq_cmd_word0_t vq_cmd_w0; 2013 vq_cmd_word3_t vq_cmd_w3; 2014 opcode_info_t opcode; 2015 uint8_t *in_buffer; 2016 uint32_t g_size_bytes, s_size_bytes; 2017 uint64_t dptr_dma, rptr_dma; 2018 sg_comp_t *gather_comp; 2019 sg_comp_t *scatter_comp; 2020 2021 buf_p = ¶ms->meta_buf; 2022 m_vaddr = buf_p->vaddr; 2023 m_dma = buf_p->dma_addr; 2024 2025 encr_offset = ENCR_OFFSET(d_offs) / 8; 2026 auth_offset = AUTH_OFFSET(d_offs) / 8; 2027 encr_data_len = ENCR_DLEN(d_lens); 2028 auth_data_len = AUTH_DLEN(d_lens); 2029 2030 cpt_ctx = params->ctx_buf.vaddr; 2031 flags = cpt_ctx->zsk_flags; 2032 mac_len = cpt_ctx->mac_len; 2033 2034 if (flags == 0x0) 2035 iv_s = params->iv_buf; 2036 else 2037 iv_s = params->auth_iv_buf; 2038 2039 dir = iv_s[8] & 0x1; 2040 2041 /* 2042 * Save initial space that followed app data for completion code & 2043 * alternate completion code to fall in same cache line as app data 2044 */ 2045 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2046 m_dma += COMPLETION_CODE_SIZE; 2047 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2048 (uint8_t *)m_vaddr; 2049 2050 c_vaddr = (uint8_t *)m_vaddr + size; 2051 c_dma = m_dma + size; 2052 size += sizeof(cpt_res_s_t); 2053 2054 m_vaddr = (uint8_t *)m_vaddr + size; 2055 m_dma += size; 2056 2057 /* Reserve memory for cpt request info */ 2058 req = m_vaddr; 2059 2060 size = sizeof(struct cpt_request_info); 2061 m_vaddr = (uint8_t *)m_vaddr + size; 2062 m_dma += size; 2063 2064 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2065 2066 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2067 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2068 (dir << 4) | (0 << 3) | (flags & 0x7)); 2069 2070 /* 2071 * GP op header, lengths are expected in bits. 2072 */ 2073 vq_cmd_w0.u64 = 0; 2074 vq_cmd_w0.s.param1 = encr_data_len; 2075 vq_cmd_w0.s.param2 = auth_data_len; 2076 vq_cmd_w0.s.opcode = opcode.flags; 2077 2078 /* consider iv len */ 2079 if (flags == 0x0) { 2080 encr_offset += iv_len; 2081 auth_offset += iv_len; 2082 } 2083 2084 /* save space for offset ctrl and iv */ 2085 offset_vaddr = m_vaddr; 2086 offset_dma = m_dma; 2087 2088 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2089 m_dma += OFF_CTRL_LEN + iv_len; 2090 2091 /* DPTR has SG list */ 2092 in_buffer = m_vaddr; 2093 dptr_dma = m_dma; 2094 2095 ((uint16_t *)in_buffer)[0] = 0; 2096 ((uint16_t *)in_buffer)[1] = 0; 2097 2098 /* TODO Add error check if space will be sufficient */ 2099 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2100 2101 /* 2102 * Input Gather List 2103 */ 2104 i = 0; 2105 2106 /* Offset control word followed by iv */ 2107 2108 if (flags == 0x0) { 2109 inputlen = encr_offset + (RTE_ALIGN(encr_data_len, 8) / 8); 2110 outputlen = inputlen; 2111 /* iv offset is 0 */ 2112 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2113 } else { 2114 inputlen = auth_offset + (RTE_ALIGN(auth_data_len, 8) / 8); 2115 outputlen = mac_len; 2116 /* iv offset is 0 */ 2117 *offset_vaddr = rte_cpu_to_be_64((uint64_t)auth_offset); 2118 } 2119 2120 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2121 2122 /* IV */ 2123 iv_d = (uint8_t *)offset_vaddr + OFF_CTRL_LEN; 2124 memcpy(iv_d, iv_s, iv_len); 2125 2126 /* input data */ 2127 size = inputlen - iv_len; 2128 if (size) { 2129 i = fill_sg_comp_from_iov(gather_comp, i, 2130 params->src_iov, 0, 2131 &size, NULL, 0); 2132 2133 if (unlikely(size)) { 2134 CPT_LOG_DP_ERR("Insufficient buffer space," 2135 " size %d needed", size); 2136 return; 2137 } 2138 } 2139 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2140 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2141 2142 /* 2143 * Output Scatter List 2144 */ 2145 2146 i = 0; 2147 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2148 2149 if (flags == 0x1) { 2150 /* IV in SLIST only for F8 */ 2151 iv_len = 0; 2152 } 2153 2154 /* IV */ 2155 if (iv_len) { 2156 i = fill_sg_comp(scatter_comp, i, 2157 offset_dma + OFF_CTRL_LEN, 2158 iv_len); 2159 } 2160 2161 /* Add output data */ 2162 if (req_flags & VALID_MAC_BUF) { 2163 size = outputlen - iv_len - mac_len; 2164 if (size) { 2165 i = fill_sg_comp_from_iov(scatter_comp, i, 2166 params->dst_iov, 0, 2167 &size, NULL, 0); 2168 2169 if (unlikely(size)) { 2170 CPT_LOG_DP_ERR("Insufficient buffer space," 2171 " size %d needed", size); 2172 return; 2173 } 2174 } 2175 2176 /* mac data */ 2177 if (mac_len) { 2178 i = fill_sg_comp_from_buf(scatter_comp, i, 2179 ¶ms->mac_buf); 2180 } 2181 } else { 2182 /* Output including mac */ 2183 size = outputlen - iv_len; 2184 if (size) { 2185 i = fill_sg_comp_from_iov(scatter_comp, i, 2186 params->dst_iov, 0, 2187 &size, NULL, 0); 2188 2189 if (unlikely(size)) { 2190 CPT_LOG_DP_ERR("Insufficient buffer space," 2191 " size %d needed", size); 2192 return; 2193 } 2194 } 2195 } 2196 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2197 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2198 2199 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2200 2201 /* This is DPTR len incase of SG mode */ 2202 vq_cmd_w0.s.dlen = size; 2203 2204 m_vaddr = (uint8_t *)m_vaddr + size; 2205 m_dma += size; 2206 2207 /* cpt alternate completion address saved earlier */ 2208 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2209 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2210 rptr_dma = c_dma - 8; 2211 2212 req->ist.ei1 = dptr_dma; 2213 req->ist.ei2 = rptr_dma; 2214 2215 /* vq command w3 */ 2216 vq_cmd_w3.u64 = 0; 2217 vq_cmd_w3.s.grp = 0; 2218 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2219 offsetof(struct cpt_ctx, k_ctx); 2220 2221 /* 16 byte aligned cpt res address */ 2222 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2223 *req->completion_addr = COMPLETION_CODE_INIT; 2224 req->comp_baddr = c_dma; 2225 2226 /* Fill microcode part of instruction */ 2227 req->ist.ei0 = vq_cmd_w0.u64; 2228 req->ist.ei3 = vq_cmd_w3.u64; 2229 2230 req->op = op; 2231 2232 *prep_req = req; 2233 return; 2234 } 2235 2236 static __rte_always_inline void 2237 cpt_kasumi_dec_prep(uint64_t d_offs, 2238 uint64_t d_lens, 2239 fc_params_t *params, 2240 void *op, 2241 void **prep_req) 2242 { 2243 uint32_t size; 2244 int32_t inputlen = 0, outputlen; 2245 struct cpt_ctx *cpt_ctx; 2246 uint8_t i = 0, iv_len = 8; 2247 struct cpt_request_info *req; 2248 buf_ptr_t *buf_p; 2249 uint32_t encr_offset; 2250 uint32_t encr_data_len; 2251 int flags; 2252 uint8_t dir = 0; 2253 void *m_vaddr, *c_vaddr; 2254 uint64_t m_dma, c_dma; 2255 uint64_t *offset_vaddr, offset_dma; 2256 vq_cmd_word0_t vq_cmd_w0; 2257 vq_cmd_word3_t vq_cmd_w3; 2258 opcode_info_t opcode; 2259 uint8_t *in_buffer; 2260 uint32_t g_size_bytes, s_size_bytes; 2261 uint64_t dptr_dma, rptr_dma; 2262 sg_comp_t *gather_comp; 2263 sg_comp_t *scatter_comp; 2264 2265 buf_p = ¶ms->meta_buf; 2266 m_vaddr = buf_p->vaddr; 2267 m_dma = buf_p->dma_addr; 2268 2269 encr_offset = ENCR_OFFSET(d_offs) / 8; 2270 encr_data_len = ENCR_DLEN(d_lens); 2271 2272 cpt_ctx = params->ctx_buf.vaddr; 2273 flags = cpt_ctx->zsk_flags; 2274 /* 2275 * Save initial space that followed app data for completion code & 2276 * alternate completion code to fall in same cache line as app data 2277 */ 2278 m_vaddr = (uint8_t *)m_vaddr + COMPLETION_CODE_SIZE; 2279 m_dma += COMPLETION_CODE_SIZE; 2280 size = (uint8_t *)RTE_PTR_ALIGN((uint8_t *)m_vaddr, 16) - 2281 (uint8_t *)m_vaddr; 2282 2283 c_vaddr = (uint8_t *)m_vaddr + size; 2284 c_dma = m_dma + size; 2285 size += sizeof(cpt_res_s_t); 2286 2287 m_vaddr = (uint8_t *)m_vaddr + size; 2288 m_dma += size; 2289 2290 /* Reserve memory for cpt request info */ 2291 req = m_vaddr; 2292 2293 size = sizeof(struct cpt_request_info); 2294 m_vaddr = (uint8_t *)m_vaddr + size; 2295 m_dma += size; 2296 2297 opcode.s.major = CPT_MAJOR_OP_KASUMI | CPT_DMA_MODE; 2298 2299 /* indicates ECB/CBC, direction, ctx from cptr, iv from dptr */ 2300 opcode.s.minor = ((1 << 6) | (cpt_ctx->k_ecb << 5) | 2301 (dir << 4) | (0 << 3) | (flags & 0x7)); 2302 2303 /* 2304 * GP op header, lengths are expected in bits. 2305 */ 2306 vq_cmd_w0.u64 = 0; 2307 vq_cmd_w0.s.param1 = encr_data_len; 2308 vq_cmd_w0.s.opcode = opcode.flags; 2309 2310 /* consider iv len */ 2311 encr_offset += iv_len; 2312 2313 inputlen = iv_len + (RTE_ALIGN(encr_data_len, 8) / 8); 2314 outputlen = inputlen; 2315 2316 /* save space for offset ctrl & iv */ 2317 offset_vaddr = m_vaddr; 2318 offset_dma = m_dma; 2319 2320 m_vaddr = (uint8_t *)m_vaddr + OFF_CTRL_LEN + iv_len; 2321 m_dma += OFF_CTRL_LEN + iv_len; 2322 2323 /* DPTR has SG list */ 2324 in_buffer = m_vaddr; 2325 dptr_dma = m_dma; 2326 2327 ((uint16_t *)in_buffer)[0] = 0; 2328 ((uint16_t *)in_buffer)[1] = 0; 2329 2330 /* TODO Add error check if space will be sufficient */ 2331 gather_comp = (sg_comp_t *)((uint8_t *)m_vaddr + 8); 2332 2333 /* 2334 * Input Gather List 2335 */ 2336 i = 0; 2337 2338 /* Offset control word followed by iv */ 2339 *offset_vaddr = rte_cpu_to_be_64((uint64_t)encr_offset << 16); 2340 2341 i = fill_sg_comp(gather_comp, i, offset_dma, OFF_CTRL_LEN + iv_len); 2342 2343 /* IV */ 2344 memcpy((uint8_t *)offset_vaddr + OFF_CTRL_LEN, 2345 params->iv_buf, iv_len); 2346 2347 /* Add input data */ 2348 size = inputlen - iv_len; 2349 if (size) { 2350 i = fill_sg_comp_from_iov(gather_comp, i, 2351 params->src_iov, 2352 0, &size, NULL, 0); 2353 if (unlikely(size)) { 2354 CPT_LOG_DP_ERR("Insufficient buffer space," 2355 " size %d needed", size); 2356 return; 2357 } 2358 } 2359 ((uint16_t *)in_buffer)[2] = rte_cpu_to_be_16(i); 2360 g_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2361 2362 /* 2363 * Output Scatter List 2364 */ 2365 2366 i = 0; 2367 scatter_comp = (sg_comp_t *)((uint8_t *)gather_comp + g_size_bytes); 2368 2369 /* IV */ 2370 i = fill_sg_comp(scatter_comp, i, 2371 offset_dma + OFF_CTRL_LEN, 2372 iv_len); 2373 2374 /* Add output data */ 2375 size = outputlen - iv_len; 2376 if (size) { 2377 i = fill_sg_comp_from_iov(scatter_comp, i, 2378 params->dst_iov, 0, 2379 &size, NULL, 0); 2380 if (unlikely(size)) { 2381 CPT_LOG_DP_ERR("Insufficient buffer space," 2382 " size %d needed", size); 2383 return; 2384 } 2385 } 2386 ((uint16_t *)in_buffer)[3] = rte_cpu_to_be_16(i); 2387 s_size_bytes = ((i + 3) / 4) * sizeof(sg_comp_t); 2388 2389 size = g_size_bytes + s_size_bytes + SG_LIST_HDR_SIZE; 2390 2391 /* This is DPTR len incase of SG mode */ 2392 vq_cmd_w0.s.dlen = size; 2393 2394 m_vaddr = (uint8_t *)m_vaddr + size; 2395 m_dma += size; 2396 2397 /* cpt alternate completion address saved earlier */ 2398 req->alternate_caddr = (uint64_t *)((uint8_t *)c_vaddr - 8); 2399 *req->alternate_caddr = ~((uint64_t)COMPLETION_CODE_INIT); 2400 rptr_dma = c_dma - 8; 2401 2402 req->ist.ei1 = dptr_dma; 2403 req->ist.ei2 = rptr_dma; 2404 2405 /* vq command w3 */ 2406 vq_cmd_w3.u64 = 0; 2407 vq_cmd_w3.s.grp = 0; 2408 vq_cmd_w3.s.cptr = params->ctx_buf.dma_addr + 2409 offsetof(struct cpt_ctx, k_ctx); 2410 2411 /* 16 byte aligned cpt res address */ 2412 req->completion_addr = (uint64_t *)((uint8_t *)c_vaddr); 2413 *req->completion_addr = COMPLETION_CODE_INIT; 2414 req->comp_baddr = c_dma; 2415 2416 /* Fill microcode part of instruction */ 2417 req->ist.ei0 = vq_cmd_w0.u64; 2418 req->ist.ei3 = vq_cmd_w3.u64; 2419 2420 req->op = op; 2421 2422 *prep_req = req; 2423 return; 2424 } 2425 2426 static __rte_always_inline void * 2427 cpt_fc_dec_hmac_prep(uint32_t flags, 2428 uint64_t d_offs, 2429 uint64_t d_lens, 2430 fc_params_t *fc_params, 2431 void *op) 2432 { 2433 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2434 uint8_t fc_type; 2435 void *prep_req = NULL; 2436 2437 fc_type = ctx->fc_type; 2438 2439 if (likely(fc_type == FC_GEN)) { 2440 cpt_dec_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2441 &prep_req); 2442 } else if (fc_type == ZUC_SNOW3G) { 2443 cpt_zuc_snow3g_dec_prep(flags, d_offs, d_lens, fc_params, op, 2444 &prep_req); 2445 } else if (fc_type == KASUMI) { 2446 cpt_kasumi_dec_prep(d_offs, d_lens, fc_params, op, &prep_req); 2447 } 2448 2449 /* 2450 * For AUTH_ONLY case, 2451 * MC only supports digest generation and verification 2452 * should be done in software by memcmp() 2453 */ 2454 2455 return prep_req; 2456 } 2457 2458 static __rte_always_inline void *__hot 2459 cpt_fc_enc_hmac_prep(uint32_t flags, uint64_t d_offs, uint64_t d_lens, 2460 fc_params_t *fc_params, void *op) 2461 { 2462 struct cpt_ctx *ctx = fc_params->ctx_buf.vaddr; 2463 uint8_t fc_type; 2464 void *prep_req = NULL; 2465 2466 fc_type = ctx->fc_type; 2467 2468 /* Common api for rest of the ops */ 2469 if (likely(fc_type == FC_GEN)) { 2470 cpt_enc_hmac_prep(flags, d_offs, d_lens, fc_params, op, 2471 &prep_req); 2472 } else if (fc_type == ZUC_SNOW3G) { 2473 cpt_zuc_snow3g_enc_prep(flags, d_offs, d_lens, fc_params, op, 2474 &prep_req); 2475 } else if (fc_type == KASUMI) { 2476 cpt_kasumi_enc_prep(flags, d_offs, d_lens, fc_params, op, 2477 &prep_req); 2478 } else if (fc_type == HASH_HMAC) { 2479 cpt_digest_gen_prep(flags, d_lens, fc_params, op, &prep_req); 2480 } 2481 2482 return prep_req; 2483 } 2484 2485 static __rte_always_inline int 2486 cpt_fc_auth_set_key(void *ctx, auth_type_t type, const uint8_t *key, 2487 uint16_t key_len, uint16_t mac_len) 2488 { 2489 struct cpt_ctx *cpt_ctx = ctx; 2490 mc_fc_context_t *fctx = &cpt_ctx->fctx; 2491 2492 if ((type >= ZUC_EIA3) && (type <= KASUMI_F9_ECB)) { 2493 uint32_t keyx[4]; 2494 2495 if (key_len != 16) 2496 return -1; 2497 /* No support for AEAD yet */ 2498 if (cpt_ctx->enc_cipher) 2499 return -1; 2500 /* For ZUC/SNOW3G/Kasumi */ 2501 switch (type) { 2502 case SNOW3G_UIA2: 2503 cpt_ctx->snow3g = 1; 2504 gen_key_snow3g(key, keyx); 2505 memcpy(cpt_ctx->zs_ctx.ci_key, keyx, key_len); 2506 cpt_ctx->fc_type = ZUC_SNOW3G; 2507 cpt_ctx->zsk_flags = 0x1; 2508 break; 2509 case ZUC_EIA3: 2510 cpt_ctx->snow3g = 0; 2511 memcpy(cpt_ctx->zs_ctx.ci_key, key, key_len); 2512 memcpy(cpt_ctx->zs_ctx.zuc_const, zuc_d, 32); 2513 cpt_ctx->fc_type = ZUC_SNOW3G; 2514 cpt_ctx->zsk_flags = 0x1; 2515 break; 2516 case KASUMI_F9_ECB: 2517 /* Kasumi ECB mode */ 2518 cpt_ctx->k_ecb = 1; 2519 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2520 cpt_ctx->fc_type = KASUMI; 2521 cpt_ctx->zsk_flags = 0x1; 2522 break; 2523 case KASUMI_F9_CBC: 2524 memcpy(cpt_ctx->k_ctx.ci_key, key, key_len); 2525 cpt_ctx->fc_type = KASUMI; 2526 cpt_ctx->zsk_flags = 0x1; 2527 break; 2528 default: 2529 return -1; 2530 } 2531 cpt_ctx->mac_len = 4; 2532 cpt_ctx->hash_type = type; 2533 return 0; 2534 } 2535 2536 if (!(cpt_ctx->fc_type == FC_GEN && !type)) { 2537 if (!cpt_ctx->fc_type || !cpt_ctx->enc_cipher) 2538 cpt_ctx->fc_type = HASH_HMAC; 2539 } 2540 2541 if (cpt_ctx->fc_type == FC_GEN && key_len > 64) 2542 return -1; 2543 2544 /* For GMAC auth, cipher must be NULL */ 2545 if (type == GMAC_TYPE) 2546 fctx->enc.enc_cipher = 0; 2547 2548 fctx->enc.hash_type = cpt_ctx->hash_type = type; 2549 fctx->enc.mac_len = cpt_ctx->mac_len = mac_len; 2550 2551 if (key_len) { 2552 cpt_ctx->hmac = 1; 2553 memset(cpt_ctx->auth_key, 0, sizeof(cpt_ctx->auth_key)); 2554 memcpy(cpt_ctx->auth_key, key, key_len); 2555 cpt_ctx->auth_key_len = key_len; 2556 memset(fctx->hmac.ipad, 0, sizeof(fctx->hmac.ipad)); 2557 memset(fctx->hmac.opad, 0, sizeof(fctx->hmac.opad)); 2558 2559 if (key_len <= 64) 2560 memcpy(fctx->hmac.opad, key, key_len); 2561 fctx->enc.auth_input_type = 1; 2562 } 2563 return 0; 2564 } 2565 2566 static __rte_always_inline int 2567 fill_sess_aead(struct rte_crypto_sym_xform *xform, 2568 struct cpt_sess_misc *sess) 2569 { 2570 struct rte_crypto_aead_xform *aead_form; 2571 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2572 auth_type_t auth_type = 0; /* NULL Auth type */ 2573 uint32_t cipher_key_len = 0; 2574 uint8_t aes_gcm = 0; 2575 aead_form = &xform->aead; 2576 void *ctx = SESS_PRIV(sess); 2577 2578 if (aead_form->op == RTE_CRYPTO_AEAD_OP_ENCRYPT && 2579 aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) { 2580 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2581 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2582 } else if (aead_form->op == RTE_CRYPTO_AEAD_OP_DECRYPT && 2583 aead_form->algo == RTE_CRYPTO_AEAD_AES_GCM) { 2584 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2585 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2586 } else { 2587 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2588 return -1; 2589 } 2590 switch (aead_form->algo) { 2591 case RTE_CRYPTO_AEAD_AES_GCM: 2592 enc_type = AES_GCM; 2593 cipher_key_len = 16; 2594 aes_gcm = 1; 2595 break; 2596 case RTE_CRYPTO_AEAD_AES_CCM: 2597 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2598 aead_form->algo); 2599 return -1; 2600 default: 2601 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2602 aead_form->algo); 2603 return -1; 2604 } 2605 if (aead_form->key.length < cipher_key_len) { 2606 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2607 (unsigned int long)aead_form->key.length); 2608 return -1; 2609 } 2610 sess->zsk_flag = 0; 2611 sess->aes_gcm = aes_gcm; 2612 sess->mac_len = aead_form->digest_length; 2613 sess->iv_offset = aead_form->iv.offset; 2614 sess->iv_length = aead_form->iv.length; 2615 sess->aad_length = aead_form->aad_length; 2616 2617 if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, aead_form->key.data, 2618 aead_form->key.length, NULL))) 2619 return -1; 2620 2621 if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, 2622 aead_form->digest_length))) 2623 return -1; 2624 2625 return 0; 2626 } 2627 2628 static __rte_always_inline int 2629 fill_sess_cipher(struct rte_crypto_sym_xform *xform, 2630 struct cpt_sess_misc *sess) 2631 { 2632 struct rte_crypto_cipher_xform *c_form; 2633 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2634 uint32_t cipher_key_len = 0; 2635 uint8_t zsk_flag = 0, aes_ctr = 0, is_null = 0; 2636 2637 c_form = &xform->cipher; 2638 2639 if (c_form->op == RTE_CRYPTO_CIPHER_OP_ENCRYPT) 2640 sess->cpt_op |= CPT_OP_CIPHER_ENCRYPT; 2641 else if (c_form->op == RTE_CRYPTO_CIPHER_OP_DECRYPT) 2642 sess->cpt_op |= CPT_OP_CIPHER_DECRYPT; 2643 else { 2644 CPT_LOG_DP_ERR("Unknown cipher operation\n"); 2645 return -1; 2646 } 2647 2648 switch (c_form->algo) { 2649 case RTE_CRYPTO_CIPHER_AES_CBC: 2650 enc_type = AES_CBC; 2651 cipher_key_len = 16; 2652 break; 2653 case RTE_CRYPTO_CIPHER_3DES_CBC: 2654 enc_type = DES3_CBC; 2655 cipher_key_len = 24; 2656 break; 2657 case RTE_CRYPTO_CIPHER_DES_CBC: 2658 /* DES is implemented using 3DES in hardware */ 2659 enc_type = DES3_CBC; 2660 cipher_key_len = 8; 2661 break; 2662 case RTE_CRYPTO_CIPHER_AES_CTR: 2663 enc_type = AES_CTR; 2664 cipher_key_len = 16; 2665 aes_ctr = 1; 2666 break; 2667 case RTE_CRYPTO_CIPHER_NULL: 2668 enc_type = 0; 2669 is_null = 1; 2670 break; 2671 case RTE_CRYPTO_CIPHER_KASUMI_F8: 2672 enc_type = KASUMI_F8_ECB; 2673 cipher_key_len = 16; 2674 zsk_flag = K_F8; 2675 break; 2676 case RTE_CRYPTO_CIPHER_SNOW3G_UEA2: 2677 enc_type = SNOW3G_UEA2; 2678 cipher_key_len = 16; 2679 zsk_flag = ZS_EA; 2680 break; 2681 case RTE_CRYPTO_CIPHER_ZUC_EEA3: 2682 enc_type = ZUC_EEA3; 2683 cipher_key_len = 16; 2684 zsk_flag = ZS_EA; 2685 break; 2686 case RTE_CRYPTO_CIPHER_AES_XTS: 2687 enc_type = AES_XTS; 2688 cipher_key_len = 16; 2689 break; 2690 case RTE_CRYPTO_CIPHER_3DES_ECB: 2691 enc_type = DES3_ECB; 2692 cipher_key_len = 24; 2693 break; 2694 case RTE_CRYPTO_CIPHER_AES_ECB: 2695 enc_type = AES_ECB; 2696 cipher_key_len = 16; 2697 break; 2698 case RTE_CRYPTO_CIPHER_3DES_CTR: 2699 case RTE_CRYPTO_CIPHER_AES_F8: 2700 case RTE_CRYPTO_CIPHER_ARC4: 2701 CPT_LOG_DP_ERR("Crypto: Unsupported cipher algo %u", 2702 c_form->algo); 2703 return -1; 2704 default: 2705 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2706 c_form->algo); 2707 return -1; 2708 } 2709 2710 if (c_form->key.length < cipher_key_len) { 2711 CPT_LOG_DP_ERR("Invalid cipher params keylen %lu", 2712 (unsigned long) c_form->key.length); 2713 return -1; 2714 } 2715 2716 sess->zsk_flag = zsk_flag; 2717 sess->aes_gcm = 0; 2718 sess->aes_ctr = aes_ctr; 2719 sess->iv_offset = c_form->iv.offset; 2720 sess->iv_length = c_form->iv.length; 2721 sess->is_null = is_null; 2722 2723 if (unlikely(cpt_fc_ciph_set_key(SESS_PRIV(sess), enc_type, 2724 c_form->key.data, c_form->key.length, NULL))) 2725 return -1; 2726 2727 return 0; 2728 } 2729 2730 static __rte_always_inline int 2731 fill_sess_auth(struct rte_crypto_sym_xform *xform, 2732 struct cpt_sess_misc *sess) 2733 { 2734 struct rte_crypto_auth_xform *a_form; 2735 auth_type_t auth_type = 0; /* NULL Auth type */ 2736 uint8_t zsk_flag = 0, aes_gcm = 0, is_null = 0; 2737 2738 a_form = &xform->auth; 2739 2740 if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2741 sess->cpt_op |= CPT_OP_AUTH_VERIFY; 2742 else if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2743 sess->cpt_op |= CPT_OP_AUTH_GENERATE; 2744 else { 2745 CPT_LOG_DP_ERR("Unknown auth operation"); 2746 return -1; 2747 } 2748 2749 switch (a_form->algo) { 2750 case RTE_CRYPTO_AUTH_SHA1_HMAC: 2751 /* Fall through */ 2752 case RTE_CRYPTO_AUTH_SHA1: 2753 auth_type = SHA1_TYPE; 2754 break; 2755 case RTE_CRYPTO_AUTH_SHA256_HMAC: 2756 case RTE_CRYPTO_AUTH_SHA256: 2757 auth_type = SHA2_SHA256; 2758 break; 2759 case RTE_CRYPTO_AUTH_SHA512_HMAC: 2760 case RTE_CRYPTO_AUTH_SHA512: 2761 auth_type = SHA2_SHA512; 2762 break; 2763 case RTE_CRYPTO_AUTH_AES_GMAC: 2764 auth_type = GMAC_TYPE; 2765 aes_gcm = 1; 2766 break; 2767 case RTE_CRYPTO_AUTH_SHA224_HMAC: 2768 case RTE_CRYPTO_AUTH_SHA224: 2769 auth_type = SHA2_SHA224; 2770 break; 2771 case RTE_CRYPTO_AUTH_SHA384_HMAC: 2772 case RTE_CRYPTO_AUTH_SHA384: 2773 auth_type = SHA2_SHA384; 2774 break; 2775 case RTE_CRYPTO_AUTH_MD5_HMAC: 2776 case RTE_CRYPTO_AUTH_MD5: 2777 auth_type = MD5_TYPE; 2778 break; 2779 case RTE_CRYPTO_AUTH_KASUMI_F9: 2780 auth_type = KASUMI_F9_ECB; 2781 /* 2782 * Indicate that direction needs to be taken out 2783 * from end of src 2784 */ 2785 zsk_flag = K_F9; 2786 break; 2787 case RTE_CRYPTO_AUTH_SNOW3G_UIA2: 2788 auth_type = SNOW3G_UIA2; 2789 zsk_flag = ZS_IA; 2790 break; 2791 case RTE_CRYPTO_AUTH_ZUC_EIA3: 2792 auth_type = ZUC_EIA3; 2793 zsk_flag = ZS_IA; 2794 break; 2795 case RTE_CRYPTO_AUTH_NULL: 2796 auth_type = 0; 2797 is_null = 1; 2798 break; 2799 case RTE_CRYPTO_AUTH_AES_XCBC_MAC: 2800 case RTE_CRYPTO_AUTH_AES_CMAC: 2801 case RTE_CRYPTO_AUTH_AES_CBC_MAC: 2802 CPT_LOG_DP_ERR("Crypto: Unsupported hash algo %u", 2803 a_form->algo); 2804 return -1; 2805 default: 2806 CPT_LOG_DP_ERR("Crypto: Undefined Hash algo %u specified", 2807 a_form->algo); 2808 return -1; 2809 } 2810 2811 sess->zsk_flag = zsk_flag; 2812 sess->aes_gcm = aes_gcm; 2813 sess->mac_len = a_form->digest_length; 2814 sess->is_null = is_null; 2815 if (zsk_flag) { 2816 sess->auth_iv_offset = a_form->iv.offset; 2817 sess->auth_iv_length = a_form->iv.length; 2818 } 2819 if (unlikely(cpt_fc_auth_set_key(SESS_PRIV(sess), auth_type, 2820 a_form->key.data, a_form->key.length, 2821 a_form->digest_length))) 2822 return -1; 2823 2824 return 0; 2825 } 2826 2827 static __rte_always_inline int 2828 fill_sess_gmac(struct rte_crypto_sym_xform *xform, 2829 struct cpt_sess_misc *sess) 2830 { 2831 struct rte_crypto_auth_xform *a_form; 2832 cipher_type_t enc_type = 0; /* NULL Cipher type */ 2833 auth_type_t auth_type = 0; /* NULL Auth type */ 2834 void *ctx = SESS_PRIV(sess); 2835 2836 a_form = &xform->auth; 2837 2838 if (a_form->op == RTE_CRYPTO_AUTH_OP_GENERATE) 2839 sess->cpt_op |= CPT_OP_ENCODE; 2840 else if (a_form->op == RTE_CRYPTO_AUTH_OP_VERIFY) 2841 sess->cpt_op |= CPT_OP_DECODE; 2842 else { 2843 CPT_LOG_DP_ERR("Unknown auth operation"); 2844 return -1; 2845 } 2846 2847 switch (a_form->algo) { 2848 case RTE_CRYPTO_AUTH_AES_GMAC: 2849 enc_type = AES_GCM; 2850 auth_type = GMAC_TYPE; 2851 break; 2852 default: 2853 CPT_LOG_DP_ERR("Crypto: Undefined cipher algo %u specified", 2854 a_form->algo); 2855 return -1; 2856 } 2857 2858 sess->zsk_flag = 0; 2859 sess->aes_gcm = 0; 2860 sess->is_gmac = 1; 2861 sess->iv_offset = a_form->iv.offset; 2862 sess->iv_length = a_form->iv.length; 2863 sess->mac_len = a_form->digest_length; 2864 2865 if (unlikely(cpt_fc_ciph_set_key(ctx, enc_type, a_form->key.data, 2866 a_form->key.length, NULL))) 2867 return -1; 2868 2869 if (unlikely(cpt_fc_auth_set_key(ctx, auth_type, NULL, 0, 2870 a_form->digest_length))) 2871 return -1; 2872 2873 return 0; 2874 } 2875 2876 static __rte_always_inline void * 2877 alloc_op_meta(struct rte_mbuf *m_src, 2878 buf_ptr_t *buf, 2879 int32_t len, 2880 struct rte_mempool *cpt_meta_pool) 2881 { 2882 uint8_t *mdata; 2883 2884 #ifndef CPT_ALWAYS_USE_SEPARATE_BUF 2885 if (likely(m_src && (m_src->nb_segs == 1))) { 2886 int32_t tailroom; 2887 phys_addr_t mphys; 2888 2889 /* Check if tailroom is sufficient to hold meta data */ 2890 tailroom = rte_pktmbuf_tailroom(m_src); 2891 if (likely(tailroom > len + 8)) { 2892 mdata = (uint8_t *)m_src->buf_addr + m_src->buf_len; 2893 mphys = m_src->buf_physaddr + m_src->buf_len; 2894 mdata -= len; 2895 mphys -= len; 2896 buf->vaddr = mdata; 2897 buf->dma_addr = mphys; 2898 buf->size = len; 2899 /* Indicate that this is a mbuf allocated mdata */ 2900 mdata = (uint8_t *)((uint64_t)mdata | 1ull); 2901 return mdata; 2902 } 2903 } 2904 #else 2905 RTE_SET_USED(m_src); 2906 #endif 2907 2908 if (unlikely(rte_mempool_get(cpt_meta_pool, (void **)&mdata) < 0)) 2909 return NULL; 2910 2911 buf->vaddr = mdata; 2912 buf->dma_addr = rte_mempool_virt2iova(mdata); 2913 buf->size = len; 2914 2915 return mdata; 2916 } 2917 2918 /** 2919 * cpt_free_metabuf - free metabuf to mempool. 2920 * @param instance: pointer to instance. 2921 * @param objp: pointer to the metabuf. 2922 */ 2923 static __rte_always_inline void 2924 free_op_meta(void *mdata, struct rte_mempool *cpt_meta_pool) 2925 { 2926 bool nofree = ((uintptr_t)mdata & 1ull); 2927 2928 if (likely(nofree)) 2929 return; 2930 rte_mempool_put(cpt_meta_pool, mdata); 2931 } 2932 2933 static __rte_always_inline uint32_t 2934 prepare_iov_from_pkt(struct rte_mbuf *pkt, 2935 iov_ptr_t *iovec, uint32_t start_offset) 2936 { 2937 uint16_t index = 0; 2938 void *seg_data = NULL; 2939 phys_addr_t seg_phys; 2940 int32_t seg_size = 0; 2941 2942 if (!pkt) { 2943 iovec->buf_cnt = 0; 2944 return 0; 2945 } 2946 2947 if (!start_offset) { 2948 seg_data = rte_pktmbuf_mtod(pkt, void *); 2949 seg_phys = rte_pktmbuf_mtophys(pkt); 2950 seg_size = pkt->data_len; 2951 } else { 2952 while (start_offset >= pkt->data_len) { 2953 start_offset -= pkt->data_len; 2954 pkt = pkt->next; 2955 } 2956 2957 seg_data = rte_pktmbuf_mtod_offset(pkt, void *, start_offset); 2958 seg_phys = rte_pktmbuf_mtophys_offset(pkt, start_offset); 2959 seg_size = pkt->data_len - start_offset; 2960 if (!seg_size) 2961 return 1; 2962 } 2963 2964 /* first seg */ 2965 iovec->bufs[index].vaddr = seg_data; 2966 iovec->bufs[index].dma_addr = seg_phys; 2967 iovec->bufs[index].size = seg_size; 2968 index++; 2969 pkt = pkt->next; 2970 2971 while (unlikely(pkt != NULL)) { 2972 seg_data = rte_pktmbuf_mtod(pkt, void *); 2973 seg_phys = rte_pktmbuf_mtophys(pkt); 2974 seg_size = pkt->data_len; 2975 if (!seg_size) 2976 break; 2977 2978 iovec->bufs[index].vaddr = seg_data; 2979 iovec->bufs[index].dma_addr = seg_phys; 2980 iovec->bufs[index].size = seg_size; 2981 2982 index++; 2983 2984 pkt = pkt->next; 2985 } 2986 2987 iovec->buf_cnt = index; 2988 return 0; 2989 } 2990 2991 static __rte_always_inline uint32_t 2992 prepare_iov_from_pkt_inplace(struct rte_mbuf *pkt, 2993 fc_params_t *param, 2994 uint32_t *flags) 2995 { 2996 uint16_t index = 0; 2997 void *seg_data = NULL; 2998 phys_addr_t seg_phys; 2999 uint32_t seg_size = 0; 3000 iov_ptr_t *iovec; 3001 3002 seg_data = rte_pktmbuf_mtod(pkt, void *); 3003 seg_phys = rte_pktmbuf_mtophys(pkt); 3004 seg_size = pkt->data_len; 3005 3006 /* first seg */ 3007 if (likely(!pkt->next)) { 3008 uint32_t headroom, tailroom; 3009 3010 *flags |= SINGLE_BUF_INPLACE; 3011 headroom = rte_pktmbuf_headroom(pkt); 3012 tailroom = rte_pktmbuf_tailroom(pkt); 3013 if (likely((headroom >= 24) && 3014 (tailroom >= 8))) { 3015 /* In 83XX this is prerequivisit for Direct mode */ 3016 *flags |= SINGLE_BUF_HEADTAILROOM; 3017 } 3018 param->bufs[0].vaddr = seg_data; 3019 param->bufs[0].dma_addr = seg_phys; 3020 param->bufs[0].size = seg_size; 3021 return 0; 3022 } 3023 iovec = param->src_iov; 3024 iovec->bufs[index].vaddr = seg_data; 3025 iovec->bufs[index].dma_addr = seg_phys; 3026 iovec->bufs[index].size = seg_size; 3027 index++; 3028 pkt = pkt->next; 3029 3030 while (unlikely(pkt != NULL)) { 3031 seg_data = rte_pktmbuf_mtod(pkt, void *); 3032 seg_phys = rte_pktmbuf_mtophys(pkt); 3033 seg_size = pkt->data_len; 3034 3035 if (!seg_size) 3036 break; 3037 3038 iovec->bufs[index].vaddr = seg_data; 3039 iovec->bufs[index].dma_addr = seg_phys; 3040 iovec->bufs[index].size = seg_size; 3041 3042 index++; 3043 3044 pkt = pkt->next; 3045 } 3046 3047 iovec->buf_cnt = index; 3048 return 0; 3049 } 3050 3051 static __rte_always_inline int 3052 fill_fc_params(struct rte_crypto_op *cop, 3053 struct cpt_sess_misc *sess_misc, 3054 struct cpt_qp_meta_info *m_info, 3055 void **mdata_ptr, 3056 void **prep_req) 3057 { 3058 uint32_t space = 0; 3059 struct rte_crypto_sym_op *sym_op = cop->sym; 3060 void *mdata = NULL; 3061 uintptr_t *op; 3062 uint32_t mc_hash_off; 3063 uint32_t flags = 0; 3064 uint64_t d_offs, d_lens; 3065 struct rte_mbuf *m_src, *m_dst; 3066 uint8_t cpt_op = sess_misc->cpt_op; 3067 #ifdef CPT_ALWAYS_USE_SG_MODE 3068 uint8_t inplace = 0; 3069 #else 3070 uint8_t inplace = 1; 3071 #endif 3072 fc_params_t fc_params; 3073 char src[SRC_IOV_SIZE]; 3074 char dst[SRC_IOV_SIZE]; 3075 uint32_t iv_buf[4]; 3076 int ret; 3077 3078 if (likely(sess_misc->iv_length)) { 3079 flags |= VALID_IV_BUF; 3080 fc_params.iv_buf = rte_crypto_op_ctod_offset(cop, 3081 uint8_t *, sess_misc->iv_offset); 3082 if (sess_misc->aes_ctr && 3083 unlikely(sess_misc->iv_length != 16)) { 3084 memcpy((uint8_t *)iv_buf, 3085 rte_crypto_op_ctod_offset(cop, 3086 uint8_t *, sess_misc->iv_offset), 12); 3087 iv_buf[3] = rte_cpu_to_be_32(0x1); 3088 fc_params.iv_buf = iv_buf; 3089 } 3090 } 3091 3092 if (sess_misc->zsk_flag) { 3093 fc_params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3094 uint8_t *, 3095 sess_misc->auth_iv_offset); 3096 if (sess_misc->zsk_flag != ZS_EA) 3097 inplace = 0; 3098 } 3099 m_src = sym_op->m_src; 3100 m_dst = sym_op->m_dst; 3101 3102 if (sess_misc->aes_gcm) { 3103 uint8_t *salt; 3104 uint8_t *aad_data; 3105 uint16_t aad_len; 3106 3107 d_offs = sym_op->aead.data.offset; 3108 d_lens = sym_op->aead.data.length; 3109 mc_hash_off = sym_op->aead.data.offset + 3110 sym_op->aead.data.length; 3111 3112 aad_data = sym_op->aead.aad.data; 3113 aad_len = sess_misc->aad_length; 3114 if (likely((aad_data + aad_len) == 3115 rte_pktmbuf_mtod_offset(m_src, 3116 uint8_t *, 3117 sym_op->aead.data.offset))) { 3118 d_offs = (d_offs - aad_len) | (d_offs << 16); 3119 d_lens = (d_lens + aad_len) | (d_lens << 32); 3120 } else { 3121 fc_params.aad_buf.vaddr = sym_op->aead.aad.data; 3122 fc_params.aad_buf.dma_addr = sym_op->aead.aad.phys_addr; 3123 fc_params.aad_buf.size = aad_len; 3124 flags |= VALID_AAD_BUF; 3125 inplace = 0; 3126 d_offs = d_offs << 16; 3127 d_lens = d_lens << 32; 3128 } 3129 3130 salt = fc_params.iv_buf; 3131 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3132 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3133 sess_misc->salt = *(uint32_t *)salt; 3134 } 3135 fc_params.iv_buf = salt + 4; 3136 if (likely(sess_misc->mac_len)) { 3137 struct rte_mbuf *m = (cpt_op & CPT_OP_ENCODE) ? m_dst : 3138 m_src; 3139 3140 if (!m) 3141 m = m_src; 3142 3143 /* hmac immediately following data is best case */ 3144 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3145 mc_hash_off != 3146 (uint8_t *)sym_op->aead.digest.data)) { 3147 flags |= VALID_MAC_BUF; 3148 fc_params.mac_buf.size = sess_misc->mac_len; 3149 fc_params.mac_buf.vaddr = 3150 sym_op->aead.digest.data; 3151 fc_params.mac_buf.dma_addr = 3152 sym_op->aead.digest.phys_addr; 3153 inplace = 0; 3154 } 3155 } 3156 } else { 3157 d_offs = sym_op->cipher.data.offset; 3158 d_lens = sym_op->cipher.data.length; 3159 mc_hash_off = sym_op->cipher.data.offset + 3160 sym_op->cipher.data.length; 3161 d_offs = (d_offs << 16) | sym_op->auth.data.offset; 3162 d_lens = (d_lens << 32) | sym_op->auth.data.length; 3163 3164 if (mc_hash_off < (sym_op->auth.data.offset + 3165 sym_op->auth.data.length)){ 3166 mc_hash_off = (sym_op->auth.data.offset + 3167 sym_op->auth.data.length); 3168 } 3169 /* for gmac, salt should be updated like in gcm */ 3170 if (unlikely(sess_misc->is_gmac)) { 3171 uint8_t *salt; 3172 salt = fc_params.iv_buf; 3173 if (unlikely(*(uint32_t *)salt != sess_misc->salt)) { 3174 cpt_fc_salt_update(SESS_PRIV(sess_misc), salt); 3175 sess_misc->salt = *(uint32_t *)salt; 3176 } 3177 fc_params.iv_buf = salt + 4; 3178 } 3179 if (likely(sess_misc->mac_len)) { 3180 struct rte_mbuf *m; 3181 3182 m = (cpt_op & CPT_OP_ENCODE) ? m_dst : m_src; 3183 if (!m) 3184 m = m_src; 3185 3186 /* hmac immediately following data is best case */ 3187 if (unlikely(rte_pktmbuf_mtod(m, uint8_t *) + 3188 mc_hash_off != 3189 (uint8_t *)sym_op->auth.digest.data)) { 3190 flags |= VALID_MAC_BUF; 3191 fc_params.mac_buf.size = 3192 sess_misc->mac_len; 3193 fc_params.mac_buf.vaddr = 3194 sym_op->auth.digest.data; 3195 fc_params.mac_buf.dma_addr = 3196 sym_op->auth.digest.phys_addr; 3197 inplace = 0; 3198 } 3199 } 3200 } 3201 fc_params.ctx_buf.vaddr = SESS_PRIV(sess_misc); 3202 fc_params.ctx_buf.dma_addr = sess_misc->ctx_dma_addr; 3203 3204 if (unlikely(sess_misc->is_null || sess_misc->cpt_op == CPT_OP_DECODE)) 3205 inplace = 0; 3206 3207 if (likely(!m_dst && inplace)) { 3208 /* Case of single buffer without AAD buf or 3209 * separate mac buf in place and 3210 * not air crypto 3211 */ 3212 fc_params.dst_iov = fc_params.src_iov = (void *)src; 3213 3214 if (unlikely(prepare_iov_from_pkt_inplace(m_src, 3215 &fc_params, 3216 &flags))) { 3217 CPT_LOG_DP_ERR("Prepare inplace src iov failed"); 3218 ret = -EINVAL; 3219 goto err_exit; 3220 } 3221 3222 } else { 3223 /* Out of place processing */ 3224 fc_params.src_iov = (void *)src; 3225 fc_params.dst_iov = (void *)dst; 3226 3227 /* Store SG I/O in the api for reuse */ 3228 if (prepare_iov_from_pkt(m_src, fc_params.src_iov, 0)) { 3229 CPT_LOG_DP_ERR("Prepare src iov failed"); 3230 ret = -EINVAL; 3231 goto err_exit; 3232 } 3233 3234 if (unlikely(m_dst != NULL)) { 3235 uint32_t pkt_len; 3236 3237 /* Try to make room as much as src has */ 3238 pkt_len = rte_pktmbuf_pkt_len(m_dst); 3239 3240 if (unlikely(pkt_len < rte_pktmbuf_pkt_len(m_src))) { 3241 pkt_len = rte_pktmbuf_pkt_len(m_src) - pkt_len; 3242 if (!rte_pktmbuf_append(m_dst, pkt_len)) { 3243 CPT_LOG_DP_ERR("Not enough space in " 3244 "m_dst %p, need %u" 3245 " more", 3246 m_dst, pkt_len); 3247 ret = -EINVAL; 3248 goto err_exit; 3249 } 3250 } 3251 3252 if (prepare_iov_from_pkt(m_dst, fc_params.dst_iov, 0)) { 3253 CPT_LOG_DP_ERR("Prepare dst iov failed for " 3254 "m_dst %p", m_dst); 3255 ret = -EINVAL; 3256 goto err_exit; 3257 } 3258 } else { 3259 fc_params.dst_iov = (void *)src; 3260 } 3261 } 3262 3263 if (likely(flags & SINGLE_BUF_HEADTAILROOM)) 3264 mdata = alloc_op_meta(m_src, &fc_params.meta_buf, 3265 m_info->lb_mlen, m_info->pool); 3266 else 3267 mdata = alloc_op_meta(NULL, &fc_params.meta_buf, 3268 m_info->sg_mlen, m_info->pool); 3269 3270 if (unlikely(mdata == NULL)) { 3271 CPT_LOG_DP_ERR("Error allocating meta buffer for request"); 3272 ret = -ENOMEM; 3273 goto err_exit; 3274 } 3275 3276 op = (uintptr_t *)((uintptr_t)mdata & (uintptr_t)~1ull); 3277 op[0] = (uintptr_t)mdata; 3278 op[1] = (uintptr_t)cop; 3279 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3280 space += 4 * sizeof(uint64_t); 3281 3282 fc_params.meta_buf.vaddr = (uint8_t *)op + space; 3283 fc_params.meta_buf.dma_addr += space; 3284 fc_params.meta_buf.size -= space; 3285 3286 /* Finally prepare the instruction */ 3287 if (cpt_op & CPT_OP_ENCODE) 3288 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, 3289 &fc_params, op); 3290 else 3291 *prep_req = cpt_fc_dec_hmac_prep(flags, d_offs, d_lens, 3292 &fc_params, op); 3293 3294 if (unlikely(*prep_req == NULL)) { 3295 CPT_LOG_DP_ERR("Preparing request failed due to bad input arg"); 3296 ret = -EINVAL; 3297 goto free_mdata_and_exit; 3298 } 3299 3300 *mdata_ptr = mdata; 3301 3302 return 0; 3303 3304 free_mdata_and_exit: 3305 free_op_meta(mdata, m_info->pool); 3306 err_exit: 3307 return ret; 3308 } 3309 3310 static __rte_always_inline void 3311 compl_auth_verify(struct rte_crypto_op *op, 3312 uint8_t *gen_mac, 3313 uint64_t mac_len) 3314 { 3315 uint8_t *mac; 3316 struct rte_crypto_sym_op *sym_op = op->sym; 3317 3318 if (sym_op->auth.digest.data) 3319 mac = sym_op->auth.digest.data; 3320 else 3321 mac = rte_pktmbuf_mtod_offset(sym_op->m_src, 3322 uint8_t *, 3323 sym_op->auth.data.length + 3324 sym_op->auth.data.offset); 3325 if (!mac) { 3326 op->status = RTE_CRYPTO_OP_STATUS_ERROR; 3327 return; 3328 } 3329 3330 if (memcmp(mac, gen_mac, mac_len)) 3331 op->status = RTE_CRYPTO_OP_STATUS_AUTH_FAILED; 3332 else 3333 op->status = RTE_CRYPTO_OP_STATUS_SUCCESS; 3334 } 3335 3336 static __rte_always_inline int 3337 instance_session_cfg(struct rte_crypto_sym_xform *xform, void *sess) 3338 { 3339 struct rte_crypto_sym_xform *chain; 3340 3341 CPT_PMD_INIT_FUNC_TRACE(); 3342 3343 if (cpt_is_algo_supported(xform)) 3344 goto err; 3345 3346 chain = xform; 3347 while (chain) { 3348 switch (chain->type) { 3349 case RTE_CRYPTO_SYM_XFORM_AEAD: 3350 if (fill_sess_aead(chain, sess)) 3351 goto err; 3352 break; 3353 case RTE_CRYPTO_SYM_XFORM_CIPHER: 3354 if (fill_sess_cipher(chain, sess)) 3355 goto err; 3356 break; 3357 case RTE_CRYPTO_SYM_XFORM_AUTH: 3358 if (chain->auth.algo == RTE_CRYPTO_AUTH_AES_GMAC) { 3359 if (fill_sess_gmac(chain, sess)) 3360 goto err; 3361 } else { 3362 if (fill_sess_auth(chain, sess)) 3363 goto err; 3364 } 3365 break; 3366 default: 3367 CPT_LOG_DP_ERR("Invalid crypto xform type"); 3368 break; 3369 } 3370 chain = chain->next; 3371 } 3372 3373 return 0; 3374 3375 err: 3376 return -1; 3377 } 3378 3379 static __rte_always_inline void 3380 find_kasumif9_direction_and_length(uint8_t *src, 3381 uint32_t counter_num_bytes, 3382 uint32_t *addr_length_in_bits, 3383 uint8_t *addr_direction) 3384 { 3385 uint8_t found = 0; 3386 uint32_t pos; 3387 uint8_t last_byte; 3388 while (!found && counter_num_bytes > 0) { 3389 counter_num_bytes--; 3390 if (src[counter_num_bytes] == 0x00) 3391 continue; 3392 pos = rte_bsf32(src[counter_num_bytes]); 3393 if (pos == 7) { 3394 if (likely(counter_num_bytes > 0)) { 3395 last_byte = src[counter_num_bytes - 1]; 3396 *addr_direction = last_byte & 0x1; 3397 *addr_length_in_bits = counter_num_bytes * 8 3398 - 1; 3399 } 3400 } else { 3401 last_byte = src[counter_num_bytes]; 3402 *addr_direction = (last_byte >> (pos + 1)) & 0x1; 3403 *addr_length_in_bits = counter_num_bytes * 8 3404 + (8 - (pos + 2)); 3405 } 3406 found = 1; 3407 } 3408 } 3409 3410 /* 3411 * This handles all auth only except AES_GMAC 3412 */ 3413 static __rte_always_inline int 3414 fill_digest_params(struct rte_crypto_op *cop, 3415 struct cpt_sess_misc *sess, 3416 struct cpt_qp_meta_info *m_info, 3417 void **mdata_ptr, 3418 void **prep_req) 3419 { 3420 uint32_t space = 0; 3421 struct rte_crypto_sym_op *sym_op = cop->sym; 3422 void *mdata; 3423 phys_addr_t mphys; 3424 uint64_t *op; 3425 uint32_t auth_range_off; 3426 uint32_t flags = 0; 3427 uint64_t d_offs = 0, d_lens; 3428 struct rte_mbuf *m_src, *m_dst; 3429 uint16_t auth_op = sess->cpt_op & CPT_OP_AUTH_MASK; 3430 uint16_t mac_len = sess->mac_len; 3431 fc_params_t params; 3432 char src[SRC_IOV_SIZE]; 3433 uint8_t iv_buf[16]; 3434 int ret; 3435 3436 memset(¶ms, 0, sizeof(fc_params_t)); 3437 3438 m_src = sym_op->m_src; 3439 3440 /* For just digest lets force mempool alloc */ 3441 mdata = alloc_op_meta(NULL, ¶ms.meta_buf, m_info->sg_mlen, 3442 m_info->pool); 3443 if (mdata == NULL) { 3444 ret = -ENOMEM; 3445 goto err_exit; 3446 } 3447 3448 mphys = params.meta_buf.dma_addr; 3449 3450 op = mdata; 3451 op[0] = (uintptr_t)mdata; 3452 op[1] = (uintptr_t)cop; 3453 op[2] = op[3] = 0; /* Used to indicate auth verify */ 3454 space += 4 * sizeof(uint64_t); 3455 3456 auth_range_off = sym_op->auth.data.offset; 3457 3458 flags = VALID_MAC_BUF; 3459 params.src_iov = (void *)src; 3460 if (unlikely(sess->zsk_flag)) { 3461 /* 3462 * Since for Zuc, Kasumi, Snow3g offsets are in bits 3463 * we will send pass through even for auth only case, 3464 * let MC handle it 3465 */ 3466 d_offs = auth_range_off; 3467 auth_range_off = 0; 3468 params.auth_iv_buf = rte_crypto_op_ctod_offset(cop, 3469 uint8_t *, sess->auth_iv_offset); 3470 if (sess->zsk_flag == K_F9) { 3471 uint32_t length_in_bits, num_bytes; 3472 uint8_t *src, direction = 0; 3473 3474 memcpy(iv_buf, rte_pktmbuf_mtod(cop->sym->m_src, 3475 uint8_t *), 8); 3476 /* 3477 * This is kasumi f9, take direction from 3478 * source buffer 3479 */ 3480 length_in_bits = cop->sym->auth.data.length; 3481 num_bytes = (length_in_bits >> 3); 3482 src = rte_pktmbuf_mtod(cop->sym->m_src, uint8_t *); 3483 find_kasumif9_direction_and_length(src, 3484 num_bytes, 3485 &length_in_bits, 3486 &direction); 3487 length_in_bits -= 64; 3488 cop->sym->auth.data.offset += 64; 3489 d_offs = cop->sym->auth.data.offset; 3490 auth_range_off = d_offs / 8; 3491 cop->sym->auth.data.length = length_in_bits; 3492 3493 /* Store it at end of auth iv */ 3494 iv_buf[8] = direction; 3495 params.auth_iv_buf = iv_buf; 3496 } 3497 } 3498 3499 d_lens = sym_op->auth.data.length; 3500 3501 params.ctx_buf.vaddr = SESS_PRIV(sess); 3502 params.ctx_buf.dma_addr = sess->ctx_dma_addr; 3503 3504 if (auth_op == CPT_OP_AUTH_GENERATE) { 3505 if (sym_op->auth.digest.data) { 3506 /* 3507 * Digest to be generated 3508 * in separate buffer 3509 */ 3510 params.mac_buf.size = 3511 sess->mac_len; 3512 params.mac_buf.vaddr = 3513 sym_op->auth.digest.data; 3514 params.mac_buf.dma_addr = 3515 sym_op->auth.digest.phys_addr; 3516 } else { 3517 uint32_t off = sym_op->auth.data.offset + 3518 sym_op->auth.data.length; 3519 int32_t dlen, space; 3520 3521 m_dst = sym_op->m_dst ? 3522 sym_op->m_dst : sym_op->m_src; 3523 dlen = rte_pktmbuf_pkt_len(m_dst); 3524 3525 space = off + mac_len - dlen; 3526 if (space > 0) 3527 if (!rte_pktmbuf_append(m_dst, space)) { 3528 CPT_LOG_DP_ERR("Failed to extend " 3529 "mbuf by %uB", space); 3530 ret = -EINVAL; 3531 goto free_mdata_and_exit; 3532 } 3533 3534 params.mac_buf.vaddr = 3535 rte_pktmbuf_mtod_offset(m_dst, void *, off); 3536 params.mac_buf.dma_addr = 3537 rte_pktmbuf_mtophys_offset(m_dst, off); 3538 params.mac_buf.size = mac_len; 3539 } 3540 } else { 3541 /* Need space for storing generated mac */ 3542 params.mac_buf.vaddr = (uint8_t *)mdata + space; 3543 params.mac_buf.dma_addr = mphys + space; 3544 params.mac_buf.size = mac_len; 3545 space += RTE_ALIGN_CEIL(mac_len, 8); 3546 op[2] = (uintptr_t)params.mac_buf.vaddr; 3547 op[3] = mac_len; 3548 } 3549 3550 params.meta_buf.vaddr = (uint8_t *)mdata + space; 3551 params.meta_buf.dma_addr = mphys + space; 3552 params.meta_buf.size -= space; 3553 3554 /* Out of place processing */ 3555 params.src_iov = (void *)src; 3556 3557 /*Store SG I/O in the api for reuse */ 3558 if (prepare_iov_from_pkt(m_src, params.src_iov, auth_range_off)) { 3559 CPT_LOG_DP_ERR("Prepare src iov failed"); 3560 ret = -EINVAL; 3561 goto free_mdata_and_exit; 3562 } 3563 3564 *prep_req = cpt_fc_enc_hmac_prep(flags, d_offs, d_lens, ¶ms, op); 3565 if (unlikely(*prep_req == NULL)) { 3566 ret = -EINVAL; 3567 goto free_mdata_and_exit; 3568 } 3569 3570 *mdata_ptr = mdata; 3571 3572 return 0; 3573 3574 free_mdata_and_exit: 3575 free_op_meta(mdata, m_info->pool); 3576 err_exit: 3577 return ret; 3578 } 3579 3580 #endif /*_CPT_UCODE_H_ */ 3581